Merge conflicts

tasks/elasticsearch-config.yml

@@ -51,6 +51,18 @@
 - name: Copy Logging.yml File for Instance
   template: src=logging.yml.j2 dest={{conf_dir}}/logging.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   notify: restart elasticsearch
+  when: es_version | version_compare('5.0', '<')
+
+- name: Copy log4j2.properties File for Instance
+  template: src=log4j2.properties.j2 dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  notify: restart elasticsearch
+  when: es_version | version_compare('5.0', '>=')
+
+# Copy the JVM Options (5.x only)
+- name: Copy jvm.options File for Instance
+  template: src=jvm.options.j2 dest={{conf_dir}}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  notify: restart elasticsearch
+  when: es_version | version_compare('5.0', '>=')
 
 #Clean up un-wanted package scripts to avoid confusion
 
@@ -74,3 +86,8 @@
 - name: Delete Default Logging File
   file: dest=/etc/elasticsearch/logging.yml state=absent
 
+- name: Delete Default Logging File (5.x)
+  file: dest=/etc/elasticsearch/log4j2.properties state=absent
+
+- name: Delete Default JVM Options File (5.x)
+  file: dest=/etc/elasticsearch/jvm.options state=absent

tasks/elasticsearch-parameters.yml

@@ -18,8 +18,8 @@
   when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined
 
 #If the user attempts to lock memory they must specify a heap size
-- fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified"
-  when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined
+- fail: msg="If locking memory with bootstrap.mlockall (or bootstrap.memory_lock) a heap size must be specified"
+  when: (es_config['bootstrap.mlockall'] is defined or es_config['bootstrap.memory_lock'] is defined) and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined
 
 #Don't support xpack on versions < 2.0
 - fail: msg="Use of the xpack notation is not supported on versions < 2.0.  Marvel-agent and watcher can be installed as plugins.  Version > 2.0 is required for shield."
@@ -32,7 +32,7 @@
 - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}}
 - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}}
 - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}}
-- set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }}
+- set_fact: m_lock_enabled={{ (es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True) or (es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True) }}
 
 #Use systemd for the following distributions:
 #Ubuntu 15 and up

tasks/elasticsearch-plugins.yml

@@ -11,7 +11,8 @@
   when: es_version | version_compare('2.0', '<')
 
 #List currently installed plugins - ignore xpack if > v 2.0
-- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}"
+- name: Check installed elasticsearch plugins
+  shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin {{list_command}}{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %}{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}"
   register: installed_plugins
   failed_when: "'ERROR' in installed_plugins.stdout"
   changed_when: False
@@ -20,22 +21,21 @@
     CONF_DIR: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
-#This needs to removes any currently installed plugins
+# This removes any currently installed plugins (to prevent errors when reinstalling)
 - name: Remove elasticsearch plugins
-  command: "{{es_home}}/bin/plugin remove {{item}} --silent"
+  command: "{{es_home}}/bin/{{ es_binary_prefix }}plugin remove {{item}} --silent"
   ignore_errors: yes
   with_items: "{{ installed_plugins.stdout_lines | default([]) }}"
   when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0]
   notify: restart elasticsearch
-  register: plugin_installed
+  register: plugin_removed
   environment:
     CONF_DIR: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 - name: Install elasticsearch plugins
-  #debug: var=item
   command: >
-    {{es_home}}/bin/plugin install {{ item.plugin }}{% if item.version is defined and item.version != '' %}/{{ item.version }}{% endif %} {% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -DproxyHost={{ item.proxy_host }} -DproxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -DproxyHost={{ es_proxy_host }} -DproxyPort={{ es_proxy_port }} {% endif %} --silent
+    {{es_home}}/bin/{{ es_binary_prefix }}plugin install {{ item.plugin }}{% if item.version is defined and item.version != '' %}/{{ item.version }}{% endif %} {% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -DproxyHost={{ item.proxy_host }} -DproxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -DproxyHost={{ es_proxy_host }} -DproxyPort={{ es_proxy_port }} {% endif %} --silent
   register: plugin_installed
   failed_when: "'ERROR' in plugin_installed.stdout"
   changed_when: plugin_installed.rc == 0

tasks/elasticsearch.yml

@@ -1,5 +1,10 @@
 ---
 
+- set_fact: es_binary_prefix=""
+
+- set_fact: es_binary_prefix="elasticsearch-"
+  when: es_version | version_compare('5.0', '>=')
+
 - name: Include optional user and group creation.
   when: (es_user_id is defined) and (es_group_id is defined)
   include: elasticsearch-optional-user.yml

tasks/xpack/elasticsearch-xpack-install.yml

@@ -1,7 +1,7 @@
 ---
 
 #Test if feature is installed
-- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep {{item}}"
+- shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin list{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %} | grep {{item}}"
   register: feature_installed
   changed_when: False
   failed_when: "'ERROR' in feature_installed.stdout"
@@ -13,8 +13,7 @@
 
 #Remove Plugin if installed and its not been requested or the ES version has changed
 - name: Remove {{item}} plugin
-  command: >
-    {{es_home}}/bin/plugin remove {{item}}
+  command: {{es_home}}/bin/plugin remove {{item}}
   register: xpack_state
   failed_when: "'ERROR' in xpack_state.stdout"
   changed_when: xpack_state.rc == 0
@@ -28,7 +27,7 @@
 #Install plugin if not installed, or the es version has changed (so removed above), and its been requested
 - name: Install {{item}} plugin
   command: >
-    {{es_home}}/bin/plugin install {{item}}
+    {{es_home}}/bin/{{ es_binary_prefix }}plugin install {{item}}
   register: xpack_state
   failed_when: "'ERROR' in xpack_state.stdout"
   changed_when: xpack_state.rc == 0

tasks/xpack/elasticsearch-xpack.yml

@@ -7,7 +7,7 @@
 #Check if license is installed
 - name: Check License plugin is installed
   shell: >
-    {{es_home}}/bin/plugin list | tail -n +2 | grep license
+    {{es_home}}/bin/{{ es_binary_prefix }}plugin list | tail -n +2 | grep license
   register: license_installed
   ignore_errors: yes
   failed_when: "'ERROR' in license_installed.stdout"
@@ -19,7 +19,7 @@
 #Remove license if installed and xpack not enabled
 - name: Remove license plugin
   command: >
-    {{es_home}}/bin/plugin remove license
+    {{es_home}}/bin/{{ es_binary_prefix }}plugin remove license
   register: xpack_state
   failed_when: "'ERROR' in xpack_state.stdout"
   changed_when: xpack_state.rc == 0
@@ -32,7 +32,7 @@
 #Install License if not installed, or it needs to be reinstalled due to ES change (above task will have removed), and its been requested.
 - name: Install license plugin
   command: >
-    {{es_home}}/bin/plugin install license
+    {{es_home}}/bin/{{ es_binary_prefix }}plugin install license
   register: xpack_state
   failed_when: "'ERROR' in xpack_state.stdout"
   changed_when: xpack_state.rc == 0

templates/elasticsearch.j2

@@ -14,17 +14,21 @@ DATA_DIR={{ data_dirs | array_to_str }}
 # Elasticsearch logs directory
 LOG_DIR={{log_dir}}
 
+{% if es_version | version_compare('5.0', '<') %}
 # Elasticsearch work directory
 WORK_DIR={{work_dir}}
+{% endif %}
 
 # Elasticsearch PID directory
 PID_DIR={{pid_dir}}
 
+{% if es_version | version_compare('5.0', '<') %}
 # Heap size defaults to 256m min, 1g max
 # Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g
 {% if es_heap_size is defined %}
 ES_HEAP_SIZE={{es_heap_size}}
 {% endif %}
+{% endif %}
 
 # Heap new generation
 #ES_HEAP_NEWSIZE=
@@ -72,7 +76,7 @@ MAX_OPEN_FILES={{es_max_open_files}}
 {% endif %}
 
 # The maximum number of bytes of memory that may be locked into RAM
-# Set to "unlimited" if you use the 'bootstrap.mlockall: true' option
+# Set to "unlimited" if you use the 'bootstrap.{% if es_version | version_compare('5.0', '<=') %}memory_lock{% else %}mlockall{% endif %}: true' option
 # in elasticsearch.yml (ES_HEAP_SIZE  must also be set).
 # When using Systemd, the LimitMEMLOCK property must be set
 # in /usr/lib/systemd/system/elasticsearch.service

templates/elasticsearch.yml.j2

@@ -18,6 +18,8 @@ path.conf: {{ conf_dir }}
 
 path.data: {{ data_dirs | array_to_str }}
 
+{% if es_version | version_compare('5.0', '<') %}
 path.work: {{ work_dir }}
+{% endif %}
 
 path.logs: {{ log_dir }}

templates/init/debian/elasticsearch.j2

@@ -51,9 +51,11 @@ ES_GROUP={{es_group}}
 # Directory where the Elasticsearch binary distribution resides
 ES_HOME={{es_home}}
 
+{% if es_version | version_compare('5.0', '<') %}
 # Heap size defaults to 256m min, 1g max
 # Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g
 #ES_HEAP_SIZE=2g
+{% endif %}
 
 # Heap new generation
 #ES_HEAP_NEWSIZE=
@@ -104,7 +106,11 @@ fi
 # Define other required variables
 PID_FILE="$PID_DIR/$NAME.pid"
 DAEMON={{es_home}}/bin/elasticsearch
+{% if es_version | version_compare('5.0', '<') %}
 DAEMON_OPTS="-d -p $PID_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR"
+{% else %}
+DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR"
+{% endif %}
 
 export ES_HEAP_SIZE
 export ES_HEAP_NEWSIZE
@@ -133,10 +139,12 @@ case "$1" in
   start)
 	checkJava
 
+{% if es_version | version_compare('5.0', '<') %}
 	if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then
 		log_failure_msg "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set"
 		exit 1
 	fi
+{% endif %}
 
 	log_daemon_msg "Starting $DESC"
 

templates/init/redhat/elasticsearch.j2

@@ -90,10 +90,12 @@ checkJava() {
 start() {
     checkJava
     [ -x $exec ] || exit 5
+{% if es_version | version_compare('5.0', '<') %}
     if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then
         echo "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set"
         return 7
     fi
+{% endif %}
     if [ -n "$MAX_OPEN_FILES" ]; then
         ulimit -n $MAX_OPEN_FILES
     fi
@@ -116,7 +118,11 @@ start() {
     cd $ES_HOME
     echo -n $"Starting $prog: "
     # if not running, start it up here, usually something like "daemon $exec"
+{% if es_version | version_compare('5.0', '<') %}
     daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -Des.default.path.conf=$CONF_DIR
+{% else %}
+    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=${DATA_DIR} -Edefault.path.conf=$CONF_DIR
+{% endif %}
     retval=$?
     echo
     [ $retval -eq 0 ] && touch $lockfile

templates/jvm.options.j2

@@ -0,0 +1,100 @@
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms{{ es_heap_size }}
+-Xmx{{ es_heap_size }}
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# disable calls to System#gc
+-XX:+DisableExplicitGC
+
+# pre-touch memory pages used by the JVM during initialization
+-XX:+AlwaysPreTouch
+
+## basic
+
+# force the server VM
+-server
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+-Djna.nosys=true
+
+# flags to keep Netty from being unsafe
+-Dio.netty.noUnsafe=true
+-Dio.netty.noKeySetOptimization=true
+
+# log4j 2
+-Dlog4j.shutdownHookEnabled=false
+-Dlog4j2.disable.jmx=true
+-Dlog4j.skipJansi=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${heap.dump.path}
+
+## GC logging
+
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${loggc}
+
+# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
+# If documents were already indexed with unquoted fields in a previous version
+# of Elasticsearch, some operations may throw errors.
+#
+# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
+# only for migration purposes.
+#-Delasticsearch.json.allow_unquoted_field_names=true

templates/log4j2.properties.j2

@@ -0,0 +1,74 @@
+status = error
+
+# log action execution errors for easier debugging
+logger.action.name = org.elasticsearch.action
+logger.action.level = debug
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+
+appender.rolling.type = RollingFile
+appender.rolling.name = rolling
+appender.rolling.fileName = ${sys:es.logs}.log
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n
+appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log
+appender.rolling.policies.type = Policies
+appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.rolling.policies.time.interval = 1
+appender.rolling.policies.time.modulate = true
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console
+rootLogger.appenderRef.rolling.ref = rolling
+
+appender.deprecation_rolling.type = RollingFile
+appender.deprecation_rolling.name = deprecation_rolling
+appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log
+appender.deprecation_rolling.layout.type = PatternLayout
+appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n
+appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz
+appender.deprecation_rolling.policies.type = Policies
+appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.deprecation_rolling.policies.size.size = 1GB
+appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
+appender.deprecation_rolling.strategy.max = 4
+
+logger.deprecation.name = org.elasticsearch.deprecation
+logger.deprecation.level = warn
+logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
+logger.deprecation.additivity = false
+
+appender.index_search_slowlog_rolling.type = RollingFile
+appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
+appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log
+appender.index_search_slowlog_rolling.layout.type = PatternLayout
+appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n
+appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
+appender.index_search_slowlog_rolling.policies.type = Policies
+appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_search_slowlog_rolling.policies.time.interval = 1
+appender.index_search_slowlog_rolling.policies.time.modulate = true
+
+logger.index_search_slowlog_rolling.name = index.search.slowlog
+logger.index_search_slowlog_rolling.level = trace
+logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
+logger.index_search_slowlog_rolling.additivity = false
+
+appender.index_indexing_slowlog_rolling.type = RollingFile
+appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
+appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log
+appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
+appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n
+appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
+appender.index_indexing_slowlog_rolling.policies.type = Policies
+appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
+appender.index_indexing_slowlog_rolling.policies.time.interval = 1
+appender.index_indexing_slowlog_rolling.policies.time.modulate = true
+
+logger.index_indexing_slowlog.name = index.indexing.slowlog.index
+logger.index_indexing_slowlog.level = trace
+logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
+logger.index_indexing_slowlog.additivity = false

templates/systemd/elasticsearch.j2

@@ -22,11 +22,19 @@ ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec
 {% endif %}
 
 ExecStart={{es_home}}/bin/elasticsearch \
+{% if es_version | version_compare('5.0', '<') %}
                                         -Des.pidfile=${PID_DIR}/elasticsearch.pid \
                                         -Des.default.path.home=${ES_HOME} \
                                         -Des.default.path.logs=${LOG_DIR} \
                                         -Des.default.path.data=${DATA_DIR} \
                                         -Des.default.path.conf=${CONF_DIR}
+{% else %}
+                                        -p ${PID_DIR}/elasticsearch.pid \
+                                        -Edefault.path.home=${ES_HOME} \
+                                        -Edefault.path.logs=${LOG_DIR} \
+                                        -Edefault.path.data=${DATA_DIR} \
+                                        -Edefault.path.conf=${CONF_DIR}
+{% endif %}
 
 
 
@@ -37,7 +45,7 @@ StandardError=inherit
 LimitNOFILE={{es_max_open_files}}
 
 # Specifies the maximum number of bytes of memory that may be locked into RAM
-# Set to "infinity" if you use the 'bootstrap.mlockall: true' option
+# Set to "infinity" if you use the 'bootstrap.{% if es_version | version_compare('5.0', '<=') %}memory_lock{% else %}mlockall{% endif %}: true' option
 # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}}
 {% if m_lock_enabled %}
 LimitMEMLOCK=infinity