Merge branch 'lobsterdore-elasticsearch-template-fixes'

handlers/elasticsearch-templates.yml

@@ -6,29 +6,29 @@
 - name: Wait for elasticsearch to startup
   wait_for: host={{es_api_host}} port={{es_api_port}} delay=10
 
-- name: Get template files
-  find: paths="/etc/elasticsearch/templates" patterns="*.json"
-  register: templates
-
 - name: Install templates without auth
   uri:
-    url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item.path | filename}}"
+    url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}"
     method: PUT
     status_code: 200
     body_format: json
-    body: "{{ lookup('file', item.path) }}"
+    body: "{{ lookup('file', item) }}"
   when: not es_enable_xpack or not es_xpack_features is defined or "security" not in es_xpack_features
-  with_items: "{{ templates.files }}"
+  with_fileglob:
+    - "{{ es_templates_fileglob | default('') }}"
+  run_once: True
 
 - name: Install templates with auth
   uri:
-    url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item.path | filename}}"
+    url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}"
     method: PUT
     status_code: 200
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
     body_format: json
-    body: "{{ lookup('file', item.path) }}"
+    body: "{{ lookup('file', item) }}"
   when: es_enable_xpack and es_xpack_features is defined and "security" in es_xpack_features
-  with_items: "{{ templates.files }}"
+  with_fileglob:
+    - "{{ es_templates_fileglob | default('') }}"
+  run_once: True

handlers/main.yml

@@ -1,14 +1,3 @@
-- name: reload systemd configuration
-  command: systemctl daemon-reload
-
-# Restart service and ensure it is enabled
-- name: restart elasticsearch
-  service: name={{instance_init_script | basename}} state=restarted enabled=yes
-  when: 
-    - es_restart_on_change 
-    - es_start_service 
-    - ((plugin_installed is defined and plugin_installed.changed) or (config_updated is defined and config_updated.changed) or (xpack_state.changed) or (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed))
-  register: es_restarted
 
 #Templates are a handler as they need to come after a restart e.g. suppose user removes security on a running node and doesn't
 #specify es_api_basic_auth_username and es_api_basic_auth_password.  The templates will subsequently not be removed if we don't wait for the node to restart.
@@ -16,4 +5,6 @@
 
 - name: load-templates
   include: ./handlers/elasticsearch-templates.yml
+  #only do templates if we're starting
+  when: es_templates and es_start_service
   when: es_templates

tasks/main.yml

@@ -42,19 +42,32 @@
   tags:
       - templates
 
+- name: reload systemd configuration
+  command: systemctl daemon-reload
+
+# Restart service and ensure it is enabled
+- name: restart elasticsearch
+  service: name={{instance_init_script | basename}} state=restarted enabled=yes
+  when:
+    - es_restart_on_change
+    - es_start_service
+    - ((plugin_installed is defined and plugin_installed.changed) or (config_updated is defined and config_updated.changed) or (xpack_state.changed) or (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed))
+  register: es_restarted
+
 - meta: flush_handlers
 
 - name: Make sure elasticsearch is started
   service: name={{instance_init_script | basename}} state=started enabled=yes
+  when: es_start_service
 
 - name: Wait for elasticsearch to startup
   wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1
-  when: es_restarted is defined and es_restarted.changed
+  when: es_restarted is defined and es_restarted.changed and es_start_service
 
 - name: activate-license
   include: ./xpack/security/elasticsearch-xpack-activation.yml
-  when: es_enable_xpack and es_xpack_license is defined and es_xpack_license != ''
+  when: es_start_service and es_enable_xpack and es_xpack_license is defined and es_xpack_license != ''
 
 #perform security actions here now elasticsearch is started
 - include: ./xpack/security/elasticsearch-security-native.yml
-  when: (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
+  when: es_start_service and (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))

templates/elasticsearch.j2

@@ -5,6 +5,9 @@
 # Elasticsearch home directory
 ES_HOME={{es_home}}
 
+# Elasticsearch Java path
+#JAVA_HOME=
+
 # Elasticsearch configuration directory
 CONF_DIR={{conf_dir}}
 
@@ -56,7 +59,7 @@ MAX_OPEN_FILES={{es_max_open_files}}
 
 # The maximum number of bytes of memory that may be locked into RAM
 # Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option
-# in elasticsearch.yml (ES_HEAP_SIZE  must also be set).
+# in elasticsearch.yml
 # When using Systemd, the LimitMEMLOCK property must be set
 # in /usr/lib/systemd/system/elasticsearch.service
 #MAX_LOCKED_MEMORY=

templates/elasticsearch.repo

@@ -4,6 +4,8 @@ baseurl=https://artifacts.elastic.co/packages/{{ es_major_version }}/yum
 gpgcheck=1
 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
 enabled=1
+autorefresh=1
+type=rpm-md
 {% if es_proxy_host is defined and es_proxy_host != '' and es_proxy_port is defined %}
 proxy=http://{{ es_proxy_host }}:{{es_proxy_port}}
 {% endif %}

templates/init/debian/elasticsearch.j2

@@ -84,16 +84,30 @@ if [ ! -z "$CONF_FILE" ]; then
     exit 1
 fi
 
+if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then
+    echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]"
+fi
+
 # Define other required variables
 PID_FILE="$PID_DIR/$NAME.pid"
-DAEMON={{es_home}}/bin/elasticsearch
-DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR"
+DAEMON=$ES_HOME/bin/elasticsearch
+DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR"
 
 export ES_JAVA_OPTS
 export JAVA_HOME
 export ES_INCLUDE
 export ES_JVM_OPTIONS
 
+# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported
+if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi
+if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi
+if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi
+if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi
+if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi
+if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi
+if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi
+if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi
+
 # Check DAEMON exists
 if [ ! -x "$DAEMON" ]; then
         echo "The elasticsearch startup script does not exists or it is not executable, tried: $DAEMON"
@@ -117,13 +131,6 @@ case "$1" in
   start)
 	checkJava
 
-{% if es_version | version_compare('5.0', '<') %}
-	if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then
-		log_failure_msg "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set"
-		exit 1
-	fi
-{% endif %}
-
 	log_daemon_msg "Starting $DESC"
 
 	pid=`pidofproc -p $PID_FILE elasticsearch`
@@ -133,9 +140,6 @@ case "$1" in
 		exit 0
 	fi
 
-	# Prepare environment
-	mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR"
-
 	# Ensure that the PID_DIR exists (it is cleaned at OS startup time)
 	if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then
 		mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR"
@@ -157,7 +161,7 @@ case "$1" in
 	fi
 
 	# Start Daemon
-	start-stop-daemon -d $ES_HOME --start -b --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
+	start-stop-daemon -d $ES_HOME --start --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
 	return=$?
 	if [ $return -eq 0 ]; then
 		i=0
@@ -203,7 +207,6 @@ case "$1" in
   restart|force-reload)
 	if [ -f "$PID_FILE" ]; then
 		$0 stop
-		sleep 1
 	fi
 	$0 start
 	;;

templates/init/redhat/elasticsearch.j2

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # elasticsearch <summary>
 #
@@ -48,7 +48,6 @@ DATA_DIR={{ data_dirs | array_to_str }}
 CONF_DIR="{{conf_dir}}"
 
 PID_DIR="{{pid_dir}}"
-ES_JVM_OPTIONS="{{conf_dir}}/jvm.options"
 
 # Source the default env file
 ES_ENV_FILE="{{instance_default_file}}"
@@ -56,6 +55,10 @@ if [ -f "$ES_ENV_FILE" ]; then
     . "$ES_ENV_FILE"
 fi
 
+if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then
+    echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]"
+fi
+
 # CONF_FILE setting was removed
 if [ ! -z "$CONF_FILE" ]; then
     echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."

templates/jvm.options.j2

@@ -20,13 +20,9 @@
 # Xmx represents the maximum size of total heap space
 {% if es_heap_size is defined %}
 -Xms{{ es_heap_size }}
-{% else %}
--Xms2g
-{% endif %}
-
-{% if es_heap_size is defined %}
 -Xmx{{ es_heap_size }}
 {% else %}
+-Xms2g
 -Xmx2g
 {% endif %}
 
@@ -47,9 +43,6 @@
 
 ## optimizations
 
-# disable calls to System#gc
--XX:+DisableExplicitGC
-
 # pre-touch memory pages used by the JVM during initialization
 -XX:+AlwaysPreTouch
 
@@ -67,7 +60,10 @@
 # use our provided JNA always versus the system one
 -Djna.nosys=true
 
-# flags to keep Netty from being unsafe
+# use old-style file permissions on JDK9
+-Djdk.io.permissionsUseCanonicalPath=true
+
+# flags to configure Netty
 -Dio.netty.noUnsafe=true
 -Dio.netty.noKeySetOptimization=true
 -Dio.netty.recycler.maxCapacityPerThread=0
@@ -100,6 +96,14 @@
 # ensure the directory exists
 #-Xloggc:${loggc}
 
+
+# By default, the GC log file will not rotate.
+# By uncommenting the lines below, the GC log file
+# will be rotated every 128MB at most 32 times.
+#-XX:+UseGCLogFileRotation
+#-XX:NumberOfGCLogFiles=32
+#-XX:GCLogFileSize=128M
+
 # Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
 # If documents were already indexed with unquoted fields in a previous version
 # of Elasticsearch, some operations may throw errors.

templates/systemd/elasticsearch.j2

@@ -41,6 +41,9 @@ StandardError=inherit
 LimitNOFILE={{es_max_open_files}}
 {% endif %}
 
+# Specifies the maximum number of processes
+LimitNPROC=2048
+
 # Specifies the maximum number of bytes of memory that may be locked into RAM
 # Set to "infinity" if you use the 'bootstrap.memory_lock: true' option
 # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}}
@@ -54,6 +57,9 @@ TimeoutStopSec=0
 # SIGTERM signal is used to stop the Java process
 KillSignal=SIGTERM
 
+# Send the signal only to the JVM rather than its control group
+KillMode=process
+
 # Java process is never killed
 SendSIGKILL=no
 

test/integration/config-5x/serverspec/default_spec.rb

@@ -1,6 +1,6 @@
 require 'config_spec'
 
 describe 'Config Tests v 5.x' do
-  include_examples 'config::init', "5.2.2", ["ingest-attachment","ingest-user-agent"]
+  include_examples 'config::init', "5.5.1", ["ingest-attachment","ingest-user-agent"]
 end
 

test/integration/multi-5x/serverspec/default_spec.rb

@@ -2,7 +2,7 @@ require 'multi_spec'
 
 
 describe 'Multi Tests v 5.x' do
-  include_examples 'multi::init', "5.2.2", ["ingest-geoip"]
+  include_examples 'multi::init', "5.5.1", ["ingest-geoip"]
 end
 
 

test/integration/package-5x/serverspec/default_spec.rb

@@ -2,5 +2,5 @@ require 'package_spec'
 
 
 describe 'Package Tests v 5.x' do
-  include_examples 'package::init', "5.2.2", ["ingest-attachment","ingest-geoip"]
+  include_examples 'package::init', "5.5.1", ["ingest-attachment","ingest-geoip"]
 end

test/integration/standard-5x/serverspec/default_spec.rb

@@ -2,7 +2,7 @@ require 'standard_spec'
 
 
 describe 'Standard Tests v 5.x' do
-  include_examples 'standard::init', "5.2.2", ["ingest-geoip"]
+  include_examples 'standard::init', "5.5.1", ["ingest-geoip"]
 end
 
 

test/integration/xpack-5x/serverspec/default_spec.rb

@@ -1,5 +1,5 @@
 require 'xpack_spec'
 
 describe 'Xpack Tests v 5.x' do
-  include_examples 'xpack::init', "5.2.2", ["ingest-attachment"]
+  include_examples 'xpack::init', "5.5.1", ["ingest-attachment"]
 end