Merge pull request #412 from elastic/6.x_testing

6.x testing -  Update tests to work with dynamic es_versions

.kitchen.yml

@@ -39,7 +39,9 @@ platforms:
         - pip install jmespath
         - pip uninstall -y ansible
       use_sudo: false
-      volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+      volume: 
+        - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+        - /etc/elasticsearch
   - name: ubuntu-16.04
     driver_config:
       image: dliappis/ubuntu-devopsci:16.04
@@ -51,7 +53,9 @@ platforms:
         - pip install jmespath
         - pip uninstall -y ansible
       use_sudo: false
-      volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+      volume: 
+        - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+        - /etc/elasticsearch
       run_command: "/sbin/init"
   - name: debian-8
     driver_config:
@@ -65,7 +69,9 @@ platforms:
         - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config
         - pip install jmespath
         - pip uninstall -y ansible
-      volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+      volume: 
+        - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+        - /etc/elasticsearch
       use_sudo: false
       run_command: "/sbin/init"
   - name: centos-7
@@ -80,7 +86,9 @@ platforms:
         - yum -y remove ansible
         - yum clean all
         - pip install jmespath
-      volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+      volume: 
+        - <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
+        - /etc/elasticsearch
       run_command: "/usr/sbin/init"
       privileged: true
       use_sudo: false

Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-gem 'test-kitchen', '1.17.0'
+gem 'test-kitchen', '1.20.0'
 gem 'kitchen-docker', '2.6.0'
-gem 'kitchen-ansible', '0.47.3'
+gem 'kitchen-ansible', '0.48.1'
-gem 'net-ssh', '4.1.0'
+gem 'net-ssh', '4.2.0'

Gemfile.lock

@@ -1,44 +1,75 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    artifactory (2.8.2)
+    builder (3.2.3)
-    kitchen-ansible (0.47.3)
+    erubis (2.7.0)
-      mixlib-shellout (<= 2.2.7)
+    ffi (1.9.18)
+    gssapi (1.2.0)
+      ffi (>= 1.0.1)
+    gyoku (1.3.1)
+      builder (>= 2.1.2)
+    httpclient (2.8.3)
+    kitchen-ansible (0.48.1)
       net-ssh (>= 3)
       test-kitchen (~> 1.4)
     kitchen-docker (2.6.0)
       test-kitchen (>= 1.0.0)
-    mixlib-install (2.1.12)
+    little-plugger (1.1.4)
-      artifactory
+    logging (2.2.2)
+      little-plugger (~> 1.1)
+      multi_json (~> 1.10)
+    mixlib-install (3.9.0)
       mixlib-shellout
       mixlib-versioning
       thor
-    mixlib-shellout (2.2.7)
+    mixlib-shellout (2.3.2)
     mixlib-versioning (1.2.2)
+    multi_json (1.13.1)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
-    net-ssh (4.1.0)
+    net-ssh (4.2.0)
     net-ssh-gateway (1.3.0)
       net-ssh (>= 2.6.5)
-    safe_yaml (1.0.4)
+    nori (2.6.0)
-    test-kitchen (1.17.0)
+    rubyntlm (0.6.2)
-      mixlib-install (>= 1.2, < 3.0)
+    rubyzip (1.2.1)
+    test-kitchen (1.20.0)
+      mixlib-install (~> 3.6)
       mixlib-shellout (>= 1.2, < 3.0)
       net-scp (~> 1.1)
       net-ssh (>= 2.9, < 5.0)
       net-ssh-gateway (~> 1.2)
-      safe_yaml (~> 1.0)
       thor (~> 0.19, < 0.19.2)
+      winrm (~> 2.0)
+      winrm-elevated (~> 1.0)
+      winrm-fs (~> 1.1.0)
     thor (0.19.1)
+    winrm (2.2.3)
+      builder (>= 2.1.2)
+      erubis (~> 2.7)
+      gssapi (~> 1.2)
+      gyoku (~> 1.0)
+      httpclient (~> 2.2, >= 2.2.0.2)
+      logging (>= 1.6.1, < 3.0)
+      nori (~> 2.0)
+      rubyntlm (~> 0.6.0, >= 0.6.1)
+    winrm-elevated (1.1.0)
+      winrm (~> 2.0)
+      winrm-fs (~> 1.0)
+    winrm-fs (1.1.1)
+      erubis (~> 2.7)
+      logging (>= 1.6.1, < 3.0)
+      rubyzip (~> 1.1)
+      winrm (~> 2.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  kitchen-ansible (= 0.47.3)
+  kitchen-ansible (= 0.48.1)
   kitchen-docker (= 2.6.0)
-  net-ssh (= 4.1.0)
+  net-ssh (= 4.2.0)
-  test-kitchen (= 1.17.0)
+  test-kitchen (= 1.20.0)
 
 BUNDLED WITH
-   1.15.3
+   1.16.1

README.md

@@ -1,7 +1,7 @@
 # ansible-elasticsearch
 [![Ansible Galaxy](https://img.shields.io/badge/ansible--galaxy-elastic.elasticsearch-blue.svg)](https://galaxy.ansible.com/elastic/elasticsearch/)
 
-**THIS ROLE IS FOR 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.**
+**THIS ROLE IS FOR 6.x, 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.**
 
 Ansible role for 5.x Elasticsearch.  Currently this works on Debian and RedHat based linux systems.  Tested platforms are:
 
@@ -9,7 +9,7 @@ Ansible role for 5.x Elasticsearch.  Currently this works on Debian and RedHat b
 * Debian 8
 * Centos 7
 
-The latest Elasticsearch versions of 5.x are actively tested.  **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.**
+The latest Elasticsearch versions of 6.x are actively tested.  **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.**
 
 ##### Dependency
 This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine.
@@ -329,7 +329,7 @@ These can either be set to a user declared in the file based realm, with admin p
 
 In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behaviour. Options include:
 
-* ```es_major_version```  Should be consistent with es_version. For versions >= 5.0 this must be "5.x".
+* ```es_major_version```  Should be consistent with es_version. For versions >= 5.0 and < 6.0 this must be "5.x". For versions >= 6.0 this must be "6.x".
 * ```es_version``` (e.g. "5.1.2").  
 * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost".
 * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200**
@@ -396,7 +396,7 @@ To define proxy only for a particular plugin during its installation:
 * The role assumes the user/group exists on the server.  The elasticsearch packages create the default elasticsearch user.  If this needs to be changed, ensure the user exists.
 * The playbook relies on the inventory_name of each host to ensure its directories are unique
 * Changing an instance_name for a role application will result in the installation of a new component.  The previous component will remain.
-* KitchenCI has been used for testing.  This is used to confirm images reach the correct state after a play is first applied.  We currently test only the latest version of 5.x on
+* KitchenCI has been used for testing.  This is used to confirm images reach the correct state after a play is first applied.  We currently test only the latest version of 6.x on
 all supported platforms. 
 * The role aims to be idempotent.  Running the role multiple times, with no changes, should result in no state change on the server.  If the configuration is changed, these will be applied and 
 Elasticsearch restarted where required.
@@ -405,7 +405,7 @@ Elasticsearch restarted where required.
 
 ## IMPORTANT NOTES RE PLUGIN MANAGEMENT
 
-* If the ES version is changed, all plugins will be removed.  Those listed in the playbook will be re-installed.  This is behaviour is required in ES 5.x.
+* If the ES version is changed, all plugins will be removed.  Those listed in the playbook will be re-installed.  This is behaviour is required in ES 6.x.
 * If no plugins are listed in the playbook for a node, all currently installed plugins will be removed.
 * The role supports automatic detection of differences between installed and listed plugins - installing those listed but not installed, and removing those installed but not listed.   Should users wish to re-install plugins they should set es_plugins_reinstall to true.  This will cause all currently installed plugins to be removed and those listed to be installed.
 

defaults/main.yml

@@ -23,7 +23,7 @@ es_pid_dir: "/var/run/elasticsearch"
 es_data_dirs: "/var/lib/elasticsearch"
 es_log_dir: "/var/log/elasticsearch"
 es_max_open_files: 65536
-es_max_threads: 2048
+es_max_threads: "{{ 2048 if ( es_version | version_compare('6.0.0', '<')) else 8192 }}"
 es_max_map_count: 262144
 es_allow_downgrades: false
 es_enable_xpack: false

tasks/elasticsearch-plugins.yml

@@ -19,6 +19,7 @@
   ignore_errors: yes
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 #if es_plugins_reinstall is set to true we remove ALL plugins
@@ -47,6 +48,7 @@
   register: plugin_removed
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 - name: Install elasticsearch plugins
@@ -60,6 +62,7 @@
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
     ES_JAVA_OPTS: "{% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -Dhttp.proxyHost={{ item.proxy_host }} -Dhttp.proxyPort={{ item.proxy_port }} -Dhttps.proxyHost={{ item.proxy_host }} -Dhttps.proxyPort={{ item.proxy_port }}  {% elif es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }} {% endif %}"
   until: plugin_installed.rc == 0

tasks/java.yml

@@ -10,6 +10,20 @@
   yum: name={{ java }} state={{java_state}}
   when: ansible_os_family == 'RedHat'
 
+- name: Get the installed java path
+  shell: "update-alternatives --display java | grep '^/' | awk '{print $1}' | grep 1.8.0"
+  register: java_full_path
+  failed_when: False
+  changed_when: False
+  when: ansible_os_family == 'RedHat'
+
+- name: correct java version selected
+  alternatives:
+    name: java
+    path: "{{ java_full_path.stdout }}"
+    link: /usr/bin/java
+  when: ansible_os_family == 'RedHat' and java_full_path is defined
+
 - name: Refresh java repo
   become: yes
   apt: update_cache=yes

tasks/xpack/elasticsearch-xpack-install.yml

@@ -10,6 +10,7 @@
   ignore_errors: yes
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 
@@ -24,6 +25,7 @@
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 
@@ -42,6 +44,7 @@
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 - name: Delete x-pack zip file
@@ -59,5 +62,6 @@
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
     ES_JAVA_OPTS: "{% if es_proxy_host is defined and es_proxy_host != '' %}-Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }}{% endif %}"

tasks/xpack/elasticsearch-xpack.yml

@@ -11,3 +11,9 @@
 - name: Set Plugin Directory Permissions
   become: yes
   file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes
+
+#Make sure elasticsearch.keystore has correct Permissions
+- name: Set elasticsearch.keystore Permissions
+  become: yes
+  file: state=file path={{ conf_dir }}/elasticsearch.keystore owner={{ es_user }} group={{ es_group }}
+  when: es_enable_xpack and "security" in es_xpack_features and (es_version | version_compare('6.0.0', '>'))

tasks/xpack/security/elasticsearch-security-file.yml

@@ -21,6 +21,7 @@
   when: manage_file_users
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
 - set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }}
@@ -36,6 +37,7 @@
   no_log: True
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
 #Set passwords for all users declared - Required as the useradd will not change existing user passwords
@@ -50,6 +52,7 @@
   no_log: True
   environment:
     CONF_DIR: "{{ conf_dir }}"
+    ES_PATH_CONF: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
 - set_fact: users_roles={{es_users.file | extract_role_users () }}

tasks/xpack/security/elasticsearch-security.yml

@@ -11,6 +11,25 @@
     - es_enable_xpack and '"security" in es_xpack_features'
     - (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined)
 
+#-----------------------------Create Bootstrap User-----------------------------------
+- name: Check if bootstrap password is set
+  command: >
+   {{es_home}}/bin/elasticsearch-keystore list
+  register: list_keystore
+  changed_when: False
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"
+  when:
+    - (es_enable_xpack and '"security" in es_xpack_features') and (es_version | version_compare('6.0.0', '>')) 
+
+- name: Create Bootstrap password for elastic user
+  shell: echo "{{es_api_basic_auth_password}}" | {{es_home}}/bin/elasticsearch-keystore add -x 'bootstrap.password'
+  when:
+    - (es_enable_xpack and '"security" in es_xpack_features') and (es_version | version_compare('6.0.0', '>')) and es_api_basic_auth_username is defined and list_keystore is defined and es_api_basic_auth_username == 'elastic' and 'bootstrap.password' not in list_keystore.stdout_lines 
+  environment:
+    ES_PATH_CONF: "{{ conf_dir }}"
+  no_log: true
+
 #-----------------------------FILE BASED REALM----------------------------------------
 
 - include: elasticsearch-security-file.yml

templates/elasticsearch.j2

@@ -10,6 +10,7 @@ ES_HOME={{es_home}}
 
 # Elasticsearch configuration directory
 CONF_DIR={{conf_dir}}
+ES_PATH_CONF={{conf_dir}}
 
 # Elasticsearch data directory
 DATA_DIR={{ data_dirs | array_to_str }}
@@ -74,3 +75,9 @@ MAX_LOCKED_MEMORY=unlimited
 {% if es_max_map_count is defined %}
 MAX_MAP_COUNT={{es_max_map_count}}
 {% endif %}
+
+# Specifies the maximum number of threads that can be started.
+# Elasticsearch requires a minimum of 2048.
+{% if es_max_threads is defined %}
+MAX_THREADS={{ es_max_threads }}
+{% endif %}

templates/elasticsearch.yml.j2

@@ -14,7 +14,10 @@ node.name: {{inventory_hostname}}-{{es_instance_name}}
 #################################### Paths ####################################
 
 # Path to directory containing configuration (this file and logging.yml):
+
+{% if (es_version | version_compare('6.0.0', '<')) %}
 path.conf: {{ conf_dir }}
+{% endif %}
 
 path.data: {{ data_dirs | array_to_str }}
 

templates/init/debian/elasticsearch.j2

@@ -60,6 +60,7 @@ DATA_DIR={{ data_dirs | array_to_str }}
 
 # Elasticsearch configuration directory
 CONF_DIR={{conf_dir}}
+ES_PATH_CONF={{ conf_dir }}
 
 # Maximum number of VMA (Virtual Memory Areas) a process can own
 {% if es_max_map_count is defined %}
@@ -91,12 +92,17 @@ fi
 # Define other required variables
 PID_FILE="$PID_DIR/$NAME.pid"
 DAEMON=$ES_HOME/bin/elasticsearch
+{% if (es_version | version_compare('6.0.0', '<')) %}
 DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR"
+{% else %}
+DAEMON_OPTS="-d -p $PID_FILE"
+{% endif %}
 
 export ES_JAVA_OPTS
 export JAVA_HOME
 export ES_INCLUDE
 export ES_JVM_OPTIONS
+export ES_PATH_CONF
 
 # export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported
 if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi
@@ -156,6 +162,10 @@ case "$1" in
 		ulimit -l $MAX_LOCKED_MEMORY
 	fi
 
+	if [ -n "$MAX_THREADS" ]; then
+		ulimit -u $MAX_THREADS
+	fi
+
 	if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then
 		sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
 	fi

templates/init/redhat/elasticsearch.j2

@@ -46,6 +46,7 @@ MAX_MAP_COUNT={{es_max_map_count}}
 LOG_DIR="{{log_dir}}"
 DATA_DIR={{ data_dirs | array_to_str }}
 CONF_DIR="{{conf_dir}}"
+ES_PATH_CONF="{{ conf_dir }}"
 
 PID_DIR="{{pid_dir}}"
 
@@ -74,6 +75,7 @@ export JAVA_HOME
 export ES_INCLUDE
 export ES_JVM_OPTIONS
 export ES_STARTUP_SLEEP_TIME
+export ES_PATH_CONF
 
 # export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported
 if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi
@@ -120,6 +122,9 @@ start() {
     if [ -n "$MAX_LOCKED_MEMORY" ]; then
         ulimit -l $MAX_LOCKED_MEMORY
     fi
+    if [ -n "$MAX_THREADS" ]; then
+        ulimit -u $MAX_THREADS
+    fi
     if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then
         sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
     fi
@@ -135,7 +140,11 @@ start() {
     cd $ES_HOME
     echo -n $"Starting $prog: "
     # if not running, start it up here, usually something like "daemon $exec"
+{% if (es_version | version_compare('6.0.0', '<')) %}
     daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR
+{% else %}
+    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d
+{% endif %}
     retval=$?
     echo
     [ $retval -eq 0 ] && touch $lockfile

templates/log4j2.properties.j2

@@ -11,25 +11,52 @@ appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
 
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
+{% if (es_version | version_compare('6.0.0', '<')) %}
 appender.rolling.fileName = ${sys:es.logs}.log
+{% else %}
+appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
+{% endif %}
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+{% if (es_version | version_compare('6.0.0', '<')) %}
 appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log
+{% else %}
+appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
+{% endif %}
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.rolling.policies.time.interval = 1
 appender.rolling.policies.time.modulate = true
-
+{% if (es_version | version_compare('6.0.0', '>')) %}
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size = 128MB
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.fileIndex = nomax
+appender.rolling.strategy.action.type = Delete
+appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path}
+appender.rolling.strategy.action.condition.type = IfFileName
+appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
+appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize
+appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB
+{% endif %}
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
 rootLogger.appenderRef.rolling.ref = rolling
 
 appender.deprecation_rolling.type = RollingFile
 appender.deprecation_rolling.name = deprecation_rolling
+{% if (es_version | version_compare('6.0.0', '<')) %}
 appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log
+{% else %}
+appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
+{% endif %}
 appender.deprecation_rolling.layout.type = PatternLayout
 appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
+{% if (es_version | version_compare('6.0.0', '<')) %}
 appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz
+{% else %}
+appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
+{% endif %}
 appender.deprecation_rolling.policies.type = Policies
 appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
 appender.deprecation_rolling.policies.size.size = 1GB
@@ -43,10 +70,18 @@ logger.deprecation.additivity = false
 
 appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
+{% if (es_version | version_compare('6.0.0', '<')) %}
 appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log
+{% else %}
+appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
+{% endif %}
 appender.index_search_slowlog_rolling.layout.type = PatternLayout
 appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+{% if (es_version | version_compare('6.0.0', '<')) %}
 appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
+{% else %}
+appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
+{% endif %}
 appender.index_search_slowlog_rolling.policies.type = Policies
 appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.index_search_slowlog_rolling.policies.time.interval = 1
@@ -59,10 +94,18 @@ logger.index_search_slowlog_rolling.additivity = false
 
 appender.index_indexing_slowlog_rolling.type = RollingFile
 appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
+{% if (es_version | version_compare('6.0.0', '<')) %}
 appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log
+{% else %}
+appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
+{% endif %}
 appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
 appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
+{% if (es_version | version_compare('6.0.0', '<')) %}
 appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
+{% else %}
+appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
+{% endif %}
 appender.index_indexing_slowlog_rolling.policies.type = Policies
 appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.index_indexing_slowlog_rolling.policies.time.interval = 1

templates/systemd/elasticsearch.j2

@@ -7,6 +7,7 @@ After=network-online.target
 [Service]
 Environment=ES_HOME={{es_home}}
 Environment=CONF_DIR={{conf_dir}}
+Environment=ES_PATH_CONF={{conf_dir}}
 Environment=DATA_DIR={{ data_dirs | array_to_str }}
 Environment=LOG_DIR={{log_dir}}
 Environment=PID_DIR={{pid_dir}}
@@ -17,14 +18,18 @@ WorkingDirectory={{es_home}}
 User={{es_user}}
 Group={{es_group}}
 
+{% if (es_version | version_compare('6.0.0', '<')) %}
 ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec
+{% endif %}
 
 ExecStart={{es_home}}/bin/elasticsearch \
                                     -p ${PID_DIR}/elasticsearch.pid \
-                                    --quiet \
+{% if (es_version | version_compare('6.0.0', '<')) %}
                                     -Edefault.path.logs=${LOG_DIR} \
                                     -Edefault.path.data=${DATA_DIR} \
-                                    -Edefault.path.conf=${CONF_DIR}
+                                    -Edefault.path.conf=${CONF_DIR} \
+{% endif %}
+                                    --quiet
 
 
 # StandardOutput is configured to redirect to journalctl since
@@ -41,9 +46,6 @@ StandardError=inherit
 LimitNOFILE={{es_max_open_files}}
 {% endif %}
 
-# Specifies the maximum number of processes
-LimitNPROC=2048
-
 # Specifies the maximum number of bytes of memory that may be locked into RAM
 # Set to "infinity" if you use the 'bootstrap.memory_lock: true' option
 # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}}

test/integration/config.yml

@@ -2,6 +2,8 @@
 #Test explicit setting of parameters and variables
 - name: Elasticsearch Config initial
   hosts: localhost
+  tasks:
+    - include: elasticsearch/test/integration/debug.yml
   roles:
     #expand to all available parameters
     - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: false } }
@@ -17,6 +19,8 @@
 #Modify the above configuration. Final test should evaluate this configuration. Also tests the plugins are added and removed.
 - name: Elasticsearch Config test modify
   hosts: localhost
+  tasks:
+    - include: elasticsearch/test/integration/debug.yml
   roles:
     #expand to all available parameters
     - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9501", http.port: 9401, transport.tcp.port: 9501, node.data: true, node.master: true, bootstrap.memory_lock: true } }

test/integration/config/serverspec/default_spec.rb

@@ -1,6 +1,8 @@
 require 'config_spec'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
 
-describe 'Config Tests v 5.x' do
+describe 'Config Tests' do
-  include_examples 'config::init', "5.5.1", ["ingest-attachment","ingest-user-agent"]
+  include_examples 'config::init', vars
 end
 

test/integration/debug.yml

@@ -0,0 +1,6 @@
+---
+- name: Dump all variables to a file
+  changed_when: False
+  copy:
+    content: '{{ vars | to_nice_json }} '
+    dest: '/tmp/vars.json'

test/integration/helpers/serverspec/config_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-shared_examples 'config::init' do |es_version,plugins|
+shared_examples 'config::init' do |vars|
 
   describe user('elasticsearch') do
     it { should exist }
@@ -36,7 +36,11 @@ shared_examples 'config::init' do |es_version,plugins|
     it { should contain 'node.name: node1' }
     it { should contain 'bootstrap.memory_lock: true' }
     it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9501' }
+    if vars['es_major_version'] == '6.x'
+      it { should_not contain 'path.conf: /etc/elasticsearch/node1' }
+    else
       it { should contain 'path.conf: /etc/elasticsearch/node1' }
+    end
     it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' }
     it { should contain 'path.logs: /opt/elasticsearch/logs/localhost-node1' }
   end
@@ -77,20 +81,21 @@ shared_examples 'config::init' do |es_version,plugins|
 
 
   describe 'version check' do
-    it 'should be reported as version '+es_version do
+    it 'should be reported as version '+vars['es_version'] do
       command = command('curl -s localhost:9401 | grep number')
-      expect(command.stdout).to match(es_version)
+      expect(command.stdout).to match(vars['es_version'])
       expect(command.exit_status).to eq(0)
     end
   end
 
-  for plugin in plugins
+  for plugin in vars['es_plugins']
+    plugin = plugin['plugin']
     describe file('/usr/share/elasticsearch/plugins/'+plugin) do
       it { should be_directory }
       it { should be_owned_by 'elasticsearch' }
     end
     #confirm plugins are installed and the correct version
-    describe command('curl -s localhost:9401/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
+    describe command('curl -s localhost:9401/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do
       its(:exit_status) { should eq 0 }
     end
   end
@@ -100,7 +105,7 @@ shared_examples 'config::init' do |es_version,plugins|
     it { should_not exist }
   end
   #confirm plugins are installed and the correct version
-  describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"ingest-geoip","version":"'+es_version+'"\'') do
+  describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"ingest-geoip","version":"'+vars['es_version']+'"\'') do
     its(:exit_status) { should eq 1 }
   end
 

test/integration/helpers/serverspec/issue_test_spec.rb

@@ -1,6 +1,8 @@
 require 'spec_helper'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
 
-shared_examples 'issue_test::init' do |es_version,plugins|
+shared_examples 'issue_test::init' do |vars|
 
   #Add custom tests here for the issue-test.yml test
 

test/integration/helpers/serverspec/multi_spec.rb

@@ -1,6 +1,8 @@
 require 'spec_helper'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
 
-shared_examples 'multi::init' do  |es_version,plugins|
+shared_examples 'multi::init' do |vars|
 
   describe user('elasticsearch') do
     it { should exist }
@@ -27,7 +29,11 @@ shared_examples 'multi::init' do  |es_version,plugins|
     it { should contain 'node.master: false' }
     it { should contain 'node.name: localhost-node1' }
     it { should_not contain 'bootstrap.memory_lock: true' }
+    if vars['es_major_version'] == '6.x'
+      it { should_not contain 'path.conf: /etc/elasticsearch/node1' }
+    else
       it { should contain 'path.conf: /etc/elasticsearch/node1' }
+    end
     it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' }
     it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' }
   end
@@ -42,7 +48,11 @@ shared_examples 'multi::init' do  |es_version,plugins|
     it { should contain 'node.master: true' }
     it { should contain 'node.name: localhost-master' }
     it { should contain 'bootstrap.memory_lock: true' }
+    if vars['es_major_version'] == '6.x'
+      it { should_not contain 'path.conf: /etc/elasticsearch/master' }
+    else
       it { should contain 'path.conf: /etc/elasticsearch/master' }
+    end
     it { should contain 'path.data: /opt/elasticsearch/master/localhost-master' }
     it { should contain 'path.logs: /var/log/elasticsearch/localhost-master' }
   end
@@ -154,22 +164,23 @@ shared_examples 'multi::init' do  |es_version,plugins|
   end
 
   describe 'version check on master' do
-    it 'should be reported as version '+es_version do
+    it 'should be reported as version '+vars['es_version'] do
       command = command('curl -s localhost:9200 | grep number')
-      expect(command.stdout).to match(es_version)
+      expect(command.stdout).to match(vars['es_version'])
       expect(command.exit_status).to eq(0)
     end
   end
 
   describe 'version check on data' do
-    it 'should be reported as version '+es_version do
+    it 'should be reported as version '+vars['es_version'] do
       command = command('curl -s localhost:9201 | grep number')
-      expect(command.stdout).to match(es_version)
+      expect(command.stdout).to match(vars['es_version'])
       expect(command.exit_status).to eq(0)
     end
   end
 
-  for plugin in plugins
+  for plugin in vars['es_plugins']
+    plugin = plugin['plugin']
 
     describe command('curl -s localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do
       its(:exit_status) { should eq 0 }

test/integration/helpers/serverspec/package_spec.rb

@@ -1,6 +1,8 @@
 require 'spec_helper'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
 
-shared_examples 'package::init' do  |es_version,plugins|
+shared_examples 'package::init' do |vars|
 
   describe user('elasticsearch') do
     it { should exist }
@@ -56,9 +58,9 @@ shared_examples 'package::init' do  |es_version,plugins|
   end
 
   describe 'version check' do
-    it 'should be reported as version '+es_version do
+    it 'should be reported as version '+vars['es_version'] do
       command = command('curl -s localhost:9200 | grep number')
-      expect(command.stdout).to match(es_version)
+      expect(command.stdout).to match(vars['es_version'])
       expect(command.exit_status).to eq(0)
     end
   end
@@ -69,13 +71,14 @@ shared_examples 'package::init' do  |es_version,plugins|
   end
 
 
-  for plugin in plugins
+  for plugin in vars['es_plugins']
+    plugin = plugin['plugin']
     describe file('/usr/share/elasticsearch/plugins/'+plugin) do
       it { should be_directory }
       it { should be_owned_by 'elasticsearch' }
     end
     #confirm plugins are installed and the correct version
-    describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
+    describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do
       its(:exit_status) { should eq 0 }
     end
   end

test/integration/helpers/serverspec/spec_helper.rb

@@ -1,4 +1,7 @@
 require 'serverspec'
+require 'net/http'
+require 'json'
+
 set :backend, :exec
 
 require 'rspec/retry'
@@ -9,3 +12,15 @@ RSpec.configure do |config|
   # show exception that triggers a retry if verbose_retry is set to true
   config.display_try_failure_messages = true
 end
+
+def curl_json(uri, username=nil, password=nil)
+  uri = URI(uri)
+  req =  Net::HTTP::Get.new(uri)
+  if username && password
+    req.basic_auth username, password
+  end
+  res = Net::HTTP.start(uri.hostname, uri.port) {|http|
+    http.request(req)
+  }
+  return JSON.parse(res.body)
+end

test/integration/helpers/serverspec/standard_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-shared_examples 'standard::init' do |es_version,plugins|
+shared_examples 'standard::init' do |vars|
 
   describe user('elasticsearch') do
     it { should exist }
@@ -33,7 +33,11 @@ shared_examples 'standard::init' do |es_version,plugins|
   describe file('/etc/elasticsearch/node1/elasticsearch.yml') do
     it { should contain 'node.name: localhost-node1' }
     it { should contain 'cluster.name: elasticsearch' }
+    if vars['es_major_version'] == '6.x'
+      it { should_not contain 'path.conf: /etc/elasticsearch/node1' }
+    else
       it { should contain 'path.conf: /etc/elasticsearch/node1' }
+    end
     it { should contain 'path.data: /var/lib/elasticsearch/localhost-node1' }
     it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' }
   end
@@ -45,9 +49,9 @@ shared_examples 'standard::init' do |es_version,plugins|
   end
 
   describe 'version check' do
-    it 'should be reported as version '+es_version do
+    it 'should be reported as version '+vars['es_version'] do
       command = command('curl -s localhost:9200 | grep number')
-      expect(command.stdout).to match(es_version)
+      expect(command.stdout).to match(vars['es_version'])
       expect(command.exit_status).to eq(0)
     end
   end
@@ -76,13 +80,15 @@ shared_examples 'standard::init' do |es_version,plugins|
     it { should_not exist }
   end
 
-  for plugin in plugins
+  for plugin in vars['es_plugins']
+    plugin = plugin['plugin']
+
     describe file('/usr/share/elasticsearch/plugins/'+plugin) do
       it { should be_directory }
       it { should be_owned_by 'elasticsearch' }
     end
     #confirm plugins are installed and the correct version
-    describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
+    describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do
       its(:exit_status) { should eq 0 }
     end
   end

test/integration/helpers/serverspec/xpack_spec.rb

@@ -1,6 +1,8 @@
 require 'spec_helper'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
 
-shared_examples 'xpack::init' do |es_version,plugins|
+shared_examples 'xpack::init' do |vars|
 
   describe user('elasticsearch') do
     it { should exist }
@@ -27,7 +29,11 @@ shared_examples 'xpack::init' do |es_version,plugins|
   describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
     it { should contain 'node.name: localhost-security_node' }
     it { should contain 'cluster.name: elasticsearch' }
+    if vars['es_major_version'] == '6.x'
+      it { should_not contain 'path.conf: /etc/elasticsearch/security_node' }
+    else
       it { should contain 'path.conf: /etc/elasticsearch/security_node' }
+    end
     it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' }
     it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' }
   end
@@ -39,9 +45,9 @@ shared_examples 'xpack::init' do |es_version,plugins|
   end
 
   describe 'version check' do
-    it 'should be reported as version '+es_version do
+    it 'should be reported as version '+vars['es_version'] do
       command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number')
-      expect(command.stdout).to match(es_version)
+      expect(command.stdout).to match(vars['es_version'])
       expect(command.exit_status).to eq(0)
     end
   end
@@ -104,13 +110,15 @@ shared_examples 'xpack::init' do |es_version,plugins|
     it { should be_owned_by 'elasticsearch' }
   end
 
-  for plugin in plugins
+  for plugin in vars['es_plugins']
+    plugin = plugin['plugin']
+
     describe file('/usr/share/elasticsearch/plugins/'+plugin) do
       it { should be_directory }
       it { should be_owned_by 'elasticsearch' }
     end
 
-    describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
+    describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do
       its(:exit_status) { should eq 0 }
     end
   end
@@ -141,8 +149,11 @@ shared_examples 'xpack::init' do |es_version,plugins|
     its(:exit_status) { should eq 0 }
   end
 
-  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMeAgain | md5sum | grep 44b97844bd8b31d5573493a99ef62106') do
+  describe 'security roles' do
-    its(:exit_status) { should eq 0 }
+    it 'should list the security roles' do
+      roles = curl_json('http://localhost:9200/_xpack/security/role', username='es_admin', password='changeMeAgain')
+      expect(roles.key?('superuser'))
+    end
   end
 
   describe file('/etc/elasticsearch/templates') do
@@ -166,8 +177,10 @@ shared_examples 'xpack::init' do |es_version,plugins|
   #This is possibly subject to format changes in the response across versions so may fail in the future
   describe 'Template Contents Correct' do
     it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
-      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain | md5sum')
+      template = curl_json('http://localhost:9200/_template/basic', username='es_admin', password='changeMeAgain')
-      expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/)
+      expect(template.key?('basic'))
+      expect(template['basic']['settings']['index']['number_of_shards']).to eq("1")
+      expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false)
     end
   end
 
@@ -203,9 +216,9 @@ shared_examples 'xpack::init' do |es_version,plugins|
   #check accounts are correct i.e. we can auth and they have the correct roles
 
   describe 'kibana4_server access check' do
-    it 'should be reported as version '+es_version do
+    it 'should be reported as version '+vars['es_version'] do
       command = command('curl -s localhost:9200/ -u kibana4_server:changeMe | grep number')
-      expect(command.stdout).to match(es_version)
+      expect(command.stdout).to match(vars['es_version'])
       expect(command.exit_status).to eq(0)
     end
   end
@@ -216,9 +229,9 @@ shared_examples 'xpack::init' do |es_version,plugins|
 
 
   describe 'logstash_system access check' do
-    it 'should be reported as version '+es_version do
+    it 'should be reported as version '+vars['es_version'] do
       command = command('curl -s localhost:9200/ -u logstash_system:aNewLogstashPassword | grep number')
-      expect(command.stdout).to match(es_version)
+      expect(command.stdout).to match(vars['es_version'])
       expect(command.exit_status).to eq(0)
     end
   end
@@ -227,11 +240,12 @@ shared_examples 'xpack::init' do |es_version,plugins|
       its(:exit_status) { should eq 0 }
   end
 
+  if vars['es_major_version'] == '5.x' # kibana default password has been removed in 6.x
     describe 'kibana access check' do
-    it 'should be reported as version '+es_version do
+      it 'should be reported as version '+vars['es_version'] do
-      command = command('curl -s localhost:9200/ -u kibana:changeme | grep number')
+        result = curl_json('http://localhost:9200/', username='kibana', password='changeme')
-      expect(command.stdout).to match(es_version)
+        expect(result['version']['number']).to eq(vars['es_version'])
-      expect(command.exit_status).to eq(0)
+      end
     end
   end
 

test/integration/helpers/serverspec/xpack_standard_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-shared_examples 'xpack_standard::init' do |es_version,plugins|
+shared_examples 'xpack_standard::init' do |vars|
 
   describe user('elasticsearch') do
     it { should exist }
@@ -27,7 +27,11 @@ shared_examples 'xpack_standard::init' do |es_version,plugins|
   describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
     it { should contain 'node.name: localhost-security_node' }
     it { should contain 'cluster.name: elasticsearch' }
+    if vars['es_major_version'] == '6.x'
+      it { should_not contain 'path.conf: /etc/elasticsearch/security_node' }
+    else
       it { should contain 'path.conf: /etc/elasticsearch/security_node' }
+    end
     it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' }
     it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' }
     it { should contain 'xpack.security.enabled: false' }
@@ -42,9 +46,9 @@ shared_examples 'xpack_standard::init' do |es_version,plugins|
   end
 
   describe 'version check' do
-    it 'should be reported as version '+es_version do
+    it 'should be reported as version '+vars['es_version'] do
       command = command('curl -s localhost:9200 | grep number')
-      expect(command.stdout).to match(es_version)
+      expect(command.stdout).to match(vars['es_version'])
       expect(command.exit_status).to eq(0)
     end
   end
@@ -107,16 +111,14 @@ shared_examples 'xpack_standard::init' do |es_version,plugins|
     it { should be_owned_by 'elasticsearch' }
   end
 
-  for plugin in plugins
+  describe file('/usr/share/elasticsearch/plugins/x-pack') do
-    describe file('/usr/share/elasticsearch/plugins/'+plugin) do
     it { should be_directory }
     it { should be_owned_by 'elasticsearch' }
   end
 
-    describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
+  describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"x-pack","version":"'+vars['es_version']+'"\'') do
     its(:exit_status) { should eq 0 }
   end
-  end
 
   #Test users file, users_roles and roles.yml
   describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do

test/integration/issue-test.yml

@@ -5,6 +5,8 @@
 
 - name: Simple Example
   hosts: localhost
+  tasks:
+    - include: elasticsearch/test/integration/debug.yml
   roles:
     - {
         role: elasticsearch,

test/integration/issue-test/serverspec/default_spec.rb

@@ -1,6 +1,8 @@
 require 'issue_test_spec'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
 
 describe 'Issue Test' do
-  include_examples 'issue_test::init', "5.5.1", []
+  include_examples 'issue_test::init', vars
 end
 

test/integration/multi.yml

@@ -2,6 +2,8 @@
 #Test ability to deploy multiple instances to a machine
 - name: Elasticsearch Multi test - master on 9200
   hosts: localhost
+  tasks:
+    - include: elasticsearch/test/integration/debug.yml
   vars:
     es_scripts: true
     es_templates: true
@@ -14,6 +16,8 @@
 
 - name: Elasticsearch Multi test - data on 9201
   hosts: localhost
+  tasks:
+    - include: elasticsearch/test/integration/debug.yml
   vars:
     es_scripts: true
     es_templates: true

test/integration/multi/serverspec/default_spec.rb

@@ -1,8 +1,9 @@
 require 'multi_spec'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
 
-
+describe 'Multi Tests' do
-describe 'Multi Tests v 5.x' do
+  include_examples 'multi::init', vars
-  include_examples 'multi::init', "5.5.1", ["ingest-geoip"]
 end
 
 

test/integration/package.yml

@@ -1,6 +1,8 @@
 ---
 - name: Elasticsearch Package test intial
   hosts: localhost
+  tasks:
+    - include: elasticsearch/test/integration/debug.yml
   roles:
     - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" }
   vars:
@@ -8,7 +10,6 @@
     es_templates: true
     es_heap_size: "1g"
     es_api_port: 9200
-    es_version: "5.5.1"
     es_plugins:
       - plugin: ingest-geoip
 
@@ -16,12 +17,13 @@
 #Tests the plugins have been correctly removed and ES can be upgraded between minor versions. All plugins will be removed and re-installed.
 - name: Elasticsearch Package test modify
   hosts: localhost
+  tasks:
+    - include: elasticsearch/test/integration/debug.yml
   roles:
     - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" }
   vars:
     es_scripts: true
     es_templates: true
-    es_version: "5.5.1"
     es_heap_size: "1g"
     es_api_port: 9200
     es_plugins:

test/integration/package/serverspec/default_spec.rb

@@ -1,6 +1,7 @@
 require 'package_spec'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
 
-
+describe 'Package Tests' do
-describe 'Package Tests v 5.x' do
+  include_examples 'package::init', vars
-  include_examples 'package::init', "5.5.1", ["ingest-attachment","ingest-geoip"]
 end

test/integration/standard.yml

@@ -1,6 +1,8 @@
 ---
 - name: Standard test for single node setup. Tests idempotence.
   hosts: localhost
+  tasks:
+    - include: elasticsearch/test/integration/debug.yml
   roles:
     - { role: elasticsearch, es_instance_name: "node1" }
   vars:

test/integration/standard/serverspec/default_spec.rb

@@ -1,8 +1,9 @@
 require 'standard_spec'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
 
-
+describe 'Standard Tests' do
-describe 'Standard Tests v 5.x' do
+  include_examples 'standard::init', vars
-  include_examples 'standard::init', "5.5.1", ["ingest-geoip"]
 end
 
 

test/integration/xpack-standard.yml

@@ -2,13 +2,14 @@
 ---
 - name: Elasticsearch Xpack tests - no security and manual download
   hosts: localhost
+  tasks:
+    - include: elasticsearch/test/integration/debug.yml
   roles:
     - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "security_node" }
   vars:
-    es_version: "5.5.1"
     es_heap_size: 2g
     es_enable_xpack: true
-    es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-5.5.1.zip"
+    es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip"
     es_xpack_features:
       - monitoring
       - graph

test/integration/xpack-standard/serverspec/default_spec.rb

@@ -1,5 +1,7 @@
 require 'xpack_standard_spec'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
 
-describe 'Xpack Standard Tests v 5.x' do
+describe 'Xpack Standard Tests' do
-  include_examples 'xpack_standard::init', "5.5.1", []
+  include_examples 'xpack_standard::init', vars
 end

test/integration/xpack.yml

@@ -1,6 +1,8 @@
 ---
 - name: Elasticsearch Xpack tests initial
   hosts: localhost
+  tasks:
+    - include: elasticsearch/test/integration/debug.yml
   roles:
     - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300",
     "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 },
@@ -8,7 +10,7 @@
   vars:
     es_heap_size: "1g"
     es_templates: true
-    es_version: "5.4.0"
+    es_version: "{{ '6.1.1' if es_major_version == '6.x' else '5.4.0'}}" # This is set to an older version than the current default to force an upgrade
     es_enable_xpack: true
     es_xpack_license: "{{ lookup('file', '/tmp/license.json')  }}"
     es_plugins:
@@ -101,6 +103,8 @@
 #modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed.
 - name: Elasticsearch Xpack modify
   hosts: localhost
+  tasks:
+    - include: elasticsearch/test/integration/debug.yml
   roles:
     - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300",
     "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 },
@@ -108,7 +112,6 @@
   vars:
     es_heap_size: "1g"
     es_templates: true
-    es_version: "5.5.1"
     es_enable_xpack: true
     es_xpack_license: "{{ lookup('file', '/tmp/license.json')  }}"
     es_plugins:

test/integration/xpack/serverspec/default_spec.rb

@@ -1,5 +1,7 @@
 require 'xpack_spec'
+require 'json'
+vars = JSON.parse(File.read('/tmp/vars.json'))
 
-describe 'Xpack Tests v 5.x' do
+describe 'Xpack Tests' do
-  include_examples 'xpack::init', "5.5.1", ["ingest-attachment"]
+  include_examples 'xpack::init', vars
 end