Merge pull request #452 from elastic/6.3

Initial 6.3 support

.kitchen.yml

@@ -21,7 +21,7 @@ provisioner:
     extra_vars:
       es_major_version: "<%= ENV['VERSION'] %>"
       <% if ENV['VERSION'] == '5.x' %>
-      es_version: '5.6.9'
+      es_version: '5.6.10'
       <% end %>
   <% end %>
 
@@ -35,8 +35,8 @@ platforms:
       privileged: true
       provision_command:
         - apt-get update && apt-get install -y software-properties-common &&  add-apt-repository -y ppa:ansible/ansible
-        - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip
+        - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip python-openssl
-        - pip install jmespath
+        - pip install jmespath pyOpenSSL ndg-httpsclient
         - pip uninstall -y ansible
       use_sudo: false
       volume: 

Makefile

@@ -0,0 +1,32 @@
+default: build
+
+SHELL:=/bin/bash -eux
+VERSION := 6.x
+PATTERN := xpack-standard-ubuntu-1604
+
+.PHONY: converge cerify test login destroy list
+
+setup:
+	bundle install
+	docker ps
+
+converge:
+	bundle exec kitchen converge $(PATTERN)
+
+verify:
+	bundle exec kitchen verify $(PATTERN)
+
+test:
+	bundle exec kitchen test $(PATTERN) --destroy=always
+
+login:
+	bundle exec kitchen login $(PATTERN)
+
+destroy:
+	bundle exec kitchen destroy $(PATTERN)
+
+destroy-all:
+	bundle exec kitchen destroy
+
+list:
+	bundle exec kitchen list

README.md

@@ -1,24 +1,23 @@
 # ansible-elasticsearch
+[![Build Status](https://img.shields.io/jenkins/s/https/devops-ci.elastic.co/job/elastic+ansible-elasticsearch+master.svg)](https://devops-ci.elastic.co/job/elastic+ansible-elasticsearch+master/)
 [![Ansible Galaxy](https://img.shields.io/badge/ansible--galaxy-elastic.elasticsearch-blue.svg)](https://galaxy.ansible.com/elastic/elasticsearch/)
 
 **THIS ROLE IS FOR 6.x, 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.**
 
-Ansible role for 6.x/5.x Elasticsearch.  Currently this works on Debian and RedHat based linux systems.  Tested platforms are:
+Ansible role for 6.x/5.x Elasticsearch.  Currently this works on Debian and RedHat based linux systems. Tested platforms are:
 
 * Ubuntu 14.04/16.04
 * Debian 8
 * Centos 7
 
-The latest Elasticsearch versions of 6.x are actively tested.  **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.**
+The latest Elasticsearch versions of 6.x and 5.x are actively tested.  **Only Ansible versions > 2.4.3.0 are supported, as this is currently the only version tested.**
 
 ##### Dependency
 This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine.
 
 ## Usage
 
-Create your Ansible playbook with your own tasks, and include the role elasticsearch.  You will have to have this repository accessible within the context of playbook, e.g.
+Create your Ansible playbook with your own tasks, and include the role elasticsearch. You will have to have this repository accessible within the context of playbook.
-
-e.g. 
 
 ```
 cd /my/repos/
@@ -45,6 +44,59 @@ The above installs a single node 'node1' on the hosts 'localhost'.
 
 This role also uses [Ansible tags](http://docs.ansible.com/ansible/playbooks_tags.html). Run your playbook with the `--list-tasks` flag for more information.
 
+## Testing
+
+This playbook uses [Kitchen](https://kitchen.ci/) for CI and local testing.
+
+### Requirements
+
+* Ruby
+* Bundler
+* Docker
+* Make
+
+### Running the tests
+
+If you want to test X-Pack features with a license you will first need to export the `ES_XPACK_LICENSE_FILE` variable.
+```
+export ES_XPACK_LICENSE_FILE="$(pwd)/license.json"
+```
+
+To converge an Ubuntu 16.04 host running X-Pack
+```
+$ make converge
+```
+
+To run the tests
+```
+$ make verify
+```
+
+To list all of the different test suits
+```
+$ make list
+```
+
+The default test suite is Ubuntu 16.04 with X-Pack. If you want to test another suite you can override this with the `PATTERN` variable
+```
+$ make converge PATTERN=standard-centos-7
+```
+
+The `PATTERN` is a kitchen pattern which can match multiple suites. To run all tests for CentOS
+```
+$ make converge PATTERN=centos-7
+```
+
+The default version is 6.x If you want to test 5.x you can override it with the `VERSION` variable to test 5.x
+```
+$ make converge VERSION=5.x PATTERN=standard-centos-7
+```
+
+When you are finished testing you can clean up everything with
+```
+$ make destroy-all
+```
+
 ### Basic Elasticsearch Configuration
 
 All Elasticsearch configuration parameters are supported.  This is achieved using a configuration map parameter 'es_config' which is serialized into the elasticsearch.yml file.  
@@ -220,7 +272,7 @@ ansible-playbook -i hosts ./your-playbook.yml
 
 ### Installing X-Pack Features
 
-X-Pack features, such as Security, are supported. This feature is currently experimental.  To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`.  
+X-Pack features, such as Security, are supported. This feature is currently experimental.
 
 The parameter `es_xpack_features` by default enables all features i.e. it defaults to ["alerting","monitoring","graph","security","ml"]
 
@@ -392,7 +444,7 @@ This role ships with sample scripts and templates located in the [files/scripts/
 
 ### Proxy
 
-To define proxy globaly, set the following variables:
+To define proxy globally, set the following variables:
 
 * ```es_proxy_host``` - global proxy host
 * ```es_proxy_port``` - global proxy port
@@ -413,10 +465,8 @@ To define proxy only for a particular plugin during its installation:
 * The role assumes the user/group exists on the server.  The elasticsearch packages create the default elasticsearch user.  If this needs to be changed, ensure the user exists.
 * The playbook relies on the inventory_name of each host to ensure its directories are unique
 * Changing an instance_name for a role application will result in the installation of a new component.  The previous component will remain.
-* KitchenCI has been used for testing.  This is used to confirm images reach the correct state after a play is first applied.  We currently test only the latest version of 6.x on
+* KitchenCI has been used for testing.  This is used to confirm images reach the correct state after a play is first applied.  We currently test the latest version of 6.x and 5.x on all supported platforms. 
-all supported platforms. 
+* The role aims to be idempotent.  Running the role multiple times, with no changes, should result in no state change on the server.  If the configuration is changed, these will be applied and Elasticsearch restarted where required.
-* The role aims to be idempotent.  Running the role multiple times, with no changes, should result in no state change on the server.  If the configuration is changed, these will be applied and 
-Elasticsearch restarted where required.
 * Systemd is used for Ubuntu versions >= 15, Debian >=8, Centos >=7.  All other versions use init for service scripts.
 * In order to run x-pack tests a license file with security enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests.
 
@@ -428,4 +478,4 @@ Elasticsearch restarted where required.
 
 ## Questions on Usage
 
-We welcome questions on how to use the role.  However, in order to keep the github issues list focused on "issues" we ask the community to raise questions at https://discuss.elastic.co/c/elasticsearch.  This is monitored by the maintainers.
+We welcome questions on how to use the role.  However, in order to keep the GitHub issues list focused on "issues" we ask the community to raise questions at https://discuss.elastic.co/c/elasticsearch.  This is monitored by the maintainers.

defaults/main.yml

@@ -1,12 +1,15 @@
 ---
 es_major_version: "6.x"
-es_version: "6.2.4"
+es_version: "6.3.0"
+es_use_snapshot_release: false
+es_enable_xpack: true
+es_package_name: "elasticsearch"
 es_version_lock: false
 es_use_repository: true
 es_templates_fileglob: "files/templates/*.json"
 es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch"
-es_apt_url: "deb https://artifacts.elastic.co/packages/{{ es_major_version }}/apt stable main"
+es_apt_url: "deb https://artifacts.elastic.co/packages/{{ es_repo_name }}/apt stable main"
-es_apt_url_old: "deb http://packages.elastic.co/elasticsearch/{{ es_major_version }}/debian stable main"
+es_apt_url_old: "deb http://packages.elastic.co/elasticsearch/{{ es_repo_name }}/debian stable main"
 es_start_service: true
 es_java_install: true
 update_java: false
@@ -27,7 +30,6 @@ es_max_open_files: 65536
 es_max_threads: "{{ 2048 if ( es_version | version_compare('6.0.0', '<')) else 8192 }}"
 es_max_map_count: 262144
 es_allow_downgrades: false
-es_enable_xpack: false
 es_xpack_features: ["alerting","monitoring","graph","ml","security"]
 #These are used for internal operations performed by ansible.
 #They do not affect the current configuration

tasks/compatibility-variables.yml

@@ -0,0 +1,38 @@
+---
+
+# It is possible to set these are defaults with messy jinja templating one liners however:
+# 1. That is really hard to read and debug
+# 2. When running multiple plays with the same role the defaults are not re-evaluated. An example of this
+# can be seen in our the https://github.com/elastic/ansible-elasticsearch/blob/master/test/integration/xpack.yml
+# integration test and in the Multi Node server documentation examples https://github.com/elastic/ansible-elasticsearch/blob/master/test/integration/xpack.yml
+
+- name: Set the defaults here otherwise they can't be overriden in the same play if the role is called twice
+  set_fact:
+    es_open_xpack: true
+    es_install_xpack: false
+    es_users_path: "users"
+    es_xpack_conf_subdir: ""
+    es_repo_name: "{{ es_major_version }}"
+    es_xpack_users_command: "elasticsearch-users"
+
+- name: Detect if es_version is before X-Pack was open and included
+  set_fact:
+    es_open_xpack: false
+  when: "es_version | version_compare('6.3.0', '<')"
+
+- name: If this is an older version we need to install X-Pack as a plugin and use a differet users command
+  set_fact:
+    es_install_xpack: true
+    es_xpack_users_command: "x-pack/users"
+    es_xpack_conf_subdir: "/x-pack"
+  when: 
+    - not es_open_xpack
+    - es_enable_xpack
+
+- name: Use the oss repo and package if xpack is not being used
+  set_fact:
+    es_repo_name: "{{ 'oss-' + es_major_version }}"
+    es_package_name: "elasticsearch-oss"
+  when:
+    - es_open_xpack
+    - not es_enable_xpack

tasks/elasticsearch-Debian.yml

@@ -27,10 +27,17 @@
 
 - name: Debian - Ensure elasticsearch is installed
   become: yes
-  apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present force={{force_install}} allow_unauthenticated={{ 'no' if es_apt_key else 'yes' }} cache_valid_time=86400
+  apt:
+    name: '{{ es_package_name }}{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %}'
+    state: present
+    force: '{{ force_install }}'
+    allow_unauthenticated: "{{ 'no' if es_apt_key else 'yes' }}"
+    cache_valid_time: 86400
   when: es_use_repository
   register: debian_elasticsearch_install_from_repo
   notify: restart elasticsearch
+  environment:
+    ES_PATH_CONF: "/etc/elasticsearch"
 
 - name: Debian - Include versionlock
   include: elasticsearch-Debian-version-lock.yml

tasks/elasticsearch-RedHat.yml

@@ -6,7 +6,7 @@
 
 - name: RedHat - add Elasticsearch repo
   become: yes
-  template: src=elasticsearch.repo dest=/etc/yum.repos.d/elasticsearch-{{ es_major_version }}.repo
+  template: src=elasticsearch.repo dest=/etc/yum.repos.d/elasticsearch-{{ es_repo_name }}.repo
   when: es_use_repository
 
 - name: RedHat - include versionlock
@@ -15,13 +15,18 @@
 
 - name: RedHat - Install Elasticsearch
   become: yes
-  yum: name=elasticsearch{% if es_version is defined and es_version != ""  %}-{{ es_version }}{% endif %} state=present update_cache=yes
+  yum:
+    name: '{{ es_package_name }}{% if es_version is defined and es_version != ""  %}-{{ es_version }}{% endif %}'
+    state: present
+    update_cache: yes
   when: es_use_repository
   register: redhat_elasticsearch_install_from_repo
   notify: restart elasticsearch
   until: redhat_elasticsearch_install_from_repo.rc == 0
   retries: 5
   delay: 10
+  environment:
+    ES_PATH_CONF: "/etc/elasticsearch"
 
 - name: RedHat - Install Elasticsearch from url
   become: yes

tasks/elasticsearch-plugins.yml

@@ -49,7 +49,6 @@
 - name: Remove elasticsearch plugins
   become: yes
   command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent"
-  ignore_errors: yes
   with_items: "{{ plugins_to_remove | default([]) }}"
   notify: restart elasticsearch
   register: plugin_removed
@@ -60,9 +59,8 @@
 
 - name: Install elasticsearch plugins
   become: yes
-  command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.plugin }} --batch --silent"
+  command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.url | default(item.plugin) }} --batch --silent"
   register: plugin_installed
-  failed_when: "'ERROR' in plugin_installed.stdout"
   changed_when: plugin_installed.rc == 0
   with_items: "{{ es_plugins }}"
   when: item.plugin in plugins_to_install

tasks/main.yml

@@ -4,11 +4,20 @@
   tags:
       - always
 
+- name: set compatibility variables
+  include: compatibility-variables.yml
+  tags:
+      - always
+
 - name: check-set-parameters
   include: elasticsearch-parameters.yml
   tags:
       - always
 
+- name: use snapshot release
+  include: snapshot-release.yml
+  when: es_use_snapshot_release
+
 - name: include java.yml
   include: java.yml
   when: es_java_install

tasks/snapshot-release.yml

@@ -0,0 +1,54 @@
+# These tasks are to run ansible-elasticsearch using pre-release snapshot builds
+# This should only be used for testing purposes and can be enabled by setting
+# es_use_snapshot_release: true
+
+- name: detect if we need the .deb or .rpm
+  set_fact:
+    package_type: "{{ 'deb' if (ansible_os_family == 'Debian') else 'rpm' }}"
+
+- name: get the minor version
+  set_fact:
+    minor_version: "{{ es_version.split('.')[0:2] | join('.')}}"
+
+- name: set the package_name
+  set_fact:
+    package_name: "{{ es_package_name + '-' + es_version + '-SNAPSHOT.' + package_type }}"
+
+- name: generate the artifacts url
+  set_fact:
+    artifacts_url: "{{ 'https://artifacts-api.elastic.co/v1/search/' + minor_version + '/' + package_name }}"
+
+- name: get latest snapshot build
+  uri:
+    url: "{{ artifacts_url }}"
+    return_contents: true
+  register: snapshots
+  retries: 5
+  delay: 1
+  ignore_errors: true
+  until: "'status' in snapshots and snapshots.status == 200"
+
+- name: use the custom package url instead of the repository
+  set_fact:
+    es_custom_package_url: "{{ snapshots.json[package_name]['url'] }}"
+    es_use_repository: false
+
+- name: set snapshot urls for es_plugins when it is defined
+  when: es_plugins is defined
+  block:
+  - name: split up the snapshot url so we can create the plugin url
+    set_fact:
+      split_url: "{{ es_custom_package_url.split('/') }}"
+  
+  - name: set base plugin url
+    set_fact:
+      plugin_url: "{{ split_url[0] + '//' + split_url[2:5]|join('/') + '/elasticsearch-plugins/'}}"
+  
+  - name: create es_plugins with the snapshot url
+    set_fact:
+      es_plugins_temp: "{{ es_plugins_temp|default([]) + [{'plugin': item.plugin, 'url': plugin_url + item.plugin + '/' + item.plugin + '-' + es_version + '-SNAPSHOT.zip'}] }}"
+    with_items: "{{ es_plugins }}"
+  
+  - name: override the original es_plugins with the snapshot version
+    set_fact:
+      es_plugins: "{{ es_plugins_temp }}"

tasks/xpack/elasticsearch-xpack.yml

@@ -5,6 +5,7 @@
 
 - name: include elasticsearch-xpack-install.yml
   include: elasticsearch-xpack-install.yml
+  when: es_install_xpack
 
 #Security configuration
 - name: include security/elasticsearch-security.yml

tasks/xpack/security/elasticsearch-security-file.yml

@@ -2,10 +2,19 @@
 - name: set fact manage_file_users
   set_fact: manage_file_users=es_users is defined and es_users.file is defined and es_users.file.keys() | length > 0
 
+- name: Create the users file if it doesn't exist
+  copy:
+    content: ""
+    dest: "{{ conf_dir }}{{ es_xpack_conf_subdir }}/users"
+    force: no # this ensures it only creates it if it does not exist
+    group: "{{ es_group }}"
+    owner: "{{ es_user }}"
+    mode: 0555
+
 #List current users
 - name: List Users
   become: yes
-  shell: cat {{conf_dir}}/x-pack/users | awk -F':' '{print $1}'
+  shell: cat {{conf_dir}}{{es_xpack_conf_subdir}}/users | awk -F':' '{print $1}'
   register: current_file_users
   when: manage_file_users
   changed_when: False
@@ -18,7 +27,7 @@
 - name: Remove Users
   become: yes
   command: >
-    {{es_home}}/bin/x-pack/users userdel {{item}}
+    {{es_home}}/bin/{{es_xpack_users_command}} userdel {{item}}
   with_items: "{{users_to_remove | default([])}}"
   when: manage_file_users
   environment:
@@ -34,7 +43,7 @@
 - name: Add Users
   become: yes
   command: >
-    {{es_home}}/bin/x-pack/users useradd {{item}} -p {{es_users.file[item].password}}
+    {{es_home}}/bin/{{es_xpack_users_command}} useradd {{item}} -p {{es_users.file[item].password}}
   with_items: "{{ users_to_add | default([]) }}"
   when: manage_file_users
   no_log: True
@@ -47,7 +56,7 @@
 - name: Set User Passwords
   become: yes
   command: >
-    {{es_home}}/bin/x-pack/users passwd {{ item }} -p {{es_users.file[item].password}}
+    {{es_home}}/bin/{{es_xpack_users_command}} passwd {{ item }} -p {{es_users.file[item].password}}
   with_items: "{{ es_users.file.keys() | default([]) }}"
   when: manage_file_users
   #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip.
@@ -65,16 +74,16 @@
 #Copy Roles files
 - name: Copy roles.yml File for Instance
   become: yes
-  template: src=security/roles.yml.j2 dest={{conf_dir}}/x-pack/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=security/roles.yml.j2 dest={{conf_dir}}{{es_xpack_conf_subdir}}/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   when: es_roles is defined and es_roles.file is defined
 
 #Overwrite users_roles file
 - name: Copy User Roles
   become: yes
-  template: src=security/users_roles.j2 dest={{conf_dir}}/x-pack/users_roles mode=0644 force=yes
+  template: src=security/users_roles.j2 dest={{conf_dir}}{{es_xpack_conf_subdir}}/users_roles mode=0644 force=yes
   when: manage_file_users and users_roles | length > 0
 
 #Set permission on security directory. E.g. if 2 nodes are installed on the same machine, the second node will not get the users file created at install, causing the files being created at es_users call and then having the wrong Permissions.
 - name: Set Security Directory Permissions Recursive
   become: yes
-  file: state=directory path={{conf_dir}}/x-pack/ owner={{ es_user }} group={{ es_group }} recurse=yes
+  file: state=directory path={{conf_dir}}{{es_xpack_conf_subdir}}/ owner={{ es_user }} group={{ es_group }} recurse=yes

tasks/xpack/security/elasticsearch-security.yml

@@ -5,30 +5,41 @@
 
 #Ensure x-pack conf directory is created if necessary
 - name: Ensure x-pack conf directory exists (file)
-  file: path={{ conf_dir }}/x-pack state=directory owner={{ es_user }} group={{ es_group }}
+  file: path={{ conf_dir }}{{ es_xpack_conf_subdir }} state=directory owner={{ es_user }} group={{ es_group }}
   changed_when: False
   when:
     - es_enable_xpack and "security" in es_xpack_features
     - (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined)
 
 #-----------------------------Create Bootstrap User-----------------------------------
-- name: Check if bootstrap password is set
+### START BLOCK elasticsearch keystore ###
-  command: >
+- name: create the elasticsearch keystore
-   {{es_home}}/bin/elasticsearch-keystore list
+  when: (es_enable_xpack and "security" in es_xpack_features) and (es_version | version_compare('6.0.0', '>'))
-  register: list_keystore
+  block:
-  changed_when: False
+  - name: create the keystore if it doesn't exist yet
-  environment:
+    command: >
-    ES_PATH_CONF: "{{ conf_dir }}"
+     {{es_home}}/bin/elasticsearch-keystore create
-  when:
+    args:
-    - (es_enable_xpack and "security" in es_xpack_features) and (es_version | version_compare('6.0.0', '>'))
+      creates: "{{ conf_dir }}/elasticsearch.keystore"
+    environment:
+      ES_PATH_CONF: "{{ conf_dir }}"
+  
+  - name: Check if bootstrap password is set
+    command: >
+     {{es_home}}/bin/elasticsearch-keystore list
+    register: list_keystore
+    changed_when: False
+    environment:
+      ES_PATH_CONF: "{{ conf_dir }}"
 
-- name: Create Bootstrap password for elastic user
+  - name: Create Bootstrap password for elastic user
-  shell: echo "{{es_api_basic_auth_password}}" | {{es_home}}/bin/elasticsearch-keystore add -x 'bootstrap.password'
+    shell: echo "{{es_api_basic_auth_password}}" | {{es_home}}/bin/elasticsearch-keystore add -x 'bootstrap.password'
-  when:
+    when:
-    - (es_enable_xpack and "security" in es_xpack_features) and (es_version | version_compare('6.0.0', '>')) and es_api_basic_auth_username is defined and list_keystore is defined and es_api_basic_auth_username == 'elastic' and 'bootstrap.password' not in list_keystore.stdout_lines
+      - es_api_basic_auth_username is defined and list_keystore is defined and es_api_basic_auth_username == 'elastic' and 'bootstrap.password' not in list_keystore.stdout_lines
-  environment:
+    environment:
-    ES_PATH_CONF: "{{ conf_dir }}"
+      ES_PATH_CONF: "{{ conf_dir }}"
-  no_log: true
+    no_log: true
+### END BLOCK elasticsearch keystore ###
 
 #-----------------------------FILE BASED REALM----------------------------------------
 
@@ -40,14 +51,14 @@
 #Copy Roles files
 - name: Copy role_mapping.yml File for Instance
   become: yes
-  template: src=security/role_mapping.yml.j2 dest={{conf_dir}}/x-pack/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=security/role_mapping.yml.j2 dest={{conf_dir}}{{es_xpack_conf_subdir}}/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   when: es_role_mapping is defined
 
 #-----------------------------AUTH FILE----------------------------------------
 
 - name: Copy message auth key to elasticsearch
   become: yes
-  copy: src={{ es_message_auth_file }} dest={{conf_dir}}/x-pack/system_key owner={{ es_user }} group={{ es_group }} mode=0600 force=yes
+  copy: src={{ es_message_auth_file }} dest={{conf_dir}}{{es_xpack_conf_subdir}}/system_key owner={{ es_user }} group={{ es_group }} mode=0600 force=yes
   when: es_message_auth_file is defined
 
 #------------------------------------------------------------------------------------

templates/elasticsearch.repo

@@ -1,6 +1,6 @@
-[elasticsearch-{{ es_major_version }}]
+[elasticsearch-{{ es_repo_name }}]
-name=Elasticsearch repository for {{ es_major_version }} packages
+name=Elasticsearch repository for {{ es_repo_name }} packages
-baseurl=https://artifacts.elastic.co/packages/{{ es_major_version }}/yum
+baseurl=https://artifacts.elastic.co/packages/{{ es_repo_name }}/yum
 gpgcheck=1
 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
 enabled=1

test/integration/config.yml

@@ -8,6 +8,7 @@
     #expand to all available parameters
     - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: false } }
   vars:
+    es_enable_xpack: false
     es_scripts: false
     es_templates: false
     es_version_lock: false
@@ -25,6 +26,7 @@
     #expand to all available parameters
     - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9501", http.port: 9401, transport.tcp.port: 9501, node.data: true, node.master: true, bootstrap.memory_lock: true } }
   vars:
+    es_enable_xpack: false
     es_scripts: false
     es_templates: false
     es_version_lock: false

test/integration/helpers/serverspec/config_spec.rb

@@ -18,7 +18,7 @@ shared_examples 'config::init' do |vars|
     it { should be_running }
   end
 
-  describe package('elasticsearch') do
+  describe package(vars['es_package_name']) do
     it { should be_installed }
   end
 

test/integration/helpers/serverspec/multi_spec.rb

@@ -16,7 +16,7 @@ shared_examples 'multi::init' do |vars|
     it { should be_running }
   end
 
-  describe package('elasticsearch') do
+  describe package(vars['es_package_name']) do
     it { should be_installed }
   end
 

test/integration/helpers/serverspec/package_spec.rb

@@ -12,7 +12,7 @@ shared_examples 'package::init' do |vars|
     it { should be_running }
   end
 
-  describe package('elasticsearch') do
+  describe package(vars['es_package_name']) do
     it { should be_installed }
   end
 

test/integration/helpers/serverspec/standard_spec.rb

@@ -10,7 +10,7 @@ shared_examples 'standard::init' do |vars|
     it { should be_running }
   end
 
-  describe package('elasticsearch') do
+  describe package(vars['es_package_name']) do
     it { should be_installed }
   end
 
@@ -85,15 +85,22 @@ shared_examples 'standard::init' do |vars|
   end
 
   for plugin in vars['es_plugins']
-    plugin = plugin['plugin']
+    name = plugin['plugin']
 
-    describe file('/usr/share/elasticsearch/plugins/'+plugin) do
+    describe file('/usr/share/elasticsearch/plugins/'+name) do
       it { should be_directory }
       it { should be_owned_by 'elasticsearch' }
     end
-    #confirm plugins are installed and the correct version
+    it 'should be installed and the right version' do 
-    describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do
+      plugins = curl_json('http://localhost:9200/_nodes/plugins')
-      its(:exit_status) { should eq 0 }
+      version = nil
+      node, data = plugins['nodes'].first
+      data['plugins'].each do |plugin|
+        if plugin['name'] == name
+          version = plugin['version']
+        end
+      end
+      expect(version).to eql(vars['es_version'])
     end
   end
 

test/integration/helpers/serverspec/xpack_spec.rb

@@ -12,7 +12,7 @@ shared_examples 'xpack::init' do |vars|
     it { should be_running }
   end
 
-  describe package('elasticsearch') do
+  describe package(vars['es_package_name']) do
     it { should be_installed }
   end
 
@@ -80,10 +80,42 @@ shared_examples 'xpack::init' do |vars|
     it { should_not exist }
   end
 
-  #Xpack specific tests
+  # X-Pack is no longer installed as a plugin in elasticsearch
-  describe file('/usr/share/elasticsearch/plugins') do
+  if vars['es_major_version'] == '5.x'
-    it { should be_directory }
+    describe file('/usr/share/elasticsearch/plugins') do
-    it { should be_owned_by 'elasticsearch' }
+      it { should be_directory }
+      it { should be_owned_by 'elasticsearch' }
+    end
+
+    describe file('/usr/share/elasticsearch/plugins/x-pack') do
+      it { should be_directory }
+      it { should be_owned_by 'elasticsearch' }
+    end
+
+    describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do
+      its(:exit_status) { should eq 0 }
+    end
+
+    describe file('/usr/share/elasticsearch/plugins/x-pack') do
+      it { should be_directory }
+      it { should be_owned_by 'elasticsearch' }
+    end
+
+    describe 'xpack plugin' do
+      it 'should be installed with the correct version' do
+        plugins = curl_json('http://localhost:9200/_nodes/plugins', username='es_admin', password='changeMeAgain')
+        node, data = plugins['nodes'].first
+        version = 'plugin not found'
+        name = 'x-pack'
+
+        data['plugins'].each do |plugin|
+          if plugin['name'] == name
+            version = plugin['version']
+          end
+        end
+        expect(version).to eql(vars['es_version'])
+      end
+    end
   end
 
   #Test if x-pack is activated
@@ -95,25 +127,11 @@ shared_examples 'xpack::init' do |vars|
     end
   end
 
-  describe file('/usr/share/elasticsearch/plugins/x-pack') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do
-    its(:exit_status) { should eq 0 }
-  end
-
   describe file('/etc/elasticsearch/security_node/x-pack') do
     it { should be_directory }
     it { should be_owned_by 'elasticsearch' }
   end
 
-  describe file('/usr/share/elasticsearch/plugins/x-pack') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
   for plugin in vars['es_plugins']
     plugin = plugin['plugin']
 
@@ -128,26 +146,18 @@ shared_examples 'xpack::init' do |vars|
   end
 
   #Test users file, users_roles and roles.yml
-  describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do
+  describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/users_roles') do
     it { should be_owned_by 'elasticsearch' }
     it { should contain 'admin:es_admin' }
     it { should contain 'power_user:testUser' }
   end
 
-  describe file('/etc/elasticsearch/security_node/x-pack/users') do
+  describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/users') do
     it { should be_owned_by 'elasticsearch' }
     it { should contain 'testUser:' }
     it { should contain 'es_admin:' }
   end
 
-
-  describe file('/etc/elasticsearch/security_node/x-pack/roles.yml') do
-    it { should be_owned_by 'elasticsearch' }
-    #Test contents as expected
-    its(:md5sum) { should eq '7800182547287abd480c8b095bf26e9e' }
-  end
-
-
   describe 'security roles' do
     it 'should list the security roles' do
       roles = curl_json('http://localhost:9200/_xpack/security/role', username='es_admin', password='changeMeAgain')
@@ -192,7 +202,7 @@ shared_examples 'xpack::init' do |vars|
   end
 
   #Test contents of role_mapping.yml
-  describe file('/etc/elasticsearch/security_node/x-pack/role_mapping.yml') do
+  describe file('/etc/elasticsearch/security_node' + vars['es_xpack_conf_subdir'] + '/role_mapping.yml') do
     it { should be_owned_by 'elasticsearch' }
     it { should contain 'power_user:' }
     it { should contain '- cn=admins,dc=example,dc=com' }
@@ -200,18 +210,6 @@ shared_examples 'xpack::init' do |vars|
     it { should contain '- cn=admins,dc=example,dc=com' }
   end
 
-
-  describe file('/etc/elasticsearch/security_node/x-pack/system_key') do
-    it { should be_owned_by 'elasticsearch' }
-    it { should be_writable.by('owner') }
-    it { should be_writable.by_user('elasticsearch') }
-    it { should be_readable.by('owner') }
-    it { should be_readable.by_user('elasticsearch') }
-    it { should_not be_executable }
-    #Test contents as expected
-    its(:md5sum) { should eq '6ff0e6c4380a6ac0f6e04d871c0ca5e8' }
-  end
-
   #check accounts are correct i.e. we can auth and they have the correct roles
 
   describe 'kibana4_server access check' do

test/integration/helpers/serverspec/xpack_standard_spec.rb

@@ -10,7 +10,7 @@ shared_examples 'xpack_standard::init' do |vars|
     it { should be_running }
   end
 
-  describe package('elasticsearch') do
+  describe package(vars['es_package_name']) do
     it { should be_installed }
   end
 
@@ -96,60 +96,40 @@ shared_examples 'xpack_standard::init' do |vars|
     end
   end
 
-  describe file('/usr/share/elasticsearch/plugins/x-pack') do
+  # X-Pack is no longer installed as a plugin in elasticsearch
-    it { should be_directory }
+  if vars['es_major_version'] == '5.x'
-    it { should be_owned_by 'elasticsearch' }
+    describe file('/usr/share/elasticsearch/plugins/x-pack') do
+      it { should be_directory }
+      it { should be_owned_by 'elasticsearch' }
+    end
+
+    describe 'x-pack-core plugin' do
+      it 'should be installed with the correct version' do
+        plugins = curl_json('http://localhost:9200/_nodes/plugins')
+        node, data = plugins['nodes'].first
+        version = 'plugin not found'
+        name = 'x-pack'
+
+        data['plugins'].each do |plugin|
+          if plugin['name'] == name
+            version = plugin['version']
+          end
+        end
+        expect(version).to eql(vars['es_version'])
+      end
+    end
+
+    describe file('/etc/elasticsearch/security_node/x-pack') do
+      it { should be_directory }
+      it { should be_owned_by 'elasticsearch' }
+    end
+
   end
 
   describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do
     its(:exit_status) { should eq 0 }
   end
 
-  describe file('/etc/elasticsearch/security_node/x-pack') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe file('/usr/share/elasticsearch/plugins/x-pack') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe file('/usr/share/elasticsearch/plugins/x-pack') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe 'x-pack-core plugin' do
-    it 'should be installed with the correct version' do
-      plugins = curl_json('http://localhost:9200/_nodes/plugins')
-      node, data = plugins['nodes'].first
-      version = 'plugin not found'
-
-      if Gem::Version.new(vars['es_version']) >= Gem::Version.new('6.2')
-        name = 'x-pack-core'
-      else
-        name = 'x-pack'
-      end
-
-      data['plugins'].each do |plugin|
-        if plugin['name'] == name
-          version = plugin['version']
-        end
-      end
-      expect(version).to eql(vars['es_version'])
-    end
-  end
-
-  #Test users file, users_roles and roles.yml
-  describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe file('/etc/elasticsearch/security_node/x-pack/users') do
-    it { should be_owned_by 'elasticsearch' }
-  end
-
   describe command('curl -s localhost:9200/_xpack') do
     its(:stdout_as_json) { should include('features' => include('security' => include('enabled' => false))) }
     its(:stdout_as_json) { should include('features' => include('watcher' => include('enabled' => false))) }

test/integration/issue-test.yml

@@ -8,16 +8,15 @@
   tasks:
     - include: elasticsearch/test/integration/debug.yml
   roles:
-    - {
+    - role: elasticsearch
-        role: elasticsearch,
+      es_instance_name: "security_node"
-        es_config:
+      es_xpack_license: "{{ lookup('file', '/tmp/license.json')  }}"
-        {
+      es_config:
-          "xpack.security.authc.realms.file1.type": "file",
+        xpack.security.enabled: True
-          "xpack.security.authc.realms.file1.order": 1,
+        xpack.security.authc.realms.file1.type: "file"
-          "xpack.security.authc.realms.native1.type": "native",
+        xpack.security.authc.realms.file1.order: 1
-          "xpack.security.authc.realms.native1.order": 0
+        xpack.security.authc.realms.native1.type: "native"
-        },
+        xpack.security.authc.realms.native1.order: 0
-    es_instance_name: "security_node" }
   vars:
     es_heap_size: "1g"
     es_enable_xpack: true

test/integration/multi.yml

@@ -5,6 +5,7 @@
   tasks:
     - include: elasticsearch/test/integration/debug.yml
   vars:
+    es_enable_xpack: false
     es_scripts: true
     es_templates: true
     es_heap_size: "1g"
@@ -19,6 +20,7 @@
   tasks:
     - include: elasticsearch/test/integration/debug.yml
   vars:
+    es_enable_xpack: false
     es_scripts: true
     es_templates: true
     es_heap_size: "1g"

test/integration/package.yml

@@ -6,6 +6,7 @@
   roles:
     - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" }
   vars:
+    es_enable_xpack: false
     es_scripts: true
     es_templates: true
     es_heap_size: "1g"
@@ -22,6 +23,7 @@
   roles:
     - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" }
   vars:
+    es_enable_xpack: false
     es_scripts: true
     es_templates: true
     es_heap_size: "1g"

test/integration/standard.yml

@@ -6,7 +6,7 @@
   roles:
     - { role: elasticsearch, es_instance_name: "node1" }
   vars:
-    es_use_repository: "true"
+    es_enable_xpack: false
     es_heap_size: "1g"
     es_plugins:
       - plugin: ingest-geoip

test/integration/xpack-standard.yml

@@ -7,9 +7,9 @@
   roles:
     - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "security_node" }
   vars:
+    es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip"
     es_heap_size: 2g
     es_enable_xpack: true
-    es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip"
     es_xpack_features:
       - monitoring
       - graph

test/integration/xpack.yml

@@ -10,7 +10,7 @@
   vars:
     es_heap_size: "1g"
     es_templates: true
-    es_version: "{{ '6.1.1' if es_major_version == '6.x' else '5.4.0'}}" # This is set to an older version than the current default to force an upgrade
+    es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade
     es_enable_xpack: true
     es_xpack_license: "{{ lookup('file', '/tmp/license.json')  }}"
     es_plugins:
@@ -106,9 +106,18 @@
   tasks:
     - include: elasticsearch/test/integration/debug.yml
   roles:
-    - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300",
+    - role: elasticsearch
-    "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 },
+      es_api_port: 9200
-    es_instance_name: "security_node" }
+      es_instance_name: "security_node"
+      es_config: 
+        http.port: 9200
+        transport.tcp.port: 9300
+        discovery.zen.ping.unicast.hosts: "localhost:9300"
+        xpack.security.enabled: True
+        xpack.security.authc.realms.file1.type: "file"
+        xpack.security.authc.realms.file1.order: 0
+        xpack.security.authc.realms.native1.type: "native"
+        xpack.security.authc.realms.native1.order: 1
   vars:
     es_heap_size: "1g"
     es_templates: true

vars/main.yml

@@ -5,4 +5,4 @@ sysd_script: "/usr/lib/systemd/system/elasticsearch.service"
 init_script: "/etc/init.d/elasticsearch"
 #add supported features here
 supported_xpack_features: ["alerting","monitoring","graph","security"]
-reserved_xpack_users: ["elastic","kibana","logstash_system"]
+reserved_xpack_users: ["elastic","kibana","logstash_system"]