Initial Shield support + latest gems + single plugin dir + new port/host vars

.kitchen.yml

@@ -8,6 +8,7 @@ provisioner:
   roles_path: ../
   require_ansible_repo: true
   ansible_verbose: true
+  ansible_version: 2.0.2
   http_proxy: <%= ENV['HTTP_PROXY'] %>
   https_proxy: <%= ENV['HTTPS_PROXY'] %>
   no_proxy: localhost,127.0.0.1
@@ -19,7 +20,7 @@ platforms:
       privileged: true
       provision_command:
         - apt-get update && apt-get install -y software-properties-common &&  add-apt-repository -y ppa:ansible/ansible
-        - apt-get update && apt-get -y -q install ansible python-apt python-pycurl
+        - apt-get update && apt-get -y -q install python-apt python-pycurl
       use_sudo: false
   - name: debian-7
     driver_config:
@@ -27,7 +28,6 @@ platforms:
       privileged: true
       provision_command:
         - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml
-        - pip install ansible
         - apt-get install -y -q net-tools
       use_sudo: false
   - name: debian-8
@@ -36,7 +36,6 @@ platforms:
       privileged: true
       provision_command:
         - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget
-        - pip install ansible
         - apt-get install -y -q net-tools
         - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config
         - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config
@@ -137,3 +136,9 @@ suites:
             version: latest
     provisioner:
       playbook: test/integration/multi.yml
+  #Currently we only test shield on 2x
+  - name: xpack-2x
+    run_list:
+    attributes:
+    provisioner:
+      playbook: test/integration/xpack.yml

Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-gem 'test-kitchen', '1.4.2'
-gem "kitchen-docker", '2.1.0'
-gem 'kitchen-ansible', '0.40.1'
-gem 'net-ssh', '~> 2.0'
+gem 'test-kitchen', '1.8.0'
+gem "kitchen-docker", '2.5.0'
+gem 'kitchen-ansible', '0.44.6'
+gem 'net-ssh', '~> 3.0'

Gemfile.lock

@@ -1,30 +1,27 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    faraday (0.9.2)
-      multipart-post (>= 1.2, < 3)
-    highline (1.7.8)
-    kitchen-ansible (0.40.1)
-      librarian-ansible
+    artifactory (2.3.3)
+    kitchen-ansible (0.44.6)
+      net-ssh (~> 3.0)
       test-kitchen (~> 1.4)
-    kitchen-docker (2.1.0)
+    kitchen-docker (2.5.0)
       test-kitchen (>= 1.0.0)
-    librarian (0.1.2)
-      highline
-      thor (~> 0.15)
-    librarian-ansible (3.0.0)
-      faraday
-      librarian (~> 0.1.0)
+    mixlib-install (1.1.0)
+      artifactory
+      mixlib-shellout
+      mixlib-versioning
     mixlib-shellout (2.2.6)
-    multipart-post (2.0.0)
+    mixlib-versioning (1.1.0)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
-    net-ssh (2.9.4)
+    net-ssh (3.2.0)
     safe_yaml (1.0.4)
-    test-kitchen (1.4.2)
+    test-kitchen (1.8.0)
+      mixlib-install (~> 1.0, >= 1.0.4)
       mixlib-shellout (>= 1.2, < 3.0)
       net-scp (~> 1.1)
-      net-ssh (~> 2.7, < 2.10)
+      net-ssh (>= 2.9, < 4.0)
       safe_yaml (~> 1.0)
       thor (~> 0.18)
     thor (0.19.1)
@@ -33,10 +30,10 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  kitchen-ansible (= 0.40.1)
-  kitchen-docker (= 2.1.0)
-  net-ssh (~> 2.0)
-  test-kitchen (= 1.4.2)
+  kitchen-ansible (= 0.44.6)
+  kitchen-docker (= 2.5.0)
+  net-ssh (~> 3.0)
+  test-kitchen (= 1.8.0)
 
 BUNDLED WITH
    1.11.2

README.md

@@ -251,7 +251,6 @@ controlled by the following parameters:
 * ```es_data_dirs``` - defaults to "/var/lib/elasticsearch".  This can be a list or comma separated string e.g. ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"] or "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2"
 * ```es_log_dir``` - defaults to "/var/log/elasticsearch".
 * ```es_work_dir``` - defaults to "/tmp/elasticsearch".
-* ```es_plugin_dir``` - defaults to "/usr/share/elasticsearch/plugins".
 * ```es_restart_on_change``` - defaults to true.  If false, changes will not result in Elasticsearch being restarted.
 * ```es_plugins_reinstall``` - defaults to false.  If true, all currently installed plugins will be removed from a node.  Listed plugins will then be re-installed.  
 

defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 es_major_version: "2.x"
-es_version: "2.2.0"
+es_version: "2.3.4"
 es_version_lock: false
 es_use_repository: true
 es_start_service: true
@@ -13,13 +13,16 @@ es_templates: false
 es_user: elasticsearch
 es_group: elasticsearch
 es_config: {}
-es_install_shield: false
 #Need to provide default directories
 es_pid_dir: "/var/run/elasticsearch"
 es_data_dirs: "/var/lib/elasticsearch"
 es_log_dir: "/var/log/elasticsearch"
 es_work_dir: "/tmp/elasticsearch"
-es_plugin_dir: "/usr/share/elasticsearch/plugins"
 es_max_open_files: 65536
 es_allow_downgrades: false
-
+es_enable_xpack: false
+es_xpack_features: []
+#These are used for internal operations performed by ansible.
+#They do not effect the current configuration
+es_api_host: "localhost"
+es_api_port: 9200

filter_plugins/custom.py

@@ -19,8 +19,18 @@ def append_to_list(values=[], suffix=''):
 def array_to_str(values=[],separator=','):
     return separator.join(values)
 
+def extract_role_users(users={}):
+    role_users=[]
+    for user,details in users.iteritems():
+        if "roles" in details:
+            for role in details["roles"]:
+                role_users.append(role+":"+user)
+    return role_users
+
+
 class FilterModule(object):
     def filters(self):
         return {'modify_list': modify_list,
         'append_to_list':append_to_list,
-        'array_to_str':array_to_str}
+        'array_to_str':array_to_str,
+        'extract_role_users':extract_role_users}

handlers/main.yml

@@ -2,3 +2,7 @@
 - name: restart elasticsearch
   service: name={{instance_init_script | basename}} state=restarted enabled=yes
   when: es_restart_on_change and es_start_service and not elasticsearch_started.changed and ((plugin_installed is defined and plugin_installed.changed) or (elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed))
+
+- name: load-native-realms
+  include: ./handlers/shield/elasticsearch-shield-native.yml
+  when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)

handlers/shield/elasticsearch-shield-native.yml

@@ -0,0 +1,117 @@
+---
+- name: Wait for elasticsearch to startup
+  wait_for: port={{es_api_port}} delay=10
+
+- set_fact: manage_native_users=false
+
+- set_fact: manage_native_users=true
+  when: es_users is defined and es_users.native is defined
+
+- set_fact: manage_native_roles=false
+
+- set_fact: manage_native_roles=true
+  when: es_roles is defined and es_roles.native is defined
+
+#If the node has just has shield installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load
+
+#List current users
+- name: List Native Users
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_shield/user
+    method: GET
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+    status_code: 200
+  register: user_list_response
+  when: manage_native_users
+
+
+- set_fact: current_users={{user_list_response.json.keys() | list}}
+  when: manage_native_users
+
+#Identify non declared users
+
+- set_fact: users_to_remove={{ current_users | difference ( es_users.native.keys() ) }}
+  when: manage_native_users
+
+#Delete all non required users
+- name: Delete Native Users
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item}}
+    method: DELETE
+    status_code: 200
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+  when: manage_native_users and users_to_remove | length > 0
+  with_items: "{{users_to_remove}}"
+
+
+#Overwrite all other users
+- name: Update Native Users
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item.key}}
+    method: POST
+    body_format: json
+    body: "{{item.value | to_json}}"
+    status_code: 200
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+  when: manage_native_users and es_users.native.keys() > 0
+  with_dict: "{{es_users.native}}"
+
+#List current roles
+
+- name: List Native Roles
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_shield/role
+    method: GET
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+    status_code: 200
+  register: role_list_response
+  when: manage_native_roles
+
+#Identify undeclared roles
+
+- set_fact: current_roles={{role_list_response.json.keys() | list}}
+  when: manage_native_users
+
+- debug: msg="{{current_roles}}"
+
+- set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys()  ) }}
+  when: manage_native_roles
+
+
+#Delete all non required roles
+- name: Delete Native Roles
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item}}
+    method: DELETE
+    status_code: 200
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+  when: manage_native_roles and roles_to_remove | length > 0
+  with_items: "{{roles_to_remove}}"
+
+
+#Update other roles
+- name: Update Native Roles
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item.key}}
+    method: POST
+    body_format: json
+    body: "{{item.value | to_json}}"
+    status_code: 200
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+  when: manage_native_roles and es_roles.native.keys() > 0
+  with_dict: "{{es_roles.native}}"
+
+
+

meta/main.yml

@@ -8,7 +8,7 @@ galaxy_info:
   company: "Elastic.co"
   license: "license (Apache)"
   # Require 1.6 for apt deb install
-  min_ansible_version: 1.6
+  min_ansible_version: 2.0
   platforms:
   - name: EL
     versions:

tasks/checkParameters.yml

@@ -1,22 +0,0 @@
-# Check for mandatory parameters
-
-- fail: msg="es_instance_name must be specified and cannot be blank"
-  when: es_instance_name is not defined or es_instance_name == ''
-
-- fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined"
-  when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '')
-
-- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or (es_config['discovery.zen.ping.multicast.enabled'] is defined and es_config['discovery.zen.ping.multicast.enabled'])}}
-
-- debug: msg="WARNING - It is recommended you specify the parameter 'http.port' when multicast is disabled"
-  when: not multi_cast and es_config['http.port'] is not defined
-
-- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port' when multicast is disabled"
-  when: not multi_cast and es_config['transport.tcp.port'] is not defined
-
-- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts' when multicast is disabled"
-  when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined
-
-#If the user attempts to lock memory they must specify a heap size
-- fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified"
-  when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined

tasks/elasticsearch-config.yml

@@ -1,42 +1,6 @@
 ---
-
 # Configure Elasticsearch Node
 
-#Use systemd for the following distributions:
-#
-#Ubuntu 15 and up
-#Debian 8 and up
-#Centos 7 and up
-#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied.
-
-
-- set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }}
-  tags:
-      - always
-
-- set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}}
-  when: use_system_d
-  tags:
-      - always
-
-#For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN.
-
-- set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }}
-  tags:
-      - always
-
-- set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}}
-  tags:
-      - always
-
-- set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}}
-  tags:
-      - always
-
-- set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}}
-  tags:
-      - always
-
 #Create required directories
 - name: Create Directories
   file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }}
@@ -45,11 +9,6 @@
     - "{{work_dir}}"
     - "{{log_dir}}"
     - "{{conf_dir}}"
-    - "{{plugin_dir}}"
-
-- set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }}
-  tags:
-      - always
 
 - name: Create Data Directories
   file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }}
@@ -112,4 +71,3 @@
 - name: Delete Default Logging File
   file: dest=/etc/elasticsearch/logging.yml state=absent
 
-- debug: msg="Data Dirs {{data_dirs}}"

tasks/elasticsearch-parameters.yml

@@ -0,0 +1,50 @@
+# Check for mandatory parameters
+
+- fail: msg="es_instance_name must be specified and cannot be blank"
+  when: es_instance_name is not defined or es_instance_name == ''
+
+- fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined"
+  when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '')
+
+- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or (es_config['discovery.zen.ping.multicast.enabled'] is defined and es_config['discovery.zen.ping.multicast.enabled'])}}
+
+- debug: msg="WARNING - It is recommended you specify the parameter 'http.port' when multicast is disabled"
+  when: not multi_cast and es_config['http.port'] is not defined
+
+- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port' when multicast is disabled"
+  when: not multi_cast and es_config['transport.tcp.port'] is not defined
+
+- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts' when multicast is disabled"
+  when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined
+
+#If the user attempts to lock memory they must specify a heap size
+- fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified"
+  when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined
+
+#Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work
+- fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations"
+  when: es_enable_xpack and '"shield" in es_xpack_features' and es_api_basic_auth_username is not defined and es_api_basic_auth_username is not defined
+
+- set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}}
+- set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}}
+- set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}}
+- set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }}
+
+#Use systemd for the following distributions:
+#Ubuntu 15 and up
+#Debian 8 and up
+#Centos 7 and up
+#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied.
+
+
+- set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }}
+
+- set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}}
+  when: use_system_d
+#For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN.
+
+- set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }}
+- set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}}
+- set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}}
+- set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}}
+- set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }}

tasks/elasticsearch-plugins.yml

@@ -4,24 +4,20 @@
 # i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting.
 - set_fact: es_plugins_reinstall=true
   when: ((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none
-  tags:
-      - always
 
 - set_fact: list_command="list"
-  tags:
-      - always
+
 - set_fact: list_command="--list"
   when: es_version | version_compare('2.0', '<')
-  tags:
-      - always
 
 #List currently installed plugins
 - shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-"
   register: installed_plugins
   changed_when: False
+  ignore_errors: yes
   environment:
-      CONF_DIR: "{{ conf_dir }}"
-      ES_INCLUDE: "{{ instance_default_file }}"
+    CONF_DIR: "{{ conf_dir }}"
+    ES_INCLUDE: "{{ instance_default_file }}"
 
 #This needs to removes any currently installed plugins
 - name: Remove elasticsearch plugins
@@ -50,4 +46,4 @@
 
 #Set permissions on plugins directory
 - name: Set Plugin Directory Permissions
-  file: state=directory path={{ plugin_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes
+  file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes

tasks/elasticsearch-shield.yml

@@ -1,3 +0,0 @@
----
-
-

tasks/elasticsearch-templates.yml

@@ -12,22 +12,13 @@
   with_fileglob:
     - "{{ es_templates_fileglob }}"
 
-- set_fact: http_port=9200
-  tags:
-      - always
-
-- set_fact: http_port={{es_config['http.port']}}
-  when: es_config['http.port'] is defined
-  tags:
-      - always
-
 - name: Wait for elasticsearch to startup
-  wait_for: port={{http_port}} delay=10
+  wait_for: port={{es_api_port}} delay=10
 
 - name: Get template files 
   shell: find . -maxdepth 1 -type f | sed "s#\./##" | sed "s/.json//" chdir=/etc/elasticsearch/templates
   register: resultstemplate
 
 - name: Install template(s)
-  command: "curl -sL -XPUT http://localhost:{{http_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json"
+  command: "curl -sL -XPUT http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json"
   with_items: "{{ resultstemplate.stdout_lines }}"

tasks/elasticsearch.yml

@@ -1,23 +1,5 @@
 ---
 
-- set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}}
-  tags:
-      - always
-- set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}}
-  tags:
-      - always
-- set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}}
-  tags:
-      - always
-- set_fact: plugin_dir={{ es_plugin_dir }}/{{es_instance_name}}
-  tags:
-      - always
-- set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }}
-  tags:
-      - always
-
-- debug: msg="Node configuration {{ es_config }} "
-
 - name: Include optional user and group creation.
   when: (es_user_id is defined) and (es_group_id is defined)
   include: elasticsearch-optional-user.yml

tasks/main.yml

@@ -1,12 +1,14 @@
 ---
-- name: check-parameters
-  include: checkParameters.yml
-  tags:
-      - check
 - name: os-specific vars
   include_vars: "{{ansible_os_family}}.yml"
   tags:
       - always
+
+- name: check-set-parameters
+  include: elasticsearch-parameters.yml
+  tags:
+      - always
+
 - include: java.yml
   when: es_java_install
   tags:
@@ -25,10 +27,10 @@
   when: es_plugins is defined or es_plugins_reinstall
   tags:
       - plugins
-- include: elasticsearch-shield.yml
-  when: es_install_shield
+- include: xpack/elasticsearch-xpack.yml
+  when: es_enable_xpack
   tags:
-      - shield
+      - xpack
 - include: elasticsearch-service.yml
   tags:
       - service

tasks/xpack/elasticsearch-shield-file.yml

@@ -0,0 +1,62 @@
+---
+
+- set_fact: manage_file_users=false
+
+- set_fact: manage_file_users=true
+  when: es_users is defined and es_users.file is defined
+
+#List current users
+- name: List Users
+  shell: cat {{conf_dir}}/shield/users | awk -F':' '{print $1}'
+  register: current_file_users
+  when: manage_file_users
+
+- set_fact: users_to_remove={{ current_file_users.stdout_lines | difference ( es_users.file.keys()  ) }}
+  when: manage_file_users
+
+#Remove users
+- name: Remove Users
+  command: >
+    {{es_home}}/bin/shield/esusers userdel {{item}}
+  when: manage_file_users and (users_to_remove | length > 0)
+  with_items: "{{users_to_remove}}"
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_HOME: "{{es_home}}"
+
+
+#Add users
+- name: Add Users
+  command: >
+    {{es_home}}/bin/shield/esusers useradd {{item.key}} -p {{item.value.password}}
+  with_dict: "{{es_users.file}}"
+  when: manage_file_users and es_users.file.keys() | length > 0
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_HOME: "{{es_home}}"
+
+#Set passwords for all users declared - Required as the useradd will not change existing user passwords
+- name: Set User Passwords
+  command: >
+    {{es_home}}/bin/shield/esusers passwd {{item.key}} -p {{item.value.password}}
+  with_dict: "{{es_users.file}}"
+  when: manage_file_users and es_users.file.keys() | length > 0
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_HOME: "{{es_home}}"
+
+- set_fact: users_roles={{es_users.file | extract_role_users}}
+  when: manage_file_users
+
+#Copy Roles files
+- name: Copy roles.yml File for Instance
+  template: src=shield/roles.yml.j2 dest={{conf_dir}}/shield/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  when: es_roles is defined and es_roles.file is defined
+
+#Overwrite users_roles file
+- name: Copy User Roles
+  template: src=shield/users_roles.j2 dest={{conf_dir}}/shield/users_roles mode=0644 force=yes
+  when: manage_file_users and users_roles | length > 0
+
+#TODO: Support for mapping file
+

tasks/xpack/elasticsearch-shield.yml

@@ -0,0 +1,45 @@
+---
+
+#Test if we need to install shield
+
+- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep shield"
+  register: shield_installed
+  changed_when: False
+  ignore_errors: yes
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_INCLUDE: "{{ instance_default_file }}"
+
+
+#Install Shield if not installed
+- name: Install shield plugin
+  command: >
+    {{es_home}}/bin/plugin install shield
+  register: shield
+  failed_when: "'ERROR' in shield_installed.stdout"
+  changed_when: shield.rc == 1
+  when: shield_installed.rc == 1
+  notify: restart elasticsearch
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_INCLUDE: "{{ instance_default_file }}"
+
+#TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6
+
+
+#Ensure shield conf directory is created
+- name: Ensure shield conf directory exists
+  file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }}
+
+#-----------------------------FILE BASED REALM----------------------------------------
+
+- include: elasticsearch-shield-file.yml
+  when: (es_users is defined and es_users.file) or (es_roles is defined and es_roles.file is defined)
+
+#-----------------------------NATIVE BASED REALM----------------------------------------
+# The native realm requires the node to be started so we do as a handler
+- command: /bin/true
+  notify: load-native-realms
+  when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)
+
+

tasks/xpack/elasticsearch-xpack.yml

@@ -0,0 +1,33 @@
+---
+#Check if license is installed
+- name: Check License is installed
+  shell: >
+    {{es_home}}/bin/plugin list | tail -n +2 | grep license
+  register: license_installed
+  ignore_errors: yes
+  changed_when: False
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_INCLUDE: "{{ instance_default_file }}"
+
+#Install License if not installed
+- name: Install license plugin
+  command: >
+    {{es_home}}/bin/plugin install license
+  register: license
+  failed_when: "'ERROR' in license_installed  .stdout"
+  changed_when: license.rc == 1
+  when: license_installed.rc == 1
+  notify: restart elasticsearch
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_INCLUDE: "{{ instance_default_file }}"
+
+
+- name: Set Plugin Directory Permissions
+  file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes
+
+- include: elasticsearch-shield.yml
+  when: '"shield" in es_xpack_features'
+
+#Any other xpacks plugins requiring configuration to be entered here

templates/elasticsearch.yml.j2

@@ -21,5 +21,3 @@ path.data: {{ data_dirs | array_to_str }}
 path.work: {{ work_dir }}
 
 path.logs: {{ log_dir }}
-
-path.plugins: {{ plugin_dir }}

templates/shield/roles.yml.j2

@@ -0,0 +1 @@
+{{ es_roles.file | to_nice_yaml }}

templates/shield/users_roles.j2

@@ -0,0 +1 @@
+{{users_roles | join("\n") }}

test/integration/helpers/serverspec/multi_spec.rb

@@ -173,28 +173,7 @@ shared_examples 'multi::init' do  |es_version,plugins|
     end
   end
 
-  #Multi node plugin tests
-  describe file('/opt/elasticsearch/plugins/node1') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe file('/opt/elasticsearch/plugins/master') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-
   for plugin in plugins
-    describe file('/opt/elasticsearch/plugins/node1/'+plugin) do
-      it { should be_directory }
-      it { should be_owned_by 'elasticsearch' }
-    end
-
-    describe file('/opt/elasticsearch/plugins/master/'+plugin) do
-      it { should be_directory }
-      it { should be_owned_by 'elasticsearch' }
-    end
 
     describe command('curl -s localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do
       its(:exit_status) { should eq 0 }

test/integration/multi.yml

@@ -8,5 +8,4 @@
   vars:
     es_scripts: true
     es_templates: true
-    es_plugin_dir: "/opt/elasticsearch/plugins"
 #Plugins installed for this test are specified in .kitchen.yml under suite

test/integration/xpack-2x/serverspec/xpack_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe 'XPack Tests v 2.x' do
+
+  describe user('elasticsearch') do
+    it { should exist }
+  end
+
+end
+

test/integration/xpack-2x/xpack.yaml

@@ -0,0 +1,2 @@
+---
+- host: test-kitchen

test/integration/xpack.yml

@@ -0,0 +1,68 @@
+---
+- name: Elasticsearch Xpack tests
+  hosts: localhost
+  roles:
+    - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "shield_node" }
+  vars:
+    es_templates: false
+    es_enable_xpack: true
+    es_xpack_features:
+      - shield
+      - watcher
+    es_api_basic_auth_username: es_admin
+    es_api_basic_auth_password: changeMe
+    es_users:
+      native:
+        kibana4_server:
+          password: changeMe
+          roles:
+            - kibana4_server
+      file:
+        es_admin:
+          password: changeMe
+          roles:
+            - admin
+        testUser:
+          password: changeMeAlso!
+          roles:
+            - power_user
+            - user
+    es_roles:
+      file:
+        admin:
+          cluster:
+            - all
+          indices:
+            - names: '*'
+              privileges:
+                - all
+        power_user:
+          cluster:
+            - monitor
+          indices:
+            - names: '*'
+              privileges:
+                - all
+        user:
+          indices:
+            - names: '*'
+              privileges:
+                - read
+        kibana4_server:
+          cluster:
+              - monitor
+          indices:
+            - names: '.kibana'
+              privileges:
+                - all
+      native:
+        logstash:
+          cluster:
+            - manage_index_templates
+          indices:
+            - names: 'logstash-*'
+              privileges:
+                - write
+                - delete
+                - create_index
+