Merge pull request #234 from gingerwizard/master

5.x Compatibility

.gitignore

@@ -1,3 +1,4 @@
+/vars/Debian.yml
 .kitchen/
 license.json
 *.pyc

.kitchen.yml

@@ -64,35 +64,33 @@ platforms:
       use_sudo: false
 
 suites:
-  - name: standard-2x
+  - name: standard-5x
     provisioner:
       playbook: test/integration/standard.yml
     run_list:
     attributes:
-  - name: package-2x
+  - name: package-5x
     run_list:
     attributes:
       extra_vars:
         es_plugins:
-          - plugin: lmenezes/elasticsearch-kopf
+          - plugin: ingest-geoip
-            version: master
     provisioner:
       playbook: test/integration/package.yml
-  - name: config-2x
+  - name: config-5x
     run_list:
     attributes:
     provisioner:
       playbook: test/integration/config.yml
-  - name: multi-2x
+  - name: multi-5x
     run_list:
     attributes:
       extra_vars:
         es_plugins:
-          - plugin: lmenezes/elasticsearch-kopf
+          - plugin: ingest-geoip
-            version: master
     provisioner:
       playbook: test/integration/multi.yml
-  - name: xpack-2x
+  - name: xpack-5x
     run_list:
     attributes:
     provisioner:

README.md

@@ -1,6 +1,6 @@
 # ansible-elasticsearch
 
-**THIS ROLE IS UNDER DEVELOPMENT FOR 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.**
+**THIS ROLE IS FOR 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.**
 
 Ansible role for 5.x Elasticsearch.  Currently this works on Debian and RedHat based linux systems.  Tested platforms are:
 
@@ -49,14 +49,14 @@ The use of a map ensures the Ansible playbook does not need to be updated to ref
 
 In addition to the es_config map, several other parameters are supported for additional functions e.g. script installation.  These can be found in the role's defaults/main.yml file.
 
-The following illustrates applying configuration parameters to an Elasticsearch instance.  By default, Elasticsearch 2.4.3 is installed.
+The following illustrates applying configuration parameters to an Elasticsearch instance.  By default, Elasticsearch 5.1.2is installed.
 
 ```
 - name: Elasticsearch with custom configuration
   hosts: localhost
   roles:
     #expand to all available parameters
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", 
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", 
     es_config: {
         node.name: "node1", 
         cluster.name: "custom-cluster",
@@ -65,17 +65,18 @@ The following illustrates applying configuration parameters to an Elasticsearch
         transport.tcp.port: 9301,
         node.data: false,
         node.master: true,
-        bootstrap.mlockall: true,
+        bootstrap.memory_lock: true,
-        discovery.zen.ping.multicast.enabled: false } 
+        } 
     }
   vars:
     es_scripts: false
     es_templates: false
     es_version_lock: false
     es_heap_size: 1g
+    es_api_port:9201
 ```
 `
-The role utilises Elasticsearch version defaults.  Multicast is therefore disabled for 5.x.  The following should be set to ensure a successful cluster forms.
+The role utilises Elasticsearch version defaults.  The following should be set to ensure a successful cluster forms.
 
 * ```es_config['http.port']``` - the http port for the node
 * ```es_config['transport.tcp.port']``` - the transport port for the node
@@ -95,7 +96,7 @@ A more complex example:
   hosts: localhost
   roles:
     #expand to all available parameters
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", 
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", 
     es_config: {
         node.name: "node1", 
         cluster.name: "custom-cluster", 
@@ -104,8 +105,8 @@ A more complex example:
         transport.tcp.port: 9301, 
         node.data: false, 
         node.master: true, 
-        bootstrap.mlockall: true, 
+        bootstrap.memory_lock: true, 
-        discovery.zen.ping.multicast.enabled: false } 
+        } 
     }
   vars:
     es_scripts: false
@@ -117,27 +118,25 @@ A more complex example:
     es_version_lock: false
     es_start_service: false
     es_plugins_reinstall: false
+    es_api_port:9201
     es_plugins:
-        - plugin: license
+        - plugin: ingest-geoip
-        - plugin: marvel-agent
-        - plugin: lmenezes/elasticsearch-kopf
-          version: master
           proxy_host: proxy.example.com
           proxy_port: 8080
 ```
 
 #### Important Note
 
-The role uses es_api_host and es_api_port to communicate with the node for actions only achievable via http e.g. to install templates.  These default to "localhost" and 9200 respectively.  
+**The role uses es_api_host and es_api_port to communicate with the node for actions only achievable via http e.g. to install templates and to check the NODE IS ACTIVE.  These default to "localhost" and 9200 respectively.  
-If the node is deployed to bind on either a different host or port, these must be changed.
+If the node is deployed to bind on either a different host or port, these must be changed.**
 
 ### Multi Node Server Installations
 
 The application of the elasticsearch role results in the installation of a node on a host. Specifying the role multiple times for a host therefore results in the installation of multiple nodes for the host. 
 
-An example of a two server deployment, each with 1 node on one server and 2 nodes on another.  The first server holds the master and is thus declared first.  Whilst not mandatory, this is 
+An example of a two server deployment is shown below.  The first server holds the master and is thus declared first.  Whilst not mandatory, this is recommended in any multi node cluster configuration.  The second server hosts two data nodes.
-recommended in any multi node cluster configuration.
 
+**Note the structure of the below playbook for the data nodes.  Whilst a more succinct structures are possible which allow the same role to be applied to a host multiple times, we have found the below structure to be the most reliable with respect to var behaviour.  This is the tested approach.**
 
 ```
 - hosts: master_nodes
@@ -145,14 +144,13 @@ recommended in any multi node cluster configuration.
     - { role: elasticsearch, es_instance_name: "node1", es_heap_size: "1g",
     es_config: {
         cluster.name: "test-cluster",
-        "discovery.zen.ping.multicast.enabled": false,
         discovery.zen.ping.unicast.hosts: "elastic02:9300",
         http.port: 9200,
         transport.tcp.port: 9300,
         node.data: false,
         node.master: true,
-        bootstrap.mlockall: false,
+        bootstrap.memory_lock: false,
-        discovery.zen.ping.multicast.enabled: false }
+        }
     }
   vars:
     es_scripts: false
@@ -160,43 +158,55 @@ recommended in any multi node cluster configuration.
     es_version_lock: false
     ansible_user: ansible
     es_plugins:
-     - plugin: elasticsearch/license
+     - plugin: ingest-geoip
-       version: latest
+
+
 
 - hosts: data_nodes
   roles:
     - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch", 
     es_config: {
-        "discovery.zen.ping.multicast.enabled": false,
         discovery.zen.ping.unicast.hosts: "elastic02:9300",
         http.port: 9200,
         transport.tcp.port: 9300,
         node.data: true,
         node.master: false,
-        bootstrap.mlockall: false,
+        bootstrap.memory_lock: false,
-        cluster.name: "test-cluster",
+        cluster.name: "test-cluster"
-        discovery.zen.ping.multicast.enabled: false } 
         } 
-    - { role: elasticsearch, es_instance_name: "node2", 
-    es_config: {
-        "discovery.zen.ping.multicast.enabled": false,
-        discovery.zen.ping.unicast.hosts: "elastic02:9300",
-        http.port: 9201,
-        transport.tcp.port: 9301,
-        node.data: true,
-        node.master: false,
-        bootstrap.mlockall: false,
-        cluster.name: "test-cluster",
-        discovery.zen.ping.multicast.enabled: false } 
     }
   vars:
     es_scripts: false
     es_templates: false
     es_version_lock: false
     ansible_user: ansible
+    es_api_port: 9200
     es_plugins:
-     - plugin: elasticsearch/license
+    - plugin: ingest-geoip
-       version: latest
+    
+    
+- hosts: data_nodes
+  roles:
+    - { role: elasticsearch, es_instance_name: "node2", es_api_port:9201,
+    es_config: {
+        discovery.zen.ping.unicast.hosts: "elastic02:9300",
+        http.port: 9201,
+        transport.tcp.port: 9301,
+        node.data: true,
+        node.master: false,
+        bootstrap.memory_lock: false,
+        cluster.name: "test-cluster",
+        } 
+    }
+  vars:
+    es_scripts: false
+    es_templates: false
+    es_version_lock: false
+    es_api_port: 9201
+    ansible_user: ansible
+    es_plugins:
+    - plugin: ingest-geoip
+
 ```
 
 Parameters can additionally be assigned to hosts using the inventory file if desired.
@@ -211,10 +221,14 @@ ansible-playbook -i hosts ./your-playbook.yml
 
 ### Installing X-Pack Features
 
-X-Pack features, such as Shield, are supported for Elasticsearch 2.4 only. This feature is currently experimental.  To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`.  The following additional parameters allow X-Pack to be configured:
+X-Pack features, such as Security, are supported. This feature is currently experimental.  To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`.  
+
+The parameter `es_xpack_features` by default enables all features i.e. it defaults to ["alerting","monitoring","graph","security"]
+
+The following additional parameters allow X-Pack to be configured:
 
 * ```es_message_auth_file``` System Key field to allow message authentication. This file should be placed in the 'files' directory.
-* ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/shield/current/mapping-roles.html)
+* ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/x-pack/current/mapping-roles.html)
 
 ```
 es_role_mapping:
@@ -247,7 +261,7 @@ es_users:
 ```
             
             
-* ```es_roles``` - Elasticsearch roles can be declared here as yml. Two sub keys 'native' and 'file' determine how the role is created i.e. either through a file or http(native) call.  Beneath each key list the roles with appropriate permissions, using the file based format described [here] (https://www.elastic.co/guide/en/shield/current/_file_based_roles.html) e.g.
+* ```es_roles``` - Elasticsearch roles can be declared here as yml. Two sub keys 'native' and 'file' determine how the role is created i.e. either through a file or http(native) call.  Beneath each key list the roles with appropriate permissions, using the file based format described [here] (https://www.elastic.co/guide/en/x-pack/current/file-realm.html) e.g.
 
 ```
 es_roles:
@@ -300,19 +314,25 @@ X-Pack configuration parameters can be added to the elasticsearch.yml file using
 
 For a full example see [here](https://github.com/elastic/ansible-elasticsearch/blob/master/test/integration/xpack.yml)
 
+####Important Note for Native Realm Configuration
+
+In order for native users and roles to be configured, the role calls the Elasticsearch API.  Given security is installed this requires definition of two parameters:
+
+* ```es_api_basic_auth_username``` - admin username
+* ```es_api_basic_auth_password``` - admin password
+
+These can either be set to a user declared in the file based realm, with admin permissions, or the default "elastic" superuser (default password is changeme).
 
 
 ### Additional Configuration
 
 Additional parameters to es_config allow the customization of the Java and Elasticsearch versions, in addition to role behaviour. Options include:
 
-Following variables affect the versions installed:
-
 * ```es_major_version``` (e.g. "5.1" ). Should be consistent with es_version. For versions >= 5.0 this must be "5.x".
-* ```es_version``` (e.g. "5.1.1").  
+* ```es_version``` (e.g. "5.1.2").  
 * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost".
-* ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200.
+* ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200**
-* ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Shield is enabled. Ensure this user is admin.
+* ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Security is enabled. Ensure this user is admin.
 * ```es_api_basic_auth_password``` The password associated with the user declared in `es_api_basic_auth_username`
 * ```es_start_service``` (true (default) or false)
 * ```es_plugins_reinstall``` (true or false (default) )
@@ -320,20 +340,16 @@ Following variables affect the versions installed:
 ```yml
   es_plugins:
     - plugin: elasticsearch-cloud-aws
-      version: 2.5.0
 ```
 * ```es_allow_downgrades``` For development purposes only. (true or false (default) )
 * ```es_java_install``` If set to false, Java will not be installed. (true (default) or false)
 * ```update_java``` Updates Java to the latest version. (true or false (default))
-* ```es_java_opts``` an array of java options. E.g.:
+* ```es_max_map_count``` maximum number of VMA (Virtual Memory Areas) a process can own. Defaults to 262144.
-```yml
+* ```es_max_open_files``` the maximum file descriptor number that can be opened by this process. Defaults to 65536.
-es_java_opts:
-  - "-Djava.io.tmpdir=/data/tmp/elasticsearch"
-```
 
-Earlier examples illustrate the installation of plugins using `es_plugins`.  For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version.  For community based plugins include the full path e.g. "lmenezes/elasticsearch-kopf" and the appropriate version for the target version of Elasticsearch.  This approach should NOT be used for X-Pack related plugins e.g. Shield.  See X-Pack below for details here.
+Earlier examples illustrate the installation of plugins using `es_plugins`.  For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version.  For community based plugins include the full url.  This approach should NOT be used for the X-Pack plugin.  See X-Pack below for details here.
  
-If installing Marvel or Watcher, ensure the license plugin is also specified.  Shield configuration is currently not supported but planned for later versions.
+If installing Monitoring or Alerting, ensure the license plugin is also specified.  Security configuration is currently not supported but planned for later versions.
 
 * ```es_user``` - defaults to elasticsearch.
 * ```es_group``` - defaults to elasticsearch.
@@ -348,7 +364,6 @@ controlled by the following parameters:
 * ```es_pid_dir``` - defaults to "/var/run/elasticsearch".
 * ```es_data_dirs``` - defaults to "/var/lib/elasticsearch".  This can be a list or comma separated string e.g. ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"] or "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2"
 * ```es_log_dir``` - defaults to "/var/log/elasticsearch".
-* ```es_work_dir``` - defaults to "/tmp/elasticsearch".
 * ```es_restart_on_change``` - defaults to true.  If false, changes will not result in Elasticsearch being restarted.
 * ```es_plugins_reinstall``` - defaults to false.  If true, all currently installed plugins will be removed from a node.  Listed plugins will then be re-installed.  
 
@@ -368,7 +383,6 @@ To define proxy only for a particular plugin during its installation:
 ```
   es_plugins:
     - plugin: elasticsearch-cloud-aws
-      version: 2.5.0
       proxy_host: proxy.example.com
       proxy_port: 8080
 ```
@@ -385,7 +399,7 @@ all supported platforms.
 * The role aims to be idempotent.  Running the role multiple times, with no changes, should result in no state change on the server.  If the configuration is changed, these will be applied and 
 Elasticsearch restarted where required.
 * Systemd is used for Ubuntu versions >= 15, Debian >=8, Centos >=7.  All other versions use init for service scripts.
-* In order to run x-pack tests a license file with shield enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests.
+* In order to run x-pack tests a license file with security enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests.
 
 ## IMPORTANT NOTES RE PLUGIN MANAGEMENT
 

defaults/main.yml

@@ -1,10 +1,10 @@
 ---
-es_major_version: "2.x"
+es_major_version: "5.x"
-es_version: "2.4.3"
+es_version: "5.1.2"
 es_version_lock: false
 es_use_repository: true
-es_apt_key: "https://packages.elasticsearch.org/GPG-KEY-elasticsearch"
+es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch"
-es_apt_url: "deb https://packages.elastic.co/elasticsearch/{{ es_major_version }}/debian stable main"
+es_apt_url: "deb https://artifacts.elastic.co/packages/{{ es_major_version }}/apt stable main"
 es_apt_url_old: "deb http://packages.elastic.co/elasticsearch/{{ es_major_version }}/debian stable main"
 es_start_service: true
 es_java_install: true
@@ -20,11 +20,11 @@ es_config: {}
 es_pid_dir: "/var/run/elasticsearch"
 es_data_dirs: "/var/lib/elasticsearch"
 es_log_dir: "/var/log/elasticsearch"
-es_work_dir: "/tmp/elasticsearch"
 es_max_open_files: 65536
+es_max_map_count: 262144
 es_allow_downgrades: false
 es_enable_xpack: false
-es_xpack_features: []
+es_xpack_features: ["alerting","monitoring","graph","security"]
 #These are used for internal operations performed by ansible.
 #They do not effect the current configuration
 es_api_host: "localhost"
@@ -34,6 +34,5 @@ es_api_port: 9200
 # to allow the role to be conditionally played with a when condition.
 pid_dir: ''
 log_dir: ''
-work_dir: ''
 conf_dir: ''
 data_dirs: ''

filter_plugins/custom.py

@@ -28,10 +28,16 @@ def extract_role_users(users={}):
                 role_users.append(role+":"+user)
     return role_users
 
-
 def filename(filename=''):
     return os.path.splitext(os.path.basename(filename))[0]
 
+def filter_reserved(user_roles={}):
+    not_reserved = []
+    for user_role,details in user_roles.items():
+        if not "metadata" in details or not "_reserved" in details["metadata"] or not details["metadata"]["_reserved"]:
+            not_reserved.append(user_role)
+    return not_reserved
+
 
 class FilterModule(object):
     def filters(self):
@@ -39,5 +45,6 @@ class FilterModule(object):
         'append_to_list':append_to_list,
         'array_to_str':array_to_str,
         'extract_role_users':extract_role_users,
+        'filter_reserved':filter_reserved,
         'filename':filename}
 

handlers/elasticsearch-templates.yml

@@ -17,7 +17,7 @@
     status_code: 200
     body_format: json
     body: "{{ lookup('file', item.path) }}"
-  when: not es_enable_xpack or not es_xpack_features is defined or not '"shield" in es_xpack_features'
+  when: not es_enable_xpack or not es_xpack_features is defined or not '"security" in es_xpack_features'
   with_items: "{{ templates.files }}"
 
 - name: Install templates with auth
@@ -30,5 +30,5 @@
     force_basic_auth: yes
     body_format: json
     body: "{{ lookup('file', item.path) }}"
-  when: es_enable_xpack and es_xpack_features is defined and '"shield" in es_xpack_features'
+  when: es_enable_xpack and es_xpack_features is defined and '"security" in es_xpack_features'
   with_items: "{{ templates.files }}"

handlers/main.yml

@@ -7,10 +7,10 @@
   when: es_restart_on_change and es_start_service and ((plugin_installed is defined and plugin_installed.changed) or (config_updated is defined and config_updated.changed) or (xpack_state.changed) or (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed))
 
 # All security specific actions should go in here
-- name: activate-shield
+- name: activate-security
-  include: ./handlers/shield/elasticsearch-shield.yml
+  include: ./handlers/security/elasticsearch-security.yml
 
-#Templates are a handler as they need to come after a restart e.g. suppose user removes shield on a running node and doesn't
+#Templates are a handler as they need to come after a restart e.g. suppose user removes security on a running node and doesn't
 #specify es_api_basic_auth_username and es_api_basic_auth_password.  The templates will subsequently not be removed if we don't wait for the node to restart.
 #Templates done after restart therefore - as a handler.
 

handlers/security/elasticsearch-security-native.yml

@@ -14,12 +14,12 @@
 - name: Wait 15 seconds for the Native Relm to come up
   pause: seconds=15
 
-#If the node has just has shield installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load
+#If the node has just has security installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load
 
 #List current users
 - name: List Native Users
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_shield/user
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user
     method: GET
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
@@ -28,19 +28,18 @@
   register: user_list_response
   when: manage_native_users
 
-
+#Current users not inc. those reserved
-- set_fact: current_users={{user_list_response.json.keys() | list}}
+- set_fact: current_users={{ user_list_response.json | filter_reserved }}
   when: manage_native_users
 
 #Identify non declared users
-
 - set_fact: users_to_remove={{ current_users | difference ( es_users.native.keys() ) }}
   when: manage_native_users
 
 #Delete all non required users
 - name: Delete Native Users
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item}}
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}
     method: DELETE
     status_code: 200
     user: "{{es_api_basic_auth_username}}"
@@ -49,11 +48,10 @@
   when: manage_native_users and users_to_remove | length > 0
   with_items: "{{users_to_remove}}"
 
-
 #Overwrite all other users
 - name: Update Native Users
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item.key}}
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item.key}}
     method: POST
     body_format: json
     body: "{{item.value | to_json}}"
@@ -65,11 +63,10 @@
   no_log: True
   with_dict: "{{es_users.native}}"
 
-#List current roles
+#List current roles not. inc those reserved
-
 - name: List Native Roles
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_shield/role
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role
     method: GET
     body_format: json
     user: "{{es_api_basic_auth_username}}"
@@ -79,11 +76,8 @@
   register: role_list_response
   when: manage_native_roles
 
-#Identify undeclared roles
+- set_fact: current_roles={{ role_list_response.json | filter_reserved }}
-
+  when: manage_native_roles
-- set_fact: current_roles={{role_list_response.json.keys() | list}}
-  when: manage_native_users
-
 - debug: msg="{{current_roles}}"
 
 - set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys()  ) }}
@@ -93,7 +87,7 @@
 #Delete all non required roles
 - name: Delete Native Roles
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item}}
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}
     method: DELETE
     status_code: 200
     user: "{{es_api_basic_auth_username}}"
@@ -106,7 +100,7 @@
 #Update other roles
 - name: Update Native Roles
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item.key}}
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item.key}}
     method: POST
     body_format: json
     body: "{{item.value | to_json}}"

handlers/security/elasticsearch-security.yml

@@ -6,9 +6,9 @@
   wait_for: host={{es_api_host}} port={{es_api_port}} delay=10
   
 - name: activate-license
-  include: ./handlers/shield/elasticsearch-xpack-activation.yml
+  include: ./handlers/security/elasticsearch-xpack-activation.yml
   when: es_enable_xpack and es_xpack_license is defined and es_xpack_license != ''
 
 - name: load-native-realms
-  include: ./handlers/shield/elasticsearch-shield-native.yml
+  include: ./handlers/security/elasticsearch-security-native.yml
   when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)

handlers/security/elasticsearch-xpack-activation.yml

@@ -1,6 +1,6 @@
 ---
 
-- name: Activate ES license (without shield authentication)
+- name: Activate ES license (without security authentication)
   uri:
     method: PUT
     url: "http://{{es_api_host}}:{{es_api_port}}/_license?acknowledge=true"
@@ -9,13 +9,13 @@
     return_content: yes
   register: license_activated
   no_log: True
-  when: not '"shield" in es_xpack_features'
+  when: not '"security" in es_xpack_features'
   failed_when: >
     license_activated.status != 200 or
     license_activated.json.license_status is not defined or
     license_activated.json.license_status != 'valid'
 
-- name: Activate ES license (with shield authentication)
+- name: Activate ES license (with security authentication)
   uri:
     method: PUT
     url: "http://{{es_api_host}}:{{es_api_port}}/_license?acknowledge=true"
@@ -27,7 +27,7 @@
     return_content: yes
   register: license_activated
   no_log: True
-  when: '"shield" in es_xpack_features'
+  when: '"security" in es_xpack_features'
   failed_when: >
     license_activated.status != 200 or
     license_activated.json.license_status is not defined or

meta/main.yml

@@ -8,7 +8,7 @@ galaxy_info:
   company: "Elastic.co"
   license: "license (Apache)"
   # Require 1.6 for apt deb install
-  min_ansible_version: 2.1.0
+  min_ansible_version: 2.2.0
   platforms:
   - name: EL
     versions:

meta/main.yml.orig

@@ -1,30 +0,0 @@
----
-
-allow_duplicates: yes
-
-galaxy_info:
-  author: Robin Clarke, Jakob Reiter, Dale McDiarmid
-  description: Elasticsearch for Linux
-  company: "Elastic.co"
-  license: "license (Apache)"
-<<<<<<< HEAD
-  # Require 1.6 for apt deb install
-  min_ansible_version: 2.1.0
-=======
-  min_ansible_version: 2.0
->>>>>>> elastic/master
-  platforms:
-  - name: EL
-    versions:
-    - 6
-    - 7
-  - name: Debian
-    versions:
-    - all
-  - name: Ubuntu
-    versions:
-    - all
-  categories:
-    - system
-
-dependencies: []

tasks/elasticsearch-config.yml

@@ -6,7 +6,6 @@
   file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }}
   with_items:
     - "{{pid_dir}}"
-    - "{{work_dir}}"
     - "{{log_dir}}"
     - "{{conf_dir}}"
 
@@ -48,21 +47,14 @@
   - restart elasticsearch
 
 #Copy the logging.yml
-- name: Copy Logging.yml File for Instance
-  template: src=logging.yml.j2 dest={{conf_dir}}/logging.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
-  notify: restart elasticsearch
-  when: es_version | version_compare('5.0', '<')
 
 - name: Copy log4j2.properties File for Instance
   template: src=log4j2.properties.j2 dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   notify: restart elasticsearch
-  when: es_version | version_compare('5.0', '>=')
 
-# Copy the JVM Options (5.x only)
 - name: Copy jvm.options File for Instance
   template: src=jvm.options.j2 dest={{conf_dir}}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   notify: restart elasticsearch
-  when: es_version | version_compare('5.0', '>=')
 
 #Clean up un-wanted package scripts to avoid confusion
 
@@ -86,8 +78,8 @@
 - name: Delete Default Logging File
   file: dest=/etc/elasticsearch/logging.yml state=absent
 
-- name: Delete Default Logging File (5.x)
+- name: Delete Default Logging File
   file: dest=/etc/elasticsearch/log4j2.properties state=absent
 
-- name: Delete Default JVM Options File (5.x)
+- name: Delete Default JVM Options File
   file: dest=/etc/elasticsearch/jvm.options state=absent

tasks/elasticsearch-parameters.yml

@@ -6,33 +6,30 @@
 - fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined"
   when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '')
 
-- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or (es_config['discovery.zen.ping.multicast.enabled'] is defined and es_config['discovery.zen.ping.multicast.enabled'])}}
+- debug: msg="WARNING - It is recommended you specify the parameter 'http.port'"
+  when: es_config['http.port'] is not defined
 
-- debug: msg="WARNING - It is recommended you specify the parameter 'http.port' when multicast is disabled"
+- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port'"
-  when: not multi_cast and es_config['http.port'] is not defined
+  when: es_config['transport.tcp.port'] is not defined
 
-- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port' when multicast is disabled"
+- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts'"
-  when: not multi_cast and es_config['transport.tcp.port'] is not defined
+  when: es_config['discovery.zen.ping.unicast.hosts'] is not defined
-
-- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts' when multicast is disabled"
-  when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined
 
 #If the user attempts to lock memory they must specify a heap size
-- fail: msg="If locking memory with bootstrap.mlockall (or bootstrap.memory_lock) a heap size must be specified"
+- fail: msg="If locking memory with bootstrap.memory_lock a heap size must be specified"
-  when: (es_config['bootstrap.mlockall'] is defined or es_config['bootstrap.memory_lock'] is defined) and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined
+  when: es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True and es_heap_size is not defined
 
-#Don't support xpack on versions < 2.0
+#Check if working with security we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work
-- fail: msg="Use of the xpack notation is not supported on versions < 2.0.  Marvel-agent and watcher can be installed as plugins.  Version > 2.0 is required for shield."
+- fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations"
-  when: es_enable_xpack and es_version | version_compare('2.0', '<')
+  when: es_enable_xpack and ("security" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined
-
-#Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work
-- fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations"
-  when: es_enable_xpack and ("shield" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined
 
 - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}}
 - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}}
 - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}}
-- set_fact: m_lock_enabled={{ (es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True) or (es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True) }}
+- set_fact: m_lock_enabled={{ es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True }}
+
+#TODO - if transport.host is not local maybe error on boostrap checks
+
 
 #Use systemd for the following distributions:
 #Ubuntu 15 and up
@@ -40,7 +37,6 @@
 #Centos 7 and up
 #Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied.
 
-
 - set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }}
 
 - set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}}
@@ -50,5 +46,4 @@
 - set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }}
 - set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}}
 - set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}}
-- set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}}
 - set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }}

tasks/elasticsearch-plugins.yml

@@ -7,12 +7,9 @@
 
 - set_fact: list_command="list"
 
-- set_fact: list_command="--list"
-  when: es_version | version_compare('2.0', '<')
-
 #List currently installed plugins - ignore xpack if > v 2.0
 - name: Check installed elasticsearch plugins
-  shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin {{list_command}}{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %}{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}"
+  shell: "{{es_home}}/bin/elasticsearch-plugin list | grep -vE 'x-pack'"
   register: installed_plugins
   failed_when: "'ERROR' in installed_plugins.stdout"
   changed_when: False
@@ -23,7 +20,7 @@
 
 # This removes any currently installed plugins (to prevent errors when reinstalling)
 - name: Remove elasticsearch plugins
-  command: "{{es_home}}/bin/{{ es_binary_prefix }}plugin remove {{item}} --silent"
+  command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent"
   ignore_errors: yes
   with_items: "{{ installed_plugins.stdout_lines | default([]) }}"
   when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0]
@@ -34,8 +31,7 @@
     ES_INCLUDE: "{{ instance_default_file }}"
 
 - name: Install elasticsearch plugins
-  command: >
+  command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.plugin }} --batch --silent {% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -DproxyHost={{ item.proxy_host }} -DproxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -DproxyHost={{ es_proxy_host }} -DproxyPort={{ es_proxy_port }} {% endif %}"
-    {{es_home}}/bin/{{ es_binary_prefix }}plugin install {{ item.plugin }}{% if item.version is defined and item.version != '' %}/{{ item.version }}{% endif %} {% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -DproxyHost={{ item.proxy_host }} -DproxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -DproxyHost={{ es_proxy_host }} -DproxyPort={{ es_proxy_port }} {% endif %} --silent
   register: plugin_installed
   failed_when: "'ERROR' in plugin_installed.stdout"
   changed_when: plugin_installed.rc == 0

tasks/elasticsearch.yml

@@ -1,18 +1,9 @@
 ---
 
-- set_fact: es_binary_prefix=""
-
-- set_fact: es_binary_prefix="elasticsearch-"
-  when: es_version | version_compare('5.0', '>=')
-
 - name: Include optional user and group creation.
   when: (es_user_id is defined) and (es_group_id is defined)
   include: elasticsearch-optional-user.yml
 
-#- name: Include specific Elasticsearch
-#  include: "elasticsearch-{{ansible_os_family}}.yml"
-
-#Install OS specific elasticsearch - this can be abbreviated in version 2.0.0
 - name: Include specific Elasticsearch
   include: elasticsearch-Debian.yml
   when: ansible_os_family == 'Debian'

tasks/java.yml

@@ -1,6 +1,5 @@
 ---
 
-
 - set_fact: java_state="present"
 
 - set_fact: java_state="latest"

tasks/main.yml

@@ -13,26 +13,36 @@
   when: es_java_install
   tags:
       - java
+
 - include: elasticsearch.yml
   tags:
       - install
+
 - include: elasticsearch-config.yml
   tags:
       - config
+
 - include: elasticsearch-scripts.yml
   when: es_scripts
   tags:
       - scripts
+
 - include: elasticsearch-plugins.yml
   when: es_plugins is defined or es_plugins_reinstall
   tags:
       - plugins
+
   #We always execute xpack as we may need to remove features
 - include: xpack/elasticsearch-xpack.yml
   tags:
       - xpack
+
 - include: elasticsearch-templates.yml
   when: es_templates
   tags:
       - templates
+
 - meta: flush_handlers
+
+- name: Wait for elasticsearch to startup
+  wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1

tasks/xpack/elasticsearch-xpack-install.yml

@@ -1,23 +1,23 @@
 ---
 
 #Test if feature is installed
-- shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin list{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %} | grep {{item}}"
+- shell: "{{es_home}}/bin/elasticsearch-plugin list | grep x-pack"
-  register: feature_installed
+  register: x_pack_installed
   changed_when: False
-  failed_when: "'ERROR' in feature_installed.stdout"
+  failed_when: "'ERROR' in x_pack_installed.stdout"
   ignore_errors: yes
   environment:
     CONF_DIR: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
 
-#Remove Plugin if installed and its not been requested or the ES version has changed
+#Remove X-Pack if installed and its not been requested or the ES version has changed
-- name: Remove {{item}} plugin
+- name: Remove x-pack plugin
-  command: {{es_home}}/bin/plugin remove {{item}}
+  command: "{{es_home}}/bin/elasticsearch-plugin remove x-pack"
   register: xpack_state
   failed_when: "'ERROR' in xpack_state.stdout"
   changed_when: xpack_state.rc == 0
-  when: feature_installed.rc == 0 and (not es_enable_xpack or not '"{{item}}" in es_xpack_features' or es_version_changed)
+  when: x_pack_installed.rc == 0 and (not es_enable_xpack or es_version_changed)
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"
@@ -25,13 +25,13 @@
 
 
 #Install plugin if not installed, or the es version has changed (so removed above), and its been requested
-- name: Install {{item}} plugin
+- name: Install x-pack plugin
   command: >
-    {{es_home}}/bin/{{ es_binary_prefix }}plugin install {{item}}
+    {{es_home}}/bin/elasticsearch-plugin install --silent --batch x-pack {% if es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} {% endif %}
   register: xpack_state
   failed_when: "'ERROR' in xpack_state.stdout"
   changed_when: xpack_state.rc == 0
-  when: (feature_installed.rc == 1 or es_version_changed) and es_enable_xpack and "{{item}}" in es_xpack_features
+  when: (x_pack_installed.rc == 1 or es_version_changed) and es_enable_xpack
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"

tasks/xpack/elasticsearch-xpack.yml

@@ -2,52 +2,10 @@
 
 - set_fact: es_version_changed={{ ((elasticsearch_install_from_package is defined and (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) }}
 
-#enabling xpack installs the license. Not a xpack feature and does not need to be specified - TODO: we should append it to the list if xpack is enabled and remove this
-
-#Check if license is installed
-- name: Check License plugin is installed
-  shell: >
-    {{es_home}}/bin/{{ es_binary_prefix }}plugin list | tail -n +2 | grep license
-  register: license_installed
-  ignore_errors: yes
-  failed_when: "'ERROR' in license_installed.stdout"
-  changed_when: False
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-
-#Remove license if installed and xpack not enabled
-- name: Remove license plugin
-  command: >
-    {{es_home}}/bin/{{ es_binary_prefix }}plugin remove license
-  register: xpack_state
-  failed_when: "'ERROR' in xpack_state.stdout"
-  changed_when: xpack_state.rc == 0
-  when: license_installed.rc == 0 and (not es_enable_xpack or es_version_changed)
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-
-#Install License if not installed, or it needs to be reinstalled due to ES change (above task will have removed), and its been requested.
-- name: Install license plugin
-  command: >
-    {{es_home}}/bin/{{ es_binary_prefix }}plugin install license
-  register: xpack_state
-  failed_when: "'ERROR' in xpack_state.stdout"
-  changed_when: xpack_state.rc == 0
-  when: (license_installed.rc == 1 or es_version_changed) and es_enable_xpack
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-
-#We loop on all as we may need to remove some features.
 - include: elasticsearch-xpack-install.yml
-  with_items: "{{supported_xpack_features}}"
 
-#Shield configuration
+#Security configuration
-- include: shield/elasticsearch-shield.yml
+- include: security/elasticsearch-security.yml
 
 #Add any feature specific configuration here
 - name: Set Plugin Directory Permissions

tasks/xpack/security/elasticsearch-security-file.yml

@@ -1,15 +1,15 @@
 ---
 - set_fact: manage_file_users=es_users is defined and es_users.file is defined
 
-#Ensure shield conf directory is created
+#Ensure x-pack conf directory is created
-- name: Ensure shield conf directory exists (file)
+- name: Ensure x-pack conf directory exists (file)
-  file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }}
+  file: path={{ conf_dir }}/x-pack state=directory owner={{ es_user }} group={{ es_group }}
   changed_when: False
-  when: es_enable_xpack and '"shield" in es_xpack_features'
+  when: es_enable_xpack and '"security" in es_xpack_features'
 
 #List current users
 - name: List Users
-  shell: cat {{conf_dir}}/shield/users | awk -F':' '{print $1}'
+  shell: cat {{conf_dir}}/x-pack/users | awk -F':' '{print $1}'
   register: current_file_users
   when: manage_file_users
   changed_when: False
@@ -20,7 +20,7 @@
 #Remove users
 - name: Remove Users
   command: >
-    {{es_home}}/bin/shield/esusers userdel {{item}}
+    {{es_home}}/bin/x-pack/users userdel {{item}}
   when: manage_file_users and (users_to_remove | length > 0)
   with_items: "{{users_to_remove | default([])}}"
   environment:
@@ -34,7 +34,7 @@
 #Add users
 - name: Add Users
   command: >
-    {{es_home}}/bin/shield/esusers useradd {{item}} -p {{es_users.file[item].password}}
+    {{es_home}}/bin/x-pack/users useradd {{item}} -p {{es_users.file[item].password}}
   with_items: "{{users_to_add | default([])}}"
   when: manage_file_users and users_to_add | length > 0
   no_log: True
@@ -45,7 +45,7 @@
 #Set passwords for all users declared - Required as the useradd will not change existing user passwords
 - name: Set User Passwords
   command: >
-    {{es_home}}/bin/shield/esusers passwd {{item.key}} -p {{item.value.password}}
+    {{es_home}}/bin/x-pack/users passwd {{item.key}} -p {{item.value.password}}
   with_dict: "{{(es_users | default({'file':{}})).file}}"
   when: manage_file_users and es_users.file.keys() | length > 0
   #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip.
@@ -60,14 +60,14 @@
 
 #Copy Roles files
 - name: Copy roles.yml File for Instance
-  template: src=shield/roles.yml.j2 dest={{conf_dir}}/shield/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=security/roles.yml.j2 dest={{conf_dir}}/x-pack/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   when: es_roles is defined and es_roles.file is defined
 
 #Overwrite users_roles file
 - name: Copy User Roles
-  template: src=shield/users_roles.j2 dest={{conf_dir}}/shield/users_roles mode=0644 force=yes
+  template: src=security/users_roles.j2 dest={{conf_dir}}/x-pack/users_roles mode=0644 force=yes
   when: manage_file_users and users_roles | length > 0
 
-#Set permission on shield directory. E.g. if 2 nodes are installed on the same machine, the second node will not get the users file created at install, causing the files being created at es_users call and then having the wrong Permissions.
+#Set permission on security directory. E.g. if 2 nodes are installed on the same machine, the second node will not get the users file created at install, causing the files being created at es_users call and then having the wrong Permissions.
-- name: Set Shield Directory Permissions Recursive
+- name: Set Security Directory Permissions Recursive
-  file: state=directory path={{conf_dir}}/shield/ owner={{ es_user }} group={{ es_group }} recurse=yes
+  file: state=directory path={{conf_dir}}/x-pack/ owner={{ es_user }} group={{ es_group }} recurse=yes

tasks/xpack/security/elasticsearch-security.yml

@@ -0,0 +1,36 @@
+---
+#Security specific configuration done here
+
+#TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6
+
+#-----------------------------FILE BASED REALM----------------------------------------
+
+- include: elasticsearch-security-file.yml
+  when: (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.file) or (es_roles is defined and es_roles.file is defined))
+
+#-----------------------------NATIVE BASED REALM----------------------------------------
+# The native realm requires the node to be started so we do as a handler
+- command: /bin/true
+  notify: activate-security
+  when: (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
+
+#-----------------------------ROLE MAPPING ----------------------------------------
+
+#Copy Roles files
+- name: Copy role_mapping.yml File for Instance
+  template: src=security/role_mapping.yml.j2 dest={{conf_dir}}/x-pack/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  when: es_role_mapping is defined
+
+#-----------------------------AUTH FILE----------------------------------------
+
+- name: Copy message auth key to elasticsearch
+  copy: src={{ es_message_auth_file }} dest={{conf_dir}}/x-pack/system_key owner={{ es_user }} group={{ es_group }} mode=0600 force=yes
+  when: es_message_auth_file is defined
+
+#------------------------------------------------------------------------------------
+
+#Ensure security conf directory is created
+- name: Ensure security conf directory exists
+  file: path={{ conf_dir }}/security state=directory owner={{ es_user }} group={{ es_group }}
+  changed_when: False
+  when: es_enable_xpack and '"security" in es_xpack_features'

tasks/xpack/shield/elasticsearch-shield.yml

@@ -1,36 +0,0 @@
----
-#Shield specific configuration done here
-
-#TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6
-
-#-----------------------------FILE BASED REALM----------------------------------------
-
-- include: elasticsearch-shield-file.yml
-  when: (es_enable_xpack and '"shield" in es_xpack_features') and ((es_users is defined and es_users.file) or (es_roles is defined and es_roles.file is defined))
-
-#-----------------------------NATIVE BASED REALM----------------------------------------
-# The native realm requires the node to be started so we do as a handler
-- command: /bin/true
-  notify: activate-shield
-  when: (es_enable_xpack and '"shield" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
-
-#-----------------------------ROLE MAPPING ----------------------------------------
-
-#Copy Roles files
-- name: Copy role_mapping.yml File for Instance
-  template: src=shield/role_mapping.yml.j2 dest={{conf_dir}}/shield/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
-  when: es_role_mapping is defined
-
-#-----------------------------AUTH FILE----------------------------------------
-
-- name: Copy message auth key to elasticsearch
-  copy: src={{ es_message_auth_file }} dest={{conf_dir}}/shield/system_key owner={{ es_user }} group={{ es_group }} mode=0600 force=yes
-  when: es_message_auth_file is defined
-
-#------------------------------------------------------------------------------------
-
-#Ensure shield conf directory is created
-- name: Ensure shield conf directory exists
-  file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }}
-  changed_when: False
-  when: es_enable_xpack and '"shield" in es_xpack_features'

templates/elasticsearch.j2

@@ -14,34 +14,10 @@ DATA_DIR={{ data_dirs | array_to_str }}
 # Elasticsearch logs directory
 LOG_DIR={{log_dir}}
 
-{% if es_version | version_compare('5.0', '<') %}
-# Elasticsearch work directory
-WORK_DIR={{work_dir}}
-{% endif %}
-
 # Elasticsearch PID directory
 PID_DIR={{pid_dir}}
 
-{% if es_version | version_compare('5.0', '<') %}
+ES_JVM_OPTIONS={{conf_dir}}/jvm.options
-# Heap size defaults to 256m min, 1g max
-# Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g
-{% if es_heap_size is defined %}
-ES_HEAP_SIZE={{es_heap_size}}
-{% endif %}
-{% endif %}
-
-# Heap new generation
-#ES_HEAP_NEWSIZE=
-
-# Maximum direct memory
-#ES_DIRECT_SIZE=
-
-# Additional Java OPTS
-{% if es_java_opts is defined and es_java_opts %}
-ES_JAVA_OPTS="{{es_java_opts | default([]) | join(' ')}}"
-{% else %}
-#ES_JAVA_OPTS=
-{% endif %}
 
 # Configure restart on package upgrade (true, every other setting will lead to not restarting)
 #ES_RESTART_ON_UPGRADE=true
@@ -63,6 +39,9 @@ ES_JAVA_OPTS="{{es_java_opts | default([]) | join(' ')}}"
 ES_USER={{es_user}}
 ES_GROUP={{es_group}}
 
+# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process
+ES_STARTUP_SLEEP_TIME=5
+
 ################################
 # System properties
 ################################
@@ -76,12 +55,12 @@ MAX_OPEN_FILES={{es_max_open_files}}
 {% endif %}
 
 # The maximum number of bytes of memory that may be locked into RAM
-# Set to "unlimited" if you use the 'bootstrap.{% if es_version | version_compare('5.0', '<=') %}memory_lock{% else %}mlockall{% endif %}: true' option
+# Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option
 # in elasticsearch.yml (ES_HEAP_SIZE  must also be set).
 # When using Systemd, the LimitMEMLOCK property must be set
 # in /usr/lib/systemd/system/elasticsearch.service
-{% if m_lock_enabled %}
 #MAX_LOCKED_MEMORY=
+{% if m_lock_enabled %}
 MAX_LOCKED_MEMORY=unlimited
 {% endif %}
 
@@ -89,3 +68,6 @@ MAX_LOCKED_MEMORY=unlimited
 # When using Systemd, this setting is ignored and the 'vm.max_map_count'
 # property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf
 #MAX_MAP_COUNT=262144
+{% if es_max_map_count is defined %}
+MAX_MAP_COUNT={{es_max_map_count}}
+{% endif %}

templates/elasticsearch.repo

@@ -1,6 +1,6 @@
 [elasticsearch-{{ es_major_version }}]
 name=Elasticsearch repository for {{ es_major_version }} packages
-baseurl=http://packages.elastic.co/elasticsearch/{{ es_major_version }}/centos
+baseurl=https://artifacts.elastic.co/packages/{{ es_major_version }}/yum
 gpgcheck=1
-gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
+gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
 enabled=1

templates/elasticsearch.yml.j2

@@ -18,8 +18,16 @@ path.conf: {{ conf_dir }}
 
 path.data: {{ data_dirs | array_to_str }}
 
-{% if es_version | version_compare('5.0', '<') %}
+path.logs: {{ log_dir }}
-path.work: {{ work_dir }}
+
+{% if not "security" in es_xpack_features %}
+xpack.security.enabled: false
 {% endif %}
 
-path.logs: {{ log_dir }}
+{% if not "monitoring" in es_xpack_features %}
+xpack.monitoring.enabled: false
+{% endif %}
+
+{% if not "alerting" in es_xpack_features %}
+xpack.watcher.enabled: false
+{% endif %}

templates/init/debian/elasticsearch.j2

@@ -1,16 +1,9 @@
-#!/bin/sh
+#!/bin/bash
 #
 # /etc/init.d/elasticsearch -- startup script for Elasticsearch
 #
-# Written by Miquel van Smoorenburg <miquels@cistron.nl>.
-# Modified for Debian GNU/Linux	by Ian Murdock <imurdock@gnu.ai.mit.edu>.
-# Modified for Tomcat by Stefan Gybas <sgybas@debian.org>.
-# Modified for Tomcat6 by Thierry Carrez <thierry.carrez@ubuntu.com>.
-# Additional improvements by Jason Brittain <jason.brittain@mulesoft.com>.
-# Modified by Nicolas Huray for Elasticsearch <nicolas.huray@gmail.com>.
-#
 ### BEGIN INIT INFO
-# Provides:          {{es_instance_name}}_{{default_file | basename}}
+# Provides:          elasticsearch
 # Required-Start:    $network $remote_fs $named
 # Required-Stop:     $network $remote_fs $named
 # Default-Start:     2 3 4 5
@@ -34,14 +27,11 @@ if [ `id -u` -ne 0 ]; then
 	exit 1
 fi
 
-
 . /lib/lsb/init-functions
-
 if [ -r /etc/default/rcS ]; then
 	. /etc/default/rcS
 fi
 
-
 # The following variables can be overwritten in $DEFAULT
 
 # Run Elasticsearch as this user ID and group ID
@@ -51,26 +41,16 @@ ES_GROUP={{es_group}}
 # Directory where the Elasticsearch binary distribution resides
 ES_HOME={{es_home}}
 
-{% if es_version | version_compare('5.0', '<') %}
-# Heap size defaults to 256m min, 1g max
-# Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g
-#ES_HEAP_SIZE=2g
-{% endif %}
-
-# Heap new generation
-#ES_HEAP_NEWSIZE=
-
-# max direct memory
-#ES_DIRECT_SIZE=
-
-# Additional Java OPTS
-#ES_JAVA_OPTS=
-
 # Maximum number of open files
-MAX_OPEN_FILES=65535
+{% if es_max_open_files is defined %}
+MAX_OPEN_FILES={{es_max_open_files}}
+{% endif %}
 
 # Maximum amount of locked memory
 #MAX_LOCKED_MEMORY=
+{% if m_lock_enabled %}
+MAX_LOCKED_MEMORY=unlimited
+{% endif %}
 
 # Elasticsearch log directory
 LOG_DIR={{log_dir}}
@@ -82,14 +62,15 @@ DATA_DIR={{ data_dirs | array_to_str }}
 CONF_DIR={{conf_dir}}
 
 # Maximum number of VMA (Virtual Memory Areas) a process can own
-MAX_MAP_COUNT=262144
+{% if es_max_map_count is defined %}
-
+MAX_MAP_COUNT={{es_max_map_count}}
-# Path to the GC log file
+{% endif %}
-#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log
 
 # Elasticsearch PID file directory
 PID_DIR={{pid_dir}}
 
+ES_JVM_OPTIONS="{{conf_dir}}/jvm.options"
+
 # End of variables that can be overwritten in $DEFAULT
 
 # overwrite settings from default file
@@ -106,21 +87,18 @@ fi
 # Define other required variables
 PID_FILE="$PID_DIR/$NAME.pid"
 DAEMON={{es_home}}/bin/elasticsearch
-{% if es_version | version_compare('5.0', '<') %}
-DAEMON_OPTS="-d -p $PID_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR"
-{% else %}
 DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR"
-{% endif %}
 
-export ES_HEAP_SIZE
-export ES_HEAP_NEWSIZE
-export ES_DIRECT_SIZE
 export ES_JAVA_OPTS
-export ES_GC_LOG_FILE
 export JAVA_HOME
+export ES_INCLUDE
+export ES_JVM_OPTIONS
 
 # Check DAEMON exists
-test -x $DAEMON || exit 0
+if [ ! -x "$DAEMON" ]; then
+        echo "The elasticsearch startup script does not exists or it is not executable, tried: $DAEMON"
+        exit 1
+fi
 
 checkJava() {
 	if [ -x "$JAVA_HOME/bin/java" ]; then

templates/init/redhat/elasticsearch.j2

@@ -32,16 +32,23 @@ if [ -f /etc/rc.d/init.d/functions ]; then
 fi
 
 # Sets the default values for elasticsearch variables used in this script
-ES_USER="elasticsearch"
+ES_USER="{{es_user}}"
-ES_GROUP="elasticsearch"
+ES_GROUP="{{es_group}}"
-ES_HOME="/usr/share/elasticsearch"
+ES_HOME="{{es_home}}"
-MAX_OPEN_FILES=65535
+{% if es_max_open_files is defined %}
-MAX_MAP_COUNT=262144
+MAX_OPEN_FILES={{es_max_open_files}}
-LOG_DIR="/var/log/elasticsearch"
+{% endif %}
-DATA_DIR="/var/lib/elasticsearch"
+# Maximum number of VMA (Virtual Memory Areas) a process can own
-CONF_DIR="/etc/elasticsearch"
+{% if es_max_map_count is defined %}
+MAX_MAP_COUNT={{es_max_map_count}}
+{% endif %}
 
-PID_DIR="/var/run/elasticsearch"
+LOG_DIR="{{log_dir}}"
+DATA_DIR={{ data_dirs | array_to_str }}
+CONF_DIR="{{conf_dir}}"
+
+PID_DIR="{{pid_dir}}"
+ES_JVM_OPTIONS="{{conf_dir}}/jvm.options"
 
 # Source the default env file
 ES_ENV_FILE="{{instance_default_file}}"
@@ -56,16 +63,24 @@ if [ ! -z "$CONF_FILE" ]; then
 fi
 
 exec="$ES_HOME/bin/elasticsearch"
-prog="elasticsearch"
+prog="{{es_instance_name}}_{{default_file | basename}}"
 pidfile="$PID_DIR/${prog}.pid"
 
-export ES_HEAP_SIZE
-export ES_HEAP_NEWSIZE
-export ES_DIRECT_SIZE
 export ES_JAVA_OPTS
-export ES_GC_LOG_FILE
-export ES_STARTUP_SLEEP_TIME
 export JAVA_HOME
+export ES_INCLUDE
+export ES_JVM_OPTIONS
+export ES_STARTUP_SLEEP_TIME
+
+# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported
+if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi
+if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi
+if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi
+if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi
+if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi
+if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi
+if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi
+if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi
 
 lockfile=/var/lock/subsys/$prog
 
@@ -74,6 +89,11 @@ if [ -n $USER ] && [ -z $ES_USER ] ; then
    ES_USER=$USER
 fi
 
+if [ ! -x "$exec" ]; then
+    echo "The elasticsearch startup script does not exists or it is not executable, tried: $exec"
+    exit 1
+fi
+
 checkJava() {
     if [ -x "$JAVA_HOME/bin/java" ]; then
         JAVA="$JAVA_HOME/bin/java"
@@ -90,12 +110,7 @@ checkJava() {
 start() {
     checkJava
     [ -x $exec ] || exit 5
-{% if es_version | version_compare('5.0', '<') %}
+
-    if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then
-        echo "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set"
-        return 7
-    fi
-{% endif %}
     if [ -n "$MAX_OPEN_FILES" ]; then
         ulimit -n $MAX_OPEN_FILES
     fi
@@ -105,7 +120,6 @@ start() {
     if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then
         sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
     fi
-    export ES_GC_LOG_FILE
 
     # Ensure that the PID_DIR exists (it is cleaned at OS startup time)
     if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then
@@ -118,11 +132,7 @@ start() {
     cd $ES_HOME
     echo -n $"Starting $prog: "
     # if not running, start it up here, usually something like "daemon $exec"
-{% if es_version | version_compare('5.0', '<') %}
+    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR
-    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -Des.default.path.conf=$CONF_DIR
-{% else %}
-    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=${DATA_DIR} -Edefault.path.conf=$CONF_DIR
-{% endif %}
     retval=$?
     echo
     [ $retval -eq 0 ] && touch $lockfile

templates/jvm.options.j2

@@ -18,9 +18,17 @@
 
 # Xms represents the initial size of total heap space
 # Xmx represents the maximum size of total heap space
-
+{% if es_heap_size is defined %}
 -Xms{{ es_heap_size }}
+{% else %}
+-Xms2g
+{% endif %}
+
+{% if es_heap_size is defined %}
 -Xmx{{ es_heap_size }}
+{% else %}
+-Xms2g
+{% endif %}
 
 ################################################################
 ## Expert settings

templates/logging.yml.j2

@@ -1,68 +0,0 @@
-# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
-es.logger.level: INFO
-rootLogger: ${es.logger.level}, console, file
-logger:
-  # log action execution errors for easier debugging
-  action: DEBUG
-  # reduce the logging for aws, too much is logged under the default INFO
-  com.amazonaws: WARN
-  org.apache.http: INFO
-
-  # gateway
-  #gateway: DEBUG
-  #index.gateway: DEBUG
-
-  # peer shard recovery
-  #indices.recovery: DEBUG
-
-  # discovery
-  #discovery: TRACE
-
-  index.search.slowlog: TRACE, index_search_slow_log_file
-  index.indexing.slowlog: TRACE, index_indexing_slow_log_file
-
-additivity:
-  index.search.slowlog: false
-  index.indexing.slowlog: false
-
-appender:
-  console:
-    type: console
-    layout:
-      type: consolePattern
-      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
-
-  file:
-    type: dailyRollingFile
-    file: ${path.logs}/${cluster.name}.log
-    datePattern: "'.'yyyy-MM-dd"
-    layout:
-      type: pattern
-      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
-
-  # Use the following log4j-extras RollingFileAppender to enable gzip compression of log files.
-  # For more information see https://logging.apache.org/log4j/extras/apidocs/org/apache/log4j/rolling/RollingFileAppender.html
-  #file:
-    #type: extrasRollingFile
-    #file: ${path.logs}/${cluster.name}.log
-    #rollingPolicy: timeBased
-    #rollingPolicy.FileNamePattern: ${path.logs}/${cluster.name}.log.%d{yyyy-MM-dd}.gz
-    #layout:
-      #type: pattern
-      #conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
-
-  index_search_slow_log_file:
-    type: dailyRollingFile
-    file: ${path.logs}/${cluster.name}_index_search_slowlog.log
-    datePattern: "'.'yyyy-MM-dd"
-    layout:
-      type: pattern
-      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
-
-  index_indexing_slow_log_file:
-    type: dailyRollingFile
-    file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log
-    datePattern: "'.'yyyy-MM-dd"
-    layout:
-      type: pattern
-      conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"

templates/systemd/elasticsearch.j2

@@ -17,35 +17,32 @@ WorkingDirectory={{es_home}}
 User={{es_user}}
 Group={{es_group}}
 
-{% if es_version | version_compare('2.0', '>=') %}
 ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec
-{% endif %}
 
 ExecStart={{es_home}}/bin/elasticsearch \
-{% if es_version | version_compare('5.0', '<') %}
-                                        -Des.pidfile=${PID_DIR}/elasticsearch.pid \
-                                        -Des.default.path.home=${ES_HOME} \
-                                        -Des.default.path.logs=${LOG_DIR} \
-                                        -Des.default.path.data=${DATA_DIR} \
-                                        -Des.default.path.conf=${CONF_DIR}
-{% else %}
                                     -p ${PID_DIR}/elasticsearch.pid \
-                                        -Edefault.path.home=${ES_HOME} \
+                                    --quiet \
                                     -Edefault.path.logs=${LOG_DIR} \
                                     -Edefault.path.data=${DATA_DIR} \
                                     -Edefault.path.conf=${CONF_DIR}
-{% endif %}
-
 
 
+# StandardOutput is configured to redirect to journalctl since
+# some error messages may be logged in standard output before
+# elasticsearch logging system is initialized. Elasticsearch
+# stores its logs in /var/log/elasticsearch and does not use
+# journalctl by default. If you also want to enable journalctl
+# logging, you can simply remove the "quiet" option from ExecStart.
 StandardOutput=journal
 StandardError=inherit
 
 # Specifies the maximum file descriptor number that can be opened by this process
+{% if es_max_open_files is defined %}
 LimitNOFILE={{es_max_open_files}}
+{% endif %}
 
 # Specifies the maximum number of bytes of memory that may be locked into RAM
-# Set to "infinity" if you use the 'bootstrap.{% if es_version | version_compare('5.0', '<=') %}memory_lock{% else %}mlockall{% endif %}: true' option
+# Set to "infinity" if you use the 'bootstrap.memory_lock: true' option
 # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}}
 {% if m_lock_enabled %}
 LimitMEMLOCK=infinity

test/integration/config-2x/serverspec/default_spec.rb

@@ -1,6 +0,0 @@
-require 'config_spec'
-
-describe 'Config Tests v 2.x' do
-  include_examples 'config::init', "2.4.3"
-end
-

test/integration/config-5x/serverspec/default_spec.rb

@@ -0,0 +1,6 @@
+require 'config_spec'
+
+describe 'Config Tests v 5.x' do
+  include_examples 'config::init', "5.1.2"
+end
+

test/integration/config.yml

@@ -4,9 +4,10 @@
   hosts: localhost
   roles:
     #expand to all available parameters
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.mlockall: true, discovery.zen.ping.multicast.enabled: false } }
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: true } }
   vars:
     es_scripts: false
     es_templates: false
     es_version_lock: false
     es_heap_size: 1g
+    es_api_port: 9201

test/integration/helpers/serverspec/config_spec.rb

@@ -32,14 +32,12 @@ shared_examples 'config::init' do |es_version|
     it { should contain 'transport.tcp.port: 9301' }
     it { should contain 'node.data: false' }
     it { should contain 'node.master: true' }
-    it { should contain 'discovery.zen.ping.multicast.enabled: false' }
     it { should contain 'cluster.name: custom-cluster' }
     it { should contain 'node.name: node1' }
-    it { should contain 'bootstrap.mlockall: true' }
+    it { should contain 'bootstrap.memory_lock: true' }
     it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9301' }
     it { should contain 'path.conf: /etc/elasticsearch/node1' }
     it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' }
-    it { should contain 'path.work: /opt/elasticsearch/temp/localhost-node1' }
     it { should contain 'path.logs: /opt/elasticsearch/logs/localhost-node1' }
   end
 
@@ -64,11 +62,6 @@ shared_examples 'config::init' do |es_version|
     it { should be_owned_by 'elasticsearch' }
   end
 
-  describe file('/opt/elasticsearch/temp/localhost-node1') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
   #test we started on the correct port was used
   describe command('curl -s "localhost:9201"') do
     #TODO: This is returning an empty string

test/integration/helpers/serverspec/multi_spec.rb

@@ -25,12 +25,10 @@ shared_examples 'multi::init' do  |es_version,plugins|
     it { should contain 'transport.tcp.port: 9301' }
     it { should contain 'node.data: true' }
     it { should contain 'node.master: false' }
-    it { should contain 'discovery.zen.ping.multicast.enabled: false' }
     it { should contain 'node.name: localhost-node1' }
-    it { should_not contain 'bootstrap.mlockall: true' }
+    it { should_not contain 'bootstrap.memory_lock: true' }
     it { should contain 'path.conf: /etc/elasticsearch/node1' }
     it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' }
-    it { should contain 'path.work: /tmp/elasticsearch/localhost-node1' }
     it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' }
   end
 
@@ -42,12 +40,10 @@ shared_examples 'multi::init' do  |es_version,plugins|
     it { should contain 'transport.tcp.port: 9300' }
     it { should contain 'node.data: false' }
     it { should contain 'node.master: true' }
-    it { should contain 'discovery.zen.ping.multicast.enabled: false' }
     it { should contain 'node.name: localhost-master' }
-    it { should contain 'bootstrap.mlockall: true' }
+    it { should contain 'bootstrap.memory_lock: true' }
     it { should contain 'path.conf: /etc/elasticsearch/master' }
     it { should contain 'path.data: /opt/elasticsearch/master/localhost-master' }
-    it { should contain 'path.work: /tmp/elasticsearch/localhost-master' }
     it { should contain 'path.logs: /var/log/elasticsearch/localhost-master' }
   end
 

test/integration/helpers/serverspec/standard_spec.rb

@@ -19,7 +19,12 @@ shared_examples 'standard::init' do |es_version|
     it { should be_owned_by 'elasticsearch' }
   end
 
-  describe file('/etc/elasticsearch/node1/logging.yml') do
+  describe file('/etc/elasticsearch/node1/log4j2.properties') do
+    it { should be_file }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe file('/etc/elasticsearch/node1/jvm.options') do
     it { should be_file }
     it { should be_owned_by 'elasticsearch' }
   end
@@ -29,7 +34,6 @@ shared_examples 'standard::init' do |es_version|
     it { should contain 'cluster.name: elasticsearch' }
     it { should contain 'path.conf: /etc/elasticsearch/node1' }
     it { should contain 'path.data: /var/lib/elasticsearch/localhost-node1' }
-    it { should contain 'path.work: /tmp/elasticsearch/localhost-node1' }
     it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' }
   end
 

test/integration/helpers/serverspec/xpack_spec.rb

@@ -1,12 +1,12 @@
 require 'spec_helper'
 
-shared_examples 'xpack::init' do |es_version|
+shared_examples 'xpack::init' do |es_version,plugins|
 
   describe user('elasticsearch') do
     it { should exist }
   end
 
-  describe service('shield_node_elasticsearch') do
+  describe service('security_node_elasticsearch') do
     it { should be_running }
   end
 
@@ -14,23 +14,22 @@ shared_examples 'xpack::init' do |es_version|
     it { should be_installed }
   end
 
-  describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do
+  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
     it { should be_file }
     it { should be_owned_by 'elasticsearch' }
   end
 
-  describe file('/etc/elasticsearch/shield_node/logging.yml') do
+  describe file('/etc/elasticsearch/security_node/log4j2.properties') do
     it { should be_file }
     it { should be_owned_by 'elasticsearch' }
   end
 
-  describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do
+  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
-    it { should contain 'node.name: localhost-shield_node' }
+    it { should contain 'node.name: localhost-security_node' }
     it { should contain 'cluster.name: elasticsearch' }
-    it { should contain 'path.conf: /etc/elasticsearch/shield_node' }
+    it { should contain 'path.conf: /etc/elasticsearch/security_node' }
-    it { should contain 'path.data: /var/lib/elasticsearch/localhost-shield_node' }
+    it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' }
-    it { should contain 'path.work: /tmp/elasticsearch/localhost-shield_node' }
+    it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' }
-    it { should contain 'path.logs: /var/log/elasticsearch/localhost-shield_node' }
   end
 
   describe 'Node listening' do
@@ -77,17 +76,6 @@ shared_examples 'xpack::init' do |es_version|
     it { should be_owned_by 'elasticsearch' }
   end
 
-
-  #Check shield,watcher and license plugins are installed
-  describe file('/usr/share/elasticsearch/plugins/license') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
-  end
-
-  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep license') do
-    its(:exit_status) { should eq 0 }
-  end
-  
   #Test if x-pack is activated
   describe 'x-pack activation' do
     it 'should be activated and valid' do
@@ -97,72 +85,51 @@ shared_examples 'xpack::init' do |es_version|
     end
   end
 
-  describe file('/usr/share/elasticsearch/plugins/shield') do
+  describe file('/usr/share/elasticsearch/plugins/x-pack') do
     it { should be_directory }
     it { should be_owned_by 'elasticsearch' }
   end
 
-  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep shield') do
+  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep x-pack') do
     its(:exit_status) { should eq 0 }
   end
 
-  describe file('/etc/elasticsearch/shield_node/shield') do
+  describe file('/etc/elasticsearch/security_node/x-pack') do
     it { should be_directory }
     it { should be_owned_by 'elasticsearch' }
   end
 
-  describe file('/usr/share/elasticsearch/plugins/watcher') do
+  describe file('/usr/share/elasticsearch/plugins/x-pack') do
     it { should be_directory }
     it { should be_owned_by 'elasticsearch' }
   end
 
-  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep watcher') do
+  for plugin in plugins
+    describe file('/usr/share/elasticsearch/plugins/'+plugin) do
+      it { should be_directory }
+      it { should be_owned_by 'elasticsearch' }
+    end
+
+    describe command('curl -s -u es_admin:changeMe localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do
       its(:exit_status) { should eq 0 }
     end
-
-  describe file('/usr/share/elasticsearch/plugins/kopf') do
-    it { should be_directory }
-    it { should be_owned_by 'elasticsearch' }
   end
 
-  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep kopf') do
-    its(:exit_status) { should eq 0 }
-  end
-
-  #test we haven't installed graph or marvel-agent
-
-  describe file('/usr/share/elasticsearch/plugins/graph') do
-    it { should_not exist }
-  end
-
-  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep graph') do
-    its(:exit_status) { should eq 1 }
-  end
-
-  describe file('/usr/share/elasticsearch/plugins/marvel-agent') do
-    it { should_not exist }
-  end
-
-  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep marvel-agent') do
-    its(:exit_status) { should eq 1 }
-  end
-
-
   #Test users file, users_roles and roles.yml
-  describe file('/etc/elasticsearch/shield_node/shield/users_roles') do
+  describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do
     it { should be_owned_by 'elasticsearch' }
     it { should contain 'admin:es_admin' }
     it { should contain 'power_user:testUser' }
   end
 
-  describe file('/etc/elasticsearch/shield_node/shield/users') do
+  describe file('/etc/elasticsearch/security_node/x-pack/users') do
     it { should be_owned_by 'elasticsearch' }
     it { should contain 'testUser:' }
     it { should contain 'es_admin:' }
   end
 
 
-  describe file('/etc/elasticsearch/shield_node/shield/roles.yml') do
+  describe file('/etc/elasticsearch/security_node/x-pack/roles.yml') do
     it { should be_owned_by 'elasticsearch' }
     #Test contents as expected
     its(:md5sum) { should eq '7800182547287abd480c8b095bf26e9e' }
@@ -170,19 +137,19 @@ shared_examples 'xpack::init' do |es_version|
 
 
   #Test native roles and users are loaded
-  describe command('curl -s localhost:9200/_shield/user -u es_admin:changeMe | md5sum | grep 557a730df7136694131b5b7012a5ffad') do
+  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | md5sum | grep 243b362bd47623c0b91a1fafbce2b6f5') do
     its(:exit_status) { should eq 0 }
   end
 
-  describe command('curl -s localhost:9200/_shield/user -u es_admin:changeMe | grep "{\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{}}}"') do
+  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | grep "{\"elastic\":{\"username\":\"elastic\",\"roles\":\[\"superuser\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana\":{\"username\":\"kibana\",\"roles\":\[\"kibana\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{},\"enabled\":true}}"') do
     its(:exit_status) { should eq 0 }
   end
 
-  describe command('curl -s localhost:9200/_shield/role -u es_admin:changeMe | grep "{\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\]}}"') do
+  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | grep "{\"superuser\":{\"cluster\":\[\"all\"\],\"indices\":\[{\"names\":\[\"\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\"\*\"\],\"metadata\":{\"_reserved\":true}},\"transport_client\":{\"cluster\":\[\"transport_client\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"kibana_user\":{\"cluster\":\[\"monitor\"\],\"indices\":\[{\"names\":\[\".kibana\*\"\],\"privileges\":\[\"manage\",\"read\",\"index\",\"delete\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"monitoring_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\".monitoring-\*\"\],\"privileges\":\[\"read\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"remote_monitoring_agent\":{\"cluster\":\[\"manage_index_templates\",\"manage_ingest_pipelines\",\"monitor\"\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\"\.monitoring-\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"ingest_admin\":{\"cluster\":\[\"manage_index_templates\",\"manage_pipeline\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"reporting_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.reporting-\*\"\],\"privileges\":\[\"read\",\"write\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\],\"metadata\":{}}}"') do
     its(:exit_status) { should eq 0 }
   end
 
-  describe command('curl -s localhost:9200/_shield/role -u es_admin:changeMe | md5sum | grep 6d14f09ef1eea64adf4d4a9c04229629') do
+  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | md5sum | grep 78a0696c9c9690042cec2c1f16860cfc') do
     its(:exit_status) { should eq 0 }
   end
 
@@ -213,15 +180,15 @@ shared_examples 'xpack::init' do |es_version|
   end
 
   #Test contents of Elasticsearch.yml file
-  describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do
+  describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do
-    it { should contain 'shield.authc.realms.file1.order: 0' }
+    it { should contain 'security.authc.realms.file1.order: 0' }
-    it { should contain 'shield.authc.realms.file1.type: file' }
+    it { should contain 'security.authc.realms.file1.type: file' }
-    it { should contain 'shield.authc.realms.native1.order: 1' }
+    it { should contain 'security.authc.realms.native1.order: 1' }
-    it { should contain 'shield.authc.realms.native1.type: native' }
+    it { should contain 'security.authc.realms.native1.type: native' }
   end
 
   #Test contents of role_mapping.yml
-  describe file('/etc/elasticsearch/shield_node/shield/role_mapping.yml') do
+  describe file('/etc/elasticsearch/security_node/x-pack/role_mapping.yml') do
     it { should be_owned_by 'elasticsearch' }
     it { should contain 'power_user:' }
     it { should contain '- cn=admins,dc=example,dc=com' }
@@ -230,7 +197,7 @@ shared_examples 'xpack::init' do |es_version|
   end
 
 
-  describe file('/etc/elasticsearch/shield_node/shield/system_key') do
+  describe file('/etc/elasticsearch/security_node/x-pack/system_key') do
     it { should be_owned_by 'elasticsearch' }
     it { should be_writable.by('owner') }
     it { should be_writable.by_user('elasticsearch') }

test/integration/multi-2x/serverspec/default_spec.rb

@@ -1,8 +0,0 @@
-require 'multi_spec'
-
-
-describe 'Multi Tests v 2.x' do
-  include_examples 'multi::init', "2.4.3", ["kopf"]
-end
-
-

test/integration/multi-5x/serverspec/default_spec.rb

@@ -0,0 +1,8 @@
+require 'multi_spec'
+
+
+describe 'Multi Tests v 5.x' do
+  include_examples 'multi::init', "5.1.2", ["ingest-geoip"]
+end
+
+

test/integration/multi.yml

@@ -1,11 +1,22 @@
 ---
 #Test ability to deploy multiple instances to a machine
-- name: Elasticsearch Multi tests
+- name: Elasticsearch Multi test - master on 9200
   hosts: localhost
-  roles:
-    - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_heap_size: "1g", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.mlockall: true, discovery.zen.ping.multicast.enabled: false } }
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { "discovery.zen.ping.multicast.enabled": false,  discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false, discovery.zen.ping.multicast.enabled: false } }
   vars:
     es_scripts: true
     es_templates: true
+    es_heap_size: "1g"
+    es_api_port: 9200
+  roles:
+    - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } }
+
+- name: Elasticsearch Multi test - data on 9201
+  hosts: localhost
+  vars:
+    es_scripts: true
+    es_templates: true
+    es_heap_size: "1g"
+    es_api_port: 9201
+  roles:
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } }
 #Plugins installed for this test are specified in .kitchen.yml under suite

test/integration/package-2x/serverspec/default_spec.rb

@@ -1,6 +0,0 @@
-require 'package_spec'
-
-
-describe 'Package Tests v 2.x' do
-  include_examples 'package::init', "2.4.3", ["kopf"]
-end

test/integration/package-5x/serverspec/default_spec.rb

@@ -0,0 +1,6 @@
+require 'package_spec'
+
+
+describe 'Package Tests v 5.x' do
+  include_examples 'package::init', "5.1.2", ["ingest-geoip"]
+end

test/integration/package.yml

@@ -6,4 +6,6 @@
   vars:
     es_scripts: true
     es_templates: true
+    es_heap_size: "1g"
+    es_api_port: 9200
 #Plugins installed for this test are specified in .kitchen.yml under suite

test/integration/standard-2x/serverspec/default_spec.rb

@@ -1,9 +0,0 @@
-require 'standard_spec'
-
-
-describe 'Standard Tests v 2.x' do
-  include_examples 'standard::init', "2.4.3"
-end
-
-
-

test/integration/standard-5x/serverspec/default_spec.rb

@@ -0,0 +1,9 @@
+require 'standard_spec'
+
+
+describe 'Standard Tests v 5.x' do
+  include_examples 'standard::init', "5.1.2"
+end
+
+
+

test/integration/standard.yml

@@ -5,3 +5,4 @@
     - { role: elasticsearch, es_instance_name: "node1" }
   vars:
     es_use_repository: "true"
+    es_heap_size: "1g"

test/integration/xpack-2x/serverspec/default_spec.rb

@@ -1,5 +0,0 @@
-require 'xpack_spec'
-
-describe 'Xpack Tests v 2.x' do
-  include_examples 'xpack::init', "2.4.3"
-end

test/integration/xpack-5x/serverspec/default_spec.rb

@@ -0,0 +1,5 @@
+require 'xpack_spec'
+
+describe 'Xpack Tests v 5.x' do
+  include_examples 'xpack::init', "5.1.2", ["ingest-geoip"]
+end

test/integration/xpack.yml

@@ -2,21 +2,21 @@
 - name: Elasticsearch Xpack tests
   hosts: localhost
   roles:
-    - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300",
+    - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300",
-    "shield.authc.realms.file1.type": "file","shield.authc.realms.file1.order": 0, "shield.authc.realms.native1.type": "native","shield.authc.realms.native1.order": 1 },
+    "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 },
-    es_instance_name: "shield_node" }
+    es_instance_name: "security_node" }
   vars:
+    es_heap_size: "1g"
     es_templates: true
     es_enable_xpack: true
     es_xpack_license: "{{ lookup('file', '/tmp/license.json')  }}"
     es_plugins:
-      - plugin: lmenezes/elasticsearch-kopf
+      - plugin: ingest-geoip
-        version: master
     es_xpack_features:
-      - shield
+      - security
-      - watcher
+      - alerting
-    es_api_basic_auth_username: es_admin
+    es_api_basic_auth_username: elastic
-    es_api_basic_auth_password: changeMe
+    es_api_basic_auth_password: changeme
     es_message_auth_file: system_key
     es_role_mapping:
       power_user:

vars/Debian.yml

@@ -1,4 +1,4 @@
 ---
-java: "{% if es_java is defined %}{{es_java}}{%elif (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15.10', '>=')) %}openjdk-8-jre-headless{% else %}openjdk-7-jre-headless{% endif %}" 
+java: "{% if es_java is defined %}{{es_java}}{% else %}openjdk-8-jre-headless{% endif %}"
 default_file: "/etc/default/elasticsearch"
 es_home: "/usr/share/elasticsearch"

vars/main.yml

@@ -4,4 +4,4 @@ es_conf_dir: "/etc/elasticsearch"
 sysd_script: "/usr/lib/systemd/system/elasticsearch.service"
 init_script: "/etc/init.d/elasticsearch"
 #add supported features here
-supported_xpack_features: ["watcher","marvel-agent","graph","shield"]
+supported_xpack_features: ["alerting","monitoring","graph","security"]