Merge branch 'master' into AddSSLSupport

.github/issue_template.md

@@ -1,4 +1,3 @@
-<!
 <!--
 
 ** Please read the guidelines below. **

.github/stale.yml

@@ -0,0 +1,35 @@
+---
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 90
+
+# Number of days of inactivity before an stale issue is closed
+daysUntilClose: 30
+
+# Label to use when marking an issue as stale
+staleLabel: triage/stale
+
+issues:
+  # Comment to post when marking an issue as stale.
+  markComment: >
+    This issue has been automatically marked as stale because it has not had
+    recent activity. It will be closed if no further activity occurs. Thank you
+    for your contributions.
+  # Comment to post when closing a stale issue.
+  closeComment: >
+    This issue has been automatically closed because it has not had recent
+    activity since being marked as stale.
+pulls:
+  # Comment to post when marking a PR as stale.
+  markComment: >
+    This PR has been automatically marked as stale because it has not had
+    recent activity. It will be closed if no further activity occurs. Thank you
+    for your contributions.
+
+    To track this PR (even if closed), please open a corresponding issue if one
+    does not already exist.
+  # Comment to post when closing a stale PR.
+  closeComment: >
+    This PR has been automatically closed because it has not had recent
+    activity since being marked as stale.
+
+    Please reopen when work resumes.

.kitchen.yml

@@ -19,7 +19,7 @@ provisioner:
     extra_vars:
       es_major_version: "<%= ENV['VERSION'] %>"
       <% if ENV['VERSION'] == '6.x' %>
-      es_version: '6.8.0'
+      es_version: '6.8.1'
       <% end %>
   <% end %>
 

CHANGELOG.md

@@ -1,3 +1,27 @@
+# Changelog
+
+## 7.4.0 - 2019/10/01
+
+* 7.4.0 as default version
+* Remove compatibility with versions < 6.3
+
+| PR                                                      | Author                                                     | Title                                                     |
+| ------------------------------------------------------- | ---------------------------------------------------------- | --------------------------------------------------------- |
+|[#575](https://github.com/elastic/ansible-elasticsearch/pull/575) | [@flyinggecko](https://github.com/flyinggecko) | Fix name of Elasticsearch Ansible role |
+|[#578](https://github.com/elastic/ansible-elasticsearch/pull/578) | [@jmlrt](https://github.com/jmlrt) | Fix `dict object has no attribute dict_keys` issue with Python3 |
+|[#588](https://github.com/elastic/ansible-elasticsearch/pull/588) | [@broferek](https://github.com/broferek) | Move `userid` and `groupid` in a different place in the role |
+|[#591](https://github.com/elastic/ansible-elasticsearch/pull/591) | [@Crazybus](https://github.com/Crazybus) | Add back in `force_basic_auth` for all http requests |
+|[#582](https://github.com/elastic/ansible-elasticsearch/pull/582) | [@ktibi](https://github.com/ktibi) | Allow disable Elastic official repository setup |
+|[#593](https://github.com/elastic/ansible-elasticsearch/pull/593) | [@jmlrt](https://github.com/jmlrt) | Bunch of small fixes |
+|[#595](https://github.com/elastic/ansible-elasticsearch/pull/595) | [@broferek](https://github.com/broferek) | Set `limitMEMLOCK` for OS using Systemd|
+|[#600](https://github.com/elastic/ansible-elasticsearch/pull/600) | [@titan-architrave](https://github.com/titan-architrave) | Always gather the `es_major_version` variables|
+|[#605](https://github.com/elastic/ansible-elasticsearch/pull/605) | [@jmlrt](https://github.com/jmlrt) | Add doc for migration with data move|
+|[#601](https://github.com/elastic/ansible-elasticsearch/pull/601) | [@LukeRoz](https://github.com/LukeRoz) | Removing package version hold when `es_version_hold: false`|
+|[#612](https://github.com/elastic/ansible-elasticsearch/pull/612) | [@jmlrt](https://github.com/jmlrt) | Add Probot config to manage stale issues/pr|
+|[#614](https://github.com/elastic/ansible-elasticsearch/pull/614) | [@jmlrt](https://github.com/jmlrt) | Describe how to select a different elasticsearch version|
+|[#609](https://github.com/elastic/ansible-elasticsearch/pull/609) | [@jmlrt](https://github.com/jmlrt) | No more 6.3 compatibility + Use default files permissions from Elasticsearch package|
+|[#510](https://github.com/elastic/ansible-elasticsearch/pull/510) | [@verboEse](https://github.com/verboEse) | Don't fetch APT key if existent|
+
 ## 7.1.1 - 2019/06/04
 
 ### Breaking changes

README.md

@@ -31,7 +31,7 @@ This role uses the json_query filter which [requires jmespath](https://github.co
 Create your Ansible playbook with your own tasks, and include the role elasticsearch. You will have to have this repository accessible within the context of playbook.
 
 ```sh
-ansible-galaxy install elastic.elasticsearch,7.1.1
+ansible-galaxy install elastic.elasticsearch,7.4.0
 ```
 
 Then create your playbook yaml adding the role elasticsearch.
@@ -44,9 +44,15 @@ The simplest configuration therefore consists of:
   hosts: localhost
   roles:
     - role: elastic.elasticsearch
+  vars:
+    es_version: 7.4.0
 ```
 
-The above installs a single node 'node1' on the hosts 'localhost'.
+The above installs Elasticsearch 7.4.0 in a single node 'node1' on the hosts 'localhost'.
+
+**Note**:
+Elasticsearch default version is described in [`es_version`](defaults/main.yml#L2). You can override this variable in your playbook to install another version.
+While we are testing this role only with one 7.x and one 6.x version (respectively [7.4.0](defaults/main.yml#L2) and [6.8.1](.kitchen.yml#L22) at the time of writing), this role should work with others version also in most cases.
 
 This role also uses [Ansible tags](http://docs.ansible.com/ansible/playbooks_tags.html). Run your playbook with the `--list-tasks` flag for more information.
 
@@ -258,10 +264,6 @@ X-Pack features, such as Security, are supported.
 The parameter `es_xpack_features` allows to list xpack features to install (example: `["alerting","monitoring","graph","security","ml"]`).
 When the list is empty, it install all features available with the current licence.
 
-The following additional parameters allow X-Pack to be configured:
-
-* ```es_xpack_custom_url``` Url from which X-Pack can be downloaded. This can be used for installations in isolated environments where the elastic.co repo is not accessible. e.g. ```es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-5.5.1.zip"```
-
 * ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/x-pack/current/mapping-roles.html)
 
 
@@ -364,7 +366,7 @@ These can either be set to a user declared in the file based realm, with admin p
 In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behaviour. Options include:
 
 * ```es_enable_xpack```  Default `true`. Setting this to `false` will install the oss release of elasticsearch
-* ```es_version``` (e.g. "7.1.1").
+* ```es_version``` (e.g. "7.4.0").
 * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost".
 * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200**
 * ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Security is enabled. Ensure this user is admin.

defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-es_version: "7.1.1"
+es_version: "7.4.0"
 es_use_snapshot_release: false
 es_enable_xpack: true
 es_package_name: "elasticsearch"
@@ -29,7 +29,7 @@ es_data_dirs:
 es_log_dir: "/var/log/elasticsearch"
 es_action_auto_create_index: true
 es_max_open_files: 65536
-es_max_threads: "{{ 2048 if ( es_version is version_compare('6.0.0', '<')) else 8192 }}"
+es_max_threads: 8192
 es_max_map_count: 262144
 es_allow_downgrades: false
 es_xpack_features: []

filter_plugins/custom.py

@@ -1,11 +1,12 @@
-__author__ = 'dale mcdiarmid'
+__author__ = "dale mcdiarmid"
 
 import re
 import os.path
 from six import string_types
 
-def modify_list(values=[], pattern='', replacement='', ignorecase=False):
+
-    ''' Perform a `re.sub` on every item in the list'''
+def modify_list(values=[], pattern="", replacement="", ignorecase=False):
+    """ Perform a `re.sub` on every item in the list"""
     if ignorecase:
         flags = re.I
     else:
@@ -13,45 +14,62 @@ def modify_list(values=[], pattern='', replacement='', ignorecase=False):
     _re = re.compile(pattern, flags=flags)
     return [_re.sub(replacement, value) for value in values]
 
-def append_to_list(values=[], suffix=''):
-    if isinstance(values, string_types):
-        values = values.split(',')
-    return [str(value+suffix) for value in values]
 
-def array_to_str(values=[],separator=','):
+def append_to_list(values=[], suffix=""):
+    if isinstance(values, string_types):
+        values = values.split(",")
+    return [str(value + suffix) for value in values]
+
+
+def array_to_str(values=[], separator=","):
     return separator.join(values)
 
-def extract_role_users(users={},exclude_users=[]):
+
-    role_users=[]
+def extract_role_users(users={}, exclude_users=[]):
-    for user,details in users.iteritems():
+    role_users = []
+    for user, details in users.items():
         if user not in exclude_users and "roles" in details:
             for role in details["roles"]:
-                role_users.append(role+":"+user)
+                role_users.append(role + ":" + user)
     return role_users
 
-def filename(filename=''):
+
+def filename(filename=""):
     return os.path.splitext(os.path.basename(filename))[0]
 
+
 def remove_reserved(user_roles={}):
     not_reserved = []
-    for user_role,details in user_roles.items():
+    for user_role, details in user_roles.items():
-        if not "metadata" in details or not "_reserved" in details["metadata"] or not details["metadata"]["_reserved"]:
+        if (
+            not "metadata" in details
+            or not "_reserved" in details["metadata"]
+            or not details["metadata"]["_reserved"]
+        ):
             not_reserved.append(user_role)
     return not_reserved
 
+
 def filter_reserved(users_role={}):
     reserved = []
-    for user_role,details in users_role.items():
+    for user_role, details in users_role.items():
-        if "metadata" in details and "_reserved" in details["metadata"] and details["metadata"]["_reserved"]:
+        if (
+            "metadata" in details
+            and "_reserved" in details["metadata"]
+            and details["metadata"]["_reserved"]
+        ):
             reserved.append(user_role)
     return reserved
 
+
 class FilterModule(object):
     def filters(self):
-        return {'modify_list': modify_list,
+        return {
-        'append_to_list':append_to_list,
+            "modify_list": modify_list,
-        'filter_reserved':filter_reserved,
+            "append_to_list": append_to_list,
-        'array_to_str':array_to_str,
+            "filter_reserved": filter_reserved,
-        'extract_role_users':extract_role_users,
+            "array_to_str": array_to_str,
-        'remove_reserved':remove_reserved,
+            "extract_role_users": extract_role_users,
-        'filename':filename}
+            "remove_reserved": remove_reserved,
+            "filename": filename,
+        }

handlers/main.yml

@@ -1,7 +1,8 @@
 
 - name: reload systemd configuration
   become: yes
-  command: systemctl daemon-reload
+  systemd:
+    daemon_reload: true
 
 # Restart service and ensure it is enabled
 

tasks/compatibility-variables.yml

@@ -8,31 +8,12 @@
 
 - name: Set the defaults here otherwise they can't be overriden in the same play if the role is called twice
   set_fact:
-    es_open_xpack: true
-    es_install_xpack: false
-    es_users_path: "users"
-    es_xpack_conf_subdir: ""
     es_repo_name: "{{ es_major_version }}"
-    es_xpack_users_command: "elasticsearch-users"
     es_package_name: "elasticsearch"
     es_other_package_name: "elasticsearch-oss"
     es_other_repo_name: "{{ 'oss-' + es_major_version }}"
     es_other_apt_url: "deb {{ es_repo_base }}/packages/{{ 'oss-' + es_major_version }}/apt stable main"
 
-- name: Detect if es_version is before X-Pack was open and included
-  set_fact:
-    es_open_xpack: false
-  when: "es_version is version_compare('6.3.0', '<')"
-
-- name: If this is an older version we need to install X-Pack as a plugin and use a different users command
-  set_fact:
-    es_install_xpack: true
-    es_xpack_users_command: "x-pack/users"
-    es_xpack_conf_subdir: "/x-pack"
-  when:
-    - not es_open_xpack
-    - es_enable_xpack
-
 - name: Use the oss repo and package if xpack is not being used
   set_fact:
     es_repo_name: "{{ 'oss-' + es_major_version }}"
@@ -41,7 +22,6 @@
     es_package_name: "elasticsearch-oss"
     es_other_package_name: "elasticsearch"
   when:
-    - es_open_xpack
     - not es_enable_xpack
 
 - name: Set the URL scheme based if http ssl/tls is enabled

tasks/elasticsearch-Debian.yml

@@ -49,6 +49,7 @@
   - name: Debian - Add Elasticsearch repository key
     apt_key:
       url: '{{ es_apt_key }}'
+      id: '{{ es_apt_key_id }}'
       state: present
     when: es_add_repository and es_apt_key | string
 
@@ -92,7 +93,7 @@
   register: debian_elasticsearch_install_from_repo
   notify: restart elasticsearch
   environment:
-    ES_PATH_CONF: "/etc/elasticsearch"
+    ES_PATH_CONF: "{{ es_conf_dir }}"
 
 - name: Debian - hold elasticsearch version
   become: yes
@@ -112,4 +113,4 @@
   register: elasticsearch_install_from_package
   notify: restart elasticsearch
   environment:
-    ES_PATH_CONF: "/etc/elasticsearch"
+    ES_PATH_CONF: "{{ es_conf_dir }}"

tasks/elasticsearch-RedHat.yml

@@ -52,7 +52,7 @@
   retries: 5
   delay: 10
   environment:
-    ES_PATH_CONF: "/etc/elasticsearch"
+    ES_PATH_CONF: "{{ es_conf_dir }}"
 
 - name: RedHat - Install Elasticsearch from url
   become: yes

tasks/elasticsearch-config.yml

@@ -1,27 +1,35 @@
 ---
 # Configure Elasticsearch Node
 
-#Create required directories
+#Create conf directory
-- name: Create Directories
+- name: Create Configuration Directory
   become: yes
-  file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }}
+  file: path={{ es_conf_dir }} state=directory owner=root group={{ es_group }} mode=2750
+
+#Create pid directory
+- name: Create PID Directory
+  become: yes
+  file: path={{ es_pid_dir }} state=directory owner={{ es_user }} group={{ es_group }} mode=0755
+
+#Create required directories
+- name: Create Others Directories
+  become: yes
+  file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} mode=2750
   with_items:
-    - "{{ es_pid_dir }}"
     - "{{ es_log_dir }}"
-    - "{{ es_conf_dir }}"
     - "{{ es_data_dirs }}"
 
 #Copy the config template
 - name: Copy Configuration File
   become: yes
-  template: src=elasticsearch.yml.j2 dest={{ es_conf_dir }}/elasticsearch.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=elasticsearch.yml.j2 dest={{ es_conf_dir }}/elasticsearch.yml owner=root group={{ es_group }} mode=0660 force=yes
   register: system_change
   notify: restart elasticsearch
 
 #Copy the default file
 - name: Copy Default File
   become: yes
-  template: src=elasticsearch.j2 dest={{ default_file }} mode=0644 force=yes
+  template: src=elasticsearch.j2 dest={{ default_file }} owner=root group={{ es_group }} mode=0660 force=yes
   notify: restart elasticsearch
 
 #Copy the systemd specific file if systemd is installed
@@ -30,7 +38,7 @@
   block:
     - name: Make sure destination dir exists
       file: path={{ sysd_config_file | dirname }} state=directory mode=0755
-  
+
     - name: Copy specific ElasticSearch Systemd config file
       ini_file: path={{ sysd_config_file }} section=Service option=LimitMEMLOCK value=infinity mode=0644
       notify:
@@ -40,10 +48,10 @@
 #Copy the logging.yml
 - name: Copy log4j2.properties File
   become: yes
-  template: src={{ es_config_log4j2 }} dest={{ es_conf_dir }}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src={{ es_config_log4j2 }} dest={{ es_conf_dir }}/log4j2.properties owner=root group={{ es_group }} mode=0660 force=yes
   notify: restart elasticsearch
 
 - name: Copy jvm.options File
   become: yes
-  template: src=jvm.options.j2 dest={{ es_conf_dir }}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=jvm.options.j2 dest={{ es_conf_dir }}/jvm.options owner=root group={{ es_group }} mode=0660 force=yes
   notify: restart elasticsearch

tasks/elasticsearch-plugins.yml

@@ -17,7 +17,6 @@
   file:
     dest: "{{ es_home }}/plugins/x-pack"
     state: "absent"
-  when: es_open_xpack
 
 #List currently installed plugins. We have to list the directories as the list commmand fails if the ES version is different than the plugin version.
 - name: Check installed elasticsearch plugins
@@ -80,8 +79,3 @@
   until: plugin_installed.rc == 0
   retries: 5
   delay: 5
-
-#Set permissions on plugins directory
-- name: Set Plugin Directory Permissions
-  become: yes
-  file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes

tasks/elasticsearch-template.yml

@@ -2,13 +2,14 @@
 
 - name: ensure templates dir is created
   file:
-    path: /etc/elasticsearch/templates
+    path: "{{ es_conf_dir }}/templates"
     state: directory
-    owner: "{{ es_user }}"
+    owner: root
     group: "{{ es_group }}"
+    mode: 2750
 
 - name: Copy templates to elasticsearch
-  copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }}
+  copy: src={{ item }} dest={{ es_conf_dir }}/templates owner=root group={{ es_group }} mode=0660
   register: load_templates
   with_fileglob:
     - "{{ es_templates_fileglob | default('') }}"

tasks/xpack/elasticsearch-xpack-install.yml

@@ -1,68 +0,0 @@
----
-
-#Test if feature is installed
-- name: Test if x-pack is installed
-  shell: "{{es_home}}/bin/elasticsearch-plugin list | grep x-pack"
-  become: yes
-  register: x_pack_installed
-  changed_when: False
-  failed_when: "'ERROR' in x_pack_installed.stdout"
-  check_mode: no
-  ignore_errors: yes
-  environment:
-    CONF_DIR: "{{ es_conf_dir }}"
-    ES_PATH_CONF: "{{ es_conf_dir }}"
-    ES_INCLUDE: "{{ default_file }}"
-
-
-#Remove X-Pack if installed and its not been requested or the ES version has changed
-- name: Remove x-pack plugin
-  become: yes
-  command: "{{es_home}}/bin/elasticsearch-plugin remove x-pack"
-  register: xpack_state
-  failed_when: "'ERROR' in xpack_state.stdout"
-  changed_when: xpack_state.rc == 0
-  when: x_pack_installed.rc == 0 and (not es_enable_xpack or es_version_changed)
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ es_conf_dir }}"
-    ES_PATH_CONF: "{{ es_conf_dir }}"
-    ES_INCLUDE: "{{ default_file }}"
-
-
-#Install plugin if not installed, or the es version has changed (so removed above), and its been requested
-- name: Download x-pack from url
-  get_url: url={{ es_xpack_custom_url }} dest=/tmp/x-pack-{{ es_version }}.zip
-  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined)
-
-- name: Install x-pack plugin from local
-  become: yes
-  command: >
-    {{es_home}}/bin/elasticsearch-plugin install --silent --batch file:///tmp/x-pack-{{ es_version }}.zip
-  register: xpack_state
-  changed_when: xpack_state.rc == 0
-  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined)
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ es_conf_dir }}"
-    ES_PATH_CONF: "{{ es_conf_dir }}"
-    ES_INCLUDE: "{{ default_file }}"
-
-- name: Delete x-pack zip file
-  file: dest=/tmp/x-pack-{{ es_version }}.zip state=absent
-  when: es_xpack_custom_url is defined
-
-- name: Install x-pack plugin from elastic.co
-  become: yes
-  command: >
-    {{es_home}}/bin/elasticsearch-plugin install --silent --batch x-pack
-  register: xpack_state
-  failed_when: "'ERROR' in xpack_state.stdout"
-  changed_when: xpack_state.rc == 0
-  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is not defined)
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ es_conf_dir }}"
-    ES_PATH_CONF: "{{ es_conf_dir }}"
-    ES_INCLUDE: "{{ default_file }}"
-    ES_JAVA_OPTS: "{% if es_proxy_host is defined and es_proxy_host != '' %}-Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }}{% endif %}"

tasks/xpack/elasticsearch-xpack.yml

@@ -1,24 +1,11 @@
 ---
 
-- name: set fact es_version_changed
-  set_fact: es_version_changed={{ ((elasticsearch_install_from_package is defined and (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) }}
-
-- name: include elasticsearch-xpack-install.yml
-  include: elasticsearch-xpack-install.yml
-  when: es_install_xpack
-
 #Security configuration
 - name: include security/elasticsearch-security.yml
   include: security/elasticsearch-security.yml
   when: es_enable_xpack
 
-#Add any feature specific configuration here
-- name: Set Plugin Directory Permissions
-  become: yes
-  file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes
-
 #Make sure elasticsearch.keystore has correct Permissions
 - name: Set elasticsearch.keystore Permissions
   become: yes
-  file: state=file path={{ es_conf_dir }}/elasticsearch.keystore owner={{ es_user }} group={{ es_group }}
+  file: state=file path={{ es_conf_dir }}/elasticsearch.keystore owner=root group={{ es_group }} mode=0660
-  when: es_enable_xpack

tasks/xpack/security/elasticsearch-security-file.yml

@@ -1,7 +1,10 @@
 ---
-- name: set fact manage_file_users
+- set_fact: manage_file_users=false
-  set_fact: manage_file_users=es_users is defined and es_users.file is defined and es_users.file.keys() | list | length > 0
 
+- set_fact: manage_file_users=true
+  when: es_users is defined and es_users.file is defined and es_users.file.keys() | list | length > 0
+
+# Users migration from elasticsearch < 6.3 versions
 - name: Check if old users file exists
   stat:
     path: '{{ es_conf_dir }}/x-pack/users'
@@ -13,22 +16,16 @@
     remote_src: yes
     force: no # only copy it if the new path doesn't exist yet
     src: "{{ es_conf_dir }}/x-pack/users"
-    dest: "{{ es_conf_dir }}{{ es_xpack_conf_subdir }}/users"
+    dest: "{{ es_conf_dir }}/users"
-  when: old_users_file.stat.exists
-
-- name: Create the users file if it doesn't exist
-  copy:
-    content: ""
-    dest: "{{ es_conf_dir }}{{ es_xpack_conf_subdir }}/users"
-    force: no # this ensures it only creates it if it does not exist
     group: "{{ es_group }}"
-    owner: "{{ es_user }}"
+    owner: root
-    mode: 0555
+  when: old_users_file.stat.exists
+# End of users migrations
 
 #List current users
 - name: List Users
   become: yes
-  shell: cat {{ es_conf_dir }}{{es_xpack_conf_subdir}}/users | awk -F':' '{print $1}'
+  shell: cat {{ es_conf_dir }}/users | awk -F':' '{print $1}'
   register: current_file_users
   when: manage_file_users
   changed_when: False
@@ -42,7 +39,7 @@
 - name: Remove Users
   become: yes
   command: >
-    {{es_home}}/bin/{{es_xpack_users_command}} userdel {{item}}
+    {{es_home}}/bin/elasticsearch-users userdel {{item}}
   with_items: "{{users_to_remove | default([])}}"
   when: manage_file_users
   environment:
@@ -58,7 +55,7 @@
 - name: Add Users
   become: yes
   command: >
-    {{es_home}}/bin/{{es_xpack_users_command}} useradd {{item}} -p {{es_users.file[item].password}}
+    {{es_home}}/bin/elasticsearch-users useradd {{item}} -p {{es_users.file[item].password}}
   with_items: "{{ users_to_add | default([]) }}"
   when: manage_file_users
   no_log: True
@@ -71,7 +68,7 @@
 - name: Set User Passwords
   become: yes
   command: >
-    {{es_home}}/bin/{{es_xpack_users_command}} passwd {{ item }} -p {{es_users.file[item].password}}
+    {{es_home}}/bin/elasticsearch-users passwd {{ item }} -p {{es_users.file[item].password}}
   with_items: "{{ es_users.file.keys() | list }}"
   when: manage_file_users
   #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip.
@@ -89,16 +86,11 @@
 #Copy Roles files
 - name: Copy roles.yml File for Instance
   become: yes
-  template: src=security/roles.yml.j2 dest={{ es_conf_dir }}{{es_xpack_conf_subdir}}/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=security/roles.yml.j2 dest={{ es_conf_dir }}/roles.yml owner=root group={{ es_group }} mode=0660 force=yes
   when: es_roles is defined and es_roles.file is defined
 
 #Overwrite users_roles file
 - name: Copy User Roles
   become: yes
-  template: src=security/users_roles.j2 dest={{ es_conf_dir }}{{es_xpack_conf_subdir}}/users_roles mode=0644 force=yes
+  template: src=security/users_roles.j2 dest={{ es_conf_dir }}/users_roles owner=root group={{ es_group }} mode=0660 force=yes
   when: manage_file_users and users_roles | length > 0
-
-#Set permission on security directory. E.g. if 2 nodes are installed on the same machine, the second node will not get the users file created at install, causing the files being created at es_users call and then having the wrong Permissions.
-- name: Set Security Directory Permissions Recursive
-  become: yes
-  file: state=directory path={{ es_conf_dir }}{{es_xpack_conf_subdir}}/ owner={{ es_user }} group={{ es_group }} recurse=yes

tasks/xpack/security/elasticsearch-security.yml

@@ -3,12 +3,6 @@
 
 #TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6
 
-#Ensure x-pack conf directory is created if necessary
-- name: Ensure x-pack conf directory exists (file)
-  file: path={{ es_conf_dir }}{{ es_xpack_conf_subdir }} state=directory owner={{ es_user }} group={{ es_group }}
-  changed_when: False
-  when: (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined)
-
 #-----------------------------Create Bootstrap User-----------------------------------
 ### START BLOCK elasticsearch keystore ###
 - name: create the elasticsearch keystore
@@ -52,13 +46,5 @@
 #Copy Roles files
 - name: Copy role_mapping.yml File for Instance
   become: yes
-  template: src=security/role_mapping.yml.j2 dest={{ es_conf_dir }}{{es_xpack_conf_subdir}}/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=security/role_mapping.yml.j2 dest={{ es_conf_dir }}/role_mapping.yml owner=root group={{ es_group }} mode=0660 force=yes
   when: es_role_mapping is defined
-
-#------------------------------------------------------------------------------------
-
-#Ensure security conf directory is created
-- name: Ensure security conf directory exists
-  become: yes
-  file: path={{ es_conf_dir }}/security state=directory owner={{ es_user }} group={{ es_group }}
-  changed_when: False

templates/elasticsearch.yml.j2

@@ -15,10 +15,6 @@ node.name: {{inventory_hostname}}
 
 # Path to directory containing configuration (this file and logging.yml):
 
-{% if (es_version is version_compare('6.0.0', '<')) %}
-path.conf: {{ es_conf_dir }}
-{% endif %}
-
 path.data: {{ es_data_dirs | array_to_str }}
 
 path.logs: {{ es_log_dir }}

templates/log4j2.properties.j2

@@ -11,23 +11,14 @@ appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
 
 appender.rolling.type = RollingFile
 appender.rolling.name = rolling
-{% if (es_version is version_compare('6.0.0', '<')) %}
-appender.rolling.fileName = ${sys:es.logs}.log
-{% else %}
 appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
-{% endif %}
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-{% if (es_version is version_compare('6.0.0', '<')) %}
-appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log
-{% else %}
 appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
-{% endif %}
 appender.rolling.policies.type = Policies
 appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.rolling.policies.time.interval = 1
 appender.rolling.policies.time.modulate = true
-{% if (es_version is version_compare('6.0.0', '>')) %}
 appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
 appender.rolling.policies.size.size = 128MB
 appender.rolling.strategy.type = DefaultRolloverStrategy
@@ -38,25 +29,16 @@ appender.rolling.strategy.action.condition.type = IfFileName
 appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
 appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize
 appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB
-{% endif %}
 rootLogger.level = info
 rootLogger.appenderRef.console.ref = console
 rootLogger.appenderRef.rolling.ref = rolling
 
 appender.deprecation_rolling.type = RollingFile
 appender.deprecation_rolling.name = deprecation_rolling
-{% if (es_version is version_compare('6.0.0', '<')) %}
-appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log
-{% else %}
 appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
-{% endif %}
 appender.deprecation_rolling.layout.type = PatternLayout
 appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-{% if (es_version is version_compare('6.0.0', '<')) %}
-appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz
-{% else %}
 appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
-{% endif %}
 appender.deprecation_rolling.policies.type = Policies
 appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
 appender.deprecation_rolling.policies.size.size = 1GB
@@ -70,18 +52,12 @@ logger.deprecation.additivity = false
 
 appender.index_search_slowlog_rolling.type = RollingFile
 appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
-{% if (es_version is version_compare('6.0.0', '<')) %}
 appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log
-{% else %}
 appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
-{% endif %}
 appender.index_search_slowlog_rolling.layout.type = PatternLayout
 appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-{% if (es_version is version_compare('6.0.0', '<')) %}
 appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
-{% else %}
 appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
-{% endif %}
 appender.index_search_slowlog_rolling.policies.type = Policies
 appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.index_search_slowlog_rolling.policies.time.interval = 1
@@ -94,18 +70,10 @@ logger.index_search_slowlog_rolling.additivity = false
 
 appender.index_indexing_slowlog_rolling.type = RollingFile
 appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
-{% if (es_version is version_compare('6.0.0', '<')) %}
-appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log
-{% else %}
 appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
-{% endif %}
 appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
 appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-{% if (es_version is version_compare('6.0.0', '<')) %}
-appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
-{% else %}
 appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
-{% endif %}
 appender.index_indexing_slowlog_rolling.policies.type = Policies
 appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
 appender.index_indexing_slowlog_rolling.policies.time.interval = 1

test/integration/helpers/serverspec/oss_spec.rb

@@ -3,11 +3,11 @@ require 'spec_helper'
 shared_examples 'oss::init' do |vars|
   describe file("/etc/elasticsearch/log4j2.properties") do
     it { should be_file }
-    it { should be_owned_by 'elasticsearch' }
+    it { should be_owned_by 'root' }
     it { should_not contain 'CUSTOM LOG4J FILE' }
   end
   describe file("/etc/elasticsearch/jvm.options") do
     it { should be_file }
-    it { should be_owned_by vars['es_user'] }
+    it { should be_owned_by 'root' }
   end
 end

test/integration/helpers/serverspec/shared_spec.rb

@@ -108,11 +108,11 @@ shared_examples 'shared::init' do |vars|
   if vars['es_templates']
     describe file('/etc/elasticsearch/templates') do
       it { should be_directory }
-      it { should be_owned_by vars['es_user'] }
+      it { should be_owned_by 'root' }
     end
     describe file('/etc/elasticsearch/templates/basic.json') do
       it { should be_file }
-      it { should be_owned_by vars['es_user'] }
+      it { should be_owned_by 'root' }
     end
     #This is possibly subject to format changes in the response across versions so may fail in the future
     describe 'Template Contents Correct' do
@@ -138,7 +138,7 @@ shared_examples 'shared::init' do |vars|
       name = plugin['plugin']
       describe file('/usr/share/elasticsearch/plugins/'+name) do
         it { should be_directory }
-        it { should be_owned_by vars['es_user'] }
+        it { should be_owned_by 'root' }
       end
       it 'should be installed and the right version' do
         plugins = curl_json("#{es_api_url}/_nodes/plugins", username=username, password=password)
@@ -152,6 +152,7 @@ shared_examples 'shared::init' do |vars|
     end
   end
   describe file("/etc/elasticsearch/elasticsearch.yml") do
+    it { should be_owned_by 'root' }
     it { should contain "node.name: localhost" }
     it { should contain 'cluster.name: elasticsearch' }
     it { should_not contain "path.conf: /etc/elasticsearch" }

test/integration/helpers/serverspec/xpack_upgrade_spec.rb

@@ -4,14 +4,14 @@ vars = JSON.parse(File.read('/tmp/vars.json'))
 
 shared_examples 'xpack_upgrade::init' do |vars|
   #Test users file, users_roles and roles.yml
-  describe file("/etc/elasticsearch/#{vars['es_xpack_conf_subdir']}/users_roles") do
+  describe file("/etc/elasticsearch/users_roles") do
-    it { should be_owned_by 'elasticsearch' }
+    it { should be_owned_by 'root' }
     it { should contain 'admin:es_admin' }
     it { should contain 'power_user:testUser' }
   end
 
-  describe file("/etc/elasticsearch/#{vars['es_xpack_conf_subdir']}/users") do
+  describe file("/etc/elasticsearch/users") do
-    it { should be_owned_by 'elasticsearch' }
+    it { should be_owned_by 'root' }
     it { should contain 'testUser:' }
     it { should contain 'es_admin:' }
   end
@@ -36,8 +36,8 @@ shared_examples 'xpack_upgrade::init' do |vars|
   end
 
   #Test contents of role_mapping.yml
-  describe file("/etc/elasticsearch/#{vars['es_xpack_conf_subdir']}/role_mapping.yml") do
+  describe file("/etc/elasticsearch/role_mapping.yml") do
-    it { should be_owned_by 'elasticsearch' }
+    it { should be_owned_by 'root' }
     it { should contain 'power_user:' }
     it { should contain '- cn=admins,dc=example,dc=com' }
     it { should contain 'user:' }

test/integration/xpack.yml

@@ -7,5 +7,4 @@
   roles:
     - elasticsearch
   vars:
-    es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip"
     es_heap_size: 2g

vars/Debian.yml

@@ -2,3 +2,4 @@
 java: "{% if es_java is defined %}{{es_java}}{% else %}openjdk-8-jre-headless{% endif %}"
 default_file: "/etc/default/elasticsearch"
 es_home: "/usr/share/elasticsearch"
+es_apt_key_id: "46095ACC8548582C1A2699A9D27D666CD88E42B4"