Merge branch 'master' into AddSSLSupport

tasks/compatibility-variables.yml

@@ -8,31 +8,12 @@
 
 - name: Set the defaults here otherwise they can't be overriden in the same play if the role is called twice
   set_fact:
-    es_open_xpack: true
-    es_install_xpack: false
-    es_users_path: "users"
-    es_xpack_conf_subdir: ""
     es_repo_name: "{{ es_major_version }}"
-    es_xpack_users_command: "elasticsearch-users"
     es_package_name: "elasticsearch"
     es_other_package_name: "elasticsearch-oss"
     es_other_repo_name: "{{ 'oss-' + es_major_version }}"
     es_other_apt_url: "deb {{ es_repo_base }}/packages/{{ 'oss-' + es_major_version }}/apt stable main"
 
-- name: Detect if es_version is before X-Pack was open and included
-  set_fact:
-    es_open_xpack: false
-  when: "es_version is version_compare('6.3.0', '<')"
-
-- name: If this is an older version we need to install X-Pack as a plugin and use a different users command
-  set_fact:
-    es_install_xpack: true
-    es_xpack_users_command: "x-pack/users"
-    es_xpack_conf_subdir: "/x-pack"
-  when:
-    - not es_open_xpack
-    - es_enable_xpack
-
 - name: Use the oss repo and package if xpack is not being used
   set_fact:
     es_repo_name: "{{ 'oss-' + es_major_version }}"
@@ -41,7 +22,6 @@
     es_package_name: "elasticsearch-oss"
     es_other_package_name: "elasticsearch"
   when:
-    - es_open_xpack
     - not es_enable_xpack
 
 - name: Set the URL scheme based if http ssl/tls is enabled

tasks/elasticsearch-Debian.yml

@@ -49,6 +49,7 @@
   - name: Debian - Add Elasticsearch repository key
     apt_key:
       url: '{{ es_apt_key }}'
+      id: '{{ es_apt_key_id }}'
       state: present
     when: es_add_repository and es_apt_key | string
 
@@ -92,7 +93,7 @@
   register: debian_elasticsearch_install_from_repo
   notify: restart elasticsearch
   environment:
-    ES_PATH_CONF: "/etc/elasticsearch"
+    ES_PATH_CONF: "{{ es_conf_dir }}"
 
 - name: Debian - hold elasticsearch version
   become: yes
@@ -112,4 +113,4 @@
   register: elasticsearch_install_from_package
   notify: restart elasticsearch
   environment:
-    ES_PATH_CONF: "/etc/elasticsearch"
+    ES_PATH_CONF: "{{ es_conf_dir }}"

tasks/elasticsearch-RedHat.yml

@@ -52,7 +52,7 @@
   retries: 5
   delay: 10
   environment:
-    ES_PATH_CONF: "/etc/elasticsearch"
+    ES_PATH_CONF: "{{ es_conf_dir }}"
 
 - name: RedHat - Install Elasticsearch from url
   become: yes

tasks/elasticsearch-config.yml

@@ -1,27 +1,35 @@
 ---
 # Configure Elasticsearch Node
 
-#Create required directories
-- name: Create Directories
+#Create conf directory
+- name: Create Configuration Directory
   become: yes
-  file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }}
+  file: path={{ es_conf_dir }} state=directory owner=root group={{ es_group }} mode=2750
+
+#Create pid directory
+- name: Create PID Directory
+  become: yes
+  file: path={{ es_pid_dir }} state=directory owner={{ es_user }} group={{ es_group }} mode=0755
+
+#Create required directories
+- name: Create Others Directories
+  become: yes
+  file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} mode=2750
   with_items:
-    - "{{ es_pid_dir }}"
     - "{{ es_log_dir }}"
-    - "{{ es_conf_dir }}"
     - "{{ es_data_dirs }}"
 
 #Copy the config template
 - name: Copy Configuration File
   become: yes
-  template: src=elasticsearch.yml.j2 dest={{ es_conf_dir }}/elasticsearch.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=elasticsearch.yml.j2 dest={{ es_conf_dir }}/elasticsearch.yml owner=root group={{ es_group }} mode=0660 force=yes
   register: system_change
   notify: restart elasticsearch
 
 #Copy the default file
 - name: Copy Default File
   become: yes
-  template: src=elasticsearch.j2 dest={{ default_file }} mode=0644 force=yes
+  template: src=elasticsearch.j2 dest={{ default_file }} owner=root group={{ es_group }} mode=0660 force=yes
   notify: restart elasticsearch
 
 #Copy the systemd specific file if systemd is installed
@@ -30,7 +38,7 @@
   block:
     - name: Make sure destination dir exists
       file: path={{ sysd_config_file | dirname }} state=directory mode=0755
-  
+
     - name: Copy specific ElasticSearch Systemd config file
       ini_file: path={{ sysd_config_file }} section=Service option=LimitMEMLOCK value=infinity mode=0644
       notify:
@@ -40,10 +48,10 @@
 #Copy the logging.yml
 - name: Copy log4j2.properties File
   become: yes
-  template: src={{ es_config_log4j2 }} dest={{ es_conf_dir }}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src={{ es_config_log4j2 }} dest={{ es_conf_dir }}/log4j2.properties owner=root group={{ es_group }} mode=0660 force=yes
   notify: restart elasticsearch
 
 - name: Copy jvm.options File
   become: yes
-  template: src=jvm.options.j2 dest={{ es_conf_dir }}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=jvm.options.j2 dest={{ es_conf_dir }}/jvm.options owner=root group={{ es_group }} mode=0660 force=yes
   notify: restart elasticsearch

tasks/elasticsearch-plugins.yml

@@ -17,7 +17,6 @@
   file:
     dest: "{{ es_home }}/plugins/x-pack"
     state: "absent"
-  when: es_open_xpack
 
 #List currently installed plugins. We have to list the directories as the list commmand fails if the ES version is different than the plugin version.
 - name: Check installed elasticsearch plugins
@@ -80,8 +79,3 @@
   until: plugin_installed.rc == 0
   retries: 5
   delay: 5
-
-#Set permissions on plugins directory
-- name: Set Plugin Directory Permissions
-  become: yes
-  file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes

tasks/elasticsearch-template.yml

@@ -2,13 +2,14 @@
 
 - name: ensure templates dir is created
   file:
-    path: /etc/elasticsearch/templates
+    path: "{{ es_conf_dir }}/templates"
     state: directory
-    owner: "{{ es_user }}"
+    owner: root
     group: "{{ es_group }}"
+    mode: 2750
 
 - name: Copy templates to elasticsearch
-  copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }}
+  copy: src={{ item }} dest={{ es_conf_dir }}/templates owner=root group={{ es_group }} mode=0660
   register: load_templates
   with_fileglob:
     - "{{ es_templates_fileglob | default('') }}"

tasks/xpack/elasticsearch-xpack-install.yml

@@ -1,68 +0,0 @@
----
-
-#Test if feature is installed
-- name: Test if x-pack is installed
-  shell: "{{es_home}}/bin/elasticsearch-plugin list | grep x-pack"
-  become: yes
-  register: x_pack_installed
-  changed_when: False
-  failed_when: "'ERROR' in x_pack_installed.stdout"
-  check_mode: no
-  ignore_errors: yes
-  environment:
-    CONF_DIR: "{{ es_conf_dir }}"
-    ES_PATH_CONF: "{{ es_conf_dir }}"
-    ES_INCLUDE: "{{ default_file }}"
-
-
-#Remove X-Pack if installed and its not been requested or the ES version has changed
-- name: Remove x-pack plugin
-  become: yes
-  command: "{{es_home}}/bin/elasticsearch-plugin remove x-pack"
-  register: xpack_state
-  failed_when: "'ERROR' in xpack_state.stdout"
-  changed_when: xpack_state.rc == 0
-  when: x_pack_installed.rc == 0 and (not es_enable_xpack or es_version_changed)
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ es_conf_dir }}"
-    ES_PATH_CONF: "{{ es_conf_dir }}"
-    ES_INCLUDE: "{{ default_file }}"
-
-
-#Install plugin if not installed, or the es version has changed (so removed above), and its been requested
-- name: Download x-pack from url
-  get_url: url={{ es_xpack_custom_url }} dest=/tmp/x-pack-{{ es_version }}.zip
-  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined)
-
-- name: Install x-pack plugin from local
-  become: yes
-  command: >
-    {{es_home}}/bin/elasticsearch-plugin install --silent --batch file:///tmp/x-pack-{{ es_version }}.zip
-  register: xpack_state
-  changed_when: xpack_state.rc == 0
-  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined)
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ es_conf_dir }}"
-    ES_PATH_CONF: "{{ es_conf_dir }}"
-    ES_INCLUDE: "{{ default_file }}"
-
-- name: Delete x-pack zip file
-  file: dest=/tmp/x-pack-{{ es_version }}.zip state=absent
-  when: es_xpack_custom_url is defined
-
-- name: Install x-pack plugin from elastic.co
-  become: yes
-  command: >
-    {{es_home}}/bin/elasticsearch-plugin install --silent --batch x-pack
-  register: xpack_state
-  failed_when: "'ERROR' in xpack_state.stdout"
-  changed_when: xpack_state.rc == 0
-  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is not defined)
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ es_conf_dir }}"
-    ES_PATH_CONF: "{{ es_conf_dir }}"
-    ES_INCLUDE: "{{ default_file }}"
-    ES_JAVA_OPTS: "{% if es_proxy_host is defined and es_proxy_host != '' %}-Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }}{% endif %}"

tasks/xpack/elasticsearch-xpack.yml

@@ -1,24 +1,11 @@
 ---
 
-- name: set fact es_version_changed
-  set_fact: es_version_changed={{ ((elasticsearch_install_from_package is defined and (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) }}
-
-- name: include elasticsearch-xpack-install.yml
-  include: elasticsearch-xpack-install.yml
-  when: es_install_xpack
-
 #Security configuration
 - name: include security/elasticsearch-security.yml
   include: security/elasticsearch-security.yml
   when: es_enable_xpack
 
-#Add any feature specific configuration here
-- name: Set Plugin Directory Permissions
-  become: yes
-  file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes
-
 #Make sure elasticsearch.keystore has correct Permissions
 - name: Set elasticsearch.keystore Permissions
   become: yes
-  file: state=file path={{ es_conf_dir }}/elasticsearch.keystore owner={{ es_user }} group={{ es_group }}
-  when: es_enable_xpack
+  file: state=file path={{ es_conf_dir }}/elasticsearch.keystore owner=root group={{ es_group }} mode=0660

tasks/xpack/security/elasticsearch-security-file.yml

@@ -1,7 +1,10 @@
 ---
-- name: set fact manage_file_users
-  set_fact: manage_file_users=es_users is defined and es_users.file is defined and es_users.file.keys() | list | length > 0
+- set_fact: manage_file_users=false
 
+- set_fact: manage_file_users=true
+  when: es_users is defined and es_users.file is defined and es_users.file.keys() | list | length > 0
+
+# Users migration from elasticsearch < 6.3 versions
 - name: Check if old users file exists
   stat:
     path: '{{ es_conf_dir }}/x-pack/users'
@@ -13,22 +16,16 @@
     remote_src: yes
     force: no # only copy it if the new path doesn't exist yet
     src: "{{ es_conf_dir }}/x-pack/users"
-    dest: "{{ es_conf_dir }}{{ es_xpack_conf_subdir }}/users"
-  when: old_users_file.stat.exists
-
-- name: Create the users file if it doesn't exist
-  copy:
-    content: ""
-    dest: "{{ es_conf_dir }}{{ es_xpack_conf_subdir }}/users"
-    force: no # this ensures it only creates it if it does not exist
+    dest: "{{ es_conf_dir }}/users"
     group: "{{ es_group }}"
-    owner: "{{ es_user }}"
-    mode: 0555
+    owner: root
+  when: old_users_file.stat.exists
+# End of users migrations
 
 #List current users
 - name: List Users
   become: yes
-  shell: cat {{ es_conf_dir }}{{es_xpack_conf_subdir}}/users | awk -F':' '{print $1}'
+  shell: cat {{ es_conf_dir }}/users | awk -F':' '{print $1}'
   register: current_file_users
   when: manage_file_users
   changed_when: False
@@ -42,7 +39,7 @@
 - name: Remove Users
   become: yes
   command: >
-    {{es_home}}/bin/{{es_xpack_users_command}} userdel {{item}}
+    {{es_home}}/bin/elasticsearch-users userdel {{item}}
   with_items: "{{users_to_remove | default([])}}"
   when: manage_file_users
   environment:
@@ -58,7 +55,7 @@
 - name: Add Users
   become: yes
   command: >
-    {{es_home}}/bin/{{es_xpack_users_command}} useradd {{item}} -p {{es_users.file[item].password}}
+    {{es_home}}/bin/elasticsearch-users useradd {{item}} -p {{es_users.file[item].password}}
   with_items: "{{ users_to_add | default([]) }}"
   when: manage_file_users
   no_log: True
@@ -71,7 +68,7 @@
 - name: Set User Passwords
   become: yes
   command: >
-    {{es_home}}/bin/{{es_xpack_users_command}} passwd {{ item }} -p {{es_users.file[item].password}}
+    {{es_home}}/bin/elasticsearch-users passwd {{ item }} -p {{es_users.file[item].password}}
   with_items: "{{ es_users.file.keys() | list }}"
   when: manage_file_users
   #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip.
@@ -89,16 +86,11 @@
 #Copy Roles files
 - name: Copy roles.yml File for Instance
   become: yes
-  template: src=security/roles.yml.j2 dest={{ es_conf_dir }}{{es_xpack_conf_subdir}}/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=security/roles.yml.j2 dest={{ es_conf_dir }}/roles.yml owner=root group={{ es_group }} mode=0660 force=yes
   when: es_roles is defined and es_roles.file is defined
 
 #Overwrite users_roles file
 - name: Copy User Roles
   become: yes
-  template: src=security/users_roles.j2 dest={{ es_conf_dir }}{{es_xpack_conf_subdir}}/users_roles mode=0644 force=yes
+  template: src=security/users_roles.j2 dest={{ es_conf_dir }}/users_roles owner=root group={{ es_group }} mode=0660 force=yes
   when: manage_file_users and users_roles | length > 0
-
-#Set permission on security directory. E.g. if 2 nodes are installed on the same machine, the second node will not get the users file created at install, causing the files being created at es_users call and then having the wrong Permissions.
-- name: Set Security Directory Permissions Recursive
-  become: yes
-  file: state=directory path={{ es_conf_dir }}{{es_xpack_conf_subdir}}/ owner={{ es_user }} group={{ es_group }} recurse=yes

tasks/xpack/security/elasticsearch-security.yml

@@ -3,12 +3,6 @@
 
 #TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6
 
-#Ensure x-pack conf directory is created if necessary
-- name: Ensure x-pack conf directory exists (file)
-  file: path={{ es_conf_dir }}{{ es_xpack_conf_subdir }} state=directory owner={{ es_user }} group={{ es_group }}
-  changed_when: False
-  when: (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined)
-
 #-----------------------------Create Bootstrap User-----------------------------------
 ### START BLOCK elasticsearch keystore ###
 - name: create the elasticsearch keystore
@@ -52,13 +46,5 @@
 #Copy Roles files
 - name: Copy role_mapping.yml File for Instance
   become: yes
-  template: src=security/role_mapping.yml.j2 dest={{ es_conf_dir }}{{es_xpack_conf_subdir}}/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src=security/role_mapping.yml.j2 dest={{ es_conf_dir }}/role_mapping.yml owner=root group={{ es_group }} mode=0660 force=yes
   when: es_role_mapping is defined
-
-#------------------------------------------------------------------------------------
-
-#Ensure security conf directory is created
-- name: Ensure security conf directory exists
-  become: yes
-  file: path={{ es_conf_dir }}/security state=directory owner={{ es_user }} group={{ es_group }}
-  changed_when: False