Merge branch 'master' into become-yes

tasks/elasticsearch-parameters.yml

@@ -23,6 +23,13 @@
 - fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations"
   when: es_enable_xpack and ("security" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined
 
+- set_fact: file_reserved_users={{ es_users.file.keys() | intersect (reserved_xpack_users) }}
+  when: es_users is defined and es_users.file is defined and (es_users.file.keys() | length > 0) and (es_users.file.keys() | intersect (reserved_xpack_users) | length > 0)
+
+- fail:
+     msg: "ERROR: INVALID CONFIG - YOU CANNOT CHANGE RESERVED USERS THROUGH THE FILE REALM. THE FOLLOWING CANNOT BE CHANGED: {{file_reserved_users}}. USE THE NATIVE REALM."
+  when: file_reserved_users | default([]) | length > 0
+
 - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}}
 - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}}
 - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}}

tasks/elasticsearch-template.yml

@@ -8,15 +8,6 @@
   with_fileglob:
     - "{{ es_templates_fileglob | default('') }}"
 
-
-- name: Ensure elasticsearch is started
-  service: name={{instance_init_script | basename}} state=started enabled=yes
-  when: es_start_service and load_templates.changed
-
-- name: Wait for elasticsearch to startup
-  wait_for: host={{es_api_host}} port={{es_api_port}} delay=10
-  when: es_start_service and load_templates.changed
-
 - name: Install templates without auth
   uri:
     url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}"

tasks/main.yml

@@ -39,12 +39,6 @@
 
 - meta: flush_handlers
 
-#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password.  The templates will subsequently not be removed if we don't wait for the node to restart.
-- include: elasticsearch-template.yml
-  when: es_templates
-  tags:
-      - templates
-
 - name: Make sure elasticsearch is started
   service: name={{instance_init_script | basename}} state=started enabled=yes
   when: es_start_service
@@ -53,10 +47,27 @@
   wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1
   when: es_restarted is defined and es_restarted.changed and es_start_service
 
+- set_fact: manage_native_realm=false
+
+- set_fact: manage_native_realm=true
+  when: es_start_service and (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
+
+# If playbook runs too fast, Native commands could fail as the Native Realm is not yet up
+- name: Wait 15 seconds for the Native Relm to come up
+  pause: seconds=15
+  when: manage_native_realm
+
 - name: activate-license
   include: ./xpack/security/elasticsearch-xpack-activation.yml
   when: es_start_service and es_enable_xpack and es_xpack_license is defined and es_xpack_license != ''
 
 #perform security actions here now elasticsearch is started
 - include: ./xpack/security/elasticsearch-security-native.yml
-  when: es_start_service and (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
+  when: manage_native_realm
+
+#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password.  The templates will subsequently not be removed if we don't wait for the node to restart.
+#We also do after the native realm to ensure any changes are applied here first and its denf up.
+- include: elasticsearch-template.yml
+  when: es_templates
+  tags:
+      - templates

tasks/xpack/security/elasticsearch-security-file.yml

@@ -1,5 +1,5 @@
 ---
-- set_fact: manage_file_users=es_users is defined and es_users.file is defined
+- set_fact: manage_file_users=es_users is defined and es_users.file is defined and es_users.file.keys() | length > 0
 
 #List current users
 - name: List Users
@@ -18,12 +18,11 @@
   command: >
     {{es_home}}/bin/x-pack/users userdel {{item}}
   with_items: "{{users_to_remove | default([])}}"
-  when: manage_file_users and (users_to_remove | length > 0)
+  when: manage_file_users
   environment:
     CONF_DIR: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
-
 - set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }}
   when: manage_file_users
 
@@ -32,8 +31,8 @@
   become: yes
   command: >
     {{es_home}}/bin/x-pack/users useradd {{item}} -p {{es_users.file[item].password}}
-  with_items: "{{users_to_add | default([])}}"
-  when: manage_file_users and users_to_add | length > 0
+  with_items: "{{ users_to_add | default([]) }}"
+  when: manage_file_users
   no_log: True
   environment:
     CONF_DIR: "{{ conf_dir }}"
@@ -43,9 +42,9 @@
 - name: Set User Passwords
   become: yes
   command: >
-    {{es_home}}/bin/x-pack/users passwd {{item.key}} -p {{item.value.password}}
-  with_dict: "{{(es_users | default({'file':{}})).file}}"
-  when: manage_file_users and es_users.file.keys() | length > 0
+    {{es_home}}/bin/x-pack/users passwd {{ item }} -p {{es_users.file[item].password}}
+  with_items: "{{ es_users.file.keys() | default([]) }}"
+  when: manage_file_users
   #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip.
   changed_when: False
   no_log: True
@@ -53,7 +52,7 @@
     CONF_DIR: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
-- set_fact: users_roles={{es_users.file | extract_role_users}}
+- set_fact: users_roles={{es_users.file | extract_role_users () }}
   when: manage_file_users
 
 #Copy Roles files

tasks/xpack/security/elasticsearch-security-native.yml

@@ -1,18 +1,15 @@
 ---
+- set_fact: change_api_password=false
 
 - set_fact: manage_native_users=false
 
 - set_fact: manage_native_users=true
-  when: es_users is defined and es_users.native is defined
+  when: es_users is defined and es_users.native is defined and es_users.native.keys() | length > 0
 
 - set_fact: manage_native_roles=false
 
 - set_fact: manage_native_roles=true
-  when: es_roles is defined and es_roles.native is defined
-
-# If playbook runs too fast, Native commands could fail as the Native Realm is not yet up
-- name: Wait 15 seconds for the Native Realm to come up
-  pause: seconds=15
+  when: es_roles is defined and es_roles.native is defined and es_roles.native.keys() | length > 0
 
 #If the node has just has security installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load
 
@@ -28,15 +25,40 @@
   register: user_list_response
   when: manage_native_users
 
+- set_fact: reserved_users={{ user_list_response.json | filter_reserved }}
+  when: manage_native_users
+
 #Current users not inc. those reserved
-- set_fact: current_users={{ user_list_response.json | filter_reserved }}
+- set_fact: current_users={{ user_list_response.json.keys() | difference (reserved_users) }}
   when: manage_native_users
 
-#Identify non declared users
-- set_fact: users_to_remove={{ current_users | difference ( es_users.native.keys() ) }}
+#We are changing the es_api_basic_auth_username password, so we need to do it first and update the param
+- set_fact: native_users={{ es_users.native }}
   when: manage_native_users
 
-#Delete all non required users
+- set_fact: change_api_password=true
+  when: manage_native_users and es_api_basic_auth_username in native_users and native_users[es_api_basic_auth_username].password is defined
+
+- name: Update API User Password
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{es_api_basic_auth_username}}/_password
+    method: POST
+    body_format: json
+    body: "{ \"password\":\"{{native_users[es_api_basic_auth_username].password}}\" }"
+    status_code: 200
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+  when: change_api_password
+
+- set_fact: es_api_basic_auth_password={{native_users[es_api_basic_auth_username].password}}
+  when: change_api_password
+
+#Identify users that are present in ES but not declared and thus should be removed
+- set_fact: users_to_remove={{ current_users | difference ( native_users.keys() ) }}
+  when: manage_native_users
+
+#Delete all non required users NOT inc. reserved
 - name: Delete Native Users
   uri:
     url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}
@@ -45,26 +67,50 @@
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_users and users_to_remove | length > 0
-  with_items: "{{users_to_remove | default([]) }}"
+  when: manage_native_users
+  with_items: "{{ users_to_remove | default([]) }}"
 
-- set_fact: native_users={{ es_users.native }}
-  when: manage_native_users and es_users.native.keys() > 0
+- set_fact: users_to_ignore={{ native_users.keys() | intersect (reserved_users) }}
+  when: manage_native_users
 
-#Overwrite all other users
-- name: Update Native Users
+- debug:
+    msg: "WARNING: YOU CAN ONLY CHANGE THE PASSWORD FOR RESERVED USERS IN THE NATIVE REALM. ANY ROLE CHANGES WILL BE IGNORED: {{users_to_ignore}}"
+  when: manage_native_users and users_to_ignore | length > 0
+
+#Update password on all reserved users
+- name: Update Reserved User Passwords
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item.key}}
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}/_password
     method: POST
     body_format: json
-    body: "{{item.value | to_json}}"
+    body: "{ \"password\":\"{{native_users[item].password}}\" }"
     status_code: 200
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_users and native_users.keys() > 0
+  when: native_users[item].password is defined
   no_log: True
-  with_dict: "{{native_users | default({}) }}"
+  with_items: "{{ users_to_ignore | default([]) }}"
+
+- set_fact: users_to_modify={{ native_users.keys() | difference (reserved_users) }}
+  when: manage_native_users
+
+#Overwrite all other users NOT inc. those reserved
+- name: Update Non-Reserved Native User Details
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}
+    method: POST
+    body_format: json
+    body: "{{ native_users[item] | to_json }}"
+    status_code: 200
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+  when: manage_native_users
+  no_log: True
+  with_items: "{{ users_to_modify | default([]) }}"
+
+## ROLE CHANGES
 
 #List current roles not. inc those reserved
 - name: List Native Roles
@@ -79,16 +125,23 @@
   register: role_list_response
   when: manage_native_roles
 
-- set_fact: current_roles={{ role_list_response.json | filter_reserved }}
+- set_fact: reserved_roles={{ role_list_response.json | filter_reserved }}
   when: manage_native_roles
 
-- debug: msg="{{current_roles}}"
+- set_fact: current_roles={{ role_list_response.json.keys() | difference (reserved_roles) }}
   when: manage_native_roles
 
+- set_fact: roles_to_ignore={{ es_roles.native.keys() | intersect (reserved_roles) | default([]) }}
+  when: manage_native_roles
+
+- debug:
+    msg: "WARNING: YOU CANNOT CHANGE RESERVED ROLES. THE FOLLOWING WILL BE IGNORED: {{roles_to_ignore}}"
+  when: manage_native_roles and roles_to_ignore | length > 0
+
 - set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys()  ) }}
   when: manage_native_roles
 
-#Delete all non required roles
+#Delete all non required roles NOT inc. reserved
 - name: Delete Native Roles
   uri:
     url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}
@@ -97,23 +150,22 @@
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_roles and roles_to_remove | length > 0
+  when: manage_native_roles
   with_items: "{{roles_to_remove | default([]) }}"
 
+- set_fact: roles_to_modify={{ es_roles.native.keys() | difference (reserved_roles) }}
+  when: manage_native_roles
 
-- set_fact: native_roles={{ es_roles.native }}
-  when: manage_native_roles and es_roles.native.keys() > 0
-
-#Update other roles
+#Update other roles - NOT inc. reserved roles
 - name: Update Native Roles
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item.key}}
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}
     method: POST
     body_format: json
-    body: "{{item.value | to_json}}"
+    body: "{{ es_roles.native[item] | to_json}}"
     status_code: 200
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_roles and native_roles.keys() > 0
-  with_dict: "{{ native_roles | default({})}}"
+  when: manage_native_roles
+  with_items: "{{ roles_to_modify | default([]) }}"