Merge branch 'master' into become-yes

.kitchen.yml

@@ -116,4 +116,4 @@ suites:
     attributes:
     provisioner:
       playbook: test/integration/issue-test.yml
-      idempotency_test: true
+      idempotency_test: false

filter_plugins/custom.py

@@ -21,10 +21,10 @@ def append_to_list(values=[], suffix=''):
 def array_to_str(values=[],separator=','):
     return separator.join(values)
 
-def extract_role_users(users={}):
+def extract_role_users(users={},exclude_users=[]):
     role_users=[]
     for user,details in users.iteritems():
-        if "roles" in details:
+        if user not in exclude_users and "roles" in details:
             for role in details["roles"]:
                 role_users.append(role+":"+user)
     return role_users
@@ -32,19 +32,26 @@ def extract_role_users(users={}):
 def filename(filename=''):
     return os.path.splitext(os.path.basename(filename))[0]
 
-def filter_reserved(user_roles={}):
+def remove_reserved(user_roles={}):
     not_reserved = []
     for user_role,details in user_roles.items():
         if not "metadata" in details or not "_reserved" in details["metadata"] or not details["metadata"]["_reserved"]:
             not_reserved.append(user_role)
     return not_reserved
 
+def filter_reserved(users_role={}):
+    reserved = []
+    for user_role,details in users_role.items():
+        if "metadata" in details and "_reserved" in details["metadata"] and details["metadata"]["_reserved"]:
+            reserved.append(user_role)
+    return reserved
 
 class FilterModule(object):
     def filters(self):
         return {'modify_list': modify_list,
         'append_to_list':append_to_list,
+        'filter_reserved':filter_reserved,
         'array_to_str':array_to_str,
         'extract_role_users':extract_role_users,
-        'filter_reserved':filter_reserved,
+        'remove_reserved':remove_reserved,
         'filename':filename}

tasks/elasticsearch-parameters.yml

@@ -23,6 +23,13 @@
 - fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations"
   when: es_enable_xpack and ("security" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined
 
+- set_fact: file_reserved_users={{ es_users.file.keys() | intersect (reserved_xpack_users) }}
+  when: es_users is defined and es_users.file is defined and (es_users.file.keys() | length > 0) and (es_users.file.keys() | intersect (reserved_xpack_users) | length > 0)
+
+- fail:
+     msg: "ERROR: INVALID CONFIG - YOU CANNOT CHANGE RESERVED USERS THROUGH THE FILE REALM. THE FOLLOWING CANNOT BE CHANGED: {{file_reserved_users}}. USE THE NATIVE REALM."
+  when: file_reserved_users | default([]) | length > 0
+
 - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}}
 - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}}
 - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}}

tasks/elasticsearch-template.yml

@@ -8,15 +8,6 @@
   with_fileglob:
     - "{{ es_templates_fileglob | default('') }}"
 
-
-- name: Ensure elasticsearch is started
-  service: name={{instance_init_script | basename}} state=started enabled=yes
-  when: es_start_service and load_templates.changed
-
-- name: Wait for elasticsearch to startup
-  wait_for: host={{es_api_host}} port={{es_api_port}} delay=10
-  when: es_start_service and load_templates.changed
-
 - name: Install templates without auth
   uri:
     url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}"

tasks/main.yml

@@ -39,12 +39,6 @@
 
 - meta: flush_handlers
 
-#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password.  The templates will subsequently not be removed if we don't wait for the node to restart.
-- include: elasticsearch-template.yml
-  when: es_templates
-  tags:
-      - templates
-
 - name: Make sure elasticsearch is started
   service: name={{instance_init_script | basename}} state=started enabled=yes
   when: es_start_service
@@ -53,10 +47,27 @@
   wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1
   when: es_restarted is defined and es_restarted.changed and es_start_service
 
+- set_fact: manage_native_realm=false
+
+- set_fact: manage_native_realm=true
+  when: es_start_service and (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
+
+# If playbook runs too fast, Native commands could fail as the Native Realm is not yet up
+- name: Wait 15 seconds for the Native Relm to come up
+  pause: seconds=15
+  when: manage_native_realm
+
 - name: activate-license
   include: ./xpack/security/elasticsearch-xpack-activation.yml
   when: es_start_service and es_enable_xpack and es_xpack_license is defined and es_xpack_license != ''
 
 #perform security actions here now elasticsearch is started
 - include: ./xpack/security/elasticsearch-security-native.yml
-  when: es_start_service and (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
+  when: manage_native_realm
+
+#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password.  The templates will subsequently not be removed if we don't wait for the node to restart.
+#We also do after the native realm to ensure any changes are applied here first and its denf up.
+- include: elasticsearch-template.yml
+  when: es_templates
+  tags:
+      - templates

tasks/xpack/security/elasticsearch-security-file.yml

@@ -1,5 +1,5 @@
 ---
-- set_fact: manage_file_users=es_users is defined and es_users.file is defined
+- set_fact: manage_file_users=es_users is defined and es_users.file is defined and es_users.file.keys() | length > 0
 
 #List current users
 - name: List Users
@@ -18,12 +18,11 @@
   command: >
     {{es_home}}/bin/x-pack/users userdel {{item}}
   with_items: "{{users_to_remove | default([])}}"
-  when: manage_file_users and (users_to_remove | length > 0)
+  when: manage_file_users
   environment:
     CONF_DIR: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
-
 - set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }}
   when: manage_file_users
 
@@ -32,8 +31,8 @@
   become: yes
   command: >
     {{es_home}}/bin/x-pack/users useradd {{item}} -p {{es_users.file[item].password}}
-  with_items: "{{users_to_add | default([])}}"
+  with_items: "{{ users_to_add | default([]) }}"
-  when: manage_file_users and users_to_add | length > 0
+  when: manage_file_users
   no_log: True
   environment:
     CONF_DIR: "{{ conf_dir }}"
@@ -43,9 +42,9 @@
 - name: Set User Passwords
   become: yes
   command: >
-    {{es_home}}/bin/x-pack/users passwd {{item.key}} -p {{item.value.password}}
+    {{es_home}}/bin/x-pack/users passwd {{ item }} -p {{es_users.file[item].password}}
-  with_dict: "{{(es_users | default({'file':{}})).file}}"
+  with_items: "{{ es_users.file.keys() | default([]) }}"
-  when: manage_file_users and es_users.file.keys() | length > 0
+  when: manage_file_users
   #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip.
   changed_when: False
   no_log: True
@@ -53,7 +52,7 @@
     CONF_DIR: "{{ conf_dir }}"
     ES_HOME: "{{es_home}}"
 
-- set_fact: users_roles={{es_users.file | extract_role_users}}
+- set_fact: users_roles={{es_users.file | extract_role_users () }}
   when: manage_file_users
 
 #Copy Roles files

tasks/xpack/security/elasticsearch-security-native.yml

@@ -1,18 +1,15 @@
 ---
+- set_fact: change_api_password=false
 
 - set_fact: manage_native_users=false
 
 - set_fact: manage_native_users=true
-  when: es_users is defined and es_users.native is defined
+  when: es_users is defined and es_users.native is defined and es_users.native.keys() | length > 0
 
 - set_fact: manage_native_roles=false
 
 - set_fact: manage_native_roles=true
-  when: es_roles is defined and es_roles.native is defined
+  when: es_roles is defined and es_roles.native is defined and es_roles.native.keys() | length > 0
-
-# If playbook runs too fast, Native commands could fail as the Native Realm is not yet up
-- name: Wait 15 seconds for the Native Realm to come up
-  pause: seconds=15
 
 #If the node has just has security installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load
 
@@ -28,15 +25,40 @@
   register: user_list_response
   when: manage_native_users
 
+- set_fact: reserved_users={{ user_list_response.json | filter_reserved }}
+  when: manage_native_users
+
 #Current users not inc. those reserved
-- set_fact: current_users={{ user_list_response.json | filter_reserved }}
+- set_fact: current_users={{ user_list_response.json.keys() | difference (reserved_users) }}
   when: manage_native_users
 
-#Identify non declared users
+#We are changing the es_api_basic_auth_username password, so we need to do it first and update the param
-- set_fact: users_to_remove={{ current_users | difference ( es_users.native.keys() ) }}
+- set_fact: native_users={{ es_users.native }}
   when: manage_native_users
 
-#Delete all non required users
+- set_fact: change_api_password=true
+  when: manage_native_users and es_api_basic_auth_username in native_users and native_users[es_api_basic_auth_username].password is defined
+
+- name: Update API User Password
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{es_api_basic_auth_username}}/_password
+    method: POST
+    body_format: json
+    body: "{ \"password\":\"{{native_users[es_api_basic_auth_username].password}}\" }"
+    status_code: 200
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+  when: change_api_password
+
+- set_fact: es_api_basic_auth_password={{native_users[es_api_basic_auth_username].password}}
+  when: change_api_password
+
+#Identify users that are present in ES but not declared and thus should be removed
+- set_fact: users_to_remove={{ current_users | difference ( native_users.keys() ) }}
+  when: manage_native_users
+
+#Delete all non required users NOT inc. reserved
 - name: Delete Native Users
   uri:
     url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}
@@ -45,26 +67,50 @@
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_users and users_to_remove | length > 0
+  when: manage_native_users
-  with_items: "{{users_to_remove | default([]) }}"
+  with_items: "{{ users_to_remove | default([]) }}"
 
-- set_fact: native_users={{ es_users.native }}
+- set_fact: users_to_ignore={{ native_users.keys() | intersect (reserved_users) }}
-  when: manage_native_users and es_users.native.keys() > 0
+  when: manage_native_users
 
-#Overwrite all other users
+- debug:
-- name: Update Native Users
+    msg: "WARNING: YOU CAN ONLY CHANGE THE PASSWORD FOR RESERVED USERS IN THE NATIVE REALM. ANY ROLE CHANGES WILL BE IGNORED: {{users_to_ignore}}"
+  when: manage_native_users and users_to_ignore | length > 0
+
+#Update password on all reserved users
+- name: Update Reserved User Passwords
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item.key}}
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}/_password
     method: POST
     body_format: json
-    body: "{{item.value | to_json}}"
+    body: "{ \"password\":\"{{native_users[item].password}}\" }"
     status_code: 200
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_users and native_users.keys() > 0
+  when: native_users[item].password is defined
   no_log: True
-  with_dict: "{{native_users | default({}) }}"
+  with_items: "{{ users_to_ignore | default([]) }}"
+
+- set_fact: users_to_modify={{ native_users.keys() | difference (reserved_users) }}
+  when: manage_native_users
+
+#Overwrite all other users NOT inc. those reserved
+- name: Update Non-Reserved Native User Details
+  uri:
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}
+    method: POST
+    body_format: json
+    body: "{{ native_users[item] | to_json }}"
+    status_code: 200
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+  when: manage_native_users
+  no_log: True
+  with_items: "{{ users_to_modify | default([]) }}"
+
+## ROLE CHANGES
 
 #List current roles not. inc those reserved
 - name: List Native Roles
@@ -79,16 +125,23 @@
   register: role_list_response
   when: manage_native_roles
 
-- set_fact: current_roles={{ role_list_response.json | filter_reserved }}
+- set_fact: reserved_roles={{ role_list_response.json | filter_reserved }}
   when: manage_native_roles
 
-- debug: msg="{{current_roles}}"
+- set_fact: current_roles={{ role_list_response.json.keys() | difference (reserved_roles) }}
   when: manage_native_roles
 
+- set_fact: roles_to_ignore={{ es_roles.native.keys() | intersect (reserved_roles) | default([]) }}
+  when: manage_native_roles
+
+- debug:
+    msg: "WARNING: YOU CANNOT CHANGE RESERVED ROLES. THE FOLLOWING WILL BE IGNORED: {{roles_to_ignore}}"
+  when: manage_native_roles and roles_to_ignore | length > 0
+
 - set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys()  ) }}
   when: manage_native_roles
 
-#Delete all non required roles
+#Delete all non required roles NOT inc. reserved
 - name: Delete Native Roles
   uri:
     url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}
@@ -97,23 +150,22 @@
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_roles and roles_to_remove | length > 0
+  when: manage_native_roles
   with_items: "{{roles_to_remove | default([]) }}"
 
+- set_fact: roles_to_modify={{ es_roles.native.keys() | difference (reserved_roles) }}
+  when: manage_native_roles
 
-- set_fact: native_roles={{ es_roles.native }}
+#Update other roles - NOT inc. reserved roles
-  when: manage_native_roles and es_roles.native.keys() > 0
-
-#Update other roles
 - name: Update Native Roles
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item.key}}
+    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}
     method: POST
     body_format: json
-    body: "{{item.value | to_json}}"
+    body: "{{ es_roles.native[item] | to_json}}"
     status_code: 200
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     force_basic_auth: yes
-  when: manage_native_roles and native_roles.keys() > 0
+  when: manage_native_roles
-  with_dict: "{{ native_roles | default({})}}"
+  with_items: "{{ roles_to_modify | default([]) }}"

test/integration/helpers/serverspec/xpack_spec.rb

@@ -199,5 +199,45 @@ shared_examples 'xpack::init' do |es_version,plugins|
     #Test contents as expected
     its(:md5sum) { should eq '6ff0e6c4380a6ac0f6e04d871c0ca5e8' }
   end
+
+  #check accounts are correct i.e. we can auth and they have the correct roles
+
+  describe 'kibana4_server access check' do
+    it 'should be reported as version '+es_version do
+      command = command('curl -s localhost:9200/ -u kibana4_server:changeMe | grep number')
+      expect(command.stdout).to match(es_version)
+      expect(command.exit_status).to eq(0)
+    end
+  end
+
+  describe command('curl -s localhost:9200/_xpack/security/user/kibana4_server -u elastic:elasticChanged | md5sum | grep f0548742161d9e50c7c7fbe2e061a1fa') do
+      its(:exit_status) { should eq 0 }
+  end
+
+
+  describe 'logstash_system access check' do
+    it 'should be reported as version '+es_version do
+      command = command('curl -s localhost:9200/ -u logstash_system:aNewLogstashPassword | grep number')
+      expect(command.stdout).to match(es_version)
+      expect(command.exit_status).to eq(0)
+    end
+  end
+
+  describe command('curl -s localhost:9200/_xpack/security/user/logstash_system -u elastic:elasticChanged | md5sum | grep 98d361ddfa5156abd33542a493b4fd85') do
+      its(:exit_status) { should eq 0 }
+  end
+
+  describe 'kibana access check' do
+    it 'should be reported as version '+es_version do
+      command = command('curl -s localhost:9200/ -u kibana:changeme | grep number')
+      expect(command.stdout).to match(es_version)
+      expect(command.exit_status).to eq(0)
+    end
+  end
+
+  describe command('curl -s localhost:9200/_xpack/security/user/kibana -u elastic:elasticChanged | md5sum | grep 34190c64eb3c7cfb002fa789df5fad20') do
+      its(:exit_status) { should eq 0 }
+  end
+
 end
 

test/integration/issue-test.yml

@@ -2,10 +2,20 @@
 #Modify the playbook  below and test with kitchen i.e. `kitchen test issue-test`
 #To add custom tests modify the serverspec file ./helpers/serverspec/issue_test_spec.rb
 #Idempot test is enabled for this test
+
 - name: Simple Example
   hosts: localhost
   roles:
-    - { role: elasticsearch, es_config:  {  "xpack.security.authc.realms.file1.type": "file", "xpack.security.authc.realms.file1.order": 1, "xpack.security.authc.realms.native1.type": "native", "xpack.security.authc.realms.native1.order": 0 }, es_instance_name: "security_node" }
+    - {
+        role: elasticsearch,
+        es_config:
+        {
+          "xpack.security.authc.realms.file1.type": "file",
+          "xpack.security.authc.realms.file1.order": 1,
+          "xpack.security.authc.realms.native1.type": "native",
+          "xpack.security.authc.realms.native1.order": 0
+        },
+    es_instance_name: "security_node" }
   vars:
     es_heap_size: "1g"
     es_enable_xpack: true
@@ -17,8 +27,23 @@
     es_api_basic_auth_username: elastic
     es_api_basic_auth_password: changeme
     es_users:
-      native:
+      file:
-        testUser:
+        test_user:
           password: changeme
           roles:
-            - kibana_user
+            - kibana_system
+      native:
+        kibana:
+          password: changeme
+          roles:
+            - kibana_system
+        elastic:
+          password: aNewPassWord
+    es_roles:
+      native:
+        logstash:
+          cluster:
+            - manage_index_templates
+        logstash_system:
+          cluster:
+            - manage_index_templates

test/integration/xpack.yml

@@ -31,6 +31,14 @@
           password: changeMe
           roles:
             - kibana4_server
+        logstash_system:
+          #this should be successfully modified
+          password: aNewLogstashPassword
+          #this will be ignored
+          roles:
+            - kibana4_server
+        elastic:
+          password: elasticChanged
       file:
         es_admin:
           password: changeMe
@@ -79,6 +87,16 @@
                 - write
                 - delete
                 - create_index
+        #this will be ignored - its reserved
+        logstash_system:
+          cluster:
+            - manage_index_templates
+          indices:
+            - names: 'logstash-*'
+              privileges:
+                - write
+                - delete
+                - create_index
 
 #modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed.
 - name: Elasticsearch Xpack modify
@@ -99,7 +117,7 @@
       - security
       - alerting
     es_api_basic_auth_username: elastic
-    es_api_basic_auth_password: changeme
+    es_api_basic_auth_password: elasticChanged
     es_role_mapping:
       power_user:
         - "cn=admins,dc=example,dc=com"
@@ -112,6 +130,10 @@
           password: changeMe
           roles:
             - kibana4_server
+        logstash_system:
+          #this will be ignored
+          roles:
+            - kibana4_server
       file:
         es_admin:
           password: changeMeAgain

vars/main.yml

@@ -5,3 +5,4 @@ sysd_script: "/usr/lib/systemd/system/elasticsearch.service"
 init_script: "/etc/init.d/elasticsearch"
 #add supported features here
 supported_xpack_features: ["alerting","monitoring","graph","security"]
+reserved_xpack_users: ["elastic","kibana","logstash_system"]