diff --git a/.kitchen.yml b/.kitchen.yml index 1c41c19..4d05763 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -116,4 +116,4 @@ suites: attributes: provisioner: playbook: test/integration/issue-test.yml - idempotency_test: true \ No newline at end of file + idempotency_test: false \ No newline at end of file diff --git a/filter_plugins/custom.py b/filter_plugins/custom.py index ad22133..22177cd 100644 --- a/filter_plugins/custom.py +++ b/filter_plugins/custom.py @@ -21,10 +21,10 @@ def append_to_list(values=[], suffix=''): def array_to_str(values=[],separator=','): return separator.join(values) -def extract_role_users(users={}): +def extract_role_users(users={},exclude_users=[]): role_users=[] for user,details in users.iteritems(): - if "roles" in details: + if user not in exclude_users and "roles" in details: for role in details["roles"]: role_users.append(role+":"+user) return role_users @@ -32,19 +32,26 @@ def extract_role_users(users={}): def filename(filename=''): return os.path.splitext(os.path.basename(filename))[0] -def filter_reserved(user_roles={}): +def remove_reserved(user_roles={}): not_reserved = [] for user_role,details in user_roles.items(): if not "metadata" in details or not "_reserved" in details["metadata"] or not details["metadata"]["_reserved"]: not_reserved.append(user_role) return not_reserved +def filter_reserved(users_role={}): + reserved = [] + for user_role,details in users_role.items(): + if "metadata" in details and "_reserved" in details["metadata"] and details["metadata"]["_reserved"]: + reserved.append(user_role) + return reserved class FilterModule(object): def filters(self): return {'modify_list': modify_list, 'append_to_list':append_to_list, + 'filter_reserved':filter_reserved, 'array_to_str':array_to_str, 'extract_role_users':extract_role_users, - 'filter_reserved':filter_reserved, + 'remove_reserved':remove_reserved, 'filename':filename} \ No newline at end of file diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 0a6dce0..1ac92ff 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -23,6 +23,13 @@ - fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" when: es_enable_xpack and ("security" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined +- set_fact: file_reserved_users={{ es_users.file.keys() | intersect (reserved_xpack_users) }} + when: es_users is defined and es_users.file is defined and (es_users.file.keys() | length > 0) and (es_users.file.keys() | intersect (reserved_xpack_users) | length > 0) + +- fail: + msg: "ERROR: INVALID CONFIG - YOU CANNOT CHANGE RESERVED USERS THROUGH THE FILE REALM. THE FOLLOWING CANNOT BE CHANGED: {{file_reserved_users}}. USE THE NATIVE REALM." + when: file_reserved_users | default([]) | length > 0 + - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} diff --git a/tasks/elasticsearch-template.yml b/tasks/elasticsearch-template.yml index e524043..08a97d1 100644 --- a/tasks/elasticsearch-template.yml +++ b/tasks/elasticsearch-template.yml @@ -8,15 +8,6 @@ with_fileglob: - "{{ es_templates_fileglob | default('') }}" - -- name: Ensure elasticsearch is started - service: name={{instance_init_script | basename}} state=started enabled=yes - when: es_start_service and load_templates.changed - -- name: Wait for elasticsearch to startup - wait_for: host={{es_api_host}} port={{es_api_port}} delay=10 - when: es_start_service and load_templates.changed - - name: Install templates without auth uri: url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}" diff --git a/tasks/main.yml b/tasks/main.yml index 5b3953d..0f3de03 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -39,12 +39,6 @@ - meta: flush_handlers -#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed if we don't wait for the node to restart. -- include: elasticsearch-template.yml - when: es_templates - tags: - - templates - - name: Make sure elasticsearch is started service: name={{instance_init_script | basename}} state=started enabled=yes when: es_start_service @@ -53,10 +47,27 @@ wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1 when: es_restarted is defined and es_restarted.changed and es_start_service +- set_fact: manage_native_realm=false + +- set_fact: manage_native_realm=true + when: es_start_service and (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)) + +# If playbook runs too fast, Native commands could fail as the Native Realm is not yet up +- name: Wait 15 seconds for the Native Relm to come up + pause: seconds=15 + when: manage_native_realm + - name: activate-license include: ./xpack/security/elasticsearch-xpack-activation.yml when: es_start_service and es_enable_xpack and es_xpack_license is defined and es_xpack_license != '' #perform security actions here now elasticsearch is started - include: ./xpack/security/elasticsearch-security-native.yml - when: es_start_service and (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)) + when: manage_native_realm + +#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed if we don't wait for the node to restart. +#We also do after the native realm to ensure any changes are applied here first and its denf up. +- include: elasticsearch-template.yml + when: es_templates + tags: + - templates \ No newline at end of file diff --git a/tasks/xpack/security/elasticsearch-security-file.yml b/tasks/xpack/security/elasticsearch-security-file.yml index cfd8240..885cd03 100644 --- a/tasks/xpack/security/elasticsearch-security-file.yml +++ b/tasks/xpack/security/elasticsearch-security-file.yml @@ -1,5 +1,5 @@ --- -- set_fact: manage_file_users=es_users is defined and es_users.file is defined +- set_fact: manage_file_users=es_users is defined and es_users.file is defined and es_users.file.keys() | length > 0 #List current users - name: List Users @@ -18,12 +18,11 @@ command: > {{es_home}}/bin/x-pack/users userdel {{item}} with_items: "{{users_to_remove | default([])}}" - when: manage_file_users and (users_to_remove | length > 0) + when: manage_file_users environment: CONF_DIR: "{{ conf_dir }}" ES_HOME: "{{es_home}}" - - set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }} when: manage_file_users @@ -32,8 +31,8 @@ become: yes command: > {{es_home}}/bin/x-pack/users useradd {{item}} -p {{es_users.file[item].password}} - with_items: "{{users_to_add | default([])}}" - when: manage_file_users and users_to_add | length > 0 + with_items: "{{ users_to_add | default([]) }}" + when: manage_file_users no_log: True environment: CONF_DIR: "{{ conf_dir }}" @@ -43,9 +42,9 @@ - name: Set User Passwords become: yes command: > - {{es_home}}/bin/x-pack/users passwd {{item.key}} -p {{item.value.password}} - with_dict: "{{(es_users | default({'file':{}})).file}}" - when: manage_file_users and es_users.file.keys() | length > 0 + {{es_home}}/bin/x-pack/users passwd {{ item }} -p {{es_users.file[item].password}} + with_items: "{{ es_users.file.keys() | default([]) }}" + when: manage_file_users #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip. changed_when: False no_log: True @@ -53,7 +52,7 @@ CONF_DIR: "{{ conf_dir }}" ES_HOME: "{{es_home}}" -- set_fact: users_roles={{es_users.file | extract_role_users}} +- set_fact: users_roles={{es_users.file | extract_role_users () }} when: manage_file_users #Copy Roles files diff --git a/tasks/xpack/security/elasticsearch-security-native.yml b/tasks/xpack/security/elasticsearch-security-native.yml index f98fae4..c748d38 100644 --- a/tasks/xpack/security/elasticsearch-security-native.yml +++ b/tasks/xpack/security/elasticsearch-security-native.yml @@ -1,18 +1,15 @@ --- +- set_fact: change_api_password=false - set_fact: manage_native_users=false - set_fact: manage_native_users=true - when: es_users is defined and es_users.native is defined + when: es_users is defined and es_users.native is defined and es_users.native.keys() | length > 0 - set_fact: manage_native_roles=false - set_fact: manage_native_roles=true - when: es_roles is defined and es_roles.native is defined - -# If playbook runs too fast, Native commands could fail as the Native Realm is not yet up -- name: Wait 15 seconds for the Native Realm to come up - pause: seconds=15 + when: es_roles is defined and es_roles.native is defined and es_roles.native.keys() | length > 0 #If the node has just has security installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load @@ -28,15 +25,40 @@ register: user_list_response when: manage_native_users +- set_fact: reserved_users={{ user_list_response.json | filter_reserved }} + when: manage_native_users + #Current users not inc. those reserved -- set_fact: current_users={{ user_list_response.json | filter_reserved }} +- set_fact: current_users={{ user_list_response.json.keys() | difference (reserved_users) }} when: manage_native_users -#Identify non declared users -- set_fact: users_to_remove={{ current_users | difference ( es_users.native.keys() ) }} +#We are changing the es_api_basic_auth_username password, so we need to do it first and update the param +- set_fact: native_users={{ es_users.native }} when: manage_native_users -#Delete all non required users +- set_fact: change_api_password=true + when: manage_native_users and es_api_basic_auth_username in native_users and native_users[es_api_basic_auth_username].password is defined + +- name: Update API User Password + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{es_api_basic_auth_username}}/_password + method: POST + body_format: json + body: "{ \"password\":\"{{native_users[es_api_basic_auth_username].password}}\" }" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: change_api_password + +- set_fact: es_api_basic_auth_password={{native_users[es_api_basic_auth_username].password}} + when: change_api_password + +#Identify users that are present in ES but not declared and thus should be removed +- set_fact: users_to_remove={{ current_users | difference ( native_users.keys() ) }} + when: manage_native_users + +#Delete all non required users NOT inc. reserved - name: Delete Native Users uri: url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}} @@ -45,26 +67,50 @@ user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes - when: manage_native_users and users_to_remove | length > 0 - with_items: "{{users_to_remove | default([]) }}" + when: manage_native_users + with_items: "{{ users_to_remove | default([]) }}" -- set_fact: native_users={{ es_users.native }} - when: manage_native_users and es_users.native.keys() > 0 +- set_fact: users_to_ignore={{ native_users.keys() | intersect (reserved_users) }} + when: manage_native_users -#Overwrite all other users -- name: Update Native Users +- debug: + msg: "WARNING: YOU CAN ONLY CHANGE THE PASSWORD FOR RESERVED USERS IN THE NATIVE REALM. ANY ROLE CHANGES WILL BE IGNORED: {{users_to_ignore}}" + when: manage_native_users and users_to_ignore | length > 0 + +#Update password on all reserved users +- name: Update Reserved User Passwords uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item.key}} + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}/_password method: POST body_format: json - body: "{{item.value | to_json}}" + body: "{ \"password\":\"{{native_users[item].password}}\" }" status_code: 200 user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes - when: manage_native_users and native_users.keys() > 0 + when: native_users[item].password is defined no_log: True - with_dict: "{{native_users | default({}) }}" + with_items: "{{ users_to_ignore | default([]) }}" + +- set_fact: users_to_modify={{ native_users.keys() | difference (reserved_users) }} + when: manage_native_users + +#Overwrite all other users NOT inc. those reserved +- name: Update Non-Reserved Native User Details + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}} + method: POST + body_format: json + body: "{{ native_users[item] | to_json }}" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_users + no_log: True + with_items: "{{ users_to_modify | default([]) }}" + +## ROLE CHANGES #List current roles not. inc those reserved - name: List Native Roles @@ -79,16 +125,23 @@ register: role_list_response when: manage_native_roles -- set_fact: current_roles={{ role_list_response.json | filter_reserved }} +- set_fact: reserved_roles={{ role_list_response.json | filter_reserved }} when: manage_native_roles -- debug: msg="{{current_roles}}" +- set_fact: current_roles={{ role_list_response.json.keys() | difference (reserved_roles) }} when: manage_native_roles +- set_fact: roles_to_ignore={{ es_roles.native.keys() | intersect (reserved_roles) | default([]) }} + when: manage_native_roles + +- debug: + msg: "WARNING: YOU CANNOT CHANGE RESERVED ROLES. THE FOLLOWING WILL BE IGNORED: {{roles_to_ignore}}" + when: manage_native_roles and roles_to_ignore | length > 0 + - set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys() ) }} when: manage_native_roles -#Delete all non required roles +#Delete all non required roles NOT inc. reserved - name: Delete Native Roles uri: url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}} @@ -97,23 +150,22 @@ user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes - when: manage_native_roles and roles_to_remove | length > 0 + when: manage_native_roles with_items: "{{roles_to_remove | default([]) }}" +- set_fact: roles_to_modify={{ es_roles.native.keys() | difference (reserved_roles) }} + when: manage_native_roles -- set_fact: native_roles={{ es_roles.native }} - when: manage_native_roles and es_roles.native.keys() > 0 - -#Update other roles +#Update other roles - NOT inc. reserved roles - name: Update Native Roles uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item.key}} + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}} method: POST body_format: json - body: "{{item.value | to_json}}" + body: "{{ es_roles.native[item] | to_json}}" status_code: 200 user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes - when: manage_native_roles and native_roles.keys() > 0 - with_dict: "{{ native_roles | default({})}}" + when: manage_native_roles + with_items: "{{ roles_to_modify | default([]) }}" \ No newline at end of file diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 4687a66..54df75a 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -199,5 +199,45 @@ shared_examples 'xpack::init' do |es_version,plugins| #Test contents as expected its(:md5sum) { should eq '6ff0e6c4380a6ac0f6e04d871c0ca5e8' } end + + #check accounts are correct i.e. we can auth and they have the correct roles + + describe 'kibana4_server access check' do + it 'should be reported as version '+es_version do + command = command('curl -s localhost:9200/ -u kibana4_server:changeMe | grep number') + expect(command.stdout).to match(es_version) + expect(command.exit_status).to eq(0) + end + end + + describe command('curl -s localhost:9200/_xpack/security/user/kibana4_server -u elastic:elasticChanged | md5sum | grep f0548742161d9e50c7c7fbe2e061a1fa') do + its(:exit_status) { should eq 0 } + end + + + describe 'logstash_system access check' do + it 'should be reported as version '+es_version do + command = command('curl -s localhost:9200/ -u logstash_system:aNewLogstashPassword | grep number') + expect(command.stdout).to match(es_version) + expect(command.exit_status).to eq(0) + end + end + + describe command('curl -s localhost:9200/_xpack/security/user/logstash_system -u elastic:elasticChanged | md5sum | grep 98d361ddfa5156abd33542a493b4fd85') do + its(:exit_status) { should eq 0 } + end + + describe 'kibana access check' do + it 'should be reported as version '+es_version do + command = command('curl -s localhost:9200/ -u kibana:changeme | grep number') + expect(command.stdout).to match(es_version) + expect(command.exit_status).to eq(0) + end + end + + describe command('curl -s localhost:9200/_xpack/security/user/kibana -u elastic:elasticChanged | md5sum | grep 34190c64eb3c7cfb002fa789df5fad20') do + its(:exit_status) { should eq 0 } + end + end diff --git a/test/integration/issue-test.yml b/test/integration/issue-test.yml index ae5c7f3..1f7ef2a 100644 --- a/test/integration/issue-test.yml +++ b/test/integration/issue-test.yml @@ -2,10 +2,20 @@ #Modify the playbook below and test with kitchen i.e. `kitchen test issue-test` #To add custom tests modify the serverspec file ./helpers/serverspec/issue_test_spec.rb #Idempot test is enabled for this test + - name: Simple Example hosts: localhost roles: - - { role: elasticsearch, es_config: { "xpack.security.authc.realms.file1.type": "file", "xpack.security.authc.realms.file1.order": 1, "xpack.security.authc.realms.native1.type": "native", "xpack.security.authc.realms.native1.order": 0 }, es_instance_name: "security_node" } + - { + role: elasticsearch, + es_config: + { + "xpack.security.authc.realms.file1.type": "file", + "xpack.security.authc.realms.file1.order": 1, + "xpack.security.authc.realms.native1.type": "native", + "xpack.security.authc.realms.native1.order": 0 + }, + es_instance_name: "security_node" } vars: es_heap_size: "1g" es_enable_xpack: true @@ -17,8 +27,23 @@ es_api_basic_auth_username: elastic es_api_basic_auth_password: changeme es_users: - native: - testUser: + file: + test_user: password: changeme roles: - - kibana_user \ No newline at end of file + - kibana_system + native: + kibana: + password: changeme + roles: + - kibana_system + elastic: + password: aNewPassWord + es_roles: + native: + logstash: + cluster: + - manage_index_templates + logstash_system: + cluster: + - manage_index_templates \ No newline at end of file diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 6c57500..3e53349 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -31,6 +31,14 @@ password: changeMe roles: - kibana4_server + logstash_system: + #this should be successfully modified + password: aNewLogstashPassword + #this will be ignored + roles: + - kibana4_server + elastic: + password: elasticChanged file: es_admin: password: changeMe @@ -79,6 +87,16 @@ - write - delete - create_index + #this will be ignored - its reserved + logstash_system: + cluster: + - manage_index_templates + indices: + - names: 'logstash-*' + privileges: + - write + - delete + - create_index #modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed. - name: Elasticsearch Xpack modify @@ -99,7 +117,7 @@ - security - alerting es_api_basic_auth_username: elastic - es_api_basic_auth_password: changeme + es_api_basic_auth_password: elasticChanged es_role_mapping: power_user: - "cn=admins,dc=example,dc=com" @@ -112,6 +130,10 @@ password: changeMe roles: - kibana4_server + logstash_system: + #this will be ignored + roles: + - kibana4_server file: es_admin: password: changeMeAgain diff --git a/vars/main.yml b/vars/main.yml index 3d50db2..b3bd126 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -4,4 +4,5 @@ es_conf_dir: "/etc/elasticsearch" sysd_script: "/usr/lib/systemd/system/elasticsearch.service" init_script: "/etc/init.d/elasticsearch" #add supported features here -supported_xpack_features: ["alerting","monitoring","graph","security"] \ No newline at end of file +supported_xpack_features: ["alerting","monitoring","graph","security"] +reserved_xpack_users: ["elastic","kibana","logstash_system"] \ No newline at end of file