diff --git a/.kitchen.yml b/.kitchen.yml index a02d9be..e5172cc 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -8,6 +8,7 @@ provisioner: roles_path: ../ require_ansible_repo: true ansible_verbose: true + ansible_version: 2.0.2 http_proxy: <%= ENV['HTTP_PROXY'] %> https_proxy: <%= ENV['HTTPS_PROXY'] %> no_proxy: localhost,127.0.0.1 @@ -19,7 +20,7 @@ platforms: privileged: true provision_command: - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get update && apt-get -y -q install ansible python-apt python-pycurl + - apt-get update && apt-get -y -q install python-apt python-pycurl use_sudo: false - name: debian-7 driver_config: @@ -27,7 +28,6 @@ platforms: privileged: true provision_command: - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml - - pip install ansible - apt-get install -y -q net-tools use_sudo: false - name: debian-8 @@ -36,7 +36,6 @@ platforms: privileged: true provision_command: - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget - - pip install ansible - apt-get install -y -q net-tools - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config @@ -137,3 +136,9 @@ suites: version: latest provisioner: playbook: test/integration/multi.yml + #Currently we only test shield on 2x + - name: xpack-2x + run_list: + attributes: + provisioner: + playbook: test/integration/xpack.yml \ No newline at end of file diff --git a/Gemfile b/Gemfile index e0591b1..13c5458 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'test-kitchen', '1.4.2' -gem "kitchen-docker", '2.1.0' -gem 'kitchen-ansible', '0.40.1' -gem 'net-ssh', '~> 2.0' +gem 'test-kitchen', '1.8.0' +gem "kitchen-docker", '2.5.0' +gem 'kitchen-ansible', '0.44.6' +gem 'net-ssh', '~> 3.0' diff --git a/Gemfile.lock b/Gemfile.lock index 8c827c2..d5952fa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,30 +1,27 @@ GEM remote: https://rubygems.org/ specs: - faraday (0.9.2) - multipart-post (>= 1.2, < 3) - highline (1.7.8) - kitchen-ansible (0.40.1) - librarian-ansible + artifactory (2.3.3) + kitchen-ansible (0.44.6) + net-ssh (~> 3.0) test-kitchen (~> 1.4) - kitchen-docker (2.1.0) + kitchen-docker (2.5.0) test-kitchen (>= 1.0.0) - librarian (0.1.2) - highline - thor (~> 0.15) - librarian-ansible (3.0.0) - faraday - librarian (~> 0.1.0) + mixlib-install (1.1.0) + artifactory + mixlib-shellout + mixlib-versioning mixlib-shellout (2.2.6) - multipart-post (2.0.0) + mixlib-versioning (1.1.0) net-scp (1.2.1) net-ssh (>= 2.6.5) - net-ssh (2.9.4) + net-ssh (3.2.0) safe_yaml (1.0.4) - test-kitchen (1.4.2) + test-kitchen (1.8.0) + mixlib-install (~> 1.0, >= 1.0.4) mixlib-shellout (>= 1.2, < 3.0) net-scp (~> 1.1) - net-ssh (~> 2.7, < 2.10) + net-ssh (>= 2.9, < 4.0) safe_yaml (~> 1.0) thor (~> 0.18) thor (0.19.1) @@ -33,10 +30,10 @@ PLATFORMS ruby DEPENDENCIES - kitchen-ansible (= 0.40.1) - kitchen-docker (= 2.1.0) - net-ssh (~> 2.0) - test-kitchen (= 1.4.2) + kitchen-ansible (= 0.44.6) + kitchen-docker (= 2.5.0) + net-ssh (~> 3.0) + test-kitchen (= 1.8.0) BUNDLED WITH 1.11.2 diff --git a/README.md b/README.md index 1fa450d..d4e82af 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,6 @@ controlled by the following parameters: * ```es_data_dirs``` - defaults to "/var/lib/elasticsearch". This can be a list or comma separated string e.g. ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"] or "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2" * ```es_log_dir``` - defaults to "/var/log/elasticsearch". * ```es_work_dir``` - defaults to "/tmp/elasticsearch". -* ```es_plugin_dir``` - defaults to "/usr/share/elasticsearch/plugins". * ```es_restart_on_change``` - defaults to true. If false, changes will not result in Elasticsearch being restarted. * ```es_plugins_reinstall``` - defaults to false. If true, all currently installed plugins will be removed from a node. Listed plugins will then be re-installed. diff --git a/defaults/main.yml b/defaults/main.yml index 451000f..10adc43 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,6 +1,6 @@ --- es_major_version: "2.x" -es_version: "2.2.0" +es_version: "2.3.4" es_version_lock: false es_use_repository: true es_start_service: true @@ -13,13 +13,16 @@ es_templates: false es_user: elasticsearch es_group: elasticsearch es_config: {} -es_install_shield: false #Need to provide default directories es_pid_dir: "/var/run/elasticsearch" es_data_dirs: "/var/lib/elasticsearch" es_log_dir: "/var/log/elasticsearch" es_work_dir: "/tmp/elasticsearch" -es_plugin_dir: "/usr/share/elasticsearch/plugins" es_max_open_files: 65536 es_allow_downgrades: false - +es_enable_xpack: false +es_xpack_features: [] +#These are used for internal operations performed by ansible. +#They do not effect the current configuration +es_api_host: "localhost" +es_api_port: 9200 \ No newline at end of file diff --git a/filter_plugins/custom.py b/filter_plugins/custom.py index 0453a25..ecd3b97 100644 --- a/filter_plugins/custom.py +++ b/filter_plugins/custom.py @@ -19,8 +19,18 @@ def append_to_list(values=[], suffix=''): def array_to_str(values=[],separator=','): return separator.join(values) +def extract_role_users(users={}): + role_users=[] + for user,details in users.iteritems(): + if "roles" in details: + for role in details["roles"]: + role_users.append(role+":"+user) + return role_users + + class FilterModule(object): def filters(self): return {'modify_list': modify_list, 'append_to_list':append_to_list, - 'array_to_str':array_to_str} \ No newline at end of file + 'array_to_str':array_to_str, + 'extract_role_users':extract_role_users} \ No newline at end of file diff --git a/handlers/main.yml b/handlers/main.yml index 50a3a7e..aa746f0 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -2,3 +2,7 @@ - name: restart elasticsearch service: name={{instance_init_script | basename}} state=restarted enabled=yes when: es_restart_on_change and es_start_service and not elasticsearch_started.changed and ((plugin_installed is defined and plugin_installed.changed) or (elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) + +- name: load-native-realms + include: ./handlers/shield/elasticsearch-shield-native.yml + when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) \ No newline at end of file diff --git a/handlers/shield/elasticsearch-shield-native.yml b/handlers/shield/elasticsearch-shield-native.yml new file mode 100644 index 0000000..a8f0e0f --- /dev/null +++ b/handlers/shield/elasticsearch-shield-native.yml @@ -0,0 +1,117 @@ +--- +- name: Wait for elasticsearch to startup + wait_for: port={{es_api_port}} delay=10 + +- set_fact: manage_native_users=false + +- set_fact: manage_native_users=true + when: es_users is defined and es_users.native is defined + +- set_fact: manage_native_roles=false + +- set_fact: manage_native_roles=true + when: es_roles is defined and es_roles.native is defined + +#If the node has just has shield installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load + +#List current users +- name: List Native Users + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/user + method: GET + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + status_code: 200 + register: user_list_response + when: manage_native_users + + +- set_fact: current_users={{user_list_response.json.keys() | list}} + when: manage_native_users + +#Identify non declared users + +- set_fact: users_to_remove={{ current_users | difference ( es_users.native.keys() ) }} + when: manage_native_users + +#Delete all non required users +- name: Delete Native Users + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item}} + method: DELETE + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_users and users_to_remove | length > 0 + with_items: "{{users_to_remove}}" + + +#Overwrite all other users +- name: Update Native Users + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item.key}} + method: POST + body_format: json + body: "{{item.value | to_json}}" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_users and es_users.native.keys() > 0 + with_dict: "{{es_users.native}}" + +#List current roles + +- name: List Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/role + method: GET + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + status_code: 200 + register: role_list_response + when: manage_native_roles + +#Identify undeclared roles + +- set_fact: current_roles={{role_list_response.json.keys() | list}} + when: manage_native_users + +- debug: msg="{{current_roles}}" + +- set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys() ) }} + when: manage_native_roles + + +#Delete all non required roles +- name: Delete Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item}} + method: DELETE + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_roles and roles_to_remove | length > 0 + with_items: "{{roles_to_remove}}" + + +#Update other roles +- name: Update Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item.key}} + method: POST + body_format: json + body: "{{item.value | to_json}}" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_roles and es_roles.native.keys() > 0 + with_dict: "{{es_roles.native}}" + + + diff --git a/meta/main.yml b/meta/main.yml index 1a0b068..66df2a3 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -8,7 +8,7 @@ galaxy_info: company: "Elastic.co" license: "license (Apache)" # Require 1.6 for apt deb install - min_ansible_version: 1.6 + min_ansible_version: 2.0 platforms: - name: EL versions: diff --git a/tasks/checkParameters.yml b/tasks/checkParameters.yml deleted file mode 100644 index 6fc5bef..0000000 --- a/tasks/checkParameters.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Check for mandatory parameters - -- fail: msg="es_instance_name must be specified and cannot be blank" - when: es_instance_name is not defined or es_instance_name == '' - -- fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined" - when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '') - -- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or (es_config['discovery.zen.ping.multicast.enabled'] is defined and es_config['discovery.zen.ping.multicast.enabled'])}} - -- debug: msg="WARNING - It is recommended you specify the parameter 'http.port' when multicast is disabled" - when: not multi_cast and es_config['http.port'] is not defined - -- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port' when multicast is disabled" - when: not multi_cast and es_config['transport.tcp.port'] is not defined - -- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts' when multicast is disabled" - when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined - -#If the user attempts to lock memory they must specify a heap size -- fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified" - when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined \ No newline at end of file diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 77d0a5f..34943b4 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -1,42 +1,6 @@ --- - # Configure Elasticsearch Node -#Use systemd for the following distributions: -# -#Ubuntu 15 and up -#Debian 8 and up -#Centos 7 and up -#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied. - - -- set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }} - tags: - - always - -- set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}} - when: use_system_d - tags: - - always - -#For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN. - -- set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }} - tags: - - always - -- set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}} - tags: - - always - -- set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}} - tags: - - always - -- set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}} - tags: - - always - #Create required directories - name: Create Directories file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} @@ -45,11 +9,6 @@ - "{{work_dir}}" - "{{log_dir}}" - "{{conf_dir}}" - - "{{plugin_dir}}" - -- set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }} - tags: - - always - name: Create Data Directories file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} @@ -112,4 +71,3 @@ - name: Delete Default Logging File file: dest=/etc/elasticsearch/logging.yml state=absent -- debug: msg="Data Dirs {{data_dirs}}" \ No newline at end of file diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml new file mode 100644 index 0000000..3e8281a --- /dev/null +++ b/tasks/elasticsearch-parameters.yml @@ -0,0 +1,50 @@ +# Check for mandatory parameters + +- fail: msg="es_instance_name must be specified and cannot be blank" + when: es_instance_name is not defined or es_instance_name == '' + +- fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined" + when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '') + +- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or (es_config['discovery.zen.ping.multicast.enabled'] is defined and es_config['discovery.zen.ping.multicast.enabled'])}} + +- debug: msg="WARNING - It is recommended you specify the parameter 'http.port' when multicast is disabled" + when: not multi_cast and es_config['http.port'] is not defined + +- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port' when multicast is disabled" + when: not multi_cast and es_config['transport.tcp.port'] is not defined + +- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts' when multicast is disabled" + when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined + +#If the user attempts to lock memory they must specify a heap size +- fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified" + when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined + +#Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work +- fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" + when: es_enable_xpack and '"shield" in es_xpack_features' and es_api_basic_auth_username is not defined and es_api_basic_auth_username is not defined + +- set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} +- set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} +- set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} +- set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }} + +#Use systemd for the following distributions: +#Ubuntu 15 and up +#Debian 8 and up +#Centos 7 and up +#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied. + + +- set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }} + +- set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}} + when: use_system_d +#For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN. + +- set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }} +- set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}} +- set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}} +- set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}} +- set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }} \ No newline at end of file diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 7da2d8c..894533d 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -4,24 +4,20 @@ # i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting. - set_fact: es_plugins_reinstall=true when: ((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none - tags: - - always - set_fact: list_command="list" - tags: - - always + - set_fact: list_command="--list" when: es_version | version_compare('2.0', '<') - tags: - - always #List currently installed plugins - shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-" register: installed_plugins changed_when: False + ignore_errors: yes environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" #This needs to removes any currently installed plugins - name: Remove elasticsearch plugins @@ -50,4 +46,4 @@ #Set permissions on plugins directory - name: Set Plugin Directory Permissions - file: state=directory path={{ plugin_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file + file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file diff --git a/tasks/elasticsearch-shield.yml b/tasks/elasticsearch-shield.yml deleted file mode 100644 index 849169c..0000000 --- a/tasks/elasticsearch-shield.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- - - diff --git a/tasks/elasticsearch-templates.yml b/tasks/elasticsearch-templates.yml index c9de8bb..923b699 100644 --- a/tasks/elasticsearch-templates.yml +++ b/tasks/elasticsearch-templates.yml @@ -12,22 +12,13 @@ with_fileglob: - "{{ es_templates_fileglob }}" -- set_fact: http_port=9200 - tags: - - always - -- set_fact: http_port={{es_config['http.port']}} - when: es_config['http.port'] is defined - tags: - - always - - name: Wait for elasticsearch to startup - wait_for: port={{http_port}} delay=10 + wait_for: port={{es_api_port}} delay=10 - name: Get template files shell: find . -maxdepth 1 -type f | sed "s#\./##" | sed "s/.json//" chdir=/etc/elasticsearch/templates register: resultstemplate - name: Install template(s) - command: "curl -sL -XPUT http://localhost:{{http_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json" + command: "curl -sL -XPUT http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json" with_items: "{{ resultstemplate.stdout_lines }}" diff --git a/tasks/elasticsearch.yml b/tasks/elasticsearch.yml index 1e5cc89..7addb03 100644 --- a/tasks/elasticsearch.yml +++ b/tasks/elasticsearch.yml @@ -1,24 +1,6 @@ --- -- set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} - tags: - - always -- set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} - tags: - - always -- set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} - tags: - - always -- set_fact: plugin_dir={{ es_plugin_dir }}/{{es_instance_name}} - tags: - - always -- set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }} - tags: - - always - -- debug: msg="Node configuration {{ es_config }} " - -- name: Include optional user and group creation. +- name: Include optional user and group creation. when: (es_user_id is defined) and (es_group_id is defined) include: elasticsearch-optional-user.yml diff --git a/tasks/main.yml b/tasks/main.yml index 6a92155..422b0a3 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,12 +1,14 @@ --- -- name: check-parameters - include: checkParameters.yml - tags: - - check - name: os-specific vars include_vars: "{{ansible_os_family}}.yml" tags: - always + +- name: check-set-parameters + include: elasticsearch-parameters.yml + tags: + - always + - include: java.yml when: es_java_install tags: @@ -25,10 +27,10 @@ when: es_plugins is defined or es_plugins_reinstall tags: - plugins -- include: elasticsearch-shield.yml - when: es_install_shield +- include: xpack/elasticsearch-xpack.yml + when: es_enable_xpack tags: - - shield + - xpack - include: elasticsearch-service.yml tags: - service diff --git a/tasks/xpack/elasticsearch-shield-file.yml b/tasks/xpack/elasticsearch-shield-file.yml new file mode 100644 index 0000000..4a266cc --- /dev/null +++ b/tasks/xpack/elasticsearch-shield-file.yml @@ -0,0 +1,62 @@ +--- + +- set_fact: manage_file_users=false + +- set_fact: manage_file_users=true + when: es_users is defined and es_users.file is defined + +#List current users +- name: List Users + shell: cat {{conf_dir}}/shield/users | awk -F':' '{print $1}' + register: current_file_users + when: manage_file_users + +- set_fact: users_to_remove={{ current_file_users.stdout_lines | difference ( es_users.file.keys() ) }} + when: manage_file_users + +#Remove users +- name: Remove Users + command: > + {{es_home}}/bin/shield/esusers userdel {{item}} + when: manage_file_users and (users_to_remove | length > 0) + with_items: "{{users_to_remove}}" + environment: + CONF_DIR: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + + +#Add users +- name: Add Users + command: > + {{es_home}}/bin/shield/esusers useradd {{item.key}} -p {{item.value.password}} + with_dict: "{{es_users.file}}" + when: manage_file_users and es_users.file.keys() | length > 0 + environment: + CONF_DIR: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + +#Set passwords for all users declared - Required as the useradd will not change existing user passwords +- name: Set User Passwords + command: > + {{es_home}}/bin/shield/esusers passwd {{item.key}} -p {{item.value.password}} + with_dict: "{{es_users.file}}" + when: manage_file_users and es_users.file.keys() | length > 0 + environment: + CONF_DIR: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + +- set_fact: users_roles={{es_users.file | extract_role_users}} + when: manage_file_users + +#Copy Roles files +- name: Copy roles.yml File for Instance + template: src=shield/roles.yml.j2 dest={{conf_dir}}/shield/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + when: es_roles is defined and es_roles.file is defined + +#Overwrite users_roles file +- name: Copy User Roles + template: src=shield/users_roles.j2 dest={{conf_dir}}/shield/users_roles mode=0644 force=yes + when: manage_file_users and users_roles | length > 0 + +#TODO: Support for mapping file + diff --git a/tasks/xpack/elasticsearch-shield.yml b/tasks/xpack/elasticsearch-shield.yml new file mode 100644 index 0000000..455fa2f --- /dev/null +++ b/tasks/xpack/elasticsearch-shield.yml @@ -0,0 +1,45 @@ +--- + +#Test if we need to install shield + +- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep shield" + register: shield_installed + changed_when: False + ignore_errors: yes + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +#Install Shield if not installed +- name: Install shield plugin + command: > + {{es_home}}/bin/plugin install shield + register: shield + failed_when: "'ERROR' in shield_installed.stdout" + changed_when: shield.rc == 1 + when: shield_installed.rc == 1 + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + +#TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6 + + +#Ensure shield conf directory is created +- name: Ensure shield conf directory exists + file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }} + +#-----------------------------FILE BASED REALM---------------------------------------- + +- include: elasticsearch-shield-file.yml + when: (es_users is defined and es_users.file) or (es_roles is defined and es_roles.file is defined) + +#-----------------------------NATIVE BASED REALM---------------------------------------- +# The native realm requires the node to be started so we do as a handler +- command: /bin/true + notify: load-native-realms + when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) + + diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml new file mode 100644 index 0000000..aacdf7c --- /dev/null +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -0,0 +1,33 @@ +--- +#Check if license is installed +- name: Check License is installed + shell: > + {{es_home}}/bin/plugin list | tail -n +2 | grep license + register: license_installed + ignore_errors: yes + changed_when: False + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + +#Install License if not installed +- name: Install license plugin + command: > + {{es_home}}/bin/plugin install license + register: license + failed_when: "'ERROR' in license_installed .stdout" + changed_when: license.rc == 1 + when: license_installed.rc == 1 + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +- name: Set Plugin Directory Permissions + file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes + +- include: elasticsearch-shield.yml + when: '"shield" in es_xpack_features' + +#Any other xpacks plugins requiring configuration to be entered here \ No newline at end of file diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index 203a869..40ab354 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -20,6 +20,4 @@ path.data: {{ data_dirs | array_to_str }} path.work: {{ work_dir }} -path.logs: {{ log_dir }} - -path.plugins: {{ plugin_dir }} \ No newline at end of file +path.logs: {{ log_dir }} \ No newline at end of file diff --git a/templates/shield/roles.yml.j2 b/templates/shield/roles.yml.j2 new file mode 100644 index 0000000..9f211f2 --- /dev/null +++ b/templates/shield/roles.yml.j2 @@ -0,0 +1 @@ +{{ es_roles.file | to_nice_yaml }} \ No newline at end of file diff --git a/templates/shield/users_roles.j2 b/templates/shield/users_roles.j2 new file mode 100644 index 0000000..1c0acfa --- /dev/null +++ b/templates/shield/users_roles.j2 @@ -0,0 +1 @@ +{{users_roles | join("\n") }} \ No newline at end of file diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index a52bf80..983c0a7 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -173,28 +173,7 @@ shared_examples 'multi::init' do |es_version,plugins| end end - #Multi node plugin tests - describe file('/opt/elasticsearch/plugins/node1') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/opt/elasticsearch/plugins/master') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - for plugin in plugins - describe file('/opt/elasticsearch/plugins/node1/'+plugin) do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/opt/elasticsearch/plugins/master/'+plugin) do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end describe command('curl -s localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do its(:exit_status) { should eq 0 } diff --git a/test/integration/multi.yml b/test/integration/multi.yml index e291fcb..173528c 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -8,5 +8,4 @@ vars: es_scripts: true es_templates: true - es_plugin_dir: "/opt/elasticsearch/plugins" #Plugins installed for this test are specified in .kitchen.yml under suite \ No newline at end of file diff --git a/test/integration/xpack-2x/serverspec/xpack_spec.rb b/test/integration/xpack-2x/serverspec/xpack_spec.rb new file mode 100644 index 0000000..c6a7af0 --- /dev/null +++ b/test/integration/xpack-2x/serverspec/xpack_spec.rb @@ -0,0 +1,10 @@ +require 'spec_helper' + +describe 'XPack Tests v 2.x' do + + describe user('elasticsearch') do + it { should exist } + end + +end + diff --git a/test/integration/xpack-2x/xpack.yaml b/test/integration/xpack-2x/xpack.yaml new file mode 100644 index 0000000..a3c37e1 --- /dev/null +++ b/test/integration/xpack-2x/xpack.yaml @@ -0,0 +1,2 @@ +--- +- host: test-kitchen diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml new file mode 100644 index 0000000..2c6dbcb --- /dev/null +++ b/test/integration/xpack.yml @@ -0,0 +1,68 @@ +--- +- name: Elasticsearch Xpack tests + hosts: localhost + roles: + - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "shield_node" } + vars: + es_templates: false + es_enable_xpack: true + es_xpack_features: + - shield + - watcher + es_api_basic_auth_username: es_admin + es_api_basic_auth_password: changeMe + es_users: + native: + kibana4_server: + password: changeMe + roles: + - kibana4_server + file: + es_admin: + password: changeMe + roles: + - admin + testUser: + password: changeMeAlso! + roles: + - power_user + - user + es_roles: + file: + admin: + cluster: + - all + indices: + - names: '*' + privileges: + - all + power_user: + cluster: + - monitor + indices: + - names: '*' + privileges: + - all + user: + indices: + - names: '*' + privileges: + - read + kibana4_server: + cluster: + - monitor + indices: + - names: '.kibana' + privileges: + - all + native: + logstash: + cluster: + - manage_index_templates + indices: + - names: 'logstash-*' + privileges: + - write + - delete + - create_index +