From ab592724d8b15d6bf46e1963decdc489ec87246b Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Fri, 22 Jul 2016 23:44:27 +0100 Subject: [PATCH 01/24] Initial Shield support + latest gems + single plugin dir + new port/host vars --- .kitchen.yml | 11 +- Gemfile | 8 +- Gemfile.lock | 37 +++--- README.md | 1 - defaults/main.yml | 11 +- filter_plugins/custom.py | 12 +- handlers/main.yml | 4 + .../shield/elasticsearch-shield-native.yml | 117 ++++++++++++++++++ meta/main.yml | 2 +- tasks/checkParameters.yml | 22 ---- tasks/elasticsearch-config.yml | 42 ------- tasks/elasticsearch-parameters.yml | 50 ++++++++ tasks/elasticsearch-plugins.yml | 14 +-- tasks/elasticsearch-shield.yml | 3 - tasks/elasticsearch-templates.yml | 13 +- tasks/elasticsearch.yml | 20 +-- tasks/main.yml | 16 +-- tasks/xpack/elasticsearch-shield-file.yml | 62 ++++++++++ tasks/xpack/elasticsearch-shield.yml | 45 +++++++ tasks/xpack/elasticsearch-xpack.yml | 33 +++++ templates/elasticsearch.yml.j2 | 4 +- templates/shield/roles.yml.j2 | 1 + templates/shield/users_roles.j2 | 1 + .../helpers/serverspec/multi_spec.rb | 21 ---- test/integration/multi.yml | 1 - .../xpack-2x/serverspec/xpack_spec.rb | 10 ++ test/integration/xpack-2x/xpack.yaml | 2 + test/integration/xpack.yml | 68 ++++++++++ 28 files changed, 459 insertions(+), 172 deletions(-) create mode 100644 handlers/shield/elasticsearch-shield-native.yml delete mode 100644 tasks/checkParameters.yml create mode 100644 tasks/elasticsearch-parameters.yml delete mode 100644 tasks/elasticsearch-shield.yml create mode 100644 tasks/xpack/elasticsearch-shield-file.yml create mode 100644 tasks/xpack/elasticsearch-shield.yml create mode 100644 tasks/xpack/elasticsearch-xpack.yml create mode 100644 templates/shield/roles.yml.j2 create mode 100644 templates/shield/users_roles.j2 create mode 100644 test/integration/xpack-2x/serverspec/xpack_spec.rb create mode 100644 test/integration/xpack-2x/xpack.yaml create mode 100644 test/integration/xpack.yml diff --git a/.kitchen.yml b/.kitchen.yml index a02d9be..e5172cc 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -8,6 +8,7 @@ provisioner: roles_path: ../ require_ansible_repo: true ansible_verbose: true + ansible_version: 2.0.2 http_proxy: <%= ENV['HTTP_PROXY'] %> https_proxy: <%= ENV['HTTPS_PROXY'] %> no_proxy: localhost,127.0.0.1 @@ -19,7 +20,7 @@ platforms: privileged: true provision_command: - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get update && apt-get -y -q install ansible python-apt python-pycurl + - apt-get update && apt-get -y -q install python-apt python-pycurl use_sudo: false - name: debian-7 driver_config: @@ -27,7 +28,6 @@ platforms: privileged: true provision_command: - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml - - pip install ansible - apt-get install -y -q net-tools use_sudo: false - name: debian-8 @@ -36,7 +36,6 @@ platforms: privileged: true provision_command: - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget - - pip install ansible - apt-get install -y -q net-tools - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config @@ -137,3 +136,9 @@ suites: version: latest provisioner: playbook: test/integration/multi.yml + #Currently we only test shield on 2x + - name: xpack-2x + run_list: + attributes: + provisioner: + playbook: test/integration/xpack.yml \ No newline at end of file diff --git a/Gemfile b/Gemfile index e0591b1..13c5458 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'test-kitchen', '1.4.2' -gem "kitchen-docker", '2.1.0' -gem 'kitchen-ansible', '0.40.1' -gem 'net-ssh', '~> 2.0' +gem 'test-kitchen', '1.8.0' +gem "kitchen-docker", '2.5.0' +gem 'kitchen-ansible', '0.44.6' +gem 'net-ssh', '~> 3.0' diff --git a/Gemfile.lock b/Gemfile.lock index 8c827c2..d5952fa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,30 +1,27 @@ GEM remote: https://rubygems.org/ specs: - faraday (0.9.2) - multipart-post (>= 1.2, < 3) - highline (1.7.8) - kitchen-ansible (0.40.1) - librarian-ansible + artifactory (2.3.3) + kitchen-ansible (0.44.6) + net-ssh (~> 3.0) test-kitchen (~> 1.4) - kitchen-docker (2.1.0) + kitchen-docker (2.5.0) test-kitchen (>= 1.0.0) - librarian (0.1.2) - highline - thor (~> 0.15) - librarian-ansible (3.0.0) - faraday - librarian (~> 0.1.0) + mixlib-install (1.1.0) + artifactory + mixlib-shellout + mixlib-versioning mixlib-shellout (2.2.6) - multipart-post (2.0.0) + mixlib-versioning (1.1.0) net-scp (1.2.1) net-ssh (>= 2.6.5) - net-ssh (2.9.4) + net-ssh (3.2.0) safe_yaml (1.0.4) - test-kitchen (1.4.2) + test-kitchen (1.8.0) + mixlib-install (~> 1.0, >= 1.0.4) mixlib-shellout (>= 1.2, < 3.0) net-scp (~> 1.1) - net-ssh (~> 2.7, < 2.10) + net-ssh (>= 2.9, < 4.0) safe_yaml (~> 1.0) thor (~> 0.18) thor (0.19.1) @@ -33,10 +30,10 @@ PLATFORMS ruby DEPENDENCIES - kitchen-ansible (= 0.40.1) - kitchen-docker (= 2.1.0) - net-ssh (~> 2.0) - test-kitchen (= 1.4.2) + kitchen-ansible (= 0.44.6) + kitchen-docker (= 2.5.0) + net-ssh (~> 3.0) + test-kitchen (= 1.8.0) BUNDLED WITH 1.11.2 diff --git a/README.md b/README.md index 1fa450d..d4e82af 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,6 @@ controlled by the following parameters: * ```es_data_dirs``` - defaults to "/var/lib/elasticsearch". This can be a list or comma separated string e.g. ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"] or "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2" * ```es_log_dir``` - defaults to "/var/log/elasticsearch". * ```es_work_dir``` - defaults to "/tmp/elasticsearch". -* ```es_plugin_dir``` - defaults to "/usr/share/elasticsearch/plugins". * ```es_restart_on_change``` - defaults to true. If false, changes will not result in Elasticsearch being restarted. * ```es_plugins_reinstall``` - defaults to false. If true, all currently installed plugins will be removed from a node. Listed plugins will then be re-installed. diff --git a/defaults/main.yml b/defaults/main.yml index 451000f..10adc43 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,6 +1,6 @@ --- es_major_version: "2.x" -es_version: "2.2.0" +es_version: "2.3.4" es_version_lock: false es_use_repository: true es_start_service: true @@ -13,13 +13,16 @@ es_templates: false es_user: elasticsearch es_group: elasticsearch es_config: {} -es_install_shield: false #Need to provide default directories es_pid_dir: "/var/run/elasticsearch" es_data_dirs: "/var/lib/elasticsearch" es_log_dir: "/var/log/elasticsearch" es_work_dir: "/tmp/elasticsearch" -es_plugin_dir: "/usr/share/elasticsearch/plugins" es_max_open_files: 65536 es_allow_downgrades: false - +es_enable_xpack: false +es_xpack_features: [] +#These are used for internal operations performed by ansible. +#They do not effect the current configuration +es_api_host: "localhost" +es_api_port: 9200 \ No newline at end of file diff --git a/filter_plugins/custom.py b/filter_plugins/custom.py index 0453a25..ecd3b97 100644 --- a/filter_plugins/custom.py +++ b/filter_plugins/custom.py @@ -19,8 +19,18 @@ def append_to_list(values=[], suffix=''): def array_to_str(values=[],separator=','): return separator.join(values) +def extract_role_users(users={}): + role_users=[] + for user,details in users.iteritems(): + if "roles" in details: + for role in details["roles"]: + role_users.append(role+":"+user) + return role_users + + class FilterModule(object): def filters(self): return {'modify_list': modify_list, 'append_to_list':append_to_list, - 'array_to_str':array_to_str} \ No newline at end of file + 'array_to_str':array_to_str, + 'extract_role_users':extract_role_users} \ No newline at end of file diff --git a/handlers/main.yml b/handlers/main.yml index 50a3a7e..aa746f0 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -2,3 +2,7 @@ - name: restart elasticsearch service: name={{instance_init_script | basename}} state=restarted enabled=yes when: es_restart_on_change and es_start_service and not elasticsearch_started.changed and ((plugin_installed is defined and plugin_installed.changed) or (elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) + +- name: load-native-realms + include: ./handlers/shield/elasticsearch-shield-native.yml + when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) \ No newline at end of file diff --git a/handlers/shield/elasticsearch-shield-native.yml b/handlers/shield/elasticsearch-shield-native.yml new file mode 100644 index 0000000..a8f0e0f --- /dev/null +++ b/handlers/shield/elasticsearch-shield-native.yml @@ -0,0 +1,117 @@ +--- +- name: Wait for elasticsearch to startup + wait_for: port={{es_api_port}} delay=10 + +- set_fact: manage_native_users=false + +- set_fact: manage_native_users=true + when: es_users is defined and es_users.native is defined + +- set_fact: manage_native_roles=false + +- set_fact: manage_native_roles=true + when: es_roles is defined and es_roles.native is defined + +#If the node has just has shield installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load + +#List current users +- name: List Native Users + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/user + method: GET + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + status_code: 200 + register: user_list_response + when: manage_native_users + + +- set_fact: current_users={{user_list_response.json.keys() | list}} + when: manage_native_users + +#Identify non declared users + +- set_fact: users_to_remove={{ current_users | difference ( es_users.native.keys() ) }} + when: manage_native_users + +#Delete all non required users +- name: Delete Native Users + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item}} + method: DELETE + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_users and users_to_remove | length > 0 + with_items: "{{users_to_remove}}" + + +#Overwrite all other users +- name: Update Native Users + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item.key}} + method: POST + body_format: json + body: "{{item.value | to_json}}" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_users and es_users.native.keys() > 0 + with_dict: "{{es_users.native}}" + +#List current roles + +- name: List Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/role + method: GET + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + status_code: 200 + register: role_list_response + when: manage_native_roles + +#Identify undeclared roles + +- set_fact: current_roles={{role_list_response.json.keys() | list}} + when: manage_native_users + +- debug: msg="{{current_roles}}" + +- set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys() ) }} + when: manage_native_roles + + +#Delete all non required roles +- name: Delete Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item}} + method: DELETE + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_roles and roles_to_remove | length > 0 + with_items: "{{roles_to_remove}}" + + +#Update other roles +- name: Update Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item.key}} + method: POST + body_format: json + body: "{{item.value | to_json}}" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_roles and es_roles.native.keys() > 0 + with_dict: "{{es_roles.native}}" + + + diff --git a/meta/main.yml b/meta/main.yml index 1a0b068..66df2a3 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -8,7 +8,7 @@ galaxy_info: company: "Elastic.co" license: "license (Apache)" # Require 1.6 for apt deb install - min_ansible_version: 1.6 + min_ansible_version: 2.0 platforms: - name: EL versions: diff --git a/tasks/checkParameters.yml b/tasks/checkParameters.yml deleted file mode 100644 index 6fc5bef..0000000 --- a/tasks/checkParameters.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Check for mandatory parameters - -- fail: msg="es_instance_name must be specified and cannot be blank" - when: es_instance_name is not defined or es_instance_name == '' - -- fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined" - when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '') - -- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or (es_config['discovery.zen.ping.multicast.enabled'] is defined and es_config['discovery.zen.ping.multicast.enabled'])}} - -- debug: msg="WARNING - It is recommended you specify the parameter 'http.port' when multicast is disabled" - when: not multi_cast and es_config['http.port'] is not defined - -- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port' when multicast is disabled" - when: not multi_cast and es_config['transport.tcp.port'] is not defined - -- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts' when multicast is disabled" - when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined - -#If the user attempts to lock memory they must specify a heap size -- fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified" - when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined \ No newline at end of file diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 77d0a5f..34943b4 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -1,42 +1,6 @@ --- - # Configure Elasticsearch Node -#Use systemd for the following distributions: -# -#Ubuntu 15 and up -#Debian 8 and up -#Centos 7 and up -#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied. - - -- set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }} - tags: - - always - -- set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}} - when: use_system_d - tags: - - always - -#For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN. - -- set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }} - tags: - - always - -- set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}} - tags: - - always - -- set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}} - tags: - - always - -- set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}} - tags: - - always - #Create required directories - name: Create Directories file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} @@ -45,11 +9,6 @@ - "{{work_dir}}" - "{{log_dir}}" - "{{conf_dir}}" - - "{{plugin_dir}}" - -- set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }} - tags: - - always - name: Create Data Directories file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} @@ -112,4 +71,3 @@ - name: Delete Default Logging File file: dest=/etc/elasticsearch/logging.yml state=absent -- debug: msg="Data Dirs {{data_dirs}}" \ No newline at end of file diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml new file mode 100644 index 0000000..3e8281a --- /dev/null +++ b/tasks/elasticsearch-parameters.yml @@ -0,0 +1,50 @@ +# Check for mandatory parameters + +- fail: msg="es_instance_name must be specified and cannot be blank" + when: es_instance_name is not defined or es_instance_name == '' + +- fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined" + when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '') + +- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or (es_config['discovery.zen.ping.multicast.enabled'] is defined and es_config['discovery.zen.ping.multicast.enabled'])}} + +- debug: msg="WARNING - It is recommended you specify the parameter 'http.port' when multicast is disabled" + when: not multi_cast and es_config['http.port'] is not defined + +- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port' when multicast is disabled" + when: not multi_cast and es_config['transport.tcp.port'] is not defined + +- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts' when multicast is disabled" + when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined + +#If the user attempts to lock memory they must specify a heap size +- fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified" + when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined + +#Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work +- fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" + when: es_enable_xpack and '"shield" in es_xpack_features' and es_api_basic_auth_username is not defined and es_api_basic_auth_username is not defined + +- set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} +- set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} +- set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} +- set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }} + +#Use systemd for the following distributions: +#Ubuntu 15 and up +#Debian 8 and up +#Centos 7 and up +#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied. + + +- set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }} + +- set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}} + when: use_system_d +#For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN. + +- set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }} +- set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}} +- set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}} +- set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}} +- set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }} \ No newline at end of file diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 7da2d8c..894533d 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -4,24 +4,20 @@ # i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting. - set_fact: es_plugins_reinstall=true when: ((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none - tags: - - always - set_fact: list_command="list" - tags: - - always + - set_fact: list_command="--list" when: es_version | version_compare('2.0', '<') - tags: - - always #List currently installed plugins - shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-" register: installed_plugins changed_when: False + ignore_errors: yes environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" #This needs to removes any currently installed plugins - name: Remove elasticsearch plugins @@ -50,4 +46,4 @@ #Set permissions on plugins directory - name: Set Plugin Directory Permissions - file: state=directory path={{ plugin_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file + file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file diff --git a/tasks/elasticsearch-shield.yml b/tasks/elasticsearch-shield.yml deleted file mode 100644 index 849169c..0000000 --- a/tasks/elasticsearch-shield.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- - - diff --git a/tasks/elasticsearch-templates.yml b/tasks/elasticsearch-templates.yml index c9de8bb..923b699 100644 --- a/tasks/elasticsearch-templates.yml +++ b/tasks/elasticsearch-templates.yml @@ -12,22 +12,13 @@ with_fileglob: - "{{ es_templates_fileglob }}" -- set_fact: http_port=9200 - tags: - - always - -- set_fact: http_port={{es_config['http.port']}} - when: es_config['http.port'] is defined - tags: - - always - - name: Wait for elasticsearch to startup - wait_for: port={{http_port}} delay=10 + wait_for: port={{es_api_port}} delay=10 - name: Get template files shell: find . -maxdepth 1 -type f | sed "s#\./##" | sed "s/.json//" chdir=/etc/elasticsearch/templates register: resultstemplate - name: Install template(s) - command: "curl -sL -XPUT http://localhost:{{http_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json" + command: "curl -sL -XPUT http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json" with_items: "{{ resultstemplate.stdout_lines }}" diff --git a/tasks/elasticsearch.yml b/tasks/elasticsearch.yml index 1e5cc89..7addb03 100644 --- a/tasks/elasticsearch.yml +++ b/tasks/elasticsearch.yml @@ -1,24 +1,6 @@ --- -- set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} - tags: - - always -- set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} - tags: - - always -- set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} - tags: - - always -- set_fact: plugin_dir={{ es_plugin_dir }}/{{es_instance_name}} - tags: - - always -- set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }} - tags: - - always - -- debug: msg="Node configuration {{ es_config }} " - -- name: Include optional user and group creation. +- name: Include optional user and group creation. when: (es_user_id is defined) and (es_group_id is defined) include: elasticsearch-optional-user.yml diff --git a/tasks/main.yml b/tasks/main.yml index 6a92155..422b0a3 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,12 +1,14 @@ --- -- name: check-parameters - include: checkParameters.yml - tags: - - check - name: os-specific vars include_vars: "{{ansible_os_family}}.yml" tags: - always + +- name: check-set-parameters + include: elasticsearch-parameters.yml + tags: + - always + - include: java.yml when: es_java_install tags: @@ -25,10 +27,10 @@ when: es_plugins is defined or es_plugins_reinstall tags: - plugins -- include: elasticsearch-shield.yml - when: es_install_shield +- include: xpack/elasticsearch-xpack.yml + when: es_enable_xpack tags: - - shield + - xpack - include: elasticsearch-service.yml tags: - service diff --git a/tasks/xpack/elasticsearch-shield-file.yml b/tasks/xpack/elasticsearch-shield-file.yml new file mode 100644 index 0000000..4a266cc --- /dev/null +++ b/tasks/xpack/elasticsearch-shield-file.yml @@ -0,0 +1,62 @@ +--- + +- set_fact: manage_file_users=false + +- set_fact: manage_file_users=true + when: es_users is defined and es_users.file is defined + +#List current users +- name: List Users + shell: cat {{conf_dir}}/shield/users | awk -F':' '{print $1}' + register: current_file_users + when: manage_file_users + +- set_fact: users_to_remove={{ current_file_users.stdout_lines | difference ( es_users.file.keys() ) }} + when: manage_file_users + +#Remove users +- name: Remove Users + command: > + {{es_home}}/bin/shield/esusers userdel {{item}} + when: manage_file_users and (users_to_remove | length > 0) + with_items: "{{users_to_remove}}" + environment: + CONF_DIR: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + + +#Add users +- name: Add Users + command: > + {{es_home}}/bin/shield/esusers useradd {{item.key}} -p {{item.value.password}} + with_dict: "{{es_users.file}}" + when: manage_file_users and es_users.file.keys() | length > 0 + environment: + CONF_DIR: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + +#Set passwords for all users declared - Required as the useradd will not change existing user passwords +- name: Set User Passwords + command: > + {{es_home}}/bin/shield/esusers passwd {{item.key}} -p {{item.value.password}} + with_dict: "{{es_users.file}}" + when: manage_file_users and es_users.file.keys() | length > 0 + environment: + CONF_DIR: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + +- set_fact: users_roles={{es_users.file | extract_role_users}} + when: manage_file_users + +#Copy Roles files +- name: Copy roles.yml File for Instance + template: src=shield/roles.yml.j2 dest={{conf_dir}}/shield/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + when: es_roles is defined and es_roles.file is defined + +#Overwrite users_roles file +- name: Copy User Roles + template: src=shield/users_roles.j2 dest={{conf_dir}}/shield/users_roles mode=0644 force=yes + when: manage_file_users and users_roles | length > 0 + +#TODO: Support for mapping file + diff --git a/tasks/xpack/elasticsearch-shield.yml b/tasks/xpack/elasticsearch-shield.yml new file mode 100644 index 0000000..455fa2f --- /dev/null +++ b/tasks/xpack/elasticsearch-shield.yml @@ -0,0 +1,45 @@ +--- + +#Test if we need to install shield + +- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep shield" + register: shield_installed + changed_when: False + ignore_errors: yes + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +#Install Shield if not installed +- name: Install shield plugin + command: > + {{es_home}}/bin/plugin install shield + register: shield + failed_when: "'ERROR' in shield_installed.stdout" + changed_when: shield.rc == 1 + when: shield_installed.rc == 1 + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + +#TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6 + + +#Ensure shield conf directory is created +- name: Ensure shield conf directory exists + file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }} + +#-----------------------------FILE BASED REALM---------------------------------------- + +- include: elasticsearch-shield-file.yml + when: (es_users is defined and es_users.file) or (es_roles is defined and es_roles.file is defined) + +#-----------------------------NATIVE BASED REALM---------------------------------------- +# The native realm requires the node to be started so we do as a handler +- command: /bin/true + notify: load-native-realms + when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) + + diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml new file mode 100644 index 0000000..aacdf7c --- /dev/null +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -0,0 +1,33 @@ +--- +#Check if license is installed +- name: Check License is installed + shell: > + {{es_home}}/bin/plugin list | tail -n +2 | grep license + register: license_installed + ignore_errors: yes + changed_when: False + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + +#Install License if not installed +- name: Install license plugin + command: > + {{es_home}}/bin/plugin install license + register: license + failed_when: "'ERROR' in license_installed .stdout" + changed_when: license.rc == 1 + when: license_installed.rc == 1 + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +- name: Set Plugin Directory Permissions + file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes + +- include: elasticsearch-shield.yml + when: '"shield" in es_xpack_features' + +#Any other xpacks plugins requiring configuration to be entered here \ No newline at end of file diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index 203a869..40ab354 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -20,6 +20,4 @@ path.data: {{ data_dirs | array_to_str }} path.work: {{ work_dir }} -path.logs: {{ log_dir }} - -path.plugins: {{ plugin_dir }} \ No newline at end of file +path.logs: {{ log_dir }} \ No newline at end of file diff --git a/templates/shield/roles.yml.j2 b/templates/shield/roles.yml.j2 new file mode 100644 index 0000000..9f211f2 --- /dev/null +++ b/templates/shield/roles.yml.j2 @@ -0,0 +1 @@ +{{ es_roles.file | to_nice_yaml }} \ No newline at end of file diff --git a/templates/shield/users_roles.j2 b/templates/shield/users_roles.j2 new file mode 100644 index 0000000..1c0acfa --- /dev/null +++ b/templates/shield/users_roles.j2 @@ -0,0 +1 @@ +{{users_roles | join("\n") }} \ No newline at end of file diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index a52bf80..983c0a7 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -173,28 +173,7 @@ shared_examples 'multi::init' do |es_version,plugins| end end - #Multi node plugin tests - describe file('/opt/elasticsearch/plugins/node1') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/opt/elasticsearch/plugins/master') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - for plugin in plugins - describe file('/opt/elasticsearch/plugins/node1/'+plugin) do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/opt/elasticsearch/plugins/master/'+plugin) do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end describe command('curl -s localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do its(:exit_status) { should eq 0 } diff --git a/test/integration/multi.yml b/test/integration/multi.yml index e291fcb..173528c 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -8,5 +8,4 @@ vars: es_scripts: true es_templates: true - es_plugin_dir: "/opt/elasticsearch/plugins" #Plugins installed for this test are specified in .kitchen.yml under suite \ No newline at end of file diff --git a/test/integration/xpack-2x/serverspec/xpack_spec.rb b/test/integration/xpack-2x/serverspec/xpack_spec.rb new file mode 100644 index 0000000..c6a7af0 --- /dev/null +++ b/test/integration/xpack-2x/serverspec/xpack_spec.rb @@ -0,0 +1,10 @@ +require 'spec_helper' + +describe 'XPack Tests v 2.x' do + + describe user('elasticsearch') do + it { should exist } + end + +end + diff --git a/test/integration/xpack-2x/xpack.yaml b/test/integration/xpack-2x/xpack.yaml new file mode 100644 index 0000000..a3c37e1 --- /dev/null +++ b/test/integration/xpack-2x/xpack.yaml @@ -0,0 +1,2 @@ +--- +- host: test-kitchen diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml new file mode 100644 index 0000000..2c6dbcb --- /dev/null +++ b/test/integration/xpack.yml @@ -0,0 +1,68 @@ +--- +- name: Elasticsearch Xpack tests + hosts: localhost + roles: + - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "shield_node" } + vars: + es_templates: false + es_enable_xpack: true + es_xpack_features: + - shield + - watcher + es_api_basic_auth_username: es_admin + es_api_basic_auth_password: changeMe + es_users: + native: + kibana4_server: + password: changeMe + roles: + - kibana4_server + file: + es_admin: + password: changeMe + roles: + - admin + testUser: + password: changeMeAlso! + roles: + - power_user + - user + es_roles: + file: + admin: + cluster: + - all + indices: + - names: '*' + privileges: + - all + power_user: + cluster: + - monitor + indices: + - names: '*' + privileges: + - all + user: + indices: + - names: '*' + privileges: + - read + kibana4_server: + cluster: + - monitor + indices: + - names: '.kibana' + privileges: + - all + native: + logstash: + cluster: + - manage_index_templates + indices: + - names: 'logstash-*' + privileges: + - write + - delete + - create_index + From a149328ae82fb3828201e047b1982fdd254354f1 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sat, 23 Jul 2016 16:41:37 +0100 Subject: [PATCH 02/24] Test fixes + ensuring node is started for templates --- handlers/shield/elasticsearch-shield-native.yml | 4 ++++ tasks/elasticsearch-templates.yml | 7 +++++-- tasks/xpack/elasticsearch-shield-file.yml | 14 ++++++++++---- tasks/xpack/elasticsearch-shield.yml | 9 ++++----- .../config-2x/serverspec/default_spec.rb | 2 +- .../integration/helpers/serverspec/package_spec.rb | 5 ++--- .../multi-2x/serverspec/default_spec.rb | 2 +- .../package-2x/serverspec/default_spec.rb | 2 +- .../standard-2x/serverspec/default_spec.rb | 2 +- test/integration/xpack.yml | 2 +- 10 files changed, 30 insertions(+), 19 deletions(-) diff --git a/handlers/shield/elasticsearch-shield-native.yml b/handlers/shield/elasticsearch-shield-native.yml index a8f0e0f..5632bd8 100644 --- a/handlers/shield/elasticsearch-shield-native.yml +++ b/handlers/shield/elasticsearch-shield-native.yml @@ -1,4 +1,8 @@ --- + +- name: Ensure elasticsearch is started + service: name={{instance_init_script | basename}} state=started enabled=yes + - name: Wait for elasticsearch to startup wait_for: port={{es_api_port}} delay=10 diff --git a/tasks/elasticsearch-templates.yml b/tasks/elasticsearch-templates.yml index 923b699..8d5aaf0 100644 --- a/tasks/elasticsearch-templates.yml +++ b/tasks/elasticsearch-templates.yml @@ -12,13 +12,16 @@ with_fileglob: - "{{ es_templates_fileglob }}" +- name: Ensure elasticsearch is started + service: name={{instance_init_script | basename}} state=started enabled=yes + - name: Wait for elasticsearch to startup wait_for: port={{es_api_port}} delay=10 -- name: Get template files +- name: Get template files shell: find . -maxdepth 1 -type f | sed "s#\./##" | sed "s/.json//" chdir=/etc/elasticsearch/templates register: resultstemplate - name: Install template(s) command: "curl -sL -XPUT http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json" - with_items: "{{ resultstemplate.stdout_lines }}" + with_items: "{{ resultstemplate.stdout_lines }}" \ No newline at end of file diff --git a/tasks/xpack/elasticsearch-shield-file.yml b/tasks/xpack/elasticsearch-shield-file.yml index 4a266cc..932a2a9 100644 --- a/tasks/xpack/elasticsearch-shield-file.yml +++ b/tasks/xpack/elasticsearch-shield-file.yml @@ -10,8 +10,9 @@ shell: cat {{conf_dir}}/shield/users | awk -F':' '{print $1}' register: current_file_users when: manage_file_users + changed_when: False -- set_fact: users_to_remove={{ current_file_users.stdout_lines | difference ( es_users.file.keys() ) }} +- set_fact: users_to_remove={{ current_file_users.stdout_lines | difference (es_users.file.keys()) }} when: manage_file_users #Remove users @@ -25,12 +26,15 @@ ES_HOME: "{{es_home}}" +- set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }} + when: manage_file_users + #Add users - name: Add Users command: > - {{es_home}}/bin/shield/esusers useradd {{item.key}} -p {{item.value.password}} - with_dict: "{{es_users.file}}" - when: manage_file_users and es_users.file.keys() | length > 0 + {{es_home}}/bin/shield/esusers useradd {{item}} -p {{es_users.file[item].password}} + with_items: "{{users_to_add}}" + when: manage_file_users and users_to_add | length > 0 environment: CONF_DIR: "{{ conf_dir }}" ES_HOME: "{{es_home}}" @@ -41,6 +45,8 @@ {{es_home}}/bin/shield/esusers passwd {{item.key}} -p {{item.value.password}} with_dict: "{{es_users.file}}" when: manage_file_users and es_users.file.keys() | length > 0 + #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip. + changed_when: False environment: CONF_DIR: "{{ conf_dir }}" ES_HOME: "{{es_home}}" diff --git a/tasks/xpack/elasticsearch-shield.yml b/tasks/xpack/elasticsearch-shield.yml index 455fa2f..7864534 100644 --- a/tasks/xpack/elasticsearch-shield.yml +++ b/tasks/xpack/elasticsearch-shield.yml @@ -26,11 +26,6 @@ #TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6 - -#Ensure shield conf directory is created -- name: Ensure shield conf directory exists - file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }} - #-----------------------------FILE BASED REALM---------------------------------------- - include: elasticsearch-shield-file.yml @@ -43,3 +38,7 @@ when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) +#Ensure shield conf directory is created +- name: Ensure shield conf directory exists + file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }} + changed_when: False diff --git a/test/integration/config-2x/serverspec/default_spec.rb b/test/integration/config-2x/serverspec/default_spec.rb index 377fb3b..f416eed 100644 --- a/test/integration/config-2x/serverspec/default_spec.rb +++ b/test/integration/config-2x/serverspec/default_spec.rb @@ -1,6 +1,6 @@ require 'config_spec' describe 'Config Tests v 2.x' do - include_examples 'config::init', "2.2.0" + include_examples 'config::init', "2.3.4" end diff --git a/test/integration/helpers/serverspec/package_spec.rb b/test/integration/helpers/serverspec/package_spec.rb index 2d2dbbb..897135e 100644 --- a/test/integration/helpers/serverspec/package_spec.rb +++ b/test/integration/helpers/serverspec/package_spec.rb @@ -16,7 +16,6 @@ shared_examples 'package::init' do |es_version,plugins| describe file('/etc/elasticsearch/node1/elasticsearch.yml') do it { should be_file } - it { should contain 'path.plugins: /usr/share/elasticsearch/plugins/node1' } it { should contain 'http.port: 9200' } it { should contain 'transport.tcp.port: 9300' } it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9300' } @@ -66,14 +65,14 @@ shared_examples 'package::init' do |es_version,plugins| end end - describe file('/usr/share/elasticsearch/plugins/node1') do + describe file('/usr/share/elasticsearch/plugins') do it { should be_directory } it { should be_owned_by 'elasticsearch' } end for plugin in plugins - describe file('/usr/share/elasticsearch/plugins/node1/'+plugin) do + describe file('/usr/share/elasticsearch/plugins/'+plugin) do it { should be_directory } it { should be_owned_by 'elasticsearch' } end diff --git a/test/integration/multi-2x/serverspec/default_spec.rb b/test/integration/multi-2x/serverspec/default_spec.rb index 17bb9c1..6aaae25 100644 --- a/test/integration/multi-2x/serverspec/default_spec.rb +++ b/test/integration/multi-2x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'multi_spec' describe 'Multi Tests v 2.x' do - include_examples 'multi::init', "2.2.0", ["kopf","license","marvel-agent"] + include_examples 'multi::init', "2.3.4", ["kopf","license","marvel-agent"] end diff --git a/test/integration/package-2x/serverspec/default_spec.rb b/test/integration/package-2x/serverspec/default_spec.rb index bc884cc..1a4aade 100644 --- a/test/integration/package-2x/serverspec/default_spec.rb +++ b/test/integration/package-2x/serverspec/default_spec.rb @@ -2,5 +2,5 @@ require 'package_spec' describe 'Package Tests v 2.x' do - include_examples 'package::init', "2.2.0", ["kopf","license","marvel-agent"] + include_examples 'package::init', "2.3.4", ["kopf","license","marvel-agent"] end \ No newline at end of file diff --git a/test/integration/standard-2x/serverspec/default_spec.rb b/test/integration/standard-2x/serverspec/default_spec.rb index 8f45e24..86033b3 100644 --- a/test/integration/standard-2x/serverspec/default_spec.rb +++ b/test/integration/standard-2x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'standard_spec' describe 'Standard Tests v 2.x' do - include_examples 'standard::init', "2.2.0" + include_examples 'standard::init', "2.3.4" end diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 2c6dbcb..ac502bc 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -1,6 +1,6 @@ --- - name: Elasticsearch Xpack tests - hosts: localhost + hosts: localhostpost roles: - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "shield_node" } vars: From d73e515de3190a40cb86e47e1acd536eebb3c3e5 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sat, 23 Jul 2016 19:48:50 +0100 Subject: [PATCH 03/24] Test improvements for xpack + httplib2 support --- .kitchen.yml | 1 - tasks/elasticsearch-Debian.yml | 4 + tasks/elasticsearch-RedHat.yml | 4 + tasks/xpack/elasticsearch-xpack.yml | 11 +- .../elasticsearch-shield-file.yml | 0 .../{ => shield}/elasticsearch-shield.yml | 1 - .../helpers/serverspec/package_spec.rb | 2 - .../helpers/serverspec/xpack_spec.rb | 147 ++++++++++++++++++ .../xpack-2x/serverspec/default_spec.rb | 5 + .../xpack-2x/serverspec/xpack_spec.rb | 10 -- .../xpack-2x/{xpack.yaml => xpack.yml} | 0 test/integration/xpack.yml | 2 +- 12 files changed, 166 insertions(+), 21 deletions(-) rename tasks/xpack/{ => shield}/elasticsearch-shield-file.yml (100%) rename tasks/xpack/{ => shield}/elasticsearch-shield.yml (99%) create mode 100644 test/integration/helpers/serverspec/xpack_spec.rb create mode 100644 test/integration/xpack-2x/serverspec/default_spec.rb delete mode 100644 test/integration/xpack-2x/serverspec/xpack_spec.rb rename test/integration/xpack-2x/{xpack.yaml => xpack.yml} (100%) diff --git a/.kitchen.yml b/.kitchen.yml index e5172cc..99dc404 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -136,7 +136,6 @@ suites: version: latest provisioner: playbook: test/integration/multi.yml - #Currently we only test shield on 2x - name: xpack-2x run_list: attributes: diff --git a/tasks/elasticsearch-Debian.yml b/tasks/elasticsearch-Debian.yml index dba5fa5..ebda0f5 100644 --- a/tasks/elasticsearch-Debian.yml +++ b/tasks/elasticsearch-Debian.yml @@ -26,3 +26,7 @@ apt: deb=/tmp/elasticsearch-{{ es_version }}.deb when: not es_use_repository register: elasticsearch_install_from_package + +# ansible uri module requires httplib2 +- name: pip httplib2 + pip: name=httplib2 extra_args="--user" \ No newline at end of file diff --git a/tasks/elasticsearch-RedHat.yml b/tasks/elasticsearch-RedHat.yml index 0de8e71..db74280 100644 --- a/tasks/elasticsearch-RedHat.yml +++ b/tasks/elasticsearch-RedHat.yml @@ -20,3 +20,7 @@ yum: name={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.noarch.rpm{% endif %} state=present when: not es_use_repository register: elasticsearch_install_from_package + +# ansible uri module requires httplib2 +- name: pip httplib2 + pip: name=httplib2 extra_args="--user" \ No newline at end of file diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index aacdf7c..6087830 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -23,11 +23,10 @@ CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" - -- name: Set Plugin Directory Permissions - file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes - -- include: elasticsearch-shield.yml +- include: shield/elasticsearch-shield.yml when: '"shield" in es_xpack_features' -#Any other xpacks plugins requiring configuration to be entered here \ No newline at end of file +#Any other xpacks plugins requiring configuration to be entered here + +- name: Set Plugin Directory Permissions + file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file diff --git a/tasks/xpack/elasticsearch-shield-file.yml b/tasks/xpack/shield/elasticsearch-shield-file.yml similarity index 100% rename from tasks/xpack/elasticsearch-shield-file.yml rename to tasks/xpack/shield/elasticsearch-shield-file.yml diff --git a/tasks/xpack/elasticsearch-shield.yml b/tasks/xpack/shield/elasticsearch-shield.yml similarity index 99% rename from tasks/xpack/elasticsearch-shield.yml rename to tasks/xpack/shield/elasticsearch-shield.yml index 7864534..d199485 100644 --- a/tasks/xpack/elasticsearch-shield.yml +++ b/tasks/xpack/shield/elasticsearch-shield.yml @@ -37,7 +37,6 @@ notify: load-native-realms when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) - #Ensure shield conf directory is created - name: Ensure shield conf directory exists file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }} diff --git a/test/integration/helpers/serverspec/package_spec.rb b/test/integration/helpers/serverspec/package_spec.rb index 897135e..51db46e 100644 --- a/test/integration/helpers/serverspec/package_spec.rb +++ b/test/integration/helpers/serverspec/package_spec.rb @@ -26,8 +26,6 @@ shared_examples 'package::init' do |es_version,plugins| it { should be_owned_by 'elasticsearch' } end - - describe file('/etc/elasticsearch/node1/scripts/calculate-score.groovy') do it { should be_file } it { should be_owned_by 'elasticsearch' } diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb new file mode 100644 index 0000000..268dcb7 --- /dev/null +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -0,0 +1,147 @@ +require 'spec_helper' + +shared_examples 'xpack::init' do |es_version| + + describe user('elasticsearch') do + it { should exist } + end + + describe service('shield_node_elasticsearch') do + it { should be_running } + end + + describe package('elasticsearch') do + it { should be_installed } + end + + describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do + it { should be_file } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/etc/elasticsearch/shield_node/logging.yml') do + it { should be_file } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do + it { should contain 'node.name: localhost-shield_node' } + it { should contain 'cluster.name: elasticsearch' } + it { should contain 'path.conf: /etc/elasticsearch/shield_node' } + it { should contain 'path.data: /var/lib/elasticsearch/localhost-shield_node' } + it { should contain 'path.work: /tmp/elasticsearch/localhost-shield_node' } + it { should contain 'path.logs: /var/log/elasticsearch/localhost-shield_node' } + end + + describe 'Node listening' do + it 'listening in port 9200' do + expect(port 9200).to be_listening + end + end + + describe 'version check' do + it 'should be reported as version '+es_version do + command = command('curl -s localhost:9200 -u es_admin:changeMe | grep number') + expect(command.stdout).to match(es_version) + expect(command.exit_status).to eq(0) + end + end + + describe file('/etc/init.d/elasticsearch') do + it { should_not exist } + end + + describe file('/etc/default/elasticsearch') do + it { should_not exist } + end + + describe file('/etc/sysconfig/elasticsearch') do + it { should_not exist } + end + + describe file('/usr/lib/systemd/system/elasticsearch.service') do + it { should_not exist } + end + + describe file('/etc/elasticsearch/elasticsearch.yml') do + it { should_not exist } + end + + describe file('/etc/elasticsearch/logging.yml') do + it { should_not exist } + end + + #Xpack specific tests + describe file('/usr/share/elasticsearch/plugins') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + + #Check shield and license plugins are installed + describe file('/usr/share/elasticsearch/plugins/license') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep license') do + its(:exit_status) { should eq 0 } + end + + describe file('/usr/share/elasticsearch/plugins/shield') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep shield') do + its(:exit_status) { should eq 0 } + end + + describe file('/etc/elasticsearch/shield_node/shield') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + + #Test users file, users_roles and roles.yml + describe file('/etc/elasticsearch/shield_node/shield/users_roles') do + it { should be_owned_by 'elasticsearch' } + it { should contain 'admin:es_admin' } + it { should contain 'power_user:testUser' } + end + + describe file('/etc/elasticsearch/shield_node/shield/users') do + it { should be_owned_by 'elasticsearch' } + it { should contain 'testUser:' } + it { should contain 'es_admin:' } + end + + + describe file('/etc/elasticsearch/shield_node/shield/roles.yml') do + it { should be_owned_by 'elasticsearch' } + #Test contents as expected + its(:md5sum) { should eq '7800182547287abd480c8b095bf26e9e' } + end + + + #Test native roles and users are loaded + describe command('curl -s localhost:9200/_shield/user -u es_admin:changeMe | md5sum | grep 557a730df7136694131b5b7012a5ffad') do + its(:exit_status) { should eq 0 } + end + + describe command('curl -s localhost:9200/_shield/user -u es_admin:changeMe | grep "{\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{}}}"') do + its(:exit_status) { should eq 0 } + end + + describe command('curl -s localhost:9200/_shield/role -u es_admin:changeMe | grep "{\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\]}}"') do + its(:exit_status) { should eq 0 } + end + + describe command('curl -s localhost:9200/_shield/role -u es_admin:changeMe | md5sum | grep 6d14f09ef1eea64adf4d4a9c04229629') do + its(:exit_status) { should eq 0 } + end + + + #Test contents of Elasticsearch.yml file +end + diff --git a/test/integration/xpack-2x/serverspec/default_spec.rb b/test/integration/xpack-2x/serverspec/default_spec.rb new file mode 100644 index 0000000..e4ca2d8 --- /dev/null +++ b/test/integration/xpack-2x/serverspec/default_spec.rb @@ -0,0 +1,5 @@ +require 'xpack_spec' + +describe 'Xpack Tests v 2.x' do + include_examples 'xpack::init', "2.3.4" +end diff --git a/test/integration/xpack-2x/serverspec/xpack_spec.rb b/test/integration/xpack-2x/serverspec/xpack_spec.rb deleted file mode 100644 index c6a7af0..0000000 --- a/test/integration/xpack-2x/serverspec/xpack_spec.rb +++ /dev/null @@ -1,10 +0,0 @@ -require 'spec_helper' - -describe 'XPack Tests v 2.x' do - - describe user('elasticsearch') do - it { should exist } - end - -end - diff --git a/test/integration/xpack-2x/xpack.yaml b/test/integration/xpack-2x/xpack.yml similarity index 100% rename from test/integration/xpack-2x/xpack.yaml rename to test/integration/xpack-2x/xpack.yml diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index ac502bc..2c6dbcb 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -1,6 +1,6 @@ --- - name: Elasticsearch Xpack tests - hosts: localhostpost + hosts: localhost roles: - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "shield_node" } vars: From 4a86c9c482485ea290be490f1a707f17aeb442ee Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sat, 23 Jul 2016 19:51:53 +0100 Subject: [PATCH 04/24] Multi tests plugin directories correct --- test/integration/helpers/serverspec/multi_spec.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index 983c0a7..ee37abd 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -182,6 +182,11 @@ shared_examples 'multi::init' do |es_version,plugins| describe command('curl -s localhost:9201/_nodes/plugins?pretty=true | grep '+plugin) do its(:exit_status) { should eq 0 } end + + describe file('/usr/share/elasticsearch/plugins/'+plugin) do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end end describe file('/etc/init.d/elasticsearch') do From d3d9bbca1727aa4241002dbd9fb66a87ce7fec5f Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sat, 23 Jul 2016 20:18:58 +0100 Subject: [PATCH 05/24] Templates load user credentials if provided --- tasks/elasticsearch-parameters.yml | 2 +- tasks/elasticsearch-templates.yml | 10 +++++++--- test/integration/helpers/serverspec/multi_spec.rb | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 3e8281a..c05b81d 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -23,7 +23,7 @@ #Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work - fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" - when: es_enable_xpack and '"shield" in es_xpack_features' and es_api_basic_auth_username is not defined and es_api_basic_auth_username is not defined + when: es_enable_xpack and '"shield" in es_xpack_features' and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} diff --git a/tasks/elasticsearch-templates.yml b/tasks/elasticsearch-templates.yml index 8d5aaf0..7649215 100644 --- a/tasks/elasticsearch-templates.yml +++ b/tasks/elasticsearch-templates.yml @@ -22,6 +22,10 @@ shell: find . -maxdepth 1 -type f | sed "s#\./##" | sed "s/.json//" chdir=/etc/elasticsearch/templates register: resultstemplate -- name: Install template(s) - command: "curl -sL -XPUT http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json" - with_items: "{{ resultstemplate.stdout_lines }}" \ No newline at end of file +#The basic auth details here may not be required - send always if they are defined. If not needed they will be ignored. +- name: Install template(s) with auth + command: "curl -sL -XPUT http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json {% if es_api_basic_auth_username is defined and es_api_basic_auth_password is defined%}-u {{es_api_basic_auth_username}}:{{es_api_basic_auth_password}}{% endif %}" + with_items: "{{ resultstemplate.stdout_lines }}" + +#Suppose user removes shield on a running node, doesn't specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed. +#Templates should probably be done after a restart therefore - as a handler. \ No newline at end of file diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index ee37abd..4d2cc2f 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -182,7 +182,7 @@ shared_examples 'multi::init' do |es_version,plugins| describe command('curl -s localhost:9201/_nodes/plugins?pretty=true | grep '+plugin) do its(:exit_status) { should eq 0 } end - + describe file('/usr/share/elasticsearch/plugins/'+plugin) do it { should be_directory } it { should be_owned_by 'elasticsearch' } From 9a8351180182094393b2e15720b5234b3fd35ddd Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sat, 23 Jul 2016 20:23:56 +0100 Subject: [PATCH 06/24] Templates with shield tested --- .../helpers/serverspec/xpack_spec.rb | 17 +++++++++++++++++ test/integration/xpack.yml | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 268dcb7..0e9809a 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -141,6 +141,23 @@ shared_examples 'xpack::init' do |es_version| its(:exit_status) { should eq 0 } end + describe file('/etc/elasticsearch/templates') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/etc/elasticsearch/templates/basic.json') do + it { should be_file } + it { should be_owned_by 'elasticsearch' } + end + + describe 'Template Installed' do + it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do + command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe') + expect(command.stdout).to match(/basic/) + expect(command.exit_status).to eq(0) + end + end #Test contents of Elasticsearch.yml file end diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 2c6dbcb..f927ce8 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -4,7 +4,7 @@ roles: - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "shield_node" } vars: - es_templates: false + es_templates: true es_enable_xpack: true es_xpack_features: - shield From 5d3616bd201dca724deaa0b34cb8790c78d6c7ef Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sat, 23 Jul 2016 21:47:27 +0100 Subject: [PATCH 07/24] Support for removal for shield and license --- tasks/main.yml | 2 +- tasks/xpack/elasticsearch-xpack.yml | 31 +++++++++++++++------ tasks/xpack/shield/elasticsearch-shield.yml | 28 ++++++++++++++----- test/integration/xpack.yml | 1 - 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/tasks/main.yml b/tasks/main.yml index 422b0a3..cd2743b 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -27,8 +27,8 @@ when: es_plugins is defined or es_plugins_reinstall tags: - plugins + #We always execute xpack as we may need to remove features - include: xpack/elasticsearch-xpack.yml - when: es_enable_xpack tags: - xpack - include: elasticsearch-service.yml diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index 6087830..d779768 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -10,21 +10,36 @@ CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" -#Install License if not installed -- name: Install license plugin +#Remove license if installed and xpack not enabled +- name: Remove license plugin command: > - {{es_home}}/bin/plugin install license - register: license - failed_when: "'ERROR' in license_installed .stdout" - changed_when: license.rc == 1 - when: license_installed.rc == 1 + {{es_home}}/bin/plugin remove license + register: license_change + failed_when: "'ERROR' in license.stdout" + changed_when: license_change.rc == 1 + when: license_installed.rc == 0 and not es_enable_xpack notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" + +#Install License if not installed +- name: Install license plugin + command: > + {{es_home}}/bin/plugin install license + register: license_change + failed_when: "'ERROR' in license_change.stdout" + changed_when: license_change.rc == 0 + when: license_installed.rc == 1 and es_enable_xpack + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + +#Include shield as we may need to remove it - include: shield/elasticsearch-shield.yml - when: '"shield" in es_xpack_features' +# when: '"shield" in es_xpack_features' #Any other xpacks plugins requiring configuration to be entered here diff --git a/tasks/xpack/shield/elasticsearch-shield.yml b/tasks/xpack/shield/elasticsearch-shield.yml index d199485..0612033 100644 --- a/tasks/xpack/shield/elasticsearch-shield.yml +++ b/tasks/xpack/shield/elasticsearch-shield.yml @@ -1,7 +1,6 @@ --- -#Test if we need to install shield - +#Test if shield is installed - shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep shield" register: shield_installed changed_when: False @@ -11,14 +10,28 @@ ES_INCLUDE: "{{ instance_default_file }}" -#Install Shield if not installed +#Remove Shield if installed and its not been requested +- name: Remove shield plugin + command: > + {{es_home}}/bin/plugin remove shield + register: shield_change + failed_when: "'ERROR' in shield_change.stdout" + changed_when: shield_change.rc == 0 + when: shield_installed.rc == 0 and (not es_enable_xpack or not '"shield" in es_xpack_features') + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +#Install Shield if not installed and its been requested - name: Install shield plugin command: > {{es_home}}/bin/plugin install shield - register: shield - failed_when: "'ERROR' in shield_installed.stdout" - changed_when: shield.rc == 1 - when: shield_installed.rc == 1 + register: shield_change + failed_when: "'ERROR' in shield_change.stdout" + changed_when: shield_change.rc == 0 + when: shield_installed.rc == 1 and es_enable_xpack and '"shield" in es_xpack_features' notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" @@ -41,3 +54,4 @@ - name: Ensure shield conf directory exists file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }} changed_when: False + when: es_enable_xpack and '"shield" in es_xpack_features' diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index f927ce8..f6fe157 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -8,7 +8,6 @@ es_enable_xpack: true es_xpack_features: - shield - - watcher es_api_basic_auth_username: es_admin es_api_basic_auth_password: changeMe es_users: From 4f6d5b0a4a6d8883934577a4bf8e1b7cb95018eb Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sat, 23 Jul 2016 22:18:31 +0100 Subject: [PATCH 08/24] Shield now causes restart --- handlers/main.yml | 2 +- tasks/elasticsearch-plugins.yml | 4 +++- tasks/xpack/elasticsearch-xpack.yml | 12 ++++++------ tasks/xpack/shield/elasticsearch-shield.yml | 12 ++++++------ 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/handlers/main.yml b/handlers/main.yml index aa746f0..ee37b75 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -1,7 +1,7 @@ - name: restart elasticsearch service: name={{instance_init_script | basename}} state=restarted enabled=yes - when: es_restart_on_change and es_start_service and not elasticsearch_started.changed and ((plugin_installed is defined and plugin_installed.changed) or (elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) + when: es_restart_on_change and es_start_service and not elasticsearch_started.changed and ((plugin_installed is defined and plugin_installed.changed) or (xpack_state.changed) or (elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) - name: load-native-realms include: ./handlers/shield/elasticsearch-shield-native.yml diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 894533d..826ae08 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -26,6 +26,7 @@ with_items: "{{ installed_plugins.stdout_lines }}" when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0] notify: restart elasticsearch + register: plugin_installed environment: CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" @@ -46,4 +47,5 @@ #Set permissions on plugins directory - name: Set Plugin Directory Permissions - file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file + file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes + when: es_enable_xpack \ No newline at end of file diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index d779768..5ce2a27 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -14,9 +14,9 @@ - name: Remove license plugin command: > {{es_home}}/bin/plugin remove license - register: license_change - failed_when: "'ERROR' in license.stdout" - changed_when: license_change.rc == 1 + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 when: license_installed.rc == 0 and not es_enable_xpack notify: restart elasticsearch environment: @@ -28,9 +28,9 @@ - name: Install license plugin command: > {{es_home}}/bin/plugin install license - register: license_change - failed_when: "'ERROR' in license_change.stdout" - changed_when: license_change.rc == 0 + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 when: license_installed.rc == 1 and es_enable_xpack notify: restart elasticsearch environment: diff --git a/tasks/xpack/shield/elasticsearch-shield.yml b/tasks/xpack/shield/elasticsearch-shield.yml index 0612033..dad8525 100644 --- a/tasks/xpack/shield/elasticsearch-shield.yml +++ b/tasks/xpack/shield/elasticsearch-shield.yml @@ -14,9 +14,9 @@ - name: Remove shield plugin command: > {{es_home}}/bin/plugin remove shield - register: shield_change - failed_when: "'ERROR' in shield_change.stdout" - changed_when: shield_change.rc == 0 + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 when: shield_installed.rc == 0 and (not es_enable_xpack or not '"shield" in es_xpack_features') notify: restart elasticsearch environment: @@ -28,9 +28,9 @@ - name: Install shield plugin command: > {{es_home}}/bin/plugin install shield - register: shield_change - failed_when: "'ERROR' in shield_change.stdout" - changed_when: shield_change.rc == 0 + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 when: shield_installed.rc == 1 and es_enable_xpack and '"shield" in es_xpack_features' notify: restart elasticsearch environment: From 048fd636025a00379d2549c36f8b4bd271a8f832 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sat, 23 Jul 2016 22:37:22 +0100 Subject: [PATCH 09/24] Change shield versions if ES changes version --- tasks/xpack/elasticsearch-xpack.yml | 11 +++++++---- tasks/xpack/shield/elasticsearch-shield-file.yml | 6 +----- tasks/xpack/shield/elasticsearch-shield.yml | 14 ++++++++------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index 5ce2a27..c68b79a 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -1,4 +1,7 @@ --- + +- set_fact: es_version_changed=((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) + #Check if license is installed - name: Check License is installed shell: > @@ -17,27 +20,27 @@ register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 - when: license_installed.rc == 0 and not es_enable_xpack + when: license_installed.rc == 0 and (not es_enable_xpack or es_version_changed) notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" -#Install License if not installed +#Install License if not installed, or it needs to be reinstalled due to ES change (above task will have removed), and its been requested. - name: Install license plugin command: > {{es_home}}/bin/plugin install license register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 - when: license_installed.rc == 1 and es_enable_xpack + when: (license_installed.rc == 1 or es_version_changed) and es_enable_xpack notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" -#Include shield as we may need to remove it +#Include shield as we may need to remove it or change it due to es_version_changed - include: shield/elasticsearch-shield.yml # when: '"shield" in es_xpack_features' diff --git a/tasks/xpack/shield/elasticsearch-shield-file.yml b/tasks/xpack/shield/elasticsearch-shield-file.yml index 932a2a9..0746f4b 100644 --- a/tasks/xpack/shield/elasticsearch-shield-file.yml +++ b/tasks/xpack/shield/elasticsearch-shield-file.yml @@ -1,9 +1,5 @@ --- - -- set_fact: manage_file_users=false - -- set_fact: manage_file_users=true - when: es_users is defined and es_users.file is defined +- set_fact: manage_file_users=es_users is defined and es_users.file is defined #List current users - name: List Users diff --git a/tasks/xpack/shield/elasticsearch-shield.yml b/tasks/xpack/shield/elasticsearch-shield.yml index dad8525..f3bd306 100644 --- a/tasks/xpack/shield/elasticsearch-shield.yml +++ b/tasks/xpack/shield/elasticsearch-shield.yml @@ -10,28 +10,28 @@ ES_INCLUDE: "{{ instance_default_file }}" -#Remove Shield if installed and its not been requested +#Remove Shield if installed and its not been requested or the ES version has changed - name: Remove shield plugin command: > {{es_home}}/bin/plugin remove shield register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 - when: shield_installed.rc == 0 and (not es_enable_xpack or not '"shield" in es_xpack_features') + when: shield_installed.rc == 0 and (not es_enable_xpack or not '"shield" in es_xpack_features' or es_version_changed) notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" -#Install Shield if not installed and its been requested +#Install Shield if not installed, or the es version has changed (so removed above), and its been requested - name: Install shield plugin command: > {{es_home}}/bin/plugin install shield register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 - when: shield_installed.rc == 1 and es_enable_xpack and '"shield" in es_xpack_features' + when: (shield_installed.rc == 1 or es_version_changed) and es_enable_xpack and '"shield" in es_xpack_features' notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" @@ -42,13 +42,15 @@ #-----------------------------FILE BASED REALM---------------------------------------- - include: elasticsearch-shield-file.yml - when: (es_users is defined and es_users.file) or (es_roles is defined and es_roles.file is defined) + when: (es_enable_xpack and '"shield" in es_xpack_features') and ((es_users is defined and es_users.file) or (es_roles is defined and es_roles.file is defined)) #-----------------------------NATIVE BASED REALM---------------------------------------- # The native realm requires the node to be started so we do as a handler - command: /bin/true notify: load-native-realms - when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) + when: (es_enable_xpack and '"shield" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)) + +#--------------------------------------------------------------------- #Ensure shield conf directory is created - name: Ensure shield conf directory exists From 31cc54ddbcac1809177249a4f742f06bf462e943 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sun, 24 Jul 2016 01:10:07 +0100 Subject: [PATCH 10/24] Templates now invoked as handler to cover edge cases --- handlers/elasticsearch-templates.yml | 16 ++++++++++++++++ handlers/main.yml | 11 ++++++++++- tasks/elasticsearch-templates.yml | 22 +++------------------- 3 files changed, 29 insertions(+), 20 deletions(-) create mode 100644 handlers/elasticsearch-templates.yml diff --git a/handlers/elasticsearch-templates.yml b/handlers/elasticsearch-templates.yml new file mode 100644 index 0000000..9c46eed --- /dev/null +++ b/handlers/elasticsearch-templates.yml @@ -0,0 +1,16 @@ +--- + +- name: Ensure elasticsearch is started + service: name={{instance_init_script | basename}} state=started enabled=yes + +- name: Wait for elasticsearch to startup + wait_for: port={{es_api_port}} delay=10 + +- name: Get template files + shell: find . -maxdepth 1 -type f | sed "s#\./##" | sed "s/.json//" chdir=/etc/elasticsearch/templates + register: resultstemplate + +#The basic auth details here may not be required - send always if they are defined. If not needed they will be ignored. +- name: Install template(s) with auth + command: "curl -sL -XPUT http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json {% if es_api_basic_auth_username is defined and es_api_basic_auth_password is defined%}-u {{es_api_basic_auth_username}}:{{es_api_basic_auth_password}}{% endif %}" + with_items: "{{ resultstemplate.stdout_lines }}" \ No newline at end of file diff --git a/handlers/main.yml b/handlers/main.yml index ee37b75..e98d041 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -5,4 +5,13 @@ - name: load-native-realms include: ./handlers/shield/elasticsearch-shield-native.yml - when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) \ No newline at end of file + when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) + + +#Templates are a handler as they need to come after a restart e.g. suppose user removes shield on a running node and doesn't +#specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed if we don't wait for the node to restart. +#Templates done after restart therefore - as a handler. + +- name: load-templates + include: ./handlers/elasticsearch-templates.yml + when: es_templates \ No newline at end of file diff --git a/tasks/elasticsearch-templates.yml b/tasks/elasticsearch-templates.yml index 7649215..76a471d 100644 --- a/tasks/elasticsearch-templates.yml +++ b/tasks/elasticsearch-templates.yml @@ -4,28 +4,12 @@ - name: Copy default templates to elasticsearch copy: src=templates dest=/etc/elasticsearch/ owner={{ es_user }} group={{ es_group }} + notify: load-templates when: es_templates_fileglob is not defined - name: Copy templates to elasticsearch copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }} when: es_templates_fileglob is defined + notify: load-templates with_fileglob: - - "{{ es_templates_fileglob }}" - -- name: Ensure elasticsearch is started - service: name={{instance_init_script | basename}} state=started enabled=yes - -- name: Wait for elasticsearch to startup - wait_for: port={{es_api_port}} delay=10 - -- name: Get template files - shell: find . -maxdepth 1 -type f | sed "s#\./##" | sed "s/.json//" chdir=/etc/elasticsearch/templates - register: resultstemplate - -#The basic auth details here may not be required - send always if they are defined. If not needed they will be ignored. -- name: Install template(s) with auth - command: "curl -sL -XPUT http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json {% if es_api_basic_auth_username is defined and es_api_basic_auth_password is defined%}-u {{es_api_basic_auth_username}}:{{es_api_basic_auth_password}}{% endif %}" - with_items: "{{ resultstemplate.stdout_lines }}" - -#Suppose user removes shield on a running node, doesn't specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed. -#Templates should probably be done after a restart therefore - as a handler. \ No newline at end of file + - "{{ es_templates_fileglob }}" \ No newline at end of file From fdf1bda1554bcc0dcb2ebc7f1994687a540fdeb7 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sun, 24 Jul 2016 12:25:34 +0100 Subject: [PATCH 11/24] Shield config tests + prevent use of xpack in version < 2.0 --- tasks/elasticsearch-parameters.yml | 4 ++++ test/integration/helpers/serverspec/xpack_spec.rb | 7 +++++++ test/integration/xpack.yml | 4 +++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index c05b81d..3cac7cb 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -21,6 +21,10 @@ - fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified" when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined +#Don't support xpack on versions < 2.0 +- fail: msg="Use of the xpack notation is not supported on versions < 2.0. Marvel-agent and watcher can be installed as plugins. Version > 2.0 is required for shield." + when: es_enable_xpack and version_compare('2.0', '<') + #Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work - fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" when: es_enable_xpack and '"shield" in es_xpack_features' and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 0e9809a..0772435 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -160,5 +160,12 @@ shared_examples 'xpack::init' do |es_version| end #Test contents of Elasticsearch.yml file + describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do + it { should contain 'shield.authc.realms.file1.order: 0' } + it { should contain 'shield.authc.realms.file1.type: file' } + it { should contain 'shield.authc.realms.native1.order: 1' } + it { should contain 'shield.authc.realms.native1.type: native' } + end + end diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index f6fe157..ec8c7a5 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -2,7 +2,9 @@ - name: Elasticsearch Xpack tests hosts: localhost roles: - - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "shield_node" } + - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", + "shield.authc.realms.file1.type": "file","shield.authc.realms.file1.order": 0, "shield.authc.realms.native1.type": "native","shield.authc.realms.native1.order": 1 }, + es_instance_name: "shield_node" } vars: es_templates: true es_enable_xpack: true From 57fa9e432bb728141dc34d60520ea028d56613b1 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sun, 24 Jul 2016 15:25:32 +0100 Subject: [PATCH 12/24] Support for all xpack features through generic install + improved tests --- .kitchen.yml | 4 -- tasks/elasticsearch-parameters.yml | 2 +- tasks/elasticsearch-plugins.yml | 4 +- tasks/xpack/elasticsearch-xpack-install.yml | 40 +++++++++++++++++++ tasks/xpack/elasticsearch-xpack.yml | 14 ++++--- tasks/xpack/shield/elasticsearch-shield.yml | 38 +----------------- .../helpers/serverspec/xpack_spec.rb | 29 +++++++++++++- .../multi-2x/serverspec/default_spec.rb | 2 +- .../package-2x/serverspec/default_spec.rb | 2 +- test/integration/xpack.yml | 1 + vars/main.yml | 4 +- 11 files changed, 87 insertions(+), 53 deletions(-) create mode 100644 tasks/xpack/elasticsearch-xpack-install.yml diff --git a/.kitchen.yml b/.kitchen.yml index 99dc404..2b29495 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -74,8 +74,6 @@ suites: es_plugins: - plugin: lmenezes/elasticsearch-kopf version: master - - plugin: license - - plugin: marvel-agent provisioner: playbook: test/integration/package.yml - name: config-2x @@ -90,8 +88,6 @@ suites: es_plugins: - plugin: lmenezes/elasticsearch-kopf version: master - - plugin: license - - plugin: marvel-agent provisioner: playbook: test/integration/multi.yml - name: standard-1x diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 3cac7cb..0f1e8e5 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -23,7 +23,7 @@ #Don't support xpack on versions < 2.0 - fail: msg="Use of the xpack notation is not supported on versions < 2.0. Marvel-agent and watcher can be installed as plugins. Version > 2.0 is required for shield." - when: es_enable_xpack and version_compare('2.0', '<') + when: es_enable_xpack and es_version | version_compare('2.0', '<') #Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work - fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 826ae08..925f2bb 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -10,8 +10,8 @@ - set_fact: list_command="--list" when: es_version | version_compare('2.0', '<') -#List currently installed plugins -- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-" +#List currently installed plugins - ignore xpack if > v 2.0 +- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-{% if {{es_version}} | version_compare('2.0', '>') %} | grep -vE 'shield|watcher|marvel-agent|graph'{% endif %}" register: installed_plugins changed_when: False ignore_errors: yes diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml new file mode 100644 index 0000000..397aaed --- /dev/null +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -0,0 +1,40 @@ +--- + +- set_fact: es_version_changed=((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) + +#Test if feature is installed +- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep {{item}}" + register: feature_installed + changed_when: False + ignore_errors: yes + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +#Remove Plugin if installed and its not been requested or the ES version has changed +- name: Remove {{item}} plugin + command: > + {{es_home}}/bin/plugin remove shield + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 + when: feature_installed.rc == 0 and (not es_enable_xpack or not '"{{item}}" in es_xpack_features' or es_version_changed) + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +#Install plugin if not installed, or the es version has changed (so removed above), and its been requested +- name: Install {{item}} plugin + command: > + {{es_home}}/bin/plugin install {{item}} + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 + when: (feature_installed.rc == 1 or es_version_changed) and es_enable_xpack and "{{item}}" in es_xpack_features + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" \ No newline at end of file diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index c68b79a..db0a112 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -2,6 +2,8 @@ - set_fact: es_version_changed=((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) +#enabling xpack installs the license. Not a xpack feature and does not need to be specified + #Check if license is installed - name: Check License is installed shell: > @@ -26,7 +28,6 @@ CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" - #Install License if not installed, or it needs to be reinstalled due to ES change (above task will have removed), and its been requested. - name: Install license plugin command: > @@ -40,11 +41,14 @@ CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" -#Include shield as we may need to remove it or change it due to es_version_changed -- include: shield/elasticsearch-shield.yml -# when: '"shield" in es_xpack_features' +#We loop on all as we may need to remove some features +- include: elasticsearch-xpack-install.yml + with_items: "{{supported_xpack_features}}" -#Any other xpacks plugins requiring configuration to be entered here +#Shield configuration +- include: shield/elasticsearch-shield.yml + +#Add any feature specific configuration here - name: Set Plugin Directory Permissions file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file diff --git a/tasks/xpack/shield/elasticsearch-shield.yml b/tasks/xpack/shield/elasticsearch-shield.yml index f3bd306..ca16402 100644 --- a/tasks/xpack/shield/elasticsearch-shield.yml +++ b/tasks/xpack/shield/elasticsearch-shield.yml @@ -1,41 +1,5 @@ --- - -#Test if shield is installed -- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep shield" - register: shield_installed - changed_when: False - ignore_errors: yes - environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - - -#Remove Shield if installed and its not been requested or the ES version has changed -- name: Remove shield plugin - command: > - {{es_home}}/bin/plugin remove shield - register: xpack_state - failed_when: "'ERROR' in xpack_state.stdout" - changed_when: xpack_state.rc == 0 - when: shield_installed.rc == 0 and (not es_enable_xpack or not '"shield" in es_xpack_features' or es_version_changed) - notify: restart elasticsearch - environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - - -#Install Shield if not installed, or the es version has changed (so removed above), and its been requested -- name: Install shield plugin - command: > - {{es_home}}/bin/plugin install shield - register: xpack_state - failed_when: "'ERROR' in xpack_state.stdout" - changed_when: xpack_state.rc == 0 - when: (shield_installed.rc == 1 or es_version_changed) and es_enable_xpack and '"shield" in es_xpack_features' - notify: restart elasticsearch - environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" +#Shield specific configuration done here #TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6 diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 0772435..5d68f75 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -78,7 +78,7 @@ shared_examples 'xpack::init' do |es_version| end - #Check shield and license plugins are installed + #Check shield,watcher and license plugins are installed describe file('/usr/share/elasticsearch/plugins/license') do it { should be_directory } it { should be_owned_by 'elasticsearch' } @@ -102,6 +102,33 @@ shared_examples 'xpack::init' do |es_version| it { should be_owned_by 'elasticsearch' } end + describe file('/usr/share/elasticsearch/plugins/watcher') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep watcher') do + its(:exit_status) { should eq 0 } + end + + #test we haven't installed graph or marvel-agent + + describe file('/usr/share/elasticsearch/plugins/graph') do + it { should_not exist } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep graph') do + its(:exit_status) { should eq 1 } + end + + describe file('/usr/share/elasticsearch/plugins/marvel-agent') do + it { should_not exist } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep marvel-agent') do + its(:exit_status) { should eq 1 } + end + #Test users file, users_roles and roles.yml describe file('/etc/elasticsearch/shield_node/shield/users_roles') do diff --git a/test/integration/multi-2x/serverspec/default_spec.rb b/test/integration/multi-2x/serverspec/default_spec.rb index 6aaae25..81637c4 100644 --- a/test/integration/multi-2x/serverspec/default_spec.rb +++ b/test/integration/multi-2x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'multi_spec' describe 'Multi Tests v 2.x' do - include_examples 'multi::init', "2.3.4", ["kopf","license","marvel-agent"] + include_examples 'multi::init', "2.3.4", ["kopf"] end diff --git a/test/integration/package-2x/serverspec/default_spec.rb b/test/integration/package-2x/serverspec/default_spec.rb index 1a4aade..417df47 100644 --- a/test/integration/package-2x/serverspec/default_spec.rb +++ b/test/integration/package-2x/serverspec/default_spec.rb @@ -2,5 +2,5 @@ require 'package_spec' describe 'Package Tests v 2.x' do - include_examples 'package::init', "2.3.4", ["kopf","license","marvel-agent"] + include_examples 'package::init', "2.3.4", ["kopf"] end \ No newline at end of file diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index ec8c7a5..71096c0 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -10,6 +10,7 @@ es_enable_xpack: true es_xpack_features: - shield + - watcher es_api_basic_auth_username: es_admin es_api_basic_auth_password: changeMe es_users: diff --git a/vars/main.yml b/vars/main.yml index e22b2cc..fd29447 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -2,4 +2,6 @@ es_package_url: "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch" es_conf_dir: "/etc/elasticsearch" sysd_script: "/usr/lib/systemd/system/elasticsearch.service" -init_script: "/etc/init.d/elasticsearch" \ No newline at end of file +init_script: "/etc/init.d/elasticsearch" +#add supported features here +supported_xpack_features: ["watcher","marvel-agent","graph","shield"] \ No newline at end of file From 0a269a3d82a2ca57df8ec25efc7d2048642e396e Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sun, 24 Jul 2016 16:17:39 +0100 Subject: [PATCH 13/24] Fix for plugin version check --- tasks/elasticsearch-plugins.yml | 2 +- test/integration/helpers/serverspec/xpack_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 925f2bb..aa8c291 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -11,7 +11,7 @@ when: es_version | version_compare('2.0', '<') #List currently installed plugins - ignore xpack if > v 2.0 -- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-{% if {{es_version}} | version_compare('2.0', '>') %} | grep -vE 'shield|watcher|marvel-agent|graph'{% endif %}" +- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}'{% endif %}" register: installed_plugins changed_when: False ignore_errors: yes diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 5d68f75..ef871a5 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -110,7 +110,7 @@ shared_examples 'xpack::init' do |es_version| describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep watcher') do its(:exit_status) { should eq 0 } end - + #test we haven't installed graph or marvel-agent describe file('/usr/share/elasticsearch/plugins/graph') do From 9eacd3a5e0155ff5982e539f4feeee7079e5ff08 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sun, 24 Jul 2016 17:39:44 +0100 Subject: [PATCH 14/24] Fix for detecting es version changed --- tasks/elasticsearch-plugins.yml | 3 ++- tasks/xpack/elasticsearch-xpack-install.yml | 2 -- tasks/xpack/elasticsearch-xpack.yml | 9 ++++++--- test/integration/helpers/serverspec/xpack_spec.rb | 9 +++++++++ test/integration/xpack.yml | 3 +++ 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index aa8c291..63488b2 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -11,8 +11,9 @@ when: es_version | version_compare('2.0', '<') #List currently installed plugins - ignore xpack if > v 2.0 -- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}'{% endif %}" +- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}" register: installed_plugins + failed_when: "'ERROR' in installed_plugins.stdout" changed_when: False ignore_errors: yes environment: diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml index 397aaed..dfe7962 100644 --- a/tasks/xpack/elasticsearch-xpack-install.yml +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -1,7 +1,5 @@ --- -- set_fact: es_version_changed=((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) - #Test if feature is installed - shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep {{item}}" register: feature_installed diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index db0a112..d83fa38 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -1,8 +1,10 @@ --- -- set_fact: es_version_changed=((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) +- set_fact: es_version_changed={{((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed))}} -#enabling xpack installs the license. Not a xpack feature and does not need to be specified + +- debug: msg="{{es_version_changed}}" +#enabling xpack installs the license. Not a xpack feature and does not need to be specified - TODO: we should append it to the list if xpack is enabled and remove this #Check if license is installed - name: Check License is installed @@ -10,6 +12,7 @@ {{es_home}}/bin/plugin list | tail -n +2 | grep license register: license_installed ignore_errors: yes + failed_when: "'ERROR' in license_installed.stdout" changed_when: False environment: CONF_DIR: "{{ conf_dir }}" @@ -41,7 +44,7 @@ CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" -#We loop on all as we may need to remove some features +#We loop on all as we may need to remove some features. - include: elasticsearch-xpack-install.yml with_items: "{{supported_xpack_features}}" diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index ef871a5..57117e6 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -111,6 +111,15 @@ shared_examples 'xpack::init' do |es_version| its(:exit_status) { should eq 0 } end + describe file('/usr/share/elasticsearch/plugins/kopf') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep kopf') do + its(:exit_status) { should eq 0 } + end + #test we haven't installed graph or marvel-agent describe file('/usr/share/elasticsearch/plugins/graph') do diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 71096c0..fb22b60 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -8,6 +8,9 @@ vars: es_templates: true es_enable_xpack: true + es_plugins: + - plugin: lmenezes/elasticsearch-kopf + version: master es_xpack_features: - shield - watcher From 53535241370db11c77fac379f547f65fb49b1a13 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sun, 24 Jul 2016 17:48:42 +0100 Subject: [PATCH 15/24] Remove debug statements --- tasks/elasticsearch-plugins.yml | 5 +++-- tasks/xpack/elasticsearch-xpack.yml | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 63488b2..0a0f36d 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -2,8 +2,9 @@ # es_plugins_reinstall will be set to true if elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed # i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting. -- set_fact: es_plugins_reinstall=true - when: ((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none +- set_fact: es_plugins_reinstall={{ ((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none }} + +- debug: msg="{{es_plugins_reinstall}}" - set_fact: list_command="list" diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index d83fa38..46be973 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -1,9 +1,7 @@ --- -- set_fact: es_version_changed={{((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed))}} +- set_fact: es_version_changed={{ ((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) }} - -- debug: msg="{{es_version_changed}}" #enabling xpack installs the license. Not a xpack feature and does not need to be specified - TODO: we should append it to the list if xpack is enabled and remove this #Check if license is installed From 605baf9a96be9109db46ff1c162565d8f7dfde94 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sun, 24 Jul 2016 19:18:04 +0100 Subject: [PATCH 16/24] More tests --- handlers/elasticsearch-templates.yml | 26 ++++++++++++++++--- .../helpers/serverspec/xpack_spec.rb | 8 ++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/handlers/elasticsearch-templates.yml b/handlers/elasticsearch-templates.yml index 9c46eed..797ec62 100644 --- a/handlers/elasticsearch-templates.yml +++ b/handlers/elasticsearch-templates.yml @@ -10,7 +10,25 @@ shell: find . -maxdepth 1 -type f | sed "s#\./##" | sed "s/.json//" chdir=/etc/elasticsearch/templates register: resultstemplate -#The basic auth details here may not be required - send always if they are defined. If not needed they will be ignored. -- name: Install template(s) with auth - command: "curl -sL -XPUT http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json {% if es_api_basic_auth_username is defined and es_api_basic_auth_password is defined%}-u {{es_api_basic_auth_username}}:{{es_api_basic_auth_password}}{% endif %}" - with_items: "{{ resultstemplate.stdout_lines }}" \ No newline at end of file +- name: Install templates without auth + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} + method: PUT + status_code: 200 + body_format: json + body: "{{ lookup('file', '/etc/elasticsearch/templates/'+item+'.json') }}" + when: not es_enable_xpack or not es_xpack_features is defined or not '"shield" in es_xpack_features' + with_items: "{{ resultstemplate.stdout_lines }}" + +- name: Install templates with auth + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} + method: PUT + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + body_format: json + body: "{{ lookup('file', '/etc/elasticsearch/templates/'+item+'.json') }}" + when: es_enable_xpack and es_xpack_features is defined and '"shield" in es_xpack_features' + with_items: "{{ resultstemplate.stdout_lines }}" diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 57117e6..69583ed 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -195,6 +195,14 @@ shared_examples 'xpack::init' do |es_version| end end + #This is possibly subject to format changes in the response across versions so may fail in the future + describe 'Template Contents Correct' do + it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do + command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe | md5sum') + expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/) + end + end + #Test contents of Elasticsearch.yml file describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do it { should contain 'shield.authc.realms.file1.order: 0' } From 208ccada51f035999960f1d1b6cdc78e3010feb2 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 25 Jul 2016 15:40:59 +0300 Subject: [PATCH 17/24] Disable EPEL repo in kitchen CI The EPEL repo referencing download.fedoraproject.org is frequently timing out on yum install operations. Disable the EPEL and puppetlabs repo with kitchen during CI builds. --- .kitchen.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.kitchen.yml b/.kitchen.yml index 2b29495..0d2582c 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -55,6 +55,7 @@ platforms: - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config + - rm /etc/yum.repos.d/epel*repo /etc/yum.repos.d/puppetlabs-pc1.repo - yum -y install initscripts - yum clean all run_command: "/usr/sbin/init" @@ -136,4 +137,4 @@ suites: run_list: attributes: provisioner: - playbook: test/integration/xpack.yml \ No newline at end of file + playbook: test/integration/xpack.yml From e3c71a7fd656b1a990b9a3f722cbaaca53d3b384 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 29 Aug 2016 16:34:14 +0100 Subject: [PATCH 18/24] Using python-httplib2 + retries for yum delays --- handlers/shield/elasticsearch-shield-native.yml | 1 + tasks/elasticsearch-Debian.yml | 5 ++++- tasks/elasticsearch-RedHat.yml | 7 +++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/handlers/shield/elasticsearch-shield-native.yml b/handlers/shield/elasticsearch-shield-native.yml index 5632bd8..52083a5 100644 --- a/handlers/shield/elasticsearch-shield-native.yml +++ b/handlers/shield/elasticsearch-shield-native.yml @@ -72,6 +72,7 @@ uri: url: http://{{es_api_host}}:{{es_api_port}}/_shield/role method: GET + body_format: json user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes diff --git a/tasks/elasticsearch-Debian.yml b/tasks/elasticsearch-Debian.yml index ebda0f5..1457789 100644 --- a/tasks/elasticsearch-Debian.yml +++ b/tasks/elasticsearch-Debian.yml @@ -17,6 +17,9 @@ apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present force={{force_install}} cache_valid_time=86400 when: es_use_repository register: elasticsearch_install_from_repo + until: '"failed" not in result' + retries: 5 + delay: 10 - name: Debian - Download elasticsearch from url get_url: url={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.deb{% endif %} dest=/tmp/elasticsearch-{{ es_version }}.deb validate_certs=no @@ -29,4 +32,4 @@ # ansible uri module requires httplib2 - name: pip httplib2 - pip: name=httplib2 extra_args="--user" \ No newline at end of file + apt: deb=python-httplib2 \ No newline at end of file diff --git a/tasks/elasticsearch-RedHat.yml b/tasks/elasticsearch-RedHat.yml index db74280..4b124ca 100644 --- a/tasks/elasticsearch-RedHat.yml +++ b/tasks/elasticsearch-RedHat.yml @@ -15,12 +15,15 @@ yum: name=elasticsearch{% if es_version is defined and es_version != "" %}-{{ es_version }}{% endif %} state=present update_cache=yes when: es_use_repository register: elasticsearch_install_from_repo + until: '"failed" not in result' + retries: 5 + delay: 10 - name: RedHat - Install Elasticsearch from url yum: name={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.noarch.rpm{% endif %} state=present when: not es_use_repository register: elasticsearch_install_from_package -# ansible uri module requires httplib2 +# ansible uri module requires python-httplib2 - name: pip httplib2 - pip: name=httplib2 extra_args="--user" \ No newline at end of file + yum: name=python-httplib2 \ No newline at end of file From dad4ce7512cfb5cc55bd48cb7c667cded0c22c30 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 29 Aug 2016 16:51:02 +0100 Subject: [PATCH 19/24] Version changes --- README.md | 6 +++--- meta/main.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d4e82af..80be213 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Ansible role for Elasticsearch. Currently this works on Debian and RedHat based * Centos 6 * Centos 7 -The latest Elasticsearch versions of 1.7.x and 2.x are actively tested. **Only Ansible versions 2.x are supported.** +The latest Elasticsearch versions of 1.7.x and 2.x are actively tested. **Only Ansible versions > 2.1.2 are supported.** ## Usage @@ -49,7 +49,7 @@ The use of a map ensures the Ansible playbook does not need to be updated to ref In addition to the es_config map, several other parameters are supported for additional functions e.g. script installation. These can be found in the role's defaults/main.yml file. -The following illustrates applying configuration parameters to an Elasticsearch instance. By default, Elasticsearch 2.1.0 is installed. +The following illustrates applying configuration parameters to an Elasticsearch instance. By default, Elasticsearch 2.3.4 is installed. ``` - name: Elasticsearch with custom configuration @@ -282,7 +282,7 @@ To define proxy only for a particular plugin during its installation: * The role assumes the user/group exists on the server. The elasticsearch packages create the default elasticsearch user. If this needs to be changed, ensure the user exists. * The playbook relies on the inventory_name of each host to ensure its directories are unique * Changing an instance_name for a role application will result in the installation of a new component. The previous component will remain. -* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test only the latest version of each major release i.e. 1.7.3 and 2.1.0 on +* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test only the latest version of each major release i.e. 1.7.3 and 2.3.4 on all supported platforms. * The role aims to be idempotent. Running the role multiple times, with no changes, should result in no state change on the server. If the configuration is changed, these will be applied and Elasticsearch restarted where required. diff --git a/meta/main.yml b/meta/main.yml index 66df2a3..ebe4fab 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -8,7 +8,7 @@ galaxy_info: company: "Elastic.co" license: "license (Apache)" # Require 1.6 for apt deb install - min_ansible_version: 2.0 + min_ansible_version: 2.1.2 platforms: - name: EL versions: From aa284de28107eb2a9f7ee67aafd1be550a48c887 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 29 Aug 2016 17:04:12 +0100 Subject: [PATCH 20/24] Fix for variable rery --- tasks/elasticsearch-Debian.yml | 3 --- tasks/elasticsearch-RedHat.yml | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tasks/elasticsearch-Debian.yml b/tasks/elasticsearch-Debian.yml index 1457789..1bc3de0 100644 --- a/tasks/elasticsearch-Debian.yml +++ b/tasks/elasticsearch-Debian.yml @@ -17,9 +17,6 @@ apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present force={{force_install}} cache_valid_time=86400 when: es_use_repository register: elasticsearch_install_from_repo - until: '"failed" not in result' - retries: 5 - delay: 10 - name: Debian - Download elasticsearch from url get_url: url={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.deb{% endif %} dest=/tmp/elasticsearch-{{ es_version }}.deb validate_certs=no diff --git a/tasks/elasticsearch-RedHat.yml b/tasks/elasticsearch-RedHat.yml index 4b124ca..67a32a3 100644 --- a/tasks/elasticsearch-RedHat.yml +++ b/tasks/elasticsearch-RedHat.yml @@ -15,7 +15,7 @@ yum: name=elasticsearch{% if es_version is defined and es_version != "" %}-{{ es_version }}{% endif %} state=present update_cache=yes when: es_use_repository register: elasticsearch_install_from_repo - until: '"failed" not in result' + until: '"failed" not in es_use_repository' retries: 5 delay: 10 From 0aa00b477b702484811b9d1356a7e483d55195bd Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 29 Aug 2016 17:41:35 +0100 Subject: [PATCH 21/24] Plugin listed error detection improved --- tasks/elasticsearch-Debian.yml | 4 ++-- tasks/elasticsearch-RedHat.yml | 4 ++-- tasks/xpack/elasticsearch-xpack-install.yml | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tasks/elasticsearch-Debian.yml b/tasks/elasticsearch-Debian.yml index 1bc3de0..b84573e 100644 --- a/tasks/elasticsearch-Debian.yml +++ b/tasks/elasticsearch-Debian.yml @@ -28,5 +28,5 @@ register: elasticsearch_install_from_package # ansible uri module requires httplib2 -- name: pip httplib2 - apt: deb=python-httplib2 \ No newline at end of file +- name: python-httplib2 + apt: name=python-httplib2 \ No newline at end of file diff --git a/tasks/elasticsearch-RedHat.yml b/tasks/elasticsearch-RedHat.yml index 67a32a3..da058eb 100644 --- a/tasks/elasticsearch-RedHat.yml +++ b/tasks/elasticsearch-RedHat.yml @@ -15,7 +15,7 @@ yum: name=elasticsearch{% if es_version is defined and es_version != "" %}-{{ es_version }}{% endif %} state=present update_cache=yes when: es_use_repository register: elasticsearch_install_from_repo - until: '"failed" not in es_use_repository' + until: '"failed" not in elasticsearch_install_from_repo' retries: 5 delay: 10 @@ -25,5 +25,5 @@ register: elasticsearch_install_from_package # ansible uri module requires python-httplib2 -- name: pip httplib2 +- name: python-httplib2 yum: name=python-httplib2 \ No newline at end of file diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml index dfe7962..77ddc58 100644 --- a/tasks/xpack/elasticsearch-xpack-install.yml +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -4,6 +4,7 @@ - shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep {{item}}" register: feature_installed changed_when: False + failed_when: "'ERROR' in feature_installed.stdout" ignore_errors: yes environment: CONF_DIR: "{{ conf_dir }}" From fbfbb66d564e5c1696e2803f2310d5d7d866b6e4 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 30 Aug 2016 10:38:54 +0100 Subject: [PATCH 22/24] Template management with files - no nasty shell commands --- filter_plugins/custom.py | 10 +++++++++- handlers/elasticsearch-templates.yml | 16 ++++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/filter_plugins/custom.py b/filter_plugins/custom.py index ecd3b97..6ed3f2a 100644 --- a/filter_plugins/custom.py +++ b/filter_plugins/custom.py @@ -1,6 +1,7 @@ __author__ = 'dale mcdiarmid' import re +import os.path def modify_list(values=[], pattern='', replacement='', ignorecase=False): ''' Perform a `re.sub` on every item in the list''' @@ -28,9 +29,16 @@ def extract_role_users(users={}): return role_users +def filename(filename=''): + return os.path.splitext(os.path.basename(filename))[0] + + class FilterModule(object): def filters(self): return {'modify_list': modify_list, 'append_to_list':append_to_list, 'array_to_str':array_to_str, - 'extract_role_users':extract_role_users} \ No newline at end of file + 'extract_role_users':extract_role_users, + 'filename':filename} + +print filename('/etc/elasticsearch/templates/basic.json') \ No newline at end of file diff --git a/handlers/elasticsearch-templates.yml b/handlers/elasticsearch-templates.yml index 797ec62..3441d83 100644 --- a/handlers/elasticsearch-templates.yml +++ b/handlers/elasticsearch-templates.yml @@ -7,28 +7,28 @@ wait_for: port={{es_api_port}} delay=10 - name: Get template files - shell: find . -maxdepth 1 -type f | sed "s#\./##" | sed "s/.json//" chdir=/etc/elasticsearch/templates - register: resultstemplate + find: paths="/etc/elasticsearch/templates" patterns="*.json" + register: templates - name: Install templates without auth uri: - url: http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} + url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item.path | filename}}" method: PUT status_code: 200 body_format: json - body: "{{ lookup('file', '/etc/elasticsearch/templates/'+item+'.json') }}" + body: "{{ lookup('file', item.path) }}" when: not es_enable_xpack or not es_xpack_features is defined or not '"shield" in es_xpack_features' - with_items: "{{ resultstemplate.stdout_lines }}" + with_items: "{{ templates.files }}" - name: Install templates with auth uri: - url: http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} + url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item.path | filename}}" method: PUT status_code: 200 user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" force_basic_auth: yes body_format: json - body: "{{ lookup('file', '/etc/elasticsearch/templates/'+item+'.json') }}" + body: "{{ lookup('file', item.path) }}" when: es_enable_xpack and es_xpack_features is defined and '"shield" in es_xpack_features' - with_items: "{{ resultstemplate.stdout_lines }}" + with_items: "{{ templates.files }}" From 8915d5f9a533c28a97554e54a4b861e07a542675 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 30 Aug 2016 11:37:24 +0100 Subject: [PATCH 23/24] httplib2 through pip for now --- tasks/elasticsearch-RedHat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/elasticsearch-RedHat.yml b/tasks/elasticsearch-RedHat.yml index da058eb..8a9be8b 100644 --- a/tasks/elasticsearch-RedHat.yml +++ b/tasks/elasticsearch-RedHat.yml @@ -26,4 +26,4 @@ # ansible uri module requires python-httplib2 - name: python-httplib2 - yum: name=python-httplib2 \ No newline at end of file + pip: name=httplib2 \ No newline at end of file From d94808d7e24fe63ada0fb0e3e1c7ea50c7da8af6 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 30 Aug 2016 11:58:36 +0100 Subject: [PATCH 24/24] Merge remote-tracking branch 'elastic/master' # Conflicts: # meta/main.yml --- meta/main.yml | 2 +- tasks/elasticsearch-Debian.yml | 4 ++++ tasks/elasticsearch-version-lock.yml | 10 ---------- templates/init/debian/elasticsearch.j2 | 18 +++++++++--------- vars/Debian.yml | 4 ++-- 5 files changed, 16 insertions(+), 22 deletions(-) delete mode 100644 tasks/elasticsearch-version-lock.yml diff --git a/meta/main.yml b/meta/main.yml index ebe4fab..fe062a5 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -8,7 +8,7 @@ galaxy_info: company: "Elastic.co" license: "license (Apache)" # Require 1.6 for apt deb install - min_ansible_version: 2.1.2 + min_ansible_version: 2.1.0 platforms: - name: EL versions: diff --git a/tasks/elasticsearch-Debian.yml b/tasks/elasticsearch-Debian.yml index b84573e..07e3088 100644 --- a/tasks/elasticsearch-Debian.yml +++ b/tasks/elasticsearch-Debian.yml @@ -13,6 +13,10 @@ apt_repository: repo="deb http://packages.elastic.co/elasticsearch/{{ es_major_version }}/debian stable main" state=present when: es_use_repository +- name: Debian - include versionlock + include: elasticsearch-Debian-version-lock.yml + when: es_version_lock + - name: Debian - Ensure elasticsearch is installed apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present force={{force_install}} cache_valid_time=86400 when: es_use_repository diff --git a/tasks/elasticsearch-version-lock.yml b/tasks/elasticsearch-version-lock.yml deleted file mode 100644 index 7189203..0000000 --- a/tasks/elasticsearch-version-lock.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -# Trigger Debian section -- name: Include Debian specific Elasticsearch - include: elasticsearch-Debian-version-lock.yml - when: ansible_os_family == 'Debian' - -# Trigger Redhat section -- name: Include RedHat specific Elasticsearch - include: elasticsearch-RedHat-version-lock.yml - when: ansible_os_family == 'RedHat' diff --git a/templates/init/debian/elasticsearch.j2 b/templates/init/debian/elasticsearch.j2 index 1c47643..a19153e 100755 --- a/templates/init/debian/elasticsearch.j2 +++ b/templates/init/debian/elasticsearch.j2 @@ -45,11 +45,11 @@ fi # The following variables can be overwritten in $DEFAULT # Run Elasticsearch as this user ID and group ID -ES_USER=elasticsearch -ES_GROUP=elasticsearch +ES_USER={{es_user}} +ES_GROUP={{es_group}} # Directory where the Elasticsearch binary distribution resides -ES_HOME=/usr/share/$NAME +ES_HOME={{es_home}} # Heap size defaults to 256m min, 1g max # Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g @@ -71,13 +71,13 @@ MAX_OPEN_FILES=65535 #MAX_LOCKED_MEMORY= # Elasticsearch log directory -LOG_DIR=/var/log/$NAME +LOG_DIR={{log_dir}} # Elasticsearch data directory -DATA_DIR=/var/lib/$NAME +DATA_DIR={{ data_dirs | array_to_str }} # Elasticsearch configuration directory -CONF_DIR=/etc/$NAME +CONF_DIR={{conf_dir}} # Maximum number of VMA (Virtual Memory Areas) a process can own MAX_MAP_COUNT=262144 @@ -86,7 +86,7 @@ MAX_MAP_COUNT=262144 #ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log # Elasticsearch PID file directory -PID_DIR="/var/run/elasticsearch" +PID_DIR={{pid_dir}} # End of variables that can be overwritten in $DEFAULT @@ -103,7 +103,7 @@ fi # Define other required variables PID_FILE="$PID_DIR/$NAME.pid" -DAEMON=$ES_HOME/bin/elasticsearch +DAEMON={{es_home}}/bin/elasticsearch DAEMON_OPTS="-d -p $PID_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR" export ES_HEAP_SIZE @@ -227,4 +227,4 @@ case "$1" in ;; esac -exit 0 \ No newline at end of file +exit 0 diff --git a/vars/Debian.yml b/vars/Debian.yml index 7725f49..cfa73ae 100644 --- a/vars/Debian.yml +++ b/vars/Debian.yml @@ -1,4 +1,4 @@ --- -java: "{{ es_java | default('openjdk-7-jre-headless') }}" +java: "{% if es_java is defined %}{{es_java}}{%elif (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15.10', '>=')) %}openjdk-8-jre-headless{% else %}openjdk-7-jre-headless{% endif %}" default_file: "/etc/default/elasticsearch" -es_home: "/usr/share/elasticsearch" \ No newline at end of file +es_home: "/usr/share/elasticsearch"