From 57fa9e432bb728141dc34d60520ea028d56613b1 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Sun, 24 Jul 2016 15:25:32 +0100 Subject: [PATCH] Support for all xpack features through generic install + improved tests --- .kitchen.yml | 4 -- tasks/elasticsearch-parameters.yml | 2 +- tasks/elasticsearch-plugins.yml | 4 +- tasks/xpack/elasticsearch-xpack-install.yml | 40 +++++++++++++++++++ tasks/xpack/elasticsearch-xpack.yml | 14 ++++--- tasks/xpack/shield/elasticsearch-shield.yml | 38 +----------------- .../helpers/serverspec/xpack_spec.rb | 29 +++++++++++++- .../multi-2x/serverspec/default_spec.rb | 2 +- .../package-2x/serverspec/default_spec.rb | 2 +- test/integration/xpack.yml | 1 + vars/main.yml | 4 +- 11 files changed, 87 insertions(+), 53 deletions(-) create mode 100644 tasks/xpack/elasticsearch-xpack-install.yml diff --git a/.kitchen.yml b/.kitchen.yml index 99dc404..2b29495 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -74,8 +74,6 @@ suites: es_plugins: - plugin: lmenezes/elasticsearch-kopf version: master - - plugin: license - - plugin: marvel-agent provisioner: playbook: test/integration/package.yml - name: config-2x @@ -90,8 +88,6 @@ suites: es_plugins: - plugin: lmenezes/elasticsearch-kopf version: master - - plugin: license - - plugin: marvel-agent provisioner: playbook: test/integration/multi.yml - name: standard-1x diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 3cac7cb..0f1e8e5 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -23,7 +23,7 @@ #Don't support xpack on versions < 2.0 - fail: msg="Use of the xpack notation is not supported on versions < 2.0. Marvel-agent and watcher can be installed as plugins. Version > 2.0 is required for shield." - when: es_enable_xpack and version_compare('2.0', '<') + when: es_enable_xpack and es_version | version_compare('2.0', '<') #Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work - fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 826ae08..925f2bb 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -10,8 +10,8 @@ - set_fact: list_command="--list" when: es_version | version_compare('2.0', '<') -#List currently installed plugins -- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-" +#List currently installed plugins - ignore xpack if > v 2.0 +- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-{% if {{es_version}} | version_compare('2.0', '>') %} | grep -vE 'shield|watcher|marvel-agent|graph'{% endif %}" register: installed_plugins changed_when: False ignore_errors: yes diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml new file mode 100644 index 0000000..397aaed --- /dev/null +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -0,0 +1,40 @@ +--- + +- set_fact: es_version_changed=((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) + +#Test if feature is installed +- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep {{item}}" + register: feature_installed + changed_when: False + ignore_errors: yes + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +#Remove Plugin if installed and its not been requested or the ES version has changed +- name: Remove {{item}} plugin + command: > + {{es_home}}/bin/plugin remove shield + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 + when: feature_installed.rc == 0 and (not es_enable_xpack or not '"{{item}}" in es_xpack_features' or es_version_changed) + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +#Install plugin if not installed, or the es version has changed (so removed above), and its been requested +- name: Install {{item}} plugin + command: > + {{es_home}}/bin/plugin install {{item}} + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 + when: (feature_installed.rc == 1 or es_version_changed) and es_enable_xpack and "{{item}}" in es_xpack_features + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" \ No newline at end of file diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index c68b79a..db0a112 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -2,6 +2,8 @@ - set_fact: es_version_changed=((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) +#enabling xpack installs the license. Not a xpack feature and does not need to be specified + #Check if license is installed - name: Check License is installed shell: > @@ -26,7 +28,6 @@ CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" - #Install License if not installed, or it needs to be reinstalled due to ES change (above task will have removed), and its been requested. - name: Install license plugin command: > @@ -40,11 +41,14 @@ CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" -#Include shield as we may need to remove it or change it due to es_version_changed -- include: shield/elasticsearch-shield.yml -# when: '"shield" in es_xpack_features' +#We loop on all as we may need to remove some features +- include: elasticsearch-xpack-install.yml + with_items: "{{supported_xpack_features}}" -#Any other xpacks plugins requiring configuration to be entered here +#Shield configuration +- include: shield/elasticsearch-shield.yml + +#Add any feature specific configuration here - name: Set Plugin Directory Permissions file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file diff --git a/tasks/xpack/shield/elasticsearch-shield.yml b/tasks/xpack/shield/elasticsearch-shield.yml index f3bd306..ca16402 100644 --- a/tasks/xpack/shield/elasticsearch-shield.yml +++ b/tasks/xpack/shield/elasticsearch-shield.yml @@ -1,41 +1,5 @@ --- - -#Test if shield is installed -- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep shield" - register: shield_installed - changed_when: False - ignore_errors: yes - environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - - -#Remove Shield if installed and its not been requested or the ES version has changed -- name: Remove shield plugin - command: > - {{es_home}}/bin/plugin remove shield - register: xpack_state - failed_when: "'ERROR' in xpack_state.stdout" - changed_when: xpack_state.rc == 0 - when: shield_installed.rc == 0 and (not es_enable_xpack or not '"shield" in es_xpack_features' or es_version_changed) - notify: restart elasticsearch - environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - - -#Install Shield if not installed, or the es version has changed (so removed above), and its been requested -- name: Install shield plugin - command: > - {{es_home}}/bin/plugin install shield - register: xpack_state - failed_when: "'ERROR' in xpack_state.stdout" - changed_when: xpack_state.rc == 0 - when: (shield_installed.rc == 1 or es_version_changed) and es_enable_xpack and '"shield" in es_xpack_features' - notify: restart elasticsearch - environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" +#Shield specific configuration done here #TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6 diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 0772435..5d68f75 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -78,7 +78,7 @@ shared_examples 'xpack::init' do |es_version| end - #Check shield and license plugins are installed + #Check shield,watcher and license plugins are installed describe file('/usr/share/elasticsearch/plugins/license') do it { should be_directory } it { should be_owned_by 'elasticsearch' } @@ -102,6 +102,33 @@ shared_examples 'xpack::init' do |es_version| it { should be_owned_by 'elasticsearch' } end + describe file('/usr/share/elasticsearch/plugins/watcher') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep watcher') do + its(:exit_status) { should eq 0 } + end + + #test we haven't installed graph or marvel-agent + + describe file('/usr/share/elasticsearch/plugins/graph') do + it { should_not exist } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep graph') do + its(:exit_status) { should eq 1 } + end + + describe file('/usr/share/elasticsearch/plugins/marvel-agent') do + it { should_not exist } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep marvel-agent') do + its(:exit_status) { should eq 1 } + end + #Test users file, users_roles and roles.yml describe file('/etc/elasticsearch/shield_node/shield/users_roles') do diff --git a/test/integration/multi-2x/serverspec/default_spec.rb b/test/integration/multi-2x/serverspec/default_spec.rb index 6aaae25..81637c4 100644 --- a/test/integration/multi-2x/serverspec/default_spec.rb +++ b/test/integration/multi-2x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'multi_spec' describe 'Multi Tests v 2.x' do - include_examples 'multi::init', "2.3.4", ["kopf","license","marvel-agent"] + include_examples 'multi::init', "2.3.4", ["kopf"] end diff --git a/test/integration/package-2x/serverspec/default_spec.rb b/test/integration/package-2x/serverspec/default_spec.rb index 1a4aade..417df47 100644 --- a/test/integration/package-2x/serverspec/default_spec.rb +++ b/test/integration/package-2x/serverspec/default_spec.rb @@ -2,5 +2,5 @@ require 'package_spec' describe 'Package Tests v 2.x' do - include_examples 'package::init', "2.3.4", ["kopf","license","marvel-agent"] + include_examples 'package::init', "2.3.4", ["kopf"] end \ No newline at end of file diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index ec8c7a5..71096c0 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -10,6 +10,7 @@ es_enable_xpack: true es_xpack_features: - shield + - watcher es_api_basic_auth_username: es_admin es_api_basic_auth_password: changeMe es_users: diff --git a/vars/main.yml b/vars/main.yml index e22b2cc..fd29447 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -2,4 +2,6 @@ es_package_url: "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch" es_conf_dir: "/etc/elasticsearch" sysd_script: "/usr/lib/systemd/system/elasticsearch.service" -init_script: "/etc/init.d/elasticsearch" \ No newline at end of file +init_script: "/etc/init.d/elasticsearch" +#add supported features here +supported_xpack_features: ["watcher","marvel-agent","graph","shield"] \ No newline at end of file