Support for all xpack features through generic install + improved tests

.kitchen.yml

@@ -74,8 +74,6 @@ suites:
         es_plugins:
           - plugin: lmenezes/elasticsearch-kopf
             version: master
-          - plugin: license
-          - plugin: marvel-agent
     provisioner:
       playbook: test/integration/package.yml
   - name: config-2x
@@ -90,8 +88,6 @@ suites:
         es_plugins:
           - plugin: lmenezes/elasticsearch-kopf
             version: master
-          - plugin: license
-          - plugin: marvel-agent
     provisioner:
       playbook: test/integration/multi.yml
   - name: standard-1x

tasks/elasticsearch-parameters.yml

@@ -23,7 +23,7 @@
 
 #Don't support xpack on versions < 2.0
 - fail: msg="Use of the xpack notation is not supported on versions < 2.0.  Marvel-agent and watcher can be installed as plugins.  Version > 2.0 is required for shield."
-  when: es_enable_xpack and version_compare('2.0', '<')
+  when: es_enable_xpack and es_version | version_compare('2.0', '<')
 
 #Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work
 - fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations"

tasks/elasticsearch-plugins.yml

@@ -10,8 +10,8 @@
 - set_fact: list_command="--list"
   when: es_version | version_compare('2.0', '<')
 
-#List currently installed plugins
+#List currently installed plugins - ignore xpack if > v 2.0
-- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-"
+- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-{% if {{es_version}} | version_compare('2.0', '>') %} | grep -vE 'shield|watcher|marvel-agent|graph'{% endif %}"
   register: installed_plugins
   changed_when: False
   ignore_errors: yes

tasks/xpack/elasticsearch-xpack-install.yml

@@ -0,0 +1,40 @@
+---
+
+- set_fact: es_version_changed=((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed))
+
+#Test if feature is installed
+- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep {{item}}"
+  register: feature_installed
+  changed_when: False
+  ignore_errors: yes
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_INCLUDE: "{{ instance_default_file }}"
+
+
+#Remove Plugin if installed and its not been requested or the ES version has changed
+- name: Remove {{item}} plugin
+  command: >
+    {{es_home}}/bin/plugin remove shield
+  register: xpack_state
+  failed_when: "'ERROR' in xpack_state.stdout"
+  changed_when: xpack_state.rc == 0
+  when: feature_installed.rc == 0 and (not es_enable_xpack or not '"{{item}}" in es_xpack_features' or es_version_changed)
+  notify: restart elasticsearch
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_INCLUDE: "{{ instance_default_file }}"
+
+
+#Install plugin if not installed, or the es version has changed (so removed above), and its been requested
+- name: Install {{item}} plugin
+  command: >
+    {{es_home}}/bin/plugin install {{item}}
+  register: xpack_state
+  failed_when: "'ERROR' in xpack_state.stdout"
+  changed_when: xpack_state.rc == 0
+  when: (feature_installed.rc == 1 or es_version_changed) and es_enable_xpack and "{{item}}" in es_xpack_features
+  notify: restart elasticsearch
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_INCLUDE: "{{ instance_default_file }}"

tasks/xpack/elasticsearch-xpack.yml

@@ -2,6 +2,8 @@
 
 - set_fact: es_version_changed=((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed))
 
+#enabling xpack installs the license. Not a xpack feature and does not need to be specified
+
 #Check if license is installed
 - name: Check License is installed
   shell: >
@@ -26,7 +28,6 @@
     CONF_DIR: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
-
 #Install License if not installed, or it needs to be reinstalled due to ES change (above task will have removed), and its been requested.
 - name: Install license plugin
   command: >
@@ -40,11 +41,14 @@
     CONF_DIR: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
-#Include shield as we may need to remove it or change it due to es_version_changed
+#We loop on all as we may need to remove some features
-- include: shield/elasticsearch-shield.yml
+- include: elasticsearch-xpack-install.yml
-#  when: '"shield" in es_xpack_features'
+  with_items: "{{supported_xpack_features}}"
 
-#Any other xpacks plugins requiring configuration to be entered here
+#Shield configuration
+- include: shield/elasticsearch-shield.yml
+
+#Add any feature specific configuration here
 
 - name: Set Plugin Directory Permissions
   file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes

tasks/xpack/shield/elasticsearch-shield.yml

@@ -1,41 +1,5 @@
 ---
-
+#Shield specific configuration done here
-#Test if shield is installed
-- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep shield"
-  register: shield_installed
-  changed_when: False
-  ignore_errors: yes
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-
-
-#Remove Shield if installed and its not been requested or the ES version has changed
-- name: Remove shield plugin
-  command: >
-    {{es_home}}/bin/plugin remove shield
-  register: xpack_state
-  failed_when: "'ERROR' in xpack_state.stdout"
-  changed_when: xpack_state.rc == 0
-  when: shield_installed.rc == 0 and (not es_enable_xpack or not '"shield" in es_xpack_features' or es_version_changed)
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-
-
-#Install Shield if not installed, or the es version has changed (so removed above), and its been requested
-- name: Install shield plugin
-  command: >
-    {{es_home}}/bin/plugin install shield
-  register: xpack_state
-  failed_when: "'ERROR' in xpack_state.stdout"
-  changed_when: xpack_state.rc == 0
-  when: (shield_installed.rc == 1 or es_version_changed) and es_enable_xpack and '"shield" in es_xpack_features'
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
 
 #TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6
 

test/integration/helpers/serverspec/xpack_spec.rb

@@ -78,7 +78,7 @@ shared_examples 'xpack::init' do |es_version|
   end
 
 
-  #Check shield and license plugins are installed
+  #Check shield,watcher and license plugins are installed
   describe file('/usr/share/elasticsearch/plugins/license') do
     it { should be_directory }
     it { should be_owned_by 'elasticsearch' }
@@ -102,6 +102,33 @@ shared_examples 'xpack::init' do |es_version|
     it { should be_owned_by 'elasticsearch' }
   end
 
+  describe file('/usr/share/elasticsearch/plugins/watcher') do
+    it { should be_directory }
+    it { should be_owned_by 'elasticsearch' }
+  end
+
+  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep watcher') do
+    its(:exit_status) { should eq 0 }
+  end
+  
+  #test we haven't installed graph or marvel-agent
+
+  describe file('/usr/share/elasticsearch/plugins/graph') do
+    it { should_not exist }
+  end
+
+  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep graph') do
+    its(:exit_status) { should eq 1 }
+  end
+
+  describe file('/usr/share/elasticsearch/plugins/marvel-agent') do
+    it { should_not exist }
+  end
+
+  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep marvel-agent') do
+    its(:exit_status) { should eq 1 }
+  end
+
 
   #Test users file, users_roles and roles.yml
   describe file('/etc/elasticsearch/shield_node/shield/users_roles') do

test/integration/multi-2x/serverspec/default_spec.rb

@@ -2,7 +2,7 @@ require 'multi_spec'
 
 
 describe 'Multi Tests v 2.x' do
-  include_examples 'multi::init', "2.3.4", ["kopf","license","marvel-agent"]
+  include_examples 'multi::init', "2.3.4", ["kopf"]
 end
 
 

test/integration/package-2x/serverspec/default_spec.rb

@@ -2,5 +2,5 @@ require 'package_spec'
 
 
 describe 'Package Tests v 2.x' do
-  include_examples 'package::init', "2.3.4", ["kopf","license","marvel-agent"]
+  include_examples 'package::init', "2.3.4", ["kopf"]
 end

test/integration/xpack.yml

@@ -10,6 +10,7 @@
     es_enable_xpack: true
     es_xpack_features:
       - shield
+      - watcher
     es_api_basic_auth_username: es_admin
     es_api_basic_auth_password: changeMe
     es_users:

vars/main.yml

@@ -2,4 +2,6 @@
 es_package_url: "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch"
 es_conf_dir: "/etc/elasticsearch"
 sysd_script: "/usr/lib/systemd/system/elasticsearch.service"
-init_script: "/etc/init.d/elasticsearch"
+init_script: "/etc/init.d/elasticsearch"
+#add supported features here
+supported_xpack_features: ["watcher","marvel-agent","graph","shield"]