Support for plugin diffs + improved testing on minor upgrades + fixes

.kitchen.yml

@@ -12,7 +12,6 @@ provisioner:
   http_proxy: <%= ENV['HTTP_PROXY'] %>
   https_proxy: <%= ENV['HTTPS_PROXY'] %>
   no_proxy: localhost,127.0.0.1
-  idempotency_test: true
 
 platforms:
   - name: ubuntu-14.04
@@ -67,15 +66,13 @@ platforms:
 suites:
   - name: standard-5x
     provisioner:
+      idempotency_test: true
       playbook: test/integration/standard.yml
     run_list:
     attributes:
   - name: package-5x
     run_list:
     attributes:
-      extra_vars:
-        es_plugins:
-          - plugin: ingest-geoip
     provisioner:
       playbook: test/integration/package.yml
   - name: config-5x
@@ -91,6 +88,7 @@ suites:
           - plugin: ingest-geoip
     provisioner:
       playbook: test/integration/multi.yml
+      idempotency_test: true
   - name: xpack-5x
     run_list:
     attributes:

defaults/main.yml

@@ -1,6 +1,6 @@
 ---
 es_major_version: "5.x"
-es_version: "5.1.2"
+es_version: "5.2.2"
 es_version_lock: false
 es_use_repository: true
 es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch"

tasks/elasticsearch-config.yml

@@ -47,7 +47,6 @@
   - restart elasticsearch
 
 #Copy the logging.yml
-
 - name: Copy log4j2.properties File for Instance
   template: src=log4j2.properties.j2 dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   notify: restart elasticsearch

tasks/elasticsearch-plugins.yml

@@ -5,25 +5,35 @@
 - set_fact: es_plugins_reinstall=true
   when: (((debian_elasticsearch_install_from_repo is defined and debian_elasticsearch_install_from_repo.changed) or (redhat_elasticsearch_install_from_repo is defined and redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none
 
-- set_fact: list_command="list"
+- set_fact: list_command=""
+#If we are reinstalling all plugins, e.g. to a version change, we need to remove all plugins (inc. x-pack) to install any plugins. Otherwise we don't consider x-pack so the role stays idempotent.
+- set_fact: list_command="| grep -vE 'x-pack'"
+  when: not es_plugins_reinstall
 
-#List currently installed plugins - ignore xpack if > v 2.0
+#List currently installed plugins. We have to list the directories as the list commmand fails if the ES version is different than the plugin version.
 - name: Check installed elasticsearch plugins
-  shell: "{{es_home}}/bin/elasticsearch-plugin list | grep -vE 'x-pack'"
+  shell: "ls {{es_home}}/plugins {{list_command}}"
   register: installed_plugins
-  failed_when: "'ERROR' in installed_plugins.stdout"
   changed_when: False
   ignore_errors: yes
   environment:
     CONF_DIR: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
+#if es_plugins_reinstall is set to true we remove ALL plugins
+- set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | default([]) }}"
+  when: es_plugins_reinstall
+
+#if the plugins listed are different than those requested, we remove those installed but not listed in the config
+- set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('es_plugins[*].plugin')) | default([]) }}"
+  when: not es_plugins_reinstall
+
 # This removes any currently installed plugins (to prevent errors when reinstalling)
 - name: Remove elasticsearch plugins
   command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent"
   ignore_errors: yes
-  with_items: "{{ installed_plugins.stdout_lines | default([]) }}"
-  when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0]
+  with_items: "{{ plugins_to_remove | default([]) }}"
+  when: es_plugins_reinstall and plugins_to_remove | length > 0
   notify: restart elasticsearch
   register: plugin_removed
   environment:

tasks/elasticsearch-scripts.yml

@@ -10,7 +10,7 @@
       - always
 
 - name: Create script dir
-  file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }}
+  file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes
 
 - name: Copy default scripts to elasticsearch
   copy: src=scripts dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }}

test/integration/config-5x/serverspec/default_spec.rb

@@ -1,6 +1,6 @@
 require 'config_spec'
 
 describe 'Config Tests v 5.x' do
-  include_examples 'config::init', "5.1.2"
+  include_examples 'config::init', "5.2.2", ["ingest-attachment","ingest-user-agent"]
 end
 

test/integration/config.yml

@@ -1,6 +1,6 @@
 ---
 #Test explicit setting of parameters and variables
-- name: Elasticsearch Config tests
+- name: Elasticsearch Config initial
   hosts: localhost
   roles:
     #expand to all available parameters
@@ -11,3 +11,21 @@
     es_version_lock: false
     es_heap_size: 1g
     es_api_port: 9201
+    es_plugins:
+      - plugin: docke
+
+#Modify the above configuration. Final test should evaluate this configuration. Also tests the plugins are added and removed.
+- name: Elasticsearch Config test modify
+  hosts: localhost
+  roles:
+    #expand to all available parameters
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9501", http.port: 9401, transport.tcp.port: 9501, node.data: true, node.master: true, bootstrap.memory_lock: false } }
+  vars:
+    es_scripts: false
+    es_templates: false
+    es_version_lock: false
+    es_heap_size: 1g
+    es_api_port: 9401
+    es_plugins:
+      - plugin: ingest-attachment
+      - plugin: ingest-user-agent

test/integration/helpers/serverspec/config_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-shared_examples 'config::init' do |es_version|
+shared_examples 'config::init' do |es_version,plugins|
 
   describe user('elasticsearch') do
     it { should exist }
@@ -28,14 +28,14 @@ shared_examples 'config::init' do |es_version|
 
   #test configuration parameters have been set - test all appropriately set in config file
   describe file('/etc/elasticsearch/node1/elasticsearch.yml') do
-    it { should contain 'http.port: 9201' }
-    it { should contain 'transport.tcp.port: 9301' }
-    it { should contain 'node.data: false' }
+    it { should contain 'http.port: 9401' }
+    it { should contain 'transport.tcp.port: 9501' }
+    it { should contain 'node.data: true' }
     it { should contain 'node.master: true' }
     it { should contain 'cluster.name: custom-cluster' }
     it { should contain 'node.name: node1' }
-    it { should contain 'bootstrap.memory_lock: true' }
-    it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9301' }
+    it { should contain 'bootstrap.memory_lock: false' }
+    it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9501' }
     it { should contain 'path.conf: /etc/elasticsearch/node1' }
     it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' }
     it { should contain 'path.logs: /opt/elasticsearch/logs/localhost-node1' }
@@ -63,27 +63,47 @@ shared_examples 'config::init' do |es_version|
   end
 
   #test we started on the correct port was used
-  describe command('curl -s "localhost:9201"') do
+  describe command('curl -s "localhost:9401"') do
     #TODO: This is returning an empty string
     #its(:stdout) { should match /\"status\" : 200/ }
     its(:exit_status) { should eq 0 }
   end
 
   #test to make sure mlock was applied
-  describe command('curl -s "localhost:9201/_nodes/process?pretty" | grep mlockall') do
-    its(:stdout) { should match /true/ }
+  describe command('curl -s "localhost:9401/_nodes/process?pretty" | grep mlockall') do
+    its(:stdout) { should match /false/ }
     its(:exit_status) { should eq 0 }
   end
 
 
   describe 'version check' do
     it 'should be reported as version '+es_version do
-      command = command('curl -s localhost:9201 | grep number')
+      command = command('curl -s localhost:9401 | grep number')
       expect(command.stdout).to match(es_version)
       expect(command.exit_status).to eq(0)
     end
   end
 
+  for plugin in plugins
+    describe file('/usr/share/elasticsearch/plugins/'+plugin) do
+      it { should be_directory }
+      it { should be_owned_by 'elasticsearch' }
+    end
+    #confirm plugins are installed and the correct version
+    describe command('curl -s localhost:9401/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
+      its(:exit_status) { should eq 0 }
+    end
+  end
+
+  #explit test to make sure ingest-geoip is not installed
+  describe file('/usr/share/elasticsearch/plugins/ingest-geoip') do
+    it { should_not exist }
+  end
+  #confirm plugins are installed and the correct version
+  describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"ingest-geoip","version":"'+es_version+'"\'') do
+    its(:exit_status) { should eq 1 }
+  end
+
   describe file('/etc/init.d/elasticsearch') do
     it { should_not exist }
   end

test/integration/helpers/serverspec/package_spec.rb

@@ -74,12 +74,13 @@ shared_examples 'package::init' do  |es_version,plugins|
       it { should be_directory }
       it { should be_owned_by 'elasticsearch' }
     end
-
-    describe command('curl -s localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do
+    #confirm plugins are installed and the correct version
+    describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
       its(:exit_status) { should eq 0 }
     end
   end
 
+
   describe file('/etc/init.d/elasticsearch') do
     it { should_not exist }
   end

test/integration/helpers/serverspec/xpack_spec.rb

@@ -40,7 +40,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
 
   describe 'version check' do
     it 'should be reported as version '+es_version do
-      command = command('curl -s localhost:9200 -u es_admin:changeMe | grep number')
+      command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number')
       expect(command.stdout).to match(es_version)
       expect(command.exit_status).to eq(0)
     end
@@ -79,7 +79,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
   #Test if x-pack is activated
   describe 'x-pack activation' do
     it 'should be activated and valid' do
-      command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMe')
+      command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMeAgain')
       expect(command.stdout).to match('"status" : "active"')
       expect(command.exit_status).to eq(0)
     end
@@ -90,7 +90,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
     it { should be_owned_by 'elasticsearch' }
   end
 
-  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep x-pack') do
+  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do
     its(:exit_status) { should eq 0 }
   end
 
@@ -110,7 +110,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
       it { should be_owned_by 'elasticsearch' }
     end
 
-    describe command('curl -s -u es_admin:changeMe localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do
+    describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
       its(:exit_status) { should eq 0 }
     end
   end
@@ -137,19 +137,11 @@ shared_examples 'xpack::init' do |es_version,plugins|
 
 
   #Test native roles and users are loaded
-  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | md5sum | grep 243b362bd47623c0b91a1fafbce2b6f5') do
+  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMeAgain | md5sum | grep 74bcc9f9534b253c1204e264df21496c') do
     its(:exit_status) { should eq 0 }
   end
 
-  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | grep "{\"elastic\":{\"username\":\"elastic\",\"roles\":\[\"superuser\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana\":{\"username\":\"kibana\",\"roles\":\[\"kibana\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{},\"enabled\":true}}"') do
-    its(:exit_status) { should eq 0 }
-  end
-
-  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | grep "{\"superuser\":{\"cluster\":\[\"all\"\],\"indices\":\[{\"names\":\[\"\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\"\*\"\],\"metadata\":{\"_reserved\":true}},\"transport_client\":{\"cluster\":\[\"transport_client\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"kibana_user\":{\"cluster\":\[\"monitor\"\],\"indices\":\[{\"names\":\[\".kibana\*\"\],\"privileges\":\[\"manage\",\"read\",\"index\",\"delete\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"monitoring_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\".monitoring-\*\"\],\"privileges\":\[\"read\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"remote_monitoring_agent\":{\"cluster\":\[\"manage_index_templates\",\"manage_ingest_pipelines\",\"monitor\"\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\"\.monitoring-\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"ingest_admin\":{\"cluster\":\[\"manage_index_templates\",\"manage_pipeline\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"reporting_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.reporting-\*\"\],\"privileges\":\[\"read\",\"write\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\],\"metadata\":{}}}"') do
-    its(:exit_status) { should eq 0 }
-  end
-
-  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | md5sum | grep 78a0696c9c9690042cec2c1f16860cfc') do
+  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMeAgain | md5sum | grep 2bf3ffbb9cabf26bb25de6334c4da323') do
     its(:exit_status) { should eq 0 }
   end
 
@@ -165,7 +157,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
 
   describe 'Template Installed' do
     it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
-      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe')
+      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain')
       expect(command.stdout).to match(/basic/)
       expect(command.exit_status).to eq(0)
     end
@@ -174,7 +166,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
   #This is possibly subject to format changes in the response across versions so may fail in the future
   describe 'Template Contents Correct' do
     it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
-      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe | md5sum')
+      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain | md5sum')
       expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/)
     end
   end

test/integration/multi-5x/serverspec/default_spec.rb

@@ -2,7 +2,7 @@ require 'multi_spec'
 
 
 describe 'Multi Tests v 5.x' do
-  include_examples 'multi::init', "5.1.2", ["ingest-geoip"]
+  include_examples 'multi::init', "5.2.2", ["ingest-geoip"]
 end
 
 

test/integration/package-5x/serverspec/default_spec.rb

@@ -2,5 +2,5 @@ require 'package_spec'
 
 
 describe 'Package Tests v 5.x' do
-  include_examples 'package::init', "5.1.2", ["ingest-geoip"]
+  include_examples 'package::init', "5.2.2", ["ingest-attachment","ingest-geoip"]
 end

test/integration/package.yml

@@ -1,5 +1,5 @@
 ---
-- name: Elasticsearch Package tests
+- name: Elasticsearch Package test intiial
   hosts: localhost
   roles:
     - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" }
@@ -8,4 +8,22 @@
     es_templates: true
     es_heap_size: "1g"
     es_api_port: 9200
-#Plugins installed for this test are specified in .kitchen.yml under suite
+    es_version: "5.1.2"
+    es_plugins:
+      - plugin: ingest-geoip
+
+
+#Tests the plugins have been correctly removed and ES can be upgraded between minor versions. All plugins will be removed and re-installed.
+- name: Elasticsearch Package test modify
+  hosts: localhost
+  roles:
+    - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" }
+  vars:
+    es_scripts: true
+    es_templates: true
+    es_version: "5.2.2"
+    es_heap_size: "1g"
+    es_api_port: 9200
+    es_plugins:
+      - plugin: ingest-attachment
+      - plugin: ingest-geoip

test/integration/standard-5x/serverspec/default_spec.rb

@@ -2,7 +2,7 @@ require 'standard_spec'
 
 
 describe 'Standard Tests v 5.x' do
-  include_examples 'standard::init', "5.1.2"
+  include_examples 'standard::init', "5.2.2"
 end
 
 

test/integration/standard.yml

@@ -1,8 +1,10 @@
 ---
-- name: wrapper playbook for kitchen testing "elasticsearch"
+- name: Standard test for single node setup. Tests idempotence.
   hosts: localhost
   roles:
     - { role: elasticsearch, es_instance_name: "node1" }
   vars:
     es_use_repository: "true"
     es_heap_size: "1g"
+
+#Do not add tests here. This test is run twice and confirms idempotency.

test/integration/xpack-5x/serverspec/default_spec.rb

@@ -1,5 +1,5 @@
 require 'xpack_spec'
 
 describe 'Xpack Tests v 5.x' do
-  include_examples 'xpack::init', "5.1.2", ["ingest-geoip"]
+  include_examples 'xpack::init', "5.2.2", ["ingest-attachment"]
 end

test/integration/xpack.yml

@@ -1,5 +1,5 @@
 ---
-- name: Elasticsearch Xpack tests
+- name: Elasticsearch Xpack tests initial
   hosts: localhost
   roles:
     - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300",
@@ -8,6 +8,7 @@
   vars:
     es_heap_size: "1g"
     es_templates: true
+    es_version: "5.1.2"
     es_enable_xpack: true
     es_xpack_license: "{{ lookup('file', '/tmp/license.json')  }}"
     es_plugins:
@@ -79,3 +80,45 @@
                 - delete
                 - create_index
 
+#modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed.
+- name: Elasticsearch Xpack modify
+  hosts: localhost
+  roles:
+    - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300",
+    "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 },
+    es_instance_name: "security_node" }
+  vars:
+    es_heap_size: "1g"
+    es_templates: true
+    es_version: "5.2.2"
+    es_enable_xpack: true
+    es_xpack_license: "{{ lookup('file', '/tmp/license.json')  }}"
+    es_plugins:
+      - plugin: ingest-attachment
+    es_xpack_features:
+      - security
+      - alerting
+    es_api_basic_auth_username: elastic
+    es_api_basic_auth_password: changeme
+    es_role_mapping:
+      power_user:
+        - "cn=admins,dc=example,dc=com"
+      user:
+        - "cn=users,dc=example,dc=com"
+        - "cn=admins,dc=example,dc=com"
+    es_users:
+      native:
+        kibana4_server:
+          password: changeMe
+          roles:
+            - kibana4_server
+      file:
+        es_admin:
+          password: changeMeAgain
+          roles:
+            - admin
+        testUser:
+          password: changeMeAlso!
+          roles:
+            - power_user
+            - user