From 1bc817a378ec2f5c860f4b23f2e4ab22f154afe4 Mon Sep 17 00:00:00 2001
From: Dale McDiarmid <dalem@elastic.co>
Date: Thu, 16 Mar 2017 21:06:17 +0000
Subject: [PATCH 1/5] Support for plugin diffs + improved testing on minor
 upgrades + fixes

---
 .kitchen.yml                                  |  8 ++--
 defaults/main.yml                             |  2 +-
 tasks/elasticsearch-config.yml                |  1 -
 tasks/elasticsearch-plugins.yml               | 22 ++++++---
 tasks/elasticsearch-scripts.yml               |  2 +-
 .../config-5x/serverspec/default_spec.rb      |  2 +-
 test/integration/config.yml                   | 22 ++++++++-
 .../helpers/serverspec/config_spec.rb         | 40 ++++++++++++-----
 .../helpers/serverspec/package_spec.rb        |  5 ++-
 .../helpers/serverspec/xpack_spec.rb          | 24 ++++------
 .../multi-5x/serverspec/default_spec.rb       |  2 +-
 .../package-5x/serverspec/default_spec.rb     |  2 +-
 test/integration/package.yml                  | 22 ++++++++-
 .../standard-5x/serverspec/default_spec.rb    |  2 +-
 test/integration/standard.yml                 |  6 ++-
 .../xpack-5x/serverspec/default_spec.rb       |  2 +-
 test/integration/xpack.yml                    | 45 ++++++++++++++++++-
 17 files changed, 155 insertions(+), 54 deletions(-)

diff --git a/.kitchen.yml b/.kitchen.yml
index 0bd7a15..044a77a 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -12,7 +12,6 @@ provisioner:
   http_proxy: <%= ENV['HTTP_PROXY'] %>
   https_proxy: <%= ENV['HTTPS_PROXY'] %>
   no_proxy: localhost,127.0.0.1
-  idempotency_test: true
 
 platforms:
   - name: ubuntu-14.04
@@ -67,15 +66,13 @@ platforms:
 suites:
   - name: standard-5x
     provisioner:
+      idempotency_test: true
       playbook: test/integration/standard.yml
     run_list:
     attributes:
   - name: package-5x
     run_list:
     attributes:
-      extra_vars:
-        es_plugins:
-          - plugin: ingest-geoip
     provisioner:
       playbook: test/integration/package.yml
   - name: config-5x
@@ -91,8 +88,9 @@ suites:
           - plugin: ingest-geoip
     provisioner:
       playbook: test/integration/multi.yml
+      idempotency_test: true
   - name: xpack-5x
     run_list:
     attributes:
     provisioner:
-      playbook: test/integration/xpack.yml
+      playbook: test/integration/xpack.yml
\ No newline at end of file
diff --git a/defaults/main.yml b/defaults/main.yml
index 62dec45..59f01af 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -1,6 +1,6 @@
 ---
 es_major_version: "5.x"
-es_version: "5.1.2"
+es_version: "5.2.2"
 es_version_lock: false
 es_use_repository: true
 es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch"
diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml
index 9eeb5d2..6debfaf 100644
--- a/tasks/elasticsearch-config.yml
+++ b/tasks/elasticsearch-config.yml
@@ -47,7 +47,6 @@
   - restart elasticsearch
 
 #Copy the logging.yml
-
 - name: Copy log4j2.properties File for Instance
   template: src=log4j2.properties.j2 dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   notify: restart elasticsearch
diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml
index eab6109..5957783 100644
--- a/tasks/elasticsearch-plugins.yml
+++ b/tasks/elasticsearch-plugins.yml
@@ -5,25 +5,35 @@
 - set_fact: es_plugins_reinstall=true
   when: (((debian_elasticsearch_install_from_repo is defined and debian_elasticsearch_install_from_repo.changed) or (redhat_elasticsearch_install_from_repo is defined and redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none
 
-- set_fact: list_command="list"
+- set_fact: list_command=""
+#If we are reinstalling all plugins, e.g. to a version change, we need to remove all plugins (inc. x-pack) to install any plugins. Otherwise we don't consider x-pack so the role stays idempotent.
+- set_fact: list_command="| grep -vE 'x-pack'"
+  when: not es_plugins_reinstall
 
-#List currently installed plugins - ignore xpack if > v 2.0
+#List currently installed plugins. We have to list the directories as the list commmand fails if the ES version is different than the plugin version.
 - name: Check installed elasticsearch plugins
-  shell: "{{es_home}}/bin/elasticsearch-plugin list | grep -vE 'x-pack'"
+  shell: "ls {{es_home}}/plugins {{list_command}}"
   register: installed_plugins
-  failed_when: "'ERROR' in installed_plugins.stdout"
   changed_when: False
   ignore_errors: yes
   environment:
     CONF_DIR: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
 
+#if es_plugins_reinstall is set to true we remove ALL plugins
+- set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | default([]) }}"
+  when: es_plugins_reinstall
+
+#if the plugins listed are different than those requested, we remove those installed but not listed in the config
+- set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('es_plugins[*].plugin')) | default([]) }}"
+  when: not es_plugins_reinstall
+
 # This removes any currently installed plugins (to prevent errors when reinstalling)
 - name: Remove elasticsearch plugins
   command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent"
   ignore_errors: yes
-  with_items: "{{ installed_plugins.stdout_lines | default([]) }}"
-  when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0]
+  with_items: "{{ plugins_to_remove | default([]) }}"
+  when: es_plugins_reinstall and plugins_to_remove | length > 0
   notify: restart elasticsearch
   register: plugin_removed
   environment:
diff --git a/tasks/elasticsearch-scripts.yml b/tasks/elasticsearch-scripts.yml
index 40910c2..30867f5 100644
--- a/tasks/elasticsearch-scripts.yml
+++ b/tasks/elasticsearch-scripts.yml
@@ -10,7 +10,7 @@
       - always
 
 - name: Create script dir
-  file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }}
+  file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes
 
 - name: Copy default scripts to elasticsearch
   copy: src=scripts dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }}
diff --git a/test/integration/config-5x/serverspec/default_spec.rb b/test/integration/config-5x/serverspec/default_spec.rb
index 72b83d9..785614b 100644
--- a/test/integration/config-5x/serverspec/default_spec.rb
+++ b/test/integration/config-5x/serverspec/default_spec.rb
@@ -1,6 +1,6 @@
 require 'config_spec'
 
 describe 'Config Tests v 5.x' do
-  include_examples 'config::init', "5.1.2"
+  include_examples 'config::init', "5.2.2", ["ingest-attachment","ingest-user-agent"]
 end
 
diff --git a/test/integration/config.yml b/test/integration/config.yml
index 5c4d3d4..d4741db 100644
--- a/test/integration/config.yml
+++ b/test/integration/config.yml
@@ -1,6 +1,6 @@
 ---
 #Test explicit setting of parameters and variables
-- name: Elasticsearch Config tests
+- name: Elasticsearch Config initial
   hosts: localhost
   roles:
     #expand to all available parameters
@@ -10,4 +10,22 @@
     es_templates: false
     es_version_lock: false
     es_heap_size: 1g
-    es_api_port: 9201
\ No newline at end of file
+    es_api_port: 9201
+    es_plugins:
+      - plugin: docke
+
+#Modify the above configuration. Final test should evaluate this configuration. Also tests the plugins are added and removed.
+- name: Elasticsearch Config test modify
+  hosts: localhost
+  roles:
+    #expand to all available parameters
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9501", http.port: 9401, transport.tcp.port: 9501, node.data: true, node.master: true, bootstrap.memory_lock: false } }
+  vars:
+    es_scripts: false
+    es_templates: false
+    es_version_lock: false
+    es_heap_size: 1g
+    es_api_port: 9401
+    es_plugins:
+      - plugin: ingest-attachment
+      - plugin: ingest-user-agent
\ No newline at end of file
diff --git a/test/integration/helpers/serverspec/config_spec.rb b/test/integration/helpers/serverspec/config_spec.rb
index aca7f1a..c218104 100644
--- a/test/integration/helpers/serverspec/config_spec.rb
+++ b/test/integration/helpers/serverspec/config_spec.rb
@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-shared_examples 'config::init' do |es_version|
+shared_examples 'config::init' do |es_version,plugins|
 
   describe user('elasticsearch') do
     it { should exist }
@@ -28,14 +28,14 @@ shared_examples 'config::init' do |es_version|
 
   #test configuration parameters have been set - test all appropriately set in config file
   describe file('/etc/elasticsearch/node1/elasticsearch.yml') do
-    it { should contain 'http.port: 9201' }
-    it { should contain 'transport.tcp.port: 9301' }
-    it { should contain 'node.data: false' }
+    it { should contain 'http.port: 9401' }
+    it { should contain 'transport.tcp.port: 9501' }
+    it { should contain 'node.data: true' }
     it { should contain 'node.master: true' }
     it { should contain 'cluster.name: custom-cluster' }
     it { should contain 'node.name: node1' }
-    it { should contain 'bootstrap.memory_lock: true' }
-    it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9301' }
+    it { should contain 'bootstrap.memory_lock: false' }
+    it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9501' }
     it { should contain 'path.conf: /etc/elasticsearch/node1' }
     it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' }
     it { should contain 'path.logs: /opt/elasticsearch/logs/localhost-node1' }
@@ -63,27 +63,47 @@ shared_examples 'config::init' do |es_version|
   end
 
   #test we started on the correct port was used
-  describe command('curl -s "localhost:9201"') do
+  describe command('curl -s "localhost:9401"') do
     #TODO: This is returning an empty string
     #its(:stdout) { should match /\"status\" : 200/ }
     its(:exit_status) { should eq 0 }
   end
 
   #test to make sure mlock was applied
-  describe command('curl -s "localhost:9201/_nodes/process?pretty" | grep mlockall') do
-    its(:stdout) { should match /true/ }
+  describe command('curl -s "localhost:9401/_nodes/process?pretty" | grep mlockall') do
+    its(:stdout) { should match /false/ }
     its(:exit_status) { should eq 0 }
   end
 
 
   describe 'version check' do
     it 'should be reported as version '+es_version do
-      command = command('curl -s localhost:9201 | grep number')
+      command = command('curl -s localhost:9401 | grep number')
       expect(command.stdout).to match(es_version)
       expect(command.exit_status).to eq(0)
     end
   end
 
+  for plugin in plugins
+    describe file('/usr/share/elasticsearch/plugins/'+plugin) do
+      it { should be_directory }
+      it { should be_owned_by 'elasticsearch' }
+    end
+    #confirm plugins are installed and the correct version
+    describe command('curl -s localhost:9401/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
+      its(:exit_status) { should eq 0 }
+    end
+  end
+
+  #explit test to make sure ingest-geoip is not installed
+  describe file('/usr/share/elasticsearch/plugins/ingest-geoip') do
+    it { should_not exist }
+  end
+  #confirm plugins are installed and the correct version
+  describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"ingest-geoip","version":"'+es_version+'"\'') do
+    its(:exit_status) { should eq 1 }
+  end
+
   describe file('/etc/init.d/elasticsearch') do
     it { should_not exist }
   end
diff --git a/test/integration/helpers/serverspec/package_spec.rb b/test/integration/helpers/serverspec/package_spec.rb
index 51db46e..ac68bc2 100644
--- a/test/integration/helpers/serverspec/package_spec.rb
+++ b/test/integration/helpers/serverspec/package_spec.rb
@@ -74,12 +74,13 @@ shared_examples 'package::init' do  |es_version,plugins|
       it { should be_directory }
       it { should be_owned_by 'elasticsearch' }
     end
-
-    describe command('curl -s localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do
+    #confirm plugins are installed and the correct version
+    describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
       its(:exit_status) { should eq 0 }
     end
   end
 
+
   describe file('/etc/init.d/elasticsearch') do
     it { should_not exist }
   end
diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb
index 87365b4..08a7cac 100644
--- a/test/integration/helpers/serverspec/xpack_spec.rb
+++ b/test/integration/helpers/serverspec/xpack_spec.rb
@@ -40,7 +40,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
 
   describe 'version check' do
     it 'should be reported as version '+es_version do
-      command = command('curl -s localhost:9200 -u es_admin:changeMe | grep number')
+      command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number')
       expect(command.stdout).to match(es_version)
       expect(command.exit_status).to eq(0)
     end
@@ -79,7 +79,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
   #Test if x-pack is activated
   describe 'x-pack activation' do
     it 'should be activated and valid' do
-      command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMe')
+      command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMeAgain')
       expect(command.stdout).to match('"status" : "active"')
       expect(command.exit_status).to eq(0)
     end
@@ -90,7 +90,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
     it { should be_owned_by 'elasticsearch' }
   end
 
-  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep x-pack') do
+  describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do
     its(:exit_status) { should eq 0 }
   end
 
@@ -110,7 +110,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
       it { should be_owned_by 'elasticsearch' }
     end
 
-    describe command('curl -s -u es_admin:changeMe localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do
+    describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do
       its(:exit_status) { should eq 0 }
     end
   end
@@ -137,19 +137,11 @@ shared_examples 'xpack::init' do |es_version,plugins|
 
 
   #Test native roles and users are loaded
-  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | md5sum | grep 243b362bd47623c0b91a1fafbce2b6f5') do
+  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMeAgain | md5sum | grep 74bcc9f9534b253c1204e264df21496c') do
     its(:exit_status) { should eq 0 }
   end
 
-  describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | grep "{\"elastic\":{\"username\":\"elastic\",\"roles\":\[\"superuser\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana\":{\"username\":\"kibana\",\"roles\":\[\"kibana\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{},\"enabled\":true}}"') do
-    its(:exit_status) { should eq 0 }
-  end
-
-  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | grep "{\"superuser\":{\"cluster\":\[\"all\"\],\"indices\":\[{\"names\":\[\"\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\"\*\"\],\"metadata\":{\"_reserved\":true}},\"transport_client\":{\"cluster\":\[\"transport_client\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"kibana_user\":{\"cluster\":\[\"monitor\"\],\"indices\":\[{\"names\":\[\".kibana\*\"\],\"privileges\":\[\"manage\",\"read\",\"index\",\"delete\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"monitoring_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\".monitoring-\*\"\],\"privileges\":\[\"read\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"remote_monitoring_agent\":{\"cluster\":\[\"manage_index_templates\",\"manage_ingest_pipelines\",\"monitor\"\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\"\.monitoring-\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"ingest_admin\":{\"cluster\":\[\"manage_index_templates\",\"manage_pipeline\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"reporting_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.reporting-\*\"\],\"privileges\":\[\"read\",\"write\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\],\"metadata\":{}}}"') do
-    its(:exit_status) { should eq 0 }
-  end
-
-  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | md5sum | grep 78a0696c9c9690042cec2c1f16860cfc') do
+  describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMeAgain | md5sum | grep 2bf3ffbb9cabf26bb25de6334c4da323') do
     its(:exit_status) { should eq 0 }
   end
 
@@ -165,7 +157,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
 
   describe 'Template Installed' do
     it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
-      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe')
+      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain')
       expect(command.stdout).to match(/basic/)
       expect(command.exit_status).to eq(0)
     end
@@ -174,7 +166,7 @@ shared_examples 'xpack::init' do |es_version,plugins|
   #This is possibly subject to format changes in the response across versions so may fail in the future
   describe 'Template Contents Correct' do
     it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do
-      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe | md5sum')
+      command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain | md5sum')
       expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/)
     end
   end
diff --git a/test/integration/multi-5x/serverspec/default_spec.rb b/test/integration/multi-5x/serverspec/default_spec.rb
index 4abcf13..7020270 100644
--- a/test/integration/multi-5x/serverspec/default_spec.rb
+++ b/test/integration/multi-5x/serverspec/default_spec.rb
@@ -2,7 +2,7 @@ require 'multi_spec'
 
 
 describe 'Multi Tests v 5.x' do
-  include_examples 'multi::init', "5.1.2", ["ingest-geoip"]
+  include_examples 'multi::init', "5.2.2", ["ingest-geoip"]
 end
 
 
diff --git a/test/integration/package-5x/serverspec/default_spec.rb b/test/integration/package-5x/serverspec/default_spec.rb
index 4b2191b..6c553ae 100644
--- a/test/integration/package-5x/serverspec/default_spec.rb
+++ b/test/integration/package-5x/serverspec/default_spec.rb
@@ -2,5 +2,5 @@ require 'package_spec'
 
 
 describe 'Package Tests v 5.x' do
-  include_examples 'package::init', "5.1.2", ["ingest-geoip"]
+  include_examples 'package::init', "5.2.2", ["ingest-attachment","ingest-geoip"]
 end
\ No newline at end of file
diff --git a/test/integration/package.yml b/test/integration/package.yml
index 1275d2f..c5f19f7 100644
--- a/test/integration/package.yml
+++ b/test/integration/package.yml
@@ -1,5 +1,5 @@
 ---
-- name: Elasticsearch Package tests
+- name: Elasticsearch Package test intiial
   hosts: localhost
   roles:
     - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" }
@@ -8,4 +8,22 @@
     es_templates: true
     es_heap_size: "1g"
     es_api_port: 9200
-#Plugins installed for this test are specified in .kitchen.yml under suite
\ No newline at end of file
+    es_version: "5.1.2"
+    es_plugins:
+      - plugin: ingest-geoip
+
+
+#Tests the plugins have been correctly removed and ES can be upgraded between minor versions. All plugins will be removed and re-installed.
+- name: Elasticsearch Package test modify
+  hosts: localhost
+  roles:
+    - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" }
+  vars:
+    es_scripts: true
+    es_templates: true
+    es_version: "5.2.2"
+    es_heap_size: "1g"
+    es_api_port: 9200
+    es_plugins:
+      - plugin: ingest-attachment
+      - plugin: ingest-geoip
\ No newline at end of file
diff --git a/test/integration/standard-5x/serverspec/default_spec.rb b/test/integration/standard-5x/serverspec/default_spec.rb
index eaf5121..22b0976 100644
--- a/test/integration/standard-5x/serverspec/default_spec.rb
+++ b/test/integration/standard-5x/serverspec/default_spec.rb
@@ -2,7 +2,7 @@ require 'standard_spec'
 
 
 describe 'Standard Tests v 5.x' do
-  include_examples 'standard::init', "5.1.2"
+  include_examples 'standard::init', "5.2.2"
 end
 
 
diff --git a/test/integration/standard.yml b/test/integration/standard.yml
index 6f978de..dc05fac 100644
--- a/test/integration/standard.yml
+++ b/test/integration/standard.yml
@@ -1,8 +1,10 @@
 ---
-- name: wrapper playbook for kitchen testing "elasticsearch"
+- name: Standard test for single node setup. Tests idempotence.
   hosts: localhost
   roles:
     - { role: elasticsearch, es_instance_name: "node1" }
   vars:
     es_use_repository: "true"
-    es_heap_size: "1g"
\ No newline at end of file
+    es_heap_size: "1g"
+
+#Do not add tests here. This test is run twice and confirms idempotency.
\ No newline at end of file
diff --git a/test/integration/xpack-5x/serverspec/default_spec.rb b/test/integration/xpack-5x/serverspec/default_spec.rb
index bbf646c..aa7e697 100644
--- a/test/integration/xpack-5x/serverspec/default_spec.rb
+++ b/test/integration/xpack-5x/serverspec/default_spec.rb
@@ -1,5 +1,5 @@
 require 'xpack_spec'
 
 describe 'Xpack Tests v 5.x' do
-  include_examples 'xpack::init', "5.1.2", ["ingest-geoip"]
+  include_examples 'xpack::init', "5.2.2", ["ingest-attachment"]
 end
diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml
index db543e2..b468874 100644
--- a/test/integration/xpack.yml
+++ b/test/integration/xpack.yml
@@ -1,5 +1,5 @@
 ---
-- name: Elasticsearch Xpack tests
+- name: Elasticsearch Xpack tests initial
   hosts: localhost
   roles:
     - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300",
@@ -8,6 +8,7 @@
   vars:
     es_heap_size: "1g"
     es_templates: true
+    es_version: "5.1.2"
     es_enable_xpack: true
     es_xpack_license: "{{ lookup('file', '/tmp/license.json')  }}"
     es_plugins:
@@ -79,3 +80,45 @@
                 - delete
                 - create_index
 
+#modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed.
+- name: Elasticsearch Xpack modify
+  hosts: localhost
+  roles:
+    - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300",
+    "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 },
+    es_instance_name: "security_node" }
+  vars:
+    es_heap_size: "1g"
+    es_templates: true
+    es_version: "5.2.2"
+    es_enable_xpack: true
+    es_xpack_license: "{{ lookup('file', '/tmp/license.json')  }}"
+    es_plugins:
+      - plugin: ingest-attachment
+    es_xpack_features:
+      - security
+      - alerting
+    es_api_basic_auth_username: elastic
+    es_api_basic_auth_password: changeme
+    es_role_mapping:
+      power_user:
+        - "cn=admins,dc=example,dc=com"
+      user:
+        - "cn=users,dc=example,dc=com"
+        - "cn=admins,dc=example,dc=com"
+    es_users:
+      native:
+        kibana4_server:
+          password: changeMe
+          roles:
+            - kibana4_server
+      file:
+        es_admin:
+          password: changeMeAgain
+          roles:
+            - admin
+        testUser:
+          password: changeMeAlso!
+          roles:
+            - power_user
+            - user
\ No newline at end of file

From c8d0173fb8418bf5099eebae61ee7c244716b1ab Mon Sep 17 00:00:00 2001
From: Dale McDiarmid <dalem@elastic.co>
Date: Thu, 16 Mar 2017 22:23:20 +0000
Subject: [PATCH 2/5] fixing spelling mistake in test

---
 test/integration/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/config.yml b/test/integration/config.yml
index d4741db..afa5398 100644
--- a/test/integration/config.yml
+++ b/test/integration/config.yml
@@ -12,7 +12,7 @@
     es_heap_size: 1g
     es_api_port: 9201
     es_plugins:
-      - plugin: docke
+      - plugin: ingest-geoip
 
 #Modify the above configuration. Final test should evaluate this configuration. Also tests the plugins are added and removed.
 - name: Elasticsearch Config test modify

From af30f882dd1b97ac964f9c6f71b03a6d8a30650c Mon Sep 17 00:00:00 2001
From: Dale McDiarmid <dalem@elastic.co>
Date: Thu, 16 Mar 2017 23:59:13 +0000
Subject: [PATCH 3/5] Fixes for multi + config tests

---
 .kitchen.yml                    | 11 ++++++-----
 tasks/elasticsearch-plugins.yml |  2 +-
 test/integration/multi.yml      |  4 ++++
 test/integration/standard.yml   |  2 ++
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/.kitchen.yml b/.kitchen.yml
index 044a77a..1a97586 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -20,7 +20,8 @@ platforms:
       privileged: true
       provision_command:
         - apt-get update && apt-get install -y software-properties-common &&  add-apt-repository -y ppa:ansible/ansible
-        - apt-get update && apt-get -y -q install python-apt python-pycurl
+        - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip
+        - pip install jmespath
       use_sudo: false
       volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
   - name: ubuntu-16.04
@@ -30,7 +31,8 @@ platforms:
       provision_command:
         - apt-get update && apt-get install -y software-properties-common &&  add-apt-repository -y ppa:ansible/ansible
         - apt-get install -y -q net-tools
-        - apt-get update && apt-get -y -q install python-apt python-pycurl
+        - apt-get update && apt-get -y -q install python-apt python-pycurl python-pip
+        - pip install jmespath
       use_sudo: false
       volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
       run_command: "/sbin/init"
@@ -45,6 +47,7 @@ platforms:
         - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config
         - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config
         - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config
+        - pip install jmespath
       volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
       use_sudo: false
       run_command: "/sbin/init"
@@ -58,6 +61,7 @@ platforms:
         - rm /etc/yum.repos.d/epel*repo /etc/yum.repos.d/puppetlabs-pc1.repo
         - yum -y install initscripts
         - yum clean all
+        - pip install jmespath
       volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json
       run_command: "/usr/sbin/init"
       privileged: true
@@ -83,9 +87,6 @@ suites:
   - name: multi-5x
     run_list:
     attributes:
-      extra_vars:
-        es_plugins:
-          - plugin: ingest-geoip
     provisioner:
       playbook: test/integration/multi.yml
       idempotency_test: true
diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml
index 5957783..8ec9e25 100644
--- a/tasks/elasticsearch-plugins.yml
+++ b/tasks/elasticsearch-plugins.yml
@@ -25,7 +25,7 @@
   when: es_plugins_reinstall
 
 #if the plugins listed are different than those requested, we remove those installed but not listed in the config
-- set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('es_plugins[*].plugin')) | default([]) }}"
+- set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('[*].plugin')) | default([]) }}"
   when: not es_plugins_reinstall
 
 # This removes any currently installed plugins (to prevent errors when reinstalling)
diff --git a/test/integration/multi.yml b/test/integration/multi.yml
index 1d191f5..702b11f 100644
--- a/test/integration/multi.yml
+++ b/test/integration/multi.yml
@@ -7,6 +7,8 @@
     es_templates: true
     es_heap_size: "1g"
     es_api_port: 9200
+    es_plugins:
+      - plugin: ingest-geoip
   roles:
     - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } }
 
@@ -17,6 +19,8 @@
     es_templates: true
     es_heap_size: "1g"
     es_api_port: 9201
+    es_plugins:
+      - plugin: ingest-geoip
   roles:
     - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } }
 #Plugins installed for this test are specified in .kitchen.yml under suite
diff --git a/test/integration/standard.yml b/test/integration/standard.yml
index dc05fac..7d8e6cc 100644
--- a/test/integration/standard.yml
+++ b/test/integration/standard.yml
@@ -6,5 +6,7 @@
   vars:
     es_use_repository: "true"
     es_heap_size: "1g"
+    es_plugins:
+      - plugin: ingest-geoip
 
 #Do not add tests here. This test is run twice and confirms idempotency.
\ No newline at end of file

From 01992fdb777e9db1b849c17bf4c442588fca9f8d Mon Sep 17 00:00:00 2001
From: Dale McDiarmid <dalem@elastic.co>
Date: Fri, 17 Mar 2017 00:20:50 +0000
Subject: [PATCH 4/5] Fixes for mlock all test

---
 test/integration/config.yml                        | 4 ++--
 test/integration/helpers/serverspec/config_spec.rb | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/test/integration/config.yml b/test/integration/config.yml
index afa5398..9c652b2 100644
--- a/test/integration/config.yml
+++ b/test/integration/config.yml
@@ -4,7 +4,7 @@
   hosts: localhost
   roles:
     #expand to all available parameters
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: true } }
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: false } }
   vars:
     es_scripts: false
     es_templates: false
@@ -19,7 +19,7 @@
   hosts: localhost
   roles:
     #expand to all available parameters
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9501", http.port: 9401, transport.tcp.port: 9501, node.data: true, node.master: true, bootstrap.memory_lock: false } }
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9501", http.port: 9401, transport.tcp.port: 9501, node.data: true, node.master: true, bootstrap.memory_lock: true } }
   vars:
     es_scripts: false
     es_templates: false
diff --git a/test/integration/helpers/serverspec/config_spec.rb b/test/integration/helpers/serverspec/config_spec.rb
index c218104..95a4bfd 100644
--- a/test/integration/helpers/serverspec/config_spec.rb
+++ b/test/integration/helpers/serverspec/config_spec.rb
@@ -34,7 +34,7 @@ shared_examples 'config::init' do |es_version,plugins|
     it { should contain 'node.master: true' }
     it { should contain 'cluster.name: custom-cluster' }
     it { should contain 'node.name: node1' }
-    it { should contain 'bootstrap.memory_lock: false' }
+    it { should contain 'bootstrap.memory_lock: true' }
     it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9501' }
     it { should contain 'path.conf: /etc/elasticsearch/node1' }
     it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' }
@@ -71,7 +71,7 @@ shared_examples 'config::init' do |es_version,plugins|
 
   #test to make sure mlock was applied
   describe command('curl -s "localhost:9401/_nodes/process?pretty" | grep mlockall') do
-    its(:stdout) { should match /false/ }
+    its(:stdout) { should match /true/ }
     its(:exit_status) { should eq 0 }
   end
 

From cb85735fb6e5ed4ae010850dda5fba308c19d136 Mon Sep 17 00:00:00 2001
From: Jared Carey <jared.carey@elastic.co>
Date: Fri, 17 Mar 2017 11:46:07 -0600
Subject: [PATCH 5/5] Update package.yml

---
 test/integration/package.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/integration/package.yml b/test/integration/package.yml
index c5f19f7..4cfd73c 100644
--- a/test/integration/package.yml
+++ b/test/integration/package.yml
@@ -1,5 +1,5 @@
 ---
-- name: Elasticsearch Package test intiial
+- name: Elasticsearch Package test intial
   hosts: localhost
   roles:
     - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" }
@@ -26,4 +26,4 @@
     es_api_port: 9200
     es_plugins:
       - plugin: ingest-attachment
-      - plugin: ingest-geoip
\ No newline at end of file
+      - plugin: ingest-geoip