diff --git a/.kitchen.yml b/.kitchen.yml index 0bd7a15..044a77a 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -12,7 +12,6 @@ provisioner: http_proxy: <%= ENV['HTTP_PROXY'] %> https_proxy: <%= ENV['HTTPS_PROXY'] %> no_proxy: localhost,127.0.0.1 - idempotency_test: true platforms: - name: ubuntu-14.04 @@ -67,15 +66,13 @@ platforms: suites: - name: standard-5x provisioner: + idempotency_test: true playbook: test/integration/standard.yml run_list: attributes: - name: package-5x run_list: attributes: - extra_vars: - es_plugins: - - plugin: ingest-geoip provisioner: playbook: test/integration/package.yml - name: config-5x @@ -91,8 +88,9 @@ suites: - plugin: ingest-geoip provisioner: playbook: test/integration/multi.yml + idempotency_test: true - name: xpack-5x run_list: attributes: provisioner: - playbook: test/integration/xpack.yml + playbook: test/integration/xpack.yml \ No newline at end of file diff --git a/defaults/main.yml b/defaults/main.yml index 62dec45..59f01af 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,6 +1,6 @@ --- es_major_version: "5.x" -es_version: "5.1.2" +es_version: "5.2.2" es_version_lock: false es_use_repository: true es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch" diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 9eeb5d2..6debfaf 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -47,7 +47,6 @@ - restart elasticsearch #Copy the logging.yml - - name: Copy log4j2.properties File for Instance template: src=log4j2.properties.j2 dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes notify: restart elasticsearch diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index eab6109..5957783 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -5,25 +5,35 @@ - set_fact: es_plugins_reinstall=true when: (((debian_elasticsearch_install_from_repo is defined and debian_elasticsearch_install_from_repo.changed) or (redhat_elasticsearch_install_from_repo is defined and redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none -- set_fact: list_command="list" +- set_fact: list_command="" +#If we are reinstalling all plugins, e.g. to a version change, we need to remove all plugins (inc. x-pack) to install any plugins. Otherwise we don't consider x-pack so the role stays idempotent. +- set_fact: list_command="| grep -vE 'x-pack'" + when: not es_plugins_reinstall -#List currently installed plugins - ignore xpack if > v 2.0 +#List currently installed plugins. We have to list the directories as the list commmand fails if the ES version is different than the plugin version. - name: Check installed elasticsearch plugins - shell: "{{es_home}}/bin/elasticsearch-plugin list | grep -vE 'x-pack'" + shell: "ls {{es_home}}/plugins {{list_command}}" register: installed_plugins - failed_when: "'ERROR' in installed_plugins.stdout" changed_when: False ignore_errors: yes environment: CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" +#if es_plugins_reinstall is set to true we remove ALL plugins +- set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | default([]) }}" + when: es_plugins_reinstall + +#if the plugins listed are different than those requested, we remove those installed but not listed in the config +- set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('es_plugins[*].plugin')) | default([]) }}" + when: not es_plugins_reinstall + # This removes any currently installed plugins (to prevent errors when reinstalling) - name: Remove elasticsearch plugins command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent" ignore_errors: yes - with_items: "{{ installed_plugins.stdout_lines | default([]) }}" - when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0] + with_items: "{{ plugins_to_remove | default([]) }}" + when: es_plugins_reinstall and plugins_to_remove | length > 0 notify: restart elasticsearch register: plugin_removed environment: diff --git a/tasks/elasticsearch-scripts.yml b/tasks/elasticsearch-scripts.yml index 40910c2..30867f5 100644 --- a/tasks/elasticsearch-scripts.yml +++ b/tasks/elasticsearch-scripts.yml @@ -10,7 +10,7 @@ - always - name: Create script dir - file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} + file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes - name: Copy default scripts to elasticsearch copy: src=scripts dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} diff --git a/test/integration/config-5x/serverspec/default_spec.rb b/test/integration/config-5x/serverspec/default_spec.rb index 72b83d9..785614b 100644 --- a/test/integration/config-5x/serverspec/default_spec.rb +++ b/test/integration/config-5x/serverspec/default_spec.rb @@ -1,6 +1,6 @@ require 'config_spec' describe 'Config Tests v 5.x' do - include_examples 'config::init', "5.1.2" + include_examples 'config::init', "5.2.2", ["ingest-attachment","ingest-user-agent"] end diff --git a/test/integration/config.yml b/test/integration/config.yml index 5c4d3d4..d4741db 100644 --- a/test/integration/config.yml +++ b/test/integration/config.yml @@ -1,6 +1,6 @@ --- #Test explicit setting of parameters and variables -- name: Elasticsearch Config tests +- name: Elasticsearch Config initial hosts: localhost roles: #expand to all available parameters @@ -10,4 +10,22 @@ es_templates: false es_version_lock: false es_heap_size: 1g - es_api_port: 9201 \ No newline at end of file + es_api_port: 9201 + es_plugins: + - plugin: docke + +#Modify the above configuration. Final test should evaluate this configuration. Also tests the plugins are added and removed. +- name: Elasticsearch Config test modify + hosts: localhost + roles: + #expand to all available parameters + - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9501", http.port: 9401, transport.tcp.port: 9501, node.data: true, node.master: true, bootstrap.memory_lock: false } } + vars: + es_scripts: false + es_templates: false + es_version_lock: false + es_heap_size: 1g + es_api_port: 9401 + es_plugins: + - plugin: ingest-attachment + - plugin: ingest-user-agent \ No newline at end of file diff --git a/test/integration/helpers/serverspec/config_spec.rb b/test/integration/helpers/serverspec/config_spec.rb index aca7f1a..c218104 100644 --- a/test/integration/helpers/serverspec/config_spec.rb +++ b/test/integration/helpers/serverspec/config_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -shared_examples 'config::init' do |es_version| +shared_examples 'config::init' do |es_version,plugins| describe user('elasticsearch') do it { should exist } @@ -28,14 +28,14 @@ shared_examples 'config::init' do |es_version| #test configuration parameters have been set - test all appropriately set in config file describe file('/etc/elasticsearch/node1/elasticsearch.yml') do - it { should contain 'http.port: 9201' } - it { should contain 'transport.tcp.port: 9301' } - it { should contain 'node.data: false' } + it { should contain 'http.port: 9401' } + it { should contain 'transport.tcp.port: 9501' } + it { should contain 'node.data: true' } it { should contain 'node.master: true' } it { should contain 'cluster.name: custom-cluster' } it { should contain 'node.name: node1' } - it { should contain 'bootstrap.memory_lock: true' } - it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9301' } + it { should contain 'bootstrap.memory_lock: false' } + it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9501' } it { should contain 'path.conf: /etc/elasticsearch/node1' } it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' } it { should contain 'path.logs: /opt/elasticsearch/logs/localhost-node1' } @@ -63,27 +63,47 @@ shared_examples 'config::init' do |es_version| end #test we started on the correct port was used - describe command('curl -s "localhost:9201"') do + describe command('curl -s "localhost:9401"') do #TODO: This is returning an empty string #its(:stdout) { should match /\"status\" : 200/ } its(:exit_status) { should eq 0 } end #test to make sure mlock was applied - describe command('curl -s "localhost:9201/_nodes/process?pretty" | grep mlockall') do - its(:stdout) { should match /true/ } + describe command('curl -s "localhost:9401/_nodes/process?pretty" | grep mlockall') do + its(:stdout) { should match /false/ } its(:exit_status) { should eq 0 } end describe 'version check' do it 'should be reported as version '+es_version do - command = command('curl -s localhost:9201 | grep number') + command = command('curl -s localhost:9401 | grep number') expect(command.stdout).to match(es_version) expect(command.exit_status).to eq(0) end end + for plugin in plugins + describe file('/usr/share/elasticsearch/plugins/'+plugin) do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + #confirm plugins are installed and the correct version + describe command('curl -s localhost:9401/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do + its(:exit_status) { should eq 0 } + end + end + + #explit test to make sure ingest-geoip is not installed + describe file('/usr/share/elasticsearch/plugins/ingest-geoip') do + it { should_not exist } + end + #confirm plugins are installed and the correct version + describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"ingest-geoip","version":"'+es_version+'"\'') do + its(:exit_status) { should eq 1 } + end + describe file('/etc/init.d/elasticsearch') do it { should_not exist } end diff --git a/test/integration/helpers/serverspec/package_spec.rb b/test/integration/helpers/serverspec/package_spec.rb index 51db46e..ac68bc2 100644 --- a/test/integration/helpers/serverspec/package_spec.rb +++ b/test/integration/helpers/serverspec/package_spec.rb @@ -74,12 +74,13 @@ shared_examples 'package::init' do |es_version,plugins| it { should be_directory } it { should be_owned_by 'elasticsearch' } end - - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do + #confirm plugins are installed and the correct version + describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do its(:exit_status) { should eq 0 } end end + describe file('/etc/init.d/elasticsearch') do it { should_not exist } end diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 87365b4..08a7cac 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -40,7 +40,7 @@ shared_examples 'xpack::init' do |es_version,plugins| describe 'version check' do it 'should be reported as version '+es_version do - command = command('curl -s localhost:9200 -u es_admin:changeMe | grep number') + command = command('curl -s localhost:9200 -u es_admin:changeMeAgain | grep number') expect(command.stdout).to match(es_version) expect(command.exit_status).to eq(0) end @@ -79,7 +79,7 @@ shared_examples 'xpack::init' do |es_version,plugins| #Test if x-pack is activated describe 'x-pack activation' do it 'should be activated and valid' do - command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMe') + command = command('curl -s localhost:9200/_license?pretty=true -u es_admin:changeMeAgain') expect(command.stdout).to match('"status" : "active"') expect(command.exit_status).to eq(0) end @@ -90,7 +90,7 @@ shared_examples 'xpack::init' do |es_version,plugins| it { should be_owned_by 'elasticsearch' } end - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep x-pack') do + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do its(:exit_status) { should eq 0 } end @@ -110,7 +110,7 @@ shared_examples 'xpack::init' do |es_version,plugins| it { should be_owned_by 'elasticsearch' } end - describe command('curl -s -u es_admin:changeMe localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do + describe command('curl -s localhost:9200/_nodes/plugins -u es_admin:changeMeAgain | grep \'"name":"'+plugin+'","version":"'+es_version+'"\'') do its(:exit_status) { should eq 0 } end end @@ -137,19 +137,11 @@ shared_examples 'xpack::init' do |es_version,plugins| #Test native roles and users are loaded - describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | md5sum | grep 243b362bd47623c0b91a1fafbce2b6f5') do + describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMeAgain | md5sum | grep 74bcc9f9534b253c1204e264df21496c') do its(:exit_status) { should eq 0 } end - describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | grep "{\"elastic\":{\"username\":\"elastic\",\"roles\":\[\"superuser\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana\":{\"username\":\"kibana\",\"roles\":\[\"kibana\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{},\"enabled\":true}}"') do - its(:exit_status) { should eq 0 } - end - - describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | grep "{\"superuser\":{\"cluster\":\[\"all\"\],\"indices\":\[{\"names\":\[\"\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\"\*\"\],\"metadata\":{\"_reserved\":true}},\"transport_client\":{\"cluster\":\[\"transport_client\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"kibana_user\":{\"cluster\":\[\"monitor\"\],\"indices\":\[{\"names\":\[\".kibana\*\"\],\"privileges\":\[\"manage\",\"read\",\"index\",\"delete\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"monitoring_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\".monitoring-\*\"\],\"privileges\":\[\"read\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"remote_monitoring_agent\":{\"cluster\":\[\"manage_index_templates\",\"manage_ingest_pipelines\",\"monitor\"\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\"\.monitoring-\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"ingest_admin\":{\"cluster\":\[\"manage_index_templates\",\"manage_pipeline\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"reporting_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.reporting-\*\"\],\"privileges\":\[\"read\",\"write\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\],\"metadata\":{}}}"') do - its(:exit_status) { should eq 0 } - end - - describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | md5sum | grep 78a0696c9c9690042cec2c1f16860cfc') do + describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMeAgain | md5sum | grep 2bf3ffbb9cabf26bb25de6334c4da323') do its(:exit_status) { should eq 0 } end @@ -165,7 +157,7 @@ shared_examples 'xpack::init' do |es_version,plugins| describe 'Template Installed' do it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe') + command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain') expect(command.stdout).to match(/basic/) expect(command.exit_status).to eq(0) end @@ -174,7 +166,7 @@ shared_examples 'xpack::init' do |es_version,plugins| #This is possibly subject to format changes in the response across versions so may fail in the future describe 'Template Contents Correct' do it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do - command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe | md5sum') + command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMeAgain | md5sum') expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/) end end diff --git a/test/integration/multi-5x/serverspec/default_spec.rb b/test/integration/multi-5x/serverspec/default_spec.rb index 4abcf13..7020270 100644 --- a/test/integration/multi-5x/serverspec/default_spec.rb +++ b/test/integration/multi-5x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'multi_spec' describe 'Multi Tests v 5.x' do - include_examples 'multi::init', "5.1.2", ["ingest-geoip"] + include_examples 'multi::init', "5.2.2", ["ingest-geoip"] end diff --git a/test/integration/package-5x/serverspec/default_spec.rb b/test/integration/package-5x/serverspec/default_spec.rb index 4b2191b..6c553ae 100644 --- a/test/integration/package-5x/serverspec/default_spec.rb +++ b/test/integration/package-5x/serverspec/default_spec.rb @@ -2,5 +2,5 @@ require 'package_spec' describe 'Package Tests v 5.x' do - include_examples 'package::init', "5.1.2", ["ingest-geoip"] + include_examples 'package::init', "5.2.2", ["ingest-attachment","ingest-geoip"] end \ No newline at end of file diff --git a/test/integration/package.yml b/test/integration/package.yml index 1275d2f..c5f19f7 100644 --- a/test/integration/package.yml +++ b/test/integration/package.yml @@ -1,5 +1,5 @@ --- -- name: Elasticsearch Package tests +- name: Elasticsearch Package test intiial hosts: localhost roles: - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" } @@ -8,4 +8,22 @@ es_templates: true es_heap_size: "1g" es_api_port: 9200 -#Plugins installed for this test are specified in .kitchen.yml under suite \ No newline at end of file + es_version: "5.1.2" + es_plugins: + - plugin: ingest-geoip + + +#Tests the plugins have been correctly removed and ES can be upgraded between minor versions. All plugins will be removed and re-installed. +- name: Elasticsearch Package test modify + hosts: localhost + roles: + - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" } + vars: + es_scripts: true + es_templates: true + es_version: "5.2.2" + es_heap_size: "1g" + es_api_port: 9200 + es_plugins: + - plugin: ingest-attachment + - plugin: ingest-geoip \ No newline at end of file diff --git a/test/integration/standard-5x/serverspec/default_spec.rb b/test/integration/standard-5x/serverspec/default_spec.rb index eaf5121..22b0976 100644 --- a/test/integration/standard-5x/serverspec/default_spec.rb +++ b/test/integration/standard-5x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'standard_spec' describe 'Standard Tests v 5.x' do - include_examples 'standard::init', "5.1.2" + include_examples 'standard::init', "5.2.2" end diff --git a/test/integration/standard.yml b/test/integration/standard.yml index 6f978de..dc05fac 100644 --- a/test/integration/standard.yml +++ b/test/integration/standard.yml @@ -1,8 +1,10 @@ --- -- name: wrapper playbook for kitchen testing "elasticsearch" +- name: Standard test for single node setup. Tests idempotence. hosts: localhost roles: - { role: elasticsearch, es_instance_name: "node1" } vars: es_use_repository: "true" - es_heap_size: "1g" \ No newline at end of file + es_heap_size: "1g" + +#Do not add tests here. This test is run twice and confirms idempotency. \ No newline at end of file diff --git a/test/integration/xpack-5x/serverspec/default_spec.rb b/test/integration/xpack-5x/serverspec/default_spec.rb index bbf646c..aa7e697 100644 --- a/test/integration/xpack-5x/serverspec/default_spec.rb +++ b/test/integration/xpack-5x/serverspec/default_spec.rb @@ -1,5 +1,5 @@ require 'xpack_spec' describe 'Xpack Tests v 5.x' do - include_examples 'xpack::init', "5.1.2", ["ingest-geoip"] + include_examples 'xpack::init', "5.2.2", ["ingest-attachment"] end diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index db543e2..b468874 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -1,5 +1,5 @@ --- -- name: Elasticsearch Xpack tests +- name: Elasticsearch Xpack tests initial hosts: localhost roles: - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", @@ -8,6 +8,7 @@ vars: es_heap_size: "1g" es_templates: true + es_version: "5.1.2" es_enable_xpack: true es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" es_plugins: @@ -79,3 +80,45 @@ - delete - create_index +#modifies the installation. Changes es_admin password and upgrades ES. Tests confirm the correct version is installed. +- name: Elasticsearch Xpack modify + hosts: localhost + roles: + - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", + "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 }, + es_instance_name: "security_node" } + vars: + es_heap_size: "1g" + es_templates: true + es_version: "5.2.2" + es_enable_xpack: true + es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" + es_plugins: + - plugin: ingest-attachment + es_xpack_features: + - security + - alerting + es_api_basic_auth_username: elastic + es_api_basic_auth_password: changeme + es_role_mapping: + power_user: + - "cn=admins,dc=example,dc=com" + user: + - "cn=users,dc=example,dc=com" + - "cn=admins,dc=example,dc=com" + es_users: + native: + kibana4_server: + password: changeMe + roles: + - kibana4_server + file: + es_admin: + password: changeMeAgain + roles: + - admin + testUser: + password: changeMeAlso! + roles: + - power_user + - user \ No newline at end of file