From 6d82cf3142cec56d2432126eb4d276b7c4424c39 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Wed, 13 Jun 2018 08:29:45 +0200 Subject: [PATCH] WIP: 6.3 support --- .kitchen.yml | 2 + README.md | 2 +- defaults/main.yml | 6 +- tasks/elasticsearch-Debian.yml | 7 +- tasks/elasticsearch-RedHat.yml | 5 +- tasks/elasticsearch-plugins.yml | 3 +- tasks/xpack/elasticsearch-xpack.yml | 1 + .../helpers/serverspec/config_spec.rb | 2 +- .../helpers/serverspec/multi_spec.rb | 2 +- .../helpers/serverspec/package_spec.rb | 2 +- .../helpers/serverspec/standard_spec.rb | 19 ++-- .../helpers/serverspec/xpack_spec.rb | 2 +- .../helpers/serverspec/xpack_standard_spec.rb | 86 ++++++++----------- test/integration/standard.yml | 3 +- test/integration/xpack-standard.yml | 2 +- test/integration/xpack.yml | 3 +- vars/main.yml | 5 +- 17 files changed, 82 insertions(+), 70 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 6425c86..2d9173a 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -22,6 +22,8 @@ provisioner: es_major_version: "<%= ENV['VERSION'] %>" <% if ENV['VERSION'] == '5.x' %> es_version: '5.6.9' + es_package_name: 'elasticsearch' + es_use_snapshot_release: false <% end %> <% end %> diff --git a/README.md b/README.md index e4d0ac6..9051229 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Ansible role for 6.x/5.x Elasticsearch. Currently this works on Debian and RedH * Debian 8 * Centos 7 -The latest Elasticsearch versions of 6.x are actively tested. **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.** +The latest Elasticsearch versions of 6.x and 5.x are actively tested. **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.** ##### Dependency This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine. diff --git a/defaults/main.yml b/defaults/main.yml index 524f65d..5598870 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,6 +1,9 @@ --- +es_use_snapshot_release: true # TODO remove when 6.3 is released es_major_version: "6.x" -es_version: "6.2.4" +es_version: "6.3.0" +es_enable_xpack: true +es_package_name: "elasticsearch" es_version_lock: false es_use_repository: true es_templates_fileglob: "files/templates/*.json" @@ -27,7 +30,6 @@ es_max_open_files: 65536 es_max_threads: "{{ 2048 if ( es_version | version_compare('6.0.0', '<')) else 8192 }}" es_max_map_count: 262144 es_allow_downgrades: false -es_enable_xpack: false es_xpack_features: ["alerting","monitoring","graph","ml","security"] #These are used for internal operations performed by ansible. #They do not affect the current configuration diff --git a/tasks/elasticsearch-Debian.yml b/tasks/elasticsearch-Debian.yml index 3677b3e..28ba13a 100644 --- a/tasks/elasticsearch-Debian.yml +++ b/tasks/elasticsearch-Debian.yml @@ -27,7 +27,12 @@ - name: Debian - Ensure elasticsearch is installed become: yes - apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present force={{force_install}} allow_unauthenticated={{ 'no' if es_apt_key else 'yes' }} cache_valid_time=86400 + apt: + name: {{ es_package_name }}{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} + state: present + force: {{force_install}} + allow_unauthenticated: {{ 'no' if es_apt_key else 'yes' }} + cache_valid_time: 86400 when: es_use_repository register: debian_elasticsearch_install_from_repo notify: restart elasticsearch diff --git a/tasks/elasticsearch-RedHat.yml b/tasks/elasticsearch-RedHat.yml index ba35556..baf8f74 100644 --- a/tasks/elasticsearch-RedHat.yml +++ b/tasks/elasticsearch-RedHat.yml @@ -15,7 +15,10 @@ - name: RedHat - Install Elasticsearch become: yes - yum: name=elasticsearch{% if es_version is defined and es_version != "" %}-{{ es_version }}{% endif %} state=present update_cache=yes + yum: + name: {{ es_package_name }}{% if es_version is defined and es_version != "" %}-{{ es_version }}{% endif %} + state: present + update_cache: yes when: es_use_repository register: redhat_elasticsearch_install_from_repo notify: restart elasticsearch diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 5d4e2d8..5753c2a 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -60,9 +60,8 @@ - name: Install elasticsearch plugins become: yes - command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.plugin }} --batch --silent" + command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.url | default(item.plugin) }} --batch --silent" register: plugin_installed - failed_when: "'ERROR' in plugin_installed.stdout" changed_when: plugin_installed.rc == 0 with_items: "{{ es_plugins }}" when: item.plugin in plugins_to_install diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index ec82a68..2074dcb 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -5,6 +5,7 @@ - name: include elasticsearch-xpack-install.yml include: elasticsearch-xpack-install.yml + when: es_install_xpack #Security configuration - name: include security/elasticsearch-security.yml diff --git a/test/integration/helpers/serverspec/config_spec.rb b/test/integration/helpers/serverspec/config_spec.rb index 424a11a..933dc5f 100644 --- a/test/integration/helpers/serverspec/config_spec.rb +++ b/test/integration/helpers/serverspec/config_spec.rb @@ -18,7 +18,7 @@ shared_examples 'config::init' do |vars| it { should be_running } end - describe package('elasticsearch') do + describe package(vars['es_package_name']) do it { should be_installed } end diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index 6d8062d..b3ec77a 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -16,7 +16,7 @@ shared_examples 'multi::init' do |vars| it { should be_running } end - describe package('elasticsearch') do + describe package(vars['es_package_name']) do it { should be_installed } end diff --git a/test/integration/helpers/serverspec/package_spec.rb b/test/integration/helpers/serverspec/package_spec.rb index bf85760..ab37b0d 100644 --- a/test/integration/helpers/serverspec/package_spec.rb +++ b/test/integration/helpers/serverspec/package_spec.rb @@ -12,7 +12,7 @@ shared_examples 'package::init' do |vars| it { should be_running } end - describe package('elasticsearch') do + describe package(vars['es_package_name']) do it { should be_installed } end diff --git a/test/integration/helpers/serverspec/standard_spec.rb b/test/integration/helpers/serverspec/standard_spec.rb index ca4b3f2..5a01cd4 100644 --- a/test/integration/helpers/serverspec/standard_spec.rb +++ b/test/integration/helpers/serverspec/standard_spec.rb @@ -10,7 +10,7 @@ shared_examples 'standard::init' do |vars| it { should be_running } end - describe package('elasticsearch') do + describe package(vars['es_package_name']) do it { should be_installed } end @@ -85,15 +85,22 @@ shared_examples 'standard::init' do |vars| end for plugin in vars['es_plugins'] - plugin = plugin['plugin'] + name = plugin['plugin'] - describe file('/usr/share/elasticsearch/plugins/'+plugin) do + describe file('/usr/share/elasticsearch/plugins/'+name) do it { should be_directory } it { should be_owned_by 'elasticsearch' } end - #confirm plugins are installed and the correct version - describe command('curl -s localhost:9200/_nodes/plugins | grep \'"name":"'+plugin+'","version":"'+vars['es_version']+'"\'') do - its(:exit_status) { should eq 0 } + it 'should be installed and the right version' do + plugins = curl_json('http://localhost:9200/_nodes/plugins') + version = nil + node, data = plugins['nodes'].first + data['plugins'].each do |plugin| + if plugin['name'] == name + version = plugin['version'] + end + end + expect(version).to eql(vars['es_version']) end end diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index c94db3d..4812fc6 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -12,7 +12,7 @@ shared_examples 'xpack::init' do |vars| it { should be_running } end - describe package('elasticsearch') do + describe package(vars['es_package_name']) do it { should be_installed } end diff --git a/test/integration/helpers/serverspec/xpack_standard_spec.rb b/test/integration/helpers/serverspec/xpack_standard_spec.rb index 545fbbc..59d811f 100644 --- a/test/integration/helpers/serverspec/xpack_standard_spec.rb +++ b/test/integration/helpers/serverspec/xpack_standard_spec.rb @@ -10,7 +10,7 @@ shared_examples 'xpack_standard::init' do |vars| it { should be_running } end - describe package('elasticsearch') do + describe package(vars['es_package_name']) do it { should be_installed } end @@ -96,60 +96,48 @@ shared_examples 'xpack_standard::init' do |vars| end end - describe file('/usr/share/elasticsearch/plugins/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } + # X-Pack is no longer installed as a plugin in elasticsearch + if vars['es_major_version'] == '5.x' + describe file('/usr/share/elasticsearch/plugins/x-pack') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe 'x-pack-core plugin' do + it 'should be installed with the correct version' do + plugins = curl_json('http://localhost:9200/_nodes/plugins') + node, data = plugins['nodes'].first + version = 'plugin not found' + name = 'x-pack' + + data['plugins'].each do |plugin| + if plugin['name'] == name + version = plugin['version'] + end + end + expect(version).to eql(vars['es_version']) + end + end + + describe file('/etc/elasticsearch/security_node/x-pack') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + #Test users file, users_roles and roles.yml + describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do + it { should be_owned_by 'elasticsearch' } + end + + describe file('/etc/elasticsearch/security_node/x-pack/users') do + it { should be_owned_by 'elasticsearch' } + end end describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMeAgain | grep x-pack') do its(:exit_status) { should eq 0 } end - describe file('/etc/elasticsearch/security_node/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/usr/share/elasticsearch/plugins/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/usr/share/elasticsearch/plugins/x-pack') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe 'x-pack-core plugin' do - it 'should be installed with the correct version' do - plugins = curl_json('http://localhost:9200/_nodes/plugins') - node, data = plugins['nodes'].first - version = 'plugin not found' - - if Gem::Version.new(vars['es_version']) >= Gem::Version.new('6.2') - name = 'x-pack-core' - else - name = 'x-pack' - end - - data['plugins'].each do |plugin| - if plugin['name'] == name - version = plugin['version'] - end - end - expect(version).to eql(vars['es_version']) - end - end - - #Test users file, users_roles and roles.yml - describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do - it { should be_owned_by 'elasticsearch' } - end - - describe file('/etc/elasticsearch/security_node/x-pack/users') do - it { should be_owned_by 'elasticsearch' } - end - describe command('curl -s localhost:9200/_xpack') do its(:stdout_as_json) { should include('features' => include('security' => include('enabled' => false))) } its(:stdout_as_json) { should include('features' => include('watcher' => include('enabled' => false))) } diff --git a/test/integration/standard.yml b/test/integration/standard.yml index 769f2c5..6cdf68e 100644 --- a/test/integration/standard.yml +++ b/test/integration/standard.yml @@ -6,7 +6,8 @@ roles: - { role: elasticsearch, es_instance_name: "node1" } vars: - es_use_repository: "true" + es_enable_xpack: false + es_package_name: "elasticsearch-oss" es_heap_size: "1g" es_plugins: - plugin: ingest-geoip diff --git a/test/integration/xpack-standard.yml b/test/integration/xpack-standard.yml index e3eba44..6789a7e 100644 --- a/test/integration/xpack-standard.yml +++ b/test/integration/xpack-standard.yml @@ -7,9 +7,9 @@ roles: - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "security_node" } vars: + es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip" es_heap_size: 2g es_enable_xpack: true - es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip" es_xpack_features: - monitoring - graph diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index a386732..53baba4 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -10,7 +10,8 @@ vars: es_heap_size: "1g" es_templates: true - es_version: "{{ '6.1.1' if es_major_version == '6.x' else '5.4.0'}}" # This is set to an older version than the current default to force an upgrade + es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.6' }}" # This is set to an older version than the current default to force an upgrade + es_use_snapshot_release: false # TODO remove when 6.3 is released es_enable_xpack: true es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" es_plugins: diff --git a/vars/main.yml b/vars/main.yml index b3bd126..0a2f094 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -5,4 +5,7 @@ sysd_script: "/usr/lib/systemd/system/elasticsearch.service" init_script: "/etc/init.d/elasticsearch" #add supported features here supported_xpack_features: ["alerting","monitoring","graph","security"] -reserved_xpack_users: ["elastic","kibana","logstash_system"] \ No newline at end of file +reserved_xpack_users: ["elastic","kibana","logstash_system"] + +# X-Pack used to be installed as a plugin. For versions after 6.3 we don't need to install it anymore +es_install_xpack: "{{ true if ( es_version | version_compare('6.3.0', '<')) else false }}"