santerikainulainen/ansible-role-elasticsearch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[7.x] add support for elasticsearch 7.x and remove support for 5.x (#558)

Browse source 
- add support for elasticsearch 7.x
- remove support for elasticsearch 5.x
- update kitchen-ansible configuration (install ansible and jmespath dependencies using os repositories)
- replace geoip plugin in tests as this one is now embeded in elasticsearch since 6.7.0 (cf. https://www.elastic.co/guide/en/elasticsearch/plugins/6.7/ingest-geoip.html)
- update discovery configuration for 7.x (in ES 7.x, discovery.zen.ping.unicast.hosts is replaced by discovery.seed_hosts and transport.tcp.port is replaced by transport.port, also discovery.seed_hosts is disabled on master nodes to avoid "master_not_discovered_exception" error when creating templates in the same play)
- update index template structure for 7.x 
- update security realms settings for 7.x (cf. https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#include-realm-type-in-setting)
This commit is contained in:
Julien Mailleret 2019-05-09 09:06:02 +02:00 • committed by GitHub
parent adba13bcd8
commit a1c81884e2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
22 changed files with 139 additions and 175 deletions
Download patch file Download diff file Expand all files Collapse all files

24
test/integration/helpers/serverspec/multi_spec.rb
View file

@ -11,16 +11,16 @@ shared_examples 'multi::init' do |vars|
describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do
it { should be_file }
it { should contain 'http.port: 9201' }
it { should contain 'transport.tcp.port: 9301' }
if vars['es_major_version'] == '7.x'
it { should contain 'transport.port: 9301' }
else
it { should contain 'transport.tcp.port: 9301' }
end
it { should contain 'node.data: true' }
it { should contain 'node.master: false' }
it { should contain "node.name: localhost-#{vars['es_instance_name']}" }
it { should_not contain 'bootstrap.memory_lock: true' }
if vars['es_major_version'] == '6.x'
it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
else
it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
end
it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
it { should contain "path.data: /opt/elasticsearch/data-1/localhost-#{vars['es_instance_name']},/opt/elasticsearch/data-2/localhost-#{vars['es_instance_name']}" }
it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" }
end
@ -30,16 +30,16 @@ shared_examples 'multi::init' do |vars|
describe file('/etc/elasticsearch/master/elasticsearch.yml') do
it { should be_file }
it { should contain 'http.port: 9200' }
it { should contain 'transport.tcp.port: 9300' }
if vars['es_major_version'] == '7.x'
it { should contain 'transport.port: 9300' }
else
it { should contain 'transport.tcp.port: 9300' }
end
it { should contain 'node.data: false' }
it { should contain 'node.master: true' }
it { should contain 'node.name: localhost-master' }
it { should contain 'bootstrap.memory_lock: true' }
if vars['es_major_version'] == '6.x'
it { should_not contain 'path.conf: /etc/elasticsearch/master' }
else
it { should contain 'path.conf: /etc/elasticsearch/master' }
end
it { should_not contain 'path.conf: /etc/elasticsearch/master' }
it { should contain 'path.data: /opt/elasticsearch/master/localhost-master' }
it { should contain 'path.logs: /var/log/elasticsearch/localhost-master' }
end

38
test/integration/helpers/serverspec/shared_spec.rb
View file

@ -43,32 +43,6 @@ shared_examples 'shared::init' do |vars|
expect(values['enabled'] = enabled)
end
end
# X-Pack is no longer installed as a plugin in elasticsearch
if vars['es_major_version'] == '5.x'
describe file('/usr/share/elasticsearch/plugins/x-pack') do
it { should be_directory }
it { should be_owned_by vars['es_user'] }
end
describe file("/etc/elasticsearch/#{vars['es_instance_name']}/x-pack") do
it { should be_directory }
it { should be_owned_by vars['es_user'] }
end
describe 'x-pack-core plugin' do
it 'should be installed with the correct version' do
plugins = curl_json("#{es_api_url}/_nodes/plugins", username=username, password=password)
node, data = plugins['nodes'].first
version = 'plugin not found'
name = 'x-pack'
data['plugins'].each do |plugin|
if plugin['name'] == name
version = plugin['version']
end
end
expect(version).to eql(vars['es_version'])
end
end
end
end
end
describe user(vars['es_user']) do
@ -108,7 +82,11 @@ shared_examples 'shared::init' do |vars|
template = curl_json("#{es_api_url}/_template/basic", username=username, password=password)
expect(template.key?('basic'))
expect(template['basic']['settings']['index']['number_of_shards']).to eq("1")
expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false)
if vars['es_major_version'] == '7.x'
expect(template['basic']['mappings']['_source']['enabled']).to eq(false)
else
expect(template['basic']['mappings']['type1']['_source']['enabled']).to eq(false)
end
end
end
end
@ -159,11 +137,7 @@ shared_examples 'shared::init' do |vars|
describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do
it { should contain "node.name: localhost-#{vars['es_instance_name']}" }
it { should contain 'cluster.name: elasticsearch' }
if vars['es_major_version'] == '6.x'
it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
else
it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
end
it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
its(:content) { should match "path.data: #{vars['data_dirs'].join(',')}" }
its(:content) { should match "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" }
end

6
test/integration/helpers/serverspec/xpack_spec.rb
View file

@ -4,11 +4,7 @@ shared_examples 'xpack::init' do |vars|
describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do
it { should contain "node.name: localhost-#{vars['es_instance_name']}" }
it { should contain 'cluster.name: elasticsearch' }
if vars['es_major_version'] == '6.x'
it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
else
it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
end
it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
it { should contain "path.data: /var/lib/elasticsearch/localhost-#{vars['es_instance_name']}" }
it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" }
it { should contain 'xpack.security.enabled: false' }

28
test/integration/helpers/serverspec/xpack_upgrade_spec.rb
View file

@ -6,11 +6,7 @@ shared_examples 'xpack_upgrade::init' do |vars|
describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do
it { should contain "node.name: localhost-#{vars['es_instance_name']}" }
it { should contain 'cluster.name: elasticsearch' }
if vars['es_major_version'] == '6.x'
it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
else
it { should contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
end
it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
it { should contain "path.data: /var/lib/elasticsearch/localhost-#{vars['es_instance_name']}" }
it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" }
end
@ -36,10 +32,15 @@ shared_examples 'xpack_upgrade::init' do |vars|
end
describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do
it { should contain 'security.authc.realms.file1.order: 0' }
it { should contain 'security.authc.realms.file1.type: file' }
it { should contain 'security.authc.realms.native1.order: 1' }
it { should contain 'security.authc.realms.native1.type: native' }
if vars['es_major_version'] == '7.x'
it { should contain 'security.authc.realms.file.file1.order: 0' }
it { should contain 'security.authc.realms.native.native1.order: 1' }
else
it { should contain 'security.authc.realms.file1.order: 0' }
it { should contain 'security.authc.realms.file1.type: file' }
it { should contain 'security.authc.realms.native1.order: 1' }
it { should contain 'security.authc.realms.native1.type: native' }
end
end
#Test contents of role_mapping.yml
@ -91,13 +92,4 @@ shared_examples 'xpack_upgrade::init' do |vars|
expect(command.exit_status).to eq(0)
end
end
if vars['es_major_version'] == '5.x' # kibana default password has been removed in 6.x
describe 'kibana access check' do
it 'should be reported as version '+vars['es_version'] do
result = curl_json('http://localhost:9200/', username='kibana', password='changeme')
expect(result['version']['number']).to eq(vars['es_version'])
end
end
end
end

8
test/integration/issue-test.yml
View file

@ -14,14 +14,12 @@
es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}"
es_config:
xpack.security.enabled: True
xpack.security.authc.realms.file1.type: "file"
xpack.security.authc.realms.file1.order: 1
xpack.security.authc.realms.native1.type: "native"
xpack.security.authc.realms.native1.order: 0
xpack.security.authc.realms.file.file1.order: 1
xpack.security.authc.realms.native.native1.type: "native"
es_heap_size: "1g"
es_enable_xpack: true
es_plugins:
- plugin: ingest-geoip
- plugin: ingest-attachment
es_xpack_features:
- security
- alerting

24
test/integration/multi.yml
View file

@ -10,20 +10,27 @@
es_instance_name: "master"
es_data_dirs:
- "/opt/elasticsearch/master"
es_config:
es_config_6x:
discovery.zen.ping.unicast.hosts: "localhost:9300"
http.port: 9200
transport.tcp.port: 9300
node.data: false
node.master: true
bootstrap.memory_lock: true
es_config_7x:
http.port: 9200
transport.port: 9300
node.data: false
node.master: true
bootstrap.memory_lock: true
es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}"
es_enable_xpack: false
es_scripts: true
es_templates: true
es_heap_size: "1g"
es_api_port: 9200
es_plugins:
- plugin: ingest-geoip
- plugin: ingest-attachment
- name: Elasticsearch Multi test - data on 9201
hosts: localhost
@ -38,14 +45,21 @@
es_heap_size: "1g"
es_api_port: 9201
es_plugins:
- plugin: ingest-geoip
- plugin: ingest-attachment
es_instance_name: "node1"
es_data_dirs:
es_data_dirs:
- "/opt/elasticsearch/data-1"
- "/opt/elasticsearch/data-2"
es_config:
es_config_6x:
discovery.zen.ping.unicast.hosts: "localhost:9300"
http.port: 9201
transport.tcp.port: 9301
node.data: true
node.master: false
es_config_7x:
discovery.seed_hosts: "localhost:9300"
http.port: 9201
transport.port: 9301
node.data: true
node.master: false
es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}"

2
test/integration/oss-to-xpack-upgrade.yml
View file

@ -7,7 +7,7 @@
- elasticsearch
vars:
es_instance_name: "node1"
es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade
es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade
es_enable_xpack: false
es_heap_size: "1g"

2
test/integration/oss-upgrade.yml
View file

@ -7,7 +7,7 @@
- elasticsearch
vars:
es_instance_name: "node1"
es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade
es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade
es_enable_xpack: false
es_heap_size: "1g"

2
test/integration/oss.yml
View file

@ -10,6 +10,6 @@
es_enable_xpack: false
es_heap_size: "1g"
es_plugins:
- plugin: ingest-geoip
- plugin: ingest-attachment
#Do not add tests here. This test is run twice and confirms idempotency.

34
test/integration/xpack-upgrade.yml
View file

@ -8,21 +8,26 @@
vars:
es_instance_name: "node1"
es_api_port: 9200
es_config:
es_config_6x:
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: "localhost:9300"
xpack.security.authc.realms.file1.type: "file"
xpack.security.authc.realms.file1.order: 0
xpack.security.authc.realms.native1.type: "native"
xpack.security.authc.realms.file1.type: file
xpack.security.authc.realms.native1.order: 1
xpack.security.authc.realms.native1.type: native
es_config_7x:
http.port: 9200
xpack.security.enabled: True
xpack.security.authc.realms.file.file1.order: 0
xpack.security.authc.realms.native.native1.order: 1
es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}"
es_heap_size: "1g"
es_templates: true
es_version: "{{ '6.2.4' if es_major_version == '6.x' else '5.6.9' }}" # This is set to an older version than the current default to force an upgrade
es_major_version: "7.x"
es_version: "{{ '7.0.0' if es_major_version == '7.x' else '6.7.1' }}" # This is set to an older version than the current default to force an upgrade
es_enable_xpack: true
es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}"
es_plugins:
- plugin: ingest-geoip
- plugin: ingest-attachment
es_xpack_features:
- security
- alerting
@ -118,15 +123,18 @@
vars:
es_api_port: 9200
es_instance_name: "node1"
es_config:
es_config_6x:
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: "localhost:9300"
xpack.security.enabled: True
xpack.security.authc.realms.file1.type: "file"
xpack.security.authc.realms.file1.order: 0
xpack.security.authc.realms.native1.type: "native"
xpack.security.authc.realms.file1.type: file
xpack.security.authc.realms.native1.order: 1
xpack.security.authc.realms.native1.type: native
es_config_7x:
http.port: 9200
xpack.security.enabled: True
xpack.security.authc.realms.file.file1.order: 0
xpack.security.authc.realms.native.native1.order: 1
es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}"
es_heap_size: "1g"
es_templates: true
es_enable_xpack: true

2
test/integration/xpack.yml
View file

@ -11,8 +11,6 @@
es_instance_name: "node1"
es_config:
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: "localhost:9300"
es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip"
es_heap_size: 2g
es_enable_xpack: true