From 5c4c76e08876c6aef513330360e213789f7394cb Mon Sep 17 00:00:00 2001 From: Janne Salo Date: Thu, 8 Dec 2016 14:24:28 +0200 Subject: [PATCH 01/42] Updated yum repository URL for 5.x --- templates/elasticsearch.repo | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/elasticsearch.repo b/templates/elasticsearch.repo index 9b188e6..83e1920 100644 --- a/templates/elasticsearch.repo +++ b/templates/elasticsearch.repo @@ -1,6 +1,8 @@ [elasticsearch-{{ es_major_version }}] name=Elasticsearch repository for {{ es_major_version }} packages -baseurl=http://packages.elastic.co/elasticsearch/{{ es_major_version }}/centos +baseurl={% if es_version | version_compare('5.0', '>=') %}https://artifacts.elastic.co/packages/{{ es_major_version }}/yum{% else %}http://packages.elastic.co/elasticsearch/{{ es_major_version }}/centos{% endif %} + gpgcheck=1 -gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch +gpgkey={% if es_version | version_compare('5.0', '>=') %}https://artifacts.elastic.co/GPG-KEY-elasticsearch{% else %}http://packages.elastic.co/GPG-KEY-elasticsearch{% endif %} + enabled=1 From 06d156fd578fd20d0075bb892cfbd2135e83d4c7 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Fri, 6 Jan 2017 11:24:52 +0000 Subject: [PATCH 02/42] Moving to 5.x tests --- .kitchen.yml | 10 +++++----- meta/main.yml | 2 +- test/integration/{config-2x => config-5x}/config.yml | 0 .../serverspec/default_spec.rb | 0 test/integration/{multi-2x => multi-5x}/multi.yml | 0 .../{multi-2x => multi-5x}/serverspec/default_spec.rb | 0 .../{package-2x => package-5x}/package.yaml | 0 .../serverspec/default_spec.rb | 0 .../serverspec/default_spec.rb | 0 .../{standard-2x => standard-5x}/standard.yml | 0 .../{xpack-2x => xpack-5x}/serverspec/default_spec.rb | 0 test/integration/{xpack-2x => xpack-5x}/xpack.yml | 0 12 files changed, 6 insertions(+), 6 deletions(-) rename test/integration/{config-2x => config-5x}/config.yml (100%) rename test/integration/{config-2x => config-5x}/serverspec/default_spec.rb (100%) rename test/integration/{multi-2x => multi-5x}/multi.yml (100%) rename test/integration/{multi-2x => multi-5x}/serverspec/default_spec.rb (100%) rename test/integration/{package-2x => package-5x}/package.yaml (100%) rename test/integration/{package-2x => package-5x}/serverspec/default_spec.rb (100%) rename test/integration/{standard-2x => standard-5x}/serverspec/default_spec.rb (100%) rename test/integration/{standard-2x => standard-5x}/standard.yml (100%) rename test/integration/{xpack-2x => xpack-5x}/serverspec/default_spec.rb (100%) rename test/integration/{xpack-2x => xpack-5x}/xpack.yml (100%) diff --git a/.kitchen.yml b/.kitchen.yml index 0284b0b..4c09c0a 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -64,12 +64,12 @@ platforms: use_sudo: false suites: - - name: standard-2x + - name: standard-5x provisioner: playbook: test/integration/standard.yml run_list: attributes: - - name: package-2x + - name: package-5x run_list: attributes: extra_vars: @@ -78,12 +78,12 @@ suites: version: master provisioner: playbook: test/integration/package.yml - - name: config-2x + - name: config-5x run_list: attributes: provisioner: playbook: test/integration/config.yml - - name: multi-2x + - name: multi-5x run_list: attributes: extra_vars: @@ -92,7 +92,7 @@ suites: version: master provisioner: playbook: test/integration/multi.yml - - name: xpack-2x + - name: xpack-5x run_list: attributes: provisioner: diff --git a/meta/main.yml b/meta/main.yml index fe062a5..edd7295 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -8,7 +8,7 @@ galaxy_info: company: "Elastic.co" license: "license (Apache)" # Require 1.6 for apt deb install - min_ansible_version: 2.1.0 + min_ansible_version: 2.2.0 platforms: - name: EL versions: diff --git a/test/integration/config-2x/config.yml b/test/integration/config-5x/config.yml similarity index 100% rename from test/integration/config-2x/config.yml rename to test/integration/config-5x/config.yml diff --git a/test/integration/config-2x/serverspec/default_spec.rb b/test/integration/config-5x/serverspec/default_spec.rb similarity index 100% rename from test/integration/config-2x/serverspec/default_spec.rb rename to test/integration/config-5x/serverspec/default_spec.rb diff --git a/test/integration/multi-2x/multi.yml b/test/integration/multi-5x/multi.yml similarity index 100% rename from test/integration/multi-2x/multi.yml rename to test/integration/multi-5x/multi.yml diff --git a/test/integration/multi-2x/serverspec/default_spec.rb b/test/integration/multi-5x/serverspec/default_spec.rb similarity index 100% rename from test/integration/multi-2x/serverspec/default_spec.rb rename to test/integration/multi-5x/serverspec/default_spec.rb diff --git a/test/integration/package-2x/package.yaml b/test/integration/package-5x/package.yaml similarity index 100% rename from test/integration/package-2x/package.yaml rename to test/integration/package-5x/package.yaml diff --git a/test/integration/package-2x/serverspec/default_spec.rb b/test/integration/package-5x/serverspec/default_spec.rb similarity index 100% rename from test/integration/package-2x/serverspec/default_spec.rb rename to test/integration/package-5x/serverspec/default_spec.rb diff --git a/test/integration/standard-2x/serverspec/default_spec.rb b/test/integration/standard-5x/serverspec/default_spec.rb similarity index 100% rename from test/integration/standard-2x/serverspec/default_spec.rb rename to test/integration/standard-5x/serverspec/default_spec.rb diff --git a/test/integration/standard-2x/standard.yml b/test/integration/standard-5x/standard.yml similarity index 100% rename from test/integration/standard-2x/standard.yml rename to test/integration/standard-5x/standard.yml diff --git a/test/integration/xpack-2x/serverspec/default_spec.rb b/test/integration/xpack-5x/serverspec/default_spec.rb similarity index 100% rename from test/integration/xpack-2x/serverspec/default_spec.rb rename to test/integration/xpack-5x/serverspec/default_spec.rb diff --git a/test/integration/xpack-2x/xpack.yml b/test/integration/xpack-5x/xpack.yml similarity index 100% rename from test/integration/xpack-2x/xpack.yml rename to test/integration/xpack-5x/xpack.yml From dffb17e59a0c87aaa7b927e6d2b298857bd7d3d1 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Fri, 6 Jan 2017 11:34:19 +0000 Subject: [PATCH 03/42] Tests moved to 5.x --- README.md | 8 ++++---- defaults/main.yml | 4 ++-- test/integration/config-5x/serverspec/default_spec.rb | 4 ++-- test/integration/multi-5x/serverspec/default_spec.rb | 4 ++-- test/integration/package-5x/serverspec/default_spec.rb | 4 ++-- test/integration/standard-5x/serverspec/default_spec.rb | 4 ++-- test/integration/xpack-5x/serverspec/default_spec.rb | 4 ++-- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index bd5b208..3d7831e 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The use of a map ensures the Ansible playbook does not need to be updated to ref In addition to the es_config map, several other parameters are supported for additional functions e.g. script installation. These can be found in the role's defaults/main.yml file. -The following illustrates applying configuration parameters to an Elasticsearch instance. By default, Elasticsearch 2.4.3 is installed. +The following illustrates applying configuration parameters to an Elasticsearch instance. By default, Elasticsearch 5.1.1 is installed. ``` - name: Elasticsearch with custom configuration @@ -211,7 +211,7 @@ ansible-playbook -i hosts ./your-playbook.yml ### Installing X-Pack Features -X-Pack features, such as Shield, are supported for Elasticsearch 2.4 only. This feature is currently experimental. To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`. The following additional parameters allow X-Pack to be configured: +X-Pack features, such as Security, are supported. This feature is currently experimental. To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`. The following additional parameters allow X-Pack to be configured: * ```es_message_auth_file``` System Key field to allow message authentication. This file should be placed in the 'files' directory. * ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/shield/current/mapping-roles.html) @@ -320,7 +320,7 @@ Following variables affect the versions installed: ```yml es_plugins: - plugin: elasticsearch-cloud-aws - version: 2.5.0 + version: 5.0.0 ``` * ```es_allow_downgrades``` For development purposes only. (true or false (default) ) * ```es_java_install``` If set to false, Java will not be installed. (true (default) or false) @@ -368,7 +368,7 @@ To define proxy only for a particular plugin during its installation: ``` es_plugins: - plugin: elasticsearch-cloud-aws - version: 2.5.0 + version: 5.0.0 proxy_host: proxy.example.com proxy_port: 8080 ``` diff --git a/defaults/main.yml b/defaults/main.yml index dfa55da..faaffc5 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,6 +1,6 @@ --- -es_major_version: "2.x" -es_version: "2.4.3" +es_major_version: "5.x" +es_version: "5.1.1" es_version_lock: false es_use_repository: true es_apt_key: "https://packages.elasticsearch.org/GPG-KEY-elasticsearch" diff --git a/test/integration/config-5x/serverspec/default_spec.rb b/test/integration/config-5x/serverspec/default_spec.rb index c7d6685..6e9c4af 100644 --- a/test/integration/config-5x/serverspec/default_spec.rb +++ b/test/integration/config-5x/serverspec/default_spec.rb @@ -1,6 +1,6 @@ require 'config_spec' -describe 'Config Tests v 2.x' do - include_examples 'config::init', "2.4.3" +describe 'Config Tests v 5.x' do + include_examples 'config::init', "5.1.1" end diff --git a/test/integration/multi-5x/serverspec/default_spec.rb b/test/integration/multi-5x/serverspec/default_spec.rb index 2772909..4d54c16 100644 --- a/test/integration/multi-5x/serverspec/default_spec.rb +++ b/test/integration/multi-5x/serverspec/default_spec.rb @@ -1,8 +1,8 @@ require 'multi_spec' -describe 'Multi Tests v 2.x' do - include_examples 'multi::init', "2.4.3", ["kopf"] +describe 'Multi Tests v 5.x' do + include_examples 'multi::init', "5.1.1", ["kopf"] end diff --git a/test/integration/package-5x/serverspec/default_spec.rb b/test/integration/package-5x/serverspec/default_spec.rb index b452119..72cbfbc 100644 --- a/test/integration/package-5x/serverspec/default_spec.rb +++ b/test/integration/package-5x/serverspec/default_spec.rb @@ -1,6 +1,6 @@ require 'package_spec' -describe 'Package Tests v 2.x' do - include_examples 'package::init', "2.4.3", ["kopf"] +describe 'Package Tests v 5.x' do + include_examples 'package::init', "5.1.1", ["kopf"] end \ No newline at end of file diff --git a/test/integration/standard-5x/serverspec/default_spec.rb b/test/integration/standard-5x/serverspec/default_spec.rb index 853d28d..db96847 100644 --- a/test/integration/standard-5x/serverspec/default_spec.rb +++ b/test/integration/standard-5x/serverspec/default_spec.rb @@ -1,8 +1,8 @@ require 'standard_spec' -describe 'Standard Tests v 2.x' do - include_examples 'standard::init', "2.4.3" +describe 'Standard Tests v 5.x' do + include_examples 'standard::init', "5.1.1" end diff --git a/test/integration/xpack-5x/serverspec/default_spec.rb b/test/integration/xpack-5x/serverspec/default_spec.rb index e372a45..3eb0df7 100644 --- a/test/integration/xpack-5x/serverspec/default_spec.rb +++ b/test/integration/xpack-5x/serverspec/default_spec.rb @@ -1,5 +1,5 @@ require 'xpack_spec' -describe 'Xpack Tests v 2.x' do - include_examples 'xpack::init', "2.4.3" +describe 'Xpack Tests v 5.x' do + include_examples 'xpack::init', "5.1.1" end From 02bb86a9f2a404b6eb9170ee6e190a577002d187 Mon Sep 17 00:00:00 2001 From: Benjamin Daschel Date: Sun, 8 Jan 2017 15:31:11 -0800 Subject: [PATCH 04/42] quote interpolated variables in command --- tasks/xpack/elasticsearch-xpack-install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml index 72595b7..88b439f 100644 --- a/tasks/xpack/elasticsearch-xpack-install.yml +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -13,7 +13,7 @@ #Remove Plugin if installed and its not been requested or the ES version has changed - name: Remove {{item}} plugin - command: {{es_home}}/bin/plugin remove {{item}} + command: "{{es_home}}/bin/plugin remove {{item}}" register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 From 84913049c561a1601bcb0fc45f104e4f0c21b85a Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 10 Jan 2017 16:45:40 +0000 Subject: [PATCH 05/42] Role is 5.x only so java 8 --- tasks/elasticsearch.yml | 4 ---- vars/Debian.yml | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tasks/elasticsearch.yml b/tasks/elasticsearch.yml index 71ad9e1..8fe5895 100644 --- a/tasks/elasticsearch.yml +++ b/tasks/elasticsearch.yml @@ -9,10 +9,6 @@ when: (es_user_id is defined) and (es_group_id is defined) include: elasticsearch-optional-user.yml -#- name: Include specific Elasticsearch -# include: "elasticsearch-{{ansible_os_family}}.yml" - -#Install OS specific elasticsearch - this can be abbreviated in version 2.0.0 - name: Include specific Elasticsearch include: elasticsearch-Debian.yml when: ansible_os_family == 'Debian' diff --git a/vars/Debian.yml b/vars/Debian.yml index cfa73ae..071736e 100644 --- a/vars/Debian.yml +++ b/vars/Debian.yml @@ -1,4 +1,4 @@ --- -java: "{% if es_java is defined %}{{es_java}}{%elif (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15.10', '>=')) %}openjdk-8-jre-headless{% else %}openjdk-7-jre-headless{% endif %}" +java: "{% if es_java is defined %}{{es_java}}{% else %}openjdk-8-jre-headless{% endif %}" default_file: "/etc/default/elasticsearch" es_home: "/usr/share/elasticsearch" From 55444f3cbea83f4f19972f8191e73a222cce1e3e Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 10 Jan 2017 16:50:04 +0000 Subject: [PATCH 06/42] systemd file for 5.x only --- templates/systemd/elasticsearch.j2 | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/templates/systemd/elasticsearch.j2 b/templates/systemd/elasticsearch.j2 index 8043360..204fa49 100644 --- a/templates/systemd/elasticsearch.j2 +++ b/templates/systemd/elasticsearch.j2 @@ -17,26 +17,22 @@ WorkingDirectory={{es_home}} User={{es_user}} Group={{es_group}} -{% if es_version | version_compare('2.0', '>=') %} ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec -{% endif %} ExecStart={{es_home}}/bin/elasticsearch \ -{% if es_version | version_compare('5.0', '<') %} - -Des.pidfile=${PID_DIR}/elasticsearch.pid \ - -Des.default.path.home=${ES_HOME} \ - -Des.default.path.logs=${LOG_DIR} \ - -Des.default.path.data=${DATA_DIR} \ - -Des.default.path.conf=${CONF_DIR} -{% else %} - -p ${PID_DIR}/elasticsearch.pid \ - -Edefault.path.home=${ES_HOME} \ - -Edefault.path.logs=${LOG_DIR} \ - -Edefault.path.data=${DATA_DIR} \ - -Edefault.path.conf=${CONF_DIR} -{% endif %} + -p ${PID_DIR}/elasticsearch.pid \ + --quiet \ + -Edefault.path.logs=${LOG_DIR} \ + -Edefault.path.data=${DATA_DIR} \ + -Edefault.path.conf=${CONF_DIR} +# StandardOutput is configured to redirect to journalctl since +# some error messages may be logged in standard output before +# elasticsearch logging system is initialized. Elasticsearch +# stores its logs in /var/log/elasticsearch and does not use +# journalctl by default. If you also want to enable journalctl +# logging, you can simply remove the "quiet" option from ExecStart. StandardOutput=journal StandardError=inherit From 2d7b851b1fd6d963fb526cc4bec285cb62eed315 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 10 Jan 2017 17:09:43 +0000 Subject: [PATCH 07/42] Parameters refactored for 5.x --- tasks/elasticsearch-parameters.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 356dc9a..73c5c55 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -6,25 +6,19 @@ - fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined" when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '') -- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or (es_config['discovery.zen.ping.multicast.enabled'] is defined and es_config['discovery.zen.ping.multicast.enabled'])}} +- debug: msg="WARNING - It is recommended you specify the parameter 'http.port'" + when: es_config['http.port'] is not defined -- debug: msg="WARNING - It is recommended you specify the parameter 'http.port' when multicast is disabled" - when: not multi_cast and es_config['http.port'] is not defined +- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port'" + when: es_config['transport.tcp.port'] is not defined -- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port' when multicast is disabled" - when: not multi_cast and es_config['transport.tcp.port'] is not defined - -- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts' when multicast is disabled" - when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined +- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts'" + when: es_config['discovery.zen.ping.unicast.hosts'] is not defined #If the user attempts to lock memory they must specify a heap size - fail: msg="If locking memory with bootstrap.mlockall (or bootstrap.memory_lock) a heap size must be specified" when: (es_config['bootstrap.mlockall'] is defined or es_config['bootstrap.memory_lock'] is defined) and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined -#Don't support xpack on versions < 2.0 -- fail: msg="Use of the xpack notation is not supported on versions < 2.0. Marvel-agent and watcher can be installed as plugins. Version > 2.0 is required for shield." - when: es_enable_xpack and es_version | version_compare('2.0', '<') - #Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work - fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" when: es_enable_xpack and ("shield" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined @@ -40,7 +34,6 @@ #Centos 7 and up #Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied. - - set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }} - set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}} From cde17346d57117329d37487b1da0746b969a5c65 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Wed, 11 Jan 2017 10:26:57 +0000 Subject: [PATCH 08/42] Version 5.0 - no need for plugin prefix --- tasks/elasticsearch-plugins.yml | 6 +++--- tasks/elasticsearch.yml | 5 ----- tasks/java.yml | 1 - tasks/main.yml | 1 + tasks/xpack/elasticsearch-xpack-install.yml | 4 ++-- tasks/xpack/elasticsearch-xpack.yml | 6 +++--- 6 files changed, 9 insertions(+), 14 deletions(-) diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 21fd92c..dd22194 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -12,7 +12,7 @@ #List currently installed plugins - ignore xpack if > v 2.0 - name: Check installed elasticsearch plugins - shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin {{list_command}}{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %}{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}" + shell: "{{es_home}}/bin/elasticsearch-plugin {{list_command}}{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %}{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}" register: installed_plugins failed_when: "'ERROR' in installed_plugins.stdout" changed_when: False @@ -23,7 +23,7 @@ # This removes any currently installed plugins (to prevent errors when reinstalling) - name: Remove elasticsearch plugins - command: "{{es_home}}/bin/{{ es_binary_prefix }}plugin remove {{item}} --silent" + command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent" ignore_errors: yes with_items: "{{ installed_plugins.stdout_lines | default([]) }}" when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0] @@ -35,7 +35,7 @@ - name: Install elasticsearch plugins command: > - {{es_home}}/bin/{{ es_binary_prefix }}plugin install {{ item.plugin }}{% if item.version is defined and item.version != '' %}/{{ item.version }}{% endif %} {% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -DproxyHost={{ item.proxy_host }} -DproxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -DproxyHost={{ es_proxy_host }} -DproxyPort={{ es_proxy_port }} {% endif %} --silent + {{es_home}}/bin/elasticsearch-plugin install {{ item.plugin }}{% if item.version is defined and item.version != '' %}/{{ item.version }}{% endif %} {% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -DproxyHost={{ item.proxy_host }} -DproxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -DproxyHost={{ es_proxy_host }} -DproxyPort={{ es_proxy_port }} {% endif %} --silent register: plugin_installed failed_when: "'ERROR' in plugin_installed.stdout" changed_when: plugin_installed.rc == 0 diff --git a/tasks/elasticsearch.yml b/tasks/elasticsearch.yml index 8fe5895..e2361d4 100644 --- a/tasks/elasticsearch.yml +++ b/tasks/elasticsearch.yml @@ -1,10 +1,5 @@ --- -- set_fact: es_binary_prefix="" - -- set_fact: es_binary_prefix="elasticsearch-" - when: es_version | version_compare('5.0', '>=') - - name: Include optional user and group creation. when: (es_user_id is defined) and (es_group_id is defined) include: elasticsearch-optional-user.yml diff --git a/tasks/java.yml b/tasks/java.yml index ed49fb1..a940cc2 100644 --- a/tasks/java.yml +++ b/tasks/java.yml @@ -1,6 +1,5 @@ --- - - set_fact: java_state="present" - set_fact: java_state="latest" diff --git a/tasks/main.yml b/tasks/main.yml index ebf67b3..e8395da 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -13,6 +13,7 @@ when: es_java_install tags: - java + - include: elasticsearch.yml tags: - install diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml index 88b439f..c309421 100644 --- a/tasks/xpack/elasticsearch-xpack-install.yml +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -1,7 +1,7 @@ --- #Test if feature is installed -- shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin list{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %} | grep {{item}}" +- shell: "{{es_home}}/bin/elasticsearch-plugin list{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %} | grep {{item}}" register: feature_installed changed_when: False failed_when: "'ERROR' in feature_installed.stdout" @@ -27,7 +27,7 @@ #Install plugin if not installed, or the es version has changed (so removed above), and its been requested - name: Install {{item}} plugin command: > - {{es_home}}/bin/{{ es_binary_prefix }}plugin install {{item}} + {{es_home}}/bin/elasticsearch-plugin install {{item}} register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index f081e5f..4dac838 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -7,7 +7,7 @@ #Check if license is installed - name: Check License plugin is installed shell: > - {{es_home}}/bin/{{ es_binary_prefix }}plugin list | tail -n +2 | grep license + {{es_home}}/bin/elasticsearch-plugin list | tail -n +2 | grep license register: license_installed ignore_errors: yes failed_when: "'ERROR' in license_installed.stdout" @@ -19,7 +19,7 @@ #Remove license if installed and xpack not enabled - name: Remove license plugin command: > - {{es_home}}/bin/{{ es_binary_prefix }}plugin remove license + {{es_home}}/bin/elasticsearch-plugin remove license register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 @@ -32,7 +32,7 @@ #Install License if not installed, or it needs to be reinstalled due to ES change (above task will have removed), and its been requested. - name: Install license plugin command: > - {{es_home}}/bin/{{ es_binary_prefix }}plugin install license + {{es_home}}/bin/elasticsearch-plugin install license register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 From 86bc009b6053a022137151c8eb80bb95134c3ffb Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Wed, 11 Jan 2017 12:17:16 +0000 Subject: [PATCH 09/42] Logging configs for 5.0 only --- tasks/elasticsearch-config.yml | 7 ------- tasks/elasticsearch-parameters.yml | 3 +++ tasks/main.yml | 5 +++++ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 7336041..8b16414 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -48,21 +48,14 @@ - restart elasticsearch #Copy the logging.yml -- name: Copy Logging.yml File for Instance - template: src=logging.yml.j2 dest={{conf_dir}}/logging.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes - notify: restart elasticsearch - when: es_version | version_compare('5.0', '<') - name: Copy log4j2.properties File for Instance template: src=log4j2.properties.j2 dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes notify: restart elasticsearch - when: es_version | version_compare('5.0', '>=') -# Copy the JVM Options (5.x only) - name: Copy jvm.options File for Instance template: src=jvm.options.j2 dest={{conf_dir}}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes notify: restart elasticsearch - when: es_version | version_compare('5.0', '>=') #Clean up un-wanted package scripts to avoid confusion diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 73c5c55..b3f0146 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -28,6 +28,9 @@ - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} - set_fact: m_lock_enabled={{ (es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True) or (es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True) }} +#TODO - if transport.host is not local maybe error on boostrap checks + + #Use systemd for the following distributions: #Ubuntu 15 and up #Debian 8 and up diff --git a/tasks/main.yml b/tasks/main.yml index e8395da..bcd2df0 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -17,21 +17,26 @@ - include: elasticsearch.yml tags: - install + - include: elasticsearch-config.yml tags: - config + - include: elasticsearch-scripts.yml when: es_scripts tags: - scripts + - include: elasticsearch-plugins.yml when: es_plugins is defined or es_plugins_reinstall tags: - plugins + #We always execute xpack as we may need to remove features - include: xpack/elasticsearch-xpack.yml tags: - xpack + - include: elasticsearch-templates.yml when: es_templates tags: From 197cf05a0e7e94aedb0214e27776d64e14a1854c Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Wed, 11 Jan 2017 13:02:23 +0000 Subject: [PATCH 10/42] Shield to Security and other X-Pack clear up --- README.md | 13 +++-- handlers/elasticsearch-templates.yml | 4 +- handlers/main.yml | 6 +-- .../elasticsearch-security-native.yml} | 14 ++--- .../elasticsearch-security.yml} | 4 +- .../elasticsearch-xpack-activation.yml | 8 +-- tasks/elasticsearch-config.yml | 4 +- tasks/elasticsearch-parameters.yml | 6 +-- tasks/xpack/elasticsearch-xpack.yml | 4 +- ...le.yml => elasticsearch-security-file.yml} | 26 ++++----- tasks/xpack/shield/elasticsearch-security.yml | 36 +++++++++++++ tasks/xpack/shield/elasticsearch-shield.yml | 36 ------------- .../{shield => security}/role_mapping.yml.j2 | 0 templates/{shield => security}/roles.yml.j2 | 0 templates/{shield => security}/users_roles.j2 | 0 .../helpers/serverspec/xpack_spec.rb | 54 +++++++++---------- test/integration/xpack.yml | 8 +-- vars/main.yml | 2 +- 18 files changed, 112 insertions(+), 113 deletions(-) rename handlers/{shield/elasticsearch-shield-native.yml => security/elasticsearch-security-native.yml} (82%) rename handlers/{shield/elasticsearch-shield.yml => security/elasticsearch-security.yml} (78%) rename handlers/{shield => security}/elasticsearch-xpack-activation.yml (82%) rename tasks/xpack/shield/{elasticsearch-shield-file.yml => elasticsearch-security-file.yml} (62%) create mode 100644 tasks/xpack/shield/elasticsearch-security.yml delete mode 100644 tasks/xpack/shield/elasticsearch-shield.yml rename templates/{shield => security}/role_mapping.yml.j2 (100%) rename templates/{shield => security}/roles.yml.j2 (100%) rename templates/{shield => security}/users_roles.j2 (100%) diff --git a/README.md b/README.md index 3d7831e..848804c 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,6 @@ A more complex example: es_plugins_reinstall: false es_plugins: - plugin: license - - plugin: marvel-agent - plugin: lmenezes/elasticsearch-kopf version: master proxy_host: proxy.example.com @@ -214,7 +213,7 @@ ansible-playbook -i hosts ./your-playbook.yml X-Pack features, such as Security, are supported. This feature is currently experimental. To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`. The following additional parameters allow X-Pack to be configured: * ```es_message_auth_file``` System Key field to allow message authentication. This file should be placed in the 'files' directory. -* ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/shield/current/mapping-roles.html) +* ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/x-pack/current/mapping-roles.html) ``` es_role_mapping: @@ -247,7 +246,7 @@ es_users: ``` -* ```es_roles``` - Elasticsearch roles can be declared here as yml. Two sub keys 'native' and 'file' determine how the role is created i.e. either through a file or http(native) call. Beneath each key list the roles with appropriate permissions, using the file based format described [here] (https://www.elastic.co/guide/en/shield/current/_file_based_roles.html) e.g. +* ```es_roles``` - Elasticsearch roles can be declared here as yml. Two sub keys 'native' and 'file' determine how the role is created i.e. either through a file or http(native) call. Beneath each key list the roles with appropriate permissions, using the file based format described [here] (https://www.elastic.co/guide/en/x-pack/current/file-realm.html) e.g. ``` es_roles: @@ -312,7 +311,7 @@ Following variables affect the versions installed: * ```es_version``` (e.g. "5.1.1"). * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. -* ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Shield is enabled. Ensure this user is admin. +* ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Security is enabled. Ensure this user is admin. * ```es_api_basic_auth_password``` The password associated with the user declared in `es_api_basic_auth_username` * ```es_start_service``` (true (default) or false) * ```es_plugins_reinstall``` (true or false (default) ) @@ -331,9 +330,9 @@ es_java_opts: - "-Djava.io.tmpdir=/data/tmp/elasticsearch" ``` -Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full path e.g. "lmenezes/elasticsearch-kopf" and the appropriate version for the target version of Elasticsearch. This approach should NOT be used for X-Pack related plugins e.g. Shield. See X-Pack below for details here. +Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full path e.g. "lmenezes/elasticsearch-kopf" and the appropriate version for the target version of Elasticsearch. This approach should NOT be used for X-Pack related plugins e.g. Security. See X-Pack below for details here. -If installing Marvel or Watcher, ensure the license plugin is also specified. Shield configuration is currently not supported but planned for later versions. +If installing Monitoring or Alerting, ensure the license plugin is also specified. Security configuration is currently not supported but planned for later versions. * ```es_user``` - defaults to elasticsearch. * ```es_group``` - defaults to elasticsearch. @@ -385,7 +384,7 @@ all supported platforms. * The role aims to be idempotent. Running the role multiple times, with no changes, should result in no state change on the server. If the configuration is changed, these will be applied and Elasticsearch restarted where required. * Systemd is used for Ubuntu versions >= 15, Debian >=8, Centos >=7. All other versions use init for service scripts. -* In order to run x-pack tests a license file with shield enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests. +* In order to run x-pack tests a license file with security enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests. ## IMPORTANT NOTES RE PLUGIN MANAGEMENT diff --git a/handlers/elasticsearch-templates.yml b/handlers/elasticsearch-templates.yml index 4595ef6..68160e6 100644 --- a/handlers/elasticsearch-templates.yml +++ b/handlers/elasticsearch-templates.yml @@ -17,7 +17,7 @@ status_code: 200 body_format: json body: "{{ lookup('file', item.path) }}" - when: not es_enable_xpack or not es_xpack_features is defined or not '"shield" in es_xpack_features' + when: not es_enable_xpack or not es_xpack_features is defined or not '"security" in es_xpack_features' with_items: "{{ templates.files }}" - name: Install templates with auth @@ -30,5 +30,5 @@ force_basic_auth: yes body_format: json body: "{{ lookup('file', item.path) }}" - when: es_enable_xpack and es_xpack_features is defined and '"shield" in es_xpack_features' + when: es_enable_xpack and es_xpack_features is defined and '"security" in es_xpack_features' with_items: "{{ templates.files }}" diff --git a/handlers/main.yml b/handlers/main.yml index 00df18e..ff07a08 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -7,10 +7,10 @@ when: es_restart_on_change and es_start_service and ((plugin_installed is defined and plugin_installed.changed) or (config_updated is defined and config_updated.changed) or (xpack_state.changed) or (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) # All security specific actions should go in here -- name: activate-shield - include: ./handlers/shield/elasticsearch-shield.yml +- name: activate-security + include: ./handlers/security/elasticsearch-security.yml -#Templates are a handler as they need to come after a restart e.g. suppose user removes shield on a running node and doesn't +#Templates are a handler as they need to come after a restart e.g. suppose user removes security on a running node and doesn't #specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed if we don't wait for the node to restart. #Templates done after restart therefore - as a handler. diff --git a/handlers/shield/elasticsearch-shield-native.yml b/handlers/security/elasticsearch-security-native.yml similarity index 82% rename from handlers/shield/elasticsearch-shield-native.yml rename to handlers/security/elasticsearch-security-native.yml index d545394..53df16e 100644 --- a/handlers/shield/elasticsearch-shield-native.yml +++ b/handlers/security/elasticsearch-security-native.yml @@ -14,12 +14,12 @@ - name: Wait 15 seconds for the Native Relm to come up pause: seconds=15 -#If the node has just has shield installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load +#If the node has just has security installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load #List current users - name: List Native Users uri: - url: http://{{es_api_host}}:{{es_api_port}}/_shield/user + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user method: GET user: "{{es_api_basic_auth_username}}" password: "{{es_api_basic_auth_password}}" @@ -40,7 +40,7 @@ #Delete all non required users - name: Delete Native Users uri: - url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item}} + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}} method: DELETE status_code: 200 user: "{{es_api_basic_auth_username}}" @@ -53,7 +53,7 @@ #Overwrite all other users - name: Update Native Users uri: - url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item.key}} + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/{{item.key}} method: POST body_format: json body: "{{item.value | to_json}}" @@ -69,7 +69,7 @@ - name: List Native Roles uri: - url: http://{{es_api_host}}:{{es_api_port}}/_shield/role + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role method: GET body_format: json user: "{{es_api_basic_auth_username}}" @@ -93,7 +93,7 @@ #Delete all non required roles - name: Delete Native Roles uri: - url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item}} + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}} method: DELETE status_code: 200 user: "{{es_api_basic_auth_username}}" @@ -106,7 +106,7 @@ #Update other roles - name: Update Native Roles uri: - url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item.key}} + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item.key}} method: POST body_format: json body: "{{item.value | to_json}}" diff --git a/handlers/shield/elasticsearch-shield.yml b/handlers/security/elasticsearch-security.yml similarity index 78% rename from handlers/shield/elasticsearch-shield.yml rename to handlers/security/elasticsearch-security.yml index 61620b3..af52976 100644 --- a/handlers/shield/elasticsearch-shield.yml +++ b/handlers/security/elasticsearch-security.yml @@ -6,9 +6,9 @@ wait_for: host={{es_api_host}} port={{es_api_port}} delay=10 - name: activate-license - include: ./handlers/shield/elasticsearch-xpack-activation.yml + include: ./handlers/security/elasticsearch-xpack-activation.yml when: es_enable_xpack and es_xpack_license is defined and es_xpack_license != '' - name: load-native-realms - include: ./handlers/shield/elasticsearch-shield-native.yml + include: ./handlers/security/elasticsearch-security-native.yml when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) \ No newline at end of file diff --git a/handlers/shield/elasticsearch-xpack-activation.yml b/handlers/security/elasticsearch-xpack-activation.yml similarity index 82% rename from handlers/shield/elasticsearch-xpack-activation.yml rename to handlers/security/elasticsearch-xpack-activation.yml index 948bd83..afdec99 100644 --- a/handlers/shield/elasticsearch-xpack-activation.yml +++ b/handlers/security/elasticsearch-xpack-activation.yml @@ -1,6 +1,6 @@ --- -- name: Activate ES license (without shield authentication) +- name: Activate ES license (without security authentication) uri: method: PUT url: "http://{{es_api_host}}:{{es_api_port}}/_license?acknowledge=true" @@ -9,13 +9,13 @@ return_content: yes register: license_activated no_log: True - when: not '"shield" in es_xpack_features' + when: not '"security" in es_xpack_features' failed_when: > license_activated.status != 200 or license_activated.json.license_status is not defined or license_activated.json.license_status != 'valid' -- name: Activate ES license (with shield authentication) +- name: Activate ES license (with security authentication) uri: method: PUT url: "http://{{es_api_host}}:{{es_api_port}}/_license?acknowledge=true" @@ -27,7 +27,7 @@ return_content: yes register: license_activated no_log: True - when: '"shield" in es_xpack_features' + when: '"security" in es_xpack_features' failed_when: > license_activated.status != 200 or license_activated.json.license_status is not defined or diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 8b16414..98d80ea 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -79,8 +79,8 @@ - name: Delete Default Logging File file: dest=/etc/elasticsearch/logging.yml state=absent -- name: Delete Default Logging File (5.x) +- name: Delete Default Logging File file: dest=/etc/elasticsearch/log4j2.properties state=absent -- name: Delete Default JVM Options File (5.x) +- name: Delete Default JVM Options File file: dest=/etc/elasticsearch/jvm.options state=absent diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index b3f0146..56800b3 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -19,9 +19,9 @@ - fail: msg="If locking memory with bootstrap.mlockall (or bootstrap.memory_lock) a heap size must be specified" when: (es_config['bootstrap.mlockall'] is defined or es_config['bootstrap.memory_lock'] is defined) and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined -#Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work -- fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" - when: es_enable_xpack and ("shield" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined +#Check if working with security we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work +- fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" + when: es_enable_xpack and ("security" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index 4dac838..343245b 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -46,8 +46,8 @@ - include: elasticsearch-xpack-install.yml with_items: "{{supported_xpack_features}}" -#Shield configuration -- include: shield/elasticsearch-shield.yml +#Security configuration +- include: security/elasticsearch-security.yml #Add any feature specific configuration here - name: Set Plugin Directory Permissions diff --git a/tasks/xpack/shield/elasticsearch-shield-file.yml b/tasks/xpack/shield/elasticsearch-security-file.yml similarity index 62% rename from tasks/xpack/shield/elasticsearch-shield-file.yml rename to tasks/xpack/shield/elasticsearch-security-file.yml index 380f289..5da0d13 100644 --- a/tasks/xpack/shield/elasticsearch-shield-file.yml +++ b/tasks/xpack/shield/elasticsearch-security-file.yml @@ -1,15 +1,15 @@ --- - set_fact: manage_file_users=es_users is defined and es_users.file is defined -#Ensure shield conf directory is created -- name: Ensure shield conf directory exists (file) - file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }} +#Ensure x-pack conf directory is created +- name: Ensure x-pack conf directory exists (file) + file: path={{ conf_dir }}/x-pack state=directory owner={{ es_user }} group={{ es_group }} changed_when: False - when: es_enable_xpack and '"shield" in es_xpack_features' + when: es_enable_xpack and '"security" in es_xpack_features' #List current users - name: List Users - shell: cat {{conf_dir}}/shield/users | awk -F':' '{print $1}' + shell: cat {{conf_dir}}/x-pack/users | awk -F':' '{print $1}' register: current_file_users when: manage_file_users changed_when: False @@ -20,7 +20,7 @@ #Remove users - name: Remove Users command: > - {{es_home}}/bin/shield/esusers userdel {{item}} + {{es_home}}/bin/x-pack/users userdel {{item}} when: manage_file_users and (users_to_remove | length > 0) with_items: "{{users_to_remove | default([])}}" environment: @@ -34,7 +34,7 @@ #Add users - name: Add Users command: > - {{es_home}}/bin/shield/esusers useradd {{item}} -p {{es_users.file[item].password}} + {{es_home}}/bin/x-pack/users useradd {{item}} -p {{es_users.file[item].password}} with_items: "{{users_to_add | default([])}}" when: manage_file_users and users_to_add | length > 0 no_log: True @@ -45,7 +45,7 @@ #Set passwords for all users declared - Required as the useradd will not change existing user passwords - name: Set User Passwords command: > - {{es_home}}/bin/shield/esusers passwd {{item.key}} -p {{item.value.password}} + {{es_home}}/bin/x-pack/users passwd {{item.key}} -p {{item.value.password}} with_dict: "{{(es_users | default({'file':{}})).file}}" when: manage_file_users and es_users.file.keys() | length > 0 #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip. @@ -60,14 +60,14 @@ #Copy Roles files - name: Copy roles.yml File for Instance - template: src=shield/roles.yml.j2 dest={{conf_dir}}/shield/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + template: src=security/roles.yml.j2 dest={{conf_dir}}/x-pack/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes when: es_roles is defined and es_roles.file is defined #Overwrite users_roles file - name: Copy User Roles - template: src=shield/users_roles.j2 dest={{conf_dir}}/shield/users_roles mode=0644 force=yes + template: src=security/users_roles.j2 dest={{conf_dir}}/x-pack/users_roles mode=0644 force=yes when: manage_file_users and users_roles | length > 0 -#Set permission on shield directory. E.g. if 2 nodes are installed on the same machine, the second node will not get the users file created at install, causing the files being created at es_users call and then having the wrong Permissions. -- name: Set Shield Directory Permissions Recursive - file: state=directory path={{conf_dir}}/shield/ owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file +#Set permission on security directory. E.g. if 2 nodes are installed on the same machine, the second node will not get the users file created at install, causing the files being created at es_users call and then having the wrong Permissions. +- name: Set Security Directory Permissions Recursive + file: state=directory path={{conf_dir}}/x-pack/ owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file diff --git a/tasks/xpack/shield/elasticsearch-security.yml b/tasks/xpack/shield/elasticsearch-security.yml new file mode 100644 index 0000000..5b483cf --- /dev/null +++ b/tasks/xpack/shield/elasticsearch-security.yml @@ -0,0 +1,36 @@ +--- +#Security specific configuration done here + +#TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6 + +#-----------------------------FILE BASED REALM---------------------------------------- + +- include: elasticsearch-security-file.yml + when: (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.file) or (es_roles is defined and es_roles.file is defined)) + +#-----------------------------NATIVE BASED REALM---------------------------------------- +# The native realm requires the node to be started so we do as a handler +- command: /bin/true + notify: activate-security + when: (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)) + +#-----------------------------ROLE MAPPING ---------------------------------------- + +#Copy Roles files +- name: Copy role_mapping.yml File for Instance + template: src=security/role_mapping.yml.j2 dest={{conf_dir}}/x-pack/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + when: es_role_mapping is defined + +#-----------------------------AUTH FILE---------------------------------------- + +- name: Copy message auth key to elasticsearch + copy: src={{ es_message_auth_file }} dest={{conf_dir}}/x-pack/system_key owner={{ es_user }} group={{ es_group }} mode=0600 force=yes + when: es_message_auth_file is defined + +#------------------------------------------------------------------------------------ + +#Ensure security conf directory is created +- name: Ensure security conf directory exists + file: path={{ conf_dir }}/security state=directory owner={{ es_user }} group={{ es_group }} + changed_when: False + when: es_enable_xpack and '"security" in es_xpack_features' diff --git a/tasks/xpack/shield/elasticsearch-shield.yml b/tasks/xpack/shield/elasticsearch-shield.yml deleted file mode 100644 index e464c76..0000000 --- a/tasks/xpack/shield/elasticsearch-shield.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -#Shield specific configuration done here - -#TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6 - -#-----------------------------FILE BASED REALM---------------------------------------- - -- include: elasticsearch-shield-file.yml - when: (es_enable_xpack and '"shield" in es_xpack_features') and ((es_users is defined and es_users.file) or (es_roles is defined and es_roles.file is defined)) - -#-----------------------------NATIVE BASED REALM---------------------------------------- -# The native realm requires the node to be started so we do as a handler -- command: /bin/true - notify: activate-shield - when: (es_enable_xpack and '"shield" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)) - -#-----------------------------ROLE MAPPING ---------------------------------------- - -#Copy Roles files -- name: Copy role_mapping.yml File for Instance - template: src=shield/role_mapping.yml.j2 dest={{conf_dir}}/shield/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes - when: es_role_mapping is defined - -#-----------------------------AUTH FILE---------------------------------------- - -- name: Copy message auth key to elasticsearch - copy: src={{ es_message_auth_file }} dest={{conf_dir}}/shield/system_key owner={{ es_user }} group={{ es_group }} mode=0600 force=yes - when: es_message_auth_file is defined - -#------------------------------------------------------------------------------------ - -#Ensure shield conf directory is created -- name: Ensure shield conf directory exists - file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }} - changed_when: False - when: es_enable_xpack and '"shield" in es_xpack_features' diff --git a/templates/shield/role_mapping.yml.j2 b/templates/security/role_mapping.yml.j2 similarity index 100% rename from templates/shield/role_mapping.yml.j2 rename to templates/security/role_mapping.yml.j2 diff --git a/templates/shield/roles.yml.j2 b/templates/security/roles.yml.j2 similarity index 100% rename from templates/shield/roles.yml.j2 rename to templates/security/roles.yml.j2 diff --git a/templates/shield/users_roles.j2 b/templates/security/users_roles.j2 similarity index 100% rename from templates/shield/users_roles.j2 rename to templates/security/users_roles.j2 diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 2bc671d..4b17395 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -6,7 +6,7 @@ shared_examples 'xpack::init' do |es_version| it { should exist } end - describe service('shield_node_elasticsearch') do + describe service('security_node_elasticsearch') do it { should be_running } end @@ -14,23 +14,23 @@ shared_examples 'xpack::init' do |es_version| it { should be_installed } end - describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do + describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do it { should be_file } it { should be_owned_by 'elasticsearch' } end - describe file('/etc/elasticsearch/shield_node/logging.yml') do + describe file('/etc/elasticsearch/security_node/logging.yml') do it { should be_file } it { should be_owned_by 'elasticsearch' } end - describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do - it { should contain 'node.name: localhost-shield_node' } + describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do + it { should contain 'node.name: localhost-security_node' } it { should contain 'cluster.name: elasticsearch' } - it { should contain 'path.conf: /etc/elasticsearch/shield_node' } - it { should contain 'path.data: /var/lib/elasticsearch/localhost-shield_node' } - it { should contain 'path.work: /tmp/elasticsearch/localhost-shield_node' } - it { should contain 'path.logs: /var/log/elasticsearch/localhost-shield_node' } + it { should contain 'path.conf: /etc/elasticsearch/security_node' } + it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' } + it { should contain 'path.work: /tmp/elasticsearch/localhost-security_node' } + it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' } end describe 'Node listening' do @@ -78,7 +78,7 @@ shared_examples 'xpack::init' do |es_version| end - #Check shield,watcher and license plugins are installed + #Check security,watcher and license plugins are installed describe file('/usr/share/elasticsearch/plugins/license') do it { should be_directory } it { should be_owned_by 'elasticsearch' } @@ -97,16 +97,16 @@ shared_examples 'xpack::init' do |es_version| end end - describe file('/usr/share/elasticsearch/plugins/shield') do + describe file('/usr/share/elasticsearch/plugins/security') do it { should be_directory } it { should be_owned_by 'elasticsearch' } end - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep shield') do + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep security') do its(:exit_status) { should eq 0 } end - describe file('/etc/elasticsearch/shield_node/shield') do + describe file('/etc/elasticsearch/security_node/security') do it { should be_directory } it { should be_owned_by 'elasticsearch' } end @@ -149,20 +149,20 @@ shared_examples 'xpack::init' do |es_version| #Test users file, users_roles and roles.yml - describe file('/etc/elasticsearch/shield_node/shield/users_roles') do + describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do it { should be_owned_by 'elasticsearch' } it { should contain 'admin:es_admin' } it { should contain 'power_user:testUser' } end - describe file('/etc/elasticsearch/shield_node/shield/users') do + describe file('/etc/elasticsearch/security_node/x-pack/users') do it { should be_owned_by 'elasticsearch' } it { should contain 'testUser:' } it { should contain 'es_admin:' } end - describe file('/etc/elasticsearch/shield_node/shield/roles.yml') do + describe file('/etc/elasticsearch/security_node/x-pack/roles.yml') do it { should be_owned_by 'elasticsearch' } #Test contents as expected its(:md5sum) { should eq '7800182547287abd480c8b095bf26e9e' } @@ -170,19 +170,19 @@ shared_examples 'xpack::init' do |es_version| #Test native roles and users are loaded - describe command('curl -s localhost:9200/_shield/user -u es_admin:changeMe | md5sum | grep 557a730df7136694131b5b7012a5ffad') do + describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | md5sum | grep 557a730df7136694131b5b7012a5ffad') do its(:exit_status) { should eq 0 } end - describe command('curl -s localhost:9200/_shield/user -u es_admin:changeMe | grep "{\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{}}}"') do + describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | grep "{\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{}}}"') do its(:exit_status) { should eq 0 } end - describe command('curl -s localhost:9200/_shield/role -u es_admin:changeMe | grep "{\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\]}}"') do + describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | grep "{\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\]}}"') do its(:exit_status) { should eq 0 } end - describe command('curl -s localhost:9200/_shield/role -u es_admin:changeMe | md5sum | grep 6d14f09ef1eea64adf4d4a9c04229629') do + describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | md5sum | grep 6d14f09ef1eea64adf4d4a9c04229629') do its(:exit_status) { should eq 0 } end @@ -213,15 +213,15 @@ shared_examples 'xpack::init' do |es_version| end #Test contents of Elasticsearch.yml file - describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do - it { should contain 'shield.authc.realms.file1.order: 0' } - it { should contain 'shield.authc.realms.file1.type: file' } - it { should contain 'shield.authc.realms.native1.order: 1' } - it { should contain 'shield.authc.realms.native1.type: native' } + describe file('/etc/elasticsearch/security_node/elasticsearch.yml') do + it { should contain 'security.authc.realms.file1.order: 0' } + it { should contain 'security.authc.realms.file1.type: file' } + it { should contain 'security.authc.realms.native1.order: 1' } + it { should contain 'security.authc.realms.native1.type: native' } end #Test contents of role_mapping.yml - describe file('/etc/elasticsearch/shield_node/shield/role_mapping.yml') do + describe file('/etc/elasticsearch/security_node/x-pack/role_mapping.yml') do it { should be_owned_by 'elasticsearch' } it { should contain 'power_user:' } it { should contain '- cn=admins,dc=example,dc=com' } @@ -230,7 +230,7 @@ shared_examples 'xpack::init' do |es_version| end - describe file('/etc/elasticsearch/shield_node/shield/system_key') do + describe file('/etc/elasticsearch/security_node/x-pack/system_key') do it { should be_owned_by 'elasticsearch' } it { should be_writable.by('owner') } it { should be_writable.by_user('elasticsearch') } diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 25ece50..4029149 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -3,8 +3,8 @@ hosts: localhost roles: - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", - "shield.authc.realms.file1.type": "file","shield.authc.realms.file1.order": 0, "shield.authc.realms.native1.type": "native","shield.authc.realms.native1.order": 1 }, - es_instance_name: "shield_node" } + "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 }, + es_instance_name: "security_node" } vars: es_templates: true es_enable_xpack: true @@ -13,8 +13,8 @@ - plugin: lmenezes/elasticsearch-kopf version: master es_xpack_features: - - shield - - watcher + - security + - alerting es_api_basic_auth_username: es_admin es_api_basic_auth_password: changeMe es_message_auth_file: system_key diff --git a/vars/main.yml b/vars/main.yml index fd29447..3d50db2 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -4,4 +4,4 @@ es_conf_dir: "/etc/elasticsearch" sysd_script: "/usr/lib/systemd/system/elasticsearch.service" init_script: "/etc/init.d/elasticsearch" #add supported features here -supported_xpack_features: ["watcher","marvel-agent","graph","shield"] \ No newline at end of file +supported_xpack_features: ["alerting","monitoring","graph","security"] \ No newline at end of file From b2dd44e9d536f177a8b43caacc7537c81670c4ba Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Wed, 11 Jan 2017 13:05:13 +0000 Subject: [PATCH 11/42] x-pack test cleanup --- .../helpers/serverspec/xpack_spec.rb | 32 +++---------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 4b17395..6367702 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -78,7 +78,7 @@ shared_examples 'xpack::init' do |es_version| end - #Check security,watcher and license plugins are installed + #Check x-pack and license plugins are installed describe file('/usr/share/elasticsearch/plugins/license') do it { should be_directory } it { should be_owned_by 'elasticsearch' } @@ -97,29 +97,25 @@ shared_examples 'xpack::init' do |es_version| end end - describe file('/usr/share/elasticsearch/plugins/security') do + describe file('/usr/share/elasticsearch/plugins/x-pack') do it { should be_directory } it { should be_owned_by 'elasticsearch' } end - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep security') do + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep x-pack') do its(:exit_status) { should eq 0 } end - describe file('/etc/elasticsearch/security_node/security') do + describe file('/etc/elasticsearch/security_node/x-pack') do it { should be_directory } it { should be_owned_by 'elasticsearch' } end - describe file('/usr/share/elasticsearch/plugins/watcher') do + describe file('/usr/share/elasticsearch/plugins/x-pack') do it { should be_directory } it { should be_owned_by 'elasticsearch' } end - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep watcher') do - its(:exit_status) { should eq 0 } - end - describe file('/usr/share/elasticsearch/plugins/kopf') do it { should be_directory } it { should be_owned_by 'elasticsearch' } @@ -129,24 +125,6 @@ shared_examples 'xpack::init' do |es_version| its(:exit_status) { should eq 0 } end - #test we haven't installed graph or marvel-agent - - describe file('/usr/share/elasticsearch/plugins/graph') do - it { should_not exist } - end - - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep graph') do - its(:exit_status) { should eq 1 } - end - - describe file('/usr/share/elasticsearch/plugins/marvel-agent') do - it { should_not exist } - end - - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep marvel-agent') do - its(:exit_status) { should eq 1 } - end - #Test users file, users_roles and roles.yml describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do From 8772f6d605decf203090f8f167f52effbd26bd8a Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Wed, 11 Jan 2017 13:09:05 +0000 Subject: [PATCH 12/42] Changes for x-pack installation 5.x --- tasks/xpack/elasticsearch-xpack-install.yml | 20 ++++++++++---------- tasks/xpack/elasticsearch-xpack.yml | 2 -- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml index c309421..ef54301 100644 --- a/tasks/xpack/elasticsearch-xpack-install.yml +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -1,23 +1,23 @@ --- #Test if feature is installed -- shell: "{{es_home}}/bin/elasticsearch-plugin list{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %} | grep {{item}}" - register: feature_installed +- shell: "{{es_home}}/bin/elasticsearch-plugin list | grep x-pack" + register: x_pack_installed changed_when: False - failed_when: "'ERROR' in feature_installed.stdout" + failed_when: "'ERROR' in x_pack_installed.stdout" ignore_errors: yes environment: CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" -#Remove Plugin if installed and its not been requested or the ES version has changed -- name: Remove {{item}} plugin - command: "{{es_home}}/bin/plugin remove {{item}}" +#Remove X-Pack if installed and its not been requested or the ES version has changed +- name: Remove x-pack plugin + command: "{{es_home}}/bin/elasticsearch-plugin remove x-pack" register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 - when: feature_installed.rc == 0 and (not es_enable_xpack or not '"{{item}}" in es_xpack_features' or es_version_changed) + when: x_pack_installed.rc == 0 and (not es_enable_xpack or es_version_changed) notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" @@ -25,13 +25,13 @@ #Install plugin if not installed, or the es version has changed (so removed above), and its been requested -- name: Install {{item}} plugin +- name: Install x-pack plugin command: > - {{es_home}}/bin/elasticsearch-plugin install {{item}} + {{es_home}}/bin/elasticsearch-plugin install x-pack register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 - when: (feature_installed.rc == 1 or es_version_changed) and es_enable_xpack and "{{item}}" in es_xpack_features + when: (feature_installed.rc == 1 or es_version_changed) and es_enable_xpack notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index 343245b..f9386e8 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -42,9 +42,7 @@ CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" -#We loop on all as we may need to remove some features. - include: elasticsearch-xpack-install.yml - with_items: "{{supported_xpack_features}}" #Security configuration - include: security/elasticsearch-security.yml From 1a207458a6c6761a52f3e35d04b7202edb4f5d67 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Wed, 11 Jan 2017 13:09:54 +0000 Subject: [PATCH 13/42] Folder rename shield -> security --- tasks/xpack/{shield => security}/elasticsearch-security-file.yml | 0 tasks/xpack/{shield => security}/elasticsearch-security.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tasks/xpack/{shield => security}/elasticsearch-security-file.yml (100%) rename tasks/xpack/{shield => security}/elasticsearch-security.yml (100%) diff --git a/tasks/xpack/shield/elasticsearch-security-file.yml b/tasks/xpack/security/elasticsearch-security-file.yml similarity index 100% rename from tasks/xpack/shield/elasticsearch-security-file.yml rename to tasks/xpack/security/elasticsearch-security-file.yml diff --git a/tasks/xpack/shield/elasticsearch-security.yml b/tasks/xpack/security/elasticsearch-security.yml similarity index 100% rename from tasks/xpack/shield/elasticsearch-security.yml rename to tasks/xpack/security/elasticsearch-security.yml From f664580ce6e627f88c280213f04547e65a9bf679 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Wed, 11 Jan 2017 13:16:07 +0000 Subject: [PATCH 14/42] Ability to disable x-pack features --- README.md | 6 +++++- defaults/main.yml | 2 +- templates/elasticsearch.yml.j2 | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 848804c..709d806 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,11 @@ ansible-playbook -i hosts ./your-playbook.yml ### Installing X-Pack Features -X-Pack features, such as Security, are supported. This feature is currently experimental. To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`. The following additional parameters allow X-Pack to be configured: +X-Pack features, such as Security, are supported. This feature is currently experimental. To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`. + +The parameter `es_xpack_features` by default enables all features i.e. it defaults to ["alerting","monitoring","graph","security","reporting"] + +The following additional parameters allow X-Pack to be configured: * ```es_message_auth_file``` System Key field to allow message authentication. This file should be placed in the 'files' directory. * ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/x-pack/current/mapping-roles.html) diff --git a/defaults/main.yml b/defaults/main.yml index faaffc5..84b4779 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -24,7 +24,7 @@ es_work_dir: "/tmp/elasticsearch" es_max_open_files: 65536 es_allow_downgrades: false es_enable_xpack: false -es_xpack_features: [] +es_xpack_features: ["alerting","monitoring","graph","security","reporting"] #These are used for internal operations performed by ansible. #They do not effect the current configuration es_api_host: "localhost" diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index d6755ae..fb57827 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -23,3 +23,19 @@ path.work: {{ work_dir }} {% endif %} path.logs: {{ log_dir }} + +{% if not "security" in es_xpack_features %} +xpack.security.enabled: false +{% endif %} + +{% if not "monitoring" in es_xpack_features %} +xpack.monitoring.enabled: false +{% endif %} + +{% if not "alerting" in es_xpack_features %} +xpack.watcher.enabled: false +{% endif %} + +{% if not "reporting" in es_xpack_features %} +xpack.reporting.enabled: false +{% endif %} \ No newline at end of file From 2b60a217ab041433d11ba9d0d5660f38dccea9e0 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Fri, 13 Jan 2017 11:59:31 +0000 Subject: [PATCH 15/42] Debian package url fixes for 5.x --- defaults/main.yml | 6 +++--- meta/main.yml.orig | 30 ------------------------------ templates/elasticsearch.repo | 6 ++---- 3 files changed, 5 insertions(+), 37 deletions(-) delete mode 100644 meta/main.yml.orig diff --git a/defaults/main.yml b/defaults/main.yml index 84b4779..be4f132 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -3,9 +3,9 @@ es_major_version: "5.x" es_version: "5.1.1" es_version_lock: false es_use_repository: true -es_apt_key: "https://packages.elasticsearch.org/GPG-KEY-elasticsearch" -es_apt_url: "deb https://packages.elastic.co/elasticsearch/{{ es_major_version }}/debian stable main" -es_apt_url_old: "deb http://packages.elastic.co/elasticsearch/{{ es_major_version }}/debian stable main" +es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch" +es_apt_url: "deb https://artifacts.elastic.co/packages/{{ es_major_version }}/apt stable main" +es_apt_url_old: "deb https://artifacts.elastic.co/packages/{{ es_major_version }}/apt stable main" es_start_service: true es_java_install: true update_java: false diff --git a/meta/main.yml.orig b/meta/main.yml.orig deleted file mode 100644 index 27a03cf..0000000 --- a/meta/main.yml.orig +++ /dev/null @@ -1,30 +0,0 @@ ---- - -allow_duplicates: yes - -galaxy_info: - author: Robin Clarke, Jakob Reiter, Dale McDiarmid - description: Elasticsearch for Linux - company: "Elastic.co" - license: "license (Apache)" -<<<<<<< HEAD - # Require 1.6 for apt deb install - min_ansible_version: 2.1.0 -======= - min_ansible_version: 2.0 ->>>>>>> elastic/master - platforms: - - name: EL - versions: - - 6 - - 7 - - name: Debian - versions: - - all - - name: Ubuntu - versions: - - all - categories: - - system - -dependencies: [] diff --git a/templates/elasticsearch.repo b/templates/elasticsearch.repo index 83e1920..116ccf8 100644 --- a/templates/elasticsearch.repo +++ b/templates/elasticsearch.repo @@ -1,8 +1,6 @@ [elasticsearch-{{ es_major_version }}] name=Elasticsearch repository for {{ es_major_version }} packages -baseurl={% if es_version | version_compare('5.0', '>=') %}https://artifacts.elastic.co/packages/{{ es_major_version }}/yum{% else %}http://packages.elastic.co/elasticsearch/{{ es_major_version }}/centos{% endif %} - +baseurl=https://artifacts.elastic.co/packages/{{ es_major_version }}/yum gpgcheck=1 -gpgkey={% if es_version | version_compare('5.0', '>=') %}https://artifacts.elastic.co/GPG-KEY-elasticsearch{% else %}http://packages.elastic.co/GPG-KEY-elasticsearch{% endif %} - +gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch enabled=1 From 661ec8044710cd828fe95f881b03ddf13358f16b Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Fri, 13 Jan 2017 12:18:33 +0000 Subject: [PATCH 16/42] move to 5.1.2 --- README.md | 4 ++-- defaults/main.yml | 4 ++-- test/integration/config-5x/serverspec/default_spec.rb | 2 +- test/integration/multi-5x/serverspec/default_spec.rb | 2 +- test/integration/package-5x/serverspec/default_spec.rb | 2 +- test/integration/standard-5x/serverspec/default_spec.rb | 2 +- test/integration/xpack-5x/serverspec/default_spec.rb | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 709d806..9d85284 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ The use of a map ensures the Ansible playbook does not need to be updated to ref In addition to the es_config map, several other parameters are supported for additional functions e.g. script installation. These can be found in the role's defaults/main.yml file. -The following illustrates applying configuration parameters to an Elasticsearch instance. By default, Elasticsearch 5.1.1 is installed. +The following illustrates applying configuration parameters to an Elasticsearch instance. By default, Elasticsearch 5.1.2is installed. ``` - name: Elasticsearch with custom configuration @@ -312,7 +312,7 @@ Additional parameters to es_config allow the customization of the Java and Elast Following variables affect the versions installed: * ```es_major_version``` (e.g. "5.1" ). Should be consistent with es_version. For versions >= 5.0 this must be "5.x". -* ```es_version``` (e.g. "5.1.1"). +* ```es_version``` (e.g. "5.1.2"). * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. * ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Security is enabled. Ensure this user is admin. diff --git a/defaults/main.yml b/defaults/main.yml index be4f132..03cef9f 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,11 +1,11 @@ --- es_major_version: "5.x" -es_version: "5.1.1" +es_version: "5.1.2" es_version_lock: false es_use_repository: true es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch" es_apt_url: "deb https://artifacts.elastic.co/packages/{{ es_major_version }}/apt stable main" -es_apt_url_old: "deb https://artifacts.elastic.co/packages/{{ es_major_version }}/apt stable main" +es_apt_url_old: "deb http://packages.elastic.co/elasticsearch/{{ es_major_version }}/debian stable main" es_start_service: true es_java_install: true update_java: false diff --git a/test/integration/config-5x/serverspec/default_spec.rb b/test/integration/config-5x/serverspec/default_spec.rb index 6e9c4af..72b83d9 100644 --- a/test/integration/config-5x/serverspec/default_spec.rb +++ b/test/integration/config-5x/serverspec/default_spec.rb @@ -1,6 +1,6 @@ require 'config_spec' describe 'Config Tests v 5.x' do - include_examples 'config::init', "5.1.1" + include_examples 'config::init', "5.1.2" end diff --git a/test/integration/multi-5x/serverspec/default_spec.rb b/test/integration/multi-5x/serverspec/default_spec.rb index 4d54c16..d815816 100644 --- a/test/integration/multi-5x/serverspec/default_spec.rb +++ b/test/integration/multi-5x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'multi_spec' describe 'Multi Tests v 5.x' do - include_examples 'multi::init', "5.1.1", ["kopf"] + include_examples 'multi::init', "5.1.2", ["kopf"] end diff --git a/test/integration/package-5x/serverspec/default_spec.rb b/test/integration/package-5x/serverspec/default_spec.rb index 72cbfbc..156dca8 100644 --- a/test/integration/package-5x/serverspec/default_spec.rb +++ b/test/integration/package-5x/serverspec/default_spec.rb @@ -2,5 +2,5 @@ require 'package_spec' describe 'Package Tests v 5.x' do - include_examples 'package::init', "5.1.1", ["kopf"] + include_examples 'package::init', "5.1.2", ["kopf"] end \ No newline at end of file diff --git a/test/integration/standard-5x/serverspec/default_spec.rb b/test/integration/standard-5x/serverspec/default_spec.rb index db96847..eaf5121 100644 --- a/test/integration/standard-5x/serverspec/default_spec.rb +++ b/test/integration/standard-5x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'standard_spec' describe 'Standard Tests v 5.x' do - include_examples 'standard::init', "5.1.1" + include_examples 'standard::init', "5.1.2" end diff --git a/test/integration/xpack-5x/serverspec/default_spec.rb b/test/integration/xpack-5x/serverspec/default_spec.rb index 3eb0df7..eb9bef2 100644 --- a/test/integration/xpack-5x/serverspec/default_spec.rb +++ b/test/integration/xpack-5x/serverspec/default_spec.rb @@ -1,5 +1,5 @@ require 'xpack_spec' describe 'Xpack Tests v 5.x' do - include_examples 'xpack::init', "5.1.1" + include_examples 'xpack::init', "5.1.2" end From 349fd36f15f8f5dd53a8d0762d0bbd73eb56ab5a Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Fri, 13 Jan 2017 12:26:53 +0000 Subject: [PATCH 17/42] Default es_heap_size --- templates/jvm.options.j2 | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/templates/jvm.options.j2 b/templates/jvm.options.j2 index 5b9010e..8d34be6 100644 --- a/templates/jvm.options.j2 +++ b/templates/jvm.options.j2 @@ -18,9 +18,17 @@ # Xms represents the initial size of total heap space # Xmx represents the maximum size of total heap space - +{% if es_heap_size is defined %} -Xms{{ es_heap_size }} +{% else %} +-Xms2g +{% endif %} + +{% if es_heap_size is defined %} -Xmx{{ es_heap_size }} +{% else %} +-Xms2g +{% endif %} ################################################################ ## Expert settings From ffc417c766f24467b538a1ce36e077d27471137f Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Fri, 13 Jan 2017 12:51:25 +0000 Subject: [PATCH 18/42] remove path.work from tests --- tasks/xpack/elasticsearch-xpack-install.yml | 2 +- templates/logging.yml.j2 | 68 ------------------- .../helpers/serverspec/standard_spec.rb | 8 ++- 3 files changed, 7 insertions(+), 71 deletions(-) delete mode 100644 templates/logging.yml.j2 diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml index ef54301..cdff1a9 100644 --- a/tasks/xpack/elasticsearch-xpack-install.yml +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -31,7 +31,7 @@ register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 - when: (feature_installed.rc == 1 or es_version_changed) and es_enable_xpack + when: (x_pack_installed.rc == 1 or es_version_changed) and es_enable_xpack notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" diff --git a/templates/logging.yml.j2 b/templates/logging.yml.j2 deleted file mode 100644 index 538c9bd..0000000 --- a/templates/logging.yml.j2 +++ /dev/null @@ -1,68 +0,0 @@ -# you can override this using by setting a system property, for example -Des.logger.level=DEBUG -es.logger.level: INFO -rootLogger: ${es.logger.level}, console, file -logger: - # log action execution errors for easier debugging - action: DEBUG - # reduce the logging for aws, too much is logged under the default INFO - com.amazonaws: WARN - org.apache.http: INFO - - # gateway - #gateway: DEBUG - #index.gateway: DEBUG - - # peer shard recovery - #indices.recovery: DEBUG - - # discovery - #discovery: TRACE - - index.search.slowlog: TRACE, index_search_slow_log_file - index.indexing.slowlog: TRACE, index_indexing_slow_log_file - -additivity: - index.search.slowlog: false - index.indexing.slowlog: false - -appender: - console: - type: console - layout: - type: consolePattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - # Use the following log4j-extras RollingFileAppender to enable gzip compression of log files. - # For more information see https://logging.apache.org/log4j/extras/apidocs/org/apache/log4j/rolling/RollingFileAppender.html - #file: - #type: extrasRollingFile - #file: ${path.logs}/${cluster.name}.log - #rollingPolicy: timeBased - #rollingPolicy.FileNamePattern: ${path.logs}/${cluster.name}.log.%d{yyyy-MM-dd}.gz - #layout: - #type: pattern - #conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - index_search_slow_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_index_search_slowlog.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" - - index_indexing_slow_log_file: - type: dailyRollingFile - file: ${path.logs}/${cluster.name}_index_indexing_slowlog.log - datePattern: "'.'yyyy-MM-dd" - layout: - type: pattern - conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n" \ No newline at end of file diff --git a/test/integration/helpers/serverspec/standard_spec.rb b/test/integration/helpers/serverspec/standard_spec.rb index 3133493..6272ed2 100644 --- a/test/integration/helpers/serverspec/standard_spec.rb +++ b/test/integration/helpers/serverspec/standard_spec.rb @@ -19,7 +19,12 @@ shared_examples 'standard::init' do |es_version| it { should be_owned_by 'elasticsearch' } end - describe file('/etc/elasticsearch/node1/logging.yml') do + describe file('/etc/elasticsearch/node1/log4j2.properties') do + it { should be_file } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/etc/elasticsearch/node1/jvm.options') do it { should be_file } it { should be_owned_by 'elasticsearch' } end @@ -29,7 +34,6 @@ shared_examples 'standard::init' do |es_version| it { should contain 'cluster.name: elasticsearch' } it { should contain 'path.conf: /etc/elasticsearch/node1' } it { should contain 'path.data: /var/lib/elasticsearch/localhost-node1' } - it { should contain 'path.work: /tmp/elasticsearch/localhost-node1' } it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' } end From 7efec96ea96ff4d47141e349af9579be99d32662 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Fri, 13 Jan 2017 15:28:53 +0000 Subject: [PATCH 19/42] Plugin fixes for 5.x --- .kitchen.yml | 6 ++---- tasks/elasticsearch-plugins.yml | 8 ++------ tasks/xpack/elasticsearch-xpack-install.yml | 2 +- test/integration/package-5x/serverspec/default_spec.rb | 2 +- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 4c09c0a..9d2c326 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -74,8 +74,7 @@ suites: attributes: extra_vars: es_plugins: - - plugin: lmenezes/elasticsearch-kopf - version: master + - plugin: ingest-geoip provisioner: playbook: test/integration/package.yml - name: config-5x @@ -88,8 +87,7 @@ suites: attributes: extra_vars: es_plugins: - - plugin: lmenezes/elasticsearch-kopf - version: master + - plugin: ingest-geoip provisioner: playbook: test/integration/multi.yml - name: xpack-5x diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index dd22194..752911f 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -7,12 +7,9 @@ - set_fact: list_command="list" -- set_fact: list_command="--list" - when: es_version | version_compare('2.0', '<') - #List currently installed plugins - ignore xpack if > v 2.0 - name: Check installed elasticsearch plugins - shell: "{{es_home}}/bin/elasticsearch-plugin {{list_command}}{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %}{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}" + shell: "{{es_home}}/bin/elasticsearch-plugin list | grep -vE 'x-pack|license'" register: installed_plugins failed_when: "'ERROR' in installed_plugins.stdout" changed_when: False @@ -34,8 +31,7 @@ ES_INCLUDE: "{{ instance_default_file }}" - name: Install elasticsearch plugins - command: > - {{es_home}}/bin/elasticsearch-plugin install {{ item.plugin }}{% if item.version is defined and item.version != '' %}/{{ item.version }}{% endif %} {% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -DproxyHost={{ item.proxy_host }} -DproxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -DproxyHost={{ es_proxy_host }} -DproxyPort={{ es_proxy_port }} {% endif %} --silent + command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.plugin }} --batch --silent {% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -DproxyHost={{ item.proxy_host }} -DproxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -DproxyHost={{ es_proxy_host }} -DproxyPort={{ es_proxy_port }} {% endif %}" register: plugin_installed failed_when: "'ERROR' in plugin_installed.stdout" changed_when: plugin_installed.rc == 0 diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml index cdff1a9..5d9eefb 100644 --- a/tasks/xpack/elasticsearch-xpack-install.yml +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -27,7 +27,7 @@ #Install plugin if not installed, or the es version has changed (so removed above), and its been requested - name: Install x-pack plugin command: > - {{es_home}}/bin/elasticsearch-plugin install x-pack + {{es_home}}/bin/elasticsearch-plugin install --batch x-pack {% if es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} {% endif %} register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 diff --git a/test/integration/package-5x/serverspec/default_spec.rb b/test/integration/package-5x/serverspec/default_spec.rb index 156dca8..4b2191b 100644 --- a/test/integration/package-5x/serverspec/default_spec.rb +++ b/test/integration/package-5x/serverspec/default_spec.rb @@ -2,5 +2,5 @@ require 'package_spec' describe 'Package Tests v 5.x' do - include_examples 'package::init', "5.1.2", ["kopf"] + include_examples 'package::init', "5.1.2", ["ingest-geoip"] end \ No newline at end of file From 4edc56f1049407f9c1531fcce10f7ca50f260efc Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 16 Jan 2017 10:40:01 +0000 Subject: [PATCH 20/42] Move to memory_lock --- README.md | 10 +++++----- tasks/elasticsearch-parameters.yml | 6 +++--- templates/elasticsearch.j2 | 2 +- templates/systemd/elasticsearch.j2 | 2 +- test/integration/config.yml | 2 +- test/integration/helpers/serverspec/config_spec.rb | 5 ++--- test/integration/helpers/serverspec/multi_spec.rb | 8 ++++---- test/integration/multi.yml | 2 +- 8 files changed, 18 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 9d85284..9591165 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ The following illustrates applying configuration parameters to an Elasticsearch transport.tcp.port: 9301, node.data: false, node.master: true, - bootstrap.mlockall: true, + bootstrap.memory_lock: true, discovery.zen.ping.multicast.enabled: false } } vars: @@ -104,7 +104,7 @@ A more complex example: transport.tcp.port: 9301, node.data: false, node.master: true, - bootstrap.mlockall: true, + bootstrap.memory_lock: true, discovery.zen.ping.multicast.enabled: false } } vars: @@ -150,7 +150,7 @@ recommended in any multi node cluster configuration. transport.tcp.port: 9300, node.data: false, node.master: true, - bootstrap.mlockall: false, + bootstrap.memory_lock: false, discovery.zen.ping.multicast.enabled: false } } vars: @@ -172,7 +172,7 @@ recommended in any multi node cluster configuration. transport.tcp.port: 9300, node.data: true, node.master: false, - bootstrap.mlockall: false, + bootstrap.memory_lock: false, cluster.name: "test-cluster", discovery.zen.ping.multicast.enabled: false } } @@ -184,7 +184,7 @@ recommended in any multi node cluster configuration. transport.tcp.port: 9301, node.data: true, node.master: false, - bootstrap.mlockall: false, + bootstrap.memory_lock: false, cluster.name: "test-cluster", discovery.zen.ping.multicast.enabled: false } } diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 56800b3..502403d 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -16,8 +16,8 @@ when: es_config['discovery.zen.ping.unicast.hosts'] is not defined #If the user attempts to lock memory they must specify a heap size -- fail: msg="If locking memory with bootstrap.mlockall (or bootstrap.memory_lock) a heap size must be specified" - when: (es_config['bootstrap.mlockall'] is defined or es_config['bootstrap.memory_lock'] is defined) and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined +- fail: msg="If locking memory with bootstrap.memory_lock a heap size must be specified" + when: es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True and es_heap_size is not defined #Check if working with security we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work - fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" @@ -26,7 +26,7 @@ - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} -- set_fact: m_lock_enabled={{ (es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True) or (es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True) }} +- set_fact: m_lock_enabled={{ es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True }} #TODO - if transport.host is not local maybe error on boostrap checks diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index f099e23..a490cec 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -76,7 +76,7 @@ MAX_OPEN_FILES={{es_max_open_files}} {% endif %} # The maximum number of bytes of memory that may be locked into RAM -# Set to "unlimited" if you use the 'bootstrap.{% if es_version | version_compare('5.0', '<=') %}memory_lock{% else %}mlockall{% endif %}: true' option +# Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option # in elasticsearch.yml (ES_HEAP_SIZE must also be set). # When using Systemd, the LimitMEMLOCK property must be set # in /usr/lib/systemd/system/elasticsearch.service diff --git a/templates/systemd/elasticsearch.j2 b/templates/systemd/elasticsearch.j2 index 204fa49..78d79b4 100644 --- a/templates/systemd/elasticsearch.j2 +++ b/templates/systemd/elasticsearch.j2 @@ -41,7 +41,7 @@ StandardError=inherit LimitNOFILE={{es_max_open_files}} # Specifies the maximum number of bytes of memory that may be locked into RAM -# Set to "infinity" if you use the 'bootstrap.{% if es_version | version_compare('5.0', '<=') %}memory_lock{% else %}mlockall{% endif %}: true' option +# Set to "infinity" if you use the 'bootstrap.memory_lock: true' option # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}} {% if m_lock_enabled %} LimitMEMLOCK=infinity diff --git a/test/integration/config.yml b/test/integration/config.yml index fabc7cd..1399556 100644 --- a/test/integration/config.yml +++ b/test/integration/config.yml @@ -4,7 +4,7 @@ hosts: localhost roles: #expand to all available parameters - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.mlockall: true, discovery.zen.ping.multicast.enabled: false } } + - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: true, discovery.zen.ping.multicast.enabled: false } } vars: es_scripts: false es_templates: false diff --git a/test/integration/helpers/serverspec/config_spec.rb b/test/integration/helpers/serverspec/config_spec.rb index 086c063..ff1d563 100644 --- a/test/integration/helpers/serverspec/config_spec.rb +++ b/test/integration/helpers/serverspec/config_spec.rb @@ -35,11 +35,10 @@ shared_examples 'config::init' do |es_version| it { should contain 'discovery.zen.ping.multicast.enabled: false' } it { should contain 'cluster.name: custom-cluster' } it { should contain 'node.name: node1' } - it { should contain 'bootstrap.mlockall: true' } + it { should contain 'bootstrap.memory_lock: true' } it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9301' } it { should contain 'path.conf: /etc/elasticsearch/node1' } it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' } - it { should contain 'path.work: /opt/elasticsearch/temp/localhost-node1' } it { should contain 'path.logs: /opt/elasticsearch/logs/localhost-node1' } end @@ -77,7 +76,7 @@ shared_examples 'config::init' do |es_version| end #test to make sure mlock was applied - describe command('curl -s "localhost:9201/_nodes/process?pretty" | grep mlockall') do + describe command('curl -s "localhost:9201/_nodes/process?pretty" | grep memory_lock') do its(:stdout) { should match /true/ } its(:exit_status) { should eq 0 } end diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index 4d2cc2f..dba1e90 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -27,7 +27,7 @@ shared_examples 'multi::init' do |es_version,plugins| it { should contain 'node.master: false' } it { should contain 'discovery.zen.ping.multicast.enabled: false' } it { should contain 'node.name: localhost-node1' } - it { should_not contain 'bootstrap.mlockall: true' } + it { should_not contain 'bootstrap.memory_lock: true' } it { should contain 'path.conf: /etc/elasticsearch/node1' } it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' } it { should contain 'path.work: /tmp/elasticsearch/localhost-node1' } @@ -44,7 +44,7 @@ shared_examples 'multi::init' do |es_version,plugins| it { should contain 'node.master: true' } it { should contain 'discovery.zen.ping.multicast.enabled: false' } it { should contain 'node.name: localhost-master' } - it { should contain 'bootstrap.mlockall: true' } + it { should contain 'bootstrap.memory_lock: true' } it { should contain 'path.conf: /etc/elasticsearch/master' } it { should contain 'path.data: /opt/elasticsearch/master/localhost-master' } it { should contain 'path.work: /tmp/elasticsearch/localhost-master' } @@ -146,13 +146,13 @@ shared_examples 'multi::init' do |es_version,plugins| end #test to make sure mlock was applied - describe command('curl -s "localhost:9200/_nodes/localhost-master/process?pretty=true" | grep mlockall') do + describe command('curl -s "localhost:9200/_nodes/localhost-master/process?pretty=true" | grep memory_lock') do its(:stdout) { should match /true/ } its(:exit_status) { should eq 0 } end #test to make sure mlock was not applied - describe command('curl -s "localhost:9201/_nodes/localhost-node1/process?pretty=true" | grep mlockall') do + describe command('curl -s "localhost:9201/_nodes/localhost-node1/process?pretty=true" | grep memory_lock') do its(:stdout) { should match /false/ } its(:exit_status) { should eq 0 } end diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 173528c..bf04717 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -3,7 +3,7 @@ - name: Elasticsearch Multi tests hosts: localhost roles: - - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_heap_size: "1g", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.mlockall: true, discovery.zen.ping.multicast.enabled: false } } + - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_heap_size: "1g", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true, discovery.zen.ping.multicast.enabled: false } } - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false, discovery.zen.ping.multicast.enabled: false } } vars: es_scripts: true From dcbebf79c637031595211b8278b8f78f4caacd56 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 16 Jan 2017 11:18:45 +0000 Subject: [PATCH 21/42] Multi cast removed --- .gitignore | 1 + README.md | 17 +++++++---------- test/integration/config.yml | 2 +- .../helpers/serverspec/config_spec.rb | 3 +-- .../helpers/serverspec/multi_spec.rb | 2 -- test/integration/multi.yml | 4 ++-- 6 files changed, 12 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index bdcfa29..c702191 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +/vars/Debian.yml .kitchen/ license.json *.pyc diff --git a/README.md b/README.md index 9591165..8cd376e 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ The following illustrates applying configuration parameters to an Elasticsearch node.data: false, node.master: true, bootstrap.memory_lock: true, - discovery.zen.ping.multicast.enabled: false } + } } vars: es_scripts: false @@ -75,7 +75,7 @@ The following illustrates applying configuration parameters to an Elasticsearch es_heap_size: 1g ``` ` -The role utilises Elasticsearch version defaults. Multicast is therefore disabled for 5.x. The following should be set to ensure a successful cluster forms. +The role utilises Elasticsearch version defaults. The following should be set to ensure a successful cluster forms. * ```es_config['http.port']``` - the http port for the node * ```es_config['transport.tcp.port']``` - the transport port for the node @@ -105,7 +105,7 @@ A more complex example: node.data: false, node.master: true, bootstrap.memory_lock: true, - discovery.zen.ping.multicast.enabled: false } + } } vars: es_scripts: false @@ -144,14 +144,13 @@ recommended in any multi node cluster configuration. - { role: elasticsearch, es_instance_name: "node1", es_heap_size: "1g", es_config: { cluster.name: "test-cluster", - "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "elastic02:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: false, - discovery.zen.ping.multicast.enabled: false } + } } vars: es_scripts: false @@ -166,19 +165,17 @@ recommended in any multi node cluster configuration. roles: - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch", es_config: { - "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "elastic02:9300", http.port: 9200, transport.tcp.port: 9300, node.data: true, node.master: false, bootstrap.memory_lock: false, - cluster.name: "test-cluster", - discovery.zen.ping.multicast.enabled: false } + cluster.name: "test-cluster" + } } - { role: elasticsearch, es_instance_name: "node2", es_config: { - "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "elastic02:9300", http.port: 9201, transport.tcp.port: 9301, @@ -186,7 +183,7 @@ recommended in any multi node cluster configuration. node.master: false, bootstrap.memory_lock: false, cluster.name: "test-cluster", - discovery.zen.ping.multicast.enabled: false } + } } vars: es_scripts: false diff --git a/test/integration/config.yml b/test/integration/config.yml index 1399556..9a28b0e 100644 --- a/test/integration/config.yml +++ b/test/integration/config.yml @@ -4,7 +4,7 @@ hosts: localhost roles: #expand to all available parameters - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: true, discovery.zen.ping.multicast.enabled: false } } + - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: true } } vars: es_scripts: false es_templates: false diff --git a/test/integration/helpers/serverspec/config_spec.rb b/test/integration/helpers/serverspec/config_spec.rb index ff1d563..2f7aa91 100644 --- a/test/integration/helpers/serverspec/config_spec.rb +++ b/test/integration/helpers/serverspec/config_spec.rb @@ -32,7 +32,6 @@ shared_examples 'config::init' do |es_version| it { should contain 'transport.tcp.port: 9301' } it { should contain 'node.data: false' } it { should contain 'node.master: true' } - it { should contain 'discovery.zen.ping.multicast.enabled: false' } it { should contain 'cluster.name: custom-cluster' } it { should contain 'node.name: node1' } it { should contain 'bootstrap.memory_lock: true' } @@ -76,7 +75,7 @@ shared_examples 'config::init' do |es_version| end #test to make sure mlock was applied - describe command('curl -s "localhost:9201/_nodes/process?pretty" | grep memory_lock') do + describe command('curl -s "localhost:9201/_nodes/process?pretty" | grep mlockall') do its(:stdout) { should match /true/ } its(:exit_status) { should eq 0 } end diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index dba1e90..bd6ba0f 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -25,7 +25,6 @@ shared_examples 'multi::init' do |es_version,plugins| it { should contain 'transport.tcp.port: 9301' } it { should contain 'node.data: true' } it { should contain 'node.master: false' } - it { should contain 'discovery.zen.ping.multicast.enabled: false' } it { should contain 'node.name: localhost-node1' } it { should_not contain 'bootstrap.memory_lock: true' } it { should contain 'path.conf: /etc/elasticsearch/node1' } @@ -42,7 +41,6 @@ shared_examples 'multi::init' do |es_version,plugins| it { should contain 'transport.tcp.port: 9300' } it { should contain 'node.data: false' } it { should contain 'node.master: true' } - it { should contain 'discovery.zen.ping.multicast.enabled: false' } it { should contain 'node.name: localhost-master' } it { should contain 'bootstrap.memory_lock: true' } it { should contain 'path.conf: /etc/elasticsearch/master' } diff --git a/test/integration/multi.yml b/test/integration/multi.yml index bf04717..8b1ddd1 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -3,8 +3,8 @@ - name: Elasticsearch Multi tests hosts: localhost roles: - - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_heap_size: "1g", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true, discovery.zen.ping.multicast.enabled: false } } - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false, discovery.zen.ping.multicast.enabled: false } } + - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_heap_size: "1g", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } + - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } } vars: es_scripts: true es_templates: true From 3218191ee02ce5a5e05706aa9616e180f5b7f3d2 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 16 Jan 2017 11:36:05 +0000 Subject: [PATCH 22/42] Multi tests fixed to test for mlockall and no work dir --- test/integration/helpers/serverspec/multi_spec.rb | 6 ++---- test/integration/multi-5x/serverspec/default_spec.rb | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index bd6ba0f..003ffe6 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -29,7 +29,6 @@ shared_examples 'multi::init' do |es_version,plugins| it { should_not contain 'bootstrap.memory_lock: true' } it { should contain 'path.conf: /etc/elasticsearch/node1' } it { should contain 'path.data: /opt/elasticsearch/data-1/localhost-node1,/opt/elasticsearch/data-2/localhost-node1' } - it { should contain 'path.work: /tmp/elasticsearch/localhost-node1' } it { should contain 'path.logs: /var/log/elasticsearch/localhost-node1' } end @@ -45,7 +44,6 @@ shared_examples 'multi::init' do |es_version,plugins| it { should contain 'bootstrap.memory_lock: true' } it { should contain 'path.conf: /etc/elasticsearch/master' } it { should contain 'path.data: /opt/elasticsearch/master/localhost-master' } - it { should contain 'path.work: /tmp/elasticsearch/localhost-master' } it { should contain 'path.logs: /var/log/elasticsearch/localhost-master' } end @@ -144,13 +142,13 @@ shared_examples 'multi::init' do |es_version,plugins| end #test to make sure mlock was applied - describe command('curl -s "localhost:9200/_nodes/localhost-master/process?pretty=true" | grep memory_lock') do + describe command('curl -s "localhost:9200/_nodes/localhost-master/process?pretty=true" | grep mlockall') do its(:stdout) { should match /true/ } its(:exit_status) { should eq 0 } end #test to make sure mlock was not applied - describe command('curl -s "localhost:9201/_nodes/localhost-node1/process?pretty=true" | grep memory_lock') do + describe command('curl -s "localhost:9201/_nodes/localhost-node1/process?pretty=true" | grep mlockall') do its(:stdout) { should match /false/ } its(:exit_status) { should eq 0 } end diff --git a/test/integration/multi-5x/serverspec/default_spec.rb b/test/integration/multi-5x/serverspec/default_spec.rb index d815816..4abcf13 100644 --- a/test/integration/multi-5x/serverspec/default_spec.rb +++ b/test/integration/multi-5x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'multi_spec' describe 'Multi Tests v 5.x' do - include_examples 'multi::init', "5.1.2", ["kopf"] + include_examples 'multi::init', "5.1.2", ["ingest-geoip"] end From 719bd67fde42fd2ba13ca611d3f184a138e71d05 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 16 Jan 2017 12:04:10 +0000 Subject: [PATCH 23/42] X-Pack fixes - no more license file needed --- tasks/xpack/elasticsearch-xpack-install.yml | 2 +- tasks/xpack/elasticsearch-xpack.yml | 40 --------------------- test/integration/xpack.yml | 3 +- 3 files changed, 2 insertions(+), 43 deletions(-) diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml index 5d9eefb..505b6d2 100644 --- a/tasks/xpack/elasticsearch-xpack-install.yml +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -27,7 +27,7 @@ #Install plugin if not installed, or the es version has changed (so removed above), and its been requested - name: Install x-pack plugin command: > - {{es_home}}/bin/elasticsearch-plugin install --batch x-pack {% if es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} {% endif %} + {{es_home}}/bin/elasticsearch-plugin install --silent --batch x-pack {% if es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} {% endif %} register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index f9386e8..1e54782 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -2,46 +2,6 @@ - set_fact: es_version_changed={{ ((elasticsearch_install_from_package is defined and (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) }} -#enabling xpack installs the license. Not a xpack feature and does not need to be specified - TODO: we should append it to the list if xpack is enabled and remove this - -#Check if license is installed -- name: Check License plugin is installed - shell: > - {{es_home}}/bin/elasticsearch-plugin list | tail -n +2 | grep license - register: license_installed - ignore_errors: yes - failed_when: "'ERROR' in license_installed.stdout" - changed_when: False - environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - -#Remove license if installed and xpack not enabled -- name: Remove license plugin - command: > - {{es_home}}/bin/elasticsearch-plugin remove license - register: xpack_state - failed_when: "'ERROR' in xpack_state.stdout" - changed_when: xpack_state.rc == 0 - when: license_installed.rc == 0 and (not es_enable_xpack or es_version_changed) - notify: restart elasticsearch - environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - -#Install License if not installed, or it needs to be reinstalled due to ES change (above task will have removed), and its been requested. -- name: Install license plugin - command: > - {{es_home}}/bin/elasticsearch-plugin install license - register: xpack_state - failed_when: "'ERROR' in xpack_state.stdout" - changed_when: xpack_state.rc == 0 - when: (license_installed.rc == 1 or es_version_changed) and es_enable_xpack - notify: restart elasticsearch - environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" - - include: elasticsearch-xpack-install.yml #Security configuration diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 4029149..3582664 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -10,8 +10,7 @@ es_enable_xpack: true es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" es_plugins: - - plugin: lmenezes/elasticsearch-kopf - version: master + - plugin: ingest-geoip es_xpack_features: - security - alerting From 0656a19501ce8f7640af639e7c917a5710d2a523 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 16 Jan 2017 12:19:22 +0000 Subject: [PATCH 24/42] Reporting should not be declared as x-pack feature --- README.md | 2 +- defaults/main.yml | 2 +- tasks/elasticsearch-plugins.yml | 2 +- templates/elasticsearch.yml.j2 | 4 ---- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8cd376e..57819a6 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,7 @@ ansible-playbook -i hosts ./your-playbook.yml X-Pack features, such as Security, are supported. This feature is currently experimental. To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`. -The parameter `es_xpack_features` by default enables all features i.e. it defaults to ["alerting","monitoring","graph","security","reporting"] +The parameter `es_xpack_features` by default enables all features i.e. it defaults to ["alerting","monitoring","graph","security"] The following additional parameters allow X-Pack to be configured: diff --git a/defaults/main.yml b/defaults/main.yml index 03cef9f..178e84f 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -24,7 +24,7 @@ es_work_dir: "/tmp/elasticsearch" es_max_open_files: 65536 es_allow_downgrades: false es_enable_xpack: false -es_xpack_features: ["alerting","monitoring","graph","security","reporting"] +es_xpack_features: ["alerting","monitoring","graph","security"] #These are used for internal operations performed by ansible. #They do not effect the current configuration es_api_host: "localhost" diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 752911f..eab6109 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -9,7 +9,7 @@ #List currently installed plugins - ignore xpack if > v 2.0 - name: Check installed elasticsearch plugins - shell: "{{es_home}}/bin/elasticsearch-plugin list | grep -vE 'x-pack|license'" + shell: "{{es_home}}/bin/elasticsearch-plugin list | grep -vE 'x-pack'" register: installed_plugins failed_when: "'ERROR' in installed_plugins.stdout" changed_when: False diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index fb57827..2bbc5c2 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -34,8 +34,4 @@ xpack.monitoring.enabled: false {% if not "alerting" in es_xpack_features %} xpack.watcher.enabled: false -{% endif %} - -{% if not "reporting" in es_xpack_features %} -xpack.reporting.enabled: false {% endif %} \ No newline at end of file From 75d750a43ef9920027df0e582a9fa78803795926 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 16 Jan 2017 12:32:03 +0000 Subject: [PATCH 25/42] Protect reserved elastic user + doc clarification on configuring native realm --- README.md | 8 ++++++++ handlers/security/elasticsearch-security-native.yml | 5 ++--- test/integration/xpack.yml | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 57819a6..ed04414 100644 --- a/README.md +++ b/README.md @@ -300,6 +300,14 @@ X-Pack configuration parameters can be added to the elasticsearch.yml file using For a full example see [here](https://github.com/elastic/ansible-elasticsearch/blob/master/test/integration/xpack.yml) +####Important Note for Native Realm Configuration + +In order for native users and roles to be configured, the role calls the Elasticsearch API. Given security is installed this requires definition of two parameters: + +* ```es_api_basic_auth_username``` - admin username +* ```es_api_basic_auth_password``` - admin password + +These can either be set to a user declared in the file based realm, with admin permissions, or the default "elastic" superuser (default password is changeme). ### Additional Configuration diff --git a/handlers/security/elasticsearch-security-native.yml b/handlers/security/elasticsearch-security-native.yml index 53df16e..df0546d 100644 --- a/handlers/security/elasticsearch-security-native.yml +++ b/handlers/security/elasticsearch-security-native.yml @@ -28,12 +28,11 @@ register: user_list_response when: manage_native_users - -- set_fact: current_users={{user_list_response.json.keys() | list}} +#Current users not inc. the elastic user which is reserved and cannot be deleted +- set_fact: current_users={{user_list_response.json.keys() | list | difference(['elastic'] )}} when: manage_native_users #Identify non declared users - - set_fact: users_to_remove={{ current_users | difference ( es_users.native.keys() ) }} when: manage_native_users diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 3582664..3826ad9 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -14,8 +14,8 @@ es_xpack_features: - security - alerting - es_api_basic_auth_username: es_admin - es_api_basic_auth_password: changeMe + es_api_basic_auth_username: elastic + es_api_basic_auth_password: changeme es_message_auth_file: system_key es_role_mapping: power_user: From 3b0c47333cc25e3a331575309c23522d8f294529 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 16 Jan 2017 12:33:06 +0000 Subject: [PATCH 26/42] Doc cleanup --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index ed04414..d693151 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ansible-elasticsearch -**THIS ROLE IS UNDER DEVELOPMENT FOR 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.** +**THIS ROLE IS FOR 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.** Ansible role for 5.x Elasticsearch. Currently this works on Debian and RedHat based linux systems. Tested platforms are: @@ -314,8 +314,6 @@ These can either be set to a user declared in the file based realm, with admin p Additional parameters to es_config allow the customization of the Java and Elasticsearch versions, in addition to role behaviour. Options include: -Following variables affect the versions installed: - * ```es_major_version``` (e.g. "5.1" ). Should be consistent with es_version. For versions >= 5.0 this must be "5.x". * ```es_version``` (e.g. "5.1.2"). * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". From 8d84b57941c389ee15a8237e0f055c8e4b1ee8df Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 16 Jan 2017 15:17:18 +0000 Subject: [PATCH 27/42] Xpack tests + protection against reserved roles and users --- filter_plugins/custom.py | 9 ++++- .../elasticsearch-security-native.yml | 19 ++++----- .../helpers/serverspec/xpack_spec.rb | 39 +++++++------------ .../xpack-5x/serverspec/default_spec.rb | 2 +- 4 files changed, 30 insertions(+), 39 deletions(-) diff --git a/filter_plugins/custom.py b/filter_plugins/custom.py index 96570ec..82b0efa 100644 --- a/filter_plugins/custom.py +++ b/filter_plugins/custom.py @@ -28,10 +28,16 @@ def extract_role_users(users={}): role_users.append(role+":"+user) return role_users - def filename(filename=''): return os.path.splitext(os.path.basename(filename))[0] +def filter_reserved(user_roles={}): + not_reserved = [] + for user_role,details in user_roles.items(): + if not "metadata" in details or not "_reserved" in details["metadata"] or not details["metadata"]["_reserved"]: + not_reserved.append(user_role) + return not_reserved + class FilterModule(object): def filters(self): @@ -39,5 +45,6 @@ class FilterModule(object): 'append_to_list':append_to_list, 'array_to_str':array_to_str, 'extract_role_users':extract_role_users, + 'filter_reserved':filter_reserved, 'filename':filename} diff --git a/handlers/security/elasticsearch-security-native.yml b/handlers/security/elasticsearch-security-native.yml index df0546d..80546ce 100644 --- a/handlers/security/elasticsearch-security-native.yml +++ b/handlers/security/elasticsearch-security-native.yml @@ -28,8 +28,8 @@ register: user_list_response when: manage_native_users -#Current users not inc. the elastic user which is reserved and cannot be deleted -- set_fact: current_users={{user_list_response.json.keys() | list | difference(['elastic'] )}} +#Current users not inc. those reserved +- set_fact: current_users={{ user_list_response.json | filter_reserved }} when: manage_native_users #Identify non declared users @@ -48,11 +48,10 @@ when: manage_native_users and users_to_remove | length > 0 with_items: "{{users_to_remove}}" - #Overwrite all other users - name: Update Native Users uri: - url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/{{item.key}} + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item.key}} method: POST body_format: json body: "{{item.value | to_json}}" @@ -61,11 +60,10 @@ password: "{{es_api_basic_auth_password}}" force_basic_auth: yes when: manage_native_users and es_users.native.keys() > 0 - no_log: True + #no_log: True with_dict: "{{es_users.native}}" -#List current roles - +#List current roles not. inc those reserved - name: List Native Roles uri: url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role @@ -78,11 +76,8 @@ register: role_list_response when: manage_native_roles -#Identify undeclared roles - -- set_fact: current_roles={{role_list_response.json.keys() | list}} - when: manage_native_users - +- set_fact: current_roles={{ role_list_response.json | filter_reserved }} + when: manage_native_roles - debug: msg="{{current_roles}}" - set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys() ) }} diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb index 6367702..87365b4 100644 --- a/test/integration/helpers/serverspec/xpack_spec.rb +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -shared_examples 'xpack::init' do |es_version| +shared_examples 'xpack::init' do |es_version,plugins| describe user('elasticsearch') do it { should exist } @@ -19,7 +19,7 @@ shared_examples 'xpack::init' do |es_version| it { should be_owned_by 'elasticsearch' } end - describe file('/etc/elasticsearch/security_node/logging.yml') do + describe file('/etc/elasticsearch/security_node/log4j2.properties') do it { should be_file } it { should be_owned_by 'elasticsearch' } end @@ -29,7 +29,6 @@ shared_examples 'xpack::init' do |es_version| it { should contain 'cluster.name: elasticsearch' } it { should contain 'path.conf: /etc/elasticsearch/security_node' } it { should contain 'path.data: /var/lib/elasticsearch/localhost-security_node' } - it { should contain 'path.work: /tmp/elasticsearch/localhost-security_node' } it { should contain 'path.logs: /var/log/elasticsearch/localhost-security_node' } end @@ -77,17 +76,6 @@ shared_examples 'xpack::init' do |es_version| it { should be_owned_by 'elasticsearch' } end - - #Check x-pack and license plugins are installed - describe file('/usr/share/elasticsearch/plugins/license') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep license') do - its(:exit_status) { should eq 0 } - end - #Test if x-pack is activated describe 'x-pack activation' do it 'should be activated and valid' do @@ -116,16 +104,17 @@ shared_examples 'xpack::init' do |es_version| it { should be_owned_by 'elasticsearch' } end - describe file('/usr/share/elasticsearch/plugins/kopf') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end + for plugin in plugins + describe file('/usr/share/elasticsearch/plugins/'+plugin) do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end - describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep kopf') do - its(:exit_status) { should eq 0 } + describe command('curl -s -u es_admin:changeMe localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do + its(:exit_status) { should eq 0 } + end end - #Test users file, users_roles and roles.yml describe file('/etc/elasticsearch/security_node/x-pack/users_roles') do it { should be_owned_by 'elasticsearch' } @@ -148,19 +137,19 @@ shared_examples 'xpack::init' do |es_version| #Test native roles and users are loaded - describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | md5sum | grep 557a730df7136694131b5b7012a5ffad') do + describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | md5sum | grep 243b362bd47623c0b91a1fafbce2b6f5') do its(:exit_status) { should eq 0 } end - describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | grep "{\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{}}}"') do + describe command('curl -s localhost:9200/_xpack/security/user -u es_admin:changeMe | grep "{\"elastic\":{\"username\":\"elastic\",\"roles\":\[\"superuser\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana\":{\"username\":\"kibana\",\"roles\":\[\"kibana\"\],\"full_name\":null,\"email\":null,\"metadata\":{\"_reserved\":true},\"enabled\":true},\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{},\"enabled\":true}}"') do its(:exit_status) { should eq 0 } end - describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | grep "{\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\]}}"') do + describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | grep "{\"superuser\":{\"cluster\":\[\"all\"\],\"indices\":\[{\"names\":\[\"\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\"\*\"\],\"metadata\":{\"_reserved\":true}},\"transport_client\":{\"cluster\":\[\"transport_client\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"kibana_user\":{\"cluster\":\[\"monitor\"\],\"indices\":\[{\"names\":\[\".kibana\*\"\],\"privileges\":\[\"manage\",\"read\",\"index\",\"delete\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"monitoring_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\".monitoring-\*\"\],\"privileges\":\[\"read\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"remote_monitoring_agent\":{\"cluster\":\[\"manage_index_templates\",\"manage_ingest_pipelines\",\"monitor\"\],\"indices\":\[{\"names\":\[\"\.marvel-es-\*\",\"\.monitoring-\*\"\],\"privileges\":\[\"all\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"ingest_admin\":{\"cluster\":\[\"manage_index_templates\",\"manage_pipeline\"\],\"indices\":\[\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"reporting_user\":{\"cluster\":\[\],\"indices\":\[{\"names\":\[\"\.reporting-\*\"\],\"privileges\":\[\"read\",\"write\"\]}\],\"run_as\":\[\],\"metadata\":{\"_reserved\":true}},\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\],\"metadata\":{}}}"') do its(:exit_status) { should eq 0 } end - describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | md5sum | grep 6d14f09ef1eea64adf4d4a9c04229629') do + describe command('curl -s localhost:9200/_xpack/security/role -u es_admin:changeMe | md5sum | grep 78a0696c9c9690042cec2c1f16860cfc') do its(:exit_status) { should eq 0 } end diff --git a/test/integration/xpack-5x/serverspec/default_spec.rb b/test/integration/xpack-5x/serverspec/default_spec.rb index eb9bef2..bbf646c 100644 --- a/test/integration/xpack-5x/serverspec/default_spec.rb +++ b/test/integration/xpack-5x/serverspec/default_spec.rb @@ -1,5 +1,5 @@ require 'xpack_spec' describe 'Xpack Tests v 5.x' do - include_examples 'xpack::init', "5.1.2" + include_examples 'xpack::init', "5.1.2", ["ingest-geoip"] end From 7b9d4db6da4dfb54af890676fc08c1b74bb92c4d Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Mon, 16 Jan 2017 15:38:54 +0000 Subject: [PATCH 28/42] Dont log on native user install --- handlers/security/elasticsearch-security-native.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers/security/elasticsearch-security-native.yml b/handlers/security/elasticsearch-security-native.yml index 80546ce..463e095 100644 --- a/handlers/security/elasticsearch-security-native.yml +++ b/handlers/security/elasticsearch-security-native.yml @@ -60,7 +60,7 @@ password: "{{es_api_basic_auth_password}}" force_basic_auth: yes when: manage_native_users and es_users.native.keys() > 0 - #no_log: True + no_log: True with_dict: "{{es_users.native}}" #List current roles not. inc those reserved From c22ace11e00ea8466b7ef822c62ce54490b0df4f Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 10:48:26 +0000 Subject: [PATCH 29/42] Remove java opts in favour of es_jvm_options + allow max_map_count to be set. docs updated --- README.md | 7 ++----- templates/elasticsearch.j2 | 27 +++++++-------------------- 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index d693151..67a6284 100644 --- a/README.md +++ b/README.md @@ -331,11 +331,8 @@ Additional parameters to es_config allow the customization of the Java and Elast * ```es_allow_downgrades``` For development purposes only. (true or false (default) ) * ```es_java_install``` If set to false, Java will not be installed. (true (default) or false) * ```update_java``` Updates Java to the latest version. (true or false (default)) -* ```es_java_opts``` an array of java options. E.g.: -```yml -es_java_opts: - - "-Djava.io.tmpdir=/data/tmp/elasticsearch" -``` +* ```es_max_map_count``` maximum number of VMA (Virtual Memory Areas) a process can own. Default is undefined and not used. +* ```es_max_open_files``` the maximum file descriptor number that can be opened by this process. Defaults to 65536. Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full path e.g. "lmenezes/elasticsearch-kopf" and the appropriate version for the target version of Elasticsearch. This approach should NOT be used for X-Pack related plugins e.g. Security. See X-Pack below for details here. diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index a490cec..0acf7ef 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -22,26 +22,7 @@ WORK_DIR={{work_dir}} # Elasticsearch PID directory PID_DIR={{pid_dir}} -{% if es_version | version_compare('5.0', '<') %} -# Heap size defaults to 256m min, 1g max -# Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g -{% if es_heap_size is defined %} -ES_HEAP_SIZE={{es_heap_size}} -{% endif %} -{% endif %} - -# Heap new generation -#ES_HEAP_NEWSIZE= - -# Maximum direct memory -#ES_DIRECT_SIZE= - -# Additional Java OPTS -{% if es_java_opts is defined and es_java_opts %} -ES_JAVA_OPTS="{{es_java_opts | default([]) | join(' ')}}" -{% else %} -#ES_JAVA_OPTS= -{% endif %} +ES_JVM_OPTIONS={{conf_dir}}/jvm.options # Configure restart on package upgrade (true, every other setting will lead to not restarting) #ES_RESTART_ON_UPGRADE=true @@ -63,6 +44,9 @@ ES_JAVA_OPTS="{{es_java_opts | default([]) | join(' ')}}" ES_USER={{es_user}} ES_GROUP={{es_group}} +# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process +ES_STARTUP_SLEEP_TIME=5 + ################################ # System properties ################################ @@ -89,3 +73,6 @@ MAX_LOCKED_MEMORY=unlimited # When using Systemd, this setting is ignored and the 'vm.max_map_count' # property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf #MAX_MAP_COUNT=262144 +{% if es_max_map_count is defined %} +MAX_MAP_COUNT={{es_max_map_count}} +{% endif %} \ No newline at end of file From fedccafb67e615c84c6eb2157f619344c2c23e59 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 10:50:19 +0000 Subject: [PATCH 30/42] remove work dir for 5.x --- README.md | 5 ++--- defaults/main.yml | 2 -- tasks/elasticsearch-config.yml | 1 - tasks/elasticsearch-parameters.yml | 1 - templates/elasticsearch.j2 | 5 ----- templates/elasticsearch.yml.j2 | 4 ---- test/integration/config.yml | 2 +- 7 files changed, 3 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 67a6284..da0ceaa 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ The following illustrates applying configuration parameters to an Elasticsearch hosts: localhost roles: #expand to all available parameters - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", + - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", es_config: { node.name: "node1", cluster.name: "custom-cluster", @@ -95,7 +95,7 @@ A more complex example: hosts: localhost roles: #expand to all available parameters - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", + - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", es_config: { node.name: "node1", cluster.name: "custom-cluster", @@ -351,7 +351,6 @@ controlled by the following parameters: * ```es_pid_dir``` - defaults to "/var/run/elasticsearch". * ```es_data_dirs``` - defaults to "/var/lib/elasticsearch". This can be a list or comma separated string e.g. ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"] or "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2" * ```es_log_dir``` - defaults to "/var/log/elasticsearch". -* ```es_work_dir``` - defaults to "/tmp/elasticsearch". * ```es_restart_on_change``` - defaults to true. If false, changes will not result in Elasticsearch being restarted. * ```es_plugins_reinstall``` - defaults to false. If true, all currently installed plugins will be removed from a node. Listed plugins will then be re-installed. diff --git a/defaults/main.yml b/defaults/main.yml index 178e84f..6f09b9f 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -20,7 +20,6 @@ es_config: {} es_pid_dir: "/var/run/elasticsearch" es_data_dirs: "/var/lib/elasticsearch" es_log_dir: "/var/log/elasticsearch" -es_work_dir: "/tmp/elasticsearch" es_max_open_files: 65536 es_allow_downgrades: false es_enable_xpack: false @@ -34,6 +33,5 @@ es_api_port: 9200 # to allow the role to be conditionally played with a when condition. pid_dir: '' log_dir: '' -work_dir: '' conf_dir: '' data_dirs: '' diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 98d80ea..9eeb5d2 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -6,7 +6,6 @@ file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} with_items: - "{{pid_dir}}" - - "{{work_dir}}" - "{{log_dir}}" - "{{conf_dir}}" diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 502403d..9a7cae3 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -46,5 +46,4 @@ - set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }} - set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}} - set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}} -- set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}} - set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }} diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index 0acf7ef..0305e4e 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -14,11 +14,6 @@ DATA_DIR={{ data_dirs | array_to_str }} # Elasticsearch logs directory LOG_DIR={{log_dir}} -{% if es_version | version_compare('5.0', '<') %} -# Elasticsearch work directory -WORK_DIR={{work_dir}} -{% endif %} - # Elasticsearch PID directory PID_DIR={{pid_dir}} diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index 2bbc5c2..c0cc584 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -18,10 +18,6 @@ path.conf: {{ conf_dir }} path.data: {{ data_dirs | array_to_str }} -{% if es_version | version_compare('5.0', '<') %} -path.work: {{ work_dir }} -{% endif %} - path.logs: {{ log_dir }} {% if not "security" in es_xpack_features %} diff --git a/test/integration/config.yml b/test/integration/config.yml index 9a28b0e..2a05ebe 100644 --- a/test/integration/config.yml +++ b/test/integration/config.yml @@ -4,7 +4,7 @@ hosts: localhost roles: #expand to all available parameters - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: true } } + - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.memory_lock: true } } vars: es_scripts: false es_templates: false From 42c84e07a2683f104fb1652851f2e70908e4f42c Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 11:00:25 +0000 Subject: [PATCH 31/42] Updated debian scripts --- README.md | 2 +- defaults/main.yml | 1 + templates/init/debian/elasticsearch.j2 | 58 ++++++++------------------ 3 files changed, 19 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index da0ceaa..5b81475 100644 --- a/README.md +++ b/README.md @@ -331,7 +331,7 @@ Additional parameters to es_config allow the customization of the Java and Elast * ```es_allow_downgrades``` For development purposes only. (true or false (default) ) * ```es_java_install``` If set to false, Java will not be installed. (true (default) or false) * ```update_java``` Updates Java to the latest version. (true or false (default)) -* ```es_max_map_count``` maximum number of VMA (Virtual Memory Areas) a process can own. Default is undefined and not used. +* ```es_max_map_count``` maximum number of VMA (Virtual Memory Areas) a process can own. Defaults to 262144. * ```es_max_open_files``` the maximum file descriptor number that can be opened by this process. Defaults to 65536. Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full path e.g. "lmenezes/elasticsearch-kopf" and the appropriate version for the target version of Elasticsearch. This approach should NOT be used for X-Pack related plugins e.g. Security. See X-Pack below for details here. diff --git a/defaults/main.yml b/defaults/main.yml index 6f09b9f..fb31044 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -21,6 +21,7 @@ es_pid_dir: "/var/run/elasticsearch" es_data_dirs: "/var/lib/elasticsearch" es_log_dir: "/var/log/elasticsearch" es_max_open_files: 65536 +es_max_map_count: 262144 es_allow_downgrades: false es_enable_xpack: false es_xpack_features: ["alerting","monitoring","graph","security"] diff --git a/templates/init/debian/elasticsearch.j2 b/templates/init/debian/elasticsearch.j2 index 68dc12e..8312be4 100755 --- a/templates/init/debian/elasticsearch.j2 +++ b/templates/init/debian/elasticsearch.j2 @@ -1,16 +1,9 @@ -#!/bin/sh +#!/bin/bash # # /etc/init.d/elasticsearch -- startup script for Elasticsearch # -# Written by Miquel van Smoorenburg . -# Modified for Debian GNU/Linux by Ian Murdock . -# Modified for Tomcat by Stefan Gybas . -# Modified for Tomcat6 by Thierry Carrez . -# Additional improvements by Jason Brittain . -# Modified by Nicolas Huray for Elasticsearch . -# ### BEGIN INIT INFO -# Provides: {{es_instance_name}}_{{default_file | basename}} +# Provides: elasticsearch # Required-Start: $network $remote_fs $named # Required-Stop: $network $remote_fs $named # Default-Start: 2 3 4 5 @@ -34,14 +27,11 @@ if [ `id -u` -ne 0 ]; then exit 1 fi - . /lib/lsb/init-functions - if [ -r /etc/default/rcS ]; then . /etc/default/rcS fi - # The following variables can be overwritten in $DEFAULT # Run Elasticsearch as this user ID and group ID @@ -51,26 +41,16 @@ ES_GROUP={{es_group}} # Directory where the Elasticsearch binary distribution resides ES_HOME={{es_home}} -{% if es_version | version_compare('5.0', '<') %} -# Heap size defaults to 256m min, 1g max -# Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g -#ES_HEAP_SIZE=2g -{% endif %} - -# Heap new generation -#ES_HEAP_NEWSIZE= - -# max direct memory -#ES_DIRECT_SIZE= - -# Additional Java OPTS -#ES_JAVA_OPTS= - # Maximum number of open files -MAX_OPEN_FILES=65535 +{% if es_max_open_files is defined %} +MAX_OPEN_FILES={{es_max_open_files}} +{% endif %} # Maximum amount of locked memory #MAX_LOCKED_MEMORY= +{% if m_lock_enabled %} +MAX_LOCKED_MEMORY=unlimited +{% endif %} # Elasticsearch log directory LOG_DIR={{log_dir}} @@ -82,10 +62,9 @@ DATA_DIR={{ data_dirs | array_to_str }} CONF_DIR={{conf_dir}} # Maximum number of VMA (Virtual Memory Areas) a process can own -MAX_MAP_COUNT=262144 - -# Path to the GC log file -#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log +{% if es_max_map_count is defined %} +MAX_MAP_COUNT={{es_max_map_count}} +{% endif %} # Elasticsearch PID file directory PID_DIR={{pid_dir}} @@ -106,21 +85,18 @@ fi # Define other required variables PID_FILE="$PID_DIR/$NAME.pid" DAEMON={{es_home}}/bin/elasticsearch -{% if es_version | version_compare('5.0', '<') %} -DAEMON_OPTS="-d -p $PID_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR" -{% else %} DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR" -{% endif %} -export ES_HEAP_SIZE -export ES_HEAP_NEWSIZE -export ES_DIRECT_SIZE export ES_JAVA_OPTS -export ES_GC_LOG_FILE export JAVA_HOME +export ES_INCLUDE +export ES_JVM_OPTIONS # Check DAEMON exists -test -x $DAEMON || exit 0 +if [ ! -x "$DAEMON" ]; then + echo "The elasticsearch startup script does not exists or it is not executable, tried: $DAEMON" + exit 1 +fi checkJava() { if [ -x "$JAVA_HOME/bin/java" ]; then From b2efa703da30b206980af2166ed949eb3af9b3ff Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 11:21:18 +0000 Subject: [PATCH 32/42] Redhat start script updates --- templates/init/redhat/elasticsearch.j2 | 63 +++++++++++++++----------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/templates/init/redhat/elasticsearch.j2 b/templates/init/redhat/elasticsearch.j2 index bdb1ad7..b50d63b 100755 --- a/templates/init/redhat/elasticsearch.j2 +++ b/templates/init/redhat/elasticsearch.j2 @@ -32,16 +32,22 @@ if [ -f /etc/rc.d/init.d/functions ]; then fi # Sets the default values for elasticsearch variables used in this script -ES_USER="elasticsearch" -ES_GROUP="elasticsearch" -ES_HOME="/usr/share/elasticsearch" -MAX_OPEN_FILES=65535 -MAX_MAP_COUNT=262144 -LOG_DIR="/var/log/elasticsearch" -DATA_DIR="/var/lib/elasticsearch" -CONF_DIR="/etc/elasticsearch" +ES_USER="{{es_user}}" +ES_GROUP="{{es_group}}" +ES_HOME="{{es_home}}" +{% if es_max_open_files is defined %} +MAX_OPEN_FILES={{es_max_open_files}} +{% endif %} +# Maximum number of VMA (Virtual Memory Areas) a process can own +{% if es_max_map_count is defined %} +MAX_MAP_COUNT={{es_max_map_count}} +{% endif %} -PID_DIR="/var/run/elasticsearch" +LOG_DIR="{{log_dir}}" +DATA_DIR={{ data_dirs | array_to_str }} +CONF_DIR="{{conf_dir}}" + +PID_DIR="{{pid_dir}}" # Source the default env file ES_ENV_FILE="{{instance_default_file}}" @@ -56,16 +62,24 @@ if [ ! -z "$CONF_FILE" ]; then fi exec="$ES_HOME/bin/elasticsearch" -prog="elasticsearch" +prog="{{es_instance_name}}_{{default_file | basename}}" pidfile="$PID_DIR/${prog}.pid" -export ES_HEAP_SIZE -export ES_HEAP_NEWSIZE -export ES_DIRECT_SIZE export ES_JAVA_OPTS -export ES_GC_LOG_FILE -export ES_STARTUP_SLEEP_TIME export JAVA_HOME +export ES_INCLUDE +export ES_JVM_OPTIONS +export ES_STARTUP_SLEEP_TIME + +# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported +if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi +if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi +if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi +if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi +if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi +if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi +if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi +if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi lockfile=/var/lock/subsys/$prog @@ -74,6 +88,11 @@ if [ -n $USER ] && [ -z $ES_USER ] ; then ES_USER=$USER fi +if [ ! -x "$exec" ]; then + echo "The elasticsearch startup script does not exists or it is not executable, tried: $exec" + exit 1 +fi + checkJava() { if [ -x "$JAVA_HOME/bin/java" ]; then JAVA="$JAVA_HOME/bin/java" @@ -90,12 +109,7 @@ checkJava() { start() { checkJava [ -x $exec ] || exit 5 -{% if es_version | version_compare('5.0', '<') %} - if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then - echo "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set" - return 7 - fi -{% endif %} + if [ -n "$MAX_OPEN_FILES" ]; then ulimit -n $MAX_OPEN_FILES fi @@ -105,7 +119,6 @@ start() { if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT fi - export ES_GC_LOG_FILE # Ensure that the PID_DIR exists (it is cleaned at OS startup time) if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then @@ -118,11 +131,7 @@ start() { cd $ES_HOME echo -n $"Starting $prog: " # if not running, start it up here, usually something like "daemon $exec" -{% if es_version | version_compare('5.0', '<') %} - daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -Des.default.path.conf=$CONF_DIR -{% else %} - daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=${DATA_DIR} -Edefault.path.conf=$CONF_DIR -{% endif %} + daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR retval=$? echo [ $retval -eq 0 ] && touch $lockfile From 2c6d812bd559700b350d0858aa86152abff05195 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 11:24:21 +0000 Subject: [PATCH 33/42] Systemd updated --- templates/systemd/elasticsearch.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/systemd/elasticsearch.j2 b/templates/systemd/elasticsearch.j2 index 78d79b4..334310e 100644 --- a/templates/systemd/elasticsearch.j2 +++ b/templates/systemd/elasticsearch.j2 @@ -38,7 +38,9 @@ StandardOutput=journal StandardError=inherit # Specifies the maximum file descriptor number that can be opened by this process +{% if es_max_open_files is defined %} LimitNOFILE={{es_max_open_files}} +{% endif %} # Specifies the maximum number of bytes of memory that may be locked into RAM # Set to "infinity" if you use the 'bootstrap.memory_lock: true' option From b6b9d19082d03c34252bc0ffa0a7b9f988d7cb1e Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 12:00:49 +0000 Subject: [PATCH 34/42] wait for startup added --- handlers/elasticsearch-restart.yml | 8 ++++++++ handlers/main.yml | 3 +-- templates/elasticsearch.j2 | 2 +- test/integration/multi.yml | 3 ++- test/integration/package.yml | 1 + test/integration/standard.yml | 3 ++- test/integration/xpack.yml | 1 + 7 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 handlers/elasticsearch-restart.yml diff --git a/handlers/elasticsearch-restart.yml b/handlers/elasticsearch-restart.yml new file mode 100644 index 0000000..24d444b --- /dev/null +++ b/handlers/elasticsearch-restart.yml @@ -0,0 +1,8 @@ +--- + +- name: Restart Elasticsearch + service: name={{instance_init_script | basename}} state=restarted enabled=yes + when: es_restart_on_change and es_start_service and ((plugin_installed is defined and plugin_installed.changed) or (config_updated is defined and config_updated.changed) or (xpack_state.changed) or (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) + +- name: Wait for elasticsearch to startup + wait_for: host={{es_api_host}} port={{es_api_port}} delay=10 \ No newline at end of file diff --git a/handlers/main.yml b/handlers/main.yml index ff07a08..310e9f7 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -3,8 +3,7 @@ # Restart service and ensure it is enabled - name: restart elasticsearch - service: name={{instance_init_script | basename}} state=restarted enabled=yes - when: es_restart_on_change and es_start_service and ((plugin_installed is defined and plugin_installed.changed) or (config_updated is defined and config_updated.changed) or (xpack_state.changed) or (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) + include: ./handlers/elasticsearch-restart.yml # All security specific actions should go in here - name: activate-security diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index 0305e4e..0c7f4a6 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -59,8 +59,8 @@ MAX_OPEN_FILES={{es_max_open_files}} # in elasticsearch.yml (ES_HEAP_SIZE must also be set). # When using Systemd, the LimitMEMLOCK property must be set # in /usr/lib/systemd/system/elasticsearch.service -{% if m_lock_enabled %} #MAX_LOCKED_MEMORY= +{% if m_lock_enabled %} MAX_LOCKED_MEMORY=unlimited {% endif %} diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 8b1ddd1..6bfa96c 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -3,9 +3,10 @@ - name: Elasticsearch Multi tests hosts: localhost roles: - - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_heap_size: "1g", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } + - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } } vars: es_scripts: true es_templates: true + es_heap_size: "1g" #Plugins installed for this test are specified in .kitchen.yml under suite \ No newline at end of file diff --git a/test/integration/package.yml b/test/integration/package.yml index ef59d37..015f08b 100644 --- a/test/integration/package.yml +++ b/test/integration/package.yml @@ -6,4 +6,5 @@ vars: es_scripts: true es_templates: true + es_heap_size: "1g" #Plugins installed for this test are specified in .kitchen.yml under suite \ No newline at end of file diff --git a/test/integration/standard.yml b/test/integration/standard.yml index fc5798f..6f978de 100644 --- a/test/integration/standard.yml +++ b/test/integration/standard.yml @@ -4,4 +4,5 @@ roles: - { role: elasticsearch, es_instance_name: "node1" } vars: - es_use_repository: "true" \ No newline at end of file + es_use_repository: "true" + es_heap_size: "1g" \ No newline at end of file diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 3826ad9..39099ba 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -6,6 +6,7 @@ "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 }, es_instance_name: "security_node" } vars: + es_heap_size: "1g" es_templates: true es_enable_xpack: true es_xpack_license: "{{ lookup('file', '/tmp/license.json') }}" From baf3a7c9f5fc8d9388229b0790646ffebdebe355 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 12:54:29 +0000 Subject: [PATCH 35/42] Sleep on init + cleanup --- handlers/elasticsearch-restart.yml | 4 ++-- templates/init/debian/elasticsearch.j2 | 2 ++ templates/init/redhat/elasticsearch.j2 | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/handlers/elasticsearch-restart.yml b/handlers/elasticsearch-restart.yml index 24d444b..03e7b5f 100644 --- a/handlers/elasticsearch-restart.yml +++ b/handlers/elasticsearch-restart.yml @@ -1,8 +1,8 @@ --- - name: Restart Elasticsearch - service: name={{instance_init_script | basename}} state=restarted enabled=yes + service: name={{instance_init_script | basename}} state=restarted enabled=yes sleep=5 when: es_restart_on_change and es_start_service and ((plugin_installed is defined and plugin_installed.changed) or (config_updated is defined and config_updated.changed) or (xpack_state.changed) or (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) - name: Wait for elasticsearch to startup - wait_for: host={{es_api_host}} port={{es_api_port}} delay=10 \ No newline at end of file + wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1 \ No newline at end of file diff --git a/templates/init/debian/elasticsearch.j2 b/templates/init/debian/elasticsearch.j2 index 8312be4..5a21e47 100755 --- a/templates/init/debian/elasticsearch.j2 +++ b/templates/init/debian/elasticsearch.j2 @@ -69,6 +69,8 @@ MAX_MAP_COUNT={{es_max_map_count}} # Elasticsearch PID file directory PID_DIR={{pid_dir}} +ES_JVM_OPTIONS="{{conf_dir}}/jvm.options" + # End of variables that can be overwritten in $DEFAULT # overwrite settings from default file diff --git a/templates/init/redhat/elasticsearch.j2 b/templates/init/redhat/elasticsearch.j2 index b50d63b..f906074 100755 --- a/templates/init/redhat/elasticsearch.j2 +++ b/templates/init/redhat/elasticsearch.j2 @@ -48,6 +48,7 @@ DATA_DIR={{ data_dirs | array_to_str }} CONF_DIR="{{conf_dir}}" PID_DIR="{{pid_dir}}" +ES_JVM_OPTIONS="{{conf_dir}}/jvm.options" # Source the default env file ES_ENV_FILE="{{instance_default_file}}" From 7b32a17d59b181ee58564898d924838958dfaf0e Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 12:54:43 +0000 Subject: [PATCH 36/42] remove sleep --- handlers/elasticsearch-restart.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers/elasticsearch-restart.yml b/handlers/elasticsearch-restart.yml index 03e7b5f..41853bd 100644 --- a/handlers/elasticsearch-restart.yml +++ b/handlers/elasticsearch-restart.yml @@ -1,7 +1,7 @@ --- - name: Restart Elasticsearch - service: name={{instance_init_script | basename}} state=restarted enabled=yes sleep=5 + service: name={{instance_init_script | basename}} state=restarted enabled=yes when: es_restart_on_change and es_start_service and ((plugin_installed is defined and plugin_installed.changed) or (config_updated is defined and config_updated.changed) or (xpack_state.changed) or (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) - name: Wait for elasticsearch to startup From 914cb7867cbc64c33c9ebb949e5f4619b71b4bb9 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 14:42:21 +0000 Subject: [PATCH 37/42] Move restart to main from handler --- handlers/elasticsearch-restart.yml | 8 -------- handlers/main.yml | 3 ++- tasks/main.yml | 4 ++++ templates/elasticsearch.j2 | 2 +- templates/systemd/elasticsearch.j2 | 1 - 5 files changed, 7 insertions(+), 11 deletions(-) delete mode 100644 handlers/elasticsearch-restart.yml diff --git a/handlers/elasticsearch-restart.yml b/handlers/elasticsearch-restart.yml deleted file mode 100644 index 41853bd..0000000 --- a/handlers/elasticsearch-restart.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- - -- name: Restart Elasticsearch - service: name={{instance_init_script | basename}} state=restarted enabled=yes - when: es_restart_on_change and es_start_service and ((plugin_installed is defined and plugin_installed.changed) or (config_updated is defined and config_updated.changed) or (xpack_state.changed) or (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) - -- name: Wait for elasticsearch to startup - wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1 \ No newline at end of file diff --git a/handlers/main.yml b/handlers/main.yml index 310e9f7..ff07a08 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -3,7 +3,8 @@ # Restart service and ensure it is enabled - name: restart elasticsearch - include: ./handlers/elasticsearch-restart.yml + service: name={{instance_init_script | basename}} state=restarted enabled=yes + when: es_restart_on_change and es_start_service and ((plugin_installed is defined and plugin_installed.changed) or (config_updated is defined and config_updated.changed) or (xpack_state.changed) or (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) # All security specific actions should go in here - name: activate-security diff --git a/tasks/main.yml b/tasks/main.yml index bcd2df0..fe12c38 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -41,4 +41,8 @@ when: es_templates tags: - templates + - meta: flush_handlers + +- name: Wait for elasticsearch to startup + wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1 \ No newline at end of file diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index 0c7f4a6..a86fe16 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -17,7 +17,7 @@ LOG_DIR={{log_dir}} # Elasticsearch PID directory PID_DIR={{pid_dir}} -ES_JVM_OPTIONS={{conf_dir}}/jvm.options +#ES_JVM_OPTIONS={{conf_dir}}/jvm.options # Configure restart on package upgrade (true, every other setting will lead to not restarting) #ES_RESTART_ON_UPGRADE=true diff --git a/templates/systemd/elasticsearch.j2 b/templates/systemd/elasticsearch.j2 index 334310e..dafae58 100644 --- a/templates/systemd/elasticsearch.j2 +++ b/templates/systemd/elasticsearch.j2 @@ -33,7 +33,6 @@ ExecStart={{es_home}}/bin/elasticsearch \ # stores its logs in /var/log/elasticsearch and does not use # journalctl by default. If you also want to enable journalctl # logging, you can simply remove the "quiet" option from ExecStart. - StandardOutput=journal StandardError=inherit From b9c9fd109e29d38bf314274f5bc75deb1db1f532 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 15:19:50 +0000 Subject: [PATCH 38/42] set api port in config test + doc clarification --- README.md | 10 ++++++---- templates/elasticsearch.j2 | 2 +- test/integration/config.yml | 3 ++- test/integration/multi.yml | 4 ++-- test/integration/package.yml | 2 +- test/integration/xpack.yml | 2 +- 6 files changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5b81475..ac88a52 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ The following illustrates applying configuration parameters to an Elasticsearch es_templates: false es_version_lock: false es_heap_size: 1g + es_api_port:9201 ``` ` The role utilises Elasticsearch version defaults. The following should be set to ensure a successful cluster forms. @@ -117,6 +118,7 @@ A more complex example: es_version_lock: false es_start_service: false es_plugins_reinstall: false + es_api_port:9201 es_plugins: - plugin: license - plugin: lmenezes/elasticsearch-kopf @@ -127,8 +129,8 @@ A more complex example: #### Important Note -The role uses es_api_host and es_api_port to communicate with the node for actions only achievable via http e.g. to install templates. These default to "localhost" and 9200 respectively. -If the node is deployed to bind on either a different host or port, these must be changed. +**The role uses es_api_host and es_api_port to communicate with the node for actions only achievable via http e.g. to install templates and to check the NODE IS ACTIVE. These default to "localhost" and 9200 respectively. +If the node is deployed to bind on either a different host or port, these must be changed.** ### Multi Node Server Installations @@ -174,7 +176,7 @@ recommended in any multi node cluster configuration. cluster.name: "test-cluster" } } - - { role: elasticsearch, es_instance_name: "node2", + - { role: elasticsearch, es_instance_name: "node2", es_api_port:9201, es_config: { discovery.zen.ping.unicast.hosts: "elastic02:9300", http.port: 9201, @@ -317,7 +319,7 @@ Additional parameters to es_config allow the customization of the Java and Elast * ```es_major_version``` (e.g. "5.1" ). Should be consistent with es_version. For versions >= 5.0 this must be "5.x". * ```es_version``` (e.g. "5.1.2"). * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". -* ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. +* ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200** * ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Security is enabled. Ensure this user is admin. * ```es_api_basic_auth_password``` The password associated with the user declared in `es_api_basic_auth_username` * ```es_start_service``` (true (default) or false) diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index a86fe16..0c7f4a6 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -17,7 +17,7 @@ LOG_DIR={{log_dir}} # Elasticsearch PID directory PID_DIR={{pid_dir}} -#ES_JVM_OPTIONS={{conf_dir}}/jvm.options +ES_JVM_OPTIONS={{conf_dir}}/jvm.options # Configure restart on package upgrade (true, every other setting will lead to not restarting) #ES_RESTART_ON_UPGRADE=true diff --git a/test/integration/config.yml b/test/integration/config.yml index 2a05ebe..5c4d3d4 100644 --- a/test/integration/config.yml +++ b/test/integration/config.yml @@ -9,4 +9,5 @@ es_scripts: false es_templates: false es_version_lock: false - es_heap_size: 1g \ No newline at end of file + es_heap_size: 1g + es_api_port: 9201 \ No newline at end of file diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 6bfa96c..e3f1f30 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -3,8 +3,8 @@ - name: Elasticsearch Multi tests hosts: localhost roles: - - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } - - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } } + - { role: elasticsearch, es_api_port:9200, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } + - { role: elasticsearch, es_api_port:9201, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } } vars: es_scripts: true es_templates: true diff --git a/test/integration/package.yml b/test/integration/package.yml index 015f08b..a1319a7 100644 --- a/test/integration/package.yml +++ b/test/integration/package.yml @@ -2,7 +2,7 @@ - name: Elasticsearch Package tests hosts: localhost roles: - - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" } + - { role: elasticsearch, es_api_port:9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" } vars: es_scripts: true es_templates: true diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index 39099ba..a65f321 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -2,7 +2,7 @@ - name: Elasticsearch Xpack tests hosts: localhost roles: - - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", + - { role: elasticsearch, es_api_port:9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 }, es_instance_name: "security_node" } vars: From a0c0ab98373a7a5e6fcfc45939f866ca7d590eae Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 15:24:25 +0000 Subject: [PATCH 39/42] Test fix for config --- test/integration/helpers/serverspec/config_spec.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/integration/helpers/serverspec/config_spec.rb b/test/integration/helpers/serverspec/config_spec.rb index 2f7aa91..d89b7e3 100644 --- a/test/integration/helpers/serverspec/config_spec.rb +++ b/test/integration/helpers/serverspec/config_spec.rb @@ -61,12 +61,7 @@ shared_examples 'config::init' do |es_version| it { should be_directory } it { should be_owned_by 'elasticsearch' } end - - describe file('/opt/elasticsearch/temp/localhost-node1') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - + #test we started on the correct port was used describe command('curl -s "localhost:9201"') do #TODO: This is returning an empty string From 608df1ecf0bd7dd33ff74bd3e82b9fe9f8ef07e9 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Tue, 17 Jan 2017 15:33:51 +0000 Subject: [PATCH 40/42] Test fixes --- test/integration/helpers/serverspec/config_spec.rb | 2 +- test/integration/multi.yml | 4 ++-- test/integration/package.yml | 3 ++- test/integration/xpack.yml | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/integration/helpers/serverspec/config_spec.rb b/test/integration/helpers/serverspec/config_spec.rb index d89b7e3..aca7f1a 100644 --- a/test/integration/helpers/serverspec/config_spec.rb +++ b/test/integration/helpers/serverspec/config_spec.rb @@ -61,7 +61,7 @@ shared_examples 'config::init' do |es_version| it { should be_directory } it { should be_owned_by 'elasticsearch' } end - + #test we started on the correct port was used describe command('curl -s "localhost:9201"') do #TODO: This is returning an empty string diff --git a/test/integration/multi.yml b/test/integration/multi.yml index e3f1f30..c93b7e6 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -3,8 +3,8 @@ - name: Elasticsearch Multi tests hosts: localhost roles: - - { role: elasticsearch, es_api_port:9200, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } - - { role: elasticsearch, es_api_port:9201, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } } + - { role: elasticsearch, es_api_port: 9200, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } + - { role: elasticsearch, es_api_port: 9201, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } } vars: es_scripts: true es_templates: true diff --git a/test/integration/package.yml b/test/integration/package.yml index a1319a7..1275d2f 100644 --- a/test/integration/package.yml +++ b/test/integration/package.yml @@ -2,9 +2,10 @@ - name: Elasticsearch Package tests hosts: localhost roles: - - { role: elasticsearch, es_api_port:9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" } + - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300" }, es_instance_name: "node1" } vars: es_scripts: true es_templates: true es_heap_size: "1g" + es_api_port: 9200 #Plugins installed for this test are specified in .kitchen.yml under suite \ No newline at end of file diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index a65f321..db543e2 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -2,7 +2,7 @@ - name: Elasticsearch Xpack tests hosts: localhost roles: - - { role: elasticsearch, es_api_port:9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", + - { role: elasticsearch, es_api_port: 9200, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", "xpack.security.authc.realms.file1.type": "file","xpack.security.authc.realms.file1.order": 0, "xpack.security.authc.realms.native1.type": "native","xpack.security.authc.realms.native1.order": 1 }, es_instance_name: "security_node" } vars: From 0aa42b67317f1bc197cbd372477255653e922475 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Wed, 18 Jan 2017 11:20:23 +0000 Subject: [PATCH 41/42] Restructure multi test to overcome limitations with same role on same instance multiple times --- test/integration/multi.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/test/integration/multi.yml b/test/integration/multi.yml index c93b7e6..60e45f0 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -1,12 +1,20 @@ --- #Test ability to deploy multiple instances to a machine -- name: Elasticsearch Multi tests +- name: Elasticsearch Multi test - master on 9200 hosts: localhost - roles: - - { role: elasticsearch, es_api_port: 9200, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } - - { role: elasticsearch, es_api_port: 9201, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } } vars: es_scripts: true es_templates: true es_heap_size: "1g" -#Plugins installed for this test are specified in .kitchen.yml under suite \ No newline at end of file + roles: + - { role: elasticsearch, es_api_port: 9200, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } + +- name: Elasticsearch Multi test - data on 9201 + hosts: localhost + vars: + es_scripts: true + es_templates: true + es_heap_size: "1g" + roles: + - { role: elasticsearch, es_api_port: 9201, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } } +#Plugins installed for this test are specified in .kitchen.yml under suite From c294110a6be30ed4df9cfd96ea975967aabaf3bc Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Wed, 18 Jan 2017 11:40:19 +0000 Subject: [PATCH 42/42] Documentation updates for plugins and multi node usage --- README.md | 34 ++++++++++++++++++++++------------ test/integration/multi.yml | 6 ++++-- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ac88a52..b85aaf2 100644 --- a/README.md +++ b/README.md @@ -120,9 +120,7 @@ A more complex example: es_plugins_reinstall: false es_api_port:9201 es_plugins: - - plugin: license - - plugin: lmenezes/elasticsearch-kopf - version: master + - plugin: ingest-geoip proxy_host: proxy.example.com proxy_port: 8080 ``` @@ -136,9 +134,9 @@ If the node is deployed to bind on either a different host or port, these must b The application of the elasticsearch role results in the installation of a node on a host. Specifying the role multiple times for a host therefore results in the installation of multiple nodes for the host. -An example of a two server deployment, each with 1 node on one server and 2 nodes on another. The first server holds the master and is thus declared first. Whilst not mandatory, this is -recommended in any multi node cluster configuration. +An example of a two server deployment is shown below. The first server holds the master and is thus declared first. Whilst not mandatory, this is recommended in any multi node cluster configuration. The second server hosts two data nodes. +**Note the structure of the below playbook for the data nodes. Whilst a more succinct structures are possible which allow the same role to be applied to a host multiple times, we have found the below structure to be the most reliable with respect to var behaviour. This is the tested approach.** ``` - hosts: master_nodes @@ -160,8 +158,9 @@ recommended in any multi node cluster configuration. es_version_lock: false ansible_user: ansible es_plugins: - - plugin: elasticsearch/license - version: latest + - plugin: ingest-geoip + + - hosts: data_nodes roles: @@ -176,6 +175,18 @@ recommended in any multi node cluster configuration. cluster.name: "test-cluster" } } + vars: + es_scripts: false + es_templates: false + es_version_lock: false + ansible_user: ansible + es_api_port: 9200 + es_plugins: + - plugin: ingest-geoip + + +- hosts: data_nodes + roles: - { role: elasticsearch, es_instance_name: "node2", es_api_port:9201, es_config: { discovery.zen.ping.unicast.hosts: "elastic02:9300", @@ -191,10 +202,11 @@ recommended in any multi node cluster configuration. es_scripts: false es_templates: false es_version_lock: false + es_api_port: 9201 ansible_user: ansible es_plugins: - - plugin: elasticsearch/license - version: latest + - plugin: ingest-geoip + ``` Parameters can additionally be assigned to hosts using the inventory file if desired. @@ -328,7 +340,6 @@ Additional parameters to es_config allow the customization of the Java and Elast ```yml es_plugins: - plugin: elasticsearch-cloud-aws - version: 5.0.0 ``` * ```es_allow_downgrades``` For development purposes only. (true or false (default) ) * ```es_java_install``` If set to false, Java will not be installed. (true (default) or false) @@ -336,7 +347,7 @@ Additional parameters to es_config allow the customization of the Java and Elast * ```es_max_map_count``` maximum number of VMA (Virtual Memory Areas) a process can own. Defaults to 262144. * ```es_max_open_files``` the maximum file descriptor number that can be opened by this process. Defaults to 65536. -Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full path e.g. "lmenezes/elasticsearch-kopf" and the appropriate version for the target version of Elasticsearch. This approach should NOT be used for X-Pack related plugins e.g. Security. See X-Pack below for details here. +Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full url. This approach should NOT be used for the X-Pack plugin. See X-Pack below for details here. If installing Monitoring or Alerting, ensure the license plugin is also specified. Security configuration is currently not supported but planned for later versions. @@ -372,7 +383,6 @@ To define proxy only for a particular plugin during its installation: ``` es_plugins: - plugin: elasticsearch-cloud-aws - version: 5.0.0 proxy_host: proxy.example.com proxy_port: 8080 ``` diff --git a/test/integration/multi.yml b/test/integration/multi.yml index 60e45f0..1d191f5 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -6,8 +6,9 @@ es_scripts: true es_templates: true es_heap_size: "1g" + es_api_port: 9200 roles: - - { role: elasticsearch, es_api_port: 9200, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } + - { role: elasticsearch, es_instance_name: "master", es_data_dirs: ["/opt/elasticsearch/master"], es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.memory_lock: true } } - name: Elasticsearch Multi test - data on 9201 hosts: localhost @@ -15,6 +16,7 @@ es_scripts: true es_templates: true es_heap_size: "1g" + es_api_port: 9201 roles: - - { role: elasticsearch, es_api_port: 9201, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } } + - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2", es_config: { discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false } } #Plugins installed for this test are specified in .kitchen.yml under suite