From c52eed7aa11590e44d7ad54a1bc7323e5dbc4bb8 Mon Sep 17 00:00:00 2001 From: Ludovic Petetin Date: Fri, 28 Jun 2019 14:15:03 +0200 Subject: [PATCH 01/10] Set userid and groupid between ElasticSearch removal and installation --- tasks/elasticsearch-Debian.yml | 3 +++ tasks/elasticsearch-RedHat.yml | 4 ++++ tasks/elasticsearch.yml | 4 ---- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tasks/elasticsearch-Debian.yml b/tasks/elasticsearch-Debian.yml index 3b696c7..5809a45 100644 --- a/tasks/elasticsearch-Debian.yml +++ b/tasks/elasticsearch-Debian.yml @@ -54,6 +54,9 @@ - { repo: "{{ es_apt_url }}", state: "present" } - { repo: "{{ es_other_apt_url }}", state: "absent" } +- name: Include optional user and group creation. + when: (es_user_id is defined) and (es_group_id is defined) + include: elasticsearch-optional-user.yml - name: Debian - Ensure elasticsearch is installed become: yes diff --git a/tasks/elasticsearch-RedHat.yml b/tasks/elasticsearch-RedHat.yml index c872fc3..3445b00 100644 --- a/tasks/elasticsearch-RedHat.yml +++ b/tasks/elasticsearch-RedHat.yml @@ -35,6 +35,10 @@ name: '{{ es_other_package_name }}' state: 'absent' +- name: Include optional user and group creation. + when: (es_user_id is defined) and (es_group_id is defined) + include: elasticsearch-optional-user.yml + - name: RedHat - Install Elasticsearch become: yes yum: diff --git a/tasks/elasticsearch.yml b/tasks/elasticsearch.yml index e2361d4..05fd93b 100644 --- a/tasks/elasticsearch.yml +++ b/tasks/elasticsearch.yml @@ -1,9 +1,5 @@ --- -- name: Include optional user and group creation. - when: (es_user_id is defined) and (es_group_id is defined) - include: elasticsearch-optional-user.yml - - name: Include specific Elasticsearch include: elasticsearch-Debian.yml when: ansible_os_family == 'Debian' From 812faf9596587ba438f9dacff89b9c8ce4fb6657 Mon Sep 17 00:00:00 2001 From: Kevin Tibi Date: Mon, 24 Jun 2019 11:32:27 +0200 Subject: [PATCH 02/10] Add option for disable the addition of official repository --- README.md | 3 ++- defaults/main.yml | 1 + tasks/elasticsearch-Debian.yml | 3 ++- tasks/elasticsearch-RedHat.yml | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2074676..b8d8e7d 100644 --- a/README.md +++ b/README.md @@ -386,7 +386,8 @@ In addition to es_config, the following parameters allow the customization of th * ```es_max_open_files``` the maximum file descriptor number that can be opened by this process. Defaults to 65536. * ```es_max_threads``` the maximum number of threads the process can start. Defaults to 2048 (the minimum required by elasticsearch). * ```es_debian_startup_timeout``` how long Debian-family SysV init scripts wait for the service to start, in seconds. Defaults to 10 seconds. -* ```es_use_repository``` Setting this to `false` will stop Ansible from using the official Elastic package repositories. +* ```es_use_repository``` Setting this to `false` will stop Ansible from using the official Elastic package from any repository configured on the system. +* ```es_add_repository``` Setting this to `false` will stop Ansible to add the official Elastic package repositories (if es_use_repository is true) if you want to use a repo already present. * ```es_custom_package_url``` the URL to the rpm or deb package for Ansible to install. When using this you will also need to set `es_use_repository: false` and make sure that the `es_version` matches the version being installed from your custom URL. E.g. `es_custom_package_url: https://downloads.example.com/elasticsearch.rpm` Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full url. This approach should NOT be used for the X-Pack plugin. See X-Pack below for details here. diff --git a/defaults/main.yml b/defaults/main.yml index 02ae64d..8de86f4 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -5,6 +5,7 @@ es_enable_xpack: true es_package_name: "elasticsearch" es_version_lock: false es_use_repository: true +es_add_repository: true es_templates_fileglob: "files/templates-{{ es_major_version }}/*.json" es_repo_base: "https://artifacts.elastic.co" es_apt_key: "{{ es_repo_base }}/GPG-KEY-elasticsearch" diff --git a/tasks/elasticsearch-Debian.yml b/tasks/elasticsearch-Debian.yml index 3b696c7..257ea54 100644 --- a/tasks/elasticsearch-Debian.yml +++ b/tasks/elasticsearch-Debian.yml @@ -43,12 +43,13 @@ apt_key: url: '{{ es_apt_key }}' state: present - when: es_apt_key | string + when: es_add_repository and es_apt_key | string - name: Debian - Add elasticsearch repository apt_repository: repo: '{{ item.repo }}' state: '{{ item.state }}' + when: es_add_repository with_items: - { repo: "{{ es_apt_url_old }}", state: "absent" } - { repo: "{{ es_apt_url }}", state: "present" } diff --git a/tasks/elasticsearch-RedHat.yml b/tasks/elasticsearch-RedHat.yml index c872fc3..97c49a9 100644 --- a/tasks/elasticsearch-RedHat.yml +++ b/tasks/elasticsearch-RedHat.yml @@ -16,7 +16,7 @@ template: src: 'elasticsearch.repo' dest: '/etc/yum.repos.d/elasticsearch-{{ es_repo_name }}.repo' - when: es_use_repository + when: es_use_repository and es_add_repository - name: RedHat - remove unused Elasticsearch repo become: yes From 582c79c607f4a564f49b56a0b1b566eb8c0cd1c5 Mon Sep 17 00:00:00 2001 From: Michael Russell Date: Tue, 9 Jul 2019 21:26:10 +0200 Subject: [PATCH 03/10] Add back in force_basic_auth for all http requests Related: #576 This is needed when using security for 6.x. All other http requests still have this parameter added. --- tasks/elasticsearch-template.yml | 1 + tasks/xpack/security/elasticsearch-xpack-activation.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/tasks/elasticsearch-template.yml b/tasks/elasticsearch-template.yml index e1bc80d..8af780d 100644 --- a/tasks/elasticsearch-template.yml +++ b/tasks/elasticsearch-template.yml @@ -20,6 +20,7 @@ status_code: 200 user: "{{es_api_basic_auth_username | default(omit)}}" password: "{{es_api_basic_auth_password | default(omit)}}" + force_basic_auth: yes body_format: json body: "{{ lookup('file', item) }}" when: load_templates.changed and es_start_service diff --git a/tasks/xpack/security/elasticsearch-xpack-activation.yml b/tasks/xpack/security/elasticsearch-xpack-activation.yml index 996d86a..7da085e 100644 --- a/tasks/xpack/security/elasticsearch-xpack-activation.yml +++ b/tasks/xpack/security/elasticsearch-xpack-activation.yml @@ -8,6 +8,7 @@ body_format: json body: "{{ es_xpack_license }}" return_content: yes + force_basic_auth: yes register: license_activated no_log: True failed_when: > From 226df614ebc8dfb68208baaf36ad7dc38885c03a Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 12 Jul 2019 09:53:12 +0200 Subject: [PATCH 04/10] strip spaces to avoid wrong indent --- templates/elasticsearch.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index 47346ed..abb6c56 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -50,8 +50,8 @@ xpack.notification.email: auth: {{ es_mail_config['require_auth'] }} host: {{ es_mail_config['host'] }} port: {{ es_mail_config['port'] }} - {% if es_mail_config['require_auth'] == true %} + {% if es_mail_config['require_auth'] == true -%} user: {{ es_mail_config['user'] }} password: {{ es_mail_config['pass'] }} - {% endif %} + {%- endif %} {% endif %} From 718b3936f17b691500921fd66e352418548293da Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 12 Jul 2019 11:07:10 +0200 Subject: [PATCH 05/10] remove http.port recommendation --- tasks/elasticsearch-parameters.yml | 4 ---- test/integration/xpack-upgrade.yml | 4 ---- test/integration/xpack.yml | 2 -- 3 files changed, 10 deletions(-) diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index f20c2d2..a27088a 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -4,10 +4,6 @@ fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined" when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '') -- name: debug message - debug: msg="WARNING - It is recommended you specify the parameter 'http.port'" - when: es_config['http.port'] is not defined - #If the user attempts to lock memory they must specify a heap size - name: fail when heap size is not specified when using memory lock fail: msg="If locking memory with bootstrap.memory_lock a heap size must be specified" diff --git a/test/integration/xpack-upgrade.yml b/test/integration/xpack-upgrade.yml index c1944fe..1270007 100644 --- a/test/integration/xpack-upgrade.yml +++ b/test/integration/xpack-upgrade.yml @@ -7,13 +7,11 @@ - elasticsearch vars: es_config_6x: - http.port: 9200 xpack.security.authc.realms.file1.order: 0 xpack.security.authc.realms.file1.type: file xpack.security.authc.realms.native1.order: 1 xpack.security.authc.realms.native1.type: native es_config_7x: - http.port: 9200 xpack.security.authc.realms.file.file1.order: 0 xpack.security.authc.realms.native.native1.order: 1 es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" @@ -117,13 +115,11 @@ - elasticsearch vars: es_config_6x: - http.port: 9200 xpack.security.authc.realms.file1.order: 0 xpack.security.authc.realms.file1.type: file xpack.security.authc.realms.native1.order: 1 xpack.security.authc.realms.native1.type: native es_config_7x: - http.port: 9200 xpack.security.authc.realms.file.file1.order: 0 xpack.security.authc.realms.native.native1.order: 1 es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}" diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml index aa02d61..e12064d 100644 --- a/test/integration/xpack.yml +++ b/test/integration/xpack.yml @@ -7,7 +7,5 @@ roles: - elasticsearch vars: - es_config: - http.port: 9200 es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip" es_heap_size: 2g From 6ca0f063720c68db9bf66d754e8307719269969a Mon Sep 17 00:00:00 2001 From: Julien Mailleret Date: Fri, 12 Jul 2019 11:12:45 +0200 Subject: [PATCH 06/10] fix variables names --- docs/multi-instance.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/multi-instance.md b/docs/multi-instance.md index b934904..11872a9 100644 --- a/docs/multi-instance.md +++ b/docs/multi-instance.md @@ -10,11 +10,11 @@ If you have single-instances hosts and want to upgrade from previous versions of 1. Override these variables to match previous values: ```yaml -es_conf_dir: /etc/elasticsearch/{{ instance_name }} +es_conf_dir: /etc/elasticsearch/{{ es_instance_name }} es_data_dirs: - - /var/lib/elasticsearch/{{ node_name }}-{{ instance_name }} -es_log_dir: /var/log/elasticsearch/{{ node_name }}-{{ instance_name }} -es_pid_dir: /var/run/elasticsearch/{{ node_name }}-{{ instance_name }} + - /var/lib/elasticsearch/{{ inventory_hostname }}-{{ es_instance_name }} +es_log_dir: /var/log/elasticsearch/{{ inventory_hostname }}-{{ es_instance_name }} +es_pid_dir: /var/run/elasticsearch/{{ inventory_hostname }}-{{ es_instance_name }} ``` 2. Deploy ansible-role. **Even if these variables are overrided, Elasticsearch config file and default option file will change, which imply an Elasticsearch restart.** From b73cd06c1491515bdaf7912a97ee3865208b8f36 Mon Sep 17 00:00:00 2001 From: Ludovic Petetin Date: Mon, 22 Jul 2019 12:28:40 +0200 Subject: [PATCH 07/10] Set limitMEMLOCK for OS using systemd --- tasks/elasticsearch-config.yml | 12 ++++++++++++ tasks/elasticsearch-parameters.yml | 4 ++++ vars/main.yml | 1 + 3 files changed, 17 insertions(+) diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 3d4f304..9b7d14f 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -24,6 +24,18 @@ template: src=elasticsearch.j2 dest={{ default_file }} mode=0644 force=yes notify: restart elasticsearch +#Copy the systemd specific file if systemd is installed +- name: Make sure destination dir exists + file: path={{ sysd_config_file | dirname }} state=directory recurse=yes mode=0755 + +- name: Copy Systemd File for Instance + become: yes + copy: src=systemd/elasticsearch_override.conf dest={{ sysd_config_file }} mode=0644 force=yes + when: use_system_d and m_lock_enabled + notify: + - reload systemd configuration + - restart elasticsearch + #Copy the logging.yml - name: Copy log4j2.properties File become: yes diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index a27088a..36c3fe1 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -28,3 +28,7 @@ - name: set fact m_lock_enabled set_fact: m_lock_enabled={{ es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True }} + +- name: set fact use_system_d + set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version is version_compare('8', '>=')) or (ansible_distribution in ['RedHat','CentOS'] and ansible_distribution_version is version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version_compare('15', '>=')) }} + diff --git a/vars/main.yml b/vars/main.yml index d640fc4..91b7b69 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -1,3 +1,4 @@ --- es_package_url: "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch" reserved_xpack_users: ["elastic","kibana","logstash_system"] +sysd_config_file: "/etc/systemd/system/elasticsearch.service.d/override.conf" From 2142c6f8f92fc58ad6103e11442072f3daf470c5 Mon Sep 17 00:00:00 2001 From: Ludovic Petetin Date: Mon, 22 Jul 2019 12:30:43 +0200 Subject: [PATCH 08/10] adding systemd limitMEMLOCK override file --- files/systemd/elasticsearch_override.conf | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 files/systemd/elasticsearch_override.conf diff --git a/files/systemd/elasticsearch_override.conf b/files/systemd/elasticsearch_override.conf new file mode 100644 index 0000000..bf02203 --- /dev/null +++ b/files/systemd/elasticsearch_override.conf @@ -0,0 +1,2 @@ +[Service] +LimitMEMLOCK=infinity From 5994840c75320e460056dd44b8af021d389adf0f Mon Sep 17 00:00:00 2001 From: Ludovic Petetin Date: Mon, 22 Jul 2019 14:42:26 +0200 Subject: [PATCH 09/10] Create overriding systemd directory only if using systemd and memory lock is enabled --- tasks/elasticsearch-config.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 9b7d14f..f3508a7 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -25,16 +25,17 @@ notify: restart elasticsearch #Copy the systemd specific file if systemd is installed -- name: Make sure destination dir exists - file: path={{ sysd_config_file | dirname }} state=directory recurse=yes mode=0755 +- when: use_system_d and m_lock_enabled + block: + - name: Make sure destination dir exists + file: path={{ sysd_config_file | dirname }} state=directory recurse=yes mode=0755 -- name: Copy Systemd File for Instance - become: yes - copy: src=systemd/elasticsearch_override.conf dest={{ sysd_config_file }} mode=0644 force=yes - when: use_system_d and m_lock_enabled - notify: - - reload systemd configuration - - restart elasticsearch + - name: Copy Systemd File for Instance + become: yes + copy: src=systemd/elasticsearch_override.conf dest={{ sysd_config_file }} mode=0644 force=yes + notify: + - reload systemd configuration + - restart elasticsearch #Copy the logging.yml - name: Copy log4j2.properties File From 5bac5a4dbf07f748c77175fbc032ac6052d45cd3 Mon Sep 17 00:00:00 2001 From: Ludovic Petetin Date: Mon, 22 Jul 2019 15:19:00 +0200 Subject: [PATCH 10/10] Use ini_file module for systemd overriding config file, better task naming and fix variable indentation --- tasks/elasticsearch-config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index f3508a7..e3437f2 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -26,13 +26,13 @@ #Copy the systemd specific file if systemd is installed - when: use_system_d and m_lock_enabled + become: yes block: - name: Make sure destination dir exists - file: path={{ sysd_config_file | dirname }} state=directory recurse=yes mode=0755 + file: path={{ sysd_config_file | dirname }} state=directory mode=0755 - - name: Copy Systemd File for Instance - become: yes - copy: src=systemd/elasticsearch_override.conf dest={{ sysd_config_file }} mode=0644 force=yes + - name: Copy specific ElasticSearch Systemd config file + ini_file: path={{ sysd_config_file }} section=Service option=LimitMEMLOCK value=infinity mode=0644 notify: - reload systemd configuration - restart elasticsearch @@ -40,7 +40,7 @@ #Copy the logging.yml - name: Copy log4j2.properties File become: yes - template: src={{es_config_log4j2}} dest={{ es_conf_dir }}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + template: src={{ es_config_log4j2 }} dest={{ es_conf_dir }}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes notify: restart elasticsearch - name: Copy jvm.options File