Merge branch 'master' into become-yes

tasks/elasticsearch-Debian.yml

@@ -32,6 +32,7 @@
   apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present force={{force_install}} allow_unauthenticated={{ 'no' if es_apt_key else 'yes' }} cache_valid_time=86400
   when: es_use_repository
   register: debian_elasticsearch_install_from_repo
+  notify: restart elasticsearch
 
 - name: Debian - Download elasticsearch from url
   get_url: url={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.deb{% endif %} dest=/tmp/elasticsearch-{{ es_version }}.deb validate_certs=no
@@ -42,3 +43,4 @@
   apt: deb=/tmp/elasticsearch-{{ es_version }}.deb
   when: not es_use_repository
   register: elasticsearch_install_from_package
+  notify: restart elasticsearch

tasks/elasticsearch-RedHat.yml

@@ -18,6 +18,7 @@
   yum: name=elasticsearch{% if es_version is defined and es_version != ""  %}-{{ es_version }}{% endif %} state=present update_cache=yes
   when: es_use_repository
   register: redhat_elasticsearch_install_from_repo
+  notify: restart elasticsearch
   until: '"failed" not in redhat_elasticsearch_install_from_repo'
   retries: 5
   delay: 10
@@ -27,3 +28,4 @@
   yum: name={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.noarch.rpm{% endif %} state=present
   when: not es_use_repository
   register: elasticsearch_install_from_package
+  notify: restart elasticsearch

tasks/elasticsearch-config.yml

@@ -21,7 +21,7 @@
 - name: Copy Configuration File
   become: yes
   template: src=elasticsearch.yml.j2 dest={{conf_dir}}/elasticsearch.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
-  register: config_updated
+  register: system_change
   notify: restart elasticsearch
 
 #Copy the instance specific default file
@@ -56,7 +56,7 @@
 #Copy the logging.yml
 - name: Copy log4j2.properties File for Instance
   become: yes
-  template: src=log4j2.properties.j2 dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
+  template: src={{es_config_log4j2}} dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   notify: restart elasticsearch
 
 - name: Copy jvm.options File for Instance

tasks/elasticsearch-optional-user.yml

@@ -2,6 +2,8 @@
 #Add the elasticsearch user before installing from packages.
 - name: Ensure optional elasticsearch group is created with the correct id.
   become: yes
+  #Restart if these change
+  notify: restart elasticsearch
   group:
     state: present
     name: "{{ es_group }}"
@@ -10,6 +12,8 @@
 
 - name: Ensure optional elasticsearch user is created with the correct id.
   become: yes
+  #Restart if these change
+  notify: restart elasticsearch
   user:
     state: present
     name: "{{ es_user }}"

tasks/elasticsearch-plugins.yml

@@ -29,13 +29,20 @@
 - set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('[*].plugin')) | default([]) }}"
   when: not es_plugins_reinstall
 
+#if es_plugins_reinstall is set to true we (re)install ALL plugins
+- set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | default([]) }}"
+  when: es_plugins_reinstall
+
+#if the plugins listed are different than those requested, we install those not installed but listed in the config
+- set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | difference(installed_plugins.stdout_lines) | default([]) }}"
+  when: not es_plugins_reinstall
+
 # This removes any currently installed plugins (to prevent errors when reinstalling)
 - name: Remove elasticsearch plugins
   become: yes
   command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent"
   ignore_errors: yes
   with_items: "{{ plugins_to_remove | default([]) }}"
-  when: es_plugins_reinstall and plugins_to_remove | length > 0
   notify: restart elasticsearch
   register: plugin_removed
   environment:
@@ -44,16 +51,17 @@
 
 - name: Install elasticsearch plugins
   become: yes
-  command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.plugin }} --batch --silent {% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -DproxyHost={{ item.proxy_host }} -DproxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -DproxyHost={{ es_proxy_host }} -DproxyPort={{ es_proxy_port }} {% endif %}"
+  command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.plugin }} --batch --silent"
   register: plugin_installed
   failed_when: "'ERROR' in plugin_installed.stdout"
   changed_when: plugin_installed.rc == 0
-  with_items: "{{ es_plugins | default([]) }}"
-  when: not es_plugins is none and es_plugins_reinstall
+  with_items: "{{ es_plugins }}"
+  when: item.plugin in plugins_to_install
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
+    ES_JAVA_OPTS: "{% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -Dhttp.proxyHost={{ item.proxy_host }} -Dhttp.proxyPort={{ item.proxy_port }} -Dhttps.proxyHost={{ item.proxy_host }} -Dhttps.proxyPort={{ item.proxy_port }}  {% elif es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }} {% endif %}"
   until: plugin_installed.rc == 0
   retries: 5
   delay: 5

tasks/elasticsearch-template.yml

@@ -0,0 +1,45 @@
+---
+
+- file: path=/etc/elasticsearch/templates state=directory owner={{ es_user }} group={{ es_group }}
+
+- name: Copy templates to elasticsearch
+  copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }}
+  register: load_templates
+  with_fileglob:
+    - "{{ es_templates_fileglob | default('') }}"
+
+
+- name: Ensure elasticsearch is started
+  service: name={{instance_init_script | basename}} state=started enabled=yes
+  when: es_start_service and load_templates.changed
+
+- name: Wait for elasticsearch to startup
+  wait_for: host={{es_api_host}} port={{es_api_port}} delay=10
+  when: es_start_service and load_templates.changed
+
+- name: Install templates without auth
+  uri:
+    url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}"
+    method: PUT
+    status_code: 200
+    body_format: json
+    body: "{{ lookup('file', item) }}"
+  when: load_templates.changed and es_start_service and not es_enable_xpack or not es_xpack_features is defined or "security" not in es_xpack_features
+  with_fileglob:
+    - "{{ es_templates_fileglob | default('') }}"
+  run_once: True
+
+- name: Install templates with auth
+  uri:
+    url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}"
+    method: PUT
+    status_code: 200
+    user: "{{es_api_basic_auth_username}}"
+    password: "{{es_api_basic_auth_password}}"
+    force_basic_auth: yes
+    body_format: json
+    body: "{{ lookup('file', item) }}"
+  when: load_templates.changed and es_start_service and es_enable_xpack and es_xpack_features is defined and "security" in es_xpack_features
+  with_fileglob:
+    - "{{ es_templates_fileglob | default('') }}"
+  run_once: True

tasks/elasticsearch-templates.yml

@@ -1,17 +0,0 @@
----
-
-- become: yes
-  file: path=/etc/elasticsearch/templates state=directory owner={{ es_user }} group={{ es_group }}
-
-- name: Copy default templates to elasticsearch
-  become: yes
-  copy: src=templates dest=/etc/elasticsearch/ owner={{ es_user }} group={{ es_group }}
-  notify: load-templates
-  when: es_templates_fileglob is not defined
-
-- name: Copy templates to elasticsearch
-  become: yes
-  copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }}
-  notify: load-templates
-  with_fileglob:
-    - "{{ es_templates_fileglob | default('') }}"

tasks/main.yml

@@ -37,20 +37,26 @@
   tags:
       - xpack
 
-- include: elasticsearch-templates.yml
+- meta: flush_handlers
+
+#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password.  The templates will subsequently not be removed if we don't wait for the node to restart.
+- include: elasticsearch-template.yml
   when: es_templates
   tags:
       - templates
 
-- meta: flush_handlers
+- name: Make sure elasticsearch is started
+  service: name={{instance_init_script | basename}} state=started enabled=yes
+  when: es_start_service
 
 - name: Wait for elasticsearch to startup
   wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1
+  when: es_restarted is defined and es_restarted.changed and es_start_service
 
 - name: activate-license
   include: ./xpack/security/elasticsearch-xpack-activation.yml
-  when: es_enable_xpack and es_xpack_license is defined and es_xpack_license != ''
+  when: es_start_service and es_enable_xpack and es_xpack_license is defined and es_xpack_license != ''
 
 #perform security actions here now elasticsearch is started
 - include: ./xpack/security/elasticsearch-security-native.yml
-  when: (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
+  when: es_start_service and (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))

tasks/xpack/elasticsearch-xpack-install.yml

@@ -28,15 +28,36 @@
 
 
 #Install plugin if not installed, or the es version has changed (so removed above), and its been requested
-- name: Install x-pack plugin
+- name: Download x-pack from url
+  get_url: url={{ es_xpack_custom_url }} dest=/tmp/x-pack-{{ es_version }}.zip
+  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined)
+
+- name: Install x-pack plugin from local
   become: yes
   command: >
-    {{es_home}}/bin/elasticsearch-plugin install --silent --batch x-pack {% if es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} {% endif %}
+    {{es_home}}/bin/elasticsearch-plugin install --silent --batch file:///tmp/x-pack-{{ es_version }}.zip
   register: xpack_state
-  failed_when: "'ERROR' in xpack_state.stdout"
   changed_when: xpack_state.rc == 0
-  when: (x_pack_installed.rc == 1 or es_version_changed) and es_enable_xpack
+  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined)
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
+
+- name: Delete x-pack zip file
+  file: dest=/tmp/x-pack-{{ es_version }}.zip state=absent
+  when: es_xpack_custom_url is defined
+
+- name: Install x-pack plugin from elastic.co
+  become: yes
+  command: >
+    {{es_home}}/bin/elasticsearch-plugin install --silent --batch x-pack
+  register: xpack_state
+  failed_when: "'ERROR' in xpack_state.stdout"
+  changed_when: xpack_state.rc == 0
+  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is not defined)
+  notify: restart elasticsearch
+  environment:
+    CONF_DIR: "{{ conf_dir }}"
+    ES_INCLUDE: "{{ instance_default_file }}"
+    ES_JAVA_OPTS: "{% if es_proxy_host is defined and es_proxy_host != '' %}-Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }}{% endif %}"

tasks/xpack/security/elasticsearch-security-file.yml

@@ -1,13 +1,6 @@
 ---
 - set_fact: manage_file_users=es_users is defined and es_users.file is defined
 
-#Ensure x-pack conf directory is created
-- name: Ensure x-pack conf directory exists (file)
-  become: yes
-  file: path={{ conf_dir }}/x-pack state=directory owner={{ es_user }} group={{ es_group }}
-  changed_when: False
-  when: es_enable_xpack and '"security" in es_xpack_features'
-
 #List current users
 - name: List Users
   become: yes

tasks/xpack/security/elasticsearch-security-native.yml

@@ -81,12 +81,13 @@
 
 - set_fact: current_roles={{ role_list_response.json | filter_reserved }}
   when: manage_native_roles
+
 - debug: msg="{{current_roles}}"
+  when: manage_native_roles
 
 - set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys()  ) }}
   when: manage_native_roles
 
-
 #Delete all non required roles
 - name: Delete Native Roles
   uri:

tasks/xpack/security/elasticsearch-security.yml

@@ -3,10 +3,18 @@
 
 #TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6
 
+#Ensure x-pack conf directory is created if necessary
+- name: Ensure x-pack conf directory exists (file)
+  file: path={{ conf_dir }}/x-pack state=directory owner={{ es_user }} group={{ es_group }}
+  changed_when: False
+  when:
+    - es_enable_xpack and '"security" in es_xpack_features'
+    - (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined)
+
 #-----------------------------FILE BASED REALM----------------------------------------
 
 - include: elasticsearch-security-file.yml
-  when: (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.file) or (es_roles is defined and es_roles.file is defined))
+  when: (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined))
 
 #-----------------------------ROLE MAPPING ----------------------------------------
 

tasks/xpack/security/elasticsearch-xpack-activation.yml

@@ -3,7 +3,7 @@
 - name: Activate ES license (without security authentication)
   uri:
     method: PUT
-    url: "http://{{es_api_host}}:{{es_api_port}}/_license?acknowledge=true"
+    url: "http://{{es_api_host}}:{{es_api_port}}/_xpack/license?acknowledge=true"
     body_format: json
     body: "{{ es_xpack_license }}"
     return_content: yes
@@ -18,7 +18,7 @@
 - name: Activate ES license (with security authentication)
   uri:
     method: PUT
-    url: "http://{{es_api_host}}:{{es_api_port}}/_license?acknowledge=true"
+    url: "http://{{es_api_host}}:{{es_api_port}}/_xpack/license?acknowledge=true"
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     body_format: json
@@ -34,4 +34,4 @@
     license_activated.json.license_status != 'valid'
 
 - debug:
-    msg: "License: {{ license_activated.content }}"
+    msg: "License: {{ license_activated }}"