Added more x-pack config options

README.md

@@ -339,6 +339,8 @@ In addition to es_config, the following parameters allow the customization of th
 * ```es_plugins_reinstall``` (true or false (default) )
 * ```es_plugins``` an array of plugin definitions e.g.:
 * ```es_path_repo``` Sets the whitelist for allowing local back-up repositories
+* ```es_action_auto_create_index ``` Sets the value for auto index creation, use the syntax below for specifying indexes (else true/false):
+     es_action_auto_create_index: '[".watches", ".triggered_watches", ".watcher-history-*"]'
 ```yml
   es_plugins:
     - plugin: ingest-geoip 

defaults/main.yml

@@ -22,6 +22,7 @@ es_config_log4j2: log4j2.properties.j2
 es_pid_dir: "/var/run/elasticsearch"
 es_data_dirs: "/var/lib/elasticsearch"
 es_log_dir: "/var/log/elasticsearch"
+es_action_auto_create_index: true
 es_max_open_files: 65536
 es_max_threads: "{{ 2048 if ( es_version | version_compare('6.0.0', '<')) else 8192 }}"
 es_max_map_count: 262144
@@ -33,7 +34,7 @@ es_xpack_features: ["alerting","monitoring","graph","ml","security"]
 es_api_host: "localhost"
 es_api_port: 9200
 es_debian_startup_timeout: 10
-
+es_http_type: http
 # Since ansible 2.2 the following variables need to be defined
 # to allow the role to be conditionally played with a when condition.
 pid_dir: ''

tasks/elasticsearch-Debian-version-lock.yml

@@ -1,6 +1,10 @@
 ---
-- name: Debian - hold elasticsearch version
+- name: Debian - hold elasticsearch version with apt
   become: yes
   command: apt-mark hold elasticsearch
   register: hold_elasticsearch_result
   changed_when: "hold_elasticsearch_result.stdout != 'elasticsearch was already set on hold.'"
+- name: Debian - hold elasticsearch version with aptitude
+  become: yes
+  command: aptitude hold elasticsearch
+  register: aptitude_hold_elasticsearch_result

tasks/xpack/elasticsearch-xpack-install.yml

@@ -66,3 +66,6 @@
     ES_PATH_CONF: "{{ conf_dir }}"
     ES_INCLUDE: "{{ instance_default_file }}"
     ES_JAVA_OPTS: "{% if es_proxy_host is defined and es_proxy_host != '' %}-Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }}{% endif %}"
+
+- name: Generate SSL/TLS environment
+  include: ./elasticsearch-xpack-ssl.yml

tasks/xpack/elasticsearch-xpack-ssl.yml

@@ -0,0 +1,146 @@
+#### Install SSL/TLS certificates when platinum license is present
+#ES_PATH_CONF="/etc/elasticsearch/ases1" && export ES_PATH_CONF
+#/usr/share/elasticsearch/bin/x-pack/setup-passwords auto --url https://localhost:9200
+
+- name: Check if /etc/ssl/elasticsearch folder exists
+  file:
+    path: /etc/ssl/elasticsearch
+    state: directory
+    owner: elasticsearch
+    group: elasticsearch
+    mode: 0755
+  register: es_ssl_folder
+
+- name: Verify if elastic CA keys are present
+  stat:
+    path: "/etc/ssl/elasticsearch/elastic-ca.p12"
+  become: true
+  become_user: elasticsearch
+  register: es_ssl_ca_present
+  when: es_ssl_folder
+
+- name: Generate SSL/TLS CA Authority (required for platinum license)
+  environment: 
+    - ES_PATH_CONF: "{{conf_dir}}"
+  command: /usr/share/elasticsearch/bin/x-pack/certutil ca --silent -out /etc/ssl/elasticsearch/elastic-ca.p12 -pass "test"
+  become: true
+  become_user: elasticsearch
+  register: elastic_ca
+  when: es_platinum_license is defined and es_ssl_config["enabled"] is defined and not es_ssl_ca_present.stat.exists
+
+- name: Check if /usr/local/share/ca-certificates/local-elastic-ca folder exists
+  file:
+    path: /usr/local/share/ca-certificates/local-elastic-ca
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  register: es_ca_folder
+
+- name: Extract CA certificate to the trusted CA folder
+  command: openssl pkcs12 -in /etc/ssl/elasticsearch/elastic-ca.p12 -clcerts -nokeys -out /usr/local/share/ca-certificates/local-elastic-ca/elastic-ca.crt -passin pass:test
+  when: es_ca_folder
+
+- name: Add CA certificate to cacerts
+  become: true
+  command: update-ca-certificates
+
+- name: Verify if elastic Cert keys are present
+  stat:
+    path: "{{conf_dir}}/ssl/{{ es_ssl_config['dns'] }}.p12"
+  become: true
+  become_user: elasticsearch
+  register: es_ssl_cert_present
+
+- name: Generate SSL/TLS certificate for ES (required for platinum license)
+  environment: 
+    - ES_PATH_CONF: "{{conf_dir}}"
+  command: /usr/share/elasticsearch/bin/x-pack/certutil cert --silent --ca /etc/ssl/elasticsearch/elastic-ca.p12 --ca-pass "test" --ip {{ es_ssl_config['ip'] }} --dns {{ es_ssl_config['dns'] }},localhost --out {{conf_dir}}/ssl/{{ es_ssl_config['dns'] }}.p12 --pass "test"
+  become: true
+  become_user: elasticsearch
+  when: es_ssl_regen_cert is defined or not es_ssl_cert_present.stat.exists
+
+- name: Add SSL/TLS keystore password to ES keystore (required for platinum license)
+  environment: 
+    - ES_PATH_CONF: "{{conf_dir}}"
+  shell: echo "test" | /usr/share/elasticsearch/bin/elasticsearch-keystore add --silent xpack.security.http.ssl.keystore.secure_password -x --force
+  become: true
+  become_user: elasticsearch
+  when: elastic_ca
+
+- name: Add SSL/TLS truststore password to ES keystore (required for platinum license)
+  environment: 
+    - ES_PATH_CONF: "{{conf_dir}}"
+  shell: echo "test" | /usr/share/elasticsearch/bin/elasticsearch-keystore add --silent xpack.security.http.ssl.truststore.secure_password -x --force
+  become: true
+  become_user: elasticsearch
+  when: elastic_ca
+
+- name: set fact es_http_type to HTTPS
+  set_fact: es_http_type=https
+  when: elastic_ca
+
+##################################### Kibana certificates #####################################
+
+- name: Verify if kibana Cert keys are present
+  stat:
+    path: "/etc/ssl/elasticsearch/kb-{{ item }}.zip"
+  become: true
+  become_user: elasticsearch
+  register: es_kb_ssl_cert_present
+  loop: "{{ groups['kibana_droplets'] }}"
+
+- name: Generate SSL/TLS certificate for Kibana
+  environment: 
+    - ES_PATH_CONF: "{{conf_dir}}"
+  command: /usr/share/elasticsearch/bin/x-pack/certutil cert --silent --ca /etc/ssl/elasticsearch/elastic-ca.p12 --ca-pass "test" --ip "{{ hostvars[item.item].ansible_host }}" --dns {{ item.item }},localhost --out /etc/ssl/elasticsearch/kb-{{ item.item }}.zip --pem --name {{ item.item }}
+  become: true
+  become_user: elasticsearch
+  when: es_ssl_regen_certs is defined or not item.stat.exists
+  loop: "{{ es_kb_ssl_cert_present.results }}"
+
+- name: Find kibana certificates in /etc/ssl/elasticsearch
+  find:
+    paths: /etc/ssl/elasticsearch
+    patterns: 'kb-.*\.zip'
+    use_regex: yes
+  register: es_kb_ssl_cert_archives
+
+- debug:
+    msg: "Found Kibana certificate ZIP files: {{ es_kb_ssl_cert_archives }}"
+
+# - name: Unarchive certificates for Kibana
+  # unarchive:
+  #   src: "{{item.path}}"
+  #   dest: "{{item.path | regex_replace('(.*).zip$', '\\1')}}"
+  #   remote_src: yes
+  # become: true
+  # become_user: elasticsearch
+  # loop: "{{ es_kb_ssl_cert_archives.files }}"
+
+# - name: Unarchive certificates for Kibana
+#   command: unzip {{item.path}}
+#   become: true
+#   become_user: elasticsearch
+#   loop: "{{ es_kb_ssl_cert_archives.files }}"
+
+# - name: Find kibana certificate directories in /etc/ssl/elasticsearch
+#   find:
+#     paths: /etc/ssl/elasticsearch
+#     patterns: 'kb-.*'
+#     use_regex: yes
+#     file_type: directory
+#   register: es_kb_ssl_cert_folder
+
+- name: Fetch certificates for Kibana
+  fetch:
+    src: "{{item.path}}"
+    dest: /tmp/certs/
+    flat: true
+  loop: "{{ es_kb_ssl_cert_archives.files }}"
+
+- name: Fetch CA certificate from primary ES server
+  fetch:
+    src: /usr/local/share/ca-certificates/local-elastic-ca/elastic-ca.crt
+    dest: /tmp/certs/
+    flat: true

tasks/xpack/security/elasticsearch-security-native.yml

@@ -21,7 +21,7 @@
 #List current users
 - name: List Native Users
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user
+    url: "{{es_http_type}}://{{es_api_host}}:{{es_api_port}}/_xpack/security/user"
     method: GET
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
@@ -50,7 +50,7 @@
 
 - name: Update API User Password
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{es_api_basic_auth_username}}/_password
+    url: "{{es_http_type}}://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{es_api_basic_auth_username}}/_password"
     method: POST
     body_format: json
     body: "{ \"password\":\"{{native_users[es_api_basic_auth_username].password}}\" }"
@@ -72,7 +72,7 @@
 #Delete all non required users NOT inc. reserved
 - name: Delete Native Users
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}
+    url: "{{es_http_type}}://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}"
     method: DELETE
     status_code: 200
     user: "{{es_api_basic_auth_username}}"
@@ -93,7 +93,7 @@
 #Update password on all reserved users
 - name: Update Reserved User Passwords
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}/_password
+    url: "{{es_http_type}}://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}/_password"
     method: POST
     body_format: json
     body: "{ \"password\":\"{{native_users[item].password}}\" }"
@@ -112,7 +112,7 @@
 #Overwrite all other users NOT inc. those reserved
 - name: Update Non-Reserved Native User Details
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}
+    url: "{{es_http_type}}://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}"
     method: POST
     body_format: json
     body: "{{ native_users[item] | to_json }}"
@@ -129,7 +129,7 @@
 #List current roles not. inc those reserved
 - name: List Native Roles
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role
+    url: "{{es_http_type}}://{{es_api_host}}:{{es_api_port}}/_xpack/security/role"
     method: GET
     body_format: json
     user: "{{es_api_basic_auth_username}}"
@@ -163,7 +163,7 @@
 #Delete all non required roles NOT inc. reserved
 - name: Delete Native Roles
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}
+    url: "{{es_http_type}}://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}"
     method: DELETE
     status_code: 200
     user: "{{es_api_basic_auth_username}}"
@@ -179,7 +179,7 @@
 #Update other roles - NOT inc. reserved roles
 - name: Update Native Roles
   uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}
+    url: "{{es_http_type}}://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}"
     method: POST
     body_format: json
     body: "{{ es_roles.native[item] | to_json}}"

tasks/xpack/security/elasticsearch-xpack-activation.yml

@@ -18,7 +18,7 @@
 - name: Activate ES license (with security authentication)
   uri:
     method: PUT
-    url: "http://{{es_api_host}}:{{es_api_port}}/_xpack/license?acknowledge=true"
+    url: "{{es_http_type}}://{{es_api_host}}:{{es_api_port}}/_xpack/license?acknowledge=true"
     user: "{{es_api_basic_auth_username}}"
     password: "{{es_api_basic_auth_password}}"
     body_format: json
@@ -26,7 +26,7 @@
     body: "{{ es_xpack_license }}"
     return_content: yes
   register: license_activated
-  no_log: True
+  no_log: false
   when: "'security' in es_xpack_features"
   failed_when: >
     license_activated.status != 200 or

templates/elasticsearch.yml.j2

@@ -23,11 +23,17 @@ path.data: {{ data_dirs | array_to_str }}
 
 path.logs: {{ log_dir }}
 
-{% if es_path_repo %}
+{% if es_path_repo is defined %}
 path.repo: {{ es_path_repo }}
 {% endif %}
 
-action.auto_create_index: {{ action_auto_create_index }}
+{% if es_action_auto_create_index == true %}
+action.auto_create_index: true
+{% elif not es_action_auto_create_index %}
+action.auto_create_index: false
+{% else %}
+action.auto_create_index: {{ es_action_auto_create_index }}
+{% endif %}
 
 {% if es_enable_xpack %}
 {% if not "security" in es_xpack_features %}
@@ -51,6 +57,18 @@ xpack.graph.enabled: false
 {% endif %}
 {% endif %}
 
+{% if es_ssl_config['enabled'] is defined %}
+xpack.security.transport.ssl.enabled: true
+xpack.security.http.ssl.enabled: true
+xpack.security.http.ssl.keystore.path: {{conf_dir}}/ssl/{{es_ssl_config['dns']}}.p12
+#Legacy config required for /usr/share/elasticsearch/bin/x-pack/setup-passwords
+xpack.security.http.ssl.keystore.password: "test"
+xpack.security.http.ssl.truststore.path: {{conf_dir}}/ssl/{{es_ssl_config['dns']}}.p12
+#Legacy config required for /usr/share/elasticsearch/bin/x-pack/setup-passwords
+xpack.security.http.ssl.truststore.password: "test"
+xpack.security.http.ssl.client_authentication: none
+{% endif %}
+
 {% if es_mail_config['account'] is defined %}
 xpack.notification.email:
   account: