[xpack] use elasticsearch default xpack features (#560)

- Stop forcing es_xpack_features variable in order to let elasticsearch install default features described in http://localhost:9200/_xpack
- Change xpack test scope to be able to test default xpack install
  - xpack scenario will test xpack install with default features
  - xpack upgrade scenario will fully test security feature
  - oss-to-xpack-upgrade will test installing only other specific features
- Cleanup some duplicate serverspec tests
- Remove `system_key`feature (deprecated in 5.6 and removed in 6.0 - [Breaking Changes 6.0.0](https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking-6.0.0-xes.html))
- Cleanup some ansible code (especially in `when` conditions)

README.md

@@ -272,14 +272,15 @@ ansible-playbook -i hosts ./your-playbook.yml
 
 ### Installing X-Pack Features
 
-X-Pack features, such as Security, are supported. This feature is currently experimental.
+X-Pack features, such as Security, are supported.
 
-The parameter `es_xpack_features` by default enables all features i.e. it defaults to ["alerting","monitoring","graph","security","ml"]
+The parameter `es_xpack_features` allows to list xpack features to install (example: `["alerting","monitoring","graph","security","ml"]`).
+When the list is empty, it install all features available with the current licence.
 
 The following additional parameters allow X-Pack to be configured:
 
-* ```es_message_auth_file``` System Key field to allow message authentication. This file should be placed in the 'files' directory.
 * ```es_xpack_custom_url``` Url from which X-Pack can be downloaded. This can be used for installations in isolated environments where the elastic.co repo is not accessible. e.g. ```es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-5.5.1.zip"```
+
 * ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/x-pack/current/mapping-roles.html)
 
 

defaults/main.yml

@@ -29,7 +29,7 @@ es_max_open_files: 65536
 es_max_threads: "{{ 2048 if ( es_version is version_compare('6.0.0', '<')) else 8192 }}"
 es_max_map_count: 262144
 es_allow_downgrades: false
-es_xpack_features: ["alerting","monitoring","graph","ml","security"]
+es_xpack_features: []
 #These are used for internal operations performed by ansible.
 #They do not affect the current configuration
 es_api_host: "localhost"

tasks/elasticsearch-parameters.yml

@@ -20,7 +20,10 @@
 #Check if working with security we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work
 - name: fail when api credentials are not declared when using security
   fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations"
-  when: es_enable_xpack and ("security" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined
+  when:
+    - es_enable_xpack and "security" in es_xpack_features
+    - es_api_basic_auth_username is not defined
+    - es_api_basic_auth_password is not defined
 
 - name: set fact file_reserved_users
   set_fact: file_reserved_users={{ es_users.file.keys() | intersect (reserved_xpack_users) }}

tasks/main.yml

@@ -68,7 +68,10 @@
 
 - name: set fact manage_native_realm to true
   set_fact: manage_native_realm=true
-  when: es_start_service and (es_enable_xpack and "security" in es_xpack_features) and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
+  when:
+   - es_start_service
+   - es_enable_xpack
+   - (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)
 
 # If playbook runs too fast, Native commands could fail as the Native Realm is not yet up
 - name: Wait 15 seconds for the Native Relm to come up

tasks/xpack/elasticsearch-xpack.yml

@@ -10,6 +10,7 @@
 #Security configuration
 - name: include security/elasticsearch-security.yml
   include: security/elasticsearch-security.yml
+  when: es_enable_xpack
 
 #Add any feature specific configuration here
 - name: Set Plugin Directory Permissions
@@ -20,4 +21,4 @@
 - name: Set elasticsearch.keystore Permissions
   become: yes
   file: state=file path={{ conf_dir }}/elasticsearch.keystore owner={{ es_user }} group={{ es_group }}
-  when: es_enable_xpack and "security" in es_xpack_features and (es_version is version_compare('6.0.0', '>'))
+  when: es_enable_xpack

tasks/xpack/security/elasticsearch-security.yml

@@ -7,14 +7,11 @@
 - name: Ensure x-pack conf directory exists (file)
   file: path={{ conf_dir }}{{ es_xpack_conf_subdir }} state=directory owner={{ es_user }} group={{ es_group }}
   changed_when: False
-  when:
-    - es_enable_xpack and "security" in es_xpack_features
-    - (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined)
+  when: (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined)
 
 #-----------------------------Create Bootstrap User-----------------------------------
 ### START BLOCK elasticsearch keystore ###
 - name: create the elasticsearch keystore
-  when: (es_enable_xpack and "security" in es_xpack_features) and (es_version is version_compare('6.0.0', '>'))
   block:
   - name: create the keystore if it doesn't exist yet
     become: yes
@@ -48,7 +45,7 @@
 #-----------------------------FILE BASED REALM----------------------------------------
 
 - include: elasticsearch-security-file.yml
-  when: (es_enable_xpack and "security" in es_xpack_features) and ((es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined))
+  when: (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined)
 
 #-----------------------------ROLE MAPPING ----------------------------------------
 
@@ -58,13 +55,6 @@
   template: src=security/role_mapping.yml.j2 dest={{conf_dir}}{{es_xpack_conf_subdir}}/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   when: es_role_mapping is defined
 
-#-----------------------------AUTH FILE----------------------------------------
-
-- name: Copy message auth key to elasticsearch
-  become: yes
-  copy: src={{ es_message_auth_file }} dest={{conf_dir}}{{es_xpack_conf_subdir}}/system_key owner={{ es_user }} group={{ es_group }} mode=0600 force=yes
-  when: es_message_auth_file is defined
-
 #------------------------------------------------------------------------------------
 
 #Ensure security conf directory is created
@@ -72,4 +62,3 @@
   become: yes
   file: path={{ conf_dir }}/security state=directory owner={{ es_user }} group={{ es_group }}
   changed_when: False
-  when: es_enable_xpack and "security" in es_xpack_features

tasks/xpack/security/elasticsearch-xpack-activation.yml

@@ -9,7 +9,7 @@
     return_content: yes
   register: license_activated
   no_log: True
-  when: not "security" in es_xpack_features
+  when: es_api_basic_auth_username is not defined or es_api_basic_auth_password is not defined
   failed_when: >
     license_activated.status != 200 or
     license_activated.json.license_status is not defined or
@@ -27,7 +27,7 @@
     return_content: yes
   register: license_activated
   no_log: True
-  when: "'security' in es_xpack_features"
+  when: es_api_basic_auth_username is defined and es_api_basic_auth_password is defined
   failed_when: >
     license_activated.status != 200 or
     license_activated.json.license_status is not defined or

templates/elasticsearch.yml.j2

@@ -35,26 +35,8 @@ action.auto_create_index: false
 action.auto_create_index: {{ es_action_auto_create_index }}
 {% endif %}
 
-{% if es_enable_xpack %}
-{% if not "security" in es_xpack_features %}
-xpack.security.enabled: false
-{% endif %}
-
-{% if not "monitoring" in es_xpack_features %}
-xpack.monitoring.enabled: false
-{% endif %}
-
-{% if not "alerting" in es_xpack_features %}
-xpack.watcher.enabled: false
-{% endif %}
-
-{% if not "ml" in es_xpack_features %}
-xpack.ml.enabled: false
-{% endif %}
-
-{% if not "graph" in es_xpack_features %}
-xpack.graph.enabled: false
-{% endif %}
+{% if es_enable_xpack and es_api_basic_auth_username is defined and es_api_basic_auth_password is defined %}
+xpack.security.enabled: true
 {% endif %}
 
 {% if es_mail_config is defined %}
@@ -71,5 +53,5 @@ xpack.notification.email:
         {% if es_mail_config['require_auth'] == true %}
         user: {{ es_mail_config['user'] }}
         password: {{ es_mail_config['pass'] }}
-        {% endif %}  
+        {% endif %}
 {% endif %}

test/integration/helpers/serverspec/multi_spec.rb

@@ -16,13 +16,7 @@ shared_examples 'multi::init' do |vars|
     else
       it { should contain 'transport.tcp.port: 9301' }
     end
-    it { should contain 'node.data: true' }
-    it { should contain 'node.master: false' }
-    it { should contain "node.name: localhost-#{vars['es_instance_name']}" }
     it { should_not contain 'bootstrap.memory_lock: true' }
-    it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
-    it { should contain "path.data: /opt/elasticsearch/data-1/localhost-#{vars['es_instance_name']},/opt/elasticsearch/data-2/localhost-#{vars['es_instance_name']}" }
-    it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" }
   end
 
 

test/integration/helpers/serverspec/shared_spec.rb

@@ -21,6 +21,22 @@ es_api_url = "http://localhost:#{vars['es_api_port']}"
 username = vars['es_api_basic_auth_username']
 password = vars['es_api_basic_auth_password']
 
+# Sample of default features status
+features = {
+  'monitoring' => {
+    'enabled' => 'true',
+    'available' => 'true'
+  },
+  'ml' => {
+    'enabled' => 'true',
+    'available' => 'false'
+  },
+  'sql' => {
+    'enabled' => 'true',
+    'available' => 'true'
+  }
+}
+
 shared_examples 'shared::init' do |vars|
   describe 'version check' do
     it 'should be reported as version '+vars['es_version'] do
@@ -35,12 +51,34 @@ shared_examples 'shared::init' do |vars|
       it 'xpack should be activated' do
         expect(curl_json("#{es_api_url}/_license", username=username, password=password)['license']['status']).to eq('active')
       end
-      features = curl_json("#{es_api_url}/_xpack", username=username, password=password)
-      curl_json("#{es_api_url}/_xpack", username=username, password=password)['features'].each do |feature,values|
-        enabled = vars['es_xpack_features'].include? feature
-		status = if enabled then 'enabled' else 'disabled' end
-		it "the xpack feature '#{feature}' to be #{status}" do
-          expect(values['enabled'] = enabled)
+      if vars.key?('es_xpack_features')
+        curl_json("#{es_api_url}/_xpack", username=username, password=password)['features'].each do |feature,values|
+          enabled = vars['es_xpack_features'].include? feature
+          status = if enabled then 'enabled' else 'disabled' end
+          it "the xpack feature '#{feature}' to be #{status}" do
+            expect(values['enabled'] = enabled)
+          end
+        end
+      else
+        features.each do |feature, status|
+          feature_available = curl_json("#{es_api_url}/_xpack", username=username, password=password)['features'][feature]['available']
+          if feature_available == "true"
+            status = "available"
+          else
+            status = "unavailable"
+          end
+          it "the xpack feature '#{feature}' to be #{status}" do
+            expect(feature_available = status['available'])
+          end
+          feature_enabled = curl_json("#{es_api_url}/_xpack", username=username, password=password)['features'][feature]['enabled']
+          if feature_enabled == "true"
+            status = "enabled"
+          else
+            status = "disabled"
+          end
+          it "the xpack feature '#{feature}' to be #{status}" do
+            expect(feature_available = status['enabled'])
+          end
         end
       end
     end

test/integration/helpers/serverspec/xpack_spec.rb

@@ -1,13 +0,0 @@
-require 'spec_helper'
-
-shared_examples 'xpack::init' do |vars|
-  describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do
-    it { should contain "node.name: localhost-#{vars['es_instance_name']}" }
-    it { should contain 'cluster.name: elasticsearch' }
-    it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
-    it { should contain "path.data: /var/lib/elasticsearch/localhost-#{vars['es_instance_name']}" }
-    it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" }
-    it { should contain 'xpack.security.enabled: false' }
-    it { should contain 'xpack.watcher.enabled: false' }
-  end
-end

test/integration/helpers/serverspec/xpack_upgrade_spec.rb

@@ -3,14 +3,6 @@ require 'json'
 vars = JSON.parse(File.read('/tmp/vars.json'))
 
 shared_examples 'xpack_upgrade::init' do |vars|
-  describe file("/etc/elasticsearch/#{vars['es_instance_name']}/elasticsearch.yml") do
-    it { should contain "node.name: localhost-#{vars['es_instance_name']}" }
-    it { should contain 'cluster.name: elasticsearch' }
-    it { should_not contain "path.conf: /etc/elasticsearch/#{vars['es_instance_name']}" }
-    it { should contain "path.data: /var/lib/elasticsearch/localhost-#{vars['es_instance_name']}" }
-    it { should contain "path.logs: /var/log/elasticsearch/localhost-#{vars['es_instance_name']}" }
-  end
-
   #Test users file, users_roles and roles.yml
   describe file("/etc/elasticsearch/#{vars['es_instance_name']}#{vars['es_xpack_conf_subdir']}/users_roles") do
     it { should be_owned_by 'elasticsearch' }
@@ -39,7 +31,7 @@ shared_examples 'xpack_upgrade::init' do |vars|
       it { should contain 'security.authc.realms.file1.order: 0' }
       it { should contain 'security.authc.realms.file1.type: file' }
       it { should contain 'security.authc.realms.native1.order: 1' }
-      it { should contain 'security.authc.realms.native1.type: native' } 
+      it { should contain 'security.authc.realms.native1.type: native' }
     end
   end
 

test/integration/issue-test.yml

@@ -13,7 +13,6 @@
     es_instance_name: "security_node"
     es_xpack_license: "{{ lookup('file', '/tmp/license.json')  }}"
     es_config:
-      xpack.security.enabled: True
       xpack.security.authc.realms.file.file1.order: 1
       xpack.security.authc.realms.native.native1.type: "native"
     es_heap_size: "1g"

test/integration/oss-to-xpack-upgrade.yml

@@ -20,8 +20,8 @@
   vars:
     es_instance_name: "node1"
     es_enable_xpack: true
-    es_api_basic_auth_username: elastic
-    es_api_basic_auth_password: changeme
     es_heap_size: "1g"
     es_xpack_features:
-      - security
+      - monitoring
+      - graph
+      - ml

test/integration/xpack-upgrade.yml

@@ -16,7 +16,6 @@
       xpack.security.authc.realms.native1.type: native
     es_config_7x:
       http.port: 9200
-      xpack.security.enabled: True
       xpack.security.authc.realms.file.file1.order: 0
       xpack.security.authc.realms.native.native1.order: 1
     es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}"
@@ -33,7 +32,6 @@
       - alerting
     es_api_basic_auth_username: elastic
     es_api_basic_auth_password: changeme
-    es_message_auth_file: system_key
     es_role_mapping:
       power_user:
         - "cn=admins,dc=example,dc=com"
@@ -131,7 +129,6 @@
       xpack.security.authc.realms.native1.type: native
     es_config_7x:
       http.port: 9200
-      xpack.security.enabled: True
       xpack.security.authc.realms.file.file1.order: 0
       xpack.security.authc.realms.native.native1.order: 1
     es_config: "{{ es_config_7x if es_major_version == '7.x' else es_config_6x }}"

test/integration/xpack.yml

@@ -14,7 +14,3 @@
     es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-{{ es_version }}.zip"
     es_heap_size: 2g
     es_enable_xpack: true
-    es_xpack_features:
-      - monitoring
-      - graph
-      - ml

test/integration/xpack/serverspec/default_spec.rb

@@ -1,9 +1,7 @@
-require 'xpack_spec'
 require 'shared_spec'
 require 'json'
 vars = JSON.parse(File.read('/tmp/vars.json'))
 
 describe 'Xpack upgrade Tests' do
   include_examples 'shared::init', vars
-  include_examples 'xpack::init', vars
 end

vars/main.yml

@@ -3,6 +3,4 @@ es_package_url: "https://artifacts.elastic.co/downloads/elasticsearch/elasticsea
 es_conf_dir: "/etc/elasticsearch"
 sysd_script: "/usr/lib/systemd/system/elasticsearch.service"
 init_script: "/etc/init.d/elasticsearch"
-#add supported features here
-supported_xpack_features: ["alerting","monitoring","graph","security"]
 reserved_xpack_users: ["elastic","kibana","logstash_system"]