[xpack] use elasticsearch default xpack features (#560)

- Stop forcing es_xpack_features variable in order to let elasticsearch install default features described in http://localhost:9200/_xpack
- Change xpack test scope to be able to test default xpack install
  - xpack scenario will test xpack install with default features
  - xpack upgrade scenario will fully test security feature
  - oss-to-xpack-upgrade will test installing only other specific features
- Cleanup some duplicate serverspec tests
- Remove `system_key`feature (deprecated in 5.6 and removed in 6.0 - [Breaking Changes 6.0.0](https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking-6.0.0-xes.html))
- Cleanup some ansible code (especially in `when` conditions)

tasks/elasticsearch-parameters.yml

@@ -20,7 +20,10 @@
 #Check if working with security we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work
 - name: fail when api credentials are not declared when using security
   fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations"
-  when: es_enable_xpack and ("security" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined
+  when:
+    - es_enable_xpack and "security" in es_xpack_features
+    - es_api_basic_auth_username is not defined
+    - es_api_basic_auth_password is not defined
 
 - name: set fact file_reserved_users
   set_fact: file_reserved_users={{ es_users.file.keys() | intersect (reserved_xpack_users) }}

tasks/main.yml

@@ -68,7 +68,10 @@
 
 - name: set fact manage_native_realm to true
   set_fact: manage_native_realm=true
-  when: es_start_service and (es_enable_xpack and "security" in es_xpack_features) and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
+  when:
+   - es_start_service
+   - es_enable_xpack
+   - (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)
 
 # If playbook runs too fast, Native commands could fail as the Native Realm is not yet up
 - name: Wait 15 seconds for the Native Relm to come up

tasks/xpack/elasticsearch-xpack.yml

@@ -10,6 +10,7 @@
 #Security configuration
 - name: include security/elasticsearch-security.yml
   include: security/elasticsearch-security.yml
+  when: es_enable_xpack
 
 #Add any feature specific configuration here
 - name: Set Plugin Directory Permissions
@@ -20,4 +21,4 @@
 - name: Set elasticsearch.keystore Permissions
   become: yes
   file: state=file path={{ conf_dir }}/elasticsearch.keystore owner={{ es_user }} group={{ es_group }}
-  when: es_enable_xpack and "security" in es_xpack_features and (es_version is version_compare('6.0.0', '>'))
+  when: es_enable_xpack

tasks/xpack/security/elasticsearch-security.yml

@@ -7,14 +7,11 @@
 - name: Ensure x-pack conf directory exists (file)
   file: path={{ conf_dir }}{{ es_xpack_conf_subdir }} state=directory owner={{ es_user }} group={{ es_group }}
   changed_when: False
-  when:
-    - es_enable_xpack and "security" in es_xpack_features
-    - (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined)
+  when: (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined)
 
 #-----------------------------Create Bootstrap User-----------------------------------
 ### START BLOCK elasticsearch keystore ###
 - name: create the elasticsearch keystore
-  when: (es_enable_xpack and "security" in es_xpack_features) and (es_version is version_compare('6.0.0', '>'))
   block:
   - name: create the keystore if it doesn't exist yet
     become: yes
@@ -48,7 +45,7 @@
 #-----------------------------FILE BASED REALM----------------------------------------
 
 - include: elasticsearch-security-file.yml
-  when: (es_enable_xpack and "security" in es_xpack_features) and ((es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined))
+  when: (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined)
 
 #-----------------------------ROLE MAPPING ----------------------------------------
 
@@ -58,13 +55,6 @@
   template: src=security/role_mapping.yml.j2 dest={{conf_dir}}{{es_xpack_conf_subdir}}/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
   when: es_role_mapping is defined
 
-#-----------------------------AUTH FILE----------------------------------------
-
-- name: Copy message auth key to elasticsearch
-  become: yes
-  copy: src={{ es_message_auth_file }} dest={{conf_dir}}{{es_xpack_conf_subdir}}/system_key owner={{ es_user }} group={{ es_group }} mode=0600 force=yes
-  when: es_message_auth_file is defined
-
 #------------------------------------------------------------------------------------
 
 #Ensure security conf directory is created
@@ -72,4 +62,3 @@
   become: yes
   file: path={{ conf_dir }}/security state=directory owner={{ es_user }} group={{ es_group }}
   changed_when: False
-  when: es_enable_xpack and "security" in es_xpack_features

tasks/xpack/security/elasticsearch-xpack-activation.yml

@@ -9,7 +9,7 @@
     return_content: yes
   register: license_activated
   no_log: True
-  when: not "security" in es_xpack_features
+  when: es_api_basic_auth_username is not defined or es_api_basic_auth_password is not defined
   failed_when: >
     license_activated.status != 200 or
     license_activated.json.license_status is not defined or
@@ -27,7 +27,7 @@
     return_content: yes
   register: license_activated
   no_log: True
-  when: "'security' in es_xpack_features"
+  when: es_api_basic_auth_username is defined and es_api_basic_auth_password is defined
   failed_when: >
     license_activated.status != 200 or
     license_activated.json.license_status is not defined or