diff --git a/.kitchen.yml b/.kitchen.yml index a02d9be..2b29495 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -8,6 +8,7 @@ provisioner: roles_path: ../ require_ansible_repo: true ansible_verbose: true + ansible_version: 2.0.2 http_proxy: <%= ENV['HTTP_PROXY'] %> https_proxy: <%= ENV['HTTPS_PROXY'] %> no_proxy: localhost,127.0.0.1 @@ -19,7 +20,7 @@ platforms: privileged: true provision_command: - apt-get update && apt-get install -y software-properties-common && add-apt-repository -y ppa:ansible/ansible - - apt-get update && apt-get -y -q install ansible python-apt python-pycurl + - apt-get update && apt-get -y -q install python-apt python-pycurl use_sudo: false - name: debian-7 driver_config: @@ -27,7 +28,6 @@ platforms: privileged: true provision_command: - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml - - pip install ansible - apt-get install -y -q net-tools use_sudo: false - name: debian-8 @@ -36,7 +36,6 @@ platforms: privileged: true provision_command: - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget - - pip install ansible - apt-get install -y -q net-tools - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config @@ -75,8 +74,6 @@ suites: es_plugins: - plugin: lmenezes/elasticsearch-kopf version: master - - plugin: license - - plugin: marvel-agent provisioner: playbook: test/integration/package.yml - name: config-2x @@ -91,8 +88,6 @@ suites: es_plugins: - plugin: lmenezes/elasticsearch-kopf version: master - - plugin: license - - plugin: marvel-agent provisioner: playbook: test/integration/multi.yml - name: standard-1x @@ -137,3 +132,8 @@ suites: version: latest provisioner: playbook: test/integration/multi.yml + - name: xpack-2x + run_list: + attributes: + provisioner: + playbook: test/integration/xpack.yml \ No newline at end of file diff --git a/Gemfile b/Gemfile index e0591b1..13c5458 100644 --- a/Gemfile +++ b/Gemfile @@ -1,6 +1,6 @@ source 'https://rubygems.org' -gem 'test-kitchen', '1.4.2' -gem "kitchen-docker", '2.1.0' -gem 'kitchen-ansible', '0.40.1' -gem 'net-ssh', '~> 2.0' +gem 'test-kitchen', '1.8.0' +gem "kitchen-docker", '2.5.0' +gem 'kitchen-ansible', '0.44.6' +gem 'net-ssh', '~> 3.0' diff --git a/Gemfile.lock b/Gemfile.lock index 8c827c2..d5952fa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,30 +1,27 @@ GEM remote: https://rubygems.org/ specs: - faraday (0.9.2) - multipart-post (>= 1.2, < 3) - highline (1.7.8) - kitchen-ansible (0.40.1) - librarian-ansible + artifactory (2.3.3) + kitchen-ansible (0.44.6) + net-ssh (~> 3.0) test-kitchen (~> 1.4) - kitchen-docker (2.1.0) + kitchen-docker (2.5.0) test-kitchen (>= 1.0.0) - librarian (0.1.2) - highline - thor (~> 0.15) - librarian-ansible (3.0.0) - faraday - librarian (~> 0.1.0) + mixlib-install (1.1.0) + artifactory + mixlib-shellout + mixlib-versioning mixlib-shellout (2.2.6) - multipart-post (2.0.0) + mixlib-versioning (1.1.0) net-scp (1.2.1) net-ssh (>= 2.6.5) - net-ssh (2.9.4) + net-ssh (3.2.0) safe_yaml (1.0.4) - test-kitchen (1.4.2) + test-kitchen (1.8.0) + mixlib-install (~> 1.0, >= 1.0.4) mixlib-shellout (>= 1.2, < 3.0) net-scp (~> 1.1) - net-ssh (~> 2.7, < 2.10) + net-ssh (>= 2.9, < 4.0) safe_yaml (~> 1.0) thor (~> 0.18) thor (0.19.1) @@ -33,10 +30,10 @@ PLATFORMS ruby DEPENDENCIES - kitchen-ansible (= 0.40.1) - kitchen-docker (= 2.1.0) - net-ssh (~> 2.0) - test-kitchen (= 1.4.2) + kitchen-ansible (= 0.44.6) + kitchen-docker (= 2.5.0) + net-ssh (~> 3.0) + test-kitchen (= 1.8.0) BUNDLED WITH 1.11.2 diff --git a/README.md b/README.md index 1fa450d..d4e82af 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,6 @@ controlled by the following parameters: * ```es_data_dirs``` - defaults to "/var/lib/elasticsearch". This can be a list or comma separated string e.g. ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"] or "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2" * ```es_log_dir``` - defaults to "/var/log/elasticsearch". * ```es_work_dir``` - defaults to "/tmp/elasticsearch". -* ```es_plugin_dir``` - defaults to "/usr/share/elasticsearch/plugins". * ```es_restart_on_change``` - defaults to true. If false, changes will not result in Elasticsearch being restarted. * ```es_plugins_reinstall``` - defaults to false. If true, all currently installed plugins will be removed from a node. Listed plugins will then be re-installed. diff --git a/defaults/main.yml b/defaults/main.yml index 451000f..10adc43 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,6 +1,6 @@ --- es_major_version: "2.x" -es_version: "2.2.0" +es_version: "2.3.4" es_version_lock: false es_use_repository: true es_start_service: true @@ -13,13 +13,16 @@ es_templates: false es_user: elasticsearch es_group: elasticsearch es_config: {} -es_install_shield: false #Need to provide default directories es_pid_dir: "/var/run/elasticsearch" es_data_dirs: "/var/lib/elasticsearch" es_log_dir: "/var/log/elasticsearch" es_work_dir: "/tmp/elasticsearch" -es_plugin_dir: "/usr/share/elasticsearch/plugins" es_max_open_files: 65536 es_allow_downgrades: false - +es_enable_xpack: false +es_xpack_features: [] +#These are used for internal operations performed by ansible. +#They do not effect the current configuration +es_api_host: "localhost" +es_api_port: 9200 \ No newline at end of file diff --git a/filter_plugins/custom.py b/filter_plugins/custom.py index 0453a25..ecd3b97 100644 --- a/filter_plugins/custom.py +++ b/filter_plugins/custom.py @@ -19,8 +19,18 @@ def append_to_list(values=[], suffix=''): def array_to_str(values=[],separator=','): return separator.join(values) +def extract_role_users(users={}): + role_users=[] + for user,details in users.iteritems(): + if "roles" in details: + for role in details["roles"]: + role_users.append(role+":"+user) + return role_users + + class FilterModule(object): def filters(self): return {'modify_list': modify_list, 'append_to_list':append_to_list, - 'array_to_str':array_to_str} \ No newline at end of file + 'array_to_str':array_to_str, + 'extract_role_users':extract_role_users} \ No newline at end of file diff --git a/handlers/elasticsearch-templates.yml b/handlers/elasticsearch-templates.yml new file mode 100644 index 0000000..797ec62 --- /dev/null +++ b/handlers/elasticsearch-templates.yml @@ -0,0 +1,34 @@ +--- + +- name: Ensure elasticsearch is started + service: name={{instance_init_script | basename}} state=started enabled=yes + +- name: Wait for elasticsearch to startup + wait_for: port={{es_api_port}} delay=10 + +- name: Get template files + shell: find . -maxdepth 1 -type f | sed "s#\./##" | sed "s/.json//" chdir=/etc/elasticsearch/templates + register: resultstemplate + +- name: Install templates without auth + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} + method: PUT + status_code: 200 + body_format: json + body: "{{ lookup('file', '/etc/elasticsearch/templates/'+item+'.json') }}" + when: not es_enable_xpack or not es_xpack_features is defined or not '"shield" in es_xpack_features' + with_items: "{{ resultstemplate.stdout_lines }}" + +- name: Install templates with auth + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_template/{{item}} + method: PUT + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + body_format: json + body: "{{ lookup('file', '/etc/elasticsearch/templates/'+item+'.json') }}" + when: es_enable_xpack and es_xpack_features is defined and '"shield" in es_xpack_features' + with_items: "{{ resultstemplate.stdout_lines }}" diff --git a/handlers/main.yml b/handlers/main.yml index 50a3a7e..e98d041 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -1,4 +1,17 @@ - name: restart elasticsearch service: name={{instance_init_script | basename}} state=restarted enabled=yes - when: es_restart_on_change and es_start_service and not elasticsearch_started.changed and ((plugin_installed is defined and plugin_installed.changed) or (elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) + when: es_restart_on_change and es_start_service and not elasticsearch_started.changed and ((plugin_installed is defined and plugin_installed.changed) or (xpack_state.changed) or (elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) + +- name: load-native-realms + include: ./handlers/shield/elasticsearch-shield-native.yml + when: (es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined) + + +#Templates are a handler as they need to come after a restart e.g. suppose user removes shield on a running node and doesn't +#specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed if we don't wait for the node to restart. +#Templates done after restart therefore - as a handler. + +- name: load-templates + include: ./handlers/elasticsearch-templates.yml + when: es_templates \ No newline at end of file diff --git a/handlers/shield/elasticsearch-shield-native.yml b/handlers/shield/elasticsearch-shield-native.yml new file mode 100644 index 0000000..5632bd8 --- /dev/null +++ b/handlers/shield/elasticsearch-shield-native.yml @@ -0,0 +1,121 @@ +--- + +- name: Ensure elasticsearch is started + service: name={{instance_init_script | basename}} state=started enabled=yes + +- name: Wait for elasticsearch to startup + wait_for: port={{es_api_port}} delay=10 + +- set_fact: manage_native_users=false + +- set_fact: manage_native_users=true + when: es_users is defined and es_users.native is defined + +- set_fact: manage_native_roles=false + +- set_fact: manage_native_roles=true + when: es_roles is defined and es_roles.native is defined + +#If the node has just has shield installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load + +#List current users +- name: List Native Users + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/user + method: GET + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + status_code: 200 + register: user_list_response + when: manage_native_users + + +- set_fact: current_users={{user_list_response.json.keys() | list}} + when: manage_native_users + +#Identify non declared users + +- set_fact: users_to_remove={{ current_users | difference ( es_users.native.keys() ) }} + when: manage_native_users + +#Delete all non required users +- name: Delete Native Users + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item}} + method: DELETE + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_users and users_to_remove | length > 0 + with_items: "{{users_to_remove}}" + + +#Overwrite all other users +- name: Update Native Users + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/user/{{item.key}} + method: POST + body_format: json + body: "{{item.value | to_json}}" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_users and es_users.native.keys() > 0 + with_dict: "{{es_users.native}}" + +#List current roles + +- name: List Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/role + method: GET + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + status_code: 200 + register: role_list_response + when: manage_native_roles + +#Identify undeclared roles + +- set_fact: current_roles={{role_list_response.json.keys() | list}} + when: manage_native_users + +- debug: msg="{{current_roles}}" + +- set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys() ) }} + when: manage_native_roles + + +#Delete all non required roles +- name: Delete Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item}} + method: DELETE + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_roles and roles_to_remove | length > 0 + with_items: "{{roles_to_remove}}" + + +#Update other roles +- name: Update Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_shield/role/{{item.key}} + method: POST + body_format: json + body: "{{item.value | to_json}}" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_roles and es_roles.native.keys() > 0 + with_dict: "{{es_roles.native}}" + + + diff --git a/meta/main.yml b/meta/main.yml index 1a0b068..66df2a3 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -8,7 +8,7 @@ galaxy_info: company: "Elastic.co" license: "license (Apache)" # Require 1.6 for apt deb install - min_ansible_version: 1.6 + min_ansible_version: 2.0 platforms: - name: EL versions: diff --git a/tasks/checkParameters.yml b/tasks/checkParameters.yml deleted file mode 100644 index 6fc5bef..0000000 --- a/tasks/checkParameters.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Check for mandatory parameters - -- fail: msg="es_instance_name must be specified and cannot be blank" - when: es_instance_name is not defined or es_instance_name == '' - -- fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined" - when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '') - -- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or (es_config['discovery.zen.ping.multicast.enabled'] is defined and es_config['discovery.zen.ping.multicast.enabled'])}} - -- debug: msg="WARNING - It is recommended you specify the parameter 'http.port' when multicast is disabled" - when: not multi_cast and es_config['http.port'] is not defined - -- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port' when multicast is disabled" - when: not multi_cast and es_config['transport.tcp.port'] is not defined - -- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts' when multicast is disabled" - when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined - -#If the user attempts to lock memory they must specify a heap size -- fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified" - when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined \ No newline at end of file diff --git a/tasks/elasticsearch-Debian.yml b/tasks/elasticsearch-Debian.yml index dba5fa5..ebda0f5 100644 --- a/tasks/elasticsearch-Debian.yml +++ b/tasks/elasticsearch-Debian.yml @@ -26,3 +26,7 @@ apt: deb=/tmp/elasticsearch-{{ es_version }}.deb when: not es_use_repository register: elasticsearch_install_from_package + +# ansible uri module requires httplib2 +- name: pip httplib2 + pip: name=httplib2 extra_args="--user" \ No newline at end of file diff --git a/tasks/elasticsearch-RedHat.yml b/tasks/elasticsearch-RedHat.yml index 0de8e71..db74280 100644 --- a/tasks/elasticsearch-RedHat.yml +++ b/tasks/elasticsearch-RedHat.yml @@ -20,3 +20,7 @@ yum: name={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.noarch.rpm{% endif %} state=present when: not es_use_repository register: elasticsearch_install_from_package + +# ansible uri module requires httplib2 +- name: pip httplib2 + pip: name=httplib2 extra_args="--user" \ No newline at end of file diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 77d0a5f..34943b4 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -1,42 +1,6 @@ --- - # Configure Elasticsearch Node -#Use systemd for the following distributions: -# -#Ubuntu 15 and up -#Debian 8 and up -#Centos 7 and up -#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied. - - -- set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }} - tags: - - always - -- set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}} - when: use_system_d - tags: - - always - -#For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN. - -- set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }} - tags: - - always - -- set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}} - tags: - - always - -- set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}} - tags: - - always - -- set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}} - tags: - - always - #Create required directories - name: Create Directories file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} @@ -45,11 +9,6 @@ - "{{work_dir}}" - "{{log_dir}}" - "{{conf_dir}}" - - "{{plugin_dir}}" - -- set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }} - tags: - - always - name: Create Data Directories file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} @@ -112,4 +71,3 @@ - name: Delete Default Logging File file: dest=/etc/elasticsearch/logging.yml state=absent -- debug: msg="Data Dirs {{data_dirs}}" \ No newline at end of file diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml new file mode 100644 index 0000000..0f1e8e5 --- /dev/null +++ b/tasks/elasticsearch-parameters.yml @@ -0,0 +1,54 @@ +# Check for mandatory parameters + +- fail: msg="es_instance_name must be specified and cannot be blank" + when: es_instance_name is not defined or es_instance_name == '' + +- fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined" + when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '') + +- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or (es_config['discovery.zen.ping.multicast.enabled'] is defined and es_config['discovery.zen.ping.multicast.enabled'])}} + +- debug: msg="WARNING - It is recommended you specify the parameter 'http.port' when multicast is disabled" + when: not multi_cast and es_config['http.port'] is not defined + +- debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port' when multicast is disabled" + when: not multi_cast and es_config['transport.tcp.port'] is not defined + +- debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts' when multicast is disabled" + when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined + +#If the user attempts to lock memory they must specify a heap size +- fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified" + when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined + +#Don't support xpack on versions < 2.0 +- fail: msg="Use of the xpack notation is not supported on versions < 2.0. Marvel-agent and watcher can be installed as plugins. Version > 2.0 is required for shield." + when: es_enable_xpack and es_version | version_compare('2.0', '<') + +#Check if working with shield we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work +- fail: msg="Enabling shield requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" + when: es_enable_xpack and '"shield" in es_xpack_features' and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined + +- set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} +- set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} +- set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} +- set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }} + +#Use systemd for the following distributions: +#Ubuntu 15 and up +#Debian 8 and up +#Centos 7 and up +#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied. + + +- set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }} + +- set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}} + when: use_system_d +#For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN. + +- set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }} +- set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}} +- set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}} +- set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}} +- set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }} \ No newline at end of file diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 7da2d8c..0a0f36d 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -2,26 +2,24 @@ # es_plugins_reinstall will be set to true if elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed # i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting. -- set_fact: es_plugins_reinstall=true - when: ((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none - tags: - - always +- set_fact: es_plugins_reinstall={{ ((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none }} + +- debug: msg="{{es_plugins_reinstall}}" - set_fact: list_command="list" - tags: - - always + - set_fact: list_command="--list" when: es_version | version_compare('2.0', '<') - tags: - - always -#List currently installed plugins -- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-" +#List currently installed plugins - ignore xpack if > v 2.0 +- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}" register: installed_plugins + failed_when: "'ERROR' in installed_plugins.stdout" changed_when: False + ignore_errors: yes environment: - CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" #This needs to removes any currently installed plugins - name: Remove elasticsearch plugins @@ -30,6 +28,7 @@ with_items: "{{ installed_plugins.stdout_lines }}" when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0] notify: restart elasticsearch + register: plugin_installed environment: CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" @@ -50,4 +49,5 @@ #Set permissions on plugins directory - name: Set Plugin Directory Permissions - file: state=directory path={{ plugin_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file + file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes + when: es_enable_xpack \ No newline at end of file diff --git a/tasks/elasticsearch-shield.yml b/tasks/elasticsearch-shield.yml deleted file mode 100644 index 849169c..0000000 --- a/tasks/elasticsearch-shield.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- - - diff --git a/tasks/elasticsearch-templates.yml b/tasks/elasticsearch-templates.yml index c9de8bb..76a471d 100644 --- a/tasks/elasticsearch-templates.yml +++ b/tasks/elasticsearch-templates.yml @@ -4,30 +4,12 @@ - name: Copy default templates to elasticsearch copy: src=templates dest=/etc/elasticsearch/ owner={{ es_user }} group={{ es_group }} + notify: load-templates when: es_templates_fileglob is not defined - name: Copy templates to elasticsearch copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }} when: es_templates_fileglob is defined + notify: load-templates with_fileglob: - - "{{ es_templates_fileglob }}" - -- set_fact: http_port=9200 - tags: - - always - -- set_fact: http_port={{es_config['http.port']}} - when: es_config['http.port'] is defined - tags: - - always - -- name: Wait for elasticsearch to startup - wait_for: port={{http_port}} delay=10 - -- name: Get template files - shell: find . -maxdepth 1 -type f | sed "s#\./##" | sed "s/.json//" chdir=/etc/elasticsearch/templates - register: resultstemplate - -- name: Install template(s) - command: "curl -sL -XPUT http://localhost:{{http_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json" - with_items: "{{ resultstemplate.stdout_lines }}" + - "{{ es_templates_fileglob }}" \ No newline at end of file diff --git a/tasks/elasticsearch.yml b/tasks/elasticsearch.yml index 1e5cc89..7addb03 100644 --- a/tasks/elasticsearch.yml +++ b/tasks/elasticsearch.yml @@ -1,24 +1,6 @@ --- -- set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} - tags: - - always -- set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} - tags: - - always -- set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} - tags: - - always -- set_fact: plugin_dir={{ es_plugin_dir }}/{{es_instance_name}} - tags: - - always -- set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }} - tags: - - always - -- debug: msg="Node configuration {{ es_config }} " - -- name: Include optional user and group creation. +- name: Include optional user and group creation. when: (es_user_id is defined) and (es_group_id is defined) include: elasticsearch-optional-user.yml diff --git a/tasks/main.yml b/tasks/main.yml index 6a92155..cd2743b 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,12 +1,14 @@ --- -- name: check-parameters - include: checkParameters.yml - tags: - - check - name: os-specific vars include_vars: "{{ansible_os_family}}.yml" tags: - always + +- name: check-set-parameters + include: elasticsearch-parameters.yml + tags: + - always + - include: java.yml when: es_java_install tags: @@ -25,10 +27,10 @@ when: es_plugins is defined or es_plugins_reinstall tags: - plugins -- include: elasticsearch-shield.yml - when: es_install_shield + #We always execute xpack as we may need to remove features +- include: xpack/elasticsearch-xpack.yml tags: - - shield + - xpack - include: elasticsearch-service.yml tags: - service diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml new file mode 100644 index 0000000..dfe7962 --- /dev/null +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -0,0 +1,38 @@ +--- + +#Test if feature is installed +- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep {{item}}" + register: feature_installed + changed_when: False + ignore_errors: yes + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +#Remove Plugin if installed and its not been requested or the ES version has changed +- name: Remove {{item}} plugin + command: > + {{es_home}}/bin/plugin remove shield + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 + when: feature_installed.rc == 0 and (not es_enable_xpack or not '"{{item}}" in es_xpack_features' or es_version_changed) + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +#Install plugin if not installed, or the es version has changed (so removed above), and its been requested +- name: Install {{item}} plugin + command: > + {{es_home}}/bin/plugin install {{item}} + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 + when: (feature_installed.rc == 1 or es_version_changed) and es_enable_xpack and "{{item}}" in es_xpack_features + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" \ No newline at end of file diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml new file mode 100644 index 0000000..46be973 --- /dev/null +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -0,0 +1,55 @@ +--- + +- set_fact: es_version_changed={{ ((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) }} + +#enabling xpack installs the license. Not a xpack feature and does not need to be specified - TODO: we should append it to the list if xpack is enabled and remove this + +#Check if license is installed +- name: Check License is installed + shell: > + {{es_home}}/bin/plugin list | tail -n +2 | grep license + register: license_installed + ignore_errors: yes + failed_when: "'ERROR' in license_installed.stdout" + changed_when: False + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + +#Remove license if installed and xpack not enabled +- name: Remove license plugin + command: > + {{es_home}}/bin/plugin remove license + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 + when: license_installed.rc == 0 and (not es_enable_xpack or es_version_changed) + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + +#Install License if not installed, or it needs to be reinstalled due to ES change (above task will have removed), and its been requested. +- name: Install license plugin + command: > + {{es_home}}/bin/plugin install license + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 + when: (license_installed.rc == 1 or es_version_changed) and es_enable_xpack + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + +#We loop on all as we may need to remove some features. +- include: elasticsearch-xpack-install.yml + with_items: "{{supported_xpack_features}}" + +#Shield configuration +- include: shield/elasticsearch-shield.yml + +#Add any feature specific configuration here + +- name: Set Plugin Directory Permissions + file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file diff --git a/tasks/xpack/shield/elasticsearch-shield-file.yml b/tasks/xpack/shield/elasticsearch-shield-file.yml new file mode 100644 index 0000000..0746f4b --- /dev/null +++ b/tasks/xpack/shield/elasticsearch-shield-file.yml @@ -0,0 +1,64 @@ +--- +- set_fact: manage_file_users=es_users is defined and es_users.file is defined + +#List current users +- name: List Users + shell: cat {{conf_dir}}/shield/users | awk -F':' '{print $1}' + register: current_file_users + when: manage_file_users + changed_when: False + +- set_fact: users_to_remove={{ current_file_users.stdout_lines | difference (es_users.file.keys()) }} + when: manage_file_users + +#Remove users +- name: Remove Users + command: > + {{es_home}}/bin/shield/esusers userdel {{item}} + when: manage_file_users and (users_to_remove | length > 0) + with_items: "{{users_to_remove}}" + environment: + CONF_DIR: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + + +- set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }} + when: manage_file_users + +#Add users +- name: Add Users + command: > + {{es_home}}/bin/shield/esusers useradd {{item}} -p {{es_users.file[item].password}} + with_items: "{{users_to_add}}" + when: manage_file_users and users_to_add | length > 0 + environment: + CONF_DIR: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + +#Set passwords for all users declared - Required as the useradd will not change existing user passwords +- name: Set User Passwords + command: > + {{es_home}}/bin/shield/esusers passwd {{item.key}} -p {{item.value.password}} + with_dict: "{{es_users.file}}" + when: manage_file_users and es_users.file.keys() | length > 0 + #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip. + changed_when: False + environment: + CONF_DIR: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + +- set_fact: users_roles={{es_users.file | extract_role_users}} + when: manage_file_users + +#Copy Roles files +- name: Copy roles.yml File for Instance + template: src=shield/roles.yml.j2 dest={{conf_dir}}/shield/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + when: es_roles is defined and es_roles.file is defined + +#Overwrite users_roles file +- name: Copy User Roles + template: src=shield/users_roles.j2 dest={{conf_dir}}/shield/users_roles mode=0644 force=yes + when: manage_file_users and users_roles | length > 0 + +#TODO: Support for mapping file + diff --git a/tasks/xpack/shield/elasticsearch-shield.yml b/tasks/xpack/shield/elasticsearch-shield.yml new file mode 100644 index 0000000..ca16402 --- /dev/null +++ b/tasks/xpack/shield/elasticsearch-shield.yml @@ -0,0 +1,23 @@ +--- +#Shield specific configuration done here + +#TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6 + +#-----------------------------FILE BASED REALM---------------------------------------- + +- include: elasticsearch-shield-file.yml + when: (es_enable_xpack and '"shield" in es_xpack_features') and ((es_users is defined and es_users.file) or (es_roles is defined and es_roles.file is defined)) + +#-----------------------------NATIVE BASED REALM---------------------------------------- +# The native realm requires the node to be started so we do as a handler +- command: /bin/true + notify: load-native-realms + when: (es_enable_xpack and '"shield" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)) + +#--------------------------------------------------------------------- + +#Ensure shield conf directory is created +- name: Ensure shield conf directory exists + file: path={{ conf_dir }}/shield state=directory owner={{ es_user }} group={{ es_group }} + changed_when: False + when: es_enable_xpack and '"shield" in es_xpack_features' diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index 203a869..40ab354 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -20,6 +20,4 @@ path.data: {{ data_dirs | array_to_str }} path.work: {{ work_dir }} -path.logs: {{ log_dir }} - -path.plugins: {{ plugin_dir }} \ No newline at end of file +path.logs: {{ log_dir }} \ No newline at end of file diff --git a/templates/shield/roles.yml.j2 b/templates/shield/roles.yml.j2 new file mode 100644 index 0000000..9f211f2 --- /dev/null +++ b/templates/shield/roles.yml.j2 @@ -0,0 +1 @@ +{{ es_roles.file | to_nice_yaml }} \ No newline at end of file diff --git a/templates/shield/users_roles.j2 b/templates/shield/users_roles.j2 new file mode 100644 index 0000000..1c0acfa --- /dev/null +++ b/templates/shield/users_roles.j2 @@ -0,0 +1 @@ +{{users_roles | join("\n") }} \ No newline at end of file diff --git a/test/integration/config-2x/serverspec/default_spec.rb b/test/integration/config-2x/serverspec/default_spec.rb index 377fb3b..f416eed 100644 --- a/test/integration/config-2x/serverspec/default_spec.rb +++ b/test/integration/config-2x/serverspec/default_spec.rb @@ -1,6 +1,6 @@ require 'config_spec' describe 'Config Tests v 2.x' do - include_examples 'config::init', "2.2.0" + include_examples 'config::init', "2.3.4" end diff --git a/test/integration/helpers/serverspec/multi_spec.rb b/test/integration/helpers/serverspec/multi_spec.rb index a52bf80..4d2cc2f 100644 --- a/test/integration/helpers/serverspec/multi_spec.rb +++ b/test/integration/helpers/serverspec/multi_spec.rb @@ -173,28 +173,7 @@ shared_examples 'multi::init' do |es_version,plugins| end end - #Multi node plugin tests - describe file('/opt/elasticsearch/plugins/node1') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/opt/elasticsearch/plugins/master') do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - for plugin in plugins - describe file('/opt/elasticsearch/plugins/node1/'+plugin) do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end - - describe file('/opt/elasticsearch/plugins/master/'+plugin) do - it { should be_directory } - it { should be_owned_by 'elasticsearch' } - end describe command('curl -s localhost:9200/_nodes/plugins?pretty=true | grep '+plugin) do its(:exit_status) { should eq 0 } @@ -203,6 +182,11 @@ shared_examples 'multi::init' do |es_version,plugins| describe command('curl -s localhost:9201/_nodes/plugins?pretty=true | grep '+plugin) do its(:exit_status) { should eq 0 } end + + describe file('/usr/share/elasticsearch/plugins/'+plugin) do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end end describe file('/etc/init.d/elasticsearch') do diff --git a/test/integration/helpers/serverspec/package_spec.rb b/test/integration/helpers/serverspec/package_spec.rb index 2d2dbbb..51db46e 100644 --- a/test/integration/helpers/serverspec/package_spec.rb +++ b/test/integration/helpers/serverspec/package_spec.rb @@ -16,7 +16,6 @@ shared_examples 'package::init' do |es_version,plugins| describe file('/etc/elasticsearch/node1/elasticsearch.yml') do it { should be_file } - it { should contain 'path.plugins: /usr/share/elasticsearch/plugins/node1' } it { should contain 'http.port: 9200' } it { should contain 'transport.tcp.port: 9300' } it { should contain 'discovery.zen.ping.unicast.hosts: localhost:9300' } @@ -27,8 +26,6 @@ shared_examples 'package::init' do |es_version,plugins| it { should be_owned_by 'elasticsearch' } end - - describe file('/etc/elasticsearch/node1/scripts/calculate-score.groovy') do it { should be_file } it { should be_owned_by 'elasticsearch' } @@ -66,14 +63,14 @@ shared_examples 'package::init' do |es_version,plugins| end end - describe file('/usr/share/elasticsearch/plugins/node1') do + describe file('/usr/share/elasticsearch/plugins') do it { should be_directory } it { should be_owned_by 'elasticsearch' } end for plugin in plugins - describe file('/usr/share/elasticsearch/plugins/node1/'+plugin) do + describe file('/usr/share/elasticsearch/plugins/'+plugin) do it { should be_directory } it { should be_owned_by 'elasticsearch' } end diff --git a/test/integration/helpers/serverspec/xpack_spec.rb b/test/integration/helpers/serverspec/xpack_spec.rb new file mode 100644 index 0000000..69583ed --- /dev/null +++ b/test/integration/helpers/serverspec/xpack_spec.rb @@ -0,0 +1,215 @@ +require 'spec_helper' + +shared_examples 'xpack::init' do |es_version| + + describe user('elasticsearch') do + it { should exist } + end + + describe service('shield_node_elasticsearch') do + it { should be_running } + end + + describe package('elasticsearch') do + it { should be_installed } + end + + describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do + it { should be_file } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/etc/elasticsearch/shield_node/logging.yml') do + it { should be_file } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do + it { should contain 'node.name: localhost-shield_node' } + it { should contain 'cluster.name: elasticsearch' } + it { should contain 'path.conf: /etc/elasticsearch/shield_node' } + it { should contain 'path.data: /var/lib/elasticsearch/localhost-shield_node' } + it { should contain 'path.work: /tmp/elasticsearch/localhost-shield_node' } + it { should contain 'path.logs: /var/log/elasticsearch/localhost-shield_node' } + end + + describe 'Node listening' do + it 'listening in port 9200' do + expect(port 9200).to be_listening + end + end + + describe 'version check' do + it 'should be reported as version '+es_version do + command = command('curl -s localhost:9200 -u es_admin:changeMe | grep number') + expect(command.stdout).to match(es_version) + expect(command.exit_status).to eq(0) + end + end + + describe file('/etc/init.d/elasticsearch') do + it { should_not exist } + end + + describe file('/etc/default/elasticsearch') do + it { should_not exist } + end + + describe file('/etc/sysconfig/elasticsearch') do + it { should_not exist } + end + + describe file('/usr/lib/systemd/system/elasticsearch.service') do + it { should_not exist } + end + + describe file('/etc/elasticsearch/elasticsearch.yml') do + it { should_not exist } + end + + describe file('/etc/elasticsearch/logging.yml') do + it { should_not exist } + end + + #Xpack specific tests + describe file('/usr/share/elasticsearch/plugins') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + + #Check shield,watcher and license plugins are installed + describe file('/usr/share/elasticsearch/plugins/license') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep license') do + its(:exit_status) { should eq 0 } + end + + describe file('/usr/share/elasticsearch/plugins/shield') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep shield') do + its(:exit_status) { should eq 0 } + end + + describe file('/etc/elasticsearch/shield_node/shield') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/usr/share/elasticsearch/plugins/watcher') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep watcher') do + its(:exit_status) { should eq 0 } + end + + describe file('/usr/share/elasticsearch/plugins/kopf') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep kopf') do + its(:exit_status) { should eq 0 } + end + + #test we haven't installed graph or marvel-agent + + describe file('/usr/share/elasticsearch/plugins/graph') do + it { should_not exist } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep graph') do + its(:exit_status) { should eq 1 } + end + + describe file('/usr/share/elasticsearch/plugins/marvel-agent') do + it { should_not exist } + end + + describe command('curl -s localhost:9200/_nodes/plugins?pretty=true -u es_admin:changeMe | grep marvel-agent') do + its(:exit_status) { should eq 1 } + end + + + #Test users file, users_roles and roles.yml + describe file('/etc/elasticsearch/shield_node/shield/users_roles') do + it { should be_owned_by 'elasticsearch' } + it { should contain 'admin:es_admin' } + it { should contain 'power_user:testUser' } + end + + describe file('/etc/elasticsearch/shield_node/shield/users') do + it { should be_owned_by 'elasticsearch' } + it { should contain 'testUser:' } + it { should contain 'es_admin:' } + end + + + describe file('/etc/elasticsearch/shield_node/shield/roles.yml') do + it { should be_owned_by 'elasticsearch' } + #Test contents as expected + its(:md5sum) { should eq '7800182547287abd480c8b095bf26e9e' } + end + + + #Test native roles and users are loaded + describe command('curl -s localhost:9200/_shield/user -u es_admin:changeMe | md5sum | grep 557a730df7136694131b5b7012a5ffad') do + its(:exit_status) { should eq 0 } + end + + describe command('curl -s localhost:9200/_shield/user -u es_admin:changeMe | grep "{\"kibana4_server\":{\"username\":\"kibana4_server\",\"roles\":\[\"kibana4_server\"\],\"full_name\":null,\"email\":null,\"metadata\":{}}}"') do + its(:exit_status) { should eq 0 } + end + + describe command('curl -s localhost:9200/_shield/role -u es_admin:changeMe | grep "{\"logstash\":{\"cluster\":\[\"manage_index_templates\"\],\"indices\":\[{\"names\":\[\"logstash-\*\"\],\"privileges\":\[\"write\",\"delete\",\"create_index\"\]}\],\"run_as\":\[\]}}"') do + its(:exit_status) { should eq 0 } + end + + describe command('curl -s localhost:9200/_shield/role -u es_admin:changeMe | md5sum | grep 6d14f09ef1eea64adf4d4a9c04229629') do + its(:exit_status) { should eq 0 } + end + + describe file('/etc/elasticsearch/templates') do + it { should be_directory } + it { should be_owned_by 'elasticsearch' } + end + + describe file('/etc/elasticsearch/templates/basic.json') do + it { should be_file } + it { should be_owned_by 'elasticsearch' } + end + + describe 'Template Installed' do + it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do + command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe') + expect(command.stdout).to match(/basic/) + expect(command.exit_status).to eq(0) + end + end + + #This is possibly subject to format changes in the response across versions so may fail in the future + describe 'Template Contents Correct' do + it 'should be reported as being installed', :retry => 3, :retry_wait => 10 do + command = command('curl -s "localhost:9200/_template/basic" -u es_admin:changeMe | md5sum') + expect(command.stdout).to match(/153b1a45daf48ccee80395b85c61e332/) + end + end + + #Test contents of Elasticsearch.yml file + describe file('/etc/elasticsearch/shield_node/elasticsearch.yml') do + it { should contain 'shield.authc.realms.file1.order: 0' } + it { should contain 'shield.authc.realms.file1.type: file' } + it { should contain 'shield.authc.realms.native1.order: 1' } + it { should contain 'shield.authc.realms.native1.type: native' } + end + +end + diff --git a/test/integration/multi-2x/serverspec/default_spec.rb b/test/integration/multi-2x/serverspec/default_spec.rb index 17bb9c1..81637c4 100644 --- a/test/integration/multi-2x/serverspec/default_spec.rb +++ b/test/integration/multi-2x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'multi_spec' describe 'Multi Tests v 2.x' do - include_examples 'multi::init', "2.2.0", ["kopf","license","marvel-agent"] + include_examples 'multi::init', "2.3.4", ["kopf"] end diff --git a/test/integration/multi.yml b/test/integration/multi.yml index e291fcb..173528c 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -8,5 +8,4 @@ vars: es_scripts: true es_templates: true - es_plugin_dir: "/opt/elasticsearch/plugins" #Plugins installed for this test are specified in .kitchen.yml under suite \ No newline at end of file diff --git a/test/integration/package-2x/serverspec/default_spec.rb b/test/integration/package-2x/serverspec/default_spec.rb index bc884cc..417df47 100644 --- a/test/integration/package-2x/serverspec/default_spec.rb +++ b/test/integration/package-2x/serverspec/default_spec.rb @@ -2,5 +2,5 @@ require 'package_spec' describe 'Package Tests v 2.x' do - include_examples 'package::init', "2.2.0", ["kopf","license","marvel-agent"] + include_examples 'package::init', "2.3.4", ["kopf"] end \ No newline at end of file diff --git a/test/integration/standard-2x/serverspec/default_spec.rb b/test/integration/standard-2x/serverspec/default_spec.rb index 8f45e24..86033b3 100644 --- a/test/integration/standard-2x/serverspec/default_spec.rb +++ b/test/integration/standard-2x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'standard_spec' describe 'Standard Tests v 2.x' do - include_examples 'standard::init', "2.2.0" + include_examples 'standard::init', "2.3.4" end diff --git a/test/integration/xpack-2x/serverspec/default_spec.rb b/test/integration/xpack-2x/serverspec/default_spec.rb new file mode 100644 index 0000000..e4ca2d8 --- /dev/null +++ b/test/integration/xpack-2x/serverspec/default_spec.rb @@ -0,0 +1,5 @@ +require 'xpack_spec' + +describe 'Xpack Tests v 2.x' do + include_examples 'xpack::init', "2.3.4" +end diff --git a/test/integration/xpack-2x/xpack.yml b/test/integration/xpack-2x/xpack.yml new file mode 100644 index 0000000..a3c37e1 --- /dev/null +++ b/test/integration/xpack-2x/xpack.yml @@ -0,0 +1,2 @@ +--- +- host: test-kitchen diff --git a/test/integration/xpack.yml b/test/integration/xpack.yml new file mode 100644 index 0000000..fb22b60 --- /dev/null +++ b/test/integration/xpack.yml @@ -0,0 +1,73 @@ +--- +- name: Elasticsearch Xpack tests + hosts: localhost + roles: + - { role: elasticsearch, es_config: { "http.port": 9200, "transport.tcp.port":9300, discovery.zen.ping.unicast.hosts: "localhost:9300", + "shield.authc.realms.file1.type": "file","shield.authc.realms.file1.order": 0, "shield.authc.realms.native1.type": "native","shield.authc.realms.native1.order": 1 }, + es_instance_name: "shield_node" } + vars: + es_templates: true + es_enable_xpack: true + es_plugins: + - plugin: lmenezes/elasticsearch-kopf + version: master + es_xpack_features: + - shield + - watcher + es_api_basic_auth_username: es_admin + es_api_basic_auth_password: changeMe + es_users: + native: + kibana4_server: + password: changeMe + roles: + - kibana4_server + file: + es_admin: + password: changeMe + roles: + - admin + testUser: + password: changeMeAlso! + roles: + - power_user + - user + es_roles: + file: + admin: + cluster: + - all + indices: + - names: '*' + privileges: + - all + power_user: + cluster: + - monitor + indices: + - names: '*' + privileges: + - all + user: + indices: + - names: '*' + privileges: + - read + kibana4_server: + cluster: + - monitor + indices: + - names: '.kibana' + privileges: + - all + native: + logstash: + cluster: + - manage_index_templates + indices: + - names: 'logstash-*' + privileges: + - write + - delete + - create_index + diff --git a/vars/main.yml b/vars/main.yml index e22b2cc..fd29447 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -2,4 +2,6 @@ es_package_url: "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch" es_conf_dir: "/etc/elasticsearch" sysd_script: "/usr/lib/systemd/system/elasticsearch.service" -init_script: "/etc/init.d/elasticsearch" \ No newline at end of file +init_script: "/etc/init.d/elasticsearch" +#add supported features here +supported_xpack_features: ["watcher","marvel-agent","graph","shield"] \ No newline at end of file