From b2997cc0bcca40f2f84a5fae1aa1c9b5db325800 Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Thu, 26 Nov 2015 17:26:28 +0000 Subject: [PATCH] Additional tests + support for empty es_config + documentation --- README.md | 140 +++++++++++------- defaults/main.yml | 2 +- tasks/checkParameters.yml | 8 +- tasks/elasticsearch-config.yml | 1 + templates/elasticsearch.yml.j2 | 3 + test/integration/config.yml | 2 +- test/integration/multi.yml | 2 +- .../multi/serverspec/default_spec.rb | 4 +- test/integration/standard.yml | 2 +- 9 files changed, 101 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 77801fd..e26197b 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,7 @@ Ansible playbook / roles / tasks for Elasticsearch. Currently it will work on D ## Usage -Create your ansible playbook with your own tasks, and include the role elasticsearch. -You will have to have this repository accessible within the context of playbook, e.g. +Create your Ansible playbook with your own tasks, and include the role elasticsearch. You will have to have this repository accessible within the context of playbook, e.g. e.g. @@ -17,23 +16,46 @@ mkdir -p roles ln -s /my/repos/ansible-elasticsearch ./roles/elasticsearch ``` -Then create your playbook yaml adding the role elasticsearch and overriding any variables you wish. It can be as simple as this to take all the defaults: +Then create your playbook yaml adding the role elasticsearch. By default, the user is only required to specify a unique es_instance_name per role application. +The simplest configuration therefore consists of: ``` --- -hosts: my_host +- name: Simple Example + hosts: localhost roles: - - { role: elasticsearch, es_multicast_enabled: true} - tasks: - - .... your tasks ... + - { role: elasticsearch, es_instance_name: "node1" } + vars: ``` -By default es_multicast_enabled is false and the user is required to specify the following additional parameters: +All Elasticsearch configuration parameters are supported. This is achieved using a configuration map parameter 'es_config' which is serialized into the elasticsearch.yml file. +The use of a map ensures the Ansible playbook does not need to be updated to reflect new/deprecated/plugin configuration parameters. -1. es_http_port - the http port for the node -2. es_transport_tcp_port - the transport port for the node -3. es_unicast_hosts - the unicast discovery list, in the comma separated format ":,:" (typically the clusters dedicated masters) +In addition to the es_config map, several other parameters are supported for additional functions e.g. script installation. These can be found in the role's defaults/main.yml file. + +The following illustrates applying configuration parameters to an Elasticsearch instance. + +``` +- name: Elasticsearch with custom configuration + hosts: localhost + roles: + #expand to all available parameters + - { role: elasticsearch, es_instance_name: "node1", es_data_dir: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.mlockall: true, discovery.zen.ping.multicast.enabled: false } } + vars: + es_scripts: false + es_templates: false + es_version_lock: false + es_heap_size: 1g +``` + + + +The playbook utilises Elasticsearch version defaults. By default, therefore, multicast is enabled for 1.x. If disabled, the user user is required to specify the following additional parameters: + +1. es_config['http.port'] - the http port for the node +2. es_config['transport.tcp.port'] - the transport port for the node +3. es_config['discovery.zen.ping.unicast.hosts'] - the unicast discovery list, in the comma separated format ":,:" (typically the clusters dedicated masters) If set to true, the ports will be auto defined and node discovery will be performed using multicast. @@ -42,65 +64,75 @@ A more complex example: ``` --- -hosts: localhost -roles: - - { role: elasticsearch, es_unicast_hosts: "localhost:9301", es_http_port: "9201", es_transport_tcp_port: "9301", es_data_node: false, es_master_node: true, es_m_lock_enabled: true, es_multicast_enabled: false, es_node_name_prefix: "node1_", es_cluster_name: "custom-cluster" } -vars: - es_scripts: false - es_templates: false - es_version_lock: false - es_m_lock_enabled: true - es_start_service: false - es_plugins_reinstall: false - es_plugins: - - plugin: elasticsearch/elasticsearch-cloud-aws - version: 2.5.0 - - plugin: elasticsearch/marvel - version: latest - - plugin: elasticsearch/license - version: latest - - plugin: elasticsearch/shield - version: latest - - plugin: elasticsearch/elasticsearch-support-diagnostics - version: latest - - plugin: lmenezes/elasticsearch-kopf - version: master -tasks: - - .... your tasks ... +- name: Elasticsearch with custom configuration + hosts: localhost + roles: + #expand to all available parameters + - { role: elasticsearch, es_instance_name: "node1", es_data_dir: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.mlockall: true, discovery.zen.ping.multicast.enabled: false } } + vars: + es_scripts: false + es_templates: false + es_version_lock: false + es_heap_size: 1g + es_scripts: false + es_templates: false + es_version_lock: false + es_start_service: false + es_plugins_reinstall: false + es_plugins: + - plugin: elasticsearch/elasticsearch-cloud-aws + version: 2.5.0 + - plugin: elasticsearch/marvel + version: latest + - plugin: elasticsearch/license + version: latest + - plugin: elasticsearch/shield + version: latest + - plugin: elasticsearch/elasticsearch-support-diagnostics + version: latest + - plugin: lmenezes/elasticsearch-kopf + version: master ``` -The above example illustrates the ability to control the configuration. +The application of a role results in the installation of a node on a host. Multiple roles equates to multiple nodes for a single host. -The application of a role results in the installation of a node on a host. Multiple roles equates to multiple nodes for a single host. If specifying multiple roles for a host, and thus multiple nodes, the user must: - -1. Provide a es_node_name_prefix. This is used to ensure seperation of data, log, config and init scripts. -2. Ensure those playbooks responsible for installing and starting master eligble roles are specified first. These are required for cluster initalization. +In any multi node cluster configuration it is recommened the user list the master eligble roles first - especially if these are used a unicast hosts off which other nodes are 'booted' An example of a two server deployment, each with 1 node on one server and 2 nodes on another. The first server holds the master and is thus declared first. ``` --- -hosts: masters -roles: - - { role: elasticsearch, es_node_name_prefix: "node1_", es_unicast_hosts: "localhost:9300", es_http_port: "9201", es_transport_tcp_port: "9301", es_data_node: true, es_master_node: false, es_m_lock_enabled: true, es_multicast_enabled: false } -vars: - es_scripts: false - es_templates: false - es_version_lock: false - es_cluster_name: example-cluster - m_lock_enabled: false -- hosts: data_nodes +- hosts: master_nodes roles: - - { role: elasticsearch, es_data_node: true, es_master_node: false, es_m_lock_enabled: true, es_multicast_enabled: false, es_node_name_prefix: "node1_" } - - { role: elasticsearch, es_data_node: true, es_master_node: false, es_m_lock_enabled: true, es_multicast_enabled: false, es_node_name_prefix: "node2_" } + # one master per host + - { role: elasticsearch, es_instance_name: "node1", es_heap_size: "1g", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "elastic02:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.mlockall: false, discovery.zen.ping.multicast.enabled: false } } vars: es_scripts: false es_templates: false es_version_lock: false - es_cluster_name: example-cluster + es_cluster_name: test-cluster m_lock_enabled: true + ansible_user: ansible es_plugins: + - plugin: elasticsearch/license + version: latest + +- hosts: data_nodes + roles: + # two nodes per host + - { role: elasticsearch, es_instance_name: "node1", es_data_dir: "/opt/elasticsearch", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "elastic02:9300", http.port: 9200, transport.tcp.port: 9300, node.data: true, node.master: false, bootstrap.mlockall: false, discovery.zen.ping.multicast.enabled: false } } + - { role: elasticsearch, es_instance_name: "node2", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "elastic02:9300", http.port: 9201, transport.tcp.port: 9301, node.data: true, node.master: false, bootstrap.mlockall: false, discovery.zen.ping.multicast.enabled: false } } + vars: + es_scripts: false + es_templates: false + es_version_lock: false + es_cluster_name: test-cluster + m_lock_enabled: false + ansible_user: ansible + es_plugins: + - plugin: elasticsearch/license + version: latest ``` Parameters can additionally be assigned to hosts using the inventory file if desired. diff --git a/defaults/main.yml b/defaults/main.yml index 96f0e88..86ac56d 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -9,7 +9,7 @@ es_scripts: false es_templates: false es_user: elasticsearch es_group: elasticsearch - +es_config: {} #Need to provide default directories es_pid_dir: "/var/run/elasticsearch" es_data_dir: "/var/lib/elasticsearch" diff --git a/tasks/checkParameters.yml b/tasks/checkParameters.yml index 34d52d3..d2a03a4 100644 --- a/tasks/checkParameters.yml +++ b/tasks/checkParameters.yml @@ -3,14 +3,16 @@ - fail: msg="es_instance_name must be specified" when: es_instance_name is not defined +- set_fact: multi_cast={{ (es_version | version_compare('2.0', '<') and es_config['discovery.zen.ping.multicast.enabled'] is not defined) or es_config['discovery.zen.ping.multicast.enabled']}} + - fail: msg="Parameter 'http.port' must be defined when multicast is disabled" - when: (es_config['discovery.zen.ping.multicast.enabled'] == False or (es_version | version_compare('2.0', '>=') and es_config['discovery.zen.ping.multicast.enabled'] is not defined)) and es_config['http.port'] is not defined + when: not multi_cast and es_config['http.port'] is not defined - fail: msg="Parameter 'transport.tcp.port' must be defined when multicast is disabled" - when: (es_config['discovery.zen.ping.multicast.enabled'] == False or (es_version | version_compare('2.0', '>=') and es_config['discovery.zen.ping.multicast.enabled'] is not defined)) and es_config['transport.tcp.port'] is not defined + when: not multi_cast and es_config['transport.tcp.port'] is not defined - fail: msg="Parameter 'discovery.zen.ping.unicast.hosts' must be defined when multicast is disabled" - when: (es_config['discovery.zen.ping.multicast.enabled'] == False or (es_version | version_compare('2.0', '>=') and es_config['discovery.zen.ping.multicast.enabled'] is not defined)) and es_config['discovery.zen.ping.unicast.hosts'] is not defined + when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined #If the user attempts to lock memory they must specify a heap size - fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified" diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index a26ef55..ebceb25 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -128,6 +128,7 @@ - stat: path=/usr/lib/systemd/system/elasticsearch.service register: systemd_service + when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True - name: Configure systemd lineinfile: dest=/usr/lib/systemd/system/elasticsearch.service regexp="^LimitMEMLOCK" insertafter="^#LimitMEMLOCK" line="LimitMEMLOCK=infinity" diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index 275c93c..c31d7da 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -1,4 +1,7 @@ + +{% if es_config %} {{ es_config | to_nice_yaml }} +{% endif %} {% if es_config['cluster.name'] is not defined %} cluster.name: elasticsearch diff --git a/test/integration/config.yml b/test/integration/config.yml index 357752d..9bd8936 100644 --- a/test/integration/config.yml +++ b/test/integration/config.yml @@ -1,6 +1,6 @@ --- #Test explicit setting of parameters and variables -- name: Elasticsearch Package tests +- name: Elasticsearch Config tests hosts: localhost roles: #expand to all available parameters diff --git a/test/integration/multi.yml b/test/integration/multi.yml index e138f7b..34652b6 100644 --- a/test/integration/multi.yml +++ b/test/integration/multi.yml @@ -1,6 +1,6 @@ --- #Test ability to deploy multiple instances to a machine -- name: Elasticsearch Config tests +- name: Elasticsearch Multi tests hosts: localhost roles: - { role: elasticsearch, es_instance_name: "master", es_data_dir: "/opt/elasticsearch", es_heap_size: "1g", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "localhost:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.mlockall: true, discovery.zen.ping.multicast.enabled: false } } diff --git a/test/integration/multi/serverspec/default_spec.rb b/test/integration/multi/serverspec/default_spec.rb index 3447f4a..38f8334 100644 --- a/test/integration/multi/serverspec/default_spec.rb +++ b/test/integration/multi/serverspec/default_spec.rb @@ -88,13 +88,13 @@ context "basic tests" do end #test to make sure mlock was applied - describe command('curl "localhost:9200/_nodes/process?pretty" | grep mlockall') do + describe command('curl "localhost:9200/_nodes/localhost-master/process?pretty" | grep mlockall') do its(:stdout) { should match /\"mlockall\" : true/ } its(:exit_status) { should eq 0 } end #test to make sure mlock was not applied - describe command('curl "localhost:9201/_nodes/process?pretty" | grep mlockall') do + describe command('curl "localhost:9201/_nodes/localhost-node1/process?pretty" | grep mlockall') do its(:stdout) { should match /\"mlockall\" : false/ } its(:exit_status) { should eq 0 } end diff --git a/test/integration/standard.yml b/test/integration/standard.yml index 89236d7..66fd90e 100644 --- a/test/integration/standard.yml +++ b/test/integration/standard.yml @@ -2,7 +2,7 @@ - name: wrapper playbook for kitchen testing "elasticsearch" hosts: localhost roles: - - { role: elasticsearch, es_config: { "discovery.zen.ping.multicast.enabled": true }, es_instance_name: "node1" } + - { role: elasticsearch, es_instance_name: "node1" } vars: es_use_repository: "true" es_plugins: