Merge branch 'master' of github.com:elastic/ansible-elasticsearch

.gitignore

@@ -1,5 +1,5 @@
 .kitchen/
-*.lock
+*.pyc
 .vendor
 .bundle
 Converging

.kitchen.yml

@@ -15,7 +15,7 @@ provisioner:
 platforms:
   - name: ubuntu-14.04
     driver_config:
-      image: dliappis/ubuntu:14.04
+      image: dliappis/ubuntu-devopsci:14.04
       privileged: true
       provision_command:
         - apt-get update && apt-get install -y software-properties-common &&  add-apt-repository -y ppa:ansible/ansible
@@ -23,7 +23,7 @@ platforms:
       use_sudo: false
   - name: debian-7
     driver_config:
-      image: debian:7
+      image: dliappis/debian-devopsci:7
       privileged: true
       provision_command:
         - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml
@@ -32,7 +32,7 @@ platforms:
       use_sudo: false
   - name: debian-8
     driver_config:
-      image: debian:8
+      image: dliappis/debian-devopsci:8
       privileged: true
       provision_command:
         - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget
@@ -45,21 +45,14 @@ platforms:
       run_command: "/sbin/init"
   - name: centos-6
     driver_config:
-      image:  dliappis/centos:6
+      image: dliappis/centos-devopsci:6
       privileged: true
       provision_command:
-        - yum remove -y ansible
-        - yum -y install gmp-devel python-devel python-pip
-        - pip install ansible
       use_sudo: false
   - name: centos-7
     driver_config:
-      image: dliappis/centos:7
+      image: dliappis/centos-devopsci:7
       provision_command:
-        - yum remove -y ansible
-        - yum -y install gmp-devel python-devel python-pip
-        - yum -y groupinstall "Development Tools"
-        - pip install ansible
         - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config
         - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config
         - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config

Gemfile.lock

@@ -0,0 +1,42 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    highline (1.7.8)
+    kitchen-ansible (0.40.1)
+      librarian-ansible
+      test-kitchen (~> 1.4)
+    kitchen-docker (2.1.0)
+      test-kitchen (>= 1.0.0)
+    librarian (0.1.2)
+      highline
+      thor (~> 0.15)
+    librarian-ansible (3.0.0)
+      faraday
+      librarian (~> 0.1.0)
+    mixlib-shellout (2.2.6)
+    multipart-post (2.0.0)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.4)
+    safe_yaml (1.0.4)
+    test-kitchen (1.4.2)
+      mixlib-shellout (>= 1.2, < 3.0)
+      net-scp (~> 1.1)
+      net-ssh (~> 2.7, < 2.10)
+      safe_yaml (~> 1.0)
+      thor (~> 0.18)
+    thor (0.19.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  kitchen-ansible (= 0.40.1)
+  kitchen-docker (= 2.1.0)
+  net-ssh (~> 2.0)
+  test-kitchen (= 1.4.2)
+
+BUNDLED WITH
+   1.11.2

README.md

@@ -40,6 +40,8 @@ The simplest configuration therefore consists of:
 
 The above installs a single node 'node1' on the hosts 'localhost'.
 
+This role also uses [Ansible tags](http://docs.ansible.com/ansible/playbooks_tags.html). Run your playbook with the `--list-tasks` flag for more information.
+
 ### Basic Elasticsearch Configuration
 
 All Elasticsearch configuration parameters are supported.  This is achieved using a configuration map parameter 'es_config' which is serialized into the elasticsearch.yml file.  
@@ -137,12 +139,22 @@ recommended in any multi node cluster configuration.
 ```
 - hosts: master_nodes
   roles:
-    - { role: elasticsearch, es_instance_name: "node1", es_heap_size: "1g", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "elastic02:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.mlockall: false, discovery.zen.ping.multicast.enabled: false } }
+    - { role: elasticsearch, es_instance_name: "node1", es_heap_size: "1g",
+    es_config: {
+        cluster.name: "test-cluster",
+        "discovery.zen.ping.multicast.enabled": false,
+        discovery.zen.ping.unicast.hosts: "elastic02:9300",
+        http.port: 9200,
+        transport.tcp.port: 9300,
+        node.data: false,
+        node.master: true,
+        bootstrap.mlockall: false,
+        discovery.zen.ping.multicast.enabled: false }
+    }
   vars:
     es_scripts: false
     es_templates: false
     es_version_lock: false
-    es_cluster_name: test-cluster
     ansible_user: ansible
     es_plugins:
      - plugin: elasticsearch/license
@@ -159,6 +171,7 @@ recommended in any multi node cluster configuration.
         node.data: true,
         node.master: false,
         bootstrap.mlockall: false,
+        cluster.name: "test-cluster",
         discovery.zen.ping.multicast.enabled: false } 
     }
     - { role: elasticsearch, es_instance_name: "node2", 
@@ -170,13 +183,13 @@ recommended in any multi node cluster configuration.
         node.data: true,
         node.master: false,
         bootstrap.mlockall: false,
+        cluster.name: "test-cluster",
         discovery.zen.ping.multicast.enabled: false } 
     }
   vars:
     es_scripts: false
     es_templates: false
     es_version_lock: false
-    es_cluster_name: test-cluster
     ansible_user: ansible
     es_plugins:
      - plugin: elasticsearch/license
@@ -204,6 +217,11 @@ Following variables affect the versions installed:
 * ```es_start_service``` (true (default) or false)
 * ```es_plugins_reinstall``` (true or false (default) )
 * ```es_plugins``` (an array of plugin definitions e.g.:
+* ```es_allow_downgrades``` For development purposes only. (true or false (default) )
+* ```es_java_install``` If set to false, Java will not be installed. (true (default) or false)
+* ```update_java``` Updates Java to the latest version. (true or false (default))
+
+
 
 ```
   es_plugins:
@@ -221,6 +239,10 @@ If installing Marvel or Watcher, ensure the license plugin is also specified.  S
 
 * ```es_user``` - defaults to elasticsearch.
 * ```es_group``` - defaults to elasticsearch.
+* ```es_user_id``` - default is undefined.
+* ```es_group_id``` - default is undefined.
+
+Both ```es_user_id``` and ```es_group_id``` must be set for the user and group ids to be set. 
 
 By default, each node on a host will be installed to use unique pid, plugin, work, data and log directories.  These directories are created, using the instance and host name, beneath default locations ]
 controlled by the following parameters:

defaults/main.yml

@@ -4,6 +4,8 @@ es_version: "2.2.0"
 es_version_lock: false
 es_use_repository: true
 es_start_service: true
+es_java_install: true
+update_java: false
 es_restart_on_change: true
 es_plugins_reinstall: false
 es_scripts: false
@@ -11,9 +13,13 @@ es_templates: false
 es_user: elasticsearch
 es_group: elasticsearch
 es_config: {}
+es_install_shield: false
 #Need to provide default directories
 es_pid_dir: "/var/run/elasticsearch"
 es_data_dirs: "/var/lib/elasticsearch"
 es_log_dir: "/var/log/elasticsearch"
 es_work_dir: "/tmp/elasticsearch"
 es_plugin_dir: "/usr/share/elasticsearch/plugins"
+es_max_open_files: 65536
+es_allow_downgrades: false
+

handlers/main.yml

@@ -1,4 +1,4 @@
 
 - name: restart elasticsearch
   service: name={{instance_init_script | basename}} state=restarted enabled=yes
-  when: es_restart_on_change and es_start_service and (not elasticsearch_started.changed or (plugin_installed is defined and plugin_installed.changed) or elasticsearch_install.changed)
+  when: es_restart_on_change and es_start_service and not elasticsearch_started.changed and ((plugin_installed is defined and plugin_installed.changed) or (elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed))

tasks/elasticsearch-Debian.yml

@@ -1,4 +1,10 @@
 ---
+
+- set_fact: force_install=no
+
+- set_fact: force_install=yes
+  when: es_allow_downgrades
+
 - name: Debian - Add Elasticsearch repository key
   apt_key: url="http://packages.elasticsearch.org/GPG-KEY-elasticsearch" state=present
   when: es_use_repository
@@ -8,9 +14,9 @@
   when: es_use_repository
 
 - name: Debian - Ensure elasticsearch is installed
-  apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present cache_valid_time=86400
+  apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present force={{force_install}} cache_valid_time=86400
   when: es_use_repository
-  register: elasticsearch_install
+  register: elasticsearch_install_from_repo
 
 - name: Debian - Download elasticsearch from url
   get_url: url={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.deb{% endif %} dest=/tmp/elasticsearch-{{ es_version }}.deb validate_certs=no
@@ -19,4 +25,4 @@
 - name: Debian - Ensure elasticsearch is installed from downloaded package
   apt: deb=/tmp/elasticsearch-{{ es_version }}.deb
   when: not es_use_repository
-  register: elasticsearch_install
+  register: elasticsearch_install_from_package

tasks/elasticsearch-RedHat.yml

@@ -14,9 +14,9 @@
 - name: RedHat - Install Elasticsearch
   yum: name=elasticsearch{% if es_version is defined and es_version != ""  %}-{{ es_version }}{% endif %} state=present update_cache=yes
   when: es_use_repository
-  register: elasticsearch_install
+  register: elasticsearch_install_from_repo
 
 - name: RedHat - Install Elasticsearch from url
   yum: name={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.noarch.rpm{% endif %} state=present
   when: not es_use_repository
-  register: elasticsearch_install
+  register: elasticsearch_install_from_package

tasks/elasticsearch-config.yml

@@ -11,19 +11,31 @@
 
 
 - set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }}
+  tags:
+      - always
 
 - set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}}
   when: use_system_d
+  tags:
+      - always
 
 #For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN.
 
 - set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }}
+  tags:
+      - always
 
 - set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}}
+  tags:
+      - always
 
 - set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}}
+  tags:
+      - always
 
 - set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}}
+  tags:
+      - always
 
 #Create required directories
 - name: Create Directories
@@ -36,6 +48,8 @@
     - "{{plugin_dir}}"
 
 - set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }}
+  tags:
+      - always
 
 - name: Create Data Directories
   file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }}

tasks/elasticsearch-optional-user.yml

@@ -0,0 +1,18 @@
+---
+#Add the elasticsearch user before installing from packages.
+- name: Ensure optional elasticsearch group is created with the correct id.
+  group:
+    state: present
+    name: "{{ es_group }}"
+    system: yes
+    gid: "{{ es_group_id }}"
+
+- name: Ensure optional elasticsearch user is created with the correct id.
+  user:
+    state: present
+    name: "{{ es_user }}"
+    comment: elasticsearch system user
+    system: yes
+    createhome: no
+    uid: "{{ es_user_id }}"
+    group: "{{ es_group }}"

tasks/elasticsearch-plugins.yml

@@ -1,12 +1,19 @@
 ---
 
-#es_plugins_reinstall will be set to true if elasticsearch_install.changed i.e. we have changed ES version, or if no plugins listed. Otherwise it is false and requires explicitly setting.
+# es_plugins_reinstall will be set to true if elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed
+# i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting.
 - set_fact: es_plugins_reinstall=true
-  when: elasticsearch_install.changed or es_plugins is not defined or es_plugins is none
+  when: ((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none
+  tags:
+      - always
 
 - set_fact: list_command="list"
+  tags:
+      - always
 - set_fact: list_command="--list"
   when: es_version | version_compare('2.0', '<')
+  tags:
+      - always
 
 #List currently installed plugins
 - shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-"
@@ -20,7 +27,7 @@
 - name: Remove elasticsearch plugins
   command: "{{es_home}}/bin/plugin remove {{item}} --silent"
   ignore_errors: yes
-  with_items: installed_plugins.stdout_lines
+  with_items: "{{ installed_plugins.stdout_lines }}"
   when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0]
   notify: restart elasticsearch
   environment:
@@ -34,8 +41,8 @@
   register: plugin_installed
   failed_when: "'ERROR' in plugin_installed.stdout"
   changed_when: plugin_installed.rc == 0
-  with_items: es_plugins
+  with_items: "{{ es_plugins }}"
-  when: es_plugins is defined and not es_plugins is none
+  when: es_plugins is defined and not es_plugins is none and es_plugins_reinstall
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"

tasks/elasticsearch-scripts.yml

@@ -1,9 +1,13 @@
 ---
 
 - set_fact: es_script_dir={{ es_conf_dir }}/{{es_instance_name}}
+  tags:
+      - always
 
 - set_fact: es_script_dir={{es_config['path.scripts']}}
   when: es_config['path.scripts'] is defined
+  tags:
+      - always
 
 - name: Create script dir
   file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }}

tasks/elasticsearch-shield.yml

@@ -0,0 +1,3 @@
+---
+
+

tasks/elasticsearch-templates.yml

@@ -7,13 +7,19 @@
   when: es_templates_fileglob is not defined
 
 - name: Copy templates to elasticsearch
-  copy: src={{ item }} dest=/etc/elasticsearch/ owner={{ es_user }} group={{ es_group }}
+  copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }}
-  with_fileglob: es_templates_fileglob
+  when: es_templates_fileglob is defined
+  with_fileglob:
+    - "{{ es_templates_fileglob }}"
 
 - set_fact: http_port=9200
+  tags:
+      - always
 
 - set_fact: http_port={{es_config['http.port']}}
   when: es_config['http.port'] is defined
+  tags:
+      - always
 
 - name: Wait for elasticsearch to startup
   wait_for: port={{http_port}} delay=10
@@ -24,4 +30,4 @@
 
 - name: Install template(s)
   command: "curl -sL -XPUT http://localhost:{{http_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json"
-  with_items: resultstemplate.stdout_lines
+  with_items: "{{ resultstemplate.stdout_lines }}"

tasks/elasticsearch.yml

@@ -1,13 +1,27 @@
 ---
 
 - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}}
+  tags:
+      - always
 - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}}
+  tags:
+      - always
 - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}}
+  tags:
+      - always
 - set_fact: plugin_dir={{ es_plugin_dir }}/{{es_instance_name}}
+  tags:
+      - always
 - set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }}
+  tags:
+      - always
 
 - debug: msg="Node configuration {{ es_config }} "
 
+- name: Include optional user and group creation. 
+  when: (es_user_id is defined) and (es_group_id is defined)
+  include: elasticsearch-optional-user.yml
+
 #- name: Include specific Elasticsearch
 #  include: "elasticsearch-{{ansible_os_family}}.yml"
 

tasks/java.yml

@@ -1,8 +1,15 @@
 ---
+
+
+- set_fact: java_state="present"
+
+- set_fact: java_state="latest"
+  when: update_java == true
+
 - name: RedHat - Ensure Java is installed
-  yum: name={{ java }} state=latest
+  yum: name={{ java }} state={{java_state}}
   when: ansible_os_family == 'RedHat'
   
 - name: Debian - Ensure Java is installed
-  apt: name={{ java }} state=present update_cache=yes force=yes
+  apt: name={{ java }} state={{java_state}} update_cache=yes force=yes
   when: ansible_os_family == 'Debian'

tasks/main.yml

@@ -1,16 +1,39 @@
 ---
 - name: check-parameters
   include: checkParameters.yml
+  tags:
+      - check
 - name: os-specific vars
   include_vars: "{{ansible_os_family}}.yml"
+  tags:
+      - always
 - include: java.yml
+  when: es_java_install
+  tags:
+      - java
 - include: elasticsearch.yml
+  tags:
+      - install
 - include: elasticsearch-config.yml
+  tags:
+      - config
 - include: elasticsearch-scripts.yml
   when: es_scripts
+  tags:
+      - scripts
 - include: elasticsearch-plugins.yml
   when: es_plugins is defined or es_plugins_reinstall
+  tags:
+      - plugins
+- include: elasticsearch-shield.yml
+  when: es_install_shield
+  tags:
+      - shield
 - include: elasticsearch-service.yml
+  tags:
+      - service
 - include: elasticsearch-templates.yml
   when: es_templates
+  tags:
+      - templates
 - meta: flush_handlers

templates/systemd/elasticsearch.j2

@@ -31,7 +31,7 @@ StandardOutput=null
 StandardError=journal
 
 # Specifies the maximum file descriptor number that can be opened by this process
-LimitNOFILE=65535
+LimitNOFILE={{es_max_open_files}}
 
 # Specifies the maximum number of bytes of memory that may be locked into RAM
 # Set to "infinity" if you use the 'bootstrap.mlockall: true' option

test/integration/config.yml

@@ -4,7 +4,7 @@
   hosts: localhost
   roles:
     #expand to all available parameters
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.mlockall: true, discovery.zen.ping.multicast.enabled: false } }
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.mlockall: true, discovery.zen.ping.multicast.enabled: false } }
   vars:
     es_scripts: false
     es_templates: false

test/integration/helpers/serverspec/config_spec.rb

@@ -6,6 +6,14 @@ shared_examples 'config::init' do |es_version|
     it { should exist }
   end
   
+  describe group('elasticsearch') do
+    it { should have_gid 333 }
+  end
+
+  describe user('elasticsearch') do
+    it { should have_uid 333 }
+  end
+
   describe service('node1_elasticsearch') do
     it { should be_running }
   end

vars/Debian.yml

@@ -1,4 +1,4 @@
 ---
-java: "openjdk-7-jre-headless"
+java: "{{ es_java | default('openjdk-7-jre-headless') }}"
 default_file: "/etc/default/elasticsearch"
 es_home: "/usr/share/elasticsearch"

vars/RedHat.yml

@@ -1,4 +1,4 @@
 ---
-java: "java-1.8.0-openjdk.x86_64"
+java: "{{ es_java | default('java-1.8.0-openjdk.x86_64') }}"
 default_file: "/etc/sysconfig/elasticsearch"
 es_home: "/usr/share/elasticsearch"