Merge branch 'master' of github.com:elastic/ansible-elasticsearch

.gitignore

@@ -1,8 +1,8 @@
 .kitchen/
-*.lock
+*.pyc
 .vendor
 .bundle
 Converging
 TODO
 .idea/
-elasticsearch.iml
+elasticsearch.iml

.kitchen.yml

@@ -15,7 +15,7 @@ provisioner:
 platforms:
   - name: ubuntu-14.04
     driver_config:
-      image: dliappis/ubuntu:14.04
+      image: dliappis/ubuntu-devopsci:14.04
       privileged: true
       provision_command:
         - apt-get update && apt-get install -y software-properties-common &&  add-apt-repository -y ppa:ansible/ansible
@@ -23,7 +23,7 @@ platforms:
       use_sudo: false
   - name: debian-7
     driver_config:
-      image: debian:7
+      image: dliappis/debian-devopsci:7
       privileged: true
       provision_command:
         - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml
@@ -32,7 +32,7 @@ platforms:
       use_sudo: false
   - name: debian-8
     driver_config:
-      image: debian:8
+      image: dliappis/debian-devopsci:8
       privileged: true
       provision_command:
         - apt-get update && apt-get -y install python python-dev python-pip build-essential libyaml-dev python-yaml curl wget
@@ -45,21 +45,14 @@ platforms:
       run_command: "/sbin/init"
   - name: centos-6
     driver_config:
-      image:  dliappis/centos:6
+      image: dliappis/centos-devopsci:6
       privileged: true
       provision_command:
-        - yum remove -y ansible
-        - yum -y install gmp-devel python-devel python-pip
-        - pip install ansible
       use_sudo: false
   - name: centos-7
     driver_config:
-      image: dliappis/centos:7
+      image: dliappis/centos-devopsci:7
       provision_command:
-        - yum remove -y ansible
-        - yum -y install gmp-devel python-devel python-pip
-        - yum -y groupinstall "Development Tools"
-        - pip install ansible
         - sed -ri 's/^#?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config
         - sed -ri 's/^#?PasswordAuthentication .*/PasswordAuthentication yes/' /etc/ssh/sshd_config
         - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config

Gemfile.lock

@@ -0,0 +1,42 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    highline (1.7.8)
+    kitchen-ansible (0.40.1)
+      librarian-ansible
+      test-kitchen (~> 1.4)
+    kitchen-docker (2.1.0)
+      test-kitchen (>= 1.0.0)
+    librarian (0.1.2)
+      highline
+      thor (~> 0.15)
+    librarian-ansible (3.0.0)
+      faraday
+      librarian (~> 0.1.0)
+    mixlib-shellout (2.2.6)
+    multipart-post (2.0.0)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.4)
+    safe_yaml (1.0.4)
+    test-kitchen (1.4.2)
+      mixlib-shellout (>= 1.2, < 3.0)
+      net-scp (~> 1.1)
+      net-ssh (~> 2.7, < 2.10)
+      safe_yaml (~> 1.0)
+      thor (~> 0.18)
+    thor (0.19.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  kitchen-ansible (= 0.40.1)
+  kitchen-docker (= 2.1.0)
+  net-ssh (~> 2.0)
+  test-kitchen (= 1.4.2)
+
+BUNDLED WITH
+   1.11.2

README.md

@@ -40,6 +40,8 @@ The simplest configuration therefore consists of:
 
 The above installs a single node 'node1' on the hosts 'localhost'.
 
+This role also uses [Ansible tags](http://docs.ansible.com/ansible/playbooks_tags.html). Run your playbook with the `--list-tasks` flag for more information.
+
 ### Basic Elasticsearch Configuration
 
 All Elasticsearch configuration parameters are supported.  This is achieved using a configuration map parameter 'es_config' which is serialized into the elasticsearch.yml file.  
@@ -137,12 +139,22 @@ recommended in any multi node cluster configuration.
 ```
 - hosts: master_nodes
   roles:
-    - { role: elasticsearch, es_instance_name: "node1", es_heap_size: "1g", es_config: { "discovery.zen.ping.multicast.enabled": false, discovery.zen.ping.unicast.hosts: "elastic02:9300", http.port: 9200, transport.tcp.port: 9300, node.data: false, node.master: true, bootstrap.mlockall: false, discovery.zen.ping.multicast.enabled: false } }
+    - { role: elasticsearch, es_instance_name: "node1", es_heap_size: "1g",
+    es_config: {
+        cluster.name: "test-cluster",
+        "discovery.zen.ping.multicast.enabled": false,
+        discovery.zen.ping.unicast.hosts: "elastic02:9300",
+        http.port: 9200,
+        transport.tcp.port: 9300,
+        node.data: false,
+        node.master: true,
+        bootstrap.mlockall: false,
+        discovery.zen.ping.multicast.enabled: false }
+    }
   vars:
     es_scripts: false
     es_templates: false
     es_version_lock: false
-    es_cluster_name: test-cluster
     ansible_user: ansible
     es_plugins:
      - plugin: elasticsearch/license
@@ -159,6 +171,7 @@ recommended in any multi node cluster configuration.
         node.data: true,
         node.master: false,
         bootstrap.mlockall: false,
+        cluster.name: "test-cluster",
         discovery.zen.ping.multicast.enabled: false } 
     }
     - { role: elasticsearch, es_instance_name: "node2", 
@@ -170,13 +183,13 @@ recommended in any multi node cluster configuration.
         node.data: true,
         node.master: false,
         bootstrap.mlockall: false,
+        cluster.name: "test-cluster",
         discovery.zen.ping.multicast.enabled: false } 
     }
   vars:
     es_scripts: false
     es_templates: false
     es_version_lock: false
-    es_cluster_name: test-cluster
     ansible_user: ansible
     es_plugins:
      - plugin: elasticsearch/license
@@ -204,6 +217,11 @@ Following variables affect the versions installed:
 * ```es_start_service``` (true (default) or false)
 * ```es_plugins_reinstall``` (true or false (default) )
 * ```es_plugins``` (an array of plugin definitions e.g.:
+* ```es_allow_downgrades``` For development purposes only. (true or false (default) )
+* ```es_java_install``` If set to false, Java will not be installed. (true (default) or false)
+* ```update_java``` Updates Java to the latest version. (true or false (default))
+
+
 
 ```
   es_plugins:
@@ -221,6 +239,10 @@ If installing Marvel or Watcher, ensure the license plugin is also specified.  S
 
 * ```es_user``` - defaults to elasticsearch.
 * ```es_group``` - defaults to elasticsearch.
+* ```es_user_id``` - default is undefined.
+* ```es_group_id``` - default is undefined.
+
+Both ```es_user_id``` and ```es_group_id``` must be set for the user and group ids to be set. 
 
 By default, each node on a host will be installed to use unique pid, plugin, work, data and log directories.  These directories are created, using the instance and host name, beneath default locations ]
 controlled by the following parameters:
@@ -271,4 +293,4 @@ Elasticsearch restarted where required.
 
 * If the ES version is changed, all plugins will be removed.  Those listed in the playbook will be re-installed.  This is behaviour is required in ES 2.x.
 * If no plugins are listed in the playbook for a node, all currently installed plugins will be removed.
-* The role does not currently support automatic detection of differences between installed and listed plugins (other than if none are listed).   Should users wish to change installed plugins should set es_plugins_reinstall to true.  This will cause all currently installed plugins to be removed and those listed to be installed.  Change detection will be implemented in future releases.
+* The role does not currently support automatic detection of differences between installed and listed plugins (other than if none are listed).   Should users wish to change installed plugins should set es_plugins_reinstall to true.  This will cause all currently installed plugins to be removed and those listed to be installed.  Change detection will be implemented in future releases.

defaults/main.yml

@@ -4,6 +4,8 @@ es_version: "2.2.0"
 es_version_lock: false
 es_use_repository: true
 es_start_service: true
+es_java_install: true
+update_java: false
 es_restart_on_change: true
 es_plugins_reinstall: false
 es_scripts: false
@@ -11,9 +13,13 @@ es_templates: false
 es_user: elasticsearch
 es_group: elasticsearch
 es_config: {}
+es_install_shield: false
 #Need to provide default directories
 es_pid_dir: "/var/run/elasticsearch"
 es_data_dirs: "/var/lib/elasticsearch"
 es_log_dir: "/var/log/elasticsearch"
 es_work_dir: "/tmp/elasticsearch"
-es_plugin_dir: "/usr/share/elasticsearch/plugins"
+es_plugin_dir: "/usr/share/elasticsearch/plugins"
+es_max_open_files: 65536
+es_allow_downgrades: false
+

handlers/main.yml

@@ -1,4 +1,4 @@
 
 - name: restart elasticsearch
   service: name={{instance_init_script | basename}} state=restarted enabled=yes
-  when: es_restart_on_change and es_start_service and (not elasticsearch_started.changed or (plugin_installed is defined and plugin_installed.changed) or elasticsearch_install.changed)
+  when: es_restart_on_change and es_start_service and not elasticsearch_started.changed and ((plugin_installed is defined and plugin_installed.changed) or (elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed))

tasks/elasticsearch-Debian.yml

@@ -1,4 +1,10 @@
 ---
+
+- set_fact: force_install=no
+
+- set_fact: force_install=yes
+  when: es_allow_downgrades
+
 - name: Debian - Add Elasticsearch repository key
   apt_key: url="http://packages.elasticsearch.org/GPG-KEY-elasticsearch" state=present
   when: es_use_repository
@@ -8,9 +14,9 @@
   when: es_use_repository
 
 - name: Debian - Ensure elasticsearch is installed
-  apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present cache_valid_time=86400
+  apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present force={{force_install}} cache_valid_time=86400
   when: es_use_repository
-  register: elasticsearch_install
+  register: elasticsearch_install_from_repo
 
 - name: Debian - Download elasticsearch from url
   get_url: url={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.deb{% endif %} dest=/tmp/elasticsearch-{{ es_version }}.deb validate_certs=no
@@ -19,4 +25,4 @@
 - name: Debian - Ensure elasticsearch is installed from downloaded package
   apt: deb=/tmp/elasticsearch-{{ es_version }}.deb
   when: not es_use_repository
-  register: elasticsearch_install
+  register: elasticsearch_install_from_package

tasks/elasticsearch-RedHat.yml

@@ -14,9 +14,9 @@
 - name: RedHat - Install Elasticsearch
   yum: name=elasticsearch{% if es_version is defined and es_version != ""  %}-{{ es_version }}{% endif %} state=present update_cache=yes
   when: es_use_repository
-  register: elasticsearch_install
+  register: elasticsearch_install_from_repo
 
 - name: RedHat - Install Elasticsearch from url
   yum: name={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.noarch.rpm{% endif %} state=present
   when: not es_use_repository
-  register: elasticsearch_install
+  register: elasticsearch_install_from_package

tasks/elasticsearch-config.yml

@@ -11,19 +11,31 @@
 
 
 - set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution == 'CentOS' and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }}
+  tags:
+      - always
 
 - set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}}
   when: use_system_d
+  tags:
+      - always
 
 #For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN.
 
 - set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }}
+  tags:
+      - always
 
 - set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}}
+  tags:
+      - always
 
 - set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}}
+  tags:
+      - always
 
 - set_fact: work_dir={{ es_work_dir }}/{{instance_suffix}}
+  tags:
+      - always
 
 #Create required directories
 - name: Create Directories
@@ -36,6 +48,8 @@
     - "{{plugin_dir}}"
 
 - set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }}
+  tags:
+      - always
 
 - name: Create Data Directories
   file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }}

tasks/elasticsearch-optional-user.yml

@@ -0,0 +1,18 @@
+---
+#Add the elasticsearch user before installing from packages.
+- name: Ensure optional elasticsearch group is created with the correct id.
+  group:
+    state: present
+    name: "{{ es_group }}"
+    system: yes
+    gid: "{{ es_group_id }}"
+
+- name: Ensure optional elasticsearch user is created with the correct id.
+  user:
+    state: present
+    name: "{{ es_user }}"
+    comment: elasticsearch system user
+    system: yes
+    createhome: no
+    uid: "{{ es_user_id }}"
+    group: "{{ es_group }}"

tasks/elasticsearch-plugins.yml

@@ -1,12 +1,19 @@
 ---
 
-#es_plugins_reinstall will be set to true if elasticsearch_install.changed i.e. we have changed ES version, or if no plugins listed. Otherwise it is false and requires explicitly setting.
+# es_plugins_reinstall will be set to true if elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed
+# i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting.
 - set_fact: es_plugins_reinstall=true
-  when: elasticsearch_install.changed or es_plugins is not defined or es_plugins is none
+  when: ((elasticsearch_install_from_package is defined and elasticsearch_install_from_repo.changed) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none
+  tags:
+      - always
 
 - set_fact: list_command="list"
+  tags:
+      - always
 - set_fact: list_command="--list"
   when: es_version | version_compare('2.0', '<')
+  tags:
+      - always
 
 #List currently installed plugins
 - shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-"
@@ -20,7 +27,7 @@
 - name: Remove elasticsearch plugins
   command: "{{es_home}}/bin/plugin remove {{item}} --silent"
   ignore_errors: yes
-  with_items: installed_plugins.stdout_lines
+  with_items: "{{ installed_plugins.stdout_lines }}"
   when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0]
   notify: restart elasticsearch
   environment:
@@ -34,8 +41,8 @@
   register: plugin_installed
   failed_when: "'ERROR' in plugin_installed.stdout"
   changed_when: plugin_installed.rc == 0
-  with_items: es_plugins
-  when: es_plugins is defined and not es_plugins is none
+  with_items: "{{ es_plugins }}"
+  when: es_plugins is defined and not es_plugins is none and es_plugins_reinstall
   notify: restart elasticsearch
   environment:
     CONF_DIR: "{{ conf_dir }}"

tasks/elasticsearch-scripts.yml

@@ -1,9 +1,13 @@
 ---
 
 - set_fact: es_script_dir={{ es_conf_dir }}/{{es_instance_name}}
+  tags:
+      - always
 
 - set_fact: es_script_dir={{es_config['path.scripts']}}
   when: es_config['path.scripts'] is defined
+  tags:
+      - always
 
 - name: Create script dir
   file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }}

tasks/elasticsearch-shield.yml

@@ -0,0 +1,3 @@
+---
+
+

tasks/elasticsearch-templates.yml

@@ -7,13 +7,19 @@
   when: es_templates_fileglob is not defined
 
 - name: Copy templates to elasticsearch
-  copy: src={{ item }} dest=/etc/elasticsearch/ owner={{ es_user }} group={{ es_group }}
-  with_fileglob: es_templates_fileglob
+  copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }}
+  when: es_templates_fileglob is defined
+  with_fileglob:
+    - "{{ es_templates_fileglob }}"
 
 - set_fact: http_port=9200
+  tags:
+      - always
 
 - set_fact: http_port={{es_config['http.port']}}
   when: es_config['http.port'] is defined
+  tags:
+      - always
 
 - name: Wait for elasticsearch to startup
   wait_for: port={{http_port}} delay=10
@@ -24,4 +30,4 @@
 
 - name: Install template(s)
   command: "curl -sL -XPUT http://localhost:{{http_port}}/_template/{{item}} -d @/etc/elasticsearch/templates/{{item}}.json"
-  with_items: resultstemplate.stdout_lines
+  with_items: "{{ resultstemplate.stdout_lines }}"

tasks/elasticsearch.yml

@@ -1,13 +1,27 @@
 ---
 
 - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}}
+  tags:
+      - always
 - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}}
+  tags:
+      - always
 - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}}
+  tags:
+      - always
 - set_fact: plugin_dir={{ es_plugin_dir }}/{{es_instance_name}}
+  tags:
+      - always
 - set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }}
+  tags:
+      - always
 
 - debug: msg="Node configuration {{ es_config }} "
 
+- name: Include optional user and group creation. 
+  when: (es_user_id is defined) and (es_group_id is defined)
+  include: elasticsearch-optional-user.yml
+
 #- name: Include specific Elasticsearch
 #  include: "elasticsearch-{{ansible_os_family}}.yml"
 

tasks/java.yml

@@ -1,8 +1,15 @@
 ---
+
+
+- set_fact: java_state="present"
+
+- set_fact: java_state="latest"
+  when: update_java == true
+
 - name: RedHat - Ensure Java is installed
-  yum: name={{ java }} state=latest
+  yum: name={{ java }} state={{java_state}}
   when: ansible_os_family == 'RedHat'
   
 - name: Debian - Ensure Java is installed
-  apt: name={{ java }} state=present update_cache=yes force=yes
+  apt: name={{ java }} state={{java_state}} update_cache=yes force=yes
   when: ansible_os_family == 'Debian'

tasks/main.yml

@@ -1,16 +1,39 @@
 ---
 - name: check-parameters
   include: checkParameters.yml
+  tags:
+      - check
 - name: os-specific vars
   include_vars: "{{ansible_os_family}}.yml"
+  tags:
+      - always
 - include: java.yml
+  when: es_java_install
+  tags:
+      - java
 - include: elasticsearch.yml
+  tags:
+      - install
 - include: elasticsearch-config.yml
+  tags:
+      - config
 - include: elasticsearch-scripts.yml
   when: es_scripts
+  tags:
+      - scripts
 - include: elasticsearch-plugins.yml
   when: es_plugins is defined or es_plugins_reinstall
+  tags:
+      - plugins
+- include: elasticsearch-shield.yml
+  when: es_install_shield
+  tags:
+      - shield
 - include: elasticsearch-service.yml
+  tags:
+      - service
 - include: elasticsearch-templates.yml
   when: es_templates
-- meta: flush_handlers
+  tags:
+      - templates
+- meta: flush_handlers

templates/systemd/elasticsearch.j2

@@ -31,7 +31,7 @@ StandardOutput=null
 StandardError=journal
 
 # Specifies the maximum file descriptor number that can be opened by this process
-LimitNOFILE=65535
+LimitNOFILE={{es_max_open_files}}
 
 # Specifies the maximum number of bytes of memory that may be locked into RAM
 # Set to "infinity" if you use the 'bootstrap.mlockall: true' option
@@ -53,4 +53,4 @@ SendSIGKILL=no
 SuccessExitStatus=143
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target

test/integration/config.yml

@@ -4,7 +4,7 @@
   hosts: localhost
   roles:
     #expand to all available parameters
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.mlockall: true, discovery.zen.ping.multicast.enabled: false } }
+    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"], es_log_dir: "/opt/elasticsearch/logs", es_work_dir: "/opt/elasticsearch/temp", es_user_id: 333, es_group_id: 333, es_config: {node.name: "node1", cluster.name: "custom-cluster", discovery.zen.ping.unicast.hosts: "localhost:9301", http.port: 9201, transport.tcp.port: 9301, node.data: false, node.master: true, bootstrap.mlockall: true, discovery.zen.ping.multicast.enabled: false } }
   vars:
     es_scripts: false
     es_templates: false

test/integration/helpers/serverspec/config_spec.rb

@@ -5,6 +5,14 @@ shared_examples 'config::init' do |es_version|
   describe user('elasticsearch') do
     it { should exist }
   end
+  
+  describe group('elasticsearch') do
+    it { should have_gid 333 }
+  end
+
+  describe user('elasticsearch') do
+    it { should have_uid 333 }
+  end
 
   describe service('node1_elasticsearch') do
     it { should be_running }

vars/Debian.yml

@@ -1,4 +1,4 @@
 ---
-java: "openjdk-7-jre-headless"
+java: "{{ es_java | default('openjdk-7-jre-headless') }}"
 default_file: "/etc/default/elasticsearch"
 es_home: "/usr/share/elasticsearch"

vars/RedHat.yml

@@ -1,4 +1,4 @@
 ---
-java: "java-1.8.0-openjdk.x86_64"
+java: "{{ es_java | default('java-1.8.0-openjdk.x86_64') }}"
 default_file: "/etc/sysconfig/elasticsearch"
 es_home: "/usr/share/elasticsearch"