diff --git a/.gitignore b/.gitignore index 9f43345..ff93046 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ Converging TODO .idea/ elasticsearch.iml +!/vars/RedHat.yml diff --git a/.kitchen.yml b/.kitchen.yml index 3727dd4..44d127f 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -66,10 +66,9 @@ platforms: - sed -ri 's/^#?UsePAM .*/UsePAM no/' /etc/ssh/sshd_config - rm /etc/yum.repos.d/epel*repo /etc/yum.repos.d/puppetlabs-pc1.repo - yum -y install initscripts - - yum clean all - - pip install --upgrade pip - - pip install jmespath - yum -y remove ansible + - yum clean all + - pip install jmespath volume: <%=ENV['ES_XPACK_LICENSE_FILE']%>:/tmp/license.json run_command: "/usr/sbin/init" privileged: true diff --git a/defaults/main.yml b/defaults/main.yml index 59f01af..f264ae1 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,8 +1,9 @@ --- es_major_version: "5.x" -es_version: "5.2.2" +es_version: "5.5.1" es_version_lock: false es_use_repository: true +es_templates_fileglob: "files/templates/*.json" es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch" es_apt_url: "deb https://artifacts.elastic.co/packages/{{ es_major_version }}/apt stable main" es_apt_url_old: "deb http://packages.elastic.co/elasticsearch/{{ es_major_version }}/debian stable main" diff --git a/handlers/elasticsearch-templates.yml b/handlers/elasticsearch-templates.yml deleted file mode 100644 index b1ff63a..0000000 --- a/handlers/elasticsearch-templates.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- - -- name: Ensure elasticsearch is started - service: name={{instance_init_script | basename}} state=started enabled=yes - -- name: Wait for elasticsearch to startup - wait_for: host={{es_api_host}} port={{es_api_port}} delay=10 - -- name: Get template files - find: paths="/etc/elasticsearch/templates" patterns="*.json" - register: templates - -- name: Install templates without auth - uri: - url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item.path | filename}}" - method: PUT - status_code: 200 - body_format: json - body: "{{ lookup('file', item.path) }}" - when: not es_enable_xpack or not es_xpack_features is defined or "security" not in es_xpack_features - with_items: "{{ templates.files }}" - -- name: Install templates with auth - uri: - url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item.path | filename}}" - method: PUT - status_code: 200 - user: "{{es_api_basic_auth_username}}" - password: "{{es_api_basic_auth_password}}" - force_basic_auth: yes - body_format: json - body: "{{ lookup('file', item.path) }}" - when: es_enable_xpack and es_xpack_features is defined and "security" in es_xpack_features - with_items: "{{ templates.files }}" diff --git a/handlers/main.yml b/handlers/main.yml index e233aae..d44c24d 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -1,19 +1,13 @@ + - name: reload systemd configuration command: systemctl daemon-reload # Restart service and ensure it is enabled + - name: restart elasticsearch service: name={{instance_init_script | basename}} state=restarted enabled=yes - when: - - es_restart_on_change - - es_start_service + when: + - es_restart_on_change + - es_start_service - ((plugin_installed is defined and plugin_installed.changed) or (config_updated is defined and config_updated.changed) or (xpack_state.changed) or (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed)) register: es_restarted - -#Templates are a handler as they need to come after a restart e.g. suppose user removes security on a running node and doesn't -#specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed if we don't wait for the node to restart. -#Templates done after restart therefore - as a handler. - -- name: load-templates - include: ./handlers/elasticsearch-templates.yml - when: es_templates diff --git a/meta/main.yml b/meta/main.yml index edd7295..aeecec7 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -7,8 +7,7 @@ galaxy_info: description: Elasticsearch for Linux company: "Elastic.co" license: "license (Apache)" - # Require 1.6 for apt deb install - min_ansible_version: 2.2.0 + min_ansible_version: 2.3.2 platforms: - name: EL versions: diff --git a/tasks/elasticsearch-template.yml b/tasks/elasticsearch-template.yml new file mode 100644 index 0000000..e524043 --- /dev/null +++ b/tasks/elasticsearch-template.yml @@ -0,0 +1,45 @@ +--- + +- file: path=/etc/elasticsearch/templates state=directory owner={{ es_user }} group={{ es_group }} + +- name: Copy templates to elasticsearch + copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }} + register: load_templates + with_fileglob: + - "{{ es_templates_fileglob | default('') }}" + + +- name: Ensure elasticsearch is started + service: name={{instance_init_script | basename}} state=started enabled=yes + when: es_start_service and load_templates.changed + +- name: Wait for elasticsearch to startup + wait_for: host={{es_api_host}} port={{es_api_port}} delay=10 + when: es_start_service and load_templates.changed + +- name: Install templates without auth + uri: + url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}" + method: PUT + status_code: 200 + body_format: json + body: "{{ lookup('file', item) }}" + when: load_templates.changed and es_start_service and not es_enable_xpack or not es_xpack_features is defined or "security" not in es_xpack_features + with_fileglob: + - "{{ es_templates_fileglob | default('') }}" + run_once: True + +- name: Install templates with auth + uri: + url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}" + method: PUT + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + body_format: json + body: "{{ lookup('file', item) }}" + when: load_templates.changed and es_start_service and es_enable_xpack and es_xpack_features is defined and "security" in es_xpack_features + with_fileglob: + - "{{ es_templates_fileglob | default('') }}" + run_once: True diff --git a/tasks/elasticsearch-templates.yml b/tasks/elasticsearch-templates.yml deleted file mode 100644 index 27c0c42..0000000 --- a/tasks/elasticsearch-templates.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- - -- file: path=/etc/elasticsearch/templates state=directory owner={{ es_user }} group={{ es_group }} - -- name: Copy default templates to elasticsearch - copy: src=templates dest=/etc/elasticsearch/ owner={{ es_user }} group={{ es_group }} - notify: load-templates - when: es_templates_fileglob is not defined - -- name: Copy templates to elasticsearch - copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }} - notify: load-templates - with_fileglob: - - "{{ es_templates_fileglob | default('') }}" \ No newline at end of file diff --git a/tasks/main.yml b/tasks/main.yml index 19d9633..5b3953d 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -37,24 +37,26 @@ tags: - xpack -- include: elasticsearch-templates.yml +- meta: flush_handlers + +#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password. The templates will subsequently not be removed if we don't wait for the node to restart. +- include: elasticsearch-template.yml when: es_templates tags: - templates -- meta: flush_handlers - - name: Make sure elasticsearch is started service: name={{instance_init_script | basename}} state=started enabled=yes + when: es_start_service - name: Wait for elasticsearch to startup wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1 - when: es_restarted is defined and es_restarted.changed + when: es_restarted is defined and es_restarted.changed and es_start_service - name: activate-license include: ./xpack/security/elasticsearch-xpack-activation.yml - when: es_enable_xpack and es_xpack_license is defined and es_xpack_license != '' + when: es_start_service and es_enable_xpack and es_xpack_license is defined and es_xpack_license != '' #perform security actions here now elasticsearch is started - include: ./xpack/security/elasticsearch-security-native.yml - when: (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)) + when: es_start_service and (es_enable_xpack and '"security" in es_xpack_features') and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)) diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index 0c7f4a6..cb2341a 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -5,6 +5,9 @@ # Elasticsearch home directory ES_HOME={{es_home}} +# Elasticsearch Java path +#JAVA_HOME= + # Elasticsearch configuration directory CONF_DIR={{conf_dir}} @@ -56,7 +59,7 @@ MAX_OPEN_FILES={{es_max_open_files}} # The maximum number of bytes of memory that may be locked into RAM # Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option -# in elasticsearch.yml (ES_HEAP_SIZE must also be set). +# in elasticsearch.yml # When using Systemd, the LimitMEMLOCK property must be set # in /usr/lib/systemd/system/elasticsearch.service #MAX_LOCKED_MEMORY= diff --git a/templates/elasticsearch.repo b/templates/elasticsearch.repo index 562f74e..b629904 100644 --- a/templates/elasticsearch.repo +++ b/templates/elasticsearch.repo @@ -4,6 +4,8 @@ baseurl=https://artifacts.elastic.co/packages/{{ es_major_version }}/yum gpgcheck=1 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch enabled=1 +autorefresh=1 +type=rpm-md {% if es_proxy_host is defined and es_proxy_host != '' and es_proxy_port is defined %} proxy=http://{{ es_proxy_host }}:{{es_proxy_port}} {% endif %} diff --git a/templates/init/debian/elasticsearch.j2 b/templates/init/debian/elasticsearch.j2 index 5a21e47..64c2f0a 100755 --- a/templates/init/debian/elasticsearch.j2 +++ b/templates/init/debian/elasticsearch.j2 @@ -84,16 +84,30 @@ if [ ! -z "$CONF_FILE" ]; then exit 1 fi +if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then + echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]" +fi + # Define other required variables PID_FILE="$PID_DIR/$NAME.pid" -DAEMON={{es_home}}/bin/elasticsearch -DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR" +DAEMON=$ES_HOME/bin/elasticsearch +DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR" export ES_JAVA_OPTS export JAVA_HOME export ES_INCLUDE export ES_JVM_OPTIONS +# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported +if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi +if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi +if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi +if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi +if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi +if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi +if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi +if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi + # Check DAEMON exists if [ ! -x "$DAEMON" ]; then echo "The elasticsearch startup script does not exists or it is not executable, tried: $DAEMON" @@ -117,13 +131,6 @@ case "$1" in start) checkJava -{% if es_version | version_compare('5.0', '<') %} - if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then - log_failure_msg "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set" - exit 1 - fi -{% endif %} - log_daemon_msg "Starting $DESC" pid=`pidofproc -p $PID_FILE elasticsearch` @@ -133,9 +140,6 @@ case "$1" in exit 0 fi - # Prepare environment - mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR" - # Ensure that the PID_DIR exists (it is cleaned at OS startup time) if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR" @@ -157,7 +161,7 @@ case "$1" in fi # Start Daemon - start-stop-daemon -d $ES_HOME --start -b --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS + start-stop-daemon -d $ES_HOME --start --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS return=$? if [ $return -eq 0 ]; then i=0 @@ -203,7 +207,6 @@ case "$1" in restart|force-reload) if [ -f "$PID_FILE" ]; then $0 stop - sleep 1 fi $0 start ;; diff --git a/templates/init/redhat/elasticsearch.j2 b/templates/init/redhat/elasticsearch.j2 index f906074..e093a85 100755 --- a/templates/init/redhat/elasticsearch.j2 +++ b/templates/init/redhat/elasticsearch.j2 @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # elasticsearch # @@ -48,7 +48,6 @@ DATA_DIR={{ data_dirs | array_to_str }} CONF_DIR="{{conf_dir}}" PID_DIR="{{pid_dir}}" -ES_JVM_OPTIONS="{{conf_dir}}/jvm.options" # Source the default env file ES_ENV_FILE="{{instance_default_file}}" @@ -56,6 +55,10 @@ if [ -f "$ES_ENV_FILE" ]; then . "$ES_ENV_FILE" fi +if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then + echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]" +fi + # CONF_FILE setting was removed if [ ! -z "$CONF_FILE" ]; then echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed." diff --git a/templates/jvm.options.j2 b/templates/jvm.options.j2 index 0cf7394..ad30851 100644 --- a/templates/jvm.options.j2 +++ b/templates/jvm.options.j2 @@ -20,13 +20,9 @@ # Xmx represents the maximum size of total heap space {% if es_heap_size is defined %} -Xms{{ es_heap_size }} -{% else %} --Xms2g -{% endif %} - -{% if es_heap_size is defined %} -Xmx{{ es_heap_size }} {% else %} +-Xms2g -Xmx2g {% endif %} @@ -47,9 +43,6 @@ ## optimizations -# disable calls to System#gc --XX:+DisableExplicitGC - # pre-touch memory pages used by the JVM during initialization -XX:+AlwaysPreTouch @@ -67,7 +60,10 @@ # use our provided JNA always versus the system one -Djna.nosys=true -# flags to keep Netty from being unsafe +# use old-style file permissions on JDK9 +-Djdk.io.permissionsUseCanonicalPath=true + +# flags to configure Netty -Dio.netty.noUnsafe=true -Dio.netty.noKeySetOptimization=true -Dio.netty.recycler.maxCapacityPerThread=0 @@ -100,6 +96,14 @@ # ensure the directory exists #-Xloggc:${loggc} + +# By default, the GC log file will not rotate. +# By uncommenting the lines below, the GC log file +# will be rotated every 128MB at most 32 times. +#-XX:+UseGCLogFileRotation +#-XX:NumberOfGCLogFiles=32 +#-XX:GCLogFileSize=128M + # Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. # If documents were already indexed with unquoted fields in a previous version # of Elasticsearch, some operations may throw errors. diff --git a/templates/systemd/elasticsearch.j2 b/templates/systemd/elasticsearch.j2 index dafae58..6473fa3 100644 --- a/templates/systemd/elasticsearch.j2 +++ b/templates/systemd/elasticsearch.j2 @@ -41,6 +41,9 @@ StandardError=inherit LimitNOFILE={{es_max_open_files}} {% endif %} +# Specifies the maximum number of processes +LimitNPROC=2048 + # Specifies the maximum number of bytes of memory that may be locked into RAM # Set to "infinity" if you use the 'bootstrap.memory_lock: true' option # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}} @@ -54,6 +57,9 @@ TimeoutStopSec=0 # SIGTERM signal is used to stop the Java process KillSignal=SIGTERM +# Send the signal only to the JVM rather than its control group +KillMode=process + # Java process is never killed SendSIGKILL=no diff --git a/test/integration/config-5x/serverspec/default_spec.rb b/test/integration/config-5x/serverspec/default_spec.rb index 785614b..ade4b54 100644 --- a/test/integration/config-5x/serverspec/default_spec.rb +++ b/test/integration/config-5x/serverspec/default_spec.rb @@ -1,6 +1,6 @@ require 'config_spec' describe 'Config Tests v 5.x' do - include_examples 'config::init', "5.2.2", ["ingest-attachment","ingest-user-agent"] + include_examples 'config::init', "5.5.1", ["ingest-attachment","ingest-user-agent"] end diff --git a/test/integration/multi-5x/serverspec/default_spec.rb b/test/integration/multi-5x/serverspec/default_spec.rb index 7020270..fcf6ee5 100644 --- a/test/integration/multi-5x/serverspec/default_spec.rb +++ b/test/integration/multi-5x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'multi_spec' describe 'Multi Tests v 5.x' do - include_examples 'multi::init', "5.2.2", ["ingest-geoip"] + include_examples 'multi::init', "5.5.1", ["ingest-geoip"] end diff --git a/test/integration/package-5x/serverspec/default_spec.rb b/test/integration/package-5x/serverspec/default_spec.rb index 6c553ae..225541a 100644 --- a/test/integration/package-5x/serverspec/default_spec.rb +++ b/test/integration/package-5x/serverspec/default_spec.rb @@ -2,5 +2,5 @@ require 'package_spec' describe 'Package Tests v 5.x' do - include_examples 'package::init', "5.2.2", ["ingest-attachment","ingest-geoip"] + include_examples 'package::init', "5.5.1", ["ingest-attachment","ingest-geoip"] end \ No newline at end of file diff --git a/test/integration/package.yml b/test/integration/package.yml index 4cfd73c..77168c4 100644 --- a/test/integration/package.yml +++ b/test/integration/package.yml @@ -8,7 +8,7 @@ es_templates: true es_heap_size: "1g" es_api_port: 9200 - es_version: "5.1.2" + es_version: "5.5.1" es_plugins: - plugin: ingest-geoip @@ -21,7 +21,7 @@ vars: es_scripts: true es_templates: true - es_version: "5.2.2" + es_version: "5.5.1" es_heap_size: "1g" es_api_port: 9200 es_plugins: diff --git a/test/integration/standard-5x/serverspec/default_spec.rb b/test/integration/standard-5x/serverspec/default_spec.rb index f219dfc..729b306 100644 --- a/test/integration/standard-5x/serverspec/default_spec.rb +++ b/test/integration/standard-5x/serverspec/default_spec.rb @@ -2,7 +2,7 @@ require 'standard_spec' describe 'Standard Tests v 5.x' do - include_examples 'standard::init', "5.2.2", ["ingest-geoip"] + include_examples 'standard::init', "5.5.1", ["ingest-geoip"] end diff --git a/test/integration/xpack-5x/serverspec/default_spec.rb b/test/integration/xpack-5x/serverspec/default_spec.rb index aa7e697..ab0946b 100644 --- a/test/integration/xpack-5x/serverspec/default_spec.rb +++ b/test/integration/xpack-5x/serverspec/default_spec.rb @@ -1,5 +1,5 @@ require 'xpack_spec' describe 'Xpack Tests v 5.x' do - include_examples 'xpack::init', "5.2.2", ["ingest-attachment"] + include_examples 'xpack::init', "5.5.1", ["ingest-attachment"] end