From 2771eb2a92e8a7f421d57280aef0a01ec87abae5 Mon Sep 17 00:00:00 2001 From: Maarten Bezemer Date: Wed, 2 Nov 2016 14:29:11 +0100 Subject: [PATCH 1/8] Add binary prefix for version 5.x --- tasks/elasticsearch-plugins.yml | 17 +++++++++++------ tasks/xpack/elasticsearch-xpack-install.yml | 8 ++++---- tasks/xpack/elasticsearch-xpack.yml | 8 ++++---- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 14f6a33..67d5517 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -10,8 +10,14 @@ - set_fact: list_command="--list" when: es_version | version_compare('2.0', '<') +- set_fact: es_binary_prefix="" + +- set_fact: es_binary_prefix="elasticsearch-" + when: es_version | version_compare('2.0', '>') + #List currently installed plugins - ignore xpack if > v 2.0 -- shell: "{{es_home}}/bin/plugin {{list_command}} | sed -n '1!p' | cut -d '-' -f2-{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}" +- name: Check installed elasticsearch plugins + shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin {{list_command}}{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}" register: installed_plugins failed_when: "'ERROR' in installed_plugins.stdout" changed_when: False @@ -20,22 +26,21 @@ CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" -#This needs to removes any currently installed plugins +# This removes any currently installed plugins (to prevent errors when reinstalling) - name: Remove elasticsearch plugins - command: "{{es_home}}/bin/plugin remove {{item}} --silent" + command: "{{es_home}}/bin/{{ es_binary_prefix }}plugin remove {{item}} --silent" ignore_errors: yes with_items: "{{ installed_plugins.stdout_lines | default([]) }}" when: es_plugins_reinstall and installed_plugins.stdout_lines | length > 0 and not 'No plugin detected' in installed_plugins.stdout_lines[0] notify: restart elasticsearch - register: plugin_installed + register: plugin_removed environment: CONF_DIR: "{{ conf_dir }}" ES_INCLUDE: "{{ instance_default_file }}" - name: Install elasticsearch plugins - #debug: var=item command: > - {{es_home}}/bin/plugin install {{ item.plugin }}{% if item.version is defined and item.version != '' %}/{{ item.version }}{% endif %} {% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -DproxyHost={{ item.proxy_host }} -DproxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -DproxyHost={{ es_proxy_host }} -DproxyPort={{ es_proxy_port }} {% endif %} --silent + {{es_home}}/bin/{{ es_binary_prefix }}plugin install {{ item.plugin }}{% if item.version is defined and item.version != '' %}/{{ item.version }}{% endif %} {% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -DproxyHost={{ item.proxy_host }} -DproxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -DproxyHost={{ es_proxy_host }} -DproxyPort={{ es_proxy_port }} {% endif %} --silent register: plugin_installed failed_when: "'ERROR' in plugin_installed.stdout" changed_when: plugin_installed.rc == 0 diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml index 77ddc58..9fe3a2d 100644 --- a/tasks/xpack/elasticsearch-xpack-install.yml +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -1,7 +1,7 @@ --- #Test if feature is installed -- shell: "{{es_home}}/bin/plugin list | sed -n '1!p' | grep {{item}}" +- shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin list | sed -n '1!p' | grep {{item}}" register: feature_installed changed_when: False failed_when: "'ERROR' in feature_installed.stdout" @@ -14,7 +14,7 @@ #Remove Plugin if installed and its not been requested or the ES version has changed - name: Remove {{item}} plugin command: > - {{es_home}}/bin/plugin remove shield + {{es_home}}/bin/{{ es_binary_prefix }}plugin remove shield register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 @@ -28,7 +28,7 @@ #Install plugin if not installed, or the es version has changed (so removed above), and its been requested - name: Install {{item}} plugin command: > - {{es_home}}/bin/plugin install {{item}} + {{es_home}}/bin/{{ es_binary_prefix }}plugin install {{item}} register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 @@ -36,4 +36,4 @@ notify: restart elasticsearch environment: CONF_DIR: "{{ conf_dir }}" - ES_INCLUDE: "{{ instance_default_file }}" \ No newline at end of file + ES_INCLUDE: "{{ instance_default_file }}" diff --git a/tasks/xpack/elasticsearch-xpack.yml b/tasks/xpack/elasticsearch-xpack.yml index 46be973..87282fc 100644 --- a/tasks/xpack/elasticsearch-xpack.yml +++ b/tasks/xpack/elasticsearch-xpack.yml @@ -7,7 +7,7 @@ #Check if license is installed - name: Check License is installed shell: > - {{es_home}}/bin/plugin list | tail -n +2 | grep license + {{es_home}}/bin/{{ es_binary_prefix }}plugin list | tail -n +2 | grep license register: license_installed ignore_errors: yes failed_when: "'ERROR' in license_installed.stdout" @@ -19,7 +19,7 @@ #Remove license if installed and xpack not enabled - name: Remove license plugin command: > - {{es_home}}/bin/plugin remove license + {{es_home}}/bin/{{ es_binary_prefix }}plugin remove license register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 @@ -32,7 +32,7 @@ #Install License if not installed, or it needs to be reinstalled due to ES change (above task will have removed), and its been requested. - name: Install license plugin command: > - {{es_home}}/bin/plugin install license + {{es_home}}/bin/{{ es_binary_prefix }}plugin install license register: xpack_state failed_when: "'ERROR' in xpack_state.stdout" changed_when: xpack_state.rc == 0 @@ -52,4 +52,4 @@ #Add any feature specific configuration here - name: Set Plugin Directory Permissions - file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes \ No newline at end of file + file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes From 523270405eff3a02f800988bbac3b567aa15c63b Mon Sep 17 00:00:00 2001 From: Maarten Bezemer Date: Thu, 3 Nov 2016 15:24:17 +0100 Subject: [PATCH 2/8] Add support for Elasticsearch 5.x * Use log4j2.properties instead of logging.yml * Add jvm.options * Use bootstrap.memory_lock instead of bootstrap.mlockall * Get rid of work directory * Executable does not accept `-D` anymore, replace with `-E` and `-p` * Get rid of 5.x default (file)s --- tasks/elasticsearch-config.yml | 17 +++++ tasks/elasticsearch-parameters.yml | 6 +- templates/elasticsearch.j2 | 6 +- templates/elasticsearch.yml.j2 | 2 + templates/init/debian/elasticsearch.j2 | 4 + templates/init/redhat/elasticsearch.j2 | 4 + templates/jvm.options.j2 | 100 +++++++++++++++++++++++++ templates/log4j2.properties.j2 | 74 ++++++++++++++++++ templates/systemd/elasticsearch.j2 | 10 ++- 9 files changed, 218 insertions(+), 5 deletions(-) create mode 100644 templates/jvm.options.j2 create mode 100644 templates/log4j2.properties.j2 diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 34943b4..2b05d39 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -48,6 +48,18 @@ - name: Copy Logging.yml File for Instance template: src=logging.yml.j2 dest={{conf_dir}}/logging.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes notify: restart elasticsearch + when: es_version | version_compare('2.0', '<=') + +- name: Copy log4j2.properties File for Instance + template: src=log4j2.properties.j2 dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + notify: restart elasticsearch + when: es_version | version_compare('2.0', '>') + +# Copy the JVM Options (5.x only) +- name: Copy jvm.options File for Instance + template: src=jvm.options.j2 dest={{conf_dir}}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + notify: restart elasticsearch + when: es_version | version_compare('2.0', '>') #Clean up un-wanted package scripts to avoid confusion @@ -71,3 +83,8 @@ - name: Delete Default Logging File file: dest=/etc/elasticsearch/logging.yml state=absent +- name: Delete Default Logging File (5.x) + file: dest=/etc/elasticsearch/log4j2.properties state=absent + +- name: Delete Default JVM Options File (5.x) + file: dest=/etc/elasticsearch/jvm.options state=absent diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 0f1e8e5..2d5f94c 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -18,8 +18,8 @@ when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined #If the user attempts to lock memory they must specify a heap size -- fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified" - when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined +- fail: msg="If locking memory with bootstrap.mlockall (or bootstrap.memory_lock) a heap size must be specified" + when: (es_config['bootstrap.mlockall'] is defined or es_config['bootstrap.memory_lock'] is defined) and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined #Don't support xpack on versions < 2.0 - fail: msg="Use of the xpack notation is not supported on versions < 2.0. Marvel-agent and watcher can be installed as plugins. Version > 2.0 is required for shield." @@ -32,7 +32,7 @@ - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} -- set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }} +- set_fact: m_lock_enabled={{ (es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True) or (es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True) }} #Use systemd for the following distributions: #Ubuntu 15 and up diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index d934c67..674b170 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -14,17 +14,21 @@ DATA_DIR={{ data_dirs | array_to_str }} # Elasticsearch logs directory LOG_DIR={{log_dir}} +{% if es_version | version_compare('2.0', '<=') %} # Elasticsearch work directory WORK_DIR={{work_dir}} +{% endif %} # Elasticsearch PID directory PID_DIR={{pid_dir}} +{% if es_version | version_compare('2.0', '<=') %} # Heap size defaults to 256m min, 1g max # Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g {% if es_heap_size is defined %} ES_HEAP_SIZE={{es_heap_size}} {% endif %} +{% endif %} # Heap new generation #ES_HEAP_NEWSIZE= @@ -74,7 +78,7 @@ MAX_OPEN_FILES={{es_max_open_files}} {% endif %} # The maximum number of bytes of memory that may be locked into RAM -# Set to "unlimited" if you use the 'bootstrap.mlockall: true' option +# Set to "unlimited" if you use the 'bootstrap.{% if es_version | version_compare('2.0', '>') %}memory_lock{% else %}mlockall{% endif %}: true' option # in elasticsearch.yml (ES_HEAP_SIZE must also be set). # When using Systemd, the LimitMEMLOCK property must be set # in /usr/lib/systemd/system/elasticsearch.service diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index 40ab354..da1e7ae 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -18,6 +18,8 @@ path.conf: {{ conf_dir }} path.data: {{ data_dirs | array_to_str }} +{% if es_version | version_compare('2.0', '<=') %} path.work: {{ work_dir }} +{% endif %} path.logs: {{ log_dir }} \ No newline at end of file diff --git a/templates/init/debian/elasticsearch.j2 b/templates/init/debian/elasticsearch.j2 index a19153e..b29b7da 100755 --- a/templates/init/debian/elasticsearch.j2 +++ b/templates/init/debian/elasticsearch.j2 @@ -104,7 +104,11 @@ fi # Define other required variables PID_FILE="$PID_DIR/$NAME.pid" DAEMON={{es_home}}/bin/elasticsearch +{% if es_version | version_compare('2.0', '<=') %} DAEMON_OPTS="-d -p $PID_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR" +{% else %} +DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR" +{% endif %} export ES_HEAP_SIZE export ES_HEAP_NEWSIZE diff --git a/templates/init/redhat/elasticsearch.j2 b/templates/init/redhat/elasticsearch.j2 index 7925231..4166354 100755 --- a/templates/init/redhat/elasticsearch.j2 +++ b/templates/init/redhat/elasticsearch.j2 @@ -116,7 +116,11 @@ start() { cd $ES_HOME echo -n $"Starting $prog: " # if not running, start it up here, usually something like "daemon $exec" +{% if es_version | version_compare('2.0', '<=') %} daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -Des.default.path.conf=$CONF_DIR +{% else %} + daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=${DATA_DIR} -Edefault.path.conf=$CONF_DIR +{% endif %} retval=$? echo [ $retval -eq 0 ] && touch $lockfile diff --git a/templates/jvm.options.j2 b/templates/jvm.options.j2 new file mode 100644 index 0000000..5b9010e --- /dev/null +++ b/templates/jvm.options.j2 @@ -0,0 +1,100 @@ +## JVM configuration + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## You should always set the min and max JVM heap +## size to the same value. For example, to set +## the heap to 4 GB, set: +## +## -Xms4g +## -Xmx4g +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html +## for more information +## +################################################################ + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms{{ es_heap_size }} +-Xmx{{ es_heap_size }} + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +-XX:+UseConcMarkSweepGC +-XX:CMSInitiatingOccupancyFraction=75 +-XX:+UseCMSInitiatingOccupancyOnly + +## optimizations + +# disable calls to System#gc +-XX:+DisableExplicitGC + +# pre-touch memory pages used by the JVM during initialization +-XX:+AlwaysPreTouch + +## basic + +# force the server VM +-server + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +-Djna.nosys=true + +# flags to keep Netty from being unsafe +-Dio.netty.noUnsafe=true +-Dio.netty.noKeySetOptimization=true + +# log4j 2 +-Dlog4j.shutdownHookEnabled=false +-Dlog4j2.disable.jmx=true +-Dlog4j.skipJansi=true + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps +# ensure the directory exists and has sufficient space +#-XX:HeapDumpPath=${heap.dump.path} + +## GC logging + +#-XX:+PrintGCDetails +#-XX:+PrintGCTimeStamps +#-XX:+PrintGCDateStamps +#-XX:+PrintClassHistogram +#-XX:+PrintTenuringDistribution +#-XX:+PrintGCApplicationStoppedTime + +# log GC status to a file with time stamps +# ensure the directory exists +#-Xloggc:${loggc} + +# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. +# If documents were already indexed with unquoted fields in a previous version +# of Elasticsearch, some operations may throw errors. +# +# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided +# only for migration purposes. +#-Delasticsearch.json.allow_unquoted_field_names=true diff --git a/templates/log4j2.properties.j2 b/templates/log4j2.properties.j2 new file mode 100644 index 0000000..9a3147f --- /dev/null +++ b/templates/log4j2.properties.j2 @@ -0,0 +1,74 @@ +status = error + +# log action execution errors for easier debugging +logger.action.name = org.elasticsearch.action +logger.action.level = debug + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs}.log +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n +appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.rolling.ref = rolling + +appender.deprecation_rolling.type = RollingFile +appender.deprecation_rolling.name = deprecation_rolling +appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log +appender.deprecation_rolling.layout.type = PatternLayout +appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n +appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz +appender.deprecation_rolling.policies.type = Policies +appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling.policies.size.size = 1GB +appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling.strategy.max = 4 + +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = warn +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.additivity = false + +appender.index_search_slowlog_rolling.type = RollingFile +appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log +appender.index_search_slowlog_rolling.layout.type = PatternLayout +appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n +appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log +appender.index_search_slowlog_rolling.policies.type = Policies +appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.index_search_slowlog_rolling.policies.time.interval = 1 +appender.index_search_slowlog_rolling.policies.time.modulate = true + +logger.index_search_slowlog_rolling.name = index.search.slowlog +logger.index_search_slowlog_rolling.level = trace +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.additivity = false + +appender.index_indexing_slowlog_rolling.type = RollingFile +appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log +appender.index_indexing_slowlog_rolling.layout.type = PatternLayout +appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n +appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log +appender.index_indexing_slowlog_rolling.policies.type = Policies +appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling.policies.time.interval = 1 +appender.index_indexing_slowlog_rolling.policies.time.modulate = true + +logger.index_indexing_slowlog.name = index.indexing.slowlog.index +logger.index_indexing_slowlog.level = trace +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.additivity = false diff --git a/templates/systemd/elasticsearch.j2 b/templates/systemd/elasticsearch.j2 index 34d212b..f14b82e 100644 --- a/templates/systemd/elasticsearch.j2 +++ b/templates/systemd/elasticsearch.j2 @@ -22,11 +22,19 @@ ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec {% endif %} ExecStart={{es_home}}/bin/elasticsearch \ +{% if es_version | version_compare('2.0', '<=') %} -Des.pidfile=${PID_DIR}/elasticsearch.pid \ -Des.default.path.home=${ES_HOME} \ -Des.default.path.logs=${LOG_DIR} \ -Des.default.path.data=${DATA_DIR} \ -Des.default.path.conf=${CONF_DIR} +{% else %} + -p ${PID_DIR}/elasticsearch.pid \ + -Edefault.path.home=${ES_HOME} \ + -Edefault.path.logs=${LOG_DIR} \ + -Edefault.path.data=${DATA_DIR} \ + -Edefault.path.conf=${CONF_DIR} +{% endif %} @@ -37,7 +45,7 @@ StandardError=inherit LimitNOFILE={{es_max_open_files}} # Specifies the maximum number of bytes of memory that may be locked into RAM -# Set to "infinity" if you use the 'bootstrap.mlockall: true' option +# Set to "infinity" if you use the 'bootstrap.{% if es_version | version_compare('2.0', '>') %}memory_lock{% else %}mlockall{% endif %}: true' option # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}} {% if m_lock_enabled %} LimitMEMLOCK=infinity From bfb74b96a449d5dbf060bea01afd2772701709c9 Mon Sep 17 00:00:00 2001 From: Maarten Bezemer Date: Fri, 4 Nov 2016 15:55:46 +0100 Subject: [PATCH 3/8] Fix version check for plugin prefix --- tasks/elasticsearch-plugins.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 67d5517..6e80637 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -13,7 +13,7 @@ - set_fact: es_binary_prefix="" - set_fact: es_binary_prefix="elasticsearch-" - when: es_version | version_compare('2.0', '>') + when: es_version | version_compare('5.0', '>=') #List currently installed plugins - ignore xpack if > v 2.0 - name: Check installed elasticsearch plugins From d341a4c0eaec9c39d366bc6c2e6ef64551671396 Mon Sep 17 00:00:00 2001 From: Maarten Bezemer Date: Fri, 4 Nov 2016 15:56:27 +0100 Subject: [PATCH 4/8] Version 2.X does need the filter to parse the plugin output --- tasks/elasticsearch-plugins.yml | 2 +- tasks/xpack/elasticsearch-xpack-install.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index 6e80637..b99cfcc 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -17,7 +17,7 @@ #List currently installed plugins - ignore xpack if > v 2.0 - name: Check installed elasticsearch plugins - shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin {{list_command}}{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}" + shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin {{list_command}}{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %}{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}" register: installed_plugins failed_when: "'ERROR' in installed_plugins.stdout" changed_when: False diff --git a/tasks/xpack/elasticsearch-xpack-install.yml b/tasks/xpack/elasticsearch-xpack-install.yml index 9fe3a2d..ff64c0b 100644 --- a/tasks/xpack/elasticsearch-xpack-install.yml +++ b/tasks/xpack/elasticsearch-xpack-install.yml @@ -1,7 +1,7 @@ --- #Test if feature is installed -- shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin list | sed -n '1!p' | grep {{item}}" +- shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin list{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %} | grep {{item}}" register: feature_installed changed_when: False failed_when: "'ERROR' in feature_installed.stdout" From 65238c74bd115b61efa54d4bfcfeb121eeb14a2e Mon Sep 17 00:00:00 2001 From: Maarten Bezemer Date: Fri, 4 Nov 2016 16:04:31 +0100 Subject: [PATCH 5/8] Fix version_compare usage --- tasks/elasticsearch-config.yml | 6 +++--- templates/elasticsearch.j2 | 6 +++--- templates/elasticsearch.yml.j2 | 4 ++-- templates/init/debian/elasticsearch.j2 | 2 +- templates/init/redhat/elasticsearch.j2 | 4 ++-- templates/systemd/elasticsearch.j2 | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 2b05d39..83586ae 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -48,18 +48,18 @@ - name: Copy Logging.yml File for Instance template: src=logging.yml.j2 dest={{conf_dir}}/logging.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes notify: restart elasticsearch - when: es_version | version_compare('2.0', '<=') + when: es_version | version_compare('5.0', '<') - name: Copy log4j2.properties File for Instance template: src=log4j2.properties.j2 dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes notify: restart elasticsearch - when: es_version | version_compare('2.0', '>') + when: es_version | version_compare('5.0', '>=') # Copy the JVM Options (5.x only) - name: Copy jvm.options File for Instance template: src=jvm.options.j2 dest={{conf_dir}}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes notify: restart elasticsearch - when: es_version | version_compare('2.0', '>') + when: es_version | version_compare('5.0', '>=') #Clean up un-wanted package scripts to avoid confusion diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index 674b170..58eb282 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -14,7 +14,7 @@ DATA_DIR={{ data_dirs | array_to_str }} # Elasticsearch logs directory LOG_DIR={{log_dir}} -{% if es_version | version_compare('2.0', '<=') %} +{% if es_version | version_compare('5.0', '<') %} # Elasticsearch work directory WORK_DIR={{work_dir}} {% endif %} @@ -22,7 +22,7 @@ WORK_DIR={{work_dir}} # Elasticsearch PID directory PID_DIR={{pid_dir}} -{% if es_version | version_compare('2.0', '<=') %} +{% if es_version | version_compare('5.0', '<') %} # Heap size defaults to 256m min, 1g max # Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g {% if es_heap_size is defined %} @@ -78,7 +78,7 @@ MAX_OPEN_FILES={{es_max_open_files}} {% endif %} # The maximum number of bytes of memory that may be locked into RAM -# Set to "unlimited" if you use the 'bootstrap.{% if es_version | version_compare('2.0', '>') %}memory_lock{% else %}mlockall{% endif %}: true' option +# Set to "unlimited" if you use the 'bootstrap.{% if es_version | version_compare('5.0', '<=') %}memory_lock{% else %}mlockall{% endif %}: true' option # in elasticsearch.yml (ES_HEAP_SIZE must also be set). # When using Systemd, the LimitMEMLOCK property must be set # in /usr/lib/systemd/system/elasticsearch.service diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index da1e7ae..d6755ae 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -18,8 +18,8 @@ path.conf: {{ conf_dir }} path.data: {{ data_dirs | array_to_str }} -{% if es_version | version_compare('2.0', '<=') %} +{% if es_version | version_compare('5.0', '<') %} path.work: {{ work_dir }} {% endif %} -path.logs: {{ log_dir }} \ No newline at end of file +path.logs: {{ log_dir }} diff --git a/templates/init/debian/elasticsearch.j2 b/templates/init/debian/elasticsearch.j2 index b29b7da..bc6ed87 100755 --- a/templates/init/debian/elasticsearch.j2 +++ b/templates/init/debian/elasticsearch.j2 @@ -104,7 +104,7 @@ fi # Define other required variables PID_FILE="$PID_DIR/$NAME.pid" DAEMON={{es_home}}/bin/elasticsearch -{% if es_version | version_compare('2.0', '<=') %} +{% if es_version | version_compare('5.0', '<') %} DAEMON_OPTS="-d -p $PID_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR" {% else %} DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR" diff --git a/templates/init/redhat/elasticsearch.j2 b/templates/init/redhat/elasticsearch.j2 index 4166354..91d23a6 100755 --- a/templates/init/redhat/elasticsearch.j2 +++ b/templates/init/redhat/elasticsearch.j2 @@ -3,7 +3,7 @@ # elasticsearch # # chkconfig: 2345 80 20 -# description: Starts and stops a single elasticsearch instance on this system +# description: Starts and stops a single elasticsearch instance on this system # ### BEGIN INIT INFO @@ -116,7 +116,7 @@ start() { cd $ES_HOME echo -n $"Starting $prog: " # if not running, start it up here, usually something like "daemon $exec" -{% if es_version | version_compare('2.0', '<=') %} +{% if es_version | version_compare('5.0', '<') %} daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -Des.default.path.conf=$CONF_DIR {% else %} daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=${DATA_DIR} -Edefault.path.conf=$CONF_DIR diff --git a/templates/systemd/elasticsearch.j2 b/templates/systemd/elasticsearch.j2 index f14b82e..8043360 100644 --- a/templates/systemd/elasticsearch.j2 +++ b/templates/systemd/elasticsearch.j2 @@ -22,7 +22,7 @@ ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec {% endif %} ExecStart={{es_home}}/bin/elasticsearch \ -{% if es_version | version_compare('2.0', '<=') %} +{% if es_version | version_compare('5.0', '<') %} -Des.pidfile=${PID_DIR}/elasticsearch.pid \ -Des.default.path.home=${ES_HOME} \ -Des.default.path.logs=${LOG_DIR} \ @@ -45,7 +45,7 @@ StandardError=inherit LimitNOFILE={{es_max_open_files}} # Specifies the maximum number of bytes of memory that may be locked into RAM -# Set to "infinity" if you use the 'bootstrap.{% if es_version | version_compare('2.0', '>') %}memory_lock{% else %}mlockall{% endif %}: true' option +# Set to "infinity" if you use the 'bootstrap.{% if es_version | version_compare('5.0', '<=') %}memory_lock{% else %}mlockall{% endif %}: true' option # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}} {% if m_lock_enabled %} LimitMEMLOCK=infinity From 8f989c41a75f004e9b7d2d3c3f142b70ada0bf53 Mon Sep 17 00:00:00 2001 From: Maarten Bezemer Date: Mon, 21 Nov 2016 11:31:25 +0100 Subject: [PATCH 6/8] Disable check if ES_HEAP_SIZE exists for ES 5.x ES_HEAP_SIZE is not used for ES5.x anymore, instead ES_JAVA_OPTS 'Xms' and 'Xmx' are used --- templates/init/debian/elasticsearch.j2 | 4 ++++ templates/init/redhat/elasticsearch.j2 | 2 ++ 2 files changed, 6 insertions(+) diff --git a/templates/init/debian/elasticsearch.j2 b/templates/init/debian/elasticsearch.j2 index bc6ed87..68dc12e 100755 --- a/templates/init/debian/elasticsearch.j2 +++ b/templates/init/debian/elasticsearch.j2 @@ -51,9 +51,11 @@ ES_GROUP={{es_group}} # Directory where the Elasticsearch binary distribution resides ES_HOME={{es_home}} +{% if es_version | version_compare('5.0', '<') %} # Heap size defaults to 256m min, 1g max # Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g #ES_HEAP_SIZE=2g +{% endif %} # Heap new generation #ES_HEAP_NEWSIZE= @@ -137,10 +139,12 @@ case "$1" in start) checkJava +{% if es_version | version_compare('5.0', '<') %} if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then log_failure_msg "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set" exit 1 fi +{% endif %} log_daemon_msg "Starting $DESC" diff --git a/templates/init/redhat/elasticsearch.j2 b/templates/init/redhat/elasticsearch.j2 index 91d23a6..bdb1ad7 100755 --- a/templates/init/redhat/elasticsearch.j2 +++ b/templates/init/redhat/elasticsearch.j2 @@ -90,10 +90,12 @@ checkJava() { start() { checkJava [ -x $exec ] || exit 5 +{% if es_version | version_compare('5.0', '<') %} if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then echo "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set" return 7 fi +{% endif %} if [ -n "$MAX_OPEN_FILES" ]; then ulimit -n $MAX_OPEN_FILES fi From 49dfa81684a351b2259fcfa747366136217842b9 Mon Sep 17 00:00:00 2001 From: Maarten Bezemer Date: Wed, 7 Dec 2016 22:54:33 +0100 Subject: [PATCH 7/8] Make prefix available when es_plugins is not defined --- tasks/elasticsearch-plugins.yml | 5 ----- tasks/elasticsearch.yml | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tasks/elasticsearch-plugins.yml b/tasks/elasticsearch-plugins.yml index b99cfcc..ae53943 100644 --- a/tasks/elasticsearch-plugins.yml +++ b/tasks/elasticsearch-plugins.yml @@ -10,11 +10,6 @@ - set_fact: list_command="--list" when: es_version | version_compare('2.0', '<') -- set_fact: es_binary_prefix="" - -- set_fact: es_binary_prefix="elasticsearch-" - when: es_version | version_compare('5.0', '>=') - #List currently installed plugins - ignore xpack if > v 2.0 - name: Check installed elasticsearch plugins shell: "{{es_home}}/bin/{{ es_binary_prefix }}plugin {{list_command}}{% if es_version | version_compare('5.0', '<') %} | sed -n '1!p' | cut -d '-' -f2-{% endif %}{% if es_version | version_compare('2.0', '>') %} | grep -vE '{{supported_xpack_features | join('|')}}|license'{% endif %}" diff --git a/tasks/elasticsearch.yml b/tasks/elasticsearch.yml index 7addb03..71ad9e1 100644 --- a/tasks/elasticsearch.yml +++ b/tasks/elasticsearch.yml @@ -1,5 +1,10 @@ --- +- set_fact: es_binary_prefix="" + +- set_fact: es_binary_prefix="elasticsearch-" + when: es_version | version_compare('5.0', '>=') + - name: Include optional user and group creation. when: (es_user_id is defined) and (es_group_id is defined) include: elasticsearch-optional-user.yml From d8d78f33753845eca94d120501a54e57d34bd45c Mon Sep 17 00:00:00 2001 From: Dale McDiarmid Date: Fri, 6 Jan 2017 11:03:12 +0000 Subject: [PATCH 8/8] Readme fix for 5.x --- README.md | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index aa18750..2d93f81 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,14 @@ # ansible-elasticsearch -Ansible role for Elasticsearch. Currently this works on Debian and RedHat based linux systems. Tested platforms are: +THIS ROLE IS UNDER DEVELOPMENT FOR 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH. -* Ubuntu 14.04 +Ansible role for 5.x Elasticsearch. Currently this works on Debian and RedHat based linux systems. Tested platforms are: + +* Ubuntu 14.04/16.04 * Debian 8 * Centos 7 -The latest Elasticsearch versions of 2.x are actively tested. **Only Ansible versions > 2.1.2 are supported.**. 5.x will be available shortly. +The latest Elasticsearch versions of 5.x are actively tested. **Only Ansible versions > 2.2.0 are supported.**. ## Usage @@ -73,9 +75,7 @@ The following illustrates applying configuration parameters to an Elasticsearch es_heap_size: 1g ``` ` -The role utilises Elasticsearch version defaults. Multicast is therefore enabled for 1.x (legacy) and disabled for 2.x (plugin required in 2.x). If using 1.x it is strongly recommended you disable -multicast and specify the required uni-cast settings for a production environment. -When not utilizing multicast, the following should be set to ensure a successful cluster forms. +The role utilises Elasticsearch version defaults. Multicast is therefore disabled for 5.x. The following should be set to ensure a successful cluster forms. * ```es_config['http.port']``` - the http port for the node * ```es_config['transport.tcp.port']``` - the transport port for the node @@ -308,8 +308,8 @@ Additional parameters to es_config allow the customization of the Java and Elast Following variables affect the versions installed: -* ```es_major_version``` (e.g. "2.4" ). Should be consistent with es_version. For versions >= 2.0 this must be "2.x". -* ```es_version``` (e.g. "2.4.2"). +* ```es_major_version``` (e.g. "5.1" ). Should be consistent with es_version. For versions >= 5.0 this must be "5.x". +* ```es_version``` (e.g. "5.1.1"). * ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". * ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. * ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Shield is enabled. Ensure this user is admin. @@ -331,10 +331,7 @@ es_java_opts: - "-Djava.io.tmpdir=/data/tmp/elasticsearch" ``` -Earlier examples illustrate the installation of plugins for 2.x. The correct use of this parameter varies depending on the version of Elasticsearch being installed: - - - 2.x. - For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. - For community based plugins include the full path e.g. "lmenezes/elasticsearch-kopf" and the appropriate version for the target version of Elasticsearch. This approach should NOT be used for X-Pack related plugins e.g. Shield. See X-Pack below for details here. +Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full path e.g. "lmenezes/elasticsearch-kopf" and the appropriate version for the target version of Elasticsearch. This approach should NOT be used for X-Pack related plugins e.g. Shield. See X-Pack below for details here. If installing Marvel or Watcher, ensure the license plugin is also specified. Shield configuration is currently not supported but planned for later versions. @@ -383,15 +380,16 @@ To define proxy only for a particular plugin during its installation: * The role assumes the user/group exists on the server. The elasticsearch packages create the default elasticsearch user. If this needs to be changed, ensure the user exists. * The playbook relies on the inventory_name of each host to ensure its directories are unique * Changing an instance_name for a role application will result in the installation of a new component. The previous component will remain. -* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test only the latest version of 2.x on +* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test only the latest version of 5.x on all supported platforms. * The role aims to be idempotent. Running the role multiple times, with no changes, should result in no state change on the server. If the configuration is changed, these will be applied and Elasticsearch restarted where required. * Systemd is used for Ubuntu versions >= 15, Debian >=8, Centos >=7. All other versions use init for service scripts. +* In order to run x-pack tests a license file with shield enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests. ## IMPORTANT NOTES RE PLUGIN MANAGEMENT -* If the ES version is changed, all plugins will be removed. Those listed in the playbook will be re-installed. This is behaviour is required in ES 2.x. +* If the ES version is changed, all plugins will be removed. Those listed in the playbook will be re-installed. This is behaviour is required in ES 5.x. * If no plugins are listed in the playbook for a node, all currently installed plugins will be removed. * The role does not currently support automatic detection of differences between installed and listed plugins (other than if none are listed). Should users wish to change installed plugins should set es_plugins_reinstall to true. This will cause all currently installed plugins to be removed and those listed to be installed. Change detection will be implemented in future releases.