diff --git a/tasks/elasticsearch-config.yml b/tasks/elasticsearch-config.yml index 278f094..ac90fdd 100644 --- a/tasks/elasticsearch-config.yml +++ b/tasks/elasticsearch-config.yml @@ -49,6 +49,18 @@ - name: Copy Logging.yml File for Instance template: src=logging.yml.j2 dest={{conf_dir}}/logging.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes notify: restart elasticsearch + when: es_version | version_compare('5.0', '<') + +- name: Copy log4j2.properties File for Instance + template: src=log4j2.properties.j2 dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + notify: restart elasticsearch + when: es_version | version_compare('5.0', '>=') + +# Copy the JVM Options (5.x only) +- name: Copy jvm.options File for Instance + template: src=jvm.options.j2 dest={{conf_dir}}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + notify: restart elasticsearch + when: es_version | version_compare('5.0', '>=') #Clean up un-wanted package scripts to avoid confusion @@ -72,3 +84,8 @@ - name: Delete Default Logging File file: dest=/etc/elasticsearch/logging.yml state=absent +- name: Delete Default Logging File (5.x) + file: dest=/etc/elasticsearch/log4j2.properties state=absent + +- name: Delete Default JVM Options File (5.x) + file: dest=/etc/elasticsearch/jvm.options state=absent diff --git a/tasks/elasticsearch-parameters.yml b/tasks/elasticsearch-parameters.yml index 771be3e..356dc9a 100644 --- a/tasks/elasticsearch-parameters.yml +++ b/tasks/elasticsearch-parameters.yml @@ -18,8 +18,8 @@ when: not multi_cast and es_config['discovery.zen.ping.unicast.hosts'] is not defined #If the user attempts to lock memory they must specify a heap size -- fail: msg="If locking memory with bootstrap.mlockall a heap size must be specified" - when: es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined +- fail: msg="If locking memory with bootstrap.mlockall (or bootstrap.memory_lock) a heap size must be specified" + when: (es_config['bootstrap.mlockall'] is defined or es_config['bootstrap.memory_lock'] is defined) and es_config['bootstrap.mlockall'] == True and es_heap_size is not defined #Don't support xpack on versions < 2.0 - fail: msg="Use of the xpack notation is not supported on versions < 2.0. Marvel-agent and watcher can be installed as plugins. Version > 2.0 is required for shield." @@ -32,7 +32,7 @@ - set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} - set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} - set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} -- set_fact: m_lock_enabled={{ es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True }} +- set_fact: m_lock_enabled={{ (es_config['bootstrap.mlockall'] is defined and es_config['bootstrap.mlockall'] == True) or (es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True) }} #Use systemd for the following distributions: #Ubuntu 15 and up diff --git a/templates/elasticsearch.j2 b/templates/elasticsearch.j2 index e33657f..f099e23 100644 --- a/templates/elasticsearch.j2 +++ b/templates/elasticsearch.j2 @@ -14,17 +14,21 @@ DATA_DIR={{ data_dirs | array_to_str }} # Elasticsearch logs directory LOG_DIR={{log_dir}} +{% if es_version | version_compare('5.0', '<') %} # Elasticsearch work directory WORK_DIR={{work_dir}} +{% endif %} # Elasticsearch PID directory PID_DIR={{pid_dir}} +{% if es_version | version_compare('5.0', '<') %} # Heap size defaults to 256m min, 1g max # Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g {% if es_heap_size is defined %} ES_HEAP_SIZE={{es_heap_size}} {% endif %} +{% endif %} # Heap new generation #ES_HEAP_NEWSIZE= @@ -72,7 +76,7 @@ MAX_OPEN_FILES={{es_max_open_files}} {% endif %} # The maximum number of bytes of memory that may be locked into RAM -# Set to "unlimited" if you use the 'bootstrap.mlockall: true' option +# Set to "unlimited" if you use the 'bootstrap.{% if es_version | version_compare('5.0', '<=') %}memory_lock{% else %}mlockall{% endif %}: true' option # in elasticsearch.yml (ES_HEAP_SIZE must also be set). # When using Systemd, the LimitMEMLOCK property must be set # in /usr/lib/systemd/system/elasticsearch.service diff --git a/templates/elasticsearch.yml.j2 b/templates/elasticsearch.yml.j2 index 40ab354..d6755ae 100644 --- a/templates/elasticsearch.yml.j2 +++ b/templates/elasticsearch.yml.j2 @@ -18,6 +18,8 @@ path.conf: {{ conf_dir }} path.data: {{ data_dirs | array_to_str }} +{% if es_version | version_compare('5.0', '<') %} path.work: {{ work_dir }} +{% endif %} -path.logs: {{ log_dir }} \ No newline at end of file +path.logs: {{ log_dir }} diff --git a/templates/init/debian/elasticsearch.j2 b/templates/init/debian/elasticsearch.j2 index a19153e..68dc12e 100755 --- a/templates/init/debian/elasticsearch.j2 +++ b/templates/init/debian/elasticsearch.j2 @@ -51,9 +51,11 @@ ES_GROUP={{es_group}} # Directory where the Elasticsearch binary distribution resides ES_HOME={{es_home}} +{% if es_version | version_compare('5.0', '<') %} # Heap size defaults to 256m min, 1g max # Set ES_HEAP_SIZE to 50% of available RAM, but no more than 31g #ES_HEAP_SIZE=2g +{% endif %} # Heap new generation #ES_HEAP_NEWSIZE= @@ -104,7 +106,11 @@ fi # Define other required variables PID_FILE="$PID_DIR/$NAME.pid" DAEMON={{es_home}}/bin/elasticsearch +{% if es_version | version_compare('5.0', '<') %} DAEMON_OPTS="-d -p $PID_FILE --default.path.home=$ES_HOME --default.path.logs=$LOG_DIR --default.path.data=$DATA_DIR --default.path.conf=$CONF_DIR" +{% else %} +DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR" +{% endif %} export ES_HEAP_SIZE export ES_HEAP_NEWSIZE @@ -133,10 +139,12 @@ case "$1" in start) checkJava +{% if es_version | version_compare('5.0', '<') %} if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then log_failure_msg "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set" exit 1 fi +{% endif %} log_daemon_msg "Starting $DESC" diff --git a/templates/init/redhat/elasticsearch.j2 b/templates/init/redhat/elasticsearch.j2 index 7925231..bdb1ad7 100755 --- a/templates/init/redhat/elasticsearch.j2 +++ b/templates/init/redhat/elasticsearch.j2 @@ -3,7 +3,7 @@ # elasticsearch # # chkconfig: 2345 80 20 -# description: Starts and stops a single elasticsearch instance on this system +# description: Starts and stops a single elasticsearch instance on this system # ### BEGIN INIT INFO @@ -90,10 +90,12 @@ checkJava() { start() { checkJava [ -x $exec ] || exit 5 +{% if es_version | version_compare('5.0', '<') %} if [ -n "$MAX_LOCKED_MEMORY" -a -z "$ES_HEAP_SIZE" ]; then echo "MAX_LOCKED_MEMORY is set - ES_HEAP_SIZE must also be set" return 7 fi +{% endif %} if [ -n "$MAX_OPEN_FILES" ]; then ulimit -n $MAX_OPEN_FILES fi @@ -116,7 +118,11 @@ start() { cd $ES_HOME echo -n $"Starting $prog: " # if not running, start it up here, usually something like "daemon $exec" +{% if es_version | version_compare('5.0', '<') %} daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Des.default.path.home=$ES_HOME -Des.default.path.logs=$LOG_DIR -Des.default.path.data=$DATA_DIR -Des.default.path.conf=$CONF_DIR +{% else %} + daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.home=$ES_HOME -Edefault.path.logs=$LOG_DIR -Edefault.path.data=${DATA_DIR} -Edefault.path.conf=$CONF_DIR +{% endif %} retval=$? echo [ $retval -eq 0 ] && touch $lockfile diff --git a/templates/jvm.options.j2 b/templates/jvm.options.j2 new file mode 100644 index 0000000..5b9010e --- /dev/null +++ b/templates/jvm.options.j2 @@ -0,0 +1,100 @@ +## JVM configuration + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## You should always set the min and max JVM heap +## size to the same value. For example, to set +## the heap to 4 GB, set: +## +## -Xms4g +## -Xmx4g +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html +## for more information +## +################################################################ + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms{{ es_heap_size }} +-Xmx{{ es_heap_size }} + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +-XX:+UseConcMarkSweepGC +-XX:CMSInitiatingOccupancyFraction=75 +-XX:+UseCMSInitiatingOccupancyOnly + +## optimizations + +# disable calls to System#gc +-XX:+DisableExplicitGC + +# pre-touch memory pages used by the JVM during initialization +-XX:+AlwaysPreTouch + +## basic + +# force the server VM +-server + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +-Djna.nosys=true + +# flags to keep Netty from being unsafe +-Dio.netty.noUnsafe=true +-Dio.netty.noKeySetOptimization=true + +# log4j 2 +-Dlog4j.shutdownHookEnabled=false +-Dlog4j2.disable.jmx=true +-Dlog4j.skipJansi=true + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps +# ensure the directory exists and has sufficient space +#-XX:HeapDumpPath=${heap.dump.path} + +## GC logging + +#-XX:+PrintGCDetails +#-XX:+PrintGCTimeStamps +#-XX:+PrintGCDateStamps +#-XX:+PrintClassHistogram +#-XX:+PrintTenuringDistribution +#-XX:+PrintGCApplicationStoppedTime + +# log GC status to a file with time stamps +# ensure the directory exists +#-Xloggc:${loggc} + +# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. +# If documents were already indexed with unquoted fields in a previous version +# of Elasticsearch, some operations may throw errors. +# +# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided +# only for migration purposes. +#-Delasticsearch.json.allow_unquoted_field_names=true diff --git a/templates/log4j2.properties.j2 b/templates/log4j2.properties.j2 new file mode 100644 index 0000000..9a3147f --- /dev/null +++ b/templates/log4j2.properties.j2 @@ -0,0 +1,74 @@ +status = error + +# log action execution errors for easier debugging +logger.action.name = org.elasticsearch.action +logger.action.level = debug + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs}.log +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n +appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.rolling.ref = rolling + +appender.deprecation_rolling.type = RollingFile +appender.deprecation_rolling.name = deprecation_rolling +appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log +appender.deprecation_rolling.layout.type = PatternLayout +appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.10000m%n +appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz +appender.deprecation_rolling.policies.type = Policies +appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling.policies.size.size = 1GB +appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling.strategy.max = 4 + +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = warn +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.additivity = false + +appender.index_search_slowlog_rolling.type = RollingFile +appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log +appender.index_search_slowlog_rolling.layout.type = PatternLayout +appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n +appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log +appender.index_search_slowlog_rolling.policies.type = Policies +appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.index_search_slowlog_rolling.policies.time.interval = 1 +appender.index_search_slowlog_rolling.policies.time.modulate = true + +logger.index_search_slowlog_rolling.name = index.search.slowlog +logger.index_search_slowlog_rolling.level = trace +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.additivity = false + +appender.index_indexing_slowlog_rolling.type = RollingFile +appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log +appender.index_indexing_slowlog_rolling.layout.type = PatternLayout +appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.10000m%n +appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log +appender.index_indexing_slowlog_rolling.policies.type = Policies +appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling.policies.time.interval = 1 +appender.index_indexing_slowlog_rolling.policies.time.modulate = true + +logger.index_indexing_slowlog.name = index.indexing.slowlog.index +logger.index_indexing_slowlog.level = trace +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.additivity = false diff --git a/templates/systemd/elasticsearch.j2 b/templates/systemd/elasticsearch.j2 index 34d212b..8043360 100644 --- a/templates/systemd/elasticsearch.j2 +++ b/templates/systemd/elasticsearch.j2 @@ -22,11 +22,19 @@ ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec {% endif %} ExecStart={{es_home}}/bin/elasticsearch \ +{% if es_version | version_compare('5.0', '<') %} -Des.pidfile=${PID_DIR}/elasticsearch.pid \ -Des.default.path.home=${ES_HOME} \ -Des.default.path.logs=${LOG_DIR} \ -Des.default.path.data=${DATA_DIR} \ -Des.default.path.conf=${CONF_DIR} +{% else %} + -p ${PID_DIR}/elasticsearch.pid \ + -Edefault.path.home=${ES_HOME} \ + -Edefault.path.logs=${LOG_DIR} \ + -Edefault.path.data=${DATA_DIR} \ + -Edefault.path.conf=${CONF_DIR} +{% endif %} @@ -37,7 +45,7 @@ StandardError=inherit LimitNOFILE={{es_max_open_files}} # Specifies the maximum number of bytes of memory that may be locked into RAM -# Set to "infinity" if you use the 'bootstrap.mlockall: true' option +# Set to "infinity" if you use the 'bootstrap.{% if es_version | version_compare('5.0', '<=') %}memory_lock{% else %}mlockall{% endif %}: true' option # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}} {% if m_lock_enabled %} LimitMEMLOCK=infinity