Merge pull request #637 from jmlrt/use-default-config-files

Use default config files

README.md

@@ -405,7 +405,6 @@ In addition to es_config, the following parameters allow the customization of th
 * ```update_java``` Updates Java to the latest version. (true or false (default))
 * ```es_max_map_count``` maximum number of VMA (Virtual Memory Areas) a process can own. Defaults to 262144.
 * ```es_max_open_files``` the maximum file descriptor number that can be opened by this process. Defaults to 65536.
-* ```es_max_threads``` the maximum number of threads the process can start. Defaults to 2048 (the minimum required by elasticsearch).
 * ```es_debian_startup_timeout``` how long Debian-family SysV init scripts wait for the service to start, in seconds. Defaults to 10 seconds.
 * ```es_use_repository``` Setting this to `false` will stop Ansible from using the official Elastic package from any repository configured on the system.
 * ```es_add_repository``` Setting this to `false` will stop Ansible to add the official Elastic package repositories (if es_use_repository is true) if you want to use a repo already present.

defaults/main.yml

@@ -20,7 +20,9 @@ es_templates: false
 es_user: elasticsearch
 es_group: elasticsearch
 es_config: {}
-es_config_log4j2: log4j2.properties.j2
+es_config_default: "elasticsearch.j2"
+es_config_log4j2: ""
+es_config_jvm: "jvm.options.j2"
 #Need to provide default directories
 es_conf_dir: "/etc/elasticsearch"
 es_pid_dir: "/var/run/elasticsearch"
@@ -29,7 +31,6 @@ es_data_dirs:
 es_log_dir: "/var/log/elasticsearch"
 es_action_auto_create_index: true
 es_max_open_files: 65536
-es_max_threads: 8192
 es_max_map_count: 262144
 es_allow_downgrades: false
 es_xpack_features: []

files/logging/log4j2.properties.custom.j2

@@ -1,76 +0,0 @@
-#CUSTOM LOG4J FILE
-
-status = error
-
-# log action execution errors for easier debugging
-logger.action.name = org.elasticsearch.action
-logger.action.level = info
-
-appender.console.type = Console
-appender.console.name = console
-appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
-
-appender.rolling.type = RollingFile
-appender.rolling.name = rolling
-appender.rolling.fileName = ${sys:es.logs}.log
-appender.rolling.layout.type = PatternLayout
-appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log
-appender.rolling.policies.type = Policies
-appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.rolling.policies.time.interval = 1
-appender.rolling.policies.time.modulate = true
-
-rootLogger.level = debug
-rootLogger.appenderRef.console.ref = console
-rootLogger.appenderRef.rolling.ref = rolling
-
-appender.deprecation_rolling.type = RollingFile
-appender.deprecation_rolling.name = deprecation_rolling
-appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log
-appender.deprecation_rolling.layout.type = PatternLayout
-appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz
-appender.deprecation_rolling.policies.type = Policies
-appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
-appender.deprecation_rolling.policies.size.size = 10mb
-appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
-appender.deprecation_rolling.strategy.max = 4
-
-logger.deprecation.name = org.elasticsearch.deprecation
-logger.deprecation.level = debug
-logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
-logger.deprecation.additivity = false
-
-appender.index_search_slowlog_rolling.type = RollingFile
-appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
-appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log
-appender.index_search_slowlog_rolling.layout.type = PatternLayout
-appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
-appender.index_search_slowlog_rolling.policies.type = Policies
-appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.index_search_slowlog_rolling.policies.time.interval = 1
-appender.index_search_slowlog_rolling.policies.time.modulate = true
-
-logger.index_search_slowlog_rolling.name = index.search.slowlog
-logger.index_search_slowlog_rolling.level = debug
-logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
-logger.index_search_slowlog_rolling.additivity = false
-
-appender.index_indexing_slowlog_rolling.type = RollingFile
-appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
-appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log
-appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
-appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
-appender.index_indexing_slowlog_rolling.policies.type = Policies
-appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.index_indexing_slowlog_rolling.policies.time.interval = 1
-appender.index_indexing_slowlog_rolling.policies.time.modulate = true
-
-logger.index_indexing_slowlog.name = index.indexing.slowlog.index
-logger.index_indexing_slowlog.level = debug
-logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
-logger.index_indexing_slowlog.additivity = false

tasks/elasticsearch-config.yml

@@ -4,17 +4,32 @@
 #Create conf directory
 - name: Create Configuration Directory
   become: yes
-  file: path={{ es_conf_dir }} state=directory owner=root group={{ es_group }} mode=2750
+  file:
+    path: "{{ es_conf_dir }}"
+    state: directory
+    owner: root
+    group: "{{ es_group }}"
+    mode: "2750"
 
 #Create pid directory
 - name: Create PID Directory
   become: yes
-  file: path={{ es_pid_dir }} state=directory owner={{ es_user }} group={{ es_group }} mode=0755
+  file:
+    path: "{{ es_pid_dir }}"
+    state: directory
+    owner: "{{ es_user }}"
+    group: "{{ es_group }}"
+    mode: "755"
 
 #Create required directories
 - name: Create Others Directories
   become: yes
-  file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} mode=2750
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ es_user }}"
+    group: "{{ es_group }}"
+    mode: "2750"
   with_items:
     - "{{ es_log_dir }}"
     - "{{ es_data_dirs }}"
@@ -22,14 +37,26 @@
 #Copy the config template
 - name: Copy Configuration File
   become: yes
-  template: src=elasticsearch.yml.j2 dest={{ es_conf_dir }}/elasticsearch.yml owner=root group={{ es_group }} mode=0660 force=yes
+  template:
+    src: elasticsearch.yml.j2
+    dest: "{{ es_conf_dir }}/elasticsearch.yml"
+    owner: root
+    group: "{{ es_group }}"
+    mode: "660"
+    force: yes
   register: system_change
   notify: restart elasticsearch
 
 #Copy the default file
 - name: Copy Default File
   become: yes
-  template: src=elasticsearch.j2 dest={{ default_file }} owner=root group={{ es_group }} mode=0660 force=yes
+  template:
+    src: "{{ es_config_default }}"
+    dest: "{{ default_file }}"
+    owner: root
+    group: "{{ es_group }}"
+    mode: "660"
+    force: yes
   notify: restart elasticsearch
 
 #Copy the systemd specific file if systemd is installed
@@ -37,21 +64,41 @@
   become: yes
   block:
     - name: Make sure destination dir exists
-      file: path={{ sysd_config_file | dirname }} state=directory mode=0755
+      file:
+        path: "{{ sysd_config_file | dirname }}"
+        state: directory
+        mode: "755"
 
     - name: Copy specific ElasticSearch Systemd config file
-      ini_file: path={{ sysd_config_file }} section=Service option=LimitMEMLOCK value=infinity mode=0644
+      ini_file:
+        path: "{{ sysd_config_file }}"
+        section: Service
+        option: LimitMEMLOCK
+        value: infinity
+        mode: "644"
       notify:
        - reload systemd configuration
        - restart elasticsearch
 
-#Copy the logging.yml
-- name: Copy log4j2.properties File
-  become: yes
-  template: src={{ es_config_log4j2 }} dest={{ es_conf_dir }}/log4j2.properties owner=root group={{ es_group }} mode=0660 force=yes
-  notify: restart elasticsearch
-
 - name: Copy jvm.options File
   become: yes
-  template: src=jvm.options.j2 dest={{ es_conf_dir }}/jvm.options owner=root group={{ es_group }} mode=0660 force=yes
+  template:
+    src: "{{ es_config_jvm }}"
+    dest: "{{ es_conf_dir }}/jvm.options"
+    owner: root
+    group: "{{ es_group }}"
+    mode: "660"
+    force: yes
   notify: restart elasticsearch
+
+- name: Copy log4j2.properties File
+  become: yes
+  template:
+    src: "{{ es_config_log4j2 }}"
+    dest: "{{ es_conf_dir }}/log4j2.properties"
+    owner: root
+    group: "{{ es_group }}"
+    mode: "660"
+    force: yes
+  notify: restart elasticsearch
+  when: es_config_log4j2 != ''

tasks/elasticsearch-ssl.yml

@@ -13,7 +13,7 @@
     state: directory
     owner: root
     group: "{{ es_group }}"
-    mode: 0750
+    mode: "750"
 
 - name: Upload SSL/TLS keystore
   copy:
@@ -21,7 +21,7 @@
     dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_keystore | basename }}"
     owner: "{{ es_user }}"
     group: "{{ es_group }}"
-    mode: 0640
+    mode: "640"
   when: es_ssl_keystore and es_ssl_truststore
   notify: restart elasticsearch
   register: copy_keystore
@@ -32,7 +32,7 @@
     dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_truststore | basename }}"
     owner: "{{ es_user }}"
     group: "{{ es_group }}"
-    mode: 0640
+    mode: "640"
   when: es_ssl_keystore and es_ssl_truststore
   notify: restart elasticsearch
   register: copy_truststore
@@ -43,7 +43,7 @@
     dest: "{{ es_ssl_certificate_path }}/{{ item | basename }}"
     owner: "{{ es_user }}"
     group: "{{ es_group }}"
-    mode: 0640
+    mode: "640"
   with_items:
     - "{{ es_ssl_key }}"
     - "{{ es_ssl_certificate }}"
@@ -58,7 +58,7 @@
     dest: "{{ es_ssl_certificate_path }}/{{ es_ssl_certificate_authority | basename }}"
     owner: "{{ es_user }}"
     group: "{{ es_group }}"
-    mode: 0640
+    mode: "640"
   #Restart if this changes
   notify: restart elasticsearch
   when: es_ssl_certificate_authority | bool

tasks/elasticsearch-template.yml

@@ -6,7 +6,7 @@
     state: directory
     owner: root
     group: "{{ es_group }}"
-    mode: 2750
+    mode: "2750"
 
 - name: Copy templates to elasticsearch
   copy: src={{ item }} dest={{ es_conf_dir }}/templates owner=root group={{ es_group }} mode=0660

templates/elasticsearch.j2

@@ -9,30 +9,23 @@ ES_HOME={{es_home}}
 #JAVA_HOME=
 
 # Elasticsearch configuration directory
-CONF_DIR={{ es_conf_dir }}
 ES_PATH_CONF={{ es_conf_dir }}
 
-# Elasticsearch data directory
-DATA_DIR={{ es_data_dirs | array_to_str }}
-
-# Elasticsearch logs directory
-LOG_DIR={{ es_log_dir }}
-
 # Elasticsearch PID directory
 PID_DIR={{ es_pid_dir }}
 
-ES_JVM_OPTIONS={{ es_conf_dir }}/jvm.options
+# Additional Java OPTS
+ES_JAVA_OPTS=
 
 # Configure restart on package upgrade (true, every other setting will lead to not restarting)
-#ES_RESTART_ON_UPGRADE=true
+#RESTART_ON_UPGRADE=true
-
-# Path to the GC log file
-#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log
 
 ################################
 # Elasticsearch service
 ################################
 
+# SysV init.d
+#
 # The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process
 ES_STARTUP_SLEEP_TIME=5
 
@@ -43,17 +36,17 @@ ES_STARTUP_SLEEP_TIME=5
 # Specifies the maximum file descriptor number that can be opened by this process
 # When using Systemd, this setting is ignored and the LimitNOFILE defined in
 # /usr/lib/systemd/system/elasticsearch.service takes precedence
+#MAX_OPEN_FILES=65535
 {% if es_max_open_files is defined %}
-#MAX_OPEN_FILES
 MAX_OPEN_FILES={{es_max_open_files}}
 {% endif %}
 
 # The maximum number of bytes of memory that may be locked into RAM
 # Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option
-# in elasticsearch.yml
+# in elasticsearch.yml.
-# When using Systemd, the LimitMEMLOCK property must be set
+# When using systemd, LimitMEMLOCK must be set in a unit file such as
-# in /usr/lib/systemd/system/elasticsearch.service
+# /etc/systemd/system/elasticsearch.service.d/override.conf.
-#MAX_LOCKED_MEMORY=
+#MAX_LOCKED_MEMORY=unlimited
 {% if m_lock_enabled %}
 MAX_LOCKED_MEMORY=unlimited
 {% endif %}
@@ -65,9 +58,3 @@ MAX_LOCKED_MEMORY=unlimited
 {% if es_max_map_count is defined %}
 MAX_MAP_COUNT={{es_max_map_count}}
 {% endif %}
-
-# Specifies the maximum number of threads that can be started.
-# Elasticsearch requires a minimum of 2048.
-{% if es_max_threads is defined %}
-MAX_THREADS={{ es_max_threads }}
-{% endif %}

templates/jvm.options.j2

@@ -41,6 +41,28 @@
 -XX:CMSInitiatingOccupancyFraction=75
 -XX:+UseCMSInitiatingOccupancyOnly
 
+## G1GC Configuration
+# NOTE: G1GC is only supported on JDK version 10 or later.
+# To use G1GC uncomment the lines below.
+# 10-:-XX:-UseConcMarkSweepGC
+# 10-:-XX:-UseCMSInitiatingOccupancyOnly
+# 10-:-XX:+UseG1GC
+{% if es_version is version('7.3.0', '<=') %}
+# 10-:-XX:InitiatingHeapOccupancyPercent=75
+{% else %}
+# 10-:-XX:G1ReservePercent=25
+# 10-:-XX:InitiatingHeapOccupancyPercent=30
+{% endif %}
+
+## DNS cache policy
+# cache ttl in seconds for positive DNS lookups noting that this overrides the
+# JDK security property networkaddress.cache.ttl; set to -1 to cache forever
+-Des.networkaddress.cache.ttl=60
+# cache ttl in seconds for negative DNS lookups noting that this overrides the
+# JDK security property networkaddress.cache.negative ttl; set to -1 to cache
+# forever
+-Des.networkaddress.cache.negative.ttl=10
+
 ## optimizations
 
 # pre-touch memory pages used by the JVM during initialization
@@ -48,8 +70,8 @@
 
 ## basic
 
-# force the server VM
+# explicitly set the stack size
--server
+-Xss1m
 
 # set to headless, just in case
 -Djava.awt.headless=true
@@ -60,18 +82,23 @@
 # use our provided JNA always versus the system one
 -Djna.nosys=true
 
-# use old-style file permissions on JDK9
+# turn off a JDK optimization that throws away stack traces for common
--Djdk.io.permissionsUseCanonicalPath=true
+# exceptions because stack traces are important for debugging
+-XX:-OmitStackTraceInFastThrow
 
 # flags to configure Netty
 -Dio.netty.noUnsafe=true
 -Dio.netty.noKeySetOptimization=true
 -Dio.netty.recycler.maxCapacityPerThread=0
+{% if es_version is version('7.4.0', '>=') %}
+-Dio.netty.allocator.numDirectArenas=0
+{% endif %}
 
 # log4j 2
 -Dlog4j.shutdownHookEnabled=false
 -Dlog4j2.disable.jmx=true
--Dlog4j.skipJansi=true
+
+-Djava.io.tmpdir=${ES_TMPDIR}
 
 ## heap dumps
 
@@ -79,30 +106,35 @@
 # heap dumps are created in the working directory of the JVM
 -XX:+HeapDumpOnOutOfMemoryError
 
-# specify an alternative path for heap dumps
+# specify an alternative path for heap dumps; ensure the directory exists and
-# ensure the directory exists and has sufficient space
+# has sufficient space
-#-XX:HeapDumpPath=${heap.dump.path}
+-XX:HeapDumpPath=${heap.dump.path}
 
-## GC logging
+# specify an alternative path for JVM fatal error logs
+-XX:ErrorFile={{ es_log_dir }}/hs_err_pid%p.log
 
-#-XX:+PrintGCDetails
+## JDK 8 GC logging
-#-XX:+PrintGCTimeStamps
-#-XX:+PrintGCDateStamps
-#-XX:+PrintClassHistogram
-#-XX:+PrintTenuringDistribution
-#-XX:+PrintGCApplicationStoppedTime
 
-# log GC status to a file with time stamps
+8:-XX:+PrintGCDetails
-# ensure the directory exists
+8:-XX:+PrintGCDateStamps
-#-Xloggc:${loggc}
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:{{ es_log_dir }}/gc.log
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
 
+# JDK 9+ GC logging
+9-:-Xlog:gc*,gc+age=trace,safepoint:file={{ es_log_dir }}/gc.log:utctime,pid,tags:filecount=32,filesize=64m
+# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise
+# time/date parsing will break in an incompatible way for some date patterns and locals
+9-:-Djava.locale.providers=COMPAT
+
+{% if es_major_version == "6.x" %}
+# temporary workaround for C2 bug with JDK 10 on hardware with AVX-512
+10-:-XX:UseAVX=2
+{% endif %}
 
-# By default, the GC log file will not rotate.
-# By uncommenting the lines below, the GC log file
-# will be rotated every 128MB at most 32 times.
-#-XX:+UseGCLogFileRotation
-#-XX:NumberOfGCLogFiles=32
-#-XX:GCLogFileSize=128M
 {% if es_jvm_custom_parameters !='' %}
 {% for item in es_jvm_custom_parameters %}
 {{ item }}

templates/log4j2.properties.j2

@@ -1,85 +0,0 @@
-status = error
-
-# log action execution errors for easier debugging
-logger.action.name = org.elasticsearch.action
-logger.action.level = debug
-
-appender.console.type = Console
-appender.console.name = console
-appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
-
-appender.rolling.type = RollingFile
-appender.rolling.name = rolling
-appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
-appender.rolling.layout.type = PatternLayout
-appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
-appender.rolling.policies.type = Policies
-appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.rolling.policies.time.interval = 1
-appender.rolling.policies.time.modulate = true
-appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
-appender.rolling.policies.size.size = 128MB
-appender.rolling.strategy.type = DefaultRolloverStrategy
-appender.rolling.strategy.fileIndex = nomax
-appender.rolling.strategy.action.type = Delete
-appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path}
-appender.rolling.strategy.action.condition.type = IfFileName
-appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
-appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize
-appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB
-rootLogger.level = info
-rootLogger.appenderRef.console.ref = console
-rootLogger.appenderRef.rolling.ref = rolling
-
-appender.deprecation_rolling.type = RollingFile
-appender.deprecation_rolling.name = deprecation_rolling
-appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
-appender.deprecation_rolling.layout.type = PatternLayout
-appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
-appender.deprecation_rolling.policies.type = Policies
-appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
-appender.deprecation_rolling.policies.size.size = 1GB
-appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
-appender.deprecation_rolling.strategy.max = 4
-
-logger.deprecation.name = org.elasticsearch.deprecation
-logger.deprecation.level = warn
-logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
-logger.deprecation.additivity = false
-
-appender.index_search_slowlog_rolling.type = RollingFile
-appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
-appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log
-appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
-appender.index_search_slowlog_rolling.layout.type = PatternLayout
-appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
-appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
-appender.index_search_slowlog_rolling.policies.type = Policies
-appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.index_search_slowlog_rolling.policies.time.interval = 1
-appender.index_search_slowlog_rolling.policies.time.modulate = true
-
-logger.index_search_slowlog_rolling.name = index.search.slowlog
-logger.index_search_slowlog_rolling.level = trace
-logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
-logger.index_search_slowlog_rolling.additivity = false
-
-appender.index_indexing_slowlog_rolling.type = RollingFile
-appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
-appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
-appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
-appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
-appender.index_indexing_slowlog_rolling.policies.type = Policies
-appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.index_indexing_slowlog_rolling.policies.time.interval = 1
-appender.index_indexing_slowlog_rolling.policies.time.modulate = true
-
-logger.index_indexing_slowlog.name = index.indexing.slowlog.index
-logger.index_indexing_slowlog.level = trace
-logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
-logger.index_indexing_slowlog.additivity = false

test/integration/files/custom_config/elasticsearch

@@ -0,0 +1,7 @@
+# Elasticsearch CUSTOM FILE
+
+ES_HOME=/usr/share/elasticsearch
+ES_PATH_CONF=/etc/elasticsearch
+PID_DIR=/var/run/elasticsearch
+ES_JAVA_OPTS=
+ES_STARTUP_SLEEP_TIME=5

test/integration/files/custom_config/jvm.options

@@ -0,0 +1,34 @@
+## JVM configuration CUSTOM FILE
+
+-Xms2g
+-Xmx2g
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+-Des.networkaddress.cache.ttl=60
+-Des.networkaddress.cache.negative.ttl=10
+-XX:+AlwaysPreTouch
+-Xss1m
+-Djava.awt.headless=true
+-Dfile.encoding=UTF-8
+-Djna.nosys=true
+-XX:-OmitStackTraceInFastThrow
+-Dio.netty.noUnsafe=true
+-Dio.netty.noKeySetOptimization=true
+-Dio.netty.recycler.maxCapacityPerThread=0
+-Dlog4j.shutdownHookEnabled=false
+-Dlog4j2.disable.jmx=true
+-Djava.io.tmpdir=${ES_TMPDIR}
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:HeapDumpPath=${heap.dump.path}
+-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
+8:-XX:+PrintGCDetails
+8:-XX:+PrintGCDateStamps
+8:-XX:+PrintTenuringDistribution
+8:-XX:+PrintGCApplicationStoppedTime
+8:-Xloggc:/var/log/elasticsearch/gc.log
+8:-XX:+UseGCLogFileRotation
+8:-XX:NumberOfGCLogFiles=32
+8:-XX:GCLogFileSize=64m
+9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m
+9-:-Djava.locale.providers=COMPAT

test/integration/files/custom_config/log4j2.properties

@@ -0,0 +1,86 @@
+{% raw %}
+# Log4j CUSTOM FILE
+
+status = error
+logger.action.name = org.elasticsearch.action
+logger.action.level = debug
+appender.rolling.type = Console
+appender.rolling.name = rolling
+appender.rolling.layout.type = ESJsonLayout
+appender.rolling.layout.type_name = server
+rootLogger.level = info
+rootLogger.appenderRef.rolling.ref = rolling
+appender.deprecation_rolling.type = Console
+appender.deprecation_rolling.name = deprecation_rolling
+appender.deprecation_rolling.layout.type = ESJsonLayout
+appender.deprecation_rolling.layout.type_name = deprecation
+appender.deprecation_rolling.layout.esmessagefields=x-opaque-id
+logger.deprecation.name = org.elasticsearch.deprecation
+logger.deprecation.level = warn
+logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
+logger.deprecation.additivity = false
+appender.index_search_slowlog_rolling.type = Console
+appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
+appender.index_search_slowlog_rolling.layout.type = ESJsonLayout
+appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog
+appender.index_search_slowlog_rolling.layout.esmessagefields=message,took,took_millis,total_hits,stats,search_type,total_shards,source,id
+logger.index_search_slowlog_rolling.name = index.search.slowlog
+logger.index_search_slowlog_rolling.level = trace
+logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
+logger.index_search_slowlog_rolling.additivity = false
+appender.index_indexing_slowlog_rolling.type = Console
+appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
+appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout
+appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog
+appender.index_indexing_slowlog_rolling.layout.esmessagefields=message,took,took_millis,doc_type,id,routing,source
+logger.index_indexing_slowlog.name = index.indexing.slowlog.index
+logger.index_indexing_slowlog.level = trace
+logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
+logger.index_indexing_slowlog.additivity = false
+appender.audit_rolling.type = Console
+appender.audit_rolling.name = audit_rolling
+appender.audit_rolling.layout.type = PatternLayout
+appender.audit_rolling.layout.pattern = {\
+"type": "audit", \
+"timestamp":"%d{yyyy-MM-dd'T'HH:mm:ss,SSSZ}"\
+%varsNotEmpty{, "node.name":"%enc{%map{node.name}}{JSON}"}\
+%varsNotEmpty{, "node.id":"%enc{%map{node.id}}{JSON}"}\
+%varsNotEmpty{, "host.name":"%enc{%map{host.name}}{JSON}"}\
+%varsNotEmpty{, "host.ip":"%enc{%map{host.ip}}{JSON}"}\
+%varsNotEmpty{, "event.type":"%enc{%map{event.type}}{JSON}"}\
+%varsNotEmpty{, "event.action":"%enc{%map{event.action}}{JSON}"}\
+%varsNotEmpty{, "user.name":"%enc{%map{user.name}}{JSON}"}\
+%varsNotEmpty{, "user.run_by.name":"%enc{%map{user.run_by.name}}{JSON}"}\
+%varsNotEmpty{, "user.run_as.name":"%enc{%map{user.run_as.name}}{JSON}"}\
+%varsNotEmpty{, "user.realm":"%enc{%map{user.realm}}{JSON}"}\
+%varsNotEmpty{, "user.run_by.realm":"%enc{%map{user.run_by.realm}}{JSON}"}\
+%varsNotEmpty{, "user.run_as.realm":"%enc{%map{user.run_as.realm}}{JSON}"}\
+%varsNotEmpty{, "user.roles":%map{user.roles}}\
+%varsNotEmpty{, "origin.type":"%enc{%map{origin.type}}{JSON}"}\
+%varsNotEmpty{, "origin.address":"%enc{%map{origin.address}}{JSON}"}\
+%varsNotEmpty{, "realm":"%enc{%map{realm}}{JSON}"}\
+%varsNotEmpty{, "url.path":"%enc{%map{url.path}}{JSON}"}\
+%varsNotEmpty{, "url.query":"%enc{%map{url.query}}{JSON}"}\
+%varsNotEmpty{, "request.method":"%enc{%map{request.method}}{JSON}"}\
+%varsNotEmpty{, "request.body":"%enc{%map{request.body}}{JSON}"}\
+%varsNotEmpty{, "request.id":"%enc{%map{request.id}}{JSON}"}\
+%varsNotEmpty{, "action":"%enc{%map{action}}{JSON}"}\
+%varsNotEmpty{, "request.name":"%enc{%map{request.name}}{JSON}"}\
+%varsNotEmpty{, "indices":%map{indices}}\
+%varsNotEmpty{, "opaque_id":"%enc{%map{opaque_id}}{JSON}"}\
+%varsNotEmpty{, "x_forwarded_for":"%enc{%map{x_forwarded_for}}{JSON}"}\
+%varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}\
+%varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}\
+%varsNotEmpty{, "event.category":"%enc{%map{event.category}}{JSON}"}\
+}%n
+logger.xpack_security_audit_logfile.name = org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail
+logger.xpack_security_audit_logfile.level = info
+logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rolling
+logger.xpack_security_audit_logfile.additivity = false
+logger.xmlsig.name = org.apache.xml.security.signature.XMLSignature
+logger.xmlsig.level = error
+logger.samlxml_decrypt.name = org.opensaml.xmlsec.encryption.support.Decrypter
+logger.samlxml_decrypt.level = fatal
+logger.saml2_decrypt.name = org.opensaml.saml.saml2.encryption.Decrypter
+logger.saml2_decrypt.level = fatal
+{% endraw %}

test/integration/helpers/serverspec/oss_spec.rb

@@ -1,13 +1,20 @@
 require 'spec_helper'
+require 'shared_spec'
 
 shared_examples 'oss::init' do |vars|
   describe file("/etc/elasticsearch/log4j2.properties") do
     it { should be_file }
     it { should be_owned_by 'root' }
-    it { should_not contain 'CUSTOM LOG4J FILE' }
+    it { should contain 'Log4j CUSTOM FILE' }
   end
   describe file("/etc/elasticsearch/jvm.options") do
     it { should be_file }
     it { should be_owned_by 'root' }
+    it { should contain 'JVM configuration CUSTOM FILE' }
+  end
+  describe file($family['defaults_path']) do
+    it { should be_file }
+    it { should be_owned_by 'root' }
+    it { should contain 'Elasticsearch CUSTOM FILE' }
   end
 end

test/integration/helpers/serverspec/shared_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 require 'json'
 vars = JSON.parse(File.read('/tmp/vars.json'))
 
-families = {
+$families = {
   'Debian' => {
     'shell'    => '/bin/false',
     'password' => '*',
@@ -15,7 +15,7 @@ families = {
   }
 }
 
-family = families[vars['ansible_os_family']]
+$family = $families[vars['ansible_os_family']]
 
 es_api_url = "#{vars['es_api_scheme']}://localhost:#{vars['es_api_port']}"
 username = vars['es_api_basic_auth_username']
@@ -88,9 +88,9 @@ shared_examples 'shared::init' do |vars|
     it { should belong_to_group vars['es_group'] }
     it { should have_uid vars['es_user_id'] } if vars.key?('es_user_id')
 
-    it { should have_login_shell family['shell'] }
+    it { should have_login_shell $family['shell'] }
 
-    its(:encrypted_password) { should eq(family['password']) }
+    its(:encrypted_password) { should eq($family['password']) }
   end
 
   describe package(vars['es_package_name']) do
@@ -129,7 +129,7 @@ shared_examples 'shared::init' do |vars|
     end
   end
 
-  describe file(family['defaults_path']) do
+  describe file($family['defaults_path']) do
     its(:content) { should match '' }
   end
 

test/integration/oss.yml

@@ -10,5 +10,8 @@
     es_heap_size: "1g"
     es_plugins:
       - plugin: ingest-attachment
+    es_config_default: "test/integration/files/custom_config/elasticsearch"
+    es_config_log4j2: "test/integration/files/custom_config/log4j2.properties"
+    es_config_jvm: "test/integration/files/custom_config/jvm.options"
 
 #Do not add tests here. This test is run twice and confirms idempotency.