From 68535e3d360fd886fb8d57393cb6e4209c59be93 Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Tue, 23 Sep 2025 12:27:11 +0200 Subject: [PATCH 01/10] Rename `elasticstack_full_stack` to `elasticstack_collection_managed` This fixes #316 --- README.md | 3 +++ docs/role-beats.md | 4 ++-- docs/role-kibana.md | 6 +++--- docs/role-logstash.md | 4 ++-- molecule/beats_default/converge.yml | 2 +- molecule/beats_peculiar/converge.yml | 2 +- molecule/elasticsearch_default/converge.yml | 2 +- molecule/elasticsearch_test_modules/converge.yml | 2 +- molecule/elasticstack_default/converge.yml | 2 +- molecule/kibana_default/converge.yml | 2 +- molecule/logstash_full_stack-oss/converge.yml | 2 +- molecule/logstash_pipelines/converge.yml | 2 +- molecule/logstash_specific_version/converge.yml | 2 +- molecule/repos_default/converge.yml | 2 +- roles/beats/tasks/auditbeat.yml | 8 ++++---- roles/beats/tasks/filebeat.yml | 8 ++++---- roles/beats/tasks/main.yml | 4 ++-- roles/beats/tasks/metricbeat.yml | 8 ++++---- roles/beats/templates/auditbeat.yml.j2 | 6 +++--- roles/beats/templates/filebeat.yml.j2 | 6 +++--- roles/beats/templates/metricbeat.yml.j2 | 6 +++--- roles/elasticsearch/handlers/main.yml | 2 +- .../tasks/elasticsearch-rolling-upgrade.yml | 12 ++++++------ roles/elasticsearch/tasks/main.yml | 4 ++-- roles/elasticstack/defaults/main.yml | 2 +- roles/kibana/tasks/main.yml | 8 ++++---- roles/kibana/templates/kibana.yml.j2 | 2 +- roles/logstash/tasks/main.yml | 10 +++++----- .../logstash/templates/elasticsearch-output.conf.j2 | 4 ++-- roles/logstash/templates/logstash.yml.j2 | 2 +- 30 files changed, 66 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 63f4c98e..f3ed40fc 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,9 @@ Every role is documented with all variables, please refer to the documentation f ## Global variables +* `elasticstack_collection_managed`: When set to true, roles inside this collection can fully rely on each other’s default configuration, naming conventions, and managed resources. This means the collection handles repositories, package names, and other components internally, leaving no room for external tools or custom configurations to override these defaults. +When set to false, you can provide your own repositories, package names, or other configurations, but roles cannot assume defaults from other roles anymore. (Default: `true`) + * `elasticstack_force_pip`: Will force installation of required Python modules via `pip`. This is useful if your package manager doesn't provide current versions of modules. (Default: `false`) See [PEP668](https://peps.python.org/pep-0668/) for more details. * `elasticstack_manage_pip`: Will install `pip` on your system. (Default: `false`) diff --git a/docs/role-beats.md b/docs/role-beats.md index 5d742c82..49ffcc8b 100644 --- a/docs/role-beats.md +++ b/docs/role-beats.md @@ -79,7 +79,7 @@ beats_filebeat_journald_inputs: * *beats_metricbeat_modules*: List of modules to enable. (Default: `- system`) * *beats_metricbeat_loadbalance*: Enable loadbalancing for Metricbeats Logstash output (default: `true`) -* *beats_security*: Activate TLS for connections to targets. Can either be use with our other roles and `elasticstack_full_stack` to automatically create certificates or `beats_tls*` variables for custom certificates. (default: `false`) +* *beats_security*: Activate TLS for connections to targets. Can either be use with our other roles and `elasticstack_collection_managed` to automatically create certificates or `beats_tls*` variables for custom certificates. (default: `false`) * *beats_target_hosts*: Only use when this role is used standalone. When used in combination with our other roles, the target hosts will be determined automatically. Use a YAML list. (default: `- localhost`) * *elasticstack_elasticsearch_http_port*: Port of Elasticsearch to send events to (Default: `9200`) * *elasticstack_beats_port*: Port of Logstash to send events to (Default: `5044`) @@ -90,7 +90,7 @@ beats_filebeat_journald_inputs: The following variables only apply if you use this role together with our other Elastic Stack roles. -* *elasticstack_full_stack*: Use `elasticsearch` as well (default: `false`) +* *elasticstack_collection_managed*: Use `elasticsearch` as well (default: `false`) * *elasticstack_variant*: Define which variant of elastic stack to use. (default: `elastic`) * *elasticstack_ca_dir*: Directory where on the Elasticsearch CA host certificates are stored. This is only useful in connection with out other Elastic Stack related roles. (default: `/opt/es-ca`) * *elasticstack_ca_pass*: Password for Elasticsearch CA (default: `PleaseChangeMe`) diff --git a/docs/role-kibana.md b/docs/role-kibana.md index f1df2640..6e4bdceb 100644 --- a/docs/role-kibana.md +++ b/docs/role-kibana.md @@ -15,11 +15,11 @@ Role Variables * *kibana_tls_key*: Path to the key Kibana should use when communicating with clients (default: `/etc/kibana/certs/key.pem`) * *kibana_extra_config*: You can add arbitraty configuration options with this option. Just start it with `|-` and indent the following lines. So you can add as many lines and options to `kibana.yml` as you like. (default: none) -* *kibana_security*: Activate TLS and authentication when connecting to Elasticsearch. **Note**: Only works when `elasticstack_full_stack` is enabled. (default: `true`) +* *kibana_security*: Activate TLS and authentication when connecting to Elasticsearch. **Note**: Only works when `elasticstack_collection_managed` is enabled. (default: `true`) These variables are identical over all our elastic related roles, hence the different naming scheme. -* *elasticstack_full_stack*: Use `ansible-role-elasticsearch` as well (default: `false`). If you set to `true`, you will be able to use the following variables: +* *elasticstack_collection_managed*: Use `ansible-role-elasticsearch` as well (default: `false`). If you set to `true`, you will be able to use the following variables: * *kibana_elasticsearch_hosts*: A list of DNS resolvable hostnames of Elasticsearch hosts to connect your Kibana instance to. (default: `- localhost`) * *elasticstack_elasticsearch_http_port*: Port of Elasticsearch http (Default: `9200`) * *kibana_sniff_on_start*: Attempt to find other Elasticsearch nodes on startup (default: `false`) @@ -50,7 +50,7 @@ If you use `localhost` in `kibana_elasticsearch_hosts` , certificate verificatio - netways.elasticstack hosts: kibana-host vars: - elasticstack_full_stack: true + elasticstack_collection_managed: true elasticstack_variant: oss roles: - repos diff --git a/docs/role-logstash.md b/docs/role-logstash.md index 1b4e4b78..c318f504 100644 --- a/docs/role-logstash.md +++ b/docs/role-logstash.md @@ -86,7 +86,7 @@ Aside from `logstash.yml` we can manage Logstashs pipelines. * *logstash_sniffing*: Enable sniffing (Default: `false`). * *logstash_sniffing_delay*: How long to wait, in seconds, between sniffing attempts (Default: `not set`). * *logstash_sniffing_path*: HTTP Path to be used for the sniffing requests (Default: `not set`). -* *logstash_legacy_monitoring*: Enables legacy monitoring - ignored when `elasticstack_full_stack` is not set. (default: `true`) +* *logstash_legacy_monitoring*: Enables legacy monitoring - ignored when `elasticstack_collection_managed` is not set. (default: `true`) * *logstash_redis_password*: If set this will use this password when connecting our simple inputs and outputs to Redis. (default: not set) * *logstash_mermaid*: Print overview over Logstash pipelines in Mermaid syntax. (default: `true`) @@ -114,7 +114,7 @@ The following variables are identical over all our elastic related roles, hence The following variables only apply if you use this role together with our Elasticsearch and Kibana roles. -* *elasticstack_full_stack*: Use `ansible-role-elasticsearch` as well (default: `false`) +* *elasticstack_collection_managed*: Use `ansible-role-elasticsearch` as well (default: `false`) * *elasticstack_ca_dir*: Directory where the CA and certificates lie on the main Elasticsearch host (default: `/opt/es-ca`) * *elasticstack_elasticsearch_http_port*: Port of Elasticsearch to send events to (Default: `9200`) * *elasticstack_initial_passwords*: File where initial passwords are stored on the main Elasticsearch host (default: `/usr/share/elasticsearch/initial_passwords`) diff --git a/molecule/beats_default/converge.yml b/molecule/beats_default/converge.yml index 8164cd84..2644f7a0 100644 --- a/molecule/beats_default/converge.yml +++ b/molecule/beats_default/converge.yml @@ -8,7 +8,7 @@ collections: - netways.elasticstack vars: - elasticstack_full_stack: false + elasticstack_collection_managed: false elasticsearch_jna_workaround: true elasticsearch_disable_systemcallfilterchecks: true elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" diff --git a/molecule/beats_peculiar/converge.yml b/molecule/beats_peculiar/converge.yml index e5a46244..17339726 100644 --- a/molecule/beats_peculiar/converge.yml +++ b/molecule/beats_peculiar/converge.yml @@ -23,7 +23,7 @@ testfield2: thisisanothertest elasticsearch_jna_workaround: true elasticsearch_disable_systemcallfilterchecks: true - elasticstack_full_stack: false + elasticstack_collection_managed: false elasticstack_no_log: false beats_filebeat_mysql_slowlog_input: true beats_auditbeat: true diff --git a/molecule/elasticsearch_default/converge.yml b/molecule/elasticsearch_default/converge.yml index 1e836b59..c6865b32 100644 --- a/molecule/elasticsearch_default/converge.yml +++ b/molecule/elasticsearch_default/converge.yml @@ -7,7 +7,7 @@ hosts: all vars: #elasticsearch_security: true # needed for tests of > 7 releases - elasticstack_full_stack: false + elasticstack_collection_managed: false elasticsearch_jna_workaround: true elasticsearch_disable_systemcallfilterchecks: true elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" diff --git a/molecule/elasticsearch_test_modules/converge.yml b/molecule/elasticsearch_test_modules/converge.yml index 77ffdd66..4fab30ce 100644 --- a/molecule/elasticsearch_test_modules/converge.yml +++ b/molecule/elasticsearch_test_modules/converge.yml @@ -6,7 +6,7 @@ - netways.elasticstack hosts: all vars: - elasticstack_full_stack: false + elasticstack_collection_managed: false elasticsearch_jna_workaround: true elasticsearch_disable_systemcallfilterchecks: true elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index 694db61d..54f7aa2d 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -15,7 +15,7 @@ elasticsearch_monitoring_enabled: false elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" elasticsearch_heap: "2" - elasticstack_full_stack: true + elasticstack_collection_managed: true elasticstack_no_log: false logstash_pipeline_unsafe_shutdown: true logstash_redis_password: "{{ lookup('ansible.builtin.password', '/tmp/redispassword', chars=['ascii_letters'], length=15) }}" diff --git a/molecule/kibana_default/converge.yml b/molecule/kibana_default/converge.yml index 4b0f6a1e..381f47a2 100644 --- a/molecule/kibana_default/converge.yml +++ b/molecule/kibana_default/converge.yml @@ -6,7 +6,7 @@ - name: Converge hosts: all vars: - elasticstack_full_stack: false + elasticstack_collection_managed: false elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" elasticstack_no_log: false collections: diff --git a/molecule/logstash_full_stack-oss/converge.yml b/molecule/logstash_full_stack-oss/converge.yml index 5f163e7a..6116c9f6 100644 --- a/molecule/logstash_full_stack-oss/converge.yml +++ b/molecule/logstash_full_stack-oss/converge.yml @@ -14,7 +14,7 @@ collections: - netways.elasticstack vars: - elasticstack_full_stack: true + elasticstack_collection_managed: true elasticstack_variant: oss logstash_security: false logstash_pipeline_unsafe_shutdown: true diff --git a/molecule/logstash_pipelines/converge.yml b/molecule/logstash_pipelines/converge.yml index 34b1308d..53c3654b 100644 --- a/molecule/logstash_pipelines/converge.yml +++ b/molecule/logstash_pipelines/converge.yml @@ -32,7 +32,7 @@ logstash_pipeline_unsafe_shutdown: true redis_requirepass: "{{ logstash_redis_password }}" elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" - elasticstack_full_stack: false + elasticstack_collection_managed: false elasticstack_no_log: false tasks: - name: "Include Elastics repos role" diff --git a/molecule/logstash_specific_version/converge.yml b/molecule/logstash_specific_version/converge.yml index cabf8ae4..f847a187 100644 --- a/molecule/logstash_specific_version/converge.yml +++ b/molecule/logstash_specific_version/converge.yml @@ -14,7 +14,7 @@ logstash_pipeline_identifier: false logstash_pipeline_unsafe_shutdown: true elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" - elasticstack_full_stack: false + elasticstack_collection_managed: false elasticstack_no_log: false tasks: diff --git a/molecule/repos_default/converge.yml b/molecule/repos_default/converge.yml index 76c90b70..80864563 100644 --- a/molecule/repos_default/converge.yml +++ b/molecule/repos_default/converge.yml @@ -5,7 +5,7 @@ - netways.elasticstack vars: elasticstack_rpm_workaround: true - elasticstack_full_stack: false + elasticstack_collection_managed: false elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" elasticstack_no_log: false tasks: diff --git a/roles/beats/tasks/auditbeat.yml b/roles/beats/tasks/auditbeat.yml index 46a84a00..2a8985d3 100644 --- a/roles/beats/tasks/auditbeat.yml +++ b/roles/beats/tasks/auditbeat.yml @@ -20,7 +20,7 @@ - Restart Auditbeat when: - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - name: Install Auditbeat - rpm - standalone ansible.builtin.package: @@ -29,7 +29,7 @@ - Restart Auditbeat when: - ansible_os_family == "RedHat" - - not elasticstack_full_stack | bool + - not elasticstack_collection_managed | bool - name: Install Auditbeat - deb ansible.builtin.package: @@ -54,7 +54,7 @@ - elasticstack_version is defined - elasticstack_version == "latest" - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - name: Install Auditbeat latest version - rpm - standalone ansible.builtin.package: @@ -66,7 +66,7 @@ - elasticstack_version is defined - elasticstack_version == "latest" - ansible_os_family == "RedHat" - - not elasticstack_full_stack | bool + - not elasticstack_collection_managed | bool - name: Install Auditbeat latest version - deb ansible.builtin.package: diff --git a/roles/beats/tasks/filebeat.yml b/roles/beats/tasks/filebeat.yml index 0fbce0cb..2f0cb54a 100644 --- a/roles/beats/tasks/filebeat.yml +++ b/roles/beats/tasks/filebeat.yml @@ -19,7 +19,7 @@ - Restart Filebeat when: - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - name: Install Filebeat - rpm - standalone ansible.builtin.package: @@ -28,7 +28,7 @@ - Restart Filebeat when: - ansible_os_family == "RedHat" - - not elasticstack_full_stack | bool + - not elasticstack_collection_managed | bool - name: Install Filebeat - deb ansible.builtin.package: @@ -50,7 +50,7 @@ - elasticstack_version is defined - elasticstack_version == "latest" - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - name: Install Filebeat latest version - rpm - standalone ansible.builtin.package: @@ -62,7 +62,7 @@ - elasticstack_version is defined - elasticstack_version == "latest" - ansible_os_family == "RedHat" - - not elasticstack_full_stack | bool + - not elasticstack_collection_managed | bool - name: Install Filebeat latest version - deb ansible.builtin.package: diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml index 7739e7e5..7b08cc2e 100644 --- a/roles/beats/tasks/main.yml +++ b/roles/beats/tasks/main.yml @@ -14,7 +14,7 @@ - name: Prepare for whole stack roles if used when: - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool block: - name: Enable security if enabled for whole stack ansible.builtin.set_fact: @@ -58,7 +58,7 @@ ansible.builtin.import_tasks: beats-security.yml when: - beats_security | bool - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - elasticstack_variant == "elastic" tags: - certificates diff --git a/roles/beats/tasks/metricbeat.yml b/roles/beats/tasks/metricbeat.yml index 4ca61cfb..f4b91a99 100644 --- a/roles/beats/tasks/metricbeat.yml +++ b/roles/beats/tasks/metricbeat.yml @@ -20,7 +20,7 @@ - Restart Metricbeat when: - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - name: Install Metricbeat - rpm - standalone ansible.builtin.package: @@ -29,7 +29,7 @@ - Restart Metricbeat when: - ansible_os_family == "RedHat" - - not elasticstack_full_stack | bool + - not elasticstack_collection_managed | bool - name: Install Metricbeat - deb ansible.builtin.package: @@ -51,7 +51,7 @@ - elasticstack_version is defined - elasticstack_version == "latest" - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - name: Install Metricbeat latest version - rpm - standalone ansible.builtin.package: @@ -63,7 +63,7 @@ - elasticstack_version is defined - elasticstack_version == "latest" - ansible_os_family == "RedHat" - - not elasticstack_full_stack | bool + - not elasticstack_collection_managed | bool - name: Install Metricbeat latest version - deb diff --git a/roles/beats/templates/auditbeat.yml.j2 b/roles/beats/templates/auditbeat.yml.j2 index 85a085b8..511a606c 100644 --- a/roles/beats/templates/auditbeat.yml.j2 +++ b/roles/beats/templates/auditbeat.yml.j2 @@ -32,7 +32,7 @@ setup.kibana: {% if beats_auditbeat_output == "elasticsearch" %} output.elasticsearch: {% if beats_security | bool %} -{% if elasticstack_full_stack | bool %} +{% if elasticstack_collection_managed | bool %} hosts: [ {% for host in groups[elasticstack_elasticsearch_group_name] %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] {% else %} hosts: [ {% for host in beats_target_hosts %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] @@ -43,7 +43,7 @@ output.elasticsearch: ssl.verification_mode: none ssl.certificate_authorities: ["/etc/beats/certs/ca.crt"] {% else %} -{% if elasticstack_full_stack | bool %} +{% if elasticstack_collection_managed | bool %} hosts: [ {% for host in groups[elasticstack_elasticsearch_group_name] %}"http://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] {% else %} hosts: [ {% for host in beats_target_hosts %}"http://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] @@ -52,7 +52,7 @@ output.elasticsearch: {% endif %} {% if beats_auditbeat_output == "logstash" %} output.logstash: -{% if elasticstack_full_stack | bool %} +{% if elasticstack_collection_managed | bool %} hosts: [ {% for host in groups[elasticstack_logstash_group_name] %}"{{ host }}:{{ elasticstack_beats_port }}"{% if not loop.last %},{% endif %}{% endfor %}] {% else %} hosts: [ {% for host in beats_target_hosts %}"{{ host }}:{{ elasticstack_beats_port }}"{% if not loop.last %},{% endif %}{% endfor %}] diff --git a/roles/beats/templates/filebeat.yml.j2 b/roles/beats/templates/filebeat.yml.j2 index b77bd641..48c59ce1 100644 --- a/roles/beats/templates/filebeat.yml.j2 +++ b/roles/beats/templates/filebeat.yml.j2 @@ -107,7 +107,7 @@ setup.kibana: {% if beats_filebeat_output == "elasticsearch" %} output.elasticsearch: {% if beats_security | bool %} -{% if elasticstack_full_stack | bool %} +{% if elasticstack_collection_managed | bool %} hosts: [ {% for host in groups[elasticstack_elasticsearch_group_name] %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] {% else %} hosts: [ {% for host in beats_target_hosts %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] @@ -118,7 +118,7 @@ output.elasticsearch: ssl.verification_mode: none ssl.certificate_authorities: ["/etc/beats/certs/ca.crt"] {% else %} -{% if elasticstack_full_stack | bool %} +{% if elasticstack_collection_managed | bool %} hosts: [ {% for host in groups[elasticstack_elasticsearch_group_name] %}"http://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] {% else %} hosts: [ {% for host in beats_target_hosts %}"http://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] @@ -127,7 +127,7 @@ output.elasticsearch: {% endif %} {% if beats_filebeat_output == "logstash" %} output.logstash: -{% if elasticstack_full_stack | bool %} +{% if elasticstack_collection_managed | bool %} hosts: [ {% for host in groups[elasticstack_logstash_group_name] %}"{{ host }}:{{ elasticstack_beats_port }}"{% if not loop.last %},{% endif %}{% endfor %}] {% else %} hosts: [ {% for host in beats_target_hosts %}"{{ host }}:{{ elasticstack_beats_port }}"{% if not loop.last %},{% endif %}{% endfor %}] diff --git a/roles/beats/templates/metricbeat.yml.j2 b/roles/beats/templates/metricbeat.yml.j2 index a8d45b04..8e227828 100644 --- a/roles/beats/templates/metricbeat.yml.j2 +++ b/roles/beats/templates/metricbeat.yml.j2 @@ -9,7 +9,7 @@ setup.kibana: {% if beats_metricbeat_output == "elasticsearch" %} output.elasticsearch: {% if beats_security | bool %} -{% if elasticstack_full_stack | bool %} +{% if elasticstack_collection_managed | bool %} hosts: [ {% for host in groups[elasticstack_elasticsearch_group_name] %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] {% else %} hosts: [ {% for host in beats_target_hosts %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] @@ -20,7 +20,7 @@ output.elasticsearch: ssl.verification_mode: none ssl.certificate_authorities: ["/etc/beats/certs/ca.crt"] {% else %} -{% if elasticstack_full_stack | bool %} +{% if elasticstack_collection_managed | bool %} hosts: [ {% for host in groups[elasticstack_elasticsearch_group_name] %}"http://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] {% else %} hosts: [ {% for host in beats_target_hosts %}"http://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] @@ -29,7 +29,7 @@ output.elasticsearch: {% endif %} {% if beats_metricbeat_output == "logstash" %} output.logstash: -{% if elasticstack_full_stack | bool %} +{% if elasticstack_collection_managed | bool %} hosts: [ {% for host in groups[elasticstack_logstash_group_name] %}"{{ host }}:{{ elasticstack_beats_port }}"{% if not loop.last %},{% endif %}{% endfor %}] {% else %} hosts: [ {% for host in beats_target_hosts %}"{{ host }}:{{ elasticstack_beats_port }}"{% if not loop.last %},{% endif %}{% endfor %}] diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml index 78169736..bde0880a 100644 --- a/roles/elasticsearch/handlers/main.yml +++ b/roles/elasticsearch/handlers/main.yml @@ -14,7 +14,7 @@ ansible.builtin.include_tasks: handlers/restart_kibana.yml with_items: "{{ groups[elasticstack_kibana_group_name] }}" when: - - elasticstack_full_stack: true + - elasticstack_collection_managed: true - "not 'renew_ca' in ansible_run_tags" - "not elasticstack_ca_will_expire_soon | bool" - elasticstack_kibana_group_name is defined diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 19801a76..4c8d9cec 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -31,14 +31,14 @@ - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' when: - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - name: Update stopped Elasticsearch - deb or unmanaged repositories rpm ansible.builtin.package: name: "{{ elasticsearch_package }}" when: - ansible_os_family == "Debian" or - not elasticstack_full_stack | bool + not elasticstack_collection_managed | bool - name: Update single instances without extra caution when: @@ -49,7 +49,7 @@ name: "{{ elasticsearch_package }}" when: - ansible_os_family == "Debian" or - not elasticstack_full_stack | bool + not elasticstack_collection_managed | bool notify: - Restart Elasticsearch @@ -60,7 +60,7 @@ - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' when: - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool notify: - Restart Elasticsearch @@ -138,14 +138,14 @@ - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' when: - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - name: Update Elasticsearch - deb or unmanaged repositories rpm ansible.builtin.package: name: "{{ elasticsearch_package }}" when: - ansible_os_family == "Debian" or - not elasticstack_full_stack | bool + not elasticstack_collection_managed | bool - name: Start elasticsearch ansible.builtin.service: diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index 7537197b..e2d538ab 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -155,14 +155,14 @@ - 'elastic-{% if elasticstack_variant == "oss" %}oss-{% endif %}{{ elasticstack_release }}.x' when: - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - name: Install Elasticsearch - rpm - standalone ansible.builtin.package: name: "{{ elasticsearch_package }}" when: - ansible_os_family == "RedHat" - - not elasticstack_full_stack | bool + - not elasticstack_collection_managed | bool - name: Install Elasticsearch - deb ansible.builtin.package: diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index ee14ca30..132c0345 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -14,7 +14,7 @@ elasticstack_ca_validity_period: 1095 elasticstack_ca_will_expire_soon: false elasticstack_elasticsearch_http_port: 9200 elasticstack_enable_repos: true -elasticstack_full_stack: true +elasticstack_collection_managed: true elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords elasticstack_kibana_port: 5601 elasticstack_override_beats_tls: false diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index 194f8344..eb96d0d7 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -60,7 +60,7 @@ - Restart Kibana when: - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - name: Install Kibana - rpm - standalone ansible.builtin.package: @@ -69,7 +69,7 @@ - Restart Kibana when: - ansible_os_family == "RedHat" - - not elasticstack_full_stack | bool + - not elasticstack_collection_managed | bool - name: Install Kibana - deb ansible.builtin.package: @@ -82,8 +82,8 @@ - name: Import security related tasks ansible.builtin.import_tasks: kibana-security.yml when: - - elasticstack_full_stack is defined - - elasticstack_full_stack | bool + - elasticstack_collection_managed is defined + - elasticstack_collection_managed | bool - kibana_security | bool - elasticstack_variant == "elastic" tags: diff --git a/roles/kibana/templates/kibana.yml.j2 b/roles/kibana/templates/kibana.yml.j2 index b401caa0..03d098cd 100644 --- a/roles/kibana/templates/kibana.yml.j2 +++ b/roles/kibana/templates/kibana.yml.j2 @@ -1,7 +1,7 @@ server.host: "0.0.0.0" server.publicBaseUrl: "http{% if kibana_tls | bool %}s{% endif %}://{{ elasticstack_kibana_host | default( ansible_fqdn ) }}:{{ elasticstack_kibana_port }}" -{% if elasticstack_full_stack is defined and elasticstack_full_stack | bool and elasticstack_variant == "elastic" %} +{% if elasticstack_collection_managed is defined and elasticstack_collection_managed | bool and elasticstack_variant == "elastic" %} {%- if kibana_sniff_on_start | bool %} elasticsearch.sniffOnStart: true {%- endif %} diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index 4a1ab279..d427b326 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -14,7 +14,7 @@ - name: Prepare for whole stack roles if used when: - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool block: - name: Set Elasticsearch hosts if used with other roles @@ -50,7 +50,7 @@ logstash_security: true when: - logstash_security is undefined or elasticstack_security | bool - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - elasticstack_variant == "elastic" tags: - certificates @@ -93,7 +93,7 @@ - Restart Logstash when: - ansible_os_family == "RedHat" - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - name: Install Logstash - rpm - standalone ansible.builtin.package: @@ -102,7 +102,7 @@ - Restart Logstash when: - ansible_os_family == "RedHat" - - not elasticstack_full_stack | bool + - not elasticstack_collection_managed | bool - name: Install Logstash - deb ansible.builtin.package: @@ -115,7 +115,7 @@ - name: Import Logstash Security tasks ansible.builtin.import_tasks: logstash-security.yml when: - - elasticstack_full_stack | bool + - elasticstack_collection_managed | bool - logstash_security is defined and logstash_security | bool - elasticstack_variant == "elastic" tags: diff --git a/roles/logstash/templates/elasticsearch-output.conf.j2 b/roles/logstash/templates/elasticsearch-output.conf.j2 index 86a8988e..e0fd9ae7 100644 --- a/roles/logstash/templates/elasticsearch-output.conf.j2 +++ b/roles/logstash/templates/elasticsearch-output.conf.j2 @@ -22,7 +22,7 @@ output { elasticsearch { hosts => [ {% for host in logstash_elasticsearch %}"{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %}] validate_after_inactivity => {{ logstash_validate_after_inactivity }} -{% if elasticstack_full_stack | bool and elasticstack_variant == "elastic" and logstash_sniffing | bool %} +{% if elasticstack_collection_managed | bool and elasticstack_variant == "elastic" and logstash_sniffing | bool %} sniffing => true {% if logstash_sniffing_delay is defined %} sniffing_delay => {{ logstash_sniffing_delay }} @@ -31,7 +31,7 @@ output { {% endif %} {% endif %} {% endif %} -{% if elasticstack_full_stack | bool and logstash_security is defined and logstash_security | bool and elasticstack_variant == "elastic" %} +{% if elasticstack_collection_managed | bool and logstash_security is defined and logstash_security | bool and elasticstack_variant == "elastic" %} keystore => "{{ logstash_certs_dir }}/keystore.pfx" keystore_password => "{{ logstash_tls_key_passphrase }}" cacert => "{{ logstash_certs_dir }}/ca.crt" diff --git a/roles/logstash/templates/logstash.yml.j2 b/roles/logstash/templates/logstash.yml.j2 index c7d6aced..f440d6e9 100644 --- a/roles/logstash/templates/logstash.yml.j2 +++ b/roles/logstash/templates/logstash.yml.j2 @@ -16,7 +16,7 @@ pipeline.ecs_compatibility: {{ logstash_global_ecs }} {% if logstash_pipeline_unsafe_shutdown is defined %} pipeline.unsafe_shutdown: {{ logstash_pipeline_unsafe_shutdown }} {% endif %} -{% if logstash_legacy_monitoring | bool and elasticstack_full_stack | bool and elasticstack_variant == "elastic" and elasticstack_release | int < 8 %} +{% if logstash_legacy_monitoring | bool and elasticstack_collection_managed | bool and elasticstack_variant == "elastic" and elasticstack_release | int < 8 %} xpack.monitoring.enabled: true xpack.monitoring.elasticsearch.hosts: [ {% for host in logstash_elasticsearch %}"https://{{ host }}:{{ elasticstack_elasticsearch_http_port }}"{% if not loop.last %},{% endif %}{% endfor %} ] xpack.monitoring.elasticsearch.username: elastic From 0ccd9693c03ba84102645ee7a0994b25b4735a4b Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Tue, 23 Sep 2025 12:47:05 +0200 Subject: [PATCH 02/10] Sort documentation --- README.md | 18 +++++++++--------- ...etting-started.md => 02-getting-started.md} | 10 +++++----- docs/{role-repos.md => 10-role-repos.md} | 0 ...asticsearch.md => 11-role-elasticsearch.md} | 0 docs/{role-logstash.md => 12-role-logstash.md} | 4 ++-- ...h-pipelines.md => 13-logstash-pipelines.md} | 0 docs/{role-beats.md => 14-role-beats.md} | 0 docs/{role-kibana.md => 15-role-kibana.md} | 0 ...role.md => 20-module-elasticsearch_role.md} | 0 ...user.md => 21-module-elasticsearch_user.md} | 0 10 files changed, 16 insertions(+), 16 deletions(-) rename docs/{getting-started.md => 02-getting-started.md} (88%) rename docs/{role-repos.md => 10-role-repos.md} (100%) rename docs/{role-elasticsearch.md => 11-role-elasticsearch.md} (100%) rename docs/{role-logstash.md => 12-role-logstash.md} (99%) rename docs/{logstash-pipelines.md => 13-logstash-pipelines.md} (100%) rename docs/{role-beats.md => 14-role-beats.md} (100%) rename docs/{role-kibana.md => 15-role-kibana.md} (100%) rename docs/{module-elasticsearch_role.md => 20-module-elasticsearch_role.md} (100%) rename docs/{module-elasticsearch_user.md => 21-module-elasticsearch_user.md} (100%) diff --git a/README.md b/README.md index f3ed40fc..ffc0b7f8 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This collection installs and manages the Elastic Stack. It provides roles for every component of the Stack. Furthermore, it is possible to differentiate between Enterprise or OSS releases. -Every role is documented with all variables, please refer to the documentation found in **[Getting-Started](./docs/getting-started.md)** +Every role is documented with all variables, please refer to the documentation found in **[Getting-Started](./docs/02-getting-started.md)** **Please note**: If you are already using this collection before version `1.0.0`, please note that we had to rename a significant amount of variables due to naming schema changes made by Ansible. Please review the variables you have set in your playbooks and variable files. @@ -16,16 +16,16 @@ Every role is documented with all variables, please refer to the documentation f > [!NOTE] > Some roles have fixed requirements that you must observe. Please have a look at the [requirements](docs/01-requirements.md) before using the collection. (There is a high probability that some of them will be refactored soon) -* [Beats](docs/role-beats.md) -* [Elasticsearch](docs/role-elasticsearch.md) -* [Kibana](docs/role-kibana.md) -* [Logstash](docs/role-logstash.md) -* [Repos](docs/role-repos.md) +* [Beats](docs/14-role-beats.md) +* [Elasticsearch](docs/11-role-elasticsearch.md) +* [Kibana](docs/15-role-kibana.md) +* [Logstash](docs/12-role-logstash.md) +* [Repos](docs/10-role-repos.md) ## Modules documentation -* [elasticsearch_role](docs/module-elasticsearch_role.md) -* [elasticsearch_user](docs/module-elasticsearch_user.md) +* [elasticsearch_role](docs/20-module-elasticsearch_role.md) +* [elasticsearch_user](docs/21-module-elasticsearch_user.md) ## Global variables @@ -86,7 +86,7 @@ We know from personal experience, that the collections work in following combina ## Usage -Every role is documented with all variables, please refer to the documentation found in **[Getting-Started](./docs/getting-started.md)** +Every role is documented with all variables, please refer to the documentation found in **[Getting-Started](./docs/02-getting-started.md)** Make sure all hosts that should be configured are part of your playbook. (See below for details on groups etc.). The collection is built to first collect all facts from all hosts (including those only running beats) and then use facts like hostnames or ip addresses to connect the tools to each other. diff --git a/docs/getting-started.md b/docs/02-getting-started.md similarity index 88% rename from docs/getting-started.md rename to docs/02-getting-started.md index 79569b50..62b00aca 100644 --- a/docs/getting-started.md +++ b/docs/02-getting-started.md @@ -47,11 +47,11 @@ Or refer to the role with the FQCN of the role. Roles ------- -* [Beats](role-beats.md) -* [Elasticsearch](role-elasticsearch.md) -* [Kibana](role-kibana.md) -* [Logstash](role-logstash.md) -* [Repos](role-repos.md) +* [Beats](14-role-beats.md) +* [Elasticsearch](11-role-elasticsearch.md) +* [Kibana](15-role-kibana.md) +* [Logstash](12-role-logstash.md) +* [Repos](10-role-repos.md) Variables diff --git a/docs/role-repos.md b/docs/10-role-repos.md similarity index 100% rename from docs/role-repos.md rename to docs/10-role-repos.md diff --git a/docs/role-elasticsearch.md b/docs/11-role-elasticsearch.md similarity index 100% rename from docs/role-elasticsearch.md rename to docs/11-role-elasticsearch.md diff --git a/docs/role-logstash.md b/docs/12-role-logstash.md similarity index 99% rename from docs/role-logstash.md rename to docs/12-role-logstash.md index c318f504..f098dc6a 100644 --- a/docs/role-logstash.md +++ b/docs/12-role-logstash.md @@ -9,7 +9,7 @@ It can optionally configure two types of Logstash pipelines: * Pipeline configuration managed in an external git repository * A default pipeline which will read from different Redis keys and write into Elasticsearch -For details on how to configure pipelines please refer to our [docs about pipelines](./logstash-pipelines.md). +For details on how to configure pipelines please refer to our [docs about pipelines](./13-logstash-pipelines.md). Details about configured pipelines will be written into `pipelines.yml` as comments. Same goes for logging configuration in `log4j.options`. @@ -24,7 +24,7 @@ You will need these packages / libraries installed. Some very basic packages lik * `passlib` Python library if you do not disable password hashing for logstash user. It should be installed with pip on the Ansible controller. -You need to have the Elastic Repos configured on your system. You can use our [role](./role-repos.md) +You need to have the Elastic Repos configured on your system. You can use our [role](./14-role-repos.md) If you want to use the default pipeline configuration you need to have `git` available. diff --git a/docs/logstash-pipelines.md b/docs/13-logstash-pipelines.md similarity index 100% rename from docs/logstash-pipelines.md rename to docs/13-logstash-pipelines.md diff --git a/docs/role-beats.md b/docs/14-role-beats.md similarity index 100% rename from docs/role-beats.md rename to docs/14-role-beats.md diff --git a/docs/role-kibana.md b/docs/15-role-kibana.md similarity index 100% rename from docs/role-kibana.md rename to docs/15-role-kibana.md diff --git a/docs/module-elasticsearch_role.md b/docs/20-module-elasticsearch_role.md similarity index 100% rename from docs/module-elasticsearch_role.md rename to docs/20-module-elasticsearch_role.md diff --git a/docs/module-elasticsearch_user.md b/docs/21-module-elasticsearch_user.md similarity index 100% rename from docs/module-elasticsearch_user.md rename to docs/21-module-elasticsearch_user.md From 56c57519c44e95152261f635ddde3aeeb978acfe Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Wed, 24 Sep 2025 09:35:04 +0200 Subject: [PATCH 03/10] Centralize package installation in `repos` role --- docs/10-role-repos.md | 10 ++++++---- roles/repos/tasks/debian.yml | 7 ------- roles/repos/tasks/main.yml | 23 +++++++++++++++++++++++ roles/repos/tasks/redhat.yml | 7 ------- roles/repos/tasks/suse.yml | 6 ------ roles/repos/vars/Debian.yml | 5 +++++ roles/repos/vars/RedHat.yml | 3 +++ roles/repos/vars/Suse.yml | 3 +++ roles/repos/vars/default.yml | 1 + 9 files changed, 41 insertions(+), 24 deletions(-) create mode 100644 roles/repos/vars/Debian.yml create mode 100644 roles/repos/vars/RedHat.yml create mode 100644 roles/repos/vars/Suse.yml create mode 100644 roles/repos/vars/default.yml diff --git a/docs/10-role-repos.md b/docs/10-role-repos.md index a47e62ce..237cbab3 100644 --- a/docs/10-role-repos.md +++ b/docs/10-role-repos.md @@ -8,10 +8,12 @@ The role adds Elastic repositories to the package manager. It's main use is in c Requirements ------------ -* You need `gpg` to be installed because packages / repositories are digitally signed and verified. -* Debian and Ubuntu hosts need to have `apt-transport-https` installed to deal with Elastics repositories. -* Ubuntu hosts also need to have `gpg-agent` installed. -* For SuSE hosts you need the Ansible collection `community.general` on your Ansible controller. +GPG needs to be installed on the systems to verify the package signature. This will be installed as part of the role. Below you can find a list of packages that will be installed. +* Debian family: `apt-transport-https`, `gpg` and `gpg-agent` +* RedHat family: `gnupg` +* Suse family: `gpg2` + +For SuSE hosts you will need the Ansible collection `community.general` on your Ansible controller. Role Variables -------------- diff --git a/roles/repos/tasks/debian.yml b/roles/repos/tasks/debian.yml index a25effbb..d9abbcf3 100644 --- a/roles/repos/tasks/debian.yml +++ b/roles/repos/tasks/debian.yml @@ -1,11 +1,4 @@ --- -- name: Ensure gpg exists, for signing keys - ansible.builtin.apt: - name: - - gpg - - gpg-agent - state: present - - name: Ensure Elastic Stack key is removed (Debian legacy format) ansible.builtin.apt_key: url: "{{ elasticstack_repo_key }}" diff --git a/roles/repos/tasks/main.yml b/roles/repos/tasks/main.yml index d3390966..67609d13 100644 --- a/roles/repos/tasks/main.yml +++ b/roles/repos/tasks/main.yml @@ -5,6 +5,29 @@ name: netways.elasticstack.elasticstack when: not hostvars[inventory_hostname]._elasticstack_role_imported | default(false) +- name: Include OS specific vars + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" + vars: + params: + files: + - "{{ ansible_os_family }}-{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml" + - "{{ ansible_os_family }}-{{ ansible_distribution_major_version }}.yml" + - "{{ ansible_os_family }}-{{ ansible_distribution }}.yml" + - "{{ ansible_os_family }}-{{ ansible_lsb.id if ansible_lsb.id is defined else ansible_distribution }}.yml" + - "{{ ansible_os_family }}.yml" + - default.yml + paths: + - "{{ role_path }}/vars" + +- name: Install package dependencies + ansible.builtin.package: + name: "{{ item }}" + state: present + loop: "{{ repos_packages }}" + when: + - repos_packages is defined + - repos_packages | length > 0 + - name: Import RedHat tasks ansible.builtin.import_tasks: redhat.yml when: ansible_os_family == 'RedHat' diff --git a/roles/repos/tasks/redhat.yml b/roles/repos/tasks/redhat.yml index 01388e8d..bd4b82ea 100644 --- a/roles/repos/tasks/redhat.yml +++ b/roles/repos/tasks/redhat.yml @@ -1,13 +1,6 @@ --- - # See https://github.com/elastic/elasticsearch/issues/85876 # for more information why this is needed - -- name: Ensure gpg exists, for signing keys - ansible.builtin.package: - name: gnupg - state: present - - name: Workaround for EL > 8 when: - ansible_distribution_major_version >= "9" diff --git a/roles/repos/tasks/suse.yml b/roles/repos/tasks/suse.yml index ae9d1ead..6bfbca1a 100644 --- a/roles/repos/tasks/suse.yml +++ b/roles/repos/tasks/suse.yml @@ -1,10 +1,4 @@ --- - -- name: Ensure gpg exists, for signing keys - community.general.zypper: - name: gpg2 - state: present - - name: Ensure Elastic Stack yum repository is configured (SuSE) community.general.zypper_repository: name: elastic-{{ elasticstack_release }}.x diff --git a/roles/repos/vars/Debian.yml b/roles/repos/vars/Debian.yml new file mode 100644 index 00000000..92549600 --- /dev/null +++ b/roles/repos/vars/Debian.yml @@ -0,0 +1,5 @@ +--- +repos_packages: + - gpg + - gpg-agent + - apt-transport-https diff --git a/roles/repos/vars/RedHat.yml b/roles/repos/vars/RedHat.yml new file mode 100644 index 00000000..0e9181d6 --- /dev/null +++ b/roles/repos/vars/RedHat.yml @@ -0,0 +1,3 @@ +--- +repos_packages: + - gnupg diff --git a/roles/repos/vars/Suse.yml b/roles/repos/vars/Suse.yml new file mode 100644 index 00000000..36f9c68b --- /dev/null +++ b/roles/repos/vars/Suse.yml @@ -0,0 +1,3 @@ +--- +repos_packages: + - gpg2 diff --git a/roles/repos/vars/default.yml b/roles/repos/vars/default.yml new file mode 100644 index 00000000..ed97d539 --- /dev/null +++ b/roles/repos/vars/default.yml @@ -0,0 +1 @@ +--- From c5b90a3252762d8ce32af88a1755d35d4d1b868b Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Wed, 24 Sep 2025 10:39:04 +0200 Subject: [PATCH 04/10] Remove workaround for EL > 8 There were an known issue with EL 9 that has been solved by elastic itself. We dont need it anymore. https://github.com/elastic/elasticsearch/issues/85876 --- molecule/elasticstack_default/converge.yml | 4 ---- molecule/repos_default/converge.yml | 1 - molecule/repos_oss/converge.yml | 1 - roles/elasticstack/defaults/main.yml | 1 - roles/repos/tasks/redhat.yml | 28 ---------------------- 5 files changed, 35 deletions(-) diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml index 54f7aa2d..6f808869 100644 --- a/molecule/elasticstack_default/converge.yml +++ b/molecule/elasticstack_default/converge.yml @@ -28,10 +28,6 @@ kibana_extra_config: |- ops.interval: 5000 tasks: - - name: Enable Elastic installation on RHEL 9 - ansible.builtin.set_fact: - elasticstack_rpm_workaround: true - when: ansible_os_family == 'RedHat' and ansible_distribution_major_version >= "9" - name: Include Redis ansible.builtin.include_role: name: geerlingguy.redis diff --git a/molecule/repos_default/converge.yml b/molecule/repos_default/converge.yml index 80864563..26d926c4 100644 --- a/molecule/repos_default/converge.yml +++ b/molecule/repos_default/converge.yml @@ -4,7 +4,6 @@ collections: - netways.elasticstack vars: - elasticstack_rpm_workaround: true elasticstack_collection_managed: false elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}" elasticstack_no_log: false diff --git a/molecule/repos_oss/converge.yml b/molecule/repos_oss/converge.yml index 49416223..df70203e 100644 --- a/molecule/repos_oss/converge.yml +++ b/molecule/repos_oss/converge.yml @@ -7,7 +7,6 @@ - netways.elasticstack vars: elasticstack_variant: oss - elasticstack_rpm_workaround: true elasticstack_release: 7 elasticstack_no_log: false tasks: diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index 132c0345..9bdc8dbc 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -20,7 +20,6 @@ elasticstack_kibana_port: 5601 elasticstack_override_beats_tls: false elasticstack_release: 8 elasticstack_repo_key: https://artifacts.elastic.co/GPG-KEY-elasticsearch -elasticstack_rpm_workaround: false elasticstack_security: true elasticstack_variant: elastic elasticstack_force_pip: false diff --git a/roles/repos/tasks/redhat.yml b/roles/repos/tasks/redhat.yml index bd4b82ea..a798c6c0 100644 --- a/roles/repos/tasks/redhat.yml +++ b/roles/repos/tasks/redhat.yml @@ -1,32 +1,4 @@ --- -# See https://github.com/elastic/elasticsearch/issues/85876 -# for more information why this is needed -- name: Workaround for EL > 8 - when: - - ansible_distribution_major_version >= "9" - block: - - - name: Show a warning - ansible.builtin.debug: - msg: "For this workaround to work, please set elasticstack_rpm_workaround to true" - when: - - not elasticstack_rpm_workaround | bool - - - name: Enable workaround for rpm keys - when: - - elasticstack_rpm_workaround | bool - block: - - - name: Install crypto-policies-scripts - ansible.builtin.package: - name: crypto-policies-scripts - - # since we don't expect to have that workaround for long - # we can skip having idempotency checks fixed - - name: Set Crypto policies to legacy - ansible.builtin.command: "update-crypto-policies --set LEGACY" - changed_when: false - - name: Ensure Elastic repository key is available (RedHat) ansible.builtin.rpm_key: key: "{{ elasticstack_repo_key }}" From 2e7d8081546e5fc3b18821e57eaa09dff77fcae7 Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Wed, 24 Sep 2025 12:31:48 +0200 Subject: [PATCH 05/10] Move and rename `elasticstack_enable_repos` to the `repos` role This variable is only part of the `repos` role. There is no need to define it inside the `elasticstack` role. --- docs/10-role-repos.md | 39 ++++++++++++++-------------- roles/elasticstack/defaults/main.yml | 1 - roles/repos/tasks/redhat.yml | 4 +-- roles/repos/tasks/suse.yml | 4 +-- 4 files changed, 23 insertions(+), 25 deletions(-) diff --git a/docs/10-role-repos.md b/docs/10-role-repos.md index 237cbab3..e45c945c 100644 --- a/docs/10-role-repos.md +++ b/docs/10-role-repos.md @@ -1,12 +1,8 @@ -Elastic Repos -========= +# Role `repos` -![Test Role repos](https://github.com/netways/ansible-collection-elasticstack/actions/workflows/test_role_repos.yml/badge.svg) +This role will add the Elastic repositories to the package manager (dnf/yum. apt or zypper). It will not install packages (these can be done with the role `elasticsearch`). -The role adds Elastic repositories to the package manager. It's main use is in connection with other roles that provide installation and configuration of the Elastic Stack. - -Requirements ------------- +## Requirements GPG needs to be installed on the systems to verify the package signature. This will be installed as part of the role. Below you can find a list of packages that will be installed. * Debian family: `apt-transport-https`, `gpg` and `gpg-agent` @@ -15,25 +11,28 @@ GPG needs to be installed on the systems to verify the package signature. This w For SuSE hosts you will need the Ansible collection `community.general` on your Ansible controller. -Role Variables --------------- +## Variables + +There are variables that are needed in more than one role of the collection. These are defined inside the "meta" role `elasticstack`. The role `elasticstack` is imported in this role. -* *elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`). `7` and `8` are supported. -* *elasticstack_variant*: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`). -* *elasticstack_enable_repos*: Enable repositories after creating them. (default: `true`) Only works on RPM based distributions! +**List of variables from `elasticsearch` used inside `repos`:** +* `elasticstack_release`: Major release version of Elastic stack to configure. (default: `7`). `7` and `8` are supported. +* `elasticstack_variant`: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`). Please note that no `oss` versions are available for Elastic Stack later than `7`. This role will fail if you try to install them. -Usage --------- +**Variables that are explicity defined inside `repos`:** +* `repos_enable_repos`: Enable repositories after creating them. Only works on RPM based distributions (default: `true`) + +## Usage -Upgrades -======== +### Upgrades -If you want to be able to update your operating system without worrying about accidentally upgrading Elastic Stack, set `elasticstack_enable_repos` to `false`. The roles in this collection will enable the repository in case they need it. Keep in mind that this will only work on rpm based distributions. +If you want to be able to update your operating system without worrying about accidentally upgrading Elastic Stack, set `repos_enable_repos` to `false`. The roles in this collection will enable the repository in case they need it. Keep in mind that this will only work on rpm based distributions. -Example playbook -================ +### Example playbook + +The following playbook will add the Elastic repository to the package manager. ``` - hosts: all @@ -42,4 +41,4 @@ Example playbook - netways.elasticstack roles: - repos -``` +``` \ No newline at end of file diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index 9bdc8dbc..1e1afe7c 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -13,7 +13,6 @@ elasticstack_ca_pass: PleaseChangeMe elasticstack_ca_validity_period: 1095 elasticstack_ca_will_expire_soon: false elasticstack_elasticsearch_http_port: 9200 -elasticstack_enable_repos: true elasticstack_collection_managed: true elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords elasticstack_kibana_port: 5601 diff --git a/roles/repos/tasks/redhat.yml b/roles/repos/tasks/redhat.yml index a798c6c0..b644a363 100644 --- a/roles/repos/tasks/redhat.yml +++ b/roles/repos/tasks/redhat.yml @@ -12,7 +12,7 @@ baseurl: https://artifacts.elastic.co/packages/{{ elasticstack_release }}.x/yum gpgcheck: yes gpgkey: "{{ elasticstack_repo_key }}" - enabled: "{{ elasticstack_enable_repos | bool }}" + enabled: "{{ repos_enable_repos | bool }}" when: elasticstack_variant == "elastic" - name: Ensure Elastic Stack OSS yum repository is configured (RedHat) @@ -23,5 +23,5 @@ baseurl: https://artifacts.elastic.co/packages/oss-{{ elasticstack_release }}.x/yum gpgcheck: yes gpgkey: "{{ elasticstack_repo_key }}" - enabled: "{{ elasticstack_enable_repos | bool }}" + enabled: "{{ repos_enable_repos | bool }}" when: elasticstack_variant == "oss" diff --git a/roles/repos/tasks/suse.yml b/roles/repos/tasks/suse.yml index 6bfbca1a..344c0811 100644 --- a/roles/repos/tasks/suse.yml +++ b/roles/repos/tasks/suse.yml @@ -5,7 +5,7 @@ description: Elastic Release {{ elasticstack_release }}.x repo: https://artifacts.elastic.co/packages/{{ elasticstack_release }}.x/yum auto_import_keys: true - enabled: "{{ elasticstack_enable_repos | bool }}" + enabled: "{{ repos_enable_repos | bool }}" when: elasticstack_variant == "elastic" - name: Ensure Elastic Stack OSS yum repository is configured (SuSE) @@ -14,5 +14,5 @@ description: Elastic OSS Release {{ elasticstack_release }}.x repo: https://artifacts.elastic.co/packages/oss-{{ elasticstack_release }}.x/yum auto_import_keys: true - enabled: "{{ elasticstack_enable_repos | bool }}" + enabled: "{{ repos_enable_repos | bool }}" when: elasticstack_variant == "oss" From 4aa385fc259a362b9b89dc40df25609d5d8e26f9 Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Wed, 24 Sep 2025 12:37:52 +0200 Subject: [PATCH 06/10] Introduce `repos_deb_remove_legacy_key` `repos_deb_remove_legacy_key` can be used to remove an existing key from the trusted.gpg keyring --- docs/10-role-repos.md | 1 + roles/repos/defaults/main.yml | 2 ++ roles/repos/tasks/debian.yml | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 roles/repos/defaults/main.yml diff --git a/docs/10-role-repos.md b/docs/10-role-repos.md index e45c945c..0ae23fa8 100644 --- a/docs/10-role-repos.md +++ b/docs/10-role-repos.md @@ -23,6 +23,7 @@ Please note that no `oss` versions are available for Elastic Stack later than `7 **Variables that are explicity defined inside `repos`:** * `repos_enable_repos`: Enable repositories after creating them. Only works on RPM based distributions (default: `true`) +* `repos_deb_remove_legacy_key`: Remove repository key from legacy trusted.gpg keyring (Only needed on older distributions with apt as package manager. Details can be found in [this](https://askubuntu.com/questions/1286545/what-commands-exactly-should-replace-the-deprecated-apt-key) post) (Default: `false`) ## Usage diff --git a/roles/repos/defaults/main.yml b/roles/repos/defaults/main.yml new file mode 100644 index 00000000..8c276089 --- /dev/null +++ b/roles/repos/defaults/main.yml @@ -0,0 +1,2 @@ +--- +repos_deb_remove_legacy_key: false diff --git a/roles/repos/tasks/debian.yml b/roles/repos/tasks/debian.yml index d9abbcf3..5f9f8a0f 100644 --- a/roles/repos/tasks/debian.yml +++ b/roles/repos/tasks/debian.yml @@ -3,6 +3,7 @@ ansible.builtin.apt_key: url: "{{ elasticstack_repo_key }}" state: absent + when: repos_deb_remove_legacy_key | bool - name: Ensure Elastic Stack key is available (Debian) ansible.builtin.get_url: @@ -19,6 +20,7 @@ - "oss-7" - "8" - "oss-8" + when: repos_deb_remove_legacy_key | bool - name: Ensure Elastic Stack apt repository is configured (Debian) ansible.builtin.apt_repository: From e61ca32bff85a5607eebc82ad0be4a1d9f1e32bc Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Wed, 24 Sep 2025 12:41:26 +0200 Subject: [PATCH 07/10] Add missing definition for `repos_enable_repos` --- roles/repos/vars/RedHat.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/repos/vars/RedHat.yml b/roles/repos/vars/RedHat.yml index 0e9181d6..deeaf096 100644 --- a/roles/repos/vars/RedHat.yml +++ b/roles/repos/vars/RedHat.yml @@ -1,3 +1,4 @@ --- +repos_enable_repos: true repos_packages: - gnupg From 79c30099eb13e5c2de8e19a94b72571ac33c9e21 Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Wed, 24 Sep 2025 13:17:14 +0200 Subject: [PATCH 08/10] Streamline yaml sytnax, comments and newlines Keep an straight line for comments in templates, newlines in yaml files and yaml syntxa in general --- roles/beats/defaults/main.yml | 1 - roles/beats/handlers/main.yml | 2 -- roles/beats/tasks/auditbeat.yml | 1 - roles/beats/tasks/filebeat.yml | 1 - roles/beats/tasks/main.yml | 1 - roles/beats/tasks/metricbeat.yml | 1 - roles/beats/templates/auditbeat.yml.j2 | 2 ++ roles/beats/templates/filebeat-system.yml.j2 | 2 ++ roles/beats/templates/filebeat.yml.j2 | 2 ++ roles/beats/templates/metricbeat.yml.j2 | 2 ++ roles/elasticsearch/defaults/main.yml | 1 - roles/elasticsearch/handlers/main.yml | 1 - roles/elasticsearch/tasks/elasticsearch-keystore.yml | 1 - .../elasticsearch/tasks/elasticsearch-rolling-upgrade.yml | 7 ++----- roles/elasticsearch/tasks/elasticsearch-security.yml | 1 - roles/elasticsearch/tasks/main.yml | 1 - roles/elasticsearch/tasks/wait_for_instance.yml | 1 - roles/elasticsearch/templates/elasticsearch.yml.j2 | 1 - roles/elasticsearch/templates/elasticsearch_initialized.j2 | 3 ++- roles/elasticsearch/templates/instances.j2 | 2 ++ .../templates/jvm.options.d/custom.options.j2 | 2 +- .../elasticsearch/templates/jvm.options.d/heap.options.j2 | 3 ++- .../elasticsearch/templates/jvm.options.d/paths.options.j2 | 2 +- roles/elasticsearch/vars/Debian.yml | 1 - roles/elasticsearch/vars/RedHat.yml | 1 - roles/elasticsearch/vars/main.yml | 1 - roles/elasticstack/defaults/main.yml | 1 - roles/elasticstack/tasks/elasticstack-passwords.yml | 1 - roles/elasticstack/tasks/elasticstack-versions.yml | 1 - roles/elasticstack/vars/Debian.yml | 1 - roles/elasticstack/vars/RedHat.yml | 1 - roles/elasticstack/vars/main.yml | 1 - roles/kibana/defaults/main.yml | 1 - roles/kibana/handlers/main.yml | 1 - roles/kibana/meta/main.yml | 1 + roles/kibana/tasks/kibana-security.yml | 1 - roles/kibana/tasks/main.yml | 1 - roles/kibana/templates/kibana.yml.j2 | 2 ++ roles/logstash/defaults/main.yml | 1 - roles/logstash/handlers/main.yml | 1 - roles/logstash/tasks/logstash-mermaid.yml | 1 - roles/logstash/tasks/logstash-security.yml | 1 - roles/logstash/tasks/main.yml | 1 - roles/logstash/tasks/manage_pipeline.yml | 1 - roles/logstash/templates/beats-input.conf.j2 | 2 ++ roles/logstash/templates/elasticsearch-output.conf.j2 | 2 ++ roles/logstash/templates/log4j2.properties.j2 | 5 ++--- roles/logstash/templates/logstash.yml.j2 | 2 ++ roles/logstash/templates/pipelines.mermaid.j2 | 3 +-- roles/logstash/templates/pipelines.yml.j2 | 3 +-- roles/logstash/templates/redis-input.conf.j2 | 2 ++ roles/logstash/templates/redis-output.conf.j2 | 2 ++ roles/logstash/templates/simple-input.conf.j2 | 2 ++ roles/logstash/templates/simple-output.conf.j2 | 2 ++ roles/repos/meta/main.yml | 1 + roles/repos/tasks/main.yml | 1 - 56 files changed, 40 insertions(+), 50 deletions(-) diff --git a/roles/beats/defaults/main.yml b/roles/beats/defaults/main.yml index ae7b8d9b..2a9cb68e 100644 --- a/roles/beats/defaults/main.yml +++ b/roles/beats/defaults/main.yml @@ -1,5 +1,4 @@ --- -# defaults file for beats beats_security: false beats_filebeat: true beats_auditbeat: false diff --git a/roles/beats/handlers/main.yml b/roles/beats/handlers/main.yml index a584dbb6..43fc1d10 100644 --- a/roles/beats/handlers/main.yml +++ b/roles/beats/handlers/main.yml @@ -1,6 +1,4 @@ --- -# handlers file for beats -# - name: Restart Filebeat ansible.builtin.service: name: filebeat diff --git a/roles/beats/tasks/auditbeat.yml b/roles/beats/tasks/auditbeat.yml index 2a8985d3..a690da9e 100644 --- a/roles/beats/tasks/auditbeat.yml +++ b/roles/beats/tasks/auditbeat.yml @@ -1,5 +1,4 @@ --- - - name: Construct exact name of Auditbeat package ansible.builtin.set_fact: beats_auditbeat_package: >- diff --git a/roles/beats/tasks/filebeat.yml b/roles/beats/tasks/filebeat.yml index 2f0cb54a..5676bcba 100644 --- a/roles/beats/tasks/filebeat.yml +++ b/roles/beats/tasks/filebeat.yml @@ -1,5 +1,4 @@ --- - - name: Construct exact name of Filebeat package ansible.builtin.set_fact: beats_filebeat_package: >- diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml index 7b08cc2e..7817a3ee 100644 --- a/roles/beats/tasks/main.yml +++ b/roles/beats/tasks/main.yml @@ -1,5 +1,4 @@ --- - - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack diff --git a/roles/beats/tasks/metricbeat.yml b/roles/beats/tasks/metricbeat.yml index f4b91a99..9545fd52 100644 --- a/roles/beats/tasks/metricbeat.yml +++ b/roles/beats/tasks/metricbeat.yml @@ -1,5 +1,4 @@ --- - - name: Construct exact name of Metricbeat package ansible.builtin.set_fact: beats_metricbeat_package: >- diff --git a/roles/beats/templates/auditbeat.yml.j2 b/roles/beats/templates/auditbeat.yml.j2 index 511a606c..e9a7de6e 100644 --- a/roles/beats/templates/auditbeat.yml.j2 +++ b/roles/beats/templates/auditbeat.yml.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + auditbeat.modules: - module: auditd audit_rule_files: [ '${path.config}/audit.rules.d/*.conf' ] diff --git a/roles/beats/templates/filebeat-system.yml.j2 b/roles/beats/templates/filebeat-system.yml.j2 index af69b2b4..f87a4f64 100644 --- a/roles/beats/templates/filebeat-system.yml.j2 +++ b/roles/beats/templates/filebeat-system.yml.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + - module: system syslog: enabled: true diff --git a/roles/beats/templates/filebeat.yml.j2 b/roles/beats/templates/filebeat.yml.j2 index 48c59ce1..26fbe2ad 100644 --- a/roles/beats/templates/filebeat.yml.j2 +++ b/roles/beats/templates/filebeat.yml.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + filebeat.inputs: {% if beats_filebeat_log_input | bool %} diff --git a/roles/beats/templates/metricbeat.yml.j2 b/roles/beats/templates/metricbeat.yml.j2 index 8e227828..c933aceb 100644 --- a/roles/beats/templates/metricbeat.yml.j2 +++ b/roles/beats/templates/metricbeat.yml.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + --- metricbeat.config.modules: path: ${path.config}/modules.d/*.yml diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml index 29aaa0c6..1cb65355 100644 --- a/roles/elasticsearch/defaults/main.yml +++ b/roles/elasticsearch/defaults/main.yml @@ -1,5 +1,4 @@ --- -# defaults file for elasticsearch elasticsearch_enable: true elasticsearch_ml_enabled: true elasticsearch_config_backup: false diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml index bde0880a..be9ec69d 100644 --- a/roles/elasticsearch/handlers/main.yml +++ b/roles/elasticsearch/handlers/main.yml @@ -1,5 +1,4 @@ --- -# handlers file for elasticsearch - name: Restart Elasticsearch ansible.builtin.service: name: elasticsearch diff --git a/roles/elasticsearch/tasks/elasticsearch-keystore.yml b/roles/elasticsearch/tasks/elasticsearch-keystore.yml index 31481fc8..5df763db 100644 --- a/roles/elasticsearch/tasks/elasticsearch-keystore.yml +++ b/roles/elasticsearch/tasks/elasticsearch-keystore.yml @@ -1,5 +1,4 @@ --- - - name: Create keystore ansible.builtin.command: /usr/share/elasticsearch/bin/elasticsearch-keystore create args: diff --git a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml index 4c8d9cec..9b049018 100644 --- a/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml +++ b/roles/elasticsearch/tasks/elasticsearch-rolling-upgrade.yml @@ -1,13 +1,10 @@ -# Ansible -# +--- # Rolling Upgrade of Elasticsearch with security on # Source from: author: Jeff Steinmetz, @jeffsteinmetz; Bin Li, @holysoros # Modifications: author: Daniel Neuberger @netways.de # More modifications: NETWAYS Professional Services GmbH # latest tested with Ansible 2.9 and later - ---- - +# # For now we support upgrade only for clusters with security enabled # If you positively need support for safely upgrading clusters without security, # feel free to open an issue at https://github.com/NETWAYS/ansible-collection-elasticstack/issues diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index d1fd178a..7d045fa2 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -1,5 +1,4 @@ --- - - name: Ensure ca exists ansible.builtin.stat: path: "{{ elasticstack_ca_dir }}/elastic-stack-ca.p12" diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index e2d538ab..b8e74189 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -1,5 +1,4 @@ --- - - name: Check for versions ansible.builtin.fail: msg: "No OSS package with version later than 7 is available for Elasticsearch" diff --git a/roles/elasticsearch/tasks/wait_for_instance.yml b/roles/elasticsearch/tasks/wait_for_instance.yml index 6bcd90df..1cdd2ad9 100644 --- a/roles/elasticsearch/tasks/wait_for_instance.yml +++ b/roles/elasticsearch/tasks/wait_for_instance.yml @@ -1,5 +1,4 @@ --- - - name: Wait for instance ansible.builtin.wait_for: host: "{{ elasticsearch_api_host }}" diff --git a/roles/elasticsearch/templates/elasticsearch.yml.j2 b/roles/elasticsearch/templates/elasticsearch.yml.j2 index 1d4238cd..ea486c22 100644 --- a/roles/elasticsearch/templates/elasticsearch.yml.j2 +++ b/roles/elasticsearch/templates/elasticsearch.yml.j2 @@ -1,4 +1,3 @@ -# test {{ ansible_managed | comment }} node.name: "{{ elasticsearch_nodename }}" diff --git a/roles/elasticsearch/templates/elasticsearch_initialized.j2 b/roles/elasticsearch/templates/elasticsearch_initialized.j2 index c7c8f395..4bda1b53 100644 --- a/roles/elasticsearch/templates/elasticsearch_initialized.j2 +++ b/roles/elasticsearch/templates/elasticsearch_initialized.j2 @@ -1,2 +1,3 @@ -{{ ansible_managed }} +{{ ansible_managed | comment }} + green diff --git a/roles/elasticsearch/templates/instances.j2 b/roles/elasticsearch/templates/instances.j2 index 226f81a4..4f776326 100644 --- a/roles/elasticsearch/templates/instances.j2 +++ b/roles/elasticsearch/templates/instances.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + instances: {% for host in groups[elasticstack_elasticsearch_group_name] %} - name: "{{ host }}" diff --git a/roles/elasticsearch/templates/jvm.options.d/custom.options.j2 b/roles/elasticsearch/templates/jvm.options.d/custom.options.j2 index 0135ca0a..a32a2f07 100644 --- a/roles/elasticsearch/templates/jvm.options.d/custom.options.j2 +++ b/roles/elasticsearch/templates/jvm.options.d/custom.options.j2 @@ -1,4 +1,4 @@ -# {{ ansible_managed }} +{{ ansible_managed | comment }} {% for item in elasticsearch_jvm_custom_parameters %} {{ item }} diff --git a/roles/elasticsearch/templates/jvm.options.d/heap.options.j2 b/roles/elasticsearch/templates/jvm.options.d/heap.options.j2 index 238cdbdf..16713282 100644 --- a/roles/elasticsearch/templates/jvm.options.d/heap.options.j2 +++ b/roles/elasticsearch/templates/jvm.options.d/heap.options.j2 @@ -1,4 +1,5 @@ -# {{ ansible_managed }} +{{ ansible_managed | comment }} + # Xms represents the initial size of total heap space # Xmx represents the maximum size of total heap space -Xms{{ elasticsearch_heap }}g diff --git a/roles/elasticsearch/templates/jvm.options.d/paths.options.j2 b/roles/elasticsearch/templates/jvm.options.d/paths.options.j2 index b12bcd81..e0046170 100644 --- a/roles/elasticsearch/templates/jvm.options.d/paths.options.j2 +++ b/roles/elasticsearch/templates/jvm.options.d/paths.options.j2 @@ -1,4 +1,4 @@ -# {{ ansible_managed }} +{{ ansible_managed | comment }} # specify an alternative path for heap dumps; ensure the directory exists and # has sufficient space diff --git a/roles/elasticsearch/vars/Debian.yml b/roles/elasticsearch/vars/Debian.yml index bb0878c1..edec644a 100644 --- a/roles/elasticsearch/vars/Debian.yml +++ b/roles/elasticsearch/vars/Debian.yml @@ -1,3 +1,2 @@ --- - elasticsearch_sysconfig_file: /etc/default/elasticsearch diff --git a/roles/elasticsearch/vars/RedHat.yml b/roles/elasticsearch/vars/RedHat.yml index f0dbc02a..93488480 100644 --- a/roles/elasticsearch/vars/RedHat.yml +++ b/roles/elasticsearch/vars/RedHat.yml @@ -1,3 +1,2 @@ --- - elasticsearch_sysconfig_file: /etc/sysconfig/elasticsearch diff --git a/roles/elasticsearch/vars/main.yml b/roles/elasticsearch/vars/main.yml index 140916a9..ed97d539 100644 --- a/roles/elasticsearch/vars/main.yml +++ b/roles/elasticsearch/vars/main.yml @@ -1,2 +1 @@ --- -# vars file for elasticsearch diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index 1e1afe7c..b47b2b16 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -1,5 +1,4 @@ --- - elasticstack_elasticsearch_group_name: elasticsearch elasticstack_logstash_group_name: logstash elasticstack_kibana_group_name: kibana diff --git a/roles/elasticstack/tasks/elasticstack-passwords.yml b/roles/elasticstack/tasks/elasticstack-passwords.yml index 9a285685..f171131f 100644 --- a/roles/elasticstack/tasks/elasticstack-passwords.yml +++ b/roles/elasticstack/tasks/elasticstack-passwords.yml @@ -1,5 +1,4 @@ --- - - name: Check for passwords being set ansible.builtin.stat: path: "{{ elasticstack_initial_passwords }}" diff --git a/roles/elasticstack/tasks/elasticstack-versions.yml b/roles/elasticstack/tasks/elasticstack-versions.yml index c135ac56..0ca01990 100644 --- a/roles/elasticstack/tasks/elasticstack-versions.yml +++ b/roles/elasticstack/tasks/elasticstack-versions.yml @@ -1,5 +1,4 @@ --- - - name: Gather package facts ansible.builtin.package_facts: manager: auto diff --git a/roles/elasticstack/vars/Debian.yml b/roles/elasticstack/vars/Debian.yml index 3d9e31b5..bf8a928d 100644 --- a/roles/elasticstack/vars/Debian.yml +++ b/roles/elasticstack/vars/Debian.yml @@ -1,3 +1,2 @@ --- - elasticstack_versionseparator: "=" diff --git a/roles/elasticstack/vars/RedHat.yml b/roles/elasticstack/vars/RedHat.yml index a8d601fe..f7fa008c 100644 --- a/roles/elasticstack/vars/RedHat.yml +++ b/roles/elasticstack/vars/RedHat.yml @@ -1,3 +1,2 @@ --- - elasticstack_versionseparator: "-" diff --git a/roles/elasticstack/vars/main.yml b/roles/elasticstack/vars/main.yml index ed456934..ed97d539 100644 --- a/roles/elasticstack/vars/main.yml +++ b/roles/elasticstack/vars/main.yml @@ -1,2 +1 @@ --- -# vars file for logstash diff --git a/roles/kibana/defaults/main.yml b/roles/kibana/defaults/main.yml index 725d70f5..d99c4f78 100644 --- a/roles/kibana/defaults/main.yml +++ b/roles/kibana/defaults/main.yml @@ -1,5 +1,4 @@ --- -# defaults file for kibana kibana_enable: true kibana_config_backup: true kibana_manage_yaml: true diff --git a/roles/kibana/handlers/main.yml b/roles/kibana/handlers/main.yml index 532d014a..0e4454e3 100644 --- a/roles/kibana/handlers/main.yml +++ b/roles/kibana/handlers/main.yml @@ -1,5 +1,4 @@ --- -# handlers file for kibana - name: Restart Kibana ansible.builtin.service: name: kibana diff --git a/roles/kibana/meta/main.yml b/roles/kibana/meta/main.yml index 3e42337f..24e6a544 100644 --- a/roles/kibana/meta/main.yml +++ b/roles/kibana/meta/main.yml @@ -1,3 +1,4 @@ +--- galaxy_info: role_name: kibana author: netways diff --git a/roles/kibana/tasks/kibana-security.yml b/roles/kibana/tasks/kibana-security.yml index db4479ed..05560a0d 100644 --- a/roles/kibana/tasks/kibana-security.yml +++ b/roles/kibana/tasks/kibana-security.yml @@ -1,5 +1,4 @@ --- - - name: Ensure kibana certificate exists ansible.builtin.stat: path: "/etc/kibana/certs/{{ ansible_hostname }}-kibana.p12" diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index eb96d0d7..5b8e39ff 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -1,5 +1,4 @@ --- - - name: Check for versions ansible.builtin.fail: msg: "No OSS package with version later than 7 is available for Kibana" diff --git a/roles/kibana/templates/kibana.yml.j2 b/roles/kibana/templates/kibana.yml.j2 index 03d098cd..0fc5217b 100644 --- a/roles/kibana/templates/kibana.yml.j2 +++ b/roles/kibana/templates/kibana.yml.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + server.host: "0.0.0.0" server.publicBaseUrl: "http{% if kibana_tls | bool %}s{% endif %}://{{ elasticstack_kibana_host | default( ansible_fqdn ) }}:{{ elasticstack_kibana_port }}" diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml index 88eead19..adc4a7f6 100644 --- a/roles/logstash/defaults/main.yml +++ b/roles/logstash/defaults/main.yml @@ -1,5 +1,4 @@ --- -# defaults file for logstash logstash_enable: true logstash_config_backup: no logstash_manage_yaml: true diff --git a/roles/logstash/handlers/main.yml b/roles/logstash/handlers/main.yml index afe5a438..07d8da60 100644 --- a/roles/logstash/handlers/main.yml +++ b/roles/logstash/handlers/main.yml @@ -1,5 +1,4 @@ --- -# handlers file for logstash - name: Restart Logstash ansible.builtin.service: name: logstash diff --git a/roles/logstash/tasks/logstash-mermaid.yml b/roles/logstash/tasks/logstash-mermaid.yml index 5d904237..4319b109 100644 --- a/roles/logstash/tasks/logstash-mermaid.yml +++ b/roles/logstash/tasks/logstash-mermaid.yml @@ -1,5 +1,4 @@ --- - - name: Print Logstash pipelines in Mermaid syntax on Logstash hosts ansible.builtin.template: src: pipelines.mermaid.j2 diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index 4e8e1901..ffa9139a 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -1,5 +1,4 @@ --- - - name: Ensure logstash certificate exists ansible.builtin.stat: path: "{{ logstash_certs_dir }}/{{ ansible_hostname }}-ls.p12" diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml index d427b326..af76faab 100644 --- a/roles/logstash/tasks/main.yml +++ b/roles/logstash/tasks/main.yml @@ -1,5 +1,4 @@ --- - - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack diff --git a/roles/logstash/tasks/manage_pipeline.yml b/roles/logstash/tasks/manage_pipeline.yml index e11f4a2a..b7dfed74 100644 --- a/roles/logstash/tasks/manage_pipeline.yml +++ b/roles/logstash/tasks/manage_pipeline.yml @@ -1,5 +1,4 @@ --- - - name: Check if Logstash pipeline already exists ansible.builtin.stat: path: "/etc/logstash/conf.d/{{ pipelinename.name }}" diff --git a/roles/logstash/templates/beats-input.conf.j2 b/roles/logstash/templates/beats-input.conf.j2 index c37e7211..099cda34 100644 --- a/roles/logstash/templates/beats-input.conf.j2 +++ b/roles/logstash/templates/beats-input.conf.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + input { beats { port => {{ elasticstack_beats_port }} diff --git a/roles/logstash/templates/elasticsearch-output.conf.j2 b/roles/logstash/templates/elasticsearch-output.conf.j2 index e0fd9ae7..4f0a60ec 100644 --- a/roles/logstash/templates/elasticsearch-output.conf.j2 +++ b/roles/logstash/templates/elasticsearch-output.conf.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + {% if logstash_pipeline_identifier | bool and logstash_pipeline_identifier_defaults | bool %} filter { mutate { diff --git a/roles/logstash/templates/log4j2.properties.j2 b/roles/logstash/templates/log4j2.properties.j2 index 78a025e9..44acd2df 100644 --- a/roles/logstash/templates/log4j2.properties.j2 +++ b/roles/logstash/templates/log4j2.properties.j2 @@ -1,6 +1,5 @@ -# Managed by Ansible Role -# https://github.com/netways/ansible-role-logstash -# +{{ ansible_managed | comment }} + # Logging to logfile: {% if logstash_logging_file | bool %}true{% else %}false{% endif %} # Logging to console/syslog/journald: {% if logstash_logging_console | bool %}true{% else %}false{% endif %} diff --git a/roles/logstash/templates/logstash.yml.j2 b/roles/logstash/templates/logstash.yml.j2 index f440d6e9..6bab17f1 100644 --- a/roles/logstash/templates/logstash.yml.j2 +++ b/roles/logstash/templates/logstash.yml.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + path.data: {{ logstash_config_path_data }} config.reload.automatic: {{ logstash_config_autoreload | lower }} {% if logstash_config_autoreload | bool and logstash_config_autoreload_interval is defined %} diff --git a/roles/logstash/templates/pipelines.mermaid.j2 b/roles/logstash/templates/pipelines.mermaid.j2 index adc30895..adb8625d 100644 --- a/roles/logstash/templates/pipelines.mermaid.j2 +++ b/roles/logstash/templates/pipelines.mermaid.j2 @@ -1,5 +1,4 @@ -# Managed via Ansible role -# https://github.com/netways/ansible-role-logstash +{{ ansible_managed | comment }} # Use the following code with your favorite Mermaid editor # Or paste into: https://mermaid.live/ diff --git a/roles/logstash/templates/pipelines.yml.j2 b/roles/logstash/templates/pipelines.yml.j2 index 901088e6..81ad2973 100644 --- a/roles/logstash/templates/pipelines.yml.j2 +++ b/roles/logstash/templates/pipelines.yml.j2 @@ -1,7 +1,6 @@ --- +{{ ansible_managed | comment }} -# Managed via Ansible role -# https://github.com/netways/ansible-role-logstash {% if logstash_beats_input_congestion is defined %} # global congestion threshold: {{ logstash_beats_input_congestion }} {% endif %} diff --git a/roles/logstash/templates/redis-input.conf.j2 b/roles/logstash/templates/redis-input.conf.j2 index 0481dc53..0d64ae3f 100644 --- a/roles/logstash/templates/redis-input.conf.j2 +++ b/roles/logstash/templates/redis-input.conf.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + input { redis { host => "localhost" diff --git a/roles/logstash/templates/redis-output.conf.j2 b/roles/logstash/templates/redis-output.conf.j2 index 59c5a578..231aae67 100644 --- a/roles/logstash/templates/redis-output.conf.j2 +++ b/roles/logstash/templates/redis-output.conf.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + {% if logstash_pipeline_identifier | bool and logstash_pipeline_identifier_defaults | bool %} filter { mutate { diff --git a/roles/logstash/templates/simple-input.conf.j2 b/roles/logstash/templates/simple-input.conf.j2 index ce871c7f..840f383c 100644 --- a/roles/logstash/templates/simple-input.conf.j2 +++ b/roles/logstash/templates/simple-input.conf.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + input { {% for input in pipelinename.input %} diff --git a/roles/logstash/templates/simple-output.conf.j2 b/roles/logstash/templates/simple-output.conf.j2 index 4b3ace63..7a75234d 100644 --- a/roles/logstash/templates/simple-output.conf.j2 +++ b/roles/logstash/templates/simple-output.conf.j2 @@ -1,3 +1,5 @@ +{{ ansible_managed | comment }} + {% if logstash_pipeline_identifier | bool %} filter { mutate { diff --git a/roles/repos/meta/main.yml b/roles/repos/meta/main.yml index cf90ea63..58a4a6d9 100644 --- a/roles/repos/meta/main.yml +++ b/roles/repos/meta/main.yml @@ -1,3 +1,4 @@ +--- galaxy_info: role_name: elasticstack_repos namespace: netways diff --git a/roles/repos/tasks/main.yml b/roles/repos/tasks/main.yml index 67609d13..eea48005 100644 --- a/roles/repos/tasks/main.yml +++ b/roles/repos/tasks/main.yml @@ -1,5 +1,4 @@ --- - - name: Include global role ansible.builtin.import_role: name: netways.elasticstack.elasticstack From 62e746333a301ff0b2a1d9e74b9d744cae090f0a Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Wed, 24 Sep 2025 15:35:52 +0200 Subject: [PATCH 09/10] Update issue templates to use types --- .github/ISSUE_TEMPLATE/bug_report.yaml | 3 ++- .github/ISSUE_TEMPLATE/feature_request.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 8c2bd837..8411cbbd 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -1,7 +1,8 @@ name: Bug Report description: File a bug report title: "[Bug]: " -labels: ["bug", "needs-triage"] +labels: ["needs-triage"] +type: ["bug"] body: - type: checkboxes id: terms diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 12c9e2e5..258f2c42 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -1,7 +1,8 @@ name: Feature Request description: Request a feature or enhancement title: "[Feature]: " -labels: ["feature", "needs-triage"] +labels: ["needs-triage"] +type: ["feature"] body: - type: markdown attributes: From 1fe1c2a704e2fff52ab7b9e5a8c815db6d1ecc77 Mon Sep 17 00:00:00 2001 From: Tobias Bauriedel Date: Thu, 25 Sep 2025 11:09:20 +0200 Subject: [PATCH 10/10] Implement global handlers There are tasks that are doing things with other tools of the collection (e.g: after changing certificates in elasticsearch, we need to restart kibana). This tasks will be moved to the global role "elasticstack" and can be called by via 'notify'. Also, global handlers will only do something when `elasticstack_collection_managed` is true (means the whole stack is managed by the collecion) --- roles/elasticsearch/handlers/main.yml | 10 ---------- roles/elasticsearch/handlers/restart_kibana.yml | 12 ------------ .../elasticsearch/tasks/elasticsearch-security.yml | 4 ++-- roles/elasticstack/defaults/main.yml | 4 ++++ roles/elasticstack/handlers/main.yml | 7 +++++++ roles/elasticstack/tasks/global/restart_kibana.yml | 14 ++++++++++++++ 6 files changed, 27 insertions(+), 24 deletions(-) delete mode 100644 roles/elasticsearch/handlers/restart_kibana.yml create mode 100644 roles/elasticstack/handlers/main.yml create mode 100644 roles/elasticstack/tasks/global/restart_kibana.yml diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml index be9ec69d..00f55939 100644 --- a/roles/elasticsearch/handlers/main.yml +++ b/roles/elasticsearch/handlers/main.yml @@ -8,13 +8,3 @@ - elasticsearch_enable | bool - not elasticsearch_freshstart.changed | bool - not elasticsearch_freshstart_security.changed | bool - -- name: Restart kibana if available for elasticsearch certificates - ansible.builtin.include_tasks: handlers/restart_kibana.yml - with_items: "{{ groups[elasticstack_kibana_group_name] }}" - when: - - elasticstack_collection_managed: true - - "not 'renew_ca' in ansible_run_tags" - - "not elasticstack_ca_will_expire_soon | bool" - - elasticstack_kibana_group_name is defined - - groups[elasticstack_kibana_group_name] is defined diff --git a/roles/elasticsearch/handlers/restart_kibana.yml b/roles/elasticsearch/handlers/restart_kibana.yml deleted file mode 100644 index 00e02ea7..00000000 --- a/roles/elasticsearch/handlers/restart_kibana.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Check if kibana is installed - ansible.builtin.package_facts: - manager: "auto" - changed_when: false - -- name: Restart Kibana - ansible.builtin.service: - name: kibana - state: restarted - delegate_to: "{{ item }}" - when: "'kibana' in ansible_facts.packages" diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml index b6157369..4a640354 100644 --- a/roles/elasticsearch/tasks/elasticsearch-security.yml +++ b/roles/elasticsearch/tasks/elasticsearch-security.yml @@ -303,7 +303,7 @@ mode: 0640 notify: - Restart Elasticsearch - - Restart kibana if available for elasticsearch certificates + - "Global - Restart Kibana" tags: - certificates - renew_ca @@ -319,7 +319,7 @@ mode: 0640 notify: - Restart Elasticsearch - - Restart kibana if available for elasticsearch certificates + - Global - Restart Kibana tags: - certificates - renew_ca diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index b47b2b16..acef096c 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -3,6 +3,10 @@ elasticstack_elasticsearch_group_name: elasticsearch elasticstack_logstash_group_name: logstash elasticstack_kibana_group_name: kibana +elasticstack_hosts_elasticsearch: "{{ groups[elasticstack_elasticsearch_group_name] }}" +elasticstack_hosts_logstash: "{{ groups[elasticstack_logstash_group_name] }}" +elasticstack_hosts_kibana: "{{ groups[elasticstack_kibana_group_name] }}" + elasticstack_beats_port: 5044 elasticstack_ca_host: "{{ groups[elasticstack_elasticsearch_group_name][0] }}" elasticstack_ca_dir: /opt/es-ca diff --git a/roles/elasticstack/handlers/main.yml b/roles/elasticstack/handlers/main.yml new file mode 100644 index 00000000..76dc90a9 --- /dev/null +++ b/roles/elasticstack/handlers/main.yml @@ -0,0 +1,7 @@ +--- +- name: "Global - Restart Kibana" + ansible.builtin.include_tasks: tasks/global/restart_kibana.yml + with_items: "{{ elasticstack_hosts_kibana }}" + when: + - elasticstack_collection_managed | bool # Only restart if all is managed by the collection + - elasticstack_hosts_kibana | length > 0 diff --git a/roles/elasticstack/tasks/global/restart_kibana.yml b/roles/elasticstack/tasks/global/restart_kibana.yml new file mode 100644 index 00000000..59930637 --- /dev/null +++ b/roles/elasticstack/tasks/global/restart_kibana.yml @@ -0,0 +1,14 @@ +--- +- name: Restart Kibana (if installed) + delegate_to: "{{ item }}" + block: + - name: Fetch installed packages + ansible.builtin.package_facts: + manager: "auto" + changed_when: false + + - name: Restart Kibana + ansible.builtin.service: + name: kibana + state: restarted + when: "'kibana' in ansible_facts.packages"