Workaround for mismatch in Ansible and Python versions

.github/workflows/test_full_stack.yml

@@ -31,6 +31,7 @@ jobs:
 
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         distro:
           - rockylinux8

.github/workflows/test_role_repos.yml

@@ -37,10 +37,11 @@ jobs:
 
     strategy:
       fail-fast: false
+      max-parallel: 5
       matrix:
 
         distro:
-          - centos7
+          # centos7 removed - no longer supported
           - debian10
           - debian11
           - rockylinux8

.github/workflows/test_roles_pr.yml

@@ -34,7 +34,6 @@ jobs:
       matrix:
         distro:
           - rockylinux8
-          - rockylinux9
           - ubuntu2204
         scenario:
           - elasticstack_default

.gitignore

@@ -2,3 +2,7 @@
 *.swp
 __pycache__*
 .vscode
+.roo
+.ansible
+.rooignore
+.venv

README.md

@@ -9,8 +9,6 @@ This collection installs and manages the Elastic Stack. It provides roles for ev
 
 Every role is documented with all variables, please refer to the documentation found in **[Getting-Started](./docs/getting-started.md)**
 
-**Please note**: If you are already using this collection before version `1.0.0`, please note that we had to rename a significant amount of variables due to naming schema changes made by Ansible. Please review the variables you have set in your playbooks and variable files.
-
 ## Roles documentation
 
 > [!NOTE]
@@ -78,8 +76,15 @@ We know from personal experience, that the collections work in following combina
 
 * CentOS 7 - Elastic Stack 7
 
-### Known Issues
+## Caveats and information for long time users
+
+### Variable renaming
+
+**Please note**: If you are already using this collection before version `1.0.0`, please note that we had to rename a significant amount of variables due to naming schema changes made by Ansible. Please review the variables you have set in your playbooks and variable files.
+
+### Ulimit management for Elasticsearch
 
+**Warning**: The role no longer configures `ulimit`. Ensure the system's open file limit is set correctly (e.g., `ulimit -n`). Packages usually handle this, but verify with `ulimit -n` or checking `/proc/<pid>/limits`. If your version does not, open an issue at the collection's issue tracker: https://github.com/netways/ansible-collection-elasticstack/issues
 
 ## Usage
 

docs/role-elasticsearch.md

@@ -29,7 +29,6 @@ Role Variables
 * *elasticsearch_disable_systemcallfilterchecks*: Disable system call filter checks. This has a security impact but is necessary on some systems. Please refer to the [docs](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/_system_call_filter_check.html) for details. (default: `false`)
 * *elasticsearch_http_publish_host*: (String) The network address for HTTP clients to contact the node using sniffing. Accepts an IP address, a hostname, or a special value. (default: `not set`)
 * *elasticsearch_http_publish_port*: (integer) The port of the HTTP publish address. Configure this setting only if you need the publish port to be different from http.port. (default: `not set`)
-* *elasticsearch_pamlimits*: Set pam_limits neccessary for Elasticsearch. (Default: `true`)
 * *elasticsearch_check_calculation*: End play in checks (Default: `false`)
 * *elasticsearch_network_host*: You can configure multipe network addresses where the networking is bind to. You can assign IP addresses or interfaces by their names. You can also use elasticsearch internal variabels as it set as default. Example: `"_ens190_,_local_"` (Default: `"_local_,"_site_"`) (Optional; if not defined `default` is used)
 * *elasticsearch_api_host*: Hostname or IP elasticsearch is listening on. Only used for connection checks by ansible role. (Default: `localhost`)
@@ -83,3 +82,7 @@ These variables are identical over all our elastic related roles, hence the diff
     - repos
     - elasticsearch
 ```
+
+## Caveats
+
+> **Note**: The role no longer configures `ulimit`. Ensure the system's open file limit is set correctly (e.g., `ulimit -n`). Packages usually handle this, but verify with `ulimit -n` or checking `/proc/<pid>/limits`. If your version does not, open an issue at the collection's issue tracker: https://github.com/netways/ansible-collection-elasticstack/issues

molecule/beats_default/molecule.yml

@@ -4,8 +4,8 @@ dependency:
 driver:
   name: docker
 platforms:
-  - name: beats_default_${MOLECULE_DISTRO:-debian11}
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+  - name: beats_default_${MOLECULE_DISTRO:-debian12}
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/beats_peculiar/molecule.yml

@@ -4,8 +4,8 @@ dependency:
 driver:
   name: docker
 platforms:
-  - name: beats_peculiar_${MOLECULE_DISTRO:-debian11}
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+  - name: beats_peculiar_${MOLECULE_DISTRO:-debian12}
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/elasticsearch_cluster-oss/molecule.yml

@@ -9,7 +9,7 @@ platforms:
   - name: elasticsearch-cluster1
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -19,7 +19,7 @@ platforms:
   - name: elasticsearch-cluster2
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/elasticsearch_default/molecule.yml

@@ -9,7 +9,7 @@ platforms:
   - name: elasticsearch_default1
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -19,7 +19,7 @@ platforms:
   - name: elasticsearch_default2
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/elasticsearch_no-security/molecule.yml

@@ -9,7 +9,7 @@ platforms:
   - name: elasticsearch-nosecurity1
     groups:
       - elasticsearchXYZ
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -19,7 +19,7 @@ platforms:
   - name: elasticsearch-nosecurity2
     groups:
       - elasticsearchXYZ
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/elasticsearch_roles_calculation/molecule.yml

@@ -9,7 +9,7 @@ platforms:
   - name: elasticsearch-cluster1
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -19,7 +19,7 @@ platforms:
   - name: elasticsearch-cluster2
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -29,7 +29,7 @@ platforms:
   - name: elasticsearch-cluster3
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/elasticsearch_test_modules/molecule.yml

@@ -9,7 +9,7 @@ platforms:
   - name: elasticsearch_default
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/elasticstack_default/converge.yml

@@ -14,7 +14,7 @@
     elasticsearch_disable_systemcallfilterchecks: true
     elasticsearch_monitoring_enabled: false
     elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
-    elasticsearch_heap: "2"
+    elasticsearch_heap: "1"
     elasticstack_full_stack: true
     elasticstack_no_log: false
     logstash_pipeline_unsafe_shutdown: true

molecule/elasticstack_default/molecule.yml

@@ -11,7 +11,7 @@ platforms:
       - beats
       - logstash
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -23,7 +23,7 @@ platforms:
       - beats
       - kibana
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/kibana_default/molecule.yml

@@ -5,7 +5,7 @@ driver:
   name: docker
 platforms:
   - name: kibana_default
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/logstash_full_stack-oss/molecule.yml

@@ -11,7 +11,7 @@ platforms:
       - elasticsearch
       - logstash
       - filebeat
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/logstash_pipelines/molecule.yml

@@ -7,7 +7,7 @@ driver:
   name: docker
 platforms:
   - name: ansible-role-logstash_pipelines
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/logstash_specific_version/molecule.yml

@@ -7,7 +7,7 @@ driver:
   name: docker
 platforms:
   - name: elasticstack_version
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/repos_default/molecule.yml

@@ -7,7 +7,7 @@ driver:
   name: docker
 platforms:
   - name: elastic-repos-default
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

molecule/repos_oss/molecule.yml

@@ -7,7 +7,7 @@ driver:
   name: docker
 platforms:
   - name: elastic-repos-default-oss
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian12}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

requirements-test.txt

@@ -1,4 +1,4 @@
-ansible
+ansible>=2.15
 ansible-lint
 molecule
 molecule-plugins[docker]

roles/beats/meta/main.yml

@@ -23,4 +23,3 @@ galaxy_info:
     - beats
     - logging
     - system
-dependencies: []

roles/elasticsearch/handlers/main.yml

@@ -12,10 +12,10 @@
 
 - name: Restart kibana if available for elasticsearch certificates
   ansible.builtin.include_tasks: handlers/restart_kibana.yml
-  with_items: "{{ groups[elasticstack_kibana_group_name] }}"
+  with_items: "{{ groups[elasticstack_kibana_group_name] | default([]) }}"
   when:
-    - elasticstack_full_stack: true
-    - "not 'renew_ca' in ansible_run_tags"
-    - "not elasticstack_ca_will_expire_soon | bool"
+    - elasticstack_full_stack | bool
+    - "'renew_ca' not in ansible_run_tags"
+    - not elasticstack_ca_will_expire_soon | bool
     - elasticstack_kibana_group_name is defined
-    - groups[elasticstack_kibana_group_name] is defined
+    - groups[elasticstack_kibana_group_name] | default([]) | length > 0

roles/elasticsearch/meta/main.yml

@@ -1,7 +1,8 @@
+---
 galaxy_info:
+  role_name: elasticsearch
   author: Netways GmbH
   description: Elasticsearch Ansible role
-  role_name: elasticsearch
   company: Netways GmbH
   namespace: netways
   license: GPL-3.0-or-later
@@ -18,4 +19,5 @@ galaxy_info:
         - focal
         - jammy
   galaxy_tags: []
-dependencies: []
+collections:
+  - netways.elasticstack

roles/elasticsearch/tasks/elasticsearch-security.yml

@@ -1,24 +1,24 @@
 ---
 
 - name: Ensure ca exists
   ansible.builtin.stat:
     path: "{{ elasticstack_ca_dir }}/elastic-stack-ca.p12"
   register: elasticstack_ca_exists
   when: inventory_hostname == elasticstack_ca_host
 
 - name: Get CA informations
   cert_info:
     path: "{{ elasticstack_ca_dir }}/elastic-stack-ca.p12"
     passphrase: "{{ elasticstack_ca_pass | default(omit, true) }}"
   register: elasticstack_ca_infos
   when: inventory_hostname == elasticstack_ca_host and elasticstack_ca_exists.stat.exists | bool
 
 - name: Set the ca expiration date in days
   ansible.builtin.set_fact:
     elasticstack_ca_expiration_days: "{{ ((elasticstack_ca_infos.not_valid_after | to_datetime()) - (ansible_date_time.date | to_datetime('%Y-%m-%d'))).days }}"
   when: inventory_hostname == elasticstack_ca_host and elasticstack_ca_infos.skipped is not defined
 
 - name: Set ca will expire soon to true
   ansible.builtin.set_fact:
     elasticstack_ca_will_expire_soon: true
   when: >
@@ -40,12 +40,12 @@
   ansible.builtin.service:
     name: logstash
     state: stopped
-  with_items: "{{ groups[elasticstack_logstash_group_name] }}"
+  with_items: "{{ groups[elasticstack_logstash_group_name] | default([]) }}"
   delegate_to: "{{ item }}"
   when:
     - "'renew_ca' in ansible_run_tags or elasticstack_ca_will_expire_soon | bool"
     - elasticstack_logstash_group_name is defined
-    - groups[elasticstack_logstash_group_name] is defined
+    - groups[elasticstack_logstash_group_name] | default([]) | length > 0
 
 - name: Backup ca directory on elasticsearch ca host then remove
   when: (inventory_hostname == elasticstack_ca_host) and ('renew_ca' in "ansible_run_tags" or elasticstack_ca_will_expire_soon | bool)
@@ -138,7 +138,7 @@
       register: elasticsearch_check_cert_path
 
     - name: Move cert directory on elasticsearch node
       ansible.builtin.copy:
         src: /etc/elasticsearch/certs
         dest: "/etc/elasticsearch/certs_{{ ansible_date_time.iso8601_micro }}"
         mode: preserve
@@ -408,7 +408,7 @@
   retries: 5
   delay: 10
 
 - name: Fetch Elastic password # noqa: risky-shell-pipe
   ansible.builtin.shell: >
     if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
     grep "PASSWORD elastic" {{ elasticstack_initial_passwords }} |

roles/elasticsearch/tasks/main.yml

@@ -117,16 +117,6 @@
     name: openssl
   when: elasticsearch_security | bool
 
-# the following should be done by the rpm but failed with 7.4
-- name: Set ulimits for Elasticsearch
-  community.general.pam_limits:
-    limit_item: nofile
-    domain: elasticsearch
-    value: 65535
-    limit_type: hard
-  when:
-    - elasticsearch_pamlimits | bool
-
 - name: Construct exact name of Elasticsearch package
   ansible.builtin.set_fact:
     elasticsearch_package: >
@@ -212,7 +202,7 @@
     mode: "660"
     force: yes
   notify: Restart Elasticsearch
-  when: (elasticsearch_heap)
+  when: elasticsearch_heap | bool
 
 - name: Set jvm paths
   ansible.builtin.template:
@@ -233,7 +223,7 @@
     mode: "660"
     force: yes
   notify: Restart Elasticsearch
-  when: (elasticsearch_jvm_custom_parameters)
+  when: elasticsearch_jvm_custom_parameters | bool
 
 # On containerized Debian 10 systemd will not recognize elasticsearch service
 - name: Force systemd to reread configs on container
@@ -275,7 +265,7 @@
     - name: Check for cluster status without security
       ansible.builtin.uri:
 # kics-scan ignore-line
         url: "http://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty"
       register: elasticsearch_cluster_status
       ignore_errors: "{{ ansible_check_mode }}"
       until: elasticsearch_cluster_status.json.status == "green"
@@ -311,7 +301,7 @@
 - name: Show Info about heap
   ansible.builtin.debug:
     msg: "Using {{ elasticsearch_heap | int * 1024 }} of {{ ansible_memtotal_mb }} MB as heap for Elasticsearch"
-  when: (elasticsearch_heap)
+  when: elasticsearch_heap | bool
 
 - name: Show hint about passwords
   ansible.builtin.debug:

roles/kibana/meta/main.yml

@@ -1,10 +1,10 @@
+---
 galaxy_info:
   role_name: kibana
   author: netways
   description: Ansible role for Kibana
   company: Netways GmbH
   license: GPL-3.0-or-later
-
   min_ansible_version: "2.4"
   platforms:
     - name: EL
@@ -18,6 +18,5 @@ galaxy_info:
         - focal
         - jammy
   galaxy_tags: []
-dependencies: []
-  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
-  # if you add dependencies to this list.
+collections:
+  - netways.elasticstack