diff --git a/molecule/elasticstack_default/molecule.yml b/molecule/elasticstack_default/molecule.yml index 1131b0e6..1efd1575 100644 --- a/molecule/elasticstack_default/molecule.yml +++ b/molecule/elasticstack_default/molecule.yml @@ -12,6 +12,7 @@ platforms: - beats - logstash - elasticsearch + - kibana image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian13}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: @@ -25,6 +26,7 @@ platforms: - beats - kibana - elasticsearch + - logstash image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian13}-ansible:latest" command: ${MOLECULE_DOCKER_COMMAND:-""} volumes: diff --git a/molecule/elasticstack_default/prepare.yml b/molecule/elasticstack_default/prepare.yml index f07307ae..8d8c3819 100644 --- a/molecule/elasticstack_default/prepare.yml +++ b/molecule/elasticstack_default/prepare.yml @@ -64,3 +64,20 @@ state: latest update_cache: yes when: ansible_os_family == "Debian" + + # Temporary CI workaround: containers cannot resolve each other by name + # (the runner's resolver is Azure's 168.63.129.16, which does not know the + # container hostnames). Seed /etc/hosts from gathered facts so name-based + # connections work. Remove once #524 (elasticstack_address_mode) lands and + # molecule sets elasticstack_address_mode: ip. + - name: Seed /etc/hosts from facts for inter-container name resolution + ansible.builtin.blockinfile: + path: /etc/hosts + marker: "# {mark} ANSIBLE CLUSTER HOSTS" + # /etc/hosts is a Docker bind-mount, so the default atomic temp+rename + # write fails with EBUSY; write in place instead. + unsafe_writes: true + block: |- + {% for h in groups['all'] %} + {{ hostvars[h].ansible_default_ipv4.address | default(hostvars[h].ansible_all_ipv4_addresses[0]) }} {{ hostvars[h].ansible_fqdn }} {{ hostvars[h].ansible_hostname }} {{ h }} + {% endfor %} diff --git a/roles/beats/README.md b/roles/beats/README.md index b5197631..7816e301 100644 --- a/roles/beats/README.md +++ b/roles/beats/README.md @@ -47,7 +47,7 @@ Run only parts of the role with `--tags`: | `beats_auditbeat` | `bool` | `false` | — | Install and manage Auditbeat. | | `beats_metricbeat` | `bool` | `false` | — | Install and manage Metricbeat. | | `beats_target_hosts` | `list` of `str` | `['localhost']` | — | Hosts the Beats ship to. Only used when the role runs standalone; with the other Elastic Stack roles the targets are determined automatically. | -| `beats_fields` | `list` of `str` | N/A | — | Fields added to every input, given as a list of "key: value" strings (the global counterpart to the per-input fields). Unset by default. See the Filebeat inputs documentation. | +| `beats_fields` | `list` of `str` | N/A | — | Global fields added to the log and syslog (tcp/udp) inputs (not to the mysql, journald or docker inputs), given as a list of "key: value" strings — the global counterpart to the per-input fields. Unset by default. See the Filebeat inputs documentation. | | `beats_logging` | `str` | `"file"` | — | Where the Beats log. Set to "file" to log into beats_logpath; any other value leaves the Beats built-in logging. | | `beats_loglevel` | `str` | `"info"` | — | Log level for all Beats. | | `beats_logpath` | `str` | `"/var/log/beats"` | — | Directory for the log files when beats_logging is "file". | @@ -69,18 +69,18 @@ Run only parts of the role with `--tags`: | `beats_filebeat_modules` | `list` of `str` | N/A | — | List of Filebeat modules to enable (experimental). Unset by default. | | `beats_auditbeat_enable` | `bool` | `true` | — | Start and enable the Auditbeat service. | | `beats_auditbeat_output` | `str` | `"elasticsearch"` | `logstash`, `elasticsearch` | Where Auditbeat sends its events. | -| `beats_auditbeat_setup` | `bool` | `true` | — | Run the Auditbeat setup (index management and pipelines). Only effective with the elasticsearch output. | +| `beats_auditbeat_setup` | `bool` | `true` | — | Run the Auditbeat setup (index management, ingest pipelines and Kibana dashboards). Only effective with the elasticsearch output; loading the dashboards additionally requires Kibana to be reachable. | | `beats_auditbeat_loadbalance` | `bool` | `true` | — | Enable load balancing for the Auditbeat Logstash output. | | `beats_metricbeat_enable` | `bool` | `true` | — | Start and enable the Metricbeat service. | | `beats_metricbeat_output` | `str` | `"elasticsearch"` | `logstash`, `elasticsearch` | Where Metricbeat sends its events. | | `beats_metricbeat_modules` | `list` of `str` | `['system']` | — | Metricbeat modules to enable. | | `beats_metricbeat_loadbalance` | `bool` | `true` | — | Enable load balancing for the Metricbeat Logstash output. | | `beats_security` | `bool` | `false` | — | Activate TLS for the connections to the targets. Works with the other roles and elasticstack_full_stack to create certificates automatically, or with the beats_tls_* variables for custom certificates. | -| `beats_ca_dir` | `str` | N/A | — | Base directory for custom CA certificates and keys. Unset by default; the role fills it depending on the stack variant (/opt/ca standalone, or /etc/beats/certs in a full stack). | -| `beats_tls_key` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.key"` | — | Path to the private key file for custom certificates. | -| `beats_tls_cert` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.crt"` | — | Path to the certificate for custom certificates. | -| `beats_tls_cacert` | `str` | `"{{ beats_ca_dir \| default('') }}/ca.crt"` | — | Path to the CA certificate for custom certificates. | -| `beats_tls_key_passphrase` | `str` | `"BeatsChangeMe"` | — | Passphrase of the private key. | +| `beats_ca_dir` | `str` | N/A | — | Base directory for the Beats certificate, key and CA file. The role fills it per variant (/opt/ca standalone, /etc/beats/certs in a full stack) if you did not set it. | +| `beats_tls_key` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.key"` | — | Path to the Beats private key. Defaults under beats_ca_dir; set it to bring your own. | +| `beats_tls_cert` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.crt"` | — | Path to the Beats certificate. Defaults under beats_ca_dir; set it to bring your own. | +| `beats_tls_cacert` | `str` | `"{{ beats_ca_dir \| default('') }}/ca.crt"` | — | Path to the CA certificate Beats trusts. Defaults under beats_ca_dir; set it to bring your own. | +| `beats_tls_key_passphrase` | `str` | `"BeatsChangeMe"` | — | Passphrase of the Beats private key. | | `beats_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. | | `beats_cert_expiration_buffer` | `str` | `"+30d"` | — | Renew the certificate when it would expire within this period. Uses the community.crypto check_period format (e.g. "+30d"), not a plain number. | | `beats_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Beats certificate. Alternatively run the playbook with the renew_beats_cert tag. | diff --git a/roles/beats/meta/argument_specs.yml b/roles/beats/meta/argument_specs.yml index e36abaee..c6ae8aa9 100644 --- a/roles/beats/meta/argument_specs.yml +++ b/roles/beats/meta/argument_specs.yml @@ -53,9 +53,10 @@ argument_specs: type: list elements: str description: >- - Fields added to every input, given as a list of "key: value" strings - (the global counterpart to the per-input fields). Unset by default. - See the Filebeat inputs documentation. + Global fields added to the log and syslog (tcp/udp) inputs (not to the + mysql, journald or docker inputs), given as a list of "key: value" + strings — the global counterpart to the per-input fields. Unset by + default. See the Filebeat inputs documentation. # ----- Logging ----- beats_logging: @@ -187,7 +188,10 @@ argument_specs: beats_auditbeat_setup: type: bool default: true - description: Run the Auditbeat setup (index management and pipelines). Only effective with the elasticsearch output. + description: >- + Run the Auditbeat setup (index management, ingest pipelines and Kibana + dashboards). Only effective with the elasticsearch output; loading the + dashboards additionally requires Kibana to be reachable. beats_auditbeat_loadbalance: type: bool @@ -232,29 +236,29 @@ argument_specs: beats_ca_dir: type: str description: >- - Base directory for custom CA certificates and keys. Unset by default; the - role fills it depending on the stack variant (/opt/ca standalone, or - /etc/beats/certs in a full stack). + Base directory for the Beats certificate, key and CA file. The role + fills it per variant (/opt/ca standalone, /etc/beats/certs in a full + stack) if you did not set it. beats_tls_key: type: str default: "{{ beats_ca_dir | default('') }}/{{ inventory_hostname }}-beats.key" - description: Path to the private key file for custom certificates. + description: Path to the Beats private key. Defaults under beats_ca_dir; set it to bring your own. beats_tls_cert: type: str default: "{{ beats_ca_dir | default('') }}/{{ inventory_hostname }}-beats.crt" - description: Path to the certificate for custom certificates. + description: Path to the Beats certificate. Defaults under beats_ca_dir; set it to bring your own. beats_tls_cacert: type: str default: "{{ beats_ca_dir | default('') }}/ca.crt" - description: Path to the CA certificate for custom certificates. + description: Path to the CA certificate Beats trusts. Defaults under beats_ca_dir; set it to bring your own. beats_tls_key_passphrase: type: str default: BeatsChangeMe - description: Passphrase of the private key. + description: Passphrase of the Beats private key. beats_cert_validity_period: type: int diff --git a/roles/beats/tasks/auditbeat.yml b/roles/beats/tasks/auditbeat.yml index 7142168b..3db33713 100644 --- a/roles/beats/tasks/auditbeat.yml +++ b/roles/beats/tasks/auditbeat.yml @@ -57,8 +57,8 @@ - beats_configuration - name: Setup Auditbeat in Elasticsearch - ansible.builtin.command: > - /usr/bin/auditbeat setup --pipelines --index-management && + ansible.builtin.shell: > + /usr/bin/auditbeat setup --pipelines --index-management --dashboards && /usr/bin/auditbeat version > /etc/auditbeat/pipeline_created run_once: true args: diff --git a/roles/beats/tasks/filebeat.yml b/roles/beats/tasks/filebeat.yml index 2cd28406..9d5a5185 100644 --- a/roles/beats/tasks/filebeat.yml +++ b/roles/beats/tasks/filebeat.yml @@ -81,7 +81,7 @@ - name: Enable Ingest Pipelines ansible.builtin.shell: > - /usr/bin/filebeat setup --pipelines && + /usr/bin/filebeat setup --pipelines --dashboards && /usr/bin/filebeat version > /etc/filebeat/{{ item }}_pipeline_created args: creates: "/etc/filebeat/{{ item }}_pipeline_created" diff --git a/roles/beats/tasks/metricbeat.yml b/roles/beats/tasks/metricbeat.yml index 3501ce01..89841bed 100644 --- a/roles/beats/tasks/metricbeat.yml +++ b/roles/beats/tasks/metricbeat.yml @@ -60,14 +60,14 @@ when: beats_metricbeat_modules is defined - name: Enable Ingest Pipelines - ansible.builtin.command: > + ansible.builtin.shell: > metricbeat setup && metricbeat version > /etc/metricbeat/pipelines_created args: creates: "/etc/metricbeat/pipelines_created" notify: - Restart Metricbeat - with_items: "{{ beats_metricbeat_modules }}" + run_once: true when: - beats_metricbeat_modules is defined - beats_metricbeat_output == "elasticsearch" diff --git a/roles/beats/templates/auditbeat.yml.j2 b/roles/beats/templates/auditbeat.yml.j2 index 85a085b8..2485e741 100644 --- a/roles/beats/templates/auditbeat.yml.j2 +++ b/roles/beats/templates/auditbeat.yml.j2 @@ -19,7 +19,9 @@ auditbeat.modules: - host # General host information, e.g. uptime, IPs - login # User logins, logouts, and system boots. - process # Started and stopped processes - - socket # Opened and closed sockets +{% if ansible_architecture == 'x86_64' %} + - socket # Opened and closed sockets (metricset not available in the arm64/aarch64 auditbeat build) +{% endif %} - user # User information state.period: 12h user.detect_password_changes: true @@ -29,6 +31,13 @@ auditbeat.modules: setup.template.settings: index.number_of_shards: 1 setup.kibana: +{% if elasticstack_full_stack | bool and groups[elasticstack_kibana_group_name] | default([]) | length > 0 %} + host: "http://{{ groups[elasticstack_kibana_group_name] | first }}:{{ elasticstack_kibana_port }}" +{% if beats_security | bool %} + username: "elastic" + password: "{{ beats_writer_password.stdout }}" +{% endif %} +{% endif %} {% if beats_auditbeat_output == "elasticsearch" %} output.elasticsearch: {% if beats_security | bool %} diff --git a/roles/beats/templates/filebeat.yml.j2 b/roles/beats/templates/filebeat.yml.j2 index cbddb8aa..a7a89cfc 100644 --- a/roles/beats/templates/filebeat.yml.j2 +++ b/roles/beats/templates/filebeat.yml.j2 @@ -109,6 +109,13 @@ filebeat.config.modules: setup.template.settings: index.number_of_shards: 1 setup.kibana: +{% if elasticstack_full_stack | bool and groups[elasticstack_kibana_group_name] | default([]) | length > 0 %} + host: "http://{{ groups[elasticstack_kibana_group_name] | first }}:{{ elasticstack_kibana_port }}" +{% if beats_security | bool %} + username: "elastic" + password: "{{ beats_writer_password.stdout }}" +{% endif %} +{% endif %} {% if beats_filebeat_output == "elasticsearch" %} output.elasticsearch: {% if beats_security | bool %} diff --git a/roles/beats/templates/metricbeat.yml.j2 b/roles/beats/templates/metricbeat.yml.j2 index a8d45b04..2218fd15 100644 --- a/roles/beats/templates/metricbeat.yml.j2 +++ b/roles/beats/templates/metricbeat.yml.j2 @@ -6,6 +6,13 @@ metricbeat.config.modules: setup.template.settings: index.number_of_shards: 1 setup.kibana: +{% if elasticstack_full_stack | bool and groups[elasticstack_kibana_group_name] | default([]) | length > 0 %} + host: "http://{{ groups[elasticstack_kibana_group_name] | first }}:{{ elasticstack_kibana_port }}" +{% if beats_security | bool %} + username: "elastic" + password: "{{ beats_writer_password.stdout }}" +{% endif %} +{% endif %} {% if beats_metricbeat_output == "elasticsearch" %} output.elasticsearch: {% if beats_security | bool %} diff --git a/roles/elasticstack/README.md b/roles/elasticstack/README.md index 4e3e417b..132a2cb5 100644 --- a/roles/elasticstack/README.md +++ b/roles/elasticstack/README.md @@ -39,7 +39,7 @@ The role runs automatically the first time any component role is applied to a ho | `elasticstack_version` | `str` | N/A | — | Exact package version to install (e.g. "8.8.1"). Unset by default: the collection then installs the latest, or reuses the version already present on the CA host. | | `elasticstack_full_stack` | `bool` | `true` | — | Treat the hosts as one coordinated stack (roles read each other's hosts, share the CA and wire the connections). Set to false to run a role standalone. | | `elasticstack_security` | `bool` | `true` | — | Enable X-Pack security across the stack. Only effective with the elastic variant. | -| `elasticstack_override_beats_tls` | `bool` | `false` | — | Override the automatically derived TLS setting for the Beats input. | +| `elasticstack_override_beats_tls` | `bool` | `false` | — | When true, do not auto-enable TLS between Beats and Logstash in a full stack (also skips Beats certificate generation). | | `elasticstack_elasticsearch_group_name` | `str` | `"elasticsearch"` | — | Inventory group that holds the Elasticsearch hosts. | | `elasticstack_logstash_group_name` | `str` | `"logstash"` | — | Inventory group that holds the Logstash hosts. | | `elasticstack_kibana_group_name` | `str` | `"kibana"` | — | Inventory group that holds the Kibana hosts. | diff --git a/roles/elasticstack/meta/argument_specs.yml b/roles/elasticstack/meta/argument_specs.yml index 02310e13..52e912f9 100644 --- a/roles/elasticstack/meta/argument_specs.yml +++ b/roles/elasticstack/meta/argument_specs.yml @@ -63,7 +63,7 @@ argument_specs: elasticstack_override_beats_tls: type: bool default: false - description: Override the automatically derived TLS setting for the Beats input. + description: When true, do not auto-enable TLS between Beats and Logstash in a full stack (also skips Beats certificate generation). # ----- Inventory host groups ----- elasticstack_elasticsearch_group_name: diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml index 4ea72c04..8a58acda 100644 --- a/roles/kibana/tasks/main.yml +++ b/roles/kibana/tasks/main.yml @@ -134,17 +134,38 @@ port: 5601 timeout: 300 register: kibana_wait + + # An open port only means Kibana is starting; /api/status returns 503 ("not + # ready yet") until it is fully up, so downstream consumers (e.g. Beats + # dashboard setup) must not race it. Poll unauthenticated: a ready Kibana + # answers 200 (anonymous status allowed) or 401 (auth required but server up) + # depending on status.allowAnonymous / the version — treat both as ready. + - name: Wait for Kibana API to be ready + ansible.builtin.uri: + url: "{{ 'https' if kibana_tls | bool else 'http' }}://localhost:{{ elasticstack_kibana_port }}/api/status" + method: GET + validate_certs: false + status_code: [200, 401, 503] + register: kibana_api + until: kibana_api.status in [200, 401] + retries: 60 + delay: 5 + changed_when: false rescue: - - name: Debug Kibana status on failure + - name: Show why the Kibana readiness check failed ansible.builtin.debug: - msg: "Kibana failed to start: {{ kibana_wait }}" - - name: Show Kibana logs + msg: >- + Kibana readiness failed. + API check: status={{ kibana_api.status | default('n/a') }}, + msg={{ kibana_api.msg | default('n/a') }}. + Port opened after {{ kibana_wait.elapsed | default('n/a') }}s. + - name: Show Kibana warnings and errors ansible.builtin.command: - cmd: journalctl -u kibana -n 100 + cmd: journalctl -u kibana -n 100 -p warning register: kibana_logs changed_when: false no_log: true - - name: Output Kibana logs + - name: Output Kibana warnings and errors ansible.builtin.debug: var: kibana_logs.stdout