diff --git a/assets/optional/observability/02-cluster-role.yaml b/assets/optional/observability/02-cluster-role.yaml index 17b8723acd..55472a81b7 100644 --- a/assets/optional/observability/02-cluster-role.yaml +++ b/assets/optional/observability/02-cluster-role.yaml @@ -7,10 +7,11 @@ rules: - apiGroups: [""] resources: ["nodes/stats"] verbs: ["get"] - # OpenTelemetry Component: k8s_event receiver + # OpenTelemetry Component: k8s_event receiver, prometheus receiver (endpoints SD) - apiGroups: - "" resources: + - endpoints - events - namespaces - namespaces/status @@ -63,4 +64,9 @@ rules: verbs: - get - list - - watch \ No newline at end of file + - watch + # OpenTelemetry Component: prometheus receiver (kube-rbac-proxy authorization) + - nonResourceURLs: + - /metrics + verbs: + - get \ No newline at end of file diff --git a/packaging/observability/opentelemetry-collector-large.yaml b/packaging/observability/opentelemetry-collector-large.yaml index 208dd4932c..3757ae804e 100644 --- a/packaging/observability/opentelemetry-collector-large.yaml +++ b/packaging/observability/opentelemetry-collector-large.yaml @@ -4,6 +4,7 @@ # - Host's CPU, memory, disk, and network usage metrics # - System journals for selected MicroShift services and dependencies, priority < Info # - Metrics exposed by Pods that have "prometheus.io/scrape": "true" annotation +# - Metrics from optional component scrape configs (scrape.d drop-ins) receivers: kubeletstats: @@ -40,6 +41,8 @@ receivers: priority: info prometheus: config: + scrape_config_files: + - /etc/microshift/observability/scrape.d/*.yaml scrape_configs: - job_name: k8s scrape_interval: 10s @@ -109,7 +112,7 @@ service: receivers: [ journald ] processors: [ resourcedetection/system ] exporters: [ otlp ] - metrics/pods: + metrics/scrape: receivers: [ prometheus ] processors: [ batch ] exporters: [ otlp ] diff --git a/packaging/observability/opentelemetry-collector-medium.yaml b/packaging/observability/opentelemetry-collector-medium.yaml index cc2b6f1677..af1950bd08 100644 --- a/packaging/observability/opentelemetry-collector-medium.yaml +++ b/packaging/observability/opentelemetry-collector-medium.yaml @@ -2,8 +2,13 @@ # - Resource usage metrics of MicroShift's Containers, Pods, Volumes, and Node # - Kubernetes Events # - System journal for MicroShift service, priority < Info +# - Metrics from optional component scrape configs (scrape.d drop-ins) receivers: + prometheus: + config: + scrape_config_files: + - /etc/microshift/observability/scrape.d/*.yaml kubeletstats: auth_type: tls ca_file: /var/lib/microshift/certs/ca-bundle/client-ca.crt @@ -48,6 +53,10 @@ extensions: service: extensions: [ file_storage ] pipelines: + metrics/scrape: + receivers: [ prometheus ] + processors: [ batch ] + exporters: [ otlp ] metrics/kubeletstats: receivers: [ kubeletstats ] processors: [ batch ] diff --git a/packaging/observability/opentelemetry-collector-small.yaml b/packaging/observability/opentelemetry-collector-small.yaml index 588ea38707..6f96b8ea40 100644 --- a/packaging/observability/opentelemetry-collector-small.yaml +++ b/packaging/observability/opentelemetry-collector-small.yaml @@ -1,8 +1,13 @@ # Following file contains OpenTelemetry Collector configuration that exports: # - Resource usage metrics of MicroShift's Containers, Pods, Volumes, and Node # - Kubernetes Events +# - Metrics from optional component scrape configs (scrape.d drop-ins) receivers: + prometheus: + config: + scrape_config_files: + - /etc/microshift/observability/scrape.d/*.yaml kubeletstats: auth_type: tls ca_file: /var/lib/microshift/certs/ca-bundle/client-ca.crt @@ -43,6 +48,10 @@ extensions: service: extensions: [ file_storage ] pipelines: + metrics/scrape: + receivers: [ prometheus ] + processors: [ batch ] + exporters: [ otlp ] metrics/kubeletstats: receivers: [ kubeletstats ] processors: [ batch ] diff --git a/packaging/observability/scrape.d/kube-state-metrics.yaml b/packaging/observability/scrape.d/kube-state-metrics.yaml new file mode 100644 index 0000000000..06a23f770c --- /dev/null +++ b/packaging/observability/scrape.d/kube-state-metrics.yaml @@ -0,0 +1,37 @@ +scrape_configs: + - job_name: kube-state-metrics + scrape_interval: 10s + scheme: https + tls_config: + ca_file: /var/lib/microshift/certs/service-ca/ca.crt + cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt + key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key + server_name: kube-state-metrics.openshift-monitoring.svc + kubernetes_sd_configs: + - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig + role: endpoints + namespaces: + names: + - openshift-monitoring + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: kube-state-metrics;https-main + - job_name: kube-state-metrics-self + scrape_interval: 10s + scheme: https + tls_config: + ca_file: /var/lib/microshift/certs/service-ca/ca.crt + cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt + key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key + server_name: kube-state-metrics.openshift-monitoring.svc + kubernetes_sd_configs: + - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig + role: endpoints + namespaces: + names: + - openshift-monitoring + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: kube-state-metrics;https-self diff --git a/packaging/observability/scrape.d/metrics-server.yaml b/packaging/observability/scrape.d/metrics-server.yaml new file mode 100644 index 0000000000..b653dac0c3 --- /dev/null +++ b/packaging/observability/scrape.d/metrics-server.yaml @@ -0,0 +1,19 @@ +scrape_configs: + - job_name: metrics-server + scrape_interval: 10s + scheme: https + tls_config: + ca_file: /var/lib/microshift/certs/service-ca/ca.crt + cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt + key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key + server_name: metrics-server.openshift-monitoring.svc + kubernetes_sd_configs: + - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig + role: endpoints + namespaces: + names: + - openshift-monitoring + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: metrics-server;https diff --git a/packaging/observability/scrape.d/node-exporter.yaml b/packaging/observability/scrape.d/node-exporter.yaml new file mode 100644 index 0000000000..66626da522 --- /dev/null +++ b/packaging/observability/scrape.d/node-exporter.yaml @@ -0,0 +1,19 @@ +scrape_configs: + - job_name: node-exporter + scrape_interval: 10s + scheme: https + tls_config: + ca_file: /var/lib/microshift/certs/service-ca/ca.crt + cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt + key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key + server_name: node-exporter.openshift-monitoring.svc + kubernetes_sd_configs: + - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig + role: endpoints + namespaces: + names: + - openshift-monitoring + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: node-exporter;https diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index 0dbbc931a6..960d5769f7 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -303,6 +303,7 @@ Requires: microshift-release-info = %{version} The microshift-metrics-node-exporter-release-info package provides release information files for this release. These files contain the list of container image references used by node-exporter and can be used to embed those images into osbuilder blueprints or bootc containerfiles. + %package metrics-kube-state Summary: Kubernetes kube-state-metrics for MicroShift ExclusiveArch: x86_64 aarch64 @@ -623,6 +624,7 @@ install -p -m644 assets/optional/ai-model-serving/release-ai-model-serving-x86_6 # observability install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability +install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d install -p -m644 packaging/observability/*.yaml -D %{buildroot}%{_sysconfdir}/microshift/observability/ # Explicit copy of large config as default. Not using symlink to avoid accidental package upgrade overwriting user config if the user edits the config without copying (i.e. edits the target of symlink). install -p -m644 packaging/observability/opentelemetry-collector-large.yaml -D %{buildroot}%{_sysconfdir}/microshift/observability/opentelemetry-collector.yaml @@ -681,6 +683,9 @@ cat assets/optional/metrics-server/kustomization.aarch64.yaml >> %{buildroot}/%{ cat assets/optional/metrics-server/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server/kustomization.yaml %endif +install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d +install -p -m644 packaging/observability/scrape.d/metrics-server.yaml %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d/ + # metrics-server-release-info mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/metrics-server/release-metrics-server-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ @@ -705,6 +710,9 @@ cat assets/optional/node-exporter/kustomization.aarch64.yaml >> %{buildroot}/%{_ cat assets/optional/node-exporter/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter/kustomization.yaml %endif +install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d +install -p -m644 packaging/observability/scrape.d/node-exporter.yaml %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d/ + # node-exporter-release-info mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/node-exporter/release-node-exporter-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ @@ -728,6 +736,9 @@ cat assets/optional/kube-state-metrics/kustomization.aarch64.yaml >> %{buildroot cat assets/optional/kube-state-metrics/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/kustomization.yaml %endif +install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d +install -p -m644 packaging/observability/scrape.d/kube-state-metrics.yaml %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d/ + # kube-state-metrics-release-info mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/kube-state-metrics/release-kube-state-metrics-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ @@ -923,6 +934,7 @@ fi %files observability %dir %{_prefix}/lib/microshift/manifests.d/003-microshift-observability %dir %{_sysconfdir}/microshift/observability/ +%dir %{_sysconfdir}/microshift/observability/scrape.d/ %{_unitdir}/microshift-observability.service %config(noreplace) %{_sysconfdir}/microshift/observability/opentelemetry-collector.yaml %{_sysconfdir}/microshift/observability/opentelemetry-collector-*.yaml @@ -938,6 +950,9 @@ fi %files metrics-server %dir %{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server %{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server/* +%dir %{_sysconfdir}/microshift/observability +%dir %{_sysconfdir}/microshift/observability/scrape.d +%config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/metrics-server.yaml %files metrics-server-release-info %{_datadir}/microshift/release/release-metrics-server-{x86_64,aarch64}.json @@ -945,12 +960,19 @@ fi %files metrics-node-exporter %dir %{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter %{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter/* +%dir %{_sysconfdir}/microshift/observability +%dir %{_sysconfdir}/microshift/observability/scrape.d +%config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/node-exporter.yaml %files metrics-node-exporter-release-info %{_datadir}/microshift/release/release-node-exporter-{x86_64,aarch64}.json + %files metrics-kube-state %dir %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/* +%dir %{_sysconfdir}/microshift/observability +%dir %{_sysconfdir}/microshift/observability/scrape.d +%config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/kube-state-metrics.yaml %files metrics-kube-state-release-info %{_datadir}/microshift/release/release-kube-state-metrics-{x86_64,aarch64}.json diff --git a/test/assets/observability/otel_config.yaml b/test/assets/observability/otel_config.yaml index 4565f82077..9d3763eae3 100644 --- a/test/assets/observability/otel_config.yaml +++ b/test/assets/observability/otel_config.yaml @@ -1,5 +1,9 @@ # This OTEL config is only for testing purposes receivers: + prometheus: + config: + scrape_config_files: + - /etc/microshift/observability/scrape.d/*.yaml hostmetrics: root_path: / collection_interval: 10s @@ -67,6 +71,10 @@ extensions: service: extensions: [ file_storage ] pipelines: + metrics/scrape: + receivers: [ prometheus ] + processors: [ resourcedetection/system ] + exporters: [ prometheus, prometheusremotewrite ] metrics/hostmetrics: receivers: [ hostmetrics ] processors: [ resourcedetection/system ] diff --git a/test/suites/optional/metrics.robot b/test/suites/optional/metrics.robot index 68cfdef75a..e172c13508 100644 --- a/test/suites/optional/metrics.robot +++ b/test/suites/optional/metrics.robot @@ -109,14 +109,10 @@ Scrape Metrics From ... with mTLS client certs. Resolves the IP from the Endpoints object ... to validate that the headless Service selector matches the pod. [Arguments] ${metrics_service} ${port} - ${ep_ip}= Oc Get JsonPath - ... endpoints - ... ${METRICS_NS} - ... ${metrics_service} - ... .subsets[0].addresses[0].ip - ${ep_ip}= Get Regexp Matches ${ep_ip} \\d+\\.\\d+\\.\\d+\\.\\d+ - Length Should Be ${ep_ip} 1 - VAR ${ep_ip}= ${ep_ip}[0] + # Oc Get JsonPath uses Run With Kubeconfig which merges stderr into stdout; + # the v1 Endpoints deprecation warning from K8s 1.33+ pollutes the IP value. + ${ep_ip}= Command Should Work + ... oc get endpoints -n ${METRICS_NS} ${metrics_service} -o jsonpath\='{.subsets[0].addresses[0].ip}' --kubeconfig\=/var/lib/microshift/resources/kubeadmin/kubeconfig VAR ${svc_host}= ${metrics_service}.${METRICS_NS}.svc ${stdout}= Command Should Work ... curl -s --resolve ${svc_host}:${port}:${ep_ip} --cacert ${SERVICE_CA} --cert ${CLIENT_CERT} --key ${CLIENT_KEY} https://${svc_host}:${port}/metrics diff --git a/test/suites/optional/observability.robot b/test/suites/optional/observability.robot index d1ffb51be0..5952a6113a 100644 --- a/test/suites/optional/observability.robot +++ b/test/suites/optional/observability.robot @@ -57,6 +57,30 @@ Kube Events Logs Are Exported Wait Until Keyword Succeeds 10x 5s ... Check Loki Query ${LOKI_HOST} ${LOKI_PORT} {service_name="kube_events"} +KSM Metrics Are Exported Via Scrape Drop-In + [Documentation] The prometheus receiver should scrape kube-state-metrics via the + ... scrape.d drop-in and export kube_node_info to the prometheus exporter. + + VAR ${METRIC} kube_node_info scope=TEST + Check Prometheus Query ${PROMETHEUS_HOST} ${PROMETHEUS_PORT} ${METRIC} + Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} ${METRIC} + +Node Exporter Metrics Are Exported Via Scrape Drop-In + [Documentation] The prometheus receiver should scrape node-exporter via the + ... scrape.d drop-in and export node_cpu_seconds_total to the prometheus exporter. + + VAR ${METRIC} node_cpu_seconds_total scope=TEST + Check Prometheus Query ${PROMETHEUS_HOST} ${PROMETHEUS_PORT} ${METRIC} + Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} ${METRIC} + +Metrics Server Metrics Are Exported Via Scrape Drop-In + [Documentation] The prometheus receiver should scrape metrics-server via the + ... scrape.d drop-in and export metrics to the prometheus exporter. + + VAR ${METRIC} metrics_server_kubelet_request_total scope=TEST + Check Prometheus Query ${PROMETHEUS_HOST} ${PROMETHEUS_PORT} ${METRIC} + Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} ${METRIC} + Logs Should Not Contain Receiver Errors [Documentation] Internal receiver errors are not treated as fatal. Typically these are due to a misconfiguration ... and thus indicate the provided default config should be reviewed. @@ -71,7 +95,11 @@ Setup Suite And Prepare Test Host [Documentation] The service starts after MicroShift starts and thus will start generating pertinent log data ... right away. When the suite is executed, immediately get the cursor for the microshift-observability unit. Setup Suite - Setup MicroShift With Optionals 003-microshift-observability + Setup MicroShift With Optionals + ... 003-microshift-observability + ... 080-microshift-metrics-server + ... 081-microshift-kube-state-metrics + ... 082-microshift-node-exporter ${ns} Create Unique Namespace VAR ${NAMESPACE} ${ns} scope=SUITE Configure Firewall And Observability