feat: add more support for different workload types

rules/apps.libsonnet

@@ -205,6 +205,28 @@
       {
         name: 'k8s.rules.pod_owner',
         rules: [
+          // workload aggregation for replicasets
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by (%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  label_replace(
+                    kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"},
+                    "replicaset", "$1", "owner_name", "(.*)"
+                  ) * on (%(clusterLabel)s, replicaset, namespace) group_left(owner_name) topk by(%(clusterLabel)s, replicaset, namespace) (
+                    1, max by (%(clusterLabel)s, replicaset, namespace, owner_name) (
+                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind=""}
+                    )
+                  ),
+                  "workload", "$1", "replicaset", "(.*)"
+                )
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'replicaset',
+            },
+          },
           // workload aggregation for deployments
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
@@ -214,9 +236,9 @@
                   label_replace(
                     kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"},
                     "replicaset", "$1", "owner_name", "(.*)"
-                  ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (
-                    1, max by (replicaset, namespace, owner_name) (
-                      kube_replicaset_owner{%(kubeStateMetricsSelector)s}
+                  ) * on(replicaset, namespace, %(clusterLabel)s) group_left(owner_name) topk by(%(clusterLabel)s, replicaset, namespace) (
+                    1, max by (%(clusterLabel)s, replicaset, namespace, owner_name) (
+                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind="Deployment"}
                     )
                   ),
                   "workload", "$1", "owner_name", "(.*)"
@@ -227,6 +249,7 @@
               workload_type: 'deployment',
             },
           },
+          // workload aggregation for daemonsets
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
@@ -241,6 +264,7 @@
               workload_type: 'daemonset',
             },
           },
+          // workload aggregation for statefulsets
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
@@ -255,20 +279,103 @@
               workload_type: 'statefulset',
             },
           },
+          // backwards compatibility for jobs
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
-              max by (%(clusterLabel)s, namespace, workload, pod) (
+              group by(%(clusterLabel)s, namespace, workload, pod) (
                 label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Job"},
-                  "workload", "$1", "owner_name", "(.*)"
-                )
+                  label_join(
+                    group by (%(clusterLabel)s, namespace, job_name, pod, owner_name) (
+                      label_join(
+                        kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Job"}
+                      , "job_name", "", "owner_name")
+                    )
+                    * on (%(clusterLabel)s, namespace, job_name) group_left()
+                    group by (%(clusterLabel)s, namespace, job_name) (
+                      kube_job_owner{%(kubeStateMetricsSelector)s, owner_kind=""}
+                    )
+                  , "workload", "", "owner_name")
+                , "workload_type", "job", "", "")
               )
             ||| % $._config,
             labels: {
               workload_type: 'job',
             },
           },
+          // workload aggregation for barepods
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, pod, workload) (
+                label_replace(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="", owner_name=""},
+                  "workload", "$1", "pod", "(.+)"
+                )
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'barepod',
+            },
+          },
+          // workload aggregation for staticpods
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Node"},
+                  "workload", "$1", "owner_name", "(.+)"
+                )
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'staticpod',
+            },
+          },
+          // workload aggregation for non-standard types (jobs, replicasets)
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              group by(%(clusterLabel)s, namespace, workload, workload_type, pod) (
+                label_join(
+                    label_join(
+                        group by (%(clusterLabel)s, namespace, job_name, pod) (
+                            label_join(
+                                kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Job"}
+                            , "job_name", "", "owner_name")
+                        )
+                        * on (%(clusterLabel)s, namespace, job_name) group_left(owner_kind, owner_name)
+                        group by (%(clusterLabel)s, namespace, job_name, owner_kind, owner_name) (
+                            kube_job_owner{%(kubeStateMetricsSelector)s, owner_kind!="Pod", owner_kind!=""}
+                        )
+                    , "workload", "", "owner_name")
+                , "workload_type", "", "owner_kind")
+                OR
+                label_replace(
+                  label_replace(
+                    label_replace(
+                      kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"}
+                      , "replicaset", "$1", "owner_name", "(.+)"
+                    )
+                    * on(%(clusterLabel)s, namespace, replicaset) group_left(owner_kind, owner_name)
+                    group by (%(clusterLabel)s, namespace, replicaset, owner_kind, owner_name) (
+                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind!="Deployment", owner_kind!=""}
+                    )
+                    , "workload", "$1", "owner_name", "(.+)"
+                  )
+                  OR
+                  label_replace(
+                    group by (%(clusterLabel)s, namespace, pod, owner_name, owner_kind) (
+                      kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
+                    )
+                    , "workload", "$1", "owner_name", "(.+)"
+                  )
+                  , "workload_type", "$1", "owner_kind", "(.+)"
+                )
+              )
+            ||| % $._config
+          }
         ],
       },
     ],

tests/rules-pod-owner-test.yaml

@@ -0,0 +1,161 @@
+rule_files:
+- ../prometheus_rules.yaml
+
+evaluation_interval: 1m
+
+tests:
+# deployment
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
+    values: '1 1'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns2",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
+    values: '1'
+  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
+    values: '1 1'
+  - series: 'kube_replicaset_owner{instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns2",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="ds-7cc77d965f-cgsdv", workload="ds", workload_type="deployment"}'
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes", namespace="ns2", pod="ds-7cc77d965f-cgsdv", workload="ds", workload_type="deployment"}'
+
+# bare pod
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:8443"}'
+    values: '1'
+  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:1234"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="openshift", namespace="default", pod="nginx", workload="nginx", workload_type="barepod"}'
+
+# static pod
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="staticpod"}'
+
+# statefulset
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="statefulset"}'
+
+# daemonset
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1 1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="daemonset"}'
+
+# replicaset
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
+    values: '1 '
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10013",pod="gke-duplicate-default-pool-10013-na4g"}'
+    values: '1'
+  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",replicaset="gke-duplicate-default-pool-10012"}'
+    values: '1'
+  - series: 'kube_replicaset_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10013",replicaset="gke-duplicate-default-pool-10013"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="gke-duplicate-default-pool-10012-kx2v", workload="gke-duplicate-default-pool-10012", workload_type="replicaset"}'
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="gke-duplicate-default-pool-10013-na4g", workload="gke-duplicate-default-pool-10013", workload_type="replicaset"}'
+
+# non-standard job
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
+    values: '1'
+  - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
+    values: '1'
+  - series: 'kube_pod_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120881", pod="cuda-vectoradd-29120881-dka0c", uid="8e3586c6-12f7-488e-b748-4f096192802m"}'
+    values: '1'
+  - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120881", namespace="monitoring", owner_is_controller="true", owner_kind="ScaledJob", owner_name="cuda-vectoradd"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120880-jfl5c", workload="cuda-vectoradd", workload_type="CronJob"}'
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120881-dka0c", workload="cuda-vectoradd", workload_type="ScaledJob"}'
+
+# job
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="kube-proxy-gke-duplicate-default-pool-f72d976h",pod="kube-proxy-gke-duplicate-default-pool-f72d976h-632a"}'
+    values: '1'
+  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",job_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
+    values: '1'
+  - series: 'kube_job_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",job_name="kube-proxy-gke-duplicate-default-pool-f72d976h"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="kube-proxy-gke-duplicate-default-pool-e20e250a", workload_type="job"}'
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-f72d976h-632a", workload="kube-proxy-gke-duplicate-default-pool-f72d976h", workload_type="job"}'
+
+# non-standard replicaset
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="ReplicaSet", owner_name="rollouts-demo-687d76d795", pod="rollouts-demo-687d76d795-5mlvh", uid="491a64cd-6a5f-4c64-b4b8-f95ee6a6b9c0"}'
+    values: '1'
+  - series: 'kube_replicaset_owner{cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="Rollout", owner_name="rollouts-demo", replicaset="rollouts-demo-687d76d795"}'
+    values: '1'
+  - series: 'kube_pod_owner{cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="ReplicaSet", owner_name="configmap-demo-687d76d795", pod="configmap-demo-687d76d795-5mlvh", uid="491a64cd-6a5f-4c64-b4b8-f95ee6a6b9c0"}'
+    values: '1'
+  - series: 'kube_replicaset_owner{cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="ConfigMap", owner_name="configmap-demo", replicaset="configmap-demo-687d76d795"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="k8sdev/aws/helm",namespace="default", pod="rollouts-demo-687d76d795-5mlvh", workload="rollouts-demo", workload_type="Rollout"}'
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="k8sdev/aws/helm",namespace="default", pod="configmap-demo-687d76d795-5mlvh", workload="configmap-demo", workload_type="ConfigMap"}'