From 1261213b7f018b28afea9ac0d43c2b61ab7692f5 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Wed, 23 Apr 2025 21:23:50 +0200
Subject: [PATCH 01/42] feat: add more support for different workload tpes

---
 rules/apps.libsonnet | 63 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 62 insertions(+), 1 deletion(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index decc0125c..c87c27636 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -205,6 +205,28 @@
       {
         name: 'k8s.rules.pod_owner',
         rules: [
+          // workload aggregation for replicasets
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by (%(clusterLabel)s, namespace, workload, pod, workload_type) (
+                label_replace(
+                  label_replace(
+                    kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"},
+                    "replicaset", "$1", "owner_name", "(.*)"
+                  ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (
+                    1, max by (replicaset, namespace, owner_name) (
+                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind=""}
+                    )
+                  ),
+                  "workload", "$1", "owner_name", "(.*)"
+                )
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'deployment',
+            },
+          },
           // workload aggregation for deployments
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
@@ -216,7 +238,7 @@
                     "replicaset", "$1", "owner_name", "(.*)"
                   ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (
                     1, max by (replicaset, namespace, owner_name) (
-                      kube_replicaset_owner{%(kubeStateMetricsSelector)s}
+                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind!=""}
                     )
                   ),
                   "workload", "$1", "owner_name", "(.*)"
@@ -227,6 +249,7 @@
               workload_type: 'deployment',
             },
           },
+          // workload aggregation for daemonsets
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
@@ -241,6 +264,7 @@
               workload_type: 'daemonset',
             },
           },
+          // workload aggregation for statefulsets
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
@@ -255,6 +279,7 @@
               workload_type: 'statefulset',
             },
           },
+          // workload aggregation for jobs
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
@@ -269,6 +294,42 @@
               workload_type: 'job',
             },
           },
+          // workload aggregation for barepods
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  max by(cluster, namespace, pod, asserts_env, asserts_site) (
+                    last_over_time(
+                      kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind=""}[$__range:]
+                    )
+                  )
+                ), "workload", "$1", "barepod", "(.+)"
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'barepod',
+            },
+          },
+          // workload aggregation for staticpods
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  max by (cluster, namespace, pod, asserts_env, asserts_site) (
+                    last_over_time(
+                      kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Node"}[$__range:]
+                    )
+                  )
+                ), "workload", "$1", "pod", "(.+)"
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'staticpod',
+            },
+          },
         ],
       },
     ],

From b26540f5f6839ac83708b83cc7fa45624bcba3e3 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 24 Apr 2025 15:13:48 +0200
Subject: [PATCH 02/42] chore: add non-standard workload types

---
 rules/apps.libsonnet | 128 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 116 insertions(+), 12 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index c87c27636..c7e1d0860 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -209,7 +209,7 @@
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
-              max by (%(clusterLabel)s, namespace, workload, pod, workload_type) (
+              max by (%(clusterLabel)s, namespace, workload, pod) (
                 label_replace(
                   label_replace(
                     kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"},
@@ -238,7 +238,7 @@
                     "replicaset", "$1", "owner_name", "(.*)"
                   ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (
                     1, max by (replicaset, namespace, owner_name) (
-                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind!=""}
+                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind="Deployment"}
                     )
                   ),
                   "workload", "$1", "owner_name", "(.*)"
@@ -300,11 +300,7 @@
             expr: |||
               max by(%(clusterLabel)s, namespace, workload, pod) (
                 label_replace(
-                  max by(cluster, namespace, pod, asserts_env, asserts_site) (
-                    last_over_time(
-                      kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind=""}[$__range:]
-                    )
-                  )
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind=""}
                 ), "workload", "$1", "barepod", "(.+)"
               )
             ||| % $._config,
@@ -318,11 +314,7 @@
             expr: |||
               max by(%(clusterLabel)s, namespace, workload, pod) (
                 label_replace(
-                  max by (cluster, namespace, pod, asserts_env, asserts_site) (
-                    last_over_time(
-                      kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Node"}[$__range:]
-                    )
-                  )
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Node"}
                 ), "workload", "$1", "pod", "(.+)"
               )
             ||| % $._config,
@@ -330,6 +322,118 @@
               workload_type: 'staticpod',
             },
           },
+          // workload aggregation for StrimziPodSet
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="StrimziPodSet"}
+                ), "workload", "$1", "pod", "(.+)"
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'strimzipodset',
+            },
+          },
+          // workload aggregation for Rollout
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Rollout"}
+                ), "workload", "$1", "pod", "(.+)"
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'rollout',
+            },
+          },
+          // workload aggregation for CatalogSource
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="CatalogSource"}
+                ), "workload", "$1", "pod", "(.+)"
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'catalogsource',
+            },
+          },
+          // workload aggregation for ClusterPolicy
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ClusterPolicy"}
+                ), "workload", "$1", "pod", "(.+)"
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'clusterpolicy',
+            },
+          },
+          // workload aggregation for ConfigMap
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ConfigMap"}
+                ), "workload", "$1", "pod", "(.+)"
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'configmap',
+            },
+          },
+          // workload aggregation for EphemeralRunner
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="EphemeralRunner"}
+                ), "workload", "$1", "pod", "(.+)"
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'ephemeralrunner',
+            },
+          },  
+          // workload aggregation for AutoscalingListener
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="AutoscalingListener"}
+                ), "workload", "$1", "pod", "(.+)"
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'autoscalinglistener',
+            },
+          },
+          // workload aggregation for Workflow
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by(%(clusterLabel)s, namespace, workload, pod) (
+                label_replace(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Workflow"}
+                ), "workload", "$1", "pod", "(.+)"
+              )
+            ||| % $._config,
+            labels: {
+              workload_type: 'workflow',
+            },
+          },
         ],
       },
     ],

From 9fd7c568bca99de4bc0228d29f8a09bf09e84238 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 24 Apr 2025 15:34:06 +0200
Subject: [PATCH 03/42] chore: fix wrong worklaod type in replicaset

---
 rules/apps.libsonnet | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index c7e1d0860..77b7b4d32 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -224,7 +224,7 @@
               )
             ||| % $._config,
             labels: {
-              workload_type: 'deployment',
+              workload_type: 'replicaset',
             },
           },
           // workload aggregation for deployments

From 2ca3af3c878f3960b4196d59f5a6bd482270237a Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 24 Apr 2025 21:20:00 +0200
Subject: [PATCH 04/42] chore: add dynamic support for non-standard workload
 types

---
 rules/apps.libsonnet | 117 ++++---------------------------------------
 1 file changed, 11 insertions(+), 106 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 77b7b4d32..d15677de3 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -322,117 +322,22 @@
               workload_type: 'staticpod',
             },
           },
-          // workload aggregation for StrimziPodSet
+          // workload aggregation for non-standard workloads
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
-              max by(%(clusterLabel)s, namespace, workload, pod) (
-                label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="StrimziPodSet"}
-                ), "workload", "$1", "pod", "(.+)"
-              )
-            ||| % $._config,
-            labels: {
-              workload_type: 'strimzipodset',
-            },
-          },
-          // workload aggregation for Rollout
-          {
-            record: 'namespace_workload_pod:kube_pod_owner:relabel',
-            expr: |||
-              max by(%(clusterLabel)s, namespace, workload, pod) (
-                label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Rollout"}
-                ), "workload", "$1", "pod", "(.+)"
-              )
-            ||| % $._config,
-            labels: {
-              workload_type: 'rollout',
-            },
-          },
-          // workload aggregation for CatalogSource
-          {
-            record: 'namespace_workload_pod:kube_pod_owner:relabel',
-            expr: |||
-              max by(%(clusterLabel)s, namespace, workload, pod) (
-                label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="CatalogSource"}
-                ), "workload", "$1", "pod", "(.+)"
-              )
-            ||| % $._config,
-            labels: {
-              workload_type: 'catalogsource',
-            },
-          },
-          // workload aggregation for ClusterPolicy
-          {
-            record: 'namespace_workload_pod:kube_pod_owner:relabel',
-            expr: |||
-              max by(%(clusterLabel)s, namespace, workload, pod) (
-                label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ClusterPolicy"}
-                ), "workload", "$1", "pod", "(.+)"
-              )
-            ||| % $._config,
-            labels: {
-              workload_type: 'clusterpolicy',
-            },
-          },
-          // workload aggregation for ConfigMap
-          {
-            record: 'namespace_workload_pod:kube_pod_owner:relabel',
-            expr: |||
-              max by(%(clusterLabel)s, namespace, workload, pod) (
-                label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ConfigMap"}
-                ), "workload", "$1", "pod", "(.+)"
-              )
-            ||| % $._config,
-            labels: {
-              workload_type: 'configmap',
-            },
-          },
-          // workload aggregation for EphemeralRunner
-          {
-            record: 'namespace_workload_pod:kube_pod_owner:relabel',
-            expr: |||
-              max by(%(clusterLabel)s, namespace, workload, pod) (
-                label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="EphemeralRunner"}
-                ), "workload", "$1", "pod", "(.+)"
-              )
-            ||| % $._config,
-            labels: {
-              workload_type: 'ephemeralrunner',
-            },
-          },  
-          // workload aggregation for AutoscalingListener
-          {
-            record: 'namespace_workload_pod:kube_pod_owner:relabel',
-            expr: |||
-              max by(%(clusterLabel)s, namespace, workload, pod) (
-                label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="AutoscalingListener"}
-                ), "workload", "$1", "pod", "(.+)"
-              )
-            ||| % $._config,
-            labels: {
-              workload_type: 'autoscalinglistener',
-            },
-          },
-          // workload aggregation for Workflow
-          {
-            record: 'namespace_workload_pod:kube_pod_owner:relabel',
-            expr: |||
-              max by(%(clusterLabel)s, namespace, workload, pod) (
-                label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Workflow"}
-                ), "workload", "$1", "pod", "(.+)"
+              max by(%(clusterLabel)s, namespace, workload, pod, workload_type) (
+                label_join(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!~"ReplicaSet|Deployment|DaemonSet|StatefulSet|Job||Node"},
+                  "workload_type", "", "owner_kind"
+                )
+                or
+                label_join(
+                  kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind!~"Deployment"},
+                  "workload_type", "", "owner_kind"
+                )
               )
             ||| % $._config,
-            labels: {
-              workload_type: 'workflow',
-            },
           },
         ],
       },

From ba8220fce3bc78f307ce3cc0e323c1bf55a02c94 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Mon, 28 Apr 2025 09:51:12 +0200
Subject: [PATCH 05/42] chore: add cluster labels to workload rules

---
 rules/apps.libsonnet | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index d15677de3..705d28cd9 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -214,8 +214,8 @@
                   label_replace(
                     kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"},
                     "replicaset", "$1", "owner_name", "(.*)"
-                  ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (
-                    1, max by (replicaset, namespace, owner_name) (
+                  ) * on(replicaset, namespace) group_left(owner_name) topk by(%(clusterLabel)s, replicaset, namespace) (
+                    1, max by (%(clusterLabel)s, replicaset, namespace, owner_name) (
                       kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind=""}
                     )
                   ),
@@ -236,8 +236,8 @@
                   label_replace(
                     kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"},
                     "replicaset", "$1", "owner_name", "(.*)"
-                  ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) (
-                    1, max by (replicaset, namespace, owner_name) (
+                  ) * on(replicaset, namespace, %(clusterLabel)s) group_left(owner_name) topk by(%(clusterLabel)s, replicaset, namespace) (
+                    1, max by (%(clusterLabel)s, replicaset, namespace, owner_name) (
                       kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind="Deployment"}
                     )
                   ),

From ea4b487373da7a429cce6de534f253e27a23af66 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Tue, 29 Apr 2025 23:12:38 +0200
Subject: [PATCH 06/42] chore: fix non standard workload type query

---
 rules/apps.libsonnet | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 705d28cd9..306d77fc7 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -298,10 +298,8 @@
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
-              max by(%(clusterLabel)s, namespace, workload, pod) (
-                label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind=""}
-                ), "workload", "$1", "barepod", "(.+)"
+              max by(%(clusterLabel)s, namespace, pod) (
+                kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="", owner_name=""}
               )
             ||| % $._config,
             labels: {
@@ -326,16 +324,13 @@
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
-              max by(%(clusterLabel)s, namespace, workload, pod, workload_type) (
-                label_join(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!~"ReplicaSet|Deployment|DaemonSet|StatefulSet|Job||Node"},
-                  "workload_type", "", "owner_kind"
-                )
-                or
-                label_join(
-                  kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind!~"Deployment"},
-                  "workload_type", "", "owner_kind"
-                )
+              label_replace(
+                kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!~"ReplicaSet|Deployment|DaemonSet|StatefulSet|Job|Node|"}
+                , "replicaset", "$1", "owner_name", "(.+)"
+              )
+              * on(%(clusterLabels)s, namespace, replicaset) group_left(owner_kind)
+              group by(%(clusterLabels)s, namespace, replicaset, owner_kind) (
+                kube_replicaset_owner{job!="", owner_kind!~"Deployment|"}
               )
             ||| % $._config,
           },

From 72af1fe7f3e91ef3585d7548f96d306ab1beb601 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Wed, 30 Apr 2025 14:15:47 +0200
Subject: [PATCH 07/42] chore: fix non-standard rule

---
 rules/apps.libsonnet | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 306d77fc7..9856923c4 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -325,7 +325,7 @@
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
               label_replace(
-                kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!~"ReplicaSet|Deployment|DaemonSet|StatefulSet|Job|Node|"}
+                kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!~"Deployment|DaemonSet|StatefulSet|Job|Node|"}
                 , "replicaset", "$1", "owner_name", "(.+)"
               )
               * on(%(clusterLabels)s, namespace, replicaset) group_left(owner_kind)

From d6c2fd37ade9e28724b86b0e2b16f32985c5791a Mon Sep 17 00:00:00 2001
From: Serena <26805916+sleepyfoodie@users.noreply.github.com>
Date: Thu, 1 May 2025 09:06:00 +0200
Subject: [PATCH 08/42] Update rules/apps.libsonnet

Co-authored-by: Stephen Lang <skl@users.noreply.github.com>
---
 rules/apps.libsonnet | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 9856923c4..3e3bef282 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -325,7 +325,7 @@
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
               label_replace(
-                kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!~"Deployment|DaemonSet|StatefulSet|Job|Node|"}
+                kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!="Deployment", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
                 , "replicaset", "$1", "owner_name", "(.+)"
               )
               * on(%(clusterLabels)s, namespace, replicaset) group_left(owner_kind)

From 875b51909966d65f39c55ee113a8421798c53cf2 Mon Sep 17 00:00:00 2001
From: Serena <26805916+sleepyfoodie@users.noreply.github.com>
Date: Thu, 1 May 2025 09:06:26 +0200
Subject: [PATCH 09/42] Update rules/apps.libsonnet

Co-authored-by: Stephen Lang <skl@users.noreply.github.com>
---
 rules/apps.libsonnet | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 3e3bef282..1bfaba0fc 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -330,7 +330,7 @@
               )
               * on(%(clusterLabels)s, namespace, replicaset) group_left(owner_kind)
               group by(%(clusterLabels)s, namespace, replicaset, owner_kind) (
-                kube_replicaset_owner{job!="", owner_kind!~"Deployment|"}
+                kube_replicaset_owner{job!="", owner_kind!="Deployment", owner_kind!=""}
               )
             ||| % $._config,
           },

From 47a80db9cac9da82ec591be8b06a0e861beffe53 Mon Sep 17 00:00:00 2001
From: Serena <26805916+sleepyfoodie@users.noreply.github.com>
Date: Thu, 1 May 2025 09:06:35 +0200
Subject: [PATCH 10/42] Update rules/apps.libsonnet

Co-authored-by: Stephen Lang <skl@users.noreply.github.com>
---
 rules/apps.libsonnet | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 1bfaba0fc..552ec7fd3 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -214,7 +214,7 @@
                   label_replace(
                     kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"},
                     "replicaset", "$1", "owner_name", "(.*)"
-                  ) * on(replicaset, namespace) group_left(owner_name) topk by(%(clusterLabel)s, replicaset, namespace) (
+                  ) * on (%(clusterLabel), replicaset, namespace) group_left(owner_name) topk by(%(clusterLabel)s, replicaset, namespace) (
                     1, max by (%(clusterLabel)s, replicaset, namespace, owner_name) (
                       kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind=""}
                     )

From 77b22270178b4d4253d1d109fef955d809405d97 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 1 May 2025 10:47:36 +0200
Subject: [PATCH 11/42] chore: fix non-standard labels from comments

---
 rules/apps.libsonnet | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 552ec7fd3..170ed7612 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -324,13 +324,24 @@
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
-              label_replace(
-                kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!="Deployment", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
-                , "replicaset", "$1", "owner_name", "(.+)"
-              )
-              * on(%(clusterLabels)s, namespace, replicaset) group_left(owner_kind)
-              group by(%(clusterLabels)s, namespace, replicaset, owner_kind) (
-                kube_replicaset_owner{job!="", owner_kind!="Deployment", owner_kind!=""}
+              max by (%(clusterLabel)s, namespace, pod, workload, workload_type) (
+                label_replace(
+                  label_replace(
+                    kube_pod_owner{job!="", owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
+                  , "workload", "$1", "owner_name", "(.+)")
+                  * on(%(clusterLabel)s, namespace, replicaset) group_left(owner_kind)
+                  label_replace(
+                    group by (%(clusterLabel)s, namespace, replicaset, owner_kind) (
+                      kube_replicaset_owner{job!="", owner_kind!="Deployment", owner_kind!=""}
+                    )
+                  , "workload", "$1", "replicaset", "(.+)")
+                  OR
+                  label_replace(
+                    group by (%(clusterLabel)s, namespace, pod, owner_name, owner_kind) (
+                      kube_pod_owner{ owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
+                    )
+                  , "workload", "$1", "owner_name", "(.+)")
+                , "workload_type", "$1", "owner_kind", "(.+)")
               )
             ||| % $._config,
           },

From 34dd76df6ebd1018bc661eb4ce1bcd2e273c03ac Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 1 May 2025 11:26:02 +0200
Subject: [PATCH 12/42] chore: fix barepod to include workload but with pod
 name

---
 rules/apps.libsonnet | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 170ed7612..6e777f503 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -299,7 +299,10 @@
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
               max by(%(clusterLabel)s, namespace, pod) (
-                kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="", owner_name=""}
+                label_replace(
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="", owner_name=""},
+                  "workload", "$1", "pod", "(.+)"
+                )
               )
             ||| % $._config,
             labels: {
@@ -312,8 +315,9 @@
             expr: |||
               max by(%(clusterLabel)s, namespace, workload, pod) (
                 label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Node"}
-                ), "workload", "$1", "pod", "(.+)"
+                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Node"},
+                  "workload", "$1", "pod", "(.+)"
+                )
               )
             ||| % $._config,
             labels: {

From 4c7f7e77b757305c786134fc739e4eeb96f2b678 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 1 May 2025 12:30:43 +0200
Subject: [PATCH 13/42] fix: add CRDs non standard workload type

---
 rules/apps.libsonnet | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 6e777f503..232249f6c 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -324,7 +324,7 @@
               workload_type: 'staticpod',
             },
           },
-          // workload aggregation for non-standard workloads
+          // workload aggregation for non-standard workloads for replicaset
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
@@ -339,7 +339,16 @@
                       kube_replicaset_owner{job!="", owner_kind!="Deployment", owner_kind!=""}
                     )
                   , "workload", "$1", "replicaset", "(.+)")
-                  OR
+                , "workload_type", "$1", "owner_kind", "(.+)")
+              )
+            ||| % $._config,
+          },
+          // workload aggregation for non-standard workloads for CRDs
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              max by (%(clusterLabel)s, namespace, pod, workload, workload_type) (
+                label_replace(
                   label_replace(
                     group by (%(clusterLabel)s, namespace, pod, owner_name, owner_kind) (
                       kube_pod_owner{ owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}

From cabd5252880dadd356508d9d109ac6e98df303b6 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 1 May 2025 12:43:56 +0200
Subject: [PATCH 14/42] chore: push prev lost changes... not sure what
 happened?

---
 rules/apps.libsonnet | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 232249f6c..5e638dc1d 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -331,30 +331,25 @@
               max by (%(clusterLabel)s, namespace, pod, workload, workload_type) (
                 label_replace(
                   label_replace(
-                    kube_pod_owner{job!="", owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
-                  , "workload", "$1", "owner_name", "(.+)")
-                  * on(%(clusterLabel)s, namespace, replicaset) group_left(owner_kind)
+                    kube_pod_owner{job!="", owner_kind="ReplicaSet"}
+                    , "workload", "$1", "owner_name", "(.+)"
+                  )
+                  * on(%(clusterLabel)s, namespace, workload) group_left(owner_kind)
                   label_replace(
-                    group by (%(clusterLabel)s, namespace, replicaset, owner_kind) (
+                    group by (%(clusterLabel)s, namespace, replicaset, owner_kind, owner_name) (
                       kube_replicaset_owner{job!="", owner_kind!="Deployment", owner_kind!=""}
                     )
-                  , "workload", "$1", "replicaset", "(.+)")
-                , "workload_type", "$1", "owner_kind", "(.+)")
-              )
-            ||| % $._config,
-          },
-          // workload aggregation for non-standard workloads for CRDs
-          {
-            record: 'namespace_workload_pod:kube_pod_owner:relabel',
-            expr: |||
-              max by (%(clusterLabel)s, namespace, pod, workload, workload_type) (
-                label_replace(
+                    , "workload", "$1", "replicaset", "(.+)"
+                  )
+                  OR
                   label_replace(
-                    group by (%(clusterLabel)s, namespace, pod, owner_name, owner_kind) (
+                    group by (cluster, namespace, pod, owner_name, owner_kind) (
                       kube_pod_owner{ owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
                     )
-                  , "workload", "$1", "owner_name", "(.+)")
-                , "workload_type", "$1", "owner_kind", "(.+)")
+                    , "workload", "$1", "owner_name", "(.+)"
+                  )
+                  , "workload_type", "$1", "owner_kind", "(.+)"
+                )
               )
             ||| % $._config,
           },

From 1ea1d1515ac09d32f8c235f6046dc9d88f85e94f Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 1 May 2025 12:47:03 +0200
Subject: [PATCH 15/42] chore: modify barepod to include workload type

---
 rules/apps.libsonnet | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 5e638dc1d..6c64ff4a2 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -298,7 +298,7 @@
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
-              max by(%(clusterLabel)s, namespace, pod) (
+              max by(%(clusterLabel)s, namespace, pod, workload) (
                 label_replace(
                   kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="", owner_name=""},
                   "workload", "$1", "pod", "(.+)"
@@ -316,7 +316,7 @@
               max by(%(clusterLabel)s, namespace, workload, pod) (
                 label_replace(
                   kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Node"},
-                  "workload", "$1", "pod", "(.+)"
+                  "workload", "$1", "owner_name", "(.+)"
                 )
               )
             ||| % $._config,

From 728a650006ce60e1ab65912f31111352cd8a3b07 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Wed, 7 May 2025 12:12:32 -0400
Subject: [PATCH 16/42] test: wip

---
 tests/rules-pod-owner-test.yaml | 24 ++++++++++++++++++++++++
 tests/tests.yaml                | 23 -----------------------
 2 files changed, 24 insertions(+), 23 deletions(-)
 create mode 100644 tests/rules-pod-owner-test.yaml

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
new file mode 100644
index 000000000..41b36f304
--- /dev/null
+++ b/tests/rules-pod-owner-test.yaml
@@ -0,0 +1,24 @@
+rules_files:
+- ../prometheus_rules.yaml
+
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
+    values: '1 1'
+  - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
+    values: '1 stale'
+  - series: 'kube_replicaset_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
+    values: '1 1'
+  - series: 'kube_replicaset_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
+    values: '1 stale'
+  promql_expr_test:
+  - eval_time: 0m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="ds-7cc77d965f-cgsdv", workload="ds", workload_type="deployment"}'
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="ds-7cc77d965f-cgsdv", workload="ds", workload_type="deployment"}'
diff --git a/tests/tests.yaml b/tests/tests.yaml
index 16607f881..c9d4b3b5b 100644
--- a/tests/tests.yaml
+++ b/tests/tests.yaml
@@ -772,29 +772,6 @@ tests:
     - value: 1
       labels: 'node:node_num_cpu:sum{cluster="kubernetes",node="node-2"}'
 
-
-- interval: 1m
-  input_series:
-  - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
-    values: '1 1'
-  - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
-    values: '1 stale'
-  - series: 'kube_replicaset_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
-    values: '1 1'
-  - series: 'kube_replicaset_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
-    values: '1 stale'
-  promql_expr_test:
-  - eval_time: 0m
-    expr: namespace_workload_pod:kube_pod_owner:relabel
-    exp_samples:
-    - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="ds-7cc77d965f-cgsdv", workload="ds", workload_type="deployment"}'
-  - eval_time: 1m
-    expr: namespace_workload_pod:kube_pod_owner:relabel
-    exp_samples:
-    - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="ds-7cc77d965f-cgsdv", workload="ds", workload_type="deployment"}'
-
 - interval: 1m
   input_series:
   - series: 'kube_pod_status_phase{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",phase="Pending",pod="pod-ds-7cc77d965f-cgsdv",service="ksm"}'

From 81368695339ff563f5c6dc080be6acd0fc287809 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Fri, 9 May 2025 12:50:33 -0400
Subject: [PATCH 17/42] ci: add test for new workload type support

---
 tests/rules-pod-owner-test.yaml | 54 +++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 41b36f304..64780db1e 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -22,3 +22,57 @@ rules_files:
     exp_samples:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="ds-7cc77d965f-cgsdv", workload="ds", workload_type="deployment"}'
+
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="false",owner_kind="",owner_name="bare-pod-xyz123",pod="bare-pod-xyz123",service="ksm"}'
+    values: '1 1'
+  - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns2",owner_is_controller="false",owner_kind="",owner_name="bare-pod-xyz123",pod="bare-pod-xyz123",service="ksm"}'
+    values: '1 stale'
+  promql_expr_test:
+  - eval_time: 0m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="bare-pod-xyz123", workload="bare-pod-xyz123", workload_type="barepod"}'
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="bare-pod-xyz123", workload="bare-pod-xyz123", workload_type="barepod"}'
+
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Node",owner_name="etcd-node1",pod="etcd-node1",service="ksm"}'
+    values: '1 1'
+  - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns2",owner_is_controller="true",owner_kind="Node",owner_name="etcd-node1",pod="etcd-node1",service="ksm"}'
+    values: '1 stale'
+  promql_expr_test:
+  - eval_time: 0m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="etcd-node1", workload="etcd-node1", workload_type="staticpod"}'
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="etcd-node1", workload="etcd-node1", workload_type="staticpod"}'
+
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="CustomController",owner_name="myapp-controller",pod="myapp-controller-abc123",service="ksm"}'
+    values: '1 1'
+  - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="CustomController",owner_name="myapp-controller",pod="myapp-controller-abc123",service="ksm"}'
+    values: '1 stale'
+  promql_expr_test:
+  - eval_time: 0m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="myapp-controller-abc123", workload="myapp-controller", workload_type="customcontroller"}'
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="myapp-controller-abc123", workload="myapp-controller", workload_type="customcontroller"}'
\ No newline at end of file

From 50e2b758c2c3295ad1638573267aed51cee8f9be Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Mon, 12 May 2025 09:10:00 -0400
Subject: [PATCH 18/42] chore: add fields for tests

---
 tests/rules-pod-owner-test.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 64780db1e..f13187a1f 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -1,6 +1,9 @@
 rules_files:
 - ../prometheus_rules.yaml
 
+evaluation_interval: 1m
+
+tests:
 - interval: 1m
   input_series:
   - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'

From b1236a3539059f28cff7999b7236707fcec82619 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Mon, 12 May 2025 14:03:42 -0400
Subject: [PATCH 19/42] chore: wip - got is nil

---
 rules/apps.libsonnet            |  4 +-
 tests/rules-pod-owner-test.yaml | 69 ++++++++++++++++++---------------
 2 files changed, 40 insertions(+), 33 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 6c64ff4a2..787151a69 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -205,7 +205,7 @@
       {
         name: 'k8s.rules.pod_owner',
         rules: [
-          // workload aggregation for replicasets
+          // // workload aggregation for replicasets
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
@@ -214,7 +214,7 @@
                   label_replace(
                     kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"},
                     "replicaset", "$1", "owner_name", "(.*)"
-                  ) * on (%(clusterLabel), replicaset, namespace) group_left(owner_name) topk by(%(clusterLabel)s, replicaset, namespace) (
+                  ) * on (%(clusterLabel)s, replicaset, namespace) group_left(owner_name) topk by(%(clusterLabel)s, replicaset, namespace) (
                     1, max by (%(clusterLabel)s, replicaset, namespace, owner_name) (
                       kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind=""}
                     )
diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index f13187a1f..2d4550da5 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -1,4 +1,5 @@
-rules_files:
+rule_files:
+- ../prometheus_alerts.yaml
 - ../prometheus_rules.yaml
 
 evaluation_interval: 1m
@@ -26,56 +27,62 @@ tests:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="ds-7cc77d965f-cgsdv", workload="ds", workload_type="deployment"}'
 
+# bare pod
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="false",owner_kind="",owner_name="bare-pod-xyz123",pod="bare-pod-xyz123",service="ksm"}'
-    values: '1 1'
-  - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns2",owner_is_controller="false",owner_kind="",owner_name="bare-pod-xyz123",pod="bare-pod-xyz123",service="ksm"}'
-    values: '1 stale'
+  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="integrations/kubernetes/kube-state-metrics", instance="10.128.0.135:8443"}'
+    values: '1'
+  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:8443"}'
+    values: '1'
   promql_expr_test:
-  - eval_time: 0m
-    expr: namespace_workload_pod:kube_pod_owner:relabel
-    exp_samples:
-    - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="bare-pod-xyz123", workload="bare-pod-xyz123", workload_type="barepod"}'
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="bare-pod-xyz123", workload="bare-pod-xyz123", workload_type="barepod"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="openshift", namespace="default", pod="nginx", workload="nginx", workload_type="barepod"}'
 
+# static pod
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Node",owner_name="etcd-node1",pod="etcd-node1",service="ksm"}'
-    values: '1 1'
-  - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns2",owner_is_controller="true",owner_kind="Node",owner_name="etcd-node1",pod="etcd-node1",service="ksm"}'
-    values: '1 stale'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
   promql_expr_test:
-  - eval_time: 0m
-    expr: namespace_workload_pod:kube_pod_owner:relabel
-    exp_samples:
-    - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="etcd-node1", workload="etcd-node1", workload_type="staticpod"}'
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="etcd-node1", workload="etcd-node1", workload_type="staticpod"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="staticpod"}'
 
+# replicaset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="CustomController",owner_name="myapp-controller",pod="myapp-controller-abc123",service="ksm"}'
-    values: '1 1'
-  - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="CustomController",owner_name="myapp-controller",pod="myapp-controller-abc123",service="ksm"}'
-    values: '1 stale'
+  - series: 'kube_pod_owner{owner_kind="ReplicaSet", cluster="duplicate", namespace="kube-system", pod="kube-dns-9db7cdc57-26g7k", owner_name="kube-dns-9db7cdc57", job="integrations/kubernetes/kube-state-metrics", instance="10.128.0.135:8443"}'
+    values: '1'
+  - series: 'kube_pod_owner{owner_kind="ReplicaSet", cluster="duplicate", namespace="kube-system", pod="kube-dns-9db7cdc57-26g7k", owner_name="kube-dns-9db7cdc57", job="kube-state-metrics", instance="10.128.0.135:8443"}'
+    values: '1'
+  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",replicaset="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
   promql_expr_test:
-  - eval_time: 0m
-    expr: namespace_workload_pod:kube_pod_owner:relabel
-    exp_samples:
-    - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="myapp-controller-abc123", workload="myapp-controller", workload_type="customcontroller"}'
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="myapp-controller-abc123", workload="myapp-controller", workload_type="customcontroller"}'
\ No newline at end of file
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="kube-system", pod="kube-dns-9db7cdc57-26g7k", workload="kube-dns-9db7cdc57", workload_type="ReplicaSet"}'
+
+# non-standard replicaset
+# - interval: 1m
+#   input_series:
+#   - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="custom-controller",pod="custom-controller-abc123",service="ksm"}'
+#     values: '1 1'
+#   - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="replicaset-123",pod="replicaset-123",service="ksm"}'
+#     values: '1 stale'
+#   - series: 'kube_replicaset_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="CustomController",owner_name="custom-controller",pod="custom-controller-abc123",replicaset="custom-controller",service="ksm"}'
+#     values: '1 1'
+#   promql_expr_test:
+#   - eval_time: 1m
+#     expr: namespace_workload_pod:kube_pod_owner:relabel
+#     exp_samples:
+#     - value: 1
+#       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="custom-controller-abc123", workload="custom-controller", workload_type="customcontroller"}'
\ No newline at end of file

From 2a7cd1dc99e4f76fedd700c901a30203e9759c18 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Mon, 12 May 2025 14:26:31 -0400
Subject: [PATCH 20/42] fix: test for new rules

---
 tests/rules-pod-owner-test.yaml | 42 ++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 2d4550da5..6ecda6daf 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -58,31 +58,31 @@ tests:
 # replicaset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{owner_kind="ReplicaSet", cluster="duplicate", namespace="kube-system", pod="kube-dns-9db7cdc57-26g7k", owner_name="kube-dns-9db7cdc57", job="integrations/kubernetes/kube-state-metrics", instance="10.128.0.135:8443"}'
-    values: '1'
-  - series: 'kube_pod_owner{owner_kind="ReplicaSet", cluster="duplicate", namespace="kube-system", pod="kube-dns-9db7cdc57-26g7k", owner_name="kube-dns-9db7cdc57", job="kube-state-metrics", instance="10.128.0.135:8443"}'
-    values: '1'
-  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",replicaset="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
-    values: '1'
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1 1'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics2",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1 stale'
+  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",replicaset="gke-duplicate-default-pool-10012"}'
+    values: '1 1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="kube-system", pod="kube-dns-9db7cdc57-26g7k", workload="kube-dns-9db7cdc57", workload_type="ReplicaSet"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-10012", workload_type="replicaset"}'
 
 # non-standard replicaset
-# - interval: 1m
-#   input_series:
-#   - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="custom-controller",pod="custom-controller-abc123",service="ksm"}'
-#     values: '1 1'
-#   - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="replicaset-123",pod="replicaset-123",service="ksm"}'
-#     values: '1 stale'
-#   - series: 'kube_replicaset_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="CustomController",owner_name="custom-controller",pod="custom-controller-abc123",replicaset="custom-controller",service="ksm"}'
-#     values: '1 1'
-#   promql_expr_test:
-#   - eval_time: 1m
-#     expr: namespace_workload_pod:kube_pod_owner:relabel
-#     exp_samples:
-#     - value: 1
-#       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="custom-controller-abc123", workload="custom-controller", workload_type="customcontroller"}'
\ No newline at end of file
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="custom-controller",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1 1'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics2",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Deployment",owner_name="custom-controller",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1 stale'
+  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CustomController",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",replicaset="custom-controller"}'
+    values: '1 1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-controller", workload_type="CustomController"}'
\ No newline at end of file

From 76cf7aeb9b6a6402336ced271a3d3e8ec74af597 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Mon, 12 May 2025 15:55:33 -0400
Subject: [PATCH 21/42] chore: add tests for all recording rules under
 kube_pod_owner

---
 tests/rules-pod-owner-test.yaml | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 6ecda6daf..200604889 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -5,6 +5,7 @@ rule_files:
 evaluation_interval: 1m
 
 tests:
+# deployment
 - interval: 1m
   input_series:
   - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
@@ -55,6 +56,34 @@ tests:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="staticpod"}'
 
+# statefulset
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="statefulset"}'
+
+# daemonset
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Daemonset",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Daemonset",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="Daemonset"}'
+
 # replicaset
 - interval: 1m
   input_series:
@@ -85,4 +114,4 @@ tests:
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-controller", workload_type="CustomController"}'
\ No newline at end of file
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-controller", workload_type="CustomController"}'

From c6bcfa0a487e175c2ca069619ebc782ccf84673b Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Mon, 12 May 2025 16:13:45 -0400
Subject: [PATCH 22/42] chore: add test case for new job rule

---
 rules/apps.libsonnet            | 25 ++++++++++++++++++++++++-
 tests/rules-pod-owner-test.yaml | 17 +++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 787151a69..118882324 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -279,7 +279,30 @@
               workload_type: 'statefulset',
             },
           },
-          // workload aggregation for jobs
+          // workload aggregation for jobs (cronjobs, jobs, etc)
+          {
+            record: 'namespace_workload_pod:kube_pod_owner:relabel',
+            expr: |||
+              group by(%(clusterLabel)s, namespace, workload, workload_type, pod) (
+                label_join(
+                  label_join(
+                    group by (%(clusterLabel)s, namespace, job_name, pod) (label_join(
+                        kube_pod_owner{%(kubeStateMetricsSelector)s,job!="", owner_kind="Job"}
+                    , "job_name", "", "owner_name"))
+                    * on (%(clusterLabel)s, namespace, job_name) group_left(owner_kind, owner_name)
+                    group by (%(clusterLabel)s, namespace, job_name, owner_kind, owner_name) (
+                        kube_job_owner{%(kubeStateMetricsSelector)s,job!="", owner_kind!="Pod", owner_kind!=""}
+                    )
+                    OR on (%(clusterLabel)s, namespace, pod)
+                    group by (%(clusterLabel)s, namespace, owner_name, owner_kind, pod) (
+                        kube_pod_owner{%(kubeStateMetricsSelector)s, job!="", owner_kind="Job"}
+                    )
+                  , "workload", "", "owner_name")
+                , "workload_type", "", "owner_kind")
+              )
+            ||| % $._config,
+          }
+          // backwards compatibility for jobs
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 200604889..3b7d659fb 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -100,6 +100,23 @@ tests:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-10012", workload_type="replicaset"}'
 
+# job
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",owner_name="custom-job",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1 1'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics2",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",owner_name="custom-job",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1 stale'
+  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",job_name="custom-job"}'
+    values: '1 1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-job", workload_type="CronJob"}'
+
+
 # non-standard replicaset
 - interval: 1m
   input_series:

From 66525209935e661710f5d2b28bec34696fc1351b Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Tue, 13 May 2025 16:32:16 -0400
Subject: [PATCH 23/42] chore: add new test cases for job

---
 tests/rules-pod-owner-test.yaml | 69 +++++++++++++++++++++------------
 1 file changed, 45 insertions(+), 24 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 3b7d659fb..b7aff9121 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -70,62 +70,83 @@ tests:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="statefulset"}'
 
-# daemonset
+# # daemonset
+# - interval: 1m
+#   input_series:
+#   - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Daemonset",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+#     values: '1'
+#   - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Daemonset",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+#     values: '1'
+#   promql_expr_test:
+#   - eval_time: 1m
+#     expr: namespace_workload_pod:kube_pod_owner:relabel
+#     exp_samples:
+#     - value: 1
+#       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="daemonset"}'
+
+# replicaset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Daemonset",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1 '
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics2",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Daemonset",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",replicaset="gke-duplicate-default-pool-10012"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="Daemonset"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-10012", workload_type="replicaset"}'
 
-# replicaset
+# CronJob
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
-    values: '1 1'
-  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics2",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
-    values: '1 stale'
-  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",replicaset="gke-duplicate-default-pool-10012"}'
-    values: '1 1'
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",owner_name="custom-job123",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",owner_name="custom-job123", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",job_name="custom-job"}'
+    values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-10012", workload_type="replicaset"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-job123", workload_type="CronJob"}'
 
-# job
+# ScaledJob
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",owner_name="custom-job",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
-    values: '1 1'
-  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics2",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",owner_name="custom-job",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
-    values: '1 stale'
-  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",job_name="custom-job"}'
-    values: '1 1'
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ScaledJob",owner_name="custom-job123",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ScaledJob",owner_name="custom-job123", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",job_name="custom-job"}'
+    values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-job", workload_type="CronJob"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-job123", workload_type="ScaledJob"}'
 
+# job
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="custom-job123",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-job123", workload_type="job"}'
 
 # non-standard replicaset
 - interval: 1m
   input_series:
   - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="custom-controller",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
-    values: '1 1'
-  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics2",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Deployment",owner_name="custom-controller",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
-    values: '1 stale'
+    values: '1'
   - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CustomController",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",replicaset="custom-controller"}'
-    values: '1 1'
+    values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel

From ac59da10bb3a1069e7726887c26791a38417363c Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Tue, 13 May 2025 17:01:06 -0400
Subject: [PATCH 24/42] fix: add missing label in nonstandard replicaset input
 serires

---
 tests/rules-pod-owner-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index b7aff9121..e66c63ceb 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -145,7 +145,7 @@ tests:
   input_series:
   - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="custom-controller",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CustomController",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",replicaset="custom-controller"}'
+  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CustomController", owner_name="custom-controller", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",replicaset="custom-controller"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m

From 3a9dc6b48fa7eacaf4803b2421423f15c0b734a6 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Wed, 14 May 2025 07:34:28 -0400
Subject: [PATCH 25/42] chore: fix hashtag comments

---
 rules/apps.libsonnet            |  2 +-
 tests/rules-pod-owner-test.yaml | 26 +++++++++++++-------------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 118882324..84406a4ed 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -205,7 +205,7 @@
       {
         name: 'k8s.rules.pod_owner',
         rules: [
-          // // workload aggregation for replicasets
+          // workload aggregation for replicasets
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index e66c63ceb..a098d6487 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -70,19 +70,19 @@ tests:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="statefulset"}'
 
-# # daemonset
-# - interval: 1m
-#   input_series:
-#   - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Daemonset",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
-#     values: '1'
-#   - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Daemonset",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
-#     values: '1'
-#   promql_expr_test:
-#   - eval_time: 1m
-#     expr: namespace_workload_pod:kube_pod_owner:relabel
-#     exp_samples:
-#     - value: 1
-#       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="daemonset"}'
+# daemonset
+- interval: 1m
+  input_series:
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="daemonset"}'
 
 # replicaset
 - interval: 1m

From 2b6a802249244eeaa15a3fe857ee09d5f932954c Mon Sep 17 00:00:00 2001
From: Serena <26805916+sleepyfoodie@users.noreply.github.com>
Date: Wed, 14 May 2025 11:59:26 -0400
Subject: [PATCH 26/42] Update tests/rules-pod-owner-test.yaml

Co-authored-by: Stephen Lang <skl@users.noreply.github.com>
---
 tests/rules-pod-owner-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index a098d6487..cde66b411 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -117,7 +117,7 @@ tests:
 # ScaledJob
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ScaledJob",owner_name="custom-job123",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="custom-job123",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ScaledJob",owner_name="custom-job123", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",job_name="custom-job"}'
     values: '1'

From 63ef8150d1445935745f11c286bef3e39b42ba86 Mon Sep 17 00:00:00 2001
From: Serena <26805916+sleepyfoodie@users.noreply.github.com>
Date: Wed, 14 May 2025 12:02:49 -0400
Subject: [PATCH 27/42] Update tests/rules-pod-owner-test.yaml

Co-authored-by: Stephen Lang <skl@users.noreply.github.com>
---
 tests/rules-pod-owner-test.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index cde66b411..95d5742dc 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -103,9 +103,9 @@ tests:
 # CronJob
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",owner_name="custom-job123",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="cron-job-1234",pod="cron-job-1243-abcd"}'
     values: '1'
-  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",owner_name="custom-job123", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",job_name="custom-job"}'
+  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",owner_name="cron-job",job_name="cron-job-1234"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m

From 453d418a708a99618c5795800da5ed422376913d Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Wed, 14 May 2025 17:10:18 -0400
Subject: [PATCH 28/42] fix: add owner_name into non-standard replicaset
 group_left, cronjob test wip

---
 rules/apps.libsonnet            | 38 +++++++++++--------
 tests/rules-pod-owner-test.yaml | 65 +++++++++++++++++----------------
 2 files changed, 56 insertions(+), 47 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 84406a4ed..954742ec3 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -212,14 +212,14 @@
               max by (%(clusterLabel)s, namespace, workload, pod) (
                 label_replace(
                   label_replace(
-                    kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"},
+                    kube_pod_owner{%(kubeStateMetricsSelector)s, job!="", owner_kind="ReplicaSet"},
                     "replicaset", "$1", "owner_name", "(.*)"
                   ) * on (%(clusterLabel)s, replicaset, namespace) group_left(owner_name) topk by(%(clusterLabel)s, replicaset, namespace) (
                     1, max by (%(clusterLabel)s, replicaset, namespace, owner_name) (
-                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind=""}
+                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, job!="", owner_kind=""}
                     )
                   ),
-                  "workload", "$1", "owner_name", "(.*)"
+                  "workload", "$1", "replicaset", "(.*)"
                 )
               )
             ||| % $._config,
@@ -286,16 +286,13 @@
               group by(%(clusterLabel)s, namespace, workload, workload_type, pod) (
                 label_join(
                   label_join(
-                    group by (%(clusterLabel)s, namespace, job_name, pod) (label_join(
-                        kube_pod_owner{%(kubeStateMetricsSelector)s,job!="", owner_kind="Job"}
-                    , "job_name", "", "owner_name"))
+                    group by (%(clusterLabel)s, namespace, job_name, pod) (
+                      label_join(
+                          kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Job"}
+                      , "job_name", "", "owner_name"))
                     * on (%(clusterLabel)s, namespace, job_name) group_left(owner_kind, owner_name)
                     group by (%(clusterLabel)s, namespace, job_name, owner_kind, owner_name) (
-                        kube_job_owner{%(kubeStateMetricsSelector)s,job!="", owner_kind!="Pod", owner_kind!=""}
-                    )
-                    OR on (%(clusterLabel)s, namespace, pod)
-                    group by (%(clusterLabel)s, namespace, owner_name, owner_kind, pod) (
-                        kube_pod_owner{%(kubeStateMetricsSelector)s, job!="", owner_kind="Job"}
+                        kube_job_owner{%(kubeStateMetricsSelector)s, owner_kind!="Pod", owner_kind!=""}
                     )
                   , "workload", "", "owner_name")
                 , "workload_type", "", "owner_kind")
@@ -306,11 +303,20 @@
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
-              max by (%(clusterLabel)s, namespace, workload, pod) (
+              group by(%(clusterLabel)s, namespace, workload, workload_type, pod) (
                 label_replace(
-                  kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Job"},
-                  "workload", "$1", "owner_name", "(.*)"
-                )
+                  label_join(
+                    group by (%(clusterLabel)s, namespace, job_name, pod, owner_name) (
+                      label_join(
+                        kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Job"}
+                      , "job_name", "", "owner_name")
+                    )
+                    * on (%(clusterLabel)s, namespace, job_name) group_left()
+                    group by (%(clusterLabel)s, namespace, job_name) (
+                      kube_job_owner{%(kubeStateMetricsSelector)s, owner_kind=""}
+                    )
+                  , "workload", "", "owner_name")
+                , "workload_type", "job", "", "")
               )
             ||| % $._config,
             labels: {
@@ -357,7 +363,7 @@
                     kube_pod_owner{job!="", owner_kind="ReplicaSet"}
                     , "workload", "$1", "owner_name", "(.+)"
                   )
-                  * on(%(clusterLabel)s, namespace, workload) group_left(owner_kind)
+                  * on(%(clusterLabel)s, namespace, workload) group_left(owner_kind, owner_name)
                   label_replace(
                     group by (%(clusterLabel)s, namespace, replicaset, owner_kind, owner_name) (
                       kube_replicaset_owner{job!="", owner_kind!="Deployment", owner_kind!=""}
diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 95d5742dc..196440153 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -11,11 +11,11 @@ tests:
   - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
     values: '1 1'
   - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
-    values: '1 stale'
+    values: '1'
   - series: 'kube_replicaset_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
     values: '1 1'
   - series: 'kube_replicaset_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
-    values: '1 stale'
+    values: '1'
   promql_expr_test:
   - eval_time: 0m
     expr: namespace_workload_pod:kube_pod_owner:relabel
@@ -100,39 +100,42 @@ tests:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-10012", workload_type="replicaset"}'
 
-# CronJob
-- interval: 1m
-  input_series:
-  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="cron-job-1234",pod="cron-job-1243-abcd"}'
-    values: '1'
-  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CronJob",owner_name="cron-job",job_name="cron-job-1234"}'
-    values: '1'
-  promql_expr_test:
-  - eval_time: 1m
-    expr: namespace_workload_pod:kube_pod_owner:relabel
-    exp_samples:
-    - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-job123", workload_type="CronJob"}'
+# # CronJob
+# - interval: 1m
+#   input_series:
+#   - series: '{__name__="kube_pod_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="integrations/kubernetes/kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
+#     values: '1'
+#   - series: '{__name__="kube_job_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="integrations/kubernetes/kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
+#     values: '1'
+#   promql_expr_test:
+#   - eval_time: 1m
+#     expr: namespace_workload_pod:kube_pod_owner:relabel
+#     exp_samples:
+#     - value: 1
+#       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120880-jfl5c", workload="cuda-vectoradd", workload_type="CronJob"}'
 
-# ScaledJob
-- interval: 1m
-  input_series:
-  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="custom-job123",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
-    values: '1'
-  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ScaledJob",owner_name="custom-job123", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",job_name="custom-job"}'
-    values: '1'
-  promql_expr_test:
-  - eval_time: 1m
-    expr: namespace_workload_pod:kube_pod_owner:relabel
-    exp_samples:
-    - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-job123", workload_type="ScaledJob"}'
+# # ScaledJob
+# - interval: 1m
+#   input_series:
+#   - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="custom-job123",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+#     values: '1'
+#   - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ScaledJob",owner_name="custom-job123", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",job_name="custom-job"}'
+#     values: '1'
+#   promql_expr_test:
+#   - eval_time: 1m
+#     expr: namespace_workload_pod:kube_pod_owner:relabel
+#     exp_samples:
+#     - value: 1
+#       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-job123", workload_type="ScaledJob"}'
 
 # job
 - interval: 1m
   input_series:
   - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="custom-job123",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
+  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="custom-job123"}'
+  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="custom-job123"}'
+    values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
@@ -143,13 +146,13 @@ tests:
 # non-standard replicaset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="custom-controller",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner", cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="integrations/kubernetes/kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="ReplicaSet", owner_name="rollouts-demo-687d76d795", pod="rollouts-demo-687d76d795-5mlvh", uid="491a64cd-6a5f-4c64-b4b8-f95ee6a6b9c0"}'
     values: '1'
-  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="CustomController", owner_name="custom-controller", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",replicaset="custom-controller"}'
+  - series: 'kube_replicaset_owner{__name__="kube_replicaset_owner", cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="integrations/kubernetes/kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="Rollout", owner_name="rollouts-demo", replicaset="rollouts-demo-687d76d795"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-controller", workload_type="CustomController"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="k8sdev/aws/helm",namespace="default", pod="rollouts-demo-687d76d795-5mlvh", workload="rollouts-demo-687d76d795", workload_type="Rollout"}'

From b5c9ff1bad4e9e805158624c7edcfd9d6118c907 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 15 May 2025 07:20:30 -0400
Subject: [PATCH 29/42] fix: owner name in test cases

---
 tests/rules-pod-owner-test.yaml | 61 +++++++++++++++++----------------
 1 file changed, 31 insertions(+), 30 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 196440153..be39ef544 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -45,74 +45,74 @@ tests:
 # static pod
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="staticpod"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="staticpod"}'
 
 # statefulset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="statefulset"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="statefulset"}'
 
 # daemonset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a-kx2v",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a-kx2v", workload_type="daemonset"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="daemonset"}'
 
 # replicaset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
     values: '1 '
-  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics2",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics2",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
     values: '1'
-  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",replicaset="gke-duplicate-default-pool-10012"}'
+  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",replicaset="gke-duplicate-default-pool-10012"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-10012", workload_type="replicaset"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="gke-duplicate-default-pool-10012-kx2v", workload="gke-duplicate-default-pool-10012", workload_type="replicaset"}'
 
-# # CronJob
-# - interval: 1m
-#   input_series:
-#   - series: '{__name__="kube_pod_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="integrations/kubernetes/kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
-#     values: '1'
-#   - series: '{__name__="kube_job_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="integrations/kubernetes/kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
-#     values: '1'
-#   promql_expr_test:
-#   - eval_time: 1m
-#     expr: namespace_workload_pod:kube_pod_owner:relabel
-#     exp_samples:
-#     - value: 1
-#       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120880-jfl5c", workload="cuda-vectoradd", workload_type="CronJob"}'
+# CronJob
+- interval: 1m
+  input_series:
+  - series: '{__name__="kube_pod_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="integrations/kubernetes/kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
+    values: '1'
+  - series: '{__name__="kube_job_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="integrations/kubernetes/kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120880-jfl5c", workload="cuda-vectoradd", workload_type="CronJob"}'
 
 # # ScaledJob
 # - interval: 1m
@@ -131,17 +131,18 @@ tests:
 # job
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="custom-job123",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+    values: '1'
+  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
     values: '1'
-  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="custom-job123"}'
-  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="custom-job123"}'
+  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-job123", workload_type="job"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="kube-proxy-gke-duplicate-default-pool-e20e250a", workload_type="job"}'
 
 # non-standard replicaset
 - interval: 1m

From c00d26bd3b73d539b54d5fc9316c7af8ef8ce98c Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 15 May 2025 07:31:46 -0400
Subject: [PATCH 30/42] chore: modify default job name in cronjob

---
 tests/rules-pod-owner-test.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index be39ef544..e7cfdc36c 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -103,9 +103,9 @@ tests:
 # CronJob
 - interval: 1m
   input_series:
-  - series: '{__name__="kube_pod_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="integrations/kubernetes/kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
+  - series: '{__name__="kube_pod_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
     values: '1'
-  - series: '{__name__="kube_job_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="integrations/kubernetes/kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
+  - series: '{__name__="kube_job_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -147,9 +147,9 @@ tests:
 # non-standard replicaset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{__name__="kube_pod_owner", cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="integrations/kubernetes/kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="ReplicaSet", owner_name="rollouts-demo-687d76d795", pod="rollouts-demo-687d76d795-5mlvh", uid="491a64cd-6a5f-4c64-b4b8-f95ee6a6b9c0"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner", cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="ReplicaSet", owner_name="rollouts-demo-687d76d795", pod="rollouts-demo-687d76d795-5mlvh", uid="491a64cd-6a5f-4c64-b4b8-f95ee6a6b9c0"}'
     values: '1'
-  - series: 'kube_replicaset_owner{__name__="kube_replicaset_owner", cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="integrations/kubernetes/kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="Rollout", owner_name="rollouts-demo", replicaset="rollouts-demo-687d76d795"}'
+  - series: 'kube_replicaset_owner{__name__="kube_replicaset_owner", cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="Rollout", owner_name="rollouts-demo", replicaset="rollouts-demo-687d76d795"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m

From 0b29b38cb2dab1cf083cda28f00fed3a0bb8f776 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 15 May 2025 09:29:24 -0400
Subject: [PATCH 31/42] fix recording rule to only have one dynamic owner_kind

---
 rules/apps.libsonnet            | 68 +++++++++++++++------------------
 tests/rules-pod-owner-test.yaml | 26 ++++++-------
 2 files changed, 44 insertions(+), 50 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 954742ec3..5fb943356 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -279,26 +279,6 @@
               workload_type: 'statefulset',
             },
           },
-          // workload aggregation for jobs (cronjobs, jobs, etc)
-          {
-            record: 'namespace_workload_pod:kube_pod_owner:relabel',
-            expr: |||
-              group by(%(clusterLabel)s, namespace, workload, workload_type, pod) (
-                label_join(
-                  label_join(
-                    group by (%(clusterLabel)s, namespace, job_name, pod) (
-                      label_join(
-                          kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Job"}
-                      , "job_name", "", "owner_name"))
-                    * on (%(clusterLabel)s, namespace, job_name) group_left(owner_kind, owner_name)
-                    group by (%(clusterLabel)s, namespace, job_name, owner_kind, owner_name) (
-                        kube_job_owner{%(kubeStateMetricsSelector)s, owner_kind!="Pod", owner_kind!=""}
-                    )
-                  , "workload", "", "owner_name")
-                , "workload_type", "", "owner_kind")
-              )
-            ||| % $._config,
-          }
           // backwards compatibility for jobs
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
@@ -353,35 +333,49 @@
               workload_type: 'staticpod',
             },
           },
-          // workload aggregation for non-standard workloads for replicaset
+          // workload aggregation for non-standard types (jobs, replicasets)
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
-              max by (%(clusterLabel)s, namespace, pod, workload, workload_type) (
+              group by(%(clusterLabel)s, namespace, workload, workload_type, pod) (
+                label_join(
+                    label_join(
+                        group by (%(clusterLabel)s, namespace, job_name, pod) (
+                            label_join(
+                                kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="Job"}
+                            , "job_name", "", "owner_name")
+                        )
+                        * on (%(clusterLabel)s, namespace, job_name) group_left(owner_kind, owner_name)
+                        group by (%(clusterLabel)s, namespace, job_name, owner_kind, owner_name) (
+                            kube_job_owner{%(kubeStateMetricsSelector)s, owner_kind!="Pod", owner_kind!=""}
+                        )
+                    , "workload", "", "owner_name")
+                , "workload_type", "", "owner_kind")
+                OR
                 label_replace(
-                  label_replace(
-                    kube_pod_owner{job!="", owner_kind="ReplicaSet"}
+                    label_replace(
+                    kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"}
                     , "workload", "$1", "owner_name", "(.+)"
-                  )
-                  * on(%(clusterLabel)s, namespace, workload) group_left(owner_kind, owner_name)
-                  label_replace(
+                    )
+                    * on(%(clusterLabel)s, namespace, workload) group_left(owner_kind, owner_name)
+                    label_replace(
                     group by (%(clusterLabel)s, namespace, replicaset, owner_kind, owner_name) (
-                      kube_replicaset_owner{job!="", owner_kind!="Deployment", owner_kind!=""}
+                        kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind!="Deployment", owner_kind!=""}
                     )
                     , "workload", "$1", "replicaset", "(.+)"
-                  )
-                  OR
-                  label_replace(
-                    group by (cluster, namespace, pod, owner_name, owner_kind) (
-                      kube_pod_owner{ owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
+                    )
+                    OR
+                    label_replace(
+                    group by (%(clusterLabel)s, namespace, pod, owner_name, owner_kind) (
+                        kube_pod_owner{ owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
                     )
                     , "workload", "$1", "owner_name", "(.+)"
-                  )
-                  , "workload_type", "$1", "owner_kind", "(.+)"
+                    )
+                    , "workload_type", "$1", "owner_kind", "(.+)"
                 )
               )
-            ||| % $._config,
-          },
+            ||| % $._config
+          }
         ],
       },
     ],
diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index e7cfdc36c..c82590c86 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -114,19 +114,19 @@ tests:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120880-jfl5c", workload="cuda-vectoradd", workload_type="CronJob"}'
 
-# # ScaledJob
-# - interval: 1m
-#   input_series:
-#   - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="custom-job123",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
-#     values: '1'
-#   - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ScaledJob",owner_name="custom-job123", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v",job_name="custom-job"}'
-#     values: '1'
-#   promql_expr_test:
-#   - eval_time: 1m
-#     expr: namespace_workload_pod:kube_pod_owner:relabel
-#     exp_samples:
-#     - value: 1
-#       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="custom-job123", workload_type="ScaledJob"}'
+# ScaledJob
+- interval: 1m
+  input_series:
+  - series: '{__name__="kube_pod_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
+    values: '1'
+  - series: '{__name__="kube_job_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="ScaledJob", owner_name="cuda-vectoradd"}'
+    values: '1'
+  promql_expr_test:
+  - eval_time: 1m
+    expr: namespace_workload_pod:kube_pod_owner:relabel
+    exp_samples:
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120880-jfl5c", workload="cuda-vectoradd", workload_type="ScaledJob"}'
 
 # job
 - interval: 1m

From 5ce6db95ff3515301518f25a23fb3af2e23b770f Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 15 May 2025 09:50:18 -0400
Subject: [PATCH 32/42] chore: fix test name consistency

---
 tests/rules-pod-owner-test.yaml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index c82590c86..9798bab56 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -31,7 +31,7 @@ tests:
 # bare pod
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="integrations/kubernetes/kube-state-metrics", instance="10.128.0.135:8443"}'
+  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:8443"}'
     values: '1'
   - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:8443"}'
     values: '1'
@@ -45,44 +45,44 @@ tests:
 # static pod
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="staticpod"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="staticpod"}'
 
 # statefulset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="statefulset"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="statefulset"}'
 
 # daemonset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="daemonset"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", namespace="kube-system", pod="gke-duplicate-default-pool-e20e250a-kx2v", workload="gke-duplicate-default-pool-e20e250a", workload_type="daemonset"}'
 
 # replicaset
 - interval: 1m

From 13374819a873ac9ce978f7889aaaa4ed3e47399e Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 15 May 2025 09:54:10 -0400
Subject: [PATCH 33/42] remove alert import

---
 tests/rules-pod-owner-test.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 9798bab56..bbfc603e5 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -1,5 +1,4 @@
 rule_files:
-- ../prometheus_alerts.yaml
 - ../prometheus_rules.yaml
 
 evaluation_interval: 1m

From 405034754b1fc2f5169b531256128a83134681fd Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 15 May 2025 12:06:43 -0400
Subject: [PATCH 34/42] chore: remove job labels where it should be using
 config defaults

---
 rules/apps.libsonnet            |  6 +++---
 tests/rules-pod-owner-test.yaml | 12 ++++++------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index 5fb943356..d104be460 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -212,11 +212,11 @@
               max by (%(clusterLabel)s, namespace, workload, pod) (
                 label_replace(
                   label_replace(
-                    kube_pod_owner{%(kubeStateMetricsSelector)s, job!="", owner_kind="ReplicaSet"},
+                    kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"},
                     "replicaset", "$1", "owner_name", "(.*)"
                   ) * on (%(clusterLabel)s, replicaset, namespace) group_left(owner_name) topk by(%(clusterLabel)s, replicaset, namespace) (
                     1, max by (%(clusterLabel)s, replicaset, namespace, owner_name) (
-                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, job!="", owner_kind=""}
+                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind=""}
                     )
                   ),
                   "workload", "$1", "replicaset", "(.*)"
@@ -367,7 +367,7 @@
                     OR
                     label_replace(
                     group by (%(clusterLabel)s, namespace, pod, owner_name, owner_kind) (
-                        kube_pod_owner{ owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
+                        kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
                     )
                     , "workload", "$1", "owner_name", "(.+)"
                     )
diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index bbfc603e5..de7e4be0c 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -32,7 +32,7 @@ tests:
   input_series:
   - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:8443"}'
     values: '1'
-  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:8443"}'
+  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:1234"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -44,7 +44,7 @@ tests:
 # static pod
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
@@ -58,7 +58,7 @@ tests:
 # statefulset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
@@ -72,10 +72,10 @@ tests:
 # daemonset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="integrations/kubernetes/kube-state-metrics2",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
-    values: '1'
+    values: '1 1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
@@ -88,7 +88,7 @@ tests:
   input_series:
   - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
     values: '1 '
-  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics2",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
     values: '1'
   - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",replicaset="gke-duplicate-default-pool-10012"}'
     values: '1'

From 278cd4f23996e43a00c176c57cfbc2846b0e105d Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 15 May 2025 12:10:14 -0400
Subject: [PATCH 35/42] chore: add __name__ in test data

---
 tests/rules-pod-owner-test.yaml | 36 ++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index de7e4be0c..9450b4ea3 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -7,13 +7,13 @@ tests:
 # deployment
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
     values: '1 1'
-  - series: 'kube_pod_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
     values: '1'
-  - series: 'kube_replicaset_owner{endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
+  - series: 'kube_replicaset_owner{__name__="kube_replicaset_owner",endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
     values: '1 1'
-  - series: 'kube_replicaset_owner{endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
+  - series: 'kube_replicaset_owner{__name__="kube_replicaset_owner",endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
     values: '1'
   promql_expr_test:
   - eval_time: 0m
@@ -30,9 +30,9 @@ tests:
 # bare pod
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:8443"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:8443"}'
     values: '1'
-  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:1234"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:1234"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -44,9 +44,9 @@ tests:
 # static pod
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -58,9 +58,9 @@ tests:
 # statefulset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -72,9 +72,9 @@ tests:
 # daemonset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1 1'
   promql_expr_test:
   - eval_time: 1m
@@ -86,11 +86,11 @@ tests:
 # replicaset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
     values: '1 '
-  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
     values: '1'
-  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",replicaset="gke-duplicate-default-pool-10012"}'
+  - series: 'kube_replicaset_owner{__name__="kube_replicaset_owner",instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",replicaset="gke-duplicate-default-pool-10012"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -130,11 +130,11 @@ tests:
 # job
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
+  - series: 'kube_job_owner{__name__="kube_job_owner",instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
     values: '1'
-  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
+  - series: 'kube_job_owner{__name__="kube_job_owner",instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m

From bc237b6881607f8abcb1fd28af4e14aef19d54c5 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 15 May 2025 12:57:36 -0400
Subject: [PATCH 36/42] chore: fix test incorrect data

---
 tests/rules-pod-owner-test.yaml | 48 ++++++++++++++++-----------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 9450b4ea3..731089f60 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -7,13 +7,13 @@ tests:
 # deployment
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
     values: '1 1'
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
     values: '1'
-  - series: 'kube_replicaset_owner{__name__="kube_replicaset_owner",endpoint="https",instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
+  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
     values: '1 1'
-  - series: 'kube_replicaset_owner{__name__="kube_replicaset_owner",endpoint="https",instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
+  - series: 'kube_replicaset_owner{instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
     values: '1'
   promql_expr_test:
   - eval_time: 0m
@@ -30,9 +30,9 @@ tests:
 # bare pod
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:8443"}'
+  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:8443"}'
     values: '1'
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:1234"}'
+  - series: 'kube_pod_owner{owner_kind="", cluster="openshift", namespace="default", pod="nginx", job="kube-state-metrics", instance="10.128.0.135:1234"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -44,9 +44,9 @@ tests:
 # static pod
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="Node",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -58,9 +58,9 @@ tests:
 # statefulset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="StatefulSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -72,9 +72,9 @@ tests:
 # daemonset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="grafana-k8s-monitoring-opencost.monitoring.svc:9003",job="kube-state-metrics",cluster="duplicate",namespace="kube-system",owner_is_controller="true",owner_kind="DaemonSet",owner_name="gke-duplicate-default-pool-e20e250a",pod="gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1 1'
   promql_expr_test:
   - eval_time: 1m
@@ -86,11 +86,11 @@ tests:
 # replicaset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
     values: '1 '
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
     values: '1'
-  - series: 'kube_replicaset_owner{__name__="kube_replicaset_owner",instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",replicaset="gke-duplicate-default-pool-10012"}'
+  - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",replicaset="gke-duplicate-default-pool-10012"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -102,9 +102,9 @@ tests:
 # CronJob
 - interval: 1m
   input_series:
-  - series: '{__name__="kube_pod_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
+  - series: 'kube_pod_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
     values: '1'
-  - series: '{__name__="kube_job_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
+  - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -116,9 +116,9 @@ tests:
 # ScaledJob
 - interval: 1m
   input_series:
-  - series: '{__name__="kube_pod_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
+  - series: 'kube_pod_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
     values: '1'
-  - series: '{__name__="kube_job_owner", cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="ScaledJob", owner_name="cuda-vectoradd"}'
+  - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="ScaledJob", owner_name="cuda-vectoradd"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -130,11 +130,11 @@ tests:
 # job
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{__name__="kube_pod_owner",instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
+  - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_job_owner{__name__="kube_job_owner",instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
+  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
     values: '1'
-  - series: 'kube_job_owner{__name__="kube_job_owner",instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
+  - series: 'kube_job_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="kube-proxy-gke-duplicate-default-pool-a24e250a"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -146,9 +146,9 @@ tests:
 # non-standard replicaset
 - interval: 1m
   input_series:
-  - series: 'kube_pod_owner{__name__="kube_pod_owner", cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="ReplicaSet", owner_name="rollouts-demo-687d76d795", pod="rollouts-demo-687d76d795-5mlvh", uid="491a64cd-6a5f-4c64-b4b8-f95ee6a6b9c0"}'
+  - series: 'kube_pod_owner{cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="ReplicaSet", owner_name="rollouts-demo-687d76d795", pod="rollouts-demo-687d76d795-5mlvh", uid="491a64cd-6a5f-4c64-b4b8-f95ee6a6b9c0"}'
     values: '1'
-  - series: 'kube_replicaset_owner{__name__="kube_replicaset_owner", cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="Rollout", owner_name="rollouts-demo", replicaset="rollouts-demo-687d76d795"}'
+  - series: 'kube_replicaset_owner{cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="Rollout", owner_name="rollouts-demo", replicaset="rollouts-demo-687d76d795"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m

From f0ba0a8363c070a008596b05f01d7106a717ff90 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 15 May 2025 13:03:13 -0400
Subject: [PATCH 37/42] chore: add a cronjob to job test

---
 tests/rules-pod-owner-test.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 731089f60..d28c290cb 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -136,6 +136,8 @@ tests:
     values: '1'
   - series: 'kube_job_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="kube-proxy-gke-duplicate-default-pool-a24e250a"}'
     values: '1'
+  - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
+    values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel

From 2589e669bb62683a0c0bb8de19314a966e754284 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 15 May 2025 13:13:14 -0400
Subject: [PATCH 38/42] chore: fix job test

---
 tests/rules-pod-owner-test.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index d28c290cb..be90da8c4 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -136,11 +136,11 @@ tests:
     values: '1'
   - series: 'kube_job_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="kube-proxy-gke-duplicate-default-pool-a24e250a"}'
     values: '1'
-  - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
+  - series: 'kube_job_owner{instance="instance1",cluster="duplicate", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="default", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
-    expr: namespace_workload_pod:kube_pod_owner:relabel
+    expr: namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", workload_type="job"}
     exp_samples:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="kube-proxy-gke-duplicate-default-pool-e20e250a", workload_type="job"}'

From bc2cd2f9f62eed6c761d32f0ca478178947fb5aa Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Thu, 15 May 2025 14:37:30 -0400
Subject: [PATCH 39/42] chore: combine non-standard ones together

---
 tests/rules-pod-owner-test.yaml | 41 ++++++++++++++-------------------
 1 file changed, 17 insertions(+), 24 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index be90da8c4..7ac035fd2 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -9,23 +9,20 @@ tests:
   input_series:
   - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
     values: '1 1'
-  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns2",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="ds-7cc77d965f",pod="ds-7cc77d965f-cgsdv",service="ksm"}'
     values: '1'
   - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
     values: '1 1'
-  - series: 'kube_replicaset_owner{instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns1",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
+  - series: 'kube_replicaset_owner{instance="instance2",job="kube-state-metrics",cluster="kubernetes",namespace="ns2",owner_is_controller="true",owner_kind="Deployment",owner_name="ds",pod="ds-777f6bf798-kq7tj",replicaset="ds-7cc77d965f",service="ksm"}'
     values: '1'
   promql_expr_test:
-  - eval_time: 0m
-    expr: namespace_workload_pod:kube_pod_owner:relabel
-    exp_samples:
-    - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="ds-7cc77d965f-cgsdv", workload="ds", workload_type="deployment"}'
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes",namespace="ns1", pod="ds-7cc77d965f-cgsdv", workload="ds", workload_type="deployment"}'
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="kubernetes", namespace="ns2", pod="ds-7cc77d965f-cgsdv", workload="ds", workload_type="deployment"}'
 
 # bare pod
 - interval: 1m
@@ -99,33 +96,25 @@ tests:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="gke-duplicate-default-pool-10012-kx2v", workload="gke-duplicate-default-pool-10012", workload_type="replicaset"}'
 
-# CronJob
+# non-standard job
 - interval: 1m
   input_series:
   - series: 'kube_pod_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
     values: '1'
   - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
     values: '1'
-  promql_expr_test:
-  - eval_time: 1m
-    expr: namespace_workload_pod:kube_pod_owner:relabel
-    exp_samples:
-    - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120880-jfl5c", workload="cuda-vectoradd", workload_type="CronJob"}'
-
-# ScaledJob
-- interval: 1m
-  input_series:
-  - series: 'kube_pod_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120880", pod="cuda-vectoradd-29120880-jfl5c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
+  - series: 'kube_pod_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120881", pod="cuda-vectoradd-29120881-jfl4c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
     values: '1'
-  - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="ScaledJob", owner_name="cuda-vectoradd"}'
+  - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120881", namespace="monitoring", owner_is_controller="true", owner_kind="ScaledJob", owner_name="cuda-vectoradd"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120880-jfl5c", workload="cuda-vectoradd", workload_type="ScaledJob"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120880-jfl5c", workload="cuda-vectoradd", workload_type="CronJob"}'
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120881-jfl4c", workload="cuda-vectoradd", workload_type="ScaledJob"}'
 
 # job
 - interval: 1m
@@ -136,11 +125,9 @@ tests:
     values: '1'
   - series: 'kube_job_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="kube-proxy-gke-duplicate-default-pool-a24e250a"}'
     values: '1'
-  - series: 'kube_job_owner{instance="instance1",cluster="duplicate", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="default", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
-    values: '1'
   promql_expr_test:
   - eval_time: 1m
-    expr: namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate", workload_type="job"}
+    expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="kube-proxy-gke-duplicate-default-pool-e20e250a", workload_type="job"}'
@@ -152,9 +139,15 @@ tests:
     values: '1'
   - series: 'kube_replicaset_owner{cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="Rollout", owner_name="rollouts-demo", replicaset="rollouts-demo-687d76d795"}'
     values: '1'
+  - series: 'kube_pod_owner{cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="ReplicaSet", owner_name="configmap-demo-687d76d795", pod="configmap-demo-687d76d795-5mlvh", uid="491a64cd-6a5f-4c64-b4b8-f95ee6a6b9c0"}'
+    values: '1'
+  - series: 'kube_replicaset_owner{cluster="k8sdev/aws/helm", instance="grafana-k8s-monitoring-kube-state-metrics.default.svc:8080", job="kube-state-metrics", namespace="default", owner_is_controller="true", owner_kind="ConfigMap", owner_name="configmap-demo", replicaset="configmap-demo-687d76d795"}'
+    values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="k8sdev/aws/helm",namespace="default", pod="rollouts-demo-687d76d795-5mlvh", workload="rollouts-demo-687d76d795", workload_type="Rollout"}'
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="k8sdev/aws/helm",namespace="default", pod="configmap-demo-687d76d795-5mlvh", workload="configmap-demo-687d76d795", workload_type="ConfigMap"}'

From 1550cc924c8ca0a3cdd2376b222fd83692a67b2c Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Fri, 16 May 2025 08:41:19 -0400
Subject: [PATCH 40/42] fix: bug in non-standard workload replicasets rule

---
 rules/apps.libsonnet            | 26 +++++++++++++-------------
 tests/rules-pod-owner-test.yaml | 20 ++++++++++----------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
index d104be460..0af825df3 100644
--- a/rules/apps.libsonnet
+++ b/rules/apps.libsonnet
@@ -283,7 +283,7 @@
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',
             expr: |||
-              group by(%(clusterLabel)s, namespace, workload, workload_type, pod) (
+              group by(%(clusterLabel)s, namespace, workload, pod) (
                 label_replace(
                   label_join(
                     group by (%(clusterLabel)s, namespace, job_name, pod, owner_name) (
@@ -353,25 +353,25 @@
                 , "workload_type", "", "owner_kind")
                 OR
                 label_replace(
+                  label_replace(
                     label_replace(
-                    kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"}
-                    , "workload", "$1", "owner_name", "(.+)"
+                      kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind="ReplicaSet"}
+                      , "replicaset", "$1", "owner_name", "(.+)"
                     )
-                    * on(%(clusterLabel)s, namespace, workload) group_left(owner_kind, owner_name)
-                    label_replace(
+                    * on(%(clusterLabel)s, namespace, replicaset) group_left(owner_kind, owner_name)
                     group by (%(clusterLabel)s, namespace, replicaset, owner_kind, owner_name) (
-                        kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind!="Deployment", owner_kind!=""}
-                    )
-                    , "workload", "$1", "replicaset", "(.+)"
+                      kube_replicaset_owner{%(kubeStateMetricsSelector)s, owner_kind!="Deployment", owner_kind!=""}
                     )
-                    OR
-                    label_replace(
+                    , "workload", "$1", "owner_name", "(.+)"
+                  )
+                  OR
+                  label_replace(
                     group by (%(clusterLabel)s, namespace, pod, owner_name, owner_kind) (
-                        kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
+                      kube_pod_owner{%(kubeStateMetricsSelector)s, owner_kind!="ReplicaSet", owner_kind!="DaemonSet", owner_kind!="StatefulSet", owner_kind!="Job", owner_kind!="Node", owner_kind!=""}
                     )
                     , "workload", "$1", "owner_name", "(.+)"
-                    )
-                    , "workload_type", "$1", "owner_kind", "(.+)"
+                  )
+                  , "workload_type", "$1", "owner_kind", "(.+)"
                 )
               )
             ||| % $._config
diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index 7ac035fd2..dafee5ce7 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -103,27 +103,27 @@ tests:
     values: '1'
   - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
     values: '1'
-  - series: 'kube_pod_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120881", pod="cuda-vectoradd-29120881-jfl4c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
-    values: '1'
-  - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120881", namespace="monitoring", owner_is_controller="true", owner_kind="ScaledJob", owner_name="cuda-vectoradd"}'
-    values: '1'
+#   - series: 'kube_pod_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120881", pod="cuda-vectoradd-29120881-jfl4c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
+#     values: '1'
+#   - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120881", namespace="monitoring", owner_is_controller="true", owner_kind="ScaledJob", owner_name="cuda-vectoradd"}'
+#     values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120880-jfl5c", workload="cuda-vectoradd", workload_type="CronJob"}'
-    - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120881-jfl4c", workload="cuda-vectoradd", workload_type="ScaledJob"}'
+#     - value: 1
+#       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120881-jfl4c", workload="cuda-vectoradd", workload_type="ScaledJob"}'
 
 # job
 - interval: 1m
   input_series:
   - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
-  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
+  - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",job_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
     values: '1'
-  - series: 'kube_job_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",job_name="kube-proxy-gke-duplicate-default-pool-a24e250a"}'
+  - series: 'kube_job_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",job_name="kube-proxy-gke-duplicate-default-pool-a24e250a"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -148,6 +148,6 @@ tests:
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="k8sdev/aws/helm",namespace="default", pod="rollouts-demo-687d76d795-5mlvh", workload="rollouts-demo-687d76d795", workload_type="Rollout"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="k8sdev/aws/helm",namespace="default", pod="rollouts-demo-687d76d795-5mlvh", workload="rollouts-demo", workload_type="Rollout"}'
     - value: 1
-      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="k8sdev/aws/helm",namespace="default", pod="configmap-demo-687d76d795-5mlvh", workload="configmap-demo-687d76d795", workload_type="ConfigMap"}'
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="k8sdev/aws/helm",namespace="default", pod="configmap-demo-687d76d795-5mlvh", workload="configmap-demo", workload_type="ConfigMap"}'

From 09e9c438da26a963a82c5e6472157699145f20c8 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Fri, 16 May 2025 09:16:42 -0400
Subject: [PATCH 41/42] chore: add data to have consistent numbers in pod_owner
 and job/replicaset_owner

---
 tests/rules-pod-owner-test.yaml | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index dafee5ce7..a0a54f3ca 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -89,6 +89,8 @@ tests:
     values: '1'
   - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",replicaset="gke-duplicate-default-pool-10012"}'
     values: '1'
+  # - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",replicaset="gke-duplicate-default-pool-10012"}'
+  #   values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
@@ -103,27 +105,29 @@ tests:
     values: '1'
   - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120880", namespace="monitoring", owner_is_controller="true", owner_kind="CronJob", owner_name="cuda-vectoradd"}'
     values: '1'
-#   - series: 'kube_pod_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120881", pod="cuda-vectoradd-29120881-jfl4c", uid="8e3586c6-12f7-488e-b748-3f094193802c"}'
-#     values: '1'
-#   - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120881", namespace="monitoring", owner_is_controller="true", owner_kind="ScaledJob", owner_name="cuda-vectoradd"}'
-#     values: '1'
+  - series: 'kube_pod_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", namespace="monitoring", owner_is_controller="true", owner_kind="Job", owner_name="cuda-vectoradd-29120881", pod="cuda-vectoradd-29120881-dka0c", uid="8e3586c6-12f7-488e-b748-4f096192802m"}'
+    values: '1'
+  - series: 'kube_job_owner{cluster="duplicate", instance="grafana-k8s-monitoring-kube-state-metrics.monitoring.svc:8080", job="kube-state-metrics", job_name="cuda-vectoradd-29120881", namespace="monitoring", owner_is_controller="true", owner_kind="ScaledJob", owner_name="cuda-vectoradd"}'
+    values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120880-jfl5c", workload="cuda-vectoradd", workload_type="CronJob"}'
-#     - value: 1
-#       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120881-jfl4c", workload="cuda-vectoradd", workload_type="ScaledJob"}'
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="monitoring", pod="cuda-vectoradd-29120881-dka0c", workload="cuda-vectoradd", workload_type="ScaledJob"}'
 
 # job
 - interval: 1m
   input_series:
   - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="kube-proxy-gke-duplicate-default-pool-e20e250a",pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v"}'
     values: '1'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="Job",owner_name="kube-proxy-gke-duplicate-default-pool-f72d976h",pod="kube-proxy-gke-duplicate-default-pool-f72d976h-632a"}'
+    values: '1'
   - series: 'kube_job_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",job_name="kube-proxy-gke-duplicate-default-pool-e20e250a"}'
     values: '1'
-  - series: 'kube_job_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",job_name="kube-proxy-gke-duplicate-default-pool-a24e250a"}'
+  - series: 'kube_job_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",job_name="kube-proxy-gke-duplicate-default-pool-f72d976h"}'
     values: '1'
   promql_expr_test:
   - eval_time: 1m
@@ -131,6 +135,8 @@ tests:
     exp_samples:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-e20e250a-kx2v", workload="kube-proxy-gke-duplicate-default-pool-e20e250a", workload_type="job"}'
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="kube-proxy-gke-duplicate-default-pool-f72d976h-632a", workload="kube-proxy-gke-duplicate-default-pool-f72d976h", workload_type="job"}'
 
 # non-standard replicaset
 - interval: 1m

From 57ec7385062b8c2bf28ceba976a5adbf870a3f06 Mon Sep 17 00:00:00 2001
From: Serena K <serena.kei@grafana.com>
Date: Fri, 16 May 2025 09:25:15 -0400
Subject: [PATCH 42/42] chore: fix test data for replicaset

---
 tests/rules-pod-owner-test.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/rules-pod-owner-test.yaml b/tests/rules-pod-owner-test.yaml
index a0a54f3ca..28d0d3fdb 100644
--- a/tests/rules-pod-owner-test.yaml
+++ b/tests/rules-pod-owner-test.yaml
@@ -85,18 +85,20 @@ tests:
   input_series:
   - series: 'kube_pod_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
     values: '1 '
-  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10012",pod="gke-duplicate-default-pool-10012-kx2v"}'
+  - series: 'kube_pod_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="ReplicaSet",owner_name="gke-duplicate-default-pool-10013",pod="gke-duplicate-default-pool-10013-na4g"}'
     values: '1'
   - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",replicaset="gke-duplicate-default-pool-10012"}'
     values: '1'
-  # - series: 'kube_replicaset_owner{instance="instance1",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10012",replicaset="gke-duplicate-default-pool-10012"}'
-  #   values: '1'
+  - series: 'kube_replicaset_owner{instance="instance2",job="kube-state-metrics",cluster="duplicate",namespace="default",owner_is_controller="true",owner_kind="",owner_name="gke-duplicate-default-pool-10013",replicaset="gke-duplicate-default-pool-10013"}'
+    values: '1'
   promql_expr_test:
   - eval_time: 1m
     expr: namespace_workload_pod:kube_pod_owner:relabel
     exp_samples:
     - value: 1
       labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="gke-duplicate-default-pool-10012-kx2v", workload="gke-duplicate-default-pool-10012", workload_type="replicaset"}'
+    - value: 1
+      labels: 'namespace_workload_pod:kube_pod_owner:relabel{cluster="duplicate",namespace="default", pod="gke-duplicate-default-pool-10013-na4g", workload="gke-duplicate-default-pool-10013", workload_type="replicaset"}'
 
 # non-standard job
 - interval: 1m