From 8ec12fae4d0b78f2cb4049e2ac65c06803947645 Mon Sep 17 00:00:00 2001 From: Arun S Date: Wed, 15 Jul 2026 13:11:06 +0530 Subject: [PATCH 1/5] collector: add tainted collector for /proc/sys/kernel/tainted Signed-off-by: Arun S --- collector/fixtures/proc/sys/kernel/tainted | 1 + collector/tainted_linux.go | 76 +++++++++++++++ collector/tainted_linux_test.go | 103 +++++++++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 collector/fixtures/proc/sys/kernel/tainted create mode 100644 collector/tainted_linux.go create mode 100644 collector/tainted_linux_test.go diff --git a/collector/fixtures/proc/sys/kernel/tainted b/collector/fixtures/proc/sys/kernel/tainted new file mode 100644 index 0000000000..63fce6863a --- /dev/null +++ b/collector/fixtures/proc/sys/kernel/tainted @@ -0,0 +1 @@ +12288 diff --git a/collector/tainted_linux.go b/collector/tainted_linux.go new file mode 100644 index 0000000000..85c386c214 --- /dev/null +++ b/collector/tainted_linux.go @@ -0,0 +1,76 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "fmt" + "log/slog" + "strconv" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs" +) + +type taintedCollector struct { + logger *slog.Logger + desc *prometheus.Desc +} + +func init() { + registerCollector("tainted", defaultDisabled, NewTaintedCollector) +} + +// NewTaintedCollector returns a Collector exposing kernel taint flags from +// /proc/sys/kernel/tainted as a labelled gauge. +// See https://www.kernel.org/doc/html/latest/admin-guide/tainted-kernels.html +func NewTaintedCollector(logger *slog.Logger) (Collector, error) { + return &taintedCollector{ + logger: logger, + desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "kernel", "tainted"), + "Taint flags set on the running Linux kernel, as reported by /proc/sys/kernel/tainted. "+ + "Value is 1 if the flag is set, 0 otherwise. "+ + "See https://www.kernel.org/doc/html/latest/admin-guide/tainted-kernels.html for flag meanings.", + []string{"bit", "flag"}, + nil, + ), + }, nil +} + +func (c *taintedCollector) Update(ch chan<- prometheus.Metric) error { + fs, err := procfs.NewFS(*procPath) + if err != nil { + return fmt.Errorf("failed to open procfs: %w", err) + } + + tainted, err := fs.KernelTainted() + if err != nil { + return fmt.Errorf("couldn't read kernel tainted state: %w", err) + } + + for _, b := range tainted.Bits { + var val float64 + if b.Set { + val = 1.0 + } + ch <- prometheus.MustNewConstMetric( + c.desc, + prometheus.GaugeValue, + val, + strconv.Itoa(b.Index), + b.Flag, + ) + } + return nil +} diff --git a/collector/tainted_linux_test.go b/collector/tainted_linux_test.go new file mode 100644 index 0000000000..8e06e09dee --- /dev/null +++ b/collector/tainted_linux_test.go @@ -0,0 +1,103 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !notainted + +package collector + +import ( + "io" + "log/slog" + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +func TestTaintedCollector(t *testing.T) { + *procPath = "fixtures/proc" + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + c, err := NewTaintedCollector(logger) + if err != nil { + t.Fatalf("failed to create tainted collector: %v", err) + } + + reg := prometheus.NewPedanticRegistry() + reg.MustRegister(&taintedCollectorWrapper{c.(*taintedCollector)}) + + mfs, err := reg.Gather() + if err != nil { + t.Fatalf("gather failed: %v", err) + } + if len(mfs) != 1 { + t.Fatalf("expected 1 metric family, got %d", len(mfs)) + } + + mf := mfs[0] + if got := mf.GetName(); got != "node_kernel_tainted" { + t.Errorf("metric name: want node_kernel_tainted, got %s", got) + } + + // Expect one series per known taint bit (20 defined by the kernel). + const wantBits = 20 + if got := len(mf.GetMetric()); got != wantBits { + t.Errorf("metric count: want %d, got %d", wantBits, got) + } + + // Build bit → value map for assertion. + // Fixture is 12288 = bit 12 (O) + bit 13 (E). + // Build flag → value map for assertion (labels: bit, flag). + flagVals := make(map[string]float64) + for _, m := range mf.GetMetric() { + // Each metric has exactly 2 labels: bit and flag. + for _, lp := range m.GetLabel() { + if lp.GetName() == "flag" { + flagVals[lp.GetValue()] = m.GetGauge().GetValue() + } + } + } + + // Fixture is 12288 = bit 12 (O) + bit 13 (E). + for _, tc := range []struct { + flag string + want float64 + }{ + {"O", 1}, // Externally-built (out-of-tree) module — set + {"E", 1}, // Unsigned module — set + {"L", 0}, // Soft lockup — must be clear + {"P", 0}, + {"T", 0}, + } { + got, ok := flagVals[tc.flag] + if !ok { + t.Errorf("flag %q not found in metrics", tc.flag) + continue + } + if got != tc.want { + t.Errorf("flag %q: want %.0f, got %.0f", tc.flag, tc.want, got) + } + } +} + +// taintedCollectorWrapper adapts taintedCollector to prometheus.Collector. +type taintedCollectorWrapper struct { + c *taintedCollector +} + +func (w *taintedCollectorWrapper) Describe(ch chan<- *prometheus.Desc) { + ch <- w.c.desc +} + +func (w *taintedCollectorWrapper) Collect(ch chan<- prometheus.Metric) { + _ = w.c.Update(ch) +} From 72962a5a34ef601c1c2f4a394bfd7468175ff425 Mon Sep 17 00:00:00 2001 From: Arun S Date: Fri, 17 Jul 2026 06:29:15 +0530 Subject: [PATCH 2/5] go.mod: bump procfs to pick up KernelTainted() (#844) Signed-off-by: Arun S --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7c4458e399..405ccb01f8 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.70.1 github.com/prometheus/exporter-toolkit v0.17.1 - github.com/prometheus/procfs v0.21.1 + github.com/prometheus/procfs v0.21.2-0.20260716175001-1de9cf374cc1 github.com/safchain/ethtool v0.7.0 golang.org/x/sys v0.47.0 howett.net/plist v1.0.1 diff --git a/go.sum b/go.sum index 58c02fe429..9a3b61424c 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= github.com/prometheus/exporter-toolkit v0.17.1 h1:psKN4wM7shBL/BxZkDHgm6YZJ3fAVG36+r86An/+7q0= github.com/prometheus/exporter-toolkit v0.17.1/go.mod h1:dabwPJvxsC5+tsp2iolQrqBWZh+QlISKlYRpj9Hh5xk= -github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= -github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/prometheus/procfs v0.21.2-0.20260716175001-1de9cf374cc1 h1:CZ7I0vh6VJG2gZsl/V98AKob5uR0SwteYxLFpIHqVvw= +github.com/prometheus/procfs v0.21.2-0.20260716175001-1de9cf374cc1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/safchain/ethtool v0.7.0 h1:rlJzfDetsVvT61uz8x1YIcFn12akMfuPulHtZjtb7Is= github.com/safchain/ethtool v0.7.0/go.mod h1:MenQKEjXdfkjD3mp2QdCk8B/hwvkrlOTm/FD4gTpFxQ= github.com/siebenmann/go-kstat v0.0.0-20210513183136-173c9b0a9973 h1:GfSdC6wKfTGcgCS7BtzF5694Amne1pGCSTY252WhlEY= From 19cd78d77e0bda2ef9b34b42036c2f78e2239b1b Mon Sep 17 00:00:00 2001 From: Arun S Date: Fri, 17 Jul 2026 12:38:47 +0530 Subject: [PATCH 3/5] go.mod: bump procfs to pick up linux build tag fix (#847) Signed-off-by: Arun S --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 405ccb01f8..67147b838c 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.70.1 github.com/prometheus/exporter-toolkit v0.17.1 - github.com/prometheus/procfs v0.21.2-0.20260716175001-1de9cf374cc1 + github.com/prometheus/procfs v0.21.2-0.20260717070424-0cd18237af6e github.com/safchain/ethtool v0.7.0 golang.org/x/sys v0.47.0 howett.net/plist v1.0.1 diff --git a/go.sum b/go.sum index 9a3b61424c..295885f0ce 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= github.com/prometheus/exporter-toolkit v0.17.1 h1:psKN4wM7shBL/BxZkDHgm6YZJ3fAVG36+r86An/+7q0= github.com/prometheus/exporter-toolkit v0.17.1/go.mod h1:dabwPJvxsC5+tsp2iolQrqBWZh+QlISKlYRpj9Hh5xk= -github.com/prometheus/procfs v0.21.2-0.20260716175001-1de9cf374cc1 h1:CZ7I0vh6VJG2gZsl/V98AKob5uR0SwteYxLFpIHqVvw= -github.com/prometheus/procfs v0.21.2-0.20260716175001-1de9cf374cc1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/prometheus/procfs v0.21.2-0.20260717070424-0cd18237af6e h1:sm1MuWauLFbQsFVJn/5vKTMrolG02N/DT10IPcCYJA4= +github.com/prometheus/procfs v0.21.2-0.20260717070424-0cd18237af6e/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/safchain/ethtool v0.7.0 h1:rlJzfDetsVvT61uz8x1YIcFn12akMfuPulHtZjtb7Is= github.com/safchain/ethtool v0.7.0/go.mod h1:MenQKEjXdfkjD3mp2QdCk8B/hwvkrlOTm/FD4gTpFxQ= github.com/siebenmann/go-kstat v0.0.0-20210513183136-173c9b0a9973 h1:GfSdC6wKfTGcgCS7BtzF5694Amne1pGCSTY252WhlEY= From 7747d62e35681d76330402ed8d9f11cf2f1cd304 Mon Sep 17 00:00:00 2001 From: Arun S Date: Sat, 8 Aug 2026 11:24:36 +0530 Subject: [PATCH 4/5] collector/tainted: use year-less copyright header Co-authored-by: Cursor Signed-off-by: Arun S --- collector/tainted_linux.go | 2 +- collector/tainted_linux_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/collector/tainted_linux.go b/collector/tainted_linux.go index 85c386c214..0ad98ee4ca 100644 --- a/collector/tainted_linux.go +++ b/collector/tainted_linux.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/collector/tainted_linux_test.go b/collector/tainted_linux_test.go index 8e06e09dee..0ce7e3b621 100644 --- a/collector/tainted_linux_test.go +++ b/collector/tainted_linux_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Prometheus Authors +// Copyright The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at From f6a97bb8758d8621c155e15b1fab5d653b8252b9 Mon Sep 17 00:00:00 2001 From: Arun S Date: Sat, 22 Aug 2026 19:00:40 +0530 Subject: [PATCH 5/5] collector/tainted: address review feedback and document collector Store procfs.FS on the collector instead of opening it each scrape. Add matching !notainted build tag on the implementation file. Document the tainted collector and node_kernel_tainted metrics in README. Signed-off-by: Arun S --- README.md | 30 ++++++++++++++++++++++++++++++ collector/tainted_linux.go | 15 +++++++++------ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8abf2cdf0f..3eab7a9799 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,7 @@ softirqs | Exposes detailed softirq statistics from `/proc/softirqs`. | Linux sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(-info)` to configure. | Linux swap | Expose swap information from `/proc/swaps`. | Linux systemd | Exposes service and system status from [systemd](http://www.freedesktop.org/wiki/Software/systemd/). | Linux +tainted | Exposes kernel taint flags from `/proc/sys/kernel/tainted`. | Linux tcpstat | Exposes TCP connection status information from `/proc/net/tcp` and `/proc/net/tcp6`. (Warning: the current version has potential performance issues in high load situations.) | Linux wifi | Exposes WiFi device and station statistics. | Linux xfrm | Exposes statistics from `/proc/net/xfrm_stat` | Linux @@ -321,6 +322,35 @@ node_sysctl_info{key="kernel.seccomp.actions_avail", index="1", value="kill_thre ... ``` +### Tainted Collector + +The `tainted` collector can be enabled with `--collector.tainted`. It reads the +integer bitmask from `/proc/sys/kernel/tainted` and exposes each known [kernel +taint flag](https://www.kernel.org/doc/html/latest/admin-guide/tainted-kernels.html) +as a separate gauge. The raw bitmask is not exported; each series indicates +whether a single flag is active. + +Metric: `node_kernel_tainted{bit="N", flag="X"}` — value `1` if the flag is +set, `0` otherwise. Labels: + +* `bit` — zero-based bit position in the taint bitmask (0–19). +* `flag` — kernel letter code for the taint (e.g. `O` for an out-of-tree module, + `E` for an unsigned module). + +All 20 defined taint flags are reported on every scrape, even when clear. + +#### Example + +If `/proc/sys/kernel/tainted` is `12288` (out-of-tree and unsigned modules +loaded), the collector exposes: + +``` +node_kernel_tainted{bit="12",flag="O"} 1 +node_kernel_tainted{bit="13",flag="E"} 1 +node_kernel_tainted{bit="0",flag="P"} 0 +... +``` + ### Textfile Collector The `textfile` collector is similar to the [Pushgateway](https://github.com/prometheus/pushgateway), diff --git a/collector/tainted_linux.go b/collector/tainted_linux.go index 0ad98ee4ca..3ab3f25c6e 100644 --- a/collector/tainted_linux.go +++ b/collector/tainted_linux.go @@ -11,6 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !notainted + package collector import ( @@ -23,6 +25,7 @@ import ( ) type taintedCollector struct { + fs procfs.FS logger *slog.Logger desc *prometheus.Desc } @@ -35,7 +38,12 @@ func init() { // /proc/sys/kernel/tainted as a labelled gauge. // See https://www.kernel.org/doc/html/latest/admin-guide/tainted-kernels.html func NewTaintedCollector(logger *slog.Logger) (Collector, error) { + fs, err := procfs.NewFS(*procPath) + if err != nil { + return nil, fmt.Errorf("failed to open procfs: %w", err) + } return &taintedCollector{ + fs: fs, logger: logger, desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "kernel", "tainted"), @@ -49,12 +57,7 @@ func NewTaintedCollector(logger *slog.Logger) (Collector, error) { } func (c *taintedCollector) Update(ch chan<- prometheus.Metric) error { - fs, err := procfs.NewFS(*procPath) - if err != nil { - return fmt.Errorf("failed to open procfs: %w", err) - } - - tainted, err := fs.KernelTainted() + tainted, err := c.fs.KernelTainted() if err != nil { return fmt.Errorf("couldn't read kernel tainted state: %w", err) }