From af2863de510f4ee883c8c1733016dee5937288e1 Mon Sep 17 00:00:00 2001 From: dongjiang Date: Fri, 31 Jul 2026 18:20:45 +0800 Subject: [PATCH 1/2] timex: filter unreasonable offset values from kernel adjtimex() overflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On KVM/pvclock guests, a race condition in the kernel NTP PLL can cause adjtimex() to return 2^32 ns (4.294967296s) as a transient overflow value, triggering false-positive NodeClockSkewDetected alerts hundreds of times per day despite NTP being correctly synchronized. Add a ±1.0s sanity bound in the timex collector: values exceeding this threshold are discarded (set to 0) with a warning log, as real NTP sync never produces offsets of this magnitude. Apply the same bound in the NodeClockSkewDetected alert expression to provide defense-in-depth at the rule layer. Fixes #3764 Signed-off-by: dongjiang --- collector/timex.go | 9 ++++++++- docs/node-mixin/alerts/alerts.libsonnet | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/collector/timex.go b/collector/timex.go index 6494726ef7..109b6be719 100644 --- a/collector/timex.go +++ b/collector/timex.go @@ -185,8 +185,15 @@ func (c *timexCollector) Update(ch chan<- prometheus.Metric) error { divisor = microSeconds } + offsetSec := float64(timex.Offset) / divisor + if offsetSec > 1.0 || offsetSec < -1.0 { + c.logger.Warn("Discarding unreasonable timex offset value", + "offset_seconds", offsetSec, "status", status) + offsetSec = 0 + } + ch <- c.syncStatus.mustNewConstMetric(syncStatus) - ch <- c.offset.mustNewConstMetric(float64(timex.Offset) / divisor) + ch <- c.offset.mustNewConstMetric(offsetSec) ch <- c.freq.mustNewConstMetric(1 + float64(timex.Freq)/ppm16frac) ch <- c.maxerror.mustNewConstMetric(float64(timex.Maxerror) / microSeconds) ch <- c.esterror.mustNewConstMetric(float64(timex.Esterror) / microSeconds) diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet index 29c934f57b..01d04c7f8d 100644 --- a/docs/node-mixin/alerts/alerts.libsonnet +++ b/docs/node-mixin/alerts/alerts.libsonnet @@ -215,12 +215,16 @@ expr: ||| ( node_timex_offset_seconds{%(nodeExporterSelector)s} > 0.05 + and + node_timex_offset_seconds{%(nodeExporterSelector)s} < 1.0 and deriv(node_timex_offset_seconds{%(nodeExporterSelector)s}[5m]) >= 0 ) or ( node_timex_offset_seconds{%(nodeExporterSelector)s} < -0.05 + and + node_timex_offset_seconds{%(nodeExporterSelector)s} > -1.0 and deriv(node_timex_offset_seconds{%(nodeExporterSelector)s}[5m]) <= 0 ) From 9341e7202dcc34c15286fed69db3cdb09d00a847 Mon Sep 17 00:00:00 2001 From: dongjiang Date: Fri, 21 Aug 2026 14:04:05 +0800 Subject: [PATCH 2/2] timex: add overflow diagnostic counter and relax alert threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback from @SuperQ: 1. Remove silent offset filtering at collection time - node_exporter should faithfully report kernel values - Do not mask or replace offset values 2. Add node_timex_offset_overflow_total counter - Monotonically counts readings where |offset| > 4s - Provides diagnostic signal for KVM/pvclock overflow artifacts - Does not modify the raw offset metric 3. Relax NodeClockSkewDetected alert threshold to ±60s - Previous ±1.0s was too tight and risked false negatives - 60s provides margin above the 4.29s overflow artifact - Still catches genuinely broken clocks (minutes/hours off) Fixes #3764 Signed-off-by: dongjiang --- collector/timex.go | 23 +++++++++++++++++------ docs/node-mixin/alerts/alerts.libsonnet | 4 ++-- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/collector/timex.go b/collector/timex.go index 109b6be719..c249911db8 100644 --- a/collector/timex.go +++ b/collector/timex.go @@ -39,6 +39,11 @@ const ( // See NOTES in adjtimex(2). ppm16frac = 1000000.0 * 65536.0 + + // offsetOverflowThresholdSec: |offset| exceeding this indicates a likely + // kernel adjtimex() overflow artifact (2^32 ns ≈ 4.29s on KVM/pvclock). + // See https://github.com/prometheus/node_exporter/issues/3764 + offsetOverflowThresholdSec = 4.0 ) type timexCollector struct { @@ -58,8 +63,10 @@ type timexCollector struct { errcnt, stbcnt, tai, - syncStatus typedDesc - logger *slog.Logger + syncStatus, + overflow typedDesc + logger *slog.Logger + overflowCount uint64 } func init() { @@ -156,6 +163,11 @@ func NewTimexCollector(logger *slog.Logger) (Collector, error) { "Is clock synchronized to a reliable server (1 = yes, 0 = no).", nil, nil, ), prometheus.GaugeValue}, + overflow: typedDesc{prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "offset_overflow_total"), + "Count of adjtimex() offset readings indicating kernel overflow (|offset| > 4s). See https://github.com/prometheus/node_exporter/issues/3764", + nil, nil, + ), prometheus.CounterValue}, logger: logger, }, nil } @@ -186,14 +198,13 @@ func (c *timexCollector) Update(ch chan<- prometheus.Metric) error { } offsetSec := float64(timex.Offset) / divisor - if offsetSec > 1.0 || offsetSec < -1.0 { - c.logger.Warn("Discarding unreasonable timex offset value", - "offset_seconds", offsetSec, "status", status) - offsetSec = 0 + if offsetSec > offsetOverflowThresholdSec || offsetSec < -offsetOverflowThresholdSec { + c.overflowCount++ } ch <- c.syncStatus.mustNewConstMetric(syncStatus) ch <- c.offset.mustNewConstMetric(offsetSec) + ch <- c.overflow.mustNewConstMetric(float64(c.overflowCount)) ch <- c.freq.mustNewConstMetric(1 + float64(timex.Freq)/ppm16frac) ch <- c.maxerror.mustNewConstMetric(float64(timex.Maxerror) / microSeconds) ch <- c.esterror.mustNewConstMetric(float64(timex.Esterror) / microSeconds) diff --git a/docs/node-mixin/alerts/alerts.libsonnet b/docs/node-mixin/alerts/alerts.libsonnet index 01d04c7f8d..2b654135a5 100644 --- a/docs/node-mixin/alerts/alerts.libsonnet +++ b/docs/node-mixin/alerts/alerts.libsonnet @@ -216,7 +216,7 @@ ( node_timex_offset_seconds{%(nodeExporterSelector)s} > 0.05 and - node_timex_offset_seconds{%(nodeExporterSelector)s} < 1.0 + node_timex_offset_seconds{%(nodeExporterSelector)s} < 60 and deriv(node_timex_offset_seconds{%(nodeExporterSelector)s}[5m]) >= 0 ) @@ -224,7 +224,7 @@ ( node_timex_offset_seconds{%(nodeExporterSelector)s} < -0.05 and - node_timex_offset_seconds{%(nodeExporterSelector)s} > -1.0 + node_timex_offset_seconds{%(nodeExporterSelector)s} > -60 and deriv(node_timex_offset_seconds{%(nodeExporterSelector)s}[5m]) <= 0 )