From a3ece2bb316a578c5d0fd0df4279077c87887f57 Mon Sep 17 00:00:00 2001 From: "emlautarom1-agent[bot]" <292495798+emlautarom1-agent[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:42:08 -0300 Subject: [PATCH 1/3] test(compose): gate the sched log topic in mixed clusters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Charon's infosync exchange with pluto peers completes cleanly over /charon/priority/2.0.0, so no warning is logged under topic=sched and the Warn Log Rate gate has nothing to suppress. Measured across three mixed_2_charon_2_pluto runs: app_log_warn_total{topic="sched"} is never created on the charon nodes, no sched-topic WARN or ERROR line appears in their logs, and charon never logs "P2P sending failing" — the warning the exclusion named. mixed_2_charon_2_pluto was the exclusion's only user, so AlertWarnExcludeTopics goes with it. The two remaining warn-topic exclusions are pinned by test instead, since both name charon components and neither occurs on pluto — pluto sets a log topic on almost nothing, so its warnings carry no topic label and stay gated. Closes #583 --- test-infra/compose/README.md | 1 - test-infra/compose/config.go | 7 ------- test-infra/compose/define.go | 13 +++++++------ test-infra/compose/rules_internal_test.go | 14 ++++++-------- test-infra/compose/smoke/smoke_test.go | 6 ------ 5 files changed, 13 insertions(+), 28 deletions(-) diff --git a/test-infra/compose/README.md b/test-infra/compose/README.md index 95eb6380..c70750a3 100644 --- a/test-infra/compose/README.md +++ b/test-infra/compose/README.md @@ -116,7 +116,6 @@ Scenarios that intentionally degrade the cluster tune the gate via config, not t | Config knob | Effect | Used by | |-------------|--------|---------| | `AlertExcludeJobs` | exempt a node from the per-node rules (never from `Pluto Down`) | `1_of_4_down`, `1_of_3_down` | -| `AlertWarnExcludeTopics` | extra warn-topic exclusions | `mixed_2_charon_2_pluto` (excludes `sched` until pluto serves infosync) | | `AlertDisableRules` | drop an entire rule | `1_of_3_down` (disables the error-rate gates — a downed round-1 leader makes every third proposer duty unrecoverable on the mock) | ## Versioning diff --git a/test-infra/compose/config.go b/test-infra/compose/config.go index d6f5915c..c41c782b 100644 --- a/test-infra/compose/config.go +++ b/test-infra/compose/config.go @@ -163,13 +163,6 @@ type Config struct { // clean. AlertExcludeJobs []string `json:"alert_exclude_jobs,omitempty"` - // AlertWarnExcludeTopics appends log topics to the Warn Log Rate - // exclusion list on top of the built-in charon mock noise (see - // writeAlertRules). Scenario-scoped escape hatch: e.g. mixed - // charon/pluto clusters exclude "sched" because charon's infosync warns - // each epoch until pluto serves /charon/priority/2.0.0 (#402B). - AlertWarnExcludeTopics []string `json:"alert_warn_exclude_topics,omitempty"` - // AlertDisableRules drops entire alert rules (by name, see // alertRuleNames) from the generated rules. Last-resort scenario knob // for cluster-wide degradation that per-job exclusion cannot express: diff --git a/test-infra/compose/define.go b/test-infra/compose/define.go index ad200a0e..60edbc93 100644 --- a/test-infra/compose/define.go +++ b/test-infra/compose/define.go @@ -432,9 +432,8 @@ var alertRuleNames = map[string]bool{ // writeAlertRules writes the prometheus alert rules evaluated by the smoke // tests. Rules are generated (not static) because the expressions depend on // config: scenarios that deliberately degrade a node exempt its job via -// conf.AlertExcludeJobs, mixed-impl scenarios extend the warn-topic -// exclusions via conf.AlertWarnExcludeTopics, and cluster-wide degradations -// drop whole rules via conf.AlertDisableRules. +// conf.AlertExcludeJobs, and cluster-wide degradations drop whole rules via +// conf.AlertDisableRules. // // Charon's "Outstanding Duty Rate" rule (core_bcast_broadcast_total - // core_scheduler_duty_total > 50) is deliberately not ported: a node cannot @@ -477,9 +476,11 @@ func writeAlertRules(dir string, conf Config) error { // every successful proposal epoch warns "Broadcasted block/attestation // never included on-chain" (the better the cluster works, the more it // warns). - // Both are mock artifacts, not node behavior; all other warn topics stay - // gated unless a scenario opts out via AlertWarnExcludeTopics. - warnTopics := strings.Join(append([]string{"vmock", "tracker"}, conf.AlertWarnExcludeTopics...), "|") + // Both are mock artifacts, not node behavior; every other warn topic stays + // gated. Both name charon components and only ever occur on charon nodes: + // pluto sets a log topic on almost nothing, so its warnings carry no topic + // label and this exclusion never applies to them. + const warnTopics = "vmock|tracker" // The broadcast-liveness expression must fail when a node exposes NO // core_bcast_broadcast_total series at all: the counter is created on diff --git a/test-infra/compose/rules_internal_test.go b/test-infra/compose/rules_internal_test.go index 08b4c263..c73b086a 100644 --- a/test-infra/compose/rules_internal_test.go +++ b/test-infra/compose/rules_internal_test.go @@ -67,14 +67,12 @@ func TestWriteAlertRulesExcludesDegradedJobs(t *testing.T) { require.Contains(t, content, "expr: up == 0") } -// TestWriteAlertRulesWarnTopicExtension asserts scenario-scoped warn-topic -// exclusions append to the built-in mock-noise list. -func TestWriteAlertRulesWarnTopicExtension(t *testing.T) { - conf := NewDefaultConfig() - conf.AlertWarnExcludeTopics = []string{"sched"} - - content := writeRules(t, conf) - require.Contains(t, content, `increase(app_log_warn_total{topic!~"vmock|tracker|sched"}[30s]) > 2`) +// TestWriteAlertRulesWarnTopics asserts the Warn Log Rate gate excludes exactly +// the two charon mock-noise topics, so every other topic — including the empty +// one pluto emits — stays gated. +func TestWriteAlertRulesWarnTopics(t *testing.T) { + content := writeRules(t, NewDefaultConfig()) + require.Contains(t, content, `increase(app_log_warn_total{topic!~"vmock|tracker"}[30s]) > 2`) } // TestWriteAlertRulesDropsOutstandingDuty pins the removal of charon's dead diff --git a/test-infra/compose/smoke/smoke_test.go b/test-infra/compose/smoke/smoke_test.go index 141b55b9..84ee48cb 100644 --- a/test-infra/compose/smoke/smoke_test.go +++ b/test-infra/compose/smoke/smoke_test.go @@ -236,12 +236,6 @@ func smokeScenarios() []smokeScenario { } // `pluto run` fails fast on --synthetic-block-proposals. conf.SyntheticBlockProposals = false - // Charon triggers infosync (/charon/priority/2.0.0) every - // epoch; pluto does not serve the protocol yet (#402B), so - // charon nodes warn "P2P sending failing" under topic=sched - // twice per epoch. Exempt that topic in mixed clusters until - // the protocol lands; drop this with #402B. - conf.AlertWarnExcludeTopics = []string{"sched"} }, }, { From addc64eb56cdb1fdbb028996253822821366a6fc Mon Sep 17 00:00:00 2001 From: "emlautarom1-agent[bot]" <292495798+emlautarom1-agent[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:42:08 -0300 Subject: [PATCH 2/3] chore(nix): add go, gopls and delve to the dev shell --- flake.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake.nix b/flake.nix index aeb2e905..1d303513 100644 --- a/flake.nix +++ b/flake.nix @@ -32,6 +32,9 @@ cargo-machete protobuf oas3-gen + go + gopls + delve ]; shellHook = '' From 5007a3cc8d2f821925f5a98549647747e6dec358 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 6 Aug 2026 10:29:46 -0300 Subject: [PATCH 3/3] refactor: simplify comments in alert rules and tests for clarity --- test-infra/compose/define.go | 4 +--- test-infra/compose/rules_internal_test.go | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/test-infra/compose/define.go b/test-infra/compose/define.go index 60edbc93..0836687d 100644 --- a/test-infra/compose/define.go +++ b/test-infra/compose/define.go @@ -477,9 +477,7 @@ func writeAlertRules(dir string, conf Config) error { // never included on-chain" (the better the cluster works, the more it // warns). // Both are mock artifacts, not node behavior; every other warn topic stays - // gated. Both name charon components and only ever occur on charon nodes: - // pluto sets a log topic on almost nothing, so its warnings carry no topic - // label and this exclusion never applies to them. + // gated. const warnTopics = "vmock|tracker" // The broadcast-liveness expression must fail when a node exposes NO diff --git a/test-infra/compose/rules_internal_test.go b/test-infra/compose/rules_internal_test.go index c73b086a..99ef19a8 100644 --- a/test-infra/compose/rules_internal_test.go +++ b/test-infra/compose/rules_internal_test.go @@ -68,8 +68,7 @@ func TestWriteAlertRulesExcludesDegradedJobs(t *testing.T) { } // TestWriteAlertRulesWarnTopics asserts the Warn Log Rate gate excludes exactly -// the two charon mock-noise topics, so every other topic — including the empty -// one pluto emits — stays gated. +// the two charon mock-noise topics. func TestWriteAlertRulesWarnTopics(t *testing.T) { content := writeRules(t, NewDefaultConfig()) require.Contains(t, content, `increase(app_log_warn_total{topic!~"vmock|tracker"}[30s]) > 2`)