From 88db2400a5624a0423c7f2eb9d7ba7fa01665e2b Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 5 Aug 2026 10:20:59 +0000 Subject: [PATCH 1/2] test(e2e): tolerate intermittent EAI-7423 XPASS on the Strix Halo Ubuntu lane (EAI-7853) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lemonade managed-serve bug EAI-7423 no longer reproduces reliably on the Strix Halo Ubuntu runner: which of its six xfail'd scenarios pass varies run to run (2, 3, and 6 XPASS across three recent runs). Because those rows are deterministic xfails, the reconciler reports stale XPASS and fails the lane on PRs that touch neither serve nor chat. Mark the EAI-7423 lemonade-linux rows `flaky`, which is what that flag is for: either outcome is tolerated while the bug stays open, matching how the EAI-7333 rows already handle an intermittent known bug. This deliberately does not claim EAI-7423 is fixed — the inference scenarios still fail on some runs, so the bug stays open and the rows stay in place. Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/expectations.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index dde70512..1cece3dd 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -45,6 +45,14 @@ reason = "vLLM service reports ready (/v1/models 200) but POST /v1/chat/completi serve_timeout_secs = 90 flaky = true +# EAI-7423 is INTERMITTENT on this host as of 2026-08-05, not fixed: which of its +# scenarios pass varies run to run (run 30995473578: 2 XPASS; run 30993213188: +# 3 XPASS; run 30939967076: all 6). A deterministic-XPASS row fails the lane on +# unrelated PRs, so every EAI-7423 lemonade-linux row below is `flaky = true` — +# either outcome is tolerated while the bug stays open. See EAI-7853; if the +# lemonade 11.5.1 upgrade turns out to fix EAI-7423, drop these rows outright +# rather than leaving them flaky. + # On a lemonade-native LINUX host (Strix Halo Ubuntu) the lemonade managed serve # loads the model and reaches ready on :8001, then is shut down ~0.08s later, so # the endpoint never becomes reachable (EAI-7423, root-caused on the box @@ -56,6 +64,7 @@ when = { effective_engine = "lemonade", os = "linux" } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the endpoint never becomes reachable." serve_timeout_secs = 90 +flaky = true # On a lemonade-native LINUX host the served model's endpoint drops before # inference: the lemonade managed serve reaches ready then is shut down ~0.08s @@ -66,6 +75,7 @@ when = { effective_engine = "lemonade", os = "linux" } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." serve_timeout_secs = 90 +flaky = true # EAI-7333 (CLI healthcheck reports ready off /v1/models before inference works) # on the vLLM path. This scenario is FLAKY, not fixed: it XPASS'd on MI300X once @@ -89,6 +99,7 @@ when = { effective_engine = "lemonade", os = "linux" } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the ready-to-serve contract fails." serve_timeout_secs = 90 +flaky = true # --- EAI-7423: on a lemonade LINUX host the managed serve reaches ready then is # shut down immediately, so inference never happens (root-caused on the box @@ -100,6 +111,7 @@ when = { effective_engine = "lemonade", os = "linux" } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." serve_timeout_secs = 90 +flaky = true # --- EAI-7223: chat requests with tool definitions rejected (vLLM path) --- [["chat-tool-definitions-accepted"]] @@ -119,6 +131,7 @@ when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." serve_timeout_secs = 90 +flaky = true # EAI-7221 (TUI chat sends hardcoded 'local-model' instead of querying /v1/models) # was previously mapped here as a vLLM-path xfail, but that bug is TUI-only and this @@ -135,6 +148,7 @@ when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*" } bug = "EAI-7423" reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." serve_timeout_secs = 90 +flaky = true # --- EAI-7383: `rocm help` lists subcommands in declaration order, not # alphabetically. Platform-independent (pure CLI help output). --- From b9dca0a940a5fb902a97fc9a92075297eb10309b Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 5 Aug 2026 14:22:03 +0000 Subject: [PATCH 2/2] test(e2e): correct the flaky-xfail notes in expectations.toml (EAI-7853) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on the EAI-7423 flaky rows: - Document `flaky` in the file header grammar and qualify the "an XPASS is a stale entry, remove it" rule, which was false for 8 of 13 rows. - Drop the "if the lemonade 11.5.1 upgrade fixes EAI-7423, delete these rows" note. The install gate is a file-existence check with no version comparison and the install root persists across jobs, so which lemonade build a run used is not readable from this lane — the attribution was untestable and acting on it would have deleted six live rows. - Name the host (Strix Halo Ubuntu) instead of "this host", state that `flaky` is per-row and not derived from `bug`, and attach the note to the rows it describes. Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/expectations.toml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 1cece3dd..0316bf62 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -10,7 +10,9 @@ # 1. A scenario whose @requires-gpu / @requires-engine can't be satisfied on # this host is SKIPPED (not-applicable) — never listed here. # 2. Otherwise, the FIRST matching condition below marks it expected-fail -# (xfail); a scenario that then PASSES is an XPASS (stale entry — remove it). +# (xfail); a scenario that then PASSES is an XPASS — a stale entry to be +# removed, UNLESS the row is `flaky = true`, which tolerates either result +# while the bug stays open (see the grammar below). # 3. No matching condition → expected-pass. # # Condition predicate grammar (deliberately tiny): @@ -20,6 +22,8 @@ # bug = "EAI-NNNN" ticket reference (shown in the report) # reason = "..." human explanation (shown in the report) # serve_timeout_secs = 90 optional: shorter serve wait for a known bug +# flaky = true optional: bug reproduces only intermittently, +# so an XPASS is tolerated (reported, not stale) # # An empty `when = {}` means "always xfail" (platform-independent bug). @@ -45,14 +49,16 @@ reason = "vLLM service reports ready (/v1/models 200) but POST /v1/chat/completi serve_timeout_secs = 90 flaky = true -# EAI-7423 is INTERMITTENT on this host as of 2026-08-05, not fixed: which of its -# scenarios pass varies run to run (run 30995473578: 2 XPASS; run 30993213188: -# 3 XPASS; run 30939967076: all 6). A deterministic-XPASS row fails the lane on -# unrelated PRs, so every EAI-7423 lemonade-linux row below is `flaky = true` — -# either outcome is tolerated while the bug stays open. See EAI-7853; if the -# lemonade 11.5.1 upgrade turns out to fix EAI-7423, drop these rows outright -# rather than leaving them flaky. - +# EAI-7423 was INTERMITTENT on Strix Halo Ubuntu as of 2026-08-05, not fixed: +# which of its scenarios pass varied run to run (run 30995473578: 2 XPASS; run +# 30993213188: 3 XPASS; run 30939967076: all 6). A deterministic-XPASS row fails +# that lane on unrelated PRs, so the EAI-7423 lemonade-linux rows below carry +# `flaky = true` — either outcome is tolerated while the bug stays open. The flag +# is per-row and is NOT derived from `bug`, so a new EAI-7423 row needs it added +# explicitly. Revisit under EAI-7853 before deleting any of them: which lemonade +# build a given run used cannot currently be read back from this lane, so an +# XPASS streak here is not by itself evidence that the bug is fixed. +# # On a lemonade-native LINUX host (Strix Halo Ubuntu) the lemonade managed serve # loads the model and reaches ready on :8001, then is shut down ~0.08s later, so # the endpoint never becomes reachable (EAI-7423, root-caused on the box