From de9a95cd96307065267639bd88cacafcbc18ab3d Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Sun, 27 Sep 2026 12:46:51 -0400 Subject: [PATCH 1/2] test: preserve machine gate settings when pinning run models --- cmd/codeaf/do_engine_test.go | 47 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/cmd/codeaf/do_engine_test.go b/cmd/codeaf/do_engine_test.go index 90df20d0dc..f3384da34d 100644 --- a/cmd/codeaf/do_engine_test.go +++ b/cmd/codeaf/do_engine_test.go @@ -149,6 +149,23 @@ func beltRunEnv(t *testing.T) string { return home } +// beltModelPins preserves the fixture's resource settings while changing the +// seats a test intends to exercise. Replacing the profile wholesale would +// silently re-enable the real machine gate on a busy test host (#1525). +func beltModelPins(t *testing.T, profileDir string, pins map[string]string) { + t.Helper() + settings := config.NewSettings(config.SettingsOptions{ProfileDir: profileDir}) + for key, value := range pins { + row, ok := settings.Row(key) + if !ok { + t.Fatalf("missing setting %s", key) + } + if err := row.Apply(value); err != nil { + t.Fatal(err) + } + } +} + // beltPlandbDoor is the real plandb CLI behind the resolver's override, built // once for the package. THE LOOP ENDS IN THE STORE: a worker's task is done // when `plandb done` marks it so and no other way, so a scripted worker that @@ -376,16 +393,10 @@ func TestDoOnTheRunEngineSeatsEveryLaunchOnTheDoorsModels(t *testing.T) { if err := os.MkdirAll(profileDir, 0o700); err != nil { t.Fatal(err) } - rows, err := json.Marshal(map[string]string{ + beltModelPins(t, profileDir, map[string]string{ config.KeyTierWorkerModel: "vendor/profile-worker", config.KeyTierMastermindModel: "vendor/profile-thinking", }) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(config.BudgetConfigPath(profileDir), rows, 0o600); err != nil { - t.Fatal(err) - } workspace := beltRepoWorkspace(t) const ( @@ -433,7 +444,7 @@ func TestDoOnTheRunEngineSeatsEveryLaunchOnTheDoorsModels(t *testing.T) { } var stdout, stderr strings.Builder - err = doErrand(doRequest{ + err := doErrand(doRequest{ task: "write out.txt and say what you did", workspace: workspace, asJSON: true, timeout: 60 * time.Second, slots: bound(1), model: workModel, planModel: planModel, checkModel: planModel, stdout: &stdout, stderr: &stderr, newBeltCompleter: newBelt, @@ -668,18 +679,12 @@ func TestDoOnTheRunEngineSeatsACheckOnTheCheckModel(t *testing.T) { if err := os.MkdirAll(profileDir, 0o700); err != nil { t.Fatal(err) } - rows, err := json.Marshal(map[string]string{ + beltModelPins(t, profileDir, map[string]string{ config.KeyTierLowModel: "vendor/profile-small", config.KeyTierWorkerModel: "vendor/profile-worker", config.KeyTierHighModel: "vendor/profile-careful", config.KeyTierMastermindModel: "vendor/profile-thinking", }) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(config.BudgetConfigPath(profileDir), rows, 0o600); err != nil { - t.Fatal(err) - } workspace := beltRepoWorkspace(t) const ( @@ -722,7 +727,7 @@ func TestDoOnTheRunEngineSeatsACheckOnTheCheckModel(t *testing.T) { } var stdout, stderr strings.Builder - err = doErrand(doRequest{ + err := doErrand(doRequest{ task: "write out.txt and say what you did", workspace: workspace, asJSON: true, timeout: 60 * time.Second, slots: bound(1), model: workModel, planModel: planModel, checkModel: checkModel, @@ -767,17 +772,11 @@ func TestDoOnTheRunEngineSeatsAnUnpinnedCheckOnTheCrewsChecker(t *testing.T) { if err := os.MkdirAll(profileDir, 0o700); err != nil { t.Fatal(err) } - rows, err := json.Marshal(map[string]string{ + beltModelPins(t, profileDir, map[string]string{ config.KeyTierWorkerModel: "vendor/profile-worker", config.KeyTierHighModel: "vendor/profile-careful", config.KeyTierMastermindModel: "vendor/profile-thinking", }) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(config.BudgetConfigPath(profileDir), rows, 0o600); err != nil { - t.Fatal(err) - } workspace := beltRepoWorkspace(t) // The same one-seat shape as the two tests above. @@ -813,7 +812,7 @@ func TestDoOnTheRunEngineSeatsAnUnpinnedCheckOnTheCrewsChecker(t *testing.T) { } var stdout, stderr strings.Builder - err = doErrand(doRequest{ + err := doErrand(doRequest{ task: "write out.txt and say what you did", workspace: workspace, asJSON: true, timeout: 60 * time.Second, slots: bound(1), stdout: &stdout, stderr: &stderr, newBeltCompleter: newBelt, From 53877754b0835743d5ff4b4572766222bffb6967 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Sun, 27 Sep 2026 12:47:21 -0400 Subject: [PATCH 2/2] docs: record PR 1615 test fixture correction --- .../unreleased/1615-run-model-test-settings.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 docs/changes/unreleased/1615-run-model-test-settings.md diff --git a/docs/changes/unreleased/1615-run-model-test-settings.md b/docs/changes/unreleased/1615-run-model-test-settings.md new file mode 100644 index 0000000000..b30c249e31 --- /dev/null +++ b/docs/changes/unreleased/1615-run-model-test-settings.md @@ -0,0 +1,11 @@ +--- +kind: internal +title: Run-model tests preserve their fixture resource settings +pr: 1615 +surface: [build] +invalidates: + - "Three model-pinning tests overwrote the fixture profile and re-enabled the real machine gate. Model pins now preserve the fixture's disabled load and memory limits." +--- + +This removes one source of load-dependent failures tracked by #1525 without +changing product resource limits or tests that exercise those limits.