Skip to content

test: preserve machine gate settings when pinning run models #1615

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
santoshkumarradha wants to merge 2 commits into from
Closed

test: preserve machine gate settings when pinning run models #1615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
de9a95c
test: preserve machine gate settings when pinning run models
santoshkumarradha Sep 27, 2026
5387775
docs: record PR 1615 test fixture correction
santoshkumarradha Sep 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 23 additions & 24 deletions cmd/codeaf/do_engine_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,23 @@ func beltRunEnv(t *testing.T) string {
return home
}

// beltModelPins preserves the fixture's resource settings while changing the
// seats a test intends to exercise. Replacing the profile wholesale would
// silently re-enable the real machine gate on a busy test host (#1525).
func beltModelPins(t *testing.T, profileDir string, pins map[string]string) {
t.Helper()
settings := config.NewSettings(config.SettingsOptions{ProfileDir: profileDir})
for key, value := range pins {
row, ok := settings.Row(key)
if !ok {
t.Fatalf("missing setting %s", key)
}
if err := row.Apply(value); err != nil {
t.Fatal(err)
}
}
}

// beltPlandbDoor is the real plandb CLI behind the resolver's override, built
// once for the package. THE LOOP ENDS IN THE STORE: a worker's task is done
// when `plandb done` marks it so and no other way, so a scripted worker that
Expand Down Expand Up @@ -376,16 +393,10 @@ func TestDoOnTheRunEngineSeatsEveryLaunchOnTheDoorsModels(t *testing.T) {
if err := os.MkdirAll(profileDir, 0o700); err != nil {
t.Fatal(err)
}
rows, err := json.Marshal(map[string]string{
beltModelPins(t, profileDir, map[string]string{
config.KeyTierWorkerModel: "vendor/profile-worker",
config.KeyTierMastermindModel: "vendor/profile-thinking",
})
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(config.BudgetConfigPath(profileDir), rows, 0o600); err != nil {
t.Fatal(err)
}

workspace := beltRepoWorkspace(t)
const (
Expand Down Expand Up @@ -433,7 +444,7 @@ func TestDoOnTheRunEngineSeatsEveryLaunchOnTheDoorsModels(t *testing.T) {
}

var stdout, stderr strings.Builder
err = doErrand(doRequest{
err := doErrand(doRequest{
task: "write out.txt and say what you did", workspace: workspace, asJSON: true,
timeout: 60 * time.Second, slots: bound(1), model: workModel, planModel: planModel, checkModel: planModel,
stdout: &stdout, stderr: &stderr, newBeltCompleter: newBelt,
Expand Down Expand Up @@ -668,18 +679,12 @@ func TestDoOnTheRunEngineSeatsACheckOnTheCheckModel(t *testing.T) {
if err := os.MkdirAll(profileDir, 0o700); err != nil {
t.Fatal(err)
}
rows, err := json.Marshal(map[string]string{
beltModelPins(t, profileDir, map[string]string{
config.KeyTierLowModel: "vendor/profile-small",
config.KeyTierWorkerModel: "vendor/profile-worker",
config.KeyTierHighModel: "vendor/profile-careful",
config.KeyTierMastermindModel: "vendor/profile-thinking",
})
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(config.BudgetConfigPath(profileDir), rows, 0o600); err != nil {
t.Fatal(err)
}

workspace := beltRepoWorkspace(t)
const (
Expand Down Expand Up @@ -722,7 +727,7 @@ func TestDoOnTheRunEngineSeatsACheckOnTheCheckModel(t *testing.T) {
}

var stdout, stderr strings.Builder
err = doErrand(doRequest{
err := doErrand(doRequest{
task: "write out.txt and say what you did", workspace: workspace, asJSON: true,
timeout: 60 * time.Second, slots: bound(1),
model: workModel, planModel: planModel, checkModel: checkModel,
Expand Down Expand Up @@ -767,17 +772,11 @@ func TestDoOnTheRunEngineSeatsAnUnpinnedCheckOnTheCrewsChecker(t *testing.T) {
if err := os.MkdirAll(profileDir, 0o700); err != nil {
t.Fatal(err)
}
rows, err := json.Marshal(map[string]string{
beltModelPins(t, profileDir, map[string]string{
config.KeyTierWorkerModel: "vendor/profile-worker",
config.KeyTierHighModel: "vendor/profile-careful",
config.KeyTierMastermindModel: "vendor/profile-thinking",
})
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(config.BudgetConfigPath(profileDir), rows, 0o600); err != nil {
t.Fatal(err)
}

workspace := beltRepoWorkspace(t)
// The same one-seat shape as the two tests above.
Expand Down Expand Up @@ -813,7 +812,7 @@ func TestDoOnTheRunEngineSeatsAnUnpinnedCheckOnTheCrewsChecker(t *testing.T) {
}

var stdout, stderr strings.Builder
err = doErrand(doRequest{
err := doErrand(doRequest{
task: "write out.txt and say what you did", workspace: workspace, asJSON: true,
timeout: 60 * time.Second, slots: bound(1),
stdout: &stdout, stderr: &stderr, newBeltCompleter: newBelt,
Expand Down
11 changes: 11 additions & 0 deletions docs/changes/unreleased/1615-run-model-test-settings.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
---
kind: internal
title: Run-model tests preserve their fixture resource settings
pr: 1615
surface: [build]
invalidates:
- "Three model-pinning tests overwrote the fixture profile and re-enabled the real machine gate. Model pins now preserve the fixture's disabled load and memory limits."
---

This removes one source of load-dependent failures tracked by #1525 without
changing product resource limits or tests that exercise those limits.
Loading