From 0118ec785fa03bd49c6576b5d05f2efe3c2301ce Mon Sep 17 00:00:00 2001 From: max-braintrust Date: Tue, 16 Jun 2026 15:47:04 -0700 Subject: [PATCH 1/3] validate origin input --- js/src/framework.test.ts | 62 ++++++++++++++++++++++++++++++++++++++++ js/src/framework.ts | 7 ++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/js/src/framework.test.ts b/js/src/framework.test.ts index 12bcbb8c1..87135dd28 100644 --- a/js/src/framework.test.ts +++ b/js/src/framework.test.ts @@ -166,6 +166,68 @@ function makeTestScorer( } describe("runEvaluator", () => { + test("preserves a valid inline origin", async () => { + const origin: { + object_type: "dataset"; + object_id: string; + id: string; + _xact_id: string; + created: string; + } = { + object_type: "dataset", + object_id: "00000000-0000-0000-0000-000000000001", + id: "dataset-row-1", + _xact_id: "100", + created: "2026-06-01T00:00:00.000Z", + }; + + const out = await runEvaluator( + null, + { + projectName: "proj", + evalName: "eval", + data: [{ input: 1, origin }], + task: async (input: number) => input * 2, + scores: [], + }, + new NoopProgressReporter(), + [], + undefined, + ); + + expect(out.results[0].origin).toEqual(origin); + }); + + test("rejects an invalid inline origin", async () => { + const origin: { + object_type: "dataset"; + object_id: string; + id: string; + } = { + object_type: "dataset", + object_id: "not-a-uuid", + id: "dataset-row-1", + }; + const task = vi.fn(async (input: number) => input * 2); + + await expect( + runEvaluator( + null, + { + projectName: "proj", + evalName: "eval", + data: [{ input: 1, origin }], + task, + scores: [], + }, + new NoopProgressReporter(), + [], + undefined, + ), + ).rejects.toThrow(); + expect(task).not.toHaveBeenCalled(); + }); + describe("errors", () => { test("task errors generate no scores", async () => { const out = await runEvaluator( diff --git a/js/src/framework.ts b/js/src/framework.ts index 3183a575a..4f0ec0c89 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -10,6 +10,7 @@ import { } from "../util/index"; import { type GitMetadataSettingsType as GitMetadataSettings, + ObjectReference as ObjectReferenceSchema, type ObjectReferenceType as ObjectReference, type RepoInfoType as RepoInfo, type SSEProgressEventDataType as SSEProgressEventData, @@ -1179,8 +1180,12 @@ async function runEvaluatorInternal( : Dataset.isDataset(evaluator.data) ? evaluator.data : undefined; + const inlineOrigin = + datum.origin === undefined + ? undefined + : ObjectReferenceSchema.parse(datum.origin); const origin = - datum.origin ?? + inlineOrigin ?? (eventDataset && datum.id && datum._xact_id ? { object_type: "dataset", From 782a1b0b1db8481f4b58c2463b7df75bba9002bd Mon Sep 17 00:00:00 2001 From: max-braintrust Date: Tue, 16 Jun 2026 16:00:27 -0700 Subject: [PATCH 2/3] add changeset --- .changeset/olive-keys-cross.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/olive-keys-cross.md diff --git a/.changeset/olive-keys-cross.md b/.changeset/olive-keys-cross.md new file mode 100644 index 000000000..d02919a45 --- /dev/null +++ b/.changeset/olive-keys-cross.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +Adds inline origin validation From a54b893b47e7d8b4481c6c58902558440d5218e6 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Wed, 17 Jun 2026 17:12:06 +0200 Subject: [PATCH 3/3] cs --- .changeset/olive-keys-cross.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/olive-keys-cross.md b/.changeset/olive-keys-cross.md index d02919a45..c3d00036f 100644 --- a/.changeset/olive-keys-cross.md +++ b/.changeset/olive-keys-cross.md @@ -1,5 +1,5 @@ --- -"braintrust": minor +"braintrust": patch --- -Adds inline origin validation +fix: Validate inline row origin for evals