diff --git a/.changeset/goofy-hotels-care.md b/.changeset/goofy-hotels-care.md new file mode 100644 index 000000000..41bd0fd96 --- /dev/null +++ b/.changeset/goofy-hotels-care.md @@ -0,0 +1,6 @@ +--- +"braintrust": minor +--- + +- (feat) Add dataset snapshot/environment selection support to `init()` and `initDataset()`, including snapshot CRUD helpers and `DatasetSnapshot` type exports. +- (feat) Update `braintrust/dev` to respect `dataset_version` and `dataset_environment` when resolving datasets for evals. diff --git a/js/dev/server.ts b/js/dev/server.ts index 5499fe208..ebce5c3f5 100644 --- a/js/dev/server.ts +++ b/js/dev/server.ts @@ -318,6 +318,8 @@ async function getDataset( state, project: data.project_name, dataset: data.dataset_name, + version: data.dataset_version ?? undefined, + environment: data.dataset_environment ?? undefined, _internal_btql: data._internal_btql ?? undefined, }); } else if ("dataset_id" in data) { @@ -329,6 +331,8 @@ async function getDataset( state, projectId: datasetInfo.projectId, dataset: datasetInfo.dataset, + version: data.dataset_version ?? undefined, + environment: data.dataset_environment ?? undefined, _internal_btql: data._internal_btql ?? undefined, }); } else { diff --git a/js/src/exports.ts b/js/src/exports.ts index 158479847..4a4a30a8d 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -8,6 +8,9 @@ export type { CompiledPromptParams, CompletionPrompt, ContextParentSpanIds, + DatasetRestorePreviewResult, + DatasetRestoreResult, + DatasetSnapshot, DataSummary, DatasetSummary, DefaultMetadataType, diff --git a/js/src/logger.test.ts b/js/src/logger.test.ts index fc4f14f7b..b971db70a 100644 --- a/js/src/logger.test.ts +++ b/js/src/logger.test.ts @@ -4,6 +4,7 @@ import { vi, expect, test, describe, beforeEach, afterEach } from "vitest"; import { _exportsForTestingOnly, init, + initDataset, initLogger, Prompt, BraintrustState, @@ -453,6 +454,571 @@ test("init accepts dataset with id and version", () => { expect(datasetWithVersion.version).toBe("v2"); }); +test("init accepts dataset with id and environment", () => { + const datasetWithEnvironment = { + id: "dataset-id-123", + environment: "production", + }; + + expect(datasetWithEnvironment.id).toBe("dataset-id-123"); + expect(datasetWithEnvironment.environment).toBe("production"); +}); + +test("init accepts dataset with id and snapshotName", () => { + const datasetWithSnapshot = { + id: "dataset-id-123", + snapshotName: "123", + }; + + expect(datasetWithSnapshot.id).toBe("dataset-id-123"); + expect(datasetWithSnapshot.snapshotName).toBe("123"); +}); + +function mockInitGitMetadata() { + vi.spyOn(_exportsForTestingOnly.isomorph, "getRepoInfo").mockResolvedValue( + undefined, + ); + vi.spyOn( + _exportsForTestingOnly.isomorph, + "getPastNAncestors", + ).mockResolvedValue([]); +} + +test("initDataset prefers version over environment in eval data", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + vi.spyOn(state.appConn(), "post_json").mockResolvedValue({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + dataset: { + id: "00000000-0000-0000-0000-000000000002", + name: "test-dataset", + }, + }); + + const dataset = initDataset({ + project: "test-project", + dataset: "test-dataset", + version: "123", + environment: "production", + state, + }); + + await expect(dataset.toEvalData()).resolves.toEqual({ + dataset_id: "00000000-0000-0000-0000-000000000002", + dataset_version: "123", + }); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + +test("dataset.toEvalData preserves dataset_environment", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + vi.spyOn(state.apiConn(), "get_json").mockResolvedValue({ + object_version: "123", + }); + vi.spyOn(state.appConn(), "post_json").mockResolvedValue({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + dataset: { + id: "00000000-0000-0000-0000-000000000002", + name: "test-dataset", + }, + }); + + const dataset = initDataset({ + project: "test-project", + dataset: "test-dataset", + environment: "production", + state, + }); + + await expect(dataset.toEvalData()).resolves.toEqual({ + dataset_id: "00000000-0000-0000-0000-000000000002", + dataset_environment: "production", + }); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + +test("dataset.toEvalData preserves dataset_snapshot_name", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + const postJson = vi + .spyOn(state.appConn(), "post_json") + .mockResolvedValueOnce({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + dataset: { + id: "00000000-0000-0000-0000-000000000002", + name: "test-dataset", + }, + }) + .mockResolvedValueOnce([ + { + id: "00000000-0000-0000-0000-000000000004", + dataset_id: "00000000-0000-0000-0000-000000000002", + name: "123", + description: null, + xact_id: "456", + created: "2026-03-31T00:00:00.000Z", + }, + ]); + + const dataset = initDataset({ + project: "test-project", + dataset: "test-dataset", + snapshotName: "123", + state, + }); + + await expect(dataset.toEvalData()).resolves.toEqual({ + dataset_id: "00000000-0000-0000-0000-000000000002", + dataset_snapshot_name: "123", + }); + expect(postJson).toHaveBeenNthCalledWith(2, "api/dataset_snapshot/get", { + dataset_id: "00000000-0000-0000-0000-000000000002", + }); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + +test("dataset.createSnapshot forwards update when requested", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + const postJson = vi + .spyOn(state.appConn(), "post_json") + .mockResolvedValueOnce({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + dataset: { + id: "00000000-0000-0000-0000-000000000002", + name: "test-dataset", + }, + }) + .mockResolvedValueOnce({ + dataset_snapshot: { + id: "00000000-0000-0000-0000-000000000004", + dataset_id: "00000000-0000-0000-0000-000000000002", + name: "snapshot", + description: "updated description", + xact_id: "123", + created: "2026-03-31T00:00:00.000Z", + }, + found_existing: true, + }); + + const dataset = initDataset({ + project: "test-project", + dataset: "test-dataset", + version: "123", + state, + }); + + await expect( + dataset.createSnapshot({ + name: "snapshot", + description: "updated description", + update: true, + }), + ).resolves.toMatchObject({ + id: "00000000-0000-0000-0000-000000000004", + xact_id: "123", + }); + + expect(postJson).toHaveBeenNthCalledWith(2, "api/dataset_snapshot/register", { + dataset_id: "00000000-0000-0000-0000-000000000002", + dataset_snapshot_name: "snapshot", + description: "updated description", + xact_id: "123", + update: true, + }); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + +test("dataset.updateSnapshot patches snapshot metadata by id", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + const postJson = vi + .spyOn(state.appConn(), "post_json") + .mockResolvedValueOnce({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + dataset: { + id: "00000000-0000-0000-0000-000000000002", + name: "test-dataset", + }, + }) + .mockResolvedValueOnce({ + id: "00000000-0000-0000-0000-000000000004", + dataset_id: "00000000-0000-0000-0000-000000000002", + name: "renamed snapshot", + description: null, + xact_id: "123", + created: "2026-03-31T00:00:00.000Z", + }); + + const dataset = initDataset({ + project: "test-project", + dataset: "test-dataset", + state, + }); + + await expect( + dataset.updateSnapshot("00000000-0000-0000-0000-000000000004", { + name: "renamed snapshot", + description: null, + }), + ).resolves.toMatchObject({ + id: "00000000-0000-0000-0000-000000000004", + name: "renamed snapshot", + description: null, + }); + + expect(postJson).toHaveBeenNthCalledWith(2, "api/dataset_snapshot/patch_id", { + id: "00000000-0000-0000-0000-000000000004", + name: "renamed snapshot", + description: null, + }); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + +test("dataset.restorePreview posts restore preview request", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + vi.spyOn(state.appConn(), "post_json").mockResolvedValueOnce({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + dataset: { + id: "00000000-0000-0000-0000-000000000002", + name: "test-dataset", + }, + }); + const postJson = vi + .spyOn(state.apiConn(), "post_json") + .mockResolvedValueOnce({ + rows_to_restore: 3, + rows_to_delete: 1, + }); + + const dataset = initDataset({ + project: "test-project", + dataset: "test-dataset", + state, + }); + + await expect( + dataset.restorePreview({ + version: "123", + }), + ).resolves.toEqual({ + rows_to_restore: 3, + rows_to_delete: 1, + }); + + expect(postJson).toHaveBeenNthCalledWith( + 1, + "v1/dataset/00000000-0000-0000-0000-000000000002/restore/preview", + { + version: "123", + }, + ); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + +test("dataset.restore posts restore request", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + vi.spyOn(state.appConn(), "post_json").mockResolvedValueOnce({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + dataset: { + id: "00000000-0000-0000-0000-000000000002", + name: "test-dataset", + }, + }); + const postJson = vi + .spyOn(state.apiConn(), "post_json") + .mockResolvedValueOnce({ + xact_id: "456", + rows_restored: 3, + rows_deleted: 1, + }); + + const dataset = initDataset({ + project: "test-project", + dataset: "test-dataset", + state, + }); + + await expect( + dataset.restore({ + version: "123", + }), + ).resolves.toEqual({ + xact_id: "456", + rows_restored: 3, + rows_deleted: 1, + }); + + expect(postJson).toHaveBeenNthCalledWith( + 1, + "v1/dataset/00000000-0000-0000-0000-000000000002/restore", + { + version: "123", + }, + ); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + +test("init keeps plain dataset refs attached to the experiment", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + mockInitGitMetadata(); + vi.spyOn(state.appConn(), "post_json").mockResolvedValue({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + experiment: { + id: "00000000-0000-0000-0000-000000000003", + project_id: "00000000-0000-0000-0000-000000000001", + name: "test-experiment", + public: false, + }, + }); + + const experiment = init({ + project: "test-project", + experiment: "test-experiment", + dataset: { + id: "00000000-0000-0000-0000-000000000002", + }, + setCurrent: false, + state, + }); + + await experiment.id; + + expect(experiment.dataset).toMatchObject({ + id: "00000000-0000-0000-0000-000000000002", + }); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + +test("init resolves dataset environment before experiment registration", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + mockInitGitMetadata(); + const getJson = vi.spyOn(state.apiConn(), "get_json").mockResolvedValue({ + object_version: "123", + }); + const postJson = vi.spyOn(state.appConn(), "post_json").mockResolvedValue({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + experiment: { + id: "00000000-0000-0000-0000-000000000003", + project_id: "00000000-0000-0000-0000-000000000001", + name: "test-experiment", + public: false, + }, + }); + + const experiment = init({ + project: "test-project", + experiment: "test-experiment", + dataset: { + id: "00000000-0000-0000-0000-000000000002", + environment: "production", + }, + setCurrent: false, + state, + }); + + await experiment.id; + + expect(getJson).toHaveBeenCalledWith( + "environment-object/dataset/00000000-0000-0000-0000-000000000002/production", + { + org_name: "test-org-name", + }, + ); + expect(experiment.dataset).toMatchObject({ + id: "00000000-0000-0000-0000-000000000002", + environment: "production", + }); + expect(postJson).toHaveBeenCalledWith( + "api/experiment/register", + expect.objectContaining({ + dataset_id: "00000000-0000-0000-0000-000000000002", + dataset_version: "123", + }), + ); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + +test("init prefers dataset version over environment before experiment registration", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + mockInitGitMetadata(); + const getJson = vi.spyOn(state.apiConn(), "get_json"); + const postJson = vi.spyOn(state.appConn(), "post_json").mockResolvedValue({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + experiment: { + id: "00000000-0000-0000-0000-000000000003", + project_id: "00000000-0000-0000-0000-000000000001", + name: "test-experiment", + public: false, + }, + }); + + const experiment = init({ + project: "test-project", + experiment: "test-experiment", + dataset: { + id: "00000000-0000-0000-0000-000000000002", + version: "123", + environment: "production", + }, + setCurrent: false, + state, + }); + + await experiment.id; + + expect(getJson).not.toHaveBeenCalled(); + expect(postJson).toHaveBeenCalledWith( + "api/experiment/register", + expect.objectContaining({ + dataset_id: "00000000-0000-0000-0000-000000000002", + dataset_version: "123", + }), + ); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + +test("init resolves dataset snapshots before experiment registration", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + mockInitGitMetadata(); + const postJson = vi + .spyOn(state.appConn(), "post_json") + .mockResolvedValueOnce([ + { + id: "00000000-0000-0000-0000-000000000004", + dataset_id: "00000000-0000-0000-0000-000000000002", + name: "123", + description: null, + xact_id: "456", + created: "2026-03-31T00:00:00.000Z", + }, + ]) + .mockResolvedValueOnce({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + experiment: { + id: "00000000-0000-0000-0000-000000000003", + project_id: "00000000-0000-0000-0000-000000000001", + name: "test-experiment", + public: false, + }, + }); + + const experiment = init({ + project: "test-project", + experiment: "test-experiment", + dataset: { + id: "00000000-0000-0000-0000-000000000002", + snapshotName: "123", + }, + setCurrent: false, + state, + }); + + await experiment.id; + + expect(postJson).toHaveBeenNthCalledWith(1, "api/dataset_snapshot/get", { + dataset_id: "00000000-0000-0000-0000-000000000002", + }); + expect(postJson).toHaveBeenNthCalledWith( + 2, + "api/experiment/register", + expect.objectContaining({ + dataset_id: "00000000-0000-0000-0000-000000000002", + dataset_version: "456", + }), + ); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + +test("init surfaces dataset environment lookup errors instead of falling back to latest", async () => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + vi.spyOn(state, "login").mockResolvedValue(state); + mockInitGitMetadata(); + vi.spyOn(state.apiConn(), "get_json").mockRejectedValue( + new Error("environment lookup failed"), + ); + const postJson = vi.spyOn(state.appConn(), "post_json"); + + const experiment = init({ + project: "test-project", + experiment: "test-experiment", + dataset: { + id: "00000000-0000-0000-0000-000000000002", + environment: "production", + }, + setCurrent: false, + state, + }); + + await expect(experiment.id).rejects.toThrow("environment lookup failed"); + expect(postJson).not.toHaveBeenCalled(); + + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); +}); + describe("loader version precedence", () => { let state: BraintrustState; let getJson: ReturnType; diff --git a/js/src/logger.ts b/js/src/logger.ts index 2dd3d5150..c75b2b825 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -69,6 +69,8 @@ import { type GitMetadataSettingsType as GitMetadataSettings, type ChatCompletionMessageParamType as Message, type ChatCompletionOpenAIMessageParamType as OpenAIMessage, + DatasetSnapshot as datasetSnapshotSchema, + type DatasetSnapshotType as DatasetSnapshot, PromptData as promptDataSchema, type PromptDataType as PromptData, Prompt as promptSchema, @@ -90,6 +92,28 @@ const RESET_CONTEXT_MANAGER_STATE = Symbol.for( // 6 MB for the AWS lambda gateway (from our own testing). export const DEFAULT_MAX_REQUEST_SIZE = 6 * 1024 * 1024; +export type { DatasetSnapshot }; + +const datasetSnapshotRegisterResponseSchema = z.object({ + dataset_snapshot: datasetSnapshotSchema, + found_existing: z.boolean().optional(), +}); + +const datasetRestorePreviewResultSchema = z.object({ + rows_to_restore: z.number(), + rows_to_delete: z.number(), +}); +export type DatasetRestorePreviewResult = z.infer< + typeof datasetRestorePreviewResultSchema +>; + +const datasetRestoreResultSchema = z.object({ + xact_id: z.string().nullable(), + rows_restored: z.number(), + rows_deleted: z.number(), +}); +export type DatasetRestoreResult = z.infer; + const parametersRowSchema = z.object({ id: z.string().uuid(), _xact_id: z.string(), @@ -3390,12 +3414,23 @@ type InitOpenOption = { }; /** - * Reference to a dataset by ID and optional version. + * Reference to a dataset by ID and optional explicit selector. */ -export interface DatasetRef { - id: string; +type DatasetSelection = { version?: string; -} + environment?: string; + snapshotName?: string; +}; + +type DatasetPinState = { + lazyPinnedVersion?: LazyValue; + pinnedEnvironment?: string; + pinnedSnapshotName?: string; +}; + +export type DatasetRef = { + id: string; +} & DatasetSelection; export interface ParametersRef { id: string; @@ -3621,20 +3656,13 @@ export function init( } if (dataset !== undefined) { - if ( - "id" in dataset && - typeof dataset.id === "string" && - !("__braintrust_dataset_marker" in dataset) - ) { - // Simple {id: ..., version?: ...} object - args["dataset_id"] = dataset.id; - if ("version" in dataset && dataset.version !== undefined) { - args["dataset_version"] = dataset.version; - } - } else { - // Full Dataset object - args["dataset_id"] = await (dataset as AnyDataset).id; - args["dataset_version"] = await (dataset as AnyDataset).version(); + const datasetSelection = await serializeDatasetForExperiment({ + dataset, + state, + }); + args["dataset_id"] = datasetSelection.datasetId; + if (datasetSelection.datasetVersion !== undefined) { + args["dataset_version"] = datasetSelection.datasetVersion; } } @@ -3704,9 +3732,7 @@ export function init( const ret = new Experiment( state, lazyMetadata, - dataset !== undefined && "version" in dataset - ? (dataset as AnyDataset) - : undefined, + dataset !== undefined ? (dataset as AnyDataset) : undefined, ); if (options.setCurrent ?? true) { state.currentExperiment = ret; @@ -3795,6 +3821,8 @@ export type InitDatasetOptions = dataset?: string; description?: string; version?: string; + environment?: string; + snapshotName?: string; projectId?: string; metadata?: Record; state?: BraintrustState; @@ -3805,6 +3833,187 @@ export type FullInitDatasetOptions = { project?: string; } & InitDatasetOptions; +async function getDatasetSnapshots({ + state, + datasetId, +}: { + state: BraintrustState; + datasetId: string; +}): Promise { + return datasetSnapshotSchema.array().parse( + await state.appConn().post_json("api/dataset_snapshot/get", { + dataset_id: datasetId, + }), + ); +} + +function normalizeDatasetSelection({ + version, + environment, + snapshotName, +}: DatasetSelection): DatasetSelection { + if (version !== undefined) { + return { version }; + } + + if (snapshotName !== undefined) { + return { snapshotName }; + } + + if (environment !== undefined) { + return { environment }; + } + + return {}; +} + +async function resolveDatasetSnapshotName({ + state, + datasetId, + snapshotName, +}: { + state: BraintrustState; + datasetId: string; + snapshotName: string; +}): Promise { + const snapshots = await getDatasetSnapshots({ state, datasetId }); + const match = snapshots.find((snapshot) => snapshot.name === snapshotName); + if (match === undefined) { + throw new Error( + `Dataset snapshot "${snapshotName}" not found for ${datasetId}`, + ); + } + return match.xact_id; +} + +async function resolveDatasetSnapshotNameForMetadata({ + state, + lazyMetadata, + snapshotName, +}: { + state: BraintrustState; + lazyMetadata: LazyValue; + snapshotName: string; +}): Promise { + const metadata = await lazyMetadata.get(); + return await resolveDatasetSnapshotName({ + state, + datasetId: metadata.dataset.id, + snapshotName, + }); +} + +async function resolveDatasetEnvironment({ + state, + datasetId, + environment, +}: { + state: BraintrustState; + datasetId: string; + environment: string; +}): Promise { + const response = await state + .apiConn() + .get_json( + `environment-object/dataset/${datasetId}/${encodeURIComponent(environment)}`, + { + org_name: state.orgName ?? undefined, + }, + ); + return z.object({ object_version: z.string() }).parse(response) + .object_version; +} + +async function resolveDatasetEnvironmentForMetadata({ + state, + lazyMetadata, + environment, +}: { + state: BraintrustState; + lazyMetadata: LazyValue; + environment: string; +}): Promise { + const metadata = await lazyMetadata.get(); + return await resolveDatasetEnvironment({ + state, + datasetId: metadata.dataset.id, + environment, + }); +} + +async function serializeDatasetForExperiment({ + dataset, + state, +}: { + dataset: AnyDataset | DatasetRef; + state: BraintrustState; +}): Promise<{ datasetId: string; datasetVersion?: string }> { + if (!Dataset.isDataset(dataset)) { + const selection = normalizeDatasetSelection(dataset); + + if (selection.version !== undefined) { + return { + datasetId: dataset.id, + datasetVersion: selection.version, + }; + } + + if (selection.snapshotName !== undefined) { + return { + datasetId: dataset.id, + datasetVersion: await resolveDatasetSnapshotName({ + state, + datasetId: dataset.id, + snapshotName: selection.snapshotName, + }), + }; + } + + if (selection.environment !== undefined) { + return { + datasetId: dataset.id, + datasetVersion: await resolveDatasetEnvironment({ + state, + datasetId: dataset.id, + environment: selection.environment, + }), + }; + } + + return { + datasetId: dataset.id, + }; + } + + const evalData = await dataset.toEvalData(); + const selection = normalizeDatasetSelection({ + version: evalData.dataset_version, + environment: evalData.dataset_environment, + snapshotName: evalData.dataset_snapshot_name, + }); + + if (selection.version !== undefined) { + return { + datasetId: evalData.dataset_id, + datasetVersion: selection.version, + }; + } + + if ( + selection.environment !== undefined || + selection.snapshotName !== undefined + ) { + return { + datasetId: evalData.dataset_id, + datasetVersion: await dataset.version(), + }; + } + + return { + datasetId: evalData.dataset_id, + }; +} + /** * Create a new dataset in a specified project. If the project does not exist, it will be created. * @@ -3812,6 +4021,9 @@ export type FullInitDatasetOptions = { * @param options.project The name of the project to create the dataset in. Must specify at least one of `project` or `projectId`. * @param options.dataset The name of the dataset to create. If not specified, a name will be generated automatically. * @param options.description An optional description of the dataset. + * @param options.version Pin the dataset to a specific version xact_id. If `snapshotName` or `environment` are also provided, `version` takes precedence. + * @param options.snapshotName Pin the dataset to the version captured by this named snapshot. If `environment` is also provided, `snapshotName` takes precedence. + * @param options.environment Pin the dataset to the version tagged with this environment slug. * @param options.appUrl The URL of the Braintrust App. Defaults to https://www.braintrust.dev. * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API key is specified, will prompt the user to login. * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple. @@ -3868,6 +4080,8 @@ export function initDataset< dataset, description, version, + snapshotName, + environment, appUrl, apiKey, orgName, @@ -3879,6 +4093,14 @@ export function initDataset< state: stateArg, _internal_btql, } = options; + const selection = normalizeDatasetSelection({ + version, + environment, + snapshotName, + }); + const normalizedVersion = selection.version; + const normalizedEnvironment = selection.environment; + const normalizedSnapshotName = selection.snapshotName; const state = stateArg ?? _globalState; @@ -3919,13 +4141,57 @@ export function initDataset< }, ); - return new Dataset( + const resolvedVersion = + normalizedVersion !== undefined + ? normalizedVersion + : normalizedSnapshotName !== undefined + ? new LazyValue(async () => { + return await resolveDatasetSnapshotNameForMetadata({ + state, + lazyMetadata, + snapshotName: normalizedSnapshotName, + }); + }) + : normalizedEnvironment !== undefined + ? new LazyValue(async () => { + return await resolveDatasetEnvironmentForMetadata({ + state, + lazyMetadata, + environment: normalizedEnvironment, + }); + }) + : undefined; + + const datasetObject = new Dataset( stateArg ?? _globalState, lazyMetadata, - version, + typeof resolvedVersion === "string" ? resolvedVersion : undefined, legacy, _internal_btql, + resolvedVersion instanceof LazyValue || + normalizedEnvironment !== undefined || + normalizedSnapshotName !== undefined + ? { + ...(resolvedVersion instanceof LazyValue + ? { + lazyPinnedVersion: resolvedVersion, + } + : {}), + ...(normalizedEnvironment !== undefined + ? { + pinnedEnvironment: normalizedEnvironment, + } + : {}), + ...(normalizedSnapshotName !== undefined + ? { + pinnedSnapshotName: normalizedSnapshotName, + } + : {}), + } + : undefined, ); + + return datasetObject; } /** @@ -5706,6 +5972,18 @@ export class ObjectFetcher implements AsyncIterable< throw new Error("ObjectFetcher subclasses must have a 'getState' method"); } + protected getPinnedVersion(): string | undefined { + return this.pinnedVersion; + } + + protected setPinnedVersion(pinnedVersion: string | undefined): void { + this.pinnedVersion = pinnedVersion; + } + + protected getInternalBtql(): Record | undefined { + return this._internal_btql; + } + private async *fetchRecordsFromApi( batchSize: number | undefined, ): AsyncGenerator> { @@ -6840,6 +7118,9 @@ export class Dataset< IsLegacyDataset extends boolean = typeof DEFAULT_IS_LEGACY_DATASET, > extends ObjectFetcher> { private readonly lazyMetadata: LazyValue; + private lazyPinnedVersion?: LazyValue; + private pinnedEnvironment?: string; + private pinnedSnapshotName?: string; private readonly __braintrust_dataset_marker = true; private newRecords = 0; @@ -6849,6 +7130,7 @@ export class Dataset< pinnedVersion?: string, legacy?: IsLegacyDataset, _internal_btql?: Record, + pinState?: DatasetPinState, ) { // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const isLegacyDataset = (legacy ?? @@ -6872,6 +7154,9 @@ export class Dataset< _internal_btql, ); this.lazyMetadata = lazyMetadata; + this.lazyPinnedVersion = pinState?.lazyPinnedVersion; + this.pinnedEnvironment = pinState?.pinnedEnvironment; + this.pinnedSnapshotName = pinState?.pinnedSnapshotName; } public get id(): Promise { @@ -6896,12 +7181,59 @@ export class Dataset< return this.state; } + public async toEvalData(): Promise<{ + dataset_id: string; + dataset_version?: string; + dataset_environment?: string; + dataset_snapshot_name?: string; + _internal_btql?: Record; + }> { + await this.getState(); + const metadata = await this.lazyMetadata.get(); + const pinnedVersion = this.getPinnedVersion(); + const internalBtql = this.getInternalBtql(); + + return { + dataset_id: metadata.dataset.id, + ...(this.pinnedEnvironment !== undefined + ? { + dataset_environment: this.pinnedEnvironment, + } + : {}), + ...(this.pinnedEnvironment === undefined && + this.pinnedSnapshotName !== undefined + ? { + dataset_snapshot_name: this.pinnedSnapshotName, + } + : {}), + ...(this.pinnedEnvironment === undefined && + this.pinnedSnapshotName === undefined && + pinnedVersion !== undefined + ? { + dataset_version: pinnedVersion, + } + : {}), + ...(internalBtql !== undefined ? { _internal_btql: internalBtql } : {}), + }; + } + protected async getState(): Promise { // Ensure the login state is populated by awaiting lazyMetadata. await this.lazyMetadata.get(); + if ( + this.lazyPinnedVersion !== undefined && + this.getPinnedVersion() === undefined + ) { + this.setPinnedVersion(await this.lazyPinnedVersion.get()); + } return this.state; } + public override async version(options?: { batchSize?: number }) { + await this.getState(); + return await super.version(options); + } + private validateEvent({ metadata, expected, @@ -7078,6 +7410,104 @@ export class Dataset< return id; } + public async createSnapshot({ + name, + description, + update, + }: { + readonly name: string; + readonly description?: string; + readonly update?: boolean; + }): Promise { + await this.flush(); + const state = await this.getState(); + const datasetId = await this.id; + const currentVersion = await this.version(); + if (currentVersion === undefined) { + throw new Error("Cannot create snapshot: dataset has no version"); + } + const response = await state + .appConn() + .post_json("api/dataset_snapshot/register", { + dataset_id: datasetId, + dataset_snapshot_name: name, + description, + xact_id: currentVersion, + update, + }); + return datasetSnapshotRegisterResponseSchema.parse(response) + .dataset_snapshot; + } + + public async listSnapshots(): Promise { + const state = await this.getState(); + return await getDatasetSnapshots({ + state, + datasetId: await this.id, + }); + } + + public async updateSnapshot( + snapshotId: string, + { + name, + description, + }: { + readonly name?: string; + readonly description?: string | null; + }, + ): Promise { + const state = await this.getState(); + return datasetSnapshotSchema.parse( + await state.appConn().post_json("api/dataset_snapshot/patch_id", { + id: snapshotId, + name, + description, + }), + ); + } + + public async deleteSnapshot(snapshotId: string): Promise { + const state = await this.getState(); + return datasetSnapshotSchema.parse( + await state.appConn().post_json("api/dataset_snapshot/delete_id", { + id: snapshotId, + }), + ); + } + + public async restorePreview({ + version, + }: { + readonly version: string; + }): Promise { + await this.flush(); + const state = await this.getState(); + const datasetId = await this.id; + return datasetRestorePreviewResultSchema.parse( + await state + .apiConn() + .post_json(`v1/dataset/${datasetId}/restore/preview`, { + version, + }), + ); + } + + public async restore({ + version, + }: { + readonly version: string; + }): Promise { + await this.flush(); + const state = await this.getState(); + const datasetId = await this.id; + return datasetRestoreResultSchema.parse( + await state.apiConn().post_json(`v1/dataset/${datasetId}/restore`, { + version, + }), + ); + } + /** * Summarize the dataset, including high level metrics about its size and other metadata. * @param summarizeData Whether to summarize the data. If false, only the metadata will be returned.