From bd49da87f8f27de2270c0bdea782201ea200da73 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 17 Aug 2026 21:29:50 -0400 Subject: [PATCH] feat(api): generate Experiments REST bindings Implements step 3 of #683 and fixes #639. This adds the second public generated REST resource: - `BraintrustOpenApiClient.experiments`: all 10 operations selected by the Experiments OpenAPI tag, including `get_experiment_id_summarize` - `braintrust.api.types`: public experiment request and response types Codegen now partitions models across multiple resources. Models reached by one resource remain in its resource-specific module, while shared definitions are emitted once in `models/common.py` and imported explicitly. Against the pinned specification, Projects plus Experiments generates 15 operations and 45 reachable component schemas. Logical POST reads use a reviewed `safe_reads` allowlist, while generated GETs retain mechanical `SAFE_READ` classification and writes remain non-retrying unless explicitly classified. `Experiment.summarize()` now uses the generated summarize binding. Successful and intentionally skipped summaries are represented by `SummarySuccess` and `SummarySkipped`, with `comparison` as the primary result. The deprecated read-only `scores` and `metrics` bridges remain serialized for compatibility. Summary retrieval errors are no longer swallowed: transient failures retry through the policy-aware transport and final failures raise typed API errors. Structured summaries support tagged deep deserialization and legacy payloads containing only top-level score and metric maps. Coverage includes deterministic multi-resource codegen, retry-policy validation, exact wire behavior for all generated Experiments methods, additive responses, retry exhaustion, framework propagation, static and runtime typing, structured-summary round trips, and real-backend VCR flows for implicit and explicit comparison selection. --- openapi/README.md | 13 +- openapi/config.json | 6 +- py/scripts/openapi_codegen.py | 207 ++++- .../braintrust/api/_generated/experiments.py | 485 ++++++++++ .../api/_generated/models/__init__.py | 126 ++- .../api/_generated/models/common.py | 60 ++ .../api/_generated/models/experiments.py | 850 ++++++++++++++++++ .../api/_generated/models/projects.py | 63 +- py/src/braintrust/api/_generated/projects.py | 17 +- ...nt_summarize_with_real_backend[False].yaml | 433 +++++++++ ...ent_summarize_with_real_backend[True].yaml | 495 ++++++++++ py/src/braintrust/api/client.py | 2 + py/src/braintrust/api/test_experiments.py | 97 ++ .../braintrust/api/test_generated_models.py | 17 +- py/src/braintrust/api/test_projects.py | 135 +-- py/src/braintrust/api/types/__init__.py | 29 +- py/src/braintrust/framework.py | 4 +- py/src/braintrust/logger.py | 195 ++-- py/src/braintrust/test_framework.py | 29 +- .../braintrust/type_tests/test_api_client.py | 33 +- .../type_tests/test_experiment_summary.py | 73 ++ py/tests/api_codegen/conftest.py | 1 + py/tests/api_codegen/test_generation.py | 55 +- py/tests/api_codegen/test_validation.py | 46 +- 24 files changed, 3131 insertions(+), 340 deletions(-) create mode 100644 py/src/braintrust/api/_generated/experiments.py create mode 100644 py/src/braintrust/api/_generated/models/common.py create mode 100644 py/src/braintrust/api/_generated/models/experiments.py create mode 100644 py/src/braintrust/api/cassettes/test_experiment_summarize_with_real_backend[False].yaml create mode 100644 py/src/braintrust/api/cassettes/test_experiment_summarize_with_real_backend[True].yaml create mode 100644 py/src/braintrust/api/test_experiments.py create mode 100644 py/src/braintrust/type_tests/test_experiment_summary.py diff --git a/openapi/README.md b/openapi/README.md index a41583c0..0420ffb4 100644 --- a/openapi/README.md +++ b/openapi/README.md @@ -15,12 +15,13 @@ make check-api-client-codegen The check regenerates into a temporary directory and does not modify the worktree. Endpoint bindings are rolled out explicitly through `endpoint_generator.generated_tags`. The current rollout supports -exactly one selected OpenAPI tag and emits its operation registry and resource class together in -`projects.py`, with reachable types in `models/projects.py`; unreachable models are omitted. Add -explicit cross-resource model partitioning before selecting a second tag. Public resource method and -inline response type names are derived mechanically from each `operationId`, and generated methods -forward request fields and parameters without implicit defaults. Writes that are safe to retry are -listed declaratively in `endpoint_generator.idempotent_writes`; reads and all other writes use +the Projects and Experiments tags and emits one operation registry/resource class per tag. Reachable +models used by one resource live in that resource's model module; models shared by multiple resources +live once in `models/common.py` and are imported explicitly. Unreachable models are omitted. Public +resource method and inline response type names are derived mechanically from each `operationId`, and +generated methods forward request fields and parameters without implicit defaults. Logical POST reads +that are safe to retry are listed in `endpoint_generator.safe_reads`, while verified idempotent writes +are listed in `endpoint_generator.idempotent_writes`; GET/HEAD reads and all other writes use mechanical retry defaults. To fetch the configured upstream commit explicitly: diff --git a/openapi/config.json b/openapi/config.json index 0a3fe36c..877be3a4 100644 --- a/openapi/config.json +++ b/openapi/config.json @@ -30,7 +30,11 @@ "endpoint_generator": { "schema_version": 1, "generated_tags": [ - "Projects" + "Projects", + "Experiments" + ], + "safe_reads": [ + "postExperimentIdFetch" ], "idempotent_writes": [ "postProject" diff --git a/py/scripts/openapi_codegen.py b/py/scripts/openapi_codegen.py index e50d8508..f25a4f69 100644 --- a/py/scripts/openapi_codegen.py +++ b/py/scripts/openapi_codegen.py @@ -1,5 +1,6 @@ """Shared validation and generation helpers for the pinned Braintrust OpenAPI spec.""" +import ast import copy import difflib import hashlib @@ -289,19 +290,119 @@ def _with_inline_models( return model_spec -def _single_model_module(operations: Sequence[GeneratedOperation]) -> str: - tags = {operation.tag for operation in operations} - if len(tags) != 1: - raise CodegenError( - "Model generation currently requires exactly one generated OpenAPI tag; " - "add explicit cross-resource model partitioning before enabling another tag" +_NON_MODEL_ANNOTATION_NAMES = {"Any", "Literal", "Mapping", "None", "Sequence"} + + +def _operation_annotation_names(operation: GeneratedOperation) -> Set[str]: + annotation_names: Set[str] = set() + for type_name in [ + operation.request_body_type, + operation.response_type, + *(parameter.type_name for parameter in operation.parameters), + ]: + if type_name: + annotation_names.update(re.findall(r"\b[A-Z][A-Za-z0-9_]*\b", type_name)) + return annotation_names + + +def _operation_model_roots(operations: Sequence[GeneratedOperation]) -> Dict[str, Set[str]]: + roots: Dict[str, Set[str]] = {} + for operation in operations: + roots.setdefault(operation.tag, set()).update( + _operation_annotation_names(operation) - _NON_MODEL_ANNOTATION_NAMES ) - return _snake_case(next(iter(tags))) + return roots + + +def _partition_model_source( + source: str, operations: Sequence[GeneratedOperation] +) -> Tuple[Dict[str, str], Dict[str, str]]: + """Partition one deterministic model-generator output by resource dependency closure. + + Definitions reached by more than one generated tag live in ``common.py``. Resource-specific + modules import those shared definitions explicitly, avoiding duplicate runtime type identities. + """ + tree = ast.parse(source) + imports: List[ast.stmt] = [] + definitions: List[Tuple[str, List[ast.stmt]]] = [] + for node in tree.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + imports.append(node) + continue + if isinstance(node, ast.ClassDef): + names = [node.name] + elif isinstance(node, (ast.Assign, ast.AnnAssign)): + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + names = [target.id for target in targets if isinstance(target, ast.Name)] + elif isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): + if definitions: + definitions[-1][1].append(node) + continue + else: + raise CodegenError(f"Unsupported generated model statement: {type(node).__name__}") + if len(names) != 1: + raise CodegenError("Generated model definitions must bind exactly one public name") + definitions.append((names[0], [node])) + + definition_names = {name for name, _ in definitions} + dependencies: Dict[str, Set[str]] = {} + for name, nodes in definitions: + dependencies[name] = { + child.id + for node in nodes + for child in ast.walk(node) + if isinstance(child, ast.Name) and child.id in definition_names and child.id != name + } + owners: Dict[str, Set[str]] = {name: set() for name in definition_names} + for tag, roots in _operation_model_roots(operations).items(): + pending = list(roots) + seen: Set[str] = set() + while pending: + name = pending.pop() + if name in seen: + continue + if name not in definition_names: + raise CodegenError(f"Generated resource {tag!r} references unknown model {name!r}") + seen.add(name) + owners[name].add(tag) + pending.extend(dependencies[name]) + + unreachable = sorted(name for name, tags in owners.items() if not tags) + if unreachable: + raise CodegenError(f"Generated models are unreachable from resource methods: {unreachable}") + + common_names = {name for name, tags in owners.items() if len(tags) > 1} + module_for_name = { + name: "common" if name in common_names else _snake_case(next(iter(tags))) for name, tags in owners.items() + } -def _model_modules(spec: Mapping[str, Any], module: str) -> Dict[str, str]: - schemas = spec.get("components", {}).get("schemas", {}) - return {_python_type_name(name): module for name in schemas} + def source_for(node: ast.stmt) -> str: + segment = ast.get_source_segment(source, node) + if segment is None: + raise CodegenError(f"Could not recover generated model source for {type(node).__name__}") + return segment + + import_source = "\n".join(source_for(node) for node in imports) + bodies: Dict[str, List[str]] = {} + for name, nodes in definitions: + bodies.setdefault(module_for_name[name], []).append("\n".join(source_for(node) for node in nodes)) + + modules: Dict[str, str] = {} + for module, blocks in sorted(bodies.items()): + common_imports = sorted( + dependency + for name, _ in definitions + if module_for_name[name] == module + for dependency in dependencies[name] + if dependency in common_names + ) + sections = [import_source] + if common_imports and module != "common": + sections.append(f"from .common import {', '.join(dict.fromkeys(common_imports))}") + sections.append("\n\n".join(blocks)) + modules[module] = "\n\n".join(section for section in sections if section) + "\n" + return modules, module_for_name def generate_tree(output_root: Path, config: Mapping[str, Any], spec: Mapping[str, Any]) -> ValidationReport: @@ -310,21 +411,29 @@ def generate_tree(output_root: Path, config: Mapping[str, Any], spec: Mapping[st operations, inline_models = _collect_generated_operations(spec, config) selected_spec = _slice_model_spec(spec, {operation.operation_id for operation in operations}) model_spec = _with_inline_models(selected_spec, inline_models) - model_module = _single_model_module(operations) - model_modules = _model_modules(model_spec, model_module) output_root.mkdir(parents=True, exist_ok=True) selected_spec_path = output_root.parent / "selected-spec.json" + monolithic_models_path = output_root.parent / "models.py" selected_spec_path.write_text( json.dumps(model_spec, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n", encoding="utf-8" ) try: - _generate_models(selected_spec_path, output_root / "models" / f"{model_module}.py", config) + _generate_models(selected_spec_path, monolithic_models_path, config) + model_sources, model_modules = _partition_model_source(monolithic_models_path.read_text(), operations) finally: selected_spec_path.unlink(missing_ok=True) + monolithic_models_path.unlink(missing_ok=True) + model_paths = [] + for module, body in model_sources.items(): + model_path = output_root / "models" / f"{module}.py" + model_path.parent.mkdir(parents=True, exist_ok=True) + _write_generated_file(model_path, body, config) + model_paths.append(model_path) _write_generated_file(output_root / "__init__.py", _GENERATED_INIT_BODY, config) - _write_generated_file(output_root / "models" / "__init__.py", '"""Generated private model types."""\n', config) + model_init_path = output_root / "models" / "__init__.py" + _write_generated_file(model_init_path, _model_package_source(model_modules), config) resource_files = _generate_resources(output_root, operations, model_modules, config) - _format_generated_files(resource_files) + _format_generated_files([*model_paths, model_init_path, *resource_files]) return report @@ -459,6 +568,20 @@ def _generated_header(config: Mapping[str, Any], content_hash: str) -> str: ''' +def _model_package_source(model_modules: Mapping[str, str]) -> str: + by_module: Dict[str, List[str]] = {} + for name, module in model_modules.items(): + by_module.setdefault(module, []).append(name) + + lines = ['"""Generated private model types with stable package-level imports."""', ""] + for module, names in sorted(by_module.items()): + lines.append(f"from .{module} import {', '.join(sorted(names))}") + lines.extend(["", "", "__all__ = ["]) + lines.extend(f" {name!r}," for name in sorted(model_modules)) + lines.extend(["]", ""]) + return "\n".join(lines) + + def _validate_selected_operations( operations: Sequence[Tuple[str, str, Any, Mapping[str, Any], Mapping[str, Any]]], endpoint: Mapping[str, Any], @@ -474,11 +597,31 @@ def _validate_selected_operations( if missing_tags: raise CodegenError(f"endpoint_generator.generated_tags contains unknown tags: {sorted(missing_tags)}") + safe_reads = set(endpoint["safe_reads"]) + stale_safe_reads = safe_reads - set(supported) + if stale_safe_reads: + operation_id = sorted(stale_safe_reads)[0] + raise CodegenError(f"endpoint_generator.safe_reads references non-generated operation {operation_id!r}") + non_post_safe_reads = sorted( + operation_id for method, _, operation_id, _, _ in operations if operation_id in safe_reads and method != "post" + ) + if non_post_safe_reads: + raise CodegenError( + f"endpoint_generator.safe_reads must reference POST operations; got {non_post_safe_reads[0]!r}" + ) + idempotent_writes = set(endpoint["idempotent_writes"]) stale_idempotent_writes = idempotent_writes - set(supported) if stale_idempotent_writes: operation_id = sorted(stale_idempotent_writes)[0] raise CodegenError(f"endpoint_generator.idempotent_writes references non-generated operation {operation_id!r}") + overlapping_retry_modes = safe_reads & idempotent_writes + if overlapping_retry_modes: + operation_id = sorted(overlapping_retry_modes)[0] + raise CodegenError( + f"Operation {operation_id!r} cannot appear in both endpoint_generator.safe_reads " + "and endpoint_generator.idempotent_writes" + ) non_writes = sorted( operation_id for method, _, operation_id, _, _ in operations @@ -488,8 +631,8 @@ def _validate_selected_operations( raise CodegenError(f"endpoint_generator.idempotent_writes references read operation {non_writes[0]!r}") -def _operation_retry_mode(method: str, operation_id: str, idempotent_writes: Set[str]) -> str: - if method in {"get", "head"}: +def _operation_retry_mode(method: str, operation_id: str, safe_reads: Set[str], idempotent_writes: Set[str]) -> str: + if method in {"get", "head"} or operation_id in safe_reads: return "SAFE_READ" if operation_id in idempotent_writes: return "IDEMPOTENT_WRITE" @@ -500,6 +643,7 @@ def _collect_generated_operations( spec: Mapping[str, Any], config: Mapping[str, Any] ) -> Tuple[List[GeneratedOperation], List[Tuple[str, Mapping[str, Any]]]]: endpoint = _endpoint_config(config) + safe_reads = set(endpoint["safe_reads"]) idempotent_writes = set(endpoint["idempotent_writes"]) operations: List[GeneratedOperation] = [] inline_models: Dict[str, Mapping[str, Any]] = {} @@ -528,7 +672,7 @@ def _collect_generated_operations( response_type=response_type, success_statuses=statuses, json_success_statuses=json_statuses, - retry_mode=_operation_retry_mode(method, operation_id, idempotent_writes), + retry_mode=_operation_retry_mode(method, operation_id, safe_reads, idempotent_writes), ) ) return operations, list(inline_models.items()) @@ -663,18 +807,10 @@ def _generate_resources( def _resource_module_source( tag: str, operations: Sequence[GeneratedOperation], model_modules: Mapping[str, str] ) -> str: - annotation_names: Set[str] = set() - for operation in operations: - for type_name in [ - operation.request_body_type, - operation.response_type, - *(parameter.type_name for parameter in operation.parameters), - ]: - if type_name: - annotation_names.update(re.findall(r"\b[A-Z][A-Za-z0-9_]*\b", type_name)) + annotation_names = set().union(*(_operation_annotation_names(operation) for operation in operations)) collections_imports = sorted(annotation_names & {"Mapping", "Sequence"}) typing_imports = sorted(annotation_names & {"Any", "Literal"}) - model_type_names = annotation_names - {"Any", "Literal", "Mapping", "None", "Sequence"} + model_type_names = annotation_names - _NON_MODEL_ANNOTATION_NAMES model_imports: Dict[str, Set[str]] = {} for type_name in model_type_names: module = model_modules.get(type_name) @@ -834,13 +970,14 @@ def _endpoint_config(config: Mapping[str, Any]) -> Mapping[str, Any]: or len(generated_tags) != len(set(generated_tags)) ): raise CodegenError("endpoint_generator.generated_tags must be a unique list of non-empty strings") - idempotent_writes = endpoint.get("idempotent_writes") - if ( - not isinstance(idempotent_writes, list) - or not all(isinstance(value, str) and value for value in idempotent_writes) - or len(idempotent_writes) != len(set(idempotent_writes)) - ): - raise CodegenError("endpoint_generator.idempotent_writes must be a unique list of non-empty strings") + for key in ("safe_reads", "idempotent_writes"): + values = endpoint.get(key) + if ( + not isinstance(values, list) + or not all(isinstance(value, str) and value for value in values) + or len(values) != len(set(values)) + ): + raise CodegenError(f"endpoint_generator.{key} must be a unique list of non-empty strings") for key in ("supported_request_media_types", "supported_response_media_types", "supported_success_statuses"): values = endpoint.get(key) if not isinstance(values, list) or not values or not all(isinstance(value, str) for value in values): diff --git a/py/src/braintrust/api/_generated/experiments.py b/py/src/braintrust/api/_generated/experiments.py new file mode 100644 index 00000000..1462cc6e --- /dev/null +++ b/py/src/braintrust/api/_generated/experiments.py @@ -0,0 +1,485 @@ +# Generated by scripts/generate-api-client.py. DO NOT EDIT. +# OpenAPI commit: 9daf27f19d9e0340304d7a3e7d0edb28380b94c6 +# OpenAPI spec SHA-256: 5ec753c0263c0c44cd04f741edfc7e8bad491cc25a2113d029e84edc076520f0 +# datamodel-code-generator: 0.72.4 +# ruff: 0.15.21 +# Generator Python: 3.14 +# Content SHA-256: c3bcd5de70f3b9110426fd4557e7c056637cfda9f219ef6c18f23b17ee66b6c0 + +"""Generated Experiments REST operations and resource.""" + +from typing import cast + +from .._service import Operation, Parameter, ResourceAPI +from ..policies import RetryMode +from .models.common import EndingBefore, Ids, OrgName, ProjectName, StartingAfter +from .models.experiments import ( + AppLimitWithDefaultParam, + ComparisonExperimentId, + CreateExperiment, + Experiment, + ExperimentIdParam, + ExperimentName, + FeedbackExperimentEventRequest, + FeedbackResponseSchema, + FetchEventsRequest, + FetchExperimentEventsResponse, + FetchLimitParam, + GetExperimentResponse, + InsertEventsResponse, + InsertExperimentEventRequest, + MaxRootSpanId, + MaxXactId, + PatchExperiment, + ProjectIdQuery, + SummarizeExperimentResponse, + SummarizeScores, + Version, +) + + +POST_EXPERIMENT = Operation( + operation_id="postExperiment", + method="POST", + path="/v1/experiment", + parameters=(), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +GET_EXPERIMENT = Operation( + operation_id="getExperiment", + method="GET", + path="/v1/experiment", + parameters=( + Parameter( + argument_name="limit", + name="limit", + location="query", + required=False, + ), + Parameter( + argument_name="starting_after", + name="starting_after", + location="query", + required=False, + ), + Parameter( + argument_name="ending_before", + name="ending_before", + location="query", + required=False, + ), + Parameter( + argument_name="ids", + name="ids", + location="query", + required=False, + ), + Parameter( + argument_name="experiment_name", + name="experiment_name", + location="query", + required=False, + ), + Parameter( + argument_name="project_name", + name="project_name", + location="query", + required=False, + ), + Parameter( + argument_name="project_id", + name="project_id", + location="query", + required=False, + ), + Parameter( + argument_name="org_name", + name="org_name", + location="query", + required=False, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +GET_EXPERIMENT_ID = Operation( + operation_id="getExperimentId", + method="GET", + path="/v1/experiment/{experiment_id}", + parameters=( + Parameter( + argument_name="experiment_id", + name="experiment_id", + location="path", + required=True, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +PATCH_EXPERIMENT_ID = Operation( + operation_id="patchExperimentId", + method="PATCH", + path="/v1/experiment/{experiment_id}", + parameters=( + Parameter( + argument_name="experiment_id", + name="experiment_id", + location="path", + required=True, + ), + ), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +DELETE_EXPERIMENT_ID = Operation( + operation_id="deleteExperimentId", + method="DELETE", + path="/v1/experiment/{experiment_id}", + parameters=( + Parameter( + argument_name="experiment_id", + name="experiment_id", + location="path", + required=True, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +POST_EXPERIMENT_ID_INSERT = Operation( + operation_id="postExperimentIdInsert", + method="POST", + path="/v1/experiment/{experiment_id}/insert", + parameters=( + Parameter( + argument_name="experiment_id", + name="experiment_id", + location="path", + required=True, + ), + ), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +POST_EXPERIMENT_ID_FETCH = Operation( + operation_id="postExperimentIdFetch", + method="POST", + path="/v1/experiment/{experiment_id}/fetch", + parameters=( + Parameter( + argument_name="experiment_id", + name="experiment_id", + location="path", + required=True, + ), + ), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +GET_EXPERIMENT_ID_FETCH = Operation( + operation_id="getExperimentIdFetch", + method="GET", + path="/v1/experiment/{experiment_id}/fetch", + parameters=( + Parameter( + argument_name="experiment_id", + name="experiment_id", + location="path", + required=True, + ), + Parameter( + argument_name="limit", + name="limit", + location="query", + required=False, + ), + Parameter( + argument_name="max_xact_id", + name="max_xact_id", + location="query", + required=False, + ), + Parameter( + argument_name="max_root_span_id", + name="max_root_span_id", + location="query", + required=False, + ), + Parameter( + argument_name="version", + name="version", + location="query", + required=False, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +POST_EXPERIMENT_ID_FEEDBACK = Operation( + operation_id="postExperimentIdFeedback", + method="POST", + path="/v1/experiment/{experiment_id}/feedback", + parameters=( + Parameter( + argument_name="experiment_id", + name="experiment_id", + location="path", + required=True, + ), + ), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +GET_EXPERIMENT_ID_SUMMARIZE = Operation( + operation_id="getExperimentIdSummarize", + method="GET", + path="/v1/experiment/{experiment_id}/summarize", + parameters=( + Parameter( + argument_name="experiment_id", + name="experiment_id", + location="path", + required=True, + ), + Parameter( + argument_name="summarize_scores", + name="summarize_scores", + location="query", + required=False, + ), + Parameter( + argument_name="comparison_experiment_id", + name="comparison_experiment_id", + location="query", + required=False, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +OPERATIONS = { + "postExperiment": POST_EXPERIMENT, + "getExperiment": GET_EXPERIMENT, + "getExperimentId": GET_EXPERIMENT_ID, + "patchExperimentId": PATCH_EXPERIMENT_ID, + "deleteExperimentId": DELETE_EXPERIMENT_ID, + "postExperimentIdInsert": POST_EXPERIMENT_ID_INSERT, + "postExperimentIdFetch": POST_EXPERIMENT_ID_FETCH, + "getExperimentIdFetch": GET_EXPERIMENT_ID_FETCH, + "postExperimentIdFeedback": POST_EXPERIMENT_ID_FEEDBACK, + "getExperimentIdSummarize": GET_EXPERIMENT_ID_SUMMARIZE, +} + + +class ExperimentsAPI(ResourceAPI): + """Generated Experiments REST API.""" + + def post_experiment( + self, + *, + body: "CreateExperiment | None" = None, + ) -> "Experiment": + return cast( + "Experiment", + self.execute( + POST_EXPERIMENT, + body=body, + ), + ) + + def get_experiment( + self, + *, + limit: "AppLimitWithDefaultParam | None" = None, + starting_after: "StartingAfter | None" = None, + ending_before: "EndingBefore | None" = None, + ids: "Ids | None" = None, + experiment_name: "ExperimentName | None" = None, + project_name: "ProjectName | None" = None, + project_id: "ProjectIdQuery | None" = None, + org_name: "OrgName | None" = None, + ) -> "GetExperimentResponse": + return cast( + "GetExperimentResponse", + self.execute( + GET_EXPERIMENT, + query_parameters={ + "limit": limit, + "starting_after": starting_after, + "ending_before": ending_before, + "ids": ids, + "experiment_name": experiment_name, + "project_name": project_name, + "project_id": project_id, + "org_name": org_name, + }, + ), + ) + + def get_experiment_id( + self, + experiment_id: "ExperimentIdParam", + ) -> "Experiment": + return cast( + "Experiment", + self.execute( + GET_EXPERIMENT_ID, + path_parameters={"experiment_id": experiment_id}, + ), + ) + + def patch_experiment_id( + self, + experiment_id: "ExperimentIdParam", + *, + body: "PatchExperiment | None" = None, + ) -> "Experiment": + return cast( + "Experiment", + self.execute( + PATCH_EXPERIMENT_ID, + path_parameters={"experiment_id": experiment_id}, + body=body, + ), + ) + + def delete_experiment_id( + self, + experiment_id: "ExperimentIdParam", + ) -> "Experiment": + return cast( + "Experiment", + self.execute( + DELETE_EXPERIMENT_ID, + path_parameters={"experiment_id": experiment_id}, + ), + ) + + def post_experiment_id_insert( + self, + experiment_id: "ExperimentIdParam", + *, + body: "InsertExperimentEventRequest | None" = None, + ) -> "InsertEventsResponse": + return cast( + "InsertEventsResponse", + self.execute( + POST_EXPERIMENT_ID_INSERT, + path_parameters={"experiment_id": experiment_id}, + body=body, + ), + ) + + def post_experiment_id_fetch( + self, + experiment_id: "ExperimentIdParam", + *, + body: "FetchEventsRequest | None" = None, + ) -> "FetchExperimentEventsResponse": + return cast( + "FetchExperimentEventsResponse", + self.execute( + POST_EXPERIMENT_ID_FETCH, + path_parameters={"experiment_id": experiment_id}, + body=body, + ), + ) + + def get_experiment_id_fetch( + self, + experiment_id: "ExperimentIdParam", + *, + limit: "FetchLimitParam | None" = None, + max_xact_id: "MaxXactId | None" = None, + max_root_span_id: "MaxRootSpanId | None" = None, + version: "Version | None" = None, + ) -> "FetchExperimentEventsResponse": + return cast( + "FetchExperimentEventsResponse", + self.execute( + GET_EXPERIMENT_ID_FETCH, + path_parameters={"experiment_id": experiment_id}, + query_parameters={ + "limit": limit, + "max_xact_id": max_xact_id, + "max_root_span_id": max_root_span_id, + "version": version, + }, + ), + ) + + def post_experiment_id_feedback( + self, + experiment_id: "ExperimentIdParam", + *, + body: "FeedbackExperimentEventRequest | None" = None, + ) -> "FeedbackResponseSchema": + return cast( + "FeedbackResponseSchema", + self.execute( + POST_EXPERIMENT_ID_FEEDBACK, + path_parameters={"experiment_id": experiment_id}, + body=body, + ), + ) + + def get_experiment_id_summarize( + self, + experiment_id: "ExperimentIdParam", + *, + summarize_scores: "SummarizeScores | None" = None, + comparison_experiment_id: "ComparisonExperimentId | None" = None, + ) -> "SummarizeExperimentResponse": + return cast( + "SummarizeExperimentResponse", + self.execute( + GET_EXPERIMENT_ID_SUMMARIZE, + path_parameters={"experiment_id": experiment_id}, + query_parameters={ + "summarize_scores": summarize_scores, + "comparison_experiment_id": comparison_experiment_id, + }, + ), + ) diff --git a/py/src/braintrust/api/_generated/models/__init__.py b/py/src/braintrust/api/_generated/models/__init__.py index e7736608..4c917afb 100644 --- a/py/src/braintrust/api/_generated/models/__init__.py +++ b/py/src/braintrust/api/_generated/models/__init__.py @@ -4,6 +4,128 @@ # datamodel-code-generator: 0.72.4 # ruff: 0.15.21 # Generator Python: 3.14 -# Content SHA-256: cc2dbd2430abddbceff7f693feaffa844fa536663621674e67d06a78aca61f17 +# Content SHA-256: ad43048e1178347f0d1c31f64d9886cb6e0d91c2e6f901e8d0a3a7f08e8348af -"""Generated private model types.""" +"""Generated private model types with stable package-level imports.""" + +from .common import EndingBefore, FunctionTypeEnum, Ids, OrgName, ProjectName, StartingAfter +from .experiments import ( + AppLimitWithDefaultParam, + Classification, + ComparisonExperimentId, + Context, + CreateExperiment, + Experiment, + ExperimentEvent, + ExperimentIdParam, + ExperimentName, + FeedbackExperimentEventRequest, + FeedbackExperimentItem, + FeedbackResponseSchema, + FetchEventsRequest, + FetchExperimentEventsResponse, + FetchLimit, + FetchLimitParam, + FetchPaginationCursor, + FieldArrayDeleteItem, + GetExperimentResponse, + InsertEventsResponse, + InsertExperimentEvent, + InsertExperimentEventRequest, + InternalMetadata, + MaxRootSpanId, + MaxXactId, + Metadata, + MetricSummary, + Metrics, + ObjectReferenceNullish, + PatchExperiment, + ProjectIdQuery, + RepoInfo, + SavedFunctionId, + SavedFunctionId1, + SavedFunctionId2, + ScoreSummary, + SpanAttributes, + SpanType, + SummarizeExperimentResponse, + SummarizeScores, + Version, +) +from .projects import ( + AppLimitParam, + CreateProject, + GetProjectResponse, + NullableSavedFunctionId, + NullableSavedFunctionId1, + NullableSavedFunctionId2, + PatchProject, + Project, + ProjectIdParam, + ProjectSettings, + RemoteEvalSource, + SpanFieldOrderItem, +) + + +__all__ = [ + "AppLimitParam", + "AppLimitWithDefaultParam", + "Classification", + "ComparisonExperimentId", + "Context", + "CreateExperiment", + "CreateProject", + "EndingBefore", + "Experiment", + "ExperimentEvent", + "ExperimentIdParam", + "ExperimentName", + "FeedbackExperimentEventRequest", + "FeedbackExperimentItem", + "FeedbackResponseSchema", + "FetchEventsRequest", + "FetchExperimentEventsResponse", + "FetchLimit", + "FetchLimitParam", + "FetchPaginationCursor", + "FieldArrayDeleteItem", + "FunctionTypeEnum", + "GetExperimentResponse", + "GetProjectResponse", + "Ids", + "InsertEventsResponse", + "InsertExperimentEvent", + "InsertExperimentEventRequest", + "InternalMetadata", + "MaxRootSpanId", + "MaxXactId", + "Metadata", + "MetricSummary", + "Metrics", + "NullableSavedFunctionId", + "NullableSavedFunctionId1", + "NullableSavedFunctionId2", + "ObjectReferenceNullish", + "OrgName", + "PatchExperiment", + "PatchProject", + "Project", + "ProjectIdParam", + "ProjectIdQuery", + "ProjectName", + "ProjectSettings", + "RemoteEvalSource", + "RepoInfo", + "SavedFunctionId", + "SavedFunctionId1", + "SavedFunctionId2", + "ScoreSummary", + "SpanAttributes", + "SpanFieldOrderItem", + "SpanType", + "StartingAfter", + "SummarizeExperimentResponse", + "SummarizeScores", + "Version", +] diff --git a/py/src/braintrust/api/_generated/models/common.py b/py/src/braintrust/api/_generated/models/common.py new file mode 100644 index 00000000..2704e160 --- /dev/null +++ b/py/src/braintrust/api/_generated/models/common.py @@ -0,0 +1,60 @@ +# Generated by scripts/generate-api-client.py. DO NOT EDIT. +# OpenAPI commit: 9daf27f19d9e0340304d7a3e7d0edb28380b94c6 +# OpenAPI spec SHA-256: 5ec753c0263c0c44cd04f741edfc7e8bad491cc25a2113d029e84edc076520f0 +# datamodel-code-generator: 0.72.4 +# ruff: 0.15.21 +# Generator Python: 3.14 +# Content SHA-256: 4f44bc62a969364392528ed2e8919cb5ba98b5a57aa742fe184571dcc61443da + +from typing import Any, Literal, TypeAlias, TypedDict +from collections.abc import Mapping, Sequence +from typing_extensions import NotRequired + +EndingBefore: TypeAlias = str +""" +Pagination cursor id. + +For example, if the initial item in the last page you fetched had an id of `foo`, pass `ending_before=foo` to fetch the previous page. Note: you may only pass one of `starting_after` and `ending_before` +""" + +FunctionTypeEnum: TypeAlias = ( + Literal[ + "llm", + "scorer", + "task", + "tool", + "custom_view", + "preprocessor", + "facet", + "classifier", + "tag", + "parameters", + "sandbox", + ] + | None +) +""" +The type of global function. Defaults to 'scorer'. +""" + +Ids: TypeAlias = str | Sequence[str] +""" +Filter search results to a particular set of object IDs. To specify a list of IDs, include the query param multiple times +""" + +OrgName: TypeAlias = str +""" +Filter search results to within a particular organization +""" + +ProjectName: TypeAlias = str +""" +Name of the project to search for +""" + +StartingAfter: TypeAlias = str +""" +Pagination cursor id. + +For example, if the final item in the last page you fetched had an id of `foo`, pass `starting_after=foo` to fetch the next page. Note: you may only pass one of `starting_after` and `ending_before` +""" diff --git a/py/src/braintrust/api/_generated/models/experiments.py b/py/src/braintrust/api/_generated/models/experiments.py new file mode 100644 index 00000000..963d9fa3 --- /dev/null +++ b/py/src/braintrust/api/_generated/models/experiments.py @@ -0,0 +1,850 @@ +# Generated by scripts/generate-api-client.py. DO NOT EDIT. +# OpenAPI commit: 9daf27f19d9e0340304d7a3e7d0edb28380b94c6 +# OpenAPI spec SHA-256: 5ec753c0263c0c44cd04f741edfc7e8bad491cc25a2113d029e84edc076520f0 +# datamodel-code-generator: 0.72.4 +# ruff: 0.15.21 +# Generator Python: 3.14 +# Content SHA-256: 8217275b6af7a615da1333759bd9958a66bfbd003548dc35a70cc3430c27b119 + +from typing import Any, Literal, TypeAlias, TypedDict +from collections.abc import Mapping, Sequence +from typing_extensions import NotRequired + +from .common import FunctionTypeEnum + +AppLimitWithDefaultParam: TypeAlias = int | None +""" +Limit the number of objects to return +""" + +ComparisonExperimentId: TypeAlias = str +""" +The experiment to compare against, if summarizing scores and metrics. If omitted, will fall back to the `base_exp_id` stored in the experiment metadata, and then to the most recent experiment run in the same project. Must pass `summarize_scores=true` for this id to be used +""" + + +class InternalMetadata(TypedDict): + dataset_filter: NotRequired[Mapping[str, Any] | None] + """ + BTQL filter payload used to evaluate a subset of a linked dataset. + """ + + +class Context(TypedDict): + caller_filename: NotRequired[str | None] + """ + Name of the file in code where the experiment event was created + """ + caller_functionname: NotRequired[str | None] + """ + The function in code which created the experiment event + """ + caller_lineno: NotRequired[int | None] + """ + Line of code where the experiment event was created + """ + + +class Metadata(TypedDict): + model: NotRequired[str | None] + """ + The model used for this example + """ + + +class Metrics(TypedDict): + caller_filename: NotRequired[Any | None] + """ + This metric is deprecated + """ + caller_functionname: NotRequired[Any | None] + """ + This metric is deprecated + """ + caller_lineno: NotRequired[Any | None] + """ + This metric is deprecated + """ + completion_tokens: NotRequired[int | None] + """ + The number of tokens in the completion generated by the model (only set if this is an LLM span) + """ + end: NotRequired[float | None] + """ + A unix timestamp recording when the section of code which produced the experiment event finished + """ + prompt_tokens: NotRequired[int | None] + """ + The number of tokens in the prompt used to generate the experiment event (only set if this is an LLM span) + """ + start: NotRequired[float | None] + """ + A unix timestamp recording when the section of code which produced the experiment event started + """ + tokens: NotRequired[int | None] + """ + The total number of tokens in the input and output of the experiment event. + """ + + +ExperimentIdParam: TypeAlias = str +""" +Experiment id +""" + +ExperimentName: TypeAlias = str +""" +Name of the experiment to search for +""" + + +class FeedbackExperimentItem(TypedDict): + comment: NotRequired[str | None] + """ + An optional comment string to log about the experiment event + """ + expected: NotRequired[Any | None] + """ + The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not + """ + id: str + """ + The id of the experiment event to log feedback for. This is the row `id` returned by `POST /v1/experiment/{experiment_id}/insert` + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + A dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI. Note, this metadata does not correspond to the main event itself, but rather the audit log attached to the event. + """ + scores: NotRequired[Mapping[str, float | None] | None] + """ + A dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the experiment event + """ + source: NotRequired[Literal["app", "api", "external"] | None] + """ + The source of the feedback. Must be one of "external" (default), "app", or "api" + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags to log + """ + + +class FeedbackResponseSchema(TypedDict): + status: Literal["success"] + + +FetchLimit: TypeAlias = int | None +""" +limit the number of traces fetched + +Fetch queries may be paginated if the total result size is expected to be large (e.g. project_logs which accumulate over a long time). Note that fetch queries only support pagination in descending time order (from latest to earliest `_xact_id`. Furthermore, later pages may return rows which showed up in earlier pages, except with an earlier `_xact_id`. This happens because pagination occurs over the whole version history of the event log. You will most likely want to exclude any such duplicate, outdated rows (by `id`) from your combined result set. + +The `limit` parameter controls the number of full traces to return. So you may end up with more individual rows than the specified limit if you are fetching events containing traces. +""" + +FetchLimitParam: TypeAlias = int | None +""" +limit the number of traces fetched + +Fetch queries may be paginated if the total result size is expected to be large (e.g. project_logs which accumulate over a long time). Note that fetch queries only support pagination in descending time order (from latest to earliest `_xact_id`. Furthermore, later pages may return rows which showed up in earlier pages, except with an earlier `_xact_id`. This happens because pagination occurs over the whole version history of the event log. You will most likely want to exclude any such duplicate, outdated rows (by `id`) from your combined result set. + +The `limit` parameter controls the number of full traces to return. So you may end up with more individual rows than the specified limit if you are fetching events containing traces. +""" + +FetchPaginationCursor: TypeAlias = str | None +""" +An opaque string to be used as a cursor for the next page of results, in order from latest to earliest. + +The string can be obtained directly from the `cursor` property of the previous fetch query +""" + + +class InsertEventsResponse(TypedDict): + row_ids: Sequence[str] + """ + The ids of all rows that were inserted, aligning one-to-one with the rows provided as input + """ + + +class FieldArrayDeleteItem(TypedDict): + delete: Sequence[Any] + path: Sequence[str] + + +MaxRootSpanId: TypeAlias = str +""" +DEPRECATION NOTICE: The manually-constructed pagination cursor is deprecated in favor of the explicit 'cursor' returned by object fetch requests. Please prefer the 'cursor' argument going forwards. + +Together, `max_xact_id` and `max_root_span_id` form a pagination cursor + +Since a paginated fetch query returns results in order from latest to earliest, the cursor for the next page can be found as the row with the minimum (earliest) value of the tuple `(_xact_id, root_span_id)`. See the documentation of `limit` for an overview of paginating fetch queries. +""" + +MaxXactId: TypeAlias = str +""" +DEPRECATION NOTICE: The manually-constructed pagination cursor is deprecated in favor of the explicit 'cursor' returned by object fetch requests. Please prefer the 'cursor' argument going forwards. + +Together, `max_xact_id` and `max_root_span_id` form a pagination cursor + +Since a paginated fetch query returns results in order from latest to earliest, the cursor for the next page can be found as the row with the minimum (earliest) value of the tuple `(_xact_id, root_span_id)`. See the documentation of `limit` for an overview of paginating fetch queries. +""" + + +class MetricSummary(TypedDict): + diff: NotRequired[float] + """ + Difference in metric between the current and comparison experiment + """ + improvements: int + """ + Number of improvements in the metric + """ + metric: float + """ + Average metric across all examples + """ + name: str + """ + Name of the metric + """ + regressions: int + """ + Number of regressions in the metric + """ + unit: str + """ + Unit label for the metric + """ + + +class ObjectReferenceNullish(TypedDict): + field_xact_id: NotRequired[str | None] + """ + Transaction ID of the original event. + """ + created: NotRequired[str | None] + """ + Created timestamp of the original event. Used to help sort in the UI + """ + id: str + """ + ID of the original event. + """ + object_id: str + """ + ID of the object the event is originating from. + """ + object_type: Literal["project_logs", "experiment", "dataset", "prompt", "function", "prompt_session"] + """ + Type of the object the event is originating from. + """ + + +ProjectIdQuery: TypeAlias = str +""" +Project id +""" + + +class RepoInfo(TypedDict): + author_email: NotRequired[str | None] + """ + Email of the author of the most recent commit + """ + author_name: NotRequired[str | None] + """ + Name of the author of the most recent commit + """ + branch: NotRequired[str | None] + """ + Name of the branch the most recent commit belongs to + """ + commit: NotRequired[str | None] + """ + SHA of most recent commit + """ + commit_message: NotRequired[str | None] + """ + Most recent commit message + """ + commit_time: NotRequired[str | None] + """ + Time of the most recent commit + """ + dirty: NotRequired[bool | None] + """ + Whether or not the repo had uncommitted changes when snapshotted + """ + git_diff: NotRequired[str | None] + """ + If the repo was dirty when run, this includes the diff between the current state of the repo and the most recent commit. + """ + tag: NotRequired[str | None] + """ + Name of the tag on the most recent commit + """ + + +class SavedFunctionId1(TypedDict): + id: str + type: Literal["function"] + version: NotRequired[str] + """ + The version of the function + """ + + +class SavedFunctionId2(TypedDict): + function_type: NotRequired[FunctionTypeEnum] + name: str + type: Literal["global"] + + +SavedFunctionId: TypeAlias = SavedFunctionId1 | SavedFunctionId2 | None +""" +Optional function identifier that produced the classification +""" + + +class ScoreSummary(TypedDict): + diff: NotRequired[float] + """ + Difference in score between the current and comparison experiment + """ + improvements: int + """ + Number of improvements in the score + """ + name: str + """ + Name of the score + """ + regressions: int + """ + Number of regressions in the score + """ + score: float + """ + Average score across all examples + """ + + +SpanType: TypeAlias = ( + Literal[ + "llm", + "score", + "function", + "eval", + "task", + "tool", + "automation", + "facet", + "preprocessor", + "classifier", + "review", + ] + | None +) +""" +Type of the span, for display purposes only +""" + + +class SummarizeExperimentResponse(TypedDict): + comparison_experiment_name: NotRequired[str | None] + """ + The experiment which scores are baselined against + """ + experiment_name: str + """ + Name of the experiment + """ + experiment_url: str + """ + URL to the experiment's page in the Braintrust app + """ + metrics: NotRequired[Mapping[str, MetricSummary] | None] + """ + Summary of the experiment's metrics + """ + project_name: str + """ + Name of the project that the experiment belongs to + """ + project_url: str + """ + URL to the project's page in the Braintrust app + """ + scores: NotRequired[Mapping[str, ScoreSummary] | None] + """ + Summary of the experiment's scores + """ + + +SummarizeScores: TypeAlias = bool | None +""" +Whether to summarize the scores and metrics. If false (or omitted), only the metadata will be returned. +""" + +Version: TypeAlias = str +""" +Retrieve a snapshot of events from a past time + +The version id is essentially a filter on the latest event transaction id. You can use the `max_xact_id` returned by a past fetch as the version to reproduce that exact fetch. +""" + + +class CreateExperiment(TypedDict): + base_exp_id: NotRequired[str | None] + """ + Id of default base experiment to compare against when viewing this experiment + """ + dataset_id: NotRequired[str | None] + """ + Identifier of the linked dataset, or null if the experiment is not linked to a dataset + """ + dataset_version: NotRequired[str | None] + """ + Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified. + """ + description: NotRequired[str | None] + """ + Textual description of the experiment + """ + ensure_new: NotRequired[bool | None] + """ + Normally, creating an experiment with the same name as an existing experiment will return the existing one un-modified. But if `ensure_new` is true, registration will generate a new experiment with a unique name in case of a conflict. + """ + internal_metadata: NotRequired[InternalMetadata | None] + """ + Braintrust-controlled metadata about the experiment. + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + User-controlled metadata about the experiment + """ + name: NotRequired[str | None] + """ + Name of the experiment. Within a project, experiment names are unique + """ + parameters_id: NotRequired[str | None] + """ + Identifier of the linked saved parameters object, or null if the experiment is not linked to saved parameters + """ + parameters_version: NotRequired[str | None] + """ + Version number of the linked saved parameters object the experiment was run against. + """ + project_id: str + """ + Unique identifier for the project that the experiment belongs under + """ + public: NotRequired[bool | None] + """ + Whether or not the experiment is public. Public experiments can be viewed by anybody inside or outside the organization + """ + repo_info: NotRequired[RepoInfo | None] + tags: NotRequired[Sequence[str] | None] + """ + A list of tags for the experiment + """ + + +class Experiment(TypedDict): + base_exp_id: NotRequired[str | None] + """ + Id of default base experiment to compare against when viewing this experiment + """ + commit: NotRequired[str | None] + """ + Commit, taken directly from `repo_info.commit` + """ + created: NotRequired[str | None] + """ + Date of experiment creation + """ + dataset_id: NotRequired[str | None] + """ + Identifier of the linked dataset, or null if the experiment is not linked to a dataset + """ + dataset_version: NotRequired[str | None] + """ + Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified. + """ + deleted_at: NotRequired[str | None] + """ + Date of experiment deletion, or null if the experiment is still active + """ + description: NotRequired[str | None] + """ + Textual description of the experiment + """ + id: str + """ + Unique identifier for the experiment + """ + internal_metadata: NotRequired[InternalMetadata | None] + """ + Braintrust-controlled metadata about the experiment. + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + User-controlled metadata about the experiment + """ + name: str + """ + Name of the experiment. Within a project, experiment names are unique + """ + parameters_id: NotRequired[str | None] + """ + Identifier of the linked saved parameters object, or null if the experiment is not linked to saved parameters + """ + parameters_version: NotRequired[str | None] + """ + Version number of the linked saved parameters object the experiment was run against. + """ + project_id: str + """ + Unique identifier for the project that the experiment belongs under + """ + public: bool + """ + Whether or not the experiment is public. Public experiments can be viewed by anybody inside or outside the organization + """ + repo_info: NotRequired[RepoInfo | None] + tags: NotRequired[Sequence[str] | None] + """ + A list of tags for the experiment + """ + user_id: NotRequired[str | None] + """ + Identifies the user who created the experiment + """ + + +class Classification(TypedDict): + confidence: NotRequired[float | None] + """ + Optional confidence score for the classification + """ + id: str + """ + Stable classification identifier + """ + label: NotRequired[str] + """ + Original label of the classification item, which is useful for search and indexing purposes + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + Optional metadata associated with the classification + """ + source: NotRequired[SavedFunctionId] + + +class FeedbackExperimentEventRequest(TypedDict): + feedback: Sequence[FeedbackExperimentItem] + """ + A list of experiment feedback items + """ + + +class FetchEventsRequest(TypedDict): + cursor: NotRequired[FetchPaginationCursor | None] + limit: NotRequired[FetchLimit | None] + max_root_span_id: NotRequired[MaxRootSpanId | None] + max_xact_id: NotRequired[MaxXactId | None] + version: NotRequired[Version | None] + + +class GetExperimentResponse(TypedDict): + objects: Sequence[Experiment] + """ + A list of experiment objects + """ + + +class PatchExperiment(TypedDict): + base_exp_id: NotRequired[str | None] + """ + Id of default base experiment to compare against when viewing this experiment + """ + dataset_id: NotRequired[str | None] + """ + Identifier of the linked dataset, or null if the experiment is not linked to a dataset + """ + dataset_version: NotRequired[str | None] + """ + Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified. + """ + description: NotRequired[str | None] + """ + Textual description of the experiment + """ + internal_metadata: NotRequired[InternalMetadata | None] + """ + Braintrust-controlled metadata about the experiment. + """ + metadata: NotRequired[Mapping[str, Any] | None] + """ + User-controlled metadata about the experiment + """ + name: NotRequired[str | None] + """ + Name of the experiment. Within a project, experiment names are unique + """ + parameters_id: NotRequired[str | None] + """ + Identifier of the linked saved parameters object, or null if the experiment is not linked to saved parameters + """ + parameters_version: NotRequired[str | None] + """ + Version number of the linked saved parameters object the experiment was run against. + """ + public: NotRequired[bool | None] + """ + Whether or not the experiment is public. Public experiments can be viewed by anybody inside or outside the organization + """ + repo_info: NotRequired[RepoInfo | None] + tags: NotRequired[Sequence[str] | None] + """ + A list of tags for the experiment + """ + + +class SpanAttributes(TypedDict): + name: NotRequired[str | None] + """ + Name of the span, for display purposes only + """ + purpose: NotRequired[Literal["scorer"] | None] + """ + A special value that indicates the span was generated by a scoring automation + """ + type: NotRequired[SpanType | None] + + +class ExperimentEvent(TypedDict): + field_pagination_key: NotRequired[str | None] + """ + A stable, time-ordered key that can be used to paginate over experiment events. This field is auto-generated by Braintrust and only exists in Brainstore. + """ + field_xact_id: str + """ + The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the experiment (see the `version` parameter) + """ + audit_data: NotRequired[Sequence[Any] | None] + """ + Optional list of audit entries attached to this event + """ + classifications: NotRequired[Mapping[str, Sequence[Classification]] | None] + """ + Classifications for this event (dictionary from classification name to items) + """ + comments: NotRequired[Sequence[Any] | None] + """ + Optional list of comments attached to this event + """ + context: NotRequired[Context | None] + """ + Context is additional information about the code that produced the experiment event. It is essentially the textual counterpart to `metrics`. Use the `caller_*` attributes to track the location in code which produced the experiment event + """ + created: str + """ + The timestamp the experiment event was created + """ + error: NotRequired[Any | None] + """ + The error that occurred, if any. + """ + expected: NotRequired[Any | None] + """ + The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models + """ + experiment_id: str + """ + Unique identifier for the experiment + """ + facets: NotRequired[Mapping[str, str | None] | None] + """ + Facets for categorization (dictionary from facet id to value) + """ + id: str + """ + A unique identifier for the experiment event. If you don't provide one, Braintrust will generate one for you + """ + input: NotRequired[Any | None] + """ + The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical + """ + is_root: NotRequired[bool | None] + """ + Whether this span is a root span + """ + metadata: NotRequired[Metadata | None] + """ + A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings + """ + metrics: NotRequired[Metrics | None] + """ + Metrics are numerical measurements tracking the execution of the code that produced the experiment event. Use "start" and "end" to track the time span over which the experiment event was produced + """ + origin: NotRequired[ObjectReferenceNullish | None] + output: NotRequired[Any | None] + """ + The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question + """ + project_id: str + """ + Unique identifier for the project that the experiment belongs under + """ + root_span_id: str + """ + A unique identifier for the trace this experiment event belongs to + """ + scores: NotRequired[Mapping[str, float | None] | None] + """ + A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments + """ + span_attributes: NotRequired[SpanAttributes | None] + span_id: str + """ + A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing + """ + span_parents: NotRequired[Sequence[str] | None] + """ + An array of the parent `span_ids` of this experiment event. This should be empty for the root span of a trace, and should most often contain just one parent element for subspans + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags to log + """ + + +class FetchExperimentEventsResponse(TypedDict): + cursor: NotRequired[str | None] + """ + Pagination cursor + + Pass this string directly as the `cursor` param to your next fetch request to get the next page of results. Not provided if the returned result set is empty. + """ + events: Sequence[ExperimentEvent] + """ + A list of fetched events + """ + + +class InsertExperimentEvent(TypedDict): + field_array_delete: NotRequired[Sequence[FieldArrayDeleteItem] | None] + """ + The `_array_delete` field allows removing specific values from array fields. It is an array of objects with `path` and `delete` properties. + + For example, to remove tags "foo" and "bar" from an existing row: `{"_is_merge": true, "_array_delete": [{"path": ["tags"], "delete": ["foo", "bar"]}]}`. For nested fields like `metadata.categories`, use `[{"path": ["metadata", "categories"], "delete": ["value"]}]`. This will remove those specific values from the array while preserving others. + """ + field_is_merge: NotRequired[bool | None] + """ + The `_is_merge` field controls how the row is merged with any existing row with the same id in the DB. By default (or when set to `false`), the existing row is completely replaced by the new row. When set to `true`, the new row is deep-merged into the existing row, if one is found. If no existing row is found, the new row is inserted as is. + + For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": 5, "b": 10}}`. If we merge a new row as `{"_is_merge": true, "id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"a": 5, "b": 11, "c": 20}}`. If we replace the new row as `{"id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"b": 11, "c": 20}}` + """ + field_merge_paths: NotRequired[Sequence[Sequence[str]] | None] + """ + The `_merge_paths` field allows controlling the depth of the merge, when `_is_merge=true`. `_merge_paths` is a list of paths, where each path is a list of field names. The deep merge will not descend below any of the specified merge paths. + + For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": {"b": 10}, "c": {"d": 20}}, "output": {"a": 20}}`. If we merge a new row as `{"_is_merge": true, "_merge_paths": [["input", "a"], ["output"]], "input": {"a": {"q": 30}, "c": {"e": 30}, "bar": "baz"}, "output": {"d": 40}}`, the new row will be `{"id": "foo": "input": {"a": {"q": 30}, "c": {"d": 20, "e": 30}, "bar": "baz"}, "output": {"d": 40}}`. In this case, due to the merge paths, we have replaced `input.a` and `output`, but have still deep-merged `input` and `input.c`. + """ + field_object_delete: NotRequired[bool | None] + """ + Pass `_object_delete=true` to mark the experiment event deleted. Deleted events will not show up in subsequent fetches for this experiment + """ + field_parent_id: NotRequired[str | None] + """ + DEPRECATED: The `_parent_id` field is deprecated and should not be used. Support for `_parent_id` will be dropped in a future version of Braintrust. Log `span_id`, `root_span_id`, and `span_parents` explicitly instead. + + Use the `_parent_id` field to create this row as a subspan of an existing row. Tracking hierarchical relationships are important for tracing (see the [guide](https://www.braintrust.dev/docs/instrument) for full details). + + For example, say we have logged a row `{"id": "abc", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"_parent_id": "abc", "id": "llm_call", "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. + + If the row is being merged into an existing row, this field will be ignored. + """ + context: NotRequired[Context | None] + """ + Context is additional information about the code that produced the experiment event. It is essentially the textual counterpart to `metrics`. Use the `caller_*` attributes to track the location in code which produced the experiment event + """ + created: NotRequired[str | None] + """ + The timestamp the experiment event was created + """ + error: NotRequired[Any | None] + """ + The error that occurred, if any. + """ + expected: NotRequired[Any | None] + """ + The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models + """ + facets: NotRequired[Mapping[str, str | None] | None] + """ + Facets for categorization (dictionary from facet id to value) + """ + id: NotRequired[str | None] + """ + A unique identifier for the experiment event. If you don't provide one, Braintrust will generate one for you + """ + input: NotRequired[Any | None] + """ + The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical + """ + metadata: NotRequired[Metadata | None] + """ + A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings + """ + metrics: NotRequired[Metrics | None] + """ + Metrics are numerical measurements tracking the execution of the code that produced the experiment event. Use "start" and "end" to track the time span over which the experiment event was produced + """ + origin: NotRequired[ObjectReferenceNullish | None] + output: NotRequired[Any | None] + """ + The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question + """ + root_span_id: NotRequired[str | None] + """ + Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. + + For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. + + If the row is being merged into an existing row, this field will be ignored. + """ + scores: NotRequired[Mapping[str, float | None] | None] + """ + A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments + """ + span_attributes: NotRequired[SpanAttributes | None] + span_id: NotRequired[str | None] + """ + Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. + + For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. + + If the row is being merged into an existing row, this field will be ignored. + """ + span_parents: NotRequired[Sequence[str] | None] + """ + Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. + + For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. + + If the row is being merged into an existing row, this field will be ignored. + """ + tags: NotRequired[Sequence[str] | None] + """ + A list of tags to log + """ + + +class InsertExperimentEventRequest(TypedDict): + events: Sequence[InsertExperimentEvent] + """ + A list of experiment events to insert + """ diff --git a/py/src/braintrust/api/_generated/models/projects.py b/py/src/braintrust/api/_generated/models/projects.py index 781797a6..16355ba6 100644 --- a/py/src/braintrust/api/_generated/models/projects.py +++ b/py/src/braintrust/api/_generated/models/projects.py @@ -4,12 +4,13 @@ # datamodel-code-generator: 0.72.4 # ruff: 0.15.21 # Generator Python: 3.14 -# Content SHA-256: 9736eda7e5e0dd80b0e149a4275c4a94e8da78149af79cc692006076f109107f +# Content SHA-256: 430c31ef9773d7cd389d86e7ef06d075e8db6c74cd7e6b7ee095ec87ef435a9e -from typing import Literal, TypeAlias, TypedDict +from typing import Any, Literal, TypeAlias, TypedDict +from collections.abc import Mapping, Sequence from typing_extensions import NotRequired -from collections.abc import Sequence +from .common import FunctionTypeEnum AppLimitParam: TypeAlias = int | None """ @@ -32,41 +33,6 @@ class CreateProject(TypedDict): """ -EndingBefore: TypeAlias = str -""" -Pagination cursor id. - -For example, if the initial item in the last page you fetched had an id of `foo`, pass `ending_before=foo` to fetch the previous page. Note: you may only pass one of `starting_after` and `ending_before` -""" - - -FunctionTypeEnum: TypeAlias = ( - Literal[ - "llm", - "scorer", - "task", - "tool", - "custom_view", - "preprocessor", - "facet", - "classifier", - "tag", - "parameters", - "sandbox", - ] - | None -) -""" -The type of global function. Defaults to 'scorer'. -""" - - -Ids: TypeAlias = str | Sequence[str] -""" -Filter search results to a particular set of object IDs. To specify a list of IDs, include the query param multiple times -""" - - class NullableSavedFunctionId1(TypedDict): id: str type: Literal["function"] @@ -87,25 +53,12 @@ class NullableSavedFunctionId2(TypedDict): Default preprocessor for this project. When set, functions that use preprocessors will use this instead of their built-in default. """ - -OrgName: TypeAlias = str -""" -Filter search results to within a particular organization -""" - - ProjectIdParam: TypeAlias = str """ Project id """ -ProjectName: TypeAlias = str -""" -Name of the project to search for -""" - - class RemoteEvalSource(TypedDict): description: NotRequired[str | None] name: NotRequired[str | None] @@ -143,14 +96,6 @@ class ProjectSettings(TypedDict): """ -StartingAfter: TypeAlias = str -""" -Pagination cursor id. - -For example, if the final item in the last page you fetched had an id of `foo`, pass `starting_after=foo` to fetch the next page. Note: you may only pass one of `starting_after` and `ending_before` -""" - - class PatchProject(TypedDict): description: NotRequired[str | None] name: NotRequired[str | None] diff --git a/py/src/braintrust/api/_generated/projects.py b/py/src/braintrust/api/_generated/projects.py index 5d7f57d6..c40ab5c4 100644 --- a/py/src/braintrust/api/_generated/projects.py +++ b/py/src/braintrust/api/_generated/projects.py @@ -4,7 +4,7 @@ # datamodel-code-generator: 0.72.4 # ruff: 0.15.21 # Generator Python: 3.14 -# Content SHA-256: f3acc80e226167922acb76737729f612236d5b2d5445d1cb3af5392726fd78e2 +# Content SHA-256: 4b64219218b76122324149ec0bf037bf1254affc40f8acb2f27e0481937da4e1 """Generated Projects REST operations and resource.""" @@ -12,19 +12,8 @@ from .._service import Operation, Parameter, ResourceAPI from ..policies import RetryMode -from .models.projects import ( - AppLimitParam, - CreateProject, - EndingBefore, - GetProjectResponse, - Ids, - OrgName, - PatchProject, - Project, - ProjectIdParam, - ProjectName, - StartingAfter, -) +from .models.common import EndingBefore, Ids, OrgName, ProjectName, StartingAfter +from .models.projects import AppLimitParam, CreateProject, GetProjectResponse, PatchProject, Project, ProjectIdParam POST_PROJECT = Operation( diff --git a/py/src/braintrust/api/cassettes/test_experiment_summarize_with_real_backend[False].yaml b/py/src/braintrust/api/cassettes/test_experiment_summarize_with_real_backend[False].yaml new file mode 100644 index 00000000..f144a345 --- /dev/null +++ b/py/src/braintrust/api/cassettes/test_experiment_summarize_with_real_backend[False].yaml @@ -0,0 +1,433 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ODZlM2Y4NWItODEzYy00MTkzLTliODEtMjc1ODlhMjA4MDVl'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:29 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - ODZlM2Y4NWItODEzYy00MTkzLTliODEtMjc1ODlhMjA4MDVl + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::gdz4q-1787019509522-e2c6818d6e4c + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-generated-experiments-vcr", "project_id": + null, "org_id": "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": + "generated-experiments-base", "repo_info": {"commit": null, "branch": null, + "tag": null, "dirty": null, "author_name": null, "author_email": null, "commit_message": + null, "commit_time": null, "git_diff": null}, "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '389' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"c70d5397-daab-4085-b722-3693fd8126c3","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-generated-experiments-vcr","description":null,"created":"2026-08-18T01:12:52.122Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"067c3728-da43-4dac-9d9e-555ae9873dde","project_id":"c70d5397-daab-4085-b722-3693fd8126c3","name":"generated-experiments-base","description":null,"created":"2026-08-18T01:12:52.122Z","repo_info":{"commit":null,"branch":null,"tag":null,"dirty":null,"author_name":null,"author_email":null,"commit_message":null,"commit_time":null},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '895' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-YzY1ZDVmNjQtZTk2My00MjM5LTlhNjQtYWNlNzhjYThhZGJj'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:29 GMT + Etag: + - '"nxu3jjwuqyov"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - YzY1ZDVmNjQtZTk2My00MjM5LTlhNjQtYWNlNzhjYThhZGJj + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::5n47g-1787019509729-3d616f794da7 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-generated-experiments-vcr", "project_id": + null, "org_id": "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": + "generated-experiments-candidate", "repo_info": {"commit": null, "branch": null, + "tag": null, "dirty": null, "author_name": null, "author_email": null, "commit_message": + null, "commit_time": null, "git_diff": null}, "base_exp_id": "067c3728-da43-4dac-9d9e-555ae9873dde", + "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '449' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"c70d5397-daab-4085-b722-3693fd8126c3","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-generated-experiments-vcr","description":null,"created":"2026-08-18T01:12:52.122Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"b28b747b-63bb-4e5f-b65b-d40e1970f213","project_id":"c70d5397-daab-4085-b722-3693fd8126c3","name":"generated-experiments-candidate","description":null,"created":"2026-08-18T01:12:52.485Z","repo_info":{"commit":null,"branch":null,"tag":null,"dirty":null,"author_name":null,"author_email":null,"commit_message":null,"commit_time":null},"commit":null,"base_exp_id":"067c3728-da43-4dac-9d9e-555ae9873dde","deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '934' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZDZkZThlMmMtODZlYi00NDUzLTk0NTMtMjY3MjQ1MTRhYWY2'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:30 GMT + Etag: + - '"yxampe8u7hpy"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - ZDZkZThlMmMtODZlYi00NDUzLTk0NTMtMjY3MjQ1MTRhYWY2 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::tgm48-1787019509952-42c3ab6d47bf + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/version + response: + body: + string: '{"version":"2.11.0","date_version":"20260814","ff_version":46,"commit":"f5426a800ba05c662bd543c09093ff0a4f1ec650","deployment_mode":"lambda","deployment_type":"custom","loop_runtime_enabled":true,"brainstore_default":"force","brainstore_can_contain_row_refs":true,"skip_pg_config":"all","has_realtime_wal_bucket":true,"brainstore_wal_footer_version":"v3","brainstore_wal_use_efficient_format":true,"has_logs2":true,"brainstore_export_enabled":true,"js":true,"universal":true,"code_execution":true,"logs3_payload_max_bytes":5242880,"control_plane_telemetry":["status","memprof","usage"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:30 GMT + Via: + - 1.1 c998b153551d50824c68d959194353f6.cloudfront.net (CloudFront), 1.1 cfcfb1d8fbf5ce2b107182799687a614.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - TgrMT3Do-mf6JCNsZRVplsUHaBUlHUEiSWlNdHBE1gZkY4QRswBqqA== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a83c0f6-1a87c0c67ce3d87018240fb3;Parent=1a5c018334594d4e;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '586' + etag: + - W/"24a-yTgET7S6YwpVn3uhTcruZxcYxRo" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - CRsWjG05IAMEMCg= + x-amzn-Remapped-content-length: + - '586' + x-amzn-RequestId: + - 03cf14d9-57bc-4fa8-a90d-51c037bfba64 + x-bt-internal-trace-id: + - 6a83c0f6000000004dd2a88b0ba6477c + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"context": {"caller_filename": "/Users/abhijeetprasad/workspace/braintrust-sdk-python/py/.nox/test_core/lib/python3.14/site-packages/_pytest/python.py", + "caller_functionname": "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": + {"instrumentation": {"name": "braintrust-python-logger"}, "name": "braintrust.sdk.python", + "version": "0.34.0"}}, "created": "2026-08-18T02:18:29.771219+00:00", "expected": + "correct", "experiment_id": "067c3728-da43-4dac-9d9e-555ae9873dde", "id": "generated-experiments-base-row", + "input": {"question": "What is the answer?"}, "metrics": {"completion_tokens": + 20, "end": 1787019509.773406, "start": 1787019509.3503711}, "output": "incorrect", + "root_span_id": "396027cdc9eeedbc3270356539768f90", "scores": {"accuracy": 0}, + "span_attributes": {"exec_counter": 1, "name": "root", "type": "eval"}, "span_id": + "364615cea5006bf5", "span_parents": null},{"context": {"caller_filename": "/Users/abhijeetprasad/workspace/braintrust-sdk-python/py/.nox/test_core/lib/python3.14/site-packages/_pytest/python.py", + "caller_functionname": "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": + {"instrumentation": {"name": "braintrust-python-logger"}, "name": "braintrust.sdk.python", + "version": "0.34.0"}}, "created": "2026-08-18T02:18:29.773566+00:00", "expected": + "correct", "experiment_id": "b28b747b-63bb-4e5f-b65b-d40e1970f213", "id": "generated-experiments-candidate-row", + "input": {"question": "What is the answer?"}, "metrics": {"completion_tokens": + 10, "end": 1787019509.7762828, "start": 1787019509.771082}, "output": "correct", + "root_span_id": "2dcbb1d0968aef3a520021e5e6563cef", "scores": {"accuracy": 1}, + "span_attributes": {"exec_counter": 2, "name": "root", "type": "eval"}, "span_id": + "52e6b75f3b56d9cc", "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '1798' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["generated-experiments-base-row","generated-experiments-candidate-row"],"xact_id":"1000197706363624146"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:30 GMT + Via: + - 1.1 8757e4d26d0f26e2f05769a88e8a5ace.cloudfront.net (CloudFront), 1.1 a7af18c87ffc07d74544efce5f2b0f9c.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - Z2QgtO4MKxT2q2Zbo5aP696iCJX3j1Tza5zXL34SjRR6pZb1L0OJqQ== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a83c0f6-202af55815698faa61bacacc;Parent=30f3fa2185a7e88f;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '112' + etag: + - W/"70-6fmDoW7vV1/dj+EbWPDqco1IaIM" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CRsWkGqcIAMEndw= + x-amzn-RequestId: + - a96ebf00-be91-45a2-aef2-0da7a6281104 + x-bt-internal-trace-id: + - 6a83c0f6000000005a23d0cd687ec440 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/experiment/b28b747b-63bb-4e5f-b65b-d40e1970f213/summarize?summarize_scores=true + response: + body: + string: '{"project_name":"python-sdk-generated-experiments-vcr","project_url":"https://www.braintrust.dev/app/Braintrust%20SDKs/p/python-sdk-generated-experiments-vcr","experiment_name":"generated-experiments-candidate","experiment_url":"https://www.braintrust.dev/app/Braintrust%20SDKs/p/python-sdk-generated-experiments-vcr/experiments/generated-experiments-candidate","comparison_experiment_name":"generated-experiments-base","scores":{"accuracy":{"name":"accuracy","score":1,"diff":1,"improvements":1,"regressions":0}},"metrics":{"llm_calls":{"name":"llm_calls","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"tool_calls":{"name":"tool_calls","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"errors":{"name":"errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"llm_errors":{"name":"llm_errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"tool_errors":{"name":"tool_errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"prompt_tokens":{"name":"prompt_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cached_tokens":{"name":"prompt_cached_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_tokens":{"name":"prompt_cache_creation_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_5m_tokens":{"name":"prompt_cache_creation_5m_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_1h_tokens":{"name":"prompt_cache_creation_1h_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"completion_tokens":{"name":"completion_tokens","metric":10,"unit":"tok","diff":-10,"improvements":1,"regressions":0},"completion_reasoning_tokens":{"name":"completion_reasoning_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"total_tokens":{"name":"total_tokens","metric":10,"unit":"tok","diff":-10,"improvements":1,"regressions":0},"duration":{"name":"duration","metric":0.005200862884521484,"unit":"s","diff":-0.4178340435028076,"improvements":1,"regressions":0}}}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:32 GMT + Via: + - 1.1 8757e4d26d0f26e2f05769a88e8a5ace.cloudfront.net (CloudFront), 1.1 4f3eaee3896fb5ad2377261bd0d773c8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - dn2FfZ1hUSXW3LhCX4ynrXQsDqbY5Eq-NKe0s9_ZvpvUYA63JarzLg== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a83c0f6-308ef2a35d7b8aa7560c49d0;Parent=6abced0bcab894f5;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '2152' + etag: + - W/"868-USRSYn7wA12cyoWizlx3EmhY6H4" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CRsWoHg8IAMEl0g= + x-amzn-RequestId: + - 0502640f-80f1-4f3a-a00c-729b7beec933 + x-bt-internal-trace-id: + - 6a83c0f6000000000bedf323003124a5 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/api/cassettes/test_experiment_summarize_with_real_backend[True].yaml b/py/src/braintrust/api/cassettes/test_experiment_summarize_with_real_backend[True].yaml new file mode 100644 index 00000000..b19666e1 --- /dev/null +++ b/py/src/braintrust/api/cassettes/test_experiment_summarize_with_real_backend[True].yaml @@ -0,0 +1,495 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-M2RmZmYwMDMtODU3NC00OGEzLWE2YjYtYTRkZGJhYTRiNjhh'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:33 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - M2RmZmYwMDMtODU3NC00OGEzLWE2YjYtYTRkZGJhYTRiNjhh + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::z4sgq-1787019512971-9f3db9fd59b0 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-generated-experiments-vcr", "project_id": + null, "org_id": "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": + "generated-experiments-base", "repo_info": {"commit": null, "branch": null, + "tag": null, "dirty": null, "author_name": null, "author_email": null, "commit_message": + null, "commit_time": null, "git_diff": null}, "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '389' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"c70d5397-daab-4085-b722-3693fd8126c3","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-generated-experiments-vcr","description":null,"created":"2026-08-18T01:12:52.122Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"067c3728-da43-4dac-9d9e-555ae9873dde","project_id":"c70d5397-daab-4085-b722-3693fd8126c3","name":"generated-experiments-base","description":null,"created":"2026-08-18T01:12:52.122Z","repo_info":{"commit":null,"branch":null,"tag":null,"dirty":null,"author_name":null,"author_email":null,"commit_message":null,"commit_time":null},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '895' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZGRjOGExMzMtOWQxYi00ZjExLTliOWYtZWY4M2VkZTMyY2U0'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:33 GMT + Etag: + - '"nxu3jjwuqyov"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - ZGRjOGExMzMtOWQxYi00ZjExLTliOWYtZWY4M2VkZTMyY2U0 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::rxxkp-1787019513174-ae32305039b5 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/version + response: + body: + string: '{"version":"2.11.0","date_version":"20260814","ff_version":46,"commit":"f5426a800ba05c662bd543c09093ff0a4f1ec650","deployment_mode":"lambda","deployment_type":"custom","loop_runtime_enabled":true,"brainstore_default":"force","brainstore_can_contain_row_refs":true,"skip_pg_config":"all","has_realtime_wal_bucket":true,"brainstore_wal_footer_version":"v3","brainstore_wal_use_efficient_format":true,"has_logs2":true,"brainstore_export_enabled":true,"js":true,"universal":true,"code_execution":true,"logs3_payload_max_bytes":5242880,"control_plane_telemetry":["status","memprof","usage"]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:33 GMT + Via: + - 1.1 76283331d5eee8cee95c8c29a2095f48.cloudfront.net (CloudFront), 1.1 71eaa9eb77c2eecb57c03cdcdad1cf76.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - ShEogi-Go0xs714hgM9sTfZdbxXqoFNgeedoYFJM5g3y3EOL05l7-w== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a83c0f9-766b6b7133f4b83b6f68322f;Parent=3a7395bac9ce4c0d;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '586' + etag: + - W/"24a-yTgET7S6YwpVn3uhTcruZxcYxRo" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin + x-amz-apigw-id: + - CRsXDH7lIAMEnyw= + x-amzn-Remapped-content-length: + - '586' + x-amzn-RequestId: + - df19e58e-6294-449d-9334-4a9f61c40272 + x-bt-internal-trace-id: + - 6a83c0f9000000000334d741cd436532 + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"context": {"caller_filename": "/Users/abhijeetprasad/workspace/braintrust-sdk-python/py/.nox/test_core/lib/python3.14/site-packages/_pytest/python.py", + "caller_functionname": "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": + {"instrumentation": {"name": "braintrust-python-logger"}, "name": "braintrust.sdk.python", + "version": "0.34.0"}}, "created": "2026-08-18T02:18:33.216493+00:00", "expected": + "correct", "experiment_id": "067c3728-da43-4dac-9d9e-555ae9873dde", "id": "generated-experiments-base-row", + "input": {"question": "What is the answer?"}, "metrics": {"completion_tokens": + 20, "end": 1787019513.217482, "start": 1787019512.765023}, "output": "incorrect", + "root_span_id": "8c1946fab980d8b6507c19d8c28630a8", "scores": {"accuracy": 0}, + "span_attributes": {"exec_counter": 3, "name": "root", "type": "eval"}, "span_id": + "d79b2d5a5507536d", "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '911' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["generated-experiments-base-row"],"xact_id":"1000197706363828055"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:33 GMT + Via: + - 1.1 c998b153551d50824c68d959194353f6.cloudfront.net (CloudFront), 1.1 dcd16c430149132ea12a5783d54ff114.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - mugzOotXD58_i9QGVVWuwn6pobbEfl5kUUX3SBIGImkk9jOS0t8KzA== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a83c0f9-51bf5dd33e12e55c5a628cbe;Parent=635770d246c41d7e;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '74' + etag: + - W/"4a-sza+4/y0V1B/xKlSNSa6/UI8CLY" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CRsXEHlvIAMEfMQ= + x-amzn-RequestId: + - cd0a03d8-1921-44b5-8955-0f3f65a7c146 + x-bt-internal-trace-id: + - 6a83c0f9000000003ede5b2034d3f0e9 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "python-sdk-generated-experiments-vcr", "project_id": + null, "org_id": "5abfae3a-7aa7-4653-a9c8-b3efcb18f584", "update": true, "experiment_name": + "generated-experiments-candidate", "repo_info": {"commit": null, "branch": null, + "tag": null, "dirty": null, "author_name": null, "author_email": null, "commit_message": + null, "commit_time": null, "git_diff": null}, "base_exp_id": "067c3728-da43-4dac-9d9e-555ae9873dde", + "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '449' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"c70d5397-daab-4085-b722-3693fd8126c3","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-generated-experiments-vcr","description":null,"created":"2026-08-18T01:12:52.122Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"b28b747b-63bb-4e5f-b65b-d40e1970f213","project_id":"c70d5397-daab-4085-b722-3693fd8126c3","name":"generated-experiments-candidate","description":null,"created":"2026-08-18T01:12:52.485Z","repo_info":{"commit":null,"branch":null,"tag":null,"dirty":null,"author_name":null,"author_email":null,"commit_message":null,"commit_time":null},"commit":null,"base_exp_id":"067c3728-da43-4dac-9d9e-555ae9873dde","deleted_at":null,"dataset_id":null,"dataset_version":null,"internal_metadata":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '934' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-NjZkNGU2MzMtYWI3Zi00Yzg0LWEyYzYtMjY1YjM2ZjRhNTY5'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:34 GMT + Etag: + - '"yxampe8u7hpy"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - NjZkNGU2MzMtYWI3Zi00Yzg0LWEyYzYtMjY1YjM2ZjRhNTY5 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::jz544-1787019513906-b7b9c8e275b4 + status: + code: 200 + message: OK +- request: + body: '{"rows": [{"context": {"caller_filename": "/Users/abhijeetprasad/workspace/braintrust-sdk-python/py/.nox/test_core/lib/python3.14/site-packages/_pytest/python.py", + "caller_functionname": "pytest_pyfunc_call", "caller_lineno": 167, "span_origin": + {"instrumentation": {"name": "braintrust-python-logger"}, "name": "braintrust.sdk.python", + "version": "0.34.0"}}, "created": "2026-08-18T02:18:33.217530+00:00", "expected": + "correct", "experiment_id": "b28b747b-63bb-4e5f-b65b-d40e1970f213", "id": "generated-experiments-candidate-row", + "input": {"question": "What is the answer?"}, "metrics": {"completion_tokens": + 10, "end": 1787019513.219148, "start": 1787019513.21645}, "output": "correct", + "root_span_id": "b6860c41f21ab4104418560772539aed", "scores": {"accuracy": 1}, + "span_attributes": {"exec_counter": 4, "name": "root", "type": "eval"}, "span_id": + "cf63303e4a16ff07", "span_parents": null}], "api_version": 2}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '913' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/logs3 + response: + body: + string: '{"ids":["generated-experiments-candidate-row"],"xact_id":"1000197706363894779"}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:34 GMT + Via: + - 1.1 76283331d5eee8cee95c8c29a2095f48.cloudfront.net (CloudFront), 1.1 8e6145785e47042f882be946f6c05880.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - LEH3z1bjxWLg_MHXPzDFXjM1nWVuuRDvtagNyZcDkvwUtomkOl8Ilw== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a83c0fa-1191b6ad060dbdad4af269db;Parent=13be43660738d3b1;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '79' + etag: + - W/"4f-uj0ON/RrGhszXOxDVA5N/NFsJ4c" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CRsXKGEpoAMEUwg= + x-amzn-RequestId: + - 1430b2b4-7bbd-4608-ba9d-fe360af53207 + x-bt-internal-trace-id: + - 6a83c0fa0000000063bee9af5809ff34 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/experiment/b28b747b-63bb-4e5f-b65b-d40e1970f213/summarize?summarize_scores=true&comparison_experiment_id=067c3728-da43-4dac-9d9e-555ae9873dde + response: + body: + string: '{"project_name":"python-sdk-generated-experiments-vcr","project_url":"https://www.braintrust.dev/app/Braintrust%20SDKs/p/python-sdk-generated-experiments-vcr","experiment_name":"generated-experiments-candidate","experiment_url":"https://www.braintrust.dev/app/Braintrust%20SDKs/p/python-sdk-generated-experiments-vcr/experiments/generated-experiments-candidate","comparison_experiment_name":"generated-experiments-base","scores":{"accuracy":{"name":"accuracy","score":1,"diff":1,"improvements":1,"regressions":0}},"metrics":{"llm_calls":{"name":"llm_calls","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"tool_calls":{"name":"tool_calls","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"errors":{"name":"errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"llm_errors":{"name":"llm_errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"tool_errors":{"name":"tool_errors","metric":0,"unit":"","diff":0,"improvements":0,"regressions":0},"prompt_tokens":{"name":"prompt_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cached_tokens":{"name":"prompt_cached_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_tokens":{"name":"prompt_cache_creation_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_5m_tokens":{"name":"prompt_cache_creation_5m_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"prompt_cache_creation_1h_tokens":{"name":"prompt_cache_creation_1h_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"completion_tokens":{"name":"completion_tokens","metric":10,"unit":"tok","diff":-10,"improvements":1,"regressions":0},"completion_reasoning_tokens":{"name":"completion_reasoning_tokens","metric":0,"unit":"tok","diff":0,"improvements":0,"regressions":0},"total_tokens":{"name":"total_tokens","metric":10,"unit":"tok","diff":-10,"improvements":1,"regressions":0},"duration":{"name":"duration","metric":0.0026979446411132812,"unit":"s","diff":-0.44976115226745605,"improvements":1,"regressions":0}}}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Tue, 18 Aug 2026 02:18:36 GMT + Via: + - 1.1 561ea23d1fbb47b35216fa7040bfaa74.cloudfront.net (CloudFront), 1.1 36c050103b969d83a8b90ba7cba12542.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - yGbWRLr0f_cZHvMa1jWlNXt02FhT_nvEGTtjqQhr8itcKG1nCwvjPw== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a83c0fa-4d996f5b2884018312289e1c;Parent=282aca4dbfa90efd;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '2154' + etag: + - W/"86a-5DkNfliqQIyuovAA986xcjmrD8o" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CRsXNHsdoAMEk7w= + x-amzn-RequestId: + - e6686a98-2419-4fda-b071-cd0d5d74dd99 + x-bt-internal-trace-id: + - 6a83c0fa000000003f99cba7fbacd493 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/api/client.py b/py/src/braintrust/api/client.py index b645fc91..b77c8b87 100644 --- a/py/src/braintrust/api/client.py +++ b/py/src/braintrust/api/client.py @@ -154,9 +154,11 @@ def from_transport( return client def _initialize_services(self, api_key: str) -> None: + from ._generated.experiments import ExperimentsAPI from ._generated.projects import ProjectsAPI self.api_key = api_key + self.experiments = ExperimentsAPI(self.transport, self.router, api_key) self.projects = ProjectsAPI(self.transport, self.router, api_key) def close(self) -> None: diff --git a/py/src/braintrust/api/test_experiments.py b/py/src/braintrust/api/test_experiments.py new file mode 100644 index 00000000..16f31caa --- /dev/null +++ b/py/src/braintrust/api/test_experiments.py @@ -0,0 +1,97 @@ +import os + +import braintrust +import pytest +from braintrust.api._generated.experiments import OPERATIONS +from braintrust.api.policies import RetryMode +from braintrust.git_fields import RepoInfo +from braintrust.logger import SummarySuccess + + +def test_all_experiment_operations_have_complete_retry_classification(): + assert set(OPERATIONS) == { + "postExperiment", + "getExperiment", + "getExperimentId", + "patchExperimentId", + "deleteExperimentId", + "postExperimentIdInsert", + "postExperimentIdFetch", + "getExperimentIdFetch", + "postExperimentIdFeedback", + "getExperimentIdSummarize", + } + assert {name: operation.retry_mode for name, operation in OPERATIONS.items()} == { + "postExperiment": RetryMode.NONE, + "getExperiment": RetryMode.SAFE_READ, + "getExperimentId": RetryMode.SAFE_READ, + "patchExperimentId": RetryMode.NONE, + "deleteExperimentId": RetryMode.NONE, + "postExperimentIdInsert": RetryMode.NONE, + "postExperimentIdFetch": RetryMode.SAFE_READ, + "getExperimentIdFetch": RetryMode.SAFE_READ, + "postExperimentIdFeedback": RetryMode.NONE, + "getExperimentIdSummarize": RetryMode.SAFE_READ, + } + + +def _api_key(): + return os.environ.get("BRAINTRUST_API_KEY", "sk-dummy-for-vcr-replay") + + +@pytest.mark.vcr +@pytest.mark.parametrize("explicit_comparison", [False, True]) +def test_experiment_summarize_with_real_backend(explicit_comparison): + project_name = "python-sdk-generated-experiments-vcr" + base = braintrust.init( + project=project_name, + experiment="generated-experiments-base", + api_key=_api_key(), + update=True, + set_current=False, + repo_info=RepoInfo(), + ) + candidate = braintrust.init( + project=project_name, + experiment="generated-experiments-candidate", + api_key=_api_key(), + base_experiment_id=base.id, + update=True, + set_current=False, + repo_info=RepoInfo(), + ) + comparison_input = {"question": "What is the answer?"} + base.log( + id="generated-experiments-base-row", + input=comparison_input, + output="incorrect", + expected="correct", + scores={"accuracy": 0}, + metrics={"completion_tokens": 20}, + ) + candidate.log( + id="generated-experiments-candidate-row", + input=comparison_input, + output="correct", + expected="correct", + scores={"accuracy": 1}, + metrics={"completion_tokens": 10}, + ) + + summary = candidate.summarize(comparison_experiment_id=base.id if explicit_comparison else None) + + assert isinstance(summary.comparison, SummarySuccess) + assert summary.comparison_experiment_name == base.name + assert summary.project_name == project_name + assert summary.experiment_name == candidate.name + assert summary.scores["accuracy"].score == 1 + assert summary.scores["accuracy"].diff == 1 + assert summary.scores["accuracy"].improvements == 1 + assert summary.metrics["completion_tokens"].metric == 10 + assert summary.metrics["completion_tokens"].diff == -10 + assert summary.metrics["completion_tokens"].unit == "tok" + serialized = summary.as_dict() + assert serialized["comparison"]["status"] == "success" + assert serialized["scores"]["accuracy"]["score"] == 1 + assert serialized["metrics"]["completion_tokens"]["metric"] == 10 + assert f"{candidate.name} compared to {base.name}" in str(summary) diff --git a/py/src/braintrust/api/test_generated_models.py b/py/src/braintrust/api/test_generated_models.py index 4f29a3b7..329b318f 100644 --- a/py/src/braintrust/api/test_generated_models.py +++ b/py/src/braintrust/api/test_generated_models.py @@ -19,12 +19,16 @@ def test_import_braintrust_is_lazy_about_generated_api_modules(): def test_generated_models_import_on_supported_python(): + from braintrust.api._generated import experiments as experiment_bindings + from braintrust.api._generated import models from braintrust.api._generated import projects as project_bindings - from braintrust.api._generated.models import projects - assert is_typeddict(projects.Project) - assert projects.ProjectIdParam is str - assert get_type_hints(project_bindings.ProjectsAPI.get_project)["return"] is projects.GetProjectResponse + assert is_typeddict(models.Experiment) + assert is_typeddict(models.Project) + assert models.ExperimentIdParam is str + assert models.ProjectIdParam is str + assert get_type_hints(experiment_bindings.ExperimentsAPI.get_experiment)["return"] is models.GetExperimentResponse + assert get_type_hints(project_bindings.ProjectsAPI.get_project)["return"] is models.GetProjectResponse def test_generated_package_content_is_installed(): @@ -32,7 +36,10 @@ def test_generated_package_content_is_installed(): assert generated.joinpath("__init__.py").is_file() assert generated.joinpath("models", "__init__.py").is_file() + assert generated.joinpath("models", "common.py").is_file() + assert generated.joinpath("models", "experiments.py").is_file() assert generated.joinpath("models", "projects.py").is_file() + assert generated.joinpath("experiments.py").is_file() assert generated.joinpath("projects.py").is_file() @@ -42,4 +49,4 @@ def test_rest_and_logging_type_surfaces_have_reviewed_overlap(): overlap = set(generated_types.__all__) & set(types.__all__) - assert overlap == {"Project"} + assert overlap == {"Experiment", "Project"} diff --git a/py/src/braintrust/api/test_projects.py b/py/src/braintrust/api/test_projects.py index 71ce6418..a813755d 100644 --- a/py/src/braintrust/api/test_projects.py +++ b/py/src/braintrust/api/test_projects.py @@ -1,121 +1,7 @@ -import contextlib -import json import os -from urllib.parse import urlsplit import pytest -from braintrust.api import BraintrustClient, BraintrustHTTPError, BraintrustOpenApiClient -from braintrust.api._test_server import scripted_server -from braintrust.api._transport import Transport - - -@contextlib.contextmanager -def project_server(): - project = { - "id": "project-id", - "org_id": "test-org-id", - "name": "project/name", - "new_backend_field": {"preserved": True}, - } - - def respond(method, request_path, _body, _headers): - path = urlsplit(request_path).path - response = ( - {"objects": [project], "new_list_field": ["preserved"]} - if path == "/v1/project" and method == "GET" - else project - ) - return 200, {"Content-Type": "application/json"}, json.dumps(response).encode() - - with scripted_server(respond) as server: - yield server - - -def _client_for_server(url, transport): - return BraintrustOpenApiClient(api_key="test-key", api_url=url, transport=transport) - - -def test_generated_operation_rejects_undeclared_success_statuses_as_http_errors(): - response_body = b'{"queued": true}' - headers = {"Content-Type": "application/json", "x-request-id": "unexpected-status-id"} - with scripted_server([(202, headers, response_body)]) as (url, _): - projects = _client_for_server(url, Transport()).projects - with pytest.raises(BraintrustHTTPError) as exc_info: - projects.get_project() - - assert exc_info.value.status_code == 202 - assert exc_info.value.response_body == response_body.decode() - assert exc_info.value.request_id == "unexpected-status-id" - assert exc_info.value.request_id_header == "x-request-id" - - -def test_projects_use_generated_bindings_with_exact_wire_shape_and_additive_responses(): - with project_server() as (url, handler): - projects = _client_for_server(url, Transport()).projects - create_project = {"name": "project/name", "description": "created"} - created = projects.post_project(body=create_project) - listed = projects.get_project( - limit=2, - ids=["project-id", "other/id"], - project_name="project/name", - ) - projects.get_project_id("project/id") - projects.patch_project_id("project/id", body={"description": "updated"}) - projects.delete_project_id("project/id") - - assert create_project == {"name": "project/name", "description": "created"} - assert created["new_backend_field"] == {"preserved": True} - assert listed["new_list_field"] == ["preserved"] - - assert handler.requests == [ - ( - "POST", - "/v1/project", - b'{"name": "project/name", "description": "created"}', - "Bearer test-key", - ), - ( - "GET", - "/v1/project?limit=2&ids=project-id&ids=other%2Fid&project_name=project%2Fname", - b"", - "Bearer test-key", - ), - ("GET", "/v1/project/project%2Fid", b"", "Bearer test-key"), - ("PATCH", "/v1/project/project%2Fid", b'{"description": "updated"}', "Bearer test-key"), - ("DELETE", "/v1/project/project%2Fid", b"", "Bearer test-key"), - ] - - -def test_projects_allow_explicit_organization_overrides(): - with project_server() as (url, handler): - projects = _client_for_server(url, Transport()).projects - projects.post_project(body={"name": "project/name", "org_name": "other org"}) - projects.get_project(org_name="other org") - - assert handler.requests == [ - ( - "POST", - "/v1/project", - b'{"name": "project/name", "org_name": "other org"}', - "Bearer test-key", - ), - ("GET", "/v1/project?org_name=other%20org", b"", "Bearer test-key"), - ] - - -def test_projects_percent_encode_query_values(): - with project_server() as (url, handler): - projects = _client_for_server(url, Transport()).projects - projects.get_project(project_name="R&D#research/v1+beta?x=y", org_name="org&name#one + two") - - assert handler.requests == [ - ( - "GET", - "/v1/project?project_name=R%26D%23research%2Fv1%2Bbeta%3Fx%3Dy&org_name=org%26name%23one%20%2B%20two", - b"", - "Bearer test-key", - ) - ] +from braintrust.api import BraintrustClient def _api_key(): @@ -125,15 +11,15 @@ def _api_key(): @pytest.mark.vcr def test_projects_end_to_end_with_real_backend(): project_name = "python-sdk-generated-projects-vcr" + create_project = { + "name": project_name, + "description": "created by the Python SDK VCR test", + } + with BraintrustClient(api_key=_api_key()) as client: discovery = client.auth.login() - created = client.openapi.projects.post_project( - body={ - "name": project_name, - "description": "created by the Python SDK VCR test", - "org_name": discovery.organization.name, - } - ) + create_project["org_name"] = discovery.organization.name + created = client.openapi.projects.post_project(body=create_project) listed = client.openapi.projects.get_project( project_name=project_name, org_name=discovery.organization.name, @@ -144,6 +30,11 @@ def test_projects_end_to_end_with_real_backend(): ) deleted = client.openapi.projects.delete_project_id(created["id"]) + assert create_project == { + "name": project_name, + "description": "created by the Python SDK VCR test", + "org_name": discovery.organization.name, + } assert created["name"] == project_name assert [project["id"] for project in listed["objects"]] == [created["id"]] assert fetched["id"] == created["id"] diff --git a/py/src/braintrust/api/types/__init__.py b/py/src/braintrust/api/types/__init__.py index 3365668a..ac3d4b75 100644 --- a/py/src/braintrust/api/types/__init__.py +++ b/py/src/braintrust/api/types/__init__.py @@ -1,11 +1,38 @@ """Public types for the synchronous Braintrust REST API.""" -from .._generated.models.projects import CreateProject, GetProjectResponse, PatchProject, Project +from .._generated.models import ( + CreateExperiment, + CreateProject, + Experiment, + FeedbackExperimentEventRequest, + FeedbackResponseSchema, + FetchEventsRequest, + FetchExperimentEventsResponse, + GetExperimentResponse, + GetProjectResponse, + InsertEventsResponse, + InsertExperimentEventRequest, + PatchExperiment, + PatchProject, + Project, + SummarizeExperimentResponse, +) __all__ = [ + "CreateExperiment", "CreateProject", + "Experiment", + "FeedbackExperimentEventRequest", + "FeedbackResponseSchema", + "FetchEventsRequest", + "FetchExperimentEventsResponse", + "GetExperimentResponse", "GetProjectResponse", + "InsertEventsResponse", + "InsertExperimentEventRequest", + "PatchExperiment", "PatchProject", "Project", + "SummarizeExperimentResponse", ] diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py index c73a5e77..e848d365 100644 --- a/py/src/braintrust/framework.py +++ b/py/src/braintrust/framework.py @@ -37,6 +37,7 @@ Metadata, ScoreSummary, Span, + SummarySuccess, parent_context, start_span, stringify_exception, @@ -1952,8 +1953,7 @@ def build_local_summary( project_url=None, experiment_url=None, comparison_experiment_name=None, - scores=avg_scores, - metrics={}, + comparison=SummarySuccess(scores=avg_scores, metrics={}), ) diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index 35c2a5db..5759902d 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -45,7 +45,7 @@ from .api._transport import HTTPConnection from .api._transport import RetryRequestExceptionsAdapter as RetryRequestExceptionsAdapter from .api.client import BraintrustClient, BraintrustOpenApiClient -from .api.errors import BraintrustHTTPError +from .api.errors import BraintrustAPIError, BraintrustHTTPError from .bt_json import bt_dumps, bt_safe_deep_copy from .db_fields import ( AUDIT_METADATA_FIELD, @@ -2274,7 +2274,7 @@ def summarize(summarize_scores: bool = True, comparison_experiment_id: str | Non Summarize the current experiment, including the scores (compared to the closest reference experiment) and metadata. :param summarize_scores: Whether to summarize the scores. If False, only the metadata will be returned. - :param comparison_experiment_id: The experiment to compare against. If None, the most recent experiment on the comparison_commit will be used. + :param comparison_experiment_id: The experiment to compare against. If None, the backend uses the experiment's stored base experiment, then the most recent experiment in the same project. :returns: `ExperimentSummary` """ eprint( @@ -4230,7 +4230,7 @@ def summarize( Summarize the experiment, including the scores (compared to the closest reference experiment) and metadata. :param summarize_scores: Whether to summarize the scores. If False, only the metadata will be returned. - :param comparison_experiment_id: The experiment to compare against. If None, the most recent experiment on the origin's main branch will be used. + :param comparison_experiment_id: The experiment to compare against. If None, the backend uses the experiment's stored base experiment, then the most recent experiment in the same project. :returns: `ExperimentSummary` """ # Flush our events to the API, and to the data warehouse, to ensure that the link we print @@ -4241,49 +4241,38 @@ def summarize( project_url = f"{state.app_public_url}/app/{encode_uri_component(state.org_name)}/p/{encode_uri_component(self.project.name)}" experiment_url = f"{project_url}/experiments/{encode_uri_component(self.name)}" - score_summary = {} - metric_summary = {} comparison_experiment_name = None - if summarize_scores: - # Get the comparison experiment - if comparison_experiment_id is None: - base_experiment = self.fetch_base_experiment() - if base_experiment: - comparison_experiment_id = base_experiment.id - comparison_experiment_name = base_experiment.name - else: - try: - comparison_experiment = state.api_conn().get_json(f"v1/experiment/{comparison_experiment_id}") - comparison_experiment_name = comparison_experiment.get("name") - except Exception: - pass - + if not summarize_scores: + comparison: SummaryResult = SummarySkipped(reason="Score summarization was disabled") + else: try: - summary_items = state.api_conn().get_json( - "experiment-comparison2", - args={ - "experiment_id": self.id, - "base_experiment_id": comparison_experiment_id, - }, + summary_items = state.api_client().experiments.get_experiment_id_summarize( + self.id, + summarize_scores=True, + comparison_experiment_id=comparison_experiment_id, ) - except Exception as e: + except BraintrustAPIError as e: _logger.warning( - f"Failed to fetch experiment scores and metrics: {e}\n\nView complete results in Braintrust or run experiment.summarize() again." + "Failed to fetch experiment scores and metrics: %s\n\n" + "View complete results in Braintrust or run experiment.summarize() again.", + e, ) - summary_items = {} - - score_items = summary_items.get("scores", {}) - metric_items = summary_items.get("metrics", {}) + comparison = SummarySkipped(reason="Experiment comparison could not be retrieved") + else: + comparison_experiment_name = summary_items.get("comparison_experiment_name") + score_items = summary_items.get("scores") or {} + metric_items = summary_items.get("metrics") or {} - longest_score_name = max(len(k) for k in score_items.keys()) if score_items else 0 - score_summary = { - k: ScoreSummary(_longest_score_name=longest_score_name, **v) for (k, v) in score_items.items() - } + longest_score_name = max((len(k) for k in score_items), default=0) + score_summary = { + k: ScoreSummary(_longest_score_name=longest_score_name, **v) for (k, v) in score_items.items() + } - longest_metric_name = max(len(k) for k in metric_items.keys()) if metric_items else 0 - metric_summary = { - k: MetricSummary(_longest_metric_name=longest_metric_name, **v) for (k, v) in metric_items.items() - } + longest_metric_name = max((len(k) for k in metric_items), default=0) + metric_summary = { + k: MetricSummary(_longest_metric_name=longest_metric_name, **v) for (k, v) in metric_items.items() + } + comparison = SummarySuccess(scores=score_summary, metrics=metric_summary) return ExperimentSummary( project_name=self.project.name, @@ -4293,8 +4282,7 @@ def summarize( project_url=project_url, experiment_url=experiment_url, comparison_experiment_name=comparison_experiment_name, - scores=score_summary, - metrics=metric_summary, + comparison=comparison, ) def export(self) -> str: @@ -5862,9 +5850,34 @@ def __str__(self): ) +@dataclasses.dataclass(frozen=True) +class SummarySuccess(SerializableDataClass): + """A successfully retrieved experiment comparison.""" + + scores: dict[str, ScoreSummary] + metrics: dict[str, MetricSummary] + status: Literal["success"] = dataclasses.field(default="success", init=False) + + +@dataclasses.dataclass(frozen=True) +class SummarySkipped(SerializableDataClass): + """An experiment comparison that was skipped or could not be retrieved.""" + + reason: str + status: Literal["skipped"] = dataclasses.field(default="skipped", init=False) + + +SummaryResult = SummarySuccess | SummarySkipped + + @dataclasses.dataclass class ExperimentSummary(SerializableDataClass): - """Summary of an experiment's scores and metadata.""" + """Summary of an experiment's metadata and comparison result. + + Use :attr:`comparison` to consume scores and metrics. The top-level + :attr:`scores` and :attr:`metrics` properties exist only for backwards + compatibility. + """ project_name: str """Name of the project that the experiment belongs to.""" @@ -5880,29 +5893,95 @@ class ExperimentSummary(SerializableDataClass): """URL to the experiment's page in the Braintrust app.""" comparison_experiment_name: str | None """The experiment scores are baselined against.""" - scores: dict[str, ScoreSummary] - """Summary of the experiment's scores.""" - metrics: dict[str, MetricSummary] - """Summary of the experiment's metrics.""" + comparison: SummaryResult + """The primary result to inspect when consuming an experiment summary. + + A successful comparison contains the score and metric maps. A skipped + comparison explains why comparison data is unavailable, so callers do not + mistake it for a successful summary with no scores. + """ + + @property + def scores(self) -> dict[str, ScoreSummary]: + """Deprecated. Use ``comparison.scores`` after checking its status.""" + return self.comparison.scores if isinstance(self.comparison, SummarySuccess) else {} + + @property + def metrics(self) -> dict[str, MetricSummary]: + """Deprecated. Use ``comparison.metrics`` after checking its status.""" + return self.comparison.metrics if isinstance(self.comparison, SummarySuccess) else {} + + def as_dict(self): + serialized = super().as_dict() + # Preserve the legacy serialized fields until a future major version. + serialized["scores"] = {name: dataclasses.asdict(score) for name, score in self.scores.items()} + serialized["metrics"] = {name: dataclasses.asdict(metric) for name, metric in self.metrics.items()} + return serialized + + @classmethod + def from_dict_deep(cls, d: dict): + def deserialize_scores(values: Mapping[str, Any]) -> dict[str, ScoreSummary]: + longest_name = max((len(name) for name in values), default=0) + return { + name: value + if isinstance(value, ScoreSummary) + else ScoreSummary.from_dict({"name": name, "_longest_score_name": longest_name, **value}) + for name, value in values.items() + } + + def deserialize_metrics(values: Mapping[str, Any]) -> dict[str, MetricSummary]: + longest_name = max((len(name) for name in values), default=0) + return { + name: value + if isinstance(value, MetricSummary) + else MetricSummary.from_dict({"name": name, "_longest_metric_name": longest_name, **value}) + for name, value in values.items() + } + + raw_comparison = d.get("comparison") + if isinstance(raw_comparison, (SummarySuccess, SummarySkipped)): + comparison: SummaryResult = raw_comparison + elif raw_comparison is None: + comparison = SummarySuccess( + scores=deserialize_scores(d.get("scores", {})), + metrics=deserialize_metrics(d.get("metrics", {})), + ) + elif isinstance(raw_comparison, Mapping): + status = raw_comparison.get("status") + if status == "success": + comparison = SummarySuccess( + scores=deserialize_scores(raw_comparison.get("scores", {})), + metrics=deserialize_metrics(raw_comparison.get("metrics", {})), + ) + elif status == "skipped": + comparison = SummarySkipped(reason=raw_comparison["reason"]) + else: + raise ValueError(f"Unknown experiment summary comparison status: {status!r}") + else: + raise TypeError("Experiment summary comparison must be a mapping") + + return cls.from_dict({**d, "comparison": comparison}) def __str__(self): comparison_line = "" if self.comparison_experiment_name: comparison_line = f"""{self.experiment_name} compared to {self.comparison_experiment_name}:\n""" - return ( - f"""\n=========================SUMMARY=========================\n{comparison_line}""" - + "\n".join([str(score) for score in self.scores.values()]) - + ("\n\n" if self.scores else "") - + "\n".join([str(metric) for metric in self.metrics.values()]) - + ("\n\n" if self.metrics else "") - + ( - textwrap.dedent( - f"""\ + if isinstance(self.comparison, SummarySkipped): + result = f"Summary skipped: {self.comparison.reason}\n\n" + else: + result = ( + "\n".join([str(score) for score in self.comparison.scores.values()]) + + ("\n\n" if self.comparison.scores else "") + + "\n".join([str(metric) for metric in self.comparison.metrics.values()]) + + ("\n\n" if self.comparison.metrics else "") + ) + return f"""\n=========================SUMMARY=========================\n{comparison_line}{result}""" + ( + textwrap.dedent( + f"""\ See results for {self.experiment_name} at {self.experiment_url}""" - ) - if self.experiment_url is not None - else "" ) + if self.experiment_url is not None + else "" ) diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py index 4e7994f5..74dfb510 100644 --- a/py/src/braintrust/test_framework.py +++ b/py/src/braintrust/test_framework.py @@ -309,33 +309,6 @@ def exact_match(input_value, output, expected): ) -def test_experiment_summarize_resolves_explicit_comparison_name(with_memory_logger, with_simulate_login): - exp = init_test_exp("test-evaluator", "test-project") - mock_conn = MagicMock() - - def get_json(path, args=None): - if path == "v1/experiment/base-exp-id": - return {"name": "base-exp"} - if path == "experiment-comparison2": - return {"scores": {}, "metrics": {}} - raise AssertionError(f"Unexpected get_json call: {path}, {args}") - - mock_conn.get_json.side_effect = get_json - - with patch.object(exp.state, "api_conn", return_value=mock_conn): - summary = exp.summarize(comparison_experiment_id="base-exp-id") - - assert summary.comparison_experiment_name == "base-exp" - mock_conn.get_json.assert_any_call("v1/experiment/base-exp-id") - mock_conn.get_json.assert_any_call( - "experiment-comparison2", - args={ - "experiment_id": "test-evaluator", - "base_experiment_id": "base-exp-id", - }, - ) - - @pytest.mark.asyncio @pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed") async def test_run_evaluator_exposes_validated_parameter_values_to_hooks(): @@ -774,6 +747,7 @@ def purpose_scorer(input_value, output, expected): scores=[purpose_scorer], experiment_name="test-scorer-purpose", metadata=None, + summarize_scores=False, ) # Create experiment so spans get logged @@ -934,6 +908,7 @@ async def test_classifier_spans_are_logged(with_memory_logger, with_simulate_log ], experiment_name="test-classifier-span", metadata=None, + summarize_scores=False, ) exp = init_test_exp("test-classifier-span", "test-project") diff --git a/py/src/braintrust/type_tests/test_api_client.py b/py/src/braintrust/type_tests/test_api_client.py index 6ef3343d..91e1639d 100644 --- a/py/src/braintrust/type_tests/test_api_client.py +++ b/py/src/braintrust/type_tests/test_api_client.py @@ -3,7 +3,19 @@ from typing import TYPE_CHECKING from braintrust.api import BraintrustClient, BraintrustOpenApiClient, EndpointRouter, RequestTarget -from braintrust.api.types import CreateProject, GetProjectResponse, PatchProject, Project +from braintrust.api.types import ( + CreateExperiment, + CreateProject, + Experiment, + FetchEventsRequest, + FetchExperimentEventsResponse, + GetExperimentResponse, + GetProjectResponse, + PatchExperiment, + PatchProject, + Project, + SummarizeExperimentResponse, +) if TYPE_CHECKING: @@ -23,6 +35,25 @@ updated_project: Project = openapi_client.projects.patch_project_id(project["id"], body=patch_project) deleted_project: Project = openapi_client.projects.delete_project_id(project["id"]) + create_experiment: CreateExperiment = {"project_id": project["id"], "name": "typed-experiment"} + experiment: Experiment = openapi_client.experiments.post_experiment(body=create_experiment) + experiments: GetExperimentResponse = openapi_client.experiments.get_experiment( + ids=[experiment["id"]], project_id=project["id"] + ) + fetched_experiment: Experiment = openapi_client.experiments.get_experiment_id(experiment["id"]) + patch_experiment: PatchExperiment = {"description": "updated"} + updated_experiment: Experiment = openapi_client.experiments.patch_experiment_id( + experiment["id"], body=patch_experiment + ) + fetch_request: FetchEventsRequest = {"limit": 10} + fetched_events: FetchExperimentEventsResponse = openapi_client.experiments.post_experiment_id_fetch( + experiment["id"], body=fetch_request + ) + summary: SummarizeExperimentResponse = openapi_client.experiments.get_experiment_id_summarize( + experiment["id"], summarize_scores=True + ) + deleted_experiment: Experiment = openapi_client.experiments.delete_experiment_id(experiment["id"]) + def test_api_client_router() -> None: router = EndpointRouter(app_url="https://app.example.com", api_url="https://api.example.com") diff --git a/py/src/braintrust/type_tests/test_experiment_summary.py b/py/src/braintrust/type_tests/test_experiment_summary.py new file mode 100644 index 00000000..1fe0bd2f --- /dev/null +++ b/py/src/braintrust/type_tests/test_experiment_summary.py @@ -0,0 +1,73 @@ +"""Static and runtime checks for structured experiment summaries.""" + +from typing import TYPE_CHECKING + +from braintrust import ExperimentSummary, MetricSummary, ScoreSummary, SummarySkipped, SummarySuccess + + +if TYPE_CHECKING: + + def check_summary_narrowing(summary: ExperimentSummary) -> None: + comparison = summary.comparison + if comparison.status == "success": + scores: dict[str, ScoreSummary] = comparison.scores + metrics: dict[str, MetricSummary] = comparison.metrics + else: + reason: str = comparison.reason + + legacy_scores: dict[str, ScoreSummary] = summary.scores + legacy_metrics: dict[str, MetricSummary] = summary.metrics + + +def _summary(comparison: SummarySuccess | SummarySkipped) -> ExperimentSummary: + return ExperimentSummary( + project_name="project", + project_id="project-id", + experiment_id="experiment-id", + experiment_name="experiment", + project_url="https://example.com/project", + experiment_url="https://example.com/experiment", + comparison_experiment_name=None, + comparison=comparison, + ) + + +def test_structured_experiment_summary_public_types() -> None: + skipped = _summary(SummarySkipped(reason="disabled")) + success = SummarySuccess(scores={}, metrics={}) + + assert isinstance(skipped.comparison, SummarySkipped) + assert skipped.comparison.reason == "disabled" + assert success.status == "success" + + +def test_structured_experiment_summary_deep_deserialization() -> None: + success = _summary( + SummarySuccess( + scores={ + "accuracy": ScoreSummary( + name="accuracy", + score=0.9, + improvements=2, + regressions=1, + diff=0.1, + _longest_score_name=len("accuracy"), + ) + }, + metrics={}, + ) + ) + success_payload = success.as_dict() + + restored_success = ExperimentSummary.from_dict_deep(success_payload) + restored_skipped = ExperimentSummary.from_dict_deep(_summary(SummarySkipped(reason="disabled")).as_dict()) + legacy_payload = {key: value for key, value in success_payload.items() if key != "comparison"} + restored_legacy = ExperimentSummary.from_dict_deep(legacy_payload) + + assert isinstance(restored_success.comparison, SummarySuccess) + assert restored_success.as_dict() == success_payload + assert str(restored_success) == str(success) + assert isinstance(restored_skipped.comparison, SummarySkipped) + assert restored_skipped.comparison.reason == "disabled" + assert isinstance(restored_legacy.comparison, SummarySuccess) + assert restored_legacy.scores["accuracy"].score == 0.9 diff --git a/py/tests/api_codegen/conftest.py b/py/tests/api_codegen/conftest.py index d3659c64..c3bbf88d 100644 --- a/py/tests/api_codegen/conftest.py +++ b/py/tests/api_codegen/conftest.py @@ -14,6 +14,7 @@ def codegen_config(): config = copy.deepcopy(load_config(CONFIG_PATH)) config["endpoint_generator"]["generated_tags"] = ["Widgets"] + config["endpoint_generator"]["safe_reads"] = [] config["endpoint_generator"]["idempotent_writes"] = [] return config diff --git a/py/tests/api_codegen/test_generation.py b/py/tests/api_codegen/test_generation.py index 499beea0..ca6fbcb0 100644 --- a/py/tests/api_codegen/test_generation.py +++ b/py/tests/api_codegen/test_generation.py @@ -30,6 +30,26 @@ def test_generation_selects_generated_tag_regardless_of_tag_order(tmp_path, code assert "def get_widget(" in (generated / "widgets.py").read_text() +def test_declarative_post_reads_use_safe_read_retry_mode(tmp_path, codegen_config, minimal_spec): + minimal_spec["paths"]["/widgets"] = { + "post": { + "operationId": "postWidgetFetch", + "tags": ["Widgets"], + "responses": { + "200": { + "description": "OK", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Widget"}}}, + } + }, + } + } + codegen_config["endpoint_generator"]["safe_reads"] = ["postWidgetFetch"] + + generated = _generate(tmp_path, "safe-read", codegen_config, minimal_spec) + + assert "retry_mode=RetryMode.SAFE_READ" in (generated / "widgets.py").read_text() + + def test_idempotent_writes_use_idempotent_write_retry_mode(tmp_path, codegen_config, minimal_spec): minimal_spec["paths"]["/widgets"] = { "post": { @@ -51,8 +71,24 @@ def test_idempotent_writes_use_idempotent_write_retry_mode(tmp_path, codegen_con assert "retry_mode=RetryMode.IDEMPOTENT_WRITE" in bindings -def test_multiple_generated_resource_tags_require_explicit_partitioning(tmp_path, codegen_config, minimal_spec): +def test_multiple_generated_resources_partition_shared_models_deterministically( + tmp_path, codegen_config, minimal_spec +): spec = copy.deepcopy(minimal_spec) + spec["components"]["schemas"]["Widget"]["properties"]["details"] = {"$ref": "#/components/schemas/WidgetDetails"} + spec["components"]["schemas"]["WidgetDetails"] = { + "type": "object", + "properties": {"count": {"type": "integer"}}, + "required": ["count"], + } + spec["components"]["schemas"]["Gadget"] = { + "type": "object", + "properties": { + "widget": {"$ref": "#/components/schemas/Widget"}, + "serial": {"type": "string"}, + }, + "required": ["widget", "serial"], + } spec["paths"]["/gadgets"] = { "get": { "operationId": "getGadget", @@ -60,15 +96,26 @@ def test_multiple_generated_resource_tags_require_explicit_partitioning(tmp_path "responses": { "200": { "description": "OK", - "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Widget"}}}, + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Gadget"}}}, } }, } } codegen_config["endpoint_generator"]["generated_tags"] = ["Widgets", "Gadgets"] - with pytest.raises(CodegenError, match="exactly one generated OpenAPI tag"): - _generate(tmp_path, "multiple-model-resources", codegen_config, spec) + generated = _generate(tmp_path, "multiple-model-resources", codegen_config, spec) + + common_models = (generated / "models" / "common.py").read_text() + assert "class Widget(TypedDict):" in common_models + assert "class WidgetDetails(TypedDict):" in common_models + assert "from .common import" not in common_models + assert "class Gadget(TypedDict):" in (generated / "models" / "gadgets.py").read_text() + assert "from .common import Widget" in (generated / "models" / "gadgets.py").read_text() + assert "from .models.common import Widget" in (generated / "widgets.py").read_text() + assert "from .models.gadgets import Gadget" in (generated / "gadgets.py").read_text() + model_exports = (generated / "models" / "__init__.py").read_text() + assert "from .common import Widget, WidgetDetails" in model_exports + assert "from .gadgets import Gadget" in model_exports def test_unreachable_models_are_omitted_but_transitive_references_are_kept(tmp_path, codegen_config, minimal_spec): diff --git a/py/tests/api_codegen/test_validation.py b/py/tests/api_codegen/test_validation.py index bbcd079c..3bec3288 100644 --- a/py/tests/api_codegen/test_validation.py +++ b/py/tests/api_codegen/test_validation.py @@ -129,6 +129,35 @@ def test_media_types_and_success_statuses_are_validated(minimal_spec, codegen_co validate_spec(spec, codegen_config) +def test_safe_reads_must_reference_generated_post_operations(minimal_spec, codegen_config): + codegen_config["endpoint_generator"]["safe_reads"] = ["missingOperation"] + + with pytest.raises(CodegenError, match="safe_reads.*missingOperation"): + validate_spec(minimal_spec, codegen_config) + + codegen_config["endpoint_generator"]["safe_reads"] = ["getWidget"] + with pytest.raises(CodegenError, match="safe_reads must reference POST operations.*getWidget"): + validate_spec(minimal_spec, codegen_config) + + operation = minimal_spec["paths"]["/widgets/{widget_id}"].pop("get") + operation["operationId"] = "patchWidget" + minimal_spec["paths"]["/widgets/{widget_id}"]["patch"] = operation + codegen_config["endpoint_generator"]["safe_reads"] = ["patchWidget"] + with pytest.raises(CodegenError, match="safe_reads must reference POST operations.*patchWidget"): + validate_spec(minimal_spec, codegen_config) + + +def test_retry_mode_allowlists_cannot_overlap(minimal_spec, codegen_config): + operation = minimal_spec["paths"]["/widgets/{widget_id}"].pop("get") + operation["operationId"] = "postWidgetFetch" + minimal_spec["paths"]["/widgets/{widget_id}"]["post"] = operation + codegen_config["endpoint_generator"]["safe_reads"] = ["postWidgetFetch"] + codegen_config["endpoint_generator"]["idempotent_writes"] = ["postWidgetFetch"] + + with pytest.raises(CodegenError, match="cannot appear in both.*safe_reads.*idempotent_writes"): + validate_spec(minimal_spec, codegen_config) + + def test_idempotent_writes_must_reference_generated_operations(minimal_spec, codegen_config): codegen_config["endpoint_generator"]["idempotent_writes"] = ["missingOperation"] @@ -232,10 +261,21 @@ def test_malformed_specs_and_configs_raise_actionable_errors(minimal_spec, codeg with pytest.raises(CodegenError, match="must be an object"): validate_spec(spec, codegen_config) - for key in ("supported_request_media_types", "supported_response_media_types", "supported_success_statuses"): + for key in ( + "safe_reads", + "idempotent_writes", + "supported_request_media_types", + "supported_response_media_types", + "supported_success_statuses", + ): broken = copy.deepcopy(codegen_config) del broken["endpoint_generator"][key] - with pytest.raises(CodegenError, match=f"endpoint_generator.{key} must be a non-empty list"): + message = ( + f"endpoint_generator.{key} must be a unique list" + if key in {"safe_reads", "idempotent_writes"} + else f"endpoint_generator.{key} must be a non-empty list" + ) + with pytest.raises(CodegenError, match=message): validate_spec(minimal_spec, broken) - with pytest.raises(CodegenError, match=f"endpoint_generator.{key} must be a non-empty list"): + with pytest.raises(CodegenError, match=message): validate_config(broken, check_installed_tools=False)