diff --git a/openapi/README.md b/openapi/README.md index fa2064c2..a41583c0 100644 --- a/openapi/README.md +++ b/openapi/README.md @@ -3,8 +3,8 @@ `spec.json` is a committed snapshot of the public specification from [`braintrustdata/braintrust-openapi`](https://github.com/braintrustdata/braintrust-openapi). `config.json` pins the full upstream commit, snapshot SHA-256, generator tools, generator flags, and -explicit endpoint exclusions. The generator scripts live in `py/scripts/`. Builds and package installation use the committed generated source and never fetch or run -code generation. +selected endpoint tags. The generator scripts live in `py/scripts/`. Builds and package installation +use the committed generated source and never fetch or run code generation. From `py/`, validate and regenerate the private models offline with: @@ -13,7 +13,15 @@ make generate-api-client make check-api-client-codegen ``` -The check regenerates into a temporary directory and does not modify the worktree. +The check regenerates into a temporary directory and does not modify the worktree. Endpoint bindings +are rolled out explicitly through `endpoint_generator.generated_tags`. The current rollout supports +exactly one selected OpenAPI tag and emits its operation registry and resource class together in +`projects.py`, with reachable types in `models/projects.py`; unreachable models are omitted. Add +explicit cross-resource model partitioning before selecting a second tag. Public resource method and +inline response type names are derived mechanically from each `operationId`, and generated methods +forward request fields and parameters without implicit defaults. Writes that are safe to retry are +listed declaratively in `endpoint_generator.idempotent_writes`; reads and all other writes use +mechanical retry defaults. To fetch the configured upstream commit explicitly: @@ -29,5 +37,5 @@ BRAINTRUST_OPENAPI_ROOT=../../braintrust-openapi make fetch-openapi-spec ``` To update the snapshot, first update the commit and SHA-256 in `config.json`, then fetch, regenerate, -and review both the upstream spec diff and generated model diff. Fix specification defects upstream -rather than adding Python-side normalization beyond CORS `OPTIONS` removal and configured exclusions. +and review both the upstream spec diff and generated model diff. Only operations selected through +`endpoint_generator.generated_tags` and their reachable schemas are validated and generated. diff --git a/openapi/config.json b/openapi/config.json index b043e3d5..0a3fe36c 100644 --- a/openapi/config.json +++ b/openapi/config.json @@ -21,7 +21,7 @@ "--use-generic-container-types", "--use-field-description", "--strict-nullable", - "--parent-scoped-naming", + "--naming-strategy=primary-first", "--no-use-closed-typed-dict", "--disable-future-imports", "--formatters=ruff-format" @@ -29,19 +29,12 @@ }, "endpoint_generator": { "schema_version": 1, - "skip_tags": { - "Proxy": { - "reason": "Proxy endpoints stream provider-specific payloads and remain on the specialized proxy path.", - "operation_ids": [ - "proxychatCompletions", - "proxycompletions", - "proxyauto", - "proxyembeddings", - "proxycredentials", - "proxy{path+}" - ] - } - }, + "generated_tags": [ + "Projects" + ], + "idempotent_writes": [ + "postProject" + ], "supported_success_statuses": [ "200", "201", @@ -52,8 +45,7 @@ "application/json" ], "supported_response_media_types": [ - "application/json", - "text/plain" + "application/json" ] } } diff --git a/py/scripts/openapi_codegen.py b/py/scripts/openapi_codegen.py index 7c5faea2..e50d8508 100644 --- a/py/scripts/openapi_codegen.py +++ b/py/scripts/openapi_codegen.py @@ -5,13 +5,14 @@ import hashlib import importlib.metadata import json +import keyword import os import re import shutil import subprocess import sys from pathlib import Path -from typing import Any, Dict, FrozenSet, Iterator, List, Mapping, NamedTuple, Sequence, Set, Tuple +from typing import Any, Dict, Iterator, List, Mapping, NamedTuple, Sequence, Set, Tuple REPO_ROOT = Path(__file__).resolve().parents[2] @@ -47,17 +48,33 @@ class CodegenError(RuntimeError): class ValidationReport(NamedTuple): operation_count: int - options_operation_count: int schema_count: int - skip_ids: FrozenSet[str] def __str__(self) -> str: - return ( - f"{self.operation_count} supported operations, " - f"{self.options_operation_count} CORS OPTIONS operations removed, " - f"{len(self.skip_ids)} explicitly skipped operations, " - f"{self.schema_count} schemas" - ) + return f"{self.operation_count} selected operations, {self.schema_count} reachable schemas" + + +class GeneratedParameter(NamedTuple): + argument_name: str + name: str + location: str + type_name: str + required: bool + + +class GeneratedOperation(NamedTuple): + operation_id: str + constant_name: str + method: str + path: str + tag: str + parameters: Tuple[GeneratedParameter, ...] + request_body_type: str | None + request_body_required: bool + response_type: str | None + success_statuses: Tuple[int, ...] + json_success_statuses: Tuple[int, ...] + retry_mode: str def load_config(path: Path = CONFIG_PATH) -> Dict[str, Any]: @@ -130,79 +147,184 @@ def validate_spec(spec: Mapping[str, Any], config: Mapping[str, Any]) -> Validat components = spec.get("components", {}) if not isinstance(components, dict) or not isinstance(components.get("schemas", {}), dict): raise CodegenError("OpenAPI spec components.schemas must be an object") - schemas = components.get("schemas", {}) - - _validate_refs(spec) - operations = list(_iter_operations(spec)) - # Uniqueness is checked before the skip set is resolved: skipped and supported operations share - # one operationId namespace, so a collision between them would otherwise drop the supported - # operation from the normalized spec without any error. - _validate_unique_operation_ids(operations) + endpoint = _endpoint_config(config) - skip_ids = _validate_skip_set(operations, endpoint) + all_operations = list(_iter_operations(spec)) + _validate_unique_operation_ids(all_operations) + operations = _selected_operations(all_operations, endpoint["generated_tags"]) + reference_roots = [] + for _, _, _, operation, path_item in operations: + reference_roots.append(operation) + reference_roots.extend(path_item.get("parameters", [])) + _validate_refs(reference_roots, spec) generated_names: Dict[str, str] = {} - supported_count = 0 - options_count = 0 + method_identifiers: Dict[str, str] = {} + constant_identifiers: Dict[str, str] = {} for method, path, operation_id, operation, path_item in operations: - if method == "options": - if operation.get("tags") != ["CORS"]: - raise CodegenError( - f"OPTIONS {path} is not tagged only as CORS and cannot be removed during normalization" - ) - options_count += 1 - continue - if operation_id in skip_ids: - continue if not operation_id or not OPERATION_ID_RE.fullmatch(operation_id): raise CodegenError(f"Operation {method.upper()} {path} has an invalid operationId: {operation_id!r}") tags = operation.get("tags") - if not isinstance(tags, list) or len(tags) != 1 or not isinstance(tags[0], str) or not tags[0].strip(): - raise CodegenError(f"Operation {operation_id!r} must have exactly one usable tag") + if not isinstance(tags, list) or not tags or not all(isinstance(tag, str) and tag.strip() for tag in tags): + raise CodegenError(f"Operation {operation_id!r} must have usable tags") + if len(_generated_operation_tags(operation, endpoint["generated_tags"])) != 1: + raise CodegenError(f"Operation {operation_id!r} must have exactly one generated OpenAPI tag") generated_name = _python_type_name(operation_id) previous = generated_names.setdefault(generated_name, operation_id) if previous != operation_id: raise CodegenError( f"Inline operation name collision: {previous!r} and {operation_id!r} both generate {generated_name!r}" ) + for namespace, identifier, seen in ( + ("method", _snake_case(operation_id), method_identifiers), + ("constant", _snake_case(operation_id).upper(), constant_identifiers), + ): + previous = seen.setdefault(identifier, operation_id) + if previous != operation_id: + raise CodegenError( + f"Generated operation identifier collision: {previous!r} and {operation_id!r} " + f"both emit {namespace} {identifier!r}" + ) _validate_operation_media(operation_id, operation, endpoint, spec) - _validate_path_parameters(operation_id, path, path_item, operation, spec) - supported_count += 1 + _validate_parameters(operation_id, path, path_item, operation, spec) + _validate_selected_operations(operations, endpoint) + operation_ids = {operation_id for _, _, operation_id, _, _ in operations} + selected_spec = _slice_model_spec(spec, operation_ids) + schemas = selected_spec.get("components", {}).get("schemas", {}) _validate_component_names(schemas) - _validate_json_values_and_types(spec) - return ValidationReport(supported_count, options_count, len(schemas), frozenset(skip_ids)) + _validate_json_values_and_types(selected_spec) + return ValidationReport(len(operations), len(schemas)) -def normalize_spec(spec: Mapping[str, Any], skip_ids: FrozenSet[str]) -> Dict[str, Any]: - """Remove only CORS OPTIONS and the exact configured skip set.""" - normalized = copy.deepcopy(spec) - for path, path_item in list(normalized["paths"].items()): - for method, operation in list(path_item.items()): - lower_method = method.lower() - if lower_method in HTTP_METHODS and ( - lower_method == "options" or operation.get("operationId") in skip_ids - ): - del path_item[method] - if not any(key.lower() in HTTP_METHODS for key in path_item): - del normalized["paths"][path] - return normalized +def _generated_operation_tags(operation: Mapping[str, Any], generated_tags: Sequence[str]) -> List[str]: + tags = operation.get("tags") + if not isinstance(tags, list): + return [] + selected_tags = set(generated_tags) + return [tag for tag in tags if isinstance(tag, str) and tag in selected_tags] + + +def _selected_operations( + operations: Sequence[Tuple[str, str, Any, Mapping[str, Any], Mapping[str, Any]]], + generated_tags: Sequence[str], +) -> List[Tuple[str, str, Any, Mapping[str, Any], Mapping[str, Any]]]: + return [ + operation_entry + for operation_entry in operations + if operation_entry[0] != "options" and _generated_operation_tags(operation_entry[3], generated_tags) + ] + + +def _slice_model_spec(spec: Mapping[str, Any], operation_ids: Set[str]) -> Dict[str, Any]: + """Keep selected operations and the transitive component closure they reference.""" + selected_paths: Dict[str, Any] = {} + for path, path_item in spec.get("paths", {}).items(): + selected_item = { + key: copy.deepcopy(value) + for key, value in path_item.items() + if key.lower() not in HTTP_METHODS or value.get("operationId") in operation_ids + } + if any(key.lower() in HTTP_METHODS for key in selected_item): + selected_paths[path] = selected_item + + selected_components: Dict[str, Dict[str, Any]] = {} + seen_components: Set[Tuple[str, str]] = set() + + def collect_components(value: Any) -> None: + if isinstance(value, dict): + reference = value.get("$ref") + component_key = _component_key_from_ref(reference) if isinstance(reference, str) else None + if component_key is not None and component_key not in seen_components: + seen_components.add(component_key) + component_type, component_name = component_key + components = spec.get("components", {}) + component_group = components.get(component_type, {}) + if not isinstance(component_group, dict) or component_name not in component_group: + raise CodegenError(f"Unresolved OpenAPI reference {reference!r}") + component = component_group[component_name] + selected_components.setdefault(component_type, {})[component_name] = copy.deepcopy(component) + collect_components(component) + for key, child in value.items(): + if key != "$ref": + collect_components(child) + elif isinstance(value, list): + for child in value: + collect_components(child) + + collect_components(selected_paths) + model_spec = {key: copy.deepcopy(spec[key]) for key in ("openapi", "info", "jsonSchemaDialect") if key in spec} + model_spec["paths"] = selected_paths + model_spec["components"] = selected_components + return model_spec + + +def _component_key_from_ref(reference: str) -> Tuple[str, str] | None: + prefix = "#/components/" + if not reference.startswith(prefix): + return None + parts = reference[len(prefix) :].split("/", 2) + if len(parts) < 2: + return None + return tuple(part.replace("~1", "/").replace("~0", "~") for part in parts[:2]) + + +def _with_inline_models( + spec: Mapping[str, Any], inline_models: Sequence[Tuple[str, Mapping[str, Any]]] +) -> Dict[str, Any]: + """Expose named inline response schemas to the existing model generator.""" + model_spec = copy.deepcopy(spec) + schemas = model_spec.setdefault("components", {}).setdefault("schemas", {}) + generated_names = {_python_type_name(name): name for name in schemas} + for name, schema in inline_models: + generated_name = _python_type_name(name) + existing_name = generated_names.get(generated_name) + if existing_name is not None: + raise CodegenError( + f"Inline response model {name!r} collides with component schema {existing_name!r}; " + f"both generate Python type {generated_name!r}" + ) + schemas[name] = copy.deepcopy(schema) + generated_names[generated_name] = name + return model_spec + + +def _single_model_module(operations: Sequence[GeneratedOperation]) -> str: + tags = {operation.tag for operation in operations} + if len(tags) != 1: + raise CodegenError( + "Model generation currently requires exactly one generated OpenAPI tag; " + "add explicit cross-resource model partitioning before enabling another tag" + ) + return _snake_case(next(iter(tags))) + + +def _model_modules(spec: Mapping[str, Any], module: str) -> Dict[str, str]: + schemas = spec.get("components", {}).get("schemas", {}) + return {_python_type_name(name): module for name in schemas} def generate_tree(output_root: Path, config: Mapping[str, Any], spec: Mapping[str, Any]) -> ValidationReport: validate_config(config) report = validate_spec(spec, config) - normalized = normalize_spec(spec, report.skip_ids) + operations, inline_models = _collect_generated_operations(spec, config) + selected_spec = _slice_model_spec(spec, {operation.operation_id for operation in operations}) + model_spec = _with_inline_models(selected_spec, inline_models) + model_module = _single_model_module(operations) + model_modules = _model_modules(model_spec, model_module) output_root.mkdir(parents=True, exist_ok=True) - normalized_path = output_root.parent / "normalized-spec.json" - normalized_path.write_text( - json.dumps(normalized, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n", encoding="utf-8" + selected_spec_path = output_root.parent / "selected-spec.json" + selected_spec_path.write_text( + json.dumps(model_spec, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n", encoding="utf-8" ) try: - _generate_models(normalized_path, output_root / "models.py", config) + _generate_models(selected_spec_path, output_root / "models" / f"{model_module}.py", config) finally: - normalized_path.unlink(missing_ok=True) + selected_spec_path.unlink(missing_ok=True) _write_generated_file(output_root / "__init__.py", _GENERATED_INIT_BODY, config) + _write_generated_file(output_root / "models" / "__init__.py", '"""Generated private model types."""\n', config) + resource_files = _generate_resources(output_root, operations, model_modules, config) + _format_generated_files(resource_files) return report @@ -278,8 +400,8 @@ def _prune_empty_directories(root: Path) -> None: def _generate_models(spec_path: Path, output_path: Path, config: Mapping[str, Any]) -> None: - placeholder = "CONTENT_HASH_PLACEHOLDER" - header = _generated_header(config, placeholder).rstrip() + output_path.parent.mkdir(parents=True, exist_ok=True) + header = _generated_header(config, "CONTENT_HASH_PLACEHOLDER").rstrip() command = [ sys.executable, "-m", @@ -297,19 +419,12 @@ def _generate_models(spec_path: Path, output_path: Path, config: Mapping[str, An except subprocess.CalledProcessError as exc: detail = exc.stderr.strip() or exc.stdout.strip() or str(exc) raise CodegenError(f"datamodel-code-generator failed: {detail}") from exc - # datamodel-code-generator's own `--formatters=ruff-format` pass is not a fixed point; running - # the pinned ruff again is what makes the committed output stable. - subprocess.run( - [sys.executable, "-m", "ruff", "format", str(output_path)], check=True, capture_output=True, text=True - ) - generated = output_path.read_text(encoding="utf-8") - marker = f"# Content SHA-256: {placeholder}" - if marker not in generated: - raise CodegenError("Generated models did not contain the expected content hash marker") - body = generated.split(marker, 1)[1].lstrip("\n") - content_hash = hashlib.sha256(body.encode()).hexdigest() - # Substituting the hash only rewrites characters inside a comment, so the file stays formatted. - _write_checked(output_path, generated.replace(placeholder, content_hash, 1)) + if not output_path.is_file(): + raise CodegenError("datamodel-code-generator did not emit the model module") + model_paths = [output_path] + # datamodel-code-generator's own formatter pass is not a fixed point; one pinned Ruff pass over + # the complete module tree makes the committed output stable and finalizes each content hash. + _format_generated_files(model_paths) def _write_generated_file(path: Path, body: str, config: Mapping[str, Any]) -> None: @@ -344,6 +459,355 @@ def _generated_header(config: Mapping[str, Any], content_hash: str) -> str: ''' +def _validate_selected_operations( + operations: Sequence[Tuple[str, str, Any, Mapping[str, Any], Mapping[str, Any]]], + endpoint: Mapping[str, Any], +) -> None: + supported = {operation_id: operation for _, _, operation_id, operation, _ in operations} + configured_tags = set(endpoint["generated_tags"]) + actual_tags = { + tag + for operation in supported.values() + for tag in _generated_operation_tags(operation, endpoint["generated_tags"]) + } + missing_tags = configured_tags - actual_tags + if missing_tags: + raise CodegenError(f"endpoint_generator.generated_tags contains unknown tags: {sorted(missing_tags)}") + + idempotent_writes = set(endpoint["idempotent_writes"]) + stale_idempotent_writes = idempotent_writes - set(supported) + if stale_idempotent_writes: + operation_id = sorted(stale_idempotent_writes)[0] + raise CodegenError(f"endpoint_generator.idempotent_writes references non-generated operation {operation_id!r}") + non_writes = sorted( + operation_id + for method, _, operation_id, _, _ in operations + if operation_id in idempotent_writes and method in {"get", "head"} + ) + if non_writes: + raise CodegenError(f"endpoint_generator.idempotent_writes references read operation {non_writes[0]!r}") + + +def _operation_retry_mode(method: str, operation_id: str, idempotent_writes: Set[str]) -> str: + if method in {"get", "head"}: + return "SAFE_READ" + if operation_id in idempotent_writes: + return "IDEMPOTENT_WRITE" + return "NONE" + + +def _collect_generated_operations( + spec: Mapping[str, Any], config: Mapping[str, Any] +) -> Tuple[List[GeneratedOperation], List[Tuple[str, Mapping[str, Any]]]]: + endpoint = _endpoint_config(config) + idempotent_writes = set(endpoint["idempotent_writes"]) + operations: List[GeneratedOperation] = [] + inline_models: Dict[str, Mapping[str, Any]] = {} + for method, path, operation_id, operation, path_item in _iter_operations(spec): + operation_generated_tags = _generated_operation_tags(operation, endpoint["generated_tags"]) + if method == "options" or not operation_generated_tags: + continue + parameters = _operation_parameters(path_item, operation, spec) + request_body_type, request_body_required = _operation_request_body(operation, spec) + response_type, statuses, json_statuses, inline_schema = _operation_response(operation_id, operation, spec) + if inline_schema is not None: + inline_model_name = _python_type_name(operation_id + "Response") + previous_schema = inline_models.setdefault(inline_model_name, inline_schema) + if previous_schema != inline_schema: + raise CodegenError(f"Inline response model {inline_model_name!r} has conflicting schemas") + operations.append( + GeneratedOperation( + operation_id=operation_id, + constant_name=_snake_case(operation_id).upper(), + method=method.upper(), + path=path, + tag=operation_generated_tags[0], + parameters=tuple(parameters), + request_body_type=request_body_type, + request_body_required=request_body_required, + response_type=response_type, + success_statuses=statuses, + json_success_statuses=json_statuses, + retry_mode=_operation_retry_mode(method, operation_id, idempotent_writes), + ) + ) + return operations, list(inline_models.items()) + + +def _operation_parameters( + path_item: Mapping[str, Any], operation: Mapping[str, Any], spec: Mapping[str, Any] +) -> List[GeneratedParameter]: + by_key: Dict[Tuple[str, str], Mapping[str, Any]] = {} + for raw_parameter in [*path_item.get("parameters", []), *operation.get("parameters", [])]: + parameter = _resolve_object(raw_parameter, spec) + location = parameter.get("in") + name = parameter.get("name") + if not isinstance(location, str) or not isinstance(name, str): + raise CodegenError("Generated operation parameter must have string in and name fields") + by_key[(location, name)] = parameter + generated = [] + for (location, name), parameter in by_key.items(): + generated.append( + GeneratedParameter( + argument_name=_python_argument_name(name), + name=name, + location=location, + type_name=_schema_annotation(parameter.get("schema", {}), spec), + required=bool(parameter.get("required", False)), + ) + ) + return generated + + +def _operation_request_body(operation: Mapping[str, Any], spec: Mapping[str, Any]) -> Tuple[str | None, bool]: + request_body = operation.get("requestBody") + if request_body is None: + return None, False + request_body = _resolve_object(request_body, spec) + media = request_body["content"]["application/json"] + return _schema_annotation(media.get("schema", {}), spec), bool(request_body.get("required", False)) + + +def _operation_response( + operation_id: str, + operation: Mapping[str, Any], + spec: Mapping[str, Any], +) -> Tuple[str | None, Tuple[int, ...], Tuple[int, ...], Mapping[str, Any] | None]: + successes = [ + (int(status), response) for status, response in operation["responses"].items() if str(status).startswith("2") + ] + successes.sort(key=lambda item: item[0]) + response_types: List[str | None] = [] + json_statuses: List[int] = [] + inline_schema: Mapping[str, Any] | None = None + for status, raw_response in successes: + response = _resolve_object(raw_response, spec) + content = response.get("content", {}) + if not content: + response_types.append(None) + continue + media_type = "application/json" if "application/json" in content else sorted(content)[0] + json_statuses.append(status) + schema = content[media_type].get("schema", {}) + if "$ref" in schema: + response_type = _schema_annotation(schema, spec) + elif schema: + response_type = _python_type_name(operation_id + "Response") + if inline_schema is not None and inline_schema != schema: + raise CodegenError(f"Operation {operation_id!r} has conflicting inline success response schemas") + inline_schema = schema + else: + response_type = "Any" + response_types.append(response_type) + unique_types = list(dict.fromkeys(response_types)) + if len(unique_types) == 1: + response_type = unique_types[0] + elif len(unique_types) == 2 and None in unique_types: + response_type = f"{next(type_name for type_name in unique_types if type_name is not None)} | None" + else: + response_type = "Any" + return ( + response_type, + tuple(status for status, _ in successes), + tuple(json_statuses), + inline_schema if response_type != "Any" else None, + ) + + +def _schema_annotation(schema: Mapping[str, Any], spec: Mapping[str, Any]) -> str: + if "$ref" in schema: + return _python_type_name(str(schema["$ref"]).rsplit("/", 1)[-1]) + resolved = _resolve_object(schema, spec) + if "oneOf" in resolved or "anyOf" in resolved: + choices = resolved.get("oneOf", resolved.get("anyOf", [])) + annotation = " | ".join(_schema_annotation(choice, spec) for choice in choices) or "Any" + elif "allOf" in resolved: + choices = resolved["allOf"] + annotation = _schema_annotation(choices[0], spec) if len(choices) == 1 else "Any" + else: + schema_type = resolved.get("type") + if schema_type == "array": + annotation = f"Sequence[{_schema_annotation(resolved.get('items', {}), spec)}]" + elif schema_type == "object": + annotation = "Mapping[str, Any]" + else: + annotation = { + "boolean": "bool", + "integer": "int", + "number": "float", + "string": "str", + }.get(schema_type, "Any") + if resolved.get("nullable") is True and "None" not in annotation: + annotation += " | None" + return annotation + + +def _generate_resources( + root: Path, + operations: Sequence[GeneratedOperation], + model_modules: Mapping[str, str], + config: Mapping[str, Any], +) -> List[Path]: + by_tag: Dict[str, List[GeneratedOperation]] = {} + for operation in operations: + by_tag.setdefault(operation.tag, []).append(operation) + + generated_paths = [] + for tag, tag_operations in sorted(by_tag.items()): + resource_path = root / f"{_snake_case(tag)}.py" + _write_generated_file(resource_path, _resource_module_source(tag, tag_operations, model_modules), config) + generated_paths.append(resource_path) + return generated_paths + + +def _resource_module_source( + tag: str, operations: Sequence[GeneratedOperation], model_modules: Mapping[str, str] +) -> str: + annotation_names: Set[str] = set() + for operation in operations: + for type_name in [ + operation.request_body_type, + operation.response_type, + *(parameter.type_name for parameter in operation.parameters), + ]: + if type_name: + annotation_names.update(re.findall(r"\b[A-Z][A-Za-z0-9_]*\b", type_name)) + collections_imports = sorted(annotation_names & {"Mapping", "Sequence"}) + typing_imports = sorted(annotation_names & {"Any", "Literal"}) + model_type_names = annotation_names - {"Any", "Literal", "Mapping", "None", "Sequence"} + model_imports: Dict[str, Set[str]] = {} + for type_name in model_type_names: + module = model_modules.get(type_name) + if module is None: + raise CodegenError(f"Generated resource {tag!r} references unknown model {type_name!r}") + model_imports.setdefault(module, set()).add(type_name) + + lines = [f'"""Generated {tag} REST operations and resource."""', ""] + if collections_imports: + lines.extend([f"from collections.abc import {', '.join(collections_imports)}", ""]) + lines.append(f"from typing import {', '.join([*typing_imports, 'cast'])}") + lines.extend( + [ + "", + "from .._service import Operation, Parameter, ResourceAPI", + "from ..policies import RetryMode", + ] + ) + for module, names in sorted(model_imports.items()): + lines.append(f"from .models.{module} import {', '.join(sorted(names))}") + for operation in operations: + lines.extend(["", "", *_operation_definition_source(operation)]) + lines.extend(["", "", "OPERATIONS = {"]) + lines.extend(f" {operation.operation_id!r}: {operation.constant_name}," for operation in operations) + lines.append("}") + lines.extend(["", "", f"class {_python_type_name(tag)}API(ResourceAPI):", f' """Generated {tag} REST API."""']) + for operation in operations: + lines.extend(["", *_resource_method_source(operation)]) + return "\n".join(lines) + "\n" + + +def _operation_definition_source(operation: GeneratedOperation) -> List[str]: + lines = [ + f"{operation.constant_name} = Operation(", + f" operation_id={operation.operation_id!r},", + f" method={operation.method!r},", + f" path={operation.path!r},", + " parameters=(", + ] + for parameter in operation.parameters: + lines.extend( + [ + " Parameter(", + f" argument_name={parameter.argument_name!r},", + f" name={parameter.name!r},", + f" location={parameter.location!r},", + f" required={parameter.required!r},", + " ),", + ] + ) + lines.extend( + [ + " ),", + f" has_request_body={operation.request_body_type is not None!r},", + f" success_statuses={operation.success_statuses!r},", + f" json_success_statuses={operation.json_success_statuses!r},", + f" retry_mode=RetryMode.{operation.retry_mode},", + ")", + ] + ) + return lines + + +def _resource_method_source(operation: GeneratedOperation) -> List[str]: + required_parameters = [parameter for parameter in operation.parameters if parameter.required] + optional_parameters = [parameter for parameter in operation.parameters if not parameter.required] + arguments = ["self"] + arguments.extend(f'{parameter.argument_name}: "{parameter.type_name}"' for parameter in required_parameters) + keyword_arguments = [ + f'{parameter.argument_name}: "{parameter.type_name} | None" = None' for parameter in optional_parameters + ] + if operation.request_body_type is not None: + default = "" if operation.request_body_required else " = None" + body_type = ( + operation.request_body_type if operation.request_body_required else f"{operation.request_body_type} | None" + ) + keyword_arguments.append(f'body: "{body_type}"{default}') + if keyword_arguments: + arguments.append("*") + arguments.extend(keyword_arguments) + return_type = operation.response_type or "None" + lines = [ + f" def {_snake_case(operation.operation_id)}(", + *(f" {argument}," for argument in arguments), + f' ) -> "{return_type}":', + ] + call_arguments = [operation.constant_name] + for location, keyword_name in (("path", "path_parameters"), ("query", "query_parameters")): + parameters = [parameter for parameter in operation.parameters if parameter.location == location] + if parameters: + values = ", ".join(f"{parameter.argument_name!r}: {parameter.argument_name}" for parameter in parameters) + call_arguments.append(f"{keyword_name}={{{values}}}") + if operation.request_body_type is not None: + call_arguments.append("body=body") + lines.append(f' return cast("{return_type}", self.execute(') + lines.extend(f" {argument}," for argument in call_arguments) + lines.append(" ))") + return lines + + +def _format_generated_files(paths: Sequence[Path]) -> None: + if not paths: + return + subprocess.run( + [sys.executable, "-m", "ruff", "format", *(str(path) for path in paths)], + check=True, + capture_output=True, + text=True, + ) + for path in paths: + generated = path.read_text(encoding="utf-8") + marker_match = re.search(r"^# Content SHA-256: ([^\n]+)$", generated, flags=re.MULTILINE) + if marker_match is None: + raise CodegenError(f"Formatted generated file {path} lost its content hash") + body_after_marker = generated[marker_match.end() :].lstrip("\n") + content_hash = hashlib.sha256(body_after_marker.encode()).hexdigest() + _write_checked(path, generated[: marker_match.start(1)] + content_hash + generated[marker_match.end(1) :]) + + +def _python_argument_name(value: str) -> str: + result = re.sub(r"\W", "_", value) + if not result or result[0].isdigit(): + result = "_" + result + if keyword.iskeyword(result): + result += "_" + return result + + +def _snake_case(value: str) -> str: + value = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", value) + return re.sub(r"[^A-Za-z0-9]+", "_", value).strip("_").lower() + + def _iter_operations( spec: Mapping[str, Any], ) -> Iterator[Tuple[str, str, Any, Mapping[str, Any], Mapping[str, Any]]]: @@ -363,8 +827,20 @@ def _endpoint_config(config: Mapping[str, Any]) -> Mapping[str, Any]: endpoint = config.get("endpoint_generator") if not isinstance(endpoint, dict) or endpoint.get("schema_version") != 1: raise CodegenError("Unsupported endpoint_generator schema_version; expected 1") - if not isinstance(endpoint.get("skip_tags"), dict): - raise CodegenError("endpoint_generator.skip_tags must be an object") + generated_tags = endpoint.get("generated_tags") + if ( + not isinstance(generated_tags, list) + or not all(isinstance(value, str) and value for value in generated_tags) + or len(generated_tags) != len(set(generated_tags)) + ): + raise CodegenError("endpoint_generator.generated_tags must be a unique list of non-empty strings") + idempotent_writes = endpoint.get("idempotent_writes") + if ( + not isinstance(idempotent_writes, list) + or not all(isinstance(value, str) and value for value in idempotent_writes) + or len(idempotent_writes) != len(set(idempotent_writes)) + ): + raise CodegenError("endpoint_generator.idempotent_writes must be a unique list of non-empty strings") for key in ("supported_request_media_types", "supported_response_media_types", "supported_success_statuses"): values = endpoint.get(key) if not isinstance(values, list) or not values or not all(isinstance(value, str) for value in values): @@ -385,55 +861,18 @@ def _validate_unique_operation_ids( raise CodegenError(f"Duplicate operationId {operation_id!r} on {previous} and {location}") -def _validate_skip_set( - operations: Sequence[Tuple[str, str, Any, Mapping[str, Any], Mapping[str, Any]]], endpoint: Mapping[str, Any] -) -> Set[str]: - skip_tags = endpoint["skip_tags"] - configured_ids: Set[str] = set() - operation_tags: Dict[str, Set[str]] = {} - for method, path, operation_id, operation, _ in operations: - if method == "options": - continue - if operation_id is not None: - operation_tags[operation_id] = set(operation.get("tags", [])) - for tag, skip_config in skip_tags.items(): - if not isinstance(tag, str) or not isinstance(skip_config, dict): - raise CodegenError("Each endpoint_generator.skip_tags entry must be an object keyed by a tag") - reason = skip_config.get("reason") - ids = skip_config.get("operation_ids") - if ( - not isinstance(reason, str) - or not reason.strip() - or not isinstance(ids, list) - or not all(isinstance(value, str) for value in ids) - ): - raise CodegenError(f"Skip tag {tag!r} must have a reason and an operation_ids list") - if len(ids) != len(set(ids)): - raise CodegenError(f"Skip tag {tag!r} contains duplicate operation IDs") - actual_ids = {operation_id for operation_id, tags in operation_tags.items() if tag in tags} - expected_ids = set(ids) - if actual_ids != expected_ids: - missing = sorted(actual_ids - expected_ids) - stale = sorted(expected_ids - actual_ids) - raise CodegenError( - f"Skip tag {tag!r} does not match the spec exactly; unlisted={missing}, stale={stale}. " - "Update the explicit skip set and review each operation." - ) - overlap = configured_ids & expected_ids - if overlap: - raise CodegenError(f"Operations occur in more than one skip tag: {', '.join(sorted(overlap))}") - configured_ids.update(expected_ids) - return configured_ids - - -def _validate_refs(spec: Mapping[str, Any]) -> None: - for value in _walk_values(spec): - if not isinstance(value, dict) or "$ref" not in value: +def _validate_refs(value: Any, spec: Mapping[str, Any], seen_refs: Set[str] | None = None) -> None: + seen_refs = seen_refs if seen_refs is not None else set() + for child in _walk_values(value): + if not isinstance(child, dict) or "$ref" not in child: continue - reference = value["$ref"] + reference = child["$ref"] if not isinstance(reference, str) or not reference.startswith("#/"): raise CodegenError(f"Only local OpenAPI references are supported, got {reference!r}") - _resolve_ref(reference, spec) + resolved = _resolve_ref(reference, spec) + if reference not in seen_refs: + seen_refs.add(reference) + _validate_refs(resolved, spec, seen_refs) def _resolve_ref(reference: str, spec: Mapping[str, Any]) -> Any: @@ -514,7 +953,7 @@ def _validate_operation_media( raise CodegenError(f"Operation {operation_id!r} has no supported success response") -def _validate_path_parameters( +def _validate_parameters( operation_id: str, path: str, path_item: Mapping[str, Any], @@ -527,17 +966,36 @@ def _validate_path_parameters( if not isinstance(parameter, dict): raise CodegenError(f"Operation {operation_id!r} has an invalid parameter") parameter = _resolve_object(parameter, spec) - if parameter.get("in") == "path": + location = parameter.get("in") + if location not in {"path", "query"}: + raise CodegenError(f"Operation {operation_id!r} has unsupported parameter location {location!r}") + if location == "path": name = parameter.get("name") if not isinstance(name, str): raise CodegenError(f"Operation {operation_id!r} has a path parameter without a name") path_parameters[name] = parameter + continue + + style = parameter.get("style", "form") + if style != "form": + raise CodegenError(f"Operation {operation_id!r} has unsupported query parameter style {style!r}") + schema = parameter.get("schema") + if not isinstance(schema, dict): + raise CodegenError(f"Operation {operation_id!r} query parameter must define a schema") + schema_kinds = _parameter_schema_kinds(schema, spec) + if "array" in schema_kinds and parameter.get("explode", True) is not True: + raise CodegenError(f"Operation {operation_id!r} query array parameters must be exploded") if template_names != set(path_parameters): raise CodegenError( f"Operation {operation_id!r} path template/parameter mismatch: " f"template={sorted(template_names)}, declared={sorted(path_parameters)}" ) for name, parameter in path_parameters.items(): + style = parameter.get("style", "simple") + if style != "simple": + raise CodegenError(f"Operation {operation_id!r} has unsupported path parameter style {style!r}") + if parameter.get("explode", False) is not False: + raise CodegenError(f"Operation {operation_id!r} path parameters cannot be exploded") if parameter.get("required") is not True: raise CodegenError(f"Operation {operation_id!r} path parameter {name!r} must be required") schema = parameter.get("schema") @@ -548,6 +1006,28 @@ def _validate_path_parameters( raise CodegenError(f"Operation {operation_id!r} path parameter {name!r} must be scalar") +def _parameter_schema_kinds(schema: Mapping[str, Any], spec: Mapping[str, Any]) -> Set[str]: + schema = _resolve_object(schema, spec) + choices = schema.get("oneOf", schema.get("anyOf")) + if isinstance(choices, list): + kinds: Set[str] = set() + for choice in choices: + if not isinstance(choice, dict): + raise CodegenError("Query parameter alternatives must be schemas") + kinds.update(_parameter_schema_kinds(choice, spec)) + return kinds + + schema_type = schema.get("type") + if schema_type in {"boolean", "integer", "number", "string"}: + return {"scalar"} + if schema_type == "array": + items = schema.get("items") + if not isinstance(items, dict) or _parameter_schema_kinds(items, spec) != {"scalar"}: + raise CodegenError("Query parameter arrays must contain scalar values") + return {"array"} + raise CodegenError(f"Unsupported query parameter schema type {schema_type!r}") + + def _validate_component_names(schemas: Mapping[str, Any]) -> None: names: Dict[str, str] = {} for schema_name in schemas: diff --git a/py/src/braintrust/api/__init__.py b/py/src/braintrust/api/__init__.py index b83591de..41ade786 100644 --- a/py/src/braintrust/api/__init__.py +++ b/py/src/braintrust/api/__init__.py @@ -1,47 +1,31 @@ """Public Braintrust API client package.""" from ._routing import EndpointRouter, RequestTarget -from ._service import ClientContext -from .attachments import AttachmentsAPI -from .auth import AuthAPI, LoginResult, OrganizationInfo -from .client import BraintrustClient -from .datasets import DatasetsAPI +from .auth import LoginResult, OrganizationInfo +from .client import BraintrustClient, BraintrustOpenApiClient from .errors import ( BraintrustAPIError, BraintrustHTTPError, - BraintrustResponseError, + BraintrustJSONDecodeError, BraintrustRetryExhaustedError, BraintrustTransportError, BraintrustTransportRetryExhaustedError, ) -from .experiments import ExperimentsAPI -from .functions import FunctionsAPI from .policies import RetryMode, RetryPolicy -from .projects import ProjectsAPI -from .prompts import PromptsAPI -from .queries import QueriesAPI __all__ = [ - "AttachmentsAPI", - "AuthAPI", "BraintrustAPIError", "BraintrustClient", + "BraintrustOpenApiClient", "BraintrustHTTPError", - "BraintrustResponseError", + "BraintrustJSONDecodeError", "BraintrustRetryExhaustedError", "BraintrustTransportError", "BraintrustTransportRetryExhaustedError", - "ClientContext", - "DatasetsAPI", "EndpointRouter", - "ExperimentsAPI", - "FunctionsAPI", "LoginResult", "OrganizationInfo", - "ProjectsAPI", - "PromptsAPI", - "QueriesAPI", "RequestTarget", "RetryMode", "RetryPolicy", diff --git a/py/src/braintrust/api/_generated/models.py b/py/src/braintrust/api/_generated/models.py deleted file mode 100644 index 305a2f16..00000000 --- a/py/src/braintrust/api/_generated/models.py +++ /dev/null @@ -1,6729 +0,0 @@ -# Generated by scripts/generate-api-client.py. DO NOT EDIT. -# OpenAPI commit: 9daf27f19d9e0340304d7a3e7d0edb28380b94c6 -# OpenAPI spec SHA-256: 5ec753c0263c0c44cd04f741edfc7e8bad491cc25a2113d029e84edc076520f0 -# datamodel-code-generator: 0.72.4 -# ruff: 0.15.21 -# Generator Python: 3.14 -# Content SHA-256: 8624c6f9c7bfc2e7c8f70127fd25e25269976b285526b0dc5af8223bbbdc4eb7 - -from typing_extensions import NotRequired -from typing import Any, Literal, TypeAlias, TypedDict -from collections.abc import Mapping, Sequence - - -class AISecret(TypedDict): - created: NotRequired[str | None] - """ - Date of AI secret creation - """ - id: str - """ - Unique identifier for the AI secret - """ - metadata: NotRequired[Mapping[str, Any] | None] - name: str - """ - Name of the AI secret - """ - org_id: str - """ - Unique identifier for the organization - """ - preview_secret: NotRequired[str | None] - secret_updated_at: NotRequired[str | None] - """ - Date of last update to the encrypted secret value itself - """ - secret_updated_by_user_id: NotRequired[str | None] - """ - User id of the last update to the encrypted secret value - """ - type: NotRequired[str | None] - updated_at: NotRequired[str | None] - """ - Date of last AI secret update - """ - - -AISecretType: TypeAlias = str | Sequence[str] - - -AclIdParam: TypeAlias = str -""" -Acl id -""" - - -AclListGroupId: TypeAlias = str -""" -Id of the group the ACL applies to. Exactly one of `user_id` and `group_id` will be provided -""" - - -AclListOrgObjectId: TypeAlias = str -""" -The id of the object the ACL applies to -""" - - -AclListOrgObjectType: TypeAlias = Literal[ - "organization", - "project", - "experiment", - "dataset", - "prompt", - "prompt_session", - "group", - "role", - "org_member", - "project_log", - "org_project", - "org_audit_logs", -] -""" -The object type that the ACL applies to -""" - - -AclListPermission: TypeAlias = Literal[ - "create", - "read", - "update", - "delete", - "create_acls", - "read_acls", - "update_acls", - "delete_acls", -] -""" -Each permission permits a certain type of operation on an object in the system - -Permissions can be assigned to to objects on an individual basis, or grouped into roles -""" - - -AclListRestrictObjectType: TypeAlias = Literal[ - "organization", - "project", - "experiment", - "dataset", - "prompt", - "prompt_session", - "group", - "role", - "org_member", - "project_log", - "org_project", - "org_audit_logs", -] -""" -The object type that the ACL applies to -""" - - -AclListRoleId: TypeAlias = str -""" -Id of the role the ACL grants. Exactly one of `permission` and `role_id` will be provided -""" - - -AclListUserId: TypeAlias = str -""" -Id of the user the ACL applies to. Exactly one of `user_id` and `group_id` will be provided -""" - - -AclObjectId: TypeAlias = str -""" -The id of the object the ACL applies to -""" - - -AclObjectType: TypeAlias = Literal[ - "organization", - "project", - "experiment", - "dataset", - "prompt", - "prompt_session", - "group", - "role", - "org_member", - "project_log", - "org_project", - "org_audit_logs", -] -""" -The object type that the ACL applies to -""" - - -class Agent(TypedDict): - created: NotRequired[str | None] - """ - Date of agent creation - """ - description: NotRequired[str | None] - """ - Textual description of the agent - """ - id: str - """ - Unique identifier for the agent - """ - kind: str - """ - Agent classification: 'custom' for customer-defined agents, 'loop' for built-in Loop agents. - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - User-controlled metadata about the agent - """ - name: str - """ - Name of the agent. Within a project, agent names are unique - """ - project_id: str - """ - Unique identifier for the project that the agent belongs under - """ - slug: str - """ - Stable, URL-safe identifier for the agent, unique within its project. - """ - user_id: str - - -AgentIdParam: TypeAlias = str -""" -Agent id -""" - - -AgentName: TypeAlias = str -""" -Name of the agent to search for -""" - - -AiSecretIdParam: TypeAlias = str -""" -AiSecret id -""" - - -AiSecretName: TypeAlias = str -""" -Name of the ai_secret to search for -""" - - -class ApiKey(TypedDict): - created: NotRequired[str | None] - """ - Date of api key creation - """ - id: str - """ - Unique identifier for the api key - """ - name: str - """ - Name of the api key - """ - org_id: NotRequired[str | None] - """ - Unique identifier for the organization - """ - preview_name: str - user_email: NotRequired[str | None] - """ - The user's email - """ - user_family_name: NotRequired[str | None] - """ - Family name of the user - """ - user_given_name: NotRequired[str | None] - """ - Given name of the user - """ - user_id: NotRequired[str | None] - """ - Unique identifier for the user - """ - - -ApiKeyIdParam: TypeAlias = str -""" -ApiKey id -""" - - -ApiKeyName: TypeAlias = str -""" -Name of the api_key to search for -""" - - -AppLimitParam: TypeAlias = int | None -""" -Limit the number of objects to return -""" - - -AppLimitWithDefaultParam: TypeAlias = int | None -""" -Limit the number of objects to return -""" - - -AutomationStatus: TypeAlias = Literal["active", "paused"] -""" -Whether the automation is active or paused. -""" - - -class BatchedFacetDataFacet(TypedDict): - embedding_model: NotRequired[str] - """ - The embedding model to use for vectorizing facet results. - """ - model: NotRequired[str] - """ - The model to use for facet extraction - """ - name: str - """ - The name of the facet - """ - no_match_pattern: NotRequired[str] - """ - Regex pattern to identify outputs that do not match the facet. If the output matches, the facet will be saved as 'no_match' - """ - prompt: str - """ - The prompt to use for LLM extraction. The preprocessed text will be provided as context. - """ - - -class PreprocessorPreprocessor(TypedDict): - id: str - type: Literal["function"] - version: NotRequired[str] - """ - The version of the function - """ - - -class PreprocessorPreprocessor2(TypedDict): - pass - - -class PreprocessorPreprocessor3(PreprocessorPreprocessor, PreprocessorPreprocessor2): - pass - - -class ChatCompletionContentPartFileFile(TypedDict): - file_data: NotRequired[str] - file_id: NotRequired[str] - filename: NotRequired[str] - - -class ChatCompletionContentPartFileWithTitle(TypedDict): - file: ChatCompletionContentPartFileFile - type: Literal["file"] - - -class ChatCompletionContentPartImageWithTitleImageUrl(TypedDict): - detail: NotRequired[Literal["auto"] | Literal["low"] | Literal["high"]] - url: str - - -class ChatCompletionContentPartImageWithTitle(TypedDict): - image_url: ChatCompletionContentPartImageWithTitleImageUrl - type: Literal["image_url"] - - -class ChatCompletionContentPartTextCacheControl(TypedDict): - type: Literal["ephemeral"] - - -class ChatCompletionContentPartText(TypedDict): - cache_control: NotRequired[ChatCompletionContentPartTextCacheControl] - text: NotRequired[str] - type: Literal["text"] - - -class ChatCompletionContentPartTextWithTitleCacheControl(TypedDict): - type: Literal["ephemeral"] - - -class ChatCompletionContentPartTextWithTitle(TypedDict): - cache_control: NotRequired[ChatCompletionContentPartTextWithTitleCacheControl] - text: NotRequired[str] - type: Literal["text"] - - -class ChatCompletionMessageParamChatCompletionMessageParam(TypedDict): - content: NotRequired[str | Sequence[ChatCompletionContentPartText]] - name: NotRequired[str] - role: Literal["system"] - - -class ChatCompletionMessageParamChatCompletionMessageParam2FunctionCall(TypedDict): - arguments: str - name: str - - -class ChatCompletionMessageParamChatCompletionMessageParam3(TypedDict): - content: NotRequired[str | Sequence[ChatCompletionContentPartText]] - role: Literal["tool"] - tool_call_id: NotRequired[str] - - -class ChatCompletionMessageParamChatCompletionMessageParam4(TypedDict): - content: str | None - name: str - role: Literal["function"] - - -class ChatCompletionMessageParamChatCompletionMessageParam5(TypedDict): - content: NotRequired[str | Sequence[ChatCompletionContentPartText]] - name: NotRequired[str] - role: Literal["developer"] - - -class ChatCompletionMessageParamChatCompletionMessageParam6(TypedDict): - content: NotRequired[str | None] - role: Literal["model"] - - -class ChatCompletionMessageReasoning(TypedDict): - content: NotRequired[str | None] - id: NotRequired[str | None] - - -class ChatCompletionMessageToolCallFunction(TypedDict): - arguments: str - name: str - - -class ChatCompletionMessageToolCall(TypedDict): - function: ChatCompletionMessageToolCallFunction - id: str - type: Literal["function"] - - -class CodeBundleLocationPosition(TypedDict): - type: Literal["task"] - - -class CodeBundleLocationPosition1(TypedDict): - index: int - type: Literal["scorer"] - - -class CodeBundleLocationPosition2(TypedDict): - index: int - type: Literal["classifier"] - - -class CodeBundleLocation(TypedDict): - eval_name: str - position: CodeBundleLocationPosition | CodeBundleLocationPosition1 | CodeBundleLocationPosition2 - type: Literal["experiment"] - - -class CodeBundleLocation1(TypedDict): - index: int - type: Literal["function"] - - -class CodeBundleLocation2SandboxSpec(TypedDict): - provider: Literal["modal"] - snapshot_ref: str - """ - sandbox snapshot ref - """ - - -class CodeBundleLocation2SandboxSpec1(TypedDict): - provider: Literal["lambda"] - - -class CodeBundleLocation2(TypedDict): - entrypoints: NotRequired[Sequence[str]] - """ - Which entrypoints to execute in the sandbox - """ - eval_name: str - evaluator_definition: NotRequired[Any | None] - """ - Definition of current evaluator with parameters - """ - parameters: NotRequired[Mapping[str, Any]] - """ - Parameter values for sandbox eval execution - """ - sandbox_spec: CodeBundleLocation2SandboxSpec | CodeBundleLocation2SandboxSpec1 - type: Literal["sandbox"] - - -class CodeBundleRuntimeContext(TypedDict): - runtime: Literal["node", "python", "browser", "quickjs"] - version: str - - -class CodeBundle(TypedDict): - bundle_id: NotRequired[str | None] - location: CodeBundleLocation | CodeBundleLocation1 | CodeBundleLocation2 - preview: NotRequired[str | None] - """ - A preview of the code - """ - runtime_context: CodeBundleRuntimeContext - - -ComparisonExperimentId: TypeAlias = str -""" -The experiment to compare against, if summarizing scores and metrics. If omitted, will fall back to the `base_exp_id` stored in the experiment metadata, and then to the most recent experiment run in the same project. Must pass `summarize_scores=true` for this id to be used -""" - - -class CreateAISecret(TypedDict): - metadata: NotRequired[Mapping[str, Any] | None] - name: str - """ - Name of the AI secret - """ - org_name: NotRequired[str | None] - """ - For nearly all users, this parameter should be unnecessary. But in the rare case that your API key belongs to multiple organizations, you may specify the name of the organization the AI Secret belongs in. - """ - secret: NotRequired[str | None] - """ - Secret value. If omitted in a PUT request, the existing secret value will be left intact, not replaced with null. - """ - type: NotRequired[str | None] - - -class CreateAgent(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the agent - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - User-controlled metadata about the agent - """ - name: str - """ - Name of the agent. Within a project, agent names are unique - """ - project_id: str - """ - Unique identifier for the project that the agent belongs under - """ - - -class CreateDataset(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the dataset - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - User-controlled metadata about the dataset - """ - name: str - """ - Name of the dataset. Within a project, dataset names are unique - """ - project_id: str - """ - Unique identifier for the project that the dataset belongs under - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the dataset - """ - - -class CreateDatasetSnapshot(TypedDict): - dataset_id: str - """ - Unique identifier for the dataset that this snapshot belongs to - """ - description: NotRequired[str | None] - """ - Textual description of the dataset snapshot - """ - name: str - """ - Name of the dataset snapshot - """ - xact_id: str - """ - Transaction id of the brainstore version at the time of the snapshot - """ - - -class CreateEnvironment(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the environment - """ - name: str - """ - Name of the environment - """ - org_name: NotRequired[str | None] - """ - For nearly all users, this parameter should be unnecessary. But in the rare case that your API key belongs to multiple organizations, you may specify the name of the organization the environment belongs in. - """ - slug: str - """ - A url-friendly, unique identifier for the environment within an organization - """ - - -class CreateExperimentInternalMetadata(TypedDict): - dataset_filter: NotRequired[Mapping[str, Any] | None] - """ - BTQL filter payload used to evaluate a subset of a linked dataset. - """ - - -class CreateFunctionFunctionSchema(TypedDict): - parameters: NotRequired[Any | None] - returns: NotRequired[Any | None] - - -class CreateFunctionOrigin(TypedDict): - internal: NotRequired[bool | None] - """ - The function exists for internal purposes and should not be displayed in the list of functions. - """ - object_id: str - """ - Id of the object the function is originating from - """ - object_type: AclObjectType - - -class CreateGroup(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the group - """ - member_groups: NotRequired[Sequence[str] | None] - """ - Ids of the groups this group inherits from - - An inheriting group has all the users contained in its member groups, as well as all of their inherited users - """ - member_users: NotRequired[Sequence[str] | None] - """ - Ids of users which belong to this group - """ - name: str - """ - Name of the group - """ - org_name: NotRequired[str | None] - """ - For nearly all users, this parameter should be unnecessary. But in the rare case that your API key belongs to multiple organizations, you may specify the name of the organization the group belongs in. - """ - - -class CreateMCPServer(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the MCP server - """ - name: str - """ - Name of the MCP server. Within a project, MCP server names are unique - """ - project_id: str - """ - Unique identifier for the project that the MCP server belongs under - """ - url: str - """ - URL of the MCP server endpoint - """ - - -class CreateProject(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the project - """ - name: str - """ - Name of the project - """ - org_name: NotRequired[str | None] - """ - For nearly all users, this parameter should be unnecessary. But in the rare case that your API key belongs to multiple organizations, you may specify the name of the organization the project belongs in. - """ - - -class CreateProjectAutomationConfigAction(TypedDict): - type: Literal["webhook"] - """ - The type of action to take - """ - url: str - """ - The webhook URL to send the request to - """ - - -class CreateProjectAutomationConfigAction1(TypedDict): - channel: str - """ - The Slack channel ID to post to - """ - message_template: NotRequired[str] - """ - Custom message template for the alert - """ - type: Literal["slack"] - """ - The type of action to take - """ - workspace_id: str - """ - The Slack workspace ID to post to - """ - - -class CreateProjectAutomationConfig(TypedDict): - action: CreateProjectAutomationConfigAction | CreateProjectAutomationConfigAction1 - """ - The action to take when the automation rule is triggered - """ - btql_filter: str - """ - BTQL filter to identify rows for the automation rule - """ - event_type: Literal["logs"] - """ - The type of automation. - """ - interval_seconds: float - """ - Perform the triggered action at most once in this interval of seconds - """ - - -class CreateProjectAutomationConfig1Credentials(TypedDict): - external_id: str - """ - The automation-specific external id component (auto-generated by default) - """ - role_arn: str - """ - The ARN of the IAM role to use - """ - type: Literal["aws_iam"] - - -class CreateProjectAutomationConfig1Credentials1(TypedDict): - service_account_email: str - """ - The GCP service account email to impersonate - """ - type: Literal["gcp_service_account"] - - -class CreateProjectAutomationConfig1ExportDefinition(TypedDict): - type: Literal["log_traces"] - - -class CreateProjectAutomationConfig1ExportDefinition1(TypedDict): - type: Literal["log_spans"] - - -class CreateProjectAutomationConfig1ExportDefinition2(TypedDict): - btql_query: str - """ - The BTQL query to export - """ - type: Literal["btql_query"] - - -class CreateProjectAutomationConfig2(TypedDict): - batch_size: NotRequired[int | None] - """ - The maximum number of result rows to write per async query batch - """ - created_by_user_id: str - """ - The user who submitted the async query - """ - event_type: Literal["async_query"] - """ - The type of automation. - """ - format: Literal["jsonl"] - """ - The materialized result format - """ - object_id: str - """ - The source object ID for the async query - """ - object_type: Literal["project_logs", "experiment", "dataset", "playground_logs"] - """ - The source object type for the async query - """ - query: str - """ - The SQL query to execute asynchronously - """ - status: NotRequired[AutomationStatus] - - -class CreateProjectAutomationConfig4Action(TypedDict): - type: Literal["webhook"] - """ - The type of action to take - """ - url: str - """ - The webhook URL to send the request to - """ - - -class CreateProjectAutomationConfig4Action1(TypedDict): - channel: str - """ - The Slack channel ID to post to - """ - message_template: NotRequired[str] - """ - Custom message template for the alert - """ - type: Literal["slack"] - """ - The type of action to take - """ - workspace_id: str - """ - The Slack workspace ID to post to - """ - - -class CreateProjectAutomationConfig4(TypedDict): - action: CreateProjectAutomationConfig4Action | CreateProjectAutomationConfig4Action1 - """ - The action to take when the automation rule is triggered - """ - environment_filter: NotRequired[Sequence[str]] - """ - Optional list of environment slugs to filter by - """ - event_type: Literal["environment_update"] - """ - The type of automation. - """ - - -class CreateProjectTag(TypedDict): - color: NotRequired[str | None] - """ - Color of the tag for the UI - """ - description: NotRequired[str | None] - """ - Textual description of the project tag - """ - name: str - """ - Name of the project tag - """ - project_id: str - """ - Unique identifier for the project that the project tag belongs under - """ - - -class CreateServiceTokenOutput(TypedDict): - created: NotRequired[str | None] - """ - Date of service token creation - """ - id: str - """ - Unique identifier for the service token - """ - key: str - """ - The raw service token. It will only be exposed this one time - """ - name: str - """ - Name of the service token - """ - org_id: NotRequired[str | None] - """ - Unique identifier for the organization - """ - preview_name: str - service_account_email: NotRequired[str | None] - """ - The service account email (not routable) - """ - service_account_id: NotRequired[str | None] - """ - Unique identifier for the service token - """ - service_account_name: NotRequired[str | None] - """ - The service account name - """ - - -class CreateSpanIFrame(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the span iframe - """ - name: str - """ - Name of the span iframe - """ - post_message: NotRequired[bool | None] - """ - Whether to post messages to the iframe containing the span's data. This is useful when you want to render more data than fits in the URL. - """ - project_id: str - """ - Unique identifier for the project that the span iframe belongs under - """ - url: str - """ - URL to embed the project viewer in an iframe - """ - - -class DataSummary(TypedDict): - total_records: int - """ - Total number of records in the dataset - """ - - -class Dataset(TypedDict): - created: NotRequired[str | None] - """ - Date of dataset creation - """ - deleted_at: NotRequired[str | None] - """ - Date of dataset deletion, or null if the dataset is still active - """ - description: NotRequired[str | None] - """ - Textual description of the dataset - """ - id: str - """ - Unique identifier for the dataset - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - User-controlled metadata about the dataset - """ - name: str - """ - Name of the dataset. Within a project, dataset names are unique - """ - project_id: str - """ - Unique identifier for the project that the dataset belongs under - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the dataset - """ - url_slug: str - """ - URL slug for the dataset. used to construct dataset URLs - """ - user_id: NotRequired[str | None] - """ - Identifies the user who created the dataset - """ - - -class DatasetEventMetadata(TypedDict): - model: NotRequired[str | None] - """ - The model used for this example - """ - - -DatasetIdParam: TypeAlias = str -""" -Dataset id -""" - - -DatasetName: TypeAlias = str -""" -Name of the dataset to search for -""" - - -class DatasetSnapshot(TypedDict): - created: str | None - """ - Date of dataset snapshot creation - """ - dataset_id: str - """ - Unique identifier for the dataset that this snapshot belongs to - """ - description: str | None - id: str - """ - Unique identifier for the dataset snapshot - """ - name: str - """ - Name of the dataset snapshot - """ - xact_id: str - """ - Transaction id of the brainstore version at the time of the snapshot - """ - - -DatasetSnapshotIdParam: TypeAlias = str -""" -DatasetSnapshot id -""" - - -DatasetSnapshotName: TypeAlias = str -""" -Name of the dataset_snapshot to search for -""" - - -class DeleteAISecret(TypedDict): - name: str - """ - Name of the AI secret - """ - org_name: NotRequired[str | None] - """ - For nearly all users, this parameter should be unnecessary. But in the rare case that your API key belongs to multiple organizations, you may specify the name of the organization the AI Secret belongs in. - """ - - -class DeleteServiceToken(TypedDict): - id: str - """ - Unique identifier for the service token. - """ - - -class DeleteView(TypedDict): - object_id: str - """ - The id of the object the view applies to - """ - object_type: AclObjectType - - -EndingBefore: TypeAlias = str -""" -Pagination cursor id. - -For example, if the initial item in the last page you fetched had an id of `foo`, pass `ending_before=foo` to fetch the previous page. Note: you may only pass one of `starting_after` and `ending_before` -""" - - -class EnvVar(TypedDict): - created: NotRequired[str | None] - """ - Date of environment variable creation - """ - id: str - """ - Unique identifier for the environment variable - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - Optional metadata associated with the environment variable when managed via the function secrets API - """ - name: str - """ - The name of the environment variable - """ - object_id: str - """ - The id of the object the environment variable is scoped for - """ - object_type: Literal["organization", "project", "function"] - """ - The type of the object the environment variable is scoped for - """ - preview_secret: NotRequired[str | None] - """ - Redacted preview of the stored secret value - """ - secret_category: NotRequired[Literal["env_var", "ai_provider", "sandbox_provider"]] - """ - The category of the secret: env_var for regular environment variables, ai_provider for AI provider API keys - """ - secret_type: NotRequired[str | None] - """ - Optional classification for the secret (for example, the AI provider name) - """ - secret_updated_at: NotRequired[str | None] - """ - Date of last update to the encrypted secret value itself - """ - secret_updated_by_user_id: NotRequired[str | None] - """ - User id of the last update to the encrypted secret value - """ - used: NotRequired[str | None] - """ - Date the environment variable was last used - """ - - -EnvVarIdParam: TypeAlias = str -""" -EnvVar id -""" - - -EnvVarName: TypeAlias = str -""" -Name of the env_var to search for -""" - - -EnvVarObjectId: TypeAlias = str -""" -The id of the object the environment variable is scoped for -""" - - -EnvVarObjectType: TypeAlias = Literal["organization", "project", "function"] -""" -The type of the object the environment variable is scoped for -""" - - -class Environment(TypedDict): - created: NotRequired[str | None] - """ - Date of environment creation - """ - deleted_at: NotRequired[str | None] - """ - Date of environment deletion, or null if the environment is still active - """ - description: NotRequired[str | None] - """ - Textual description of the environment - """ - id: str - """ - Unique identifier for the environment - """ - name: str - """ - Name of the environment - """ - org_id: str - """ - Unique identifier for the organization that the environment belongs under - """ - slug: str - """ - A url-friendly, unique identifier for the environment within an organization - """ - - -class ExperimentInternalMetadata(TypedDict): - dataset_filter: NotRequired[Mapping[str, Any] | None] - """ - BTQL filter payload used to evaluate a subset of a linked dataset. - """ - - -class ExperimentEventContext(TypedDict): - caller_filename: NotRequired[str | None] - """ - Name of the file in code where the experiment event was created - """ - caller_functionname: NotRequired[str | None] - """ - The function in code which created the experiment event - """ - caller_lineno: NotRequired[int | None] - """ - Line of code where the experiment event was created - """ - - -class ExperimentEventMetadata(TypedDict): - model: NotRequired[str | None] - """ - The model used for this example - """ - - -class ExperimentEventMetrics(TypedDict): - caller_filename: NotRequired[Any | None] - """ - This metric is deprecated - """ - caller_functionname: NotRequired[Any | None] - """ - This metric is deprecated - """ - caller_lineno: NotRequired[Any | None] - """ - This metric is deprecated - """ - completion_tokens: NotRequired[int | None] - """ - The number of tokens in the completion generated by the model (only set if this is an LLM span) - """ - end: NotRequired[float | None] - """ - A unix timestamp recording when the section of code which produced the experiment event finished - """ - prompt_tokens: NotRequired[int | None] - """ - The number of tokens in the prompt used to generate the experiment event (only set if this is an LLM span) - """ - start: NotRequired[float | None] - """ - A unix timestamp recording when the section of code which produced the experiment event started - """ - tokens: NotRequired[int | None] - """ - The total number of tokens in the input and output of the experiment event. - """ - - -ExperimentIdParam: TypeAlias = str -""" -Experiment id -""" - - -ExperimentName: TypeAlias = str -""" -Name of the experiment to search for -""" - - -class Preprocessor1Preprocessor1(TypedDict): - id: str - type: Literal["function"] - version: NotRequired[str] - """ - The version of the function - """ - - -class Preprocessor1Preprocessor12(TypedDict): - pass - - -class Preprocessor1Preprocessor13(Preprocessor1Preprocessor1, Preprocessor1Preprocessor12): - pass - - -class FeedbackDatasetItem(TypedDict): - comment: NotRequired[str | None] - """ - An optional comment string to log about the dataset event - """ - id: str - """ - The id of the dataset event to log feedback for. This is the row `id` returned by `POST /v1/dataset/{dataset_id}/insert` - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - A dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI. Note, this metadata does not correspond to the main event itself, but rather the audit log attached to the event. - """ - source: NotRequired[Literal["app", "api", "external"] | None] - """ - The source of the feedback. Must be one of "external" (default), "app", or "api" - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags to log - """ - - -class FeedbackExperimentItem(TypedDict): - comment: NotRequired[str | None] - """ - An optional comment string to log about the experiment event - """ - expected: NotRequired[Any | None] - """ - The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not - """ - id: str - """ - The id of the experiment event to log feedback for. This is the row `id` returned by `POST /v1/experiment/{experiment_id}/insert` - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - A dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI. Note, this metadata does not correspond to the main event itself, but rather the audit log attached to the event. - """ - scores: NotRequired[Mapping[str, float | None] | None] - """ - A dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the experiment event - """ - source: NotRequired[Literal["app", "api", "external"] | None] - """ - The source of the feedback. Must be one of "external" (default), "app", or "api" - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags to log - """ - - -class FeedbackProjectLogsItem(TypedDict): - comment: NotRequired[str | None] - """ - An optional comment string to log about the project logs event - """ - expected: NotRequired[Any | None] - """ - The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not - """ - id: str - """ - The id of the project logs event to log feedback for. This is the row `id` returned by `POST /v1/project_logs/{project_id}/insert` - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - A dictionary with additional data about the feedback. If you have a `user_id`, you can log it here and access it in the Braintrust UI. Note, this metadata does not correspond to the main event itself, but rather the audit log attached to the event. - """ - scores: NotRequired[Mapping[str, float | None] | None] - """ - A dictionary of numeric values (between 0 and 1) to log. These scores will be merged into the existing scores for the project logs event - """ - source: NotRequired[Literal["app", "api", "external"] | None] - """ - The source of the feedback. Must be one of "external" (default), "app", or "api" - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags to log - """ - - -class FeedbackResponseSchema(TypedDict): - status: Literal["success"] - - -FetchLimit: TypeAlias = int | None -""" -limit the number of traces fetched - -Fetch queries may be paginated if the total result size is expected to be large (e.g. project_logs which accumulate over a long time). Note that fetch queries only support pagination in descending time order (from latest to earliest `_xact_id`. Furthermore, later pages may return rows which showed up in earlier pages, except with an earlier `_xact_id`. This happens because pagination occurs over the whole version history of the event log. You will most likely want to exclude any such duplicate, outdated rows (by `id`) from your combined result set. - -The `limit` parameter controls the number of full traces to return. So you may end up with more individual rows than the specified limit if you are fetching events containing traces. -""" - - -FetchLimitParam: TypeAlias = int | None -""" -limit the number of traces fetched - -Fetch queries may be paginated if the total result size is expected to be large (e.g. project_logs which accumulate over a long time). Note that fetch queries only support pagination in descending time order (from latest to earliest `_xact_id`. Furthermore, later pages may return rows which showed up in earlier pages, except with an earlier `_xact_id`. This happens because pagination occurs over the whole version history of the event log. You will most likely want to exclude any such duplicate, outdated rows (by `id`) from your combined result set. - -The `limit` parameter controls the number of full traces to return. So you may end up with more individual rows than the specified limit if you are fetching events containing traces. -""" - - -FetchPaginationCursor: TypeAlias = str | None -""" -An opaque string to be used as a cursor for the next page of results, in order from latest to earliest. - -The string can be obtained directly from the `cursor` property of the previous fetch query -""" - - -class FunctionFunctionSchema(TypedDict): - parameters: NotRequired[Any | None] - returns: NotRequired[Any | None] - - -class FunctionOrigin(TypedDict): - internal: NotRequired[bool | None] - """ - The function exists for internal purposes and should not be displayed in the list of functions. - """ - object_id: str - """ - Id of the object the function is originating from - """ - object_type: AclObjectType - - -class FunctionDataFunctionData(TypedDict): - type: Literal["prompt"] - - -class Data(CodeBundle): - type: Literal["bundle"] - - -class FunctionDataFunctionData1DataRuntimeContext(TypedDict): - runtime: Literal["node", "python", "browser", "quickjs"] - version: str - - -class FunctionDataFunctionData1Data(TypedDict): - code: str - code_hash: NotRequired[str] - """ - SHA256 hash of the code, computed at save time - """ - runtime_context: FunctionDataFunctionData1DataRuntimeContext - type: Literal["inline"] - - -class FunctionDataFunctionData1(TypedDict): - data: Data | FunctionDataFunctionData1Data - type: Literal["code"] - - -class FunctionDataFunctionData2(TypedDict): - endpoint: str - eval_name: str - parameters: Mapping[str, Any] - parameters_version: NotRequired[str | None] - """ - The version (transaction ID) of the parameters being used - """ - type: Literal["remote_eval"] - - -class FunctionDataFunctionData4FieldSchema(TypedDict): - additionalProperties: NotRequired[bool] - properties: Mapping[str, Mapping[str, Any]] - required: NotRequired[Sequence[str]] - type: Literal["object"] - - -class FunctionDataFunctionData4(TypedDict): - field__schema: FunctionDataFunctionData4FieldSchema - """ - JSON Schema format for parameters - """ - data: Mapping[str, Any] - """ - The parameters data - """ - type: Literal["parameters"] - - -class FunctionDataNullishFunctionDataNullish(TypedDict): - type: Literal["prompt"] - - -class FunctionDataNullishFunctionDataNullish1DataRuntimeContext(TypedDict): - runtime: Literal["node", "python", "browser", "quickjs"] - version: str - - -class FunctionDataNullishFunctionDataNullish1Data(TypedDict): - code: str - code_hash: NotRequired[str] - """ - SHA256 hash of the code, computed at save time - """ - runtime_context: FunctionDataNullishFunctionDataNullish1DataRuntimeContext - type: Literal["inline"] - - -class FunctionDataNullishFunctionDataNullish1(TypedDict): - data: Data | FunctionDataNullishFunctionDataNullish1Data - type: Literal["code"] - - -class FunctionDataNullishFunctionDataNullish2(TypedDict): - endpoint: str - eval_name: str - parameters: Mapping[str, Any] - parameters_version: NotRequired[str | None] - """ - The version (transaction ID) of the parameters being used - """ - type: Literal["remote_eval"] - - -class FunctionDataNullishFunctionDataNullish4FieldSchema(TypedDict): - additionalProperties: NotRequired[bool] - properties: Mapping[str, Mapping[str, Any]] - required: NotRequired[Sequence[str]] - type: Literal["object"] - - -class FunctionDataNullishFunctionDataNullish4(TypedDict): - field__schema: FunctionDataNullishFunctionDataNullish4FieldSchema - """ - JSON Schema format for parameters - """ - data: Mapping[str, Any] - """ - The parameters data - """ - type: Literal["parameters"] - - -class FunctionIdFunctionId(TypedDict): - function_id: str - """ - The ID of the function - """ - version: NotRequired[str] - """ - The version of the function - """ - - -class FunctionIdFunctionId1(TypedDict): - project_name: str - """ - The name of the project containing the function - """ - slug: str - """ - The slug of the function - """ - version: NotRequired[str] - """ - The version of the function - """ - - -class FunctionIdFunctionId3(TypedDict): - prompt_session_function_id: str - """ - The ID of the function in the prompt session - """ - prompt_session_id: str - """ - The ID of the prompt session - """ - version: NotRequired[str] - """ - The version of the function - """ - - -class FunctionIdFunctionId4InlineContext(TypedDict): - runtime: Literal["node", "python", "browser", "quickjs"] - version: str - - -FunctionIdParam: TypeAlias = str -""" -Function id -""" - - -FunctionIdRef: TypeAlias = Mapping[str, Any] - - -FunctionName: TypeAlias = str -""" -Name of the function to search for -""" - - -FunctionTypeEnum: TypeAlias = ( - Literal[ - "llm", - "scorer", - "task", - "tool", - "custom_view", - "preprocessor", - "facet", - "classifier", - "tag", - "parameters", - "sandbox", - ] - | None -) -""" -The type of global function. Defaults to 'scorer'. -""" - - -FunctionTypeEnumNullish: TypeAlias = ( - Literal[ - "llm", - "scorer", - "task", - "tool", - "custom_view", - "preprocessor", - "facet", - "classifier", - "tag", - "parameters", - "sandbox", - ] - | None -) - - -class GitMetadataSettings(TypedDict): - collect: Literal["all", "none", "some"] - fields: NotRequired[ - Sequence[ - Literal[ - "commit", - "branch", - "tag", - "dirty", - "author_name", - "author_email", - "commit_message", - "commit_time", - "git_diff", - ] - ] - ] - - -class GraphEdgeSource(TypedDict): - node: str - """ - The id of the node in the graph - """ - variable: str - - -class GraphEdgeTarget(TypedDict): - node: str - """ - The id of the node in the graph - """ - variable: str - - -class GraphEdge(TypedDict): - purpose: Literal["control", "data", "messages"] - """ - The purpose of the edge - """ - source: GraphEdgeSource - target: GraphEdgeTarget - - -class GraphNodeGraphNodePosition(TypedDict): - x: float - """ - The x position of the node - """ - y: float - """ - The y position of the node - """ - - -class GraphNodeGraphNode(TypedDict): - description: NotRequired[str | None] - """ - The description of the node - """ - function: FunctionIdRef - position: NotRequired[GraphNodeGraphNodePosition | None] - """ - The position of the node - """ - type: Literal["function"] - - -class GraphNodeGraphNode1Position(TypedDict): - x: float - """ - The x position of the node - """ - y: float - """ - The y position of the node - """ - - -class GraphNodeGraphNode1(TypedDict): - description: NotRequired[str | None] - """ - The description of the node - """ - position: NotRequired[GraphNodeGraphNode1Position | None] - """ - The position of the node - """ - type: Literal["input"] - """ - The input to the graph - """ - - -class GraphNodeGraphNode2Position(TypedDict): - x: float - """ - The x position of the node - """ - y: float - """ - The y position of the node - """ - - -class GraphNodeGraphNode2(TypedDict): - description: NotRequired[str | None] - """ - The description of the node - """ - position: NotRequired[GraphNodeGraphNode2Position | None] - """ - The position of the node - """ - type: Literal["output"] - """ - The output of the graph - """ - - -class GraphNodeGraphNode3Position(TypedDict): - x: float - """ - The x position of the node - """ - y: float - """ - The y position of the node - """ - - -class GraphNodeGraphNode3(TypedDict): - description: NotRequired[str | None] - """ - The description of the node - """ - position: NotRequired[GraphNodeGraphNode3Position | None] - """ - The position of the node - """ - type: Literal["literal"] - value: NotRequired[Any | None] - """ - A literal value to be returned - """ - - -class GraphNodeGraphNode4Position(TypedDict): - x: float - """ - The x position of the node - """ - y: float - """ - The y position of the node - """ - - -class GraphNodeGraphNode4(TypedDict): - description: NotRequired[str | None] - """ - The description of the node - """ - expr: str - """ - A BTQL expression to be evaluated - """ - position: NotRequired[GraphNodeGraphNode4Position | None] - """ - The position of the node - """ - type: Literal["btql"] - - -class GraphNodeGraphNode5Position(TypedDict): - x: float - """ - The x position of the node - """ - y: float - """ - The y position of the node - """ - - -class GraphNodeGraphNode5(TypedDict): - condition: NotRequired[str | None] - """ - A BTQL expression to be evaluated - """ - description: NotRequired[str | None] - """ - The description of the node - """ - position: NotRequired[GraphNodeGraphNode5Position | None] - """ - The position of the node - """ - type: Literal["gate"] - - -class GraphNodeGraphNode6Position(TypedDict): - x: float - """ - The x position of the node - """ - y: float - """ - The y position of the node - """ - - -class GraphNodeGraphNode6(TypedDict): - description: NotRequired[str | None] - """ - The description of the node - """ - position: NotRequired[GraphNodeGraphNode6Position | None] - """ - The position of the node - """ - type: Literal["aggregator"] - - -class GraphNodeGraphNode7Position(TypedDict): - x: float - """ - The x position of the node - """ - y: float - """ - The y position of the node - """ - - -class Group(TypedDict): - created: NotRequired[str | None] - """ - Date of group creation - """ - deleted_at: NotRequired[str | None] - """ - Date of group deletion, or null if the group is still active - """ - description: NotRequired[str | None] - """ - Textual description of the group - """ - id: str - """ - Unique identifier for the group - """ - member_groups: NotRequired[Sequence[str] | None] - """ - Ids of the groups this group inherits from - - An inheriting group has all the users contained in its member groups, as well as all of their inherited users - """ - member_users: NotRequired[Sequence[str] | None] - """ - Ids of users which belong to this group - """ - name: str - """ - Name of the group - """ - org_id: str - """ - Unique id for the organization that the group belongs under - - It is forbidden to change the org after creating a group - """ - user_id: NotRequired[str | None] - """ - Identifies the user who created the group - """ - - -GroupIdParam: TypeAlias = str -""" -Group id -""" - - -GroupName: TypeAlias = str -""" -Name of the group to search for -""" - - -class GroupScope(TypedDict): - group_by: str - """ - Field path to group by, e.g. metadata.session_id - """ - idle_seconds: NotRequired[float] - """ - Optional: trigger after this many seconds of inactivity - """ - interval_seconds: NotRequired[float] - """ - Maximum time range to include when constructing a group - """ - max_traces: NotRequired[int] - """ - Maximum number of traces to include when constructing a group (default/max: 64) - """ - placement: Literal["first", "each"] - """ - Which trace or traces to write grouped scorer results to - """ - type: Literal["group"] - - -Ids: TypeAlias = str | Sequence[str] -""" -Filter search results to a particular set of object IDs. To specify a list of IDs, include the query param multiple times -""" - - -ImageRenderingMode: TypeAlias = Literal["auto", "click_to_load", "blocked"] | None -""" -Controls how images are rendered in the UI: 'auto' loads images automatically, 'click_to_load' shows a placeholder until clicked, 'blocked' prevents image loading entirely -""" - - -class InsertDatasetEventFieldArrayDeleteItem(TypedDict): - delete: Sequence[Any] - path: Sequence[str] - - -class InsertDatasetEventMetadata(TypedDict): - model: NotRequired[str | None] - """ - The model used for this example - """ - - -class InsertEventsResponse(TypedDict): - row_ids: Sequence[str] - """ - The ids of all rows that were inserted, aligning one-to-one with the rows provided as input - """ - - -class InsertExperimentEventFieldArrayDeleteItem(TypedDict): - delete: Sequence[Any] - path: Sequence[str] - - -class InsertExperimentEventContext(TypedDict): - caller_filename: NotRequired[str | None] - """ - Name of the file in code where the experiment event was created - """ - caller_functionname: NotRequired[str | None] - """ - The function in code which created the experiment event - """ - caller_lineno: NotRequired[int | None] - """ - Line of code where the experiment event was created - """ - - -class InsertExperimentEventMetadata(TypedDict): - model: NotRequired[str | None] - """ - The model used for this example - """ - - -class InsertExperimentEventMetrics(TypedDict): - caller_filename: NotRequired[Any | None] - """ - This metric is deprecated - """ - caller_functionname: NotRequired[Any | None] - """ - This metric is deprecated - """ - caller_lineno: NotRequired[Any | None] - """ - This metric is deprecated - """ - completion_tokens: NotRequired[int | None] - """ - The number of tokens in the completion generated by the model (only set if this is an LLM span) - """ - end: NotRequired[float | None] - """ - A unix timestamp recording when the section of code which produced the experiment event finished - """ - prompt_tokens: NotRequired[int | None] - """ - The number of tokens in the prompt used to generate the experiment event (only set if this is an LLM span) - """ - start: NotRequired[float | None] - """ - A unix timestamp recording when the section of code which produced the experiment event started - """ - tokens: NotRequired[int | None] - """ - The total number of tokens in the input and output of the experiment event. - """ - - -class InsertProjectLogsEventFieldArrayDeleteItem(TypedDict): - delete: Sequence[Any] - path: Sequence[str] - - -class InsertProjectLogsEventContext(TypedDict): - caller_filename: NotRequired[str | None] - """ - Name of the file in code where the project logs event was created - """ - caller_functionname: NotRequired[str | None] - """ - The function in code which created the project logs event - """ - caller_lineno: NotRequired[int | None] - """ - Line of code where the project logs event was created - """ - - -class InsertProjectLogsEventMetadata(TypedDict): - model: NotRequired[str | None] - """ - The model used for this example - """ - - -class InsertProjectLogsEventMetrics(TypedDict): - caller_filename: NotRequired[Any | None] - """ - This metric is deprecated - """ - caller_functionname: NotRequired[Any | None] - """ - This metric is deprecated - """ - caller_lineno: NotRequired[Any | None] - """ - This metric is deprecated - """ - completion_tokens: NotRequired[int | None] - """ - The number of tokens in the completion generated by the model (only set if this is an LLM span) - """ - end: NotRequired[float | None] - """ - A unix timestamp recording when the section of code which produced the project logs event finished - """ - prompt_tokens: NotRequired[int | None] - """ - The number of tokens in the prompt used to generate the project logs event (only set if this is an LLM span) - """ - start: NotRequired[float | None] - """ - A unix timestamp recording when the section of code which produced the project logs event started - """ - tokens: NotRequired[int | None] - """ - The total number of tokens in the input and output of the project logs event. - """ - - -class InvokeApiMcpAuth(TypedDict): - oauth_token: NotRequired[str] - """ - The OAuth token to use - """ - - -class InvokeParentInvokeParentRowIds(TypedDict): - id: str - """ - The id of the row - """ - root_span_id: str - """ - The root_span_id of the row - """ - span_id: str - """ - The span_id of the row - """ - - -class InvokeParentInvokeParent(TypedDict): - object_id: str - """ - The id of the container object you are logging to - """ - object_type: Literal["project_logs", "experiment", "playground_logs"] - propagated_event: NotRequired[Mapping[str, Any] | None] - """ - Include these properties in every span created under this parent - """ - row_ids: NotRequired[InvokeParentInvokeParentRowIds | None] - """ - Identifiers for the row to to log a subspan under - """ - - -InvokeParent: TypeAlias = InvokeParentInvokeParent | str -""" -Options for tracing the function call -""" - - -class MCPServer(TypedDict): - created: NotRequired[str | None] - """ - Date of MCP server creation - """ - deleted_at: NotRequired[str | None] - """ - Date of MCP server deletion, or null if the MCP server is still active - """ - description: NotRequired[str | None] - """ - Textual description of the MCP server - """ - id: str - """ - Unique identifier for the MCP server - """ - name: str - """ - Name of the MCP server. Within a project, MCP server names are unique - """ - project_id: str - """ - Unique identifier for the project that the MCP server belongs under - """ - url: str - """ - URL of the MCP server endpoint - """ - user_id: NotRequired[str | None] - """ - Identifies the user who created the MCP server - """ - - -MaxRootSpanId: TypeAlias = str -""" -DEPRECATION NOTICE: The manually-constructed pagination cursor is deprecated in favor of the explicit 'cursor' returned by object fetch requests. Please prefer the 'cursor' argument going forwards. - -Together, `max_xact_id` and `max_root_span_id` form a pagination cursor - -Since a paginated fetch query returns results in order from latest to earliest, the cursor for the next page can be found as the row with the minimum (earliest) value of the tuple `(_xact_id, root_span_id)`. See the documentation of `limit` for an overview of paginating fetch queries. -""" - - -MaxXactId: TypeAlias = str -""" -DEPRECATION NOTICE: The manually-constructed pagination cursor is deprecated in favor of the explicit 'cursor' returned by object fetch requests. Please prefer the 'cursor' argument going forwards. - -Together, `max_xact_id` and `max_root_span_id` form a pagination cursor - -Since a paginated fetch query returns results in order from latest to earliest, the cursor for the next page can be found as the row with the minimum (earliest) value of the tuple `(_xact_id, root_span_id)`. See the documentation of `limit` for an overview of paginating fetch queries. -""" - - -McpServerIdParam: TypeAlias = str -""" -McpServer id -""" - - -McpServerName: TypeAlias = str -""" -Name of the mcp_server to search for -""" - - -class MetricSummary(TypedDict): - diff: NotRequired[float] - """ - Difference in metric between the current and comparison experiment - """ - improvements: int - """ - Number of improvements in the metric - """ - metric: float - """ - Average metric across all examples - """ - name: str - """ - Name of the metric - """ - regressions: int - """ - Number of regressions in the metric - """ - unit: str - """ - Unit label for the metric - """ - - -class ModelParamsModelParamsFunctionCall(TypedDict): - name: str - - -class ModelParamsModelParamsToolChoiceFunction(TypedDict): - name: str - - -class ModelParamsModelParamsToolChoice(TypedDict): - function: ModelParamsModelParamsToolChoiceFunction - type: Literal["function"] - - -class ModelParamsModelParams1(TypedDict): - max_tokens: float - max_tokens_to_sample: NotRequired[float] - """ - This is a legacy parameter that should not be used. - """ - reasoning_budget: NotRequired[float] - reasoning_enabled: NotRequired[bool] - stop_sequences: NotRequired[Sequence[str]] - temperature: float - top_k: NotRequired[float] - top_p: NotRequired[float] - use_cache: NotRequired[bool] - - -class ModelParamsModelParams2(TypedDict): - maxOutputTokens: NotRequired[float] - reasoning_budget: NotRequired[float] - reasoning_enabled: NotRequired[bool] - temperature: NotRequired[float] - topK: NotRequired[float] - topP: NotRequired[float] - use_cache: NotRequired[bool] - - -class ModelParamsModelParams3(TypedDict): - reasoning_budget: NotRequired[float] - reasoning_enabled: NotRequired[bool] - temperature: NotRequired[float] - topK: NotRequired[float] - use_cache: NotRequired[bool] - - -class ModelParamsModelParams4(TypedDict): - reasoning_budget: NotRequired[float] - reasoning_enabled: NotRequired[bool] - use_cache: NotRequired[bool] - - -class NullableSavedFunctionIdNullableSavedFunctionId(TypedDict): - id: str - type: Literal["function"] - version: NotRequired[str] - """ - The version of the function - """ - - -class NullableSavedFunctionIdNullableSavedFunctionId1(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -NullableSavedFunctionId: TypeAlias = ( - NullableSavedFunctionIdNullableSavedFunctionId | NullableSavedFunctionIdNullableSavedFunctionId1 | None -) -""" -Default preprocessor for this project. When set, functions that use preprocessors will use this instead of their built-in default. -""" - - -class ObjectReferenceNullish(TypedDict): - field_xact_id: NotRequired[str | None] - """ - Transaction ID of the original event. - """ - created: NotRequired[str | None] - """ - Created timestamp of the original event. Used to help sort in the UI - """ - id: str - """ - ID of the original event. - """ - object_id: str - """ - ID of the object the event is originating from. - """ - object_type: Literal["project_logs", "experiment", "dataset", "prompt", "function", "prompt_session"] - """ - Type of the object the event is originating from. - """ - - -class ScorerScorer(TypedDict): - id: str - type: Literal["function"] - version: NotRequired[str] - """ - The version of the function - """ - - -class ScorerScorer1(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -class ScorerScorer2(TypedDict): - pass - - -class ScorerScorer3(ScorerScorer, ScorerScorer2): - pass - - -class ScorerScorer4(ScorerScorer1, ScorerScorer2): - pass - - -class ScorerScorer5(ScorerScorer, ScorerScorer2): - pass - - -class ScorerScorer6(ScorerScorer1, ScorerScorer2): - pass - - -Scorer: TypeAlias = ScorerScorer3 | ScorerScorer4 | ScorerScorer5 | ScorerScorer6 - - -OrgName: TypeAlias = str -""" -Filter search results to within a particular organization -""" - - -class Organization(TypedDict): - api_url: NotRequired[str | None] - created: NotRequired[str | None] - """ - Date of organization creation - """ - id: str - """ - Unique identifier for the organization - """ - image_rendering_mode: NotRequired[ImageRenderingMode | None] - is_dataplane_private: NotRequired[bool | None] - is_universal_api: NotRequired[bool | None] - name: str - """ - Name of the organization - """ - proxy_url: NotRequired[str | None] - realtime_url: NotRequired[str | None] - - -OrganizationIdParam: TypeAlias = str -""" -Organization id -""" - - -class PatchAISecret(TypedDict): - metadata: NotRequired[Mapping[str, Any] | None] - name: NotRequired[str | None] - """ - Name of the AI secret - """ - secret: NotRequired[str | None] - type: NotRequired[str | None] - - -class PatchAgent(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the agent - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - User-controlled metadata about the agent - """ - name: NotRequired[str | None] - """ - Name of the agent. Within a project, agent names are unique - """ - - -class PatchDataset(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the dataset - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - User-controlled metadata about the dataset - """ - name: NotRequired[str | None] - """ - Name of the dataset. Within a project, dataset names are unique - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the dataset - """ - - -class PatchDatasetSnapshot(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the dataset snapshot - """ - name: NotRequired[str | None] - """ - Name of the dataset snapshot - """ - - -class PatchEnvironment(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the environment - """ - name: NotRequired[str | None] - """ - Name of the environment - """ - slug: NotRequired[str | None] - """ - A url-friendly, unique identifier for the environment within an organization - """ - - -class PatchExperimentInternalMetadata(TypedDict): - dataset_filter: NotRequired[Mapping[str, Any] | None] - """ - BTQL filter payload used to evaluate a subset of a linked dataset. - """ - - -class PatchGroup(TypedDict): - add_member_groups: NotRequired[Sequence[str] | None] - """ - A list of group IDs to add to the group's inheriting-from set - """ - add_member_users: NotRequired[Sequence[str] | None] - """ - A list of user IDs to add to the group - """ - description: NotRequired[str | None] - """ - Textual description of the group - """ - name: NotRequired[str | None] - """ - Name of the group - """ - remove_member_groups: NotRequired[Sequence[str] | None] - """ - A list of group IDs to remove from the group's inheriting-from set - """ - remove_member_users: NotRequired[Sequence[str] | None] - """ - A list of user IDs to remove from the group - """ - - -class PatchMCPServer(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the MCP server - """ - name: NotRequired[str | None] - """ - Name of the MCP server. Within a project, MCP server names are unique - """ - url: NotRequired[str | None] - """ - URL of the MCP server endpoint - """ - - -class PatchOrganization(TypedDict): - api_url: NotRequired[str | None] - image_rendering_mode: NotRequired[ImageRenderingMode | None] - is_dataplane_private: NotRequired[bool | None] - is_universal_api: NotRequired[bool | None] - name: NotRequired[str | None] - """ - Name of the organization - """ - proxy_url: NotRequired[str | None] - realtime_url: NotRequired[str | None] - - -class PatchOrganizationMembersInviteUsersServiceAccount(TypedDict): - name: str - token_name: NotRequired[str | None] - """ - Optional name of an initial service token to create for the new service account. When this field is set, the request must be authenticated with a service token that has organization-owner permissions, not a user API key. - """ - - -class PatchOrganizationMembersInviteUsers(TypedDict): - emails: NotRequired[Sequence[str] | None] - """ - Emails of users to invite - """ - group_id: NotRequired[str | None] - """ - Singular form of group_ids - """ - group_ids: NotRequired[Sequence[str] | None] - """ - Optional list of group ids to add newly-invited users to. - """ - group_name: NotRequired[str | None] - """ - Singular form of group_names - """ - group_names: NotRequired[Sequence[str] | None] - """ - Optional list of group names to add newly-invited users to. - """ - ids: NotRequired[Sequence[str] | None] - """ - Ids of existing users to invite - """ - send_invite_emails: NotRequired[bool | None] - """ - If true, send invite emails to the users who wore actually added - """ - service_accounts: NotRequired[Sequence[PatchOrganizationMembersInviteUsersServiceAccount] | None] - """ - Service accounts to create. Any caller permitted to add organization members can create service accounts (but not necessarily their associated tokens). - """ - - -class PatchOrganizationMembersRemoveUsers(TypedDict): - emails: NotRequired[Sequence[str] | None] - """ - Emails of users to remove - """ - ids: NotRequired[Sequence[str] | None] - """ - Ids of users to remove - """ - - -class PatchOrganizationMembers(TypedDict): - invite_users: NotRequired[PatchOrganizationMembersInviteUsers | None] - """ - Users to invite to the organization - """ - org_id: NotRequired[str | None] - """ - For nearly all users, this parameter should be unnecessary. But in the rare case that your API key belongs to multiple organizations, or in case you want to explicitly assert the organization you are modifying, you may specify the id of the organization. - """ - org_name: NotRequired[str | None] - """ - For nearly all users, this parameter should be unnecessary. But in the rare case that your API key belongs to multiple organizations, or in case you want to explicitly assert the organization you are modifying, you may specify the name of the organization. - """ - remove_users: NotRequired[PatchOrganizationMembersRemoveUsers | None] - """ - Users to remove from the organization - """ - - -class PatchOrganizationMembersOutputAddedUser(TypedDict): - api_key: NotRequired[str | None] - email: NotRequired[str | None] - id: str - token_name: NotRequired[str | None] - - -class PatchOrganizationMembersOutput(TypedDict): - added_users: NotRequired[Sequence[PatchOrganizationMembersOutputAddedUser] | None] - """ - If service accounts with tokens were created, this will contain the added users with their API keys - """ - org_id: str - """ - The id of the org that was modified. - """ - send_email_error: NotRequired[str | None] - """ - If invite emails failed to send for some reason, the patch operation will still complete, but we will return an error message here - """ - status: Literal["success"] - - -class PatchProjectAutomationConfigAction(TypedDict): - type: Literal["webhook"] - """ - The type of action to take - """ - url: str - """ - The webhook URL to send the request to - """ - - -class PatchProjectAutomationConfigAction1(TypedDict): - channel: str - """ - The Slack channel ID to post to - """ - message_template: NotRequired[str] - """ - Custom message template for the alert - """ - type: Literal["slack"] - """ - The type of action to take - """ - workspace_id: str - """ - The Slack workspace ID to post to - """ - - -class PatchProjectAutomationConfig(TypedDict): - action: PatchProjectAutomationConfigAction | PatchProjectAutomationConfigAction1 - """ - The action to take when the automation rule is triggered - """ - btql_filter: str - """ - BTQL filter to identify rows for the automation rule - """ - event_type: Literal["logs"] - """ - The type of automation. - """ - interval_seconds: float - """ - Perform the triggered action at most once in this interval of seconds - """ - - -class PatchProjectAutomationConfig1Credentials(TypedDict): - external_id: str - """ - The automation-specific external id component (auto-generated by default) - """ - role_arn: str - """ - The ARN of the IAM role to use - """ - type: Literal["aws_iam"] - - -class PatchProjectAutomationConfig1Credentials1(TypedDict): - service_account_email: str - """ - The GCP service account email to impersonate - """ - type: Literal["gcp_service_account"] - - -class PatchProjectAutomationConfig1ExportDefinition(TypedDict): - type: Literal["log_traces"] - - -class PatchProjectAutomationConfig1ExportDefinition1(TypedDict): - type: Literal["log_spans"] - - -class PatchProjectAutomationConfig1ExportDefinition2(TypedDict): - btql_query: str - """ - The BTQL query to export - """ - type: Literal["btql_query"] - - -class PatchProjectAutomationConfig2(TypedDict): - batch_size: NotRequired[int | None] - """ - The maximum number of result rows to write per async query batch - """ - created_by_user_id: str - """ - The user who submitted the async query - """ - event_type: Literal["async_query"] - """ - The type of automation. - """ - format: Literal["jsonl"] - """ - The materialized result format - """ - object_id: str - """ - The source object ID for the async query - """ - object_type: Literal["project_logs", "experiment", "dataset", "playground_logs"] - """ - The source object type for the async query - """ - query: str - """ - The SQL query to execute asynchronously - """ - status: NotRequired[AutomationStatus] - - -class PatchProjectAutomationConfig4Action(TypedDict): - type: Literal["webhook"] - """ - The type of action to take - """ - url: str - """ - The webhook URL to send the request to - """ - - -class PatchProjectAutomationConfig4Action1(TypedDict): - channel: str - """ - The Slack channel ID to post to - """ - message_template: NotRequired[str] - """ - Custom message template for the alert - """ - type: Literal["slack"] - """ - The type of action to take - """ - workspace_id: str - """ - The Slack workspace ID to post to - """ - - -class PatchProjectAutomationConfig4(TypedDict): - action: PatchProjectAutomationConfig4Action | PatchProjectAutomationConfig4Action1 - """ - The action to take when the automation rule is triggered - """ - environment_filter: NotRequired[Sequence[str]] - """ - Optional list of environment slugs to filter by - """ - event_type: Literal["environment_update"] - """ - The type of automation. - """ - - -class PatchProjectTag(TypedDict): - color: NotRequired[str | None] - """ - Color of the tag for the UI - """ - description: NotRequired[str | None] - """ - Textual description of the project tag - """ - name: NotRequired[str | None] - """ - Name of the project tag - """ - - -class PatchSpanIFrame(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the span iframe - """ - name: NotRequired[str | None] - """ - Name of the span iframe - """ - post_message: NotRequired[bool | None] - """ - Whether to post messages to the iframe containing the span's data. This is useful when you want to render more data than fits in the URL. - """ - url: NotRequired[str | None] - """ - URL to embed the project viewer in an iframe - """ - - -Permission: TypeAlias = Literal[ - "create", - "read", - "update", - "delete", - "create_acls", - "read_acls", - "update_acls", - "delete_acls", -] -""" -Each permission permits a certain type of operation on an object in the system - -Permissions can be assigned to to objects on an individual basis, or grouped into roles -""" - - -class ProjectAutomationConfigAction(TypedDict): - type: Literal["webhook"] - """ - The type of action to take - """ - url: str - """ - The webhook URL to send the request to - """ - - -class ProjectAutomationConfigAction1(TypedDict): - channel: str - """ - The Slack channel ID to post to - """ - message_template: NotRequired[str] - """ - Custom message template for the alert - """ - type: Literal["slack"] - """ - The type of action to take - """ - workspace_id: str - """ - The Slack workspace ID to post to - """ - - -class ProjectAutomationConfig(TypedDict): - action: ProjectAutomationConfigAction | ProjectAutomationConfigAction1 - """ - The action to take when the automation rule is triggered - """ - btql_filter: str - """ - BTQL filter to identify rows for the automation rule - """ - event_type: Literal["logs"] - """ - The type of automation. - """ - interval_seconds: float - """ - Perform the triggered action at most once in this interval of seconds - """ - - -class ProjectAutomationConfig1Credentials(TypedDict): - external_id: str - """ - The automation-specific external id component (auto-generated by default) - """ - role_arn: str - """ - The ARN of the IAM role to use - """ - type: Literal["aws_iam"] - - -class ProjectAutomationConfig1Credentials1(TypedDict): - service_account_email: str - """ - The GCP service account email to impersonate - """ - type: Literal["gcp_service_account"] - - -class ProjectAutomationConfig1ExportDefinition(TypedDict): - type: Literal["log_traces"] - - -class ProjectAutomationConfig1ExportDefinition1(TypedDict): - type: Literal["log_spans"] - - -class ProjectAutomationConfig1ExportDefinition2(TypedDict): - btql_query: str - """ - The BTQL query to export - """ - type: Literal["btql_query"] - - -class ProjectAutomationConfig2(TypedDict): - batch_size: NotRequired[int | None] - """ - The maximum number of result rows to write per async query batch - """ - created_by_user_id: str - """ - The user who submitted the async query - """ - event_type: Literal["async_query"] - """ - The type of automation. - """ - format: Literal["jsonl"] - """ - The materialized result format - """ - object_id: str - """ - The source object ID for the async query - """ - object_type: Literal["project_logs", "experiment", "dataset", "playground_logs"] - """ - The source object type for the async query - """ - query: str - """ - The SQL query to execute asynchronously - """ - status: NotRequired[AutomationStatus] - - -class ProjectAutomationConfig4Action(TypedDict): - type: Literal["webhook"] - """ - The type of action to take - """ - url: str - """ - The webhook URL to send the request to - """ - - -class ProjectAutomationConfig4Action1(TypedDict): - channel: str - """ - The Slack channel ID to post to - """ - message_template: NotRequired[str] - """ - Custom message template for the alert - """ - type: Literal["slack"] - """ - The type of action to take - """ - workspace_id: str - """ - The Slack workspace ID to post to - """ - - -class ProjectAutomationConfig4(TypedDict): - action: ProjectAutomationConfig4Action | ProjectAutomationConfig4Action1 - """ - The action to take when the automation rule is triggered - """ - environment_filter: NotRequired[Sequence[str]] - """ - Optional list of environment slugs to filter by - """ - event_type: Literal["environment_update"] - """ - The type of automation. - """ - - -ProjectAutomationIdParam: TypeAlias = str -""" -ProjectAutomation id -""" - - -ProjectAutomationName: TypeAlias = str -""" -Name of the project_automation to search for -""" - - -ProjectIdParam: TypeAlias = str -""" -Project id -""" - - -ProjectIdQuery: TypeAlias = str -""" -Project id -""" - - -class ProjectLogsEventContext(TypedDict): - caller_filename: NotRequired[str | None] - """ - Name of the file in code where the project logs event was created - """ - caller_functionname: NotRequired[str | None] - """ - The function in code which created the project logs event - """ - caller_lineno: NotRequired[int | None] - """ - Line of code where the project logs event was created - """ - - -class ProjectLogsEventMetadata(TypedDict): - model: NotRequired[str | None] - """ - The model used for this example - """ - - -class ProjectLogsEventMetrics(TypedDict): - caller_filename: NotRequired[Any | None] - """ - This metric is deprecated - """ - caller_functionname: NotRequired[Any | None] - """ - This metric is deprecated - """ - caller_lineno: NotRequired[Any | None] - """ - This metric is deprecated - """ - completion_tokens: NotRequired[int | None] - """ - The number of tokens in the completion generated by the model (only set if this is an LLM span) - """ - end: NotRequired[float | None] - """ - A unix timestamp recording when the section of code which produced the project logs event finished - """ - prompt_tokens: NotRequired[int | None] - """ - The number of tokens in the prompt used to generate the project logs event (only set if this is an LLM span) - """ - start: NotRequired[float | None] - """ - A unix timestamp recording when the section of code which produced the project logs event started - """ - tokens: NotRequired[int | None] - """ - The total number of tokens in the input and output of the project logs event. - """ - - -ProjectName: TypeAlias = str -""" -Name of the project to search for -""" - - -class ProjectScoreCategory(TypedDict): - name: str - """ - Name of the category - """ - value: float - """ - Numerical value of the category. Must be between 0 and 1, inclusive - """ - - -class ProjectScoreConditionWhen(TypedDict): - clauses: NotRequired[Sequence[str] | None] - subspan_clauses: NotRequired[Sequence[str] | None] - trace_clauses: NotRequired[Sequence[str] | None] - - -class ProjectScoreCondition(TypedDict): - behavior: NotRequired[Literal["hidden"]] - when: ProjectScoreConditionWhen - - -class ProjectScoreConfigVisibility(TypedDict): - groups: NotRequired[Sequence[str] | None] - users: NotRequired[Sequence[str] | None] - - -ProjectScoreIdParam: TypeAlias = str -""" -ProjectScore id -""" - - -ProjectScoreName: TypeAlias = str -""" -Name of the project_score to search for -""" - - -ProjectScoreType: TypeAlias = Literal["slider", "categorical", "weighted", "minimum", "maximum", "online", "free-form"] -""" -The type of the configured score -""" - - -class ProjectSettingsRemoteEvalSource(TypedDict): - description: NotRequired[str | None] - name: NotRequired[str | None] - url: str - - -class ProjectSettingsSpanFieldOrderItem(TypedDict): - column_id: str - layout: NotRequired[Literal["full"] | Literal["two_column"] | None] - object_type: str - position: str - - -class ProjectSettings(TypedDict): - baseline_experiment_id: NotRequired[str | None] - """ - The id of the experiment to use as the default baseline for comparisons - """ - comparison_key: NotRequired[str | None] - """ - The key used to join two experiments (defaults to `input`) - """ - default_preprocessor: NotRequired[NullableSavedFunctionId] - disable_realtime_queries: NotRequired[bool | None] - """ - If true, disable real-time queries for this project. This can improve query performance for high-volume logs. - """ - remote_eval_sources: NotRequired[Sequence[ProjectSettingsRemoteEvalSource] | None] - """ - The remote eval sources to use for the project - """ - spanFieldOrder: NotRequired[Sequence[ProjectSettingsSpanFieldOrderItem] | None] - """ - The order of the fields to display in the trace view - """ - - -class ProjectTag(TypedDict): - color: NotRequired[str | None] - """ - Color of the tag for the UI - """ - created: NotRequired[str | None] - """ - Date of project tag creation - """ - description: NotRequired[str | None] - """ - Textual description of the project tag - """ - id: str - """ - Unique identifier for the project tag - """ - name: str - """ - Name of the project tag - """ - position: NotRequired[str | None] - """ - An optional LexoRank-based string that sets the sort position for the tag in the UI - """ - project_id: str - """ - Unique identifier for the project that the project tag belongs under - """ - user_id: str - - -ProjectTagIdParam: TypeAlias = str -""" -ProjectTag id -""" - - -ProjectTagName: TypeAlias = str -""" -Name of the project_tag to search for -""" - - -class PromptBlockDataPromptBlockData1(TypedDict): - content: str - type: Literal["completion"] - - -class PromptBlockDataNullishPromptBlockDataNullish1(TypedDict): - content: str - type: Literal["completion"] - - -class PromptDataMcp(TypedDict): - enabled_tools: NotRequired[Sequence[str] | None] - """ - If omitted, all tools are enabled - """ - id: str - is_disabled: NotRequired[bool] - type: Literal["id"] - - -class PromptDataMcp1(TypedDict): - enabled_tools: NotRequired[Sequence[str] | None] - """ - If omitted, all tools are enabled - """ - is_disabled: NotRequired[bool] - type: Literal["url"] - url: str - - -class PromptDataOrigin(TypedDict): - project_id: NotRequired[str] - prompt_id: NotRequired[str] - prompt_version: NotRequired[str] - - -class ToolFunctionToolFunction(TypedDict): - id: str - type: Literal["function"] - version: NotRequired[str] - """ - The version of the function - """ - - -class ToolFunctionToolFunction1(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -class ToolFunctionToolFunction2(TypedDict): - pass - - -class ToolFunctionToolFunction3(ToolFunctionToolFunction, ToolFunctionToolFunction2): - pass - - -class ToolFunctionToolFunction4(ToolFunctionToolFunction1, ToolFunctionToolFunction2): - pass - - -class ToolFunctionToolFunction5(ToolFunctionToolFunction, ToolFunctionToolFunction2): - pass - - -class ToolFunctionToolFunction6(ToolFunctionToolFunction1, ToolFunctionToolFunction2): - pass - - -ToolFunction: TypeAlias = ( - ToolFunctionToolFunction3 | ToolFunctionToolFunction4 | ToolFunctionToolFunction5 | ToolFunctionToolFunction6 -) - - -class PromptDataNullishMcp(TypedDict): - enabled_tools: NotRequired[Sequence[str] | None] - """ - If omitted, all tools are enabled - """ - id: str - is_disabled: NotRequired[bool] - type: Literal["id"] - - -class PromptDataNullishMcp1(TypedDict): - enabled_tools: NotRequired[Sequence[str] | None] - """ - If omitted, all tools are enabled - """ - is_disabled: NotRequired[bool] - type: Literal["url"] - url: str - - -class PromptDataNullishOrigin(TypedDict): - project_id: NotRequired[str] - prompt_id: NotRequired[str] - prompt_version: NotRequired[str] - - -class ToolFunction1ToolFunction1(TypedDict): - id: str - type: Literal["function"] - version: NotRequired[str] - """ - The version of the function - """ - - -class ToolFunction1ToolFunction11(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -class ToolFunction1ToolFunction12(TypedDict): - pass - - -class ToolFunction1ToolFunction13(ToolFunction1ToolFunction1, ToolFunction1ToolFunction12): - pass - - -class ToolFunction1ToolFunction14(ToolFunction1ToolFunction11, ToolFunction1ToolFunction12): - pass - - -class ToolFunction1ToolFunction15(ToolFunction1ToolFunction1, ToolFunction1ToolFunction12): - pass - - -class ToolFunction1ToolFunction16(ToolFunction1ToolFunction11, ToolFunction1ToolFunction12): - pass - - -ToolFunction1: TypeAlias = ( - ToolFunction1ToolFunction13 - | ToolFunction1ToolFunction14 - | ToolFunction1ToolFunction15 - | ToolFunction1ToolFunction16 -) - - -PromptEnvironment: TypeAlias = str -""" -Filter by environment slug. Cannot be used together with `version`. - -For `GET /v1/prompt`, environment resolution currently requires the request to match a single prompt. If multiple prompts match, the endpoint returns `400` (for example when `limit=1` is not set). Use `limit=1` or other filters (for example `slug`, `project_id`) to narrow results. -""" - - -PromptIdParam: TypeAlias = str -""" -Prompt id -""" - - -PromptName: TypeAlias = str -""" -Name of the prompt to search for -""" - - -class PromptParserNullish(TypedDict): - allow_no_match: NotRequired[bool] - """ - If true, adds a 'No match' option. When selected, no tag is deposited. - """ - choice: NotRequired[Sequence[str]] - """ - List of valid choices without score mapping. Used by classifiers that deposit output to tags. - """ - choice_scores: NotRequired[Mapping[str, float]] - """ - Map of choices to scores (0-1). Used by scorers. - """ - type: Literal["llm_classifier"] - use_cot: bool - - -PromptSessionIdParam: TypeAlias = str -""" -PromptSession id -""" - - -PromptSessionName: TypeAlias = str -""" -Name of the prompt_session to search for -""" - - -PromptVersion: TypeAlias = str -""" -Retrieve prompt at a specific version. - -The version id can either be a transaction id (e.g. '1000192656880881099') or a version identifier (e.g. '81cd05ee665fdfb3'). -""" - - -class RepoInfo(TypedDict): - author_email: NotRequired[str | None] - """ - Email of the author of the most recent commit - """ - author_name: NotRequired[str | None] - """ - Name of the author of the most recent commit - """ - branch: NotRequired[str | None] - """ - Name of the branch the most recent commit belongs to - """ - commit: NotRequired[str | None] - """ - SHA of most recent commit - """ - commit_message: NotRequired[str | None] - """ - Most recent commit message - """ - commit_time: NotRequired[str | None] - """ - Time of the most recent commit - """ - dirty: NotRequired[bool | None] - """ - Whether or not the repo had uncommitted changes when snapshotted - """ - git_diff: NotRequired[str | None] - """ - If the repo was dirty when run, this includes the diff between the current state of the repo and the most recent commit. - """ - tag: NotRequired[str | None] - """ - Name of the tag on the most recent commit - """ - - -class ResponseFormatJsonSchema(TypedDict): - description: NotRequired[str] - name: str - schema: NotRequired[Mapping[str, Any] | str] - strict: NotRequired[bool | None] - - -class ResponseFormatNullishResponseFormatNullish(TypedDict): - type: Literal["json_object"] - - -class ResponseFormatNullishResponseFormatNullish1(TypedDict): - json_schema: ResponseFormatJsonSchema - type: Literal["json_schema"] - - -class ResponseFormatNullishResponseFormatNullish2(TypedDict): - type: Literal["text"] - - -ResponseFormatNullish: TypeAlias = ( - ResponseFormatNullishResponseFormatNullish - | ResponseFormatNullishResponseFormatNullish1 - | ResponseFormatNullishResponseFormatNullish2 - | None -) - - -RetentionObjectType: TypeAlias = Literal["project_logs", "experiment", "dataset"] -""" -The object type that the retention policy applies to -""" - - -class RoleMemberPermission(TypedDict): - permission: Permission - restrict_object_type: NotRequired[AclObjectType | None] - - -class Role(TypedDict): - created: NotRequired[str | None] - """ - Date of role creation - """ - deleted_at: NotRequired[str | None] - """ - Date of role deletion, or null if the role is still active - """ - description: NotRequired[str | None] - """ - Textual description of the role - """ - id: str - """ - Unique identifier for the role - """ - member_permissions: NotRequired[Sequence[RoleMemberPermission] | None] - """ - (permission, restrict_object_type) tuples which belong to this role - """ - member_roles: NotRequired[Sequence[str] | None] - """ - Ids of the roles this role inherits from - - An inheriting role has all the permissions contained in its member roles, as well as all of their inherited permissions - """ - name: str - """ - Name of the role - """ - org_id: NotRequired[str | None] - """ - Unique id for the organization that the role belongs under - - A null org_id indicates a system role, which may be assigned to anybody and inherited by any other role, but cannot be edited. - - It is forbidden to change the org after creating a role - """ - user_id: NotRequired[str | None] - """ - Identifies the user who created the role - """ - - -RoleIdParam: TypeAlias = str -""" -Role id -""" - - -RoleName: TypeAlias = str -""" -Name of the role to search for -""" - - -class RunEvalData(TypedDict): - field_internal_btql: NotRequired[Mapping[str, Any] | None] - dataset_environment: NotRequired[str | None] - """ - The environment tag that resolves to the dataset version to evaluate - """ - dataset_id: str - dataset_version: NotRequired[str | None] - """ - The version of the dataset to evaluate - """ - - -class RunEvalData1(TypedDict): - field_internal_btql: NotRequired[Mapping[str, Any] | None] - dataset_environment: NotRequired[str | None] - """ - The environment tag that resolves to the dataset version to evaluate - """ - dataset_name: str - dataset_version: NotRequired[str | None] - """ - The version of the dataset to evaluate - """ - project_name: str - - -class RunEvalData2(TypedDict): - data: Sequence[Any] - - -class RunEvalMcpAuth(TypedDict): - oauth_token: NotRequired[str] - """ - The OAuth token to use - """ - - -class ParentParentRowIds(TypedDict): - id: str - """ - The id of the row - """ - root_span_id: str - """ - The root_span_id of the row - """ - span_id: str - """ - The span_id of the row - """ - - -class ParentParent(TypedDict): - object_id: str - """ - The id of the container object you are logging to - """ - object_type: Literal["project_logs", "experiment", "playground_logs"] - propagated_event: NotRequired[Mapping[str, Any] | None] - """ - Include these properties in every span created under this parent - """ - row_ids: NotRequired[ParentParentRowIds | None] - """ - Identifiers for the row to to log a subspan under - """ - - -class ParentParent1(TypedDict): - pass - - -class ParentParent2(ParentParent, ParentParent1): - pass - - -Parent: TypeAlias = ParentParent2 - - -class ScoreScore(TypedDict): - function_id: str - """ - The ID of the function - """ - version: NotRequired[str] - """ - The version of the function - """ - - -class ScoreScore1(TypedDict): - project_name: str - """ - The name of the project containing the function - """ - slug: str - """ - The slug of the function - """ - version: NotRequired[str] - """ - The version of the function - """ - - -class ScoreScore2(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - global_function: str - """ - The name of the global function. Currently, the global namespace includes the functions in autoevals - """ - - -class ScoreScore3(TypedDict): - prompt_session_function_id: str - """ - The ID of the function in the prompt session - """ - prompt_session_id: str - """ - The ID of the prompt session - """ - version: NotRequired[str] - """ - The version of the function - """ - - -class ScoreScore4InlineContext(TypedDict): - runtime: Literal["node", "python", "browser", "quickjs"] - version: str - - -class ScoreScore4(TypedDict): - code: str - """ - The inline code to execute - """ - function_type: NotRequired[FunctionTypeEnum] - inline_context: ScoreScore4InlineContext - name: NotRequired[str | None] - """ - The name of the inline code function - """ - - -class ScoreScore7(TypedDict): - pass - - -class ScoreScore8(ScoreScore, ScoreScore7): - pass - - -class ScoreScore9(ScoreScore1, ScoreScore7): - pass - - -class ScoreScore10(ScoreScore2, ScoreScore7): - pass - - -class ScoreScore11(ScoreScore3, ScoreScore7): - pass - - -class ScoreScore12(ScoreScore4, ScoreScore7): - pass - - -class SavedFunctionIdSavedFunctionId(TypedDict): - id: str - type: Literal["function"] - version: NotRequired[str] - """ - The version of the function - """ - - -class SavedFunctionIdSavedFunctionId1(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -SavedFunctionId: TypeAlias = SavedFunctionIdSavedFunctionId | SavedFunctionIdSavedFunctionId1 | None -""" -Optional function identifier that produced the classification -""" - - -class ScoreSummary(TypedDict): - diff: NotRequired[float] - """ - Difference in score between the current and comparison experiment - """ - improvements: int - """ - Number of improvements in the score - """ - name: str - """ - Name of the score - """ - regressions: int - """ - Number of regressions in the score - """ - score: float - """ - Average score across all examples - """ - - -class ServiceToken(TypedDict): - created: NotRequired[str | None] - """ - Date of service token creation - """ - id: str - """ - Unique identifier for the service token - """ - name: str - """ - Name of the service token - """ - org_id: NotRequired[str | None] - """ - Unique identifier for the organization - """ - preview_name: str - service_account_email: NotRequired[str | None] - """ - The service account email (not routable) - """ - service_account_id: NotRequired[str | None] - """ - Unique identifier for the service token - """ - service_account_name: NotRequired[str | None] - """ - The service account name - """ - - -ServiceTokenIdParam: TypeAlias = str -""" -ServiceToken id -""" - - -ServiceTokenName: TypeAlias = str -""" -Name of the service_token to search for -""" - - -Slug: TypeAlias = str -""" -Retrieve prompt with a specific slug -""" - - -class SpanIFrame(TypedDict): - created: NotRequired[str | None] - """ - Date of span iframe creation - """ - deleted_at: NotRequired[str | None] - """ - Date of span iframe deletion, or null if the span iframe is still active - """ - description: NotRequired[str | None] - """ - Textual description of the span iframe - """ - id: str - """ - Unique identifier for the span iframe - """ - name: str - """ - Name of the span iframe - """ - post_message: NotRequired[bool | None] - """ - Whether to post messages to the iframe containing the span's data. This is useful when you want to render more data than fits in the URL. - """ - project_id: str - """ - Unique identifier for the project that the span iframe belongs under - """ - url: str - """ - URL to embed the project viewer in an iframe - """ - user_id: NotRequired[str | None] - """ - Identifies the user who created the span iframe - """ - - -SpanIframeIdParam: TypeAlias = str -""" -SpanIframe id -""" - - -SpanIframeName: TypeAlias = str -""" -Name of the span_iframe to search for -""" - - -class SpanScope(TypedDict): - type: Literal["span"] - - -SpanType: TypeAlias = ( - Literal[ - "llm", - "score", - "function", - "eval", - "task", - "tool", - "automation", - "facet", - "preprocessor", - "classifier", - "review", - ] - | None -) -""" -Type of the span, for display purposes only -""" - - -StartingAfter: TypeAlias = str -""" -Pagination cursor id. - -For example, if the final item in the last page you fetched had an id of `foo`, pass `starting_after=foo` to fetch the next page. Note: you may only pass one of `starting_after` and `ending_before` -""" - - -StreamingMode: TypeAlias = Literal["auto", "parallel", "json", "text"] | None -""" -The mode format of the returned value (defaults to 'auto') -""" - - -SummarizeData: TypeAlias = bool | None -""" -Whether to summarize the data. If false (or omitted), only the metadata will be returned. -""" - - -class SummarizeDatasetResponse(TypedDict): - data_summary: NotRequired[DataSummary | None] - dataset_name: str - """ - Name of the dataset - """ - dataset_url: str - """ - URL to the dataset's page in the Braintrust app - """ - project_name: str - """ - Name of the project that the dataset belongs to - """ - project_url: str - """ - URL to the project's page in the Braintrust app - """ - - -class SummarizeExperimentResponse(TypedDict): - comparison_experiment_name: NotRequired[str | None] - """ - The experiment which scores are baselined against - """ - experiment_name: str - """ - Name of the experiment - """ - experiment_url: str - """ - URL to the experiment's page in the Braintrust app - """ - metrics: NotRequired[Mapping[str, MetricSummary] | None] - """ - Summary of the experiment's metrics - """ - project_name: str - """ - Name of the project that the experiment belongs to - """ - project_url: str - """ - URL to the project's page in the Braintrust app - """ - scores: NotRequired[Mapping[str, ScoreSummary] | None] - """ - Summary of the experiment's scores - """ - - -SummarizeScores: TypeAlias = bool | None -""" -Whether to summarize the scores and metrics. If false (or omitted), only the metadata will be returned. -""" - - -TopicAutomationConfigBackfillTimeRange = TypedDict( - "TopicAutomationConfigBackfillTimeRange", - { - "from": str, - "to": str, - }, -) - - -class FacetFunctionFacetFunction(TypedDict): - id: str - type: Literal["function"] - version: NotRequired[str] - """ - The version of the function - """ - - -class FacetFunctionFacetFunction1(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -class FacetFunctionFacetFunction2(TypedDict): - pass - - -class FacetFunctionFacetFunction3(FacetFunctionFacetFunction, FacetFunctionFacetFunction2): - pass - - -class FacetFunctionFacetFunction4(FacetFunctionFacetFunction1, FacetFunctionFacetFunction2): - pass - - -class FacetFunctionFacetFunction5(FacetFunctionFacetFunction, FacetFunctionFacetFunction2): - pass - - -class FacetFunctionFacetFunction6(FacetFunctionFacetFunction1, FacetFunctionFacetFunction2): - pass - - -FacetFunction: TypeAlias = ( - FacetFunctionFacetFunction3 - | FacetFunctionFacetFunction4 - | FacetFunctionFacetFunction5 - | FacetFunctionFacetFunction6 -) - - -class TopicAutomationDataScopeTopicAutomationDataScope(TypedDict): - type: Literal["project_logs"] - - -class TopicAutomationDataScopeTopicAutomationDataScope1(TypedDict): - type: Literal["project_experiments"] - - -class TopicAutomationDataScopeTopicAutomationDataScope2(TypedDict): - experiment_id: str - type: Literal["experiment"] - - -TopicAutomationDataScope: TypeAlias = ( - TopicAutomationDataScopeTopicAutomationDataScope - | TopicAutomationDataScopeTopicAutomationDataScope1 - | TopicAutomationDataScopeTopicAutomationDataScope2 - | None -) -""" -Optional data scope for topic automation. -""" - - -TopicAutomationFacetModel: TypeAlias = Literal["brain-facet-latest", "brain-facet-1", "brain-facet-2"] | None -""" -Optional facet model override for topic automation -""" - - -class TopicDigestAutomationConfigAction(TypedDict): - channel: str - """ - The Slack channel ID to post to - """ - message_template: NotRequired[str] - """ - Custom message template for the alert - """ - type: Literal["slack"] - """ - The type of action to take - """ - workspace_id: str - """ - The Slack workspace ID to post to - """ - - -class TopicDigestAutomationConfig(TypedDict): - action: TopicDigestAutomationConfigAction - """ - The Slack action to take when the digest is sent - """ - event_type: Literal["topic_digest"] - """ - The type of automation. - """ - scheduled_time_minutes_utc: int - """ - Minutes after midnight UTC when the digest should be sent - """ - status: NotRequired[AutomationStatus] - topic_map_function_ids: NotRequired[Sequence[str]] - """ - Optional topic map function IDs to include in the digest - """ - window_seconds: NotRequired[int] - """ - How much recent history to include in each digest - """ - - -class Function1Function1(TypedDict): - id: str - type: Literal["function"] - version: NotRequired[str] - """ - The version of the function - """ - - -class Function1Function11(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -class Function1Function12(TypedDict): - pass - - -class Function1Function13(Function1Function1, Function1Function12): - pass - - -class Function1Function14(Function1Function11, Function1Function12): - pass - - -class Function1Function15(Function1Function1, Function1Function12): - pass - - -class Function1Function16(Function1Function11, Function1Function12): - pass - - -Function1: TypeAlias = Function1Function13 | Function1Function14 | Function1Function15 | Function1Function16 - - -class TopicMapFunctionAutomation(TypedDict): - btql_filter: NotRequired[str | None] - """ - Per-topic-map BTQL filter. For trace scope, a topic map runs when max(filter) over the trace is truthy. For span scope, it runs when the current span matches. - """ - function: Function1 - - -class TopicMapGenerationSettings(TypedDict): - algorithm: Literal["hdbscan", "kmeans"] - dimension_reduction: Literal["umap", "pca", "none"] - hierarchy_threshold: NotRequired[int] - min_cluster_size: NotRequired[int] - min_samples: NotRequired[int] - n_clusters: NotRequired[int] - naming_model: NotRequired[str] - sample_size: NotRequired[int] - - -class TraceScope(TypedDict): - idle_seconds: NotRequired[float] - """ - Consider trace complete after this many seconds of inactivity (default: 30) - """ - type: Literal["trace"] - - -class User(TypedDict): - avatar_url: NotRequired[str | None] - """ - URL of the user's Avatar image - """ - created: NotRequired[str | None] - """ - Date of user creation - """ - email: NotRequired[str | None] - """ - The user's email - """ - family_name: NotRequired[str | None] - """ - Family name of the user - """ - given_name: NotRequired[str | None] - """ - Given name of the user - """ - id: str - """ - Unique identifier for the user - """ - - -UserEmail: TypeAlias = str | Sequence[str] -""" -Email of the user to search for. You may pass the param multiple times to filter for more than one email -""" - - -UserFamilyName: TypeAlias = str | Sequence[str] -""" -Family name of the user to search for. You may pass the param multiple times to filter for more than one family name -""" - - -UserGivenName: TypeAlias = str | Sequence[str] -""" -Given name of the user to search for. You may pass the param multiple times to filter for more than one given name -""" - - -UserIdParam: TypeAlias = str -""" -User id -""" - - -Version: TypeAlias = str -""" -Retrieve a snapshot of events from a past time - -The version id is essentially a filter on the latest event transaction id. You can use the `max_xact_id` returned by a past fetch as the version to reproduce that exact fetch. -""" - - -class ViewDataSearch(TypedDict): - filter: NotRequired[Sequence[Any] | None] - match: NotRequired[Sequence[Any] | None] - sort: NotRequired[Sequence[Any] | None] - tag: NotRequired[Sequence[Any] | None] - - -ViewIdParam: TypeAlias = str -""" -View id -""" - - -ViewName: TypeAlias = str -""" -Name of the view to search for -""" - - -class ViewOptionsViewOptionsOptions(TypedDict): - chartVisibility: NotRequired[Mapping[str, bool] | None] - frameEnd: NotRequired[str | None] - frameStart: NotRequired[str | None] - groupBy: NotRequired[str | None] - projectId: NotRequired[str | None] - rangeValue: NotRequired[str | None] - spanType: NotRequired[Literal["range", "frame"] | None] - type: NotRequired[Literal["project", "experiment"] | None] - tzUTC: NotRequired[bool | None] - - -class ViewOptionsViewOptions(TypedDict): - freezeColumns: NotRequired[bool | None] - options: ViewOptionsViewOptionsOptions - viewType: Literal["monitor"] - - -class ViewOptionsViewOptions1ChartAnnotation(TypedDict): - id: str - text: str - - -class ViewOptionsViewOptions1ExcludedMeasure(TypedDict): - type: Literal["none", "score", "metric", "metadata"] - value: str - - -class ViewOptionsViewOptions1SymbolGrouping(TypedDict): - type: Literal["none", "score", "metric", "metadata"] - value: str - - -ViewOptionsViewOptions1TimeRangeFilter = TypedDict( - "ViewOptionsViewOptions1TimeRangeFilter", - { - "from": str, - "to": str, - }, -) - - -class ViewOptionsViewOptions1XAxis(TypedDict): - type: Literal["none", "score", "metric", "metadata"] - value: str - - -class ViewOptionsViewOptions1YMetric(TypedDict): - type: Literal["none", "score", "metric", "metadata"] - value: str - - -class ViewOptionsViewOptions1(TypedDict): - chartAnnotations: NotRequired[Sequence[ViewOptionsViewOptions1ChartAnnotation] | None] - chartHeight: NotRequired[float | None] - cluster: NotRequired[str | None] - columnOrder: NotRequired[Sequence[str] | None] - columnSizing: NotRequired[Mapping[str, float] | None] - columnVisibility: NotRequired[Mapping[str, bool] | None] - excludedMeasures: NotRequired[Sequence[ViewOptionsViewOptions1ExcludedMeasure] | None] - freezeColumns: NotRequired[bool | None] - grouping: NotRequired[str | None] - layout: NotRequired[str | None] - queryShape: NotRequired[Literal["traces", "spans", "topics"] | None] - rowHeight: NotRequired[str | None] - symbolGrouping: NotRequired[ViewOptionsViewOptions1SymbolGrouping | None] - tallGroupRows: NotRequired[bool | None] - timeRangeFilter: NotRequired[str | ViewOptionsViewOptions1TimeRangeFilter | None] - topicMapReportKey: NotRequired[str | None] - xAxis: NotRequired[ViewOptionsViewOptions1XAxis | None] - xAxisAggregation: NotRequired[str | None] - """ - One of 'avg', 'sum', 'min', 'max', 'median', 'all' - """ - yMetric: NotRequired[ViewOptionsViewOptions1YMetric | None] - - -ViewOptions: TypeAlias = ViewOptionsViewOptions | ViewOptionsViewOptions1 | None -""" -Options for the view in the app -""" - - -ViewType: TypeAlias = ( - Literal[ - "projects", - "experiments", - "experiment", - "playgrounds", - "playground", - "datasets", - "dataset", - "prompts", - "parameters", - "tools", - "scorers", - "classifiers", - "logs", - "monitor", - "for_review_project_log", - "for_review_experiments", - "for_review_datasets", - ] - | None -) -""" -Type of object that the view corresponds to. -""" - - -class Acl(TypedDict): - field_object_org_id: str - """ - The organization the ACL's referred object belongs to - """ - created: NotRequired[str | None] - """ - Date of acl creation - """ - group_id: NotRequired[str | None] - """ - Id of the group the ACL applies to. Exactly one of `user_id` and `group_id` will be provided - """ - id: str - """ - Unique identifier for the acl - """ - object_id: str - """ - The id of the object the ACL applies to - """ - object_type: AclObjectType - permission: NotRequired[Permission | None] - """ - Permission the ACL grants. Exactly one of `permission` and `role_id` will be provided - """ - restrict_object_type: NotRequired[AclObjectType | None] - """ - When setting a permission directly, optionally restricts the permission grant to just the specified object type. Cannot be set alongside a `role_id`. - """ - role_id: NotRequired[str | None] - """ - Id of the role the ACL grants. Exactly one of `permission` and `role_id` will be provided - """ - user_id: NotRequired[str | None] - """ - Id of the user the ACL applies to. Exactly one of `user_id` and `group_id` will be provided - """ - - -class AclBatchUpdateResponse(TypedDict): - added_acls: Sequence[Acl] - removed_acls: Sequence[Acl] - - -class AclItem(TypedDict): - group_id: NotRequired[str | None] - """ - Id of the group the ACL applies to. Exactly one of `user_id` and `group_id` will be provided - """ - object_id: str - """ - The id of the object the ACL applies to - """ - object_type: AclObjectType - permission: NotRequired[Permission | None] - """ - Permission the ACL grants. Exactly one of `permission` and `role_id` will be provided - """ - restrict_object_type: NotRequired[AclObjectType | None] - """ - When setting a permission directly, optionally restricts the permission grant to just the specified object type. Cannot be set alongside a `role_id`. - """ - role_id: NotRequired[str | None] - """ - Id of the role the ACL grants. Exactly one of `permission` and `role_id` will be provided - """ - user_id: NotRequired[str | None] - """ - Id of the user the ACL applies to. Exactly one of `user_id` and `group_id` will be provided - """ - - -class PreprocessorPreprocessor1(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -class PreprocessorPreprocessor4(PreprocessorPreprocessor1, PreprocessorPreprocessor2): - pass - - -Preprocessor: TypeAlias = PreprocessorPreprocessor3 | PreprocessorPreprocessor4 - - -ChatCompletionContentPart: TypeAlias = ( - ChatCompletionContentPartTextWithTitle - | ChatCompletionContentPartImageWithTitle - | ChatCompletionContentPartFileWithTitle -) - - -class ChatCompletionMessageParamChatCompletionMessageParam1(TypedDict): - content: NotRequired[str | Sequence[ChatCompletionContentPart]] - name: NotRequired[str] - role: Literal["user"] - - -class ChatCompletionMessageParamChatCompletionMessageParam2(TypedDict): - content: NotRequired[str | Sequence[ChatCompletionContentPartText] | None] - function_call: NotRequired[ChatCompletionMessageParamChatCompletionMessageParam2FunctionCall | None] - name: NotRequired[str | None] - reasoning: NotRequired[Sequence[ChatCompletionMessageReasoning] | None] - reasoning_signature: NotRequired[str | None] - role: Literal["assistant"] - tool_calls: NotRequired[Sequence[ChatCompletionMessageToolCall] | None] - - -ChatCompletionMessageParam: TypeAlias = ( - ChatCompletionMessageParamChatCompletionMessageParam - | ChatCompletionMessageParamChatCompletionMessageParam1 - | ChatCompletionMessageParamChatCompletionMessageParam2 - | ChatCompletionMessageParamChatCompletionMessageParam3 - | ChatCompletionMessageParamChatCompletionMessageParam4 - | ChatCompletionMessageParamChatCompletionMessageParam5 - | ChatCompletionMessageParamChatCompletionMessageParam6 -) - - -class CreateExperiment(TypedDict): - base_exp_id: NotRequired[str | None] - """ - Id of default base experiment to compare against when viewing this experiment - """ - dataset_id: NotRequired[str | None] - """ - Identifier of the linked dataset, or null if the experiment is not linked to a dataset - """ - dataset_version: NotRequired[str | None] - """ - Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified. - """ - description: NotRequired[str | None] - """ - Textual description of the experiment - """ - ensure_new: NotRequired[bool | None] - """ - Normally, creating an experiment with the same name as an existing experiment will return the existing one un-modified. But if `ensure_new` is true, registration will generate a new experiment with a unique name in case of a conflict. - """ - internal_metadata: NotRequired[CreateExperimentInternalMetadata | None] - """ - Braintrust-controlled metadata about the experiment. - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - User-controlled metadata about the experiment - """ - name: NotRequired[str | None] - """ - Name of the experiment. Within a project, experiment names are unique - """ - parameters_id: NotRequired[str | None] - """ - Identifier of the linked saved parameters object, or null if the experiment is not linked to saved parameters - """ - parameters_version: NotRequired[str | None] - """ - Version number of the linked saved parameters object the experiment was run against. - """ - project_id: str - """ - Unique identifier for the project that the experiment belongs under - """ - public: NotRequired[bool | None] - """ - Whether or not the experiment is public. Public experiments can be viewed by anybody inside or outside the organization - """ - repo_info: NotRequired[RepoInfo | None] - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the experiment - """ - - -class CreateProjectAutomationConfig1(TypedDict): - batch_size: NotRequired[float | None] - """ - The number of rows to export in each batch - """ - credentials: CreateProjectAutomationConfig1Credentials | CreateProjectAutomationConfig1Credentials1 - event_type: Literal["btql_export"] - """ - The type of automation. - """ - export_definition: ( - CreateProjectAutomationConfig1ExportDefinition - | CreateProjectAutomationConfig1ExportDefinition1 - | CreateProjectAutomationConfig1ExportDefinition2 - ) - """ - The definition of what to export - """ - export_path: str - """ - The path to export the results to. It should include the storage protocol and prefix, e.g. s3://bucket-name/path/to/export - """ - format: Literal["jsonl", "parquet"] - """ - The format to export the results in - """ - interval_seconds: float - """ - Perform the triggered action at most once in this interval of seconds - """ - scope: NotRequired[SpanScope | TraceScope | GroupScope | None] - """ - Execution scope for export automation. Defaults to span-level execution. - """ - status: NotRequired[AutomationStatus] - - -class CreateProjectAutomationConfig3(TypedDict): - event_type: Literal["retention"] - """ - The type of automation. - """ - object_type: RetentionObjectType - retention_days: float - """ - The number of days to retain the object - """ - - -class CreateRoleMemberPermission(TypedDict): - permission: Permission - restrict_object_type: NotRequired[AclObjectType | None] - - -class CreateRole(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the role - """ - member_permissions: NotRequired[Sequence[CreateRoleMemberPermission] | None] - """ - (permission, restrict_object_type) tuples which belong to this role - """ - member_roles: NotRequired[Sequence[str] | None] - """ - Ids of the roles this role inherits from - - An inheriting role has all the permissions contained in its member roles, as well as all of their inherited permissions - """ - name: str - """ - Name of the role - """ - org_name: NotRequired[str | None] - """ - For nearly all users, this parameter should be unnecessary. But in the rare case that your API key belongs to multiple organizations, you may specify the name of the organization the role belongs in. - """ - - -class CrossObjectInsertResponse(TypedDict): - dataset: NotRequired[Mapping[str, InsertEventsResponse] | None] - """ - A mapping from dataset id to row ids for inserted `events` - """ - experiment: NotRequired[Mapping[str, InsertEventsResponse] | None] - """ - A mapping from experiment id to row ids for inserted `events` - """ - project_logs: NotRequired[Mapping[str, InsertEventsResponse] | None] - """ - A mapping from project id to row ids for inserted `events` - """ - - -class DatasetEventClassification(TypedDict): - confidence: NotRequired[float | None] - """ - Optional confidence score for the classification - """ - id: str - """ - Stable classification identifier - """ - label: NotRequired[str] - """ - Original label of the classification item, which is useful for search and indexing purposes - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - Optional metadata associated with the classification - """ - source: NotRequired[SavedFunctionId] - - -class DatasetEvent(TypedDict): - field_pagination_key: NotRequired[str | None] - """ - A stable, time-ordered key that can be used to paginate over dataset events. This field is auto-generated by Braintrust and only exists in Brainstore. - """ - field_xact_id: str - """ - The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the dataset (see the `version` parameter) - """ - audit_data: NotRequired[Sequence[Any] | None] - """ - Optional list of audit entries attached to this event - """ - classifications: NotRequired[Mapping[str, Sequence[DatasetEventClassification]] | None] - """ - Classifications for this event (dictionary from classification name to items) - """ - comments: NotRequired[Sequence[Any] | None] - """ - Optional list of comments attached to this event - """ - created: str - """ - The timestamp the dataset event was created - """ - dataset_id: str - """ - Unique identifier for the dataset - """ - expected: NotRequired[Any | None] - """ - The output of your application, including post-processing (an arbitrary, JSON serializable object) - """ - facets: NotRequired[Mapping[str, str | None] | None] - """ - Facets for categorization (dictionary from facet id to value) - """ - id: str - """ - A unique identifier for the dataset event. If you don't provide one, Braintrust will generate one for you - """ - input: NotRequired[Any | None] - """ - The argument that uniquely define an input case (an arbitrary, JSON serializable object) - """ - is_root: NotRequired[bool | None] - """ - Whether this span is a root span - """ - metadata: NotRequired[DatasetEventMetadata | None] - """ - A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings - """ - origin: NotRequired[ObjectReferenceNullish | None] - project_id: str - """ - Unique identifier for the project that the dataset belongs under - """ - root_span_id: str - """ - A unique identifier for the trace this dataset event belongs to - """ - span_id: str - """ - A unique identifier used to link different dataset events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags to log - """ - - -class Experiment(TypedDict): - base_exp_id: NotRequired[str | None] - """ - Id of default base experiment to compare against when viewing this experiment - """ - commit: NotRequired[str | None] - """ - Commit, taken directly from `repo_info.commit` - """ - created: NotRequired[str | None] - """ - Date of experiment creation - """ - dataset_id: NotRequired[str | None] - """ - Identifier of the linked dataset, or null if the experiment is not linked to a dataset - """ - dataset_version: NotRequired[str | None] - """ - Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified. - """ - deleted_at: NotRequired[str | None] - """ - Date of experiment deletion, or null if the experiment is still active - """ - description: NotRequired[str | None] - """ - Textual description of the experiment - """ - id: str - """ - Unique identifier for the experiment - """ - internal_metadata: NotRequired[ExperimentInternalMetadata | None] - """ - Braintrust-controlled metadata about the experiment. - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - User-controlled metadata about the experiment - """ - name: str - """ - Name of the experiment. Within a project, experiment names are unique - """ - parameters_id: NotRequired[str | None] - """ - Identifier of the linked saved parameters object, or null if the experiment is not linked to saved parameters - """ - parameters_version: NotRequired[str | None] - """ - Version number of the linked saved parameters object the experiment was run against. - """ - project_id: str - """ - Unique identifier for the project that the experiment belongs under - """ - public: bool - """ - Whether or not the experiment is public. Public experiments can be viewed by anybody inside or outside the organization - """ - repo_info: NotRequired[RepoInfo | None] - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the experiment - """ - user_id: NotRequired[str | None] - """ - Identifies the user who created the experiment - """ - - -class ExperimentEventClassification(TypedDict): - confidence: NotRequired[float | None] - """ - Optional confidence score for the classification - """ - id: str - """ - Stable classification identifier - """ - label: NotRequired[str] - """ - Original label of the classification item, which is useful for search and indexing purposes - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - Optional metadata associated with the classification - """ - source: NotRequired[SavedFunctionId] - - -class Preprocessor1Preprocessor11(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -class Preprocessor1Preprocessor14(Preprocessor1Preprocessor11, Preprocessor1Preprocessor12): - pass - - -Preprocessor1: TypeAlias = Preprocessor1Preprocessor13 | Preprocessor1Preprocessor14 - - -class FacetData(TypedDict): - embedding_model: NotRequired[str] - """ - The embedding model to use for vectorizing facet results. - """ - model: NotRequired[str] - """ - The model to use for facet extraction - """ - no_match_pattern: NotRequired[str] - """ - Regex pattern to identify outputs that do not match the facet. If the output matches, the facet will be saved as 'no_match' - """ - preprocessor: NotRequired[Preprocessor1] - prompt: str - """ - The prompt to use for LLM extraction. The preprocessed text will be provided as context. - """ - type: Literal["facet"] - - -class FeedbackDatasetEventRequest(TypedDict): - feedback: Sequence[FeedbackDatasetItem] - """ - A list of dataset feedback items - """ - - -class FeedbackExperimentEventRequest(TypedDict): - feedback: Sequence[FeedbackExperimentItem] - """ - A list of experiment feedback items - """ - - -class FeedbackProjectLogsEventRequest(TypedDict): - feedback: Sequence[FeedbackProjectLogsItem] - """ - A list of project logs feedback items - """ - - -class FetchDatasetEventsResponse(TypedDict): - cursor: NotRequired[str | None] - """ - Pagination cursor - - Pass this string directly as the `cursor` param to your next fetch request to get the next page of results. Not provided if the returned result set is empty. - """ - events: Sequence[DatasetEvent] - """ - A list of fetched events - """ - - -class FetchEventsRequest(TypedDict): - cursor: NotRequired[FetchPaginationCursor | None] - limit: NotRequired[FetchLimit | None] - max_root_span_id: NotRequired[MaxRootSpanId | None] - max_xact_id: NotRequired[MaxXactId | None] - version: NotRequired[Version | None] - - -class FunctionDataFunctionData3(TypedDict): - config: NotRequired[Mapping[str, Any] | None] - """ - Configuration options to pass to the global function (e.g., for preprocessor customization) - """ - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -class FunctionDataNullishFunctionDataNullish3(TypedDict): - config: NotRequired[Mapping[str, Any] | None] - """ - Configuration options to pass to the global function (e.g., for preprocessor customization) - """ - function_type: NotRequired[FunctionTypeEnum] - name: str - type: Literal["global"] - - -class FunctionIdFunctionId2(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - global_function: str - """ - The name of the global function. Currently, the global namespace includes the functions in autoevals - """ - - -class FunctionIdFunctionId4(TypedDict): - code: str - """ - The inline code to execute - """ - function_type: NotRequired[FunctionTypeEnum] - inline_context: FunctionIdFunctionId4InlineContext - name: NotRequired[str | None] - """ - The name of the inline code function - """ - - -class InsertDatasetEvent(TypedDict): - field_array_delete: NotRequired[Sequence[InsertDatasetEventFieldArrayDeleteItem] | None] - """ - The `_array_delete` field allows removing specific values from array fields. It is an array of objects with `path` and `delete` properties. - - For example, to remove tags "foo" and "bar" from an existing row: `{"_is_merge": true, "_array_delete": [{"path": ["tags"], "delete": ["foo", "bar"]}]}`. For nested fields like `metadata.categories`, use `[{"path": ["metadata", "categories"], "delete": ["value"]}]`. This will remove those specific values from the array while preserving others. - """ - field_is_merge: NotRequired[bool | None] - """ - The `_is_merge` field controls how the row is merged with any existing row with the same id in the DB. By default (or when set to `false`), the existing row is completely replaced by the new row. When set to `true`, the new row is deep-merged into the existing row, if one is found. If no existing row is found, the new row is inserted as is. - - For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": 5, "b": 10}}`. If we merge a new row as `{"_is_merge": true, "id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"a": 5, "b": 11, "c": 20}}`. If we replace the new row as `{"id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"b": 11, "c": 20}}` - """ - field_merge_paths: NotRequired[Sequence[Sequence[str]] | None] - """ - The `_merge_paths` field allows controlling the depth of the merge, when `_is_merge=true`. `_merge_paths` is a list of paths, where each path is a list of field names. The deep merge will not descend below any of the specified merge paths. - - For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": {"b": 10}, "c": {"d": 20}}, "output": {"a": 20}}`. If we merge a new row as `{"_is_merge": true, "_merge_paths": [["input", "a"], ["output"]], "input": {"a": {"q": 30}, "c": {"e": 30}, "bar": "baz"}, "output": {"d": 40}}`, the new row will be `{"id": "foo": "input": {"a": {"q": 30}, "c": {"d": 20, "e": 30}, "bar": "baz"}, "output": {"d": 40}}`. In this case, due to the merge paths, we have replaced `input.a` and `output`, but have still deep-merged `input` and `input.c`. - """ - field_object_delete: NotRequired[bool | None] - """ - Pass `_object_delete=true` to mark the dataset event deleted. Deleted events will not show up in subsequent fetches for this dataset - """ - field_parent_id: NotRequired[str | None] - """ - DEPRECATED: The `_parent_id` field is deprecated and should not be used. Support for `_parent_id` will be dropped in a future version of Braintrust. Log `span_id`, `root_span_id`, and `span_parents` explicitly instead. - - Use the `_parent_id` field to create this row as a subspan of an existing row. Tracking hierarchical relationships are important for tracing (see the [guide](https://www.braintrust.dev/docs/instrument) for full details). - - For example, say we have logged a row `{"id": "abc", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"_parent_id": "abc", "id": "llm_call", "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - created: NotRequired[str | None] - """ - The timestamp the dataset event was created - """ - expected: NotRequired[Any | None] - """ - The output of your application, including post-processing (an arbitrary, JSON serializable object) - """ - facets: NotRequired[Mapping[str, str | None] | None] - """ - Facets for categorization (dictionary from facet id to value) - """ - id: NotRequired[str | None] - """ - A unique identifier for the dataset event. If you don't provide one, Braintrust will generate one for you - """ - input: NotRequired[Any | None] - """ - The argument that uniquely define an input case (an arbitrary, JSON serializable object) - """ - metadata: NotRequired[InsertDatasetEventMetadata | None] - """ - A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings - """ - origin: NotRequired[ObjectReferenceNullish | None] - root_span_id: NotRequired[str | None] - """ - Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. - - For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - span_id: NotRequired[str | None] - """ - Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. - - For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - span_parents: NotRequired[Sequence[str] | None] - """ - Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. - - For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags to log - """ - - -class InsertDatasetEventRequest(TypedDict): - events: Sequence[InsertDatasetEvent] - """ - A list of dataset events to insert - """ - - -class InvokeApi(TypedDict): - expected: NotRequired[Any | None] - """ - The expected output of the function - """ - input: NotRequired[Any | None] - """ - Argument to the function, which can be any JSON serializable value - """ - mcp_auth: NotRequired[Mapping[str, InvokeApiMcpAuth]] - """ - Map of MCP server URL to auth credentials - """ - messages: NotRequired[Sequence[ChatCompletionMessageParam]] - """ - If the function is an LLM, additional messages to pass along to it - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - Any relevant metadata. This will be logged and available as the `metadata` argument. - """ - mode: NotRequired[StreamingMode | None] - overrides: NotRequired[Mapping[str, Any] | None] - """ - Partial function definition to merge with the function being invoked. Fields are validated against the function type's schema at runtime. For facets: { preprocessor?, prompt?, model? }. For prompts: { model?, ... }. - """ - parent: NotRequired[InvokeParent] - stream: NotRequired[bool | None] - """ - Whether to stream the response. If true, results will be returned in the Braintrust SSE format. - """ - strict: NotRequired[bool | None] - """ - If true, throw an error if one of the variables in the prompt is not present in the input - """ - tags: NotRequired[Sequence[str] | None] - """ - Any relevant tags to log on the span. - """ - version: NotRequired[str] - """ - The version of the function - """ - - -class ModelParamsModelParams(TypedDict): - frequency_penalty: NotRequired[float] - function_call: NotRequired[Literal["auto"] | Literal["none"] | ModelParamsModelParamsFunctionCall] - max_completion_tokens: NotRequired[float] - """ - The successor to max_tokens - """ - max_tokens: NotRequired[float] - n: NotRequired[float] - presence_penalty: NotRequired[float] - reasoning_budget: NotRequired[float] - reasoning_effort: NotRequired[Literal["none", "minimal", "low", "medium", "high"]] - reasoning_enabled: NotRequired[bool] - response_format: NotRequired[ResponseFormatNullish] - stop: NotRequired[Sequence[str]] - temperature: NotRequired[float] - tool_choice: NotRequired[ - Literal["auto"] | Literal["none"] | Literal["required"] | ModelParamsModelParamsToolChoice - ] - top_p: NotRequired[float] - use_cache: NotRequired[bool] - verbosity: NotRequired[Literal["low", "medium", "high"]] - - -ModelParams: TypeAlias = ( - ModelParamsModelParams - | ModelParamsModelParams1 - | ModelParamsModelParams2 - | ModelParamsModelParams3 - | ModelParamsModelParams4 -) - - -class OnlineScoreConfig(TypedDict): - apply_to_root_span: NotRequired[bool | None] - """ - Whether to trigger online scoring on the root span of each trace. Only applies when scope is 'span' or unset. - """ - apply_to_span_names: NotRequired[Sequence[str] | None] - """ - Trigger online scoring on any spans with a name in this list. Only applies when scope is 'span' or unset. - """ - btql_filter: NotRequired[str | None] - """ - Filter logs using BTQL - """ - sampling_rate: float - """ - The sampling rate for online scoring - """ - scope: NotRequired[SpanScope | TraceScope | GroupScope | None] - """ - The scope at which to run the functions. Defaults to span-level execution. - """ - scorers: Sequence[Scorer] - """ - The list of functions to run for online scoring. Can include scorers, facets, or other function types. - """ - skip_logging: NotRequired[bool | None] - """ - Whether to skip adding scorer spans when computing scores - """ - - -class PatchExperiment(TypedDict): - base_exp_id: NotRequired[str | None] - """ - Id of default base experiment to compare against when viewing this experiment - """ - dataset_id: NotRequired[str | None] - """ - Identifier of the linked dataset, or null if the experiment is not linked to a dataset - """ - dataset_version: NotRequired[str | None] - """ - Version number of the linked dataset the experiment was run against. This can be used to reproduce the experiment after the dataset has been modified. - """ - description: NotRequired[str | None] - """ - Textual description of the experiment - """ - internal_metadata: NotRequired[PatchExperimentInternalMetadata | None] - """ - Braintrust-controlled metadata about the experiment. - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - User-controlled metadata about the experiment - """ - name: NotRequired[str | None] - """ - Name of the experiment. Within a project, experiment names are unique - """ - parameters_id: NotRequired[str | None] - """ - Identifier of the linked saved parameters object, or null if the experiment is not linked to saved parameters - """ - parameters_version: NotRequired[str | None] - """ - Version number of the linked saved parameters object the experiment was run against. - """ - public: NotRequired[bool | None] - """ - Whether or not the experiment is public. Public experiments can be viewed by anybody inside or outside the organization - """ - repo_info: NotRequired[RepoInfo | None] - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the experiment - """ - - -class PatchProject(TypedDict): - description: NotRequired[str | None] - name: NotRequired[str | None] - """ - Name of the project - """ - settings: NotRequired[ProjectSettings] - user_id: NotRequired[str | None] - - -class PatchProjectAutomationConfig1(TypedDict): - batch_size: NotRequired[float | None] - """ - The number of rows to export in each batch - """ - credentials: PatchProjectAutomationConfig1Credentials | PatchProjectAutomationConfig1Credentials1 - event_type: Literal["btql_export"] - """ - The type of automation. - """ - export_definition: ( - PatchProjectAutomationConfig1ExportDefinition - | PatchProjectAutomationConfig1ExportDefinition1 - | PatchProjectAutomationConfig1ExportDefinition2 - ) - """ - The definition of what to export - """ - export_path: str - """ - The path to export the results to. It should include the storage protocol and prefix, e.g. s3://bucket-name/path/to/export - """ - format: Literal["jsonl", "parquet"] - """ - The format to export the results in - """ - interval_seconds: float - """ - Perform the triggered action at most once in this interval of seconds - """ - scope: NotRequired[SpanScope | TraceScope | GroupScope | None] - """ - Execution scope for export automation. Defaults to span-level execution. - """ - status: NotRequired[AutomationStatus] - - -class PatchProjectAutomationConfig3(TypedDict): - event_type: Literal["retention"] - """ - The type of automation. - """ - object_type: RetentionObjectType - retention_days: float - """ - The number of days to retain the object - """ - - -class PatchRoleAddMemberPermission(TypedDict): - permission: Permission - restrict_object_type: NotRequired[AclObjectType | None] - - -class PatchRoleRemoveMemberPermission(TypedDict): - permission: Permission - restrict_object_type: NotRequired[AclObjectType | None] - - -class PatchRole(TypedDict): - add_member_permissions: NotRequired[Sequence[PatchRoleAddMemberPermission] | None] - """ - A list of permissions to add to the role - """ - add_member_roles: NotRequired[Sequence[str] | None] - """ - A list of role IDs to add to the role's inheriting-from set - """ - description: NotRequired[str | None] - """ - Textual description of the role - """ - name: NotRequired[str | None] - """ - Name of the role - """ - remove_member_permissions: NotRequired[Sequence[PatchRoleRemoveMemberPermission] | None] - """ - A list of permissions to remove from the role - """ - remove_member_roles: NotRequired[Sequence[str] | None] - """ - A list of role IDs to remove from the role's inheriting-from set - """ - - -class Project(TypedDict): - created: NotRequired[str | None] - """ - Date of project creation - """ - deleted_at: NotRequired[str | None] - """ - Date of project deletion, or null if the project is still active - """ - description: NotRequired[str | None] - """ - Textual description of the project - """ - id: str - """ - Unique identifier for the project - """ - name: str - """ - Name of the project - """ - org_id: str - """ - Unique id for the organization that the project belongs under - """ - settings: NotRequired[ProjectSettings | None] - user_id: NotRequired[str | None] - """ - Identifies the user who created the project - """ - - -class ProjectAutomationConfig1(TypedDict): - batch_size: NotRequired[float | None] - """ - The number of rows to export in each batch - """ - credentials: ProjectAutomationConfig1Credentials | ProjectAutomationConfig1Credentials1 - event_type: Literal["btql_export"] - """ - The type of automation. - """ - export_definition: ( - ProjectAutomationConfig1ExportDefinition - | ProjectAutomationConfig1ExportDefinition1 - | ProjectAutomationConfig1ExportDefinition2 - ) - """ - The definition of what to export - """ - export_path: str - """ - The path to export the results to. It should include the storage protocol and prefix, e.g. s3://bucket-name/path/to/export - """ - format: Literal["jsonl", "parquet"] - """ - The format to export the results in - """ - interval_seconds: float - """ - Perform the triggered action at most once in this interval of seconds - """ - scope: NotRequired[SpanScope | TraceScope | GroupScope | None] - """ - Execution scope for export automation. Defaults to span-level execution. - """ - status: NotRequired[AutomationStatus] - - -class ProjectAutomationConfig3(TypedDict): - event_type: Literal["retention"] - """ - The type of automation. - """ - object_type: RetentionObjectType - retention_days: float - """ - The number of days to retain the object - """ - - -class ProjectLogsEventClassification(TypedDict): - confidence: NotRequired[float | None] - """ - Optional confidence score for the classification - """ - id: str - """ - Stable classification identifier - """ - label: NotRequired[str] - """ - Original label of the classification item, which is useful for search and indexing purposes - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - Optional metadata associated with the classification - """ - source: NotRequired[SavedFunctionId] - - -ProjectScoreCategories: TypeAlias = Sequence[ProjectScoreCategory] | Mapping[str, float] | Sequence[str] | None - - -class ProjectScoreConfig(TypedDict): - condition: NotRequired[ProjectScoreCondition | None] - destination: NotRequired[str | None] - multi_select: NotRequired[bool | None] - object_types: NotRequired[Sequence[Literal["project_logs", "dataset", "experiment"]] | None] - online: NotRequired[OnlineScoreConfig | None] - visibility: NotRequired[ProjectScoreConfigVisibility | None] - - -class PromptBlockDataPromptBlockData(TypedDict): - messages: Sequence[ChatCompletionMessageParam] - tools: NotRequired[str] - type: Literal["chat"] - - -PromptBlockData: TypeAlias = PromptBlockDataPromptBlockData | PromptBlockDataPromptBlockData1 - - -class PromptBlockDataNullishPromptBlockDataNullish(TypedDict): - messages: Sequence[ChatCompletionMessageParam] - tools: NotRequired[str] - type: Literal["chat"] - - -PromptBlockDataNullish: TypeAlias = ( - PromptBlockDataNullishPromptBlockDataNullish | PromptBlockDataNullishPromptBlockDataNullish1 | None -) - - -class PromptOptionsNullish(TypedDict): - model: NotRequired[str] - params: NotRequired[ModelParams] - position: NotRequired[str] - - -class SpanAttributes(TypedDict): - name: NotRequired[str | None] - """ - Name of the span, for display purposes only - """ - purpose: NotRequired[Literal["scorer"] | None] - """ - A special value that indicates the span was generated by a scoring automation - """ - type: NotRequired[SpanType | None] - - -class TopicAutomationConfig(TypedDict): - backfill_time_range: NotRequired[str | TopicAutomationConfigBackfillTimeRange | None] - """ - Topic window used for classification coverage and initial backfill. - """ - btql_filter: NotRequired[str | None] - """ - Optional BTQL filter applied before topic automation. - """ - data_scope: NotRequired[TopicAutomationDataScope] - event_type: Literal["topic"] - """ - The type of automation. - """ - facet_functions: Sequence[FacetFunction] - """ - Facet functions used by the topic automation - """ - facet_model: NotRequired[TopicAutomationFacetModel | None] - relabel_overlap_seconds: NotRequired[float | None] - """ - How much recent history to relabel after a new topic map version becomes active - """ - rerun_seconds: NotRequired[float | None] - """ - How often to recompute topic maps - """ - sampling_rate: float - """ - The sampling rate for topic automation - """ - scope: NotRequired[SpanScope | TraceScope | GroupScope | None] - """ - Execution scope for topic automation. Defaults to span-level execution. - """ - status: NotRequired[AutomationStatus] - topic_map_functions: Sequence[TopicMapFunctionAutomation] - """ - Topic map functions with optional per-topic-map filters - """ - - -class TopicMapData(TypedDict): - automation_btql_filter: NotRequired[str] - """ - Automation-level BTQL filter that was applied when this version was generated. Absent on versions generated before this was recorded. - """ - btql_filter: NotRequired[str] - """ - Per-topic-map BTQL filter that was applied when this version was generated. Absent on versions generated before this was recorded. - """ - bundle_key: NotRequired[str] - """ - Key of the topic map bundle in code_bundles bucket - """ - disable_reconciliation: NotRequired[bool] - """ - Whether new topic generation should ignore the previously saved report during reconciliation. Defaults to false when omitted. - """ - distance_threshold: NotRequired[float] - """ - Maximum distance to nearest centroid. If exceeded, returns no_match. - """ - embedding_model: str - """ - The embedding model to use for embedding facet values - """ - generation_settings: NotRequired[TopicMapGenerationSettings] - report_key: NotRequired[str] - """ - Key of the clustering report in code_bundles bucket - """ - source_facet: str - """ - The facet field name to use as input for classification - """ - topic_names: NotRequired[Mapping[str, str]] - """ - Mapping from topic_id to topic name - """ - type: Literal["topic_map"] - - -class ViewData(TypedDict): - custom_charts: NotRequired[Any | None] - search: NotRequired[ViewDataSearch | None] - - -class AclBatchUpdateRequest(TypedDict): - add_acls: NotRequired[Sequence[AclItem] | None] - remove_acls: NotRequired[Sequence[AclItem] | None] - - -class BatchedFacetDataTopicMap(TypedDict): - function_name: str - """ - The name of the topic map function - """ - topic_map_data: TopicMapData - topic_map_id: NotRequired[str] - """ - The id of the topic map function - """ - - -class BatchedFacetData(TypedDict): - facets: Sequence[BatchedFacetDataFacet] - preprocessor: NotRequired[Preprocessor] - topic_maps: NotRequired[Mapping[str, Sequence[BatchedFacetDataTopicMap]]] - """ - Topic maps that depend on facets in this batch, keyed by source facet name. Each source facet can have multiple topic maps. - """ - type: Literal["batched_facet"] - - -class CreateProjectAutomation(TypedDict): - config: ( - CreateProjectAutomationConfig - | CreateProjectAutomationConfig1 - | CreateProjectAutomationConfig2 - | CreateProjectAutomationConfig3 - | CreateProjectAutomationConfig4 - | TopicAutomationConfig - | TopicDigestAutomationConfig - ) - """ - The configuration for the automation rule - """ - description: NotRequired[str | None] - """ - Textual description of the project automation - """ - name: str - """ - Name of the project automation - """ - project_id: str - """ - Unique identifier for the project that the project automation belongs under - """ - - -class CreateProjectScore(TypedDict): - categories: NotRequired[ProjectScoreCategories] - config: NotRequired[ProjectScoreConfig | None] - description: NotRequired[str | None] - """ - Textual description of the project score - """ - name: str - """ - Name of the project score - """ - project_id: str - """ - Unique identifier for the project that the project score belongs under - """ - score_type: ProjectScoreType - - -class CreateView(TypedDict): - deleted_at: NotRequired[str | None] - """ - Date of role deletion, or null if the role is still active - """ - name: str - """ - Name of the view - """ - object_id: str - """ - The id of the object the view applies to - """ - object_type: AclObjectType - options: NotRequired[ViewOptions] - user_id: NotRequired[str | None] - """ - Identifies the user who created the view - """ - view_data: NotRequired[ViewData | None] - view_type: ( - Literal[ - "projects", - "experiments", - "experiment", - "playgrounds", - "playground", - "datasets", - "dataset", - "prompts", - "parameters", - "tools", - "scorers", - "classifiers", - "logs", - "monitor", - "for_review_project_log", - "for_review_experiments", - "for_review_datasets", - ] - | None - ) - """ - Type of object that the view corresponds to. - """ - - -class CrossObjectInsertRequestDataset(TypedDict): - events: NotRequired[Sequence[InsertDatasetEvent] | None] - """ - A list of dataset events to insert - """ - feedback: NotRequired[Sequence[FeedbackDatasetItem] | None] - """ - A list of dataset feedback items - """ - - -class ExperimentEvent(TypedDict): - field_pagination_key: NotRequired[str | None] - """ - A stable, time-ordered key that can be used to paginate over experiment events. This field is auto-generated by Braintrust and only exists in Brainstore. - """ - field_xact_id: str - """ - The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the experiment (see the `version` parameter) - """ - audit_data: NotRequired[Sequence[Any] | None] - """ - Optional list of audit entries attached to this event - """ - classifications: NotRequired[Mapping[str, Sequence[ExperimentEventClassification]] | None] - """ - Classifications for this event (dictionary from classification name to items) - """ - comments: NotRequired[Sequence[Any] | None] - """ - Optional list of comments attached to this event - """ - context: NotRequired[ExperimentEventContext | None] - """ - Context is additional information about the code that produced the experiment event. It is essentially the textual counterpart to `metrics`. Use the `caller_*` attributes to track the location in code which produced the experiment event - """ - created: str - """ - The timestamp the experiment event was created - """ - error: NotRequired[Any | None] - """ - The error that occurred, if any. - """ - expected: NotRequired[Any | None] - """ - The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models - """ - experiment_id: str - """ - Unique identifier for the experiment - """ - facets: NotRequired[Mapping[str, str | None] | None] - """ - Facets for categorization (dictionary from facet id to value) - """ - id: str - """ - A unique identifier for the experiment event. If you don't provide one, Braintrust will generate one for you - """ - input: NotRequired[Any | None] - """ - The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical - """ - is_root: NotRequired[bool | None] - """ - Whether this span is a root span - """ - metadata: NotRequired[ExperimentEventMetadata | None] - """ - A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings - """ - metrics: NotRequired[ExperimentEventMetrics | None] - """ - Metrics are numerical measurements tracking the execution of the code that produced the experiment event. Use "start" and "end" to track the time span over which the experiment event was produced - """ - origin: NotRequired[ObjectReferenceNullish | None] - output: NotRequired[Any | None] - """ - The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question - """ - project_id: str - """ - Unique identifier for the project that the experiment belongs under - """ - root_span_id: str - """ - A unique identifier for the trace this experiment event belongs to - """ - scores: NotRequired[Mapping[str, float | None] | None] - """ - A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments - """ - span_attributes: NotRequired[SpanAttributes | None] - span_id: str - """ - A unique identifier used to link different experiment events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing - """ - span_parents: NotRequired[Sequence[str] | None] - """ - An array of the parent `span_ids` of this experiment event. This should be empty for the root span of a trace, and should most often contain just one parent element for subspans - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags to log - """ - - -class FetchExperimentEventsResponse(TypedDict): - cursor: NotRequired[str | None] - """ - Pagination cursor - - Pass this string directly as the `cursor` param to your next fetch request to get the next page of results. Not provided if the returned result set is empty. - """ - events: Sequence[ExperimentEvent] - """ - A list of fetched events - """ - - -class GraphNodeGraphNode7(TypedDict): - description: NotRequired[str | None] - """ - The description of the node - """ - position: NotRequired[GraphNodeGraphNode7Position | None] - """ - The position of the node - """ - prompt: PromptBlockData - type: Literal["prompt_template"] - - -GraphNode: TypeAlias = ( - GraphNodeGraphNode - | GraphNodeGraphNode1 - | GraphNodeGraphNode2 - | GraphNodeGraphNode3 - | GraphNodeGraphNode4 - | GraphNodeGraphNode5 - | GraphNodeGraphNode6 - | GraphNodeGraphNode7 -) - - -class InsertExperimentEvent(TypedDict): - field_array_delete: NotRequired[Sequence[InsertExperimentEventFieldArrayDeleteItem] | None] - """ - The `_array_delete` field allows removing specific values from array fields. It is an array of objects with `path` and `delete` properties. - - For example, to remove tags "foo" and "bar" from an existing row: `{"_is_merge": true, "_array_delete": [{"path": ["tags"], "delete": ["foo", "bar"]}]}`. For nested fields like `metadata.categories`, use `[{"path": ["metadata", "categories"], "delete": ["value"]}]`. This will remove those specific values from the array while preserving others. - """ - field_is_merge: NotRequired[bool | None] - """ - The `_is_merge` field controls how the row is merged with any existing row with the same id in the DB. By default (or when set to `false`), the existing row is completely replaced by the new row. When set to `true`, the new row is deep-merged into the existing row, if one is found. If no existing row is found, the new row is inserted as is. - - For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": 5, "b": 10}}`. If we merge a new row as `{"_is_merge": true, "id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"a": 5, "b": 11, "c": 20}}`. If we replace the new row as `{"id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"b": 11, "c": 20}}` - """ - field_merge_paths: NotRequired[Sequence[Sequence[str]] | None] - """ - The `_merge_paths` field allows controlling the depth of the merge, when `_is_merge=true`. `_merge_paths` is a list of paths, where each path is a list of field names. The deep merge will not descend below any of the specified merge paths. - - For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": {"b": 10}, "c": {"d": 20}}, "output": {"a": 20}}`. If we merge a new row as `{"_is_merge": true, "_merge_paths": [["input", "a"], ["output"]], "input": {"a": {"q": 30}, "c": {"e": 30}, "bar": "baz"}, "output": {"d": 40}}`, the new row will be `{"id": "foo": "input": {"a": {"q": 30}, "c": {"d": 20, "e": 30}, "bar": "baz"}, "output": {"d": 40}}`. In this case, due to the merge paths, we have replaced `input.a` and `output`, but have still deep-merged `input` and `input.c`. - """ - field_object_delete: NotRequired[bool | None] - """ - Pass `_object_delete=true` to mark the experiment event deleted. Deleted events will not show up in subsequent fetches for this experiment - """ - field_parent_id: NotRequired[str | None] - """ - DEPRECATED: The `_parent_id` field is deprecated and should not be used. Support for `_parent_id` will be dropped in a future version of Braintrust. Log `span_id`, `root_span_id`, and `span_parents` explicitly instead. - - Use the `_parent_id` field to create this row as a subspan of an existing row. Tracking hierarchical relationships are important for tracing (see the [guide](https://www.braintrust.dev/docs/instrument) for full details). - - For example, say we have logged a row `{"id": "abc", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"_parent_id": "abc", "id": "llm_call", "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - context: NotRequired[InsertExperimentEventContext | None] - """ - Context is additional information about the code that produced the experiment event. It is essentially the textual counterpart to `metrics`. Use the `caller_*` attributes to track the location in code which produced the experiment event - """ - created: NotRequired[str | None] - """ - The timestamp the experiment event was created - """ - error: NotRequired[Any | None] - """ - The error that occurred, if any. - """ - expected: NotRequired[Any | None] - """ - The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate your experiments while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models - """ - facets: NotRequired[Mapping[str, str | None] | None] - """ - Facets for categorization (dictionary from facet id to value) - """ - id: NotRequired[str | None] - """ - A unique identifier for the experiment event. If you don't provide one, Braintrust will generate one for you - """ - input: NotRequired[Any | None] - """ - The arguments that uniquely define a test case (an arbitrary, JSON serializable object). Later on, Braintrust will use the `input` to know whether two test cases are the same between experiments, so they should not contain experiment-specific state. A simple rule of thumb is that if you run the same experiment twice, the `input` should be identical - """ - metadata: NotRequired[InsertExperimentEventMetadata | None] - """ - A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings - """ - metrics: NotRequired[InsertExperimentEventMetrics | None] - """ - Metrics are numerical measurements tracking the execution of the code that produced the experiment event. Use "start" and "end" to track the time span over which the experiment event was produced - """ - origin: NotRequired[ObjectReferenceNullish | None] - output: NotRequired[Any | None] - """ - The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question - """ - root_span_id: NotRequired[str | None] - """ - Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. - - For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - scores: NotRequired[Mapping[str, float | None] | None] - """ - A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare experiments - """ - span_attributes: NotRequired[SpanAttributes | None] - span_id: NotRequired[str | None] - """ - Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. - - For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - span_parents: NotRequired[Sequence[str] | None] - """ - Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. - - For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags to log - """ - - -class InsertExperimentEventRequest(TypedDict): - events: Sequence[InsertExperimentEvent] - """ - A list of experiment events to insert - """ - - -class InsertProjectLogsEvent(TypedDict): - field_array_delete: NotRequired[Sequence[InsertProjectLogsEventFieldArrayDeleteItem] | None] - """ - The `_array_delete` field allows removing specific values from array fields. It is an array of objects with `path` and `delete` properties. - - For example, to remove tags "foo" and "bar" from an existing row: `{"_is_merge": true, "_array_delete": [{"path": ["tags"], "delete": ["foo", "bar"]}]}`. For nested fields like `metadata.categories`, use `[{"path": ["metadata", "categories"], "delete": ["value"]}]`. This will remove those specific values from the array while preserving others. - """ - field_is_merge: NotRequired[bool | None] - """ - The `_is_merge` field controls how the row is merged with any existing row with the same id in the DB. By default (or when set to `false`), the existing row is completely replaced by the new row. When set to `true`, the new row is deep-merged into the existing row, if one is found. If no existing row is found, the new row is inserted as is. - - For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": 5, "b": 10}}`. If we merge a new row as `{"_is_merge": true, "id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"a": 5, "b": 11, "c": 20}}`. If we replace the new row as `{"id": "foo", "input": {"b": 11, "c": 20}}`, the new row will be `{"id": "foo", "input": {"b": 11, "c": 20}}` - """ - field_merge_paths: NotRequired[Sequence[Sequence[str]] | None] - """ - The `_merge_paths` field allows controlling the depth of the merge, when `_is_merge=true`. `_merge_paths` is a list of paths, where each path is a list of field names. The deep merge will not descend below any of the specified merge paths. - - For example, say there is an existing row in the DB `{"id": "foo", "input": {"a": {"b": 10}, "c": {"d": 20}}, "output": {"a": 20}}`. If we merge a new row as `{"_is_merge": true, "_merge_paths": [["input", "a"], ["output"]], "input": {"a": {"q": 30}, "c": {"e": 30}, "bar": "baz"}, "output": {"d": 40}}`, the new row will be `{"id": "foo": "input": {"a": {"q": 30}, "c": {"d": 20, "e": 30}, "bar": "baz"}, "output": {"d": 40}}`. In this case, due to the merge paths, we have replaced `input.a` and `output`, but have still deep-merged `input` and `input.c`. - """ - field_object_delete: NotRequired[bool | None] - """ - Pass `_object_delete=true` to mark the project logs event deleted. Deleted events will not show up in subsequent fetches for this project logs - """ - field_parent_id: NotRequired[str | None] - """ - DEPRECATED: The `_parent_id` field is deprecated and should not be used. Support for `_parent_id` will be dropped in a future version of Braintrust. Log `span_id`, `root_span_id`, and `span_parents` explicitly instead. - - Use the `_parent_id` field to create this row as a subspan of an existing row. Tracking hierarchical relationships are important for tracing (see the [guide](https://www.braintrust.dev/docs/instrument) for full details). - - For example, say we have logged a row `{"id": "abc", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"_parent_id": "abc", "id": "llm_call", "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - context: NotRequired[InsertProjectLogsEventContext | None] - """ - Context is additional information about the code that produced the project logs event. It is essentially the textual counterpart to `metrics`. Use the `caller_*` attributes to track the location in code which produced the project logs event - """ - created: NotRequired[str | None] - """ - The timestamp the project logs event was created - """ - error: NotRequired[Any | None] - """ - The error that occurred, if any. - """ - expected: NotRequired[Any | None] - """ - The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models. - """ - facets: NotRequired[Mapping[str, str | None] | None] - """ - Facets for categorization (dictionary from facet id to value) - """ - id: NotRequired[str | None] - """ - A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you - """ - input: NotRequired[Any | None] - """ - The arguments that uniquely define a user input (an arbitrary, JSON serializable object). - """ - metadata: NotRequired[InsertProjectLogsEventMetadata | None] - """ - A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings - """ - metrics: NotRequired[InsertProjectLogsEventMetrics | None] - """ - Metrics are numerical measurements tracking the execution of the code that produced the project logs event. Use "start" and "end" to track the time span over which the project logs event was produced - """ - origin: NotRequired[ObjectReferenceNullish | None] - output: NotRequired[Any | None] - """ - The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question. - """ - root_span_id: NotRequired[str | None] - """ - Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. - - For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - scores: NotRequired[Mapping[str, float | None] | None] - """ - A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. - """ - span_attributes: NotRequired[SpanAttributes | None] - span_id: NotRequired[str | None] - """ - Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. - - For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - span_parents: NotRequired[Sequence[str] | None] - """ - Use `span_id`, `root_span_id`, and `span_parents` instead of `_parent_id`, which is now deprecated. The span_id is a unique identifier describing the row's place in the a trace, and the root_span_id is a unique identifier for the whole trace. See the [guide](https://www.braintrust.dev/docs/instrument) for full details. - - For example, say we have logged a row `{"id": "abc", "span_id": "span0", "root_span_id": "root_span0", "input": "foo", "output": "bar", "expected": "boo", "scores": {"correctness": 0.33}}`. We can create a sub-span of the parent row by logging `{"id": "llm_call", "span_id": "span1", "root_span_id": "root_span0", "span_parents": ["span0"], "input": {"prompt": "What comes after foo?"}, "output": "bar", "metrics": {"tokens": 1}}`. In the webapp, only the root span row `"abc"` will show up in the summary view. You can view the full trace hierarchy (in this case, the `"llm_call"` row) by clicking on the "abc" row. - - If the row is being merged into an existing row, this field will be ignored. - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags to log - """ - - -class InsertProjectLogsEventRequest(TypedDict): - events: Sequence[InsertProjectLogsEvent] - """ - A list of project logs events to insert - """ - - -class PatchProjectAutomation(TypedDict): - config: NotRequired[ - PatchProjectAutomationConfig - | PatchProjectAutomationConfig1 - | PatchProjectAutomationConfig2 - | PatchProjectAutomationConfig3 - | PatchProjectAutomationConfig4 - | TopicAutomationConfig - | TopicDigestAutomationConfig - | Any - ] - """ - The configuration for the automation rule - """ - description: NotRequired[str | None] - """ - Textual description of the project automation - """ - name: NotRequired[str | None] - """ - Name of the project automation - """ - - -class PatchProjectScore(TypedDict): - categories: NotRequired[ProjectScoreCategories] - config: NotRequired[ProjectScoreConfig | None] - description: NotRequired[str | None] - """ - Textual description of the project score - """ - name: NotRequired[str | None] - """ - Name of the project score - """ - score_type: NotRequired[ProjectScoreType | None] - - -class PatchView(TypedDict): - name: NotRequired[str | None] - """ - Name of the view - """ - object_id: str - """ - The id of the object the view applies to - """ - object_type: AclObjectType - options: NotRequired[ViewOptions] - user_id: NotRequired[str | None] - """ - Identifies the user who created the view - """ - view_data: NotRequired[ViewData | None] - view_type: NotRequired[ - Literal[ - "projects", - "experiments", - "experiment", - "playgrounds", - "playground", - "datasets", - "dataset", - "prompts", - "parameters", - "tools", - "scorers", - "classifiers", - "logs", - "monitor", - "for_review_project_log", - "for_review_experiments", - "for_review_datasets", - ] - | None - ] - """ - Type of object that the view corresponds to. - """ - - -class ProjectAutomation(TypedDict): - config: ( - ProjectAutomationConfig - | ProjectAutomationConfig1 - | ProjectAutomationConfig2 - | ProjectAutomationConfig3 - | ProjectAutomationConfig4 - | TopicAutomationConfig - | TopicDigestAutomationConfig - ) - """ - The configuration for the automation rule - """ - created: NotRequired[str | None] - """ - Date of project automation creation - """ - description: NotRequired[str | None] - """ - Textual description of the project automation - """ - id: str - """ - Unique identifier for the project automation - """ - name: str - """ - Name of the project automation - """ - project_id: str - """ - Unique identifier for the project that the project automation belongs under - """ - user_id: NotRequired[str | None] - """ - Identifies the user who created the project automation - """ - - -class ProjectLogsEvent(TypedDict): - field_async_scoring_state: NotRequired[Any | None] - """ - The async scoring state for this event - """ - field_pagination_key: NotRequired[str | None] - """ - A stable, time-ordered key that can be used to paginate over project logs events. This field is auto-generated by Braintrust and only exists in Brainstore. - """ - field_xact_id: str - """ - The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the project logs (see the `version` parameter) - """ - audit_data: NotRequired[Sequence[Any] | None] - """ - Optional list of audit entries attached to this event - """ - classifications: NotRequired[Mapping[str, Sequence[ProjectLogsEventClassification]] | None] - """ - Classifications for this event (dictionary from classification name to items) - """ - comments: NotRequired[Sequence[Any] | None] - """ - Optional list of comments attached to this event - """ - context: NotRequired[ProjectLogsEventContext | None] - """ - Context is additional information about the code that produced the project logs event. It is essentially the textual counterpart to `metrics`. Use the `caller_*` attributes to track the location in code which produced the project logs event - """ - created: str - """ - The timestamp the project logs event was created - """ - error: NotRequired[Any | None] - """ - The error that occurred, if any. - """ - expected: NotRequired[Any | None] - """ - The ground truth value (an arbitrary, JSON serializable object) that you'd compare to `output` to determine if your `output` value is correct or not. Braintrust currently does not compare `output` to `expected` for you, since there are so many different ways to do that correctly. Instead, these values are just used to help you navigate while digging into analyses. However, we may later use these values to re-score outputs or fine-tune your models. - """ - facets: NotRequired[Mapping[str, str | None] | None] - """ - Facets for categorization (dictionary from facet id to value) - """ - id: str - """ - A unique identifier for the project logs event. If you don't provide one, Braintrust will generate one for you - """ - input: NotRequired[Any | None] - """ - The arguments that uniquely define a user input (an arbitrary, JSON serializable object). - """ - is_root: NotRequired[bool | None] - """ - Whether this span is a root span - """ - log_id: Literal["g"] - """ - A literal 'g' which identifies the log as a project log - """ - metadata: NotRequired[ProjectLogsEventMetadata | None] - """ - A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings - """ - metrics: NotRequired[ProjectLogsEventMetrics | None] - """ - Metrics are numerical measurements tracking the execution of the code that produced the project logs event. Use "start" and "end" to track the time span over which the project logs event was produced - """ - org_id: str - """ - Unique id for the organization that the project belongs under - """ - origin: NotRequired[ObjectReferenceNullish | None] - output: NotRequired[Any | None] - """ - The output of your application, including post-processing (an arbitrary, JSON serializable object), that allows you to determine whether the result is correct or not. For example, in an app that generates SQL queries, the `output` should be the _result_ of the SQL query generated by the model, not the query itself, because there may be multiple valid queries that answer a single question. - """ - project_id: str - """ - Unique identifier for the project - """ - root_span_id: str - """ - A unique identifier for the trace this project logs event belongs to - """ - scores: NotRequired[Mapping[str, float | None] | None] - """ - A dictionary of numeric values (between 0 and 1) to log. The scores should give you a variety of signals that help you determine how accurate the outputs are compared to what you expect and diagnose failures. For example, a summarization app might have one score that tells you how accurate the summary is, and another that measures the word similarity between the generated and grouth truth summary. The word similarity score could help you determine whether the summarization was covering similar concepts or not. You can use these scores to help you sort, filter, and compare logs. - """ - span_attributes: NotRequired[SpanAttributes | None] - span_id: str - """ - A unique identifier used to link different project logs events together as part of a full trace. See the [tracing guide](https://www.braintrust.dev/docs/instrument) for full details on tracing - """ - span_parents: NotRequired[Sequence[str] | None] - """ - An array of the parent `span_ids` of this project logs event. This should be empty for the root span of a trace, and should most often contain just one parent element for subspans - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags to log - """ - - -class ProjectScore(TypedDict): - categories: NotRequired[ProjectScoreCategories] - config: NotRequired[ProjectScoreConfig | None] - created: NotRequired[str | None] - """ - Date of project score creation - """ - description: NotRequired[str | None] - """ - Textual description of the project score - """ - id: str - """ - Unique identifier for the project score - """ - name: str - """ - Name of the project score - """ - position: NotRequired[str | None] - """ - An optional LexoRank-based string that sets the sort position for the score in the UI - """ - project_id: str - """ - Unique identifier for the project that the project score belongs under - """ - score_type: ProjectScoreType - user_id: str - - -class PromptData(TypedDict): - mcp: NotRequired[Mapping[str, PromptDataMcp | PromptDataMcp1] | None] - options: NotRequired[PromptOptionsNullish | None] - origin: NotRequired[PromptDataOrigin | None] - parser: NotRequired[PromptParserNullish | None] - prompt: NotRequired[PromptBlockDataNullish] - template_format: NotRequired[Literal["mustache", "nunjucks", "none"] | None] - tool_functions: NotRequired[Sequence[ToolFunction] | None] - - -class PromptDataNullish(TypedDict): - mcp: NotRequired[Mapping[str, PromptDataNullishMcp | PromptDataNullishMcp1] | None] - options: NotRequired[PromptOptionsNullish | None] - origin: NotRequired[PromptDataNullishOrigin | None] - parser: NotRequired[PromptParserNullish | None] - prompt: NotRequired[PromptBlockDataNullish] - template_format: NotRequired[Literal["mustache", "nunjucks", "none"] | None] - tool_functions: NotRequired[Sequence[ToolFunction1] | None] - - -class ScoreScore5(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - inline_function: Mapping[str, Any] - inline_prompt: NotRequired[PromptData] - name: NotRequired[str | None] - """ - The name of the inline function - """ - - -class ScoreScore6(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - inline_prompt: PromptData - name: NotRequired[str | None] - """ - The name of the inline prompt - """ - - -class ScoreScore13(ScoreScore5, ScoreScore7): - pass - - -class ScoreScore14(ScoreScore6, ScoreScore7): - pass - - -Score: TypeAlias = ScoreScore8 | ScoreScore9 | ScoreScore10 | ScoreScore11 | ScoreScore12 | ScoreScore13 | ScoreScore14 - - -class View(TypedDict): - created: NotRequired[str | None] - """ - Date of view creation - """ - deleted_at: NotRequired[str | None] - """ - Date of role deletion, or null if the role is still active - """ - id: str - """ - Unique identifier for the view - """ - name: str - """ - Name of the view - """ - object_id: str - """ - The id of the object the view applies to - """ - object_type: AclObjectType - options: NotRequired[ViewOptions] - user_id: NotRequired[str | None] - """ - Identifies the user who created the view - """ - view_data: NotRequired[ViewData | None] - view_type: ( - Literal[ - "projects", - "experiments", - "experiment", - "playgrounds", - "playground", - "datasets", - "dataset", - "prompts", - "parameters", - "tools", - "scorers", - "classifiers", - "logs", - "monitor", - "for_review_project_log", - "for_review_experiments", - "for_review_datasets", - ] - | None - ) - """ - Type of object that the view corresponds to. - """ - - -class CreatePrompt(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the prompt - """ - function_type: NotRequired[FunctionTypeEnumNullish | None] - name: str - """ - Name of the prompt - """ - project_id: str - """ - Unique identifier for the project that the prompt belongs under - """ - prompt_data: NotRequired[PromptDataNullish | None] - slug: str - """ - Unique identifier for the prompt - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the prompt - """ - - -class CrossObjectInsertRequestExperiment(TypedDict): - events: NotRequired[Sequence[InsertExperimentEvent] | None] - """ - A list of experiment events to insert - """ - feedback: NotRequired[Sequence[FeedbackExperimentItem] | None] - """ - A list of experiment feedback items - """ - - -class CrossObjectInsertRequestProjectLogs(TypedDict): - events: NotRequired[Sequence[InsertProjectLogsEvent] | None] - """ - A list of project logs events to insert - """ - feedback: NotRequired[Sequence[FeedbackProjectLogsItem] | None] - """ - A list of project logs feedback items - """ - - -class CrossObjectInsertRequest(TypedDict): - dataset: NotRequired[Mapping[str, CrossObjectInsertRequestDataset] | None] - """ - A mapping from dataset id to a set of log events and feedback items to insert - """ - experiment: NotRequired[Mapping[str, CrossObjectInsertRequestExperiment] | None] - """ - A mapping from experiment id to a set of log events and feedback items to insert - """ - project_logs: NotRequired[Mapping[str, CrossObjectInsertRequestProjectLogs] | None] - """ - A mapping from project id to a set of log events and feedback items to insert - """ - - -class FetchProjectLogsEventsResponse(TypedDict): - cursor: NotRequired[str | None] - """ - Pagination cursor - - Pass this string directly as the `cursor` param to your next fetch request to get the next page of results. Not provided if the returned result set is empty. - """ - events: Sequence[ProjectLogsEvent] - """ - A list of fetched events - """ - - -class FunctionIdFunctionId5(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - inline_function: Mapping[str, Any] - inline_prompt: NotRequired[PromptData] - name: NotRequired[str | None] - """ - The name of the inline function - """ - - -class FunctionIdFunctionId6(TypedDict): - function_type: NotRequired[FunctionTypeEnum] - inline_prompt: PromptData - name: NotRequired[str | None] - """ - The name of the inline prompt - """ - - -FunctionId: TypeAlias = ( - FunctionIdFunctionId - | FunctionIdFunctionId1 - | FunctionIdFunctionId2 - | FunctionIdFunctionId3 - | FunctionIdFunctionId4 - | FunctionIdFunctionId5 - | FunctionIdFunctionId6 -) -""" -The function to evaluate -""" - - -class GraphData(TypedDict): - edges: Mapping[str, GraphEdge] - nodes: Mapping[str, GraphNode] - type: Literal["graph"] - - -class PatchPrompt(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the prompt - """ - name: NotRequired[str | None] - """ - Name of the prompt - """ - prompt_data: NotRequired[PromptDataNullish | None] - slug: NotRequired[str | None] - """ - Unique identifier for the prompt - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the prompt - """ - - -class Prompt(TypedDict): - field_xact_id: str - """ - The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the prompt (see the `version` parameter) - """ - created: NotRequired[str | None] - """ - Date of prompt creation - """ - description: NotRequired[str | None] - """ - Textual description of the prompt - """ - function_type: NotRequired[FunctionTypeEnumNullish | None] - id: str - """ - Unique identifier for the prompt - """ - log_id: Literal["p"] - """ - A literal 'p' which identifies the object as a project prompt - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - User-controlled metadata about the prompt - """ - name: str - """ - Name of the prompt - """ - org_id: str - """ - Unique identifier for the organization - """ - project_id: str - """ - Unique identifier for the project that the prompt belongs under - """ - prompt_data: NotRequired[PromptDataNullish | None] - slug: str - """ - Unique identifier for the prompt - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the prompt - """ - - -class RunEval(TypedDict): - base_experiment_id: NotRequired[str | None] - """ - An optional experiment id to use as a base. If specified, the new experiment will be summarized and compared to this experiment. - """ - base_experiment_name: NotRequired[str | None] - """ - An optional experiment name to use as a base. If specified, the new experiment will be summarized and compared to this experiment. - """ - data: RunEvalData | RunEvalData1 | RunEvalData2 - """ - The dataset to use - """ - experiment_name: NotRequired[str] - """ - An optional name for the experiment created by this eval. If it conflicts with an existing experiment, it will be suffixed with a unique identifier. - """ - extra_messages: NotRequired[str] - """ - A template path of extra messages to append to the conversion. These messages will be appended to the end of the conversation, after the last message. - """ - git_metadata_settings: NotRequired[GitMetadataSettings | None] - is_public: NotRequired[bool | None] - """ - Whether the experiment should be public. Defaults to false. - """ - max_concurrency: NotRequired[float | None] - """ - The maximum number of tasks/scorers that will be run concurrently. Defaults to 10. If null is provided, no max concurrency will be used. - """ - mcp_auth: NotRequired[Mapping[str, RunEvalMcpAuth]] - metadata: NotRequired[Mapping[str, Any]] - """ - Optional experiment-level metadata to store about the evaluation. You can later use this to slice & dice across experiments. - """ - name: NotRequired[str] - """ - The name of the eval to run when multiple evals available - """ - parameters: NotRequired[Mapping[str, Any]] - """ - Values for any parameters used in the eval - """ - parent: NotRequired[Parent] - project_id: str - """ - Unique identifier for the project to run the eval in - """ - repo_info: NotRequired[RepoInfo] - scores: Sequence[Score] - """ - The functions to score the eval on - """ - stop_token: NotRequired[str | None] - """ - The token to stop the run - """ - stream: NotRequired[bool] - """ - Whether to stream the results of the eval. If true, the request will return two events: one to indicate the experiment has started, and another upon completion. If false, the request will return the evaluation's summary upon completion. - """ - strict: NotRequired[bool | None] - """ - If true, throw an error if one of the variables in the prompt is not present in the input - """ - tags: NotRequired[Sequence[str]] - """ - Optional tags that will be added to the experiment. - """ - task: FunctionId - timeout: NotRequired[float | None] - """ - The maximum duration, in milliseconds, to run the evaluation. Defaults to undefined, in which case there is no timeout. - """ - trial_count: NotRequired[float | None] - """ - The number of times to run the evaluator per input. This is useful for evaluating applications that have non-deterministic behavior and gives you both a stronger aggregate measure and a sense of the variance in the results. - """ - - -FunctionData: TypeAlias = ( - FunctionDataFunctionData - | FunctionDataFunctionData1 - | GraphData - | FunctionDataFunctionData2 - | FunctionDataFunctionData3 - | FacetData - | BatchedFacetData - | FunctionDataFunctionData4 - | TopicMapData -) - - -FunctionDataNullish: TypeAlias = ( - FunctionDataNullishFunctionDataNullish - | FunctionDataNullishFunctionDataNullish1 - | GraphData - | FunctionDataNullishFunctionDataNullish2 - | FunctionDataNullishFunctionDataNullish3 - | FacetData - | BatchedFacetData - | FunctionDataNullishFunctionDataNullish4 - | TopicMapData - | None -) - - -class PatchFunction(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the prompt - """ - function_data: NotRequired[FunctionDataNullish] - name: NotRequired[str | None] - """ - Name of the prompt - """ - prompt_data: NotRequired[PromptDataNullish | None] - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the prompt - """ - - -class CreateFunction(TypedDict): - description: NotRequired[str | None] - """ - Textual description of the prompt - """ - function_data: FunctionData - function_schema: NotRequired[CreateFunctionFunctionSchema | None] - """ - JSON schema for the function's parameters and return type - """ - function_type: NotRequired[FunctionTypeEnumNullish | None] - name: str - """ - Name of the prompt - """ - origin: NotRequired[CreateFunctionOrigin | None] - project_id: str - """ - Unique identifier for the project that the prompt belongs under - """ - prompt_data: NotRequired[PromptDataNullish | None] - slug: str - """ - Unique identifier for the prompt - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the prompt - """ - - -class Function(TypedDict): - field_xact_id: str - """ - The transaction id of an event is unique to the network operation that processed the event insertion. Transaction ids are monotonically increasing over time and can be used to retrieve a versioned snapshot of the prompt (see the `version` parameter) - """ - created: NotRequired[str | None] - """ - Date of prompt creation - """ - description: NotRequired[str | None] - """ - Textual description of the prompt - """ - function_data: FunctionData - function_schema: NotRequired[FunctionFunctionSchema | None] - """ - JSON schema for the function's parameters and return type - """ - function_type: NotRequired[FunctionTypeEnumNullish | None] - id: str - """ - Unique identifier for the prompt - """ - log_id: Literal["p"] - """ - A literal 'p' which identifies the object as a project prompt - """ - metadata: NotRequired[Mapping[str, Any] | None] - """ - User-controlled metadata about the prompt - """ - name: str - """ - Name of the prompt - """ - org_id: str - """ - Unique identifier for the organization - """ - origin: NotRequired[FunctionOrigin | None] - project_id: str - """ - Unique identifier for the project that the prompt belongs under - """ - prompt_data: NotRequired[PromptDataNullish | None] - slug: str - """ - Unique identifier for the prompt - """ - tags: NotRequired[Sequence[str] | None] - """ - A list of tags for the prompt - """ diff --git a/py/src/braintrust/api/_generated/models/__init__.py b/py/src/braintrust/api/_generated/models/__init__.py new file mode 100644 index 00000000..e7736608 --- /dev/null +++ b/py/src/braintrust/api/_generated/models/__init__.py @@ -0,0 +1,9 @@ +# Generated by scripts/generate-api-client.py. DO NOT EDIT. +# OpenAPI commit: 9daf27f19d9e0340304d7a3e7d0edb28380b94c6 +# OpenAPI spec SHA-256: 5ec753c0263c0c44cd04f741edfc7e8bad491cc25a2113d029e84edc076520f0 +# datamodel-code-generator: 0.72.4 +# ruff: 0.15.21 +# Generator Python: 3.14 +# Content SHA-256: cc2dbd2430abddbceff7f693feaffa844fa536663621674e67d06a78aca61f17 + +"""Generated private model types.""" diff --git a/py/src/braintrust/api/_generated/models/projects.py b/py/src/braintrust/api/_generated/models/projects.py new file mode 100644 index 00000000..781797a6 --- /dev/null +++ b/py/src/braintrust/api/_generated/models/projects.py @@ -0,0 +1,200 @@ +# Generated by scripts/generate-api-client.py. DO NOT EDIT. +# OpenAPI commit: 9daf27f19d9e0340304d7a3e7d0edb28380b94c6 +# OpenAPI spec SHA-256: 5ec753c0263c0c44cd04f741edfc7e8bad491cc25a2113d029e84edc076520f0 +# datamodel-code-generator: 0.72.4 +# ruff: 0.15.21 +# Generator Python: 3.14 +# Content SHA-256: 9736eda7e5e0dd80b0e149a4275c4a94e8da78149af79cc692006076f109107f + +from typing import Literal, TypeAlias, TypedDict +from typing_extensions import NotRequired +from collections.abc import Sequence + + +AppLimitParam: TypeAlias = int | None +""" +Limit the number of objects to return +""" + + +class CreateProject(TypedDict): + description: NotRequired[str | None] + """ + Textual description of the project + """ + name: str + """ + Name of the project + """ + org_name: NotRequired[str | None] + """ + For nearly all users, this parameter should be unnecessary. But in the rare case that your API key belongs to multiple organizations, you may specify the name of the organization the project belongs in. + """ + + +EndingBefore: TypeAlias = str +""" +Pagination cursor id. + +For example, if the initial item in the last page you fetched had an id of `foo`, pass `ending_before=foo` to fetch the previous page. Note: you may only pass one of `starting_after` and `ending_before` +""" + + +FunctionTypeEnum: TypeAlias = ( + Literal[ + "llm", + "scorer", + "task", + "tool", + "custom_view", + "preprocessor", + "facet", + "classifier", + "tag", + "parameters", + "sandbox", + ] + | None +) +""" +The type of global function. Defaults to 'scorer'. +""" + + +Ids: TypeAlias = str | Sequence[str] +""" +Filter search results to a particular set of object IDs. To specify a list of IDs, include the query param multiple times +""" + + +class NullableSavedFunctionId1(TypedDict): + id: str + type: Literal["function"] + version: NotRequired[str] + """ + The version of the function + """ + + +class NullableSavedFunctionId2(TypedDict): + function_type: NotRequired[FunctionTypeEnum] + name: str + type: Literal["global"] + + +NullableSavedFunctionId: TypeAlias = NullableSavedFunctionId1 | NullableSavedFunctionId2 | None +""" +Default preprocessor for this project. When set, functions that use preprocessors will use this instead of their built-in default. +""" + + +OrgName: TypeAlias = str +""" +Filter search results to within a particular organization +""" + + +ProjectIdParam: TypeAlias = str +""" +Project id +""" + + +ProjectName: TypeAlias = str +""" +Name of the project to search for +""" + + +class RemoteEvalSource(TypedDict): + description: NotRequired[str | None] + name: NotRequired[str | None] + url: str + + +class SpanFieldOrderItem(TypedDict): + column_id: str + layout: NotRequired[Literal["full"] | Literal["two_column"] | None] + object_type: str + position: str + + +class ProjectSettings(TypedDict): + baseline_experiment_id: NotRequired[str | None] + """ + The id of the experiment to use as the default baseline for comparisons + """ + comparison_key: NotRequired[str | None] + """ + The key used to join two experiments (defaults to `input`) + """ + default_preprocessor: NotRequired[NullableSavedFunctionId] + disable_realtime_queries: NotRequired[bool | None] + """ + If true, disable real-time queries for this project. This can improve query performance for high-volume logs. + """ + remote_eval_sources: NotRequired[Sequence[RemoteEvalSource] | None] + """ + The remote eval sources to use for the project + """ + spanFieldOrder: NotRequired[Sequence[SpanFieldOrderItem] | None] + """ + The order of the fields to display in the trace view + """ + + +StartingAfter: TypeAlias = str +""" +Pagination cursor id. + +For example, if the final item in the last page you fetched had an id of `foo`, pass `starting_after=foo` to fetch the next page. Note: you may only pass one of `starting_after` and `ending_before` +""" + + +class PatchProject(TypedDict): + description: NotRequired[str | None] + name: NotRequired[str | None] + """ + Name of the project + """ + settings: NotRequired[ProjectSettings] + user_id: NotRequired[str | None] + + +class Project(TypedDict): + created: NotRequired[str | None] + """ + Date of project creation + """ + deleted_at: NotRequired[str | None] + """ + Date of project deletion, or null if the project is still active + """ + description: NotRequired[str | None] + """ + Textual description of the project + """ + id: str + """ + Unique identifier for the project + """ + name: str + """ + Name of the project + """ + org_id: str + """ + Unique id for the organization that the project belongs under + """ + settings: NotRequired[ProjectSettings | None] + user_id: NotRequired[str | None] + """ + Identifies the user who created the project + """ + + +class GetProjectResponse(TypedDict): + objects: Sequence[Project] + """ + A list of project objects + """ diff --git a/py/src/braintrust/api/_generated/projects.py b/py/src/braintrust/api/_generated/projects.py new file mode 100644 index 00000000..5d7f57d6 --- /dev/null +++ b/py/src/braintrust/api/_generated/projects.py @@ -0,0 +1,235 @@ +# Generated by scripts/generate-api-client.py. DO NOT EDIT. +# OpenAPI commit: 9daf27f19d9e0340304d7a3e7d0edb28380b94c6 +# OpenAPI spec SHA-256: 5ec753c0263c0c44cd04f741edfc7e8bad491cc25a2113d029e84edc076520f0 +# datamodel-code-generator: 0.72.4 +# ruff: 0.15.21 +# Generator Python: 3.14 +# Content SHA-256: f3acc80e226167922acb76737729f612236d5b2d5445d1cb3af5392726fd78e2 + +"""Generated Projects REST operations and resource.""" + +from typing import cast + +from .._service import Operation, Parameter, ResourceAPI +from ..policies import RetryMode +from .models.projects import ( + AppLimitParam, + CreateProject, + EndingBefore, + GetProjectResponse, + Ids, + OrgName, + PatchProject, + Project, + ProjectIdParam, + ProjectName, + StartingAfter, +) + + +POST_PROJECT = Operation( + operation_id="postProject", + method="POST", + path="/v1/project", + parameters=(), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.IDEMPOTENT_WRITE, +) + + +GET_PROJECT = Operation( + operation_id="getProject", + method="GET", + path="/v1/project", + parameters=( + Parameter( + argument_name="limit", + name="limit", + location="query", + required=False, + ), + Parameter( + argument_name="starting_after", + name="starting_after", + location="query", + required=False, + ), + Parameter( + argument_name="ending_before", + name="ending_before", + location="query", + required=False, + ), + Parameter( + argument_name="ids", + name="ids", + location="query", + required=False, + ), + Parameter( + argument_name="project_name", + name="project_name", + location="query", + required=False, + ), + Parameter( + argument_name="org_name", + name="org_name", + location="query", + required=False, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +GET_PROJECT_ID = Operation( + operation_id="getProjectId", + method="GET", + path="/v1/project/{project_id}", + parameters=( + Parameter( + argument_name="project_id", + name="project_id", + location="path", + required=True, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.SAFE_READ, +) + + +PATCH_PROJECT_ID = Operation( + operation_id="patchProjectId", + method="PATCH", + path="/v1/project/{project_id}", + parameters=( + Parameter( + argument_name="project_id", + name="project_id", + location="path", + required=True, + ), + ), + has_request_body=True, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +DELETE_PROJECT_ID = Operation( + operation_id="deleteProjectId", + method="DELETE", + path="/v1/project/{project_id}", + parameters=( + Parameter( + argument_name="project_id", + name="project_id", + location="path", + required=True, + ), + ), + has_request_body=False, + success_statuses=(200,), + json_success_statuses=(200,), + retry_mode=RetryMode.NONE, +) + + +OPERATIONS = { + "postProject": POST_PROJECT, + "getProject": GET_PROJECT, + "getProjectId": GET_PROJECT_ID, + "patchProjectId": PATCH_PROJECT_ID, + "deleteProjectId": DELETE_PROJECT_ID, +} + + +class ProjectsAPI(ResourceAPI): + """Generated Projects REST API.""" + + def post_project( + self, + *, + body: "CreateProject | None" = None, + ) -> "Project": + return cast( + "Project", + self.execute( + POST_PROJECT, + body=body, + ), + ) + + def get_project( + self, + *, + limit: "AppLimitParam | None" = None, + starting_after: "StartingAfter | None" = None, + ending_before: "EndingBefore | None" = None, + ids: "Ids | None" = None, + project_name: "ProjectName | None" = None, + org_name: "OrgName | None" = None, + ) -> "GetProjectResponse": + return cast( + "GetProjectResponse", + self.execute( + GET_PROJECT, + query_parameters={ + "limit": limit, + "starting_after": starting_after, + "ending_before": ending_before, + "ids": ids, + "project_name": project_name, + "org_name": org_name, + }, + ), + ) + + def get_project_id( + self, + project_id: "ProjectIdParam", + ) -> "Project": + return cast( + "Project", + self.execute( + GET_PROJECT_ID, + path_parameters={"project_id": project_id}, + ), + ) + + def patch_project_id( + self, + project_id: "ProjectIdParam", + *, + body: "PatchProject | None" = None, + ) -> "Project": + return cast( + "Project", + self.execute( + PATCH_PROJECT_ID, + path_parameters={"project_id": project_id}, + body=body, + ), + ) + + def delete_project_id( + self, + project_id: "ProjectIdParam", + ) -> "Project": + return cast( + "Project", + self.execute( + DELETE_PROJECT_ID, + path_parameters={"project_id": project_id}, + ), + ) diff --git a/py/src/braintrust/api/_service.py b/py/src/braintrust/api/_service.py index f6522c0b..70f39dc6 100644 --- a/py/src/braintrust/api/_service.py +++ b/py/src/braintrust/api/_service.py @@ -1,37 +1,56 @@ """Shared resource service primitives.""" -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Any +from urllib.parse import quote + +import requests from ._routing import EndpointRouter, RequestTarget from ._transport import Transport +from .errors import BraintrustHTTPError +from .policies import RetryMode + + +@dataclass(frozen=True) +class Parameter: + """Serialization metadata for one generated operation parameter.""" + + argument_name: str + name: str + location: str + required: bool @dataclass(frozen=True) -class ClientContext: - """Organization and credential context shared by resource services.""" +class Operation: + """Resolved wire and runtime-policy metadata for a generated operation.""" - org_id: str - org_name: str + operation_id: str + method: str + path: str + parameters: tuple[Parameter, ...] + has_request_body: bool + success_statuses: tuple[int, ...] + json_success_statuses: tuple[int, ...] + retry_mode: RetryMode class ResourceAPI: - """Base class for synchronous resource services.""" + """Base class for synchronous resource services and generated operations.""" def __init__( self, transport: Transport, router: EndpointRouter, - context: ClientContext, api_key: str, ): self._transport = transport self._router = router - self._context = context self._api_key = api_key - def _request_json( + def _request( self, target: RequestTarget, method: str, @@ -39,13 +58,102 @@ def _request_json( *, headers: Mapping[str, str] | None = None, **kwargs: Any, - ) -> Any: + ) -> requests.Response: request_headers = {"Authorization": f"Bearer {self._api_key}"} if headers: request_headers.update(headers) - return self._transport.request_json( + return self._transport.request( method, self._router.resolve(target, path), headers=request_headers, **kwargs, ) + + def _request_json( + self, + target: RequestTarget, + method: str, + path: str, + *, + headers: Mapping[str, str] | None = None, + **kwargs: Any, + ) -> Any: + response = self._request(target, method, path, headers=headers, **kwargs) + return self._transport.decode_json_response(response, method=method, url=response.url) + + def execute( + self, + operation: Operation, + *, + path_parameters: Mapping[str, Any] | None = None, + query_parameters: Mapping[str, Any] | None = None, + body: Any = None, + ) -> Any: + """Execute a generated operation through this resource's transport.""" + + path_values = path_parameters or {} + query_values = query_parameters or {} + path = operation.path + query_parts: list[str] = [] + + for parameter in operation.parameters: + values = {"path": path_values, "query": query_values}.get(parameter.location) + if values is None: + raise ValueError(f"Unsupported generated parameter location: {parameter.location!r}") + value = values.get(parameter.argument_name) + if value is None: + if parameter.required: + raise TypeError(f"Missing required parameter: {parameter.argument_name}") + continue + if parameter.location == "path": + encoded = _encode_path_parameter(value) + path = path.replace("{" + parameter.name + "}", encoded) + else: + query_parts.extend(_encode_query_parameter(value, parameter)) + + if query_parts: + path += ("&" if "?" in path else "?") + "&".join(query_parts) + + request_kwargs: dict[str, Any] = {"retry_mode": operation.retry_mode} + if operation.has_request_body and body is not None: + request_kwargs["json"] = body + + response = self._request(RequestTarget.API, operation.method, path, **request_kwargs) + if response.status_code not in operation.success_statuses: + raise BraintrustHTTPError( + method=operation.method, + url=response.url, + status_code=response.status_code, + response_body=response.text, + response_headers=response.headers, + attempts=getattr(response, "_braintrust_attempts", 1), + retryable=False, + ) + if response.status_code in operation.json_success_statuses: + return self._transport.decode_json_response(response, method=operation.method, url=response.url) + return None + + +def _encode_path_parameter(value: Any) -> str: + if _is_array(value): + raise TypeError("Path parameters must be scalar") + return quote(_scalar_string(value), safe="") + + +def _encode_query_parameter(value: Any, parameter: Parameter) -> list[str]: + encoded_name = quote(parameter.name, safe="") + if _is_array(value): + encoded_values = [quote(_scalar_string(item), safe="") for item in value] + return [f"{encoded_name}={item}" for item in encoded_values] + encoded_value = quote(_scalar_string(value), safe="") + return [f"{encoded_name}={encoded_value}"] + + +def _is_array(value: Any) -> bool: + return isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)) + + +def _scalar_string(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + return str(value) diff --git a/py/src/braintrust/api/_test_server.py b/py/src/braintrust/api/_test_server.py new file mode 100644 index 00000000..2d577b54 --- /dev/null +++ b/py/src/braintrust/api/_test_server.py @@ -0,0 +1,74 @@ +"""Shared local HTTP server helper for API tests.""" + +import contextlib +import http.server +import socketserver +import threading +import time + + +@contextlib.contextmanager +def scripted_server(script): + """Run a local server driven by sequential actions or a request callback.""" + + class ScriptedHandler(http.server.BaseHTTPRequestHandler): + request_count = 0 + requests = [] + + def log_message(self, format, *args): + pass + + def do_GET(self): + self._handle() + + def do_POST(self): + self._handle() + + def do_PATCH(self): + self._handle() + + def do_DELETE(self): + self._handle() + + def _handle(self): + request_number = type(self).request_count + type(self).request_count += 1 + content_length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(content_length) if content_length else b"" + type(self).requests.append((self.command, self.path, body, self.headers.get("Authorization"))) + action = ( + script(self.command, self.path, body, self.headers) + if callable(script) + else script[min(request_number, len(script) - 1)] + ) + + if action == "close": + self.connection.close() + return + + if action[0] == "sleep": + _, delay, status, headers, response_body = action + time.sleep(delay) + else: + status, headers, response_body = action + + self.send_response(status) + for name, value in headers.items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(response_body))) + self.end_headers() + try: + self.wfile.write(response_body) + except BrokenPipeError: + pass + + server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), ScriptedHandler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + + try: + yield f"http://127.0.0.1:{server.server_address[1]}", ScriptedHandler + finally: + server.shutdown() + server.server_close() diff --git a/py/src/braintrust/api/_transport.py b/py/src/braintrust/api/_transport.py index 410d57c5..a6109b62 100644 --- a/py/src/braintrust/api/_transport.py +++ b/py/src/braintrust/api/_transport.py @@ -18,7 +18,7 @@ from ..util import _urljoin, response_raise_for_status from .errors import ( BraintrustHTTPError, - BraintrustResponseError, + BraintrustJSONDecodeError, BraintrustRetryExhaustedError, BraintrustTransportError, BraintrustTransportRetryExhaustedError, @@ -359,10 +359,14 @@ def request( def request_json(self, method: str, url: str, **kwargs: Any) -> Any: response = self.request(method, url, **kwargs) + return self.decode_json_response(response, method=method, url=url) + + def decode_json_response(self, response: requests.Response, *, method: str, url: str) -> Any: + """Decode a completed response while preserving transport error details.""" try: return response.json() except ValueError as exc: - error = BraintrustResponseError( + error = BraintrustJSONDecodeError( method=method.upper(), url=response.url or url, status_code=response.status_code, diff --git a/py/src/braintrust/api/attachments.py b/py/src/braintrust/api/attachments.py deleted file mode 100644 index 2c56c5f9..00000000 --- a/py/src/braintrust/api/attachments.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Attachment metadata API service.""" - -from ._service import ResourceAPI - - -class AttachmentsAPI(ResourceAPI): - """Synchronous attachment metadata operations. - - Signed object-storage traffic remains outside the routed transport. - """ diff --git a/py/src/braintrust/api/auth.py b/py/src/braintrust/api/auth.py index 81938fda..efe54475 100644 --- a/py/src/braintrust/api/auth.py +++ b/py/src/braintrust/api/auth.py @@ -5,7 +5,7 @@ from types import MappingProxyType from typing import Any -from ..env import BraintrustEnv +from ..env import BraintrustEnv, resolve_org_name from ._routing import EndpointRouter, RequestTarget from ._transport import HTTPConnection, Transport from .policies import RetryMode @@ -55,22 +55,34 @@ def optional_string(field: str) -> str | None: @dataclass(frozen=True) class LoginResult: - """Selected organization and the complete login response.""" + """Selected organization, resolved routing, and the complete login response.""" organization: OrganizationInfo + api_url: str + proxy_url: str | None response: Mapping[str, Any] class AuthAPI: """Authenticate an API key and configure an endpoint router.""" - def __init__(self, transport: Transport, router: EndpointRouter): + def __init__( + self, + transport: Transport, + router: EndpointRouter, + api_key: str, + *, + api_url: str | None = None, + proxy_url: str | None = None, + ): self._transport = transport self._router = router + self._api_key = HTTPConnection.sanitize_token(api_key) + self._api_url = api_url + self._proxy_url = proxy_url def login( self, - api_key: str, *, org_name: str | None = None, api_url: str | None = None, @@ -78,11 +90,10 @@ def login( ) -> LoginResult: """Log in, select an organization, and apply routing override precedence.""" - api_key = HTTPConnection.sanitize_token(api_key) response = self._transport.request_json( "POST", self._router.resolve(RequestTarget.APP, "/api/apikey/login"), - headers={"Authorization": f"Bearer {api_key}"}, + headers={"Authorization": f"Bearer {self._api_key}"}, retry_mode=RetryMode.SAFE_READ, ) if not isinstance(response, Mapping): @@ -92,10 +103,11 @@ def login( raise ValueError("API-key login response did not include an organization list") organizations = [OrganizationInfo.from_dict(org) for org in raw_orgs if isinstance(org, Mapping)] + org_name = resolve_org_name(org_name) organization = self._select_organization(organizations, org_name) - resolved_api_url = api_url or BraintrustEnv.API_URL.get(organization.api_url) - resolved_proxy_url = proxy_url or BraintrustEnv.PROXY_URL.get(organization.proxy_url) + resolved_api_url = api_url or self._api_url or BraintrustEnv.API_URL.get(organization.api_url) + resolved_proxy_url = proxy_url or self._proxy_url or BraintrustEnv.PROXY_URL.get(organization.proxy_url) if not resolved_api_url: if org_name: raise ValueError( @@ -109,7 +121,12 @@ def login( proxy_url=resolved_proxy_url, is_universal_api=organization.is_universal_api, ) - return LoginResult(organization=organization, response=MappingProxyType(dict(response))) + return LoginResult( + organization=organization, + api_url=resolved_api_url, + proxy_url=resolved_proxy_url, + response=MappingProxyType(dict(response)), + ) @staticmethod def _select_organization(organizations: Sequence[OrganizationInfo], org_name: str | None) -> OrganizationInfo: diff --git a/py/src/braintrust/api/cassettes/test_projects_end_to_end_with_real_backend.yaml b/py/src/braintrust/api/cassettes/test_projects_end_to_end_with_real_backend.yaml new file mode 100644 index 00000000..14202064 --- /dev/null +++ b/py/src/braintrust/api/cassettes/test_projects_end_to_end_with_real_backend.yaml @@ -0,0 +1,396 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"Braintrust + SDKs","api_url":"https://api.braintrust.dev","git_metadata":{"collect":"some","fields":["commit","branch","tag","dirty","author_name","author_email","commit_message","commit_time"]},"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '376' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MzEzZmE3Y2ItOWJiYS00OWU1LWI4OTMtOGQ4NjA2Y2Y1YjUz'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self'' + https://www.facebook.com; frame-ancestors ''self''; worker-src ''self'' blob:; + report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 14 Aug 2026 20:02:21 GMT + Etag: + - '"13vsc5ye8flag"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - MzEzZmE3Y2ItOWJiYS00OWU1LWI4OTMtOGQ4NjA2Y2Y1YjUz + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::ffbhm-1786737740734-013b054d8dac + status: + code: 200 + message: OK +- request: + body: '{"name": "python-sdk-generated-projects-vcr", "description": "created by + the Python SDK VCR test", "org_name": "Braintrust SDKs"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '129' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: POST + uri: https://api.braintrust.dev/v1/project + response: + body: + string: '{"id":"613b27b8-da58-4796-8993-d79cff1afad3","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-generated-projects-vcr","description":"created + by the Python SDK VCR test","created":"2026-08-14T20:02:21.300Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 14 Aug 2026 20:02:21 GMT + Via: + - 1.1 d38f8e8aaab4437dcb36d4adc5a35cbe.cloudfront.net (CloudFront), 1.1 70fd8dd903406754b301439f9111e256.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - n1ay16d0BJ_cvkNZS4W6Y20Die9bsM173_DAV567vyqX1oigpIlaog== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7f744d-25501c3c04858339782f8fde;Parent=575c5e29d6a53b1d;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '307' + etag: + - W/"133-J+BLvuGtfyn2Lj4IWFVWVsqLqNc" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CG8cIH6YIAMEu8g= + x-amzn-RequestId: + - 654f58b9-9d16-4c57-8616-1ee0dd15ab61 + x-bt-internal-trace-id: + - 6a7f744d0000000049c801445f2ae4ab + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/project?project_name=python-sdk-generated-projects-vcr&org_name=Braintrust%20SDKs + response: + body: + string: '{"objects":[{"id":"613b27b8-da58-4796-8993-d79cff1afad3","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-generated-projects-vcr","description":"created + by the Python SDK VCR test","created":"2026-08-14T20:02:21.300Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}]}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 14 Aug 2026 20:02:21 GMT + Via: + - 1.1 8250156022879efefd7a589c8ba8c706.cloudfront.net (CloudFront), 1.1 7293b56f3a0eb541aadcbcaa0146d528.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - f6M-kbi-SL_TPm4YD0yvWJNKubVVNE58UQd8H5TxFGv5gGRnDayRFw== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7f744d-7a079b485298fbba1d379bfe;Parent=67d61fbe4ef2b858;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '321' + etag: + - W/"141-Q/vI9D3l1GEcxigmhdgU4QDoPr4" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CG8cMH0EIAMEE6Q= + x-amzn-RequestId: + - 15063a57-e16a-432c-92ae-7a4115044629 + x-bt-internal-trace-id: + - 6a7f744d0000000006fc9365f6b00acb + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + User-Agent: + - python-requests/2.34.2 + method: GET + uri: https://api.braintrust.dev/v1/project/613b27b8-da58-4796-8993-d79cff1afad3 + response: + body: + string: '{"id":"613b27b8-da58-4796-8993-d79cff1afad3","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-generated-projects-vcr","description":"created + by the Python SDK VCR test","created":"2026-08-14T20:02:21.300Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 14 Aug 2026 20:02:21 GMT + Via: + - 1.1 d38f8e8aaab4437dcb36d4adc5a35cbe.cloudfront.net (CloudFront), 1.1 36c050103b969d83a8b90ba7cba12542.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - -Ow0f7p7F48nNhJdla7V0_UyJ5iIwurCIqdE0YDDUIv2A79IDow36Q== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7f744d-07e24fb532e649db2a93f9e3;Parent=67ee5e76f1f143bc;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '307' + etag: + - W/"133-J+BLvuGtfyn2Lj4IWFVWVsqLqNc" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CG8cPEbBIAMEv3Q= + x-amzn-RequestId: + - e77b6c34-4374-4175-9b33-210fe5c99104 + x-bt-internal-trace-id: + - 6a7f744d000000007bd2cb96555aa91a + status: + code: 200 + message: OK +- request: + body: '{"description": "updated by the Python SDK VCR test"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '53' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + method: PATCH + uri: https://api.braintrust.dev/v1/project/613b27b8-da58-4796-8993-d79cff1afad3 + response: + body: + string: '{"id":"613b27b8-da58-4796-8993-d79cff1afad3","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-generated-projects-vcr","description":"updated + by the Python SDK VCR test","created":"2026-08-14T20:02:21.300Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 14 Aug 2026 20:02:22 GMT + Via: + - 1.1 d38f8e8aaab4437dcb36d4adc5a35cbe.cloudfront.net (CloudFront), 1.1 6889869bf680fe34cca722f0a05e1106.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - sWBfxSN30owXXvYuOuBahaAAhqIi_IHRWJuI-ddXXt0qN-_sY7nQhg== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7f744e-12cd753c03328c487a650ab6;Parent=1d830992c5b5174a;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '307' + etag: + - W/"133-LM8/56xANPuebOLm3mEY13c/1pY" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CG8cRHASIAMEgcA= + x-amzn-RequestId: + - 224f1f09-1d4f-4ebe-8d7b-fdfdbc1cad5e + x-bt-internal-trace-id: + - 6a7f744e000000000fe9bcac73f7625a + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.34.2 + method: DELETE + uri: https://api.braintrust.dev/v1/project/613b27b8-da58-4796-8993-d79cff1afad3 + response: + body: + string: '{"id":"613b27b8-da58-4796-8993-d79cff1afad3","org_id":"5abfae3a-7aa7-4653-a9c8-b3efcb18f584","name":"python-sdk-generated-projects-vcr","description":"updated + by the Python SDK VCR test","created":"2026-08-14T20:02:21.300Z","deleted_at":"2026-08-14T20:02:22.434Z","user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 14 Aug 2026 20:02:22 GMT + Via: + - 1.1 561ea23d1fbb47b35216fa7040bfaa74.cloudfront.net (CloudFront), 1.1 10f12ad63ad88e4e38e4e73deb3e9570.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - TwX8S5DWN5Guvm0GuA7T-IIBL9AIcEARmMjJT1b484o_aVc-XsU8wg== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-6a7f744e-4b99ebd85091eeed2c28e93f;Parent=36e67096fb7d4942;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id,x-bt-error-origin,x-bt-used-endpoint,x-bt-overflow-url + cache-control: + - no-store, no-cache, must-revalidate, proxy-revalidate + content-length: + - '329' + etag: + - W/"149-ZEM0N6DPDhpdgarT6R15u2oVDag" + expires: + - '0' + surrogate-control: + - no-store + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - CG8cUGQboAMEv7Q= + x-amzn-RequestId: + - ff2c1211-e5b7-40ac-ab3f-3aa36933ef0b + x-bt-internal-trace-id: + - 6a7f744e0000000015b1866d584ef16c + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/api/client.py b/py/src/braintrust/api/client.py index 883592dd..b645fc91 100644 --- a/py/src/braintrust/api/client.py +++ b/py/src/braintrust/api/client.py @@ -1,36 +1,59 @@ -"""Synchronous Braintrust API client facade.""" +"""Synchronous Braintrust API clients.""" from typing import Any import requests from requests.adapters import HTTPAdapter -from ..env import BraintrustEnv, resolve_app_url, resolve_org_name +from ..env import BraintrustEnv, resolve_app_url from ._routing import EndpointRouter -from ._service import ClientContext from ._transport import HTTPConnection, Transport -from .attachments import AttachmentsAPI -from .auth import AuthAPI, LoginResult -from .datasets import DatasetsAPI -from .experiments import ExperimentsAPI -from .functions import FunctionsAPI -from .projects import ProjectsAPI -from .prompts import PromptsAPI -from .queries import QueriesAPI +from .auth import AuthAPI + + +def _resolve_api_key(api_key: str | None) -> str: + resolved_api_key = api_key or BraintrustEnv.API_KEY.get(None, use_dotenv=True) + if not resolved_api_key: + raise ValueError( + "Could not initialize the Braintrust API client. Set BRAINTRUST_API_KEY in your environment " + "or nearest .env.braintrust file, or pass api_key explicitly." + ) + return HTTPConnection.sanitize_token(resolved_api_key) + + +def _create_transport( + *, + session: requests.Session | None, + adapter: HTTPAdapter | None, + transport: Transport | None, + enable_sdk_retries: bool | None, +) -> tuple[Transport, bool]: + if transport is not None and (session is not None or adapter is not None or enable_sdk_retries is not None): + raise ValueError("transport cannot be combined with session, adapter, or enable_sdk_retries") + if transport is not None: + return transport, False + return ( + Transport( + session=session, + adapter=adapter, + enable_sdk_retries=enable_sdk_retries, + persist_cookies=False, + ), + True, + ) class BraintrustClient: - """Synchronous resource-oriented client for the Braintrust API. + """Client for generated and handwritten Braintrust API services. - The convenience constructor authenticates through the app origin, selects an - organization, and configures API and proxy routing on one shared transport. + Construction performs no network requests. Call ``client.auth.login()`` + to discover organization routing when ``api_url`` is not configured. """ def __init__( self, *, api_key: str | None = None, - org_name: str | None = None, app_url: str | None = None, api_url: str | None = None, proxy_url: str | None = None, @@ -39,41 +62,79 @@ def __init__( transport: Transport | None = None, enable_sdk_retries: bool | None = None, ): - if transport is not None and (session is not None or adapter is not None or enable_sdk_retries is not None): - raise ValueError("transport cannot be combined with session, adapter, or enable_sdk_retries") - - resolved_api_key = api_key or BraintrustEnv.API_KEY.get(None, use_dotenv=True) - if not resolved_api_key: - raise ValueError( - "Could not login to Braintrust. You may need to set BRAINTRUST_API_KEY in your environment " - "or nearest .env.braintrust file." - ) - resolved_api_key = HTTPConnection.sanitize_token(resolved_api_key) - resolved_org_name = resolve_org_name(org_name) - resolved_app_url = resolve_app_url(app_url) - - self._owns_transport = transport is None - self.transport = transport or Transport( + self.api_key = _resolve_api_key(api_key) + self.transport, self._owns_transport = _create_transport( session=session, adapter=adapter, + transport=transport, enable_sdk_retries=enable_sdk_retries, - persist_cookies=False, ) - self.router = EndpointRouter(app_url=resolved_app_url) - auth = AuthAPI(self.transport, self.router) - try: - result = auth.login( - resolved_api_key, - org_name=resolved_org_name, - api_url=api_url, - proxy_url=proxy_url, - ) - except Exception: - if self._owns_transport: - self.transport.close() - raise - - self._initialize_services(result, resolved_api_key) + self.router = EndpointRouter( + app_url=resolve_app_url(app_url), + api_url=api_url or BraintrustEnv.API_URL.get(None), + proxy_url=proxy_url or BraintrustEnv.PROXY_URL.get(None), + ) + self.auth = AuthAPI( + self.transport, + self.router, + self.api_key, + api_url=api_url, + proxy_url=proxy_url, + ) + self.openapi = BraintrustOpenApiClient.from_transport( + transport=self.transport, + router=self.router, + api_key=self.api_key, + ) + + def close(self) -> None: + """Close the transport when it was created by this client.""" + + if self._owns_transport: + self.transport.close() + + def __enter__(self) -> "BraintrustClient": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + +class BraintrustOpenApiClient: + """Synchronous resource-oriented client for the Braintrust REST API. + + Construction performs no network requests. Use :class:`BraintrustClient` + when authentication and generated resources should share one transport. + """ + + def __init__( + self, + *, + api_key: str | None = None, + api_url: str | None = None, + proxy_url: str | None = None, + session: requests.Session | None = None, + adapter: HTTPAdapter | None = None, + transport: Transport | None = None, + enable_sdk_retries: bool | None = None, + ): + resolved_api_key = _resolve_api_key(api_key) + resolved_api_url = api_url or BraintrustEnv.API_URL.get(None) + if not resolved_api_url: + raise ValueError("api_url is required when constructing BraintrustOpenApiClient") + resolved_proxy_url = proxy_url or BraintrustEnv.PROXY_URL.get(None) + self.transport, self._owns_transport = _create_transport( + session=session, + adapter=adapter, + transport=transport, + enable_sdk_retries=enable_sdk_retries, + ) + self.router = EndpointRouter( + app_url=resolve_app_url(None), + api_url=resolved_api_url, + proxy_url=resolved_proxy_url, + ) + self._initialize_services(resolved_api_key) @classmethod def from_transport( @@ -82,64 +143,21 @@ def from_transport( transport: Transport, router: EndpointRouter, api_key: str, - org_id: str, - org_name: str, - login_result: LoginResult | None = None, - ) -> "BraintrustClient": - """Build a client around an already-authenticated transport and router.""" + ) -> "BraintrustOpenApiClient": + """Build a client around an already-configured transport and router.""" client = cls.__new__(cls) client._owns_transport = False client.transport = transport client.router = router - client._initialize_services_from_context( - ClientContext(org_id=org_id, org_name=org_name), - HTTPConnection.sanitize_token(api_key), - login_result, - ) + client._initialize_services(HTTPConnection.sanitize_token(api_key)) return client - def _initialize_services(self, result: LoginResult, api_key: str) -> None: - organization = result.organization - self._initialize_services_from_context( - ClientContext(org_id=organization.id, org_name=organization.name), - api_key, - result, - ) + def _initialize_services(self, api_key: str) -> None: + from ._generated.projects import ProjectsAPI - def _initialize_services_from_context( - self, - context: ClientContext, - api_key: str, - login_result: LoginResult | None, - ) -> None: - self.context = context self.api_key = api_key - self._login_result = login_result - service_args: tuple[Any, ...] = (self.transport, self.router, context, api_key) - self.projects = ProjectsAPI(*service_args) - self.experiments = ExperimentsAPI(*service_args) - self.datasets = DatasetsAPI(*service_args) - self.prompts = PromptsAPI(*service_args) - self.functions = FunctionsAPI(*service_args) - self.queries = QueriesAPI(*service_args) - self.attachments = AttachmentsAPI(*service_args) - - @property - def login_result(self) -> LoginResult: - """Return organization discovery details for a bootstrapped client.""" - - if self._login_result is None: - raise RuntimeError("Login details are unavailable for a pre-authenticated client") - return self._login_result - - @property - def org_id(self) -> str: - return self.context.org_id - - @property - def org_name(self) -> str: - return self.context.org_name + self.projects = ProjectsAPI(self.transport, self.router, api_key) def close(self) -> None: """Close the transport when it was created by this client.""" @@ -147,7 +165,7 @@ def close(self) -> None: if self._owns_transport: self.transport.close() - def __enter__(self) -> "BraintrustClient": + def __enter__(self) -> "BraintrustOpenApiClient": return self def __exit__(self, *_: Any) -> None: diff --git a/py/src/braintrust/api/datasets.py b/py/src/braintrust/api/datasets.py deleted file mode 100644 index bc565343..00000000 --- a/py/src/braintrust/api/datasets.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Dataset API service.""" - -from ._service import ResourceAPI - - -class DatasetsAPI(ResourceAPI): - """Synchronous dataset operations. - - Endpoint methods are added as dataset call sites migrate to the API client. - """ diff --git a/py/src/braintrust/api/errors.py b/py/src/braintrust/api/errors.py index d4fca223..2a03f7ea 100644 --- a/py/src/braintrust/api/errors.py +++ b/py/src/braintrust/api/errors.py @@ -100,8 +100,8 @@ class BraintrustTransportRetryExhaustedError(BraintrustTransportError): """Transport exceptions exhausted the operation's retry policy.""" -class BraintrustResponseError(BraintrustAPIError): - """A successful HTTP response could not be decoded.""" +class BraintrustJSONDecodeError(BraintrustAPIError): + """A successful HTTP response could not be decoded as JSON.""" method: str url: str @@ -119,6 +119,7 @@ def __init__( response_body: str, response_headers: Mapping[str, str], attempts: int, + message: str | None = None, ): self.method = method self.url = url @@ -132,4 +133,4 @@ def __init__( } ) self.attempts = attempts - super().__init__(f"Could not decode the response from {method} {url} as JSON") + super().__init__(message or f"Could not decode the response from {method} {url} as JSON") diff --git a/py/src/braintrust/api/experiments.py b/py/src/braintrust/api/experiments.py deleted file mode 100644 index 6673fd1c..00000000 --- a/py/src/braintrust/api/experiments.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Experiment API service.""" - -from ._service import ResourceAPI - - -class ExperimentsAPI(ResourceAPI): - """Synchronous experiment operations. - - Endpoint methods are added as experiment call sites migrate to the API client. - """ diff --git a/py/src/braintrust/api/functions.py b/py/src/braintrust/api/functions.py deleted file mode 100644 index 94716b00..00000000 --- a/py/src/braintrust/api/functions.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Function metadata API service.""" - -from ._service import ResourceAPI - - -class FunctionsAPI(ResourceAPI): - """Synchronous function metadata operations. - - Invocation remains a specialized client and is not implemented here. - """ diff --git a/py/src/braintrust/api/projects.py b/py/src/braintrust/api/projects.py deleted file mode 100644 index bb00abbc..00000000 --- a/py/src/braintrust/api/projects.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Project API service.""" - -from ._service import ResourceAPI - - -class ProjectsAPI(ResourceAPI): - """Synchronous project operations. - - Endpoint methods are added as project call sites migrate to the API client. - """ diff --git a/py/src/braintrust/api/prompts.py b/py/src/braintrust/api/prompts.py deleted file mode 100644 index 96e21c8b..00000000 --- a/py/src/braintrust/api/prompts.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Prompt API service.""" - -from ._service import ResourceAPI - - -class PromptsAPI(ResourceAPI): - """Synchronous prompt operations. - - Endpoint methods are added as prompt call sites migrate to the API client. - """ diff --git a/py/src/braintrust/api/queries.py b/py/src/braintrust/api/queries.py deleted file mode 100644 index f84878d0..00000000 --- a/py/src/braintrust/api/queries.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Query API service.""" - -from ._service import ResourceAPI - - -class QueriesAPI(ResourceAPI): - """Synchronous query operations. - - Endpoint methods are added as query call sites migrate to the API client. - """ diff --git a/py/src/braintrust/api/test_client.py b/py/src/braintrust/api/test_client.py index ebc4ae9d..7338e6bc 100644 --- a/py/src/braintrust/api/test_client.py +++ b/py/src/braintrust/api/test_client.py @@ -7,7 +7,13 @@ import pytest import requests from braintrust import logger -from braintrust.api import BraintrustClient, BraintrustHTTPError, EndpointRouter, RequestTarget +from braintrust.api import ( + BraintrustClient, + BraintrustHTTPError, + BraintrustOpenApiClient, + EndpointRouter, + RequestTarget, +) from braintrust.api._transport import RetryRequestExceptionsAdapter from braintrust.logger import BraintrustState, login_to_state from requests.adapters import HTTPAdapter @@ -60,7 +66,7 @@ def test_endpoint_router_preserves_origins_and_proxy_fallback(): assert router.resolve(RequestTarget.PROXY, "function/invoke") == "https://universal.example.com/function/invoke" -def test_client_bootstraps_selected_org_on_one_session(monkeypatch): +def test_braintrust_client_shares_transport_across_auth_and_openapi(monkeypatch): monkeypatch.delenv("BRAINTRUST_API_URL", raising=False) monkeypatch.delenv("BRAINTRUST_PROXY_URL", raising=False) orgs = [ @@ -83,40 +89,67 @@ def test_client_bootstraps_selected_org_on_one_session(monkeypatch): session.cookies.set("existing", "yes") cookie_policy = session.cookies.get_policy() with login_server(orgs, response_headers={"Set-Cookie": "accepted=yes"}) as (app_url, handler): - client = BraintrustClient(api_key="secret\n", org_name="selected", app_url=app_url, session=session) + client = BraintrustClient(api_key="secret\n", app_url=app_url, session=session) + result = client.auth.login(org_name="selected") assert handler.request_count == 1 assert handler.authorization == "Bearer secret" - assert client.org_id == "org-2" - assert client.org_name == "selected" + assert result.organization.id == "org-2" + assert result.organization.name == "selected" assert client.router.api_url == "https://api-2.example.com" - assert client.router.is_universal_api is True + assert result.organization.is_universal_api is True assert client.router.resolve(RequestTarget.PROXY, "ping") == "https://api-2.example.com/ping" - assert client.login_result.organization.raw["new_server_field"] == {"preserved": True} - assert not hasattr(client, "auth") + assert result.organization.raw["new_server_field"] == {"preserved": True} assert "Authorization" not in session.headers assert session.cookies.get("existing") == "yes" assert session.cookies.get_policy() is cookie_policy - assert all( - service._transport is client.transport - for service in ( - client.projects, - client.experiments, - client.datasets, - client.prompts, - client.functions, - client.queries, - client.attachments, - ) - ) + assert client.openapi.transport is client.transport -def test_sdk_owned_session_rejects_response_cookies(): - orgs = [{"id": "org-1", "name": "org", "api_url": "https://api.example.com"}] - with login_server(orgs, response_headers={"Set-Cookie": "ignored=yes"}) as (app_url, _): +def test_relogin_refreshes_routing_for_selected_organization(monkeypatch): + monkeypatch.delenv("BRAINTRUST_API_URL", raising=False) + monkeypatch.delenv("BRAINTRUST_PROXY_URL", raising=False) + orgs = [ + { + "id": "org-1", + "name": "one", + "api_url": "https://api-one.example.com", + "proxy_url": "https://proxy-one.example.com", + }, + { + "id": "org-2", + "name": "two", + "api_url": "https://api-two.example.com", + "proxy_url": "https://proxy-two.example.com", + }, + ] + with login_server(orgs) as (app_url, handler): client = BraintrustClient(api_key="secret", app_url=app_url) + client.auth.login(org_name="one") + result = client.auth.login(org_name="two") + + assert handler.request_count == 2 + assert result.organization.name == "two" + assert client.openapi.router.api_url == "https://api-two.example.com" + assert client.openapi.router.proxy_url == "https://proxy-two.example.com" + + +def test_constructors_do_not_log_in(monkeypatch): + with login_server([]) as (app_url, handler): + monkeypatch.setenv("BRAINTRUST_APP_URL", app_url) + client = BraintrustClient(api_key="secret", api_url="https://api.example.com") + openapi_client = BraintrustOpenApiClient(api_key="secret", api_url="https://api.example.com") + + assert handler.request_count == 0 + assert client.router.api_url == "https://api.example.com" + assert openapi_client.router.api_url == "https://api.example.com" + + +def test_constructor_requires_api_url_without_discovery(monkeypatch): + monkeypatch.delenv("BRAINTRUST_API_URL", raising=False) - assert not client.transport.session.cookies + with pytest.raises(ValueError, match="api_url is required"): + BraintrustOpenApiClient(api_key="secret") def test_client_url_override_precedence(monkeypatch): @@ -131,25 +164,32 @@ def test_client_url_override_precedence(monkeypatch): } ] with login_server(orgs) as (app_url, _): - env_client = BraintrustClient(api_key="secret", app_url=app_url) - explicit_client = BraintrustClient( + with BraintrustClient(api_key="secret", app_url=app_url) as env_client: + env_result = env_client.auth.login() + with BraintrustClient( api_key="secret", app_url=app_url, api_url="https://api-explicit.example.com", proxy_url="https://proxy-explicit.example.com", - ) + ) as explicit_client: + explicit_result = explicit_client.auth.login() - assert env_client.router.api_url == "https://api-env.example.com" - assert env_client.router.proxy_url == "https://proxy-env.example.com" - assert explicit_client.router.api_url == "https://api-explicit.example.com" - assert explicit_client.router.proxy_url == "https://proxy-explicit.example.com" + assert env_result.api_url == "https://api-env.example.com" + assert env_result.proxy_url == "https://proxy-env.example.com" + assert env_client.router.api_url == env_result.api_url + assert env_client.router.proxy_url == env_result.proxy_url + assert explicit_result.api_url == "https://api-explicit.example.com" + assert explicit_result.proxy_url == "https://proxy-explicit.example.com" + assert explicit_client.router.api_url == explicit_result.api_url + assert explicit_client.router.proxy_url == explicit_result.proxy_url def test_custom_adapter_disables_bootstrap_retries(): orgs = [{"id": "org-1", "name": "org", "api_url": "https://api.example.com"}] with login_server(orgs, status=503) as (app_url, handler): - with pytest.raises(BraintrustHTTPError): - BraintrustClient(api_key="secret", app_url=app_url, adapter=HTTPAdapter()) + with BraintrustClient(api_key="secret", app_url=app_url, adapter=HTTPAdapter()) as client: + with pytest.raises(BraintrustHTTPError): + client.auth.login() assert handler.request_count == 1 @@ -170,7 +210,10 @@ def test_login_to_state_hydrates_isolated_legacy_connections(monkeypatch): with login_server(orgs) as (login_url, _): state = login_to_state(api_key="secret", app_url=login_url, org_name="org") - assert state.api_client().org_id == "org-1" + assert state.org_id == "org-1" + assert state._client is not None + assert state._client.openapi is state.api_client() + assert state.api_client().router.api_url == app_url assert state.git_metadata_settings is None assert state._client.transport.session is not state.api_conn().session assert state._client.transport.session is not state.app_conn().session diff --git a/py/src/braintrust/api/test_generated_models.py b/py/src/braintrust/api/test_generated_models.py index e83501d0..4f29a3b7 100644 --- a/py/src/braintrust/api/test_generated_models.py +++ b/py/src/braintrust/api/test_generated_models.py @@ -2,7 +2,7 @@ import json import subprocess import sys -from typing import is_typeddict +from typing import get_type_hints, is_typeddict def test_import_braintrust_is_lazy_about_generated_api_modules(): @@ -19,17 +19,21 @@ def test_import_braintrust_is_lazy_about_generated_api_modules(): def test_generated_models_import_on_supported_python(): - from braintrust.api._generated import models + from braintrust.api._generated import projects as project_bindings + from braintrust.api._generated.models import projects - assert is_typeddict(models.Project) - assert models.ProjectIdParam is str + assert is_typeddict(projects.Project) + assert projects.ProjectIdParam is str + assert get_type_hints(project_bindings.ProjectsAPI.get_project)["return"] is projects.GetProjectResponse def test_generated_package_content_is_installed(): generated = importlib.resources.files("braintrust.api._generated") assert generated.joinpath("__init__.py").is_file() - assert generated.joinpath("models.py").is_file() + assert generated.joinpath("models", "__init__.py").is_file() + assert generated.joinpath("models", "projects.py").is_file() + assert generated.joinpath("projects.py").is_file() def test_rest_and_logging_type_surfaces_have_reviewed_overlap(): @@ -38,4 +42,4 @@ def test_rest_and_logging_type_surfaces_have_reviewed_overlap(): overlap = set(generated_types.__all__) & set(types.__all__) - assert overlap == set() + assert overlap == {"Project"} diff --git a/py/src/braintrust/api/test_projects.py b/py/src/braintrust/api/test_projects.py new file mode 100644 index 00000000..71ce6418 --- /dev/null +++ b/py/src/braintrust/api/test_projects.py @@ -0,0 +1,151 @@ +import contextlib +import json +import os +from urllib.parse import urlsplit + +import pytest +from braintrust.api import BraintrustClient, BraintrustHTTPError, BraintrustOpenApiClient +from braintrust.api._test_server import scripted_server +from braintrust.api._transport import Transport + + +@contextlib.contextmanager +def project_server(): + project = { + "id": "project-id", + "org_id": "test-org-id", + "name": "project/name", + "new_backend_field": {"preserved": True}, + } + + def respond(method, request_path, _body, _headers): + path = urlsplit(request_path).path + response = ( + {"objects": [project], "new_list_field": ["preserved"]} + if path == "/v1/project" and method == "GET" + else project + ) + return 200, {"Content-Type": "application/json"}, json.dumps(response).encode() + + with scripted_server(respond) as server: + yield server + + +def _client_for_server(url, transport): + return BraintrustOpenApiClient(api_key="test-key", api_url=url, transport=transport) + + +def test_generated_operation_rejects_undeclared_success_statuses_as_http_errors(): + response_body = b'{"queued": true}' + headers = {"Content-Type": "application/json", "x-request-id": "unexpected-status-id"} + with scripted_server([(202, headers, response_body)]) as (url, _): + projects = _client_for_server(url, Transport()).projects + with pytest.raises(BraintrustHTTPError) as exc_info: + projects.get_project() + + assert exc_info.value.status_code == 202 + assert exc_info.value.response_body == response_body.decode() + assert exc_info.value.request_id == "unexpected-status-id" + assert exc_info.value.request_id_header == "x-request-id" + + +def test_projects_use_generated_bindings_with_exact_wire_shape_and_additive_responses(): + with project_server() as (url, handler): + projects = _client_for_server(url, Transport()).projects + create_project = {"name": "project/name", "description": "created"} + created = projects.post_project(body=create_project) + listed = projects.get_project( + limit=2, + ids=["project-id", "other/id"], + project_name="project/name", + ) + projects.get_project_id("project/id") + projects.patch_project_id("project/id", body={"description": "updated"}) + projects.delete_project_id("project/id") + + assert create_project == {"name": "project/name", "description": "created"} + assert created["new_backend_field"] == {"preserved": True} + assert listed["new_list_field"] == ["preserved"] + + assert handler.requests == [ + ( + "POST", + "/v1/project", + b'{"name": "project/name", "description": "created"}', + "Bearer test-key", + ), + ( + "GET", + "/v1/project?limit=2&ids=project-id&ids=other%2Fid&project_name=project%2Fname", + b"", + "Bearer test-key", + ), + ("GET", "/v1/project/project%2Fid", b"", "Bearer test-key"), + ("PATCH", "/v1/project/project%2Fid", b'{"description": "updated"}', "Bearer test-key"), + ("DELETE", "/v1/project/project%2Fid", b"", "Bearer test-key"), + ] + + +def test_projects_allow_explicit_organization_overrides(): + with project_server() as (url, handler): + projects = _client_for_server(url, Transport()).projects + projects.post_project(body={"name": "project/name", "org_name": "other org"}) + projects.get_project(org_name="other org") + + assert handler.requests == [ + ( + "POST", + "/v1/project", + b'{"name": "project/name", "org_name": "other org"}', + "Bearer test-key", + ), + ("GET", "/v1/project?org_name=other%20org", b"", "Bearer test-key"), + ] + + +def test_projects_percent_encode_query_values(): + with project_server() as (url, handler): + projects = _client_for_server(url, Transport()).projects + projects.get_project(project_name="R&D#research/v1+beta?x=y", org_name="org&name#one + two") + + assert handler.requests == [ + ( + "GET", + "/v1/project?project_name=R%26D%23research%2Fv1%2Bbeta%3Fx%3Dy&org_name=org%26name%23one%20%2B%20two", + b"", + "Bearer test-key", + ) + ] + + +def _api_key(): + return os.environ.get("BRAINTRUST_API_KEY", "sk-dummy-for-vcr-replay") + + +@pytest.mark.vcr +def test_projects_end_to_end_with_real_backend(): + project_name = "python-sdk-generated-projects-vcr" + with BraintrustClient(api_key=_api_key()) as client: + discovery = client.auth.login() + created = client.openapi.projects.post_project( + body={ + "name": project_name, + "description": "created by the Python SDK VCR test", + "org_name": discovery.organization.name, + } + ) + listed = client.openapi.projects.get_project( + project_name=project_name, + org_name=discovery.organization.name, + ) + fetched = client.openapi.projects.get_project_id(created["id"]) + updated = client.openapi.projects.patch_project_id( + created["id"], body={"description": "updated by the Python SDK VCR test"} + ) + deleted = client.openapi.projects.delete_project_id(created["id"]) + + assert created["name"] == project_name + assert [project["id"] for project in listed["objects"]] == [created["id"]] + assert fetched["id"] == created["id"] + assert updated["description"] == "updated by the Python SDK VCR test" + assert deleted["id"] == created["id"] diff --git a/py/src/braintrust/api/test_transport.py b/py/src/braintrust/api/test_transport.py index 96e99791..04e566f5 100644 --- a/py/src/braintrust/api/test_transport.py +++ b/py/src/braintrust/api/test_transport.py @@ -1,23 +1,19 @@ -import contextlib import datetime -import http.server import io -import socketserver -import threading -import time from email.utils import format_datetime import pytest import requests from braintrust.api import ( BraintrustHTTPError, - BraintrustResponseError, + BraintrustJSONDecodeError, BraintrustRetryExhaustedError, BraintrustTransportError, BraintrustTransportRetryExhaustedError, RetryMode, RetryPolicy, ) +from braintrust.api._test_server import scripted_server from braintrust.api._transport import Transport from braintrust.util import AugmentedHTTPError from requests.adapters import HTTPAdapter @@ -42,61 +38,6 @@ def sleep(self, delay): self.wall_time += delay -@contextlib.contextmanager -def scripted_server(script): - class ScriptedHandler(http.server.BaseHTTPRequestHandler): - request_count = 0 - requests = [] - - def log_message(self, format, *args): - pass - - def do_GET(self): - self._handle() - - def do_POST(self): - self._handle() - - def _handle(self): - request_number = type(self).request_count - type(self).request_count += 1 - content_length = int(self.headers.get("Content-Length", "0")) - body = self.rfile.read(content_length) if content_length else b"" - type(self).requests.append((self.command, self.path, body)) - action = script[min(request_number, len(script) - 1)] - - if action == "close": - self.connection.close() - return - - if action[0] == "sleep": - _, delay, status, headers, response_body = action - time.sleep(delay) - else: - status, headers, response_body = action - - self.send_response(status) - for name, value in headers.items(): - self.send_header(name, value) - self.send_header("Content-Length", str(len(response_body))) - self.end_headers() - try: - self.wfile.write(response_body) - except BrokenPipeError: - pass - - server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), ScriptedHandler) - server.daemon_threads = True - thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) - thread.start() - - try: - yield f"http://127.0.0.1:{server.server_address[1]}", ScriptedHandler - finally: - server.shutdown() - server.server_close() - - def make_transport(clock=None, adapter=None): clock = clock or FakeClock() return Transport( @@ -329,7 +270,7 @@ def test_safe_read_timeout_leaves_room_for_retry(): def test_invalid_json_is_not_retried_and_preserves_response_context(): with scripted_server([(200, {"Content-Type": "application/json"}, b"not json")]) as (url, handler): - with pytest.raises(BraintrustResponseError) as exc_info: + with pytest.raises(BraintrustJSONDecodeError) as exc_info: make_transport().request_json("GET", url, retry_mode=RetryMode.SAFE_READ) assert exc_info.value.status_code == 200 diff --git a/py/src/braintrust/api/types/__init__.py b/py/src/braintrust/api/types/__init__.py index 86d08d26..3365668a 100644 --- a/py/src/braintrust/api/types/__init__.py +++ b/py/src/braintrust/api/types/__init__.py @@ -1,7 +1,11 @@ -"""Public REST API types. +"""Public types for the synchronous Braintrust REST API.""" -No generated REST models are public yet. Types are added here deliberately as resource wrappers are -published; the private ``braintrust.api._generated`` package is not a compatibility surface. -""" +from .._generated.models.projects import CreateProject, GetProjectResponse, PatchProject, Project -__all__: list[str] = [] + +__all__ = [ + "CreateProject", + "GetProjectResponse", + "PatchProject", + "Project", +] diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index 90dd1227..35c2a5db 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -41,10 +41,10 @@ from requests.adapters import HTTPAdapter from . import context, id_gen -from .api._routing import EndpointRouter, normalize_proxy_url -from .api._transport import HTTPConnection, Transport +from .api._routing import normalize_proxy_url +from .api._transport import HTTPConnection from .api._transport import RetryRequestExceptionsAdapter as RetryRequestExceptionsAdapter -from .api.client import BraintrustClient +from .api.client import BraintrustClient, BraintrustOpenApiClient from .api.errors import BraintrustHTTPError from .bt_json import bt_dumps, bt_safe_deep_copy from .db_fields import ( @@ -644,8 +644,8 @@ def check_updated_param(varname, arg, orig): ) self.copy_state(state) - def api_client(self) -> BraintrustClient: - """Return the lazily bootstrapped resource client.""" + def api_client(self) -> BraintrustOpenApiClient: + """Return the lazily bootstrapped OpenAPI client.""" if self._client is None: with self._client_lock: @@ -653,7 +653,7 @@ def api_client(self) -> BraintrustClient: self.login() if self._client is None: raise RuntimeError("Braintrust API client was not initialized during login") - return self._client + return self._client.openapi def app_conn(self): if not self._app_conn: @@ -2179,13 +2179,12 @@ def login_to_state( } ] _check_org_info(state, test_org_info, org_name) - router = EndpointRouter(app_url=state.app_url, api_url=state.api_url, proxy_url=state.proxy_url) - state._client = BraintrustClient.from_transport( - transport=Transport(adapter=_http_adapter), - router=router, + state._client = BraintrustClient( api_key=TEST_API_KEY, - org_id=cast(str, state.org_id), - org_name=cast(str, state.org_name), + app_url=state.app_url, + api_url=state.api_url, + proxy_url=state.proxy_url, + adapter=_http_adapter, ) state.login_token = TEST_API_KEY state.logged_in = True @@ -2197,19 +2196,24 @@ def login_to_state( "or nearest .env.braintrust file." ) + client = BraintrustClient(api_key=api_key, app_url=state.app_url, adapter=_http_adapter) try: - client = BraintrustClient(api_key=api_key, org_name=org_name, app_url=state.app_url, adapter=_http_adapter) + login_result = client.auth.login(org_name=org_name) except BraintrustHTTPError as exc: + client.close() masked_api_key = mask_api_key(api_key) raise ValueError(f"Invalid API key {masked_api_key}: [{exc.status_code}] {exc.response_body}") from exc + except Exception: + client.close() + raise - organization = client.login_result.organization + organization = login_result.organization state._client = client - state.org_id = client.org_id - state.org_name = client.org_name - state.api_url = client.router.api_url - state.proxy_url = client.router.proxy_url - state.is_universal_api = client.router.is_universal_api + state.org_id = organization.id + state.org_name = organization.name + state.api_url = login_result.api_url + state.proxy_url = login_result.proxy_url + state.is_universal_api = organization.is_universal_api state.git_metadata_settings = ( GitMetadataSettings(**organization.git_metadata) if organization.git_metadata else None ) diff --git a/py/src/braintrust/type_tests/test_api_client.py b/py/src/braintrust/type_tests/test_api_client.py index aec402b0..6ef3343d 100644 --- a/py/src/braintrust/type_tests/test_api_client.py +++ b/py/src/braintrust/type_tests/test_api_client.py @@ -2,24 +2,29 @@ from typing import TYPE_CHECKING -from braintrust.api import BraintrustClient, EndpointRouter, RequestTarget +from braintrust.api import BraintrustClient, BraintrustOpenApiClient, EndpointRouter, RequestTarget +from braintrust.api.types import CreateProject, GetProjectResponse, PatchProject, Project if TYPE_CHECKING: - client = BraintrustClient(api_key="key", org_name="org") - client_with_overrides = BraintrustClient( - api_key="key", - app_url="https://app.example.com", - api_url="https://api.example.com", - proxy_url="https://proxy.example.com", + client = BraintrustClient(api_key="key", app_url="https://app.example.com") + discovery = client.auth.login(org_name="org") + openapi_client: BraintrustOpenApiClient = client.openapi + org_id: str = discovery.organization.id + org_name: str = discovery.organization.name + api_url: str | None = client.router.api_url + create_project: CreateProject = {"name": "typed-project"} + patch_project: PatchProject = {"description": "updated"} + project: Project = openapi_client.projects.post_project(body=create_project) + projects: GetProjectResponse = openapi_client.projects.get_project( + ids=[project["id"]], project_name=project["name"] ) - org_id: str = client.org_id - org_name: str = client.org_name - api_url: str | None = client_with_overrides.router.api_url + fetched_project: Project = openapi_client.projects.get_project_id(project["id"]) + updated_project: Project = openapi_client.projects.patch_project_id(project["id"], body=patch_project) + deleted_project: Project = openapi_client.projects.delete_project_id(project["id"]) -def test_api_client_public_types() -> None: +def test_api_client_router() -> None: router = EndpointRouter(app_url="https://app.example.com", api_url="https://api.example.com") assert router.resolve(RequestTarget.API, "ping") == "https://api.example.com/ping" - assert BraintrustClient.__name__ == "BraintrustClient" diff --git a/py/tests/api_codegen/conftest.py b/py/tests/api_codegen/conftest.py index 6df0bb4e..d3659c64 100644 --- a/py/tests/api_codegen/conftest.py +++ b/py/tests/api_codegen/conftest.py @@ -13,7 +13,8 @@ @pytest.fixture def codegen_config(): config = copy.deepcopy(load_config(CONFIG_PATH)) - config["endpoint_generator"]["skip_tags"] = {} + config["endpoint_generator"]["generated_tags"] = ["Widgets"] + config["endpoint_generator"]["idempotent_writes"] = [] return config diff --git a/py/tests/api_codegen/test_generation.py b/py/tests/api_codegen/test_generation.py index 203d28fe..499beea0 100644 --- a/py/tests/api_codegen/test_generation.py +++ b/py/tests/api_codegen/test_generation.py @@ -1,7 +1,8 @@ import copy import re -from openapi_codegen import atomic_replace_tree, compare_generated, generate_tree +import pytest +from openapi_codegen import CodegenError, atomic_replace_tree, compare_generated, generate_tree def _generate(tmp_path, name, config, spec): @@ -10,6 +11,10 @@ def _generate(tmp_path, name, config, spec): return output +def _models_text(generated): + return "\n".join(path.read_text() for path in sorted((generated / "models").glob("*.py"))) + + def test_generation_is_byte_for_byte_deterministic(tmp_path, codegen_config, minimal_spec): first = _generate(tmp_path, "first", codegen_config, minimal_spec) second = _generate(tmp_path, "second", codegen_config, minimal_spec) @@ -17,6 +22,151 @@ def test_generation_is_byte_for_byte_deterministic(tmp_path, codegen_config, min assert compare_generated(first, second) == [] +def test_generation_selects_generated_tag_regardless_of_tag_order(tmp_path, codegen_config, minimal_spec): + minimal_spec["paths"]["/widgets/{widget_id}"]["get"]["tags"] = ["Internal", "Widgets"] + + generated = _generate(tmp_path, "secondary-generated-tag", codegen_config, minimal_spec) + + assert "def get_widget(" in (generated / "widgets.py").read_text() + + +def test_idempotent_writes_use_idempotent_write_retry_mode(tmp_path, codegen_config, minimal_spec): + minimal_spec["paths"]["/widgets"] = { + "post": { + "operationId": "postWidget", + "tags": ["Widgets"], + "responses": { + "200": { + "description": "OK", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Widget"}}}, + } + }, + } + } + codegen_config["endpoint_generator"]["idempotent_writes"] = ["postWidget"] + + generated = _generate(tmp_path, "idempotent-write", codegen_config, minimal_spec) + bindings = (generated / "widgets.py").read_text() + + assert "retry_mode=RetryMode.IDEMPOTENT_WRITE" in bindings + + +def test_multiple_generated_resource_tags_require_explicit_partitioning(tmp_path, codegen_config, minimal_spec): + spec = copy.deepcopy(minimal_spec) + spec["paths"]["/gadgets"] = { + "get": { + "operationId": "getGadget", + "tags": ["Gadgets"], + "responses": { + "200": { + "description": "OK", + "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Widget"}}}, + } + }, + } + } + codegen_config["endpoint_generator"]["generated_tags"] = ["Widgets", "Gadgets"] + + with pytest.raises(CodegenError, match="exactly one generated OpenAPI tag"): + _generate(tmp_path, "multiple-model-resources", codegen_config, spec) + + +def test_unreachable_models_are_omitted_but_transitive_references_are_kept(tmp_path, codegen_config, minimal_spec): + minimal_spec["components"]["schemas"]["Widget"]["properties"]["details"] = { + "$ref": "#/components/schemas/WidgetDetails" + } + minimal_spec["components"]["schemas"]["WidgetDetails"] = { + "type": "object", + "properties": {"count": {"type": "integer"}}, + "required": ["count"], + } + minimal_spec["components"]["schemas"]["Unused"] = { + "type": "object", + "properties": {"value": {"type": "string"}}, + } + generated = _generate(tmp_path, "reachable-models", codegen_config, minimal_spec) + models = _models_text(generated) + + assert "class Widget(TypedDict):" in models + assert "class WidgetDetails(TypedDict):" in models + assert "class Unused(TypedDict):" not in models + + +def test_inline_response_models_use_the_model_generator(tmp_path, codegen_config, minimal_spec): + spec = copy.deepcopy(minimal_spec) + response = spec["paths"]["/widgets/{widget_id}"]["get"]["responses"]["200"] + response["content"]["application/json"]["schema"] = { + "type": "object", + "properties": { + "objects": {"type": "array", "items": {"$ref": "#/components/schemas/Widget"}}, + }, + "required": ["objects"], + } + + generated = _generate(tmp_path, "inline-response", codegen_config, spec) + models = _models_text(generated) + bindings = (generated / "widgets.py").read_text() + + assert "class GetWidgetResponse(TypedDict):" in models + assert "objects: Sequence[Widget]" in models + assert "def get_widget(" in bindings + assert ') -> "GetWidgetResponse":' in bindings + + +def test_inline_response_generated_name_collisions_fail(tmp_path, codegen_config, minimal_spec): + spec = copy.deepcopy(minimal_spec) + spec["components"]["schemas"]["get_widget_response"] = { + "type": "object", + "properties": {"id": {"type": "string"}}, + } + response = spec["paths"]["/widgets/{widget_id}"]["get"]["responses"]["200"] + response["content"]["application/json"]["schema"] = { + "type": "object", + "properties": { + "nested": {"$ref": "#/components/schemas/get_widget_response"}, + }, + } + + with pytest.raises( + CodegenError, + match="Inline response model 'GetWidgetResponse'.*component schema 'get_widget_response'", + ): + _generate(tmp_path, "inline-response-name-collision", codegen_config, spec) + + +def test_mixed_json_and_empty_success_responses_track_statuses(tmp_path, codegen_config, minimal_spec): + responses = minimal_spec["paths"]["/widgets/{widget_id}"]["get"]["responses"] + responses["200"]["content"]["application/json"]["schema"] = { + "type": "object", + "properties": {"widget": {"$ref": "#/components/schemas/Widget"}}, + } + responses["204"] = {"description": "No content"} + + generated = _generate(tmp_path, "mixed-success-responses", codegen_config, minimal_spec) + models = _models_text(generated) + bindings = (generated / "widgets.py").read_text() + + assert "class GetWidgetResponse(TypedDict):" in models + assert "success_statuses=(200, 204)" in bindings + assert "json_success_statuses=(200,)" in bindings + assert ') -> "GetWidgetResponse | None":' in bindings + + +def test_differing_inline_success_response_schemas_fail(tmp_path, codegen_config, minimal_spec): + spec = copy.deepcopy(minimal_spec) + responses = spec["paths"]["/widgets/{widget_id}"]["get"]["responses"] + responses["200"]["content"]["application/json"]["schema"] = { + "type": "object", + "properties": {"name": {"type": "string"}}, + } + responses["201"] = { + "description": "Created", + "content": {"application/json": {"schema": {"type": "object", "properties": {"id": {"type": "string"}}}}}, + } + with pytest.raises(CodegenError, match="conflicting inline success response schemas"): + _generate(tmp_path, "conflicting-inline-responses", codegen_config, spec) + + def test_stale_artifacts_are_reported_and_removed(tmp_path, codegen_config, minimal_spec): """Files the generator no longer emits count as drift, whatever their extension.""" regenerated = _generate(tmp_path, "regenerated", codegen_config, minimal_spec) @@ -49,7 +199,7 @@ def test_nullable_and_missing_fields_remain_distinct(tmp_path, codegen_config, m widget["required"] = ["required_nullable"] generated = _generate(tmp_path, "nullable", codegen_config, spec) - models = (generated / "models.py").read_text() + models = _models_text(generated) assert re.search(r"required_nullable: str \| None", models) assert re.search(r"optional_nullable: NotRequired\[str \| None\]", models) @@ -85,9 +235,12 @@ def test_composition_types_and_json_scalars_generate(tmp_path, codegen_config, m }, } ) + spec["components"]["schemas"]["Widget"]["properties"]["pet_envelope"] = { + "$ref": "#/components/schemas/PetEnvelope" + } generated = _generate(tmp_path, "composition", codegen_config, spec) - models = (generated / "models.py").read_text() + models = _models_text(generated) assert re.search(r"Pet: TypeAlias = Cat \| Dog", models) assert "class PetEnvelope" in models diff --git a/py/tests/api_codegen/test_validation.py b/py/tests/api_codegen/test_validation.py index bcaec1a2..bbcd079c 100644 --- a/py/tests/api_codegen/test_validation.py +++ b/py/tests/api_codegen/test_validation.py @@ -3,13 +3,7 @@ import json import pytest -from openapi_codegen import ( - CodegenError, - normalize_spec, - read_and_verify_spec, - validate_config, - validate_spec, -) +from openapi_codegen import CodegenError, read_and_verify_spec, validate_config, validate_spec def test_hash_and_full_commit_pin_validation(tmp_path, codegen_config, minimal_spec): @@ -28,42 +22,24 @@ def test_hash_and_full_commit_pin_validation(tmp_path, codegen_config, minimal_s validate_config(codegen_config, check_installed_tools=False) -def test_normalization_removes_only_options_and_exact_skips(minimal_spec, codegen_config): +def test_only_allowlisted_operations_are_validated(minimal_spec, codegen_config): spec = copy.deepcopy(minimal_spec) spec["paths"]["/widgets/{widget_id}"]["options"] = { "operationId": "optionsWidget", "tags": ["CORS"], - "responses": {"200": {"description": "OK", "content": {"text/plain": {"schema": {"type": "string"}}}}}, + "responses": {}, } spec["paths"]["/proxy"] = { "post": { - "operationId": "proxyRequest", + "operationId": "proxy{path+}", "tags": ["Proxy"], - "responses": { - "200": { - "description": "OK", - "content": {"application/json": {"schema": {"type": "object"}}}, - } - }, + "responses": {}, } } - codegen_config["endpoint_generator"]["skip_tags"] = { - "Proxy": {"reason": "Specialized streaming transport", "operation_ids": ["proxyRequest"]} - } - - normalized = normalize_spec(spec, validate_spec(spec, codegen_config).skip_ids) - assert set(normalized["paths"]) == {"/widgets/{widget_id}"} - assert set(normalized["paths"]["/widgets/{widget_id}"]) == {"get"} + report = validate_spec(spec, codegen_config) - non_cors_options = copy.deepcopy(spec) - non_cors_options["paths"]["/widgets/{widget_id}"]["options"]["tags"] = ["Other"] - with pytest.raises(CodegenError, match="is not tagged only as CORS"): - validate_spec(non_cors_options, codegen_config) - - codegen_config["endpoint_generator"]["skip_tags"]["Proxy"]["operation_ids"] = [] - with pytest.raises(CodegenError, match="does not match the spec exactly"): - validate_spec(spec, codegen_config) + assert report.operation_count == 1 def test_invalid_and_duplicate_operation_ids_fail(minimal_spec, codegen_config): @@ -79,16 +55,16 @@ def test_invalid_and_duplicate_operation_ids_fail(minimal_spec, codegen_config): validate_spec(spec, codegen_config) -def test_duplicate_operation_ids_fail_even_when_one_copy_is_skipped(minimal_spec, codegen_config): - """A skipped duplicate must not silently take its supported twin out of the generated client.""" +def test_duplicate_operation_ids_fail_when_only_one_operation_is_selected(minimal_spec, codegen_config): spec = copy.deepcopy(minimal_spec) duplicate = copy.deepcopy(spec["paths"]["/widgets/{widget_id}"]["get"]) duplicate["parameters"] = [] duplicate["tags"] = ["Proxy"] - spec["paths"]["/proxy"] = {"get": duplicate} - codegen_config["endpoint_generator"]["skip_tags"] = { - "Proxy": {"reason": "Specialized streaming transport", "operation_ids": ["getWidget"]} + duplicate["responses"]["200"]["content"]["application/json"]["schema"] = { + "$ref": "#/components/schemas/Unselected" } + spec["paths"]["/proxy"] = {"get": duplicate} + spec["components"]["schemas"]["Unselected"] = {"type": "string"} with pytest.raises(CodegenError, match="Duplicate operationId"): validate_spec(spec, codegen_config) @@ -104,11 +80,27 @@ def test_inline_operation_name_collisions_fail(minimal_spec, codegen_config): with pytest.raises(CodegenError, match="Inline operation name collision"): validate_spec(spec, codegen_config) + spec = copy.deepcopy(minimal_spec) + spec["paths"]["/widgets/{widget_id}"]["get"]["operationId"] = "getURL" + second = copy.deepcopy(spec["paths"]["/widgets/{widget_id}"]["get"]) + second["operationId"] = "getUrl" + second["parameters"] = [] + spec["paths"]["/other"] = {"get": second} + + with pytest.raises(CodegenError, match="Generated operation identifier collision"): + validate_spec(spec, codegen_config) + def test_schema_name_collisions_fail(minimal_spec, codegen_config): spec = copy.deepcopy(minimal_spec) spec["components"]["schemas"]["foo-bar"] = {"type": "string"} spec["components"]["schemas"]["foo_bar"] = {"type": "string"} + spec["components"]["schemas"]["Widget"]["properties"].update( + { + "first": {"$ref": "#/components/schemas/foo-bar"}, + "second": {"$ref": "#/components/schemas/foo_bar"}, + } + ) with pytest.raises(CodegenError, match="Schema name collision"): validate_spec(spec, codegen_config) @@ -137,9 +129,18 @@ def test_media_types_and_success_statuses_are_validated(minimal_spec, codegen_co validate_spec(spec, codegen_config) -def test_referenced_parameters_resolve_and_match_path(minimal_spec, codegen_config): - validate_spec(minimal_spec, codegen_config) +def test_idempotent_writes_must_reference_generated_operations(minimal_spec, codegen_config): + codegen_config["endpoint_generator"]["idempotent_writes"] = ["missingOperation"] + + with pytest.raises(CodegenError, match="idempotent_writes.*missingOperation"): + validate_spec(minimal_spec, codegen_config) + codegen_config["endpoint_generator"]["idempotent_writes"] = ["getWidget"] + with pytest.raises(CodegenError, match="idempotent_writes references read operation 'getWidget'"): + validate_spec(minimal_spec, codegen_config) + + +def test_referenced_parameters_resolve_and_match_path(minimal_spec, codegen_config): spec = copy.deepcopy(minimal_spec) spec["components"]["parameters"]["WidgetId"]["schema"] = {"type": "object"} with pytest.raises(CodegenError, match="must be scalar"): @@ -150,6 +151,38 @@ def test_referenced_parameters_resolve_and_match_path(minimal_spec, codegen_conf with pytest.raises(CodegenError, match="Unresolved OpenAPI reference"): validate_spec(spec, codegen_config) + spec = copy.deepcopy(minimal_spec) + spec["paths"]["/widgets/{widget_id}"]["get"]["parameters"].append( + {"name": "x-test", "in": "header", "schema": {"type": "string"}} + ) + with pytest.raises(CodegenError, match="unsupported parameter location"): + validate_spec(spec, codegen_config) + + spec = copy.deepcopy(minimal_spec) + spec["components"]["parameters"]["WidgetId"]["style"] = "label" + with pytest.raises(CodegenError, match="unsupported path parameter style"): + validate_spec(spec, codegen_config) + + spec = copy.deepcopy(minimal_spec) + spec["paths"]["/widgets/{widget_id}"]["get"]["parameters"].append( + {"name": "names", "in": "query", "style": "spaceDelimited", "schema": {"type": "string"}} + ) + with pytest.raises(CodegenError, match="unsupported query parameter style"): + validate_spec(spec, codegen_config) + + spec = copy.deepcopy(minimal_spec) + spec["paths"]["/widgets/{widget_id}"]["get"]["parameters"].append( + { + "name": "names", + "in": "query", + "style": "form", + "explode": False, + "schema": {"type": "array", "items": {"type": "string"}}, + } + ) + with pytest.raises(CodegenError, match="query array parameters must be exploded"): + validate_spec(spec, codegen_config) + def test_only_json_compatible_schema_types_and_values_are_supported(minimal_spec, codegen_config): spec = copy.deepcopy(minimal_spec) @@ -188,9 +221,11 @@ def test_unsupported_types_are_caught_under_every_schema_keyword(minimal_spec, c def test_malformed_specs_and_configs_raise_actionable_errors(minimal_spec, codegen_config): """Shape problems have to surface as CodegenError; a bare KeyError escapes the scripts' handler.""" # A spec without any components is empty, not malformed -- it must not blow up on a missing key. - assert validate_spec({"openapi": "3.0.3", "paths": {}}, codegen_config).schema_count == 0 + empty_config = copy.deepcopy(codegen_config) + empty_config["endpoint_generator"]["generated_tags"] = [] + assert validate_spec({"openapi": "3.0.3", "paths": {}}, empty_config).schema_count == 0 with pytest.raises(CodegenError, match="components.schemas must be an object"): - validate_spec({"openapi": "3.0.3", "paths": {}, "components": {"schemas": []}}, codegen_config) + validate_spec({"openapi": "3.0.3", "paths": {}, "components": {"schemas": []}}, empty_config) spec = copy.deepcopy(minimal_spec) spec["paths"]["/widgets/{widget_id}"]["get"]["responses"]["200"]["content"]["application/json"] = None