From 2f967e0b7fafb80974c41a904009118a7fba72de Mon Sep 17 00:00:00 2001 From: "ci.datadog-api-spec" Date: Tue, 2 Jun 2026 13:29:28 +0000 Subject: [PATCH] Regenerate client from commit 78f6b5f of spec repo --- .generator/schemas/v2/openapi.yaml | 331 +++++++++++- .../CreateLLMObsExperiment.java | 1 + .../ListLLMObsExperimentEventsV1.java | 26 + .../ListLLMObsExperimentEventsV2.java | 26 + .../UpdateLLMObsExperiment.java | 6 +- .../com/datadog/api/client/ApiClient.java | 2 + .../client/v2/api/LlmObservabilityApi.java | 495 +++++++++++++++++- ...LLMObsExperimentDataAttributesRequest.java | 64 ++- ...LMObsExperimentDataAttributesResponse.java | 357 ++++++++++++- .../LLMObsExperimentSpanDataResponse.java | 211 ++++++++ .../v2/model/LLMObsExperimentSpanType.java | 57 ++ .../model/LLMObsExperimentSpansResponse.java | 160 ++++++ .../v2/model/LLMObsExperimentStatus.java | 60 +++ ...ExperimentUpdateDataAttributesRequest.java | 128 ++++- .../client/v2/model/LLMObsExperimentUser.java | 248 +++++++++ .../client/v2/api/llm_observability.feature | 60 ++- .../com/datadog/api/client/v2/api/undo.json | 12 + 17 files changed, 2224 insertions(+), 20 deletions(-) create mode 100644 examples/v2/llm-observability/ListLLMObsExperimentEventsV1.java create mode 100644 examples/v2/llm-observability/ListLLMObsExperimentEventsV2.java create mode 100644 src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpanDataResponse.java create mode 100644 src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpanType.java create mode 100644 src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpansResponse.java create mode 100644 src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentStatus.java create mode 100644 src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentUser.java diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index ecc69ea9c12..a98155b89ea 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -46420,18 +46420,34 @@ components: description: Name of the experiment. example: "My Experiment v1" type: string + parent_experiment_id: + description: Identifier of the parent (baseline) experiment this experiment is run against. + example: "3fd6b5e0-8910-4b1c-a7d0-5b84de329012" + type: string project_id: description: Identifier of the project this experiment belongs to. example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" type: string + run_count: + description: Expected number of runs for this experiment. + format: int32 + maximum: 2147483647 + type: integer required: - project_id - - dataset_id - name type: object LLMObsExperimentDataAttributesResponse: description: Attributes of an LLM Observability experiment. properties: + aggregate_data: + additionalProperties: {} + description: >- + Pre-computed aggregate metrics for this experiment run, including eval score distributions, token costs, and error rates. + nullable: true + type: object + author: + $ref: "#/components/schemas/LLMObsExperimentUser" config: additionalProperties: {} description: Configuration parameters for the experiment. @@ -46446,11 +46462,34 @@ components: description: Identifier of the dataset used in this experiment. example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" type: string + dataset_name: + description: |- + Name of the dataset used in this experiment. + Only present when `include[dataset_names]` is `true`. + nullable: true + type: string + dataset_version: + description: Version of the dataset used in this experiment. + format: int64 + type: integer + deleted_at: + description: Timestamp when the experiment was soft-deleted, if applicable. + format: date-time + nullable: true + type: string description: description: Description of the experiment. example: "" nullable: true type: string + error: + description: Error message describing why the experiment failed, if applicable. + nullable: true + type: string + experiment: + description: Logical name of the experiment, shared across all runs of the same pipeline. + example: "my-pipeline" + type: string metadata: additionalProperties: {} description: Arbitrary metadata associated with the experiment. @@ -46460,10 +46499,22 @@ components: description: Name of the experiment. example: "My Experiment v1" type: string + parent_experiment_id: + description: Identifier of the parent (baseline) experiment this experiment was run against, if any. + example: "3fd6b5e0-8910-4b1c-a7d0-5b84de329012" + nullable: true + type: string project_id: description: Identifier of the project this experiment belongs to. example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" type: string + run_count: + description: Expected number of runs for this experiment. + format: int32 + maximum: 2147483647 + type: integer + status: + $ref: "#/components/schemas/LLMObsExperimentStatus" updated_at: description: Timestamp when the experiment was last updated. example: "2024-01-15T10:30:00Z" @@ -46818,6 +46869,22 @@ components: - duration - status type: object + LLMObsExperimentSpanDataResponse: + description: JSON:API data item wrapping a single experiment span with evaluations. + properties: + attributes: + $ref: "#/components/schemas/LLMObsExperimentSpanWithEvals" + id: + description: Unique identifier of the span. + example: "00000000-0000-0000-0000-000000000001" + type: string + type: + $ref: "#/components/schemas/LLMObsExperimentSpanType" + required: + - id + - type + - attributes + type: object LLMObsExperimentSpanError: description: Error details for an experiment span. properties: @@ -46858,6 +46925,14 @@ components: x-enum-varnames: - OK - ERROR + LLMObsExperimentSpanType: + description: Resource type for a span item in an experiment spans response. + enum: + - experiments + example: experiments + type: string + x-enum-varnames: + - EXPERIMENTS_SPAN LLMObsExperimentSpanWithEvals: description: An experiment span enriched with its associated evaluation metrics. properties: @@ -46915,6 +46990,32 @@ components: example: "abc123def456" type: string type: object + LLMObsExperimentSpansResponse: + description: >- + Response for listing experiment spans (v1). Returns only spans with their evaluation metrics. No summary metrics or pagination are included. Deprecated in favor of `ListLLMObsExperimentEventsV3`. + properties: + data: + description: List of experiment spans with their evaluation metrics. + items: + $ref: "#/components/schemas/LLMObsExperimentSpanDataResponse" + type: array + required: + - data + type: object + LLMObsExperimentStatus: + description: Execution status of an LLM Observability experiment. + enum: + - running + - completed + - failed + - interrupted + example: completed + type: string + x-enum-varnames: + - RUNNING + - COMPLETED + - FAILED + - INTERRUPTED LLMObsExperimentType: description: Resource type of an LLM Observability experiment. enum: @@ -46926,12 +47027,25 @@ components: LLMObsExperimentUpdateDataAttributesRequest: description: Attributes for updating an LLM Observability experiment. properties: + dataset_id: + description: Updated identifier of the dataset used in this experiment. + example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" + type: string description: description: Updated description of the experiment. type: string + error: + description: Error message describing why the experiment failed, if applicable. + type: string + metadata: + additionalProperties: {} + description: Updated arbitrary metadata associated with the experiment. + type: object name: description: Updated name of the experiment. type: string + status: + $ref: "#/components/schemas/LLMObsExperimentStatus" type: object LLMObsExperimentUpdateDataRequest: description: Data object for updating an LLM Observability experiment. @@ -46952,6 +47066,30 @@ components: required: - data type: object + LLMObsExperimentUser: + description: User data for the author of an experiment. Only present when `include[user_data]` is `true`. + properties: + email: + description: Email address of the user. + example: "jane.doe@example.com" + type: string + handle: + description: Username or handle associated with the user's Datadog account. + example: "jane.doe@example.com" + type: string + icon: + description: URL of the user's icon. + example: "https://example.com/icon.png" + type: string + id: + description: Unique identifier of the user. + example: "00000000-0000-0000-0000-000000000010" + type: string + name: + description: Display name of the user. + example: "Jane Doe" + type: string + type: object LLMObsExperimentationAnalyticsAggregate: description: Analytics aggregation parameters. properties: @@ -130830,16 +130968,59 @@ paths: name: filter[id] schema: type: string - - description: Use the Pagination cursor to retrieve the next page of results. + - description: Filter experiments by their exact run name. + in: query + name: filter[name] + schema: + type: string + - description: >- + Filter by logical experiment name. This is the `name` field set when creating an experiment via `POST /experiments`. Returns all experiment runs that share the same name, enabling cross-commit and cross-branch comparisons. + in: query + name: filter[experiment] + schema: + type: string + - description: |- + Filter by JSONB metadata containment. Provide a JSON object string where + experiments whose metadata contains all specified key-value pairs are returned. + For example: `{"commit":"abc123","branch":"main"}`. + in: query + name: filter[metadata] + schema: + type: string + - description: >- + Filter experiments by the ID of their parent (baseline) experiment. Returns all experiments that were run against the given baseline. Can be specified multiple times. + in: query + name: filter[parent_experiment_id] + schema: + type: string + - description: When `true`, return only soft-deleted experiments. Defaults to `false`. + in: query + name: filter[is_deleted] + schema: + type: boolean + - description: When `true`, enrich each experiment with its author's user data in the `author` field. + in: query + name: include[user_data] + schema: + type: boolean + - description: When `true`, enrich each experiment with its dataset name in the `dataset_name` field. + in: query + name: include[dataset_names] + schema: + type: boolean + - description: Use the pagination cursor returned in `meta.after` to retrieve the next page of results. in: query name: page[cursor] schema: type: string - - description: Maximum number of results to return per page. + - description: |- + Maximum number of results to return per page. Values above 5000 are clamped + to 5000. Defaults to 5000. in: query name: page[limit] schema: format: int64 + maximum: 5000 type: integer responses: "200": @@ -131110,6 +131291,76 @@ paths: **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/llm-obs/v1/experiments/{experiment_id}/events: + get: + deprecated: true + description: >- + Retrieve spans with their evaluation metrics for a given experiment. Returns spans only, with no summary metrics and no pagination. Deprecated: use `ListLLMObsExperimentEventsV3` instead. + operationId: ListLLMObsExperimentEventsV1 + parameters: + - $ref: "#/components/parameters/LLMObsExperimentIDPathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + - attributes: + duration: 1500000000.0 + eval_metrics: [] + name: llm_call + span_id: span-7a1b2c3d + start_ns: 1705314600000000000 + status: ok + tags: [] + trace_id: abc123def456 + id: 00000000-0000-0000-0000-000000000001 + type: experiments + schema: + $ref: "#/components/schemas/LLMObsExperimentSpansResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List LLM Observability experiment spans (v1) + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). post: description: Push spans and metrics for an LLM Observability experiment. operationId: CreateLLMObsExperimentEvents @@ -133195,6 +133446,80 @@ paths: x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v2/experiments/{experiment_id}/events: + get: + deprecated: true + description: >- + Retrieve spans and experiment-level summary metrics for a given experiment. Returns the full events payload without pagination. Deprecated: use `ListLLMObsExperimentEventsV3` instead. + operationId: ListLLMObsExperimentEventsV2 + parameters: + - $ref: "#/components/parameters/LLMObsExperimentIDPathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + spans: + - duration: 1500000000.0 + eval_metrics: [] + id: 00000000-0000-0000-0000-000000000002 + name: llm_call + span_id: span-7a1b2c3d + start_ns: 1705314600000000000 + status: ok + tags: [] + trace_id: abc123def456 + summary_metrics: [] + id: 00000000-0000-0000-0000-000000000001 + type: experiment_events + schema: + $ref: "#/components/schemas/LLMObsExperimentEventsV2Response" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List LLM Observability experiment events (v2) + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/llm-obs/v2/{project_id}/datasets/{dataset_id}/records/upload: post: description: |- diff --git a/examples/v2/llm-observability/CreateLLMObsExperiment.java b/examples/v2/llm-observability/CreateLLMObsExperiment.java index 10f50e2fc43..578228a43af 100644 --- a/examples/v2/llm-observability/CreateLLMObsExperiment.java +++ b/examples/v2/llm-observability/CreateLLMObsExperiment.java @@ -23,6 +23,7 @@ public static void main(String[] args) { new LLMObsExperimentDataAttributesRequest() .datasetId("9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d") .name("My Experiment v1") + .parentExperimentId("3fd6b5e0-8910-4b1c-a7d0-5b84de329012") .projectId("a33671aa-24fd-4dcd-9b33-a8ec7dde7751")) .type(LLMObsExperimentType.EXPERIMENTS)); diff --git a/examples/v2/llm-observability/ListLLMObsExperimentEventsV1.java b/examples/v2/llm-observability/ListLLMObsExperimentEventsV1.java new file mode 100644 index 00000000000..40b478f38c1 --- /dev/null +++ b/examples/v2/llm-observability/ListLLMObsExperimentEventsV1.java @@ -0,0 +1,26 @@ +// List LLM Observability experiment spans (v1) returns "OK" response + +import com.datadog.api.client.ApiClient; +import com.datadog.api.client.ApiException; +import com.datadog.api.client.v2.api.LlmObservabilityApi; +import com.datadog.api.client.v2.model.LLMObsExperimentSpansResponse; + +public class Example { + public static void main(String[] args) { + ApiClient defaultClient = ApiClient.getDefaultApiClient(); + defaultClient.setUnstableOperationEnabled("v2.listLLMObsExperimentEventsV1", true); + LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient); + + try { + LLMObsExperimentSpansResponse result = + apiInstance.listLLMObsExperimentEventsV1("3fd6b5e0-8910-4b1c-a7d0-5b84de329012"); + System.out.println(result); + } catch (ApiException e) { + System.err.println("Exception when calling LlmObservabilityApi#listLLMObsExperimentEventsV1"); + System.err.println("Status code: " + e.getCode()); + System.err.println("Reason: " + e.getResponseBody()); + System.err.println("Response headers: " + e.getResponseHeaders()); + e.printStackTrace(); + } + } +} diff --git a/examples/v2/llm-observability/ListLLMObsExperimentEventsV2.java b/examples/v2/llm-observability/ListLLMObsExperimentEventsV2.java new file mode 100644 index 00000000000..ba97f4c50cb --- /dev/null +++ b/examples/v2/llm-observability/ListLLMObsExperimentEventsV2.java @@ -0,0 +1,26 @@ +// List LLM Observability experiment events (v2) returns "OK" response + +import com.datadog.api.client.ApiClient; +import com.datadog.api.client.ApiException; +import com.datadog.api.client.v2.api.LlmObservabilityApi; +import com.datadog.api.client.v2.model.LLMObsExperimentEventsV2Response; + +public class Example { + public static void main(String[] args) { + ApiClient defaultClient = ApiClient.getDefaultApiClient(); + defaultClient.setUnstableOperationEnabled("v2.listLLMObsExperimentEventsV2", true); + LlmObservabilityApi apiInstance = new LlmObservabilityApi(defaultClient); + + try { + LLMObsExperimentEventsV2Response result = + apiInstance.listLLMObsExperimentEventsV2("3fd6b5e0-8910-4b1c-a7d0-5b84de329012"); + System.out.println(result); + } catch (ApiException e) { + System.err.println("Exception when calling LlmObservabilityApi#listLLMObsExperimentEventsV2"); + System.err.println("Status code: " + e.getCode()); + System.err.println("Reason: " + e.getResponseBody()); + System.err.println("Response headers: " + e.getResponseHeaders()); + e.printStackTrace(); + } + } +} diff --git a/examples/v2/llm-observability/UpdateLLMObsExperiment.java b/examples/v2/llm-observability/UpdateLLMObsExperiment.java index 7c6a1824f00..d7e193515ea 100644 --- a/examples/v2/llm-observability/UpdateLLMObsExperiment.java +++ b/examples/v2/llm-observability/UpdateLLMObsExperiment.java @@ -4,6 +4,7 @@ import com.datadog.api.client.ApiException; import com.datadog.api.client.v2.api.LlmObservabilityApi; import com.datadog.api.client.v2.model.LLMObsExperimentResponse; +import com.datadog.api.client.v2.model.LLMObsExperimentStatus; import com.datadog.api.client.v2.model.LLMObsExperimentType; import com.datadog.api.client.v2.model.LLMObsExperimentUpdateDataAttributesRequest; import com.datadog.api.client.v2.model.LLMObsExperimentUpdateDataRequest; @@ -19,7 +20,10 @@ public static void main(String[] args) { new LLMObsExperimentUpdateRequest() .data( new LLMObsExperimentUpdateDataRequest() - .attributes(new LLMObsExperimentUpdateDataAttributesRequest()) + .attributes( + new LLMObsExperimentUpdateDataAttributesRequest() + .datasetId("9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d") + .status(LLMObsExperimentStatus.COMPLETED)) .type(LLMObsExperimentType.EXPERIMENTS)); try { diff --git a/src/main/java/com/datadog/api/client/ApiClient.java b/src/main/java/com/datadog/api/client/ApiClient.java index c26c644775a..9a6991fa8c0 100644 --- a/src/main/java/com/datadog/api/client/ApiClient.java +++ b/src/main/java/com/datadog/api/client/ApiClient.java @@ -803,6 +803,8 @@ public class ApiClient { put("v2.listLLMObsDatasets", false); put("v2.listLLMObsDatasetVersions", false); put("v2.listLLMObsExperimentEvents", false); + put("v2.listLLMObsExperimentEventsV1", false); + put("v2.listLLMObsExperimentEventsV2", false); put("v2.listLLMObsExperiments", false); put("v2.listLLMObsIntegrationAccounts", false); put("v2.listLLMObsIntegrationModels", false); diff --git a/src/main/java/com/datadog/api/client/v2/api/LlmObservabilityApi.java b/src/main/java/com/datadog/api/client/v2/api/LlmObservabilityApi.java index 942eef32060..9771c57b8ec 100644 --- a/src/main/java/com/datadog/api/client/v2/api/LlmObservabilityApi.java +++ b/src/main/java/com/datadog/api/client/v2/api/LlmObservabilityApi.java @@ -41,6 +41,7 @@ import com.datadog.api.client.v2.model.LLMObsExperimentEventsV2Response; import com.datadog.api.client.v2.model.LLMObsExperimentRequest; import com.datadog.api.client.v2.model.LLMObsExperimentResponse; +import com.datadog.api.client.v2.model.LLMObsExperimentSpansResponse; import com.datadog.api.client.v2.model.LLMObsExperimentUpdateRequest; import com.datadog.api.client.v2.model.LLMObsExperimentationAnalyticsRequest; import com.datadog.api.client.v2.model.LLMObsExperimentationAnalyticsResponse; @@ -5826,11 +5827,372 @@ public ApiResponse listLLMObsExperimentEventsW new GenericType() {}); } + /** + * List LLM Observability experiment spans (v1). + * + *

See {@link #listLLMObsExperimentEventsV1WithHttpInfo}. + * + * @param experimentId The ID of the LLM Observability experiment. (required) + * @return LLMObsExperimentSpansResponse + * @throws ApiException if fails to make API call + * @deprecated + */ + @Deprecated + public LLMObsExperimentSpansResponse listLLMObsExperimentEventsV1(String experimentId) + throws ApiException { + return listLLMObsExperimentEventsV1WithHttpInfo(experimentId).getData(); + } + + /** + * List LLM Observability experiment spans (v1). + * + *

See {@link #listLLMObsExperimentEventsV1WithHttpInfoAsync}. + * + * @param experimentId The ID of the LLM Observability experiment. (required) + * @return CompletableFuture<LLMObsExperimentSpansResponse> + * @deprecated + */ + @Deprecated + public CompletableFuture listLLMObsExperimentEventsV1Async( + String experimentId) { + return listLLMObsExperimentEventsV1WithHttpInfoAsync(experimentId) + .thenApply( + response -> { + return response.getData(); + }); + } + + /** + * Retrieve spans with their evaluation metrics for a given experiment. Returns spans only, with + * no summary metrics and no pagination. Deprecated: use ListLLMObsExperimentEventsV3 + * instead. + * + * @param experimentId The ID of the LLM Observability experiment. (required) + * @return ApiResponse<LLMObsExperimentSpansResponse> + * @throws ApiException if fails to make API call + * @http.response.details + * + * + * + * + * + * + * + * + * + * + *
Response details
Status Code Description Response Headers
200 OK -
400 Bad Request -
401 Unauthorized -
403 Forbidden -
404 Not Found -
429 Too many requests -
500 Internal Server Error -
+ * + * @deprecated + */ + @Deprecated + public ApiResponse listLLMObsExperimentEventsV1WithHttpInfo( + String experimentId) throws ApiException { + // Check if unstable operation is enabled + String operationId = "listLLMObsExperimentEventsV1"; + if (apiClient.isUnstableOperationEnabled("v2." + operationId)) { + apiClient.getLogger().warning(String.format("Using unstable operation '%s'", operationId)); + } else { + throw new ApiException(0, String.format("Unstable operation '%s' is disabled", operationId)); + } + Object localVarPostBody = null; + + // verify the required parameter 'experimentId' is set + if (experimentId == null) { + throw new ApiException( + 400, + "Missing the required parameter 'experimentId' when calling" + + " listLLMObsExperimentEventsV1"); + } + // create path and map variables + String localVarPath = + "/api/v2/llm-obs/v1/experiments/{experiment_id}/events" + .replaceAll( + "\\{" + "experiment_id" + "\\}", apiClient.escapeString(experimentId.toString())); + + Map localVarHeaderParams = new HashMap(); + + Invocation.Builder builder = + apiClient.createBuilder( + "v2.LlmObservabilityApi.listLLMObsExperimentEventsV1", + localVarPath, + new ArrayList(), + localVarHeaderParams, + new HashMap(), + new String[] {"application/json"}, + new String[] {"apiKeyAuth", "appKeyAuth"}); + return apiClient.invokeAPI( + "GET", + builder, + localVarHeaderParams, + new String[] {}, + localVarPostBody, + new HashMap(), + false, + new GenericType() {}); + } + + /** + * List LLM Observability experiment spans (v1). + * + *

See {@link #listLLMObsExperimentEventsV1WithHttpInfo}. + * + * @param experimentId The ID of the LLM Observability experiment. (required) + * @return CompletableFuture<ApiResponse<LLMObsExperimentSpansResponse>> + * @deprecated + */ + @Deprecated + public CompletableFuture> + listLLMObsExperimentEventsV1WithHttpInfoAsync(String experimentId) { + // Check if unstable operation is enabled + String operationId = "listLLMObsExperimentEventsV1"; + if (apiClient.isUnstableOperationEnabled("v2." + operationId)) { + apiClient.getLogger().warning(String.format("Using unstable operation '%s'", operationId)); + } else { + CompletableFuture> result = + new CompletableFuture<>(); + result.completeExceptionally( + new ApiException(0, String.format("Unstable operation '%s' is disabled", operationId))); + return result; + } + Object localVarPostBody = null; + + // verify the required parameter 'experimentId' is set + if (experimentId == null) { + CompletableFuture> result = + new CompletableFuture<>(); + result.completeExceptionally( + new ApiException( + 400, + "Missing the required parameter 'experimentId' when calling" + + " listLLMObsExperimentEventsV1")); + return result; + } + // create path and map variables + String localVarPath = + "/api/v2/llm-obs/v1/experiments/{experiment_id}/events" + .replaceAll( + "\\{" + "experiment_id" + "\\}", apiClient.escapeString(experimentId.toString())); + + Map localVarHeaderParams = new HashMap(); + + Invocation.Builder builder; + try { + builder = + apiClient.createBuilder( + "v2.LlmObservabilityApi.listLLMObsExperimentEventsV1", + localVarPath, + new ArrayList(), + localVarHeaderParams, + new HashMap(), + new String[] {"application/json"}, + new String[] {"apiKeyAuth", "appKeyAuth"}); + } catch (ApiException ex) { + CompletableFuture> result = + new CompletableFuture<>(); + result.completeExceptionally(ex); + return result; + } + return apiClient.invokeAPIAsync( + "GET", + builder, + localVarHeaderParams, + new String[] {}, + localVarPostBody, + new HashMap(), + false, + new GenericType() {}); + } + + /** + * List LLM Observability experiment events (v2). + * + *

See {@link #listLLMObsExperimentEventsV2WithHttpInfo}. + * + * @param experimentId The ID of the LLM Observability experiment. (required) + * @return LLMObsExperimentEventsV2Response + * @throws ApiException if fails to make API call + * @deprecated + */ + @Deprecated + public LLMObsExperimentEventsV2Response listLLMObsExperimentEventsV2(String experimentId) + throws ApiException { + return listLLMObsExperimentEventsV2WithHttpInfo(experimentId).getData(); + } + + /** + * List LLM Observability experiment events (v2). + * + *

See {@link #listLLMObsExperimentEventsV2WithHttpInfoAsync}. + * + * @param experimentId The ID of the LLM Observability experiment. (required) + * @return CompletableFuture<LLMObsExperimentEventsV2Response> + * @deprecated + */ + @Deprecated + public CompletableFuture listLLMObsExperimentEventsV2Async( + String experimentId) { + return listLLMObsExperimentEventsV2WithHttpInfoAsync(experimentId) + .thenApply( + response -> { + return response.getData(); + }); + } + + /** + * Retrieve spans and experiment-level summary metrics for a given experiment. Returns the full + * events payload without pagination. Deprecated: use ListLLMObsExperimentEventsV3 + * instead. + * + * @param experimentId The ID of the LLM Observability experiment. (required) + * @return ApiResponse<LLMObsExperimentEventsV2Response> + * @throws ApiException if fails to make API call + * @http.response.details + * + * + * + * + * + * + * + * + * + * + *
Response details
Status Code Description Response Headers
200 OK -
400 Bad Request -
401 Unauthorized -
403 Forbidden -
404 Not Found -
429 Too many requests -
500 Internal Server Error -
+ * + * @deprecated + */ + @Deprecated + public ApiResponse listLLMObsExperimentEventsV2WithHttpInfo( + String experimentId) throws ApiException { + // Check if unstable operation is enabled + String operationId = "listLLMObsExperimentEventsV2"; + if (apiClient.isUnstableOperationEnabled("v2." + operationId)) { + apiClient.getLogger().warning(String.format("Using unstable operation '%s'", operationId)); + } else { + throw new ApiException(0, String.format("Unstable operation '%s' is disabled", operationId)); + } + Object localVarPostBody = null; + + // verify the required parameter 'experimentId' is set + if (experimentId == null) { + throw new ApiException( + 400, + "Missing the required parameter 'experimentId' when calling" + + " listLLMObsExperimentEventsV2"); + } + // create path and map variables + String localVarPath = + "/api/v2/llm-obs/v2/experiments/{experiment_id}/events" + .replaceAll( + "\\{" + "experiment_id" + "\\}", apiClient.escapeString(experimentId.toString())); + + Map localVarHeaderParams = new HashMap(); + + Invocation.Builder builder = + apiClient.createBuilder( + "v2.LlmObservabilityApi.listLLMObsExperimentEventsV2", + localVarPath, + new ArrayList(), + localVarHeaderParams, + new HashMap(), + new String[] {"application/json"}, + new String[] {"apiKeyAuth", "appKeyAuth"}); + return apiClient.invokeAPI( + "GET", + builder, + localVarHeaderParams, + new String[] {}, + localVarPostBody, + new HashMap(), + false, + new GenericType() {}); + } + + /** + * List LLM Observability experiment events (v2). + * + *

See {@link #listLLMObsExperimentEventsV2WithHttpInfo}. + * + * @param experimentId The ID of the LLM Observability experiment. (required) + * @return CompletableFuture<ApiResponse<LLMObsExperimentEventsV2Response>> + * @deprecated + */ + @Deprecated + public CompletableFuture> + listLLMObsExperimentEventsV2WithHttpInfoAsync(String experimentId) { + // Check if unstable operation is enabled + String operationId = "listLLMObsExperimentEventsV2"; + if (apiClient.isUnstableOperationEnabled("v2." + operationId)) { + apiClient.getLogger().warning(String.format("Using unstable operation '%s'", operationId)); + } else { + CompletableFuture> result = + new CompletableFuture<>(); + result.completeExceptionally( + new ApiException(0, String.format("Unstable operation '%s' is disabled", operationId))); + return result; + } + Object localVarPostBody = null; + + // verify the required parameter 'experimentId' is set + if (experimentId == null) { + CompletableFuture> result = + new CompletableFuture<>(); + result.completeExceptionally( + new ApiException( + 400, + "Missing the required parameter 'experimentId' when calling" + + " listLLMObsExperimentEventsV2")); + return result; + } + // create path and map variables + String localVarPath = + "/api/v2/llm-obs/v2/experiments/{experiment_id}/events" + .replaceAll( + "\\{" + "experiment_id" + "\\}", apiClient.escapeString(experimentId.toString())); + + Map localVarHeaderParams = new HashMap(); + + Invocation.Builder builder; + try { + builder = + apiClient.createBuilder( + "v2.LlmObservabilityApi.listLLMObsExperimentEventsV2", + localVarPath, + new ArrayList(), + localVarHeaderParams, + new HashMap(), + new String[] {"application/json"}, + new String[] {"apiKeyAuth", "appKeyAuth"}); + } catch (ApiException ex) { + CompletableFuture> result = + new CompletableFuture<>(); + result.completeExceptionally(ex); + return result; + } + return apiClient.invokeAPIAsync( + "GET", + builder, + localVarHeaderParams, + new String[] {}, + localVarPostBody, + new HashMap(), + false, + new GenericType() {}); + } + /** Manage optional parameters to listLLMObsExperiments. */ public static class ListLLMObsExperimentsOptionalParameters { private String filterProjectId; private String filterDatasetId; private String filterId; + private String filterName; + private String filterExperiment; + private String filterMetadata; + private String filterParentExperimentId; + private Boolean filterIsDeleted; + private Boolean includeUserData; + private Boolean includeDatasetNames; private String pageCursor; private Long pageLimit; @@ -5869,10 +6231,100 @@ public ListLLMObsExperimentsOptionalParameters filterId(String filterId) { return this; } + /** + * Set filterName. + * + * @param filterName Filter experiments by their exact run name. (optional) + * @return ListLLMObsExperimentsOptionalParameters + */ + public ListLLMObsExperimentsOptionalParameters filterName(String filterName) { + this.filterName = filterName; + return this; + } + + /** + * Set filterExperiment. + * + * @param filterExperiment Filter by logical experiment name. This is the name + * field set when creating an experiment via POST /experiments. Returns all + * experiment runs that share the same name, enabling cross-commit and cross-branch + * comparisons. (optional) + * @return ListLLMObsExperimentsOptionalParameters + */ + public ListLLMObsExperimentsOptionalParameters filterExperiment(String filterExperiment) { + this.filterExperiment = filterExperiment; + return this; + } + + /** + * Set filterMetadata. + * + * @param filterMetadata Filter by JSONB metadata containment. Provide a JSON object string + * where experiments whose metadata contains all specified key-value pairs are returned. For + * example: {"commit":"abc123","branch":"main"}. (optional) + * @return ListLLMObsExperimentsOptionalParameters + */ + public ListLLMObsExperimentsOptionalParameters filterMetadata(String filterMetadata) { + this.filterMetadata = filterMetadata; + return this; + } + + /** + * Set filterParentExperimentId. + * + * @param filterParentExperimentId Filter experiments by the ID of their parent (baseline) + * experiment. Returns all experiments that were run against the given baseline. Can be + * specified multiple times. (optional) + * @return ListLLMObsExperimentsOptionalParameters + */ + public ListLLMObsExperimentsOptionalParameters filterParentExperimentId( + String filterParentExperimentId) { + this.filterParentExperimentId = filterParentExperimentId; + return this; + } + + /** + * Set filterIsDeleted. + * + * @param filterIsDeleted When true, return only soft-deleted experiments. Defaults + * to false. (optional) + * @return ListLLMObsExperimentsOptionalParameters + */ + public ListLLMObsExperimentsOptionalParameters filterIsDeleted(Boolean filterIsDeleted) { + this.filterIsDeleted = filterIsDeleted; + return this; + } + + /** + * Set includeUserData. + * + * @param includeUserData When true, enrich each experiment with its author's user + * data in the author field. (optional) + * @return ListLLMObsExperimentsOptionalParameters + */ + public ListLLMObsExperimentsOptionalParameters includeUserData(Boolean includeUserData) { + this.includeUserData = includeUserData; + return this; + } + + /** + * Set includeDatasetNames. + * + * @param includeDatasetNames When true, enrich each experiment with its dataset + * name in the dataset_name field. (optional) + * @return ListLLMObsExperimentsOptionalParameters + */ + public ListLLMObsExperimentsOptionalParameters includeDatasetNames( + Boolean includeDatasetNames) { + this.includeDatasetNames = includeDatasetNames; + return this; + } + /** * Set pageCursor. * - * @param pageCursor Use the Pagination cursor to retrieve the next page of results. (optional) + * @param pageCursor Use the pagination cursor returned in meta.after to retrieve + * the next page of results. (optional) * @return ListLLMObsExperimentsOptionalParameters */ public ListLLMObsExperimentsOptionalParameters pageCursor(String pageCursor) { @@ -5883,7 +6335,8 @@ public ListLLMObsExperimentsOptionalParameters pageCursor(String pageCursor) { /** * Set pageLimit. * - * @param pageLimit Maximum number of results to return per page. (optional) + * @param pageLimit Maximum number of results to return per page. Values above 5000 are clamped + * to 5000. Defaults to 5000. (optional) * @return ListLLMObsExperimentsOptionalParameters */ public ListLLMObsExperimentsOptionalParameters pageLimit(Long pageLimit) { @@ -5981,6 +6434,13 @@ public ApiResponse listLLMObsExperimentsWithHttpInfo( String filterProjectId = parameters.filterProjectId; String filterDatasetId = parameters.filterDatasetId; String filterId = parameters.filterId; + String filterName = parameters.filterName; + String filterExperiment = parameters.filterExperiment; + String filterMetadata = parameters.filterMetadata; + String filterParentExperimentId = parameters.filterParentExperimentId; + Boolean filterIsDeleted = parameters.filterIsDeleted; + Boolean includeUserData = parameters.includeUserData; + Boolean includeDatasetNames = parameters.includeDatasetNames; String pageCursor = parameters.pageCursor; Long pageLimit = parameters.pageLimit; // create path and map variables @@ -5994,6 +6454,18 @@ public ApiResponse listLLMObsExperimentsWithHttpInfo( localVarQueryParams.addAll( apiClient.parameterToPairs("", "filter[dataset_id]", filterDatasetId)); localVarQueryParams.addAll(apiClient.parameterToPairs("", "filter[id]", filterId)); + localVarQueryParams.addAll(apiClient.parameterToPairs("", "filter[name]", filterName)); + localVarQueryParams.addAll( + apiClient.parameterToPairs("", "filter[experiment]", filterExperiment)); + localVarQueryParams.addAll(apiClient.parameterToPairs("", "filter[metadata]", filterMetadata)); + localVarQueryParams.addAll( + apiClient.parameterToPairs("", "filter[parent_experiment_id]", filterParentExperimentId)); + localVarQueryParams.addAll( + apiClient.parameterToPairs("", "filter[is_deleted]", filterIsDeleted)); + localVarQueryParams.addAll( + apiClient.parameterToPairs("", "include[user_data]", includeUserData)); + localVarQueryParams.addAll( + apiClient.parameterToPairs("", "include[dataset_names]", includeDatasetNames)); localVarQueryParams.addAll(apiClient.parameterToPairs("", "page[cursor]", pageCursor)); localVarQueryParams.addAll(apiClient.parameterToPairs("", "page[limit]", pageLimit)); @@ -6041,6 +6513,13 @@ public ApiResponse listLLMObsExperimentsWithHttpInfo( String filterProjectId = parameters.filterProjectId; String filterDatasetId = parameters.filterDatasetId; String filterId = parameters.filterId; + String filterName = parameters.filterName; + String filterExperiment = parameters.filterExperiment; + String filterMetadata = parameters.filterMetadata; + String filterParentExperimentId = parameters.filterParentExperimentId; + Boolean filterIsDeleted = parameters.filterIsDeleted; + Boolean includeUserData = parameters.includeUserData; + Boolean includeDatasetNames = parameters.includeDatasetNames; String pageCursor = parameters.pageCursor; Long pageLimit = parameters.pageLimit; // create path and map variables @@ -6054,6 +6533,18 @@ public ApiResponse listLLMObsExperimentsWithHttpInfo( localVarQueryParams.addAll( apiClient.parameterToPairs("", "filter[dataset_id]", filterDatasetId)); localVarQueryParams.addAll(apiClient.parameterToPairs("", "filter[id]", filterId)); + localVarQueryParams.addAll(apiClient.parameterToPairs("", "filter[name]", filterName)); + localVarQueryParams.addAll( + apiClient.parameterToPairs("", "filter[experiment]", filterExperiment)); + localVarQueryParams.addAll(apiClient.parameterToPairs("", "filter[metadata]", filterMetadata)); + localVarQueryParams.addAll( + apiClient.parameterToPairs("", "filter[parent_experiment_id]", filterParentExperimentId)); + localVarQueryParams.addAll( + apiClient.parameterToPairs("", "filter[is_deleted]", filterIsDeleted)); + localVarQueryParams.addAll( + apiClient.parameterToPairs("", "include[user_data]", includeUserData)); + localVarQueryParams.addAll( + apiClient.parameterToPairs("", "include[dataset_names]", includeDatasetNames)); localVarQueryParams.addAll(apiClient.parameterToPairs("", "page[cursor]", pageCursor)); localVarQueryParams.addAll(apiClient.parameterToPairs("", "page[limit]", pageLimit)); diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentDataAttributesRequest.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentDataAttributesRequest.java index 67b70d3c9dc..e0bfc648368 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentDataAttributesRequest.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentDataAttributesRequest.java @@ -26,7 +26,9 @@ LLMObsExperimentDataAttributesRequest.JSON_PROPERTY_ENSURE_UNIQUE, LLMObsExperimentDataAttributesRequest.JSON_PROPERTY_METADATA, LLMObsExperimentDataAttributesRequest.JSON_PROPERTY_NAME, - LLMObsExperimentDataAttributesRequest.JSON_PROPERTY_PROJECT_ID + LLMObsExperimentDataAttributesRequest.JSON_PROPERTY_PARENT_EXPERIMENT_ID, + LLMObsExperimentDataAttributesRequest.JSON_PROPERTY_PROJECT_ID, + LLMObsExperimentDataAttributesRequest.JSON_PROPERTY_RUN_COUNT }) @jakarta.annotation.Generated( value = "https://github.com/DataDog/datadog-api-client-java/blob/master/.generator") @@ -53,17 +55,21 @@ public class LLMObsExperimentDataAttributesRequest { public static final String JSON_PROPERTY_NAME = "name"; private String name; + public static final String JSON_PROPERTY_PARENT_EXPERIMENT_ID = "parent_experiment_id"; + private String parentExperimentId; + public static final String JSON_PROPERTY_PROJECT_ID = "project_id"; private String projectId; + public static final String JSON_PROPERTY_RUN_COUNT = "run_count"; + private Integer runCount; + public LLMObsExperimentDataAttributesRequest() {} @JsonCreator public LLMObsExperimentDataAttributesRequest( - @JsonProperty(required = true, value = JSON_PROPERTY_DATASET_ID) String datasetId, @JsonProperty(required = true, value = JSON_PROPERTY_NAME) String name, @JsonProperty(required = true, value = JSON_PROPERTY_PROJECT_ID) String projectId) { - this.datasetId = datasetId; this.name = name; this.projectId = projectId; } @@ -107,8 +113,9 @@ public LLMObsExperimentDataAttributesRequest datasetId(String datasetId) { * * @return datasetId */ + @jakarta.annotation.Nullable @JsonProperty(JSON_PROPERTY_DATASET_ID) - @JsonInclude(value = JsonInclude.Include.ALWAYS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) public String getDatasetId() { return datasetId; } @@ -229,6 +236,27 @@ public void setName(String name) { this.name = name; } + public LLMObsExperimentDataAttributesRequest parentExperimentId(String parentExperimentId) { + this.parentExperimentId = parentExperimentId; + return this; + } + + /** + * Identifier of the parent (baseline) experiment this experiment is run against. + * + * @return parentExperimentId + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_PARENT_EXPERIMENT_ID) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public String getParentExperimentId() { + return parentExperimentId; + } + + public void setParentExperimentId(String parentExperimentId) { + this.parentExperimentId = parentExperimentId; + } + public LLMObsExperimentDataAttributesRequest projectId(String projectId) { this.projectId = projectId; return this; @@ -249,6 +277,27 @@ public void setProjectId(String projectId) { this.projectId = projectId; } + public LLMObsExperimentDataAttributesRequest runCount(Integer runCount) { + this.runCount = runCount; + return this; + } + + /** + * Expected number of runs for this experiment. maximum: 2147483647 + * + * @return runCount + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_RUN_COUNT) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public Integer getRunCount() { + return runCount; + } + + public void setRunCount(Integer runCount) { + this.runCount = runCount; + } + /** * A container for additional, undeclared properties. This is a holder for any undeclared * properties as specified with the 'additionalProperties' keyword in the OAS document. @@ -313,7 +362,10 @@ public boolean equals(Object o) { && Objects.equals(this.ensureUnique, llmObsExperimentDataAttributesRequest.ensureUnique) && Objects.equals(this.metadata, llmObsExperimentDataAttributesRequest.metadata) && Objects.equals(this.name, llmObsExperimentDataAttributesRequest.name) + && Objects.equals( + this.parentExperimentId, llmObsExperimentDataAttributesRequest.parentExperimentId) && Objects.equals(this.projectId, llmObsExperimentDataAttributesRequest.projectId) + && Objects.equals(this.runCount, llmObsExperimentDataAttributesRequest.runCount) && Objects.equals( this.additionalProperties, llmObsExperimentDataAttributesRequest.additionalProperties); } @@ -328,7 +380,9 @@ public int hashCode() { ensureUnique, metadata, name, + parentExperimentId, projectId, + runCount, additionalProperties); } @@ -343,7 +397,9 @@ public String toString() { sb.append(" ensureUnique: ").append(toIndentedString(ensureUnique)).append("\n"); sb.append(" metadata: ").append(toIndentedString(metadata)).append("\n"); sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" parentExperimentId: ").append(toIndentedString(parentExperimentId)).append("\n"); sb.append(" projectId: ").append(toIndentedString(projectId)).append("\n"); + sb.append(" runCount: ").append(toIndentedString(runCount)).append("\n"); sb.append(" additionalProperties: ") .append(toIndentedString(additionalProperties)) .append("\n"); diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentDataAttributesResponse.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentDataAttributesResponse.java index f9fc15a2e44..a028ac90e73 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentDataAttributesResponse.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentDataAttributesResponse.java @@ -17,22 +17,40 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; +import org.openapitools.jackson.nullable.JsonNullable; /** Attributes of an LLM Observability experiment. */ @JsonPropertyOrder({ + LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_AGGREGATE_DATA, + LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_AUTHOR, LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_CONFIG, LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_CREATED_AT, LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_DATASET_ID, + LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_DATASET_NAME, + LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_DATASET_VERSION, + LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_DELETED_AT, LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_DESCRIPTION, + LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_ERROR, + LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_EXPERIMENT, LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_METADATA, LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_NAME, + LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_PARENT_EXPERIMENT_ID, LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_PROJECT_ID, + LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_RUN_COUNT, + LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_STATUS, LLMObsExperimentDataAttributesResponse.JSON_PROPERTY_UPDATED_AT }) @jakarta.annotation.Generated( value = "https://github.com/DataDog/datadog-api-client-java/blob/master/.generator") public class LLMObsExperimentDataAttributesResponse { @JsonIgnore public boolean unparsed = false; + public static final String JSON_PROPERTY_AGGREGATE_DATA = "aggregate_data"; + private JsonNullable> aggregateData = + JsonNullable.>undefined(); + + public static final String JSON_PROPERTY_AUTHOR = "author"; + private LLMObsExperimentUser author; + public static final String JSON_PROPERTY_CONFIG = "config"; private Map config = new HashMap(); @@ -42,18 +60,42 @@ public class LLMObsExperimentDataAttributesResponse { public static final String JSON_PROPERTY_DATASET_ID = "dataset_id"; private String datasetId; + public static final String JSON_PROPERTY_DATASET_NAME = "dataset_name"; + private JsonNullable datasetName = JsonNullable.undefined(); + + public static final String JSON_PROPERTY_DATASET_VERSION = "dataset_version"; + private Long datasetVersion; + + public static final String JSON_PROPERTY_DELETED_AT = "deleted_at"; + private JsonNullable deletedAt = JsonNullable.undefined(); + public static final String JSON_PROPERTY_DESCRIPTION = "description"; private String description; + public static final String JSON_PROPERTY_ERROR = "error"; + private JsonNullable error = JsonNullable.undefined(); + + public static final String JSON_PROPERTY_EXPERIMENT = "experiment"; + private String experiment; + public static final String JSON_PROPERTY_METADATA = "metadata"; private Map metadata = new HashMap(); public static final String JSON_PROPERTY_NAME = "name"; private String name; + public static final String JSON_PROPERTY_PARENT_EXPERIMENT_ID = "parent_experiment_id"; + private JsonNullable parentExperimentId = JsonNullable.undefined(); + public static final String JSON_PROPERTY_PROJECT_ID = "project_id"; private String projectId; + public static final String JSON_PROPERTY_RUN_COUNT = "run_count"; + private Integer runCount; + + public static final String JSON_PROPERTY_STATUS = "status"; + private LLMObsExperimentStatus status; + public static final String JSON_PROPERTY_UPDATED_AT = "updated_at"; private OffsetDateTime updatedAt; @@ -82,6 +124,74 @@ public LLMObsExperimentDataAttributesResponse( this.updatedAt = updatedAt; } + public LLMObsExperimentDataAttributesResponse aggregateData(Map aggregateData) { + this.aggregateData = JsonNullable.>of(aggregateData); + return this; + } + + public LLMObsExperimentDataAttributesResponse putAggregateDataItem( + String key, Object aggregateDataItem) { + if (this.aggregateData == null || !this.aggregateData.isPresent()) { + this.aggregateData = JsonNullable.>of(new HashMap<>()); + } + try { + this.aggregateData.get().put(key, aggregateDataItem); + } catch (java.util.NoSuchElementException e) { + // this can never happen, as we make sure above that the value is present + } + return this; + } + + /** + * Pre-computed aggregate metrics for this experiment run, including eval score distributions, + * token costs, and error rates. + * + * @return aggregateData + */ + @jakarta.annotation.Nullable + @JsonIgnore + public Map getAggregateData() { + return aggregateData.orElse(null); + } + + @JsonProperty(JSON_PROPERTY_AGGREGATE_DATA) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public JsonNullable> getAggregateData_JsonNullable() { + return aggregateData; + } + + @JsonProperty(JSON_PROPERTY_AGGREGATE_DATA) + public void setAggregateData_JsonNullable(JsonNullable> aggregateData) { + this.aggregateData = aggregateData; + } + + public void setAggregateData(Map aggregateData) { + this.aggregateData = JsonNullable.>of(aggregateData); + } + + public LLMObsExperimentDataAttributesResponse author(LLMObsExperimentUser author) { + this.author = author; + this.unparsed |= author.unparsed; + return this; + } + + /** + * User data for the author of an experiment. Only present when include[user_data] is + * true. + * + * @return author + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_AUTHOR) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public LLMObsExperimentUser getAuthor() { + return author; + } + + public void setAuthor(LLMObsExperimentUser author) { + this.author = author; + } + public LLMObsExperimentDataAttributesResponse config(Map config) { this.config = config; if (config != null) {} @@ -149,6 +259,90 @@ public void setDatasetId(String datasetId) { this.datasetId = datasetId; } + public LLMObsExperimentDataAttributesResponse datasetName(String datasetName) { + this.datasetName = JsonNullable.of(datasetName); + return this; + } + + /** + * Name of the dataset used in this experiment. Only present when include[dataset_names] + * is true. + * + * @return datasetName + */ + @jakarta.annotation.Nullable + @JsonIgnore + public String getDatasetName() { + return datasetName.orElse(null); + } + + @JsonProperty(JSON_PROPERTY_DATASET_NAME) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public JsonNullable getDatasetName_JsonNullable() { + return datasetName; + } + + @JsonProperty(JSON_PROPERTY_DATASET_NAME) + public void setDatasetName_JsonNullable(JsonNullable datasetName) { + this.datasetName = datasetName; + } + + public void setDatasetName(String datasetName) { + this.datasetName = JsonNullable.of(datasetName); + } + + public LLMObsExperimentDataAttributesResponse datasetVersion(Long datasetVersion) { + this.datasetVersion = datasetVersion; + return this; + } + + /** + * Version of the dataset used in this experiment. + * + * @return datasetVersion + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_DATASET_VERSION) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public Long getDatasetVersion() { + return datasetVersion; + } + + public void setDatasetVersion(Long datasetVersion) { + this.datasetVersion = datasetVersion; + } + + public LLMObsExperimentDataAttributesResponse deletedAt(OffsetDateTime deletedAt) { + this.deletedAt = JsonNullable.of(deletedAt); + return this; + } + + /** + * Timestamp when the experiment was soft-deleted, if applicable. + * + * @return deletedAt + */ + @jakarta.annotation.Nullable + @JsonIgnore + public OffsetDateTime getDeletedAt() { + return deletedAt.orElse(null); + } + + @JsonProperty(JSON_PROPERTY_DELETED_AT) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public JsonNullable getDeletedAt_JsonNullable() { + return deletedAt; + } + + @JsonProperty(JSON_PROPERTY_DELETED_AT) + public void setDeletedAt_JsonNullable(JsonNullable deletedAt) { + this.deletedAt = deletedAt; + } + + public void setDeletedAt(OffsetDateTime deletedAt) { + this.deletedAt = JsonNullable.of(deletedAt); + } + public LLMObsExperimentDataAttributesResponse description(String description) { this.description = description; if (description != null) {} @@ -171,6 +365,58 @@ public void setDescription(String description) { this.description = description; } + public LLMObsExperimentDataAttributesResponse error(String error) { + this.error = JsonNullable.of(error); + return this; + } + + /** + * Error message describing why the experiment failed, if applicable. + * + * @return error + */ + @jakarta.annotation.Nullable + @JsonIgnore + public String getError() { + return error.orElse(null); + } + + @JsonProperty(JSON_PROPERTY_ERROR) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public JsonNullable getError_JsonNullable() { + return error; + } + + @JsonProperty(JSON_PROPERTY_ERROR) + public void setError_JsonNullable(JsonNullable error) { + this.error = error; + } + + public void setError(String error) { + this.error = JsonNullable.of(error); + } + + public LLMObsExperimentDataAttributesResponse experiment(String experiment) { + this.experiment = experiment; + return this; + } + + /** + * Logical name of the experiment, shared across all runs of the same pipeline. + * + * @return experiment + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_EXPERIMENT) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public String getExperiment() { + return experiment; + } + + public void setExperiment(String experiment) { + this.experiment = experiment; + } + public LLMObsExperimentDataAttributesResponse metadata(Map metadata) { this.metadata = metadata; if (metadata != null) {} @@ -218,6 +464,37 @@ public void setName(String name) { this.name = name; } + public LLMObsExperimentDataAttributesResponse parentExperimentId(String parentExperimentId) { + this.parentExperimentId = JsonNullable.of(parentExperimentId); + return this; + } + + /** + * Identifier of the parent (baseline) experiment this experiment was run against, if any. + * + * @return parentExperimentId + */ + @jakarta.annotation.Nullable + @JsonIgnore + public String getParentExperimentId() { + return parentExperimentId.orElse(null); + } + + @JsonProperty(JSON_PROPERTY_PARENT_EXPERIMENT_ID) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public JsonNullable getParentExperimentId_JsonNullable() { + return parentExperimentId; + } + + @JsonProperty(JSON_PROPERTY_PARENT_EXPERIMENT_ID) + public void setParentExperimentId_JsonNullable(JsonNullable parentExperimentId) { + this.parentExperimentId = parentExperimentId; + } + + public void setParentExperimentId(String parentExperimentId) { + this.parentExperimentId = JsonNullable.of(parentExperimentId); + } + public LLMObsExperimentDataAttributesResponse projectId(String projectId) { this.projectId = projectId; return this; @@ -238,6 +515,52 @@ public void setProjectId(String projectId) { this.projectId = projectId; } + public LLMObsExperimentDataAttributesResponse runCount(Integer runCount) { + this.runCount = runCount; + return this; + } + + /** + * Expected number of runs for this experiment. maximum: 2147483647 + * + * @return runCount + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_RUN_COUNT) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public Integer getRunCount() { + return runCount; + } + + public void setRunCount(Integer runCount) { + this.runCount = runCount; + } + + public LLMObsExperimentDataAttributesResponse status(LLMObsExperimentStatus status) { + this.status = status; + this.unparsed |= !status.isValid(); + return this; + } + + /** + * Execution status of an LLM Observability experiment. + * + * @return status + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_STATUS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public LLMObsExperimentStatus getStatus() { + return status; + } + + public void setStatus(LLMObsExperimentStatus status) { + if (!status.isValid()) { + this.unparsed = true; + } + this.status = status; + } + public LLMObsExperimentDataAttributesResponse updatedAt(OffsetDateTime updatedAt) { this.updatedAt = updatedAt; return this; @@ -315,13 +638,25 @@ public boolean equals(Object o) { } LLMObsExperimentDataAttributesResponse llmObsExperimentDataAttributesResponse = (LLMObsExperimentDataAttributesResponse) o; - return Objects.equals(this.config, llmObsExperimentDataAttributesResponse.config) + return Objects.equals(this.aggregateData, llmObsExperimentDataAttributesResponse.aggregateData) + && Objects.equals(this.author, llmObsExperimentDataAttributesResponse.author) + && Objects.equals(this.config, llmObsExperimentDataAttributesResponse.config) && Objects.equals(this.createdAt, llmObsExperimentDataAttributesResponse.createdAt) && Objects.equals(this.datasetId, llmObsExperimentDataAttributesResponse.datasetId) + && Objects.equals(this.datasetName, llmObsExperimentDataAttributesResponse.datasetName) + && Objects.equals( + this.datasetVersion, llmObsExperimentDataAttributesResponse.datasetVersion) + && Objects.equals(this.deletedAt, llmObsExperimentDataAttributesResponse.deletedAt) && Objects.equals(this.description, llmObsExperimentDataAttributesResponse.description) + && Objects.equals(this.error, llmObsExperimentDataAttributesResponse.error) + && Objects.equals(this.experiment, llmObsExperimentDataAttributesResponse.experiment) && Objects.equals(this.metadata, llmObsExperimentDataAttributesResponse.metadata) && Objects.equals(this.name, llmObsExperimentDataAttributesResponse.name) + && Objects.equals( + this.parentExperimentId, llmObsExperimentDataAttributesResponse.parentExperimentId) && Objects.equals(this.projectId, llmObsExperimentDataAttributesResponse.projectId) + && Objects.equals(this.runCount, llmObsExperimentDataAttributesResponse.runCount) + && Objects.equals(this.status, llmObsExperimentDataAttributesResponse.status) && Objects.equals(this.updatedAt, llmObsExperimentDataAttributesResponse.updatedAt) && Objects.equals( this.additionalProperties, llmObsExperimentDataAttributesResponse.additionalProperties); @@ -330,13 +665,23 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash( + aggregateData, + author, config, createdAt, datasetId, + datasetName, + datasetVersion, + deletedAt, description, + error, + experiment, metadata, name, + parentExperimentId, projectId, + runCount, + status, updatedAt, additionalProperties); } @@ -345,13 +690,23 @@ public int hashCode() { public String toString() { StringBuilder sb = new StringBuilder(); sb.append("class LLMObsExperimentDataAttributesResponse {\n"); + sb.append(" aggregateData: ").append(toIndentedString(aggregateData)).append("\n"); + sb.append(" author: ").append(toIndentedString(author)).append("\n"); sb.append(" config: ").append(toIndentedString(config)).append("\n"); sb.append(" createdAt: ").append(toIndentedString(createdAt)).append("\n"); sb.append(" datasetId: ").append(toIndentedString(datasetId)).append("\n"); + sb.append(" datasetName: ").append(toIndentedString(datasetName)).append("\n"); + sb.append(" datasetVersion: ").append(toIndentedString(datasetVersion)).append("\n"); + sb.append(" deletedAt: ").append(toIndentedString(deletedAt)).append("\n"); sb.append(" description: ").append(toIndentedString(description)).append("\n"); + sb.append(" error: ").append(toIndentedString(error)).append("\n"); + sb.append(" experiment: ").append(toIndentedString(experiment)).append("\n"); sb.append(" metadata: ").append(toIndentedString(metadata)).append("\n"); sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" parentExperimentId: ").append(toIndentedString(parentExperimentId)).append("\n"); sb.append(" projectId: ").append(toIndentedString(projectId)).append("\n"); + sb.append(" runCount: ").append(toIndentedString(runCount)).append("\n"); + sb.append(" status: ").append(toIndentedString(status)).append("\n"); sb.append(" updatedAt: ").append(toIndentedString(updatedAt)).append("\n"); sb.append(" additionalProperties: ") .append(toIndentedString(additionalProperties)) diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpanDataResponse.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpanDataResponse.java new file mode 100644 index 00000000000..388da9bf354 --- /dev/null +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpanDataResponse.java @@ -0,0 +1,211 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +package com.datadog.api.client.v2.model; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** JSON:API data item wrapping a single experiment span with evaluations. */ +@JsonPropertyOrder({ + LLMObsExperimentSpanDataResponse.JSON_PROPERTY_ATTRIBUTES, + LLMObsExperimentSpanDataResponse.JSON_PROPERTY_ID, + LLMObsExperimentSpanDataResponse.JSON_PROPERTY_TYPE +}) +@jakarta.annotation.Generated( + value = "https://github.com/DataDog/datadog-api-client-java/blob/master/.generator") +public class LLMObsExperimentSpanDataResponse { + @JsonIgnore public boolean unparsed = false; + public static final String JSON_PROPERTY_ATTRIBUTES = "attributes"; + private LLMObsExperimentSpanWithEvals attributes; + + public static final String JSON_PROPERTY_ID = "id"; + private String id; + + public static final String JSON_PROPERTY_TYPE = "type"; + private LLMObsExperimentSpanType type; + + public LLMObsExperimentSpanDataResponse() {} + + @JsonCreator + public LLMObsExperimentSpanDataResponse( + @JsonProperty(required = true, value = JSON_PROPERTY_ATTRIBUTES) + LLMObsExperimentSpanWithEvals attributes, + @JsonProperty(required = true, value = JSON_PROPERTY_ID) String id, + @JsonProperty(required = true, value = JSON_PROPERTY_TYPE) LLMObsExperimentSpanType type) { + this.attributes = attributes; + this.unparsed |= attributes.unparsed; + this.id = id; + this.type = type; + this.unparsed |= !type.isValid(); + } + + public LLMObsExperimentSpanDataResponse attributes(LLMObsExperimentSpanWithEvals attributes) { + this.attributes = attributes; + this.unparsed |= attributes.unparsed; + return this; + } + + /** + * An experiment span enriched with its associated evaluation metrics. + * + * @return attributes + */ + @JsonProperty(JSON_PROPERTY_ATTRIBUTES) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public LLMObsExperimentSpanWithEvals getAttributes() { + return attributes; + } + + public void setAttributes(LLMObsExperimentSpanWithEvals attributes) { + this.attributes = attributes; + } + + public LLMObsExperimentSpanDataResponse id(String id) { + this.id = id; + return this; + } + + /** + * Unique identifier of the span. + * + * @return id + */ + @JsonProperty(JSON_PROPERTY_ID) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public LLMObsExperimentSpanDataResponse type(LLMObsExperimentSpanType type) { + this.type = type; + this.unparsed |= !type.isValid(); + return this; + } + + /** + * Resource type for a span item in an experiment spans response. + * + * @return type + */ + @JsonProperty(JSON_PROPERTY_TYPE) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public LLMObsExperimentSpanType getType() { + return type; + } + + public void setType(LLMObsExperimentSpanType type) { + if (!type.isValid()) { + this.unparsed = true; + } + this.type = type; + } + + /** + * A container for additional, undeclared properties. This is a holder for any undeclared + * properties as specified with the 'additionalProperties' keyword in the OAS document. + */ + private Map additionalProperties; + + /** + * Set the additional (undeclared) property with the specified name and value. If the property + * does not already exist, create it otherwise replace it. + * + * @param key The arbitrary key to set + * @param value The associated value + * @return LLMObsExperimentSpanDataResponse + */ + @JsonAnySetter + public LLMObsExperimentSpanDataResponse putAdditionalProperty(String key, Object value) { + if (this.additionalProperties == null) { + this.additionalProperties = new HashMap(); + } + this.additionalProperties.put(key, value); + return this; + } + + /** + * Return the additional (undeclared) property. + * + * @return The additional properties + */ + @JsonAnyGetter + public Map getAdditionalProperties() { + return additionalProperties; + } + + /** + * Return the additional (undeclared) property with the specified name. + * + * @param key The arbitrary key to get + * @return The specific additional property for the given key + */ + public Object getAdditionalProperty(String key) { + if (this.additionalProperties == null) { + return null; + } + return this.additionalProperties.get(key); + } + + /** Return true if this LLMObsExperimentSpanDataResponse object is equal to o. */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LLMObsExperimentSpanDataResponse llmObsExperimentSpanDataResponse = + (LLMObsExperimentSpanDataResponse) o; + return Objects.equals(this.attributes, llmObsExperimentSpanDataResponse.attributes) + && Objects.equals(this.id, llmObsExperimentSpanDataResponse.id) + && Objects.equals(this.type, llmObsExperimentSpanDataResponse.type) + && Objects.equals( + this.additionalProperties, llmObsExperimentSpanDataResponse.additionalProperties); + } + + @Override + public int hashCode() { + return Objects.hash(attributes, id, type, additionalProperties); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class LLMObsExperimentSpanDataResponse {\n"); + sb.append(" attributes: ").append(toIndentedString(attributes)).append("\n"); + sb.append(" id: ").append(toIndentedString(id)).append("\n"); + sb.append(" type: ").append(toIndentedString(type)).append("\n"); + sb.append(" additionalProperties: ") + .append(toIndentedString(additionalProperties)) + .append("\n"); + sb.append('}'); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpanType.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpanType.java new file mode 100644 index 00000000000..99627ab6e40 --- /dev/null +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpanType.java @@ -0,0 +1,57 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +package com.datadog.api.client.v2.model; + +import com.datadog.api.client.ModelEnum; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** Resource type for a span item in an experiment spans response. */ +@JsonSerialize(using = LLMObsExperimentSpanType.LLMObsExperimentSpanTypeSerializer.class) +public class LLMObsExperimentSpanType extends ModelEnum { + + private static final Set allowedValues = + new HashSet(Arrays.asList("experiments")); + + public static final LLMObsExperimentSpanType EXPERIMENTS_SPAN = + new LLMObsExperimentSpanType("experiments"); + + LLMObsExperimentSpanType(String value) { + super(value, allowedValues); + } + + public static class LLMObsExperimentSpanTypeSerializer + extends StdSerializer { + public LLMObsExperimentSpanTypeSerializer(Class t) { + super(t); + } + + public LLMObsExperimentSpanTypeSerializer() { + this(null); + } + + @Override + public void serialize( + LLMObsExperimentSpanType value, JsonGenerator jgen, SerializerProvider provider) + throws IOException, JsonProcessingException { + jgen.writeObject(value.value); + } + } + + @JsonCreator + public static LLMObsExperimentSpanType fromValue(String value) { + return new LLMObsExperimentSpanType(value); + } +} diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpansResponse.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpansResponse.java new file mode 100644 index 00000000000..0dfb62f8b53 --- /dev/null +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentSpansResponse.java @@ -0,0 +1,160 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +package com.datadog.api.client.v2.model; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Response for listing experiment spans (v1). Returns only spans with their evaluation metrics. No + * summary metrics or pagination are included. Deprecated in favor of + * ListLLMObsExperimentEventsV3. + */ +@JsonPropertyOrder({LLMObsExperimentSpansResponse.JSON_PROPERTY_DATA}) +@jakarta.annotation.Generated( + value = "https://github.com/DataDog/datadog-api-client-java/blob/master/.generator") +public class LLMObsExperimentSpansResponse { + @JsonIgnore public boolean unparsed = false; + public static final String JSON_PROPERTY_DATA = "data"; + private List data = new ArrayList<>(); + + public LLMObsExperimentSpansResponse() {} + + @JsonCreator + public LLMObsExperimentSpansResponse( + @JsonProperty(required = true, value = JSON_PROPERTY_DATA) + List data) { + this.data = data; + } + + public LLMObsExperimentSpansResponse data(List data) { + this.data = data; + for (LLMObsExperimentSpanDataResponse item : data) { + this.unparsed |= item.unparsed; + } + return this; + } + + public LLMObsExperimentSpansResponse addDataItem(LLMObsExperimentSpanDataResponse dataItem) { + this.data.add(dataItem); + this.unparsed |= dataItem.unparsed; + return this; + } + + /** + * List of experiment spans with their evaluation metrics. + * + * @return data + */ + @JsonProperty(JSON_PROPERTY_DATA) + @JsonInclude(value = JsonInclude.Include.ALWAYS) + public List getData() { + return data; + } + + public void setData(List data) { + this.data = data; + } + + /** + * A container for additional, undeclared properties. This is a holder for any undeclared + * properties as specified with the 'additionalProperties' keyword in the OAS document. + */ + private Map additionalProperties; + + /** + * Set the additional (undeclared) property with the specified name and value. If the property + * does not already exist, create it otherwise replace it. + * + * @param key The arbitrary key to set + * @param value The associated value + * @return LLMObsExperimentSpansResponse + */ + @JsonAnySetter + public LLMObsExperimentSpansResponse putAdditionalProperty(String key, Object value) { + if (this.additionalProperties == null) { + this.additionalProperties = new HashMap(); + } + this.additionalProperties.put(key, value); + return this; + } + + /** + * Return the additional (undeclared) property. + * + * @return The additional properties + */ + @JsonAnyGetter + public Map getAdditionalProperties() { + return additionalProperties; + } + + /** + * Return the additional (undeclared) property with the specified name. + * + * @param key The arbitrary key to get + * @return The specific additional property for the given key + */ + public Object getAdditionalProperty(String key) { + if (this.additionalProperties == null) { + return null; + } + return this.additionalProperties.get(key); + } + + /** Return true if this LLMObsExperimentSpansResponse object is equal to o. */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LLMObsExperimentSpansResponse llmObsExperimentSpansResponse = (LLMObsExperimentSpansResponse) o; + return Objects.equals(this.data, llmObsExperimentSpansResponse.data) + && Objects.equals( + this.additionalProperties, llmObsExperimentSpansResponse.additionalProperties); + } + + @Override + public int hashCode() { + return Objects.hash(data, additionalProperties); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class LLMObsExperimentSpansResponse {\n"); + sb.append(" data: ").append(toIndentedString(data)).append("\n"); + sb.append(" additionalProperties: ") + .append(toIndentedString(additionalProperties)) + .append("\n"); + sb.append('}'); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentStatus.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentStatus.java new file mode 100644 index 00000000000..69ceaec9a73 --- /dev/null +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentStatus.java @@ -0,0 +1,60 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +package com.datadog.api.client.v2.model; + +import com.datadog.api.client.ModelEnum; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** Execution status of an LLM Observability experiment. */ +@JsonSerialize(using = LLMObsExperimentStatus.LLMObsExperimentStatusSerializer.class) +public class LLMObsExperimentStatus extends ModelEnum { + + private static final Set allowedValues = + new HashSet(Arrays.asList("running", "completed", "failed", "interrupted")); + + public static final LLMObsExperimentStatus RUNNING = new LLMObsExperimentStatus("running"); + public static final LLMObsExperimentStatus COMPLETED = new LLMObsExperimentStatus("completed"); + public static final LLMObsExperimentStatus FAILED = new LLMObsExperimentStatus("failed"); + public static final LLMObsExperimentStatus INTERRUPTED = + new LLMObsExperimentStatus("interrupted"); + + LLMObsExperimentStatus(String value) { + super(value, allowedValues); + } + + public static class LLMObsExperimentStatusSerializer + extends StdSerializer { + public LLMObsExperimentStatusSerializer(Class t) { + super(t); + } + + public LLMObsExperimentStatusSerializer() { + this(null); + } + + @Override + public void serialize( + LLMObsExperimentStatus value, JsonGenerator jgen, SerializerProvider provider) + throws IOException, JsonProcessingException { + jgen.writeObject(value.value); + } + } + + @JsonCreator + public static LLMObsExperimentStatus fromValue(String value) { + return new LLMObsExperimentStatus(value); + } +} diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentUpdateDataAttributesRequest.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentUpdateDataAttributesRequest.java index 7d87f15d3cb..6263adab1c9 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentUpdateDataAttributesRequest.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentUpdateDataAttributesRequest.java @@ -18,19 +18,56 @@ /** Attributes for updating an LLM Observability experiment. */ @JsonPropertyOrder({ + LLMObsExperimentUpdateDataAttributesRequest.JSON_PROPERTY_DATASET_ID, LLMObsExperimentUpdateDataAttributesRequest.JSON_PROPERTY_DESCRIPTION, - LLMObsExperimentUpdateDataAttributesRequest.JSON_PROPERTY_NAME + LLMObsExperimentUpdateDataAttributesRequest.JSON_PROPERTY_ERROR, + LLMObsExperimentUpdateDataAttributesRequest.JSON_PROPERTY_METADATA, + LLMObsExperimentUpdateDataAttributesRequest.JSON_PROPERTY_NAME, + LLMObsExperimentUpdateDataAttributesRequest.JSON_PROPERTY_STATUS }) @jakarta.annotation.Generated( value = "https://github.com/DataDog/datadog-api-client-java/blob/master/.generator") public class LLMObsExperimentUpdateDataAttributesRequest { @JsonIgnore public boolean unparsed = false; + public static final String JSON_PROPERTY_DATASET_ID = "dataset_id"; + private String datasetId; + public static final String JSON_PROPERTY_DESCRIPTION = "description"; private String description; + public static final String JSON_PROPERTY_ERROR = "error"; + private String error; + + public static final String JSON_PROPERTY_METADATA = "metadata"; + private Map metadata = null; + public static final String JSON_PROPERTY_NAME = "name"; private String name; + public static final String JSON_PROPERTY_STATUS = "status"; + private LLMObsExperimentStatus status; + + public LLMObsExperimentUpdateDataAttributesRequest datasetId(String datasetId) { + this.datasetId = datasetId; + return this; + } + + /** + * Updated identifier of the dataset used in this experiment. + * + * @return datasetId + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_DATASET_ID) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public String getDatasetId() { + return datasetId; + } + + public void setDatasetId(String datasetId) { + this.datasetId = datasetId; + } + public LLMObsExperimentUpdateDataAttributesRequest description(String description) { this.description = description; return this; @@ -52,6 +89,57 @@ public void setDescription(String description) { this.description = description; } + public LLMObsExperimentUpdateDataAttributesRequest error(String error) { + this.error = error; + return this; + } + + /** + * Error message describing why the experiment failed, if applicable. + * + * @return error + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_ERROR) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public String getError() { + return error; + } + + public void setError(String error) { + this.error = error; + } + + public LLMObsExperimentUpdateDataAttributesRequest metadata(Map metadata) { + this.metadata = metadata; + return this; + } + + public LLMObsExperimentUpdateDataAttributesRequest putMetadataItem( + String key, Object metadataItem) { + if (this.metadata == null) { + this.metadata = new HashMap<>(); + } + this.metadata.put(key, metadataItem); + return this; + } + + /** + * Updated arbitrary metadata associated with the experiment. + * + * @return metadata + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_METADATA) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public Map getMetadata() { + return metadata; + } + + public void setMetadata(Map metadata) { + this.metadata = metadata; + } + public LLMObsExperimentUpdateDataAttributesRequest name(String name) { this.name = name; return this; @@ -73,6 +161,31 @@ public void setName(String name) { this.name = name; } + public LLMObsExperimentUpdateDataAttributesRequest status(LLMObsExperimentStatus status) { + this.status = status; + this.unparsed |= !status.isValid(); + return this; + } + + /** + * Execution status of an LLM Observability experiment. + * + * @return status + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_STATUS) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public LLMObsExperimentStatus getStatus() { + return status; + } + + public void setStatus(LLMObsExperimentStatus status) { + if (!status.isValid()) { + this.unparsed = true; + } + this.status = status; + } + /** * A container for additional, undeclared properties. This is a holder for any undeclared * properties as specified with the 'additionalProperties' keyword in the OAS document. @@ -131,8 +244,12 @@ public boolean equals(Object o) { } LLMObsExperimentUpdateDataAttributesRequest llmObsExperimentUpdateDataAttributesRequest = (LLMObsExperimentUpdateDataAttributesRequest) o; - return Objects.equals(this.description, llmObsExperimentUpdateDataAttributesRequest.description) + return Objects.equals(this.datasetId, llmObsExperimentUpdateDataAttributesRequest.datasetId) + && Objects.equals(this.description, llmObsExperimentUpdateDataAttributesRequest.description) + && Objects.equals(this.error, llmObsExperimentUpdateDataAttributesRequest.error) + && Objects.equals(this.metadata, llmObsExperimentUpdateDataAttributesRequest.metadata) && Objects.equals(this.name, llmObsExperimentUpdateDataAttributesRequest.name) + && Objects.equals(this.status, llmObsExperimentUpdateDataAttributesRequest.status) && Objects.equals( this.additionalProperties, llmObsExperimentUpdateDataAttributesRequest.additionalProperties); @@ -140,15 +257,20 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(description, name, additionalProperties); + return Objects.hash( + datasetId, description, error, metadata, name, status, additionalProperties); } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("class LLMObsExperimentUpdateDataAttributesRequest {\n"); + sb.append(" datasetId: ").append(toIndentedString(datasetId)).append("\n"); sb.append(" description: ").append(toIndentedString(description)).append("\n"); + sb.append(" error: ").append(toIndentedString(error)).append("\n"); + sb.append(" metadata: ").append(toIndentedString(metadata)).append("\n"); sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" status: ").append(toIndentedString(status)).append("\n"); sb.append(" additionalProperties: ") .append(toIndentedString(additionalProperties)) .append("\n"); diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentUser.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentUser.java new file mode 100644 index 00000000000..163d1ce1f9c --- /dev/null +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsExperimentUser.java @@ -0,0 +1,248 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2019-Present Datadog, Inc. + */ + +package com.datadog.api.client.v2.model; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * User data for the author of an experiment. Only present when include[user_data] is + * true. + */ +@JsonPropertyOrder({ + LLMObsExperimentUser.JSON_PROPERTY_EMAIL, + LLMObsExperimentUser.JSON_PROPERTY_HANDLE, + LLMObsExperimentUser.JSON_PROPERTY_ICON, + LLMObsExperimentUser.JSON_PROPERTY_ID, + LLMObsExperimentUser.JSON_PROPERTY_NAME +}) +@jakarta.annotation.Generated( + value = "https://github.com/DataDog/datadog-api-client-java/blob/master/.generator") +public class LLMObsExperimentUser { + @JsonIgnore public boolean unparsed = false; + public static final String JSON_PROPERTY_EMAIL = "email"; + private String email; + + public static final String JSON_PROPERTY_HANDLE = "handle"; + private String handle; + + public static final String JSON_PROPERTY_ICON = "icon"; + private String icon; + + public static final String JSON_PROPERTY_ID = "id"; + private String id; + + public static final String JSON_PROPERTY_NAME = "name"; + private String name; + + public LLMObsExperimentUser email(String email) { + this.email = email; + return this; + } + + /** + * Email address of the user. + * + * @return email + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_EMAIL) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email; + } + + public LLMObsExperimentUser handle(String handle) { + this.handle = handle; + return this; + } + + /** + * Username or handle associated with the user's Datadog account. + * + * @return handle + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_HANDLE) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public String getHandle() { + return handle; + } + + public void setHandle(String handle) { + this.handle = handle; + } + + public LLMObsExperimentUser icon(String icon) { + this.icon = icon; + return this; + } + + /** + * URL of the user's icon. + * + * @return icon + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_ICON) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public String getIcon() { + return icon; + } + + public void setIcon(String icon) { + this.icon = icon; + } + + public LLMObsExperimentUser id(String id) { + this.id = id; + return this; + } + + /** + * Unique identifier of the user. + * + * @return id + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_ID) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public LLMObsExperimentUser name(String name) { + this.name = name; + return this; + } + + /** + * Display name of the user. + * + * @return name + */ + @jakarta.annotation.Nullable + @JsonProperty(JSON_PROPERTY_NAME) + @JsonInclude(value = JsonInclude.Include.USE_DEFAULTS) + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + /** + * A container for additional, undeclared properties. This is a holder for any undeclared + * properties as specified with the 'additionalProperties' keyword in the OAS document. + */ + private Map additionalProperties; + + /** + * Set the additional (undeclared) property with the specified name and value. If the property + * does not already exist, create it otherwise replace it. + * + * @param key The arbitrary key to set + * @param value The associated value + * @return LLMObsExperimentUser + */ + @JsonAnySetter + public LLMObsExperimentUser putAdditionalProperty(String key, Object value) { + if (this.additionalProperties == null) { + this.additionalProperties = new HashMap(); + } + this.additionalProperties.put(key, value); + return this; + } + + /** + * Return the additional (undeclared) property. + * + * @return The additional properties + */ + @JsonAnyGetter + public Map getAdditionalProperties() { + return additionalProperties; + } + + /** + * Return the additional (undeclared) property with the specified name. + * + * @param key The arbitrary key to get + * @return The specific additional property for the given key + */ + public Object getAdditionalProperty(String key) { + if (this.additionalProperties == null) { + return null; + } + return this.additionalProperties.get(key); + } + + /** Return true if this LLMObsExperimentUser object is equal to o. */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LLMObsExperimentUser llmObsExperimentUser = (LLMObsExperimentUser) o; + return Objects.equals(this.email, llmObsExperimentUser.email) + && Objects.equals(this.handle, llmObsExperimentUser.handle) + && Objects.equals(this.icon, llmObsExperimentUser.icon) + && Objects.equals(this.id, llmObsExperimentUser.id) + && Objects.equals(this.name, llmObsExperimentUser.name) + && Objects.equals(this.additionalProperties, llmObsExperimentUser.additionalProperties); + } + + @Override + public int hashCode() { + return Objects.hash(email, handle, icon, id, name, additionalProperties); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("class LLMObsExperimentUser {\n"); + sb.append(" email: ").append(toIndentedString(email)).append("\n"); + sb.append(" handle: ").append(toIndentedString(handle)).append("\n"); + sb.append(" icon: ").append(toIndentedString(icon)).append("\n"); + sb.append(" id: ").append(toIndentedString(id)).append("\n"); + sb.append(" name: ").append(toIndentedString(name)).append("\n"); + sb.append(" additionalProperties: ") + .append(toIndentedString(additionalProperties)) + .append("\n"); + sb.append('}'); + return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces (except the first line). + */ + private String toIndentedString(Object o) { + if (o == null) { + return "null"; + } + return o.toString().replace("\n", "\n "); + } +} diff --git a/src/test/resources/com/datadog/api/client/v2/api/llm_observability.feature b/src/test/resources/com/datadog/api/client/v2/api/llm_observability.feature index ea384fa8ed4..36976da92eb 100644 --- a/src/test/resources/com/datadog/api/client/v2/api/llm_observability.feature +++ b/src/test/resources/com/datadog/api/client/v2/api/llm_observability.feature @@ -235,7 +235,7 @@ Feature: LLM Observability Scenario: Create an LLM Observability experiment returns "Bad Request" response Given operation "CreateLLMObsExperiment" enabled And new "CreateLLMObsExperiment" request - And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "parent_experiment_id": "3fd6b5e0-8910-4b1c-a7d0-5b84de329012", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} When the request is sent Then the response status is 400 Bad Request @@ -243,7 +243,7 @@ Feature: LLM Observability Scenario: Create an LLM Observability experiment returns "Created" response Given operation "CreateLLMObsExperiment" enabled And new "CreateLLMObsExperiment" request - And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "parent_experiment_id": "3fd6b5e0-8910-4b1c-a7d0-5b84de329012", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} When the request is sent Then the response status is 201 Created @@ -251,7 +251,7 @@ Feature: LLM Observability Scenario: Create an LLM Observability experiment returns "OK" response Given operation "CreateLLMObsExperiment" enabled And new "CreateLLMObsExperiment" request - And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "parent_experiment_id": "3fd6b5e0-8910-4b1c-a7d0-5b84de329012", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} When the request is sent Then the response status is 200 OK @@ -713,6 +713,54 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment events (v2) returns "Bad Request" response + Given operation "ListLLMObsExperimentEventsV2" enabled + And new "ListLLMObsExperimentEventsV2" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment events (v2) returns "Not Found" response + Given operation "ListLLMObsExperimentEventsV2" enabled + And new "ListLLMObsExperimentEventsV2" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment events (v2) returns "OK" response + Given operation "ListLLMObsExperimentEventsV2" enabled + And new "ListLLMObsExperimentEventsV2" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment spans (v1) returns "Bad Request" response + Given operation "ListLLMObsExperimentEventsV1" enabled + And new "ListLLMObsExperimentEventsV1" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment spans (v1) returns "Not Found" response + Given operation "ListLLMObsExperimentEventsV1" enabled + And new "ListLLMObsExperimentEventsV1" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: List LLM Observability experiment spans (v1) returns "OK" response + Given operation "ListLLMObsExperimentEventsV1" enabled + And new "ListLLMObsExperimentEventsV1" request + And request contains "experiment_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability experiments returns "Bad Request" response Given operation "ListLLMObsExperiments" enabled @@ -1092,7 +1140,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsExperiment" enabled And new "UpdateLLMObsExperiment" request And request contains "experiment_id" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "status": "completed"}, "type": "experiments"}} When the request is sent Then the response status is 400 Bad Request @@ -1101,7 +1149,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsExperiment" enabled And new "UpdateLLMObsExperiment" request And request contains "experiment_id" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "status": "completed"}, "type": "experiments"}} When the request is sent Then the response status is 404 Not Found @@ -1110,7 +1158,7 @@ Feature: LLM Observability Given operation "UpdateLLMObsExperiment" enabled And new "UpdateLLMObsExperiment" request And request contains "experiment_id" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {}, "type": "experiments"}} + And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "status": "completed"}, "type": "experiments"}} When the request is sent Then the response status is 200 OK diff --git a/src/test/resources/com/datadog/api/client/v2/api/undo.json b/src/test/resources/com/datadog/api/client/v2/api/undo.json index c45aea0fd2c..689ca345cd8 100644 --- a/src/test/resources/com/datadog/api/client/v2/api/undo.json +++ b/src/test/resources/com/datadog/api/client/v2/api/undo.json @@ -4090,6 +4090,12 @@ "type": "idempotent" } }, + "ListLLMObsExperimentEventsV1": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, "CreateLLMObsExperimentEvents": { "tag": "LLM Observability", "undo": { @@ -4270,6 +4276,12 @@ "type": "safe" } }, + "ListLLMObsExperimentEventsV2": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, "UploadLLMObsDatasetRecordsFile": { "tag": "LLM Observability", "undo": {