diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index c1ed29256660..b74cd858ff47 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -44653,6 +44653,142 @@ components: type: string x-enum-varnames: - DELETION_REQUEST + LLMObsDatasetBatchUpdateDataAttributesRequest: + description: Attributes for batch-updating records in an LLM Observability dataset. + properties: + create_new_version: + description: Whether to create a new dataset version when applying the batch update. Defaults to `true`. + example: true + type: boolean + delete_records: + $ref: "#/components/schemas/LLMObsDatasetBatchUpdateDeleteRecords" + insert_records: + $ref: "#/components/schemas/LLMObsDatasetBatchUpdateInsertRecords" + tags: + $ref: "#/components/schemas/LLMObsDatasetRecordTagsList" + update_records: + $ref: "#/components/schemas/LLMObsDatasetBatchUpdateUpdateRecords" + type: object + LLMObsDatasetBatchUpdateDataRequest: + description: Data object for batch-updating records in an LLM Observability dataset. + properties: + attributes: + $ref: "#/components/schemas/LLMObsDatasetBatchUpdateDataAttributesRequest" + id: + description: Unique identifier of the dataset. + example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" + type: string + type: + $ref: "#/components/schemas/LLMObsDatasetType" + required: + - id + - type + - attributes + type: object + LLMObsDatasetBatchUpdateDeleteRecords: + description: Record IDs to delete. + items: + description: A record ID to delete. + type: string + type: array + LLMObsDatasetBatchUpdateInsertRecord: + description: A record to insert as part of a batch update on an LLM Observability dataset. + properties: + expected_output: + $ref: "#/components/schemas/AnyValue" + id: + description: Optional user-provided identifier for the record. If omitted, the server generates an identifier. + example: "rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c" + type: string + input: + $ref: "#/components/schemas/AnyValue" + metadata: + additionalProperties: {} + description: Arbitrary metadata associated with the record. + type: object + tag_operations: + $ref: "#/components/schemas/LLMObsDatasetRecordTagOperations" + tags: + $ref: "#/components/schemas/LLMObsDatasetRecordTagsList" + required: + - input + type: object + LLMObsDatasetBatchUpdateInsertRecords: + description: Records to insert. + items: + $ref: "#/components/schemas/LLMObsDatasetBatchUpdateInsertRecord" + type: array + LLMObsDatasetBatchUpdateRequest: + description: Request to batch-insert, update, and delete records in an LLM Observability dataset. + properties: + data: + $ref: "#/components/schemas/LLMObsDatasetBatchUpdateDataRequest" + required: + - data + type: object + LLMObsDatasetBatchUpdateUpdateRecord: + description: A record update payload as part of a batch update on an LLM Observability dataset. + properties: + expected_output: + $ref: "#/components/schemas/AnyValue" + id: + description: Unique identifier of the record to update. + example: "rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c" + type: string + input: + $ref: "#/components/schemas/AnyValue" + metadata: + additionalProperties: {} + description: Updated metadata associated with the record. + type: object + tag_operations: + $ref: "#/components/schemas/LLMObsDatasetRecordTagOperations" + required: + - id + type: object + LLMObsDatasetBatchUpdateUpdateRecords: + description: Records to update by ID. + items: + $ref: "#/components/schemas/LLMObsDatasetBatchUpdateUpdateRecord" + type: array + LLMObsDatasetCloneDataAttributesRequest: + description: Attributes for cloning an LLM Observability dataset. + properties: + description: + description: Description of the cloned dataset. + example: "Clone of the original dataset for experimentation." + type: string + name: + description: Name of the cloned dataset. + example: "My cloned dataset" + type: string + required: + - name + type: object + LLMObsDatasetCloneDataRequest: + description: Data object for cloning an LLM Observability dataset. + properties: + attributes: + $ref: "#/components/schemas/LLMObsDatasetCloneDataAttributesRequest" + id: + description: Identifier of the source dataset to clone. + example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" + type: string + type: + $ref: "#/components/schemas/LLMObsDatasetType" + required: + - id + - type + - attributes + type: object + LLMObsDatasetCloneRequest: + description: Request to clone an LLM Observability dataset. + properties: + data: + $ref: "#/components/schemas/LLMObsDatasetCloneDataRequest" + required: + - data + type: object LLMObsDatasetDataAttributesRequest: description: Attributes for creating an LLM Observability dataset. properties: @@ -44809,6 +44945,15 @@ components: required: - id type: object + LLMObsDatasetExportFormat: + default: csv + description: Supported export format for an LLM Observability dataset. + enum: + - csv + example: csv + type: string + x-enum-varnames: + - CSV LLMObsDatasetRecordDataResponse: description: A single LLM Observability dataset record. properties: @@ -44862,6 +45007,22 @@ components: required: - input type: object + LLMObsDatasetRecordTagOperations: + description: Explicit tag operations for updating records. Operations are applied in order, Remove then Add then Set. `set` is the final override, if specified, the result of `remove` and `add` is discarded. + properties: + add: + $ref: "#/components/schemas/LLMObsDatasetRecordTagsList" + remove: + $ref: "#/components/schemas/LLMObsDatasetRecordTagsList" + set: + $ref: "#/components/schemas/LLMObsDatasetRecordTagsList" + type: object + LLMObsDatasetRecordTagsList: + description: List of tag strings. + items: + description: A tag. + type: string + type: array LLMObsDatasetRecordUpdateItem: description: A record update payload for an LLM Observability dataset. properties: @@ -44978,6 +45139,14 @@ components: required: - data type: object + LLMObsDatasetRecordsUploadFile: + description: Multipart payload for uploading dataset records from a file. + properties: + file: + description: The records file to upload. Currently only CSV is supported. The file must include an `input` column. Optional columns include `id`, `expected_output`, `metadata`, and `tags`. + format: binary + type: string + type: object LLMObsDatasetRequest: description: Request to create an LLM Observability dataset. properties: @@ -44994,6 +45163,42 @@ components: required: - data type: object + LLMObsDatasetRestoreVersionDataAttributesRequest: + description: Attributes for restoring an LLM Observability dataset to a previous version. + properties: + dataset_version: + description: Version number of the dataset to restore. Must be between 0 and the current version of the dataset, inclusive. + example: 1 + format: int32 + maximum: 2147483647 + type: integer + required: + - dataset_version + type: object + LLMObsDatasetRestoreVersionDataRequest: + description: Data object for restoring an LLM Observability dataset to a previous version. + properties: + attributes: + $ref: "#/components/schemas/LLMObsDatasetRestoreVersionDataAttributesRequest" + id: + description: Unique identifier of the dataset to restore. + example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" + type: string + type: + $ref: "#/components/schemas/LLMObsDatasetType" + required: + - id + - type + - attributes + type: object + LLMObsDatasetRestoreVersionRequest: + description: Request to restore an LLM Observability dataset to a previous version. + properties: + data: + $ref: "#/components/schemas/LLMObsDatasetRestoreVersionDataRequest" + required: + - data + type: object LLMObsDatasetType: description: Resource type of an LLM Observability dataset. enum: @@ -127650,10 +127855,10 @@ paths: x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). - /api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/draft_state: - get: - description: Retrieve the draft state of a dataset, including whether it is currently locked for editing and which user holds the lock. - operationId: GetLLMObsDatasetDraftState + /api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/batch_update: + post: + description: Insert, update, and delete records in a single dataset operation. By default, a new dataset version is created when the batch is applied. + operationId: BatchUpdateLLMObsDataset parameters: - description: The ID of the LLM Observability project. example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" @@ -127669,6 +127874,137 @@ paths: required: true schema: type: string + requestBody: + content: + application/json: + examples: + default: + value: + data: + attributes: + create_new_version: true + delete_records: + - rec-old-record-1 + insert_records: + - expected_output: + answer: "Paris" + input: + question: "What is the capital of France?" + tags: + - "topic:geography" + update_records: + - expected_output: + answer: "Paris, France" + id: rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c + id: 9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d + type: datasets + schema: + $ref: "#/components/schemas/LLMObsDatasetBatchUpdateRequest" + description: Batch update payload. + required: true + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + - records: + - created_at: "2024-01-15T10:30:00Z" + dataset_id: 9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d + expected_output: + answer: "Paris, France" + id: rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c + input: + question: "What is the capital of France?" + metadata: + updated_at: "2024-01-15T10:30:00Z" + schema: + $ref: "#/components/schemas/LLMObsDatasetRecordsMutationResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "413": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Payload Too Large + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Batch update LLM Observability dataset records + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/clone: + post: + description: Clone a dataset, copying its current records into a new dataset within the same project. + operationId: CloneLLMObsDataset + parameters: + - description: The ID of the LLM Observability project. + example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" + in: path + name: project_id + required: true + schema: + type: string + - description: The ID of the source LLM Observability dataset to clone. + example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" + in: path + name: dataset_id + required: true + schema: + type: string + requestBody: + content: + application/json: + examples: + default: + value: + data: + attributes: + description: "Clone of the original dataset for experimentation." + name: "My cloned dataset" + id: 9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d + type: datasets + schema: + $ref: "#/components/schemas/LLMObsDatasetCloneRequest" + description: Clone dataset payload. + required: true responses: "200": content: @@ -127678,16 +128014,16 @@ paths: value: data: attributes: - drafting_since: "2024-01-15T10:30:00Z" - user: - email: jane.doe@example.com - handle: jane.doe@example.com - id: 00000000-0000-0000-0000-000000000010 - name: Jane Doe - id: 9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d - type: draft_state_data + created_at: "2024-01-15T10:30:00Z" + current_version: 0 + description: "Clone of the original dataset for experimentation." + metadata: + name: "My cloned dataset" + updated_at: "2024-01-15T10:30:00Z" + id: 7c8d4e9a-1234-5678-9abc-def012345678 + type: datasets schema: - $ref: "#/components/schemas/LLMObsDatasetDraftStateResponse" + $ref: "#/components/schemas/LLMObsDatasetResponse" description: OK "400": content: @@ -127724,16 +128060,96 @@ paths: security: - apiKeyAuth: [] appKeyAuth: [] - summary: Get LLM Observability dataset draft state + summary: Clone an LLM Observability dataset tags: - LLM Observability x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). - /api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/draft_state/lock: - patch: - description: Acquire the draft lock on a dataset for the calling user. The lock prevents other users from concurrently editing the dataset draft. - operationId: LockLLMObsDatasetDraftState + /api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/draft_state: + get: + description: Retrieve the draft state of a dataset, including whether it is currently locked for editing and which user holds the lock. + operationId: GetLLMObsDatasetDraftState + parameters: + - description: The ID of the LLM Observability project. + example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" + in: path + name: project_id + required: true + schema: + type: string + - description: The ID of the LLM Observability dataset. + example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" + in: path + name: dataset_id + required: true + schema: + type: string + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + drafting_since: "2024-01-15T10:30:00Z" + user: + email: jane.doe@example.com + handle: jane.doe@example.com + id: 00000000-0000-0000-0000-000000000010 + name: Jane Doe + id: 9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d + type: draft_state_data + schema: + $ref: "#/components/schemas/LLMObsDatasetDraftStateResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Get LLM Observability dataset draft state + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/draft_state/lock: + patch: + description: Acquire the draft lock on a dataset for the calling user. The lock prevents other users from concurrently editing the dataset draft. + operationId: LockLLMObsDatasetDraftState parameters: - description: The ID of the LLM Observability project. example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" @@ -127873,6 +128289,88 @@ paths: x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/export: + get: + description: Download the contents of a dataset as a CSV file. The download is streamed and includes one row per dataset record. + operationId: ExportLLMObsDataset + parameters: + - description: The ID of the LLM Observability project. + example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" + in: path + name: project_id + required: true + schema: + type: string + - description: The ID of the LLM Observability dataset. + example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" + in: path + name: dataset_id + required: true + schema: + type: string + - description: Export format for the dataset contents. Only `csv` is currently supported. + in: query + name: format + schema: + $ref: "#/components/schemas/LLMObsDatasetExportFormat" + - description: Version of the dataset to export. If omitted, the current version is used. Must be between 0 and the current version of the dataset, inclusive. + in: query + name: version + schema: + format: int64 + maximum: 2147483647 + type: integer + responses: + "200": + content: + text/csv: + examples: + default: + value: "id,input,expected_output,metadata,tags\nrec-1,\"What is 2+2?\",\"4\",{},\"\"" + schema: + type: string + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Export an LLM Observability dataset + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/records: get: description: List all records in an LLM Observability dataset, sorted by creation date, newest first. @@ -128183,6 +128681,84 @@ paths: x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/restore: + post: + description: Restore a dataset to a previous version. The dataset's current version is bumped, and its records are replaced with the records from the specified prior version. + operationId: RestoreLLMObsDatasetVersion + parameters: + - description: The ID of the LLM Observability project. + example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" + in: path + name: project_id + required: true + schema: + type: string + - description: The ID of the LLM Observability dataset. + example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" + in: path + name: dataset_id + required: true + schema: + type: string + requestBody: + content: + application/json: + examples: + default: + value: + data: + attributes: + dataset_version: 1 + id: 9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d + type: datasets + schema: + $ref: "#/components/schemas/LLMObsDatasetRestoreVersionRequest" + description: Restore dataset version payload. + required: true + responses: + "200": + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Restore an LLM Observability dataset version + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/versions: get: description: List the active versions of a dataset. A version is created each time a dataset is referenced by an experiment run. @@ -128266,6 +128842,118 @@ paths: x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v2/{project_id}/datasets/{dataset_id}/records/upload: + post: + description: |- + Upload records to a dataset from a file. The request is a `multipart/form-data` upload containing a single `file` part. + Currently only CSV is supported. The CSV must include an `input` column. Optional columns are `id`, `expected_output`, `metadata`, and `tags`. + + The response is a Server-Sent Events stream (`text/event-stream`) emitting progress updates while records are processed. The stream emits the following named events: + - `progress`: incremental record counts written so far. + - `completed`: terminal event with a JSON body containing `records_created`. + - `error`: terminal event with a JSON body containing an error `message`. + operationId: UploadLLMObsDatasetRecordsFile + parameters: + - description: The ID of the LLM Observability project. + example: "a33671aa-24fd-4dcd-9b33-a8ec7dde7751" + in: path + name: project_id + required: true + schema: + type: string + - description: The ID of the LLM Observability dataset. + example: "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d" + in: path + name: dataset_id + required: true + schema: + type: string + - description: Whether to skip records whose `input` already exists in the dataset. Defaults to `false`. + in: query + name: deduplicate + schema: + default: false + type: boolean + - description: Whether to overwrite existing records that share the same user-provided `id`. Defaults to `true`. + in: query + name: overwrite + schema: + default: true + type: boolean + - description: Tags to apply to every uploaded record, in addition to any tags defined on individual rows. Can be repeated, e.g. `tags=env:prod&tags=team:ai`. + in: query + name: tags + schema: + items: + type: string + type: array + - description: Whether to enrich the response with user metadata. + in: query + name: "include[user_data]" + schema: + type: boolean + requestBody: + content: + multipart/form-data: + examples: + default: + value: + file: records.csv + schema: + $ref: "#/components/schemas/LLMObsDatasetRecordsUploadFile" + description: Multipart upload payload containing the records file. + required: true + responses: + "200": + content: + text/event-stream: + examples: + default: + value: "event: progress\ndata: {\"records_processed\": 100}\n\nevent: completed\ndata: {\"records_created\": 250}" + schema: + type: string + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Upload records to an LLM Observability dataset + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/llm-obs/v3/experiments/{experiment_id}/events: get: description: Retrieve spans and experiment-level summary metrics for a given experiment with cursor-based pagination. diff --git a/features/v2/llm_observability.feature b/features/v2/llm_observability.feature index 63b64631db28..ea384fa8ed44 100644 --- a/features/v2/llm_observability.feature +++ b/features/v2/llm_observability.feature @@ -109,6 +109,76 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: Batch update LLM Observability dataset records returns "Bad Request" response + Given operation "BatchUpdateLLMObsDataset" enabled + And new "BatchUpdateLLMObsDataset" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"create_new_version": true, "delete_records": [], "insert_records": [{"expected_output": null, "id": "rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c", "input": null, "tag_operations": {"add": [], "remove": [], "set": []}, "tags": []}], "tags": [], "update_records": [{"expected_output": null, "id": "rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c", "input": null, "tag_operations": {"add": [], "remove": [], "set": []}}]}, "id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "type": "datasets"}} + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Batch update LLM Observability dataset records returns "Not Found" response + Given operation "BatchUpdateLLMObsDataset" enabled + And new "BatchUpdateLLMObsDataset" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"create_new_version": true, "delete_records": [], "insert_records": [{"expected_output": null, "id": "rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c", "input": null, "tag_operations": {"add": [], "remove": [], "set": []}, "tags": []}], "tags": [], "update_records": [{"expected_output": null, "id": "rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c", "input": null, "tag_operations": {"add": [], "remove": [], "set": []}}]}, "id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "type": "datasets"}} + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Batch update LLM Observability dataset records returns "OK" response + Given operation "BatchUpdateLLMObsDataset" enabled + And new "BatchUpdateLLMObsDataset" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"create_new_version": true, "delete_records": [], "insert_records": [{"expected_output": null, "id": "rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c", "input": null, "tag_operations": {"add": [], "remove": [], "set": []}, "tags": []}], "tags": [], "update_records": [{"expected_output": null, "id": "rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c", "input": null, "tag_operations": {"add": [], "remove": [], "set": []}}]}, "id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "type": "datasets"}} + When the request is sent + Then the response status is 200 OK + + @generated @skip @team:DataDog/ml-observability + Scenario: Batch update LLM Observability dataset records returns "Payload Too Large" response + Given operation "BatchUpdateLLMObsDataset" enabled + And new "BatchUpdateLLMObsDataset" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"create_new_version": true, "delete_records": [], "insert_records": [{"expected_output": null, "id": "rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c", "input": null, "tag_operations": {"add": [], "remove": [], "set": []}, "tags": []}], "tags": [], "update_records": [{"expected_output": null, "id": "rec-7c3f5a1b-9e2d-4f8a-b1c6-3d7e9f0a2b4c", "input": null, "tag_operations": {"add": [], "remove": [], "set": []}}]}, "id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "type": "datasets"}} + When the request is sent + Then the response status is 413 Payload Too Large + + @generated @skip @team:DataDog/ml-observability + Scenario: Clone an LLM Observability dataset returns "Bad Request" response + Given operation "CloneLLMObsDataset" enabled + And new "CloneLLMObsDataset" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"description": "Clone of the original dataset for experimentation.", "name": "My cloned dataset"}, "id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "type": "datasets"}} + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Clone an LLM Observability dataset returns "Not Found" response + Given operation "CloneLLMObsDataset" enabled + And new "CloneLLMObsDataset" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"description": "Clone of the original dataset for experimentation.", "name": "My cloned dataset"}, "id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "type": "datasets"}} + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Clone an LLM Observability dataset returns "OK" response + Given operation "CloneLLMObsDataset" enabled + And new "CloneLLMObsDataset" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"description": "Clone of the original dataset for experimentation.", "name": "My cloned dataset"}, "id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "type": "datasets"}} + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability annotation queue returns "Bad Request" response Given operation "CreateLLMObsAnnotationQueue" enabled @@ -417,6 +487,33 @@ Feature: LLM Observability When the request is sent Then the response status is 404 Not Found + @generated @skip @team:DataDog/ml-observability + Scenario: Export an LLM Observability dataset returns "Bad Request" response + Given operation "ExportLLMObsDataset" enabled + And new "ExportLLMObsDataset" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Export an LLM Observability dataset returns "Not Found" response + Given operation "ExportLLMObsDataset" enabled + And new "ExportLLMObsDataset" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Export an LLM Observability dataset returns "OK" response + Given operation "ExportLLMObsDataset" enabled + And new "ExportLLMObsDataset" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Get LLM Observability dataset draft state returns "Bad Request" response Given operation "GetLLMObsDatasetDraftState" enabled @@ -770,6 +867,36 @@ Feature: LLM Observability When the request is sent Then the response status is 404 Not Found + @generated @skip @team:DataDog/ml-observability + Scenario: Restore an LLM Observability dataset version returns "Bad Request" response + Given operation "RestoreLLMObsDatasetVersion" enabled + And new "RestoreLLMObsDatasetVersion" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"dataset_version": 1}, "id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "type": "datasets"}} + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Restore an LLM Observability dataset version returns "Not Found" response + Given operation "RestoreLLMObsDatasetVersion" enabled + And new "RestoreLLMObsDatasetVersion" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"dataset_version": 1}, "id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "type": "datasets"}} + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Restore an LLM Observability dataset version returns "OK" response + Given operation "RestoreLLMObsDatasetVersion" enabled + And new "RestoreLLMObsDatasetVersion" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"dataset_version": 1}, "id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "type": "datasets"}} + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Run an LLM inference returns "Bad Request" response Given operation "CreateLLMObsIntegrationInference" enabled @@ -1040,3 +1167,30 @@ Feature: LLM Observability And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "abc-123", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}}, "type": "queues"}} When the request is sent Then the response status is 200 OK + + @generated @skip @team:DataDog/ml-observability + Scenario: Upload records to an LLM Observability dataset returns "Bad Request" response + Given operation "UploadLLMObsDatasetRecordsFile" enabled + And new "UploadLLMObsDatasetRecordsFile" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Upload records to an LLM Observability dataset returns "Not Found" response + Given operation "UploadLLMObsDatasetRecordsFile" enabled + And new "UploadLLMObsDatasetRecordsFile" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Upload records to an LLM Observability dataset returns "OK" response + Given operation "UploadLLMObsDatasetRecordsFile" enabled + And new "UploadLLMObsDatasetRecordsFile" request + And request contains "project_id" parameter from "REPLACE.ME" + And request contains "dataset_id" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK diff --git a/features/v2/undo.json b/features/v2/undo.json index d86d61c22888..9127662da889 100644 --- a/features/v2/undo.json +++ b/features/v2/undo.json @@ -3943,6 +3943,20 @@ "type": "idempotent" } }, + "BatchUpdateLLMObsDataset": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, + "CloneLLMObsDataset": { + "tag": "LLM Observability", + "undo": { + "operationId": "TODO", + "parameters": [], + "type": "unsafe" + } + }, "GetLLMObsDatasetDraftState": { "tag": "LLM Observability", "undo": { @@ -3961,6 +3975,12 @@ "type": "idempotent" } }, + "ExportLLMObsDataset": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, "ListLLMObsDatasetRecords": { "tag": "LLM Observability", "undo": { @@ -3998,12 +4018,24 @@ "type": "unsafe" } }, + "RestoreLLMObsDatasetVersion": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, "ListLLMObsDatasetVersions": { "tag": "LLM Observability", "undo": { "type": "safe" } }, + "UploadLLMObsDatasetRecordsFile": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, "ListLLMObsExperimentEvents": { "tag": "LLM Observability", "undo": { diff --git a/private/bdd_runner/src/support/scenarios_model_mapping.ts b/private/bdd_runner/src/support/scenarios_model_mapping.ts index b67bd8a6d79d..41931054e745 100644 --- a/private/bdd_runner/src/support/scenarios_model_mapping.ts +++ b/private/bdd_runner/src/support/scenarios_model_mapping.ts @@ -2965,6 +2965,36 @@ export const ScenariosModelMappings: { [key: string]: OperationMapping } = { }, operationResponseType: "LLMObsDatasetResponse", }, + "LLMObservabilityApi.V2.BatchUpdateLLMObsDataset": { + projectId: { + type: "string", + format: "", + }, + datasetId: { + type: "string", + format: "", + }, + body: { + type: "LLMObsDatasetBatchUpdateRequest", + format: "", + }, + operationResponseType: "LLMObsDatasetRecordsMutationResponse", + }, + "LLMObservabilityApi.V2.CloneLLMObsDataset": { + projectId: { + type: "string", + format: "", + }, + datasetId: { + type: "string", + format: "", + }, + body: { + type: "LLMObsDatasetCloneRequest", + format: "", + }, + operationResponseType: "LLMObsDatasetResponse", + }, "LLMObservabilityApi.V2.GetLLMObsDatasetDraftState": { projectId: { type: "string", @@ -2998,6 +3028,25 @@ export const ScenariosModelMappings: { [key: string]: OperationMapping } = { }, operationResponseType: "{}", }, + "LLMObservabilityApi.V2.ExportLLMObsDataset": { + projectId: { + type: "string", + format: "", + }, + datasetId: { + type: "string", + format: "", + }, + format: { + type: "LLMObsDatasetExportFormat", + format: "", + }, + version: { + type: "number", + format: "int64", + }, + operationResponseType: "string", + }, "LLMObservabilityApi.V2.ListLLMObsDatasetRecords": { projectId: { type: "string", @@ -3066,6 +3115,21 @@ export const ScenariosModelMappings: { [key: string]: OperationMapping } = { }, operationResponseType: "{}", }, + "LLMObservabilityApi.V2.RestoreLLMObsDatasetVersion": { + projectId: { + type: "string", + format: "", + }, + datasetId: { + type: "string", + format: "", + }, + body: { + type: "LLMObsDatasetRestoreVersionRequest", + format: "", + }, + operationResponseType: "{}", + }, "LLMObservabilityApi.V2.ListLLMObsDatasetVersions": { projectId: { type: "string", @@ -3077,6 +3141,37 @@ export const ScenariosModelMappings: { [key: string]: OperationMapping } = { }, operationResponseType: "LLMObsDatasetVersionsResponse", }, + "LLMObservabilityApi.V2.UploadLLMObsDatasetRecordsFile": { + projectId: { + type: "string", + format: "", + }, + datasetId: { + type: "string", + format: "", + }, + deduplicate: { + type: "boolean", + format: "", + }, + overwrite: { + type: "boolean", + format: "", + }, + tags: { + type: "Array", + format: "", + }, + includeUserData: { + type: "boolean", + format: "", + }, + file: { + type: "HttpFile", + format: "binary", + }, + operationResponseType: "string", + }, "LLMObservabilityApi.V2.ListLLMObsExperimentEvents": { experimentId: { type: "string", diff --git a/services/llm_observability/src/v2/LLMObservabilityApi.ts b/services/llm_observability/src/v2/LLMObservabilityApi.ts index 7ed446584aa5..1a19aaa445aa 100644 --- a/services/llm_observability/src/v2/LLMObservabilityApi.ts +++ b/services/llm_observability/src/v2/LLMObservabilityApi.ts @@ -19,8 +19,11 @@ import { ServerConfiguration, stringify, applySecurityAuthentication, + HttpFile, } from "@datadog/datadog-api-client"; +import FormData from "form-data"; + import { TypingInfo } from "./models/TypingInfo"; import { APIErrorResponse } from "./models/APIErrorResponse"; import { JSONAPIErrorResponse } from "./models/JSONAPIErrorResponse"; @@ -38,13 +41,17 @@ import { LLMObsCustomEvalConfigResponse } from "./models/LLMObsCustomEvalConfigR import { LLMObsCustomEvalConfigUpdateRequest } from "./models/LLMObsCustomEvalConfigUpdateRequest"; import { LLMObsDataDeletionRequest } from "./models/LLMObsDataDeletionRequest"; import { LLMObsDataDeletionResponse } from "./models/LLMObsDataDeletionResponse"; +import { LLMObsDatasetBatchUpdateRequest } from "./models/LLMObsDatasetBatchUpdateRequest"; +import { LLMObsDatasetCloneRequest } from "./models/LLMObsDatasetCloneRequest"; import { LLMObsDatasetDraftStateResponse } from "./models/LLMObsDatasetDraftStateResponse"; +import { LLMObsDatasetExportFormat } from "./models/LLMObsDatasetExportFormat"; import { LLMObsDatasetRecordsListResponse } from "./models/LLMObsDatasetRecordsListResponse"; import { LLMObsDatasetRecordsMutationResponse } from "./models/LLMObsDatasetRecordsMutationResponse"; import { LLMObsDatasetRecordsRequest } from "./models/LLMObsDatasetRecordsRequest"; import { LLMObsDatasetRecordsUpdateRequest } from "./models/LLMObsDatasetRecordsUpdateRequest"; import { LLMObsDatasetRequest } from "./models/LLMObsDatasetRequest"; import { LLMObsDatasetResponse } from "./models/LLMObsDatasetResponse"; +import { LLMObsDatasetRestoreVersionRequest } from "./models/LLMObsDatasetRestoreVersionRequest"; import { LLMObsDatasetsResponse } from "./models/LLMObsDatasetsResponse"; import { LLMObsDatasetUpdateRequest } from "./models/LLMObsDatasetUpdateRequest"; import { LLMObsDatasetVersionsResponse } from "./models/LLMObsDatasetVersionsResponse"; @@ -147,6 +154,154 @@ export class LLMObservabilityApiRequestFactory extends BaseAPIRequestFactory { return requestContext; } + public async batchUpdateLLMObsDataset( + projectId: string, + datasetId: string, + body: LLMObsDatasetBatchUpdateRequest, + _options?: Configuration, + ): Promise { + const _config = _options || this.configuration; + + if ( + !_config.unstableOperations[ + "LLMObservabilityApi.v2.batchUpdateLLMObsDataset" + ] + ) { + throw new Error( + "Unstable operation 'batchUpdateLLMObsDataset' is disabled. Enable it by setting `configuration.unstableOperations['LLMObservabilityApi.v2.batchUpdateLLMObsDataset'] = true`", + ); + } + + // verify required parameter 'projectId' is not null or undefined + if (projectId === null || projectId === undefined) { + throw new RequiredError("projectId", "batchUpdateLLMObsDataset"); + } + + // verify required parameter 'datasetId' is not null or undefined + if (datasetId === null || datasetId === undefined) { + throw new RequiredError("datasetId", "batchUpdateLLMObsDataset"); + } + + // verify required parameter 'body' is not null or undefined + if (body === null || body === undefined) { + throw new RequiredError("body", "batchUpdateLLMObsDataset"); + } + + // Path Params + const localVarPath = + "/api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/batch_update" + .replace("{project_id}", encodeURIComponent(String(projectId))) + .replace("{dataset_id}", encodeURIComponent(String(datasetId))); + + // Make Request Context + const { server, overrides } = _config.getServerAndOverrides( + "LLMObservabilityApi.v2.batchUpdateLLMObsDataset", + LLMObservabilityApi.operationServers, + ); + const requestContext = server.makeRequestContext( + localVarPath, + HttpMethod.POST, + overrides, + ); + requestContext.setHeaderParam("Accept", "application/json"); + requestContext.setHttpConfig(_config.httpConfig); + + // Set User-Agent + if (this.userAgent) { + requestContext.setHeaderParam("User-Agent", this.userAgent); + } + + // Body Params + const contentType = getPreferredMediaType(["application/json"]); + requestContext.setHeaderParam("Content-Type", contentType); + const serializedBody = stringify( + serialize(body, TypingInfo, "LLMObsDatasetBatchUpdateRequest", ""), + contentType, + ); + requestContext.setBody(serializedBody); + + // Apply auth methods + applySecurityAuthentication(_config, requestContext, [ + "apiKeyAuth", + "appKeyAuth", + ]); + + return requestContext; + } + + public async cloneLLMObsDataset( + projectId: string, + datasetId: string, + body: LLMObsDatasetCloneRequest, + _options?: Configuration, + ): Promise { + const _config = _options || this.configuration; + + if ( + !_config.unstableOperations["LLMObservabilityApi.v2.cloneLLMObsDataset"] + ) { + throw new Error( + "Unstable operation 'cloneLLMObsDataset' is disabled. Enable it by setting `configuration.unstableOperations['LLMObservabilityApi.v2.cloneLLMObsDataset'] = true`", + ); + } + + // verify required parameter 'projectId' is not null or undefined + if (projectId === null || projectId === undefined) { + throw new RequiredError("projectId", "cloneLLMObsDataset"); + } + + // verify required parameter 'datasetId' is not null or undefined + if (datasetId === null || datasetId === undefined) { + throw new RequiredError("datasetId", "cloneLLMObsDataset"); + } + + // verify required parameter 'body' is not null or undefined + if (body === null || body === undefined) { + throw new RequiredError("body", "cloneLLMObsDataset"); + } + + // Path Params + const localVarPath = + "/api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/clone" + .replace("{project_id}", encodeURIComponent(String(projectId))) + .replace("{dataset_id}", encodeURIComponent(String(datasetId))); + + // Make Request Context + const { server, overrides } = _config.getServerAndOverrides( + "LLMObservabilityApi.v2.cloneLLMObsDataset", + LLMObservabilityApi.operationServers, + ); + const requestContext = server.makeRequestContext( + localVarPath, + HttpMethod.POST, + overrides, + ); + requestContext.setHeaderParam("Accept", "application/json"); + requestContext.setHttpConfig(_config.httpConfig); + + // Set User-Agent + if (this.userAgent) { + requestContext.setHeaderParam("User-Agent", this.userAgent); + } + + // Body Params + const contentType = getPreferredMediaType(["application/json"]); + requestContext.setHeaderParam("Content-Type", contentType); + const serializedBody = stringify( + serialize(body, TypingInfo, "LLMObsDatasetCloneRequest", ""), + contentType, + ); + requestContext.setBody(serializedBody); + + // Apply auth methods + applySecurityAuthentication(_config, requestContext, [ + "apiKeyAuth", + "appKeyAuth", + ]); + + return requestContext; + } + public async createLLMObsAnnotationQueue( body: LLMObsAnnotationQueueRequest, _options?: Configuration, @@ -1206,6 +1361,82 @@ export class LLMObservabilityApiRequestFactory extends BaseAPIRequestFactory { return requestContext; } + public async exportLLMObsDataset( + projectId: string, + datasetId: string, + format?: LLMObsDatasetExportFormat, + version?: number, + _options?: Configuration, + ): Promise { + const _config = _options || this.configuration; + + if ( + !_config.unstableOperations["LLMObservabilityApi.v2.exportLLMObsDataset"] + ) { + throw new Error( + "Unstable operation 'exportLLMObsDataset' is disabled. Enable it by setting `configuration.unstableOperations['LLMObservabilityApi.v2.exportLLMObsDataset'] = true`", + ); + } + + // verify required parameter 'projectId' is not null or undefined + if (projectId === null || projectId === undefined) { + throw new RequiredError("projectId", "exportLLMObsDataset"); + } + + // verify required parameter 'datasetId' is not null or undefined + if (datasetId === null || datasetId === undefined) { + throw new RequiredError("datasetId", "exportLLMObsDataset"); + } + + // Path Params + const localVarPath = + "/api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/export" + .replace("{project_id}", encodeURIComponent(String(projectId))) + .replace("{dataset_id}", encodeURIComponent(String(datasetId))); + + // Make Request Context + const { server, overrides } = _config.getServerAndOverrides( + "LLMObservabilityApi.v2.exportLLMObsDataset", + LLMObservabilityApi.operationServers, + ); + const requestContext = server.makeRequestContext( + localVarPath, + HttpMethod.GET, + overrides, + ); + requestContext.setHeaderParam("Accept", "text/csv, application/json"); + requestContext.setHttpConfig(_config.httpConfig); + + // Set User-Agent + if (this.userAgent) { + requestContext.setHeaderParam("User-Agent", this.userAgent); + } + + // Query Params + if (format !== undefined) { + requestContext.setQueryParam( + "format", + serialize(format, TypingInfo, "LLMObsDatasetExportFormat", ""), + "", + ); + } + if (version !== undefined) { + requestContext.setQueryParam( + "version", + serialize(version, TypingInfo, "number", "int64"), + "", + ); + } + + // Apply auth methods + applySecurityAuthentication(_config, requestContext, [ + "apiKeyAuth", + "appKeyAuth", + ]); + + return requestContext; + } + public async getLLMObsAnnotatedInteractions( queueId: string, _options?: Configuration, @@ -2356,6 +2587,81 @@ export class LLMObservabilityApiRequestFactory extends BaseAPIRequestFactory { return requestContext; } + public async restoreLLMObsDatasetVersion( + projectId: string, + datasetId: string, + body: LLMObsDatasetRestoreVersionRequest, + _options?: Configuration, + ): Promise { + const _config = _options || this.configuration; + + if ( + !_config.unstableOperations[ + "LLMObservabilityApi.v2.restoreLLMObsDatasetVersion" + ] + ) { + throw new Error( + "Unstable operation 'restoreLLMObsDatasetVersion' is disabled. Enable it by setting `configuration.unstableOperations['LLMObservabilityApi.v2.restoreLLMObsDatasetVersion'] = true`", + ); + } + + // verify required parameter 'projectId' is not null or undefined + if (projectId === null || projectId === undefined) { + throw new RequiredError("projectId", "restoreLLMObsDatasetVersion"); + } + + // verify required parameter 'datasetId' is not null or undefined + if (datasetId === null || datasetId === undefined) { + throw new RequiredError("datasetId", "restoreLLMObsDatasetVersion"); + } + + // verify required parameter 'body' is not null or undefined + if (body === null || body === undefined) { + throw new RequiredError("body", "restoreLLMObsDatasetVersion"); + } + + // Path Params + const localVarPath = + "/api/v2/llm-obs/v1/{project_id}/datasets/{dataset_id}/restore" + .replace("{project_id}", encodeURIComponent(String(projectId))) + .replace("{dataset_id}", encodeURIComponent(String(datasetId))); + + // Make Request Context + const { server, overrides } = _config.getServerAndOverrides( + "LLMObservabilityApi.v2.restoreLLMObsDatasetVersion", + LLMObservabilityApi.operationServers, + ); + const requestContext = server.makeRequestContext( + localVarPath, + HttpMethod.POST, + overrides, + ); + requestContext.setHeaderParam("Accept", "*/*"); + requestContext.setHttpConfig(_config.httpConfig); + + // Set User-Agent + if (this.userAgent) { + requestContext.setHeaderParam("User-Agent", this.userAgent); + } + + // Body Params + const contentType = getPreferredMediaType(["application/json"]); + requestContext.setHeaderParam("Content-Type", contentType); + const serializedBody = stringify( + serialize(body, TypingInfo, "LLMObsDatasetRestoreVersionRequest", ""), + contentType, + ); + requestContext.setBody(serializedBody); + + // Apply auth methods + applySecurityAuthentication(_config, requestContext, [ + "apiKeyAuth", + "appKeyAuth", + ]); + + return requestContext; + } + public async searchLLMObsExperimentation( body: LLMObsExperimentationSearchRequest, _options?: Configuration, @@ -3100,9 +3406,115 @@ export class LLMObservabilityApiRequestFactory extends BaseAPIRequestFactory { return requestContext; } -} -export class LLMObservabilityApiResponseProcessor { + public async uploadLLMObsDatasetRecordsFile( + projectId: string, + datasetId: string, + deduplicate?: boolean, + overwrite?: boolean, + tags?: Array, + includeUserData?: boolean, + file?: HttpFile, + _options?: Configuration, + ): Promise { + const _config = _options || this.configuration; + + if ( + !_config.unstableOperations[ + "LLMObservabilityApi.v2.uploadLLMObsDatasetRecordsFile" + ] + ) { + throw new Error( + "Unstable operation 'uploadLLMObsDatasetRecordsFile' is disabled. Enable it by setting `configuration.unstableOperations['LLMObservabilityApi.v2.uploadLLMObsDatasetRecordsFile'] = true`", + ); + } + + // verify required parameter 'projectId' is not null or undefined + if (projectId === null || projectId === undefined) { + throw new RequiredError("projectId", "uploadLLMObsDatasetRecordsFile"); + } + + // verify required parameter 'datasetId' is not null or undefined + if (datasetId === null || datasetId === undefined) { + throw new RequiredError("datasetId", "uploadLLMObsDatasetRecordsFile"); + } + + // Path Params + const localVarPath = + "/api/v2/llm-obs/v2/{project_id}/datasets/{dataset_id}/records/upload" + .replace("{project_id}", encodeURIComponent(String(projectId))) + .replace("{dataset_id}", encodeURIComponent(String(datasetId))); + + // Make Request Context + const { server, overrides } = _config.getServerAndOverrides( + "LLMObservabilityApi.v2.uploadLLMObsDatasetRecordsFile", + LLMObservabilityApi.operationServers, + ); + const requestContext = server.makeRequestContext( + localVarPath, + HttpMethod.POST, + overrides, + ); + requestContext.setHeaderParam( + "Accept", + "text/event-stream, application/json", + ); + requestContext.setHttpConfig(_config.httpConfig); + + // Set User-Agent + if (this.userAgent) { + requestContext.setHeaderParam("User-Agent", this.userAgent); + } + + // Query Params + if (deduplicate !== undefined) { + requestContext.setQueryParam( + "deduplicate", + serialize(deduplicate, TypingInfo, "boolean", ""), + "", + ); + } + if (overwrite !== undefined) { + requestContext.setQueryParam( + "overwrite", + serialize(overwrite, TypingInfo, "boolean", ""), + "", + ); + } + if (tags !== undefined) { + requestContext.setQueryParam( + "tags", + serialize(tags, TypingInfo, "Array", ""), + "multi", + ); + } + if (includeUserData !== undefined) { + requestContext.setQueryParam( + "include[user_data]", + serialize(includeUserData, TypingInfo, "boolean", ""), + "", + ); + } + + // Form Params + const localVarFormParams = new FormData(); + if (file !== undefined) { + // TODO: replace .append with .set + localVarFormParams.append("file", file as any); + } + requestContext.setBody(localVarFormParams); + + // Apply auth methods + applySecurityAuthentication(_config, requestContext, [ + "apiKeyAuth", + "appKeyAuth", + ]); + + return requestContext; + } +} + +export class LLMObservabilityApiResponseProcessor { /** * Unwraps the actual response sent by the server from the response context and deserializes the response content * to the expected objects @@ -3185,6 +3597,173 @@ export class LLMObservabilityApiResponseProcessor { ); } + /** + * Unwraps the actual response sent by the server from the response context and deserializes the response content + * to the expected objects + * + * @params response Response returned by the server for a request to batchUpdateLLMObsDataset + * @throws ApiException if the response code was not in [200, 299] + */ + public async batchUpdateLLMObsDataset( + response: ResponseContext, + ): Promise { + const contentType = normalizeMediaType(response.headers["content-type"]); + if (response.httpStatusCode === 200) { + const body: LLMObsDatasetRecordsMutationResponse = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "LLMObsDatasetRecordsMutationResponse", + ) as LLMObsDatasetRecordsMutationResponse; + return body; + } + if ( + response.httpStatusCode === 400 || + response.httpStatusCode === 401 || + response.httpStatusCode === 403 || + response.httpStatusCode === 404 || + response.httpStatusCode === 413 || + response.httpStatusCode === 500 + ) { + const bodyText = parse(await response.body.text(), contentType); + let body: JSONAPIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "JSONAPIErrorResponse", + ) as JSONAPIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException( + response.httpStatusCode, + body, + ); + } + if (response.httpStatusCode === 429) { + const bodyText = parse(await response.body.text(), contentType); + let body: APIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "APIErrorResponse", + ) as APIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException(response.httpStatusCode, body); + } + + // Work around for missing responses in specification, e.g. for petstore.yaml + if (response.httpStatusCode >= 200 && response.httpStatusCode <= 299) { + const body: LLMObsDatasetRecordsMutationResponse = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "LLMObsDatasetRecordsMutationResponse", + "", + ) as LLMObsDatasetRecordsMutationResponse; + return body; + } + + const body = (await response.body.text()) || ""; + throw new ApiException( + response.httpStatusCode, + 'Unknown API Status Code!\nBody: "' + body + '"', + ); + } + + /** + * Unwraps the actual response sent by the server from the response context and deserializes the response content + * to the expected objects + * + * @params response Response returned by the server for a request to cloneLLMObsDataset + * @throws ApiException if the response code was not in [200, 299] + */ + public async cloneLLMObsDataset( + response: ResponseContext, + ): Promise { + const contentType = normalizeMediaType(response.headers["content-type"]); + if (response.httpStatusCode === 200) { + const body: LLMObsDatasetResponse = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "LLMObsDatasetResponse", + ) as LLMObsDatasetResponse; + return body; + } + if ( + response.httpStatusCode === 400 || + response.httpStatusCode === 401 || + response.httpStatusCode === 403 || + response.httpStatusCode === 404 || + response.httpStatusCode === 500 + ) { + const bodyText = parse(await response.body.text(), contentType); + let body: JSONAPIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "JSONAPIErrorResponse", + ) as JSONAPIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException( + response.httpStatusCode, + body, + ); + } + if (response.httpStatusCode === 429) { + const bodyText = parse(await response.body.text(), contentType); + let body: APIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "APIErrorResponse", + ) as APIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException(response.httpStatusCode, body); + } + + // Work around for missing responses in specification, e.g. for petstore.yaml + if (response.httpStatusCode >= 200 && response.httpStatusCode <= 299) { + const body: LLMObsDatasetResponse = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "LLMObsDatasetResponse", + "", + ) as LLMObsDatasetResponse; + return body; + } + + const body = (await response.body.text()) || ""; + throw new ApiException( + response.httpStatusCode, + 'Unknown API Status Code!\nBody: "' + body + '"', + ); + } + /** * Unwraps the actual response sent by the server from the response context and deserializes the response content * to the expected objects @@ -4398,6 +4977,87 @@ export class LLMObservabilityApiResponseProcessor { ); } + /** + * Unwraps the actual response sent by the server from the response context and deserializes the response content + * to the expected objects + * + * @params response Response returned by the server for a request to exportLLMObsDataset + * @throws ApiException if the response code was not in [200, 299] + */ + public async exportLLMObsDataset(response: ResponseContext): Promise { + const contentType = normalizeMediaType(response.headers["content-type"]); + if (response.httpStatusCode === 200) { + const body: string = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "string", + ) as string; + return body; + } + if ( + response.httpStatusCode === 400 || + response.httpStatusCode === 401 || + response.httpStatusCode === 403 || + response.httpStatusCode === 404 || + response.httpStatusCode === 500 + ) { + const bodyText = parse(await response.body.text(), contentType); + let body: JSONAPIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "JSONAPIErrorResponse", + ) as JSONAPIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException( + response.httpStatusCode, + body, + ); + } + if (response.httpStatusCode === 429) { + const bodyText = parse(await response.body.text(), contentType); + let body: APIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "APIErrorResponse", + ) as APIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException(response.httpStatusCode, body); + } + + // Work around for missing responses in specification, e.g. for petstore.yaml + if (response.httpStatusCode >= 200 && response.httpStatusCode <= 299) { + const body: string = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "string", + "", + ) as string; + return body; + } + + const body = (await response.body.text()) || ""; + throw new ApiException( + response.httpStatusCode, + 'Unknown API Status Code!\nBody: "' + body + '"', + ); + } + /** * Unwraps the actual response sent by the server from the response context and deserializes the response content * to the expected objects @@ -5708,6 +6368,78 @@ export class LLMObservabilityApiResponseProcessor { ); } + /** + * Unwraps the actual response sent by the server from the response context and deserializes the response content + * to the expected objects + * + * @params response Response returned by the server for a request to restoreLLMObsDatasetVersion + * @throws ApiException if the response code was not in [200, 299] + */ + public async restoreLLMObsDatasetVersion( + response: ResponseContext, + ): Promise { + const contentType = normalizeMediaType(response.headers["content-type"]); + if (response.httpStatusCode === 200) { + return; + } + if ( + response.httpStatusCode === 400 || + response.httpStatusCode === 401 || + response.httpStatusCode === 403 || + response.httpStatusCode === 404 || + response.httpStatusCode === 500 + ) { + const bodyText = parse(await response.body.text(), contentType); + let body: JSONAPIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "JSONAPIErrorResponse", + ) as JSONAPIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException( + response.httpStatusCode, + body, + ); + } + if (response.httpStatusCode === 429) { + const bodyText = parse(await response.body.text(), contentType); + let body: APIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "APIErrorResponse", + ) as APIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException(response.httpStatusCode, body); + } + + // Work around for missing responses in specification, e.g. for petstore.yaml + if (response.httpStatusCode >= 200 && response.httpStatusCode <= 299) { + return; + } + + const body = (await response.body.text()) || ""; + throw new ApiException( + response.httpStatusCode, + 'Unknown API Status Code!\nBody: "' + body + '"', + ); + } + /** * Unwraps the actual response sent by the server from the response context and deserializes the response content * to the expected objects @@ -6589,6 +7321,89 @@ export class LLMObservabilityApiResponseProcessor { 'Unknown API Status Code!\nBody: "' + body + '"', ); } + + /** + * Unwraps the actual response sent by the server from the response context and deserializes the response content + * to the expected objects + * + * @params response Response returned by the server for a request to uploadLLMObsDatasetRecordsFile + * @throws ApiException if the response code was not in [200, 299] + */ + public async uploadLLMObsDatasetRecordsFile( + response: ResponseContext, + ): Promise { + const contentType = normalizeMediaType(response.headers["content-type"]); + if (response.httpStatusCode === 200) { + const body: string = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "string", + ) as string; + return body; + } + if ( + response.httpStatusCode === 400 || + response.httpStatusCode === 401 || + response.httpStatusCode === 403 || + response.httpStatusCode === 404 || + response.httpStatusCode === 500 + ) { + const bodyText = parse(await response.body.text(), contentType); + let body: JSONAPIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "JSONAPIErrorResponse", + ) as JSONAPIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException( + response.httpStatusCode, + body, + ); + } + if (response.httpStatusCode === 429) { + const bodyText = parse(await response.body.text(), contentType); + let body: APIErrorResponse; + try { + body = deserialize( + bodyText, + TypingInfo, + "APIErrorResponse", + ) as APIErrorResponse; + } catch (error) { + logger.debug(`Got error deserializing error: ${error}`); + throw new ApiException( + response.httpStatusCode, + bodyText, + ); + } + throw new ApiException(response.httpStatusCode, body); + } + + // Work around for missing responses in specification, e.g. for petstore.yaml + if (response.httpStatusCode >= 200 && response.httpStatusCode <= 299) { + const body: string = deserialize( + parse(await response.body.text(), contentType), + TypingInfo, + "string", + "", + ) as string; + return body; + } + + const body = (await response.body.text()) || ""; + throw new ApiException( + response.httpStatusCode, + 'Unknown API Status Code!\nBody: "' + body + '"', + ); + } } export interface LLMObservabilityApiAggregateLLMObsExperimentationRequest { @@ -6599,6 +7414,42 @@ export interface LLMObservabilityApiAggregateLLMObsExperimentationRequest { body: LLMObsExperimentationAnalyticsRequest; } +export interface LLMObservabilityApiBatchUpdateLLMObsDatasetRequest { + /** + * The ID of the LLM Observability project. + * @type string + */ + projectId: string; + /** + * The ID of the LLM Observability dataset. + * @type string + */ + datasetId: string; + /** + * Batch update payload. + * @type LLMObsDatasetBatchUpdateRequest + */ + body: LLMObsDatasetBatchUpdateRequest; +} + +export interface LLMObservabilityApiCloneLLMObsDatasetRequest { + /** + * The ID of the LLM Observability project. + * @type string + */ + projectId: string; + /** + * The ID of the source LLM Observability dataset to clone. + * @type string + */ + datasetId: string; + /** + * Clone dataset payload. + * @type LLMObsDatasetCloneRequest + */ + body: LLMObsDatasetCloneRequest; +} + export interface LLMObservabilityApiCreateLLMObsAnnotationQueueRequest { /** * Create annotation queue payload. @@ -6782,6 +7633,29 @@ export interface LLMObservabilityApiDeleteLLMObsProjectsRequest { body: LLMObsDeleteProjectsRequest; } +export interface LLMObservabilityApiExportLLMObsDatasetRequest { + /** + * The ID of the LLM Observability project. + * @type string + */ + projectId: string; + /** + * The ID of the LLM Observability dataset. + * @type string + */ + datasetId: string; + /** + * Export format for the dataset contents. Only `csv` is currently supported. + * @type LLMObsDatasetExportFormat + */ + format?: LLMObsDatasetExportFormat; + /** + * Version of the dataset to export. If omitted, the current version is used. Must be between 0 and the current version of the dataset, inclusive. + * @type number + */ + version?: number; +} + export interface LLMObservabilityApiGetLLMObsAnnotatedInteractionsRequest { /** * The ID of the LLM Observability annotation queue. @@ -7085,6 +7959,24 @@ export interface LLMObservabilityApiLockLLMObsDatasetDraftStateRequest { datasetId: string; } +export interface LLMObservabilityApiRestoreLLMObsDatasetVersionRequest { + /** + * The ID of the LLM Observability project. + * @type string + */ + projectId: string; + /** + * The ID of the LLM Observability dataset. + * @type string + */ + datasetId: string; + /** + * Restore dataset version payload. + * @type LLMObsDatasetRestoreVersionRequest + */ + body: LLMObsDatasetRestoreVersionRequest; +} + export interface LLMObservabilityApiSearchLLMObsExperimentationRequest { /** * Experimentation search payload. @@ -7223,6 +8115,44 @@ export interface LLMObservabilityApiUpdateLLMObsProjectRequest { body: LLMObsProjectUpdateRequest; } +export interface LLMObservabilityApiUploadLLMObsDatasetRecordsFileRequest { + /** + * The ID of the LLM Observability project. + * @type string + */ + projectId: string; + /** + * The ID of the LLM Observability dataset. + * @type string + */ + datasetId: string; + /** + * Whether to skip records whose `input` already exists in the dataset. Defaults to `false`. + * @type boolean + */ + deduplicate?: boolean; + /** + * Whether to overwrite existing records that share the same user-provided `id`. Defaults to `true`. + * @type boolean + */ + overwrite?: boolean; + /** + * Tags to apply to every uploaded record, in addition to any tags defined on individual rows. Can be repeated, e.g. `tags=env:prod&tags=team:ai`. + * @type Array + */ + tags?: Array; + /** + * Whether to enrich the response with user metadata. + * @type boolean + */ + includeUserData?: boolean; + /** + * Multipart upload payload containing the records file. + * @type HttpFile + */ + file?: HttpFile; +} + export class LLMObservabilityApi { private requestFactory: LLMObservabilityApiRequestFactory; private responseProcessor: LLMObservabilityApiResponseProcessor; @@ -7267,6 +8197,54 @@ export class LLMObservabilityApi { }); } + /** + * Insert, update, and delete records in a single dataset operation. By default, a new dataset version is created when the batch is applied. + * @param param The request object + */ + public batchUpdateLLMObsDataset( + param: LLMObservabilityApiBatchUpdateLLMObsDatasetRequest, + options?: Configuration, + ): Promise { + const requestContextPromise = this.requestFactory.batchUpdateLLMObsDataset( + param.projectId, + param.datasetId, + param.body, + options, + ); + return requestContextPromise.then((requestContext) => { + return this.configuration.httpApi + .send(requestContext) + .then((responseContext) => { + return this.responseProcessor.batchUpdateLLMObsDataset( + responseContext, + ); + }); + }); + } + + /** + * Clone a dataset, copying its current records into a new dataset within the same project. + * @param param The request object + */ + public cloneLLMObsDataset( + param: LLMObservabilityApiCloneLLMObsDatasetRequest, + options?: Configuration, + ): Promise { + const requestContextPromise = this.requestFactory.cloneLLMObsDataset( + param.projectId, + param.datasetId, + param.body, + options, + ); + return requestContextPromise.then((requestContext) => { + return this.configuration.httpApi + .send(requestContext) + .then((responseContext) => { + return this.responseProcessor.cloneLLMObsDataset(responseContext); + }); + }); + } + /** * Create an annotation queue. The `name` and `project_id` fields are required. * An optional `annotation_schema` can be provided to define the labels for the queue. @@ -7646,6 +8624,30 @@ export class LLMObservabilityApi { }); } + /** + * Download the contents of a dataset as a CSV file. The download is streamed and includes one row per dataset record. + * @param param The request object + */ + public exportLLMObsDataset( + param: LLMObservabilityApiExportLLMObsDatasetRequest, + options?: Configuration, + ): Promise { + const requestContextPromise = this.requestFactory.exportLLMObsDataset( + param.projectId, + param.datasetId, + param.format, + param.version, + options, + ); + return requestContextPromise.then((requestContext) => { + return this.configuration.httpApi + .send(requestContext) + .then((responseContext) => { + return this.responseProcessor.exportLLMObsDataset(responseContext); + }); + }); + } + /** * Retrieve all interactions (traces and sessions) and their annotations for a given annotation queue. * @param param The request object @@ -8051,6 +9053,32 @@ export class LLMObservabilityApi { }); } + /** + * Restore a dataset to a previous version. The dataset's current version is bumped, and its records are replaced with the records from the specified prior version. + * @param param The request object + */ + public restoreLLMObsDatasetVersion( + param: LLMObservabilityApiRestoreLLMObsDatasetVersionRequest, + options?: Configuration, + ): Promise { + const requestContextPromise = + this.requestFactory.restoreLLMObsDatasetVersion( + param.projectId, + param.datasetId, + param.body, + options, + ); + return requestContextPromise.then((requestContext) => { + return this.configuration.httpApi + .send(requestContext) + .then((responseContext) => { + return this.responseProcessor.restoreLLMObsDatasetVersion( + responseContext, + ); + }); + }); + } + /** * Search across LLM Observability experimentation entities — projects, datasets, dataset records, experiments, and experiment runs — using cursor-based pagination. * @@ -8319,4 +9347,40 @@ export class LLMObservabilityApi { }); }); } + + /** + * Upload records to a dataset from a file. The request is a `multipart/form-data` upload containing a single `file` part. + * Currently only CSV is supported. The CSV must include an `input` column. Optional columns are `id`, `expected_output`, `metadata`, and `tags`. + * + * The response is a Server-Sent Events stream (`text/event-stream`) emitting progress updates while records are processed. The stream emits the following named events: + * - `progress`: incremental record counts written so far. + * - `completed`: terminal event with a JSON body containing `records_created`. + * - `error`: terminal event with a JSON body containing an error `message`. + * @param param The request object + */ + public uploadLLMObsDatasetRecordsFile( + param: LLMObservabilityApiUploadLLMObsDatasetRecordsFileRequest, + options?: Configuration, + ): Promise { + const requestContextPromise = + this.requestFactory.uploadLLMObsDatasetRecordsFile( + param.projectId, + param.datasetId, + param.deduplicate, + param.overwrite, + param.tags, + param.includeUserData, + param.file, + options, + ); + return requestContextPromise.then((requestContext) => { + return this.configuration.httpApi + .send(requestContext) + .then((responseContext) => { + return this.responseProcessor.uploadLLMObsDatasetRecordsFile( + responseContext, + ); + }); + }); + } } diff --git a/services/llm_observability/src/v2/index.ts b/services/llm_observability/src/v2/index.ts index ed964197e7d3..402a29f065c1 100644 --- a/services/llm_observability/src/v2/index.ts +++ b/services/llm_observability/src/v2/index.ts @@ -1,5 +1,7 @@ export { LLMObservabilityApiAggregateLLMObsExperimentationRequest, + LLMObservabilityApiBatchUpdateLLMObsDatasetRequest, + LLMObservabilityApiCloneLLMObsDatasetRequest, LLMObservabilityApiCreateLLMObsAnnotationQueueRequest, LLMObservabilityApiCreateLLMObsAnnotationQueueInteractionsRequest, LLMObservabilityApiCreateLLMObsDatasetRequest, @@ -16,6 +18,7 @@ export { LLMObservabilityApiDeleteLLMObsDatasetsRequest, LLMObservabilityApiDeleteLLMObsExperimentsRequest, LLMObservabilityApiDeleteLLMObsProjectsRequest, + LLMObservabilityApiExportLLMObsDatasetRequest, LLMObservabilityApiGetLLMObsAnnotatedInteractionsRequest, LLMObservabilityApiGetLLMObsAnnotatedInteractionsByTraceIDsRequest, LLMObservabilityApiGetLLMObsAnnotationQueueLabelSchemaRequest, @@ -32,6 +35,7 @@ export { LLMObservabilityApiListLLMObsProjectsRequest, LLMObservabilityApiListLLMObsSpansRequest, LLMObservabilityApiLockLLMObsDatasetDraftStateRequest, + LLMObservabilityApiRestoreLLMObsDatasetVersionRequest, LLMObservabilityApiSearchLLMObsExperimentationRequest, LLMObservabilityApiSearchLLMObsSpansRequest, LLMObservabilityApiSimpleSearchLLMObsExperimentationRequest, @@ -43,6 +47,7 @@ export { LLMObservabilityApiUpdateLLMObsDatasetRecordsRequest, LLMObservabilityApiUpdateLLMObsExperimentRequest, LLMObservabilityApiUpdateLLMObsProjectRequest, + LLMObservabilityApiUploadLLMObsDatasetRecordsFileRequest, LLMObservabilityApi, } from "./LLMObservabilityApi"; @@ -134,6 +139,14 @@ export { LLMObsDataDeletionResponse } from "./models/LLMObsDataDeletionResponse" export { LLMObsDataDeletionResponseAttributes } from "./models/LLMObsDataDeletionResponseAttributes"; export { LLMObsDataDeletionResponseData } from "./models/LLMObsDataDeletionResponseData"; export { LLMObsDataDeletionResponseType } from "./models/LLMObsDataDeletionResponseType"; +export { LLMObsDatasetBatchUpdateDataAttributesRequest } from "./models/LLMObsDatasetBatchUpdateDataAttributesRequest"; +export { LLMObsDatasetBatchUpdateDataRequest } from "./models/LLMObsDatasetBatchUpdateDataRequest"; +export { LLMObsDatasetBatchUpdateInsertRecord } from "./models/LLMObsDatasetBatchUpdateInsertRecord"; +export { LLMObsDatasetBatchUpdateRequest } from "./models/LLMObsDatasetBatchUpdateRequest"; +export { LLMObsDatasetBatchUpdateUpdateRecord } from "./models/LLMObsDatasetBatchUpdateUpdateRecord"; +export { LLMObsDatasetCloneDataAttributesRequest } from "./models/LLMObsDatasetCloneDataAttributesRequest"; +export { LLMObsDatasetCloneDataRequest } from "./models/LLMObsDatasetCloneDataRequest"; +export { LLMObsDatasetCloneRequest } from "./models/LLMObsDatasetCloneRequest"; export { LLMObsDatasetDataAttributesRequest } from "./models/LLMObsDatasetDataAttributesRequest"; export { LLMObsDatasetDataAttributesResponse } from "./models/LLMObsDatasetDataAttributesResponse"; export { LLMObsDatasetDataRequest } from "./models/LLMObsDatasetDataRequest"; @@ -143,6 +156,7 @@ export { LLMObsDatasetDraftStateDataAttributes } from "./models/LLMObsDatasetDra export { LLMObsDatasetDraftStateResponse } from "./models/LLMObsDatasetDraftStateResponse"; export { LLMObsDatasetDraftStateType } from "./models/LLMObsDatasetDraftStateType"; export { LLMObsDatasetDraftStateUser } from "./models/LLMObsDatasetDraftStateUser"; +export { LLMObsDatasetExportFormat } from "./models/LLMObsDatasetExportFormat"; export { LLMObsDatasetRecordDataResponse } from "./models/LLMObsDatasetRecordDataResponse"; export { LLMObsDatasetRecordItem } from "./models/LLMObsDatasetRecordItem"; export { LLMObsDatasetRecordsDataAttributesRequest } from "./models/LLMObsDatasetRecordsDataAttributesRequest"; @@ -154,9 +168,13 @@ export { LLMObsDatasetRecordsRequest } from "./models/LLMObsDatasetRecordsReques export { LLMObsDatasetRecordsUpdateDataAttributesRequest } from "./models/LLMObsDatasetRecordsUpdateDataAttributesRequest"; export { LLMObsDatasetRecordsUpdateDataRequest } from "./models/LLMObsDatasetRecordsUpdateDataRequest"; export { LLMObsDatasetRecordsUpdateRequest } from "./models/LLMObsDatasetRecordsUpdateRequest"; +export { LLMObsDatasetRecordTagOperations } from "./models/LLMObsDatasetRecordTagOperations"; export { LLMObsDatasetRecordUpdateItem } from "./models/LLMObsDatasetRecordUpdateItem"; export { LLMObsDatasetRequest } from "./models/LLMObsDatasetRequest"; export { LLMObsDatasetResponse } from "./models/LLMObsDatasetResponse"; +export { LLMObsDatasetRestoreVersionDataAttributesRequest } from "./models/LLMObsDatasetRestoreVersionDataAttributesRequest"; +export { LLMObsDatasetRestoreVersionDataRequest } from "./models/LLMObsDatasetRestoreVersionDataRequest"; +export { LLMObsDatasetRestoreVersionRequest } from "./models/LLMObsDatasetRestoreVersionRequest"; export { LLMObsDatasetsResponse } from "./models/LLMObsDatasetsResponse"; export { LLMObsDatasetType } from "./models/LLMObsDatasetType"; export { LLMObsDatasetUpdateDataAttributesRequest } from "./models/LLMObsDatasetUpdateDataAttributesRequest"; diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateDataAttributesRequest.ts b/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateDataAttributesRequest.ts new file mode 100644 index 000000000000..738b03431aa1 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateDataAttributesRequest.ts @@ -0,0 +1,79 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsDatasetBatchUpdateInsertRecord } from "./LLMObsDatasetBatchUpdateInsertRecord"; +import { LLMObsDatasetBatchUpdateUpdateRecord } from "./LLMObsDatasetBatchUpdateUpdateRecord"; + +/** + * Attributes for batch-updating records in an LLM Observability dataset. + */ +export class LLMObsDatasetBatchUpdateDataAttributesRequest { + /** + * Whether to create a new dataset version when applying the batch update. Defaults to `true`. + */ + "createNewVersion"?: boolean; + /** + * Record IDs to delete. + */ + "deleteRecords"?: Array; + /** + * Records to insert. + */ + "insertRecords"?: Array; + /** + * List of tag strings. + */ + "tags"?: Array; + /** + * Records to update by ID. + */ + "updateRecords"?: Array; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + createNewVersion: { + baseName: "create_new_version", + type: "boolean", + }, + deleteRecords: { + baseName: "delete_records", + type: "Array", + }, + insertRecords: { + baseName: "insert_records", + type: "Array", + }, + tags: { + baseName: "tags", + type: "Array", + }, + updateRecords: { + baseName: "update_records", + type: "Array", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetBatchUpdateDataAttributesRequest.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateDataRequest.ts b/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateDataRequest.ts new file mode 100644 index 000000000000..884218d1e31e --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateDataRequest.ts @@ -0,0 +1,66 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsDatasetBatchUpdateDataAttributesRequest } from "./LLMObsDatasetBatchUpdateDataAttributesRequest"; +import { LLMObsDatasetType } from "./LLMObsDatasetType"; + +/** + * Data object for batch-updating records in an LLM Observability dataset. + */ +export class LLMObsDatasetBatchUpdateDataRequest { + /** + * Attributes for batch-updating records in an LLM Observability dataset. + */ + "attributes": LLMObsDatasetBatchUpdateDataAttributesRequest; + /** + * Unique identifier of the dataset. + */ + "id": string; + /** + * Resource type of an LLM Observability dataset. + */ + "type": LLMObsDatasetType; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + attributes: { + baseName: "attributes", + type: "LLMObsDatasetBatchUpdateDataAttributesRequest", + required: true, + }, + id: { + baseName: "id", + type: "string", + required: true, + }, + type: { + baseName: "type", + type: "LLMObsDatasetType", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetBatchUpdateDataRequest.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateInsertRecord.ts b/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateInsertRecord.ts new file mode 100644 index 000000000000..977d7c92cb62 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateInsertRecord.ts @@ -0,0 +1,88 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { AnyValue } from "./AnyValue"; +import { LLMObsDatasetRecordTagOperations } from "./LLMObsDatasetRecordTagOperations"; + +/** + * A record to insert as part of a batch update on an LLM Observability dataset. + */ +export class LLMObsDatasetBatchUpdateInsertRecord { + /** + * Represents any valid JSON value. + */ + "expectedOutput"?: AnyValue; + /** + * Optional user-provided identifier for the record. If omitted, the server generates an identifier. + */ + "id"?: string; + /** + * Represents any valid JSON value. + */ + "input": AnyValue | null; + /** + * Arbitrary metadata associated with the record. + */ + "metadata"?: { [key: string]: any }; + /** + * Explicit tag operations for updating records. Operations are applied in order, Remove then Add then Set. `set` is the final override, if specified, the result of `remove` and `add` is discarded. + */ + "tagOperations"?: LLMObsDatasetRecordTagOperations; + /** + * List of tag strings. + */ + "tags"?: Array; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + expectedOutput: { + baseName: "expected_output", + type: "AnyValue", + }, + id: { + baseName: "id", + type: "string", + }, + input: { + baseName: "input", + type: "AnyValue", + required: true, + }, + metadata: { + baseName: "metadata", + type: "{ [key: string]: any; }", + }, + tagOperations: { + baseName: "tag_operations", + type: "LLMObsDatasetRecordTagOperations", + }, + tags: { + baseName: "tags", + type: "Array", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetBatchUpdateInsertRecord.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateRequest.ts b/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateRequest.ts new file mode 100644 index 000000000000..88c7b135bd13 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateRequest.ts @@ -0,0 +1,47 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsDatasetBatchUpdateDataRequest } from "./LLMObsDatasetBatchUpdateDataRequest"; + +/** + * Request to batch-insert, update, and delete records in an LLM Observability dataset. + */ +export class LLMObsDatasetBatchUpdateRequest { + /** + * Data object for batch-updating records in an LLM Observability dataset. + */ + "data": LLMObsDatasetBatchUpdateDataRequest; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + data: { + baseName: "data", + type: "LLMObsDatasetBatchUpdateDataRequest", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetBatchUpdateRequest.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateUpdateRecord.ts b/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateUpdateRecord.ts new file mode 100644 index 000000000000..8fff0b2435e6 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetBatchUpdateUpdateRecord.ts @@ -0,0 +1,80 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { AnyValue } from "./AnyValue"; +import { LLMObsDatasetRecordTagOperations } from "./LLMObsDatasetRecordTagOperations"; + +/** + * A record update payload as part of a batch update on an LLM Observability dataset. + */ +export class LLMObsDatasetBatchUpdateUpdateRecord { + /** + * Represents any valid JSON value. + */ + "expectedOutput"?: AnyValue; + /** + * Unique identifier of the record to update. + */ + "id": string; + /** + * Represents any valid JSON value. + */ + "input"?: AnyValue; + /** + * Updated metadata associated with the record. + */ + "metadata"?: { [key: string]: any }; + /** + * Explicit tag operations for updating records. Operations are applied in order, Remove then Add then Set. `set` is the final override, if specified, the result of `remove` and `add` is discarded. + */ + "tagOperations"?: LLMObsDatasetRecordTagOperations; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + expectedOutput: { + baseName: "expected_output", + type: "AnyValue", + }, + id: { + baseName: "id", + type: "string", + required: true, + }, + input: { + baseName: "input", + type: "AnyValue", + }, + metadata: { + baseName: "metadata", + type: "{ [key: string]: any; }", + }, + tagOperations: { + baseName: "tag_operations", + type: "LLMObsDatasetRecordTagOperations", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetBatchUpdateUpdateRecord.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetCloneDataAttributesRequest.ts b/services/llm_observability/src/v2/models/LLMObsDatasetCloneDataAttributesRequest.ts new file mode 100644 index 000000000000..a1dc6b82caf1 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetCloneDataAttributesRequest.ts @@ -0,0 +1,53 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * Attributes for cloning an LLM Observability dataset. + */ +export class LLMObsDatasetCloneDataAttributesRequest { + /** + * Description of the cloned dataset. + */ + "description"?: string; + /** + * Name of the cloned dataset. + */ + "name": string; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + description: { + baseName: "description", + type: "string", + }, + name: { + baseName: "name", + type: "string", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetCloneDataAttributesRequest.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetCloneDataRequest.ts b/services/llm_observability/src/v2/models/LLMObsDatasetCloneDataRequest.ts new file mode 100644 index 000000000000..1bd5433235b7 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetCloneDataRequest.ts @@ -0,0 +1,66 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsDatasetCloneDataAttributesRequest } from "./LLMObsDatasetCloneDataAttributesRequest"; +import { LLMObsDatasetType } from "./LLMObsDatasetType"; + +/** + * Data object for cloning an LLM Observability dataset. + */ +export class LLMObsDatasetCloneDataRequest { + /** + * Attributes for cloning an LLM Observability dataset. + */ + "attributes": LLMObsDatasetCloneDataAttributesRequest; + /** + * Identifier of the source dataset to clone. + */ + "id": string; + /** + * Resource type of an LLM Observability dataset. + */ + "type": LLMObsDatasetType; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + attributes: { + baseName: "attributes", + type: "LLMObsDatasetCloneDataAttributesRequest", + required: true, + }, + id: { + baseName: "id", + type: "string", + required: true, + }, + type: { + baseName: "type", + type: "LLMObsDatasetType", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetCloneDataRequest.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetCloneRequest.ts b/services/llm_observability/src/v2/models/LLMObsDatasetCloneRequest.ts new file mode 100644 index 000000000000..6f293a99053f --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetCloneRequest.ts @@ -0,0 +1,47 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsDatasetCloneDataRequest } from "./LLMObsDatasetCloneDataRequest"; + +/** + * Request to clone an LLM Observability dataset. + */ +export class LLMObsDatasetCloneRequest { + /** + * Data object for cloning an LLM Observability dataset. + */ + "data": LLMObsDatasetCloneDataRequest; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + data: { + baseName: "data", + type: "LLMObsDatasetCloneDataRequest", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetCloneRequest.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetExportFormat.ts b/services/llm_observability/src/v2/models/LLMObsDatasetExportFormat.ts new file mode 100644 index 000000000000..f1614f8ea055 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetExportFormat.ts @@ -0,0 +1,7 @@ +import { UnparsedObject } from "@datadog/datadog-api-client"; + +/** + * Supported export format for an LLM Observability dataset. + */ +export type LLMObsDatasetExportFormat = typeof CSV | UnparsedObject; +export const CSV = "csv"; diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetRecordTagOperations.ts b/services/llm_observability/src/v2/models/LLMObsDatasetRecordTagOperations.ts new file mode 100644 index 000000000000..3aab70d26b76 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetRecordTagOperations.ts @@ -0,0 +1,60 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * Explicit tag operations for updating records. Operations are applied in order, Remove then Add then Set. `set` is the final override, if specified, the result of `remove` and `add` is discarded. + */ +export class LLMObsDatasetRecordTagOperations { + /** + * List of tag strings. + */ + "add"?: Array; + /** + * List of tag strings. + */ + "remove"?: Array; + /** + * List of tag strings. + */ + "set"?: Array; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + add: { + baseName: "add", + type: "Array", + }, + remove: { + baseName: "remove", + type: "Array", + }, + set: { + baseName: "set", + type: "Array", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetRecordTagOperations.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetRestoreVersionDataAttributesRequest.ts b/services/llm_observability/src/v2/models/LLMObsDatasetRestoreVersionDataAttributesRequest.ts new file mode 100644 index 000000000000..452c6cedadf5 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetRestoreVersionDataAttributesRequest.ts @@ -0,0 +1,46 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +/** + * Attributes for restoring an LLM Observability dataset to a previous version. + */ +export class LLMObsDatasetRestoreVersionDataAttributesRequest { + /** + * Version number of the dataset to restore. Must be between 0 and the current version of the dataset, inclusive. + */ + "datasetVersion": number; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + datasetVersion: { + baseName: "dataset_version", + type: "number", + required: true, + format: "int32", + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetRestoreVersionDataAttributesRequest.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetRestoreVersionDataRequest.ts b/services/llm_observability/src/v2/models/LLMObsDatasetRestoreVersionDataRequest.ts new file mode 100644 index 000000000000..6875eb0930cc --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetRestoreVersionDataRequest.ts @@ -0,0 +1,66 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsDatasetRestoreVersionDataAttributesRequest } from "./LLMObsDatasetRestoreVersionDataAttributesRequest"; +import { LLMObsDatasetType } from "./LLMObsDatasetType"; + +/** + * Data object for restoring an LLM Observability dataset to a previous version. + */ +export class LLMObsDatasetRestoreVersionDataRequest { + /** + * Attributes for restoring an LLM Observability dataset to a previous version. + */ + "attributes": LLMObsDatasetRestoreVersionDataAttributesRequest; + /** + * Unique identifier of the dataset to restore. + */ + "id": string; + /** + * Resource type of an LLM Observability dataset. + */ + "type": LLMObsDatasetType; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + attributes: { + baseName: "attributes", + type: "LLMObsDatasetRestoreVersionDataAttributesRequest", + required: true, + }, + id: { + baseName: "id", + type: "string", + required: true, + }, + type: { + baseName: "type", + type: "LLMObsDatasetType", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetRestoreVersionDataRequest.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/LLMObsDatasetRestoreVersionRequest.ts b/services/llm_observability/src/v2/models/LLMObsDatasetRestoreVersionRequest.ts new file mode 100644 index 000000000000..af4ed552fdc1 --- /dev/null +++ b/services/llm_observability/src/v2/models/LLMObsDatasetRestoreVersionRequest.ts @@ -0,0 +1,47 @@ +import { AttributeTypeMap } from "@datadog/datadog-api-client"; + +import { LLMObsDatasetRestoreVersionDataRequest } from "./LLMObsDatasetRestoreVersionDataRequest"; + +/** + * Request to restore an LLM Observability dataset to a previous version. + */ +export class LLMObsDatasetRestoreVersionRequest { + /** + * Data object for restoring an LLM Observability dataset to a previous version. + */ + "data": LLMObsDatasetRestoreVersionDataRequest; + /** + * A container for additional, undeclared properties. + * This is a holder for any undeclared properties as specified with + * the 'additionalProperties' keyword in the OAS document. + */ + "additionalProperties"?: { [key: string]: any }; + /** + * @ignore + */ + "_unparsed"?: boolean; + + /** + * @ignore + */ + static readonly attributeTypeMap: AttributeTypeMap = { + data: { + baseName: "data", + type: "LLMObsDatasetRestoreVersionDataRequest", + required: true, + }, + additionalProperties: { + baseName: "additionalProperties", + type: "{ [key: string]: any; }", + }, + }; + + /** + * @ignore + */ + static getAttributeTypeMap(): AttributeTypeMap { + return LLMObsDatasetRestoreVersionRequest.attributeTypeMap; + } + + public constructor() {} +} diff --git a/services/llm_observability/src/v2/models/TypingInfo.ts b/services/llm_observability/src/v2/models/TypingInfo.ts index 29aa6613c5dd..19c23c17b254 100644 --- a/services/llm_observability/src/v2/models/TypingInfo.ts +++ b/services/llm_observability/src/v2/models/TypingInfo.ts @@ -67,6 +67,14 @@ import { LLMObsDataDeletionRequestData } from "./LLMObsDataDeletionRequestData"; import { LLMObsDataDeletionResponse } from "./LLMObsDataDeletionResponse"; import { LLMObsDataDeletionResponseAttributes } from "./LLMObsDataDeletionResponseAttributes"; import { LLMObsDataDeletionResponseData } from "./LLMObsDataDeletionResponseData"; +import { LLMObsDatasetBatchUpdateDataAttributesRequest } from "./LLMObsDatasetBatchUpdateDataAttributesRequest"; +import { LLMObsDatasetBatchUpdateDataRequest } from "./LLMObsDatasetBatchUpdateDataRequest"; +import { LLMObsDatasetBatchUpdateInsertRecord } from "./LLMObsDatasetBatchUpdateInsertRecord"; +import { LLMObsDatasetBatchUpdateRequest } from "./LLMObsDatasetBatchUpdateRequest"; +import { LLMObsDatasetBatchUpdateUpdateRecord } from "./LLMObsDatasetBatchUpdateUpdateRecord"; +import { LLMObsDatasetCloneDataAttributesRequest } from "./LLMObsDatasetCloneDataAttributesRequest"; +import { LLMObsDatasetCloneDataRequest } from "./LLMObsDatasetCloneDataRequest"; +import { LLMObsDatasetCloneRequest } from "./LLMObsDatasetCloneRequest"; import { LLMObsDatasetDataAttributesRequest } from "./LLMObsDatasetDataAttributesRequest"; import { LLMObsDatasetDataAttributesResponse } from "./LLMObsDatasetDataAttributesResponse"; import { LLMObsDatasetDataRequest } from "./LLMObsDatasetDataRequest"; @@ -77,6 +85,7 @@ import { LLMObsDatasetDraftStateResponse } from "./LLMObsDatasetDraftStateRespon import { LLMObsDatasetDraftStateUser } from "./LLMObsDatasetDraftStateUser"; import { LLMObsDatasetRecordDataResponse } from "./LLMObsDatasetRecordDataResponse"; import { LLMObsDatasetRecordItem } from "./LLMObsDatasetRecordItem"; +import { LLMObsDatasetRecordTagOperations } from "./LLMObsDatasetRecordTagOperations"; import { LLMObsDatasetRecordUpdateItem } from "./LLMObsDatasetRecordUpdateItem"; import { LLMObsDatasetRecordsDataAttributesRequest } from "./LLMObsDatasetRecordsDataAttributesRequest"; import { LLMObsDatasetRecordsDataRequest } from "./LLMObsDatasetRecordsDataRequest"; @@ -89,6 +98,9 @@ import { LLMObsDatasetRecordsUpdateDataRequest } from "./LLMObsDatasetRecordsUpd import { LLMObsDatasetRecordsUpdateRequest } from "./LLMObsDatasetRecordsUpdateRequest"; import { LLMObsDatasetRequest } from "./LLMObsDatasetRequest"; import { LLMObsDatasetResponse } from "./LLMObsDatasetResponse"; +import { LLMObsDatasetRestoreVersionDataAttributesRequest } from "./LLMObsDatasetRestoreVersionDataAttributesRequest"; +import { LLMObsDatasetRestoreVersionDataRequest } from "./LLMObsDatasetRestoreVersionDataRequest"; +import { LLMObsDatasetRestoreVersionRequest } from "./LLMObsDatasetRestoreVersionRequest"; import { LLMObsDatasetUpdateDataAttributesRequest } from "./LLMObsDatasetUpdateDataAttributesRequest"; import { LLMObsDatasetUpdateDataRequest } from "./LLMObsDatasetUpdateDataRequest"; import { LLMObsDatasetUpdateRequest } from "./LLMObsDatasetUpdateRequest"; @@ -259,6 +271,7 @@ export const TypingInfo: ModelTypingInfo = { LLMObsDataDeletionRequestType: ["create_deletion_req"], LLMObsDataDeletionResponseType: ["deletion_request"], LLMObsDatasetDraftStateType: ["draft_state_data"], + LLMObsDatasetExportFormat: ["csv"], LLMObsDatasetType: ["datasets"], LLMObsDatasetVersionType: ["dataset_version"], LLMObsDisplayBlockInteractionType: ["display_block"], @@ -404,6 +417,16 @@ export const TypingInfo: ModelTypingInfo = { LLMObsDataDeletionResponse: LLMObsDataDeletionResponse, LLMObsDataDeletionResponseAttributes: LLMObsDataDeletionResponseAttributes, LLMObsDataDeletionResponseData: LLMObsDataDeletionResponseData, + LLMObsDatasetBatchUpdateDataAttributesRequest: + LLMObsDatasetBatchUpdateDataAttributesRequest, + LLMObsDatasetBatchUpdateDataRequest: LLMObsDatasetBatchUpdateDataRequest, + LLMObsDatasetBatchUpdateInsertRecord: LLMObsDatasetBatchUpdateInsertRecord, + LLMObsDatasetBatchUpdateRequest: LLMObsDatasetBatchUpdateRequest, + LLMObsDatasetBatchUpdateUpdateRecord: LLMObsDatasetBatchUpdateUpdateRecord, + LLMObsDatasetCloneDataAttributesRequest: + LLMObsDatasetCloneDataAttributesRequest, + LLMObsDatasetCloneDataRequest: LLMObsDatasetCloneDataRequest, + LLMObsDatasetCloneRequest: LLMObsDatasetCloneRequest, LLMObsDatasetDataAttributesRequest: LLMObsDatasetDataAttributesRequest, LLMObsDatasetDataAttributesResponse: LLMObsDatasetDataAttributesResponse, LLMObsDatasetDataRequest: LLMObsDatasetDataRequest, @@ -415,6 +438,7 @@ export const TypingInfo: ModelTypingInfo = { LLMObsDatasetDraftStateUser: LLMObsDatasetDraftStateUser, LLMObsDatasetRecordDataResponse: LLMObsDatasetRecordDataResponse, LLMObsDatasetRecordItem: LLMObsDatasetRecordItem, + LLMObsDatasetRecordTagOperations: LLMObsDatasetRecordTagOperations, LLMObsDatasetRecordUpdateItem: LLMObsDatasetRecordUpdateItem, LLMObsDatasetRecordsDataAttributesRequest: LLMObsDatasetRecordsDataAttributesRequest, @@ -430,6 +454,11 @@ export const TypingInfo: ModelTypingInfo = { LLMObsDatasetRecordsUpdateRequest: LLMObsDatasetRecordsUpdateRequest, LLMObsDatasetRequest: LLMObsDatasetRequest, LLMObsDatasetResponse: LLMObsDatasetResponse, + LLMObsDatasetRestoreVersionDataAttributesRequest: + LLMObsDatasetRestoreVersionDataAttributesRequest, + LLMObsDatasetRestoreVersionDataRequest: + LLMObsDatasetRestoreVersionDataRequest, + LLMObsDatasetRestoreVersionRequest: LLMObsDatasetRestoreVersionRequest, LLMObsDatasetUpdateDataAttributesRequest: LLMObsDatasetUpdateDataAttributesRequest, LLMObsDatasetUpdateDataRequest: LLMObsDatasetUpdateDataRequest,