diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index be7bf771e3..aa3a8981b4 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -782,6 +782,14 @@ components: required: true schema: type: string + LLMObsEvalNamePathParameter: + description: The name of the custom LLM Observability evaluator configuration. + example: "my-custom-evaluator" + in: path + name: eval_name + required: true + schema: + type: string LLMObsExperimentIDPathParameter: description: The ID of the LLM Observability experiment. example: "3fd6b5e0-8910-4b1c-a7d0-5b84de329012" @@ -36781,6 +36789,418 @@ components: nullable: true type: string type: object + LLMObsCustomEvalConfigAssessmentCriteria: + description: Criteria used to assess the pass/fail result of a custom evaluator. + properties: + max_threshold: + description: Maximum numeric threshold for a passing result. + example: 1.0 + format: double + nullable: true + type: number + min_threshold: + description: Minimum numeric threshold for a passing result. + example: 0.7 + format: double + nullable: true + type: number + pass_values: + description: Specific output values considered as a passing result. + example: + - "pass" + - "yes" + items: + description: A value considered as a passing result. + type: string + nullable: true + type: array + pass_when: + description: When true, a boolean output of true is treated as passing. + example: true + nullable: true + type: boolean + type: object + LLMObsCustomEvalConfigAttributes: + description: Attributes of a custom LLM Observability evaluator configuration. + properties: + category: + description: Category of the evaluator. + example: "Custom" + type: string + created_at: + description: Timestamp when the evaluator configuration was created. + example: "2024-01-15T10:30:00Z" + format: date-time + type: string + created_by: + $ref: "#/components/schemas/LLMObsCustomEvalConfigUser" + eval_name: + description: Name of the custom evaluator. + example: "my-custom-evaluator" + type: string + last_updated_by: + $ref: "#/components/schemas/LLMObsCustomEvalConfigUser" + llm_judge_config: + $ref: "#/components/schemas/LLMObsCustomEvalConfigLLMJudgeConfig" + llm_provider: + $ref: "#/components/schemas/LLMObsCustomEvalConfigLLMProvider" + target: + $ref: "#/components/schemas/LLMObsCustomEvalConfigTarget" + updated_at: + description: Timestamp when the evaluator configuration was last updated. + example: "2024-01-15T10:30:00Z" + format: date-time + type: string + required: + - eval_name + - created_at + - updated_at + type: object + LLMObsCustomEvalConfigBedrockOptions: + description: AWS Bedrock-specific options for LLM provider configuration. + properties: + region: + description: AWS region for Bedrock. + example: "us-east-1" + type: string + type: object + LLMObsCustomEvalConfigData: + description: Data object for a custom LLM Observability evaluator configuration. + properties: + attributes: + $ref: "#/components/schemas/LLMObsCustomEvalConfigAttributes" + id: + description: Unique name identifier of the evaluator configuration. + example: "my-custom-evaluator" + type: string + type: + $ref: "#/components/schemas/LLMObsCustomEvalConfigType" + required: + - id + - type + - attributes + type: object + LLMObsCustomEvalConfigEvalScope: + description: Scope at which to evaluate spans. + enum: + - span + - trace + - session + example: "span" + type: string + x-enum-varnames: + - SPAN + - TRACE + - SESSION + LLMObsCustomEvalConfigInferenceParams: + description: LLM inference parameters for a custom evaluator. + properties: + frequency_penalty: + description: Frequency penalty to reduce repetition. + example: 0.0 + format: float + type: number + max_tokens: + description: Maximum number of tokens to generate. + example: 1024 + format: int64 + type: integer + presence_penalty: + description: Presence penalty to reduce repetition. + example: 0.0 + format: float + type: number + temperature: + description: Sampling temperature for the LLM. + example: 0.7 + format: float + type: number + top_k: + description: Top-k sampling parameter. + example: 50 + format: int64 + type: integer + top_p: + description: Top-p (nucleus) sampling parameter. + example: 1.0 + format: float + type: number + type: object + LLMObsCustomEvalConfigIntegrationProvider: + description: Name of the LLM integration provider. + enum: + - openai + - amazon-bedrock + - anthropic + - azure-openai + - vertex-ai + - llm-proxy + example: "openai" + type: string + x-enum-varnames: + - OPENAI + - AMAZON_BEDROCK + - ANTHROPIC + - AZURE_OPENAI + - VERTEX_AI + - LLM_PROXY + LLMObsCustomEvalConfigLLMJudgeConfig: + description: LLM judge configuration for a custom evaluator. + properties: + assessment_criteria: + $ref: "#/components/schemas/LLMObsCustomEvalConfigAssessmentCriteria" + inference_params: + $ref: "#/components/schemas/LLMObsCustomEvalConfigInferenceParams" + last_used_library_prompt_template_name: + description: Name of the last library prompt template used. + example: "sentiment-analysis-v1" + nullable: true + type: string + modified_library_prompt_template: + description: Whether the library prompt template was modified. + example: false + nullable: true + type: boolean + output_schema: + additionalProperties: {} + description: JSON schema describing the expected output format of the LLM judge. + nullable: true + type: object + parsing_type: + $ref: "#/components/schemas/LLMObsCustomEvalConfigParsingType" + prompt_template: + description: List of messages forming the LLM judge prompt template. + items: + $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage" + type: array + required: + - inference_params + type: object + LLMObsCustomEvalConfigLLMProvider: + description: LLM provider configuration for a custom evaluator. + properties: + bedrock: + $ref: "#/components/schemas/LLMObsCustomEvalConfigBedrockOptions" + integration_account_id: + description: Integration account identifier. + example: "my-account-id" + type: string + integration_provider: + $ref: "#/components/schemas/LLMObsCustomEvalConfigIntegrationProvider" + model_name: + description: Name of the LLM model. + example: "gpt-4o" + type: string + vertex_ai: + $ref: "#/components/schemas/LLMObsCustomEvalConfigVertexAIOptions" + type: object + LLMObsCustomEvalConfigParsingType: + description: Output parsing type for a custom LLM judge evaluator. + enum: + - structured_output + - json + example: "structured_output" + type: string + x-enum-varnames: + - STRUCTURED_OUTPUT + - JSON + LLMObsCustomEvalConfigPromptContent: + description: A content block within a prompt message. + properties: + type: + description: Content block type. + example: "text" + type: string + value: + $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptContentValue" + required: + - type + - value + type: object + LLMObsCustomEvalConfigPromptContentValue: + description: Value of a prompt message content block. + properties: + text: + description: Text content of the message block. + example: "What is the sentiment of this review?" + type: string + tool_call: + $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptToolCall" + tool_call_result: + $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptToolResult" + type: object + LLMObsCustomEvalConfigPromptMessage: + description: A message in the prompt template for a custom LLM judge evaluator. + properties: + content: + description: Text content of the message. + example: "Rate the quality of the following response:" + type: string + contents: + description: Multi-part content blocks for the message. + items: + $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptContent" + type: array + role: + description: Role of the message author. + example: "user" + type: string + required: + - role + type: object + LLMObsCustomEvalConfigPromptToolCall: + description: A tool call within a prompt message. + properties: + arguments: + description: JSON-encoded arguments for the tool call. + example: '{"location": "San Francisco"}' + type: string + id: + description: Unique identifier of the tool call. + example: "call_abc123" + type: string + name: + description: Name of the tool being called. + example: "get_weather" + type: string + type: + description: Type of the tool call. + example: "function" + type: string + type: object + LLMObsCustomEvalConfigPromptToolResult: + description: A tool call result within a prompt message. + properties: + name: + description: Name of the tool that produced this result. + example: "get_weather" + type: string + result: + description: The result returned by the tool. + example: "sunny, 72F" + type: string + tool_id: + description: Identifier of the tool call this result corresponds to. + example: "call_abc123" + type: string + type: + description: Type of the tool result. + example: "function" + type: string + type: object + LLMObsCustomEvalConfigResponse: + description: Response containing a custom LLM Observability evaluator configuration. + properties: + data: + $ref: "#/components/schemas/LLMObsCustomEvalConfigData" + required: + - data + type: object + LLMObsCustomEvalConfigTarget: + description: Target application configuration for a custom evaluator. + properties: + application_name: + description: Name of the ML application this evaluator targets. + example: "my-llm-app" + type: string + enabled: + description: Whether the evaluator is active for the target application. + example: true + type: boolean + eval_scope: + $ref: "#/components/schemas/LLMObsCustomEvalConfigEvalScope" + nullable: true + filter: + description: Filter expression to select which spans to evaluate. + example: "@service:my-service" + nullable: true + type: string + root_spans_only: + description: When true, only root spans are evaluated. + example: true + nullable: true + type: boolean + sampling_percentage: + description: Percentage of traces to evaluate. Must be greater than 0 and at most 100. + example: 50.0 + format: float + nullable: true + type: number + required: + - application_name + - enabled + type: object + LLMObsCustomEvalConfigType: + description: Type of the custom LLM Observability evaluator configuration resource. + enum: + - evaluator_config + example: "evaluator_config" + type: string + x-enum-varnames: + - EVALUATOR_CONFIG + LLMObsCustomEvalConfigUpdateAttributes: + description: Attributes for creating or updating a custom LLM Observability evaluator configuration. + properties: + category: + description: Category of the evaluator. + example: "Custom" + type: string + eval_name: + description: Name of the custom evaluator. If provided, must match the eval_name path parameter. + example: "my-custom-evaluator" + type: string + llm_judge_config: + $ref: "#/components/schemas/LLMObsCustomEvalConfigLLMJudgeConfig" + llm_provider: + $ref: "#/components/schemas/LLMObsCustomEvalConfigLLMProvider" + target: + $ref: "#/components/schemas/LLMObsCustomEvalConfigTarget" + required: + - target + type: object + LLMObsCustomEvalConfigUpdateData: + description: Data object for creating or updating a custom LLM Observability evaluator configuration. + properties: + attributes: + $ref: "#/components/schemas/LLMObsCustomEvalConfigUpdateAttributes" + id: + description: Name of the evaluator. If provided, must match the eval_name path parameter. + example: "my-custom-evaluator" + type: string + type: + $ref: "#/components/schemas/LLMObsCustomEvalConfigType" + required: + - type + - attributes + type: object + LLMObsCustomEvalConfigUpdateRequest: + description: Request to create or update a custom LLM Observability evaluator configuration. + properties: + data: + $ref: "#/components/schemas/LLMObsCustomEvalConfigUpdateData" + required: + - data + type: object + LLMObsCustomEvalConfigUser: + description: A Datadog user associated with a custom evaluator configuration. + properties: + email: + description: Email address of the user. + example: "user@example.com" + type: string + type: object + LLMObsCustomEvalConfigVertexAIOptions: + description: Google Vertex AI-specific options for LLM provider configuration. + properties: + location: + description: Google Cloud region. + example: "us-central1" + type: string + project: + description: Google Cloud project ID. + example: "my-gcp-project" + type: string + type: object LLMObsDatasetDataAttributesRequest: description: Attributes for creating an LLM Observability dataset. properties: @@ -78603,6 +79023,199 @@ paths: x-unstable: |- This endpoint is in Preview and may introduce breaking changes. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/unstable/llm-obs/config/evaluators/custom/{eval_name}: + delete: + description: Delete a custom LLM Observability evaluator configuration by its name. + operationId: DeleteLLMObsCustomEvalConfig + parameters: + - $ref: "#/components/parameters/LLMObsEvalNamePathParameter" + responses: + "204": + description: No Content + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Delete a custom evaluator configuration + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + get: + description: Retrieve a custom LLM Observability evaluator configuration by its name. + operationId: GetLLMObsCustomEvalConfig + parameters: + - $ref: "#/components/parameters/LLMObsEvalNamePathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + category: "Custom" + created_at: "2024-01-15T10:30:00Z" + created_by: + email: "user@example.com" + eval_name: "my-custom-evaluator" + last_updated_by: + email: "user@example.com" + llm_judge_config: + inference_params: + max_tokens: 1024 + temperature: 0.7 + parsing_type: "structured_output" + llm_provider: + integration_provider: "openai" + model_name: "gpt-4o" + target: + application_name: "my-llm-app" + enabled: true + sampling_percentage: 50.0 + updated_at: "2024-01-15T10:30:00Z" + id: "my-custom-evaluator" + type: "evaluator_config" + schema: + $ref: "#/components/schemas/LLMObsCustomEvalConfigResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Get a custom evaluator configuration + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + put: + description: Create or update a custom LLM Observability evaluator configuration by its name. + operationId: UpdateLLMObsCustomEvalConfig + parameters: + - $ref: "#/components/parameters/LLMObsEvalNamePathParameter" + requestBody: + content: + application/json: + examples: + default: + value: + data: + attributes: + llm_judge_config: + inference_params: + max_tokens: 1024 + temperature: 0.7 + parsing_type: "structured_output" + llm_provider: + integration_provider: "openai" + model_name: "gpt-4o" + target: + application_name: "my-llm-app" + enabled: true + sampling_percentage: 50.0 + id: "my-custom-evaluator" + type: "evaluator_config" + schema: + $ref: "#/components/schemas/LLMObsCustomEvalConfigUpdateRequest" + description: Custom evaluator configuration payload. + required: true + responses: + "200": + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "422": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unprocessable Entity + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Create or update a custom evaluator configuration + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/actions-datastores: get: description: Lists all datastores for the organization. diff --git a/docs/datadog_api_client.v2.model.rst b/docs/datadog_api_client.v2.model.rst index 48b85aab57..22d2fc9f5c 100644 --- a/docs/datadog_api_client.v2.model.rst +++ b/docs/datadog_api_client.v2.model.rst @@ -16517,6 +16517,167 @@ datadog\_api\_client.v2.model.llm\_obs\_cursor\_meta module :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_assessment\_criteria module +----------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_assessment_criteria + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_attributes module +------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_bedrock\_options module +------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_bedrock_options + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_data module +------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_eval\_scope module +-------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_eval_scope + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_inference\_params module +-------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_inference_params + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_integration\_provider module +------------------------------------------------------------------------------------------ + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_integration_provider + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_llm\_judge\_config module +--------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_judge_config + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_llm\_provider module +---------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_provider + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_parsing\_type module +---------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_parsing_type + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_prompt\_content module +------------------------------------------------------------------------------------ + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_prompt\_content\_value module +------------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content_value + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_prompt\_message module +------------------------------------------------------------------------------------ + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_message + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_prompt\_tool\_call module +--------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_call + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_prompt\_tool\_result module +----------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_result + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_response module +----------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_target module +--------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_target + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_type module +------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_type + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_update\_attributes module +--------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_update_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_update\_data module +--------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_update_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_update\_request module +------------------------------------------------------------------------------------ + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_update_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_user module +------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_user + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_custom\_eval\_config\_vertex\_ai\_options module +---------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_custom_eval_config_vertex_ai_options + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_dataset\_data\_attributes\_request module --------------------------------------------------------------------------------- diff --git a/examples/v2/llm-observability/DeleteLLMObsCustomEvalConfig.py b/examples/v2/llm-observability/DeleteLLMObsCustomEvalConfig.py new file mode 100644 index 0000000000..1141e977de --- /dev/null +++ b/examples/v2/llm-observability/DeleteLLMObsCustomEvalConfig.py @@ -0,0 +1,14 @@ +""" +Delete a custom evaluator configuration returns "No Content" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +configuration = Configuration() +configuration.unstable_operations["delete_llm_obs_custom_eval_config"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + api_instance.delete_llm_obs_custom_eval_config( + eval_name="eval_name", + ) diff --git a/examples/v2/llm-observability/GetLLMObsCustomEvalConfig.py b/examples/v2/llm-observability/GetLLMObsCustomEvalConfig.py new file mode 100644 index 0000000000..72d00539c4 --- /dev/null +++ b/examples/v2/llm-observability/GetLLMObsCustomEvalConfig.py @@ -0,0 +1,16 @@ +""" +Get a custom evaluator configuration returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +configuration = Configuration() +configuration.unstable_operations["get_llm_obs_custom_eval_config"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.get_llm_obs_custom_eval_config( + eval_name="eval_name", + ) + + print(response) diff --git a/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.py b/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.py new file mode 100644 index 0000000000..6ee8f657b1 --- /dev/null +++ b/examples/v2/llm-observability/UpdateLLMObsCustomEvalConfig.py @@ -0,0 +1,125 @@ +""" +Create or update a custom evaluator configuration returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi +from datadog_api_client.v2.model.llm_obs_custom_eval_config_assessment_criteria import ( + LLMObsCustomEvalConfigAssessmentCriteria, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_bedrock_options import LLMObsCustomEvalConfigBedrockOptions +from datadog_api_client.v2.model.llm_obs_custom_eval_config_eval_scope import LLMObsCustomEvalConfigEvalScope +from datadog_api_client.v2.model.llm_obs_custom_eval_config_inference_params import ( + LLMObsCustomEvalConfigInferenceParams, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_integration_provider import ( + LLMObsCustomEvalConfigIntegrationProvider, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_judge_config import LLMObsCustomEvalConfigLLMJudgeConfig +from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_provider import LLMObsCustomEvalConfigLLMProvider +from datadog_api_client.v2.model.llm_obs_custom_eval_config_parsing_type import LLMObsCustomEvalConfigParsingType +from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content import LLMObsCustomEvalConfigPromptContent +from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content_value import ( + LLMObsCustomEvalConfigPromptContentValue, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_message import LLMObsCustomEvalConfigPromptMessage +from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_call import LLMObsCustomEvalConfigPromptToolCall +from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_result import ( + LLMObsCustomEvalConfigPromptToolResult, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_target import LLMObsCustomEvalConfigTarget +from datadog_api_client.v2.model.llm_obs_custom_eval_config_type import LLMObsCustomEvalConfigType +from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_attributes import ( + LLMObsCustomEvalConfigUpdateAttributes, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_data import LLMObsCustomEvalConfigUpdateData +from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_request import LLMObsCustomEvalConfigUpdateRequest +from datadog_api_client.v2.model.llm_obs_custom_eval_config_vertex_ai_options import ( + LLMObsCustomEvalConfigVertexAIOptions, +) + +body = LLMObsCustomEvalConfigUpdateRequest( + data=LLMObsCustomEvalConfigUpdateData( + attributes=LLMObsCustomEvalConfigUpdateAttributes( + category="Custom", + eval_name="my-custom-evaluator", + llm_judge_config=LLMObsCustomEvalConfigLLMJudgeConfig( + assessment_criteria=LLMObsCustomEvalConfigAssessmentCriteria( + max_threshold=1.0, + min_threshold=0.7, + pass_values=[ + "pass", + "yes", + ], + pass_when=True, + ), + inference_params=LLMObsCustomEvalConfigInferenceParams( + frequency_penalty=0.0, + max_tokens=1024, + presence_penalty=0.0, + temperature=0.7, + top_k=50, + top_p=1.0, + ), + last_used_library_prompt_template_name="sentiment-analysis-v1", + modified_library_prompt_template=False, + output_schema=None, + parsing_type=LLMObsCustomEvalConfigParsingType.STRUCTURED_OUTPUT, + prompt_template=[ + LLMObsCustomEvalConfigPromptMessage( + content="Rate the quality of the following response:", + contents=[ + LLMObsCustomEvalConfigPromptContent( + type="text", + value=LLMObsCustomEvalConfigPromptContentValue( + text="What is the sentiment of this review?", + tool_call=LLMObsCustomEvalConfigPromptToolCall( + arguments='{"location": "San Francisco"}', + id="call_abc123", + name="get_weather", + type="function", + ), + tool_call_result=LLMObsCustomEvalConfigPromptToolResult( + name="get_weather", + result="sunny, 72F", + tool_id="call_abc123", + type="function", + ), + ), + ), + ], + role="user", + ), + ], + ), + llm_provider=LLMObsCustomEvalConfigLLMProvider( + bedrock=LLMObsCustomEvalConfigBedrockOptions( + region="us-east-1", + ), + integration_account_id="my-account-id", + integration_provider=LLMObsCustomEvalConfigIntegrationProvider.OPENAI, + model_name="gpt-4o", + vertex_ai=LLMObsCustomEvalConfigVertexAIOptions( + location="us-central1", + project="my-gcp-project", + ), + ), + target=LLMObsCustomEvalConfigTarget( + application_name="my-llm-app", + enabled=True, + eval_scope=LLMObsCustomEvalConfigEvalScope.SPAN, + filter="@service:my-service", + root_spans_only=True, + sampling_percentage=50.0, + ), + ), + id="my-custom-evaluator", + type=LLMObsCustomEvalConfigType.EVALUATOR_CONFIG, + ), +) + +configuration = Configuration() +configuration.unstable_operations["update_llm_obs_custom_eval_config"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + api_instance.update_llm_obs_custom_eval_config(eval_name="eval_name", body=body) diff --git a/src/datadog_api_client/configuration.py b/src/datadog_api_client/configuration.py index ac81a279c7..c8664c960b 100644 --- a/src/datadog_api_client/configuration.py +++ b/src/datadog_api_client/configuration.py @@ -261,6 +261,33 @@ def __init__( "v2.list_fleet_schedules": False, "v2.trigger_fleet_schedule": False, "v2.update_fleet_schedule": False, + "v2.create_llm_obs_annotation_queue": False, + "v2.create_llm_obs_annotation_queue_interactions": False, + "v2.create_llm_obs_dataset": False, + "v2.create_llm_obs_dataset_records": False, + "v2.create_llm_obs_experiment": False, + "v2.create_llm_obs_experiment_events": False, + "v2.create_llm_obs_project": False, + "v2.delete_llm_obs_annotation_queue": False, + "v2.delete_llm_obs_annotation_queue_interactions": False, + "v2.delete_llm_obs_custom_eval_config": False, + "v2.delete_llm_obs_dataset_records": False, + "v2.delete_llm_obs_datasets": False, + "v2.delete_llm_obs_experiments": False, + "v2.delete_llm_obs_projects": False, + "v2.get_llm_obs_annotated_interactions": False, + "v2.get_llm_obs_custom_eval_config": False, + "v2.list_llm_obs_annotation_queues": False, + "v2.list_llm_obs_dataset_records": False, + "v2.list_llm_obs_datasets": False, + "v2.list_llm_obs_experiments": False, + "v2.list_llm_obs_projects": False, + "v2.update_llm_obs_annotation_queue": False, + "v2.update_llm_obs_custom_eval_config": False, + "v2.update_llm_obs_dataset": False, + "v2.update_llm_obs_dataset_records": False, + "v2.update_llm_obs_experiment": False, + "v2.update_llm_obs_project": False, "v2.create_open_api": False, "v2.delete_open_api": False, "v2.get_open_api": False, @@ -409,30 +436,6 @@ def __init__( "v2.update_jira_issue_template": False, "v2.create_tenancy_config": False, "v2.get_tenancy_configs": False, - "v2.create_llm_obs_annotation_queue": False, - "v2.create_llm_obs_annotation_queue_interactions": False, - "v2.create_llm_obs_dataset": False, - "v2.create_llm_obs_dataset_records": False, - "v2.create_llm_obs_experiment": False, - "v2.create_llm_obs_experiment_events": False, - "v2.create_llm_obs_project": False, - "v2.delete_llm_obs_annotation_queue": False, - "v2.delete_llm_obs_annotation_queue_interactions": False, - "v2.delete_llm_obs_dataset_records": False, - "v2.delete_llm_obs_datasets": False, - "v2.delete_llm_obs_experiments": False, - "v2.delete_llm_obs_projects": False, - "v2.get_llm_obs_annotated_interactions": False, - "v2.list_llm_obs_annotation_queues": False, - "v2.list_llm_obs_dataset_records": False, - "v2.list_llm_obs_datasets": False, - "v2.list_llm_obs_experiments": False, - "v2.list_llm_obs_projects": False, - "v2.update_llm_obs_annotation_queue": False, - "v2.update_llm_obs_dataset": False, - "v2.update_llm_obs_dataset_records": False, - "v2.update_llm_obs_experiment": False, - "v2.update_llm_obs_project": False, "v2.add_role_to_restriction_query": False, "v2.create_restriction_query": False, "v2.delete_restriction_query": False, diff --git a/src/datadog_api_client/v2/api/llm_observability_api.py b/src/datadog_api_client/v2/api/llm_observability_api.py index 15492209ce..2cb396a694 100644 --- a/src/datadog_api_client/v2/api/llm_observability_api.py +++ b/src/datadog_api_client/v2/api/llm_observability_api.py @@ -11,6 +11,8 @@ UnsetType, unset, ) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_response import LLMObsCustomEvalConfigResponse +from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_request import LLMObsCustomEvalConfigUpdateRequest from datadog_api_client.v2.model.llm_obs_annotation_queues_response import LLMObsAnnotationQueuesResponse from datadog_api_client.v2.model.llm_obs_annotation_queue_response import LLMObsAnnotationQueueResponse from datadog_api_client.v2.model.llm_obs_annotation_queue_request import LLMObsAnnotationQueueRequest @@ -277,6 +279,29 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._delete_llm_obs_custom_eval_config_endpoint = _Endpoint( + settings={ + "response_type": None, + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/unstable/llm-obs/config/evaluators/custom/{eval_name}", + "operation_id": "delete_llm_obs_custom_eval_config", + "http_method": "DELETE", + "version": "v2", + }, + params_map={ + "eval_name": { + "required": True, + "openapi_types": (str,), + "attribute": "eval_name", + "location": "path", + }, + }, + headers_map={ + "accept": ["*/*"], + }, + api_client=api_client, + ) + self._delete_llm_obs_dataset_records_endpoint = _Endpoint( settings={ "response_type": None, @@ -398,6 +423,29 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._get_llm_obs_custom_eval_config_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsCustomEvalConfigResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/unstable/llm-obs/config/evaluators/custom/{eval_name}", + "operation_id": "get_llm_obs_custom_eval_config", + "http_method": "GET", + "version": "v2", + }, + params_map={ + "eval_name": { + "required": True, + "openapi_types": (str,), + "attribute": "eval_name", + "location": "path", + }, + }, + headers_map={ + "accept": ["application/json"], + }, + api_client=api_client, + ) + self._list_llm_obs_annotation_queues_endpoint = _Endpoint( settings={ "response_type": (LLMObsAnnotationQueuesResponse,), @@ -618,6 +666,32 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._update_llm_obs_custom_eval_config_endpoint = _Endpoint( + settings={ + "response_type": None, + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/unstable/llm-obs/config/evaluators/custom/{eval_name}", + "operation_id": "update_llm_obs_custom_eval_config", + "http_method": "PUT", + "version": "v2", + }, + params_map={ + "eval_name": { + "required": True, + "openapi_types": (str,), + "attribute": "eval_name", + "location": "path", + }, + "body": { + "required": True, + "openapi_types": (LLMObsCustomEvalConfigUpdateRequest,), + "location": "body", + }, + }, + headers_map={"accept": ["*/*"], "content_type": ["application/json"]}, + api_client=api_client, + ) + self._update_llm_obs_dataset_endpoint = _Endpoint( settings={ "response_type": (LLMObsDatasetResponse,), @@ -919,6 +993,23 @@ def delete_llm_obs_annotation_queue_interactions( return self._delete_llm_obs_annotation_queue_interactions_endpoint.call_with_http_info(**kwargs) + def delete_llm_obs_custom_eval_config( + self, + eval_name: str, + ) -> None: + """Delete a custom evaluator configuration. + + Delete a custom LLM Observability evaluator configuration by its name. + + :param eval_name: The name of the custom LLM Observability evaluator configuration. + :type eval_name: str + :rtype: None + """ + kwargs: Dict[str, Any] = {} + kwargs["eval_name"] = eval_name + + return self._delete_llm_obs_custom_eval_config_endpoint.call_with_http_info(**kwargs) + def delete_llm_obs_dataset_records( self, project_id: str, @@ -1019,6 +1110,23 @@ def get_llm_obs_annotated_interactions( return self._get_llm_obs_annotated_interactions_endpoint.call_with_http_info(**kwargs) + def get_llm_obs_custom_eval_config( + self, + eval_name: str, + ) -> LLMObsCustomEvalConfigResponse: + """Get a custom evaluator configuration. + + Retrieve a custom LLM Observability evaluator configuration by its name. + + :param eval_name: The name of the custom LLM Observability evaluator configuration. + :type eval_name: str + :rtype: LLMObsCustomEvalConfigResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["eval_name"] = eval_name + + return self._get_llm_obs_custom_eval_config_endpoint.call_with_http_info(**kwargs) + def list_llm_obs_annotation_queues( self, *, @@ -1230,6 +1338,28 @@ def update_llm_obs_annotation_queue( return self._update_llm_obs_annotation_queue_endpoint.call_with_http_info(**kwargs) + def update_llm_obs_custom_eval_config( + self, + eval_name: str, + body: LLMObsCustomEvalConfigUpdateRequest, + ) -> None: + """Create or update a custom evaluator configuration. + + Create or update a custom LLM Observability evaluator configuration by its name. + + :param eval_name: The name of the custom LLM Observability evaluator configuration. + :type eval_name: str + :param body: Custom evaluator configuration payload. + :type body: LLMObsCustomEvalConfigUpdateRequest + :rtype: None + """ + kwargs: Dict[str, Any] = {} + kwargs["eval_name"] = eval_name + + kwargs["body"] = body + + return self._update_llm_obs_custom_eval_config_endpoint.call_with_http_info(**kwargs) + def update_llm_obs_dataset( self, project_id: str, diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_assessment_criteria.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_assessment_criteria.py new file mode 100644 index 0000000000..734baddf3f --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_assessment_criteria.py @@ -0,0 +1,65 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + none_type, + unset, + UnsetType, +) + + +class LLMObsCustomEvalConfigAssessmentCriteria(ModelNormal): + @cached_property + def openapi_types(_): + return { + "max_threshold": (float, none_type), + "min_threshold": (float, none_type), + "pass_values": ([str], none_type), + "pass_when": (bool, none_type), + } + + attribute_map = { + "max_threshold": "max_threshold", + "min_threshold": "min_threshold", + "pass_values": "pass_values", + "pass_when": "pass_when", + } + + def __init__( + self_, + max_threshold: Union[float, none_type, UnsetType] = unset, + min_threshold: Union[float, none_type, UnsetType] = unset, + pass_values: Union[List[str], none_type, UnsetType] = unset, + pass_when: Union[bool, none_type, UnsetType] = unset, + **kwargs, + ): + """ + Criteria used to assess the pass/fail result of a custom evaluator. + + :param max_threshold: Maximum numeric threshold for a passing result. + :type max_threshold: float, none_type, optional + + :param min_threshold: Minimum numeric threshold for a passing result. + :type min_threshold: float, none_type, optional + + :param pass_values: Specific output values considered as a passing result. + :type pass_values: [str], none_type, optional + + :param pass_when: When true, a boolean output of true is treated as passing. + :type pass_when: bool, none_type, optional + """ + if max_threshold is not unset: + kwargs["max_threshold"] = max_threshold + if min_threshold is not unset: + kwargs["min_threshold"] = min_threshold + if pass_values is not unset: + kwargs["pass_values"] = pass_values + if pass_when is not unset: + kwargs["pass_when"] = pass_when + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_attributes.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_attributes.py new file mode 100644 index 0000000000..6a6612eafd --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_attributes.py @@ -0,0 +1,121 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + datetime, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_user import LLMObsCustomEvalConfigUser + from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_judge_config import ( + LLMObsCustomEvalConfigLLMJudgeConfig, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_provider import LLMObsCustomEvalConfigLLMProvider + from datadog_api_client.v2.model.llm_obs_custom_eval_config_target import LLMObsCustomEvalConfigTarget + + +class LLMObsCustomEvalConfigAttributes(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_user import LLMObsCustomEvalConfigUser + from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_judge_config import ( + LLMObsCustomEvalConfigLLMJudgeConfig, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_provider import ( + LLMObsCustomEvalConfigLLMProvider, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_target import LLMObsCustomEvalConfigTarget + + return { + "category": (str,), + "created_at": (datetime,), + "created_by": (LLMObsCustomEvalConfigUser,), + "eval_name": (str,), + "last_updated_by": (LLMObsCustomEvalConfigUser,), + "llm_judge_config": (LLMObsCustomEvalConfigLLMJudgeConfig,), + "llm_provider": (LLMObsCustomEvalConfigLLMProvider,), + "target": (LLMObsCustomEvalConfigTarget,), + "updated_at": (datetime,), + } + + attribute_map = { + "category": "category", + "created_at": "created_at", + "created_by": "created_by", + "eval_name": "eval_name", + "last_updated_by": "last_updated_by", + "llm_judge_config": "llm_judge_config", + "llm_provider": "llm_provider", + "target": "target", + "updated_at": "updated_at", + } + + def __init__( + self_, + created_at: datetime, + eval_name: str, + updated_at: datetime, + category: Union[str, UnsetType] = unset, + created_by: Union[LLMObsCustomEvalConfigUser, UnsetType] = unset, + last_updated_by: Union[LLMObsCustomEvalConfigUser, UnsetType] = unset, + llm_judge_config: Union[LLMObsCustomEvalConfigLLMJudgeConfig, UnsetType] = unset, + llm_provider: Union[LLMObsCustomEvalConfigLLMProvider, UnsetType] = unset, + target: Union[LLMObsCustomEvalConfigTarget, UnsetType] = unset, + **kwargs, + ): + """ + Attributes of a custom LLM Observability evaluator configuration. + + :param category: Category of the evaluator. + :type category: str, optional + + :param created_at: Timestamp when the evaluator configuration was created. + :type created_at: datetime + + :param created_by: A Datadog user associated with a custom evaluator configuration. + :type created_by: LLMObsCustomEvalConfigUser, optional + + :param eval_name: Name of the custom evaluator. + :type eval_name: str + + :param last_updated_by: A Datadog user associated with a custom evaluator configuration. + :type last_updated_by: LLMObsCustomEvalConfigUser, optional + + :param llm_judge_config: LLM judge configuration for a custom evaluator. + :type llm_judge_config: LLMObsCustomEvalConfigLLMJudgeConfig, optional + + :param llm_provider: LLM provider configuration for a custom evaluator. + :type llm_provider: LLMObsCustomEvalConfigLLMProvider, optional + + :param target: Target application configuration for a custom evaluator. + :type target: LLMObsCustomEvalConfigTarget, optional + + :param updated_at: Timestamp when the evaluator configuration was last updated. + :type updated_at: datetime + """ + if category is not unset: + kwargs["category"] = category + if created_by is not unset: + kwargs["created_by"] = created_by + if last_updated_by is not unset: + kwargs["last_updated_by"] = last_updated_by + if llm_judge_config is not unset: + kwargs["llm_judge_config"] = llm_judge_config + if llm_provider is not unset: + kwargs["llm_provider"] = llm_provider + if target is not unset: + kwargs["target"] = target + super().__init__(kwargs) + + self_.created_at = created_at + self_.eval_name = eval_name + self_.updated_at = updated_at diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_bedrock_options.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_bedrock_options.py new file mode 100644 index 0000000000..2c44cd8000 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_bedrock_options.py @@ -0,0 +1,36 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LLMObsCustomEvalConfigBedrockOptions(ModelNormal): + @cached_property + def openapi_types(_): + return { + "region": (str,), + } + + attribute_map = { + "region": "region", + } + + def __init__(self_, region: Union[str, UnsetType] = unset, **kwargs): + """ + AWS Bedrock-specific options for LLM provider configuration. + + :param region: AWS region for Bedrock. + :type region: str, optional + """ + if region is not unset: + kwargs["region"] = region + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_data.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_data.py new file mode 100644 index 0000000000..ca3c3efe55 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_data.py @@ -0,0 +1,56 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_attributes import LLMObsCustomEvalConfigAttributes + from datadog_api_client.v2.model.llm_obs_custom_eval_config_type import LLMObsCustomEvalConfigType + + +class LLMObsCustomEvalConfigData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_attributes import LLMObsCustomEvalConfigAttributes + from datadog_api_client.v2.model.llm_obs_custom_eval_config_type import LLMObsCustomEvalConfigType + + return { + "attributes": (LLMObsCustomEvalConfigAttributes,), + "id": (str,), + "type": (LLMObsCustomEvalConfigType,), + } + + attribute_map = { + "attributes": "attributes", + "id": "id", + "type": "type", + } + + def __init__( + self_, attributes: LLMObsCustomEvalConfigAttributes, id: str, type: LLMObsCustomEvalConfigType, **kwargs + ): + """ + Data object for a custom LLM Observability evaluator configuration. + + :param attributes: Attributes of a custom LLM Observability evaluator configuration. + :type attributes: LLMObsCustomEvalConfigAttributes + + :param id: Unique name identifier of the evaluator configuration. + :type id: str + + :param type: Type of the custom LLM Observability evaluator configuration resource. + :type type: LLMObsCustomEvalConfigType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_eval_scope.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_eval_scope.py new file mode 100644 index 0000000000..fcbe5cf913 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_eval_scope.py @@ -0,0 +1,41 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsCustomEvalConfigEvalScope(ModelSimple): + """ + Scope at which to evaluate spans. + + :param value: Must be one of ["span", "trace", "session"]. + :type value: str + """ + + allowed_values = { + "span", + "trace", + "session", + } + SPAN: ClassVar["LLMObsCustomEvalConfigEvalScope"] + TRACE: ClassVar["LLMObsCustomEvalConfigEvalScope"] + SESSION: ClassVar["LLMObsCustomEvalConfigEvalScope"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsCustomEvalConfigEvalScope.SPAN = LLMObsCustomEvalConfigEvalScope("span") +LLMObsCustomEvalConfigEvalScope.TRACE = LLMObsCustomEvalConfigEvalScope("trace") +LLMObsCustomEvalConfigEvalScope.SESSION = LLMObsCustomEvalConfigEvalScope("session") diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_inference_params.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_inference_params.py new file mode 100644 index 0000000000..204f30f5a0 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_inference_params.py @@ -0,0 +1,80 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LLMObsCustomEvalConfigInferenceParams(ModelNormal): + @cached_property + def openapi_types(_): + return { + "frequency_penalty": (float,), + "max_tokens": (int,), + "presence_penalty": (float,), + "temperature": (float,), + "top_k": (int,), + "top_p": (float,), + } + + attribute_map = { + "frequency_penalty": "frequency_penalty", + "max_tokens": "max_tokens", + "presence_penalty": "presence_penalty", + "temperature": "temperature", + "top_k": "top_k", + "top_p": "top_p", + } + + def __init__( + self_, + frequency_penalty: Union[float, UnsetType] = unset, + max_tokens: Union[int, UnsetType] = unset, + presence_penalty: Union[float, UnsetType] = unset, + temperature: Union[float, UnsetType] = unset, + top_k: Union[int, UnsetType] = unset, + top_p: Union[float, UnsetType] = unset, + **kwargs, + ): + """ + LLM inference parameters for a custom evaluator. + + :param frequency_penalty: Frequency penalty to reduce repetition. + :type frequency_penalty: float, optional + + :param max_tokens: Maximum number of tokens to generate. + :type max_tokens: int, optional + + :param presence_penalty: Presence penalty to reduce repetition. + :type presence_penalty: float, optional + + :param temperature: Sampling temperature for the LLM. + :type temperature: float, optional + + :param top_k: Top-k sampling parameter. + :type top_k: int, optional + + :param top_p: Top-p (nucleus) sampling parameter. + :type top_p: float, optional + """ + if frequency_penalty is not unset: + kwargs["frequency_penalty"] = frequency_penalty + if max_tokens is not unset: + kwargs["max_tokens"] = max_tokens + if presence_penalty is not unset: + kwargs["presence_penalty"] = presence_penalty + if temperature is not unset: + kwargs["temperature"] = temperature + if top_k is not unset: + kwargs["top_k"] = top_k + if top_p is not unset: + kwargs["top_p"] = top_p + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_integration_provider.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_integration_provider.py new file mode 100644 index 0000000000..0b8c063981 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_integration_provider.py @@ -0,0 +1,50 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsCustomEvalConfigIntegrationProvider(ModelSimple): + """ + Name of the LLM integration provider. + + :param value: Must be one of ["openai", "amazon-bedrock", "anthropic", "azure-openai", "vertex-ai", "llm-proxy"]. + :type value: str + """ + + allowed_values = { + "openai", + "amazon-bedrock", + "anthropic", + "azure-openai", + "vertex-ai", + "llm-proxy", + } + OPENAI: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] + AMAZON_BEDROCK: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] + ANTHROPIC: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] + AZURE_OPENAI: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] + VERTEX_AI: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] + LLM_PROXY: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsCustomEvalConfigIntegrationProvider.OPENAI = LLMObsCustomEvalConfigIntegrationProvider("openai") +LLMObsCustomEvalConfigIntegrationProvider.AMAZON_BEDROCK = LLMObsCustomEvalConfigIntegrationProvider("amazon-bedrock") +LLMObsCustomEvalConfigIntegrationProvider.ANTHROPIC = LLMObsCustomEvalConfigIntegrationProvider("anthropic") +LLMObsCustomEvalConfigIntegrationProvider.AZURE_OPENAI = LLMObsCustomEvalConfigIntegrationProvider("azure-openai") +LLMObsCustomEvalConfigIntegrationProvider.VERTEX_AI = LLMObsCustomEvalConfigIntegrationProvider("vertex-ai") +LLMObsCustomEvalConfigIntegrationProvider.LLM_PROXY = LLMObsCustomEvalConfigIntegrationProvider("llm-proxy") diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py new file mode 100644 index 0000000000..6d6b6fc885 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py @@ -0,0 +1,134 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Any, Dict, List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + date, + datetime, + none_type, + unset, + UnsetType, + UUID, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_assessment_criteria import ( + LLMObsCustomEvalConfigAssessmentCriteria, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_inference_params import ( + LLMObsCustomEvalConfigInferenceParams, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_parsing_type import LLMObsCustomEvalConfigParsingType + from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_message import ( + LLMObsCustomEvalConfigPromptMessage, + ) + + +class LLMObsCustomEvalConfigLLMJudgeConfig(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_assessment_criteria import ( + LLMObsCustomEvalConfigAssessmentCriteria, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_inference_params import ( + LLMObsCustomEvalConfigInferenceParams, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_parsing_type import ( + LLMObsCustomEvalConfigParsingType, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_message import ( + LLMObsCustomEvalConfigPromptMessage, + ) + + return { + "assessment_criteria": (LLMObsCustomEvalConfigAssessmentCriteria,), + "inference_params": (LLMObsCustomEvalConfigInferenceParams,), + "last_used_library_prompt_template_name": (str, none_type), + "modified_library_prompt_template": (bool, none_type), + "output_schema": ( + { + str: ( + bool, + date, + datetime, + dict, + float, + int, + list, + str, + UUID, + none_type, + ) + }, + none_type, + ), + "parsing_type": (LLMObsCustomEvalConfigParsingType,), + "prompt_template": ([LLMObsCustomEvalConfigPromptMessage],), + } + + attribute_map = { + "assessment_criteria": "assessment_criteria", + "inference_params": "inference_params", + "last_used_library_prompt_template_name": "last_used_library_prompt_template_name", + "modified_library_prompt_template": "modified_library_prompt_template", + "output_schema": "output_schema", + "parsing_type": "parsing_type", + "prompt_template": "prompt_template", + } + + def __init__( + self_, + inference_params: LLMObsCustomEvalConfigInferenceParams, + assessment_criteria: Union[LLMObsCustomEvalConfigAssessmentCriteria, UnsetType] = unset, + last_used_library_prompt_template_name: Union[str, none_type, UnsetType] = unset, + modified_library_prompt_template: Union[bool, none_type, UnsetType] = unset, + output_schema: Union[Dict[str, Any], none_type, UnsetType] = unset, + parsing_type: Union[LLMObsCustomEvalConfigParsingType, UnsetType] = unset, + prompt_template: Union[List[LLMObsCustomEvalConfigPromptMessage], UnsetType] = unset, + **kwargs, + ): + """ + LLM judge configuration for a custom evaluator. + + :param assessment_criteria: Criteria used to assess the pass/fail result of a custom evaluator. + :type assessment_criteria: LLMObsCustomEvalConfigAssessmentCriteria, optional + + :param inference_params: LLM inference parameters for a custom evaluator. + :type inference_params: LLMObsCustomEvalConfigInferenceParams + + :param last_used_library_prompt_template_name: Name of the last library prompt template used. + :type last_used_library_prompt_template_name: str, none_type, optional + + :param modified_library_prompt_template: Whether the library prompt template was modified. + :type modified_library_prompt_template: bool, none_type, optional + + :param output_schema: JSON schema describing the expected output format of the LLM judge. + :type output_schema: {str: (bool, date, datetime, dict, float, int, list, str, UUID, none_type,)}, none_type, optional + + :param parsing_type: Output parsing type for a custom LLM judge evaluator. + :type parsing_type: LLMObsCustomEvalConfigParsingType, optional + + :param prompt_template: List of messages forming the LLM judge prompt template. + :type prompt_template: [LLMObsCustomEvalConfigPromptMessage], optional + """ + if assessment_criteria is not unset: + kwargs["assessment_criteria"] = assessment_criteria + if last_used_library_prompt_template_name is not unset: + kwargs["last_used_library_prompt_template_name"] = last_used_library_prompt_template_name + if modified_library_prompt_template is not unset: + kwargs["modified_library_prompt_template"] = modified_library_prompt_template + if output_schema is not unset: + kwargs["output_schema"] = output_schema + if parsing_type is not unset: + kwargs["parsing_type"] = parsing_type + if prompt_template is not unset: + kwargs["prompt_template"] = prompt_template + super().__init__(kwargs) + + self_.inference_params = inference_params diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_provider.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_provider.py new file mode 100644 index 0000000000..b6f59d1be8 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_provider.py @@ -0,0 +1,94 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_bedrock_options import ( + LLMObsCustomEvalConfigBedrockOptions, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_integration_provider import ( + LLMObsCustomEvalConfigIntegrationProvider, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_vertex_ai_options import ( + LLMObsCustomEvalConfigVertexAIOptions, + ) + + +class LLMObsCustomEvalConfigLLMProvider(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_bedrock_options import ( + LLMObsCustomEvalConfigBedrockOptions, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_integration_provider import ( + LLMObsCustomEvalConfigIntegrationProvider, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_vertex_ai_options import ( + LLMObsCustomEvalConfigVertexAIOptions, + ) + + return { + "bedrock": (LLMObsCustomEvalConfigBedrockOptions,), + "integration_account_id": (str,), + "integration_provider": (LLMObsCustomEvalConfigIntegrationProvider,), + "model_name": (str,), + "vertex_ai": (LLMObsCustomEvalConfigVertexAIOptions,), + } + + attribute_map = { + "bedrock": "bedrock", + "integration_account_id": "integration_account_id", + "integration_provider": "integration_provider", + "model_name": "model_name", + "vertex_ai": "vertex_ai", + } + + def __init__( + self_, + bedrock: Union[LLMObsCustomEvalConfigBedrockOptions, UnsetType] = unset, + integration_account_id: Union[str, UnsetType] = unset, + integration_provider: Union[LLMObsCustomEvalConfigIntegrationProvider, UnsetType] = unset, + model_name: Union[str, UnsetType] = unset, + vertex_ai: Union[LLMObsCustomEvalConfigVertexAIOptions, UnsetType] = unset, + **kwargs, + ): + """ + LLM provider configuration for a custom evaluator. + + :param bedrock: AWS Bedrock-specific options for LLM provider configuration. + :type bedrock: LLMObsCustomEvalConfigBedrockOptions, optional + + :param integration_account_id: Integration account identifier. + :type integration_account_id: str, optional + + :param integration_provider: Name of the LLM integration provider. + :type integration_provider: LLMObsCustomEvalConfigIntegrationProvider, optional + + :param model_name: Name of the LLM model. + :type model_name: str, optional + + :param vertex_ai: Google Vertex AI-specific options for LLM provider configuration. + :type vertex_ai: LLMObsCustomEvalConfigVertexAIOptions, optional + """ + if bedrock is not unset: + kwargs["bedrock"] = bedrock + if integration_account_id is not unset: + kwargs["integration_account_id"] = integration_account_id + if integration_provider is not unset: + kwargs["integration_provider"] = integration_provider + if model_name is not unset: + kwargs["model_name"] = model_name + if vertex_ai is not unset: + kwargs["vertex_ai"] = vertex_ai + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py new file mode 100644 index 0000000000..b913e071ba --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py @@ -0,0 +1,38 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsCustomEvalConfigParsingType(ModelSimple): + """ + Output parsing type for a custom LLM judge evaluator. + + :param value: Must be one of ["structured_output", "json"]. + :type value: str + """ + + allowed_values = { + "structured_output", + "json", + } + STRUCTURED_OUTPUT: ClassVar["LLMObsCustomEvalConfigParsingType"] + JSON: ClassVar["LLMObsCustomEvalConfigParsingType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsCustomEvalConfigParsingType.STRUCTURED_OUTPUT = LLMObsCustomEvalConfigParsingType("structured_output") +LLMObsCustomEvalConfigParsingType.JSON = LLMObsCustomEvalConfigParsingType("json") diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_content.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_content.py new file mode 100644 index 0000000000..1c633b59fc --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_content.py @@ -0,0 +1,50 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content_value import ( + LLMObsCustomEvalConfigPromptContentValue, + ) + + +class LLMObsCustomEvalConfigPromptContent(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content_value import ( + LLMObsCustomEvalConfigPromptContentValue, + ) + + return { + "type": (str,), + "value": (LLMObsCustomEvalConfigPromptContentValue,), + } + + attribute_map = { + "type": "type", + "value": "value", + } + + def __init__(self_, type: str, value: LLMObsCustomEvalConfigPromptContentValue, **kwargs): + """ + A content block within a prompt message. + + :param type: Content block type. + :type type: str + + :param value: Value of a prompt message content block. + :type value: LLMObsCustomEvalConfigPromptContentValue + """ + super().__init__(kwargs) + + self_.type = type + self_.value = value diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_content_value.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_content_value.py new file mode 100644 index 0000000000..a80ddc7c25 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_content_value.py @@ -0,0 +1,72 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_call import ( + LLMObsCustomEvalConfigPromptToolCall, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_result import ( + LLMObsCustomEvalConfigPromptToolResult, + ) + + +class LLMObsCustomEvalConfigPromptContentValue(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_call import ( + LLMObsCustomEvalConfigPromptToolCall, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_result import ( + LLMObsCustomEvalConfigPromptToolResult, + ) + + return { + "text": (str,), + "tool_call": (LLMObsCustomEvalConfigPromptToolCall,), + "tool_call_result": (LLMObsCustomEvalConfigPromptToolResult,), + } + + attribute_map = { + "text": "text", + "tool_call": "tool_call", + "tool_call_result": "tool_call_result", + } + + def __init__( + self_, + text: Union[str, UnsetType] = unset, + tool_call: Union[LLMObsCustomEvalConfigPromptToolCall, UnsetType] = unset, + tool_call_result: Union[LLMObsCustomEvalConfigPromptToolResult, UnsetType] = unset, + **kwargs, + ): + """ + Value of a prompt message content block. + + :param text: Text content of the message block. + :type text: str, optional + + :param tool_call: A tool call within a prompt message. + :type tool_call: LLMObsCustomEvalConfigPromptToolCall, optional + + :param tool_call_result: A tool call result within a prompt message. + :type tool_call_result: LLMObsCustomEvalConfigPromptToolResult, optional + """ + if text is not unset: + kwargs["text"] = text + if tool_call is not unset: + kwargs["tool_call"] = tool_call + if tool_call_result is not unset: + kwargs["tool_call_result"] = tool_call_result + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_message.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_message.py new file mode 100644 index 0000000000..a59f12a9c6 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_message.py @@ -0,0 +1,66 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content import ( + LLMObsCustomEvalConfigPromptContent, + ) + + +class LLMObsCustomEvalConfigPromptMessage(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content import ( + LLMObsCustomEvalConfigPromptContent, + ) + + return { + "content": (str,), + "contents": ([LLMObsCustomEvalConfigPromptContent],), + "role": (str,), + } + + attribute_map = { + "content": "content", + "contents": "contents", + "role": "role", + } + + def __init__( + self_, + role: str, + content: Union[str, UnsetType] = unset, + contents: Union[List[LLMObsCustomEvalConfigPromptContent], UnsetType] = unset, + **kwargs, + ): + """ + A message in the prompt template for a custom LLM judge evaluator. + + :param content: Text content of the message. + :type content: str, optional + + :param contents: Multi-part content blocks for the message. + :type contents: [LLMObsCustomEvalConfigPromptContent], optional + + :param role: Role of the message author. + :type role: str + """ + if content is not unset: + kwargs["content"] = content + if contents is not unset: + kwargs["contents"] = contents + super().__init__(kwargs) + + self_.role = role diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_tool_call.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_tool_call.py new file mode 100644 index 0000000000..a92e908586 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_tool_call.py @@ -0,0 +1,64 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LLMObsCustomEvalConfigPromptToolCall(ModelNormal): + @cached_property + def openapi_types(_): + return { + "arguments": (str,), + "id": (str,), + "name": (str,), + "type": (str,), + } + + attribute_map = { + "arguments": "arguments", + "id": "id", + "name": "name", + "type": "type", + } + + def __init__( + self_, + arguments: Union[str, UnsetType] = unset, + id: Union[str, UnsetType] = unset, + name: Union[str, UnsetType] = unset, + type: Union[str, UnsetType] = unset, + **kwargs, + ): + """ + A tool call within a prompt message. + + :param arguments: JSON-encoded arguments for the tool call. + :type arguments: str, optional + + :param id: Unique identifier of the tool call. + :type id: str, optional + + :param name: Name of the tool being called. + :type name: str, optional + + :param type: Type of the tool call. + :type type: str, optional + """ + if arguments is not unset: + kwargs["arguments"] = arguments + if id is not unset: + kwargs["id"] = id + if name is not unset: + kwargs["name"] = name + if type is not unset: + kwargs["type"] = type + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_tool_result.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_tool_result.py new file mode 100644 index 0000000000..b531e62e2d --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_prompt_tool_result.py @@ -0,0 +1,64 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LLMObsCustomEvalConfigPromptToolResult(ModelNormal): + @cached_property + def openapi_types(_): + return { + "name": (str,), + "result": (str,), + "tool_id": (str,), + "type": (str,), + } + + attribute_map = { + "name": "name", + "result": "result", + "tool_id": "tool_id", + "type": "type", + } + + def __init__( + self_, + name: Union[str, UnsetType] = unset, + result: Union[str, UnsetType] = unset, + tool_id: Union[str, UnsetType] = unset, + type: Union[str, UnsetType] = unset, + **kwargs, + ): + """ + A tool call result within a prompt message. + + :param name: Name of the tool that produced this result. + :type name: str, optional + + :param result: The result returned by the tool. + :type result: str, optional + + :param tool_id: Identifier of the tool call this result corresponds to. + :type tool_id: str, optional + + :param type: Type of the tool result. + :type type: str, optional + """ + if name is not unset: + kwargs["name"] = name + if result is not unset: + kwargs["result"] = result + if tool_id is not unset: + kwargs["tool_id"] = tool_id + if type is not unset: + kwargs["type"] = type + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_response.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_response.py new file mode 100644 index 0000000000..0116ae1f88 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_response.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_data import LLMObsCustomEvalConfigData + + +class LLMObsCustomEvalConfigResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_data import LLMObsCustomEvalConfigData + + return { + "data": (LLMObsCustomEvalConfigData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsCustomEvalConfigData, **kwargs): + """ + Response containing a custom LLM Observability evaluator configuration. + + :param data: Data object for a custom LLM Observability evaluator configuration. + :type data: LLMObsCustomEvalConfigData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py new file mode 100644 index 0000000000..42b2d5b6ef --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py @@ -0,0 +1,86 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + none_type, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_eval_scope import LLMObsCustomEvalConfigEvalScope + + +class LLMObsCustomEvalConfigTarget(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_eval_scope import LLMObsCustomEvalConfigEvalScope + + return { + "application_name": (str,), + "enabled": (bool,), + "eval_scope": (LLMObsCustomEvalConfigEvalScope,), + "filter": (str, none_type), + "root_spans_only": (bool, none_type), + "sampling_percentage": (float, none_type), + } + + attribute_map = { + "application_name": "application_name", + "enabled": "enabled", + "eval_scope": "eval_scope", + "filter": "filter", + "root_spans_only": "root_spans_only", + "sampling_percentage": "sampling_percentage", + } + + def __init__( + self_, + application_name: str, + enabled: bool, + eval_scope: Union[LLMObsCustomEvalConfigEvalScope, UnsetType] = unset, + filter: Union[str, none_type, UnsetType] = unset, + root_spans_only: Union[bool, none_type, UnsetType] = unset, + sampling_percentage: Union[float, none_type, UnsetType] = unset, + **kwargs, + ): + """ + Target application configuration for a custom evaluator. + + :param application_name: Name of the ML application this evaluator targets. + :type application_name: str + + :param enabled: Whether the evaluator is active for the target application. + :type enabled: bool + + :param eval_scope: Scope at which to evaluate spans. + :type eval_scope: LLMObsCustomEvalConfigEvalScope, optional + + :param filter: Filter expression to select which spans to evaluate. + :type filter: str, none_type, optional + + :param root_spans_only: When true, only root spans are evaluated. + :type root_spans_only: bool, none_type, optional + + :param sampling_percentage: Percentage of traces to evaluate. Must be greater than 0 and at most 100. + :type sampling_percentage: float, none_type, optional + """ + if eval_scope is not unset: + kwargs["eval_scope"] = eval_scope + if filter is not unset: + kwargs["filter"] = filter + if root_spans_only is not unset: + kwargs["root_spans_only"] = root_spans_only + if sampling_percentage is not unset: + kwargs["sampling_percentage"] = sampling_percentage + super().__init__(kwargs) + + self_.application_name = application_name + self_.enabled = enabled diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_type.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_type.py new file mode 100644 index 0000000000..822d6125d2 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_type.py @@ -0,0 +1,35 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsCustomEvalConfigType(ModelSimple): + """ + Type of the custom LLM Observability evaluator configuration resource. + + :param value: If omitted defaults to "evaluator_config". Must be one of ["evaluator_config"]. + :type value: str + """ + + allowed_values = { + "evaluator_config", + } + EVALUATOR_CONFIG: ClassVar["LLMObsCustomEvalConfigType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsCustomEvalConfigType.EVALUATOR_CONFIG = LLMObsCustomEvalConfigType("evaluator_config") diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_update_attributes.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_update_attributes.py new file mode 100644 index 0000000000..01a77b7260 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_update_attributes.py @@ -0,0 +1,88 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_judge_config import ( + LLMObsCustomEvalConfigLLMJudgeConfig, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_provider import LLMObsCustomEvalConfigLLMProvider + from datadog_api_client.v2.model.llm_obs_custom_eval_config_target import LLMObsCustomEvalConfigTarget + + +class LLMObsCustomEvalConfigUpdateAttributes(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_judge_config import ( + LLMObsCustomEvalConfigLLMJudgeConfig, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_provider import ( + LLMObsCustomEvalConfigLLMProvider, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_target import LLMObsCustomEvalConfigTarget + + return { + "category": (str,), + "eval_name": (str,), + "llm_judge_config": (LLMObsCustomEvalConfigLLMJudgeConfig,), + "llm_provider": (LLMObsCustomEvalConfigLLMProvider,), + "target": (LLMObsCustomEvalConfigTarget,), + } + + attribute_map = { + "category": "category", + "eval_name": "eval_name", + "llm_judge_config": "llm_judge_config", + "llm_provider": "llm_provider", + "target": "target", + } + + def __init__( + self_, + target: LLMObsCustomEvalConfigTarget, + category: Union[str, UnsetType] = unset, + eval_name: Union[str, UnsetType] = unset, + llm_judge_config: Union[LLMObsCustomEvalConfigLLMJudgeConfig, UnsetType] = unset, + llm_provider: Union[LLMObsCustomEvalConfigLLMProvider, UnsetType] = unset, + **kwargs, + ): + """ + Attributes for creating or updating a custom LLM Observability evaluator configuration. + + :param category: Category of the evaluator. + :type category: str, optional + + :param eval_name: Name of the custom evaluator. If provided, must match the eval_name path parameter. + :type eval_name: str, optional + + :param llm_judge_config: LLM judge configuration for a custom evaluator. + :type llm_judge_config: LLMObsCustomEvalConfigLLMJudgeConfig, optional + + :param llm_provider: LLM provider configuration for a custom evaluator. + :type llm_provider: LLMObsCustomEvalConfigLLMProvider, optional + + :param target: Target application configuration for a custom evaluator. + :type target: LLMObsCustomEvalConfigTarget + """ + if category is not unset: + kwargs["category"] = category + if eval_name is not unset: + kwargs["eval_name"] = eval_name + if llm_judge_config is not unset: + kwargs["llm_judge_config"] = llm_judge_config + if llm_provider is not unset: + kwargs["llm_provider"] = llm_provider + super().__init__(kwargs) + + self_.target = target diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_update_data.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_update_data.py new file mode 100644 index 0000000000..77cc793b83 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_update_data.py @@ -0,0 +1,67 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_attributes import ( + LLMObsCustomEvalConfigUpdateAttributes, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_type import LLMObsCustomEvalConfigType + + +class LLMObsCustomEvalConfigUpdateData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_attributes import ( + LLMObsCustomEvalConfigUpdateAttributes, + ) + from datadog_api_client.v2.model.llm_obs_custom_eval_config_type import LLMObsCustomEvalConfigType + + return { + "attributes": (LLMObsCustomEvalConfigUpdateAttributes,), + "id": (str,), + "type": (LLMObsCustomEvalConfigType,), + } + + attribute_map = { + "attributes": "attributes", + "id": "id", + "type": "type", + } + + def __init__( + self_, + attributes: LLMObsCustomEvalConfigUpdateAttributes, + type: LLMObsCustomEvalConfigType, + id: Union[str, UnsetType] = unset, + **kwargs, + ): + """ + Data object for creating or updating a custom LLM Observability evaluator configuration. + + :param attributes: Attributes for creating or updating a custom LLM Observability evaluator configuration. + :type attributes: LLMObsCustomEvalConfigUpdateAttributes + + :param id: Name of the evaluator. If provided, must match the eval_name path parameter. + :type id: str, optional + + :param type: Type of the custom LLM Observability evaluator configuration resource. + :type type: LLMObsCustomEvalConfigType + """ + if id is not unset: + kwargs["id"] = id + super().__init__(kwargs) + + self_.attributes = attributes + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_update_request.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_update_request.py new file mode 100644 index 0000000000..deed47dcd9 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_update_request.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_data import LLMObsCustomEvalConfigUpdateData + + +class LLMObsCustomEvalConfigUpdateRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_data import LLMObsCustomEvalConfigUpdateData + + return { + "data": (LLMObsCustomEvalConfigUpdateData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsCustomEvalConfigUpdateData, **kwargs): + """ + Request to create or update a custom LLM Observability evaluator configuration. + + :param data: Data object for creating or updating a custom LLM Observability evaluator configuration. + :type data: LLMObsCustomEvalConfigUpdateData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_user.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_user.py new file mode 100644 index 0000000000..704c66b16f --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_user.py @@ -0,0 +1,36 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LLMObsCustomEvalConfigUser(ModelNormal): + @cached_property + def openapi_types(_): + return { + "email": (str,), + } + + attribute_map = { + "email": "email", + } + + def __init__(self_, email: Union[str, UnsetType] = unset, **kwargs): + """ + A Datadog user associated with a custom evaluator configuration. + + :param email: Email address of the user. + :type email: str, optional + """ + if email is not unset: + kwargs["email"] = email + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_vertex_ai_options.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_vertex_ai_options.py new file mode 100644 index 0000000000..3dc9a09dbb --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_vertex_ai_options.py @@ -0,0 +1,43 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LLMObsCustomEvalConfigVertexAIOptions(ModelNormal): + @cached_property + def openapi_types(_): + return { + "location": (str,), + "project": (str,), + } + + attribute_map = { + "location": "location", + "project": "project", + } + + def __init__(self_, location: Union[str, UnsetType] = unset, project: Union[str, UnsetType] = unset, **kwargs): + """ + Google Vertex AI-specific options for LLM provider configuration. + + :param location: Google Cloud region. + :type location: str, optional + + :param project: Google Cloud project ID. + :type project: str, optional + """ + if location is not unset: + kwargs["location"] = location + if project is not unset: + kwargs["project"] = project + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/models/__init__.py b/src/datadog_api_client/v2/models/__init__.py index e2112dcd0f..583df310f7 100644 --- a/src/datadog_api_client/v2/models/__init__.py +++ b/src/datadog_api_client/v2/models/__init__.py @@ -3024,6 +3024,43 @@ from datadog_api_client.v2.model.llm_obs_annotation_queue_update_request import LLMObsAnnotationQueueUpdateRequest from datadog_api_client.v2.model.llm_obs_annotation_queues_response import LLMObsAnnotationQueuesResponse from datadog_api_client.v2.model.llm_obs_cursor_meta import LLMObsCursorMeta +from datadog_api_client.v2.model.llm_obs_custom_eval_config_assessment_criteria import ( + LLMObsCustomEvalConfigAssessmentCriteria, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_attributes import LLMObsCustomEvalConfigAttributes +from datadog_api_client.v2.model.llm_obs_custom_eval_config_bedrock_options import LLMObsCustomEvalConfigBedrockOptions +from datadog_api_client.v2.model.llm_obs_custom_eval_config_data import LLMObsCustomEvalConfigData +from datadog_api_client.v2.model.llm_obs_custom_eval_config_eval_scope import LLMObsCustomEvalConfigEvalScope +from datadog_api_client.v2.model.llm_obs_custom_eval_config_inference_params import ( + LLMObsCustomEvalConfigInferenceParams, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_integration_provider import ( + LLMObsCustomEvalConfigIntegrationProvider, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_judge_config import LLMObsCustomEvalConfigLLMJudgeConfig +from datadog_api_client.v2.model.llm_obs_custom_eval_config_llm_provider import LLMObsCustomEvalConfigLLMProvider +from datadog_api_client.v2.model.llm_obs_custom_eval_config_parsing_type import LLMObsCustomEvalConfigParsingType +from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content import LLMObsCustomEvalConfigPromptContent +from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_content_value import ( + LLMObsCustomEvalConfigPromptContentValue, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_message import LLMObsCustomEvalConfigPromptMessage +from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_call import LLMObsCustomEvalConfigPromptToolCall +from datadog_api_client.v2.model.llm_obs_custom_eval_config_prompt_tool_result import ( + LLMObsCustomEvalConfigPromptToolResult, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_response import LLMObsCustomEvalConfigResponse +from datadog_api_client.v2.model.llm_obs_custom_eval_config_target import LLMObsCustomEvalConfigTarget +from datadog_api_client.v2.model.llm_obs_custom_eval_config_type import LLMObsCustomEvalConfigType +from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_attributes import ( + LLMObsCustomEvalConfigUpdateAttributes, +) +from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_data import LLMObsCustomEvalConfigUpdateData +from datadog_api_client.v2.model.llm_obs_custom_eval_config_update_request import LLMObsCustomEvalConfigUpdateRequest +from datadog_api_client.v2.model.llm_obs_custom_eval_config_user import LLMObsCustomEvalConfigUser +from datadog_api_client.v2.model.llm_obs_custom_eval_config_vertex_ai_options import ( + LLMObsCustomEvalConfigVertexAIOptions, +) from datadog_api_client.v2.model.llm_obs_dataset_data_attributes_request import LLMObsDatasetDataAttributesRequest from datadog_api_client.v2.model.llm_obs_dataset_data_attributes_response import LLMObsDatasetDataAttributesResponse from datadog_api_client.v2.model.llm_obs_dataset_data_request import LLMObsDatasetDataRequest @@ -9400,6 +9437,29 @@ "LLMObsAnnotationQueueUpdateRequest", "LLMObsAnnotationQueuesResponse", "LLMObsCursorMeta", + "LLMObsCustomEvalConfigAssessmentCriteria", + "LLMObsCustomEvalConfigAttributes", + "LLMObsCustomEvalConfigBedrockOptions", + "LLMObsCustomEvalConfigData", + "LLMObsCustomEvalConfigEvalScope", + "LLMObsCustomEvalConfigInferenceParams", + "LLMObsCustomEvalConfigIntegrationProvider", + "LLMObsCustomEvalConfigLLMJudgeConfig", + "LLMObsCustomEvalConfigLLMProvider", + "LLMObsCustomEvalConfigParsingType", + "LLMObsCustomEvalConfigPromptContent", + "LLMObsCustomEvalConfigPromptContentValue", + "LLMObsCustomEvalConfigPromptMessage", + "LLMObsCustomEvalConfigPromptToolCall", + "LLMObsCustomEvalConfigPromptToolResult", + "LLMObsCustomEvalConfigResponse", + "LLMObsCustomEvalConfigTarget", + "LLMObsCustomEvalConfigType", + "LLMObsCustomEvalConfigUpdateAttributes", + "LLMObsCustomEvalConfigUpdateData", + "LLMObsCustomEvalConfigUpdateRequest", + "LLMObsCustomEvalConfigUser", + "LLMObsCustomEvalConfigVertexAIOptions", "LLMObsDatasetDataAttributesRequest", "LLMObsDatasetDataAttributesResponse", "LLMObsDatasetDataRequest", diff --git a/tests/v2/features/llm_observability.feature b/tests/v2/features/llm_observability.feature index a390628998..dc0a15364b 100644 --- a/tests/v2/features/llm_observability.feature +++ b/tests/v2/features/llm_observability.feature @@ -175,6 +175,42 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability + Scenario: Create or update a custom evaluator configuration returns "Bad Request" response + Given operation "UpdateLLMObsCustomEvalConfig" enabled + And new "UpdateLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Create or update a custom evaluator configuration returns "Not Found" response + Given operation "UpdateLLMObsCustomEvalConfig" enabled + And new "UpdateLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Create or update a custom evaluator configuration returns "OK" response + Given operation "UpdateLLMObsCustomEvalConfig" enabled + And new "UpdateLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + When the request is sent + Then the response status is 200 OK + + @generated @skip @team:DataDog/ml-observability + Scenario: Create or update a custom evaluator configuration returns "Unprocessable Entity" response + Given operation "UpdateLLMObsCustomEvalConfig" enabled + And new "UpdateLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} + When the request is sent + Then the response status is 422 Unprocessable Entity + @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability dataset records returns "Bad Request" response Given operation "DeleteLLMObsDatasetRecords" enabled @@ -264,6 +300,30 @@ Feature: LLM Observability When the request is sent Then the response status is 204 No Content + @generated @skip @team:DataDog/ml-observability + Scenario: Delete a custom evaluator configuration returns "Bad Request" response + Given operation "DeleteLLMObsCustomEvalConfig" enabled + And new "DeleteLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Delete a custom evaluator configuration returns "No Content" response + Given operation "DeleteLLMObsCustomEvalConfig" enabled + And new "DeleteLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 204 No Content + + @generated @skip @team:DataDog/ml-observability + Scenario: Delete a custom evaluator configuration returns "Not Found" response + Given operation "DeleteLLMObsCustomEvalConfig" enabled + And new "DeleteLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + @generated @skip @team:DataDog/ml-observability Scenario: Delete an LLM Observability annotation queue returns "No Content" response Given operation "DeleteLLMObsAnnotationQueue" enabled @@ -307,6 +367,30 @@ Feature: LLM Observability When the request is sent Then the response status is 404 Not Found + @generated @skip @team:DataDog/ml-observability + Scenario: Get a custom evaluator configuration returns "Bad Request" response + Given operation "GetLLMObsCustomEvalConfig" enabled + And new "GetLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 400 Bad Request + + @generated @skip @team:DataDog/ml-observability + Scenario: Get a custom evaluator configuration returns "Not Found" response + Given operation "GetLLMObsCustomEvalConfig" enabled + And new "GetLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 404 Not Found + + @generated @skip @team:DataDog/ml-observability + Scenario: Get a custom evaluator configuration returns "OK" response + Given operation "GetLLMObsCustomEvalConfig" enabled + And new "GetLLMObsCustomEvalConfig" request + And request contains "eval_name" parameter from "REPLACE.ME" + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Get annotated queue interactions returns "Bad Request" response Given operation "GetLLMObsAnnotatedInteractions" enabled diff --git a/tests/v2/features/undo.json b/tests/v2/features/undo.json index 21caf3f329..774684b948 100644 --- a/tests/v2/features/undo.json +++ b/tests/v2/features/undo.json @@ -111,6 +111,31 @@ "type": "unsafe" } }, + "DeleteLLMObsCustomEvalConfig": { + "tag": "LLM Observability", + "undo": { + "operationId": "UpdateLLMObsCustomEvalConfig", + "parameters": [ + { + "name": "eval_name", + "source": "path" + } + ], + "type": "unsafe" + } + }, + "GetLLMObsCustomEvalConfig": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, + "UpdateLLMObsCustomEvalConfig": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, "ListDatastores": { "tag": "Actions Datastores", "undo": {