From 45d7ab289070083312b251fb83bb786454bba3b8 Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Mon, 20 Jul 2026 15:13:42 -0700 Subject: [PATCH] feat: GenAI Client(evals) - add create EvaluationExperiment PiperOrigin-RevId: 951078795 --- agentplatform/_genai/evals.py | 632 ++++++++++++++++++- agentplatform/_genai/types/__init__.py | 38 +- agentplatform/_genai/types/common.py | 363 ++++++++--- tests/unit/agentplatform/genai/test_evals.py | 140 ++++ vertexai/_genai/evals.py | 624 +++++++++++++++++- vertexai/_genai/types/__init__.py | 30 + vertexai/_genai/types/common.py | 207 ++++++ 7 files changed, 1950 insertions(+), 84 deletions(-) diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 1f4aa7e1de..0553eafabc 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -43,6 +43,29 @@ logger = logging.getLogger("agentplatform_genai.evals") +def _CreateEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["display_name"]) is not None: + setv(to_object, ["displayName"], getv(from_object, ["display_name"])) + + if getv(from_object, ["labels"]) is not None: + setv(to_object, ["labels"], getv(from_object, ["labels"])) + + if getv(from_object, ["merge_strategy"]) is not None: + setv(to_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"])) + + if getv(from_object, ["metadata"]) is not None: + setv(to_object, ["metadata"], getv(from_object, ["metadata"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _CreateEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -733,6 +756,20 @@ def _GenerateUserScenariosParameters_to_vertex( return to_object +def _GetEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _GetEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -793,6 +830,44 @@ def _GetEvaluationSetParameters_to_vertex( return to_object +def _ListEvaluationExperimentsConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["page_size"]) is not None: + setv(parent_object, ["_query", "pageSize"], getv(from_object, ["page_size"])) + + if getv(from_object, ["page_token"]) is not None: + setv(parent_object, ["_query", "pageToken"], getv(from_object, ["page_token"])) + + if getv(from_object, ["filter"]) is not None: + setv(parent_object, ["_query", "filter"], getv(from_object, ["filter"])) + + if getv(from_object, ["order_by"]) is not None: + setv(parent_object, ["_query", "orderBy"], getv(from_object, ["order_by"])) + + return to_object + + +def _ListEvaluationExperimentsParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["config"]) is not None: + setv( + to_object, + ["config"], + _ListEvaluationExperimentsConfig_to_vertex( + getv(from_object, ["config"]), to_object + ), + ) + + return to_object + + def _ListEvaluationMetricsConfig_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1018,6 +1093,86 @@ def _UnifiedMetric_to_vertex( class Evals(_api_module.BaseModule): + def _create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Creates an EvaluationExperiment. + """ + + parameter_model = types._CreateEvaluationExperimentParameters( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("post", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _create_evaluation_item( self, *, @@ -1754,6 +1909,78 @@ def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + def _get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _get_evaluation_metric( self, *, @@ -2015,7 +2242,77 @@ def _get_evaluation_item( response_dict = {} if not response.body else json.loads(response.body) - return_value = types.EvaluationItem._from_response( + return_value = types.EvaluationItem._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( response=response_dict, kwargs=( { @@ -3172,9 +3469,196 @@ def delete_evaluation_metric( config=config, ) + @_common.experimental_warning( + "The Vertex SDK GenAI evals.get_evaluation_experiment method is" + " experimental, and may change in future versions." + ) + def get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Retrieves an EvaluationExperiment from the resource name. + + Args: + name: The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: The optional configuration for the get operation. + + Returns: + The evaluation experiment. + + Raises: + ValueError: If the name is empty. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + return self._get_evaluation_experiment(name=name, config=config) + + @_common.experimental_warning( + "The Vertex SDK GenAI evals.list_evaluation_experiments method is" + " experimental, and may change in future versions." + ) + def list_evaluation_experiments( + self, + *, + filter: Optional[str] = None, + order_by: Optional[str] = None, + config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None, + ) -> types.ListEvaluationExperimentsResponse: + """Lists EvaluationExperiments. + + Args: + filter: An expression for filtering the results of the request. For + field names both snake_case and camelCase are supported. For more + information about filter syntax, see + `AIP-160 `_. + Example: ``'display_name="my_experiment"'``. + order_by: A comma-separated list of fields to order by, sorted in + ascending order by default. Use ``desc`` after a field name for + descending. Example: ``"create_time desc"``. + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. Top-level ``filter`` and ``order_by`` arguments + take precedence over values set in ``config``. + + Returns: + The list evaluation experiments response. + """ + if config is None: + config = types.ListEvaluationExperimentsConfig() + if isinstance(config, dict): + config = types.ListEvaluationExperimentsConfig.model_validate(config) + if filter is not None: + config.filter = filter + if order_by is not None: + config.order_by = order_by + return self._list_evaluation_experiments( + config=config, + ) + + @_common.experimental_warning( + "The Vertex SDK GenAI evals.create_evaluation_experiment method is" + " experimental, and may change in future versions." + ) + def create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Creates an EvaluationExperiment. + + Args: + display_name: The display name of the evaluation experiment. + labels: Labels for the evaluation experiment. + merge_strategy: Merge strategy for the evaluation experiment. + metadata: Metadata about the evaluation experiment, can be used by the + caller to store additional tracking information about the experiment. + config: Optional configuration for the create operation. + + Returns: + The created evaluation experiment. + """ + return self._create_evaluation_experiment( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + class AsyncEvals(_api_module.BaseModule): + async def _create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Creates an EvaluationExperiment. + """ + + parameter_model = types._CreateEvaluationExperimentParameters( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "post", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _create_evaluation_item( self, *, @@ -3929,6 +4413,80 @@ async def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + async def _get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _get_evaluation_metric( self, *, @@ -4222,6 +4780,78 @@ async def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value + async def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _list_evaluation_metrics( self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None ) -> types.ListEvaluationMetricsResponse: diff --git a/agentplatform/_genai/types/__init__.py b/agentplatform/_genai/types/__init__.py index a731e74a0c..e7612f28d1 100644 --- a/agentplatform/_genai/types/__init__.py +++ b/agentplatform/_genai/types/__init__.py @@ -36,6 +36,7 @@ from .common import _CreateAgentEngineTaskRequestParameters from .common import _CreateDatasetParameters from .common import _CreateDatasetVersionParameters +from .common import _CreateEvaluationExperimentParameters from .common import _CreateEvaluationItemParameters from .common import _CreateEvaluationMetricParameters from .common import _CreateEvaluationRunParameters @@ -90,6 +91,7 @@ from .common import _GetDatasetParameters from .common import _GetDatasetVersionParameters from .common import _GetDeleteAgentEngineRuntimeRevisionOperationParameters +from .common import _GetEvaluationExperimentParameters from .common import _GetEvaluationItemParameters from .common import _GetEvaluationMetricParameters from .common import _GetEvaluationRunParameters @@ -126,6 +128,7 @@ from .common import _ListAgentEngineTasksRequestParameters from .common import _ListDatasetsRequestParameters from .common import _ListDatasetVersionsRequestParameters +from .common import _ListEvaluationExperimentsParameters from .common import _ListEvaluationMetricsParameters from .common import _ListMultimodalDatasetsRequestParameters from .common import _ListPublisherModelsRequestParameters @@ -340,6 +343,9 @@ from .common import CreateDatasetVersionConfig from .common import CreateDatasetVersionConfigDict from .common import CreateDatasetVersionConfigOrDict +from .common import CreateEvaluationExperimentConfig +from .common import CreateEvaluationExperimentConfigDict +from .common import CreateEvaluationExperimentConfigOrDict from .common import CreateEvaluationItemConfig from .common import CreateEvaluationItemConfigDict from .common import CreateEvaluationItemConfigOrDict @@ -718,6 +724,9 @@ from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfig from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigDict from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigOrDict +from .common import GetEvaluationExperimentConfig +from .common import GetEvaluationExperimentConfigDict +from .common import GetEvaluationExperimentConfigOrDict from .common import GetEvaluationItemConfig from .common import GetEvaluationItemConfigDict from .common import GetEvaluationItemConfigOrDict @@ -891,6 +900,12 @@ from .common import ListDeployableModelsConfig from .common import ListDeployableModelsConfigDict from .common import ListDeployableModelsConfigOrDict +from .common import ListEvaluationExperimentsConfig +from .common import ListEvaluationExperimentsConfigDict +from .common import ListEvaluationExperimentsConfigOrDict +from .common import ListEvaluationExperimentsResponse +from .common import ListEvaluationExperimentsResponseDict +from .common import ListEvaluationExperimentsResponseOrDict from .common import ListEvaluationMetricsConfig from .common import ListEvaluationMetricsConfigDict from .common import ListEvaluationMetricsConfigOrDict @@ -2002,6 +2017,12 @@ "ListAgentEngineTaskEventsResponse", "ListAgentEngineTaskEventsResponseDict", "ListAgentEngineTaskEventsResponseOrDict", + "CreateEvaluationExperimentConfig", + "CreateEvaluationExperimentConfigDict", + "CreateEvaluationExperimentConfigOrDict", + "EvaluationExperiment", + "EvaluationExperimentDict", + "EvaluationExperimentOrDict", "CreateEvaluationItemConfig", "CreateEvaluationItemConfigDict", "CreateEvaluationItemConfigOrDict", @@ -2323,6 +2344,9 @@ "GenerateInstanceRubricsResponse", "GenerateInstanceRubricsResponseDict", "GenerateInstanceRubricsResponseOrDict", + "GetEvaluationExperimentConfig", + "GetEvaluationExperimentConfigDict", + "GetEvaluationExperimentConfigOrDict", "GetEvaluationMetricConfig", "GetEvaluationMetricConfigDict", "GetEvaluationMetricConfigOrDict", @@ -2335,6 +2359,12 @@ "GetEvaluationItemConfig", "GetEvaluationItemConfigDict", "GetEvaluationItemConfigOrDict", + "ListEvaluationExperimentsConfig", + "ListEvaluationExperimentsConfigDict", + "ListEvaluationExperimentsConfigOrDict", + "ListEvaluationExperimentsResponse", + "ListEvaluationExperimentsResponseDict", + "ListEvaluationExperimentsResponseOrDict", "ListEvaluationMetricsConfig", "ListEvaluationMetricsConfigDict", "ListEvaluationMetricsConfigOrDict", @@ -3538,9 +3568,6 @@ "ObservabilityEvalCase", "ObservabilityEvalCaseDict", "ObservabilityEvalCaseOrDict", - "EvaluationExperiment", - "EvaluationExperimentDict", - "EvaluationExperimentOrDict", "RubricGroup", "RubricGroupDict", "RubricGroupOrDict", @@ -3649,6 +3676,7 @@ "VersionState", "QuotaState", "FeedbackType", + "EvaluationExperimentMergeStrategy", "EvaluationItemType", "SamplingMethod", "EvaluationRunState", @@ -3658,7 +3686,6 @@ "RagFileState", "SkillRevisionState", "PromptOptimizerMethod", - "EvaluationExperimentMergeStrategy", "OptimizationMethod", "PromptData", "PromptDataDict", @@ -3688,6 +3715,7 @@ "_CreateAgentEngineTaskRequestParameters", "_AppendAgentEngineTaskEventRequestParameters", "_ListAgentEngineTaskEventsRequestParameters", + "_CreateEvaluationExperimentParameters", "_CreateEvaluationItemParameters", "_CreateEvaluationMetricParameters", "_CreateEvaluationRunParameters", @@ -3697,10 +3725,12 @@ "_GenerateUserScenariosParameters", "_GenerateLossClustersParameters", "_GenerateInstanceRubricsRequest", + "_GetEvaluationExperimentParameters", "_GetEvaluationMetricParameters", "_GetEvaluationRunParameters", "_GetEvaluationSetParameters", "_GetEvaluationItemParameters", + "_ListEvaluationExperimentsParameters", "_ListEvaluationMetricsParameters", "_OptimizeRequestParameters", "_CustomJobParameters", diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index dd57c62b1e..cd6fb70b6c 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -498,6 +498,17 @@ class FeedbackType(_common.CaseInSensitiveEnum): """Indicates a thumbs down feedback (e.g., a "thumbs down").""" +class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): + """Merge strategy for the evaluation experiment.""" + + MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" + """Unspecified merge strategy.""" + SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" + """Default. Runs are treated as an independent, sequential history.""" + SHARED_RESULT_SET = "SHARED_RESULT_SET" + """Runs are parallel iterations contributing to a shared result set.""" + + class EvaluationItemType(_common.CaseInSensitiveEnum): """The type of the EvaluationItem.""" @@ -613,17 +624,6 @@ class PromptOptimizerMethod(_common.CaseInSensitiveEnum): """The data driven prompt optimizer designer for prompts from Android core API.""" -class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): - """Merge strategy for the evaluation experiment.""" - - MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" - """Unspecified merge strategy.""" - SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" - """Default. Runs are treated as an independent, sequential history.""" - SHARED_RESULT_SET = "SHARED_RESULT_SET" - """Runs are parallel iterations contributing to a shared result set.""" - - class OptimizationMethod(_common.CaseInSensitiveEnum): """The method for data driven prompt optimization.""" @@ -1498,6 +1498,131 @@ class ListAgentEngineTaskEventsResponseDict(TypedDict, total=False): ] +class CreateEvaluationExperimentConfig(_common.BaseModel): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class CreateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + +CreateEvaluationExperimentConfigOrDict = Union[ + CreateEvaluationExperimentConfig, CreateEvaluationExperimentConfigDict +] + + +class _CreateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] = Field(default=None, description="""""") + labels: Optional[dict[str, str]] = Field(default=None, description="""""") + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""""" + ) + metadata: Optional[dict[str, Any]] = Field(default=None, description="""""") + config: Optional[CreateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _CreateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] + """""" + + labels: Optional[dict[str, str]] + """""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """""" + + metadata: Optional[dict[str, Any]] + """""" + + config: Optional[CreateEvaluationExperimentConfigDict] + """""" + + +_CreateEvaluationExperimentParametersOrDict = Union[ + _CreateEvaluationExperimentParameters, _CreateEvaluationExperimentParametersDict +] + + +class EvaluationExperiment(_common.BaseModel): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] = Field( + default=None, + description="""The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + evaluation_runs: Optional[list[str]] = Field( + default=None, + description="""The EvaluationRuns that are part of this experiment.""", + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""", + ) + create_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was created.""" + ) + update_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was last updated.""" + ) + + +class EvaluationExperimentDict(TypedDict, total=False): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + evaluation_runs: Optional[list[str]] + """The EvaluationRuns that are part of this experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""" + + create_time: Optional[datetime.datetime] + """Timestamp when this experiment was created.""" + + update_time: Optional[datetime.datetime] + """Timestamp when this experiment was last updated.""" + + +EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] + + class CreateEvaluationItemConfig(_common.BaseModel): """Config to create an evaluation item.""" @@ -5805,6 +5930,50 @@ class GenerateInstanceRubricsResponseDict(TypedDict, total=False): ] +class GetEvaluationExperimentConfig(_common.BaseModel): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class GetEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + +GetEvaluationExperimentConfigOrDict = Union[ + GetEvaluationExperimentConfig, GetEvaluationExperimentConfigDict +] + + +class _GetEvaluationExperimentParameters(_common.BaseModel): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[GetEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _GetEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[GetEvaluationExperimentConfigDict] + """""" + + +_GetEvaluationExperimentParametersOrDict = Union[ + _GetEvaluationExperimentParameters, _GetEvaluationExperimentParametersDict +] + + class GetEvaluationMetricConfig(_common.BaseModel): """Config for getting an evaluation metric.""" @@ -5971,6 +6140,111 @@ class _GetEvaluationItemParametersDict(TypedDict, total=False): ] +class ListEvaluationExperimentsConfig(_common.BaseModel): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + page_size: Optional[int] = Field(default=None, description="""""") + page_token: Optional[str] = Field(default=None, description="""""") + filter: Optional[str] = Field( + default=None, + description="""An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""", + ) + order_by: Optional[str] = Field( + default=None, + description="""A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""", + ) + + +class ListEvaluationExperimentsConfigDict(TypedDict, total=False): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + page_size: Optional[int] + """""" + + page_token: Optional[str] + """""" + + filter: Optional[str] + """An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""" + + order_by: Optional[str] + """A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""" + + +ListEvaluationExperimentsConfigOrDict = Union[ + ListEvaluationExperimentsConfig, ListEvaluationExperimentsConfigDict +] + + +class _ListEvaluationExperimentsParameters(_common.BaseModel): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfig] = Field( + default=None, description="""""" + ) + + +class _ListEvaluationExperimentsParametersDict(TypedDict, total=False): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfigDict] + """""" + + +_ListEvaluationExperimentsParametersOrDict = Union[ + _ListEvaluationExperimentsParameters, _ListEvaluationExperimentsParametersDict +] + + +class ListEvaluationExperimentsResponse(_common.BaseModel): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponse] = Field( + default=None, description="""Used to retain the full HTTP response.""" + ) + next_page_token: Optional[str] = Field(default=None, description="""""") + evaluation_experiments: Optional[list[EvaluationExperiment]] = Field( + default=None, + description="""List of evaluation experiments. + """, + ) + + +class ListEvaluationExperimentsResponseDict(TypedDict, total=False): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponse] + """Used to retain the full HTTP response.""" + + next_page_token: Optional[str] + """""" + + evaluation_experiments: Optional[list[EvaluationExperimentDict]] + """List of evaluation experiments. + """ + + +ListEvaluationExperimentsResponseOrDict = Union[ + ListEvaluationExperimentsResponse, ListEvaluationExperimentsResponseDict +] + + class ListEvaluationMetricsConfig(_common.BaseModel): """Config for listing evaluation metrics.""" @@ -24912,73 +25186,6 @@ class ObservabilityEvalCaseDict(TypedDict, total=False): ObservabilityEvalCaseOrDict = Union[ObservabilityEvalCase, ObservabilityEvalCaseDict] -class EvaluationExperiment(_common.BaseModel): - """Represents an experiment for iterating on and visualizing evaluation runs.""" - - name: Optional[str] = Field( - default=None, - description="""The resource name of the EvaluationExperiment. Format: - `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", - ) - display_name: Optional[str] = Field( - default=None, description="""The display name of the evaluation experiment.""" - ) - evaluation_runs: Optional[list[str]] = Field( - default=None, - description="""The EvaluationRuns that are part of this experiment.""", - ) - labels: Optional[dict[str, str]] = Field( - default=None, description="""Labels for the evaluation experiment.""" - ) - merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( - default=None, description="""Merge strategy for the evaluation experiment.""" - ) - metadata: Optional[dict[str, Any]] = Field( - default=None, - description="""Metadata about the evaluation experiment, can be used by the caller - to store additional tracking information about the experiment.""", - ) - create_time: Optional[datetime.datetime] = Field( - default=None, description="""Timestamp when this experiment was created.""" - ) - update_time: Optional[datetime.datetime] = Field( - default=None, description="""Timestamp when this experiment was last updated.""" - ) - - -class EvaluationExperimentDict(TypedDict, total=False): - """Represents an experiment for iterating on and visualizing evaluation runs.""" - - name: Optional[str] - """The resource name of the EvaluationExperiment. Format: - `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" - - display_name: Optional[str] - """The display name of the evaluation experiment.""" - - evaluation_runs: Optional[list[str]] - """The EvaluationRuns that are part of this experiment.""" - - labels: Optional[dict[str, str]] - """Labels for the evaluation experiment.""" - - merge_strategy: Optional[EvaluationExperimentMergeStrategy] - """Merge strategy for the evaluation experiment.""" - - metadata: Optional[dict[str, Any]] - """Metadata about the evaluation experiment, can be used by the caller - to store additional tracking information about the experiment.""" - - create_time: Optional[datetime.datetime] - """Timestamp when this experiment was created.""" - - update_time: Optional[datetime.datetime] - """Timestamp when this experiment was last updated.""" - - -EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] - - class RubricGroup(_common.BaseModel): """A group of rubrics. diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index b93b6db39b..ebcddee0b6 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -11743,3 +11743,143 @@ def test_catalog_in_sync_with_server(self): f" Server: {sorted(server_sandbox_names)}\n" f" SDK: {sorted(sdk_sandbox_names)}" ) + + +class TestGetEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.experiment_name = ( + "projects/123/locations/us-central1/evaluationExperiments/456" + ) + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": self.experiment_name, + "displayName": "my_experiment", + "evaluationRuns": [ + "projects/123/locations/us-central1/evaluationRuns/789" + ], + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_get_evaluation_experiment_returns_experiment(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + experiment = evals_module.get_evaluation_experiment(name=self.experiment_name) + + assert isinstance(experiment, agentplatform_genai_types.EvaluationExperiment) + assert experiment.name == self.experiment_name + assert experiment.display_name == "my_experiment" + assert experiment.evaluation_runs == [ + "projects/123/locations/us-central1/evaluationRuns/789" + ] + + def test_get_evaluation_experiment_uses_short_name_in_url(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.get_evaluation_experiment(name=self.experiment_name) + + self.mock_api_client.request.assert_called_once() + path = self.mock_api_client.request.call_args[0][1] + assert path == "evaluationExperiments/456" + + def test_get_evaluation_experiment_empty_name_raises(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + with pytest.raises(ValueError, match="name cannot be empty"): + evals_module.get_evaluation_experiment(name="") + + +class TestListEvaluationExperiments: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "evaluationExperiments": [ + { + "name": "projects/123/locations/us-central1/evaluationExperiments/1", + "displayName": "exp_1", + }, + { + "name": "projects/123/locations/us-central1/evaluationExperiments/2", + "displayName": "exp_2", + }, + ] + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_list_evaluation_experiments_returns_experiments(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + response = evals_module.list_evaluation_experiments() + + assert len(response.evaluation_experiments) == 2 + assert response.evaluation_experiments[0].display_name == "exp_1" + assert response.evaluation_experiments[1].display_name == "exp_2" + + def test_list_evaluation_experiments_passes_filter_and_order_by(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.list_evaluation_experiments( + filter='display_name="exp_1"', order_by="create_time desc" + ) + + self.mock_api_client.request.assert_called_once() + path = self.mock_api_client.request.call_args[0][1] + assert path.startswith("evaluationExperiments?") + assert "orderBy=create_time+desc" in path + + +class TestCreateEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": "projects/123/locations/us-central1/evaluationExperiments/456", + "displayName": "my_experiment", + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_create_evaluation_experiment_returns_experiment(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + experiment = evals_module.create_evaluation_experiment( + display_name="my_experiment" + ) + + assert isinstance(experiment, agentplatform_genai_types.EvaluationExperiment) + assert experiment.display_name == "my_experiment" + + def test_create_evaluation_experiment_posts_to_experiments(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.create_evaluation_experiment(display_name="my_experiment") + + self.mock_api_client.request.assert_called_once() + call_args = self.mock_api_client.request.call_args + assert call_args[0][0] == "post" + assert call_args[0][1] == "evaluationExperiments" + request_body = call_args[0][2] + assert request_body.get("displayName") == "my_experiment" + + def test_create_evaluation_experiment_passes_merge_strategy(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.create_evaluation_experiment( + display_name="my_experiment", + merge_strategy=agentplatform_genai_types.EvaluationExperimentMergeStrategy.SHARED_RESULT_SET, + ) + + request_body = self.mock_api_client.request.call_args[0][2] + assert request_body.get("mergeStrategy") == "SHARED_RESULT_SET" diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index eb55f9a0dd..ad54eae65a 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -140,6 +140,29 @@ def _CreateEvaluationRunParameters_to_vertex( return to_object +def _CreateEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["display_name"]) is not None: + setv(to_object, ["displayName"], getv(from_object, ["display_name"])) + + if getv(from_object, ["labels"]) is not None: + setv(to_object, ["labels"], getv(from_object, ["labels"])) + + if getv(from_object, ["merge_strategy"]) is not None: + setv(to_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"])) + + if getv(from_object, ["metadata"]) is not None: + setv(to_object, ["metadata"], getv(from_object, ["metadata"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _CreateEvaluationSetParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -721,6 +744,20 @@ def _GenerateUserScenariosParameters_to_vertex( return to_object +def _GetEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _GetEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -781,6 +818,44 @@ def _GetEvaluationSetParameters_to_vertex( return to_object +def _ListEvaluationExperimentsConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["page_size"]) is not None: + setv(parent_object, ["_query", "pageSize"], getv(from_object, ["page_size"])) + + if getv(from_object, ["page_token"]) is not None: + setv(parent_object, ["_query", "pageToken"], getv(from_object, ["page_token"])) + + if getv(from_object, ["filter"]) is not None: + setv(parent_object, ["_query", "filter"], getv(from_object, ["filter"])) + + if getv(from_object, ["order_by"]) is not None: + setv(parent_object, ["_query", "orderBy"], getv(from_object, ["order_by"])) + + return to_object + + +def _ListEvaluationExperimentsParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["config"]) is not None: + setv( + to_object, + ["config"], + _ListEvaluationExperimentsConfig_to_vertex( + getv(from_object, ["config"]), to_object + ), + ) + + return to_object + + def _ListEvaluationMetricsConfig_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1250,6 +1325,86 @@ def _create_evaluation_run( self._api_client._verify_response(return_value) return return_value + def _create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Creates an EvaluationExperiment. + """ + + parameter_model = types._CreateEvaluationExperimentParameters( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("post", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _create_evaluation_set( self, *, @@ -1740,6 +1895,78 @@ def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + def _get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _get_evaluation_metric( self, *, @@ -2001,7 +2228,77 @@ def _get_evaluation_item( response_dict = {} if not response.body else json.loads(response.body) - return_value = types.EvaluationItem._from_response( + return_value = types.EvaluationItem._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( response=response_dict, kwargs=( { @@ -2773,6 +3070,36 @@ def create_evaluation_run( config=config, ) + def create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Creates an EvaluationExperiment. + + Args: + display_name: The display name of the evaluation experiment. + labels: Labels for the evaluation experiment. + merge_strategy: Merge strategy for the evaluation experiment. + metadata: Metadata about the evaluation experiment, can be used by the + caller to store additional tracking information about the experiment. + config: Optional configuration for the create operation. + + Returns: + The created evaluation experiment. + """ + return self._create_evaluation_experiment( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + def get_evaluation_set( self, *, @@ -3103,6 +3430,73 @@ def delete_evaluation_metric( config=config, ) + def get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Retrieves an EvaluationExperiment from the resource name. + + Args: + name: The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: The optional configuration for the get operation. + + Returns: + The evaluation experiment. + + Raises: + ValueError: If the name is empty. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + return self._get_evaluation_experiment(name=name, config=config) + + @_common.experimental_warning( + "The Vertex SDK GenAI evals.list_evaluation_experiments method is" + " experimental, and may change in future versions." + ) + def list_evaluation_experiments( + self, + *, + filter: Optional[str] = None, + order_by: Optional[str] = None, + config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None, + ) -> types.ListEvaluationExperimentsResponse: + """Lists EvaluationExperiments. + + Args: + filter: An expression for filtering the results of the request. For + field names both snake_case and camelCase are supported. For more + information about filter syntax, see + `AIP-160 `_. + Example: ``'display_name="my_experiment"'``. + order_by: A comma-separated list of fields to order by, sorted in + ascending order by default. Use ``desc`` after a field name for + descending. Example: ``"create_time desc"``. + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. Top-level ``filter`` and ``order_by`` arguments + take precedence over values set in ``config``. + + Returns: + The list evaluation experiments response. + """ + if config is None: + config = types.ListEvaluationExperimentsConfig() + if isinstance(config, dict): + config = types.ListEvaluationExperimentsConfig.model_validate(config) + if filter is not None: + config.filter = filter + if order_by is not None: + config.order_by = order_by + return self._list_evaluation_experiments( + config=config, + ) + class AsyncEvals(_api_module.BaseModule): @@ -3356,6 +3750,88 @@ async def _create_evaluation_run( self._api_client._verify_response(return_value) return return_value + async def _create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Creates an EvaluationExperiment. + """ + + parameter_model = types._CreateEvaluationExperimentParameters( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "post", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _create_evaluation_set( self, *, @@ -3858,6 +4334,80 @@ async def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + async def _get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _get_evaluation_metric( self, *, @@ -4151,6 +4701,78 @@ async def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value + async def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _list_evaluation_metrics( self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None ) -> types.ListEvaluationMetricsResponse: diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py index d088967399..f692f2351d 100644 --- a/vertexai/_genai/types/__init__.py +++ b/vertexai/_genai/types/__init__.py @@ -83,6 +83,7 @@ from .common import _GetDatasetParameters from .common import _GetDatasetVersionParameters from .common import _GetDeleteAgentEngineRuntimeRevisionOperationParameters +from .common import _GetEvaluationExperimentParameters from .common import _GetEvaluationItemParameters from .common import _GetEvaluationMetricParameters from .common import _GetEvaluationRunParameters @@ -107,6 +108,7 @@ from .common import _ListAgentEngineTasksRequestParameters from .common import _ListDatasetsRequestParameters from .common import _ListDatasetVersionsRequestParameters +from .common import _ListEvaluationExperimentsParameters from .common import _ListEvaluationMetricsParameters from .common import _ListMultimodalDatasetsRequestParameters from .common import _ListSandboxEnvironmentSnapshotsRequestParameters @@ -295,6 +297,10 @@ from .common import CreateEvaluationMetricConfig from .common import CreateEvaluationMetricConfigDict from .common import CreateEvaluationMetricConfigOrDict +from .common import CreateEvaluationExperimentConfig +from .common import CreateEvaluationExperimentConfigDict +from .common import CreateEvaluationExperimentConfigOrDict +from .common import _CreateEvaluationExperimentParameters from .common import CreateEvaluationRunConfig from .common import CreateEvaluationRunConfigDict from .common import CreateEvaluationRunConfigOrDict @@ -612,6 +618,9 @@ from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfig from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigDict from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigOrDict +from .common import GetEvaluationExperimentConfig +from .common import GetEvaluationExperimentConfigDict +from .common import GetEvaluationExperimentConfigOrDict from .common import GetEvaluationItemConfig from .common import GetEvaluationItemConfigDict from .common import GetEvaluationItemConfigOrDict @@ -715,6 +724,12 @@ from .common import ListDatasetVersionsResponse from .common import ListDatasetVersionsResponseDict from .common import ListDatasetVersionsResponseOrDict +from .common import ListEvaluationExperimentsConfig +from .common import ListEvaluationExperimentsConfigDict +from .common import ListEvaluationExperimentsConfigOrDict +from .common import ListEvaluationExperimentsResponse +from .common import ListEvaluationExperimentsResponseDict +from .common import ListEvaluationExperimentsResponseOrDict from .common import ListEvaluationMetricsConfig from .common import ListEvaluationMetricsConfigDict from .common import ListEvaluationMetricsConfigOrDict @@ -1606,6 +1621,9 @@ "AnalysisConfig", "AnalysisConfigDict", "AnalysisConfigOrDict", + "CreateEvaluationExperimentConfig", + "CreateEvaluationExperimentConfigDict", + "CreateEvaluationExperimentConfigOrDict", "CreateEvaluationRunConfig", "CreateEvaluationRunConfigDict", "CreateEvaluationRunConfigOrDict", @@ -2642,6 +2660,17 @@ "EvaluationExperimentDict", "EvaluationExperimentOrDict", "EvaluationExperimentMergeStrategy", + "GetEvaluationExperimentConfig", + "GetEvaluationExperimentConfigDict", + "GetEvaluationExperimentConfigOrDict", + "_GetEvaluationExperimentParameters", + "ListEvaluationExperimentsConfig", + "ListEvaluationExperimentsConfigDict", + "ListEvaluationExperimentsConfigOrDict", + "_ListEvaluationExperimentsParameters", + "ListEvaluationExperimentsResponse", + "ListEvaluationExperimentsResponseDict", + "ListEvaluationExperimentsResponseOrDict", "EvaluateDatasetConfig", "EvaluateDatasetConfigDict", "EvaluateDatasetConfigOrDict", @@ -2776,6 +2805,7 @@ "_CreateAgentEngineTaskRequestParameters", "_AppendAgentEngineTaskEventRequestParameters", "_ListAgentEngineTaskEventsRequestParameters", + "_CreateEvaluationExperimentParameters", "_CreateEvaluationItemParameters", "_CreateEvaluationMetricParameters", "_CreateEvaluationRunParameters", diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 59490f34fa..3791b6ad13 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -1383,6 +1383,64 @@ class ListAgentEngineTaskEventsResponseDict(TypedDict, total=False): ] +class CreateEvaluationExperimentConfig(_common.BaseModel): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class CreateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +CreateEvaluationExperimentConfigOrDict = Union[ + CreateEvaluationExperimentConfig, CreateEvaluationExperimentConfigDict +] + + +class _CreateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] = Field(default=None, description="""""") + labels: Optional[dict[str, str]] = Field(default=None, description="""""") + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""""" + ) + metadata: Optional[dict[str, Any]] = Field(default=None, description="""""") + config: Optional[CreateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _CreateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] + """""" + + labels: Optional[dict[str, str]] + """""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """""" + + metadata: Optional[dict[str, Any]] + """""" + + config: Optional[CreateEvaluationExperimentConfigDict] + """""" + + +_CreateEvaluationExperimentParametersOrDict = Union[ + _CreateEvaluationExperimentParameters, _CreateEvaluationExperimentParametersDict +] + + class CreateEvaluationItemConfig(_common.BaseModel): """Config to create an evaluation item.""" @@ -19195,6 +19253,155 @@ class EvaluationExperimentDict(TypedDict, total=False): EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] +class GetEvaluationExperimentConfig(_common.BaseModel): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class GetEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +GetEvaluationExperimentConfigOrDict = Union[ + GetEvaluationExperimentConfig, GetEvaluationExperimentConfigDict +] + + +class _GetEvaluationExperimentParameters(_common.BaseModel): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[GetEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _GetEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[GetEvaluationExperimentConfigDict] + """""" + + +_GetEvaluationExperimentParametersOrDict = Union[ + _GetEvaluationExperimentParameters, _GetEvaluationExperimentParametersDict +] + + +class ListEvaluationExperimentsConfig(_common.BaseModel): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + page_size: Optional[int] = Field(default=None, description="""""") + page_token: Optional[str] = Field(default=None, description="""""") + filter: Optional[str] = Field( + default=None, + description="""An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""", + ) + order_by: Optional[str] = Field( + default=None, + description="""A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""", + ) + + +class ListEvaluationExperimentsConfigDict(TypedDict, total=False): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + page_size: Optional[int] + """""" + + page_token: Optional[str] + """""" + + filter: Optional[str] + """An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""" + + order_by: Optional[str] + """A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""" + + +ListEvaluationExperimentsConfigOrDict = Union[ + ListEvaluationExperimentsConfig, ListEvaluationExperimentsConfigDict +] + + +class _ListEvaluationExperimentsParameters(_common.BaseModel): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfig] = Field( + default=None, description="""""" + ) + + +class _ListEvaluationExperimentsParametersDict(TypedDict, total=False): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfigDict] + """""" + + +_ListEvaluationExperimentsParametersOrDict = Union[ + _ListEvaluationExperimentsParameters, _ListEvaluationExperimentsParametersDict +] + + +class ListEvaluationExperimentsResponse(_common.BaseModel): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponse] = Field( + default=None, description="""Used to retain the full HTTP response.""" + ) + next_page_token: Optional[str] = Field(default=None, description="""""") + evaluation_experiments: Optional[list[EvaluationExperiment]] = Field( + default=None, + description="""List of evaluation experiments. + """, + ) + + +class ListEvaluationExperimentsResponseDict(TypedDict, total=False): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponseDict] + """Used to retain the full HTTP response.""" + + next_page_token: Optional[str] + """""" + + evaluation_experiments: Optional[list[EvaluationExperimentDict]] + """List of evaluation experiments. + """ + + +ListEvaluationExperimentsResponseOrDict = Union[ + ListEvaluationExperimentsResponse, ListEvaluationExperimentsResponseDict +] + + class RubricGroup(_common.BaseModel): """A group of rubrics.