diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 1f4aa7e1de..a5db11bad1 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -43,6 +43,29 @@ logger = logging.getLogger("agentplatform_genai.evals") +def _CreateEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["display_name"]) is not None: + setv(to_object, ["displayName"], getv(from_object, ["display_name"])) + + if getv(from_object, ["labels"]) is not None: + setv(to_object, ["labels"], getv(from_object, ["labels"])) + + if getv(from_object, ["merge_strategy"]) is not None: + setv(to_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"])) + + if getv(from_object, ["metadata"]) is not None: + setv(to_object, ["metadata"], getv(from_object, ["metadata"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _CreateEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -201,6 +224,20 @@ def _CustomCodeExecutionSpec_to_vertex( return to_object +def _DeleteEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _DeleteEvaluationMetricParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -733,6 +770,20 @@ def _GenerateUserScenariosParameters_to_vertex( return to_object +def _GetEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _GetEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -793,6 +844,44 @@ def _GetEvaluationSetParameters_to_vertex( return to_object +def _ListEvaluationExperimentsConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["page_size"]) is not None: + setv(parent_object, ["_query", "pageSize"], getv(from_object, ["page_size"])) + + if getv(from_object, ["page_token"]) is not None: + setv(parent_object, ["_query", "pageToken"], getv(from_object, ["page_token"])) + + if getv(from_object, ["filter"]) is not None: + setv(parent_object, ["_query", "filter"], getv(from_object, ["filter"])) + + if getv(from_object, ["order_by"]) is not None: + setv(parent_object, ["_query", "orderBy"], getv(from_object, ["order_by"])) + + return to_object + + +def _ListEvaluationExperimentsParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["config"]) is not None: + setv( + to_object, + ["config"], + _ListEvaluationExperimentsConfig_to_vertex( + getv(from_object, ["config"]), to_object + ), + ) + + return to_object + + def _ListEvaluationMetricsConfig_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1016,8 +1105,130 @@ def _UnifiedMetric_to_vertex( return to_object +def _UpdateEvaluationExperimentConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["update_mask"]) is not None: + setv( + parent_object, ["_query", "updateMask"], getv(from_object, ["update_mask"]) + ) + + if getv(from_object, ["display_name"]) is not None: + setv(parent_object, ["displayName"], getv(from_object, ["display_name"])) + + if getv(from_object, ["labels"]) is not None: + setv(parent_object, ["labels"], getv(from_object, ["labels"])) + + if getv(from_object, ["merge_strategy"]) is not None: + setv(parent_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"])) + + if getv(from_object, ["metadata"]) is not None: + setv(parent_object, ["metadata"], getv(from_object, ["metadata"])) + + return to_object + + +def _UpdateEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + _UpdateEvaluationExperimentConfig_to_vertex( + getv(from_object, ["config"]), to_object + ) + + return to_object + + class Evals(_api_module.BaseModule): + def _create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Creates an EvaluationExperiment. + """ + + parameter_model = types._CreateEvaluationExperimentParameters( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("post", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _create_evaluation_item( self, *, @@ -1336,6 +1547,80 @@ def _create_evaluation_set( self._api_client._verify_response(return_value) return return_value + def _delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """ + Deletes an EvaluationExperiment. + """ + + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("delete", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationExperimentOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _delete_evaluation_metric( self, *, @@ -1754,18 +2039,18 @@ def _generate_rubrics( self._api_client._verify_response(return_value) return return_value - def _get_evaluation_metric( + def _get_evaluation_experiment( self, *, - metric_resource_name: str, - config: Optional[types.GetEvaluationMetricConfigOrDict] = None, - ) -> types.EvaluationMetric: + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: """ - Retrieves an EvaluationMetric from the resource name. + Retrieves an EvaluationExperiment from the resource name. """ - parameter_model = types._GetEvaluationMetricParameters( - metric_resource_name=metric_resource_name, + parameter_model = types._GetEvaluationExperimentParameters( + name=name, config=config, ) @@ -1775,12 +2060,84 @@ def _get_evaluation_metric( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _GetEvaluationMetricParameters_to_vertex(parameter_model) + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) request_url_dict = request_dict.get("_url") if request_url_dict: - path = "{evaluation_metric}".format_map(request_url_dict) + path = "evaluationExperiments/{name}".format_map(request_url_dict) else: - path = "{evaluation_metric}" + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + def _get_evaluation_metric( + self, + *, + metric_resource_name: str, + config: Optional[types.GetEvaluationMetricConfigOrDict] = None, + ) -> types.EvaluationMetric: + """ + Retrieves an EvaluationMetric from the resource name. + """ + + parameter_model = types._GetEvaluationMetricParameters( + metric_resource_name=metric_resource_name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationMetricParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "{evaluation_metric}".format_map(request_url_dict) + else: + path = "{evaluation_metric}" query_params = request_dict.get("_query") if query_params: @@ -2039,6 +2396,76 @@ def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value + def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _list_evaluation_metrics( self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None ) -> types.ListEvaluationMetricsResponse: @@ -2110,6 +2537,80 @@ def _list_evaluation_metrics( self._api_client._verify_response(return_value) return return_value + def _update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Updates an EvaluationExperiment. + """ + + parameter_model = types._UpdateEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("patch", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def evaluate_instances( self, *, @@ -3172,24 +3673,270 @@ def delete_evaluation_metric( config=config, ) + @_common.experimental_warning( + "The Vertex SDK GenAI evals.get_evaluation_experiment method is" + " experimental, and may change in future versions." + ) + def get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Retrieves an EvaluationExperiment from the resource name. -class AsyncEvals(_api_module.BaseModule): + Args: + name: The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: The optional configuration for the get operation. - async def _create_evaluation_item( + Returns: + The evaluation experiment. + + Raises: + ValueError: If the name is empty. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + return self._get_evaluation_experiment(name=name, config=config) + + @_common.experimental_warning( + "The Vertex SDK GenAI evals.list_evaluation_experiments method is" + " experimental, and may change in future versions." + ) + def list_evaluation_experiments( self, *, - evaluation_item_type: str, - gcs_uri: str, - display_name: Optional[str] = None, - config: Optional[types.CreateEvaluationItemConfigOrDict] = None, - ) -> types.EvaluationItem: - """ - Creates an EvaluationItem. + filter: Optional[str] = None, + order_by: Optional[str] = None, + config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None, + ) -> types.ListEvaluationExperimentsResponse: + """Lists EvaluationExperiments. + + Args: + filter: An expression for filtering the results of the request. For + field names both snake_case and camelCase are supported. For more + information about filter syntax, see + `AIP-160 `_. + Example: ``'display_name="my_experiment"'``. + order_by: A comma-separated list of fields to order by, sorted in + ascending order by default. Use ``desc`` after a field name for + descending. Example: ``"create_time desc"``. + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. Top-level ``filter`` and ``order_by`` arguments + take precedence over values set in ``config``. + + Returns: + The list evaluation experiments response. """ + if config is None: + config = types.ListEvaluationExperimentsConfig() + if isinstance(config, dict): + config = types.ListEvaluationExperimentsConfig.model_validate(config) + if filter is not None: + config.filter = filter + if order_by is not None: + config.order_by = order_by + return self._list_evaluation_experiments( + config=config, + ) - parameter_model = types._CreateEvaluationItemParameters( - evaluation_item_type=evaluation_item_type, - gcs_uri=gcs_uri, + @_common.experimental_warning( + "The Vertex SDK GenAI evals.create_evaluation_experiment method is" + " experimental, and may change in future versions." + ) + def create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Creates an EvaluationExperiment. + + Args: + display_name: The display name of the evaluation experiment. + labels: Labels for the evaluation experiment. + merge_strategy: Merge strategy for the evaluation experiment. + metadata: Metadata about the evaluation experiment, can be used by the + caller to store additional tracking information about the experiment. + config: Optional configuration for the create operation. + + Returns: + The created evaluation experiment. + """ + return self._create_evaluation_experiment( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + @_common.experimental_warning( + "The Vertex SDK GenAI evals.update_evaluation_experiment method is" + " experimental, and may change in future versions." + ) + def update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Updates an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to update. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration specifying the fields to update (e.g. + display_name, labels, merge_strategy, metadata) and the update_mask. + + Returns: + The updated evaluation experiment. + + Raises: + ValueError: If the name is empty. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + return self._update_evaluation_experiment(name=name, config=config) + + @_common.experimental_warning( + "The Vertex SDK GenAI evals.delete_evaluation_experiment method is" + " experimental, and may change in future versions." + ) + def delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """Deletes an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to delete. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration for the delete operation. + + Returns: + The delete operation. + + Raises: + ValueError: If the name is empty. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + return self._delete_evaluation_experiment(name=name, config=config) + + +class AsyncEvals(_api_module.BaseModule): + + async def _create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Creates an EvaluationExperiment. + """ + + parameter_model = types._CreateEvaluationExperimentParameters( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "post", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + async def _create_evaluation_item( + self, + *, + evaluation_item_type: str, + gcs_uri: str, + display_name: Optional[str] = None, + config: Optional[types.CreateEvaluationItemConfigOrDict] = None, + ) -> types.EvaluationItem: + """ + Creates an EvaluationItem. + """ + + parameter_model = types._CreateEvaluationItemParameters( + evaluation_item_type=evaluation_item_type, + gcs_uri=gcs_uri, display_name=display_name, config=config, ) @@ -3501,6 +4248,82 @@ async def _create_evaluation_set( self._api_client._verify_response(return_value) return return_value + async def _delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """ + Deletes an EvaluationExperiment. + """ + + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "delete", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationExperimentOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _delete_evaluation_metric( self, *, @@ -3929,6 +4752,80 @@ async def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + async def _get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _get_evaluation_metric( self, *, @@ -4222,6 +5119,78 @@ async def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value + async def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _list_evaluation_metrics( self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None ) -> types.ListEvaluationMetricsResponse: @@ -4295,6 +5264,82 @@ async def _list_evaluation_metrics( self._api_client._verify_response(return_value) return return_value + async def _update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Updates an EvaluationExperiment. + """ + + parameter_model = types._UpdateEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "patch", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def batch_evaluate( self, *, diff --git a/agentplatform/_genai/types/__init__.py b/agentplatform/_genai/types/__init__.py index a731e74a0c..6027cc3bc0 100644 --- a/agentplatform/_genai/types/__init__.py +++ b/agentplatform/_genai/types/__init__.py @@ -36,6 +36,7 @@ from .common import _CreateAgentEngineTaskRequestParameters from .common import _CreateDatasetParameters from .common import _CreateDatasetVersionParameters +from .common import _CreateEvaluationExperimentParameters from .common import _CreateEvaluationItemParameters from .common import _CreateEvaluationMetricParameters from .common import _CreateEvaluationRunParameters @@ -55,6 +56,7 @@ from .common import _DeleteAgentEngineSessionRequestParameters from .common import _DeleteAgentEngineTaskRequestParameters from .common import _DeleteDatasetRequestParameters +from .common import _DeleteEvaluationExperimentParameters from .common import _DeleteEvaluationMetricParameters from .common import _DeleteMultimodalDatasetRequestParameters from .common import _DeletePromptVersionRequestParameters @@ -90,6 +92,7 @@ from .common import _GetDatasetParameters from .common import _GetDatasetVersionParameters from .common import _GetDeleteAgentEngineRuntimeRevisionOperationParameters +from .common import _GetEvaluationExperimentParameters from .common import _GetEvaluationItemParameters from .common import _GetEvaluationMetricParameters from .common import _GetEvaluationRunParameters @@ -126,6 +129,7 @@ from .common import _ListAgentEngineTasksRequestParameters from .common import _ListDatasetsRequestParameters from .common import _ListDatasetVersionsRequestParameters +from .common import _ListEvaluationExperimentsParameters from .common import _ListEvaluationMetricsParameters from .common import _ListMultimodalDatasetsRequestParameters from .common import _ListPublisherModelsRequestParameters @@ -156,6 +160,7 @@ from .common import _UpdateAgentEngineRequestParameters from .common import _UpdateAgentEngineSessionRequestParameters from .common import _UpdateDatasetParameters +from .common import _UpdateEvaluationExperimentParameters from .common import _UpdateMultimodalDatasetParameters from .common import _UpdateRagConfigRequestParameters from .common import _UpdateRagCorpusRequestParameters @@ -340,6 +345,9 @@ from .common import CreateDatasetVersionConfig from .common import CreateDatasetVersionConfigDict from .common import CreateDatasetVersionConfigOrDict +from .common import CreateEvaluationExperimentConfig +from .common import CreateEvaluationExperimentConfigDict +from .common import CreateEvaluationExperimentConfigOrDict from .common import CreateEvaluationItemConfig from .common import CreateEvaluationItemConfigDict from .common import CreateEvaluationItemConfigOrDict @@ -434,6 +442,12 @@ from .common import DeleteAgentEngineTaskConfig from .common import DeleteAgentEngineTaskConfigDict from .common import DeleteAgentEngineTaskConfigOrDict +from .common import DeleteEvaluationExperimentConfig +from .common import DeleteEvaluationExperimentConfigDict +from .common import DeleteEvaluationExperimentConfigOrDict +from .common import DeleteEvaluationExperimentOperation +from .common import DeleteEvaluationExperimentOperationDict +from .common import DeleteEvaluationExperimentOperationOrDict from .common import DeleteEvaluationMetricConfig from .common import DeleteEvaluationMetricConfigDict from .common import DeleteEvaluationMetricConfigOrDict @@ -718,6 +732,9 @@ from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfig from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigDict from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigOrDict +from .common import GetEvaluationExperimentConfig +from .common import GetEvaluationExperimentConfigDict +from .common import GetEvaluationExperimentConfigOrDict from .common import GetEvaluationItemConfig from .common import GetEvaluationItemConfigDict from .common import GetEvaluationItemConfigOrDict @@ -891,6 +908,12 @@ from .common import ListDeployableModelsConfig from .common import ListDeployableModelsConfigDict from .common import ListDeployableModelsConfigOrDict +from .common import ListEvaluationExperimentsConfig +from .common import ListEvaluationExperimentsConfigDict +from .common import ListEvaluationExperimentsConfigOrDict +from .common import ListEvaluationExperimentsResponse +from .common import ListEvaluationExperimentsResponseDict +from .common import ListEvaluationExperimentsResponseOrDict from .common import ListEvaluationMetricsConfig from .common import ListEvaluationMetricsConfigDict from .common import ListEvaluationMetricsConfigOrDict @@ -1888,6 +1911,9 @@ from .common import UpdateAgentEngineSessionConfig from .common import UpdateAgentEngineSessionConfigDict from .common import UpdateAgentEngineSessionConfigOrDict +from .common import UpdateEvaluationExperimentConfig +from .common import UpdateEvaluationExperimentConfigDict +from .common import UpdateEvaluationExperimentConfigOrDict from .common import UpdatePromptConfig from .common import UpdatePromptConfigDict from .common import UpdatePromptConfigOrDict @@ -2002,6 +2028,12 @@ "ListAgentEngineTaskEventsResponse", "ListAgentEngineTaskEventsResponseDict", "ListAgentEngineTaskEventsResponseOrDict", + "CreateEvaluationExperimentConfig", + "CreateEvaluationExperimentConfigDict", + "CreateEvaluationExperimentConfigOrDict", + "EvaluationExperiment", + "EvaluationExperimentDict", + "EvaluationExperimentOrDict", "CreateEvaluationItemConfig", "CreateEvaluationItemConfigDict", "CreateEvaluationItemConfigOrDict", @@ -2152,6 +2184,12 @@ "EvaluationSet", "EvaluationSetDict", "EvaluationSetOrDict", + "DeleteEvaluationExperimentConfig", + "DeleteEvaluationExperimentConfigDict", + "DeleteEvaluationExperimentConfigOrDict", + "DeleteEvaluationExperimentOperation", + "DeleteEvaluationExperimentOperationDict", + "DeleteEvaluationExperimentOperationOrDict", "DeleteEvaluationMetricConfig", "DeleteEvaluationMetricConfigDict", "DeleteEvaluationMetricConfigOrDict", @@ -2323,6 +2361,9 @@ "GenerateInstanceRubricsResponse", "GenerateInstanceRubricsResponseDict", "GenerateInstanceRubricsResponseOrDict", + "GetEvaluationExperimentConfig", + "GetEvaluationExperimentConfigDict", + "GetEvaluationExperimentConfigOrDict", "GetEvaluationMetricConfig", "GetEvaluationMetricConfigDict", "GetEvaluationMetricConfigOrDict", @@ -2335,12 +2376,21 @@ "GetEvaluationItemConfig", "GetEvaluationItemConfigDict", "GetEvaluationItemConfigOrDict", + "ListEvaluationExperimentsConfig", + "ListEvaluationExperimentsConfigDict", + "ListEvaluationExperimentsConfigOrDict", + "ListEvaluationExperimentsResponse", + "ListEvaluationExperimentsResponseDict", + "ListEvaluationExperimentsResponseOrDict", "ListEvaluationMetricsConfig", "ListEvaluationMetricsConfigDict", "ListEvaluationMetricsConfigOrDict", "ListEvaluationMetricsResponse", "ListEvaluationMetricsResponseDict", "ListEvaluationMetricsResponseOrDict", + "UpdateEvaluationExperimentConfig", + "UpdateEvaluationExperimentConfigDict", + "UpdateEvaluationExperimentConfigOrDict", "OptimizeConfig", "OptimizeConfigDict", "OptimizeConfigOrDict", @@ -3538,9 +3588,6 @@ "ObservabilityEvalCase", "ObservabilityEvalCaseDict", "ObservabilityEvalCaseOrDict", - "EvaluationExperiment", - "EvaluationExperimentDict", - "EvaluationExperimentOrDict", "RubricGroup", "RubricGroupDict", "RubricGroupOrDict", @@ -3649,6 +3696,7 @@ "VersionState", "QuotaState", "FeedbackType", + "EvaluationExperimentMergeStrategy", "EvaluationItemType", "SamplingMethod", "EvaluationRunState", @@ -3658,7 +3706,6 @@ "RagFileState", "SkillRevisionState", "PromptOptimizerMethod", - "EvaluationExperimentMergeStrategy", "OptimizationMethod", "PromptData", "PromptDataDict", @@ -3688,20 +3735,25 @@ "_CreateAgentEngineTaskRequestParameters", "_AppendAgentEngineTaskEventRequestParameters", "_ListAgentEngineTaskEventsRequestParameters", + "_CreateEvaluationExperimentParameters", "_CreateEvaluationItemParameters", "_CreateEvaluationMetricParameters", "_CreateEvaluationRunParameters", "_CreateEvaluationSetParameters", + "_DeleteEvaluationExperimentParameters", "_DeleteEvaluationMetricParameters", "_EvaluateInstancesRequestParameters", "_GenerateUserScenariosParameters", "_GenerateLossClustersParameters", "_GenerateInstanceRubricsRequest", + "_GetEvaluationExperimentParameters", "_GetEvaluationMetricParameters", "_GetEvaluationRunParameters", "_GetEvaluationSetParameters", "_GetEvaluationItemParameters", + "_ListEvaluationExperimentsParameters", "_ListEvaluationMetricsParameters", + "_UpdateEvaluationExperimentParameters", "_OptimizeRequestParameters", "_CustomJobParameters", "_GetCustomJobParameters", diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index dd57c62b1e..b5da65ffb3 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -498,6 +498,17 @@ class FeedbackType(_common.CaseInSensitiveEnum): """Indicates a thumbs down feedback (e.g., a "thumbs down").""" +class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): + """Merge strategy for the evaluation experiment.""" + + MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" + """Unspecified merge strategy.""" + SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" + """Default. Runs are treated as an independent, sequential history.""" + SHARED_RESULT_SET = "SHARED_RESULT_SET" + """Runs are parallel iterations contributing to a shared result set.""" + + class EvaluationItemType(_common.CaseInSensitiveEnum): """The type of the EvaluationItem.""" @@ -613,17 +624,6 @@ class PromptOptimizerMethod(_common.CaseInSensitiveEnum): """The data driven prompt optimizer designer for prompts from Android core API.""" -class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): - """Merge strategy for the evaluation experiment.""" - - MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" - """Unspecified merge strategy.""" - SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" - """Default. Runs are treated as an independent, sequential history.""" - SHARED_RESULT_SET = "SHARED_RESULT_SET" - """Runs are parallel iterations contributing to a shared result set.""" - - class OptimizationMethod(_common.CaseInSensitiveEnum): """The method for data driven prompt optimization.""" @@ -1498,6 +1498,131 @@ class ListAgentEngineTaskEventsResponseDict(TypedDict, total=False): ] +class CreateEvaluationExperimentConfig(_common.BaseModel): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class CreateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + +CreateEvaluationExperimentConfigOrDict = Union[ + CreateEvaluationExperimentConfig, CreateEvaluationExperimentConfigDict +] + + +class _CreateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] = Field(default=None, description="""""") + labels: Optional[dict[str, str]] = Field(default=None, description="""""") + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""""" + ) + metadata: Optional[dict[str, Any]] = Field(default=None, description="""""") + config: Optional[CreateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _CreateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] + """""" + + labels: Optional[dict[str, str]] + """""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """""" + + metadata: Optional[dict[str, Any]] + """""" + + config: Optional[CreateEvaluationExperimentConfigDict] + """""" + + +_CreateEvaluationExperimentParametersOrDict = Union[ + _CreateEvaluationExperimentParameters, _CreateEvaluationExperimentParametersDict +] + + +class EvaluationExperiment(_common.BaseModel): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] = Field( + default=None, + description="""The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + evaluation_runs: Optional[list[str]] = Field( + default=None, + description="""The EvaluationRuns that are part of this experiment.""", + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""", + ) + create_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was created.""" + ) + update_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was last updated.""" + ) + + +class EvaluationExperimentDict(TypedDict, total=False): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + evaluation_runs: Optional[list[str]] + """The EvaluationRuns that are part of this experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""" + + create_time: Optional[datetime.datetime] + """Timestamp when this experiment was created.""" + + update_time: Optional[datetime.datetime] + """Timestamp when this experiment was last updated.""" + + +EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] + + class CreateEvaluationItemConfig(_common.BaseModel): """Config to create an evaluation item.""" @@ -4012,6 +4137,92 @@ class EvaluationSetDict(TypedDict, total=False): EvaluationSetOrDict = Union[EvaluationSet, EvaluationSetDict] +class DeleteEvaluationExperimentConfig(_common.BaseModel): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class DeleteEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + +DeleteEvaluationExperimentConfigOrDict = Union[ + DeleteEvaluationExperimentConfig, DeleteEvaluationExperimentConfigDict +] + + +class _DeleteEvaluationExperimentParameters(_common.BaseModel): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[DeleteEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _DeleteEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[DeleteEvaluationExperimentConfigDict] + """""" + + +_DeleteEvaluationExperimentParametersOrDict = Union[ + _DeleteEvaluationExperimentParameters, _DeleteEvaluationExperimentParametersDict +] + + +class DeleteEvaluationExperimentOperation(_common.BaseModel): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, + description="""The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""", + ) + done: Optional[bool] = Field( + default=None, + description="""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""", + ) + error: Optional[dict[str, Any]] = Field( + default=None, + description="""The error result of the operation in case of failure or cancellation.""", + ) + + +class DeleteEvaluationExperimentOperationDict(TypedDict, total=False): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] + """The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""" + + metadata: Optional[dict[str, Any]] + """Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""" + + done: Optional[bool] + """If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""" + + error: Optional[dict[str, Any]] + """The error result of the operation in case of failure or cancellation.""" + + +DeleteEvaluationExperimentOperationOrDict = Union[ + DeleteEvaluationExperimentOperation, DeleteEvaluationExperimentOperationDict +] + + class DeleteEvaluationMetricConfig(_common.BaseModel): """Config for deleting an evaluation metric.""" @@ -5805,6 +6016,50 @@ class GenerateInstanceRubricsResponseDict(TypedDict, total=False): ] +class GetEvaluationExperimentConfig(_common.BaseModel): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class GetEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + +GetEvaluationExperimentConfigOrDict = Union[ + GetEvaluationExperimentConfig, GetEvaluationExperimentConfigDict +] + + +class _GetEvaluationExperimentParameters(_common.BaseModel): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[GetEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _GetEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[GetEvaluationExperimentConfigDict] + """""" + + +_GetEvaluationExperimentParametersOrDict = Union[ + _GetEvaluationExperimentParameters, _GetEvaluationExperimentParametersDict +] + + class GetEvaluationMetricConfig(_common.BaseModel): """Config for getting an evaluation metric.""" @@ -5971,6 +6226,111 @@ class _GetEvaluationItemParametersDict(TypedDict, total=False): ] +class ListEvaluationExperimentsConfig(_common.BaseModel): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + page_size: Optional[int] = Field(default=None, description="""""") + page_token: Optional[str] = Field(default=None, description="""""") + filter: Optional[str] = Field( + default=None, + description="""An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""", + ) + order_by: Optional[str] = Field( + default=None, + description="""A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""", + ) + + +class ListEvaluationExperimentsConfigDict(TypedDict, total=False): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + page_size: Optional[int] + """""" + + page_token: Optional[str] + """""" + + filter: Optional[str] + """An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""" + + order_by: Optional[str] + """A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""" + + +ListEvaluationExperimentsConfigOrDict = Union[ + ListEvaluationExperimentsConfig, ListEvaluationExperimentsConfigDict +] + + +class _ListEvaluationExperimentsParameters(_common.BaseModel): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfig] = Field( + default=None, description="""""" + ) + + +class _ListEvaluationExperimentsParametersDict(TypedDict, total=False): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfigDict] + """""" + + +_ListEvaluationExperimentsParametersOrDict = Union[ + _ListEvaluationExperimentsParameters, _ListEvaluationExperimentsParametersDict +] + + +class ListEvaluationExperimentsResponse(_common.BaseModel): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponse] = Field( + default=None, description="""Used to retain the full HTTP response.""" + ) + next_page_token: Optional[str] = Field(default=None, description="""""") + evaluation_experiments: Optional[list[EvaluationExperiment]] = Field( + default=None, + description="""List of evaluation experiments. + """, + ) + + +class ListEvaluationExperimentsResponseDict(TypedDict, total=False): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponse] + """Used to retain the full HTTP response.""" + + next_page_token: Optional[str] + """""" + + evaluation_experiments: Optional[list[EvaluationExperimentDict]] + """List of evaluation experiments. + """ + + +ListEvaluationExperimentsResponseOrDict = Union[ + ListEvaluationExperimentsResponse, ListEvaluationExperimentsResponseDict +] + + class ListEvaluationMetricsConfig(_common.BaseModel): """Config for listing evaluation metrics.""" @@ -6076,6 +6436,85 @@ class ListEvaluationMetricsResponseDict(TypedDict, total=False): ] +class UpdateEvaluationExperimentConfig(_common.BaseModel): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + update_mask: Optional[str] = Field( + default=None, + description="""The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, description="""Metadata about the evaluation experiment.""" + ) + + +class UpdateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + update_mask: Optional[str] + """The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment.""" + + +UpdateEvaluationExperimentConfigOrDict = Union[ + UpdateEvaluationExperimentConfig, UpdateEvaluationExperimentConfigDict +] + + +class _UpdateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, description="""The resource name of the EvaluationExperiment.""" + ) + config: Optional[UpdateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _UpdateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment.""" + + config: Optional[UpdateEvaluationExperimentConfigDict] + """""" + + +_UpdateEvaluationExperimentParametersOrDict = Union[ + _UpdateEvaluationExperimentParameters, _UpdateEvaluationExperimentParametersDict +] + + class OptimizeConfig(_common.BaseModel): """Config for Prompt Optimizer.""" @@ -24912,73 +25351,6 @@ class ObservabilityEvalCaseDict(TypedDict, total=False): ObservabilityEvalCaseOrDict = Union[ObservabilityEvalCase, ObservabilityEvalCaseDict] -class EvaluationExperiment(_common.BaseModel): - """Represents an experiment for iterating on and visualizing evaluation runs.""" - - name: Optional[str] = Field( - default=None, - description="""The resource name of the EvaluationExperiment. Format: - `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", - ) - display_name: Optional[str] = Field( - default=None, description="""The display name of the evaluation experiment.""" - ) - evaluation_runs: Optional[list[str]] = Field( - default=None, - description="""The EvaluationRuns that are part of this experiment.""", - ) - labels: Optional[dict[str, str]] = Field( - default=None, description="""Labels for the evaluation experiment.""" - ) - merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( - default=None, description="""Merge strategy for the evaluation experiment.""" - ) - metadata: Optional[dict[str, Any]] = Field( - default=None, - description="""Metadata about the evaluation experiment, can be used by the caller - to store additional tracking information about the experiment.""", - ) - create_time: Optional[datetime.datetime] = Field( - default=None, description="""Timestamp when this experiment was created.""" - ) - update_time: Optional[datetime.datetime] = Field( - default=None, description="""Timestamp when this experiment was last updated.""" - ) - - -class EvaluationExperimentDict(TypedDict, total=False): - """Represents an experiment for iterating on and visualizing evaluation runs.""" - - name: Optional[str] - """The resource name of the EvaluationExperiment. Format: - `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" - - display_name: Optional[str] - """The display name of the evaluation experiment.""" - - evaluation_runs: Optional[list[str]] - """The EvaluationRuns that are part of this experiment.""" - - labels: Optional[dict[str, str]] - """Labels for the evaluation experiment.""" - - merge_strategy: Optional[EvaluationExperimentMergeStrategy] - """Merge strategy for the evaluation experiment.""" - - metadata: Optional[dict[str, Any]] - """Metadata about the evaluation experiment, can be used by the caller - to store additional tracking information about the experiment.""" - - create_time: Optional[datetime.datetime] - """Timestamp when this experiment was created.""" - - update_time: Optional[datetime.datetime] - """Timestamp when this experiment was last updated.""" - - -EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] - - class RubricGroup(_common.BaseModel): """A group of rubrics. diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index b93b6db39b..f2c847c2c6 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -4154,17 +4154,17 @@ def test_run_inference_with_litellm_parsing( @mock.patch.object(_evals_common, "_get_interactions_client") @mock.patch.object(_evals_utils, "EvalDatasetLoader") def test_run_inference_with_gemini_agent( - self, mock_eval_dataset_loader, mock_get_interactions_client, - mock_fetch_agent_config + self, + mock_eval_dataset_loader, + mock_get_interactions_client, + mock_fetch_agent_config, ): mock_fetch_agent_config.return_value = ( agentplatform_genai_types.evals.AgentConfig( agent_id="test-agent", instruction="You are helpful.", tools=[ - genai_types.Tool( - code_execution=genai_types.ToolCodeExecution() - ), + genai_types.Tool(code_execution=genai_types.ToolCodeExecution()), ], ) ) @@ -4234,7 +4234,9 @@ def make_interaction(interaction_id, prompt_text, output_text): @mock.patch.object(_evals_common, "_get_interactions_client") @mock.patch.object(_evals_utils, "EvalDatasetLoader") def test_run_inference_gemini_agent_continues_on_failure( - self, mock_eval_dataset_loader, mock_get_interactions_client, + self, + mock_eval_dataset_loader, + mock_get_interactions_client, mock_fetch_agent_config, ): mock_fetch_agent_config.return_value = ( @@ -7764,9 +7766,7 @@ def test_merge_single_dataset_with_interactions_data_source(self): ] ) - with mock.patch.object( - _evals_data_converters, "logger" - ) as mock_logger: + with mock.patch.object(_evals_data_converters, "logger") as mock_logger: merged = _evals_data_converters.merge_evaluation_datasets([dataset]) assert len(merged.eval_cases) == 1 @@ -7797,9 +7797,7 @@ def test_merge_two_datasets_with_interactions_data_source(self): ] ) - with mock.patch.object( - _evals_data_converters, "logger" - ) as mock_logger: + with mock.patch.object(_evals_data_converters, "logger") as mock_logger: merged = _evals_data_converters.merge_evaluation_datasets( [dataset_1, dataset_2] ) @@ -7845,9 +7843,7 @@ def test_merge_interactions_data_source_with_response_dataset(self): ] ) - with mock.patch.object( - _evals_data_converters, "logger" - ) as mock_logger: + with mock.patch.object(_evals_data_converters, "logger") as mock_logger: merged = _evals_data_converters.merge_evaluation_datasets( [dataset_interactions, dataset_response] ) @@ -11616,7 +11612,8 @@ def test_code_execution_expands_to_run_command(self): mock_api_client = mock.MagicMock() mock_api_client.request.return_value = self._make_api_response(agent_json) result = _evals_common._fetch_agent_config_dict( - mock_api_client, "projects/p/locations/l/agents/a", + mock_api_client, + "projects/p/locations/l/agents/a", ) assert len(result.tools) == 1 decls = result.tools[0].function_declarations @@ -11630,13 +11627,18 @@ def test_filesystem_expands_to_file_tools(self): mock_api_client = mock.MagicMock() mock_api_client.request.return_value = self._make_api_response(agent_json) result = _evals_common._fetch_agent_config_dict( - mock_api_client, "projects/p/locations/l/agents/a", + mock_api_client, + "projects/p/locations/l/agents/a", ) assert len(result.tools) == 1 names = {fd.name for fd in result.tools[0].function_declarations} assert names == { - "view_file", "create_file", "edit_file", - "list_dir", "delete_file", "move_file", + "view_file", + "create_file", + "edit_file", + "list_dir", + "delete_file", + "move_file", } def test_environment_adds_sandbox_tools(self): @@ -11648,7 +11650,8 @@ def test_environment_adds_sandbox_tools(self): mock_api_client = mock.MagicMock() mock_api_client.request.return_value = self._make_api_response(agent_json) result = _evals_common._fetch_agent_config_dict( - mock_api_client, "projects/p/locations/l/agents/a", + mock_api_client, + "projects/p/locations/l/agents/a", ) # code_execution + sandbox tool assert len(result.tools) == 2 @@ -11673,12 +11676,14 @@ def test_mcp_server_kept_as_named_declaration(self): mock_api_client = mock.MagicMock() mock_api_client.request.return_value = self._make_api_response(agent_json) result = _evals_common._fetch_agent_config_dict( - mock_api_client, "projects/p/locations/l/agents/a", + mock_api_client, + "projects/p/locations/l/agents/a", ) assert len(result.tools) == 2 assert any(t.google_search is not None for t in result.tools) mcp_tool = [ - t for t in result.tools + t + for t in result.tools if t.function_declarations and t.function_declarations[0].name == "mcp_server" ] @@ -11698,9 +11703,7 @@ def test_catalog_in_sync_with_server(self): try: from cloud.ai.platform.evaluation.utils import interaction_converter except ImportError: - pytest.skip( - "interaction_converter not available outside google3" - ) + pytest.skip("interaction_converter not available outside google3") # pylint: enable=g-import-not-at-top # --- Built-in tool types: keys must match --- @@ -11718,7 +11721,9 @@ def test_catalog_in_sync_with_server(self): for tool_type in server_builtin_keys: server_names = { fd.name - for fd in interaction_converter._BUILTIN_TOOL_FUNCTION_DECLARATIONS[tool_type] + for fd in interaction_converter._BUILTIN_TOOL_FUNCTION_DECLARATIONS[ + tool_type + ] } sdk_names = { fd.name @@ -11732,8 +11737,7 @@ def test_catalog_in_sync_with_server(self): # --- Sandbox declarations: names must match --- server_sandbox_names = { - fd.name - for fd in interaction_converter.sandbox_function_declarations() + fd.name for fd in interaction_converter.sandbox_function_declarations() } sdk_sandbox_names = { fd.name for fd in _evals_builtin_tools.SANDBOX_DECLARATIONS @@ -11743,3 +11747,217 @@ def test_catalog_in_sync_with_server(self): f" Server: {sorted(server_sandbox_names)}\n" f" SDK: {sorted(sdk_sandbox_names)}" ) + + +class TestGetEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.experiment_name = ( + "projects/123/locations/us-central1/evaluationExperiments/456" + ) + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": self.experiment_name, + "displayName": "my_experiment", + "evaluationRuns": [ + "projects/123/locations/us-central1/evaluationRuns/789" + ], + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_get_evaluation_experiment_returns_experiment(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + experiment = evals_module.get_evaluation_experiment(name=self.experiment_name) + + assert isinstance(experiment, agentplatform_genai_types.EvaluationExperiment) + assert experiment.name == self.experiment_name + assert experiment.display_name == "my_experiment" + assert experiment.evaluation_runs == [ + "projects/123/locations/us-central1/evaluationRuns/789" + ] + + def test_get_evaluation_experiment_uses_short_name_in_url(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.get_evaluation_experiment(name=self.experiment_name) + + self.mock_api_client.request.assert_called_once() + path = self.mock_api_client.request.call_args[0][1] + assert path == "evaluationExperiments/456" + + def test_get_evaluation_experiment_empty_name_raises(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + with pytest.raises(ValueError, match="name cannot be empty"): + evals_module.get_evaluation_experiment(name="") + + +class TestListEvaluationExperiments: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "evaluationExperiments": [ + { + "name": "projects/123/locations/us-central1/evaluationExperiments/1", + "displayName": "exp_1", + }, + { + "name": "projects/123/locations/us-central1/evaluationExperiments/2", + "displayName": "exp_2", + }, + ] + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_list_evaluation_experiments_returns_experiments(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + response = evals_module.list_evaluation_experiments() + + assert len(response.evaluation_experiments) == 2 + assert response.evaluation_experiments[0].display_name == "exp_1" + assert response.evaluation_experiments[1].display_name == "exp_2" + + def test_list_evaluation_experiments_passes_filter_and_order_by(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.list_evaluation_experiments( + filter='display_name="exp_1"', order_by="create_time desc" + ) + + self.mock_api_client.request.assert_called_once() + path = self.mock_api_client.request.call_args[0][1] + assert path.startswith("evaluationExperiments?") + assert "orderBy=create_time+desc" in path + + +class TestCreateEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": "projects/123/locations/us-central1/evaluationExperiments/456", + "displayName": "my_experiment", + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_create_evaluation_experiment_returns_experiment(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + experiment = evals_module.create_evaluation_experiment( + display_name="my_experiment" + ) + + assert isinstance(experiment, agentplatform_genai_types.EvaluationExperiment) + assert experiment.display_name == "my_experiment" + + def test_create_evaluation_experiment_posts_to_experiments(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.create_evaluation_experiment(display_name="my_experiment") + + self.mock_api_client.request.assert_called_once() + call_args = self.mock_api_client.request.call_args + assert call_args[0][0] == "post" + assert call_args[0][1] == "evaluationExperiments" + request_body = call_args[0][2] + assert request_body.get("displayName") == "my_experiment" + + def test_create_evaluation_experiment_passes_merge_strategy(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.create_evaluation_experiment( + display_name="my_experiment", + merge_strategy=agentplatform_genai_types.EvaluationExperimentMergeStrategy.SHARED_RESULT_SET, + ) + + request_body = self.mock_api_client.request.call_args[0][2] + assert request_body.get("mergeStrategy") == "SHARED_RESULT_SET" + + +class TestUpdateEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.experiment_name = ( + "projects/123/locations/us-central1/evaluationExperiments/456" + ) + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + {"name": self.experiment_name, "displayName": "updated_name"} + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_update_evaluation_experiment_returns_experiment(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + experiment = evals_module.update_evaluation_experiment( + name=self.experiment_name, + config={"display_name": "updated_name", "update_mask": "display_name"}, + ) + + assert isinstance(experiment, agentplatform_genai_types.EvaluationExperiment) + assert experiment.display_name == "updated_name" + + def test_update_evaluation_experiment_uses_patch_and_short_name(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.update_evaluation_experiment( + name=self.experiment_name, + config={"display_name": "updated_name", "update_mask": "display_name"}, + ) + + self.mock_api_client.request.assert_called_once() + call_args = self.mock_api_client.request.call_args + assert call_args[0][0] == "patch" + assert call_args[0][1].startswith("evaluationExperiments/456") + + def test_update_evaluation_experiment_empty_name_raises(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + with pytest.raises(ValueError, match="name cannot be empty"): + evals_module.update_evaluation_experiment(name="") + + +class TestDeleteEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.experiment_name = ( + "projects/123/locations/us-central1/evaluationExperiments/456" + ) + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps({"name": "operations/789"}) + self.mock_api_client.request.return_value = self.mock_response + + def test_delete_evaluation_experiment_uses_delete_and_short_name(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.delete_evaluation_experiment(name=self.experiment_name) + + self.mock_api_client.request.assert_called_once() + call_args = self.mock_api_client.request.call_args + assert call_args[0][0] == "delete" + assert call_args[0][1] == "evaluationExperiments/456" + + def test_delete_evaluation_experiment_empty_name_raises(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + with pytest.raises(ValueError, match="name cannot be empty"): + evals_module.delete_evaluation_experiment(name="") diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index eb55f9a0dd..5907de66cc 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -140,6 +140,85 @@ def _CreateEvaluationRunParameters_to_vertex( return to_object +def _CreateEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["display_name"]) is not None: + setv(to_object, ["displayName"], getv(from_object, ["display_name"])) + + if getv(from_object, ["labels"]) is not None: + setv(to_object, ["labels"], getv(from_object, ["labels"])) + + if getv(from_object, ["merge_strategy"]) is not None: + setv(to_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"])) + + if getv(from_object, ["metadata"]) is not None: + setv(to_object, ["metadata"], getv(from_object, ["metadata"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + +def _DeleteEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + +def _UpdateEvaluationExperimentConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["update_mask"]) is not None: + setv( + parent_object, ["_query", "updateMask"], getv(from_object, ["update_mask"]) + ) + + if getv(from_object, ["display_name"]) is not None: + setv(parent_object, ["displayName"], getv(from_object, ["display_name"])) + + if getv(from_object, ["labels"]) is not None: + setv(parent_object, ["labels"], getv(from_object, ["labels"])) + + if getv(from_object, ["merge_strategy"]) is not None: + setv(parent_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"])) + + if getv(from_object, ["metadata"]) is not None: + setv(parent_object, ["metadata"], getv(from_object, ["metadata"])) + + return to_object + + +def _UpdateEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + _UpdateEvaluationExperimentConfig_to_vertex( + getv(from_object, ["config"]), to_object + ) + + return to_object + + def _CreateEvaluationSetParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -721,6 +800,20 @@ def _GenerateUserScenariosParameters_to_vertex( return to_object +def _GetEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _GetEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -781,6 +874,44 @@ def _GetEvaluationSetParameters_to_vertex( return to_object +def _ListEvaluationExperimentsConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["page_size"]) is not None: + setv(parent_object, ["_query", "pageSize"], getv(from_object, ["page_size"])) + + if getv(from_object, ["page_token"]) is not None: + setv(parent_object, ["_query", "pageToken"], getv(from_object, ["page_token"])) + + if getv(from_object, ["filter"]) is not None: + setv(parent_object, ["_query", "filter"], getv(from_object, ["filter"])) + + if getv(from_object, ["order_by"]) is not None: + setv(parent_object, ["_query", "orderBy"], getv(from_object, ["order_by"])) + + return to_object + + +def _ListEvaluationExperimentsParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["config"]) is not None: + setv( + to_object, + ["config"], + _ListEvaluationExperimentsConfig_to_vertex( + getv(from_object, ["config"]), to_object + ), + ) + + return to_object + + def _ListEvaluationMetricsConfig_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1250,20 +1381,24 @@ def _create_evaluation_run( self._api_client._verify_response(return_value) return return_value - def _create_evaluation_set( + def _create_evaluation_experiment( self, *, - evaluation_items: list[str], display_name: Optional[str] = None, - config: Optional[types.CreateEvaluationSetConfigOrDict] = None, - ) -> types.EvaluationSet: + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: """ - Creates an EvaluationSet. + Creates an EvaluationExperiment. """ - parameter_model = types._CreateEvaluationSetParameters( - evaluation_items=evaluation_items, + parameter_model = types._CreateEvaluationExperimentParameters( display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, config=config, ) @@ -1273,12 +1408,14 @@ def _create_evaluation_set( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _CreateEvaluationSetParameters_to_vertex(parameter_model) + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) request_url_dict = request_dict.get("_url") if request_url_dict: - path = "evaluationSets".format_map(request_url_dict) + path = "evaluationExperiments".format_map(request_url_dict) else: - path = "evaluationSets" + path = "evaluationExperiments" query_params = request_dict.get("_query") if query_params: @@ -1300,7 +1437,7 @@ def _create_evaluation_set( response_dict = {} if not response.body else json.loads(response.body) - return_value = types.EvaluationSet._from_response( + return_value = types.EvaluationExperiment._from_response( response=response_dict, kwargs=( { @@ -1324,18 +1461,18 @@ def _create_evaluation_set( self._api_client._verify_response(return_value) return return_value - def _delete_evaluation_metric( + def _delete_evaluation_experiment( self, *, - metric_resource_name: str, - config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None, - ) -> types.DeleteEvaluationMetricOperation: + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: """ - Deletes an EvaluationMetric. + Deletes an EvaluationExperiment. """ - parameter_model = types._DeleteEvaluationMetricParameters( - metric_resource_name=metric_resource_name, + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, config=config, ) @@ -1345,12 +1482,14 @@ def _delete_evaluation_metric( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _DeleteEvaluationMetricParameters_to_vertex(parameter_model) + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) request_url_dict = request_dict.get("_url") if request_url_dict: - path = "{evaluation_metric}".format_map(request_url_dict) + path = "evaluationExperiments/{name}".format_map(request_url_dict) else: - path = "{evaluation_metric}" + path = "evaluationExperiments/{name}" query_params = request_dict.get("_query") if query_params: @@ -1372,7 +1511,7 @@ def _delete_evaluation_metric( response_dict = {} if not response.body else json.loads(response.body) - return_value = types.DeleteEvaluationMetricOperation._from_response( + return_value = types.DeleteEvaluationExperimentOperation._from_response( response=response_dict, kwargs=( { @@ -1396,48 +1535,18 @@ def _delete_evaluation_metric( self._api_client._verify_response(return_value) return return_value - def _evaluate_instances( + def _update_evaluation_experiment( self, *, - bleu_input: Optional[types.BleuInputOrDict] = None, - exact_match_input: Optional[types.ExactMatchInputOrDict] = None, - rouge_input: Optional[types.RougeInputOrDict] = None, - pointwise_metric_input: Optional[types.PointwiseMetricInputOrDict] = None, - pairwise_metric_input: Optional[types.PairwiseMetricInputOrDict] = None, - tool_call_valid_input: Optional[types.ToolCallValidInputOrDict] = None, - tool_name_match_input: Optional[types.ToolNameMatchInputOrDict] = None, - tool_parameter_key_match_input: Optional[ - types.ToolParameterKeyMatchInputOrDict - ] = None, - tool_parameter_kv_match_input: Optional[ - types.ToolParameterKVMatchInputOrDict - ] = None, - rubric_based_metric_input: Optional[types.RubricBasedMetricInputOrDict] = None, - autorater_config: Optional[genai_types.AutoraterConfigOrDict] = None, - metrics: Optional[list[types.MetricOrDict]] = None, - instance: Optional[types.EvaluationInstanceOrDict] = None, - metric_sources: Optional[list[types.MetricSourceOrDict]] = None, - config: Optional[types.EvaluateInstancesConfigOrDict] = None, - ) -> types.EvaluateInstancesResponse: + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: """ - Evaluates instances based on a given metric. + Updates an EvaluationExperiment. """ - parameter_model = types._EvaluateInstancesRequestParameters( - bleu_input=bleu_input, - exact_match_input=exact_match_input, - rouge_input=rouge_input, - pointwise_metric_input=pointwise_metric_input, - pairwise_metric_input=pairwise_metric_input, - tool_call_valid_input=tool_call_valid_input, - tool_name_match_input=tool_name_match_input, - tool_parameter_key_match_input=tool_parameter_key_match_input, - tool_parameter_kv_match_input=tool_parameter_kv_match_input, - rubric_based_metric_input=rubric_based_metric_input, - autorater_config=autorater_config, - metrics=metrics, - instance=instance, - metric_sources=metric_sources, + parameter_model = types._UpdateEvaluationExperimentParameters( + name=name, config=config, ) @@ -1447,14 +1556,14 @@ def _evaluate_instances( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _EvaluateInstancesRequestParameters_to_vertex( + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( parameter_model ) request_url_dict = request_dict.get("_url") if request_url_dict: - path = ":evaluateInstances".format_map(request_url_dict) + path = "evaluationExperiments/{name}".format_map(request_url_dict) else: - path = ":evaluateInstances" + path = "evaluationExperiments/{name}" query_params = request_dict.get("_query") if query_params: @@ -1472,11 +1581,11 @@ def _evaluate_instances( request_dict = _common.convert_to_dict(request_dict) request_dict = _common.encode_unserializable_types(request_dict) - response = self._api_client.request("post", path, request_dict, http_options) + response = self._api_client.request("patch", path, request_dict, http_options) response_dict = {} if not response.body else json.loads(response.body) - return_value = types.EvaluateInstancesResponse._from_response( + return_value = types.EvaluationExperiment._from_response( response=response_dict, kwargs=( { @@ -1500,29 +1609,21 @@ def _evaluate_instances( self._api_client._verify_response(return_value) return return_value - def _generate_user_scenarios( + def _create_evaluation_set( self, *, - location: Optional[str] = None, - agents: Optional[dict[str, evals_types.AgentConfigOrDict]] = None, - root_agent_id: Optional[str] = None, - user_scenario_generation_config: Optional[ - evals_types.UserScenarioGenerationConfigOrDict - ] = None, - config: Optional[types.GenerateUserScenariosConfigOrDict] = None, - allow_cross_region_model: Optional[bool] = None, - ) -> types.GenerateUserScenariosResponse: + evaluation_items: list[str], + display_name: Optional[str] = None, + config: Optional[types.CreateEvaluationSetConfigOrDict] = None, + ) -> types.EvaluationSet: """ - Generates user scenarios for agent evaluation. + Creates an EvaluationSet. """ - parameter_model = types._GenerateUserScenariosParameters( - location=location, - agents=agents, - root_agent_id=root_agent_id, - user_scenario_generation_config=user_scenario_generation_config, + parameter_model = types._CreateEvaluationSetParameters( + evaluation_items=evaluation_items, + display_name=display_name, config=config, - allow_cross_region_model=allow_cross_region_model, ) request_url_dict: Optional[dict[str, str]] @@ -1531,12 +1632,12 @@ def _generate_user_scenarios( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _GenerateUserScenariosParameters_to_vertex(parameter_model) + request_dict = _CreateEvaluationSetParameters_to_vertex(parameter_model) request_url_dict = request_dict.get("_url") if request_url_dict: - path = ":generateUserScenarios".format_map(request_url_dict) + path = "evaluationSets".format_map(request_url_dict) else: - path = ":generateUserScenarios" + path = "evaluationSets" query_params = request_dict.get("_query") if query_params: @@ -1558,7 +1659,7 @@ def _generate_user_scenarios( response_dict = {} if not response.body else json.loads(response.body) - return_value = types.GenerateUserScenariosResponse._from_response( + return_value = types.EvaluationSet._from_response( response=response_dict, kwargs=( { @@ -1582,24 +1683,18 @@ def _generate_user_scenarios( self._api_client._verify_response(return_value) return return_value - def _generate_loss_clusters( + def _delete_evaluation_metric( self, *, - location: Optional[str] = None, - evaluation_set: Optional[str] = None, - inline_results: Optional[list[types.EvaluationResultOrDict]] = None, - configs: Optional[list[types.LossAnalysisConfigOrDict]] = None, - config: Optional[types.GenerateLossClustersConfigOrDict] = None, - ) -> types.GenerateLossClustersOperation: + metric_resource_name: str, + config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None, + ) -> types.DeleteEvaluationMetricOperation: """ - Generates loss clusters from evaluation results. + Deletes an EvaluationMetric. """ - parameter_model = types._GenerateLossClustersParameters( - location=location, - evaluation_set=evaluation_set, - inline_results=inline_results, - configs=configs, + parameter_model = types._DeleteEvaluationMetricParameters( + metric_resource_name=metric_resource_name, config=config, ) @@ -1609,12 +1704,12 @@ def _generate_loss_clusters( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _GenerateLossClustersParameters_to_vertex(parameter_model) + request_dict = _DeleteEvaluationMetricParameters_to_vertex(parameter_model) request_url_dict = request_dict.get("_url") if request_url_dict: - path = ":generateLossClusters".format_map(request_url_dict) + path = "{evaluation_metric}".format_map(request_url_dict) else: - path = ":generateLossClusters" + path = "{evaluation_metric}" query_params = request_dict.get("_query") if query_params: @@ -1632,11 +1727,11 @@ def _generate_loss_clusters( request_dict = _common.convert_to_dict(request_dict) request_dict = _common.encode_unserializable_types(request_dict) - response = self._api_client.request("post", path, request_dict, http_options) + response = self._api_client.request("delete", path, request_dict, http_options) response_dict = {} if not response.body else json.loads(response.body) - return_value = types.GenerateLossClustersOperation._from_response( + return_value = types.DeleteEvaluationMetricOperation._from_response( response=response_dict, kwargs=( { @@ -1660,26 +1755,48 @@ def _generate_loss_clusters( self._api_client._verify_response(return_value) return return_value - def _generate_rubrics( + def _evaluate_instances( self, *, - contents: list[genai_types.ContentOrDict], - predefined_rubric_generation_spec: Optional[ - genai_types.PredefinedMetricSpecOrDict - ] = None, - rubric_generation_spec: Optional[genai_types.RubricGenerationSpecOrDict] = None, - metric_resource_name: Optional[str] = None, - config: Optional[types.RubricGenerationConfigOrDict] = None, - ) -> types.GenerateInstanceRubricsResponse: + bleu_input: Optional[types.BleuInputOrDict] = None, + exact_match_input: Optional[types.ExactMatchInputOrDict] = None, + rouge_input: Optional[types.RougeInputOrDict] = None, + pointwise_metric_input: Optional[types.PointwiseMetricInputOrDict] = None, + pairwise_metric_input: Optional[types.PairwiseMetricInputOrDict] = None, + tool_call_valid_input: Optional[types.ToolCallValidInputOrDict] = None, + tool_name_match_input: Optional[types.ToolNameMatchInputOrDict] = None, + tool_parameter_key_match_input: Optional[ + types.ToolParameterKeyMatchInputOrDict + ] = None, + tool_parameter_kv_match_input: Optional[ + types.ToolParameterKVMatchInputOrDict + ] = None, + rubric_based_metric_input: Optional[types.RubricBasedMetricInputOrDict] = None, + autorater_config: Optional[genai_types.AutoraterConfigOrDict] = None, + metrics: Optional[list[types.MetricOrDict]] = None, + instance: Optional[types.EvaluationInstanceOrDict] = None, + metric_sources: Optional[list[types.MetricSourceOrDict]] = None, + config: Optional[types.EvaluateInstancesConfigOrDict] = None, + ) -> types.EvaluateInstancesResponse: """ - Generates rubrics for a given prompt. + Evaluates instances based on a given metric. """ - parameter_model = types._GenerateInstanceRubricsRequest( - contents=contents, - predefined_rubric_generation_spec=predefined_rubric_generation_spec, - rubric_generation_spec=rubric_generation_spec, - metric_resource_name=metric_resource_name, + parameter_model = types._EvaluateInstancesRequestParameters( + bleu_input=bleu_input, + exact_match_input=exact_match_input, + rouge_input=rouge_input, + pointwise_metric_input=pointwise_metric_input, + pairwise_metric_input=pairwise_metric_input, + tool_call_valid_input=tool_call_valid_input, + tool_name_match_input=tool_name_match_input, + tool_parameter_key_match_input=tool_parameter_key_match_input, + tool_parameter_kv_match_input=tool_parameter_kv_match_input, + rubric_based_metric_input=rubric_based_metric_input, + autorater_config=autorater_config, + metrics=metrics, + instance=instance, + metric_sources=metric_sources, config=config, ) @@ -1689,12 +1806,14 @@ def _generate_rubrics( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _GenerateInstanceRubricsRequest_to_vertex(parameter_model) + request_dict = _EvaluateInstancesRequestParameters_to_vertex( + parameter_model + ) request_url_dict = request_dict.get("_url") if request_url_dict: - path = ":generateInstanceRubrics".format_map(request_url_dict) + path = ":evaluateInstances".format_map(request_url_dict) else: - path = ":generateInstanceRubrics" + path = ":evaluateInstances" query_params = request_dict.get("_query") if query_params: @@ -1716,7 +1835,7 @@ def _generate_rubrics( response_dict = {} if not response.body else json.loads(response.body) - return_value = types.GenerateInstanceRubricsResponse._from_response( + return_value = types.EvaluateInstancesResponse._from_response( response=response_dict, kwargs=( { @@ -1740,19 +1859,29 @@ def _generate_rubrics( self._api_client._verify_response(return_value) return return_value - def _get_evaluation_metric( + def _generate_user_scenarios( self, *, - metric_resource_name: str, - config: Optional[types.GetEvaluationMetricConfigOrDict] = None, - ) -> types.EvaluationMetric: + location: Optional[str] = None, + agents: Optional[dict[str, evals_types.AgentConfigOrDict]] = None, + root_agent_id: Optional[str] = None, + user_scenario_generation_config: Optional[ + evals_types.UserScenarioGenerationConfigOrDict + ] = None, + config: Optional[types.GenerateUserScenariosConfigOrDict] = None, + allow_cross_region_model: Optional[bool] = None, + ) -> types.GenerateUserScenariosResponse: """ - Retrieves an EvaluationMetric from the resource name. + Generates user scenarios for agent evaluation. """ - parameter_model = types._GetEvaluationMetricParameters( - metric_resource_name=metric_resource_name, + parameter_model = types._GenerateUserScenariosParameters( + location=location, + agents=agents, + root_agent_id=root_agent_id, + user_scenario_generation_config=user_scenario_generation_config, config=config, + allow_cross_region_model=allow_cross_region_model, ) request_url_dict: Optional[dict[str, str]] @@ -1761,12 +1890,12 @@ def _get_evaluation_metric( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _GetEvaluationMetricParameters_to_vertex(parameter_model) + request_dict = _GenerateUserScenariosParameters_to_vertex(parameter_model) request_url_dict = request_dict.get("_url") if request_url_dict: - path = "{evaluation_metric}".format_map(request_url_dict) + path = ":generateUserScenarios".format_map(request_url_dict) else: - path = "{evaluation_metric}" + path = ":generateUserScenarios" query_params = request_dict.get("_query") if query_params: @@ -1784,14 +1913,11 @@ def _get_evaluation_metric( request_dict = _common.convert_to_dict(request_dict) request_dict = _common.encode_unserializable_types(request_dict) - response = self._api_client.request("get", path, request_dict, http_options) + response = self._api_client.request("post", path, request_dict, http_options) response_dict = {} if not response.body else json.loads(response.body) - if self._api_client.vertexai: - response_dict = _EvaluationMetric_from_vertex(response_dict) - - return_value = types.EvaluationMetric._from_response( + return_value = types.GenerateUserScenariosResponse._from_response( response=response_dict, kwargs=( { @@ -1815,15 +1941,24 @@ def _get_evaluation_metric( self._api_client._verify_response(return_value) return return_value - def _get_evaluation_run( - self, *, name: str, config: Optional[types.GetEvaluationRunConfigOrDict] = None - ) -> types.EvaluationRun: + def _generate_loss_clusters( + self, + *, + location: Optional[str] = None, + evaluation_set: Optional[str] = None, + inline_results: Optional[list[types.EvaluationResultOrDict]] = None, + configs: Optional[list[types.LossAnalysisConfigOrDict]] = None, + config: Optional[types.GenerateLossClustersConfigOrDict] = None, + ) -> types.GenerateLossClustersOperation: """ - Retrieves an EvaluationRun from the resource name. + Generates loss clusters from evaluation results. """ - parameter_model = types._GetEvaluationRunParameters( - name=name, + parameter_model = types._GenerateLossClustersParameters( + location=location, + evaluation_set=evaluation_set, + inline_results=inline_results, + configs=configs, config=config, ) @@ -1833,12 +1968,12 @@ def _get_evaluation_run( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _GetEvaluationRunParameters_to_vertex(parameter_model) + request_dict = _GenerateLossClustersParameters_to_vertex(parameter_model) request_url_dict = request_dict.get("_url") if request_url_dict: - path = "evaluationRuns/{name}".format_map(request_url_dict) + path = ":generateLossClusters".format_map(request_url_dict) else: - path = "evaluationRuns/{name}" + path = ":generateLossClusters" query_params = request_dict.get("_query") if query_params: @@ -1856,14 +1991,11 @@ def _get_evaluation_run( request_dict = _common.convert_to_dict(request_dict) request_dict = _common.encode_unserializable_types(request_dict) - response = self._api_client.request("get", path, request_dict, http_options) + response = self._api_client.request("post", path, request_dict, http_options) response_dict = {} if not response.body else json.loads(response.body) - if self._api_client.vertexai: - response_dict = _EvaluationRun_from_vertex(response_dict) - - return_value = types.EvaluationRun._from_response( + return_value = types.GenerateLossClustersOperation._from_response( response=response_dict, kwargs=( { @@ -1887,15 +2019,26 @@ def _get_evaluation_run( self._api_client._verify_response(return_value) return return_value - def _get_evaluation_set( - self, *, name: str, config: Optional[types.GetEvaluationSetConfigOrDict] = None - ) -> types.EvaluationSet: + def _generate_rubrics( + self, + *, + contents: list[genai_types.ContentOrDict], + predefined_rubric_generation_spec: Optional[ + genai_types.PredefinedMetricSpecOrDict + ] = None, + rubric_generation_spec: Optional[genai_types.RubricGenerationSpecOrDict] = None, + metric_resource_name: Optional[str] = None, + config: Optional[types.RubricGenerationConfigOrDict] = None, + ) -> types.GenerateInstanceRubricsResponse: """ - Retrieves an EvaluationSet from the resource name. + Generates rubrics for a given prompt. """ - parameter_model = types._GetEvaluationSetParameters( - name=name, + parameter_model = types._GenerateInstanceRubricsRequest( + contents=contents, + predefined_rubric_generation_spec=predefined_rubric_generation_spec, + rubric_generation_spec=rubric_generation_spec, + metric_resource_name=metric_resource_name, config=config, ) @@ -1905,12 +2048,12 @@ def _get_evaluation_set( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _GetEvaluationSetParameters_to_vertex(parameter_model) + request_dict = _GenerateInstanceRubricsRequest_to_vertex(parameter_model) request_url_dict = request_dict.get("_url") if request_url_dict: - path = "evaluationSets/{name}".format_map(request_url_dict) + path = ":generateInstanceRubrics".format_map(request_url_dict) else: - path = "evaluationSets/{name}" + path = ":generateInstanceRubrics" query_params = request_dict.get("_query") if query_params: @@ -1928,11 +2071,11 @@ def _get_evaluation_set( request_dict = _common.convert_to_dict(request_dict) request_dict = _common.encode_unserializable_types(request_dict) - response = self._api_client.request("get", path, request_dict, http_options) + response = self._api_client.request("post", path, request_dict, http_options) response_dict = {} if not response.body else json.loads(response.body) - return_value = types.EvaluationSet._from_response( + return_value = types.GenerateInstanceRubricsResponse._from_response( response=response_dict, kwargs=( { @@ -1956,14 +2099,17 @@ def _get_evaluation_set( self._api_client._verify_response(return_value) return return_value - def _get_evaluation_item( - self, *, name: str, config: Optional[types.GetEvaluationItemConfigOrDict] = None - ) -> types.EvaluationItem: + def _get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: """ - Retrieves an EvaluationItem from the resource name. + Retrieves an EvaluationExperiment from the resource name. """ - parameter_model = types._GetEvaluationItemParameters( + parameter_model = types._GetEvaluationExperimentParameters( name=name, config=config, ) @@ -1974,12 +2120,12 @@ def _get_evaluation_item( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _GetEvaluationItemParameters_to_vertex(parameter_model) + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) request_url_dict = request_dict.get("_url") if request_url_dict: - path = "evaluationItems/{name}".format_map(request_url_dict) + path = "evaluationExperiments/{name}".format_map(request_url_dict) else: - path = "evaluationItems/{name}" + path = "evaluationExperiments/{name}" query_params = request_dict.get("_query") if query_params: @@ -2001,7 +2147,7 @@ def _get_evaluation_item( response_dict = {} if not response.body else json.loads(response.body) - return_value = types.EvaluationItem._from_response( + return_value = types.EvaluationExperiment._from_response( response=response_dict, kwargs=( { @@ -2025,14 +2171,18 @@ def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value - def _list_evaluation_metrics( - self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None - ) -> types.ListEvaluationMetricsResponse: + def _get_evaluation_metric( + self, + *, + metric_resource_name: str, + config: Optional[types.GetEvaluationMetricConfigOrDict] = None, + ) -> types.EvaluationMetric: """ - Lists EvaluationMetrics. + Retrieves an EvaluationMetric from the resource name. """ - parameter_model = types._ListEvaluationMetricsParameters( + parameter_model = types._GetEvaluationMetricParameters( + metric_resource_name=metric_resource_name, config=config, ) @@ -2042,12 +2192,12 @@ def _list_evaluation_metrics( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _ListEvaluationMetricsParameters_to_vertex(parameter_model) + request_dict = _GetEvaluationMetricParameters_to_vertex(parameter_model) request_url_dict = request_dict.get("_url") if request_url_dict: - path = "evaluationMetrics".format_map(request_url_dict) + path = "{evaluation_metric}".format_map(request_url_dict) else: - path = "evaluationMetrics" + path = "{evaluation_metric}" query_params = request_dict.get("_query") if query_params: @@ -2070,9 +2220,9 @@ def _list_evaluation_metrics( response_dict = {} if not response.body else json.loads(response.body) if self._api_client.vertexai: - response_dict = _ListEvaluationMetricsResponse_from_vertex(response_dict) + response_dict = _EvaluationMetric_from_vertex(response_dict) - return_value = types.ListEvaluationMetricsResponse._from_response( + return_value = types.EvaluationMetric._from_response( response=response_dict, kwargs=( { @@ -2096,221 +2246,102 @@ def _list_evaluation_metrics( self._api_client._verify_response(return_value) return return_value - def evaluate_instances( - self, - *, - metric_config: types._EvaluateInstancesRequestParameters, - ) -> types.EvaluateInstancesResponse: - """Evaluates an instance of a model.""" + def _get_evaluation_run( + self, *, name: str, config: Optional[types.GetEvaluationRunConfigOrDict] = None + ) -> types.EvaluationRun: + """ + Retrieves an EvaluationRun from the resource name. + """ - if isinstance(metric_config, types._EvaluateInstancesRequestParameters): - metric_config = metric_config.model_dump() # type: ignore[assignment] + parameter_model = types._GetEvaluationRunParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) else: - metric_config = dict(metric_config) + request_dict = _GetEvaluationRunParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationRuns/{name}".format_map(request_url_dict) + else: + path = "evaluationRuns/{name}" - return self._evaluate_instances( - **metric_config, - ) + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) - def run_inference( - self, - *, - src: Union[str, pd.DataFrame, types.EvaluationDataset], - model: Optional[Union[str, Callable[[Any], Any]]] = None, - agent: Optional[Union[str, types.AgentEngine, LlmAgent]] = None, - location: Optional[str] = None, - config: Optional[types.EvalRunInferenceConfigOrDict] = None, - ) -> types.EvaluationDataset: - """Runs inference on a dataset for evaluation. + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options - Args: - src: The source of the dataset. Can be a string (path to a local file, - a GCS path, or a BigQuery table), a Pandas DataFrame, or an - EvaluationDataset object. An EvaluationDataset may have either - ``eval_dataset_df`` or ``eval_cases`` populated. When - ``eval_cases`` with ``agent_data`` is provided, the last user - event in the turns is used as the current prompt and prior - events are replayed as session history for local ADK agents. - model: Optional type is experimental and may change in future versions. - The model to use for inference, optional for agent evaluations. - - For Google Gemini models, provide the model name string (e.g., "gemini-2.5-flash"). - - For third-party models via LiteLLM, use the format "provider/model_name" - (e.g., "openai/gpt-4o"). Ensure the necessary API key (e.g., OPENAI_API_KEY) - is set as an environment variable. - - For custom logic, provide a callable function that accepts a prompt and - returns a response. - agent: This field is experimental and may change in future versions - The agent engine used or local agent to run agent, optional for non-agent evaluations. - - agent engine resource name in str type, with format - `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`, - run_inference will fetch the agent engine from the resource name. - - Or `types.AgentEngine` object. - - Or ADK agent in LlMAgent type. - location: The location to use for the inference. If not specified, the - location configured in the client will be used. If specified, - this will override the location set in `vertexai.Client` only - for this API call. - config: The optional configuration for the inference run. Must be a dict or - `types.EvalRunInferenceConfig` type. - - dest: The destination path for storage of the inference results. - - prompt_template: The template string to use for constructing prompts. - - generate_content_config: The config for the Gemini generate content call. - - allow_cross_region_model: Opt-in flag to authorize cross-region routing for LLM models. + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) - Returns: - The evaluation dataset. - """ - if not config: - config = types.EvalRunInferenceConfig() - if isinstance(config, dict): - config = types.EvalRunInferenceConfig.model_validate(config) + response = self._api_client.request("get", path, request_dict, http_options) - if isinstance(src, types.EvaluationDataset): - if src.eval_dataset_df is not None: - src = src.eval_dataset_df - elif src.eval_cases: - src = _evals_common._eval_cases_to_dataframe(src.eval_cases) - else: - raise ValueError( - "EvaluationDataset must have eval_dataset_df or eval_cases" - " populated." - ) + response_dict = {} if not response.body else json.loads(response.body) - agent_engine_instance = None - agent_instance = None - if agent: - if isinstance(agent, str) or isinstance(agent, types.AgentEngine): - agent_engine_instance = agent - else: - agent_instance = agent + if self._api_client.vertexai: + response_dict = _EvaluationRun_from_vertex(response_dict) - return _evals_common._execute_inference( # type: ignore[no-any-return] - api_client=self._api_client, - model=model, - agent_engine=agent_engine_instance, - agent=agent_instance, - src=src, - dest=config.dest, - prompt_template=config.prompt_template, - location=location, - config=config.generate_content_config, - user_simulator_config=getattr(config, "user_simulator_config", None), - allow_cross_region_model=getattr(config, "allow_cross_region_model", False), + return_value = types.EvaluationRun._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), ) - def evaluate( - self, - *, - dataset: Union[ - pd.DataFrame, - types.EvaluationDatasetOrDict, - list[types.EvaluationDatasetOrDict], - ], - metrics: Optional[list[types.MetricOrDict]] = None, - location: Optional[str] = None, - config: Optional[types.EvaluateMethodConfigOrDict] = None, - **kwargs: Any, - ) -> types.EvaluationResult: - """Evaluates candidate responses in the provided dataset(s) using the specified metrics. - - Args: - dataset: The dataset(s) to evaluate. Can be a pandas DataFrame, a single - `types.EvaluationDataset` or a list of `types.EvaluationDataset`. - metrics: The list of metrics to use for evaluation. - location: The location to use for the evaluation service. If not specified, - the location configured in the client will be used. If specified, - this will override the location set in `vertexai.Client` only for - this API call. - config: Optional configuration for the evaluation. Can be a dictionary or a - `types.EvaluateMethodConfig` object. - - dataset_schema: Schema to use for the dataset. If not specified, the - dataset schema will be inferred from the dataset automatically. - - dest: Destination path for storing evaluation results. - - evaluation_service_qps: The rate limit (queries per second) for - calls to the evaluation service. Defaults to 10. Increase this - value if your project has a higher EvaluateInstances API quota. - **kwargs: Extra arguments to pass to evaluation, such as `agent_info`. + self._api_client._verify_response(return_value) + return return_value - Returns: - The evaluation result. + def _get_evaluation_set( + self, *, name: str, config: Optional[types.GetEvaluationSetConfigOrDict] = None + ) -> types.EvaluationSet: + """ + Retrieves an EvaluationSet from the resource name. """ - if not config: - config = types.EvaluateMethodConfig() - if isinstance(config, dict): - config = types.EvaluateMethodConfig.model_validate(config) - - if isinstance(dataset, pd.DataFrame): - dataset = types.EvaluationDataset(eval_dataset_df=dataset) - - if isinstance(dataset, list): - dataset = [ - ( - types.EvaluationDataset.model_validate(ds_item) - if isinstance(ds_item, dict) - else ds_item - ) - for ds_item in dataset - ] - else: - if isinstance(dataset, dict): - dataset = types.EvaluationDataset.model_validate(dataset) - if metrics is None: - metrics = [types.Metric(name="general_quality_v1")] - - # TODO: Replace kwargs with agent_info after the experimental phase. - if kwargs: - logger.warning( - "`kwargs` attribute in `evaluate` method is experimental and may change in future versions." - ) - - return _evals_common._execute_evaluation( - api_client=self._api_client, - dataset=dataset, - metrics=metrics, - dataset_schema=config.dataset_schema, - dest=config.dest, - location=location, - evaluation_service_qps=getattr(config, "evaluation_service_qps", None), - **kwargs, - ) - - def batch_evaluate( - self, - *, - dataset: types.EvaluationDatasetOrDict, - metrics: list[types.MetricOrDict], - dest: str, - config: Optional[types.EvaluateDatasetConfigOrDict] = None, - ) -> types.EvaluateDatasetOperation: - """Evaluates a dataset based on a set of given metrics.""" - resolved_metrics = _evals_common._resolve_metrics(metrics, self._api_client) - output_config = genai_types.OutputConfig( - gcs_destination=genai_types.GcsDestination(output_uri_prefix=dest) - ) - parameter_model = types.EvaluateDatasetRequestParameters( - dataset=dataset, - metrics=resolved_metrics, - output_config=output_config, + parameter_model = types._GetEvaluationSetParameters( + name=name, config=config, ) request_url_dict: Optional[dict[str, str]] if not self._api_client.vertexai: - raise ValueError("This method is only supported in the Vertex AI client.") - else: - request_dict = _evals_utils.BatchEvaluateRequestPreparer.EvaluateDatasetRequestParameters_to_vertex( - parameter_model + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) + else: + request_dict = _GetEvaluationSetParameters_to_vertex(parameter_model) request_url_dict = request_dict.get("_url") if request_url_dict: - path = ":evaluateDataset".format_map(request_url_dict) + path = "evaluationSets/{name}".format_map(request_url_dict) else: - path = ":evaluateDataset" - - request_dict = _evals_utils.BatchEvaluateRequestPreparer.prepare_metric_payload( - request_dict, resolved_metrics - ) + path = "evaluationSets/{name}" query_params = request_dict.get("_query") if query_params: @@ -2328,41 +2359,511 @@ def batch_evaluate( request_dict = _common.convert_to_dict(request_dict) request_dict = _common.encode_unserializable_types(request_dict) - response = self._api_client.request("post", path, request_dict, http_options) - - response_dict = "" if not response.body else json.loads(response.body) + response = self._api_client.request("get", path, request_dict, http_options) - if self._api_client.vertexai: - response_dict = _evals_utils.BatchEvaluateRequestPreparer.EvaluateDatasetOperation_from_vertex( - response_dict - ) + response_dict = {} if not response.body else json.loads(response.body) - return_value = types.EvaluateDatasetOperation._from_response( - response=response_dict, kwargs=parameter_model.model_dump() + return_value = types.EvaluationSet._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), ) - self._api_client._verify_response(return_value) + self._api_client._verify_response(return_value) return return_value - def generate_rubrics( - self, - *, - src: Union[str, "pd.DataFrame", types.EvaluationDataset], - rubric_group_name: str, - prompt_template: Optional[str] = None, - generator_model_config: Optional["genai_types.AutoraterConfigOrDict"] = None, - rubric_content_type: Optional["types.RubricContentType"] = None, - rubric_type_ontology: Optional[list[str]] = None, - predefined_spec_name: Optional[Union[str, "types.PrebuiltMetric"]] = None, - metric_spec_parameters: Optional[dict[str, Any]] = None, - metric: Optional[types.MetricOrDict] = None, - config: Optional[types.RubricGenerationConfigOrDict] = None, - ) -> types.EvaluationDataset: - """Generates rubrics for each prompt in the source and adds them as a new column - structured as a dictionary. - - You can generate rubrics by providing either: - 1. A `metric` to use a pre-registered metric resource. + def _get_evaluation_item( + self, *, name: str, config: Optional[types.GetEvaluationItemConfigOrDict] = None + ) -> types.EvaluationItem: + """ + Retrieves an EvaluationItem from the resource name. + """ + + parameter_model = types._GetEvaluationItemParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationItemParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationItems/{name}".format_map(request_url_dict) + else: + path = "evaluationItems/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationItem._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + def _list_evaluation_metrics( + self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None + ) -> types.ListEvaluationMetricsResponse: + """ + Lists EvaluationMetrics. + """ + + parameter_model = types._ListEvaluationMetricsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationMetricsParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationMetrics".format_map(request_url_dict) + else: + path = "evaluationMetrics" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + if self._api_client.vertexai: + response_dict = _ListEvaluationMetricsResponse_from_vertex(response_dict) + + return_value = types.ListEvaluationMetricsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + def evaluate_instances( + self, + *, + metric_config: types._EvaluateInstancesRequestParameters, + ) -> types.EvaluateInstancesResponse: + """Evaluates an instance of a model.""" + + if isinstance(metric_config, types._EvaluateInstancesRequestParameters): + metric_config = metric_config.model_dump() # type: ignore[assignment] + else: + metric_config = dict(metric_config) + + return self._evaluate_instances( + **metric_config, + ) + + def run_inference( + self, + *, + src: Union[str, pd.DataFrame, types.EvaluationDataset], + model: Optional[Union[str, Callable[[Any], Any]]] = None, + agent: Optional[Union[str, types.AgentEngine, LlmAgent]] = None, + location: Optional[str] = None, + config: Optional[types.EvalRunInferenceConfigOrDict] = None, + ) -> types.EvaluationDataset: + """Runs inference on a dataset for evaluation. + + Args: + src: The source of the dataset. Can be a string (path to a local file, + a GCS path, or a BigQuery table), a Pandas DataFrame, or an + EvaluationDataset object. An EvaluationDataset may have either + ``eval_dataset_df`` or ``eval_cases`` populated. When + ``eval_cases`` with ``agent_data`` is provided, the last user + event in the turns is used as the current prompt and prior + events are replayed as session history for local ADK agents. + model: Optional type is experimental and may change in future versions. + The model to use for inference, optional for agent evaluations. + - For Google Gemini models, provide the model name string (e.g., "gemini-2.5-flash"). + - For third-party models via LiteLLM, use the format "provider/model_name" + (e.g., "openai/gpt-4o"). Ensure the necessary API key (e.g., OPENAI_API_KEY) + is set as an environment variable. + - For custom logic, provide a callable function that accepts a prompt and + returns a response. + agent: This field is experimental and may change in future versions + The agent engine used or local agent to run agent, optional for non-agent evaluations. + - agent engine resource name in str type, with format + `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`, + run_inference will fetch the agent engine from the resource name. + - Or `types.AgentEngine` object. + - Or ADK agent in LlMAgent type. + location: The location to use for the inference. If not specified, the + location configured in the client will be used. If specified, + this will override the location set in `vertexai.Client` only + for this API call. + config: The optional configuration for the inference run. Must be a dict or + `types.EvalRunInferenceConfig` type. + - dest: The destination path for storage of the inference results. + - prompt_template: The template string to use for constructing prompts. + - generate_content_config: The config for the Gemini generate content call. + - allow_cross_region_model: Opt-in flag to authorize cross-region routing for LLM models. + + Returns: + The evaluation dataset. + """ + if not config: + config = types.EvalRunInferenceConfig() + if isinstance(config, dict): + config = types.EvalRunInferenceConfig.model_validate(config) + + if isinstance(src, types.EvaluationDataset): + if src.eval_dataset_df is not None: + src = src.eval_dataset_df + elif src.eval_cases: + src = _evals_common._eval_cases_to_dataframe(src.eval_cases) + else: + raise ValueError( + "EvaluationDataset must have eval_dataset_df or eval_cases" + " populated." + ) + + agent_engine_instance = None + agent_instance = None + if agent: + if isinstance(agent, str) or isinstance(agent, types.AgentEngine): + agent_engine_instance = agent + else: + agent_instance = agent + + return _evals_common._execute_inference( # type: ignore[no-any-return] + api_client=self._api_client, + model=model, + agent_engine=agent_engine_instance, + agent=agent_instance, + src=src, + dest=config.dest, + prompt_template=config.prompt_template, + location=location, + config=config.generate_content_config, + user_simulator_config=getattr(config, "user_simulator_config", None), + allow_cross_region_model=getattr(config, "allow_cross_region_model", False), + ) + + def evaluate( + self, + *, + dataset: Union[ + pd.DataFrame, + types.EvaluationDatasetOrDict, + list[types.EvaluationDatasetOrDict], + ], + metrics: Optional[list[types.MetricOrDict]] = None, + location: Optional[str] = None, + config: Optional[types.EvaluateMethodConfigOrDict] = None, + **kwargs: Any, + ) -> types.EvaluationResult: + """Evaluates candidate responses in the provided dataset(s) using the specified metrics. + + Args: + dataset: The dataset(s) to evaluate. Can be a pandas DataFrame, a single + `types.EvaluationDataset` or a list of `types.EvaluationDataset`. + metrics: The list of metrics to use for evaluation. + location: The location to use for the evaluation service. If not specified, + the location configured in the client will be used. If specified, + this will override the location set in `vertexai.Client` only for + this API call. + config: Optional configuration for the evaluation. Can be a dictionary or a + `types.EvaluateMethodConfig` object. + - dataset_schema: Schema to use for the dataset. If not specified, the + dataset schema will be inferred from the dataset automatically. + - dest: Destination path for storing evaluation results. + - evaluation_service_qps: The rate limit (queries per second) for + calls to the evaluation service. Defaults to 10. Increase this + value if your project has a higher EvaluateInstances API quota. + **kwargs: Extra arguments to pass to evaluation, such as `agent_info`. + + Returns: + The evaluation result. + """ + if not config: + config = types.EvaluateMethodConfig() + if isinstance(config, dict): + config = types.EvaluateMethodConfig.model_validate(config) + + if isinstance(dataset, pd.DataFrame): + dataset = types.EvaluationDataset(eval_dataset_df=dataset) + + if isinstance(dataset, list): + dataset = [ + ( + types.EvaluationDataset.model_validate(ds_item) + if isinstance(ds_item, dict) + else ds_item + ) + for ds_item in dataset + ] + else: + if isinstance(dataset, dict): + dataset = types.EvaluationDataset.model_validate(dataset) + if metrics is None: + metrics = [types.Metric(name="general_quality_v1")] + + # TODO: Replace kwargs with agent_info after the experimental phase. + if kwargs: + logger.warning( + "`kwargs` attribute in `evaluate` method is experimental and may change in future versions." + ) + + return _evals_common._execute_evaluation( + api_client=self._api_client, + dataset=dataset, + metrics=metrics, + dataset_schema=config.dataset_schema, + dest=config.dest, + location=location, + evaluation_service_qps=getattr(config, "evaluation_service_qps", None), + **kwargs, + ) + + def batch_evaluate( + self, + *, + dataset: types.EvaluationDatasetOrDict, + metrics: list[types.MetricOrDict], + dest: str, + config: Optional[types.EvaluateDatasetConfigOrDict] = None, + ) -> types.EvaluateDatasetOperation: + """Evaluates a dataset based on a set of given metrics.""" + + resolved_metrics = _evals_common._resolve_metrics(metrics, self._api_client) + output_config = genai_types.OutputConfig( + gcs_destination=genai_types.GcsDestination(output_uri_prefix=dest) + ) + parameter_model = types.EvaluateDatasetRequestParameters( + dataset=dataset, + metrics=resolved_metrics, + output_config=output_config, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError("This method is only supported in the Vertex AI client.") + else: + request_dict = _evals_utils.BatchEvaluateRequestPreparer.EvaluateDatasetRequestParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = ":evaluateDataset".format_map(request_url_dict) + else: + path = ":evaluateDataset" + + request_dict = _evals_utils.BatchEvaluateRequestPreparer.prepare_metric_payload( + request_dict, resolved_metrics + ) + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("post", path, request_dict, http_options) + + response_dict = "" if not response.body else json.loads(response.body) + + if self._api_client.vertexai: + response_dict = _evals_utils.BatchEvaluateRequestPreparer.EvaluateDatasetOperation_from_vertex( + response_dict + ) + + return_value = types.EvaluateDatasetOperation._from_response( + response=response_dict, kwargs=parameter_model.model_dump() + ) + self._api_client._verify_response(return_value) + + return return_value + + def generate_rubrics( + self, + *, + src: Union[str, "pd.DataFrame", types.EvaluationDataset], + rubric_group_name: str, + prompt_template: Optional[str] = None, + generator_model_config: Optional["genai_types.AutoraterConfigOrDict"] = None, + rubric_content_type: Optional["types.RubricContentType"] = None, + rubric_type_ontology: Optional[list[str]] = None, + predefined_spec_name: Optional[Union[str, "types.PrebuiltMetric"]] = None, + metric_spec_parameters: Optional[dict[str, Any]] = None, + metric: Optional[types.MetricOrDict] = None, + config: Optional[types.RubricGenerationConfigOrDict] = None, + ) -> types.EvaluationDataset: + """Generates rubrics for each prompt in the source and adds them as a new column + structured as a dictionary. + + You can generate rubrics by providing either: + 1. A `metric` to use a pre-registered metric resource. 2. A `predefined_spec_name` to use a Vertex AI backend recipe. 3. A `prompt_template` along with other configuration parameters (`generator_model_config`, `rubric_content_type`, `rubric_type_ontology`) @@ -2773,6 +3274,92 @@ def create_evaluation_run( config=config, ) + def create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Creates an EvaluationExperiment. + + Args: + display_name: The display name of the evaluation experiment. + labels: Labels for the evaluation experiment. + merge_strategy: Merge strategy for the evaluation experiment. + metadata: Metadata about the evaluation experiment, can be used by the + caller to store additional tracking information about the experiment. + config: Optional configuration for the create operation. + + Returns: + The created evaluation experiment. + """ + return self._create_evaluation_experiment( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + def update_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Updates an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to update. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration specifying the fields to update (e.g. + display_name, labels, merge_strategy, metadata) and the update_mask. + + Returns: + The updated evaluation experiment. + + Raises: + ValueError: If the name is empty. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + return self._update_evaluation_experiment(name=name, config=config) + + @_common.experimental_warning( + "The Vertex SDK GenAI evals.delete_evaluation_experiment method is" + " experimental, and may change in future versions." + ) + + def delete_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: + """Deletes an EvaluationExperiment. + + Args: + name: The resource name of the EvaluationExperiment to delete. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: Optional configuration for the delete operation. + + Returns: + The delete operation. + + Raises: + ValueError: If the name is empty. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + return self._delete_evaluation_experiment(name=name, config=config) + def get_evaluation_set( self, *, @@ -3031,36 +3618,122 @@ def create_evaluation_metric( metric=metric, config=config, ) - # result.name is Optional[str], but we know it's always returned on creation - return cast(str, result.name) + # result.name is Optional[str], but we know it's always returned on creation + return cast(str, result.name) + + def get_evaluation_metric( + self, + *, + metric_resource_name: str, + config: Optional[types.GetEvaluationMetricConfigOrDict] = None, + ) -> types.EvaluationMetric: + """Retrieves an EvaluationMetric from the resource name.""" + return self._get_evaluation_metric( + metric_resource_name=metric_resource_name, + config=config, + ) + + def list_evaluation_metrics( + self, + *, + filter: Optional[str] = None, + order_by: Optional[str] = None, + config: Optional[types.ListEvaluationMetricsConfigOrDict] = None, + ) -> types.ListEvaluationMetricsResponse: + """Lists EvaluationMetrics. + + Args: + filter: An expression for filtering the results of the request. For + field names both snake_case and camelCase are supported. For more + information about filter syntax, see + `AIP-160 `_. + Example: ``'display_name="my_metric"'``. + order_by: A comma-separated list of fields to order by, sorted in + ascending order by default. Use ``desc`` after a field name for + descending. Example: ``"create_time desc"``. + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. Top-level ``filter`` and ``order_by`` arguments + take precedence over values set in ``config``. + + Returns: + The list evaluation metrics response. + """ + if config is None: + config = types.ListEvaluationMetricsConfig() + if isinstance(config, dict): + config = types.ListEvaluationMetricsConfig.model_validate(config) + if filter is not None: + config.filter = filter + if order_by is not None: + config.order_by = order_by + return self._list_evaluation_metrics( + config=config, + ) - def get_evaluation_metric( + def delete_evaluation_metric( self, *, metric_resource_name: str, - config: Optional[types.GetEvaluationMetricConfigOrDict] = None, - ) -> types.EvaluationMetric: - """Retrieves an EvaluationMetric from the resource name.""" - return self._get_evaluation_metric( + config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None, + ) -> None: + """Deletes an EvaluationMetric. + + Args: + metric_resource_name: The resource name of the EvaluationMetric to delete. + Format: + `projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric}` + config: The optional configuration for the delete operation. + """ + self._delete_evaluation_metric( metric_resource_name=metric_resource_name, config=config, ) - def list_evaluation_metrics( + def get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """Retrieves an EvaluationExperiment from the resource name. + + Args: + name: The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}` + config: The optional configuration for the get operation. + + Returns: + The evaluation experiment. + + Raises: + ValueError: If the name is empty. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + return self._get_evaluation_experiment(name=name, config=config) + + @_common.experimental_warning( + "The Vertex SDK GenAI evals.list_evaluation_experiments method is" + " experimental, and may change in future versions." + ) + def list_evaluation_experiments( self, *, filter: Optional[str] = None, order_by: Optional[str] = None, - config: Optional[types.ListEvaluationMetricsConfigOrDict] = None, - ) -> types.ListEvaluationMetricsResponse: - """Lists EvaluationMetrics. + config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None, + ) -> types.ListEvaluationExperimentsResponse: + """Lists EvaluationExperiments. Args: filter: An expression for filtering the results of the request. For field names both snake_case and camelCase are supported. For more information about filter syntax, see `AIP-160 `_. - Example: ``'display_name="my_metric"'``. + Example: ``'display_name="my_experiment"'``. order_by: A comma-separated list of fields to order by, sorted in ascending order by default. Use ``desc`` after a field name for descending. Example: ``"create_time desc"``. @@ -3070,58 +3743,291 @@ def list_evaluation_metrics( take precedence over values set in ``config``. Returns: - The list evaluation metrics response. + The list evaluation experiments response. """ if config is None: - config = types.ListEvaluationMetricsConfig() + config = types.ListEvaluationExperimentsConfig() if isinstance(config, dict): - config = types.ListEvaluationMetricsConfig.model_validate(config) + config = types.ListEvaluationExperimentsConfig.model_validate(config) if filter is not None: config.filter = filter if order_by is not None: config.order_by = order_by - return self._list_evaluation_metrics( + return self._list_evaluation_experiments( + config=config, + ) + + +class AsyncEvals(_api_module.BaseModule): + + async def _create_evaluation_item( + self, + *, + evaluation_item_type: str, + gcs_uri: str, + display_name: Optional[str] = None, + config: Optional[types.CreateEvaluationItemConfigOrDict] = None, + ) -> types.EvaluationItem: + """ + Creates an EvaluationItem. + """ + + parameter_model = types._CreateEvaluationItemParameters( + evaluation_item_type=evaluation_item_type, + gcs_uri=gcs_uri, + display_name=display_name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationItemParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationItems".format_map(request_url_dict) + else: + path = "evaluationItems" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "post", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationItem._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + async def _create_evaluation_metric( + self, + *, + display_name: Optional[str] = None, + description: Optional[str] = None, + metric: Optional[types.MetricOrDict] = None, + config: Optional[types.CreateEvaluationMetricConfigOrDict] = None, + ) -> types.EvaluationMetric: + """ + Creates an EvaluationMetric. + """ + + parameter_model = types._CreateEvaluationMetricParameters( + display_name=display_name, + description=description, + metric=metric, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationMetricParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationMetrics".format_map(request_url_dict) + else: + path = "evaluationMetrics" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "post", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + if self._api_client.vertexai: + response_dict = _EvaluationMetric_from_vertex(response_dict) + + return_value = types.EvaluationMetric._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + async def _create_evaluation_run( + self, + *, + name: Optional[str] = None, + display_name: Optional[str] = None, + data_source: types.EvaluationRunDataSourceOrDict, + evaluation_config: types.EvaluationRunConfigOrDict, + labels: Optional[dict[str, str]] = None, + inference_configs: Optional[ + dict[str, types.EvaluationRunInferenceConfigOrDict] + ] = None, + config: Optional[types.CreateEvaluationRunConfigOrDict] = None, + analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, + ) -> types.EvaluationRun: + """ + Creates an EvaluationRun. + """ + + parameter_model = types._CreateEvaluationRunParameters( + name=name, + display_name=display_name, + data_source=data_source, + evaluation_config=evaluation_config, + labels=labels, + inference_configs=inference_configs, config=config, + analysis_configs=analysis_configs, ) - def delete_evaluation_metric( - self, - *, - metric_resource_name: str, - config: Optional[types.DeleteEvaluationMetricConfigOrDict] = None, - ) -> None: - """Deletes an EvaluationMetric. + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationRunParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationRuns".format_map(request_url_dict) + else: + path = "evaluationRuns" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options - Args: - metric_resource_name: The resource name of the EvaluationMetric to delete. - Format: - `projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric}` - config: The optional configuration for the delete operation. - """ - self._delete_evaluation_metric( - metric_resource_name=metric_resource_name, - config=config, + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "post", path, request_dict, http_options ) + response_dict = {} if not response.body else json.loads(response.body) -class AsyncEvals(_api_module.BaseModule): + if self._api_client.vertexai: + response_dict = _EvaluationRun_from_vertex(response_dict) - async def _create_evaluation_item( + return_value = types.EvaluationRun._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + + async def _create_evaluation_experiment( self, *, - evaluation_item_type: str, - gcs_uri: str, display_name: Optional[str] = None, - config: Optional[types.CreateEvaluationItemConfigOrDict] = None, - ) -> types.EvaluationItem: + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: """ - Creates an EvaluationItem. + Creates an EvaluationExperiment. """ - parameter_model = types._CreateEvaluationItemParameters( - evaluation_item_type=evaluation_item_type, - gcs_uri=gcs_uri, + parameter_model = types._CreateEvaluationExperimentParameters( display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, config=config, ) @@ -3131,12 +4037,14 @@ async def _create_evaluation_item( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _CreateEvaluationItemParameters_to_vertex(parameter_model) + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) request_url_dict = request_dict.get("_url") if request_url_dict: - path = "evaluationItems".format_map(request_url_dict) + path = "evaluationExperiments".format_map(request_url_dict) else: - path = "evaluationItems" + path = "evaluationExperiments" query_params = request_dict.get("_query") if query_params: @@ -3160,7 +4068,7 @@ async def _create_evaluation_item( response_dict = {} if not response.body else json.loads(response.body) - return_value = types.EvaluationItem._from_response( + return_value = types.EvaluationExperiment._from_response( response=response_dict, kwargs=( { @@ -3184,22 +4092,18 @@ async def _create_evaluation_item( self._api_client._verify_response(return_value) return return_value - async def _create_evaluation_metric( + async def _delete_evaluation_experiment( self, *, - display_name: Optional[str] = None, - description: Optional[str] = None, - metric: Optional[types.MetricOrDict] = None, - config: Optional[types.CreateEvaluationMetricConfigOrDict] = None, - ) -> types.EvaluationMetric: + name: str, + config: Optional[types.DeleteEvaluationExperimentConfigOrDict] = None, + ) -> types.DeleteEvaluationExperimentOperation: """ - Creates an EvaluationMetric. + Deletes an EvaluationExperiment. """ - parameter_model = types._CreateEvaluationMetricParameters( - display_name=display_name, - description=description, - metric=metric, + parameter_model = types._DeleteEvaluationExperimentParameters( + name=name, config=config, ) @@ -3209,12 +4113,14 @@ async def _create_evaluation_metric( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _CreateEvaluationMetricParameters_to_vertex(parameter_model) + request_dict = _DeleteEvaluationExperimentParameters_to_vertex( + parameter_model + ) request_url_dict = request_dict.get("_url") if request_url_dict: - path = "evaluationMetrics".format_map(request_url_dict) + path = "evaluationExperiments/{name}".format_map(request_url_dict) else: - path = "evaluationMetrics" + path = "evaluationExperiments/{name}" query_params = request_dict.get("_query") if query_params: @@ -3233,15 +4139,12 @@ async def _create_evaluation_metric( request_dict = _common.encode_unserializable_types(request_dict) response = await self._api_client.async_request( - "post", path, request_dict, http_options + "delete", path, request_dict, http_options ) response_dict = {} if not response.body else json.loads(response.body) - if self._api_client.vertexai: - response_dict = _EvaluationMetric_from_vertex(response_dict) - - return_value = types.EvaluationMetric._from_response( + return_value = types.DeleteEvaluationExperimentOperation._from_response( response=response_dict, kwargs=( { @@ -3265,33 +4168,19 @@ async def _create_evaluation_metric( self._api_client._verify_response(return_value) return return_value - async def _create_evaluation_run( + async def _update_evaluation_experiment( self, *, - name: Optional[str] = None, - display_name: Optional[str] = None, - data_source: types.EvaluationRunDataSourceOrDict, - evaluation_config: types.EvaluationRunConfigOrDict, - labels: Optional[dict[str, str]] = None, - inference_configs: Optional[ - dict[str, types.EvaluationRunInferenceConfigOrDict] - ] = None, - config: Optional[types.CreateEvaluationRunConfigOrDict] = None, - analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None, - ) -> types.EvaluationRun: + name: str, + config: Optional[types.UpdateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: """ - Creates an EvaluationRun. + Updates an EvaluationExperiment. """ - parameter_model = types._CreateEvaluationRunParameters( + parameter_model = types._UpdateEvaluationExperimentParameters( name=name, - display_name=display_name, - data_source=data_source, - evaluation_config=evaluation_config, - labels=labels, - inference_configs=inference_configs, config=config, - analysis_configs=analysis_configs, ) request_url_dict: Optional[dict[str, str]] @@ -3300,12 +4189,14 @@ async def _create_evaluation_run( "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." ) else: - request_dict = _CreateEvaluationRunParameters_to_vertex(parameter_model) + request_dict = _UpdateEvaluationExperimentParameters_to_vertex( + parameter_model + ) request_url_dict = request_dict.get("_url") if request_url_dict: - path = "evaluationRuns".format_map(request_url_dict) + path = "evaluationExperiments/{name}".format_map(request_url_dict) else: - path = "evaluationRuns" + path = "evaluationExperiments/{name}" query_params = request_dict.get("_query") if query_params: @@ -3324,15 +4215,12 @@ async def _create_evaluation_run( request_dict = _common.encode_unserializable_types(request_dict) response = await self._api_client.async_request( - "post", path, request_dict, http_options + "patch", path, request_dict, http_options ) response_dict = {} if not response.body else json.loads(response.body) - if self._api_client.vertexai: - response_dict = _EvaluationRun_from_vertex(response_dict) - - return_value = types.EvaluationRun._from_response( + return_value = types.EvaluationExperiment._from_response( response=response_dict, kwargs=( { @@ -3858,6 +4746,80 @@ async def _generate_rubrics( self._api_client._verify_response(return_value) return return_value + async def _get_evaluation_experiment( + self, + *, + name: str, + config: Optional[types.GetEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Retrieves an EvaluationExperiment from the resource name. + """ + + parameter_model = types._GetEvaluationExperimentParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _GetEvaluationExperimentParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments/{name}".format_map(request_url_dict) + else: + path = "evaluationExperiments/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _get_evaluation_metric( self, *, @@ -4151,6 +5113,78 @@ async def _get_evaluation_item( self._api_client._verify_response(return_value) return return_value + async def _list_evaluation_experiments( + self, *, config: Optional[types.ListEvaluationExperimentsConfigOrDict] = None + ) -> types.ListEvaluationExperimentsResponse: + """ + Lists EvaluationExperiments. + """ + + parameter_model = types._ListEvaluationExperimentsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationExperimentsParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationExperimentsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _list_evaluation_metrics( self, *, config: Optional[types.ListEvaluationMetricsConfigOrDict] = None ) -> types.ListEvaluationMetricsResponse: diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py index d088967399..a75584f31b 100644 --- a/vertexai/_genai/types/__init__.py +++ b/vertexai/_genai/types/__init__.py @@ -83,6 +83,7 @@ from .common import _GetDatasetParameters from .common import _GetDatasetVersionParameters from .common import _GetDeleteAgentEngineRuntimeRevisionOperationParameters +from .common import _GetEvaluationExperimentParameters from .common import _GetEvaluationItemParameters from .common import _GetEvaluationMetricParameters from .common import _GetEvaluationRunParameters @@ -107,6 +108,7 @@ from .common import _ListAgentEngineTasksRequestParameters from .common import _ListDatasetsRequestParameters from .common import _ListDatasetVersionsRequestParameters +from .common import _ListEvaluationExperimentsParameters from .common import _ListEvaluationMetricsParameters from .common import _ListMultimodalDatasetsRequestParameters from .common import _ListSandboxEnvironmentSnapshotsRequestParameters @@ -295,6 +297,10 @@ from .common import CreateEvaluationMetricConfig from .common import CreateEvaluationMetricConfigDict from .common import CreateEvaluationMetricConfigOrDict +from .common import CreateEvaluationExperimentConfig +from .common import CreateEvaluationExperimentConfigDict +from .common import CreateEvaluationExperimentConfigOrDict +from .common import _CreateEvaluationExperimentParameters from .common import CreateEvaluationRunConfig from .common import CreateEvaluationRunConfigDict from .common import CreateEvaluationRunConfigOrDict @@ -368,6 +374,17 @@ from .common import DeleteAgentEngineTaskConfig from .common import DeleteAgentEngineTaskConfigDict from .common import DeleteAgentEngineTaskConfigOrDict +from .common import DeleteEvaluationExperimentConfig +from .common import DeleteEvaluationExperimentConfigDict +from .common import DeleteEvaluationExperimentConfigOrDict +from .common import _DeleteEvaluationExperimentParameters +from .common import DeleteEvaluationExperimentOperation +from .common import DeleteEvaluationExperimentOperationDict +from .common import DeleteEvaluationExperimentOperationOrDict +from .common import UpdateEvaluationExperimentConfig +from .common import UpdateEvaluationExperimentConfigDict +from .common import UpdateEvaluationExperimentConfigOrDict +from .common import _UpdateEvaluationExperimentParameters from .common import DeleteEvaluationMetricConfig from .common import DeleteEvaluationMetricConfigDict from .common import DeleteEvaluationMetricConfigOrDict @@ -612,6 +629,9 @@ from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfig from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigDict from .common import GetDeleteAgentEngineRuntimeRevisionOperationConfigOrDict +from .common import GetEvaluationExperimentConfig +from .common import GetEvaluationExperimentConfigDict +from .common import GetEvaluationExperimentConfigOrDict from .common import GetEvaluationItemConfig from .common import GetEvaluationItemConfigDict from .common import GetEvaluationItemConfigOrDict @@ -715,6 +735,12 @@ from .common import ListDatasetVersionsResponse from .common import ListDatasetVersionsResponseDict from .common import ListDatasetVersionsResponseOrDict +from .common import ListEvaluationExperimentsConfig +from .common import ListEvaluationExperimentsConfigDict +from .common import ListEvaluationExperimentsConfigOrDict +from .common import ListEvaluationExperimentsResponse +from .common import ListEvaluationExperimentsResponseDict +from .common import ListEvaluationExperimentsResponseOrDict from .common import ListEvaluationMetricsConfig from .common import ListEvaluationMetricsConfigDict from .common import ListEvaluationMetricsConfigOrDict @@ -1606,6 +1632,9 @@ "AnalysisConfig", "AnalysisConfigDict", "AnalysisConfigOrDict", + "CreateEvaluationExperimentConfig", + "CreateEvaluationExperimentConfigDict", + "CreateEvaluationExperimentConfigOrDict", "CreateEvaluationRunConfig", "CreateEvaluationRunConfigDict", "CreateEvaluationRunConfigOrDict", @@ -1675,6 +1704,15 @@ "EvaluationSet", "EvaluationSetDict", "EvaluationSetOrDict", + "DeleteEvaluationExperimentConfig", + "DeleteEvaluationExperimentConfigDict", + "DeleteEvaluationExperimentConfigOrDict", + "DeleteEvaluationExperimentOperation", + "DeleteEvaluationExperimentOperationDict", + "DeleteEvaluationExperimentOperationOrDict", + "UpdateEvaluationExperimentConfig", + "UpdateEvaluationExperimentConfigDict", + "UpdateEvaluationExperimentConfigOrDict", "DeleteEvaluationMetricConfig", "DeleteEvaluationMetricConfigDict", "DeleteEvaluationMetricConfigOrDict", @@ -2642,6 +2680,17 @@ "EvaluationExperimentDict", "EvaluationExperimentOrDict", "EvaluationExperimentMergeStrategy", + "GetEvaluationExperimentConfig", + "GetEvaluationExperimentConfigDict", + "GetEvaluationExperimentConfigOrDict", + "_GetEvaluationExperimentParameters", + "ListEvaluationExperimentsConfig", + "ListEvaluationExperimentsConfigDict", + "ListEvaluationExperimentsConfigOrDict", + "_ListEvaluationExperimentsParameters", + "ListEvaluationExperimentsResponse", + "ListEvaluationExperimentsResponseDict", + "ListEvaluationExperimentsResponseOrDict", "EvaluateDatasetConfig", "EvaluateDatasetConfigDict", "EvaluateDatasetConfigOrDict", @@ -2776,10 +2825,12 @@ "_CreateAgentEngineTaskRequestParameters", "_AppendAgentEngineTaskEventRequestParameters", "_ListAgentEngineTaskEventsRequestParameters", + "_CreateEvaluationExperimentParameters", "_CreateEvaluationItemParameters", "_CreateEvaluationMetricParameters", "_CreateEvaluationRunParameters", "_CreateEvaluationSetParameters", + "_DeleteEvaluationExperimentParameters", "_DeleteEvaluationMetricParameters", "_EvaluateInstancesRequestParameters", "_GenerateUserScenariosParameters", @@ -2854,6 +2905,7 @@ "_GetMultimodalDatasetParameters", "_GetMultimodalDatasetOperationParameters", "_ListMultimodalDatasetsRequestParameters", + "_UpdateEvaluationExperimentParameters", "_UpdateMultimodalDatasetParameters", "_CreateDatasetParameters", "_CreateDatasetVersionParameters", diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 59490f34fa..ac4947c9f2 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -1383,6 +1383,64 @@ class ListAgentEngineTaskEventsResponseDict(TypedDict, total=False): ] +class CreateEvaluationExperimentConfig(_common.BaseModel): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class CreateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +CreateEvaluationExperimentConfigOrDict = Union[ + CreateEvaluationExperimentConfig, CreateEvaluationExperimentConfigDict +] + + +class _CreateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] = Field(default=None, description="""""") + labels: Optional[dict[str, str]] = Field(default=None, description="""""") + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""""" + ) + metadata: Optional[dict[str, Any]] = Field(default=None, description="""""") + config: Optional[CreateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _CreateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] + """""" + + labels: Optional[dict[str, str]] + """""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """""" + + metadata: Optional[dict[str, Any]] + """""" + + config: Optional[CreateEvaluationExperimentConfigDict] + """""" + + +_CreateEvaluationExperimentParametersOrDict = Union[ + _CreateEvaluationExperimentParameters, _CreateEvaluationExperimentParametersDict +] + + class CreateEvaluationItemConfig(_common.BaseModel): """Config to create an evaluation item.""" @@ -3819,6 +3877,92 @@ class EvaluationSetDict(TypedDict, total=False): EvaluationSetOrDict = Union[EvaluationSet, EvaluationSetDict] +class DeleteEvaluationExperimentConfig(_common.BaseModel): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class DeleteEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for deleting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +DeleteEvaluationExperimentConfigOrDict = Union[ + DeleteEvaluationExperimentConfig, DeleteEvaluationExperimentConfigDict +] + + +class _DeleteEvaluationExperimentParameters(_common.BaseModel): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[DeleteEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _DeleteEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for deleting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[DeleteEvaluationExperimentConfigDict] + """""" + + +_DeleteEvaluationExperimentParametersOrDict = Union[ + _DeleteEvaluationExperimentParameters, _DeleteEvaluationExperimentParametersDict +] + + +class DeleteEvaluationExperimentOperation(_common.BaseModel): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, + description="""The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""", + ) + done: Optional[bool] = Field( + default=None, + description="""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""", + ) + error: Optional[dict[str, Any]] = Field( + default=None, + description="""The error result of the operation in case of failure or cancellation.""", + ) + + +class DeleteEvaluationExperimentOperationDict(TypedDict, total=False): + """Operation for deleting an evaluation experiment.""" + + name: Optional[str] + """The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""" + + metadata: Optional[dict[str, Any]] + """Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""" + + done: Optional[bool] + """If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""" + + error: Optional[dict[str, Any]] + """The error result of the operation in case of failure or cancellation.""" + + +DeleteEvaluationExperimentOperationOrDict = Union[ + DeleteEvaluationExperimentOperation, DeleteEvaluationExperimentOperationDict +] + + class DeleteEvaluationMetricConfig(_common.BaseModel): """Config for deleting an evaluation metric.""" @@ -5861,6 +6005,85 @@ class ListEvaluationMetricsResponseDict(TypedDict, total=False): ] +class UpdateEvaluationExperimentConfig(_common.BaseModel): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + update_mask: Optional[str] = Field( + default=None, + description="""The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, description="""Metadata about the evaluation experiment.""" + ) + + +class UpdateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for updating an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + update_mask: Optional[str] + """The update mask to apply. For the `FieldMask` definition, see + https://protobuf.dev/reference/protobuf/google.protobuf/#field-mask.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment.""" + + +UpdateEvaluationExperimentConfigOrDict = Union[ + UpdateEvaluationExperimentConfig, UpdateEvaluationExperimentConfigDict +] + + +class _UpdateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] = Field( + default=None, description="""The resource name of the EvaluationExperiment.""" + ) + config: Optional[UpdateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _UpdateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for updating an evaluation experiment.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment.""" + + config: Optional[UpdateEvaluationExperimentConfigDict] + """""" + + +_UpdateEvaluationExperimentParametersOrDict = Union[ + _UpdateEvaluationExperimentParameters, _UpdateEvaluationExperimentParametersDict +] + + class OptimizeConfig(_common.BaseModel): """Config for Prompt Optimizer.""" @@ -19195,6 +19418,155 @@ class EvaluationExperimentDict(TypedDict, total=False): EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] +class GetEvaluationExperimentConfig(_common.BaseModel): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class GetEvaluationExperimentConfigDict(TypedDict, total=False): + """Config for getting an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +GetEvaluationExperimentConfigOrDict = Union[ + GetEvaluationExperimentConfig, GetEvaluationExperimentConfigDict +] + + +class _GetEvaluationExperimentParameters(_common.BaseModel): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[GetEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _GetEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for getting an evaluation experiment.""" + + name: Optional[str] + """""" + + config: Optional[GetEvaluationExperimentConfigDict] + """""" + + +_GetEvaluationExperimentParametersOrDict = Union[ + _GetEvaluationExperimentParameters, _GetEvaluationExperimentParametersDict +] + + +class ListEvaluationExperimentsConfig(_common.BaseModel): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + page_size: Optional[int] = Field(default=None, description="""""") + page_token: Optional[str] = Field(default=None, description="""""") + filter: Optional[str] = Field( + default=None, + description="""An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""", + ) + order_by: Optional[str] = Field( + default=None, + description="""A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""", + ) + + +class ListEvaluationExperimentsConfigDict(TypedDict, total=False): + """Config for listing evaluation experiments.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + page_size: Optional[int] + """""" + + page_token: Optional[str] + """""" + + filter: Optional[str] + """An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""" + + order_by: Optional[str] + """A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""" + + +ListEvaluationExperimentsConfigOrDict = Union[ + ListEvaluationExperimentsConfig, ListEvaluationExperimentsConfigDict +] + + +class _ListEvaluationExperimentsParameters(_common.BaseModel): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfig] = Field( + default=None, description="""""" + ) + + +class _ListEvaluationExperimentsParametersDict(TypedDict, total=False): + """Parameters for listing evaluation experiments.""" + + config: Optional[ListEvaluationExperimentsConfigDict] + """""" + + +_ListEvaluationExperimentsParametersOrDict = Union[ + _ListEvaluationExperimentsParameters, _ListEvaluationExperimentsParametersDict +] + + +class ListEvaluationExperimentsResponse(_common.BaseModel): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponse] = Field( + default=None, description="""Used to retain the full HTTP response.""" + ) + next_page_token: Optional[str] = Field(default=None, description="""""") + evaluation_experiments: Optional[list[EvaluationExperiment]] = Field( + default=None, + description="""List of evaluation experiments. + """, + ) + + +class ListEvaluationExperimentsResponseDict(TypedDict, total=False): + """Response for listing evaluation experiments.""" + + sdk_http_response: Optional[genai_types.HttpResponseDict] + """Used to retain the full HTTP response.""" + + next_page_token: Optional[str] + """""" + + evaluation_experiments: Optional[list[EvaluationExperimentDict]] + """List of evaluation experiments. + """ + + +ListEvaluationExperimentsResponseOrDict = Union[ + ListEvaluationExperimentsResponse, ListEvaluationExperimentsResponseDict +] + + class RubricGroup(_common.BaseModel): """A group of rubrics.