From 38b33babc0db5a5a4d2fde39549445da93f195b1 Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Mon, 20 Jul 2026 15:15:39 -0700 Subject: [PATCH] chore: Auto-construct `inference_configs` for `create_evaluation_run` with `agent` parameter without requiring `agent_info` PiperOrigin-RevId: 951079941 --- agentplatform/_genai/_evals_common.py | 9 +- agentplatform/_genai/_evals_constant.py | 1 + agentplatform/_genai/evals.py | 48 ++++++- tests/unit/agentplatform/genai/test_evals.py | 138 +++++++++++++++++++ 4 files changed, 188 insertions(+), 8 deletions(-) diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index d91d352dc1..e1b5b4765d 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -88,6 +88,7 @@ PARTS = _evals_constant.PARTS USER_AUTHOR = _evals_constant.USER_AUTHOR AGENT_DATA = _evals_constant.AGENT_DATA +_DEFAULT_CANDIDATE_NAME = _evals_constant.DEFAULT_CANDIDATE_NAME @contextlib.contextmanager @@ -528,11 +529,11 @@ def _resolve_inference_configs( if inference_configs is None: inference_configs = {} - # We might have used "candidate-1" as a placeholder key in the caller, - # let's migrate it to the agent name, or if it doesn't exist, just create it. - if "candidate-1" in inference_configs: + # We might have used the default candidate name as a placeholder key + # in the caller; migrate it to the agent name. + if _DEFAULT_CANDIDATE_NAME in inference_configs: inference_configs[parsed_agent_info.name] = inference_configs.pop( - "candidate-1" + _DEFAULT_CANDIDATE_NAME ) if parsed_agent_info.name not in inference_configs: diff --git a/agentplatform/_genai/_evals_constant.py b/agentplatform/_genai/_evals_constant.py index b0bb533ca1..321150310d 100644 --- a/agentplatform/_genai/_evals_constant.py +++ b/agentplatform/_genai/_evals_constant.py @@ -63,6 +63,7 @@ CONVERSATION_PLAN = "conversation_plan" HISTORY = "history" CONVERSATION_HISTORY = "conversation_history" +DEFAULT_CANDIDATE_NAME = "candidate-1" COMMON_DATASET_COLUMNS = frozenset( { diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 1f4aa7e1de..e8259dae94 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -2746,7 +2746,25 @@ def create_evaluation_run( if isinstance(config, dict): config = types.CreateEvaluationRunConfig.model_validate(config) - if agent_info and not inference_configs: + # Auto-construct inference_configs when agent_info is explicitly + # provided (existing behavior) OR an agent resource is provided + # (allows omitting agent_info for both Gemini Agents and Agent + # Engine). The server skips inference per-item when a + # CandidateResponse with a matching candidate name already exists, + # so it is safe to always send inference_configs. + _should_auto_infer = not inference_configs and (agent_info or agent) + if _should_auto_infer: + if not parsed_agent_info.name: + # Prefer the dataset's candidate_name (set by run_inference) + # so the inference_configs key matches the CandidateResponse + # and the server correctly skips already-completed items. + if ( + isinstance(dataset, types.EvaluationDataset) + and dataset.candidate_name + ): + parsed_agent_info.name = dataset.candidate_name + else: + parsed_agent_info.name = _evals_common._DEFAULT_CANDIDATE_NAME parsed_user_simulator_config = ( evals_types.UserSimulatorConfig.model_validate(user_simulator_config) if isinstance(user_simulator_config, dict) @@ -2755,7 +2773,9 @@ def create_evaluation_run( if getattr(parsed_user_simulator_config, "max_turn", None) is None: parsed_user_simulator_config.max_turn = 5 - candidate_name = parsed_agent_info.name or "candidate-1" + candidate_name = ( + parsed_agent_info.name or _evals_common._DEFAULT_CANDIDATE_NAME + ) if agent and _evals_common._is_gemini_agent_resource(agent): agent_run_config = types.AgentRunConfig( gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent), @@ -4534,7 +4554,25 @@ async def create_evaluation_run( if isinstance(config, dict): config = types.CreateEvaluationRunConfig.model_validate(config) - if agent_info and not inference_configs: + # Auto-construct inference_configs when agent_info is explicitly + # provided (existing behavior) OR an agent resource is provided + # (allows omitting agent_info for both Gemini Agents and Agent + # Engine). The server skips inference per-item when a + # CandidateResponse with a matching candidate name already exists, + # so it is safe to always send inference_configs. + _should_auto_infer = not inference_configs and (agent_info or agent) + if _should_auto_infer: + if not parsed_agent_info.name: + # Prefer the dataset's candidate_name (set by run_inference) + # so the inference_configs key matches the CandidateResponse + # and the server correctly skips already-completed items. + if ( + isinstance(dataset, types.EvaluationDataset) + and dataset.candidate_name + ): + parsed_agent_info.name = dataset.candidate_name + else: + parsed_agent_info.name = _evals_common._DEFAULT_CANDIDATE_NAME parsed_user_simulator_config = ( evals_types.UserSimulatorConfig.model_validate(user_simulator_config) if isinstance(user_simulator_config, dict) @@ -4543,7 +4581,9 @@ async def create_evaluation_run( if getattr(parsed_user_simulator_config, "max_turn", None) is None: parsed_user_simulator_config.max_turn = 5 - candidate_name = parsed_agent_info.name or "candidate-1" + candidate_name = ( + parsed_agent_info.name or _evals_common._DEFAULT_CANDIDATE_NAME + ) if agent and _evals_common._is_gemini_agent_resource(agent): agent_run_config = types.AgentRunConfig( gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent), diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index b93b6db39b..c2506f5ad2 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -10873,6 +10873,10 @@ def test_create_evaluation_run_builds_gemini_agent_config(self): == _TEST_GEMINI_AGENT ) assert "agent_engine" not in agent_run_config + # agent_info.name overrides the default candidate name. + inference_configs = request_body["inferenceConfigs"] + assert "gemini-agent" in inference_configs + assert _evals_common._DEFAULT_CANDIDATE_NAME not in inference_configs def test_create_evaluation_run_agent_engine_does_not_set_gemini(self): evals_module = evals.Evals(api_client_=self.mock_api_client) @@ -10900,6 +10904,140 @@ def test_create_evaluation_run_agent_engine_does_not_set_gemini(self): agent_run_config = self._agent_run_config(request_body) assert "gemini_agent_config" not in agent_run_config assert agent_run_config["agent_engine"] == _TEST_AGENT_ENGINE + # agent_info.name overrides the default candidate name. + inference_configs = request_body["inferenceConfigs"] + assert "ae-agent" in inference_configs + assert _evals_common._DEFAULT_CANDIDATE_NAME not in inference_configs + + def test_create_evaluation_run_gemini_agent_without_agent_info(self): + """Gemini agent resource alone triggers inference_configs auto-construction.""" + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.create_evaluation_run( + dataset=agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ), + metrics=[ + agentplatform_genai_types.EvaluationRunMetric( + metric="multi_turn_task_success_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="multi_turn_task_success_v1", + ) + ), + ) + ], + dest="gs://test-bucket/output", + agent=_TEST_GEMINI_AGENT, + # No agent_info provided. + ) + + request_body = self._get_create_run_body() + agent_run_config = self._agent_run_config(request_body) + assert ( + agent_run_config["gemini_agent_config"]["gemini_agent"] + == _TEST_GEMINI_AGENT + ) + assert "agent_engine" not in agent_run_config + # Default candidate name should match the constant. + inference_configs = request_body["inferenceConfigs"] + assert _evals_common._DEFAULT_CANDIDATE_NAME in inference_configs + + def test_create_evaluation_run_no_agent_no_agent_info_no_inference(self): + """Without agent or agent_info, no inference_configs are auto-constructed.""" + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.create_evaluation_run( + dataset=agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ), + metrics=[ + agentplatform_genai_types.EvaluationRunMetric( + metric="multi_turn_task_success_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="multi_turn_task_success_v1", + ) + ), + ) + ], + dest="gs://test-bucket/output", + # No agent, no agent_info. + ) + + request_body = self._get_create_run_body() + assert "inferenceConfigs" not in request_body or not request_body.get( + "inferenceConfigs" + ) + + def test_create_evaluation_run_agent_engine_without_agent_info(self): + """Agent Engine resource alone triggers inference_configs auto-construction.""" + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.create_evaluation_run( + dataset=agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ), + metrics=[ + agentplatform_genai_types.EvaluationRunMetric( + metric="multi_turn_task_success_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="multi_turn_task_success_v1", + ) + ), + ) + ], + dest="gs://test-bucket/output", + agent=_TEST_AGENT_ENGINE, + # No agent_info provided. + ) + + request_body = self._get_create_run_body() + agent_run_config = self._agent_run_config(request_body) + assert agent_run_config["agent_engine"] == _TEST_AGENT_ENGINE + assert "gemini_agent_config" not in agent_run_config + # Default candidate name should match the constant. + inference_configs = request_body["inferenceConfigs"] + assert _evals_common._DEFAULT_CANDIDATE_NAME in inference_configs + + @mock.patch.object(_evals_common, "_resolve_dataset") + def test_create_evaluation_run_uses_dataset_candidate_name( + self, mock_resolve_dataset + ): + """When dataset.candidate_name is set (e.g. from run_inference), the + inference_configs key should match it instead of using the default.""" + mock_resolve_dataset.return_value = ( + agentplatform_genai_types.EvaluationRunDataSource( + evaluation_set="projects/123/locations/us-central1/evaluationSets/789" + ) + ) + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.create_evaluation_run( + dataset=agentplatform_genai_types.EvaluationDataset( + eval_dataset_df=pd.DataFrame({"prompt": ["hello"]}), + candidate_name="my-agent-v2", + ), + metrics=[ + agentplatform_genai_types.EvaluationRunMetric( + metric="multi_turn_task_success_v1", + metric_config=agentplatform_genai_types.UnifiedMetric( + predefined_metric_spec=genai_types.PredefinedMetricSpec( + metric_spec_name="multi_turn_task_success_v1", + ) + ), + ) + ], + dest="gs://test-bucket/output", + agent=_TEST_GEMINI_AGENT, + # No agent_info -- candidate name should come from dataset. + ) + + request_body = self._get_create_run_body() + inference_configs = request_body["inferenceConfigs"] + assert "my-agent-v2" in inference_configs + assert _evals_common._DEFAULT_CANDIDATE_NAME not in inference_configs class TestResolveInteractionsForDisplay: