Skip to content

chore: Auto-construct inference_configs for create_evaluation_run with agent parameter without requiring agent_info #7008

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
Open

chore: Auto-construct inference_configs for create_evaluation_run with agent parameter without requiring agent_info #7008

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions agentplatform/_genai/_evals_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
PARTS = _evals_constant.PARTS
USER_AUTHOR = _evals_constant.USER_AUTHOR
AGENT_DATA = _evals_constant.AGENT_DATA
_DEFAULT_CANDIDATE_NAME = _evals_constant.DEFAULT_CANDIDATE_NAME


@contextlib.contextmanager
Expand Down Expand Up @@ -528,11 +529,11 @@ def _resolve_inference_configs(
if inference_configs is None:
inference_configs = {}

# We might have used "candidate-1" as a placeholder key in the caller,
# let's migrate it to the agent name, or if it doesn't exist, just create it.
if "candidate-1" in inference_configs:
# We might have used the default candidate name as a placeholder key
# in the caller; migrate it to the agent name.
if _DEFAULT_CANDIDATE_NAME in inference_configs:
inference_configs[parsed_agent_info.name] = inference_configs.pop(
"candidate-1"
_DEFAULT_CANDIDATE_NAME
)

if parsed_agent_info.name not in inference_configs:
Expand Down
1 change: 1 addition & 0 deletions agentplatform/_genai/_evals_constant.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
CONVERSATION_PLAN = "conversation_plan"
HISTORY = "history"
CONVERSATION_HISTORY = "conversation_history"
DEFAULT_CANDIDATE_NAME = "candidate-1"

COMMON_DATASET_COLUMNS = frozenset(
{
Expand Down
48 changes: 44 additions & 4 deletions agentplatform/_genai/evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2746,7 +2746,25 @@ def create_evaluation_run(
if isinstance(config, dict):
config = types.CreateEvaluationRunConfig.model_validate(config)

if agent_info and not inference_configs:
# Auto-construct inference_configs when agent_info is explicitly
# provided (existing behavior) OR an agent resource is provided
# (allows omitting agent_info for both Gemini Agents and Agent
# Engine). The server skips inference per-item when a
# CandidateResponse with a matching candidate name already exists,
# so it is safe to always send inference_configs.
_should_auto_infer = not inference_configs and (agent_info or agent)
if _should_auto_infer:
if not parsed_agent_info.name:
# Prefer the dataset's candidate_name (set by run_inference)
# so the inference_configs key matches the CandidateResponse
# and the server correctly skips already-completed items.
if (
isinstance(dataset, types.EvaluationDataset)
and dataset.candidate_name
):
parsed_agent_info.name = dataset.candidate_name
else:
parsed_agent_info.name = _evals_common._DEFAULT_CANDIDATE_NAME
parsed_user_simulator_config = (
evals_types.UserSimulatorConfig.model_validate(user_simulator_config)
if isinstance(user_simulator_config, dict)
Expand All @@ -2755,7 +2773,9 @@ def create_evaluation_run(
if getattr(parsed_user_simulator_config, "max_turn", None) is None:
parsed_user_simulator_config.max_turn = 5

candidate_name = parsed_agent_info.name or "candidate-1"
candidate_name = (
parsed_agent_info.name or _evals_common._DEFAULT_CANDIDATE_NAME
)
if agent and _evals_common._is_gemini_agent_resource(agent):
agent_run_config = types.AgentRunConfig(
gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent),
Expand Down Expand Up @@ -4534,7 +4554,25 @@ async def create_evaluation_run(
if isinstance(config, dict):
config = types.CreateEvaluationRunConfig.model_validate(config)

if agent_info and not inference_configs:
# Auto-construct inference_configs when agent_info is explicitly
# provided (existing behavior) OR an agent resource is provided
# (allows omitting agent_info for both Gemini Agents and Agent
# Engine). The server skips inference per-item when a
# CandidateResponse with a matching candidate name already exists,
# so it is safe to always send inference_configs.
_should_auto_infer = not inference_configs and (agent_info or agent)
if _should_auto_infer:
if not parsed_agent_info.name:
# Prefer the dataset's candidate_name (set by run_inference)
# so the inference_configs key matches the CandidateResponse
# and the server correctly skips already-completed items.
if (
isinstance(dataset, types.EvaluationDataset)
and dataset.candidate_name
):
parsed_agent_info.name = dataset.candidate_name
else:
parsed_agent_info.name = _evals_common._DEFAULT_CANDIDATE_NAME
parsed_user_simulator_config = (
evals_types.UserSimulatorConfig.model_validate(user_simulator_config)
if isinstance(user_simulator_config, dict)
Expand All @@ -4543,7 +4581,9 @@ async def create_evaluation_run(
if getattr(parsed_user_simulator_config, "max_turn", None) is None:
parsed_user_simulator_config.max_turn = 5

candidate_name = parsed_agent_info.name or "candidate-1"
candidate_name = (
parsed_agent_info.name or _evals_common._DEFAULT_CANDIDATE_NAME
)
if agent and _evals_common._is_gemini_agent_resource(agent):
agent_run_config = types.AgentRunConfig(
gemini_agent_config=types.GeminiAgentConfig(gemini_agent=agent),
Expand Down
138 changes: 138 additions & 0 deletions tests/unit/agentplatform/genai/test_evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10873,6 +10873,10 @@ def test_create_evaluation_run_builds_gemini_agent_config(self):
== _TEST_GEMINI_AGENT
)
assert "agent_engine" not in agent_run_config
# agent_info.name overrides the default candidate name.
inference_configs = request_body["inferenceConfigs"]
assert "gemini-agent" in inference_configs
assert _evals_common._DEFAULT_CANDIDATE_NAME not in inference_configs

def test_create_evaluation_run_agent_engine_does_not_set_gemini(self):
evals_module = evals.Evals(api_client_=self.mock_api_client)
Expand Down Expand Up @@ -10900,6 +10904,140 @@ def test_create_evaluation_run_agent_engine_does_not_set_gemini(self):
agent_run_config = self._agent_run_config(request_body)
assert "gemini_agent_config" not in agent_run_config
assert agent_run_config["agent_engine"] == _TEST_AGENT_ENGINE
# agent_info.name overrides the default candidate name.
inference_configs = request_body["inferenceConfigs"]
assert "ae-agent" in inference_configs
assert _evals_common._DEFAULT_CANDIDATE_NAME not in inference_configs

def test_create_evaluation_run_gemini_agent_without_agent_info(self):
"""Gemini agent resource alone triggers inference_configs auto-construction."""
evals_module = evals.Evals(api_client_=self.mock_api_client)

evals_module.create_evaluation_run(
dataset=agentplatform_genai_types.EvaluationRunDataSource(
evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
),
metrics=[
agentplatform_genai_types.EvaluationRunMetric(
metric="multi_turn_task_success_v1",
metric_config=agentplatform_genai_types.UnifiedMetric(
predefined_metric_spec=genai_types.PredefinedMetricSpec(
metric_spec_name="multi_turn_task_success_v1",
)
),
)
],
dest="gs://test-bucket/output",
agent=_TEST_GEMINI_AGENT,
# No agent_info provided.
)

request_body = self._get_create_run_body()
agent_run_config = self._agent_run_config(request_body)
assert (
agent_run_config["gemini_agent_config"]["gemini_agent"]
== _TEST_GEMINI_AGENT
)
assert "agent_engine" not in agent_run_config
# Default candidate name should match the constant.
inference_configs = request_body["inferenceConfigs"]
assert _evals_common._DEFAULT_CANDIDATE_NAME in inference_configs

def test_create_evaluation_run_no_agent_no_agent_info_no_inference(self):
"""Without agent or agent_info, no inference_configs are auto-constructed."""
evals_module = evals.Evals(api_client_=self.mock_api_client)

evals_module.create_evaluation_run(
dataset=agentplatform_genai_types.EvaluationRunDataSource(
evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
),
metrics=[
agentplatform_genai_types.EvaluationRunMetric(
metric="multi_turn_task_success_v1",
metric_config=agentplatform_genai_types.UnifiedMetric(
predefined_metric_spec=genai_types.PredefinedMetricSpec(
metric_spec_name="multi_turn_task_success_v1",
)
),
)
],
dest="gs://test-bucket/output",
# No agent, no agent_info.
)

request_body = self._get_create_run_body()
assert "inferenceConfigs" not in request_body or not request_body.get(
"inferenceConfigs"
)

def test_create_evaluation_run_agent_engine_without_agent_info(self):
"""Agent Engine resource alone triggers inference_configs auto-construction."""
evals_module = evals.Evals(api_client_=self.mock_api_client)

evals_module.create_evaluation_run(
dataset=agentplatform_genai_types.EvaluationRunDataSource(
evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
),
metrics=[
agentplatform_genai_types.EvaluationRunMetric(
metric="multi_turn_task_success_v1",
metric_config=agentplatform_genai_types.UnifiedMetric(
predefined_metric_spec=genai_types.PredefinedMetricSpec(
metric_spec_name="multi_turn_task_success_v1",
)
),
)
],
dest="gs://test-bucket/output",
agent=_TEST_AGENT_ENGINE,
# No agent_info provided.
)

request_body = self._get_create_run_body()
agent_run_config = self._agent_run_config(request_body)
assert agent_run_config["agent_engine"] == _TEST_AGENT_ENGINE
assert "gemini_agent_config" not in agent_run_config
# Default candidate name should match the constant.
inference_configs = request_body["inferenceConfigs"]
assert _evals_common._DEFAULT_CANDIDATE_NAME in inference_configs

@mock.patch.object(_evals_common, "_resolve_dataset")
def test_create_evaluation_run_uses_dataset_candidate_name(
self, mock_resolve_dataset
):
"""When dataset.candidate_name is set (e.g. from run_inference), the
inference_configs key should match it instead of using the default."""
mock_resolve_dataset.return_value = (
agentplatform_genai_types.EvaluationRunDataSource(
evaluation_set="projects/123/locations/us-central1/evaluationSets/789"
)
)
evals_module = evals.Evals(api_client_=self.mock_api_client)

evals_module.create_evaluation_run(
dataset=agentplatform_genai_types.EvaluationDataset(
eval_dataset_df=pd.DataFrame({"prompt": ["hello"]}),
candidate_name="my-agent-v2",
),
metrics=[
agentplatform_genai_types.EvaluationRunMetric(
metric="multi_turn_task_success_v1",
metric_config=agentplatform_genai_types.UnifiedMetric(
predefined_metric_spec=genai_types.PredefinedMetricSpec(
metric_spec_name="multi_turn_task_success_v1",
)
),
)
],
dest="gs://test-bucket/output",
agent=_TEST_GEMINI_AGENT,
# No agent_info -- candidate name should come from dataset.
)

request_body = self._get_create_run_body()
inference_configs = request_body["inferenceConfigs"]
assert "my-agent-v2" in inference_configs
assert _evals_common._DEFAULT_CANDIDATE_NAME not in inference_configs


class TestResolveInteractionsForDisplay:
Expand Down
Loading