Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions agentplatform/_genai/_evals_metric_handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -687,9 +687,15 @@ def _build_evaluation_instance(
for full_path in template_vars:
# Extract the root variable (e.g. 'metadata' from 'metadata.user_id')
root_var = full_path.split(".")[0].split("[")[0]

if root_var not in standard_fields and hasattr(eval_case, root_var):
val = getattr(eval_case, root_var)
field = root_var
if root_var == "history" and not hasattr(eval_case, "history"):
field = "conversation_history"

if root_var not in standard_fields and hasattr(eval_case, field):
val = getattr(eval_case, field)
if field == "conversation_history" and not val:
other_data_map[root_var] = types.evals.InstanceData(text="")
continue
# Add the root object to other_data so the backend can traverse it
other_data_map[root_var] = types.evals.InstanceData(
contents=types.evals.InstanceDataContents(
Expand Down
59 changes: 59 additions & 0 deletions tests/unit/agentplatform/genai/test_evals.py
Original file line number Diff line number Diff line change
Expand Up @@ -7351,6 +7351,65 @@ def test_build_evaluation_instance_various_field_types(self):
== "System instructions here."
)

def test_build_evaluation_instance_history_uses_conversation_history(self):
eval_case = agentplatform_genai_types.EvalCase(
prompt=genai_types.Content(parts=[genai_types.Part(text="Next")]),
responses=[
agentplatform_genai_types.ResponseCandidate(
response=genai_types.Content(parts=[genai_types.Part(text="Hi")])
)
],
conversation_history=[
agentplatform_genai_types.evals.Message(
content=genai_types.Content(
parts=[genai_types.Part(text="Turn 1 user")], role="user"
)
),
agentplatform_genai_types.evals.Message(
content=genai_types.Content(
parts=[genai_types.Part(text="Turn 1 model")], role="model"
)
),
],
)

instance = _evals_metric_handlers._build_evaluation_instance(
eval_case,
eval_case.responses[0].response,
prompt_template="{history}{prompt}{response}",
)

assert (
instance.other_data.map_instance["history"]
.contents.contents[0]
.parts[0]
.text
== "user: Turn 1 user\nmodel: Turn 1 model"
)

@pytest.mark.parametrize("variable", ["history", "conversation_history"])
@pytest.mark.parametrize("conversation_history", [None, []])
def test_build_evaluation_instance_empty_history_is_empty_text(
self, variable, conversation_history
):
eval_case = agentplatform_genai_types.EvalCase(
prompt=genai_types.Content(parts=[genai_types.Part(text="Hello")]),
responses=[
agentplatform_genai_types.ResponseCandidate(
response=genai_types.Content(parts=[genai_types.Part(text="Hi")])
)
],
conversation_history=conversation_history,
)

instance = _evals_metric_handlers._build_evaluation_instance(
eval_case,
eval_case.responses[0].response,
prompt_template=f"{{{variable}}}{{prompt}}{{response}}",
)

assert instance.other_data.map_instance[variable].text == ""

@pytest.mark.parametrize(
"context",
[["Passage one.", "Passage two."], {"source": "doc-1", "text": "Passage."}],
Expand Down
12 changes: 9 additions & 3 deletions vertexai/_genai/_evals_metric_handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -685,9 +685,15 @@ def _build_evaluation_instance(
for full_path in template_vars:
# Extract the root variable (e.g. 'metadata' from 'metadata.user_id')
root_var = full_path.split(".")[0].split("[")[0]

if root_var not in standard_fields and hasattr(eval_case, root_var):
val = getattr(eval_case, root_var)
field = root_var
if root_var == "history" and not hasattr(eval_case, "history"):
field = "conversation_history"

if root_var not in standard_fields and hasattr(eval_case, field):
val = getattr(eval_case, field)
if field == "conversation_history" and not val:
other_data_map[root_var] = types.evals.InstanceData(text="")
continue
# Add the root object to other_data so the backend can traverse it
other_data_map[root_var] = types.evals.InstanceData(
contents=types.evals.InstanceDataContents(
Expand Down
Loading