diff --git a/agentplatform/_genai/_evals_metric_handlers.py b/agentplatform/_genai/_evals_metric_handlers.py index 62b25946f7..51ba002d56 100644 --- a/agentplatform/_genai/_evals_metric_handlers.py +++ b/agentplatform/_genai/_evals_metric_handlers.py @@ -687,9 +687,15 @@ def _build_evaluation_instance( for full_path in template_vars: # Extract the root variable (e.g. 'metadata' from 'metadata.user_id') root_var = full_path.split(".")[0].split("[")[0] - - if root_var not in standard_fields and hasattr(eval_case, root_var): - val = getattr(eval_case, root_var) + field = root_var + if root_var == "history" and not hasattr(eval_case, "history"): + field = "conversation_history" + + if root_var not in standard_fields and hasattr(eval_case, field): + val = getattr(eval_case, field) + if field == "conversation_history" and not val: + other_data_map[root_var] = types.evals.InstanceData(text="") + continue # Add the root object to other_data so the backend can traverse it other_data_map[root_var] = types.evals.InstanceData( contents=types.evals.InstanceDataContents( diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index fb738cbd51..07db5e248e 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -7351,6 +7351,65 @@ def test_build_evaluation_instance_various_field_types(self): == "System instructions here." ) + def test_build_evaluation_instance_history_uses_conversation_history(self): + eval_case = agentplatform_genai_types.EvalCase( + prompt=genai_types.Content(parts=[genai_types.Part(text="Next")]), + responses=[ + agentplatform_genai_types.ResponseCandidate( + response=genai_types.Content(parts=[genai_types.Part(text="Hi")]) + ) + ], + conversation_history=[ + agentplatform_genai_types.evals.Message( + content=genai_types.Content( + parts=[genai_types.Part(text="Turn 1 user")], role="user" + ) + ), + agentplatform_genai_types.evals.Message( + content=genai_types.Content( + parts=[genai_types.Part(text="Turn 1 model")], role="model" + ) + ), + ], + ) + + instance = _evals_metric_handlers._build_evaluation_instance( + eval_case, + eval_case.responses[0].response, + prompt_template="{history}{prompt}{response}", + ) + + assert ( + instance.other_data.map_instance["history"] + .contents.contents[0] + .parts[0] + .text + == "user: Turn 1 user\nmodel: Turn 1 model" + ) + + @pytest.mark.parametrize("variable", ["history", "conversation_history"]) + @pytest.mark.parametrize("conversation_history", [None, []]) + def test_build_evaluation_instance_empty_history_is_empty_text( + self, variable, conversation_history + ): + eval_case = agentplatform_genai_types.EvalCase( + prompt=genai_types.Content(parts=[genai_types.Part(text="Hello")]), + responses=[ + agentplatform_genai_types.ResponseCandidate( + response=genai_types.Content(parts=[genai_types.Part(text="Hi")]) + ) + ], + conversation_history=conversation_history, + ) + + instance = _evals_metric_handlers._build_evaluation_instance( + eval_case, + eval_case.responses[0].response, + prompt_template=f"{{{variable}}}{{prompt}}{{response}}", + ) + + assert instance.other_data.map_instance[variable].text == "" + @pytest.mark.parametrize( "context", [["Passage one.", "Passage two."], {"source": "doc-1", "text": "Passage."}], diff --git a/vertexai/_genai/_evals_metric_handlers.py b/vertexai/_genai/_evals_metric_handlers.py index 0d5f4404c0..520e9d4224 100644 --- a/vertexai/_genai/_evals_metric_handlers.py +++ b/vertexai/_genai/_evals_metric_handlers.py @@ -685,9 +685,15 @@ def _build_evaluation_instance( for full_path in template_vars: # Extract the root variable (e.g. 'metadata' from 'metadata.user_id') root_var = full_path.split(".")[0].split("[")[0] - - if root_var not in standard_fields and hasattr(eval_case, root_var): - val = getattr(eval_case, root_var) + field = root_var + if root_var == "history" and not hasattr(eval_case, "history"): + field = "conversation_history" + + if root_var not in standard_fields and hasattr(eval_case, field): + val = getattr(eval_case, field) + if field == "conversation_history" and not val: + other_data_map[root_var] = types.evals.InstanceData(text="") + continue # Add the root object to other_data so the backend can traverse it other_data_map[root_var] = types.evals.InstanceData( contents=types.evals.InstanceDataContents(