Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions python/fi/evals/metrics/agents/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,8 @@ def compute_one(self, inputs: AgentTrajectoryInput) -> Dict[str, Any]:
else:
seen_signatures.add(call_sig)

redundancy_ratio = 1.0 - (redundant_count / total_steps) if total_steps > 0 else 1.0
total_calls = sum(len(step.tool_calls) for step in inputs.trajectory)
redundancy_ratio = 1.0 - (redundant_count / total_calls) if total_calls > 0 else 1.0
redundancy_score = redundancy_ratio * self.redundancy_weight
details["redundant_steps"] = redundant_count

Expand All @@ -259,7 +260,6 @@ def compute_one(self, inputs: AgentTrajectoryInput) -> Dict[str, Any]:
for tc in step.tool_calls
if not tc.success
)
total_calls = sum(len(step.tool_calls) for step in inputs.trajectory)
failure_ratio = 1.0 - (failed_calls / total_calls) if total_calls > 0 else 1.0
failure_score = failure_ratio * self.failure_weight
details["failed_calls"] = failed_calls
Expand Down
25 changes: 25 additions & 0 deletions python/tests/sdk/test_agents.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,31 @@ def test_redundant_steps(self):
result = metric.compute_one(input_data)
assert result["details"]["redundant_steps"] >= 3

def test_redundant_calls_within_single_step(self):
"""Multiple redundant calls in one step must not push the score below 0."""
metric = StepEfficiency()
step = AgentStep(
step_number=1,
thought="Repeating the same call",
tool_calls=[
ToolCall(
name="search",
arguments={"query": "test"},
result="result",
success=True,
)
for _ in range(5)
],
is_final=True,
)
input_data = AgentTrajectoryInput(
trajectory=[step],
task=TaskDefinition(description="Search for something")
)
result = metric.compute_one(input_data)
assert 0.0 <= result["output"] <= 1.0
assert result["details"]["redundant_steps"] == 4

def test_failed_tool_calls(self):
"""Test trajectory with failed tool calls."""
metric = StepEfficiency()
Expand Down