Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/built_in_graders/agent_graders.md
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@ Evaluates multi-step tool call sequences with step-by-step alignment against ref
**Scoring:**
- **Strict mode with Jaccard**: Intersection over union of (tool_name, parameters) pairs
- **Loose mode with Jaccard**: Intersection over union of tool names
- **Step-by-step mode**: Average F1 score across steps
- **Step-by-step mode**: Average F1 score across steps. In strict mode, repeated calls to the same tool are matched one-to-one within each step to maximize total parameter similarity; a prediction is never reused.
- Range: 0.0 (no match) to 1.0 (perfect match)

**Example:**
Expand Down
33 changes: 11 additions & 22 deletions openjudge/graders/agent/tool/tool_call_step_sequence_match.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from typing import Any, Dict, List, Set, Tuple

from loguru import logger
from scipy.optimize import linear_sum_assignment

from openjudge.evaluation_strategy import BaseEvaluationStrategy
from openjudge.graders.base_grader import (
Expand Down Expand Up @@ -259,30 +260,18 @@ def calculate_step_matching_score(
# Tool names don't match, score is 0
step_score = 0.0
else:
# Tool names match, calculate F1 score for parameters
# For each reference tool, find matching predicted tool by name and calculate F1
tool_scores = []
for gt_tool in reference_tools:
gt_name = gt_tool.get("name", "")
# Find matching predicted tool by name
matching_pred_tool = None
for pred_tool in predicted_tools:
if pred_tool.get("name", "") == gt_name:
matching_pred_tool = pred_tool
break
for tool_name in dict.fromkeys(gt_tool_names):
reference_group = [tool for tool in reference_tools if tool.get("name", "") == tool_name]
predicted_group = [tool for tool in predicted_tools if tool.get("name", "") == tool_name]
similarities = [
[self.calculate_param_similarity(reference, predicted) for predicted in predicted_group]
for reference in reference_group
]
# Assign each prediction at most once, preserving the best total parameter similarity.
rows, columns = linear_sum_assignment(similarities, maximize=True)
tool_scores.extend(similarities[row][column] for row, column in zip(rows, columns))

if matching_pred_tool is not None:
# Calculate F1 score for this tool pair
tool_f1 = self.calculate_param_similarity(
gt_tool,
matching_pred_tool,
)
tool_scores.append(tool_f1)
else:
# Ground truth tool not found in predictions, score is 0
tool_scores.append(0.0)

# Average F1 score across all reference tools in this step
step_score = sum(tool_scores) / len(tool_scores) if tool_scores else 0.0
else:
# In loose mode, calculate step score based on the ratio of matched tools
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ dependencies = [
"rouge-score>=0.1.2",
"python-Levenshtein>=0.20.0",
"scikit-learn>=1.0.0",
"scipy>=1.8.0",
"PyYAML>=6.0.0",
"python-dotenv",
]
Expand Down
79 changes: 79 additions & 0 deletions tests/graders/agent/tool/test_tool_call_step_sequence_match.py
Original file line number Diff line number Diff line change
Expand Up @@ -315,3 +315,82 @@ async def test_tool_call_sequence_match_grader_recall_vs_precision():

assert recall_result.score == 1.0
assert precision_result.score == 0.5


@pytest.mark.asyncio
@pytest.mark.parametrize(
("predicted_queries", "reference_queries", "expected_score"),
[
(["A", "B"], ["A", "B"], 1.0),
(["B", "A"], ["A", "B"], 1.0),
(["A", "B", "C"], ["C", "A", "B"], 1.0),
(["A", "B"], ["A", "A"], 0.75),
(["A", "A"], ["A", "B"], 0.75),
(["C", "B", "A"], ["A", "B"], 1.0),
(["A"], ["A", "A"], 0.0),
(["A", "A"], ["A", "A"], 1.0),
(["A"], ["A"], 1.0),
],
)
async def test_strict_steps_match_repeated_tools_once(predicted_queries, reference_queries, expected_score):
"""Match same-name calls by parameters without reusing predictions."""
grader = ToolCallStepSequenceMatchGrader(strict_mode=True, use_jaccard_similarity=False)
predicted = [{"name": "search", "arguments": {"query": query}} for query in predicted_queries]
reference = [{"name": "search", "arguments": {"query": query}} for query in reference_queries]

result = await grader.aevaluate(
messages=[{"role": "assistant", "tool_calls": predicted}],
reference_tool_calls=[reference],
)

assert result.score == pytest.approx(expected_score)


@pytest.mark.asyncio
@pytest.mark.parametrize("reverse_predictions", [False, True])
async def test_strict_steps_maximize_total_parameter_similarity(reverse_predictions):
"""An early tied match must not prevent a better total assignment."""
grader = ToolCallStepSequenceMatchGrader(strict_mode=True, use_jaccard_similarity=False)
# The first reference ties on either prediction (2/3). The second needs
# the first prediction to score 1 rather than 0.
reference = [
{"name": "search", "arguments": {"a": "A", "b": "B"}},
{"name": "search", "arguments": {"a": "A"}},
]
predicted = [
{"name": "search", "arguments": {"a": "A"}},
{"name": "search", "arguments": {"b": "B"}},
]
if reverse_predictions:
predicted.reverse()

result = await grader.aevaluate(
messages=[{"role": "assistant", "tool_calls": predicted}],
reference_tool_calls=[reference],
)

assert result.score == pytest.approx(5 / 6)


@pytest.mark.asyncio
async def test_strict_steps_keep_different_tool_names_separate():
"""Parameter matches cannot cross tool names or assistant turns."""
grader = ToolCallStepSequenceMatchGrader(strict_mode=True, use_jaccard_similarity=False)
result = await grader.aevaluate(
messages=[
{
"role": "assistant",
"tool_calls": [
{"name": "search", "arguments": {"q": "B"}},
{"name": "write", "arguments": {"q": "A"}},
],
},
{"role": "assistant", "tool_calls": [{"name": "search", "arguments": {"q": "A"}}]},
],
reference_tool_calls=[
[{"name": "search", "arguments": {"q": "A"}}, {"name": "write", "arguments": {"q": "B"}}],
[{"name": "search", "arguments": {"q": "B"}}],
],
)

assert result.score == pytest.approx(0.5)
Loading