From 72c05a43b6460b39301637c207d7464bf616046d Mon Sep 17 00:00:00 2001 From: anticipate218 <3481402509@qq.com> Date: Thu, 12 Nov 2026 21:41:24 +0800 Subject: [PATCH] fix(grader): match repeated tools one-to-one Match same-name tool calls using a maximum-weight one-to-one assignment. Keep step, count and legacy mode behavior unchanged. Assisted-by: OpenAI Codex --- docs/built_in_graders/agent_graders.md | 2 +- .../tool/tool_call_step_sequence_match.py | 33 +++----- pyproject.toml | 1 + .../test_tool_call_step_sequence_match.py | 79 +++++++++++++++++++ 4 files changed, 92 insertions(+), 23 deletions(-) diff --git a/docs/built_in_graders/agent_graders.md b/docs/built_in_graders/agent_graders.md index 4a1b54139..eca91977e 100644 --- a/docs/built_in_graders/agent_graders.md +++ b/docs/built_in_graders/agent_graders.md @@ -470,7 +470,7 @@ Evaluates multi-step tool call sequences with step-by-step alignment against ref **Scoring:** - **Strict mode with Jaccard**: Intersection over union of (tool_name, parameters) pairs - **Loose mode with Jaccard**: Intersection over union of tool names -- **Step-by-step mode**: Average F1 score across steps +- **Step-by-step mode**: Average F1 score across steps. In strict mode, repeated calls to the same tool are matched one-to-one within each step to maximize total parameter similarity; a prediction is never reused. - Range: 0.0 (no match) to 1.0 (perfect match) **Example:** diff --git a/openjudge/graders/agent/tool/tool_call_step_sequence_match.py b/openjudge/graders/agent/tool/tool_call_step_sequence_match.py index 9f3105aa2..c46dfcaf9 100644 --- a/openjudge/graders/agent/tool/tool_call_step_sequence_match.py +++ b/openjudge/graders/agent/tool/tool_call_step_sequence_match.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List, Set, Tuple from loguru import logger +from scipy.optimize import linear_sum_assignment from openjudge.evaluation_strategy import BaseEvaluationStrategy from openjudge.graders.base_grader import ( @@ -259,30 +260,18 @@ def calculate_step_matching_score( # Tool names don't match, score is 0 step_score = 0.0 else: - # Tool names match, calculate F1 score for parameters - # For each reference tool, find matching predicted tool by name and calculate F1 tool_scores = [] - for gt_tool in reference_tools: - gt_name = gt_tool.get("name", "") - # Find matching predicted tool by name - matching_pred_tool = None - for pred_tool in predicted_tools: - if pred_tool.get("name", "") == gt_name: - matching_pred_tool = pred_tool - break + for tool_name in dict.fromkeys(gt_tool_names): + reference_group = [tool for tool in reference_tools if tool.get("name", "") == tool_name] + predicted_group = [tool for tool in predicted_tools if tool.get("name", "") == tool_name] + similarities = [ + [self.calculate_param_similarity(reference, predicted) for predicted in predicted_group] + for reference in reference_group + ] + # Assign each prediction at most once, preserving the best total parameter similarity. + rows, columns = linear_sum_assignment(similarities, maximize=True) + tool_scores.extend(similarities[row][column] for row, column in zip(rows, columns)) - if matching_pred_tool is not None: - # Calculate F1 score for this tool pair - tool_f1 = self.calculate_param_similarity( - gt_tool, - matching_pred_tool, - ) - tool_scores.append(tool_f1) - else: - # Ground truth tool not found in predictions, score is 0 - tool_scores.append(0.0) - - # Average F1 score across all reference tools in this step step_score = sum(tool_scores) / len(tool_scores) if tool_scores else 0.0 else: # In loose mode, calculate step score based on the ratio of matched tools diff --git a/pyproject.toml b/pyproject.toml index 781526031..d7d41b5b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ dependencies = [ "rouge-score>=0.1.2", "python-Levenshtein>=0.20.0", "scikit-learn>=1.0.0", + "scipy>=1.8.0", "PyYAML>=6.0.0", "python-dotenv", ] diff --git a/tests/graders/agent/tool/test_tool_call_step_sequence_match.py b/tests/graders/agent/tool/test_tool_call_step_sequence_match.py index 08c0a8aa0..e15ee3589 100644 --- a/tests/graders/agent/tool/test_tool_call_step_sequence_match.py +++ b/tests/graders/agent/tool/test_tool_call_step_sequence_match.py @@ -315,3 +315,82 @@ async def test_tool_call_sequence_match_grader_recall_vs_precision(): assert recall_result.score == 1.0 assert precision_result.score == 0.5 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("predicted_queries", "reference_queries", "expected_score"), + [ + (["A", "B"], ["A", "B"], 1.0), + (["B", "A"], ["A", "B"], 1.0), + (["A", "B", "C"], ["C", "A", "B"], 1.0), + (["A", "B"], ["A", "A"], 0.75), + (["A", "A"], ["A", "B"], 0.75), + (["C", "B", "A"], ["A", "B"], 1.0), + (["A"], ["A", "A"], 0.0), + (["A", "A"], ["A", "A"], 1.0), + (["A"], ["A"], 1.0), + ], +) +async def test_strict_steps_match_repeated_tools_once(predicted_queries, reference_queries, expected_score): + """Match same-name calls by parameters without reusing predictions.""" + grader = ToolCallStepSequenceMatchGrader(strict_mode=True, use_jaccard_similarity=False) + predicted = [{"name": "search", "arguments": {"query": query}} for query in predicted_queries] + reference = [{"name": "search", "arguments": {"query": query}} for query in reference_queries] + + result = await grader.aevaluate( + messages=[{"role": "assistant", "tool_calls": predicted}], + reference_tool_calls=[reference], + ) + + assert result.score == pytest.approx(expected_score) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reverse_predictions", [False, True]) +async def test_strict_steps_maximize_total_parameter_similarity(reverse_predictions): + """An early tied match must not prevent a better total assignment.""" + grader = ToolCallStepSequenceMatchGrader(strict_mode=True, use_jaccard_similarity=False) + # The first reference ties on either prediction (2/3). The second needs + # the first prediction to score 1 rather than 0. + reference = [ + {"name": "search", "arguments": {"a": "A", "b": "B"}}, + {"name": "search", "arguments": {"a": "A"}}, + ] + predicted = [ + {"name": "search", "arguments": {"a": "A"}}, + {"name": "search", "arguments": {"b": "B"}}, + ] + if reverse_predictions: + predicted.reverse() + + result = await grader.aevaluate( + messages=[{"role": "assistant", "tool_calls": predicted}], + reference_tool_calls=[reference], + ) + + assert result.score == pytest.approx(5 / 6) + + +@pytest.mark.asyncio +async def test_strict_steps_keep_different_tool_names_separate(): + """Parameter matches cannot cross tool names or assistant turns.""" + grader = ToolCallStepSequenceMatchGrader(strict_mode=True, use_jaccard_similarity=False) + result = await grader.aevaluate( + messages=[ + { + "role": "assistant", + "tool_calls": [ + {"name": "search", "arguments": {"q": "B"}}, + {"name": "write", "arguments": {"q": "A"}}, + ], + }, + {"role": "assistant", "tool_calls": [{"name": "search", "arguments": {"q": "A"}}]}, + ], + reference_tool_calls=[ + [{"name": "search", "arguments": {"q": "A"}}, {"name": "write", "arguments": {"q": "B"}}], + [{"name": "search", "arguments": {"q": "B"}}], + ], + ) + + assert result.score == pytest.approx(0.5)