From 835948cfd9f9099b614ecbad2df0a1fef721ab4d Mon Sep 17 00:00:00 2001 From: Elon-Jiang <191222665+Elon-Jiang@users.noreply.github.com> Date: Sun, 15 Nov 2026 23:41:24 +0800 Subject: [PATCH] fix(agent): ignore tool argument key order --- docs/built_in_graders/agent_graders.md | 2 +- .../tool/tool_call_step_sequence_match.py | 4 +- .../test_tool_call_step_sequence_match.py | 38 +++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/docs/built_in_graders/agent_graders.md b/docs/built_in_graders/agent_graders.md index eca91977e..ba3693144 100644 --- a/docs/built_in_graders/agent_graders.md +++ b/docs/built_in_graders/agent_graders.md @@ -468,7 +468,7 @@ Evaluates multi-step tool call sequences with step-by-step alignment against ref | `use_jaccard_similarity` | bool | No | Use Jaccard similarity ignoring order (True) or step-by-step (False), default: True | **Scoring:** -- **Strict mode with Jaccard**: Intersection over union of (tool_name, parameters) pairs +- **Strict mode with Jaccard**: Intersection over union of (tool_name, parameters) pairs. JSON object key order is ignored, including nested objects; array order remains significant. - **Loose mode with Jaccard**: Intersection over union of tool names - **Step-by-step mode**: Average F1 score across steps. In strict mode, repeated calls to the same tool are matched one-to-one within each step to maximize total parameter similarity; a prediction is never reused. - Range: 0.0 (no match) to 1.0 (perfect match) diff --git a/openjudge/graders/agent/tool/tool_call_step_sequence_match.py b/openjudge/graders/agent/tool/tool_call_step_sequence_match.py index c46dfcaf9..b409a1db3 100644 --- a/openjudge/graders/agent/tool/tool_call_step_sequence_match.py +++ b/openjudge/graders/agent/tool/tool_call_step_sequence_match.py @@ -313,13 +313,13 @@ def calculate_jaccard_similarity_score( for tools in predicted_tool_steps.values(): for tool in tools: predicted_tool_names.add( - f"{tool.get('name', '')}: {tool.get('arguments', {})}", + f"{tool.get('name', '')}: {json.dumps(tool.get('arguments', {}), sort_keys=True)}", ) reference_tool_names = set() for tools in reference_tool_steps.values(): for tool in tools: reference_tool_names.add( - f"{tool.get('name', '')}: {tool.get('arguments', {})}", + f"{tool.get('name', '')}: {json.dumps(tool.get('arguments', {}), sort_keys=True)}", ) else: # Extract all tool names from predicted steps diff --git a/tests/graders/agent/tool/test_tool_call_step_sequence_match.py b/tests/graders/agent/tool/test_tool_call_step_sequence_match.py index e15ee3589..3901b2bde 100644 --- a/tests/graders/agent/tool/test_tool_call_step_sequence_match.py +++ b/tests/graders/agent/tool/test_tool_call_step_sequence_match.py @@ -5,6 +5,8 @@ Tests for the ToolCallStepSequenceMatchGrader class functionality. """ +import json + import pytest from openjudge.graders.agent.tool.tool_call_step_sequence_match import ( @@ -394,3 +396,39 @@ async def test_strict_steps_keep_different_tool_names_separate(): ) assert result.score == pytest.approx(0.5) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("as_json", [False, True]) +@pytest.mark.parametrize( + "predicted, reference, expected", + [ + ({"city": "杭州", "units": "metric"}, {"units": "metric", "city": "杭州"}, 1.0), + ( + {"filter": {"limit": 3, "tags": ["a", "b"]}}, + {"filter": {"tags": ["a", "b"], "limit": 3}}, + 1.0, + ), + ({"items": [{"a": 1, "b": 2}]}, {"items": [{"b": 2, "a": 1}]}, 1.0), + ({"city": "杭州"}, {"city": "上海"}, 0.0), + ({"tags": ["a", "b"]}, {"tags": ["b", "a"]}, 0.0), + ], +) +async def test_strict_jaccard_ignores_only_object_key_order(as_json, predicted, reference, expected): + """Equivalent argument objects match, while changed values and array order do not.""" + grader = ToolCallStepSequenceMatchGrader(strict_mode=True, use_jaccard_similarity=True) + result = await grader.aevaluate( + messages=[ + { + "role": "assistant", + "tool_calls": [ + {"function": {"name": "lookup", "arguments": json.dumps(predicted) if as_json else predicted}}, + ], + }, + ], + reference_tool_calls=[[{"name": "lookup", "arguments": reference}]], + ) + + assert result.score == expected + assert result.metadata["predicted_tool_count"] == 1 + assert result.metadata["reference_tool_count"] == 1