Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/built_in_graders/agent_graders.md
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,7 @@ Evaluates multi-step tool call sequences with step-by-step alignment against ref
| `use_jaccard_similarity` | bool | No | Use Jaccard similarity ignoring order (True) or step-by-step (False), default: True |

**Scoring:**
- **Strict mode with Jaccard**: Intersection over union of (tool_name, parameters) pairs
- **Strict mode with Jaccard**: Intersection over union of (tool_name, parameters) pairs. JSON object key order is ignored, including nested objects; array order remains significant.
- **Loose mode with Jaccard**: Intersection over union of tool names
- **Step-by-step mode**: Average F1 score across steps. In strict mode, repeated calls to the same tool are matched one-to-one within each step to maximize total parameter similarity; a prediction is never reused.
- Range: 0.0 (no match) to 1.0 (perfect match)
Expand Down
4 changes: 2 additions & 2 deletions openjudge/graders/agent/tool/tool_call_step_sequence_match.py
Original file line number Diff line number Diff line change
Expand Up @@ -313,13 +313,13 @@ def calculate_jaccard_similarity_score(
for tools in predicted_tool_steps.values():
for tool in tools:
predicted_tool_names.add(
f"{tool.get('name', '')}: {tool.get('arguments', {})}",
f"{tool.get('name', '')}: {json.dumps(tool.get('arguments', {}), sort_keys=True)}",
)
reference_tool_names = set()
for tools in reference_tool_steps.values():
for tool in tools:
reference_tool_names.add(
f"{tool.get('name', '')}: {tool.get('arguments', {})}",
f"{tool.get('name', '')}: {json.dumps(tool.get('arguments', {}), sort_keys=True)}",
)
else:
# Extract all tool names from predicted steps
Expand Down
38 changes: 38 additions & 0 deletions tests/graders/agent/tool/test_tool_call_step_sequence_match.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
Tests for the ToolCallStepSequenceMatchGrader class functionality.
"""

import json

import pytest

from openjudge.graders.agent.tool.tool_call_step_sequence_match import (
Expand Down Expand Up @@ -394,3 +396,39 @@ async def test_strict_steps_keep_different_tool_names_separate():
)

assert result.score == pytest.approx(0.5)


@pytest.mark.asyncio
@pytest.mark.parametrize("as_json", [False, True])
@pytest.mark.parametrize(
"predicted, reference, expected",
[
({"city": "杭州", "units": "metric"}, {"units": "metric", "city": "杭州"}, 1.0),
(
{"filter": {"limit": 3, "tags": ["a", "b"]}},
{"filter": {"tags": ["a", "b"], "limit": 3}},
1.0,
),
({"items": [{"a": 1, "b": 2}]}, {"items": [{"b": 2, "a": 1}]}, 1.0),
({"city": "杭州"}, {"city": "上海"}, 0.0),
({"tags": ["a", "b"]}, {"tags": ["b", "a"]}, 0.0),
],
)
async def test_strict_jaccard_ignores_only_object_key_order(as_json, predicted, reference, expected):
"""Equivalent argument objects match, while changed values and array order do not."""
grader = ToolCallStepSequenceMatchGrader(strict_mode=True, use_jaccard_similarity=True)
result = await grader.aevaluate(
messages=[
{
"role": "assistant",
"tool_calls": [
{"function": {"name": "lookup", "arguments": json.dumps(predicted) if as_json else predicted}},
],
},
],
reference_tool_calls=[[{"name": "lookup", "arguments": reference}]],
)

assert result.score == expected
assert result.metadata["predicted_tool_count"] == 1
assert result.metadata["reference_tool_count"] == 1
Loading