Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/built_in_graders/format.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ Performs deep structural comparison of JSON objects by recursively validating th

**Scoring:**
- **1.0**: JSON structures match completely
- JSON numbers are compared by their exact decimal values, without binary floating-point rounding. Equivalent integer and decimal forms (such as `1` and `1.0`) match.
- **0.0**: Structures differ or parse error

**Example:**
Expand Down
5 changes: 3 additions & 2 deletions openjudge/graders/format/json/json_match.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import json
from collections import deque
from decimal import Decimal
from typing import Any

from openjudge.evaluation_strategy.base_evaluation_strategy import (
Expand Down Expand Up @@ -161,7 +162,7 @@ def _compute(self, reference: str, response: str) -> tuple[bool, dict]:
# RFC 8259 defines no NaN/Infinity literals, but Python's json
# module parses them by default. Grade such responses as invalid
# instead of letting e.g. {"a": Infinity} match {"a": 1e999}.
response_json = json.loads(response, parse_constant=reject_non_finite)
response_json = json.loads(response, parse_float=Decimal, parse_constant=reject_non_finite)
except (ValueError, TypeError) as e:
return False, {
"matched": False,
Expand All @@ -171,7 +172,7 @@ def _compute(self, reference: str, response: str) -> tuple[bool, dict]:

# Parse reference JSON
try:
reference_json = json.loads(reference, parse_constant=reject_non_finite)
reference_json = json.loads(reference, parse_float=Decimal, parse_constant=reject_non_finite)
except (ValueError, TypeError) as e:
return False, {
"matched": False,
Expand Down
29 changes: 29 additions & 0 deletions tests/graders/format/test_json_match.py
Original file line number Diff line number Diff line change
Expand Up @@ -465,3 +465,32 @@ async def test_order_and_extra_key_flags(
response='[{"a":1,"b":2,"extra":3},{"a":1}]',
)
assert result.score == expected


@pytest.mark.asyncio
@pytest.mark.parametrize("strict_order", [True, False])
@pytest.mark.parametrize(
"reference, response, expected",
[
("1.0000000000000001", "1.0", 0.0),
("9007199254740992.0", "9007199254740993.0", 0.0),
("1e400", "2e400", 0.0),
("1e-400", "2e-400", 0.0),
("1e-400", "0", 0.0),
("9007199254740993", "9007199254740993.0", 1.0),
("1e400", "10e399", 1.0),
("1", "1.0", 1.0),
("-0.0", "0", 1.0),
("true", "1.0", 0.0),
],
)
async def test_json_numbers_compare_without_float_rounding(strict_order, reference, response, expected):
"""Compare original JSON numeric values in both ordered and unordered nested records."""
result = await JsonMatchGrader(strict_order=strict_order).aevaluate(
reference_response='[{"value": ' + reference + "}]",
response='[{"value": ' + response + "}]",
)

assert result.score == expected
assert result.metadata["matched"] == bool(expected)
json.dumps(result.model_dump())
Loading