Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/built_in_graders/text.md
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,10 @@ asyncio.run(main())

Check numerical calculation accuracy by comparing numbers extracted from text.

Comparisons use the original numeric tokens, so large integers and scientific notation
retain their precision when applying the absolute `tolerance`. Extracted numbers in
result metadata remain floats for compatibility and can be rounded or overflow there.

**When to use:**
- Math calculation verification
- Data report accuracy
Expand Down
33 changes: 32 additions & 1 deletion openjudge/graders/text/number_accuracy.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
"""

import re
from decimal import MAX_EMAX, MIN_EMIN, ROUND_CEILING, Context, Decimal
from typing import Any, List

from openjudge.evaluation_strategy import BaseEvaluationStrategy
Expand Down Expand Up @@ -89,6 +90,34 @@ def _extract_numbers(self, text: str) -> List[float]:
numbers = self._number_pattern.findall(text)
return [float(n) for n in numbers if n]

def _within_tolerance(self, response: str, reference: str) -> bool:
"""Compare original numeric tokens without rounding them to floats.

Args:
response: Numeric token from the generated response.
reference: Numeric token from the reference response.

Returns:
Whether the absolute difference is at most the configured tolerance.
"""
tolerance = Decimal(str(self.tolerance))
if not tolerance.is_finite():
return abs(float(response) - float(reference)) <= self.tolerance

left, right = Decimal(response), Decimal(reference)
# Round the nonnegative difference upwards with enough precision to
# represent the tolerance exactly. It cannot round across the threshold,
# even when tiny differences would disappear in the default context.
context = Context(
prec=max(28, len(tolerance.as_tuple().digits)),
rounding=ROUND_CEILING,
Emin=MIN_EMIN,
Emax=MAX_EMAX,
traps=[],
)
difference = context.subtract(max(left, right), min(left, right))
return difference <= tolerance

async def _aevaluate(self, response: str, reference_response: str) -> GraderScore:
"""
Calculate number accuracy by comparing extracted numbers from both texts.
Expand Down Expand Up @@ -160,11 +189,13 @@ async def _aevaluate(self, response: str, reference_response: str) -> GraderScor
)

# Compare numbers (match in order)
response_tokens = self._number_pattern.findall(response)
reference_tokens = self._number_pattern.findall(reference_response)
correct = 0
total = min(len(response_numbers), len(reference_response_numbers))

for i in range(total):
if abs(response_numbers[i] - reference_response_numbers[i]) <= self.tolerance:
if self._within_tolerance(response_tokens[i], reference_tokens[i]):
correct += 1

accuracy = correct / len(reference_response_numbers) if reference_response_numbers else 0.0
Expand Down
63 changes: 63 additions & 0 deletions tests/graders/text/test_number_accuracy.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
Tests for NumberAccuracyGrader
"""

from decimal import ROUND_FLOOR, Inexact, localcontext

import pytest

from openjudge.graders.text.number_accuracy import NumberAccuracyGrader
Expand Down Expand Up @@ -47,3 +49,64 @@ async def test_leading_decimal_matches(self):
reference_response="probability is 0.5",
)
assert result.score == 1.0

@pytest.mark.parametrize(
"response,reference,tolerance,expected",
[
("9007199254740992", "9007199254740993", 1e-6, 0.0),
("-9007199254740992", "-9007199254740993", 1e-6, 0.0),
("9007199254740993", "9.007199254740993e15", 0.0, 1.0),
("1e400", "10e399", 0.0, 1.0),
("1e400", "2e400", 1e-6, 0.0),
("1e-400", "0", 0.0, 0.0),
("1e-400", "2e-400", 0.0, 0.0),
("1e-400", "1e-400", 0.0, 1.0),
("0.000001", "0", 1e-6, 1.0),
("0.0000010000000000000000000000000001", "0", 1e-6, 0.0),
("0.0000009999999999999999999999999999", "0", 1e-6, 1.0),
("1.000000000000000000000000000001", "0", 1.0, 0.0),
("1", "-1e-400", 1.0, 0.0),
("1", "1e-400", 1.0, 1.0),
("1.0", "1", 0.0, 1.0),
("1.01", "1", 0.0, 0.0),
("1", "1", -1.0, 0.0),
("1", "2", float("inf"), 1.0),
("1", "1", float("-inf"), 0.0),
("1", "1", float("nan"), 0.0),
],
)
async def test_compare_original_numeric_values(
self, response: str, reference: str, tolerance: float, expected: float
) -> None:
"""Compare tokens before float rounding, retaining float-valued metadata."""
grader = NumberAccuracyGrader(tolerance=tolerance)

result = await grader.aevaluate(response=response, reference_response=reference)

assert result.score == expected
assert result.metadata["correct_numbers"] == int(expected)
assert result.metadata["response_numbers"] == [float(response)]
assert result.metadata["reference_response_numbers"] == [float(reference)]

async def test_comparison_is_independent_of_decimal_context(self) -> None:
"""A caller's decimal rounding and traps must not change the grade."""
with localcontext() as context:
context.prec = 2
context.rounding = ROUND_FLOOR
context.traps[Inexact] = True
result = await NumberAccuracyGrader(tolerance=1).aevaluate(
response="1.000000000000000000000000000001", reference_response="0"
)

assert result.score == 0.0

async def test_precise_comparison_preserves_order_and_reference_count(self) -> None:
"""Partial scores still use ordered pairs and all reference numbers."""
result = await NumberAccuracyGrader().aevaluate(
response="9007199254740992 and 2.0",
reference_response="9007199254740993 and 2 and 3",
)

assert result.score == 1 / 3
assert result.metadata["correct_numbers"] == 1
assert result.metadata["total_reference_response_numbers"] == 3
Loading