From 6ef36678c7afeda427bd0eb40679f16c8f7212c5 Mon Sep 17 00:00:00 2001 From: Elon-Jiang <191222665+Elon-Jiang@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:13:54 +0800 Subject: [PATCH] fix(text): compare unrounded numeric tokens Co-authored-by: OpenAI Codex --- docs/built_in_graders/text.md | 4 ++ openjudge/graders/text/number_accuracy.py | 33 +++++++++++- tests/graders/text/test_number_accuracy.py | 63 ++++++++++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) diff --git a/docs/built_in_graders/text.md b/docs/built_in_graders/text.md index df9eef27f..5f0a8d555 100644 --- a/docs/built_in_graders/text.md +++ b/docs/built_in_graders/text.md @@ -407,6 +407,10 @@ asyncio.run(main()) Check numerical calculation accuracy by comparing numbers extracted from text. +Comparisons use the original numeric tokens, so large integers and scientific notation +retain their precision when applying the absolute `tolerance`. Extracted numbers in +result metadata remain floats for compatibility and can be rounded or overflow there. + **When to use:** - Math calculation verification - Data report accuracy diff --git a/openjudge/graders/text/number_accuracy.py b/openjudge/graders/text/number_accuracy.py index d984413d6..6eaa63920 100644 --- a/openjudge/graders/text/number_accuracy.py +++ b/openjudge/graders/text/number_accuracy.py @@ -10,6 +10,7 @@ """ import re +from decimal import MAX_EMAX, MIN_EMIN, ROUND_CEILING, Context, Decimal from typing import Any, List from openjudge.evaluation_strategy import BaseEvaluationStrategy @@ -89,6 +90,34 @@ def _extract_numbers(self, text: str) -> List[float]: numbers = self._number_pattern.findall(text) return [float(n) for n in numbers if n] + def _within_tolerance(self, response: str, reference: str) -> bool: + """Compare original numeric tokens without rounding them to floats. + + Args: + response: Numeric token from the generated response. + reference: Numeric token from the reference response. + + Returns: + Whether the absolute difference is at most the configured tolerance. + """ + tolerance = Decimal(str(self.tolerance)) + if not tolerance.is_finite(): + return abs(float(response) - float(reference)) <= self.tolerance + + left, right = Decimal(response), Decimal(reference) + # Round the nonnegative difference upwards with enough precision to + # represent the tolerance exactly. It cannot round across the threshold, + # even when tiny differences would disappear in the default context. + context = Context( + prec=max(28, len(tolerance.as_tuple().digits)), + rounding=ROUND_CEILING, + Emin=MIN_EMIN, + Emax=MAX_EMAX, + traps=[], + ) + difference = context.subtract(max(left, right), min(left, right)) + return difference <= tolerance + async def _aevaluate(self, response: str, reference_response: str) -> GraderScore: """ Calculate number accuracy by comparing extracted numbers from both texts. @@ -160,11 +189,13 @@ async def _aevaluate(self, response: str, reference_response: str) -> GraderScor ) # Compare numbers (match in order) + response_tokens = self._number_pattern.findall(response) + reference_tokens = self._number_pattern.findall(reference_response) correct = 0 total = min(len(response_numbers), len(reference_response_numbers)) for i in range(total): - if abs(response_numbers[i] - reference_response_numbers[i]) <= self.tolerance: + if self._within_tolerance(response_tokens[i], reference_tokens[i]): correct += 1 accuracy = correct / len(reference_response_numbers) if reference_response_numbers else 0.0 diff --git a/tests/graders/text/test_number_accuracy.py b/tests/graders/text/test_number_accuracy.py index b4ea0f3d1..6651faa0a 100644 --- a/tests/graders/text/test_number_accuracy.py +++ b/tests/graders/text/test_number_accuracy.py @@ -3,6 +3,8 @@ Tests for NumberAccuracyGrader """ +from decimal import ROUND_FLOOR, Inexact, localcontext + import pytest from openjudge.graders.text.number_accuracy import NumberAccuracyGrader @@ -47,3 +49,64 @@ async def test_leading_decimal_matches(self): reference_response="probability is 0.5", ) assert result.score == 1.0 + + @pytest.mark.parametrize( + "response,reference,tolerance,expected", + [ + ("9007199254740992", "9007199254740993", 1e-6, 0.0), + ("-9007199254740992", "-9007199254740993", 1e-6, 0.0), + ("9007199254740993", "9.007199254740993e15", 0.0, 1.0), + ("1e400", "10e399", 0.0, 1.0), + ("1e400", "2e400", 1e-6, 0.0), + ("1e-400", "0", 0.0, 0.0), + ("1e-400", "2e-400", 0.0, 0.0), + ("1e-400", "1e-400", 0.0, 1.0), + ("0.000001", "0", 1e-6, 1.0), + ("0.0000010000000000000000000000000001", "0", 1e-6, 0.0), + ("0.0000009999999999999999999999999999", "0", 1e-6, 1.0), + ("1.000000000000000000000000000001", "0", 1.0, 0.0), + ("1", "-1e-400", 1.0, 0.0), + ("1", "1e-400", 1.0, 1.0), + ("1.0", "1", 0.0, 1.0), + ("1.01", "1", 0.0, 0.0), + ("1", "1", -1.0, 0.0), + ("1", "2", float("inf"), 1.0), + ("1", "1", float("-inf"), 0.0), + ("1", "1", float("nan"), 0.0), + ], + ) + async def test_compare_original_numeric_values( + self, response: str, reference: str, tolerance: float, expected: float + ) -> None: + """Compare tokens before float rounding, retaining float-valued metadata.""" + grader = NumberAccuracyGrader(tolerance=tolerance) + + result = await grader.aevaluate(response=response, reference_response=reference) + + assert result.score == expected + assert result.metadata["correct_numbers"] == int(expected) + assert result.metadata["response_numbers"] == [float(response)] + assert result.metadata["reference_response_numbers"] == [float(reference)] + + async def test_comparison_is_independent_of_decimal_context(self) -> None: + """A caller's decimal rounding and traps must not change the grade.""" + with localcontext() as context: + context.prec = 2 + context.rounding = ROUND_FLOOR + context.traps[Inexact] = True + result = await NumberAccuracyGrader(tolerance=1).aevaluate( + response="1.000000000000000000000000000001", reference_response="0" + ) + + assert result.score == 0.0 + + async def test_precise_comparison_preserves_order_and_reference_count(self) -> None: + """Partial scores still use ordered pairs and all reference numbers.""" + result = await NumberAccuracyGrader().aevaluate( + response="9007199254740992 and 2.0", + reference_response="9007199254740993 and 2 and 3", + ) + + assert result.score == 1 / 3 + assert result.metadata["correct_numbers"] == 1 + assert result.metadata["total_reference_response_numbers"] == 3