From 74028e041dcc0c768268ed107d439a93b4d5d162 Mon Sep 17 00:00:00 2001 From: Elon-Jiang <191222665+Elon-Jiang@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:20:17 +0800 Subject: [PATCH] fix(runner): accept a single callable aggregator Co-authored-by: OpenAI Codex --- docs/running_graders/run_tasks.md | 5 +++ openjudge/runner/grading_runner.py | 2 +- tests/runner/test_callable_aggregator.py | 49 ++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/runner/test_callable_aggregator.py diff --git a/docs/running_graders/run_tasks.md b/docs/running_graders/run_tasks.md index ddfc4b96e..642fa690a 100644 --- a/docs/running_graders/run_tasks.md +++ b/docs/running_graders/run_tasks.md @@ -148,6 +148,11 @@ After running multiple graders, you might want to combine their results into a s These aggregators allow you to create composite scores that reflect multiple evaluation dimensions, making it easier to compare overall performance across different models or configurations. +You can also pass a function directly as `aggregators=my_aggregator`, or include +it in a list alongside other aggregators. The function receives the dictionary of +grader results for one sample and returns a `GraderResult`; its function name is +the key used for its results. + Often you'll want to combine multiple grader results into unified scores: ```python diff --git a/openjudge/runner/grading_runner.py b/openjudge/runner/grading_runner.py index b98db8ae9..eee01d26f 100644 --- a/openjudge/runner/grading_runner.py +++ b/openjudge/runner/grading_runner.py @@ -213,7 +213,7 @@ def __init__( # Handle aggregators if not aggregators: self.aggregators = [] - elif isinstance(aggregators, BaseAggregator): + elif isinstance(aggregators, BaseAggregator) or callable(aggregators): self.aggregators = [aggregators] else: self.aggregators = aggregators diff --git a/tests/runner/test_callable_aggregator.py b/tests/runner/test_callable_aggregator.py new file mode 100644 index 000000000..30c65e513 --- /dev/null +++ b/tests/runner/test_callable_aggregator.py @@ -0,0 +1,49 @@ +"""Exercise callable aggregators through the public grading runner.""" + +import pytest + +from openjudge.graders.function_grader import FunctionGrader +from openjudge.graders.schema import GraderScore +from openjudge.runner.aggregator.weighted_sum_aggregator import WeightedSumAggregator +from openjudge.runner.grading_runner import GradingRunner + + +def copy_score(results): + """Use the only grader's score as the aggregate.""" + return GraderScore(name="copy_score", score=results["score"].score, reason="Copied score") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("multiple_datasets", [False, True]) +@pytest.mark.parametrize("kind", ["function", "function_list", "base_aggregator", "none"]) +async def test_runner_aggregator_forms(kind, multiple_datasets): + """A single function must work like its one-element-list equivalent.""" + + async def evaluate(value): + return GraderScore(name="score", score=value, reason="Supplied score") + + aggregators = { + "function": copy_score, + "function_list": [copy_score], + "base_aggregator": WeightedSumAggregator(name="weighted_sum", weights={"score": 1.0}), + "none": None, + } + runner = GradingRunner( + {"score": FunctionGrader(evaluate, name="score")}, + aggregators=aggregators[kind], + show_progress=False, + ) + dataset = [{"value": 0.25}, {"value": 0.75}] + if multiple_datasets: + runs = await runner.arun_multiple_datasets([dataset, dataset[:1]]) + else: + runs = [await runner.arun(dataset)] + + for run, expected in zip(runs, [[0.25, 0.75], [0.25]]): + assert [result.score for result in run["score"]] == expected + if kind == "none": + assert set(run) == {"score"} + else: + key = "weighted_sum" if kind == "base_aggregator" else "copy_score" + assert set(run) == {"score", key} + assert [result.score for result in run[key]] == expected