Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/running_graders/run_tasks.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,11 @@ After running multiple graders, you might want to combine their results into a s

These aggregators allow you to create composite scores that reflect multiple evaluation dimensions, making it easier to compare overall performance across different models or configurations.

You can also pass a function directly as `aggregators=my_aggregator`, or include
it in a list alongside other aggregators. The function receives the dictionary of
grader results for one sample and returns a `GraderResult`; its function name is
the key used for its results.

Often you'll want to combine multiple grader results into unified scores:

```python
Expand Down
2 changes: 1 addition & 1 deletion openjudge/runner/grading_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ def __init__(
# Handle aggregators
if not aggregators:
self.aggregators = []
elif isinstance(aggregators, BaseAggregator):
elif isinstance(aggregators, BaseAggregator) or callable(aggregators):
self.aggregators = [aggregators]
else:
self.aggregators = aggregators
Expand Down
49 changes: 49 additions & 0 deletions tests/runner/test_callable_aggregator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
"""Exercise callable aggregators through the public grading runner."""

import pytest

from openjudge.graders.function_grader import FunctionGrader
from openjudge.graders.schema import GraderScore
from openjudge.runner.aggregator.weighted_sum_aggregator import WeightedSumAggregator
from openjudge.runner.grading_runner import GradingRunner


def copy_score(results):
"""Use the only grader's score as the aggregate."""
return GraderScore(name="copy_score", score=results["score"].score, reason="Copied score")


@pytest.mark.asyncio
@pytest.mark.parametrize("multiple_datasets", [False, True])
@pytest.mark.parametrize("kind", ["function", "function_list", "base_aggregator", "none"])
async def test_runner_aggregator_forms(kind, multiple_datasets):
"""A single function must work like its one-element-list equivalent."""

async def evaluate(value):
return GraderScore(name="score", score=value, reason="Supplied score")

aggregators = {
"function": copy_score,
"function_list": [copy_score],
"base_aggregator": WeightedSumAggregator(name="weighted_sum", weights={"score": 1.0}),
"none": None,
}
runner = GradingRunner(
{"score": FunctionGrader(evaluate, name="score")},
aggregators=aggregators[kind],
show_progress=False,
)
dataset = [{"value": 0.25}, {"value": 0.75}]
if multiple_datasets:
runs = await runner.arun_multiple_datasets([dataset, dataset[:1]])
else:
runs = [await runner.arun(dataset)]

for run, expected in zip(runs, [[0.25, 0.75], [0.25]]):
assert [result.score for result in run["score"]] == expected
if kind == "none":
assert set(run) == {"score"}
else:
key = "weighted_sum" if kind == "base_aggregator" else "copy_score"
assert set(run) == {"score", key}
assert [result.score for result in run[key]] == expected
Loading