Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/built_in_graders/format.md
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ Detects and penalizes repetitive patterns in text using N-gram analysis with sup
| `penalty_rate` | float | No | Penalty rate per repetition (default: 1.0) |
| `use_soft_penalty` | bool | No | Use soft penalty mode (default: False) |
| `max_penalty` | float | No | Maximum penalty value (default: -1.0) |
| `min_scaling` | float | No | Soft penalty starts only above this repetition rate (default: 0.0) |
| `tokenizer_type` | str | No | Tokenizer type: tiktoken, jieba, simple (default: tiktoken) |
| `analyze_scope` | str | No | Analyze "thought" or "full" text (default: full) |

Expand Down
2 changes: 1 addition & 1 deletion openjudge/graders/format/ngram_repetition_penalty.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ def _calculate_penalty(self, repetition_rate: float) -> float:
)

scaling = repetition_rate
if scaling < self.min_scaling:
if scaling <= self.min_scaling:
scaling = 0.0
elif scaling > self.min_scaling:
scaling = (scaling - self.min_scaling) / (1 - self.min_scaling)
Expand Down
59 changes: 59 additions & 0 deletions tests/graders/format/test_ngram_repetition_penalty.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
# -*- coding: utf-8 -*-
"""Public scoring regressions for repetition penalty thresholds."""

import pytest

from openjudge.graders.format.ngram_repetition_penalty import (
NgramRepetitionPenaltyGrader,
)
from openjudge.utils.tokenizer import TokenizerEnum


@pytest.mark.unit
@pytest.mark.parametrize(
"n,response,min_scaling,expected_rate,expected_score",
[
(1, "alpha beta gamma alpha", 0.5, 0.25, 0.0),
(1, "alpha beta alpha beta", 0.5, 0.5, 0.0),
(2, "alpha alpha alpha alpha beta", 0.5, 0.5, 0.0),
(3, "alpha alpha alpha alpha alpha beta", 0.5, 0.5, 0.0),
(1, "alpha beta alpha beta alpha beta alpha beta", 0.75, 0.75, 0.0),
(1, "alpha beta alpha beta alpha", 0.5, 0.6, -0.4),
(1, "alpha beta alpha beta alpha beta alpha beta", 0.5, 0.75, -1.0),
(1, "alpha beta gamma delta", 0.0, 0.0, 0.0),
(1, "alpha beta alpha beta", 0.0, 0.5, -1.0),
],
)
async def test_soft_penalty_starts_above_min_scaling(
n: int, response: str, min_scaling: float, expected_rate: float, expected_score: float
) -> None:
"""Score actual tokenized responses without penalizing the threshold itself."""
grader = NgramRepetitionPenaltyGrader(
n=n,
use_soft_penalty=True,
min_scaling=min_scaling,
max_penalty=-2.0,
tokenizer_type=TokenizerEnum.simple,
)

result = await grader.aevaluate(response=response)

assert result.score == pytest.approx(expected_score)
assert result.metadata["penalty"] == pytest.approx(expected_score)
assert result.metadata["repetition_rate"] == pytest.approx(expected_rate)
assert result.metadata["penalty_mode"] == "soft"


@pytest.mark.unit
@pytest.mark.parametrize(
"response,expected_score",
[("alpha beta alpha beta", 0.0), ("alpha beta alpha beta alpha beta alpha beta", -0.25)],
)
async def test_hard_penalty_threshold_is_unchanged(response: str, expected_score: float) -> None:
"""Hard penalties still apply only above their configured threshold."""
grader = NgramRepetitionPenaltyGrader(n=1, penalty_threshold=0.5, tokenizer_type=TokenizerEnum.simple)

result = await grader.aevaluate(response=response)

assert result.score == pytest.approx(expected_score)
assert result.metadata["penalty_mode"] == "hard"
Loading