From 03099ce49b6f28c46ce3405e0468e08a2a7b5299 Mon Sep 17 00:00:00 2001 From: Chen Yufeiyang Date: Wed, 11 Nov 2026 09:19:04 +0800 Subject: [PATCH 1/4] fix(score): report gold-vs-model alpha independent of scorer trials --- pyrit/output/scorer/pretty.py | 15 +++++++++++ .../scorer_evaluation/scorer_evaluator.py | 12 +++++++++ .../score/scorer_evaluation/scorer_metrics.py | 5 ++++ tests/unit/score/test_scorer_evaluator.py | 25 +++++++++++++++++++ 4 files changed, 57 insertions(+) diff --git a/pyrit/output/scorer/pretty.py b/pyrit/output/scorer/pretty.py index 5eaa4b1bf3..72764097e3 100644 --- a/pyrit/output/scorer/pretty.py +++ b/pyrit/output/scorer/pretty.py @@ -293,6 +293,21 @@ def _render_harm_metrics(self, metrics: Any | None) -> str: ) ) + if metrics.krippendorff_alpha_gold_model is not None: + alpha_gold_color = self._get_quality_color( + metrics.krippendorff_alpha_gold_model, + higher_is_better=True, + good_threshold=0.8, + bad_threshold=0.6, + ) + lines.append( + self._format_colored( + f"{self._indent * 3}• Krippendorff Alpha (Gold vs Model): " + f"{metrics.krippendorff_alpha_gold_model:.4f}", + alpha_gold_color, + ) + ) + if metrics.krippendorff_alpha_combined is not None: alpha_color = self._get_quality_color( metrics.krippendorff_alpha_combined, higher_is_better=True, good_threshold=0.8, bad_threshold=0.6 diff --git a/pyrit/score/scorer_evaluation/scorer_evaluator.py b/pyrit/score/scorer_evaluation/scorer_evaluator.py index 20985d3ee6..6c3954ca90 100644 --- a/pyrit/score/scorer_evaluation/scorer_evaluator.py +++ b/pyrit/score/scorer_evaluation/scorer_evaluator.py @@ -779,6 +779,17 @@ def _compute_metrics( reliability_data=all_model_scores, level_of_measurement="ordinal" ) + # Human-vs-scorer agreement on the same pair that MAE compares: the gold + # label and the per-response median model score. Unlike the combined + # alpha, this has exactly two rows, so repeating the scorer trials does + # not turn agreement with itself into extra raters. + krippendorff_alpha_gold_model = None + if num_responses > 0: + krippendorff_alpha_gold_model = krippendorff_alpha( + reliability_data=np.vstack((gold_scores, median_model_scores)), + level_of_measurement="ordinal", + ) + # A scorer that ignored the response would do best by always returning the median gold score. baseline_mean_absolute_error = float(np.mean(np.abs(gold_scores - np.median(gold_scores)))) @@ -795,6 +806,7 @@ def _compute_metrics( ), krippendorff_alpha_humans=krippendorff_alpha_humans, krippendorff_alpha_model=krippendorff_alpha_model, + krippendorff_alpha_gold_model=krippendorff_alpha_gold_model, contested_threshold=contested_threshold, num_unanimous_responses=num_unanimous_responses, num_contested_responses=num_contested_responses, diff --git a/pyrit/score/scorer_evaluation/scorer_metrics.py b/pyrit/score/scorer_evaluation/scorer_metrics.py index 4701738d54..8fb9303ba4 100644 --- a/pyrit/score/scorer_evaluation/scorer_metrics.py +++ b/pyrit/score/scorer_evaluation/scorer_metrics.py @@ -197,6 +197,10 @@ class HarmScorerMetrics(ScorerMetrics): multiple human raters. This measures the agreement between human raters. krippendorff_alpha_model (float, Optional): Krippendorff's alpha for model scores, if there are multiple model scoring trials. This measures the agreement between model scoring trials. + krippendorff_alpha_gold_model (float, Optional): Krippendorff's alpha between the gold (median human) + labels and the per-response median model score. Unlike `krippendorff_alpha_combined`, it does not + count scorer trials as additional raters, so it does not rise with `num_scorer_trials` when the + scorer's answers are unchanged. None for results recorded before this field existed. contested_threshold (float, Optional): The harm score at or above which a human rating counts as harmful when deciding whether the raters agreed. Only set when there are multiple human raters. num_unanimous_responses (int, Optional): Number of responses on which every human rater fell on the @@ -228,6 +232,7 @@ class HarmScorerMetrics(ScorerMetrics): harm_definition_version: str | None = field(default=None, kw_only=True) krippendorff_alpha_humans: float | None = None krippendorff_alpha_model: float | None = None + krippendorff_alpha_gold_model: float | None = None contested_threshold: float | None = field(default=None, kw_only=True) num_unanimous_responses: int | None = field(default=None, kw_only=True) num_contested_responses: int | None = field(default=None, kw_only=True) diff --git a/tests/unit/score/test_scorer_evaluator.py b/tests/unit/score/test_scorer_evaluator.py index bc30cd650c..e4ed12441f 100644 --- a/tests/unit/score/test_scorer_evaluator.py +++ b/tests/unit/score/test_scorer_evaluator.py @@ -500,6 +500,31 @@ def test_compute_harm_metrics_perfect_agreement(mock_harm_scorer): assert metrics.krippendorff_alpha_combined == 1.0 assert metrics.krippendorff_alpha_humans == 1.0 assert metrics.krippendorff_alpha_model == 1.0 + assert metrics.krippendorff_alpha_gold_model == 1.0 + + +def test_compute_harm_metrics_gold_model_alpha_ignores_num_scorer_trials(mock_harm_scorer): + """A scorer that ignores the response must not appear more reliable just because it was run more times.""" + rng = np.random.default_rng(0) + levels = np.array([0.0, 0.25, 0.5, 0.75, 1.0]) + num_responses = 80 + human_scores = levels[rng.integers(0, len(levels), num_responses)][None, :] + scorer_output = levels[rng.integers(0, len(levels), num_responses)] + evaluator = HarmScorerEvaluator(scorer=mock_harm_scorer) + + gold_model_alphas = [] + for trials in (1, 3, 5, 10): + metrics = evaluator._compute_metrics( + all_human_scores=human_scores, + all_model_scores=np.tile(scorer_output, (trials, 1)), + num_scorer_trials=trials, + ) + gold_model_alphas.append(metrics.krippendorff_alpha_gold_model) + assert metrics.mean_absolute_error == pytest.approx(0.381, abs=1e-3) + + # The gold-vs-model alpha is two rows (gold, median model), so it is invariant. + assert gold_model_alphas == pytest.approx([gold_model_alphas[0]] * len(gold_model_alphas)) + assert metrics.krippendorff_alpha_combined > gold_model_alphas[0] def test_compute_harm_metrics_reports_constant_guess_baseline(mock_harm_scorer): From 4d49108d62d91990d95a0768161be9816318cbfb Mon Sep 17 00:00:00 2001 From: Chen Yufeiyang Date: Wed, 11 Nov 2026 09:47:37 +0800 Subject: [PATCH 2/4] fix(score): normalize float noise before gold-vs-model alpha --- pyrit/score/scorer_evaluation/scorer_evaluator.py | 5 ++++- tests/unit/score/test_scorer_evaluator.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pyrit/score/scorer_evaluation/scorer_evaluator.py b/pyrit/score/scorer_evaluation/scorer_evaluator.py index 6c3954ca90..f61ba4c631 100644 --- a/pyrit/score/scorer_evaluation/scorer_evaluator.py +++ b/pyrit/score/scorer_evaluation/scorer_evaluator.py @@ -785,8 +785,11 @@ def _compute_metrics( # not turn agreement with itself into extra raters. krippendorff_alpha_gold_model = None if num_responses > 0: + # Reuse the noise-normalized diff: floating-point medians such as + # 0.30000000000000004 must not look like disagreement with 0.3. + aligned_model_scores = gold_scores + diff krippendorff_alpha_gold_model = krippendorff_alpha( - reliability_data=np.vstack((gold_scores, median_model_scores)), + reliability_data=np.vstack((gold_scores, aligned_model_scores)), level_of_measurement="ordinal", ) diff --git a/tests/unit/score/test_scorer_evaluator.py b/tests/unit/score/test_scorer_evaluator.py index e4ed12441f..12385cab36 100644 --- a/tests/unit/score/test_scorer_evaluator.py +++ b/tests/unit/score/test_scorer_evaluator.py @@ -527,6 +527,20 @@ def test_compute_harm_metrics_gold_model_alpha_ignores_num_scorer_trials(mock_ha assert metrics.krippendorff_alpha_combined > gold_model_alphas[0] +def test_compute_harm_metrics_gold_model_alpha_ignores_float_noise(mock_harm_scorer): + """The median of 0.2 and 0.4 is 0.30000000000000004; that must still match gold 0.3.""" + evaluator = HarmScorerEvaluator(scorer=mock_harm_scorer) + all_human_scores = np.array([[0.3, 0.3, 0.3, 0.3]]) + all_model_scores = np.array([[0.2, 0.2, 0.2, 0.2], [0.4, 0.4, 0.4, 0.4]]) + metrics = evaluator._compute_metrics( + all_human_scores=all_human_scores, + all_model_scores=all_model_scores, + num_scorer_trials=2, + ) + assert metrics.mean_absolute_error == 0.0 + assert metrics.krippendorff_alpha_gold_model == 1.0 + + def test_compute_harm_metrics_reports_constant_guess_baseline(mock_harm_scorer): evaluator = HarmScorerEvaluator(scorer=mock_harm_scorer) # Gold scores 0, 0.25, 0.5, 1.0: median 0.375, so a constant guess is off by From 4523853ba5a9ff43f2c2f3e91cd6e14d5abeafc6 Mon Sep 17 00:00:00 2001 From: Chen Yufeiyang Date: Wed, 11 Nov 2026 16:18:33 +0800 Subject: [PATCH 3/4] fix(score): normalize global alpha categories within one ULP --- .../scorer_evaluation/scorer_evaluator.py | 19 +++-- .../score/scorer_evaluation/scorer_metrics.py | 4 +- tests/unit/output/scorer/test_pretty.py | 2 + tests/unit/score/test_scorer_evaluator.py | 70 ++++++++++++++++++- 4 files changed, 89 insertions(+), 6 deletions(-) diff --git a/pyrit/score/scorer_evaluation/scorer_evaluator.py b/pyrit/score/scorer_evaluation/scorer_evaluator.py index f61ba4c631..01a30bdc41 100644 --- a/pyrit/score/scorer_evaluation/scorer_evaluator.py +++ b/pyrit/score/scorer_evaluation/scorer_evaluator.py @@ -785,11 +785,22 @@ def _compute_metrics( # not turn agreement with itself into extra raters. krippendorff_alpha_gold_model = None if num_responses > 0: - # Reuse the noise-normalized diff: floating-point medians such as - # 0.30000000000000004 must not look like disagreement with 0.3. - aligned_model_scores = gold_scores + diff + # Ordinal categories are shared across items and both rows. Merge + # adjacent floating-point values (e.g. 0.3 and the median of 0.2 and + # 0.4), rather than reconstructing model scores from rounded diffs. + # Limit each group to one ULP from its first value: decimal rounding + # would discard fine score levels, and chaining could merge more. + gold_model_scores = np.vstack((gold_scores, median_model_scores)) + categories, inverse = np.unique(gold_model_scores, return_inverse=True) + normalized_categories = categories.copy() + representative = categories[0] + for index, category in enumerate(categories[1:], start=1): + if category <= np.nextafter(representative, np.inf): + normalized_categories[index] = representative + else: + representative = category krippendorff_alpha_gold_model = krippendorff_alpha( - reliability_data=np.vstack((gold_scores, aligned_model_scores)), + reliability_data=normalized_categories[inverse].reshape(gold_model_scores.shape), level_of_measurement="ordinal", ) diff --git a/pyrit/score/scorer_evaluation/scorer_metrics.py b/pyrit/score/scorer_evaluation/scorer_metrics.py index 8fb9303ba4..f52c08297b 100644 --- a/pyrit/score/scorer_evaluation/scorer_metrics.py +++ b/pyrit/score/scorer_evaluation/scorer_metrics.py @@ -200,7 +200,9 @@ class HarmScorerMetrics(ScorerMetrics): krippendorff_alpha_gold_model (float, Optional): Krippendorff's alpha between the gold (median human) labels and the per-response median model score. Unlike `krippendorff_alpha_combined`, it does not count scorer trials as additional raters, so it does not rise with `num_scorer_trials` when the - scorer's answers are unchanged. None for results recorded before this field existed. + scorer's answers are unchanged. Categories within one representable float step of the + smallest value in each group are treated as equal to suppress median roundoff; genuinely + distinct adjacent floats are therefore not distinguished. None for older results. contested_threshold (float, Optional): The harm score at or above which a human rating counts as harmful when deciding whether the raters agreed. Only set when there are multiple human raters. num_unanimous_responses (int, Optional): Number of responses on which every human rater fell on the diff --git a/tests/unit/output/scorer/test_pretty.py b/tests/unit/output/scorer/test_pretty.py index 94a80133bb..1b967c2fc5 100644 --- a/tests/unit/output/scorer/test_pretty.py +++ b/tests/unit/output/scorer/test_pretty.py @@ -56,6 +56,7 @@ def _make_harm_metrics(**overrides) -> HarmScorerMetrics: "p_value": 0.13, "krippendorff_alpha_combined": 0.85, "krippendorff_alpha_model": 0.82, + "krippendorff_alpha_gold_model": 0.75, "average_score_time_seconds": 0.8, } defaults.update(overrides) @@ -162,6 +163,7 @@ async def test_write_async_harm_with_metrics(mock_find, mock_eval_id_cls, capsys assert "Scored Responses: 1/2 (50.0% coverage)" in output assert "Krippendorff Alpha (Combined)" in output assert "Krippendorff Alpha (Model)" in output + assert "Krippendorff Alpha (Gold vs Model): 0.7500" in output mock_find.assert_called_once_with(eval_hash="harm_hash", harm_category="hate") diff --git a/tests/unit/score/test_scorer_evaluator.py b/tests/unit/score/test_scorer_evaluator.py index 12385cab36..81fb6f81ec 100644 --- a/tests/unit/score/test_scorer_evaluator.py +++ b/tests/unit/score/test_scorer_evaluator.py @@ -512,6 +512,7 @@ def test_compute_harm_metrics_gold_model_alpha_ignores_num_scorer_trials(mock_ha scorer_output = levels[rng.integers(0, len(levels), num_responses)] evaluator = HarmScorerEvaluator(scorer=mock_harm_scorer) + metrics_by_trials = {} gold_model_alphas = [] for trials in (1, 3, 5, 10): metrics = evaluator._compute_metrics( @@ -519,12 +520,14 @@ def test_compute_harm_metrics_gold_model_alpha_ignores_num_scorer_trials(mock_ha all_model_scores=np.tile(scorer_output, (trials, 1)), num_scorer_trials=trials, ) + metrics_by_trials[trials] = metrics gold_model_alphas.append(metrics.krippendorff_alpha_gold_model) assert metrics.mean_absolute_error == pytest.approx(0.381, abs=1e-3) # The gold-vs-model alpha is two rows (gold, median model), so it is invariant. assert gold_model_alphas == pytest.approx([gold_model_alphas[0]] * len(gold_model_alphas)) - assert metrics.krippendorff_alpha_combined > gold_model_alphas[0] + assert metrics_by_trials[10].krippendorff_alpha_combined > gold_model_alphas[0] + assert gold_model_alphas[0] == pytest.approx(0.0857, abs=1e-4) def test_compute_harm_metrics_gold_model_alpha_ignores_float_noise(mock_harm_scorer): @@ -541,6 +544,71 @@ def test_compute_harm_metrics_gold_model_alpha_ignores_float_noise(mock_harm_sco assert metrics.krippendorff_alpha_gold_model == 1.0 +@pytest.mark.parametrize("repetitions", [1, 2, 5]) +@pytest.mark.parametrize( + ("human_scores", "model_scores", "expected_alpha", "expected_mae"), + [ + pytest.param( + [[0.25, 0.3, 0.3, 0.3, 0.1, 0.5]], + [[0.2, 0.3, 0.3, 0.3, 0.1, 0.5], [0.4, 0.3, 0.3, 0.3, 0.1, 0.5]], + 0.8713450292397661, + 0.05 / 6, + id="cross-item-model-median-noise", + ), + pytest.param( + [[0.2, 0.9, 0.9, 0.5, 0.3]], + [[0.9, 0.9, 0.9, 0.5, 0.3]] * 3, + 0.3832167832167833, + 0.14, + id="exact-model-values", + ), + pytest.param( + [[0.2, 0.3, 0.3, 0.3, 0.1, 0.5], [0.4, 0.3, 0.3, 0.3, 0.1, 0.5]], + [[0.25, 0.3, 0.3, 0.3, 0.1, 0.5]], + 0.8713450292397661, + 0.05 / 6, + id="cross-item-gold-median-noise", + ), + pytest.param( + [[0.3, 0.3000000001, 0.3, 0.3000000001]], + [[0.3000000001, 0.3, 0.3000000001, 0.3]], + -0.75, + 1e-10, + id="preserve-fine-score-levels", + ), + ], +) +def test_compute_harm_metrics_gold_model_alpha_normalizes_categories( + mock_harm_scorer, human_scores, model_scores, expected_alpha, expected_mae, repetitions +): + """Normalize categories across both rows without reconstructing or coarsening scores.""" + evaluator = HarmScorerEvaluator(scorer=mock_harm_scorer) + all_model_scores = np.tile(model_scores, (repetitions, 1)) + metrics = evaluator._compute_metrics( + all_human_scores=np.array(human_scores), + all_model_scores=all_model_scores, + num_scorer_trials=len(all_model_scores), + ) + assert metrics.krippendorff_alpha_gold_model == pytest.approx(expected_alpha) + assert metrics.mean_absolute_error == pytest.approx(expected_mae, abs=1e-16) + + +@pytest.mark.parametrize("start", [0.0, 0.3, np.nextafter(np.nextafter(1.0, 0.0), 0.0)]) +def test_compute_harm_metrics_gold_model_alpha_does_not_chain_adjacent_values(mock_harm_scorer, start): + """A one-ULP group must not swallow a third value, including at score boundaries.""" + middle = np.nextafter(start, 1.0) + end = np.nextafter(middle, 1.0) + evaluator = HarmScorerEvaluator(scorer=mock_harm_scorer) + metrics = evaluator._compute_metrics( + all_human_scores=np.array([[start, middle, end, start]]), + all_model_scores=np.array([[end, middle, start, end]]), + num_scorer_trials=1, + ) + # start and middle share a category, but end remains distinct. Chaining + # would collapse all three and incorrectly report perfect agreement. + assert metrics.krippendorff_alpha_gold_model == pytest.approx(-0.4) + + def test_compute_harm_metrics_reports_constant_guess_baseline(mock_harm_scorer): evaluator = HarmScorerEvaluator(scorer=mock_harm_scorer) # Gold scores 0, 0.25, 0.5, 1.0: median 0.375, so a constant guess is off by From 4a90b993697ac8932d0537505bd3645e874ba953 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Thu, 12 Nov 2026 23:29:08 -0800 Subject: [PATCH 4/4] DOC: Note the gold-model alpha legitimately improves with independently noisy trials Per Allan-dq's observation on #4181: with independent per-trial noise the per-response median converges to the gold label, so the alpha rises with trials while MAE falls - unlike the #4129 artifact where MAE stayed flat. Co-Authored-By: Claude Fable 5 --- pyrit/score/scorer_evaluation/scorer_metrics.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pyrit/score/scorer_evaluation/scorer_metrics.py b/pyrit/score/scorer_evaluation/scorer_metrics.py index 08f482a6e5..a0fd74072d 100644 --- a/pyrit/score/scorer_evaluation/scorer_metrics.py +++ b/pyrit/score/scorer_evaluation/scorer_metrics.py @@ -200,9 +200,11 @@ class HarmScorerMetrics(ScorerMetrics): krippendorff_alpha_gold_model (float, Optional): Krippendorff's alpha between the gold (median human) labels and the per-response median model score. Unlike `krippendorff_alpha_combined`, it does not count scorer trials as additional raters, so it does not rise with `num_scorer_trials` when the - scorer's answers are unchanged. Categories within one representable float step of the - smallest value in each group are treated as equal to suppress median roundoff; genuinely - distinct adjacent floats are therefore not distinguished. None for older results. + scorer's answers are unchanged. It can still improve with more trials when the trials disagree + with each other, because the per-response median then becomes a better estimate of the scorer's + answer; in that case `mean_absolute_error` improves too. Categories within one representable + float step of the smallest value in each group are treated as equal to suppress median roundoff; + genuinely distinct adjacent floats are therefore not distinguished. None for older results. contested_threshold (float, Optional): The harm score at or above which a human rating counts as harmful when deciding whether the raters agreed. Only set when there are multiple human raters. num_unanimous_responses (int, Optional): Number of responses on which every human rater fell on the