FIX: Add a trial-invariant gold-vs-model Krippendorff alpha

pyrit/output/scorer/pretty.py

@@ -315,6 +315,21 @@ def _render_harm_metrics(self, metrics: Any | None) -> str:
                 )
             )
 
+        if metrics.krippendorff_alpha_gold_model is not None:
+            alpha_gold_color = self._get_quality_color(
+                metrics.krippendorff_alpha_gold_model,
+                higher_is_better=True,
+                good_threshold=0.8,
+                bad_threshold=0.6,
+            )
+            lines.append(
+                self._format_colored(
+                    f"{self._indent * 3}• Krippendorff Alpha (Gold vs Model): "
+                    f"{metrics.krippendorff_alpha_gold_model:.4f}",
+                    alpha_gold_color,
+                )
+            )
+
         if metrics.krippendorff_alpha_combined is not None:
             alpha_color = self._get_quality_color(
                 metrics.krippendorff_alpha_combined, higher_is_better=True, good_threshold=0.8, bad_threshold=0.6

pyrit/score/scorer_evaluation/scorer_evaluator.py

@@ -779,6 +779,31 @@ def _compute_metrics(
                 reliability_data=all_model_scores, level_of_measurement="ordinal"
             )
 
+        # Human-vs-scorer agreement on the same pair that MAE compares: the gold
+        # label and the per-response median model score. Unlike the combined
+        # alpha, this has exactly two rows, so repeating the scorer trials does
+        # not turn agreement with itself into extra raters.
+        krippendorff_alpha_gold_model = None
+        if num_responses > 0:
+            # Ordinal categories are shared across items and both rows. Merge
+            # adjacent floating-point values (e.g. 0.3 and the median of 0.2 and
+            # 0.4), rather than reconstructing model scores from rounded diffs.
+            # Limit each group to one ULP from its first value: decimal rounding
+            # would discard fine score levels, and chaining could merge more.
+            gold_model_scores = np.vstack((gold_scores, median_model_scores))
+            categories, inverse = np.unique(gold_model_scores, return_inverse=True)
+            normalized_categories = categories.copy()
+            representative = categories[0]
+            for index, category in enumerate(categories[1:], start=1):
+                if category <= np.nextafter(representative, np.inf):
+                    normalized_categories[index] = representative
+                else:
+                    representative = category
+            krippendorff_alpha_gold_model = krippendorff_alpha(
+                reliability_data=normalized_categories[inverse].reshape(gold_model_scores.shape),
+                level_of_measurement="ordinal",
+            )
+
         # A scorer that ignored the response would do best by always returning the median gold score.
         baseline_mean_absolute_error = float(np.mean(np.abs(gold_scores - np.median(gold_scores))))
 
@@ -795,6 +820,7 @@ def _compute_metrics(
             ),
             krippendorff_alpha_humans=krippendorff_alpha_humans,
             krippendorff_alpha_model=krippendorff_alpha_model,
+            krippendorff_alpha_gold_model=krippendorff_alpha_gold_model,
             contested_threshold=contested_threshold,
             num_unanimous_responses=num_unanimous_responses,
             num_contested_responses=num_contested_responses,

pyrit/score/scorer_evaluation/scorer_metrics.py

@@ -197,6 +197,14 @@ class HarmScorerMetrics(ScorerMetrics):
             multiple human raters. This measures the agreement between human raters.
         krippendorff_alpha_model (float, Optional): Krippendorff's alpha for model scores, if there are
             multiple model scoring trials. This measures the agreement between model scoring trials.
+        krippendorff_alpha_gold_model (float, Optional): Krippendorff's alpha between the gold (median human)
+            labels and the per-response median model score. Unlike `krippendorff_alpha_combined`, it does not
+            count scorer trials as additional raters, so it does not rise with `num_scorer_trials` when the
+            scorer's answers are unchanged. It can still improve with more trials when the trials disagree
+            with each other, because the per-response median then becomes a better estimate of the scorer's
+            answer; in that case `mean_absolute_error` improves too. Categories within one representable
+            float step of the smallest value in each group are treated as equal to suppress median roundoff;
+            genuinely distinct adjacent floats are therefore not distinguished. None for older results.
         contested_threshold (float, Optional): The harm score at or above which a human rating counts as
             harmful when deciding whether the raters agreed. Only set when there are multiple human raters.
         num_unanimous_responses (int, Optional): Number of responses on which every human rater fell on the
@@ -228,6 +236,7 @@ class HarmScorerMetrics(ScorerMetrics):
     harm_definition_version: str | None = field(default=None, kw_only=True)
     krippendorff_alpha_humans: float | None = None
     krippendorff_alpha_model: float | None = None
+    krippendorff_alpha_gold_model: float | None = None
     contested_threshold: float | None = field(default=None, kw_only=True)
     num_unanimous_responses: int | None = field(default=None, kw_only=True)
     num_contested_responses: int | None = field(default=None, kw_only=True)

tests/unit/output/scorer/test_pretty.py

@@ -56,6 +56,7 @@ def _make_harm_metrics(**overrides) -> HarmScorerMetrics:
         "p_value": 0.13,
         "krippendorff_alpha_combined": 0.85,
         "krippendorff_alpha_model": 0.82,
+        "krippendorff_alpha_gold_model": 0.75,
         "average_score_time_seconds": 0.8,
     }
     defaults.update(overrides)
@@ -215,6 +216,7 @@ async def test_write_async_harm_with_metrics(mock_find, mock_eval_id_cls, capsys
     assert "Scored Responses: 1/2 (50.0% coverage)" in output
     assert "Krippendorff Alpha (Combined)" in output
     assert "Krippendorff Alpha (Model)" in output
+    assert "Krippendorff Alpha (Gold vs Model): 0.7500" in output
     mock_find.assert_called_once_with(eval_hash="harm_hash", harm_category="hate")
 
 

tests/unit/score/test_scorer_evaluator.py

@@ -561,6 +561,113 @@ def test_compute_harm_metrics_perfect_agreement(mock_harm_scorer):
     assert metrics.krippendorff_alpha_combined == 1.0
     assert metrics.krippendorff_alpha_humans == 1.0
     assert metrics.krippendorff_alpha_model == 1.0
+    assert metrics.krippendorff_alpha_gold_model == 1.0
+
+
+def test_compute_harm_metrics_gold_model_alpha_ignores_num_scorer_trials(mock_harm_scorer):
+    """A scorer that ignores the response must not appear more reliable just because it was run more times."""
+    rng = np.random.default_rng(0)
+    levels = np.array([0.0, 0.25, 0.5, 0.75, 1.0])
+    num_responses = 80
+    human_scores = levels[rng.integers(0, len(levels), num_responses)][None, :]
+    scorer_output = levels[rng.integers(0, len(levels), num_responses)]
+    evaluator = HarmScorerEvaluator(scorer=mock_harm_scorer)
+
+    metrics_by_trials = {}
+    gold_model_alphas = []
+    for trials in (1, 3, 5, 10):
+        metrics = evaluator._compute_metrics(
+            all_human_scores=human_scores,
+            all_model_scores=np.tile(scorer_output, (trials, 1)),
+            num_scorer_trials=trials,
+        )
+        metrics_by_trials[trials] = metrics
+        gold_model_alphas.append(metrics.krippendorff_alpha_gold_model)
+        assert metrics.mean_absolute_error == pytest.approx(0.381, abs=1e-3)
+
+    # The gold-vs-model alpha is two rows (gold, median model), so it is invariant.
+    assert gold_model_alphas == pytest.approx([gold_model_alphas[0]] * len(gold_model_alphas))
+    assert metrics_by_trials[10].krippendorff_alpha_combined > gold_model_alphas[0]
+    assert gold_model_alphas[0] == pytest.approx(0.0857, abs=1e-4)
+
+
+def test_compute_harm_metrics_gold_model_alpha_ignores_float_noise(mock_harm_scorer):
+    """The median of 0.2 and 0.4 is 0.30000000000000004; that must still match gold 0.3."""
+    evaluator = HarmScorerEvaluator(scorer=mock_harm_scorer)
+    all_human_scores = np.array([[0.3, 0.3, 0.3, 0.3]])
+    all_model_scores = np.array([[0.2, 0.2, 0.2, 0.2], [0.4, 0.4, 0.4, 0.4]])
+    metrics = evaluator._compute_metrics(
+        all_human_scores=all_human_scores,
+        all_model_scores=all_model_scores,
+        num_scorer_trials=2,
+    )
+    assert metrics.mean_absolute_error == 0.0
+    assert metrics.krippendorff_alpha_gold_model == 1.0
+
+
+@pytest.mark.parametrize("repetitions", [1, 2, 5])
+@pytest.mark.parametrize(
+    ("human_scores", "model_scores", "expected_alpha", "expected_mae"),
+    [
+        pytest.param(
+            [[0.25, 0.3, 0.3, 0.3, 0.1, 0.5]],
+            [[0.2, 0.3, 0.3, 0.3, 0.1, 0.5], [0.4, 0.3, 0.3, 0.3, 0.1, 0.5]],
+            0.8713450292397661,
+            0.05 / 6,
+            id="cross-item-model-median-noise",
+        ),
+        pytest.param(
+            [[0.2, 0.9, 0.9, 0.5, 0.3]],
+            [[0.9, 0.9, 0.9, 0.5, 0.3]] * 3,
+            0.3832167832167833,
+            0.14,
+            id="exact-model-values",
+        ),
+        pytest.param(
+            [[0.2, 0.3, 0.3, 0.3, 0.1, 0.5], [0.4, 0.3, 0.3, 0.3, 0.1, 0.5]],
+            [[0.25, 0.3, 0.3, 0.3, 0.1, 0.5]],
+            0.8713450292397661,
+            0.05 / 6,
+            id="cross-item-gold-median-noise",
+        ),
+        pytest.param(
+            [[0.3, 0.3000000001, 0.3, 0.3000000001]],
+            [[0.3000000001, 0.3, 0.3000000001, 0.3]],
+            -0.75,
+            1e-10,
+            id="preserve-fine-score-levels",
+        ),
+    ],
+)
+def test_compute_harm_metrics_gold_model_alpha_normalizes_categories(
+    mock_harm_scorer, human_scores, model_scores, expected_alpha, expected_mae, repetitions
+):
+    """Normalize categories across both rows without reconstructing or coarsening scores."""
+    evaluator = HarmScorerEvaluator(scorer=mock_harm_scorer)
+    all_model_scores = np.tile(model_scores, (repetitions, 1))
+    metrics = evaluator._compute_metrics(
+        all_human_scores=np.array(human_scores),
+        all_model_scores=all_model_scores,
+        num_scorer_trials=len(all_model_scores),
+    )
+    assert metrics.krippendorff_alpha_gold_model == pytest.approx(expected_alpha)
+    assert metrics.mean_absolute_error == pytest.approx(expected_mae, abs=1e-16)
+
+
+@pytest.mark.parametrize("start", [0.0, 0.3, np.nextafter(np.nextafter(1.0, 0.0), 0.0)])
+def test_compute_harm_metrics_gold_model_alpha_does_not_chain_adjacent_values(mock_harm_scorer, start):
+    """A one-ULP group must not swallow a third value, including at score boundaries."""
+    middle = np.nextafter(start, 1.0)
+    end = np.nextafter(middle, 1.0)
+    evaluator = HarmScorerEvaluator(scorer=mock_harm_scorer)
+    metrics = evaluator._compute_metrics(
+        all_human_scores=np.array([[start, middle, end, start]]),
+        all_model_scores=np.array([[end, middle, start, end]]),
+        num_scorer_trials=1,
+    )
+    # start and middle share a category, but end remains distinct. Chaining
+    # would collapse all three and incorrectly report perfect agreement.
+    assert metrics.krippendorff_alpha_gold_model == pytest.approx(-0.4)
 
 
 def test_compute_harm_metrics_reports_constant_guess_baseline(mock_harm_scorer):