fix: isolate oversized drift evidence records

CHANGELOG.md

@@ -31,6 +31,7 @@
 
 ### Fixed
 
+- Keep drift evidence available for unrelated assertions when a scoped console record exceeds the capture limit; the affected assertion or scope remains unavailable.
 - Retain sanitized drift evidence for each test and retry attempt in CI artifacts.
 - Align Cohere streaming length-stop faults with captured tool closure, `TOOL_CALL`, and terminal `[DONE]` framing while preserving incomplete argument bytes. Object length stops retain modeled `MAX_TOKENS`; recurring comparisons distinguish valid object nontriggers from failures.
 - Preserve malformed tool-call argument strings on string-based response wires instead of silently replacing them with `{}`. Object-based wires now return an explicit error (HTTP 500, or a Gemini Live WebSocket error) before response content. Valid JSON, empty or omitted legacy arguments, and fixture validation retain their existing behavior.

scripts/drift-evidence-reporter.ts

@@ -15,6 +15,7 @@ export type EvidenceReporterRecord =
       ancestors: string[];
       title: string;
     }
+  | { kind: "console-unavailable"; taskId: string }
   | { kind: "unavailable" }
   | { kind: "complete" };
 
@@ -28,13 +29,17 @@ export default class DriftEvidenceReporter implements Reporter {
   private emit(record: EvidenceReporterRecord) {
     if (this.broken) return;
     try {
-      const encoded = JSON.stringify(record);
-      writeFileSync(
-        3,
-        (Buffer.byteLength(encoded) <= MAX_EVIDENCE_RECORD_BYTES
-          ? encoded
-          : '{"kind":"unavailable"}') + "\n",
-      );
+      let encoded = JSON.stringify(record);
+      if (Buffer.byteLength(encoded) > MAX_EVIDENCE_RECORD_BYTES) {
+        encoded = JSON.stringify(
+          record.kind === "console" && record.taskId
+            ? { kind: "console-unavailable", taskId: record.taskId }
+            : { kind: "unavailable" },
+        );
+        if (Buffer.byteLength(encoded) > MAX_EVIDENCE_RECORD_BYTES)
+          encoded = '{"kind":"unavailable"}';
+      }
+      writeFileSync(3, encoded + "\n");
     } catch {
       this.broken = true;
       // A missing completion record makes the parent mark the channel unavailable.

scripts/drift-report-collector.ts

@@ -210,6 +210,14 @@ function parseEvidenceRecord(line: string): EvidenceReporterRecord | null {
   }
   if (!value || typeof value !== "object" || !("kind" in value)) return null;
   if (value.kind === "complete" || value.kind === "unavailable") return { kind: value.kind };
+  if (
+    value.kind === "console-unavailable" &&
+    "taskId" in value &&
+    typeof value.taskId === "string" &&
+    value.taskId.length > 0
+  )
+    return { kind: "console-unavailable", taskId: value.taskId };
+
   if (
     value.kind === "console" &&
     "taskId" in value &&
@@ -257,6 +265,23 @@ function captureReporterChannel(leg: LegEvidence, stream: string) {
     records.push(record);
   }
   if (records.filter((record) => record.kind === "complete").length !== 1) return;
+  const identities = records.filter((record) => record.kind === "identity");
+  for (const record of records) {
+    if (record.kind !== "console-unavailable") continue;
+    const owners = identities.filter((identity) => identity.id === record.taskId);
+    if (owners.length !== 1) return;
+    const owner = owners[0];
+    if (
+      identities.filter(
+        (identity) =>
+          identity.entity === owner.entity &&
+          resolve(identity.file) === resolve(owner.file) &&
+          identity.title === owner.title &&
+          JSON.stringify(identity.ancestors) === JSON.stringify(owner.ancestors),
+      ).length !== 1
+    )
+      return;
+  }
   reporterChannels.set(leg, records);
 }
 
@@ -272,6 +297,23 @@ function bindReporterContext(leg: LegEvidence, result: VitestJsonResult) {
   const records = reporterChannels.get(leg) ?? [];
   const identities = records.filter((record) => record.kind === "identity");
   const logs = records.filter((record) => record.kind === "console");
+  const taintedIds = new Set(
+    records
+      .filter((record) => record.kind === "console-unavailable")
+      .map((record) => record.taskId),
+  );
+  const taintedOwners = identities.filter((identity) => taintedIds.has(identity.id));
+  const isTainted = (identity: Extract<EvidenceReporterRecord, { kind: "identity" }>) =>
+    taintedOwners.some((owner) => {
+      if (resolve(identity.file) !== resolve(owner.file)) return false;
+      if (owner.entity === "module") return true;
+      if (owner.entity === "test") return identity.id === owner.id;
+      const path = [...owner.ancestors, owner.title];
+      return (
+        identity.id === owner.id || path.every((part, index) => identity.ancestors[index] === part)
+      );
+    });
+
   const key = (file: string | undefined, ancestors: string[], title: string) =>
     JSON.stringify([file === undefined ? null : resolve(file), ancestors, title]);
   const rows = result.testResults.flatMap((file) =>
@@ -285,6 +327,7 @@ function bindReporterContext(leg: LegEvidence, result: VitestJsonResult) {
   leg.consoleScopes = [];
   for (const identity of identities) {
     if (!identity.id || identities.filter((item) => item.id === identity.id).length !== 1) continue;
+    if (isTainted(identity)) continue;
     const messages = logs.filter((log) => log.taskId === identity.id);
     if (identity.entity !== "test") {
       matchedIds.add(identity.id);

src/__tests__/drift-collector.test.ts

@@ -73,6 +73,7 @@ import type { DriftReport } from "../../scripts/drift-types.js";
 // Exercise the real CLI and installed Vitest reporter, without provider credentials.
 async function withCollectorFixture(
   mode:
+    | "console-overflow"
     | "mixed"
     | "unknown-surface"
     | "secondary-malformed"
@@ -196,7 +197,7 @@ async function withCollectorFixture(
       `,
       );
     }
-    if (mode === "bedrock-markers") {
+    if (mode === "bedrock-markers" || mode === "console-overflow") {
       const markerFixture = `
         import { it, describe, expect } from 'vitest';
         const marker = (fault, stream, source, outcome) => ({ nativeComparison: {wire:'invoke',fault,stream},source,outcome,sends:1,requestedOutputTokens:32,request:'fixture-provider-body-912',raw:process.env.FIXTURE_SECRET });
@@ -215,7 +216,7 @@ async function withCollectorFixture(
       writeFileSync(join(tests, "bedrock-misbehavior-live.drift.ts"), markerFixture);
       writeFileSync(join(tests, "unrelated.drift.ts"), markerFixture);
     }
-    if (mode === "vertex-markers") {
+    if (mode === "vertex-markers" || mode === "console-overflow") {
       const markerFixture = `
         import { it, describe, beforeAll, expect } from 'vitest';
         const cells = ['k5-object','k5-stream','k9-object','k9-stream'];
@@ -233,7 +234,7 @@ async function withCollectorFixture(
       writeFileSync(join(tests, "vertex-misbehavior.drift.ts"), markerFixture);
       writeFileSync(join(tests, "unrelated.drift.ts"), markerFixture);
     }
-    if (mode === "cohere-markers") {
+    if (mode === "cohere-markers" || mode === "console-overflow") {
       const markerFixture = `
         import { it, expect } from 'vitest';
         const marker = {cohereK5:[{stream:false,classification:'NOT_TRIGGERED',finish:'COMPLETE',usage:{tokens:{inputTokens:1,outputTokens:1}}},{stream:true,classification:'TARGET',finish:'TOOL_CALL',usage:{tokens:{inputTokens:1,outputTokens:1}}}],raw:process.env.FIXTURE_SECRET,request:'fixture-provider-body-912'};
@@ -366,6 +367,194 @@ async function withCollectorFixture(
 }
 
 describe("collector invocation evidence", async () => {
+  it.each(["test", "suite", "module"] as const)(
+    "isolates attributable console overflow at %s scope",
+    async (scope) => {
+      await withCollectorFixture("console-overflow", async ({ directory, invoke }) => {
+        const file =
+          scope === "test"
+            ? "bedrock-misbehavior-live"
+            : scope === "suite"
+              ? "vertex-misbehavior"
+              : "cohere";
+        const fixture = join(directory, `src/__tests__/drift/${file}.drift.ts`);
+        const before = readFileSync(fixture, "utf8");
+        const trigger =
+          scope === "test"
+            ? "it('invoke K5 stream=false', () => console.log(JSON.stringify(marker('K5',false,'native','TARGET_COMPARED'))));"
+            : scope === "suite"
+              ? "beforeAll(() => console.info("
+              : "const marker =";
+        expect(before).toContain(trigger);
+        writeFileSync(
+          fixture,
+          before.replace(
+            trigger,
+            scope === "test"
+              ? "it('invoke K5 stream=false', async () => { console.log('x'.repeat(262145)); await Promise.resolve(); console.log(JSON.stringify(marker('K5',false,'native','TARGET_COMPARED'))); });"
+              : scope === "suite"
+                ? "beforeAll(() => console.log('x'.repeat(262145))); beforeAll(() => console.info("
+                : "console.log('x'.repeat(262145)); const marker =",
+          ),
+        );
+        const child = await invoke();
+        expect(child.error).toBeUndefined();
+        expect(child.status).toBe(2);
+        const evidenceDirectory = join(directory, "drift-evidence");
+        const serialized = readFileSync(
+          join(evidenceDirectory, readdirSync(evidenceDirectory)[0]),
+          "utf8",
+        );
+        const evidence = JSON.parse(serialized);
+        for (const row of evidence.drift.assertions) {
+          const tainted =
+            row.file === `src/__tests__/drift/${file}.drift.ts` &&
+            (scope === "module" ||
+              (scope === "test"
+                ? row.title === "invoke K5 stream=false"
+                : row.ancestors[0] === "Vertex native recurring modeled-contract observations"));
+          expect(row.consoleContext.availability, `${row.file}: ${row.title}`).toBe(
+            tainted ? "unavailable" : "available",
+          );
+          if (tainted) {
+            expect(row.consoleContext.recordCount).toBe(0);
+            expect(row.observations).toBeUndefined();
+            expect(row.source).toBe("unknown");
+          }
+        }
+        // Vitest batches synchronous stdout: the yielded marker must survive separately.
+        if (scope === "test") expect(evidence.drift.unboundConsoleRecordCount).toBe(1);
+        if (scope === "suite")
+          expect(evidence.drift.consoleScopes).not.toContainEqual(
+            expect.objectContaining({
+              file: `src/__tests__/drift/${file}.drift.ts`,
+              title: "Vertex native recurring modeled-contract observations",
+            }),
+          );
+        expect(serialized).not.toContain("x".repeat(256));
+        expect(serialized).not.toContain("fixture-provider-body-912");
+      });
+    },
+    60_000,
+  );
+
+  it.each([
+    "missing-owner",
+    "duplicate-id",
+    "duplicate-owner",
+    "empty-id",
+    "non-string-id",
+    "missing-completion",
+    "duplicate-completion",
+    "malformed",
+    "truncated",
+    "oversized-identity",
+    "bounded-marker",
+    "unbounded-marker",
+    "unattributed-overflow",
+  ])(
+    "keeps console overflow channel validation fail-closed: %s",
+    async (fault) => {
+      await withCollectorFixture("mixed", async ({ directory, invoke }) => {
+        const config = join(directory, "vitest.config.drift.ts");
+        const reporter = resolve(import.meta.dirname, "../../scripts/drift-evidence-reporter.ts");
+        const id =
+          fault === "bounded-marker"
+            ? "i".repeat(8192)
+            : fault === "unbounded-marker"
+              ? "i".repeat(262145)
+              : "transport-owner";
+        const owner = {
+          kind: "identity",
+          id,
+          entity: "suite",
+          file: join(directory, "src/__tests__/drift/fixture.drift.ts"),
+          ancestors: [],
+          title: "transport owner",
+        };
+        const records: unknown[] = [];
+        if (!["missing-owner", "unattributed-overflow"].includes(fault)) records.push(owner);
+        if (fault === "duplicate-id") records.push({ ...owner, title: "other owner" });
+        if (fault === "duplicate-owner") records.push({ ...owner, id: "other id" });
+        if (fault === "oversized-identity") records.push({ ...owner, title: "x".repeat(262145) });
+        if (fault === "duplicate-completion") records.push({ kind: "complete" });
+        const emitsOverflow = [
+          "bounded-marker",
+          "unbounded-marker",
+          "unattributed-overflow",
+        ].includes(fault);
+        if (!emitsOverflow)
+          records.push({
+            kind: "console-unavailable",
+            taskId: fault === "empty-id" ? "" : fault === "non-string-id" ? 4 : id,
+          });
+        const transport =
+          records.map((record) => JSON.stringify(record) + "\n").join("") +
+          (fault === "malformed" ? "not json\n" : fault === "truncated" ? '{"kind":' : "");
+        writeFileSync(
+          config,
+          readFileSync(config, "utf8") +
+            `
+      import { closeSync } from 'node:fs';
+      import EvidenceReporter from ${JSON.stringify(reporter)};
+      writeSync(3, ${JSON.stringify(transport)});
+      ${emitsOverflow ? `new EvidenceReporter().onUserConsoleLog({type:'stdout',taskId:${JSON.stringify(fault === "unattributed-overflow" ? "" : id)},content:'x'.repeat(262145),time:0,size:1});` : ""}
+      ${fault === "missing-completion" ? "closeSync(3);" : ""}
+    `,
+        );
+        const child = await invoke();
+        expect(child.error).toBeUndefined();
+        expect(child.status).toBe(2);
+        const evidenceDirectory = join(directory, "drift-evidence");
+        const evidence = JSON.parse(
+          readFileSync(join(evidenceDirectory, readdirSync(evidenceDirectory)[0]), "utf8"),
+        );
+        expect(evidence.drift.assertions.length).toBeGreaterThan(0);
+        for (const row of evidence.drift.assertions) {
+          expect(row.consoleContext.availability).toBe(
+            fault === "bounded-marker" ? "available" : "unavailable",
+          );
+          expect(row.observations).toBeUndefined();
+        }
+      });
+    },
+    60_000,
+  );
+
+  it("compares tainted suite ancestors by path elements", async () => {
+    await withCollectorFixture("mixed", async ({ directory, invoke }) => {
+      writeFileSync(
+        join(directory, "src/__tests__/drift/scope.drift.ts"),
+        `
+      import {describe,it,beforeAll} from 'vitest';
+      describe('a',()=>{
+        describe('b',()=>{
+          beforeAll(()=>console.log('x'.repeat(262145)));
+          it('tainted',()=>console.log('valid child'));
+          describe('c',()=>{beforeAll(()=>console.log('valid nested scope'));it('tainted nested',()=>console.log('valid nested child'));});
+        });
+        describe('b > c',()=>{it('sibling',()=>console.log('valid sibling'));});
+      });
+      describe('a > b',()=>{it('joined name',()=>console.log('valid separate path'));});
+    `,
+      );
+      const child = await invoke();
+      expect(child.error).toBeUndefined();
+      expect(child.status).toBe(2);
+      const evidenceDirectory = join(directory, "drift-evidence");
+      const evidence = JSON.parse(
+        readFileSync(join(evidenceDirectory, readdirSync(evidenceDirectory)[0]), "utf8"),
+      );
+      for (const row of evidence.drift.assertions)
+        expect(row.consoleContext.availability).toBe(
+          row.title.startsWith("tainted") ? "unavailable" : "available",
+        );
+      expect(evidence.drift.consoleScopes).not.toContainEqual(
+        expect.objectContaining({ title: "c" }),
+      );
+    });
+  }, 60_000);
+
   it("retains Responses WS exact counts beyond the numeric cap", async () => {
     await withCollectorFixture("responses-markers", async ({ directory, invoke }) => {
       const fixture = join(directory, "src/__tests__/drift/ws-responses.drift.ts");