From 2aa1e86e3b967344257085eb6591ba4b28c2ae2c Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 9 Oct 2026 13:13:06 -0700 Subject: [PATCH 1/2] fix: isolate attributable drift console overflow --- scripts/drift-evidence-reporter.ts | 19 ++- scripts/drift-report-collector.ts | 43 ++++++ src/__tests__/drift-collector.test.ts | 195 +++++++++++++++++++++++++- 3 files changed, 247 insertions(+), 10 deletions(-) diff --git a/scripts/drift-evidence-reporter.ts b/scripts/drift-evidence-reporter.ts index 1ca15c09..176e89b4 100644 --- a/scripts/drift-evidence-reporter.ts +++ b/scripts/drift-evidence-reporter.ts @@ -15,6 +15,7 @@ export type EvidenceReporterRecord = ancestors: string[]; title: string; } + | { kind: "console-unavailable"; taskId: string } | { kind: "unavailable" } | { kind: "complete" }; @@ -28,13 +29,17 @@ export default class DriftEvidenceReporter implements Reporter { private emit(record: EvidenceReporterRecord) { if (this.broken) return; try { - const encoded = JSON.stringify(record); - writeFileSync( - 3, - (Buffer.byteLength(encoded) <= MAX_EVIDENCE_RECORD_BYTES - ? encoded - : '{"kind":"unavailable"}') + "\n", - ); + let encoded = JSON.stringify(record); + if (Buffer.byteLength(encoded) > MAX_EVIDENCE_RECORD_BYTES) { + encoded = JSON.stringify( + record.kind === "console" && record.taskId + ? { kind: "console-unavailable", taskId: record.taskId } + : { kind: "unavailable" }, + ); + if (Buffer.byteLength(encoded) > MAX_EVIDENCE_RECORD_BYTES) + encoded = '{"kind":"unavailable"}'; + } + writeFileSync(3, encoded + "\n"); } catch { this.broken = true; // A missing completion record makes the parent mark the channel unavailable. diff --git a/scripts/drift-report-collector.ts b/scripts/drift-report-collector.ts index 8d4602f3..4e1c3d90 100644 --- a/scripts/drift-report-collector.ts +++ b/scripts/drift-report-collector.ts @@ -210,6 +210,14 @@ function parseEvidenceRecord(line: string): EvidenceReporterRecord | null { } if (!value || typeof value !== "object" || !("kind" in value)) return null; if (value.kind === "complete" || value.kind === "unavailable") return { kind: value.kind }; + if ( + value.kind === "console-unavailable" && + "taskId" in value && + typeof value.taskId === "string" && + value.taskId.length > 0 + ) + return { kind: "console-unavailable", taskId: value.taskId }; + if ( value.kind === "console" && "taskId" in value && @@ -257,6 +265,23 @@ function captureReporterChannel(leg: LegEvidence, stream: string) { records.push(record); } if (records.filter((record) => record.kind === "complete").length !== 1) return; + const identities = records.filter((record) => record.kind === "identity"); + for (const record of records) { + if (record.kind !== "console-unavailable") continue; + const owners = identities.filter((identity) => identity.id === record.taskId); + if (owners.length !== 1) return; + const owner = owners[0]; + if ( + identities.filter( + (identity) => + identity.entity === owner.entity && + resolve(identity.file) === resolve(owner.file) && + identity.title === owner.title && + JSON.stringify(identity.ancestors) === JSON.stringify(owner.ancestors), + ).length !== 1 + ) + return; + } reporterChannels.set(leg, records); } @@ -272,6 +297,23 @@ function bindReporterContext(leg: LegEvidence, result: VitestJsonResult) { const records = reporterChannels.get(leg) ?? []; const identities = records.filter((record) => record.kind === "identity"); const logs = records.filter((record) => record.kind === "console"); + const taintedIds = new Set( + records + .filter((record) => record.kind === "console-unavailable") + .map((record) => record.taskId), + ); + const taintedOwners = identities.filter((identity) => taintedIds.has(identity.id)); + const isTainted = (identity: Extract) => + taintedOwners.some((owner) => { + if (resolve(identity.file) !== resolve(owner.file)) return false; + if (owner.entity === "module") return true; + if (owner.entity === "test") return identity.id === owner.id; + const path = [...owner.ancestors, owner.title]; + return ( + identity.id === owner.id || path.every((part, index) => identity.ancestors[index] === part) + ); + }); + const key = (file: string | undefined, ancestors: string[], title: string) => JSON.stringify([file === undefined ? null : resolve(file), ancestors, title]); const rows = result.testResults.flatMap((file) => @@ -285,6 +327,7 @@ function bindReporterContext(leg: LegEvidence, result: VitestJsonResult) { leg.consoleScopes = []; for (const identity of identities) { if (!identity.id || identities.filter((item) => item.id === identity.id).length !== 1) continue; + if (isTainted(identity)) continue; const messages = logs.filter((log) => log.taskId === identity.id); if (identity.entity !== "test") { matchedIds.add(identity.id); diff --git a/src/__tests__/drift-collector.test.ts b/src/__tests__/drift-collector.test.ts index 950d6ab7..44376239 100644 --- a/src/__tests__/drift-collector.test.ts +++ b/src/__tests__/drift-collector.test.ts @@ -73,6 +73,7 @@ import type { DriftReport } from "../../scripts/drift-types.js"; // Exercise the real CLI and installed Vitest reporter, without provider credentials. async function withCollectorFixture( mode: + | "console-overflow" | "mixed" | "unknown-surface" | "secondary-malformed" @@ -196,7 +197,7 @@ async function withCollectorFixture( `, ); } - if (mode === "bedrock-markers") { + if (mode === "bedrock-markers" || mode === "console-overflow") { const markerFixture = ` import { it, describe, expect } from 'vitest'; const marker = (fault, stream, source, outcome) => ({ nativeComparison: {wire:'invoke',fault,stream},source,outcome,sends:1,requestedOutputTokens:32,request:'fixture-provider-body-912',raw:process.env.FIXTURE_SECRET }); @@ -215,7 +216,7 @@ async function withCollectorFixture( writeFileSync(join(tests, "bedrock-misbehavior-live.drift.ts"), markerFixture); writeFileSync(join(tests, "unrelated.drift.ts"), markerFixture); } - if (mode === "vertex-markers") { + if (mode === "vertex-markers" || mode === "console-overflow") { const markerFixture = ` import { it, describe, beforeAll, expect } from 'vitest'; const cells = ['k5-object','k5-stream','k9-object','k9-stream']; @@ -233,7 +234,7 @@ async function withCollectorFixture( writeFileSync(join(tests, "vertex-misbehavior.drift.ts"), markerFixture); writeFileSync(join(tests, "unrelated.drift.ts"), markerFixture); } - if (mode === "cohere-markers") { + if (mode === "cohere-markers" || mode === "console-overflow") { const markerFixture = ` import { it, expect } from 'vitest'; const marker = {cohereK5:[{stream:false,classification:'NOT_TRIGGERED',finish:'COMPLETE',usage:{tokens:{inputTokens:1,outputTokens:1}}},{stream:true,classification:'TARGET',finish:'TOOL_CALL',usage:{tokens:{inputTokens:1,outputTokens:1}}}],raw:process.env.FIXTURE_SECRET,request:'fixture-provider-body-912'}; @@ -366,6 +367,194 @@ async function withCollectorFixture( } describe("collector invocation evidence", async () => { + it.each(["test", "suite", "module"] as const)( + "isolates attributable console overflow at %s scope", + async (scope) => { + await withCollectorFixture("console-overflow", async ({ directory, invoke }) => { + const file = + scope === "test" + ? "bedrock-misbehavior-live" + : scope === "suite" + ? "vertex-misbehavior" + : "cohere"; + const fixture = join(directory, `src/__tests__/drift/${file}.drift.ts`); + const before = readFileSync(fixture, "utf8"); + const trigger = + scope === "test" + ? "it('invoke K5 stream=false', () => console.log(JSON.stringify(marker('K5',false,'native','TARGET_COMPARED'))));" + : scope === "suite" + ? "beforeAll(() => console.info(" + : "const marker ="; + expect(before).toContain(trigger); + writeFileSync( + fixture, + before.replace( + trigger, + scope === "test" + ? "it('invoke K5 stream=false', async () => { console.log('x'.repeat(262145)); await Promise.resolve(); console.log(JSON.stringify(marker('K5',false,'native','TARGET_COMPARED'))); });" + : scope === "suite" + ? "beforeAll(() => console.log('x'.repeat(262145))); beforeAll(() => console.info(" + : "console.log('x'.repeat(262145)); const marker =", + ), + ); + const child = await invoke(); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(2); + const evidenceDirectory = join(directory, "drift-evidence"); + const serialized = readFileSync( + join(evidenceDirectory, readdirSync(evidenceDirectory)[0]), + "utf8", + ); + const evidence = JSON.parse(serialized); + for (const row of evidence.drift.assertions) { + const tainted = + row.file === `src/__tests__/drift/${file}.drift.ts` && + (scope === "module" || + (scope === "test" + ? row.title === "invoke K5 stream=false" + : row.ancestors[0] === "Vertex native recurring modeled-contract observations")); + expect(row.consoleContext.availability, `${row.file}: ${row.title}`).toBe( + tainted ? "unavailable" : "available", + ); + if (tainted) { + expect(row.consoleContext.recordCount).toBe(0); + expect(row.observations).toBeUndefined(); + expect(row.source).toBe("unknown"); + } + } + // Vitest batches synchronous stdout: the yielded marker must survive separately. + if (scope === "test") expect(evidence.drift.unboundConsoleRecordCount).toBe(1); + if (scope === "suite") + expect(evidence.drift.consoleScopes).not.toContainEqual( + expect.objectContaining({ + file: `src/__tests__/drift/${file}.drift.ts`, + title: "Vertex native recurring modeled-contract observations", + }), + ); + expect(serialized).not.toContain("x".repeat(256)); + expect(serialized).not.toContain("fixture-provider-body-912"); + }); + }, + 60_000, + ); + + it.each([ + "missing-owner", + "duplicate-id", + "duplicate-owner", + "empty-id", + "non-string-id", + "missing-completion", + "duplicate-completion", + "malformed", + "truncated", + "oversized-identity", + "bounded-marker", + "unbounded-marker", + "unattributed-overflow", + ])( + "keeps console overflow channel validation fail-closed: %s", + async (fault) => { + await withCollectorFixture("mixed", async ({ directory, invoke }) => { + const config = join(directory, "vitest.config.drift.ts"); + const reporter = resolve(import.meta.dirname, "../../scripts/drift-evidence-reporter.ts"); + const id = + fault === "bounded-marker" + ? "i".repeat(8192) + : fault === "unbounded-marker" + ? "i".repeat(262145) + : "transport-owner"; + const owner = { + kind: "identity", + id, + entity: "suite", + file: join(directory, "src/__tests__/drift/fixture.drift.ts"), + ancestors: [], + title: "transport owner", + }; + const records: unknown[] = []; + if (!["missing-owner", "unattributed-overflow"].includes(fault)) records.push(owner); + if (fault === "duplicate-id") records.push({ ...owner, title: "other owner" }); + if (fault === "duplicate-owner") records.push({ ...owner, id: "other id" }); + if (fault === "oversized-identity") records.push({ ...owner, title: "x".repeat(262145) }); + if (fault === "duplicate-completion") records.push({ kind: "complete" }); + const emitsOverflow = [ + "bounded-marker", + "unbounded-marker", + "unattributed-overflow", + ].includes(fault); + if (!emitsOverflow) + records.push({ + kind: "console-unavailable", + taskId: fault === "empty-id" ? "" : fault === "non-string-id" ? 4 : id, + }); + const transport = + records.map((record) => JSON.stringify(record) + "\n").join("") + + (fault === "malformed" ? "not json\n" : fault === "truncated" ? '{"kind":' : ""); + writeFileSync( + config, + readFileSync(config, "utf8") + + ` + import { closeSync } from 'node:fs'; + import EvidenceReporter from ${JSON.stringify(reporter)}; + writeSync(3, ${JSON.stringify(transport)}); + ${emitsOverflow ? `new EvidenceReporter().onUserConsoleLog({type:'stdout',taskId:${JSON.stringify(fault === "unattributed-overflow" ? "" : id)},content:'x'.repeat(262145),time:0,size:1});` : ""} + ${fault === "missing-completion" ? "closeSync(3);" : ""} + `, + ); + const child = await invoke(); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(2); + const evidenceDirectory = join(directory, "drift-evidence"); + const evidence = JSON.parse( + readFileSync(join(evidenceDirectory, readdirSync(evidenceDirectory)[0]), "utf8"), + ); + expect(evidence.drift.assertions.length).toBeGreaterThan(0); + for (const row of evidence.drift.assertions) { + expect(row.consoleContext.availability).toBe( + fault === "bounded-marker" ? "available" : "unavailable", + ); + expect(row.observations).toBeUndefined(); + } + }); + }, + 60_000, + ); + + it("compares tainted suite ancestors by path elements", async () => { + await withCollectorFixture("mixed", async ({ directory, invoke }) => { + writeFileSync( + join(directory, "src/__tests__/drift/scope.drift.ts"), + ` + import {describe,it,beforeAll} from 'vitest'; + describe('a',()=>{ + describe('b',()=>{ + beforeAll(()=>console.log('x'.repeat(262145))); + it('tainted',()=>console.log('valid child')); + describe('c',()=>{beforeAll(()=>console.log('valid nested scope'));it('tainted nested',()=>console.log('valid nested child'));}); + }); + describe('b > c',()=>{it('sibling',()=>console.log('valid sibling'));}); + }); + describe('a > b',()=>{it('joined name',()=>console.log('valid separate path'));}); + `, + ); + const child = await invoke(); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(2); + const evidenceDirectory = join(directory, "drift-evidence"); + const evidence = JSON.parse( + readFileSync(join(evidenceDirectory, readdirSync(evidenceDirectory)[0]), "utf8"), + ); + for (const row of evidence.drift.assertions) + expect(row.consoleContext.availability).toBe( + row.title.startsWith("tainted") ? "unavailable" : "available", + ); + expect(evidence.drift.consoleScopes).not.toContainEqual( + expect.objectContaining({ title: "c" }), + ); + }); + }, 60_000); + it("retains Responses WS exact counts beyond the numeric cap", async () => { await withCollectorFixture("responses-markers", async ({ directory, invoke }) => { const fixture = join(directory, "src/__tests__/drift/ws-responses.drift.ts"); From 3cea0460f7b1bc068773c5fa9e78006c4cab80ad Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Fri, 9 Oct 2026 13:15:44 -0700 Subject: [PATCH 2/2] docs: disclose scoped drift evidence overflow recovery --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47c0a93c..c3afc4c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ ### Fixed +- Keep drift evidence available for unrelated assertions when a scoped console record exceeds the capture limit; the affected assertion or scope remains unavailable. - Retain sanitized drift evidence for each test and retry attempt in CI artifacts. - Align Cohere streaming length-stop faults with captured tool closure, `TOOL_CALL`, and terminal `[DONE]` framing while preserving incomplete argument bytes. Object length stops retain modeled `MAX_TOKENS`; recurring comparisons distinguish valid object nontriggers from failures. - Preserve malformed tool-call argument strings on string-based response wires instead of silently replacing them with `{}`. Object-based wires now return an explicit error (HTTP 500, or a Gemini Live WebSocket error) before response content. Valid JSON, empty or omitted legacy arguments, and fixture validation retain their existing behavior.