镜像站点 · 本页由第三方 GitHub 只读镜像提供,非 GitHub 官方站点,不接受任何登录或凭据输入。前往 github.com
Skip to content

Commit d8fc8c9

Browse files
committed
fix(bench): reject contaminated build feedback (#614)
Share the fail-closed evidence-process classifier with the exact-parent collector, reject persistent build and test competitors before and after every focused-build scenario, and require at least 60% complete-process CPU occupancy in V2 artifacts. Preserve validation compatibility for the published V1 artifact. Validation: exact-parent self-test passed; build-feedback self-test passed; dependency audit reports 15 classified edges, 0 migration targets, and 60 classified tools. Flips 0 Test262 cases (tooling and dependency inventory only).
1 parent 8bbbf40 commit d8fc8c9

4 files changed

Lines changed: 102 additions & 50 deletions

File tree

‎docs/.data/dependency-inventory-v1.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@
214214
{ "name": "leaks", "count": 1, "edge": "macos-sdk-foundation" }
215215
],
216216
"script_extensions": [
217-
{ "extension": ".ts", "count": 65, "edge": "home-tool-runtime" }
217+
{ "extension": ".ts", "count": 66, "edge": "home-tool-runtime" }
218218
],
219219
"network_capable_scripts": [
220220
{ "path": "tools/wasm-spec/fetch.ts", "edge": "pinned-conformance-corpora" }

‎tools/build-feedback.ts‎

Lines changed: 48 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,12 @@ import {
88
temporaryDirectory,
99
writeText,
1010
} from "./lib/home";
11+
import {
12+
competingEvidenceProcesses,
13+
MINIMUM_PROCESS_CPU_OCCUPANCY,
14+
processCpuOccupancy,
15+
} from "./evidence-processes";
16+
// Inventory-visible module edge: tools/evidence-processes.ts.
1117

1218
declare const __filename: string;
1319

@@ -40,6 +46,7 @@ type Sample = {
4046
wall_seconds: number;
4147
user_seconds: number;
4248
system_seconds: number;
49+
process_cpu_occupancy?: number;
4350
peak_rss_bytes: number;
4451
build_summary: string[];
4552
stdout: string;
@@ -92,26 +99,21 @@ function commandOutput(argv: string[], fallback = "unavailable"): string {
9299
: fallback;
93100
}
94101

95-
export function competingZigProcesses(source: string, selfPid: number): string[] {
96-
const rows = source.split("\n").map((line) => {
97-
const match = /^\s*([0-9]+)\s+([0-9]+)\s+(.+)$/.exec(line);
98-
return match ? { pid: Number(match[1]), parent: Number(match[2]), command: match[3].trim() } : null;
99-
}).filter(Boolean) as { pid: number; parent: number; command: string }[];
100-
const parents = new Map(rows.map((row) => [row.pid, row.parent])), ancestors = new Set<number>();
101-
for (let pid = selfPid; pid > 0 && !ancestors.has(pid); pid = parents.get(pid) || 0) ancestors.add(pid);
102-
return rows.filter((row) => {
103-
if (ancestors.has(row.pid)) return false;
104-
const executable = row.command.split(/\s+/, 1)[0], basename = executable.slice(executable.lastIndexOf("/") + 1);
105-
return basename === "zig" || basename === "maker";
106-
}).map((row) => `${row.pid} ${row.command}`);
102+
function requireNoCompetingEvidenceProcess(phase: string): void {
103+
const listing = commandOutput(["ps", "-axo", "pid=,ppid=,command="], ""),
104+
competitors = competingEvidenceProcesses(listing, process.pid);
105+
requireValue(
106+
competitors.length === 0,
107+
`competing build/test process detected ${phase}:\n${competitors.join("\n")}`,
108+
);
107109
}
108110

109-
function requireNoCompetingZigProcess(): void {
110-
const listing = commandOutput(["ps", "-axo", "pid=,ppid=,command="], "");
111-
const competitors = competingZigProcesses(listing, process.pid);
111+
function requireSampleQuality(sample: Sample): void {
112112
requireValue(
113-
competitors.length === 0,
114-
`competing Zig build/test process detected before scenario:\n${competitors.join("\n")}`,
113+
sample.process_cpu_occupancy !== undefined &&
114+
Number.isFinite(sample.process_cpu_occupancy) &&
115+
sample.process_cpu_occupancy >= MINIMUM_PROCESS_CPU_OCCUPANCY,
116+
`${sample.scenario}: process CPU occupancy ${((sample.process_cpu_occupancy || 0) * 100).toFixed(1)}% is below ${(MINIMUM_PROCESS_CPU_OCCUPANCY * 100).toFixed(0)}%; transient competing work overlapped the scenario`,
115117
);
116118
}
117119

@@ -287,7 +289,12 @@ function runSample(
287289
env: environment,
288290
inheritEnv: true,
289291
}),
290-
timing = parseTime(result.stderr);
292+
timing = parseTime(result.stderr),
293+
process_cpu_occupancy = processCpuOccupancy(
294+
timing.wall_seconds,
295+
timing.user_seconds,
296+
timing.system_seconds,
297+
);
291298
return {
292299
scenario: scenario.name,
293300
sample,
@@ -296,6 +303,7 @@ function runSample(
296303
exit_code: result.exitCode,
297304
timed_out: result.timedOut,
298305
...timing,
306+
process_cpu_occupancy,
299307
build_summary: buildSummary(result.stderr),
300308
stdout: result.stdout,
301309
stderr: result.stderr,
@@ -362,6 +370,7 @@ export function validate(artifact: Artifact): void {
362370
if (name.startsWith("full_unit_"))
363371
requireValue(Boolean(row.unit_plan?.startsWith("shard\testimated_ms\tname\n")), `${name}: missing unit plan`);
364372
if (artifact.schema === "zig-js-build-feedback-v2") {
373+
requireSampleQuality(row);
365374
const group = scenario.cache_group!;
366375
requireValue(row.environment.ZIG_GLOBAL_CACHE_DIR.endsWith(`/${group}/global`), `${name}: global cache boundary drift`);
367376
const cacheIndex = row.command.indexOf("--cache-dir"), prefixIndex = row.command.indexOf("--prefix");
@@ -411,7 +420,10 @@ export function render(artifact: Artifact, rawName: string): string {
411420
`- power: ${artifact.identity.power.replace(/\n/g, " · ")}`,
412421
`- sampling: ${artifact.sample_count} sequential sample${artifact.sample_count === 1 ? "" : "s"} per phase; median; no outlier removal; isolated local/global caches per sample sequence`,
413422
...(artifact.schema === "zig-js-build-feedback-v2"
414-
? ["- cache boundary: library, focused-engine, combined-unit, focused-engine TSan, and combined-unit TSan groups are isolated; only adjacent phases with the same named group reuse cache state"]
423+
? [
424+
"- cache boundary: library, focused-engine, combined-unit, focused-engine TSan, and combined-unit TSan groups are isolated; only adjacent phases with the same named group reuse cache state",
425+
`- process quality: every complete build used at least ${(MINIMUM_PROCESS_CPU_OCCUPANCY * 100).toFixed(0)}% CPU occupancy; before/after snapshots reject persistent competing build and test jobs`,
426+
]
415427
: []),
416428
"",
417429
"| phase | scope | median wall | wall range | median CPU (user + system) | median peak RSS |",
@@ -516,6 +528,7 @@ function selfTest(): void {
516528
exit_code: 0,
517529
timed_out: false,
518530
...parsed,
531+
process_cpu_occupancy: 1,
519532
build_summary: ["Build Summary: 2/2 steps succeeded"],
520533
stdout: output,
521534
stderr: "",
@@ -540,17 +553,27 @@ function selfTest(): void {
540553
"100 1 /usr/bin/outer",
541554
"110 100 /cache/maker build build-feedback",
542555
"120 110 /home/home-tool run tools/build-feedback.ts",
556+
"121 120 /toolchain/zig build test-frontend",
543557
"200 1 /toolchain/zig test -Mroot=other.zig",
544558
"210 1 /cache/maker build test",
545-
"220 1 /tmp/unrelated-test",
559+
"220 1 /private/tmp/home-url-final",
560+
"230 1 /tmp/home-ts-checker/o/test --listen=-",
561+
"240 1 /repo/zig-out/bin/test262 --diag test/language",
562+
"250 1 /tmp/unrelated-test",
546563
].join("\n");
547564
requireValue(
548-
JSON.stringify(competingZigProcesses(processFixture, 120)) === JSON.stringify([
565+
JSON.stringify(competingEvidenceProcesses(processFixture, 120)) === JSON.stringify([
549566
"200 /toolchain/zig test -Mroot=other.zig",
550567
"210 /cache/maker build test",
568+
"220 /private/tmp/home-url-final",
569+
"230 /tmp/home-ts-checker/o/test --listen=-",
570+
"240 /repo/zig-out/bin/test262 --diag test/language",
551571
]),
552-
"competing Zig process classification drift",
572+
"competing evidence-process classification drift",
553573
);
574+
requireValue(processCpuOccupancy(2.5, 3, 1.25) === 1, "process CPU occupancy clamp drift");
575+
const lowOccupancy = { ...v2Samples[0], process_cpu_occupancy: 0.59 };
576+
expectFailure(() => requireSampleQuality(lowOccupancy), "transient competing work overlapped the scenario");
554577
console.log("build-feedback self-test: ok");
555578
}
556579

@@ -590,11 +613,12 @@ function main(): void {
590613
try {
591614
for (let scenarioIndex = 0; scenarioIndex < definitions.length; scenarioIndex += 1) {
592615
const scenario = definitions[scenarioIndex];
593-
requireNoCompetingZigProcess();
616+
requireNoCompetingEvidenceProcess("before scenario");
594617
console.log(`[${sample + 1}/${options.samples}] ${scenario.name}`);
595618
const row = runSample(options.zig, scenario, sample, cacheRoot);
596619
artifact.samples.push(row);
597-
requireNoCompetingZigProcess();
620+
requireNoCompetingEvidenceProcess("after scenario");
621+
requireSampleQuality(row);
598622
console.log(
599623
` exit=${row.exit_code} wall=${row.wall_seconds.toFixed(2)}s cpu=${(row.user_seconds + row.system_seconds).toFixed(2)}s rss=${formatBytes(row.peak_rss_bytes)}`,
600624
);

‎tools/evidence-processes.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/** Shared fail-closed process-quality policy for performance evidence collectors. */
2+
3+
export const MINIMUM_PROCESS_CPU_OCCUPANCY = 0.6;
4+
5+
type ProcessRow = { pid: number; parent: number; command: string };
6+
7+
function processRows(source: string): ProcessRow[] {
8+
return source.split("\n").map((line) => {
9+
const match = /^\s*([0-9]+)\s+([0-9]+)\s+(.+)$/.exec(line);
10+
return match ? { pid: Number(match[1]), parent: Number(match[2]), command: match[3].trim() } : null;
11+
}).filter(Boolean) as ProcessRow[];
12+
}
13+
14+
function processBasename(command: string): [string, string] {
15+
const executable = command.split(/\s+/, 1)[0], slash = executable.lastIndexOf("/");
16+
return [executable, executable.slice(slash + 1)];
17+
}
18+
19+
function relatedProcesses(rows: ProcessRow[], selfPid: number): Set<number> {
20+
const parents = new Map(rows.map((row) => [row.pid, row.parent])),
21+
related = new Set<number>(),
22+
descendants = new Set<number>([selfPid]);
23+
for (let pid = selfPid; pid > 0 && !related.has(pid); pid = parents.get(pid) || 0) related.add(pid);
24+
let changed = true;
25+
while (changed) {
26+
changed = false;
27+
for (const row of rows) if (!descendants.has(row.pid) && descendants.has(row.parent)) {
28+
descendants.add(row.pid);
29+
changed = true;
30+
}
31+
}
32+
for (const pid of descendants) related.add(pid);
33+
return related;
34+
}
35+
36+
export function competingEvidenceProcesses(source: string, selfPid: number): string[] {
37+
const rows = processRows(source), related = relatedProcesses(rows, selfPid);
38+
return rows.filter((row) => {
39+
if (related.has(row.pid)) return false;
40+
const [executable, basename] = processBasename(row.command);
41+
if (basename === "zig" || basename === "maker") return true;
42+
if (["home-url-final", "unit-test-parallel", "threads-test", "test262", "frontend-parse-benchmark"].includes(basename)) return true;
43+
return basename === "test" &&
44+
(executable.includes("/.zig-cache/") || executable.startsWith("/tmp/home-") || executable.startsWith("/private/tmp/home-"));
45+
}).map((row) => `${row.pid} ${row.command}`);
46+
}
47+
48+
export function processCpuOccupancy(processWall: number, user: number, system: number): number {
49+
if (![processWall, user, system].every((value) => Number.isFinite(value) && value >= 0)) return NaN;
50+
return processWall === 0 ? 1 : Math.min(1, (user + system) / processWall);
51+
}

‎tools/exact-parent-regression.ts‎

Lines changed: 2 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,37 +2,15 @@
22
import { checked, readText, removeTemporaryDirectory, run, sha256File, temporaryDirectory, writeText } from "./lib/home";
33
import { DEFAULT_SCHEMA, loadSchema, measured, unavailableMetrics, validateArtifact } from "./performance-attribution";
44
import { parseDarwinCounters, parseDarwinThermalState } from "./instrumentation-overhead";
5-
// Inventory-visible module edges: tools/performance-attribution.ts and tools/instrumentation-overhead.ts.
5+
import { competingEvidenceProcesses, MINIMUM_PROCESS_CPU_OCCUPANCY, processCpuOccupancy } from "./evidence-processes";
6+
// Inventory-visible module edges: tools/performance-attribution.ts, tools/instrumentation-overhead.ts, and tools/evidence-processes.ts.
67

78
declare const __dirname: string;
89
declare const __filename: string;
910
export const ROOT = __dirname === "tools" ? "." : __dirname.slice(0, __dirname.lastIndexOf("/tools"));
1011
const REVISION_RE = /^[0-9a-f]{40}$/;
1112
export type RunnerRow = { engine: string; mode: string; workload: string; lanes: number; jobs: number; elapsed_ns: number; checksum: number; process_wall_time_ns: number; process_cpu_user_ns: number; process_cpu_system_ns: number; process_cpu_occupancy: number; peak_rss_bytes: number; retained_rss_bytes: number | null; allocations: number | null; allocated_bytes: number | null; instructions: number; cycles: number; energy_joules: number; thermal_state: string };
1213
function requireValue(condition: boolean, message: string): void { if (!condition) throw new Error(message); }
13-
const MINIMUM_PROCESS_CPU_OCCUPANCY = 0.6;
14-
15-
type ProcessRow = { pid: number; parent: number; command: string };
16-
function processRows(source: string): ProcessRow[] { return source.split("\n").map((line) => { const match = /^\s*([0-9]+)\s+([0-9]+)\s+(.+)$/.exec(line); return match ? { pid: Number(match[1]), parent: Number(match[2]), command: match[3].trim() } : null; }).filter(Boolean) as ProcessRow[]; }
17-
function processBasename(command: string): [string, string] { const executable = command.split(/\s+/, 1)[0], slash = executable.lastIndexOf("/"); return [executable, executable.slice(slash + 1)]; }
18-
function relatedProcesses(rows: ProcessRow[], selfPid: number): Set<number> {
19-
const parents = new Map(rows.map((row) => [row.pid, row.parent])), related = new Set<number>(), descendants = new Set<number>([selfPid]);
20-
for (let pid = selfPid; pid > 0 && !related.has(pid); pid = parents.get(pid) || 0) related.add(pid);
21-
let changed = true;
22-
while (changed) { changed = false; for (const row of rows) if (!descendants.has(row.pid) && descendants.has(row.parent)) { descendants.add(row.pid); changed = true; } }
23-
for (const pid of descendants) related.add(pid);
24-
return related;
25-
}
26-
export function competingEvidenceProcesses(source: string, selfPid: number): string[] {
27-
const rows = processRows(source), related = relatedProcesses(rows, selfPid);
28-
return rows.filter((row) => {
29-
if (related.has(row.pid)) return false;
30-
const [executable, basename] = processBasename(row.command);
31-
if (basename === "zig" || basename === "maker") return true;
32-
if (["home-url-final", "unit-test-parallel", "threads-test", "test262", "frontend-parse-benchmark"].includes(basename)) return true;
33-
return basename === "test" && (executable.includes("/.zig-cache/") || executable.startsWith("/tmp/home-") || executable.startsWith("/private/tmp/home-"));
34-
}).map((row) => `${row.pid} ${row.command}`);
35-
}
3614
function requireNoCompetingEvidenceProcess(phase: string, listing = commandOutput(["ps", "-axo", "pid=,ppid=,command="], ""), selfPid = process.pid): void {
3715
const competitors = competingEvidenceProcesses(listing, selfPid);
3816
requireValue(competitors.length === 0, `competing build/test process detected ${phase}:\n${competitors.join("\n")}`);
@@ -53,7 +31,6 @@ export function parseBenchmark(stdout: string, expectedMode: string, expectedWor
5331
}
5432
export function timeMetric(stderr: string, label: string): number { const match = new RegExp(`(?:^|\\s)([0-9.]+)\\s+${label}(?:\\s|$)`).exec(stderr); requireValue(Boolean(match), `missing ${label} from /usr/bin/time -l`); return Number(match![1]); }
5533
export function parsePeakRss(stderr: string): number { return Math.round(timeMetric(stderr, "maximum resident set size")); }
56-
export function processCpuOccupancy(processWallNs: number, userNs: number, systemNs: number): number { requireValue([processWallNs, userNs, systemNs].every((value) => Number.isFinite(value) && value >= 0), "invalid process occupancy input"); return processWallNs === 0 ? 1 : Math.min(1, (userNs + systemNs) / processWallNs); }
5734
export function runnerArguments(mode: string, workload: string, jobs: number, lanes: number): string[] { if (mode === "single" || mode === "single_observed") { requireValue(lanes === 1, `${mode} mode requires one lane`); return [mode, workload, String(jobs), "1", mode === "single_observed" ? "--native-observability-telemetry" : "--darwin-rusage"]; } requireValue(["independent_steady", "independent_cold", "shared", "module_cold"].includes(mode), `unsupported benchmark mode: ${mode}`); return [mode, workload, String(jobs), "1", String(lanes), "--darwin-rusage"]; }
5835
export function parseRetainedRss(stdout: string, mode: string, workload: string, jobs: number): number {
5936
const rows = stdout.split("\n").filter((line) => line.startsWith("zig-js-native-observability\t")); requireValue(rows.length === 1, `expected one native-observability row, got ${rows.length}`);

0 commit comments

Comments
 (0)