fix: enforce per-invocation Cloud Hypervisor enclave storage limits

.github/workflows/test-cloud-hypervisor.yml

@@ -69,6 +69,24 @@ jobs:
             "$RUNNER_TEMP/microvm-supervisor.test" \
             -test.run '^TestEnclaveGuestLimitsLive$' -test.count=1 -test.v
 
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '22'
+          cache: npm
+
+      - name: Run privileged enclave host storage enforcement suite
+        # Mounts the real role-sized invocation tmpfs backing stores and
+        # host memory cgroups in a private mount namespace, then proves
+        # aggregate ENOSPC, isolation, cgroup accounting, and busy-unmount
+        # behavior. The suite is skipped without AWF_TEST_ENCLAVE_STORAGE.
+        run: |
+          npm ci
+          sudo unshare --mount --propagation private \
+            env "PATH=$PATH" AWF_TEST_ENCLAVE_STORAGE=1 \
+            npx jest --ci --runInBand \
+            src/cloud-hypervisor/enclave-storage.integration.test.ts
+
       - name: Install deterministic guest build prerequisites
         run: |
           sudo apt-get update

docs/cloud-hypervisor-foundation.md

@@ -257,13 +257,57 @@ privileges through executable metadata. Missing mounts, unsupported kernel
 controls, or verification mismatches abort startup rather than launching
 without a limit.
 
-Writable virtio-fs storage is accepted only when every writable export is on
-the same host filesystem and that filesystem's full capacity is no greater than
-the role ceiling. This deliberately strict check fails on typical larger runner
-filesystems; the future host executor must provide a genuinely size-bounded
-per-invocation backing filesystem or remain unavailable. It is not a per-folder
-quota and the guest cannot use `size=` to limit virtio-fs. Tmpfs ceilings are
-independent of this host filesystem ceiling.
+The trusted host executor mounts one invocation-private Linux **tmpfs** at the
+host-derived invocation directory, with `size=1073741824` for scripts (1 GiB) or
+`size=536870912` for agents (512 MiB), and `mode=0700,nosuid,nodev,noexec`.
+The closed `/output`, `/runtime`, and agent `/session-state` writable virtio-fs
+exports are subdirectories of that single backing store, not separate tmpfs
+mounts. Request and handoff files also consume its budget. Linux charges allocated
+pages across all exports atomically, including concurrent writes and writes into
+sparse-file holes; allocation beyond the aggregate ceiling returns `ENOSPC`.
+Sparse logical lengths do not allocate pages and do not bypass the allocation
+limit. No disk quota, loop device, or guest-only `size=` limit is required, so
+the mechanism uses the supported GitHub-hosted Linux/KVM runner's existing
+mount/virtio-fs path. Guest-internal tmpfs ceilings remain separate.
+
+Host tmpfs pages are charged to the writing virtio-fs process's memory cgroup.
+The enclave-only host `memory.max` therefore includes the fixed storage ceiling
+in addition to 768 MiB guest RAM and the existing 256 MiB VMM overhead:
+2 GiB for scripts and 1.5 GiB for agents. This avoids preempting the storage
+ceiling with the old shared-cgroup budget; it does not enlarge guest RAM or add
+a configurable resource limit. Host memory exhaustion can still terminate an
+invocation rather than return `ENOSPC`, and is treated as executor failure.
+Primary-agent cgroup budgets are unchanged.
+
+Before staging inputs and again before starting virtio-fs daemons, AWF verifies
+canonical export paths, the exact invocation mount in `/proc/self/mountinfo`,
+its tmpfs type, private mount identity, mount options, and exact role capacity
+from `statfs`. This is verification of a kernel-enforced backing store, not
+a free-space preflight. Missing, undersized, oversized, aliased, nested, or
+unverifiable mounts abort startup. There are no caller-selectable storage paths,
+sizes, classes, overrides, or fallback to the runner filesystem.
+
+The existing durable host-executor resource journal records the underlying
+directory before mounting and captures the tmpfs mount identity before use.
+Completion, timeout, cancellation, and partial startup wait for outstanding
+provisioning and VM/virtio-fs teardown before ordinary (never lazy) unmount and
+directory removal. Abandoned-run recovery first reaps the VM cleanup record,
+then verifies recorded directory/mount ownership and removes invocation storage.
+Unmount or ownership-verification failure retains the recovery record, reports
+incomplete cleanup, and keeps admissions closed; it never deletes through a
+live mount.
+
+`src/cloud-hypervisor/enclave-storage.integration.test.ts` exercises the actual
+host backing paths served by the closed writable exports, including role-sized
+aggregate `ENOSPC`, sparse and concurrent writes, invocation isolation, and busy
+unmount failure. It also fills storage in the production host cgroup budget while
+holding the guest-RAM equivalent resident, with swap disabled, to check that
+storage exhaustion is not preempted by a cgroup OOM. Run it as root in a private mount namespace with
+`AWF_TEST_ENCLAVE_STORAGE=1 npm test -- --runInBand enclave-storage.integration.test.ts`.
+The `build-test-artifacts` job in `.github/workflows/test-cloud-hypervisor.yml`
+runs this suite on every matching pull request via `sudo unshare --mount --propagation private`.
+Live guest transport conformance additionally requires the release-attested role
+artifacts and KVM runtime wiring.
 
 The rootfs build removes package-manager executables, the importable `pip` and
 `ensurepip` modules (so `python3 -m pip` cannot run or be recreated),

src/cloud-hypervisor/enclave-storage.integration.test.ts

@@ -0,0 +1,187 @@
+import { promises as fs } from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import execa from 'execa';
+import {
+  assertBoundedEnclaveWritableExports,
+  mountBoundedEnclaveStorage,
+  unmountBoundedEnclaveStorage,
+} from './enclave-storage';
+import { CLOUD_HYPERVISOR_ENCLAVE_RESOURCE_PROFILES as profiles } from './workload-profile';
+import type { CloudHypervisorDirectoryExport } from './exports';
+import { CloudHypervisorCgroup } from './launcher';
+
+const live = process.env.AWF_TEST_ENCLAVE_STORAGE === '1';
+const tools = { mount: '/usr/bin/mount', umount: '/usr/bin/umount' };
+const page = Buffer.alloc(4096, 1);
+
+// Exercise the host sources served by the closed guest-visible virtio-fs
+// exports. Live VM transport conformance is gated separately on KVM artifacts.
+(live ? describe : describe.skip)('host-enforced enclave storage', () => {
+  let root: string;
+  const mounted = new Set<string>();
+
+  beforeAll(async () => {
+    if (process.getuid?.() !== 0) throw new Error('AWF_TEST_ENCLAVE_STORAGE requires root and mount privileges');
+    root = await fs.mkdtemp(path.join(os.tmpdir(), 'awf-enclave-storage-'));
+  });
+
+  afterEach(async () => {
+    for (const directory of [...mounted].reverse()) {
+      await unmountBoundedEnclaveStorage(directory, tools);
+      mounted.delete(directory);
+    }
+    await fs.rm(root, { recursive: true, force: true });
+    root = await fs.mkdtemp(path.join(os.tmpdir(), 'awf-enclave-storage-'));
+  });
+
+  afterAll(async () => {
+    await fs.rm(root, { recursive: true, force: true });
+  });
+
+  async function provision(role: 'script' | 'agent', name: string) {
+    const directory = path.join(root, name);
+    await fs.mkdir(directory, { mode: 0o700 });
+    const profile = profiles[role];
+    await mountBoundedEnclaveStorage(
+      directory, profile.writableStorageBytes, profile.uid, profile.gid, tools,
+    );
+    mounted.add(directory);
+    const names = ['output', 'runtime', ...(role === 'agent' ? ['session-state'] : [])];
+    const exports: CloudHypervisorDirectoryExport[] = [];
+    for (const name of names) {
+      const source = path.join(directory, name);
+      await fs.mkdir(source, { mode: 0o700 });
+      exports.push({ tag: `enclave-${name}`, source, target: `/${name}`, mode: 'rw' });
+    }
+    await assertBoundedEnclaveWritableExports(exports, profile.writableStorageBytes);
+    return { directory, exports, size: profile.writableStorageBytes };
+  }
+
+  it.each(['script', 'agent'] as const)(
+    '%s writes exhaust one aggregate budget across sparse and concurrent exports',
+    async (role) => {
+      const { exports, size } = await provision(role, role);
+      const files = await Promise.all(exports.map(({ source }) => fs.open(path.join(source, 'sparse'), 'wx')));
+      try {
+        // Logical holes are not allocated storage. Writes into them must still
+        // consume the same page budget, even far beyond the capacity offset.
+        await Promise.all(files.map((file) => file.truncate(size * 4)));
+        const chunk = Buffer.alloc(1024 * 1024, 1);
+        const pagesPerChunk = chunk.length / page.length;
+        const chunksPerFile = Math.floor(size / chunk.length / files.length);
+        await Promise.all(files.map(async (file) => {
+          for (let chunkIndex = 0; chunkIndex < chunksPerFile; chunkIndex += 1) {
+            await file.write(chunk, 0, chunk.length, size * 2 + chunkIndex * chunk.length);
+          }
+        }));
+        const allocated = chunksPerFile * files.length * chunk.length;
+        const remainingPages = (size - allocated) / page.length;
+        for (let index = 0; index < remainingPages; index += 1) {
+          await files[0].write(page, 0, page.length,
+            size * 2 + (chunksPerFile * pagesPerChunk + index) * page.length);
+        }
+        const attempts = await Promise.all(files.map((file) => file.write(page, 0, page.length, 0)
+          .then(() => 'unexpected success', (error: NodeJS.ErrnoException) => error.code)));
+        expect(attempts).toEqual(files.map(() => 'ENOSPC'));
+        const stats = await Promise.all(files.map((file) => file.stat()));
+        expect(stats.reduce((total, stat) => total + stat.blocks * 512, 0)).toBe(size);
+      } finally {
+        await Promise.all(files.map((file) => file.close()));
+      }
+    }, 120_000,
+  );
+
+  it('isolates invocations and rejects an export redirected through another mount', async () => {
+    const first = await provision('agent', 'first');
+    const second = await provision('agent', 'second');
+    await fs.writeFile(path.join(first.exports[0].source, 'private'), 'first');
+    await expect(fs.readFile(path.join(second.exports[0].source, 'private'))).rejects.toMatchObject({ code: 'ENOENT' });
+    expect((await fs.stat(first.directory)).dev).not.toBe((await fs.stat(second.directory)).dev);
+    const escaped = path.join(first.directory, 'escape');
+    await fs.symlink(second.exports[0].source, escaped);
+    await expect(assertBoundedEnclaveWritableExports([
+      { ...first.exports[0], source: escaped },
+    ], first.size)).rejects.toThrow(/escapes/);
+
+    const nested = first.exports[1].source;
+    await execa(tools.mount, ['--bind', second.exports[1].source, nested]);
+    mounted.add(nested);
+    await expect(assertBoundedEnclaveWritableExports(first.exports, first.size))
+      .rejects.toThrow(/unverifiable/);
+  });
+
+  it.each(['script', 'agent'] as const)('reaches %s ENOSPC with resident guest RAM in the host memory cgroup', async (role) => {
+    const { exports, size } = await provision(role, `cgroup-${role}`);
+    const profile = profiles[role];
+    const cgroup = new CloudHypervisorCgroup(
+      `/sys/fs/cgroup/awf-cloud-hypervisor/storage-${path.basename(root)}`,
+      {
+        memoryMib: profile.memoryMiB, vcpuCount: profile.vcpuCount,
+        cpuQuotaMilli: profile.cpuQuotaMilli, writableStorageBytes: size,
+      },
+    );
+    await cgroup.setup();
+    await fs.writeFile(path.join(cgroup.cgroupPath, 'memory.swap.max'), '0');
+    // A separate process holds the guest-RAM equivalent resident while filling
+    // tmpfs. Both charges must fit the same production host memory budget.
+    const writer = execa(process.execPath, ['-e', `
+      const fs = require('fs');
+      const { sources, size, memory } = JSON.parse(process.argv[1]);
+      process.stdin.once('data', () => {
+        const guestRam = Buffer.alloc(memory, 1);
+        const chunk = Buffer.alloc(1024 * 1024, 1);
+        const files = sources.map(source => fs.openSync(source + '/charged', 'wx'));
+        try {
+          for (let index = 0; index < size / chunk.length; index++) {
+            fs.writeSync(files[index % files.length], chunk, 0, chunk.length,
+              Math.floor(index / files.length) * chunk.length);
+          }
+          for (const file of files) {
+            try {
+              fs.writeSync(file, chunk, 0, 4096, size);
+              throw new Error('storage was not bounded');
+            } catch (error) {
+              if (error.code !== 'ENOSPC') throw error;
+            }
+          }
+          if (guestRam[guestRam.length - 1] !== 1) throw new Error('guest RAM missing');
+          console.log('ENOSPC');
+        } finally {
+          files.forEach(file => fs.closeSync(file));
+        }
+      });
+    `, JSON.stringify({
+      sources: exports.map(({ source }) => source), size, memory: profile.memoryMiB * 1024 * 1024,
+    })], { stdio: ['pipe', 'pipe', 'pipe'] });
+    try {
+      if (writer.pid === undefined) throw new Error('storage writer did not start');
+      await cgroup.assign(writer.pid);
+      writer.stdin!.end('start');
+      expect((await writer).stdout.trim()).toBe('ENOSPC');
+      expect(await fs.readFile(path.join(cgroup.cgroupPath, 'memory.events'), 'utf8'))
+        .toMatch(/^oom_kill 0$/m);
+      expect(BigInt((await fs.readFile(path.join(cgroup.cgroupPath, 'memory.peak'), 'utf8')).trim()))
+        .toBeGreaterThan(BigInt((profile.memoryMiB + 256) * 1024 * 1024));
+    } finally {
+      writer.kill('SIGKILL');
+      await writer.catch(() => undefined);
+      await cgroup.cleanup();
+    }
+  }, 120_000);
+
+  it('does not report successful cleanup or remove a busy backing store', async () => {
+    const { directory, exports } = await provision('agent', 'busy');
+    const handle = await fs.open(path.join(exports[0].source, 'open'), 'wx');
+    try {
+      await expect(unmountBoundedEnclaveStorage(directory, tools)).rejects.toThrow(/unmount/);
+      await assertBoundedEnclaveWritableExports(exports, profiles.agent.writableStorageBytes);
+    } finally {
+      await handle.close();
+    }
+    await unmountBoundedEnclaveStorage(directory, tools);
+    mounted.delete(directory);
+    await fs.rm(directory, { recursive: true });
+    await expect(fs.lstat(directory)).rejects.toMatchObject({ code: 'ENOENT' });
+  });
+});