test_runner: rerun failing todo tests in suites

doc/api/test.md

@@ -160,6 +160,8 @@ in this example, there are two run attempts, with two tests defined in `test.js`
 the first test succeeded on the first attempt, and the second test succeeded on the second attempt.
 
 When the `--test-rerun-failures` option is used, the test runner will only run tests that have not yet passed.
+A `todo` test that fails counts as not yet passed and is rerun, even though its failure does not
+cause the test run to fail.
 
 ```bash
 node --test-rerun-failures /path/to/state/file

lib/internal/test_runner/reporter/rerun.js

@@ -21,7 +21,13 @@ function reportReruns(previousRuns, globalOptions) {
 
     function startTest(data) {
       const originalSuite = currentSuite;
-      currentSuite = { __proto__: null, data, parent: currentSuite, children: [] };
+      currentSuite = {
+        __proto__: null,
+        data,
+        parent: currentSuite,
+        children: [],
+        hasFailedDescendant: false,
+      };
       if (originalSuite?.children) {
         ArrayPrototypePush(originalSuite.children, currentSuite);
       }
@@ -57,7 +63,11 @@ function reportReruns(previousRuns, globalOptions) {
         } else {
           disambiguator[baseIdentifier] = 1;
         }
-        if (type === 'test:pass') {
+        if (type === 'test:fail') {
+          for (let t = currentTest.parent; t !== null; t = t.parent) {
+            t.hasFailedDescendant = true;
+          }
+        } else if (!currentTest.hasFailedDescendant) {
           const children = ArrayPrototypeMap(currentTest.children, (child) => child.data);
           obj[identifier] = {
             __proto__: null,

test/fixtures/test-runner/rerun-todo.js

@@ -0,0 +1,21 @@
+const { test, describe } = require('node:test');
+
+test('top-level todo fails on first attempt', { todo: true }, ({ attempt }) => {
+  if (attempt < 1) {
+    throw new Error('This todo test is expected to fail on the first attempt');
+  }
+});
+
+describe('suite with failing todo', () => {
+  test('nested todo fails on first attempt', { todo: true }, ({ attempt }) => {
+    if (attempt < 1) {
+      throw new Error('This todo test is expected to fail on the first attempt');
+    }
+  });
+
+  test('ok', ({ attempt }) => {
+    if (attempt > 0) {
+      throw new Error('Test should not rerun once it has passed');
+    }
+  });
+});

test/parallel/test-runner-test-rerun-failures.js

@@ -283,3 +283,73 @@ test('using `run` api', async () => {
   await setTimeout(common.platformTimeout(10)); // Wait for the stream to finish processing
   assert.deepStrictEqual(await getStateFile(), expectedStateFile);
 });
+
+test('failing todo test is rerun even when its parent suite passed', async () => {
+  const fixturePath = fixtures.path('test-runner', 'rerun-todo.js');
+  const fixtureKey = relative(process.cwd(), fixturePath).replaceAll('\\', '/');
+  const topLevelTodo = `${fixtureKey}:3:1`;
+  const suite = `${fixtureKey}:9:1`;
+  const nestedTodo = `${fixtureKey}:10:3`;
+  const ok = `${fixtureKey}:16:3`;
+
+  async function runAttempt() {
+    const results = { __proto__: null, passed: [], failed: [], summaries: [] };
+    const stream = run({ files: [fixturePath], rerunFailuresFilePath: stateFile });
+    for await (const { type, data } of stream) {
+      const { name, todo, details } = data;
+      if (type === 'test:pass') {
+        results.passed.push({ name, todo, attempt: details.attempt, passed_on_attempt: details.passed_on_attempt });
+      } else if (type === 'test:fail') {
+        results.failed.push({ name, todo, attempt: details.attempt });
+      } else if (type === 'test:summary') {
+        results.summaries.push(data);
+      }
+    }
+    await setTimeout(common.platformTimeout(10)); // Wait for the stream to finish processing
+    return results;
+  }
+
+  let results = await runAttempt();
+  assert.deepStrictEqual(results.failed, [
+    { name: 'top-level todo fails on first attempt', todo: true, attempt: 0 },
+    { name: 'nested todo fails on first attempt', todo: true, attempt: 0 },
+  ]);
+  assert.deepStrictEqual(results.passed, [
+    { name: 'ok', todo: undefined, attempt: 0, passed_on_attempt: undefined },
+    { name: 'suite with failing todo', todo: undefined, attempt: 0, passed_on_attempt: undefined },
+  ]);
+  for (const { success, counts } of results.summaries) {
+    assert.strictEqual(success, true);
+    assert.strictEqual(counts.failed, 0);
+    assert.strictEqual(counts.passed, 1);
+    assert.strictEqual(counts.todo, 2);
+  }
+  let state = await getStateFile();
+  assert.strictEqual(state.length, 1);
+  assert.deepStrictEqual(state[0], {
+    [ok]: { passed_on_attempt: 0, name: 'ok' },
+  });
+
+  results = await runAttempt();
+  assert.deepStrictEqual(results.failed, []);
+  assert.deepStrictEqual(results.passed, [
+    { name: 'top-level todo fails on first attempt', todo: true, attempt: 1, passed_on_attempt: undefined },
+    { name: 'nested todo fails on first attempt', todo: true, attempt: 1, passed_on_attempt: undefined },
+    { name: 'ok', todo: undefined, attempt: 1, passed_on_attempt: 0 },
+    { name: 'suite with failing todo', todo: undefined, attempt: 1, passed_on_attempt: undefined },
+  ]);
+  for (const { success, counts } of results.summaries) {
+    assert.strictEqual(success, true);
+    assert.strictEqual(counts.failed, 0);
+    assert.strictEqual(counts.passed, 1);
+    assert.strictEqual(counts.todo, 2);
+  }
+  state = await getStateFile();
+  assert.strictEqual(state.length, 2);
+  assert.deepStrictEqual(state[1], {
+    [topLevelTodo]: { passed_on_attempt: 1, name: 'top-level todo fails on first attempt' },
+    [nestedTodo]: { passed_on_attempt: 1, name: 'nested todo fails on first attempt' },
+    [ok]: { passed_on_attempt: 0, name: 'ok' },
+    [suite]: { passed_on_attempt: 1, name: 'suite with failing todo' },
+  });
+});