test: rerun failed tests individually (#48387)

* test: rerun failed tests individually

* rerun test up to 3 times

* test: fixup navigationHistory flake

(cherry picked from commit fa6431c368918f7439705558f6d2e08875ac4ad5)

.github/workflows/pipeline-segment-electron-test.yml

@@ -215,7 +215,7 @@ jobs:
               export ELECTRON_FORCE_TEST_SUITE_EXIT="true"
             fi
           fi
-          node script/yarn test --runners=main --trace-uncaught --enable-logging --files $tests_files
+          node script/yarn test --runners=main --enableRerun=3 --trace-uncaught --enable-logging --files $tests_files
         else
           chown :builduser .. && chmod g+w ..
           chown -R :builduser . && chmod -R g+w .

package.json

@@ -20,6 +20,7 @@
     "@types/temp": "^0.9.4",
     "@typescript-eslint/eslint-plugin": "^8.32.1",
     "@typescript-eslint/parser": "^8.7.0",
+    "@xmldom/xmldom": "^0.8.11",
     "buffer": "^6.0.3",
     "chalk": "^4.1.0",
     "check-for-leaks": "^1.2.1",

script/spec-runner.js

@@ -2,6 +2,7 @@
 
 const { ElectronVersions, Installer } = require('@electron/fiddle-core');
 
+const { DOMParser } = require('@xmldom/xmldom');
 const chalk = require('chalk');
 const { hashElement } = require('folder-hash');
 const minimist = require('minimist');
@@ -21,6 +22,7 @@ const FAILURE_STATUS_KEY = 'Electron_Spec_Runner_Failures';
 
 const args = minimist(process.argv, {
   string: ['runners', 'target', 'electronVersion'],
+  number: ['enableRerun'],
   unknown: arg => unknownFlags.push(arg)
 });
 
@@ -191,7 +193,160 @@ async function asyncSpawn (exe, runnerArgs) {
   });
 }
 
-async function runTestUsingElectron (specDir, testName) {
+function parseJUnitXML (specDir) {
+  if (!fs.existsSync(process.env.MOCHA_FILE)) {
+    console.error('JUnit XML file not found:', process.env.MOCHA_FILE);
+    return [];
+  }
+
+  const xmlContent = fs.readFileSync(process.env.MOCHA_FILE, 'utf8');
+  const parser = new DOMParser();
+  const xmlDoc = parser.parseFromString(xmlContent, 'text/xml');
+
+  const failedTests = [];
+  // find failed tests by looking for all testsuite nodes with failure > 0
+  const testSuites = xmlDoc.getElementsByTagName('testsuite');
+  for (let i = 0; i < testSuites.length; i++) {
+    const testSuite = testSuites[i];
+    const failures = testSuite.getAttribute('failures');
+    if (failures > 0) {
+      const testcases = testSuite.getElementsByTagName('testcase');
+
+      for (let i = 0; i < testcases.length; i++) {
+        const testcase = testcases[i];
+        const failures = testcase.getElementsByTagName('failure');
+        const errors = testcase.getElementsByTagName('error');
+
+        if (failures.length > 0 || errors.length > 0) {
+          const testName = testcase.getAttribute('name');
+          const filePath = testSuite.getAttribute('file');
+          const fileName = filePath ? path.relative(specDir, filePath) : 'unknown file';
+          const failureInfo = {
+            name: testName,
+            file: fileName,
+            filePath
+          };
+          if (failures.length > 0) {
+            failureInfo.failure = failures[0].textContent || failures[0].nodeValue || 'No failure message';
+          }
+
+          if (errors.length > 0) {
+            failureInfo.error = errors[0].textContent || errors[0].nodeValue || 'No error message';
+          }
+
+          failedTests.push(failureInfo);
+        }
+      }
+    }
+  }
+
+  return failedTests;
+}
+
+async function rerunFailedTest (specDir, testName, testInfo) {
+  console.log('\n========================================');
+  console.log(`Rerunning failed test: ${testInfo.name} (${testInfo.file})`);
+  console.log('========================================');
+
+  let grepPattern = testInfo.name;
+
+  // Escape special regex characters in test name
+  grepPattern = grepPattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+  const args = [];
+  if (testInfo.filePath) {
+    args.push('--files', testInfo.filePath);
+  }
+  args.push('-g', grepPattern);
+
+  const success = await runTestUsingElectron(specDir, testName, false, args);
+
+  if (success) {
+    console.log(`✅ Test passed: ${testInfo.name}`);
+    return true;
+  } else {
+    console.log(`❌ Test failed again: ${testInfo.name}`);
+    return false;
+  }
+}
+
+async function rerunFailedTests (specDir, testName) {
+  console.log('\n📋 Parsing JUnit XML for failed tests...');
+  const failedTests = parseJUnitXML(specDir);
+
+  if (failedTests.length === 0) {
+    console.log('No failed tests could be found.');
+    process.exit(1);
+    return;
+  }
+
+  // Save off the original junit xml file
+  if (fs.existsSync(process.env.MOCHA_FILE)) {
+    fs.copyFileSync(process.env.MOCHA_FILE, `${process.env.MOCHA_FILE}.save`);
+  }
+
+  console.log(`\n📊 Found ${failedTests.length} failed test(s):`);
+  failedTests.forEach((test, index) => {
+    console.log(`  ${index + 1}. ${test.name} (${test.file})`);
+  });
+
+  // Step 3: Rerun each failed test individually
+  console.log('\n🔄 Rerunning failed tests individually...\n');
+
+  const results = {
+    total: failedTests.length,
+    passed: 0,
+    failed: 0
+  };
+
+  let index = 0;
+  for (const testInfo of failedTests) {
+    let runCount = 0;
+    let success = false;
+    let retryTest = false;
+    while (!success && (runCount < args.enableRerun)) {
+      success = await rerunFailedTest(specDir, testName, testInfo);
+      if (success) {
+        results.passed++;
+      } else {
+        if (runCount === args.enableRerun - 1) {
+          results.failed++;
+        } else {
+          retryTest = true;
+          console.log(`Retrying test (${runCount + 1}/${args.enableRerun})...`);
+        }
+      }
+
+      // Add a small delay between tests
+      if (retryTest || index < failedTests.length - 1) {
+        console.log('\nWaiting 2 seconds before next test...');
+        await new Promise(resolve => setTimeout(resolve, 2000));
+      }
+      runCount++;
+    }
+    index++;
+  };
+
+  // Step 4: Summary
+  console.log('\n📈 Summary:');
+  console.log(`Total failed tests: ${results.total}`);
+  console.log(`Passed on rerun: ${results.passed}`);
+  console.log(`Still failing: ${results.failed}`);
+
+  // Restore the original junit xml file
+  if (fs.existsSync(`${process.env.MOCHA_FILE}.save`)) {
+    fs.renameSync(`${process.env.MOCHA_FILE}.save`, process.env.MOCHA_FILE);
+  }
+
+  if (results.failed === 0) {
+    console.log('🎉 All previously failed tests now pass!');
+  } else {
+    console.log(`⚠️  ${results.failed} test(s) are still failing`);
+    process.exit(1);
+  }
+}
+
+async function runTestUsingElectron (specDir, testName, shouldRerun, additionalArgs = []) {
   let exe;
   if (args.electronVersion) {
     const installer = new Installer();
@@ -199,11 +354,16 @@ async function runTestUsingElectron (specDir, testName) {
   } else {
     exe = path.resolve(BASE, utils.getElectronExec());
   }
-  const runnerArgs = [`electron/${specDir}`, ...unknownArgs.slice(2)];
+  let argsToPass = unknownArgs.slice(2);
+  if (additionalArgs.includes('--files')) {
+    argsToPass = argsToPass.filter(arg => (arg.toString().indexOf('--files') === -1 && arg.toString().indexOf('spec/') === -1));
+  }
+  const runnerArgs = [`electron/${specDir}`, ...argsToPass, ...additionalArgs];
   if (process.platform === 'linux') {
     runnerArgs.unshift(path.resolve(__dirname, 'dbus_mock.py'), exe);
     exe = 'python3';
   }
+  console.log(`Running: ${exe} ${runnerArgs.join(' ')}`);
   const { status, signal } = await asyncSpawn(exe, runnerArgs);
   if (status !== 0) {
     if (status) {
@@ -212,13 +372,22 @@ async function runTestUsingElectron (specDir, testName) {
     } else {
       console.log(`${fail} Electron tests failed with kill signal ${signal}.`);
     }
-    process.exit(1);
+    if (shouldRerun) {
+      await rerunFailedTests(specDir, testName);
+    } else {
+      return false;
+    }
   }
   console.log(`${pass} Electron ${testName} process tests passed.`);
+  return true;
 }
 
 async function runMainProcessElectronTests () {
-  await runTestUsingElectron('spec', 'main');
+  let shouldRerun = false;
+  if (args.enableRerun && args.enableRerun > 0) {
+    shouldRerun = true;
+  }
+  await runTestUsingElectron('spec', 'main', shouldRerun);
 }
 
 async function installSpecModules (dir) {

spec/api-web-contents-spec.ts

@@ -901,7 +901,7 @@ describe('webContents module', () => {
           return w.webContents.navigationHistory.restore({ index: 2, entries });
         });
 
-        expect(formValue).to.equal('Hi!');
+        await waitUntil(() => formValue === 'Hi!');
       });
 
       it('should handle invalid base64 pageState', async () => {

yarn.lock

@@ -1325,6 +1325,11 @@
   resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-2.0.5.tgz#325db42395cd49fe6c14057f9a900e427df8810e"
   integrity sha512-lqaoKnRYBdo1UgDX8uF24AfGMifWK19TxPmM5FHc2vAGxrJ/qtyUyFBWoY1tISZdelsQ5fBcOusifo5o5wSJxQ==
 
+"@xmldom/xmldom@^0.8.11":
+  version "0.8.11"
+  resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.8.11.tgz#b79de2d67389734c57c52595f7a7305e30c2d608"
+  integrity sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==
+
 "@xtuc/ieee754@^1.2.0":
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790"
@@ -7206,14 +7211,7 @@ string_decoder@~1.1.1:
   dependencies:
     safe-buffer "~5.1.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==