From 2982cd77f07e5714e9d949b80c42a025deb78b49 Mon Sep 17 00:00:00 2001 From: John Kleinschmidt Date: Sat, 27 Sep 2025 13:32:10 -0400 Subject: [PATCH] test: rerun failed tests individually (#48387) * test: rerun failed tests individually * rerun test up to 3 times * test: fixup navigationHistory flake (cherry picked from commit fa6431c368918f7439705558f6d2e08875ac4ad5) --- .../pipeline-segment-electron-test.yml | 2 +- package.json | 1 + script/spec-runner.js | 177 +++++++++++++++++- spec/api-web-contents-spec.ts | 2 +- yarn.lock | 14 +- 5 files changed, 182 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pipeline-segment-electron-test.yml b/.github/workflows/pipeline-segment-electron-test.yml index 4e440888a7e..4d6803ecc35 100644 --- a/.github/workflows/pipeline-segment-electron-test.yml +++ b/.github/workflows/pipeline-segment-electron-test.yml @@ -215,7 +215,7 @@ jobs: export ELECTRON_FORCE_TEST_SUITE_EXIT="true" fi fi - node script/yarn test --runners=main --trace-uncaught --enable-logging --files $tests_files + node script/yarn test --runners=main --enableRerun=3 --trace-uncaught --enable-logging --files $tests_files else chown :builduser .. && chmod g+w .. chown -R :builduser . && chmod -R g+w . diff --git a/package.json b/package.json index 897bd1e1a5c..78cec6e7f34 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "@types/temp": "^0.9.4", "@typescript-eslint/eslint-plugin": "^8.32.1", "@typescript-eslint/parser": "^8.7.0", + "@xmldom/xmldom": "^0.8.11", "buffer": "^6.0.3", "chalk": "^4.1.0", "check-for-leaks": "^1.2.1", diff --git a/script/spec-runner.js b/script/spec-runner.js index 3457fdb34a9..4c5e72acac0 100755 --- a/script/spec-runner.js +++ b/script/spec-runner.js @@ -2,6 +2,7 @@ const { ElectronVersions, Installer } = require('@electron/fiddle-core'); +const { DOMParser } = require('@xmldom/xmldom'); const chalk = require('chalk'); const { hashElement } = require('folder-hash'); const minimist = require('minimist'); @@ -21,6 +22,7 @@ const FAILURE_STATUS_KEY = 'Electron_Spec_Runner_Failures'; const args = minimist(process.argv, { string: ['runners', 'target', 'electronVersion'], + number: ['enableRerun'], unknown: arg => unknownFlags.push(arg) }); @@ -191,7 +193,160 @@ async function asyncSpawn (exe, runnerArgs) { }); } -async function runTestUsingElectron (specDir, testName) { +function parseJUnitXML (specDir) { + if (!fs.existsSync(process.env.MOCHA_FILE)) { + console.error('JUnit XML file not found:', process.env.MOCHA_FILE); + return []; + } + + const xmlContent = fs.readFileSync(process.env.MOCHA_FILE, 'utf8'); + const parser = new DOMParser(); + const xmlDoc = parser.parseFromString(xmlContent, 'text/xml'); + + const failedTests = []; + // find failed tests by looking for all testsuite nodes with failure > 0 + const testSuites = xmlDoc.getElementsByTagName('testsuite'); + for (let i = 0; i < testSuites.length; i++) { + const testSuite = testSuites[i]; + const failures = testSuite.getAttribute('failures'); + if (failures > 0) { + const testcases = testSuite.getElementsByTagName('testcase'); + + for (let i = 0; i < testcases.length; i++) { + const testcase = testcases[i]; + const failures = testcase.getElementsByTagName('failure'); + const errors = testcase.getElementsByTagName('error'); + + if (failures.length > 0 || errors.length > 0) { + const testName = testcase.getAttribute('name'); + const filePath = testSuite.getAttribute('file'); + const fileName = filePath ? path.relative(specDir, filePath) : 'unknown file'; + const failureInfo = { + name: testName, + file: fileName, + filePath + }; + if (failures.length > 0) { + failureInfo.failure = failures[0].textContent || failures[0].nodeValue || 'No failure message'; + } + + if (errors.length > 0) { + failureInfo.error = errors[0].textContent || errors[0].nodeValue || 'No error message'; + } + + failedTests.push(failureInfo); + } + } + } + } + + return failedTests; +} + +async function rerunFailedTest (specDir, testName, testInfo) { + console.log('\n========================================'); + console.log(`Rerunning failed test: ${testInfo.name} (${testInfo.file})`); + console.log('========================================'); + + let grepPattern = testInfo.name; + + // Escape special regex characters in test name + grepPattern = grepPattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + + const args = []; + if (testInfo.filePath) { + args.push('--files', testInfo.filePath); + } + args.push('-g', grepPattern); + + const success = await runTestUsingElectron(specDir, testName, false, args); + + if (success) { + console.log(`āœ… Test passed: ${testInfo.name}`); + return true; + } else { + console.log(`āŒ Test failed again: ${testInfo.name}`); + return false; + } +} + +async function rerunFailedTests (specDir, testName) { + console.log('\nšŸ“‹ Parsing JUnit XML for failed tests...'); + const failedTests = parseJUnitXML(specDir); + + if (failedTests.length === 0) { + console.log('No failed tests could be found.'); + process.exit(1); + return; + } + + // Save off the original junit xml file + if (fs.existsSync(process.env.MOCHA_FILE)) { + fs.copyFileSync(process.env.MOCHA_FILE, `${process.env.MOCHA_FILE}.save`); + } + + console.log(`\nšŸ“Š Found ${failedTests.length} failed test(s):`); + failedTests.forEach((test, index) => { + console.log(` ${index + 1}. ${test.name} (${test.file})`); + }); + + // Step 3: Rerun each failed test individually + console.log('\nšŸ”„ Rerunning failed tests individually...\n'); + + const results = { + total: failedTests.length, + passed: 0, + failed: 0 + }; + + let index = 0; + for (const testInfo of failedTests) { + let runCount = 0; + let success = false; + let retryTest = false; + while (!success && (runCount < args.enableRerun)) { + success = await rerunFailedTest(specDir, testName, testInfo); + if (success) { + results.passed++; + } else { + if (runCount === args.enableRerun - 1) { + results.failed++; + } else { + retryTest = true; + console.log(`Retrying test (${runCount + 1}/${args.enableRerun})...`); + } + } + + // Add a small delay between tests + if (retryTest || index < failedTests.length - 1) { + console.log('\nWaiting 2 seconds before next test...'); + await new Promise(resolve => setTimeout(resolve, 2000)); + } + runCount++; + } + index++; + }; + + // Step 4: Summary + console.log('\nšŸ“ˆ Summary:'); + console.log(`Total failed tests: ${results.total}`); + console.log(`Passed on rerun: ${results.passed}`); + console.log(`Still failing: ${results.failed}`); + + // Restore the original junit xml file + if (fs.existsSync(`${process.env.MOCHA_FILE}.save`)) { + fs.renameSync(`${process.env.MOCHA_FILE}.save`, process.env.MOCHA_FILE); + } + + if (results.failed === 0) { + console.log('šŸŽ‰ All previously failed tests now pass!'); + } else { + console.log(`āš ļø ${results.failed} test(s) are still failing`); + process.exit(1); + } +} + +async function runTestUsingElectron (specDir, testName, shouldRerun, additionalArgs = []) { let exe; if (args.electronVersion) { const installer = new Installer(); @@ -199,11 +354,16 @@ async function runTestUsingElectron (specDir, testName) { } else { exe = path.resolve(BASE, utils.getElectronExec()); } - const runnerArgs = [`electron/${specDir}`, ...unknownArgs.slice(2)]; + let argsToPass = unknownArgs.slice(2); + if (additionalArgs.includes('--files')) { + argsToPass = argsToPass.filter(arg => (arg.toString().indexOf('--files') === -1 && arg.toString().indexOf('spec/') === -1)); + } + const runnerArgs = [`electron/${specDir}`, ...argsToPass, ...additionalArgs]; if (process.platform === 'linux') { runnerArgs.unshift(path.resolve(__dirname, 'dbus_mock.py'), exe); exe = 'python3'; } + console.log(`Running: ${exe} ${runnerArgs.join(' ')}`); const { status, signal } = await asyncSpawn(exe, runnerArgs); if (status !== 0) { if (status) { @@ -212,13 +372,22 @@ async function runTestUsingElectron (specDir, testName) { } else { console.log(`${fail} Electron tests failed with kill signal ${signal}.`); } - process.exit(1); + if (shouldRerun) { + await rerunFailedTests(specDir, testName); + } else { + return false; + } } console.log(`${pass} Electron ${testName} process tests passed.`); + return true; } async function runMainProcessElectronTests () { - await runTestUsingElectron('spec', 'main'); + let shouldRerun = false; + if (args.enableRerun && args.enableRerun > 0) { + shouldRerun = true; + } + await runTestUsingElectron('spec', 'main', shouldRerun); } async function installSpecModules (dir) { diff --git a/spec/api-web-contents-spec.ts b/spec/api-web-contents-spec.ts index 442743cb45d..fc4b3314fbf 100644 --- a/spec/api-web-contents-spec.ts +++ b/spec/api-web-contents-spec.ts @@ -901,7 +901,7 @@ describe('webContents module', () => { return w.webContents.navigationHistory.restore({ index: 2, entries }); }); - expect(formValue).to.equal('Hi!'); + await waitUntil(() => formValue === 'Hi!'); }); it('should handle invalid base64 pageState', async () => { diff --git a/yarn.lock b/yarn.lock index 2effe90e4bb..06b1b0a79f7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1325,6 +1325,11 @@ resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-2.0.5.tgz#325db42395cd49fe6c14057f9a900e427df8810e" integrity sha512-lqaoKnRYBdo1UgDX8uF24AfGMifWK19TxPmM5FHc2vAGxrJ/qtyUyFBWoY1tISZdelsQ5fBcOusifo5o5wSJxQ== +"@xmldom/xmldom@^0.8.11": + version "0.8.11" + resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.8.11.tgz#b79de2d67389734c57c52595f7a7305e30c2d608" + integrity sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw== + "@xtuc/ieee754@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790" @@ -7206,14 +7211,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==