test: rerun failed tests individually (#48387)
* test: rerun failed tests individually * rerun test up to 3 times * test: fixup navigationHistory flake (cherry picked from commit fa6431c368918f7439705558f6d2e08875ac4ad5)
This commit is contained in:
John Kleinschmidt
GitHub
parent
3179f61bc1
commit
2982cd77f0
5 changed files with 182 additions and 14 deletions
|
|
@ -215,7 +215,7 @@ jobs:
|
|||
export ELECTRON_FORCE_TEST_SUITE_EXIT="true"
|
||||
fi
|
||||
fi
|
||||
node script/yarn test --runners=main --trace-uncaught --enable-logging --files $tests_files
|
||||
node script/yarn test --runners=main --enableRerun=3 --trace-uncaught --enable-logging --files $tests_files
|
||||
else
|
||||
chown :builduser .. && chmod g+w ..
|
||||
chown -R :builduser . && chmod -R g+w .
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
"@types/temp": "^0.9.4",
|
||||
"@typescript-eslint/eslint-plugin": "^8.32.1",
|
||||
"@typescript-eslint/parser": "^8.7.0",
|
||||
"@xmldom/xmldom": "^0.8.11",
|
||||
"buffer": "^6.0.3",
|
||||
"chalk": "^4.1.0",
|
||||
"check-for-leaks": "^1.2.1",
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
const { ElectronVersions, Installer } = require('@electron/fiddle-core');
|
||||
|
||||
const { DOMParser } = require('@xmldom/xmldom');
|
||||
const chalk = require('chalk');
|
||||
const { hashElement } = require('folder-hash');
|
||||
const minimist = require('minimist');
|
||||
|
|
@ -21,6 +22,7 @@ const FAILURE_STATUS_KEY = 'Electron_Spec_Runner_Failures';
|
|||
|
||||
const args = minimist(process.argv, {
|
||||
string: ['runners', 'target', 'electronVersion'],
|
||||
number: ['enableRerun'],
|
||||
unknown: arg => unknownFlags.push(arg)
|
||||
});
|
||||
|
||||
|
|
@ -191,7 +193,160 @@ async function asyncSpawn (exe, runnerArgs) {
|
|||
});
|
||||
}
|
||||
|
||||
async function runTestUsingElectron (specDir, testName) {
|
||||
function parseJUnitXML (specDir) {
|
||||
if (!fs.existsSync(process.env.MOCHA_FILE)) {
|
||||
console.error('JUnit XML file not found:', process.env.MOCHA_FILE);
|
||||
return [];
|
||||
}
|
||||
|
||||
const xmlContent = fs.readFileSync(process.env.MOCHA_FILE, 'utf8');
|
||||
const parser = new DOMParser();
|
||||
const xmlDoc = parser.parseFromString(xmlContent, 'text/xml');
|
||||
|
||||
const failedTests = [];
|
||||
// find failed tests by looking for all testsuite nodes with failure > 0
|
||||
const testSuites = xmlDoc.getElementsByTagName('testsuite');
|
||||
for (let i = 0; i < testSuites.length; i++) {
|
||||
const testSuite = testSuites[i];
|
||||
const failures = testSuite.getAttribute('failures');
|
||||
if (failures > 0) {
|
||||
const testcases = testSuite.getElementsByTagName('testcase');
|
||||
|
||||
for (let i = 0; i < testcases.length; i++) {
|
||||
const testcase = testcases[i];
|
||||
const failures = testcase.getElementsByTagName('failure');
|
||||
const errors = testcase.getElementsByTagName('error');
|
||||
|
||||
if (failures.length > 0 || errors.length > 0) {
|
||||
const testName = testcase.getAttribute('name');
|
||||
const filePath = testSuite.getAttribute('file');
|
||||
const fileName = filePath ? path.relative(specDir, filePath) : 'unknown file';
|
||||
const failureInfo = {
|
||||
name: testName,
|
||||
file: fileName,
|
||||
filePath
|
||||
};
|
||||
if (failures.length > 0) {
|
||||
failureInfo.failure = failures[0].textContent || failures[0].nodeValue || 'No failure message';
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
failureInfo.error = errors[0].textContent || errors[0].nodeValue || 'No error message';
|
||||
}
|
||||
|
||||
failedTests.push(failureInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return failedTests;
|
||||
}
|
||||
|
||||
async function rerunFailedTest (specDir, testName, testInfo) {
|
||||
console.log('\n========================================');
|
||||
console.log(`Rerunning failed test: ${testInfo.name} (${testInfo.file})`);
|
||||
console.log('========================================');
|
||||
|
||||
let grepPattern = testInfo.name;
|
||||
|
||||
// Escape special regex characters in test name
|
||||
grepPattern = grepPattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
const args = [];
|
||||
if (testInfo.filePath) {
|
||||
args.push('--files', testInfo.filePath);
|
||||
}
|
||||
args.push('-g', grepPattern);
|
||||
|
||||
const success = await runTestUsingElectron(specDir, testName, false, args);
|
||||
|
||||
if (success) {
|
||||
console.log(`✅ Test passed: ${testInfo.name}`);
|
||||
return true;
|
||||
} else {
|
||||
console.log(`❌ Test failed again: ${testInfo.name}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function rerunFailedTests (specDir, testName) {
|
||||
console.log('\n📋 Parsing JUnit XML for failed tests...');
|
||||
const failedTests = parseJUnitXML(specDir);
|
||||
|
||||
if (failedTests.length === 0) {
|
||||
console.log('No failed tests could be found.');
|
||||
process.exit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
// Save off the original junit xml file
|
||||
if (fs.existsSync(process.env.MOCHA_FILE)) {
|
||||
fs.copyFileSync(process.env.MOCHA_FILE, `${process.env.MOCHA_FILE}.save`);
|
||||
}
|
||||
|
||||
console.log(`\n📊 Found ${failedTests.length} failed test(s):`);
|
||||
failedTests.forEach((test, index) => {
|
||||
console.log(` ${index + 1}. ${test.name} (${test.file})`);
|
||||
});
|
||||
|
||||
// Step 3: Rerun each failed test individually
|
||||
console.log('\n🔄 Rerunning failed tests individually...\n');
|
||||
|
||||
const results = {
|
||||
total: failedTests.length,
|
||||
passed: 0,
|
||||
failed: 0
|
||||
};
|
||||
|
||||
let index = 0;
|
||||
for (const testInfo of failedTests) {
|
||||
let runCount = 0;
|
||||
let success = false;
|
||||
let retryTest = false;
|
||||
while (!success && (runCount < args.enableRerun)) {
|
||||
success = await rerunFailedTest(specDir, testName, testInfo);
|
||||
if (success) {
|
||||
results.passed++;
|
||||
} else {
|
||||
if (runCount === args.enableRerun - 1) {
|
||||
results.failed++;
|
||||
} else {
|
||||
retryTest = true;
|
||||
console.log(`Retrying test (${runCount + 1}/${args.enableRerun})...`);
|
||||
}
|
||||
}
|
||||
|
||||
// Add a small delay between tests
|
||||
if (retryTest || index < failedTests.length - 1) {
|
||||
console.log('\nWaiting 2 seconds before next test...');
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
}
|
||||
runCount++;
|
||||
}
|
||||
index++;
|
||||
};
|
||||
|
||||
// Step 4: Summary
|
||||
console.log('\n📈 Summary:');
|
||||
console.log(`Total failed tests: ${results.total}`);
|
||||
console.log(`Passed on rerun: ${results.passed}`);
|
||||
console.log(`Still failing: ${results.failed}`);
|
||||
|
||||
// Restore the original junit xml file
|
||||
if (fs.existsSync(`${process.env.MOCHA_FILE}.save`)) {
|
||||
fs.renameSync(`${process.env.MOCHA_FILE}.save`, process.env.MOCHA_FILE);
|
||||
}
|
||||
|
||||
if (results.failed === 0) {
|
||||
console.log('🎉 All previously failed tests now pass!');
|
||||
} else {
|
||||
console.log(`⚠️ ${results.failed} test(s) are still failing`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async function runTestUsingElectron (specDir, testName, shouldRerun, additionalArgs = []) {
|
||||
let exe;
|
||||
if (args.electronVersion) {
|
||||
const installer = new Installer();
|
||||
|
|
@ -199,11 +354,16 @@ async function runTestUsingElectron (specDir, testName) {
|
|||
} else {
|
||||
exe = path.resolve(BASE, utils.getElectronExec());
|
||||
}
|
||||
const runnerArgs = [`electron/${specDir}`, ...unknownArgs.slice(2)];
|
||||
let argsToPass = unknownArgs.slice(2);
|
||||
if (additionalArgs.includes('--files')) {
|
||||
argsToPass = argsToPass.filter(arg => (arg.toString().indexOf('--files') === -1 && arg.toString().indexOf('spec/') === -1));
|
||||
}
|
||||
const runnerArgs = [`electron/${specDir}`, ...argsToPass, ...additionalArgs];
|
||||
if (process.platform === 'linux') {
|
||||
runnerArgs.unshift(path.resolve(__dirname, 'dbus_mock.py'), exe);
|
||||
exe = 'python3';
|
||||
}
|
||||
console.log(`Running: ${exe} ${runnerArgs.join(' ')}`);
|
||||
const { status, signal } = await asyncSpawn(exe, runnerArgs);
|
||||
if (status !== 0) {
|
||||
if (status) {
|
||||
|
|
@ -212,13 +372,22 @@ async function runTestUsingElectron (specDir, testName) {
|
|||
} else {
|
||||
console.log(`${fail} Electron tests failed with kill signal ${signal}.`);
|
||||
}
|
||||
process.exit(1);
|
||||
if (shouldRerun) {
|
||||
await rerunFailedTests(specDir, testName);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
console.log(`${pass} Electron ${testName} process tests passed.`);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function runMainProcessElectronTests () {
|
||||
await runTestUsingElectron('spec', 'main');
|
||||
let shouldRerun = false;
|
||||
if (args.enableRerun && args.enableRerun > 0) {
|
||||
shouldRerun = true;
|
||||
}
|
||||
await runTestUsingElectron('spec', 'main', shouldRerun);
|
||||
}
|
||||
|
||||
async function installSpecModules (dir) {
|
||||
|
|
|
|||
|
|
@ -901,7 +901,7 @@ describe('webContents module', () => {
|
|||
return w.webContents.navigationHistory.restore({ index: 2, entries });
|
||||
});
|
||||
|
||||
expect(formValue).to.equal('Hi!');
|
||||
await waitUntil(() => formValue === 'Hi!');
|
||||
});
|
||||
|
||||
it('should handle invalid base64 pageState', async () => {
|
||||
|
|
|
|||
14
yarn.lock
14
yarn.lock
|
|
@ -1325,6 +1325,11 @@
|
|||
resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-2.0.5.tgz#325db42395cd49fe6c14057f9a900e427df8810e"
|
||||
integrity sha512-lqaoKnRYBdo1UgDX8uF24AfGMifWK19TxPmM5FHc2vAGxrJ/qtyUyFBWoY1tISZdelsQ5fBcOusifo5o5wSJxQ==
|
||||
|
||||
"@xmldom/xmldom@^0.8.11":
|
||||
version "0.8.11"
|
||||
resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.8.11.tgz#b79de2d67389734c57c52595f7a7305e30c2d608"
|
||||
integrity sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==
|
||||
|
||||
"@xtuc/ieee754@^1.2.0":
|
||||
version "1.2.0"
|
||||
resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790"
|
||||
|
|
@ -7206,14 +7211,7 @@ string_decoder@~1.1.1:
|
|||
dependencies:
|
||||
safe-buffer "~5.1.0"
|
||||
|
||||
"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
|
||||
version "6.0.1"
|
||||
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
|
||||
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
|
||||
dependencies:
|
||||
ansi-regex "^5.0.1"
|
||||
|
||||
strip-ansi@^6.0.0, strip-ansi@^6.0.1:
|
||||
"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
|
||||
version "6.0.1"
|
||||
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
|
||||
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue