test: rerun failed tests individually (#48386)

test: rerun failed tests individually (#48205)

* test: rerun failed tests individually

* ci: use screencapture-nag-remover

Needed to bypass the popup message "bash" is requesting to bypass the system private window picker and directly access your screen and audio.

* Revert "chore: test with 1st quadrant of the window"

No longer needed because of the addition of the
screencapture-nag-remover script.

This reverts commit f4a7e04c0b399aa9a85532f2c11fa35ad45bf71c.

* test: fixup navigationHistory flake

* rerun test up to 3 times
This commit is contained in:
John Kleinschmidt 2025-09-26 11:43:31 -04:00 committed by GitHub
commit a59beb5570
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 487 additions and 27 deletions

View file

@ -110,11 +110,6 @@ jobs:
configure_sys_tccdb "$values"
fi
done
# Ref: https://github.com/getsentry/sentry-cocoa/blob/main/scripts/ci-enable-permissions.sh
if [ "$OSTYPE" = "darwin24" ]; then
defaults write ~/Library/Group\ Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist "/bin/bash" -date "3024-09-23 12:00:00 +0000"
fi
- name: Turn off the unexpectedly quit dialog on macOS
if: ${{ inputs.target-platform == 'macos' }}
run: defaults write com.apple.CrashReporter DialogType server
@ -127,6 +122,12 @@ jobs:
path: src/electron
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Turn off screenshot nag on macOS
if: ${{ inputs.target-platform == 'macos' }}
run: |
defaults write ~/Library/Group\ Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist "/bin/bash" -date "3024-09-23 12:00:00 +0000"
src/electron/script/actions/screencapture-nag-remover.sh -a $(which bash)
src/electron/script/actions/screencapture-nag-remover.sh -a /opt/hca/hosted-compute-agent
- name: Setup SSH Debugging
if: ${{ inputs.target-platform == 'macos' && (inputs.enable-ssh || env.ACTIONS_STEP_DEBUG == 'true') }}
uses: ./src/electron/.github/actions/ssh-debug
@ -227,7 +228,7 @@ jobs:
export ELECTRON_FORCE_TEST_SUITE_EXIT="true"
fi
fi
node script/yarn test --runners=main --trace-uncaught --enable-logging --files $tests_files
node script/yarn test --runners=main --enableRerun=3 --trace-uncaught --enable-logging --files $tests_files
else
chown :builduser .. && chmod g+w ..
chown -R :builduser . && chmod -R g+w .

View file

@ -20,6 +20,7 @@
"@types/temp": "^0.9.4",
"@typescript-eslint/eslint-plugin": "^8.32.1",
"@typescript-eslint/parser": "^8.7.0",
"@xmldom/xmldom": "^0.8.11",
"buffer": "^6.0.3",
"chalk": "^4.1.0",
"check-for-leaks": "^1.2.1",

View file

@ -0,0 +1,297 @@
#!/bin/bash
# From https://github.com/luckman212/screencapture-nag-remover
SELF='screencapture-nag-remover'
FQPN=$(realpath "$0")
PLIST="$HOME/Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist"
AGENT_PLIST="$HOME/Library/LaunchAgents/$SELF.plist"
MDM_PROFILE="$HOME/Downloads/macOS_15.1_DisableScreenCaptureAlerts.mobileconfig"
TCC_DB='/Library/Application Support/com.apple.TCC/TCC.db'
FUTURE=$(/bin/date -j -v+100y +"%Y-%m-%d %H:%M:%S +0000")
INTERVAL=86400 #run every 24h
IFS='.' read -r MAJ MIN _ < <(/usr/bin/sw_vers --productVersion)
if (( MAJ < 15 )); then
echo >&2 "this tool requires macOS 15 (Sequoia)"
exit
fi
_os_is_151_or_higher() {
(( MAJ >= 15 )) && (( MIN > 0 ))
}
_fda_settings() {
/usr/bin/open 'x-apple.systempreferences:com.apple.preference.security?Privacy_AllFiles'
}
_open_device_management() {
/usr/bin/open 'x-apple.systempreferences:com.apple.preferences.configurationprofiles'
}
_bundleid_to_name() {
local APP_NAME
APP_NAME=$(/usr/bin/mdfind kMDItemCFBundleIdentifier == "$1" 2>/dev/null)
echo "${APP_NAME##*/}"
}
_create_plist() {
cat <<-EOF 2>/dev/null >"$PLIST"
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
</dict>
</plist>
EOF
}
_bounce_daemons() {
/usr/bin/killall -HUP replayd
/usr/bin/killall -u "$USER" cfprefsd
}
_nagblock() {
local APP_NAME
if _os_is_151_or_higher; then
if [[ -z $1 ]]; then
echo >&2 "supply the bundle ID of the app"
return 1
fi
APP_NAME=$(_bundleid_to_name "$1")
echo "disabling nag for $1${APP_NAME:+ ($APP_NAME)}"
/usr/bin/defaults write "$PLIST" "$1" -dict \
kScreenCaptureApprovalLastAlerted -date "$FUTURE" \
kScreenCaptureApprovalLastUsed -date "$FUTURE"
(( c++ ))
else
if [[ -z $1 ]]; then
echo >&2 "supply complete pathname to the binary inside the app bundle"
return 1
fi
[[ -e $1 ]] || { echo >&2 "$1 does not exist"; return 1; }
IFS='/' read -ra PARTS <<< "$1"
for p in "${PARTS[@]}"; do
if [[ $p == *.app ]]; then
APP_NAME=$p
break
fi
done
echo "disabling nag for ${APP_NAME:-$1}"
/usr/bin/defaults write "$PLIST" "$1" -date "$FUTURE"
(( c++ ))
return 0
fi
}
_enum_apps() {
[[ -e $PLIST ]] || return 1
if _os_is_151_or_higher; then
/usr/bin/plutil -convert raw -o - -- "$PLIST"
else
/usr/bin/plutil -convert xml1 -o - -- "$PLIST" |
/usr/bin/sed -n "s/.*<key>\(.*\)<\/key>.*/\1/p"
fi
}
_generate_mdm_profile() {
UUID1=$(/usr/bin/uuidgen)
UUID2=$(/usr/bin/uuidgen)
/bin/cat <<EOF >"$MDM_PROFILE"
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>PayloadContent</key>
<array>
<dict>
<key>PayloadDisplayName</key>
<string>Restrictions</string>
<key>PayloadIdentifier</key>
<string>com.apple.applicationaccess.${UUID2}</string>
<key>PayloadType</key>
<string>com.apple.applicationaccess</string>
<key>PayloadUUID</key>
<string>${UUID2}</string>
<key>PayloadVersion</key>
<integer>1</integer>
<key>forceBypassScreenCaptureAlert</key>
<true/>
</dict>
</array>
<key>PayloadDescription</key>
<string>Disables additional screen capture alerts on macOS 15.1 or higher</string>
<key>PayloadDisplayName</key>
<string>DisableScreenCaptureAlert</string>
<key>PayloadIdentifier</key>
<string>com.apple.applicationaccess.forceBypassScreenCaptureAlert</string>
<key>PayloadScope</key>
<string>System</string>
<key>PayloadType</key>
<string>Configuration</string>
<key>PayloadUUID</key>
<string>${UUID1}</string>
<key>PayloadVersion</key>
<integer>1</integer>
<key>TargetDeviceType</key>
<integer>5</integer>
</dict>
</plist>
EOF
#Apple prohibits self-installing TCC profiles, they can only be pushed via MDM
#/usr/bin/open "$MDM_PROFILE"
#_open_device_management
echo "import ${MDM_PROFILE##*/} into your MDM to provision it"
/usr/bin/open -R "$MDM_PROFILE"
}
_uninstall_launchagent() {
/bin/launchctl bootout gui/$UID "$AGENT_PLIST" 2>/dev/null
/bin/rm 2>/dev/null "$AGENT_PLIST"
echo "uninstalled $SELF LaunchAgent"
}
_install_launchagent() {
_uninstall_launchagent &>/dev/null
read -r FDA_TEST < <(/usr/bin/sqlite3 "$TCC_DB" <<-EOS
SELECT COUNT(client)
FROM access
WHERE
client = '/bin/bash' AND
service = 'kTCCServiceSystemPolicyAllFiles' AND
auth_value = 2
EOS
)
if (( FDA_TEST == 0 )); then
/bin/cat <<-EOF >&2
┌──────────────────────────────────────────────────────────────────────────────────────┐
│ For the LaunchAgent to work properly, you must grant Full Disk Access to /bin/bash │
│ │
│ The Full Disk Access settings panel will now be opened. Press the (+) button near │
│ the bottom of the window, then press [⌘cmd + ⇧shift + g] and type '/bin/bash' and │
│ click Open to get it to appear in the app list. │
│ │
│ Once that's all done, run the --install command again. │
└──────────────────────────────────────────────────────────────────────────────────────┘
EOF
sleep 3
_fda_settings
return 1
fi
/bin/cat >"$AGENT_PLIST" <<-EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>$SELF.agent</string>
<key>ProgramArguments</key>
<array>
<string>/bin/bash</string>
<string>--norc</string>
<string>--noprofile</string>
<string>$FQPN</string>
</array>
<key>StandardErrorPath</key>
<string>/private/tmp/$SELF.stderr</string>
<key>StandardOutPath</key>
<string>/private/tmp/$SELF.stdout</string>
<key>StartInterval</key>
<integer>$INTERVAL</integer>
<key>WorkingDirectory</key>
<string>/private/tmp</string>
</dict>
</plist>
EOF
/bin/chmod 644 "$PLIST"
if /bin/launchctl bootstrap gui/$UID "$AGENT_PLIST"; then
echo "installed $SELF LaunchAgent"
fi
}
_manual_add_desc() {
if _os_is_151_or_higher ; then
echo "-a,--add <bundle_id> manually create an entry"
else
echo "-a,--add <path> manually create an entry (supply full path to binary)"
fi
}
case $1 in
-h|--help)
/bin/cat <<-EOF
a tool to help suppress macOS Sequoia's persistent ScreenCapture alerts
usage: ${0##*/} [args]
-r,--reveal show ${PLIST##*/} in Finder
-p,--print print current values
$(_manual_add_desc)
--reset initialize empty ${PLIST##*/}
--generate_profile generate configuration profile for use with your MDM server
--profiles opens Device Management in System Settings
--install install LaunchAgent to ensure alerts continue to be silenced
--uninstall remove LaunchAgent
EOF
if _os_is_151_or_higher; then /bin/cat <<-EOF
┌────────────────────────────────────────────────────────────────────────────────────┐
│ macOS 15.1 introduced an official method for suppressing ScreenCapture alerts │
for ALL apps on Macs enrolled in an MDM server (Jamf, Addigy, Mosyle etc). │
│ │
│ A configuration profile to enable this can be generated using --generate_profile │
└────────────────────────────────────────────────────────────────────────────────────┘
EOF
fi
exit
;;
-r|--reveal)
if [[ -e $PLIST ]]; then
/usr/bin/open -R "$PLIST"
else
/usr/bin/open "$(/usr/bin/dirname "$PLIST")"
fi
exit
;;
-p|--print)
if [[ -e $PLIST ]]; then
/usr/bin/plutil -p "$PLIST"
else
echo >&2 "${PLIST##*/} does not exist"
fi
exit
;;
--reset) _create_plist || echo >&2 "error, could not create ${PLIST##*/}"; exit;;
--generate_profile) _generate_mdm_profile; exit;;
--profiles) _open_device_management; exit;;
--install) _install_launchagent; exit;;
--uninstall) _uninstall_launchagent; exit;;
esac
[[ -e $PLIST ]] || _create_plist
if ! /usr/bin/touch "$PLIST" 2>/dev/null; then
if [[ -n $__CFBundleIdentifier ]]; then
TERMINAL_NAME=$(_bundleid_to_name "$__CFBundleIdentifier")
fi
echo >&2 "Full Disk Access permissions are missing${TERMINAL_NAME:+ for $TERMINAL_NAME}"
exit 1
fi
case $1 in
-a|--add)
_nagblock "$2"
_bounce_daemons
exit
;;
-*) echo >&2 "invalid arg: $1"; exit 1;;
esac
c=0
while read -r APP_PATH ; do
[[ -n $APP_PATH ]] || continue
_nagblock "$APP_PATH"
done < <(_enum_apps)
#bounce daemons if any changes were made so the new settings take effect
(( c > 0 )) && _bounce_daemons
exit 0

View file

@ -2,6 +2,7 @@
const { ElectronVersions, Installer } = require('@electron/fiddle-core');
const { DOMParser } = require('@xmldom/xmldom');
const chalk = require('chalk');
const { hashElement } = require('folder-hash');
const minimist = require('minimist');
@ -21,6 +22,7 @@ const FAILURE_STATUS_KEY = 'Electron_Spec_Runner_Failures';
const args = minimist(process.argv, {
string: ['runners', 'target', 'electronVersion'],
number: ['enableRerun'],
unknown: arg => unknownFlags.push(arg)
});
@ -191,7 +193,160 @@ async function asyncSpawn (exe, runnerArgs) {
});
}
async function runTestUsingElectron (specDir, testName) {
function parseJUnitXML (specDir) {
if (!fs.existsSync(process.env.MOCHA_FILE)) {
console.error('JUnit XML file not found:', process.env.MOCHA_FILE);
return [];
}
const xmlContent = fs.readFileSync(process.env.MOCHA_FILE, 'utf8');
const parser = new DOMParser();
const xmlDoc = parser.parseFromString(xmlContent, 'text/xml');
const failedTests = [];
// find failed tests by looking for all testsuite nodes with failure > 0
const testSuites = xmlDoc.getElementsByTagName('testsuite');
for (let i = 0; i < testSuites.length; i++) {
const testSuite = testSuites[i];
const failures = testSuite.getAttribute('failures');
if (failures > 0) {
const testcases = testSuite.getElementsByTagName('testcase');
for (let i = 0; i < testcases.length; i++) {
const testcase = testcases[i];
const failures = testcase.getElementsByTagName('failure');
const errors = testcase.getElementsByTagName('error');
if (failures.length > 0 || errors.length > 0) {
const testName = testcase.getAttribute('name');
const filePath = testSuite.getAttribute('file');
const fileName = filePath ? path.relative(specDir, filePath) : 'unknown file';
const failureInfo = {
name: testName,
file: fileName,
filePath
};
if (failures.length > 0) {
failureInfo.failure = failures[0].textContent || failures[0].nodeValue || 'No failure message';
}
if (errors.length > 0) {
failureInfo.error = errors[0].textContent || errors[0].nodeValue || 'No error message';
}
failedTests.push(failureInfo);
}
}
}
}
return failedTests;
}
async function rerunFailedTest (specDir, testName, testInfo) {
console.log('\n========================================');
console.log(`Rerunning failed test: ${testInfo.name} (${testInfo.file})`);
console.log('========================================');
let grepPattern = testInfo.name;
// Escape special regex characters in test name
grepPattern = grepPattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const args = [];
if (testInfo.filePath) {
args.push('--files', testInfo.filePath);
}
args.push('-g', grepPattern);
const success = await runTestUsingElectron(specDir, testName, false, args);
if (success) {
console.log(`✅ Test passed: ${testInfo.name}`);
return true;
} else {
console.log(`❌ Test failed again: ${testInfo.name}`);
return false;
}
}
async function rerunFailedTests (specDir, testName) {
console.log('\n📋 Parsing JUnit XML for failed tests...');
const failedTests = parseJUnitXML(specDir);
if (failedTests.length === 0) {
console.log('No failed tests could be found.');
process.exit(1);
return;
}
// Save off the original junit xml file
if (fs.existsSync(process.env.MOCHA_FILE)) {
fs.copyFileSync(process.env.MOCHA_FILE, `${process.env.MOCHA_FILE}.save`);
}
console.log(`\n📊 Found ${failedTests.length} failed test(s):`);
failedTests.forEach((test, index) => {
console.log(` ${index + 1}. ${test.name} (${test.file})`);
});
// Step 3: Rerun each failed test individually
console.log('\n🔄 Rerunning failed tests individually...\n');
const results = {
total: failedTests.length,
passed: 0,
failed: 0
};
let index = 0;
for (const testInfo of failedTests) {
let runCount = 0;
let success = false;
let retryTest = false;
while (!success && (runCount < args.enableRerun)) {
success = await rerunFailedTest(specDir, testName, testInfo);
if (success) {
results.passed++;
} else {
if (runCount === args.enableRerun - 1) {
results.failed++;
} else {
retryTest = true;
console.log(`Retrying test (${runCount + 1}/${args.enableRerun})...`);
}
}
// Add a small delay between tests
if (retryTest || index < failedTests.length - 1) {
console.log('\nWaiting 2 seconds before next test...');
await new Promise(resolve => setTimeout(resolve, 2000));
}
runCount++;
}
index++;
};
// Step 4: Summary
console.log('\n📈 Summary:');
console.log(`Total failed tests: ${results.total}`);
console.log(`Passed on rerun: ${results.passed}`);
console.log(`Still failing: ${results.failed}`);
// Restore the original junit xml file
if (fs.existsSync(`${process.env.MOCHA_FILE}.save`)) {
fs.renameSync(`${process.env.MOCHA_FILE}.save`, process.env.MOCHA_FILE);
}
if (results.failed === 0) {
console.log('🎉 All previously failed tests now pass!');
} else {
console.log(`⚠️ ${results.failed} test(s) are still failing`);
process.exit(1);
}
}
async function runTestUsingElectron (specDir, testName, shouldRerun, additionalArgs = []) {
let exe;
if (args.electronVersion) {
const installer = new Installer();
@ -199,11 +354,16 @@ async function runTestUsingElectron (specDir, testName) {
} else {
exe = path.resolve(BASE, utils.getElectronExec());
}
const runnerArgs = [`electron/${specDir}`, ...unknownArgs.slice(2)];
let argsToPass = unknownArgs.slice(2);
if (additionalArgs.includes('--files')) {
argsToPass = argsToPass.filter(arg => (arg.toString().indexOf('--files') === -1 && arg.toString().indexOf('spec/') === -1));
}
const runnerArgs = [`electron/${specDir}`, ...argsToPass, ...additionalArgs];
if (process.platform === 'linux') {
runnerArgs.unshift(path.resolve(__dirname, 'dbus_mock.py'), exe);
exe = 'python3';
}
console.log(`Running: ${exe} ${runnerArgs.join(' ')}`);
const { status, signal } = await asyncSpawn(exe, runnerArgs);
if (status !== 0) {
if (status) {
@ -212,13 +372,22 @@ async function runTestUsingElectron (specDir, testName) {
} else {
console.log(`${fail} Electron tests failed with kill signal ${signal}.`);
}
process.exit(1);
if (shouldRerun) {
await rerunFailedTests(specDir, testName);
} else {
return false;
}
}
console.log(`${pass} Electron ${testName} process tests passed.`);
return true;
}
async function runMainProcessElectronTests () {
await runTestUsingElectron('spec', 'main');
let shouldRerun = false;
if (args.enableRerun && args.enableRerun > 0) {
shouldRerun = true;
}
await runTestUsingElectron('spec', 'main', shouldRerun);
}
async function installSpecModules (dir) {

View file

@ -78,13 +78,9 @@ function areColorsSimilar (
}
function displayCenter (display: Electron.Display): Electron.Point {
// On macOS, we get system prompt to ask permission for screen capture
// taking up space in the center. As a workaround, choose
// area of the application window which is not covered by the prompt.
// TODO: Remove this when the prompt situation is resolved.
return {
x: display.size.width / (process.platform === 'darwin' ? 4 : 2),
y: display.size.height / (process.platform === 'darwin' ? 4 : 2)
x: display.size.width / 2,
y: display.size.height / 2
};
}

View file

@ -1,4 +1,4 @@
import { BrowserWindow, session, ipcMain, app, WebContents, screen } from 'electron/main';
import { BrowserWindow, session, ipcMain, app, WebContents } from 'electron/main';
import * as auth from 'basic-auth';
import { expect } from 'chai';
@ -782,7 +782,6 @@ describe('<webview> tag', function () {
let w: BrowserWindow;
before(async () => {
const display = screen.getPrimaryDisplay();
w = new BrowserWindow({
webPreferences: {
webviewTag: true,
@ -790,7 +789,6 @@ describe('<webview> tag', function () {
contextIsolation: false
}
});
w.setBounds(display.bounds);
await w.loadURL(`file://${fixtures}/pages/flex-webview.html`);
w.setBackgroundColor(WINDOW_BACKGROUND_COLOR);
});

View file

@ -1345,6 +1345,11 @@
resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-2.0.5.tgz#325db42395cd49fe6c14057f9a900e427df8810e"
integrity sha512-lqaoKnRYBdo1UgDX8uF24AfGMifWK19TxPmM5FHc2vAGxrJ/qtyUyFBWoY1tISZdelsQ5fBcOusifo5o5wSJxQ==
"@xmldom/xmldom@^0.8.11":
version "0.8.11"
resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.8.11.tgz#b79de2d67389734c57c52595f7a7305e30c2d608"
integrity sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==
"@xtuc/ieee754@^1.2.0":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz#eef014a3145ae477a1cbc00cd1e552336dceb790"
@ -7347,14 +7352,7 @@ stringify-entities@^4.0.0:
character-entities-html4 "^2.0.0"
character-entities-legacy "^3.0.0"
"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
dependencies:
ansi-regex "^5.0.1"
strip-ansi@^6.0.0, strip-ansi@^6.0.1:
"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==