build: speedy tests on circle by splitting the test files into multiple jobs (#21015)

* build: let circleci divide our test suites in two

* well our tests rely on side affects, thats cool I guess

.circleci/config.yml

@@ -1221,7 +1221,8 @@ steps-tests: &steps-tests
         command: |
           cd src
           export ELECTRON_OUT_DIR=Default
-          (cd electron && node script/yarn test -- --enable-logging)
+          (cd electron && node script/yarn test --runners=main --enable-logging --files $(circleci tests glob spec-main/*-spec.ts | circleci tests split))
+          (cd electron && node script/yarn test --runners=remote --enable-logging --files $(circleci tests glob spec/*-spec.js | circleci tests split))
     - run:
         name: Check test results existence
         command: |
@@ -1766,6 +1767,7 @@ jobs:
       <<: *env-linux-medium
       <<: *env-headless-testing
       <<: *env-stack-dumping
+    parallelism: 3
     <<: *steps-tests
 
   linux-x64-testing-nan:
@@ -1815,6 +1817,7 @@ jobs:
       <<: *env-ia32
       <<: *env-headless-testing
       <<: *env-stack-dumping
+    parallelism: 3
     <<: *steps-tests
 
   linux-ia32-testing-nan:
@@ -1867,6 +1870,7 @@ jobs:
     environment:
       <<: *env-mac-large
       <<: *env-stack-dumping
+    parallelism: 2
     <<: *steps-tests
 
   osx-release-tests:
@@ -1896,6 +1900,7 @@ jobs:
     environment:
       <<: *env-mac-large
       <<: *env-stack-dumping
+    parallelism: 2
     <<: *steps-tests
 
   mas-release-tests:

spec-main/api-app-spec.ts

@@ -1,5 +1,4 @@
-import * as chai from 'chai'
+import { expect } from 'chai'
-import * as chaiAsPromised from 'chai-as-promised'
 import * as cp from 'child_process'
 import * as https from 'https'
 import * as net from 'net'
@@ -13,10 +12,6 @@ import split = require('split')
 
 const features = process.electronBinding('features')
 
-const { expect } = chai
-
-chai.use(chaiAsPromised)
-
 const fixturesPath = path.resolve(__dirname, '../spec/fixtures')
 
 describe('electron module', () => {

spec-main/api-browser-view-spec.ts

@@ -1,13 +1,9 @@
-import * as chai from 'chai'
+import { expect } from 'chai'
 import * as ChildProcess from 'child_process'
 import * as path from 'path'
 import { emittedOnce } from './events-helpers'
 import { BrowserView, BrowserWindow } from 'electron'
 import { closeWindow } from './window-helpers'
-import dirtyChai = require('dirty-chai')
-
-const { expect } = chai
-chai.use(dirtyChai)
 
 describe('BrowserView module', () => {
   const fixtures = path.resolve(__dirname, '..', 'spec', 'fixtures')

spec-main/api-browser-window-spec.ts

@@ -1,5 +1,4 @@
-import * as chai from 'chai'
+import { expect } from 'chai'
-import * as chaiAsPromised from 'chai-as-promised'
 import * as path from 'path'
 import * as fs from 'fs'
 import * as os from 'os'
@@ -11,12 +10,6 @@ import { app, BrowserWindow, BrowserView, ipcMain, OnBeforeSendHeadersListenerDe
 import { emittedOnce } from './events-helpers'
 import { ifit, ifdescribe } from './spec-helpers'
 import { closeWindow } from './window-helpers'
-import dirtyChai = require('dirty-chai')
-
-const { expect } = chai
-
-chai.use(chaiAsPromised)
-chai.use(dirtyChai)
 
 const fixtures = path.resolve(__dirname, '..', 'spec', 'fixtures')
 

spec-main/api-ipc-spec.ts

@@ -1,11 +1,6 @@
-import * as chai from 'chai'
+import { expect } from 'chai'
-import * as chaiAsPromised from 'chai-as-promised'
 import { BrowserWindow, ipcMain, IpcMainInvokeEvent } from 'electron'
 
-const { expect } = chai
-
-chai.use(chaiAsPromised)
-
 describe('ipc module', () => {
   describe('invoke', () => {
     let w = (null as unknown as BrowserWindow)

spec-main/api-web-contents-spec.ts

@@ -1,6 +1,5 @@
-import * as chai from 'chai'
+import { expect } from 'chai'
 import { AddressInfo } from 'net'
-import * as chaiAsPromised from 'chai-as-promised'
 import * as path from 'path'
 import * as fs from 'fs'
 import * as http from 'http'
@@ -10,10 +9,6 @@ import { emittedOnce } from './events-helpers'
 import { closeAllWindows } from './window-helpers'
 import { ifdescribe, ifit } from './spec-helpers'
 
-const { expect } = chai
-
-chai.use(chaiAsPromised)
-
 const fixturesPath = path.resolve(__dirname, '..', 'spec', 'fixtures')
 const features = process.electronBinding('features')
 

spec-main/chromium-spec.ts

@@ -1,5 +1,4 @@
-import * as chai from 'chai'
+import { expect } from 'chai'
-import * as chaiAsPromised from 'chai-as-promised'
 import { BrowserWindow, WebContents, session, ipcMain, app, protocol, webContents } from 'electron'
 import { emittedOnce } from './events-helpers'
 import { closeAllWindows } from './window-helpers'
@@ -14,11 +13,8 @@ import { promisify } from 'util'
 import { ifit, ifdescribe } from './spec-helpers'
 import { AddressInfo } from 'net'
 
-const { expect } = chai
-
 const features = process.electronBinding('features')
 
-chai.use(chaiAsPromised)
 const fixturesPath = path.resolve(__dirname, '..', 'spec', 'fixtures')
 
 describe('reporting api', () => {

spec-main/index.js

@@ -43,6 +43,7 @@ app.whenReady().then(() => {
 
   const argv = require('yargs')
     .boolean('ci')
+    .array('files')
     .string('g').alias('g', 'grep')
     .boolean('i').alias('i', 'invert')
     .argv
@@ -86,15 +87,27 @@ app.whenReady().then(() => {
     }
   })
 
+  const baseElectronDir = path.resolve(__dirname, '..')
+
   walker.on('end', () => {
     testFiles.sort()
-    sortToEnd(testFiles, f => f.includes('crash-reporter')).forEach((file) => mocha.addFile(file))
+    sortToEnd(testFiles, f => f.includes('crash-reporter')).forEach((file) => {
+      if (!argv.files || argv.files.includes(path.relative(baseElectronDir, file))) {
+        mocha.addFile(file)
+      }
+    })
     const cb = () => {
       // Ensure the callback is called after runner is defined
       process.nextTick(() => {
         process.exit(runner.failures)
       })
     }
+
+    // Set up chai in the correct order
+    const chai = require('chai')
+    chai.use(require('chai-as-promised'))
+    chai.use(require('dirty-chai'))
+
     const runner = mocha.run(cb)
   })
 })

spec/api-native-image-spec.js

@@ -1,13 +1,9 @@
 'use strict'
 
-const chai = require('chai')
+const { expect } = require('chai')
-const dirtyChai = require('dirty-chai')
 const { nativeImage } = require('electron')
 const path = require('path')
 
-const { expect } = chai
-chai.use(dirtyChai)
-
 describe('nativeImage module', () => {
   const ImageFormat = {
     PNG: 'png',

spec/api-remote-spec.js

@@ -1,19 +1,15 @@
 'use strict'
 
-const chai = require('chai')
+const { expect } = require('chai')
-const dirtyChai = require('dirty-chai')
 const path = require('path')
 const { resolveGetters } = require('./expect-helpers')
 const { ifdescribe } = require('./spec-helpers')
 
 const { remote, ipcRenderer } = require('electron')
 const { ipcMain, BrowserWindow } = remote
-const { expect } = chai
 
 const features = process.electronBinding('features')
 
-chai.use(dirtyChai)
-
 const comparePaths = (path1, path2) => {
   if (process.platform === 'win32') {
     path1 = path1.toLowerCase()

spec/api-shell-spec.js

@@ -1,5 +1,4 @@
-const chai = require('chai')
+const { expect } = require('chai')
-const dirtyChai = require('dirty-chai')
 
 const fs = require('fs')
 const path = require('path')
@@ -7,9 +6,6 @@ const os = require('os')
 const http = require('http')
 const { shell } = require('electron')
 
-const { expect } = chai
-chai.use(dirtyChai)
-
 describe('shell module', () => {
   const fixtures = path.resolve(__dirname, 'fixtures')
   const shortcutOptions = {

spec/api-web-frame-spec.js

@@ -1,10 +1,6 @@
-const chai = require('chai')
+const { expect } = require('chai')
-const dirtyChai = require('dirty-chai')
 const { webFrame } = require('electron')
 
-const { expect } = chai
-chai.use(dirtyChai)
-
 describe('webFrame module', function () {
   it('supports setting the visual and layout zoom level limits', function () {
     expect(() => {

spec/asar-spec.js

@@ -1,5 +1,4 @@
-const chai = require('chai')
+const { expect } = require('chai')
-const dirtyChai = require('dirty-chai')
 const ChildProcess = require('child_process')
 const fs = require('fs')
 const path = require('path')
@@ -9,9 +8,6 @@ const nativeImage = require('electron').nativeImage
 
 const features = process.electronBinding('features')
 
-const { expect } = chai
-chai.use(dirtyChai)
-
 async function expectToThrowErrorWithCode (func, code) {
   let error
   try {

spec/chromium-spec.js

@@ -1,5 +1,4 @@
-const chai = require('chai')
+const { expect } = require('chai')
-const dirtyChai = require('dirty-chai')
 const fs = require('fs')
 const http = require('http')
 const path = require('path')
@@ -11,9 +10,6 @@ const { emittedOnce } = require('./events-helpers')
 const { resolveGetters } = require('./expect-helpers')
 const features = process.electronBinding('features')
 
-const { expect } = chai
-chai.use(dirtyChai)
-
 /* Most of the APIs here don't use standard callbacks */
 /* eslint-disable standard/no-callback-literal */
 

spec/node-spec.js

@@ -1,7 +1,5 @@
 const ChildProcess = require('child_process')
-const chai = require('chai')
+const { expect } = require('chai')
-const { expect } = chai
-const dirtyChai = require('dirty-chai')
 const fs = require('fs')
 const path = require('path')
 const os = require('os')
@@ -11,8 +9,6 @@ const features = process.electronBinding('features')
 const { emittedOnce } = require('./events-helpers')
 const { ifit } = require('./spec-helpers')
 
-chai.use(dirtyChai)
-
 describe('node feature', () => {
   const fixtures = path.join(__dirname, 'fixtures')
 

spec/static/index.html

@@ -55,6 +55,8 @@
   if (query.grep) mocha.grep(query.grep)
   if (query.invert) mocha.invert()
 
+  const files = query.files ? query.files.split(',') : undefined
+
   // Read all test files.
   const walker = require('walkdir').walk(path.dirname(__dirname), {
     no_recurse: true
@@ -73,9 +75,20 @@
     }
   })
 
+  const baseElectronDir = path.resolve(__dirname, '..', '..')
+
   walker.on('end', () => {
     testFiles.sort()
-    testFiles.forEach((file) => mocha.addFile(file))
+    testFiles.forEach((file) => {
+      if (!files || files.includes(path.relative(baseElectronDir, file))) {
+        mocha.addFile(file)
+      }
+    })
+
+    // Set up chai in the correct order
+    const chai = require('chai')
+    chai.use(require('chai-as-promised'))
+    chai.use(require('dirty-chai'))
 
     const runner = mocha.run(() => {
       // Ensure the callback is called after runner is defined

spec/static/main.js

@@ -11,6 +11,7 @@ const v8 = require('v8')
 
 const argv = require('yargs')
   .boolean('ci')
+  .array('files')
   .string('g').alias('g', 'grep')
   .boolean('i').alias('i', 'invert')
   .argv
@@ -133,7 +134,8 @@ app.on('ready', function () {
   window.loadFile('static/index.html', {
     query: {
       grep: argv.grep,
-      invert: argv.invert ? 'true' : ''
+      invert: argv.invert ? 'true' : '',
+      files: argv.files ? argv.files.join(',') : undefined
     }
   })
   window.on('unresponsive', function () {

spec/webview-spec.js

@@ -1,14 +1,10 @@
-const chai = require('chai')
+const { expect } = require('chai')
-const dirtyChai = require('dirty-chai')
 const path = require('path')
 const http = require('http')
 const url = require('url')
 const { ipcRenderer } = require('electron')
 const { emittedOnce, waitForEvent } = require('./events-helpers')
 
-const { expect } = chai
-chai.use(dirtyChai)
-
 const features = process.electronBinding('features')
 const nativeModulesEnabled = process.env.ELECTRON_SKIP_NATIVE_MODULE_TESTS