Interactive test-electron

app/main.ts

@@ -2972,18 +2972,23 @@ async function showStickerCreatorWindow() {
 }
 
 if (isTestEnvironment(getEnvironment())) {
-  ipc.handle('ci:test-electron:debug', async (_event, info) => {
-    process.stdout.write(`ci:test-electron:debug=${JSON.stringify(info)}\n`);
+  ipc.on('ci:test-electron:getArgv', event => {
+    // eslint-disable-next-line no-param-reassign
+    event.returnValue = process.argv;
   });
 
-  ipc.handle('ci:test-electron:done', async (_event, info) => {
-    if (!process.env.TEST_QUIT_ON_COMPLETE) {
-      return;
-    }
-
+  ipc.handle('ci:test-electron:event', async (_event, event) => {
     process.stdout.write(
-      `ci:test-electron:done=${JSON.stringify(info)}\n`,
-      () => app.quit()
+      `ci:test-electron:event=${JSON.stringify(event)}\n`,
+      () => {
+        if (event.type !== 'end') {
+          return;
+        }
+        if (!process.env.TEST_QUIT_ON_COMPLETE) {
+          return;
+        }
+        app.quit();
+      }
     );
   });
 }

package-lock.json

@@ -225,6 +225,7 @@
         "html-webpack-plugin": "5.3.1",
         "http-server": "14.1.1",
         "json-to-ast": "2.1.0",
+        "log-symbols": "4.1.0",
         "mini-css-extract-plugin": "2.7.6",
         "mocha": "9.2.2",
         "node-gyp": "10.0.1",

package.json

@@ -307,6 +307,7 @@
     "html-webpack-plugin": "5.3.1",
     "http-server": "14.1.1",
     "json-to-ast": "2.1.0",
+    "log-symbols": "4.1.0",
     "mini-css-extract-plugin": "2.7.6",
     "mocha": "9.2.2",
     "node-gyp": "10.0.1",

test/test.js

@@ -33,29 +33,28 @@ delete window.testUtilities.prepareTests;
 window.textsecure.storage.protocol = window.getSignalProtocolStore();
 
 !(function () {
-  const passed = [];
-  const failed = [];
-
   class Reporter extends Mocha.reporters.HTML {
     constructor(runner, options) {
       super(runner, options);
 
-      runner.on('pass', test => passed.push(test.fullTitle()));
-      runner.on('fail', (test, error) => {
-        failed.push({
-          testName: test.fullTitle(),
-          error: error?.stack || String(error),
-        });
-      });
+      runner.on('pass', test => window.testUtilities.onTestEvent({
+        type: 'pass',
+        title: test.titlePath(),
+      }));
+      runner.on('fail', (test, error) => window.testUtilities.onTestEvent({
+        type: 'fail',
+        title: test.titlePath(),
+        error: error?.stack || String(error),
+      }));
 
-      runner.on('end', () =>
-        window.testUtilities.onComplete({ passed, failed })
-      );
+      runner.on('end', () => window.testUtilities.onTestEvent({ type: 'end' }));
     }
   }
 
   mocha.reporter(Reporter);
 
+  mocha.setup(window.testUtilities.setup);
+
   mocha.run();
 })();
 

ts/scripts/test-electron.ts

@@ -1,8 +1,15 @@
 // Copyright 2021 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
-import { execFileSync } from 'child_process';
-import { join } from 'path';
+import { spawn } from 'node:child_process';
+import { join } from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import z from 'zod';
+import split2 from 'split2';
+import logSymbols from 'log-symbols';
+
+import { explodePromise } from '../util/explodePromise';
+import { missingCaseError } from '../util/missingCaseError';
 
 const ROOT_DIR = join(__dirname, '..', '..');
 
@@ -16,7 +23,25 @@ const ELECTRON = join(
 const MAX_RETRIES = 3;
 const RETRIABLE_SIGNALS = ['SIGBUS'];
 
-function launchElectron(attempt: number): string {
+const failSchema = z.object({
+  type: z.literal('fail'),
+  title: z.string().array(),
+  error: z.string(),
+});
+
+const eventSchema = z
+  .object({
+    type: z.literal('pass'),
+    title: z.string().array(),
+  })
+  .or(failSchema)
+  .or(
+    z.object({
+      type: z.literal('end'),
+    })
+  );
+
+async function launchElectron(attempt: number): Promise<void> {
   if (attempt > MAX_RETRIES) {
     console.error(`Failed after ${MAX_RETRIES} retries, exiting.`);
     process.exit(1);
@@ -24,79 +49,145 @@ function launchElectron(attempt: number): string {
 
   console.log(`Launching electron for tests, attempt #${attempt}...`);
 
-  try {
-    const stdout = execFileSync(ELECTRON, [ROOT_DIR], {
-      cwd: ROOT_DIR,
-      env: {
-        ...process.env,
-        // Setting NODE_ENV to test triggers main.ts to load
-        // 'test/index.html' instead of 'background.html', which loads the tests
-        // via `test.js`
-        NODE_ENV: 'test',
-        TEST_QUIT_ON_COMPLETE: 'on',
-      },
-      // Since we run `.cmd` file on Windows - use shell
-      shell: process.platform === 'win32',
-      encoding: 'utf8',
-    });
-    return stdout;
-  } catch (error) {
-    console.error('Status', error.status);
+  const proc = spawn(ELECTRON, [ROOT_DIR, ...process.argv.slice(2)], {
+    cwd: ROOT_DIR,
+    env: {
+      ...process.env,
+      // Setting NODE_ENV to test triggers main.ts to load
+      // 'test/index.html' instead of 'background.html', which loads the tests
+      // via `test.js`
+      NODE_ENV: 'test',
+      TEST_QUIT_ON_COMPLETE: 'on',
+    },
+    // Since we run `.cmd` file on Windows - use shell
+    shell: process.platform === 'win32',
+  });
 
-    // In testing, error.signal is null, so we need to read it from stderr
-    const signalMatch = error.stderr.match(/exited with signal (\w+)/);
-    const signal = error.signal || signalMatch?.[1];
+  const { resolve, reject, promise: exitPromise } = explodePromise<void>();
 
-    console.error('Signal', signal);
-    console.error(error.output[0] ?? '');
-    console.error(error.output[1] ?? '');
+  let exitSignal: string | undefined;
+  proc.on('exit', (code, signal) => {
+    if (code === 0) {
+      resolve();
+    } else {
+      exitSignal = signal || undefined;
+      reject(new Error(`Exit code: ${code}`));
+    }
+  });
 
-    if (RETRIABLE_SIGNALS.includes(signal)) {
-      return launchElectron(attempt + 1);
+  let pass = 0;
+  const failures = new Array<z.infer<typeof failSchema>>();
+  let done = false;
+  let stack = new Array<string>();
+
+  function enter(path: ReadonlyArray<string>): void {
+    // Find the first different fragment
+    let i: number;
+    for (i = 0; i < path.length - 1; i += 1) {
+      if (stack[i] !== path[i]) {
+        break;
+      }
     }
 
+    // Separate sections
+    if (i !== stack.length) {
+      console.log('');
+
+      // Remove different fragments
+      stack = stack.slice(0, i);
+    }
+
+    for (; i < path.length - 1; i += 1) {
+      const fragment = path[i];
+
+      console.log(indent(fragment));
+      stack.push(fragment);
+    }
+  }
+
+  function indent(value: string): string {
+    return `${'  '.repeat(stack.length)}${value}`;
+  }
+
+  try {
+    await Promise.all([
+      exitPromise,
+      pipeline(
+        proc.stdout,
+        split2()
+          .resume()
+          .on('data', line => {
+            if (!line) {
+              return;
+            }
+
+            const match = line.match(/^ci:test-electron:event=(.*)/);
+            if (!match) {
+              return;
+            }
+
+            const event = eventSchema.parse(JSON.parse(match[1]));
+            if (event.type === 'pass') {
+              pass += 1;
+              enter(event.title);
+
+              console.log(
+                indent(`${logSymbols.success} ${event.title.at(-1)}`)
+              );
+            } else if (event.type === 'fail') {
+              failures.push(event);
+              enter(event.title);
+
+              console.error(
+                indent(`${logSymbols.error} ${event.title.at(-1)}`)
+              );
+              console.error('');
+              console.error(event.error);
+            } else if (event.type === 'end') {
+              done = true;
+            } else {
+              throw missingCaseError(event);
+            }
+          })
+      ),
+    ]);
+  } catch (error) {
+    if (exitSignal && RETRIABLE_SIGNALS.includes(exitSignal)) {
+      return launchElectron(attempt + 1);
+    }
+    throw error;
+  }
+
+  if (!done) {
+    throw new Error('Tests terminated early!');
+  }
+
+  if (failures.length) {
+    console.error('');
+    console.error('Failing tests:');
+    console.error('');
+    for (const { title, error } of failures) {
+      console.log(` ${logSymbols.error} ${title.join(' ')}`);
+      console.log(error);
+      console.log('');
+    }
+  }
+
+  console.log(
+    `Passed ${pass} | Failed ${failures.length} | ` +
+      `Total ${pass + failures.length}`
+  );
+
+  if (failures.length !== 0) {
     process.exit(1);
   }
 }
 
-const stdout = launchElectron(1);
-
-const debugMatch = stdout.matchAll(/ci:test-electron:debug=(.*)?\n/g);
-Array.from(debugMatch).forEach(info => {
-  try {
-    const args = JSON.parse(info[1]);
-    console.log('DEBUG:', args);
-  } catch {
-    // this section intentionally left blank
-  }
-});
-
-const match = stdout.match(/ci:test-electron:done=(.*)?\n/);
-
-if (!match) {
-  throw new Error('No test results were found in stdout');
+async function main() {
+  await launchElectron(1);
 }
 
-const {
-  passed,
-  failed,
-}: {
-  passed: Array<string>;
-  failed: Array<{ testName: string; error: string }>;
-} = JSON.parse(match[1]);
-
-const total = passed.length + failed.length;
-
-for (const { testName, error } of failed) {
-  console.error(`- ${testName}`);
+main().catch(error => {
   console.error(error);
-  console.error('');
-}
-
-console.log(
-  `Passed ${passed.length} | Failed ${failed.length} | Total ${total}`
-);
-
-if (failed.length !== 0) {
   process.exit(1);
-}
+});

ts/window.d.ts

@@ -8,6 +8,7 @@ import type * as Backbone from 'backbone';
 import type PQueue from 'p-queue/dist';
 import type { assert } from 'chai';
 import type { PhoneNumber, PhoneNumberFormat } from 'google-libphonenumber';
+import type { MochaOptions } from 'mocha';
 
 import type { ConversationModelCollectionType } from './model-types.d';
 import type { textsecure } from './textsecure';
@@ -280,9 +281,9 @@ declare global {
     RETRY_DELAY: boolean;
     assert: typeof assert;
     testUtilities: {
-      debug: (info: unknown) => void;
+      setup: MochaOptions;
+      onTestEvent: (event: unknown) => void;
       initialize: () => Promise<void>;
-      onComplete: (info: unknown) => void;
       prepareTests: () => void;
     };
   }

ts/windows/main/preload_test.ts

@@ -3,14 +3,14 @@
 
 /* eslint-disable no-console */
 
+import { inspect, parseArgs } from 'node:util';
 import { ipcRenderer as ipc } from 'electron';
 import { sync } from 'fast-glob';
-import { inspect } from 'util';
 
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { assert, config as chaiConfig } from 'chai';
 // eslint-disable-next-line import/no-extraneous-dependencies
-import { reporters } from 'mocha';
+import { reporters, type MochaOptions } from 'mocha';
 
 import { getSignalProtocolStore } from '../../SignalProtocolStore';
 import { initMessageCleanup } from '../../services/messageStateCleanup';
@@ -46,13 +46,29 @@ window.assert = assert;
 // This is a hack to let us run TypeScript tests in the renderer process. See the
 //   code in `test/test.js`.
 
-window.testUtilities = {
-  debug(info) {
-    return ipc.invoke('ci:test-electron:debug', info);
-  },
+const setup: MochaOptions = {};
 
-  onComplete(info) {
-    return ipc.invoke('ci:test-electron:done', info);
+{
+  const { values } = parseArgs({
+    args: ipc.sendSync('ci:test-electron:getArgv'),
+    options: {
+      grep: {
+        type: 'string',
+      },
+    },
+    strict: false,
+  });
+
+  if (typeof values.grep === 'string') {
+    setup.grep = values.grep;
+  }
+}
+
+window.testUtilities = {
+  setup,
+
+  onTestEvent(event: unknown) {
+    return ipc.invoke('ci:test-electron:event', event);
   },
 
   async initialize() {