Conversation open speed benchmarking for staging builds

Co-authored-by: Fedor Indutnyy <indutny@signal.org>

app/config.ts

@@ -34,6 +34,7 @@ if (getEnvironment() === Environment.Production) {
   process.env.SUPPRESS_NO_CONFIG_WARNING = '';
   process.env.NODE_TLS_REJECT_UNAUTHORIZED = '';
   process.env.SIGNAL_ENABLE_HTTP = '';
+  process.env.SIGNAL_CI_CONFIG = '';
   process.env.CUSTOM_TITLEBAR = '';
 }
 

app/main.ts

@@ -161,7 +161,7 @@ const development =
   getEnvironment() === Environment.Development ||
   getEnvironment() === Environment.Staging;
 
-const enableCI = config.get<boolean>('enableCI');
+const ciMode = config.get<'full' | 'benchmark' | false>('ciMode');
 const forcePreloadBundle = config.get<boolean>('forcePreloadBundle');
 
 const preventDisplaySleepService = new PreventDisplaySleepService(
@@ -539,8 +539,8 @@ function handleCommonWindowEvents(
   }
 }
 
-const DEFAULT_WIDTH = enableCI ? 1024 : 800;
-const DEFAULT_HEIGHT = enableCI ? 1024 : 610;
+const DEFAULT_WIDTH = ciMode ? 1024 : 800;
+const DEFAULT_HEIGHT = ciMode ? 1024 : 610;
 
 // We allow for smaller sizes because folks with OS-level zoom and HighDPI/Large Text
 //   can really cause weirdness around window pixel-sizes. The app is very broken if you
@@ -822,7 +822,7 @@ async function createWindow() {
   mainWindow.on('resize', captureWindowStats);
   mainWindow.on('move', captureWindowStats);
 
-  if (!enableCI && config.get<boolean>('openDevTools')) {
+  if (!ciMode && config.get<boolean>('openDevTools')) {
     // Open the DevTools.
     mainWindow.webContents.openDevTools();
   }
@@ -2277,8 +2277,11 @@ ipc.on('get-config', async event => {
     cdnUrl0: config.get<ConfigType>('cdn').get<string>('0'),
     cdnUrl2: config.get<ConfigType>('cdn').get<string>('2'),
     certificateAuthority: config.get<string>('certificateAuthority'),
-    environment: enableCI ? Environment.Production : getEnvironment(),
-    enableCI,
+    environment:
+      !isTestEnvironment(getEnvironment()) && ciMode
+        ? Environment.Production
+        : getEnvironment(),
+    ciMode,
     nodeVersion: process.versions.node,
     hostname: os.hostname(),
     osRelease: os.release(),

ci.js

@@ -1,8 +1,10 @@
 // Copyright 2022 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
+const CI_CONFIG = JSON.parse(process.env.SIGNAL_CI_CONFIG || '');
+
 const config = require('./app/config').default;
 
-config.util.extendDeep(config, JSON.parse(process.env.SIGNAL_CI_CONFIG || ''));
+config.util.extendDeep(config, CI_CONFIG);
 
 require('./app/main');

config/default.json

@@ -16,7 +16,7 @@
   "challengeUrl": "https://signalcaptchas.org/staging/challenge/generate.html",
   "registrationChallengeUrl": "https://signalcaptchas.org/staging/registration/generate.html",
   "updatesEnabled": false,
-  "enableCI": false,
+  "ciMode": false,
   "forcePreloadBundle": false,
   "openDevTools": false,
   "buildCreation": 0,

scripts/prepare_staging_build.js

@@ -82,6 +82,7 @@ fs.writeFileSync('./package.json', JSON.stringify(packageJson, null, '  '));
 
 const productionJson = {
   updatesEnabled: true,
+  ciMode: 'benchmark',
 };
 fs.writeFileSync(
   './config/production.json',

ts/CI.ts

@@ -15,7 +15,13 @@ export type CIType = {
   handleEvent: (event: string, data: unknown) => unknown;
   setProvisioningURL: (url: string) => unknown;
   solveChallenge: (response: ChallengeResponseType) => unknown;
-  waitForEvent: (event: string, timeout?: number) => unknown;
+  waitForEvent: (
+    event: string,
+    options: {
+      timeout?: number;
+      ignorePastEvents?: boolean;
+    }
+  ) => unknown;
 };
 
 export function getCI(deviceName: string): CIType {
@@ -26,17 +32,27 @@ export function getCI(deviceName: string): CIType {
     handleEvent(event, data);
   });
 
-  function waitForEvent(event: string, timeout = 60 * SECOND) {
-    const pendingCompleted = completedEvents.get(event) || [];
-    const pending = pendingCompleted.shift();
-    if (pending) {
-      log.info(`CI: resolving pending result for ${event}`, pending);
+  function waitForEvent(
+    event: string,
+    options: {
+      timeout?: number;
+      ignorePastEvents?: boolean;
+    } = {}
+  ) {
+    const timeout = options?.timeout ?? 60 * SECOND;
 
-      if (pendingCompleted.length === 0) {
-        completedEvents.delete(event);
+    if (!options?.ignorePastEvents) {
+      const pendingCompleted = completedEvents.get(event) || [];
+      const pending = pendingCompleted.shift();
+      if (pending) {
+        log.info(`CI: resolving pending result for ${event}`, pending);
+
+        if (pendingCompleted.length === 0) {
+          completedEvents.delete(event);
+        }
+
+        return pending;
       }
-
-      return pending;
     }
 
     log.info(`CI: waiting for event ${event}`);

ts/CI/benchmarkConversationOpen.ts

@@ -0,0 +1,229 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { v4 as uuid } from 'uuid';
+
+import { incrementMessageCounter } from '../util/incrementMessageCounter';
+import { ReadStatus } from '../messages/MessageReadStatus';
+import { UUID } from '../types/UUID';
+import { SendStatus } from '../messages/MessageSendState';
+import { BodyRange } from '../types/BodyRange';
+import { strictAssert } from '../util/assert';
+import { MINUTE } from '../util/durations';
+import { isOlderThan } from '../util/timestamp';
+import { sleep } from '../util/sleep';
+import { stats } from '../util/benchmark/stats';
+import type { StatsType } from '../util/benchmark/stats';
+import type { MessageAttributesType } from '../model-types.d';
+import * as log from '../logging/log';
+
+const BUFFER_DELAY_MS = 50;
+
+type PopulateConversationArgsType = {
+  conversationId: string;
+  messageCount: number;
+  unreadCount?: number;
+  customizeMessage?: (
+    idx: number,
+    baseMessage: MessageAttributesType
+  ) => MessageAttributesType;
+};
+
+export async function populateConversationWithMessages({
+  conversationId,
+  messageCount,
+  unreadCount = 0,
+  customizeMessage,
+}: PopulateConversationArgsType): Promise<void> {
+  strictAssert(
+    window.SignalCI,
+    'CI not enabled; ensure this is a staging build'
+  );
+  const logId = 'benchmarkConversationOpen/populateConversationWithMessages';
+  log.info(`${logId}: populating conversation`);
+
+  const ourUuid = window.textsecure.storage.user.getCheckedUuid().toString();
+  const conversation = window.ConversationController.get(conversationId);
+
+  strictAssert(
+    conversation,
+    `Conversation with id [${conversationId}] not found`
+  );
+
+  log.info(`${logId}: destroying all messages in ${conversationId}`);
+  await conversation.destroyMessages();
+
+  log.info(`${logId}: adding ${messageCount} messages to ${conversationId}`);
+  let timestamp = Date.now();
+  const messages: Array<MessageAttributesType> = [];
+  for (let i = 0; i < messageCount; i += 1) {
+    const isUnread = messageCount - i <= unreadCount;
+    const isIncoming = isUnread || i % 2 === 0;
+    const message: MessageAttributesType = {
+      body: `Message ${i}: Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam venenatis nec sapien id porttitor.`,
+      bodyRanges: [{ start: 0, length: 7, style: BodyRange.Style.BOLD }],
+      attachments: [],
+      conversationId,
+      id: uuid(),
+      type: isIncoming ? 'incoming' : 'outgoing',
+      timestamp,
+      sent_at: timestamp,
+      schemaVersion: window.Signal.Types.Message.CURRENT_SCHEMA_VERSION,
+      received_at: incrementMessageCounter(),
+      readStatus: isUnread ? ReadStatus.Unread : ReadStatus.Read,
+      sourceUuid: new UUID(isIncoming ? conversationId : ourUuid).toString(),
+      ...(isIncoming
+        ? {}
+        : {
+            sendStateByConversationId: {
+              [conversationId]: { status: SendStatus.Sent },
+            },
+          }),
+    };
+    messages.push(customizeMessage?.(i, message) ?? message);
+
+    timestamp += 1;
+  }
+
+  await window.Signal.Data.saveMessages(messages, {
+    forceSave: true,
+    ourUuid,
+  });
+
+  conversation.set('active_at', Date.now());
+  await window.Signal.Data.updateConversation(conversation.attributes);
+  log.info(`${logId}: populating conversation complete`);
+}
+
+export async function benchmarkConversationOpen({
+  conversationId,
+  messageCount = 10_000,
+  runCount = 50,
+  runCountToSkip = 0,
+  customizeMessage,
+  unreadCount,
+  testRunId,
+}: Partial<PopulateConversationArgsType> & {
+  runCount?: number;
+  runCountToSkip?: number;
+  testRunId?: string;
+} = {}): Promise<{ durations: Array<number>; stats: StatsType }> {
+  strictAssert(
+    window.SignalCI,
+    'CI not enabled; ensure this is a staging build'
+  );
+
+  // eslint-disable-next-line no-param-reassign
+  conversationId =
+    conversationId ||
+    window.reduxStore.getState().conversations.selectedConversationId;
+
+  strictAssert(conversationId, 'Must open a conversation for benchmarking');
+
+  const logId = `benchmarkConversationOpen${testRunId ? `/${testRunId}` : ''}`;
+
+  log.info(`${logId}: starting conversation open benchmarks, config:`, {
+    conversationId,
+    messageCount,
+    runCount,
+    customMessageMethod: !!customizeMessage,
+    unreadCount,
+    testRunId,
+  });
+
+  await populateConversationWithMessages({
+    conversationId,
+    messageCount,
+    unreadCount,
+    customizeMessage,
+  });
+  log.info(`${logId}: populating conversation complete`);
+
+  const durations: Array<number> = [];
+  for (let i = 0; i < runCount; i += 1) {
+    // Give some buffer between tests
+    // eslint-disable-next-line no-await-in-loop
+    await sleep(BUFFER_DELAY_MS);
+
+    log.info(`${logId}: running open test run ${i + 1}/${runCount}`);
+
+    // eslint-disable-next-line no-await-in-loop
+    const duration = await timeConversationOpen(conversationId);
+
+    if (i >= runCountToSkip) {
+      durations.push(duration);
+    }
+  }
+
+  const result = {
+    durations,
+    stats: stats(durations),
+  };
+
+  log.info(`${logId}: tests complete, results:`, result);
+  return result;
+}
+
+async function waitForSelector(
+  selector: string,
+  timeout = MINUTE
+): Promise<Node> {
+  const start = Date.now();
+
+  while (!isOlderThan(start, timeout)) {
+    const element = window.document.querySelector(selector);
+    if (element) {
+      return element;
+    }
+
+    // eslint-disable-next-line no-await-in-loop
+    await sleep(BUFFER_DELAY_MS);
+  }
+
+  throw new Error('Timed out');
+}
+
+async function timeConversationOpen(id: string): Promise<number> {
+  strictAssert(
+    window.SignalCI,
+    'CI not enabled; ensure this is a staging build'
+  );
+
+  await showEmptyInbox();
+
+  const element = await waitForSelector(`[data-id="${id}"]`);
+
+  const conversationOpenPromise = window.SignalCI.waitForEvent(
+    'conversation:open',
+    { ignorePastEvents: true }
+  );
+
+  const start = Date.now();
+
+  element.dispatchEvent(new Event('click', { bubbles: true }));
+  window.reduxActions.conversations.showConversation({
+    conversationId: id,
+  });
+
+  await conversationOpenPromise;
+  const end = Date.now();
+
+  return end - start;
+}
+
+async function showEmptyInbox() {
+  strictAssert(
+    window.SignalCI,
+    'CI not enabled; ensure this is a staging build'
+  );
+  if (!window.reduxStore.getState().conversations.selectedConversationId) {
+    return;
+  }
+  const promise = window.SignalCI.waitForEvent('empty-inbox:rendered', {
+    ignorePastEvents: true,
+  });
+  window.reduxActions.conversations.showConversation({
+    conversationId: undefined,
+  });
+  return promise;
+}

ts/components/Inbox.tsx

@@ -185,6 +185,12 @@ export function Inbox({
     setInternalHasInitialLoadCompleted(hasInitialLoadCompleted);
   }, [hasInitialLoadCompleted]);
 
+  useEffect(() => {
+    if (!selectedConversationId) {
+      window.SignalCI?.handleEvent('empty-inbox:rendered', null);
+    }
+  }, [selectedConversationId]);
+
   if (!internalHasInitialLoadCompleted) {
     let loadingProgress = 0;
     if (

ts/test-mock/benchmarks/convo_open_bench.ts

@@ -5,7 +5,8 @@
 import assert from 'assert';
 import type { PrimaryDevice } from '@signalapp/mock-server';
 
-import { Bootstrap, debug, stats, RUN_COUNT, DISCARD_COUNT } from './fixtures';
+import { Bootstrap, debug, RUN_COUNT, DISCARD_COUNT } from './fixtures';
+import { stats } from '../../util/benchmark/stats';
 
 const CONVERSATION_SIZE = 1000; // messages
 const DELAY = 50; // milliseconds

ts/test-mock/benchmarks/fixtures.ts

@@ -11,12 +11,6 @@ export const debug = createDebug('mock:benchmarks');
 export { Bootstrap };
 export { App } from '../playwright';
 
-export type StatsType = {
-  mean: number;
-  stddev: number;
-  [key: string]: number;
-};
-
 export const RUN_COUNT = process.env.RUN_COUNT
   ? parseInt(process.env.RUN_COUNT, 10)
   : 100;
@@ -29,38 +23,6 @@ export const DISCARD_COUNT = process.env.DISCARD_COUNT
   ? parseInt(process.env.DISCARD_COUNT, 10)
   : 5;
 
-export function stats(
-  list: ReadonlyArray<number>,
-  percentiles: ReadonlyArray<number> = []
-): StatsType {
-  if (list.length === 0) {
-    throw new Error('Empty list given to stats');
-  }
-
-  let mean = 0;
-  let stddev = 0;
-
-  for (const value of list) {
-    mean += value;
-    stddev += value ** 2;
-  }
-  mean /= list.length;
-  stddev /= list.length;
-
-  stddev -= mean ** 2;
-  stddev = Math.sqrt(stddev);
-
-  const sorted = list.slice().sort((a, b) => a - b);
-
-  const result: StatsType = { mean, stddev };
-
-  for (const p of percentiles) {
-    result[`p${p}`] = sorted[Math.floor((sorted.length * p) / 100)];
-  }
-
-  return result;
-}
-
 // Can happen if electron exits prematurely
 process.on('unhandledRejection', reason => {
   console.error('Unhandled rejection:');

ts/test-mock/benchmarks/group_send_bench.ts

@@ -13,11 +13,11 @@ import {
 import {
   Bootstrap,
   debug,
-  stats,
   RUN_COUNT,
   GROUP_SIZE,
   DISCARD_COUNT,
 } from './fixtures';
+import { stats } from '../../util/benchmark/stats';
 
 const CONVERSATION_SIZE = 500; // messages
 const LAST_MESSAGE = 'start sending messages now';

ts/test-mock/benchmarks/send_bench.ts

@@ -6,7 +6,8 @@ import assert from 'assert';
 
 import { ReceiptType } from '@signalapp/mock-server';
 
-import { Bootstrap, debug, stats, RUN_COUNT, DISCARD_COUNT } from './fixtures';
+import { Bootstrap, debug, RUN_COUNT, DISCARD_COUNT } from './fixtures';
+import { stats } from '../../util/benchmark/stats';
 
 const CONVERSATION_SIZE = 500; // messages
 

ts/test-mock/benchmarks/startup_bench.ts

@@ -4,7 +4,8 @@
 
 import { ReceiptType } from '@signalapp/mock-server';
 
-import { debug, Bootstrap, stats, RUN_COUNT } from './fixtures';
+import { debug, Bootstrap, RUN_COUNT } from './fixtures';
+import { stats } from '../../util/benchmark/stats';
 
 const MESSAGE_BATCH_SIZE = 1000; // messages
 

ts/test-mock/bootstrap.ts

@@ -403,7 +403,7 @@ export class Bootstrap {
       ...(await loadCertificates()),
 
       forcePreloadBundle: this.options.benchmark,
-      enableCI: true,
+      ciMode: 'full',
 
       buildExpiration: Date.now() + durations.MONTH,
       storagePath: this.storagePath,

ts/types/RendererConfig.ts

@@ -37,7 +37,7 @@ export const rendererConfigSchema = z.object({
   certificateAuthority: configRequiredStringSchema,
   contentProxyUrl: configRequiredStringSchema,
   crashDumpsPath: configRequiredStringSchema,
-  enableCI: z.boolean(),
+  ciMode: z.enum(['full', 'benchmark']).or(z.literal(false)),
   environment: environmentSchema,
   homePath: configRequiredStringSchema,
   hostname: configRequiredStringSchema,

ts/util/benchmark/stats.ts

@@ -0,0 +1,40 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+export type StatsType = {
+  mean: number;
+  stddev: number;
+  [key: string]: number;
+};
+
+export function stats(
+  list: ReadonlyArray<number>,
+  percentiles: ReadonlyArray<number> = []
+): StatsType {
+  if (list.length === 0) {
+    throw new Error('Empty list given to stats');
+  }
+
+  let mean = 0;
+  let stddev = 0;
+
+  for (const value of list) {
+    mean += value;
+    stddev += value ** 2;
+  }
+  mean /= list.length;
+  stddev /= list.length;
+
+  stddev -= mean ** 2;
+  stddev = Math.sqrt(stddev);
+
+  const sorted = list.slice().sort((a, b) => a - b);
+
+  const result: StatsType = { mean, stddev };
+
+  for (const p of percentiles) {
+    result[`p${p}`] = sorted[Math.floor((sorted.length * p) / 100)];
+  }
+
+  return result;
+}

ts/windows/main/phase1-ipc.ts

@@ -165,7 +165,7 @@ window.logAuthenticatedConnect = () => {
 window.open = () => null;
 
 // Playwright uses `eval` for `.evaluate()` API
-if (!config.enableCI && config.environment !== 'test') {
+if (config.ciMode !== 'full' && config.environment !== 'test') {
   // eslint-disable-next-line no-eval, no-multi-assign
   window.eval = global.eval = () => null;
 }

ts/windows/main/phase4-test.ts

@@ -11,8 +11,11 @@ if (config.environment === 'test') {
   console.log('Importing test infrastructure...');
   require('./preload_test');
 }
-if (config.enableCI) {
-  console.log('Importing CI infrastructure...');
+
+if (config.ciMode) {
+  console.log(
+    `Importing CI infrastructure; enabled in config, mode: ${config.ciMode}`
+  );
   const { getCI } = require('../../CI');
   window.SignalCI = getCI(window.getTitle());
 }

ts/windows/main/start.ts

@@ -21,6 +21,7 @@ import { MessageController } from '../../util/MessageController';
 import { Environment, getEnvironment } from '../../environment';
 import { isProduction } from '../../util/version';
 import { ipcInvoke } from '../../sql/channels';
+import { benchmarkConversationOpen } from '../../CI/benchmarkConversationOpen';
 
 window.addEventListener('contextmenu', e => {
   const node = e.target as Element | null;
@@ -69,6 +70,11 @@ if (!isProduction(window.SignalContext.getVersion())) {
     },
     sqlCall: (name: string, ...args: ReadonlyArray<unknown>) =>
       ipcInvoke(name, args),
+    ...(window.SignalContext.config.ciMode === 'benchmark'
+      ? {
+          benchmarkConversationOpen,
+        }
+      : {}),
   };
 
   contextBridge.exposeInMainWorld('SignalDebug', SignalDebug);
@@ -80,7 +86,7 @@ if (getEnvironment() === Environment.Test) {
   contextBridge.exposeInMainWorld('testUtilities', window.testUtilities);
 }
 
-if (process.env.SIGNAL_CI_CONFIG) {
+if (window.SignalContext.config.ciMode === 'full') {
   contextBridge.exposeInMainWorld('SignalCI', window.SignalCI);
 }