Storage service tests and benches in ts/test-mock

ts/test-mock/benchmarks/convo_open_bench.ts

@@ -0,0 +1,101 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+/* eslint-disable no-await-in-loop, no-console */
+
+import type { PrimaryDevice } from '@signalapp/mock-server';
+
+import { Bootstrap, debug, stats, RUN_COUNT, DISCARD_COUNT } from './fixtures';
+
+const CONVERSATION_SIZE = 1000; // messages
+const DELAY = 50; // milliseconds
+
+(async () => {
+  const bootstrap = new Bootstrap({
+    benchmark: true,
+  });
+
+  await bootstrap.init();
+  const app = await bootstrap.link();
+
+  try {
+    const { server, contacts, phone, desktop } = bootstrap;
+
+    const [first, second] = contacts;
+
+    const messages = new Array<Buffer>();
+    debug('encrypting');
+    // Send messages from just two contacts
+    for (const contact of [second, first]) {
+      for (let i = 0; i < CONVERSATION_SIZE; i += 1) {
+        const messageTimestamp = bootstrap.getTimestamp();
+        messages.push(
+          await contact.encryptText(
+            desktop,
+            `hello from: ${contact.profileName}`,
+            {
+              timestamp: messageTimestamp,
+              sealed: true,
+            }
+          )
+        );
+
+        messages.push(
+          await phone.encryptSyncRead(desktop, {
+            timestamp: bootstrap.getTimestamp(),
+            messages: [
+              {
+                senderUUID: contact.device.uuid,
+                timestamp: messageTimestamp,
+              },
+            ],
+          })
+        );
+      }
+    }
+
+    const sendQueue = async (): Promise<void> => {
+      await Promise.all(messages.map(message => server.send(desktop, message)));
+    };
+
+    const measure = async (): Promise<void> => {
+      const window = await app.getWindow();
+
+      const leftPane = window.locator('.left-pane-wrapper');
+
+      const openConvo = async (contact: PrimaryDevice): Promise<void> => {
+        debug('opening conversation', contact.profileName);
+        const item = leftPane.locator(
+          '_react=BaseConversationListItem' +
+            `[title = ${JSON.stringify(contact.profileName)}]`
+        );
+
+        await item.click();
+      };
+
+      const deltaList = new Array<number>();
+      for (let runId = 0; runId < RUN_COUNT + DISCARD_COUNT; runId += 1) {
+        await openConvo(runId % 2 === 0 ? first : second);
+
+        debug('waiting for timing from the app');
+        const { delta } = await app.waitForConversationOpen();
+
+        // Let render complete
+        await new Promise(resolve => setTimeout(resolve, DELAY));
+
+        if (runId >= DISCARD_COUNT) {
+          deltaList.push(delta);
+          console.log('run=%d info=%j', runId - DISCARD_COUNT, { delta });
+        } else {
+          console.log('discarded=%d info=%j', runId, { delta });
+        }
+      }
+
+      console.log('stats info=%j', { delta: stats(deltaList, [99, 99.8]) });
+    };
+
+    await Promise.all([sendQueue(), measure()]);
+  } finally {
+    await app.close();
+    await bootstrap.teardown();
+  }
+})();

ts/test-mock/benchmarks/fixtures.ts

@@ -0,0 +1,67 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+/* eslint-disable no-await-in-loop, no-console */
+
+import createDebug from 'debug';
+
+export const debug = createDebug('mock:benchmarks');
+
+export { Bootstrap } from '../bootstrap';
+export { App } from '../playwright';
+
+export type StatsType = {
+  mean: number;
+  stddev: number;
+  [key: string]: number;
+};
+
+export const RUN_COUNT = process.env.RUN_COUNT
+  ? parseInt(process.env.RUN_COUNT, 10)
+  : 100;
+
+export const GROUP_SIZE = process.env.GROUP_SIZE
+  ? parseInt(process.env.GROUP_SIZE, 10)
+  : 8;
+
+export const DISCARD_COUNT = process.env.DISCARD_COUNT
+  ? parseInt(process.env.DISCARD_COUNT, 10)
+  : 5;
+
+export function stats(
+  list: ReadonlyArray<number>,
+  percentiles: ReadonlyArray<number> = []
+): StatsType {
+  if (list.length === 0) {
+    throw new Error('Empty list given to stats');
+  }
+
+  let mean = 0;
+  let stddev = 0;
+
+  for (const value of list) {
+    mean += value;
+    stddev += value ** 2;
+  }
+  mean /= list.length;
+  stddev /= list.length;
+
+  stddev -= mean ** 2;
+  stddev = Math.sqrt(stddev);
+
+  const sorted = list.slice().sort((a, b) => a - b);
+
+  const result: StatsType = { mean, stddev };
+
+  for (const p of percentiles) {
+    result[`p${p}`] = sorted[Math.floor((sorted.length * p) / 100)];
+  }
+
+  return result;
+}
+
+// Can happen if electron exits prematurely
+process.on('unhandledRejection', reason => {
+  console.error('Unhandled rejection:');
+  console.error(reason);
+  process.exit(1);
+});

ts/test-mock/benchmarks/group_send_bench.ts

@@ -0,0 +1,184 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+/* eslint-disable no-await-in-loop, no-console */
+
+import assert from 'assert';
+
+import {
+  StorageState,
+  EnvelopeType,
+  ReceiptType,
+} from '@signalapp/mock-server';
+import {
+  Bootstrap,
+  debug,
+  stats,
+  RUN_COUNT,
+  GROUP_SIZE,
+  DISCARD_COUNT,
+} from './fixtures';
+
+const CONVERSATION_SIZE = 500; // messages
+const LAST_MESSAGE = 'start sending messages now';
+
+(async () => {
+  const bootstrap = new Bootstrap({
+    benchmark: true,
+  });
+
+  await bootstrap.init();
+
+  const { contacts, phone } = bootstrap;
+
+  const members = [...contacts].slice(0, GROUP_SIZE);
+
+  const group = await phone.createGroup({
+    title: 'Mock Group',
+    members: [phone, ...members],
+  });
+
+  await phone.setStorageState(
+    StorageState.getEmpty()
+      .addGroup(group, { whitelisted: true })
+      .pinGroup(group)
+  );
+
+  const app = await bootstrap.link();
+
+  try {
+    const { server, desktop } = bootstrap;
+    const [first] = members;
+
+    const messages = new Array<Buffer>();
+    debug('encrypting');
+    // Fill left pane
+    for (const contact of members.slice().reverse()) {
+      const messageTimestamp = bootstrap.getTimestamp();
+
+      messages.push(
+        await contact.encryptText(
+          desktop,
+          `hello from: ${contact.profileName}`,
+          {
+            timestamp: messageTimestamp,
+            sealed: true,
+          }
+        )
+      );
+      messages.push(
+        await phone.encryptSyncRead(desktop, {
+          timestamp: bootstrap.getTimestamp(),
+          messages: [
+            {
+              senderUUID: contact.device.uuid,
+              timestamp: messageTimestamp,
+            },
+          ],
+        })
+      );
+    }
+
+    // Fill group
+    for (let i = 0; i < CONVERSATION_SIZE; i += 1) {
+      const contact = members[i % members.length];
+      const messageTimestamp = bootstrap.getTimestamp();
+
+      const isLast = i === CONVERSATION_SIZE - 1;
+
+      messages.push(
+        await contact.encryptText(
+          desktop,
+          isLast ? LAST_MESSAGE : `#${i} from: ${contact.profileName}`,
+          {
+            timestamp: messageTimestamp,
+            sealed: true,
+            group,
+          }
+        )
+      );
+      messages.push(
+        await phone.encryptSyncRead(desktop, {
+          timestamp: bootstrap.getTimestamp(),
+          messages: [
+            {
+              senderUUID: contact.device.uuid,
+              timestamp: messageTimestamp,
+            },
+          ],
+        })
+      );
+    }
+    debug('encrypted');
+
+    await Promise.all(messages.map(message => server.send(desktop, message)));
+
+    const window = await app.getWindow();
+
+    debug('opening conversation');
+    {
+      const leftPane = window.locator('.left-pane-wrapper');
+
+      const item = leftPane.locator(
+        '_react=BaseConversationListItem' +
+          `[title = ${JSON.stringify(group.title)}]` +
+          `>> ${JSON.stringify(LAST_MESSAGE)}`
+      );
+      await item.click();
+    }
+
+    const timeline = window.locator(
+      '.timeline-wrapper, .ConversationView__template .react-wrapper'
+    );
+
+    const deltaList = new Array<number>();
+    for (let runId = 0; runId < RUN_COUNT + DISCARD_COUNT; runId += 1) {
+      debug('finding composition input and clicking it');
+      const composeArea = window.locator(
+        '.composition-area-wrapper, ' +
+          '.ConversationView__template .react-wrapper'
+      );
+
+      const input = composeArea.locator('_react=CompositionInput');
+
+      debug('entering message text');
+      await input.type(`my message ${runId}`);
+      await input.press('Enter');
+
+      debug('waiting for message on server side');
+      const { body, source, envelopeType } = await first.waitForMessage();
+      assert.strictEqual(body, `my message ${runId}`);
+      assert.strictEqual(source, desktop);
+      assert.strictEqual(envelopeType, EnvelopeType.SenderKey);
+
+      debug('waiting for timing from the app');
+      const { timestamp, delta } = await app.waitForMessageSend();
+
+      debug('sending delivery receipts');
+      const delivery = await first.encryptReceipt(desktop, {
+        timestamp: timestamp + 1,
+        messageTimestamps: [timestamp],
+        type: ReceiptType.Delivery,
+      });
+
+      await server.send(desktop, delivery);
+
+      debug('waiting for message state change');
+      const message = timeline.locator(
+        `_react=Message[timestamp = ${timestamp}][status = "delivered"]`
+      );
+      await message.waitFor();
+
+      if (runId >= DISCARD_COUNT) {
+        deltaList.push(delta);
+        console.log('run=%d info=%j', runId - DISCARD_COUNT, { delta });
+      } else {
+        console.log('discarded=%d info=%j', runId, { delta });
+      }
+    }
+
+    console.log('stats info=%j', { delta: stats(deltaList, [99, 99.8]) });
+  } finally {
+    await app.close();
+    await bootstrap.teardown();
+  }
+})();

ts/test-mock/benchmarks/send_bench.ts

@@ -0,0 +1,135 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+/* eslint-disable no-await-in-loop, no-console */
+
+import assert from 'assert';
+
+import { ReceiptType } from '@signalapp/mock-server';
+
+import { Bootstrap, debug, stats, RUN_COUNT, DISCARD_COUNT } from './fixtures';
+
+const CONVERSATION_SIZE = 500; // messages
+
+const LAST_MESSAGE = 'start sending messages now';
+
+(async () => {
+  const bootstrap = new Bootstrap({
+    benchmark: true,
+  });
+
+  await bootstrap.init();
+  const app = await bootstrap.link();
+
+  try {
+    const { server, contacts, phone, desktop } = bootstrap;
+
+    const [first] = contacts;
+
+    const messages = new Array<Buffer>();
+    debug('encrypting');
+    // Note: make it so that we receive the latest message from the first
+    // contact.
+    for (const contact of contacts.slice().reverse()) {
+      let count = 1;
+      if (contact === first) {
+        count = CONVERSATION_SIZE;
+      }
+
+      for (let i = 0; i < count; i += 1) {
+        const messageTimestamp = bootstrap.getTimestamp();
+
+        const isLast = i === count - 1;
+
+        messages.push(
+          await contact.encryptText(
+            desktop,
+            isLast ? LAST_MESSAGE : `#${i} from: ${contact.profileName}`,
+            {
+              timestamp: messageTimestamp,
+              sealed: true,
+            }
+          )
+        );
+        messages.push(
+          await phone.encryptSyncRead(desktop, {
+            timestamp: bootstrap.getTimestamp(),
+            messages: [
+              {
+                senderUUID: contact.device.uuid,
+                timestamp: messageTimestamp,
+              },
+            ],
+          })
+        );
+      }
+    }
+
+    await Promise.all(messages.map(message => server.send(desktop, message)));
+
+    const window = await app.getWindow();
+
+    debug('opening conversation');
+    {
+      const leftPane = window.locator('.left-pane-wrapper');
+      const item = leftPane.locator(
+        '_react=BaseConversationListItem' +
+          `[title = ${JSON.stringify(first.profileName)}]` +
+          `>> ${JSON.stringify(LAST_MESSAGE)}`
+      );
+      await item.click();
+    }
+
+    const timeline = window.locator(
+      '.timeline-wrapper, .ConversationView__template .react-wrapper'
+    );
+
+    const deltaList = new Array<number>();
+    for (let runId = 0; runId < RUN_COUNT + DISCARD_COUNT; runId += 1) {
+      debug('finding composition input and clicking it');
+      const composeArea = window.locator(
+        '.composition-area-wrapper, ' +
+          '.ConversationView__template .react-wrapper'
+      );
+      const input = composeArea.locator('_react=CompositionInput');
+
+      debug('entering message text');
+      await input.type(`my message ${runId}`);
+      await input.press('Enter');
+
+      debug('waiting for message on server side');
+      const { body, source } = await first.waitForMessage();
+      assert.strictEqual(body, `my message ${runId}`);
+      assert.strictEqual(source, desktop);
+
+      debug('waiting for timing from the app');
+      const { timestamp, delta } = await app.waitForMessageSend();
+
+      debug('sending delivery receipt');
+      const delivery = await first.encryptReceipt(desktop, {
+        timestamp: timestamp + 1,
+        messageTimestamps: [timestamp],
+        type: ReceiptType.Delivery,
+      });
+
+      await server.send(desktop, delivery);
+
+      debug('waiting for message state change');
+      const message = timeline.locator(
+        `_react=Message[timestamp = ${timestamp}][status = "delivered"]`
+      );
+      await message.waitFor();
+
+      if (runId >= DISCARD_COUNT) {
+        deltaList.push(delta);
+        console.log('run=%d info=%j', runId - DISCARD_COUNT, { delta });
+      } else {
+        console.log('discarded=%d info=%j', runId, { delta });
+      }
+    }
+
+    console.log('stats info=%j', { delta: stats(deltaList, [99, 99.8]) });
+  } finally {
+    await app.close();
+    await bootstrap.teardown();
+  }
+})();

ts/test-mock/benchmarks/startup_bench.ts

@@ -0,0 +1,133 @@
+// Copyright 2022 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+/* eslint-disable no-await-in-loop, no-console */
+
+import { ReceiptType } from '@signalapp/mock-server';
+
+import { debug, Bootstrap, stats, RUN_COUNT } from './fixtures';
+
+const MESSAGE_BATCH_SIZE = 1000; // messages
+
+const ENABLE_RECEIPTS = Boolean(process.env.ENABLE_RECEIPTS);
+
+(async () => {
+  const bootstrap = new Bootstrap({
+    benchmark: true,
+  });
+
+  await bootstrap.init();
+  await bootstrap.linkAndClose();
+
+  try {
+    const { server, contacts, phone, desktop } = bootstrap;
+
+    const messagesPerSec = new Array<number>();
+
+    for (let runId = 0; runId < RUN_COUNT; runId += 1) {
+      // Generate messages
+      const messagePromises = new Array<Promise<Buffer>>();
+      debug('started generating messages');
+
+      for (let i = 0; i < MESSAGE_BATCH_SIZE; i += 1) {
+        const contact = contacts[Math.floor(i / 2) % contacts.length];
+        const direction = i % 2 ? 'message' : 'reply';
+
+        const messageTimestamp = bootstrap.getTimestamp();
+
+        if (direction === 'message') {
+          messagePromises.push(
+            contact.encryptText(
+              desktop,
+              `Ping from mock server ${i + 1} / ${MESSAGE_BATCH_SIZE}`,
+              {
+                timestamp: messageTimestamp,
+                sealed: true,
+              }
+            )
+          );
+
+          if (ENABLE_RECEIPTS) {
+            messagePromises.push(
+              phone.encryptSyncRead(desktop, {
+                timestamp: bootstrap.getTimestamp(),
+                messages: [
+                  {
+                    senderUUID: contact.device.uuid,
+                    timestamp: messageTimestamp,
+                  },
+                ],
+              })
+            );
+          }
+          continue;
+        }
+
+        messagePromises.push(
+          phone.encryptSyncSent(
+            desktop,
+            `Pong from mock server ${i + 1} / ${MESSAGE_BATCH_SIZE}`,
+            {
+              timestamp: messageTimestamp,
+              destinationUUID: contact.device.uuid,
+            }
+          )
+        );
+
+        if (ENABLE_RECEIPTS) {
+          messagePromises.push(
+            contact.encryptReceipt(desktop, {
+              timestamp: bootstrap.getTimestamp(),
+              messageTimestamps: [messageTimestamp],
+              type: ReceiptType.Delivery,
+            })
+          );
+          messagePromises.push(
+            contact.encryptReceipt(desktop, {
+              timestamp: bootstrap.getTimestamp(),
+              messageTimestamps: [messageTimestamp],
+              type: ReceiptType.Read,
+            })
+          );
+        }
+      }
+
+      debug('ended generating messages');
+
+      const messages = await Promise.all(messagePromises);
+
+      // Open the flood gates
+      {
+        debug('got synced, sending messages');
+
+        // Queue all messages
+        const queue = async (): Promise<void> => {
+          await Promise.all(
+            messages.map(message => {
+              return server.send(desktop, message);
+            })
+          );
+        };
+
+        const run = async (): Promise<void> => {
+          const app = await bootstrap.startApp();
+          const appLoadedInfo = await app.waitUntilLoaded();
+
+          console.log('run=%d info=%j', runId, appLoadedInfo);
+
+          messagesPerSec.push(appLoadedInfo.messagesPerSec);
+
+          await app.close();
+        };
+
+        await Promise.all([queue(), run()]);
+      }
+    }
+
+    // Compute human-readable statistics
+    if (messagesPerSec.length !== 0) {
+      console.log('stats info=%j', { messagesPerSec: stats(messagesPerSec) });
+    }
+  } finally {
+    await bootstrap.teardown();
+  }
+})();