Fully move backup integration test to mock server

.github/workflows/benchmark.yml

@@ -170,6 +170,20 @@ jobs:
         # DEBUG: 'mock:benchmarks'
         ARTIFACTS_DIR: artifacts/call-history-search
 
+    - name: Run backup benchmarks
+      run: |
+        set -o pipefail
+        rm -rf /tmp/mock
+        xvfb-run --auto-servernum node \
+          ts/test-mock/benchmarks/backup_bench.js | \
+          tee benchmark-backup.log
+      timeout-minutes: 10
+      env:
+        NODE_ENV: production
+        ELECTRON_ENABLE_STACK_DUMPING: on
+        # DEBUG: 'mock:benchmarks'
+        ARTIFACTS_DIR: artifacts/backup-bench
+
     - name: Upload benchmark logs on failure
       if: failure()
       uses: actions/upload-artifact@v4
@@ -200,5 +214,6 @@ jobs:
         node ./bin/publish.js ../benchmark-large-group-send.log desktop.ci.performance.largeGroupSend
         node ./bin/publish.js ../benchmark-convo-open.log desktop.ci.performance.convoOpen
         node ./bin/publish.js ../benchmark-call-history-search.log desktop.ci.performance.callHistorySearch
+        node ./bin/publish.js ../benchmark-backup.log desktop.ci.performance.backup
       env:
         DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}

package-lock.json

@@ -126,7 +126,7 @@
         "@indutny/parallel-prettier": "3.0.0",
         "@indutny/rezip-electron": "1.3.1",
         "@indutny/symbolicate-mac": "2.3.0",
-        "@signalapp/mock-server": "7.0.1",
+        "@signalapp/mock-server": "7.1.3",
         "@storybook/addon-a11y": "8.1.11",
         "@storybook/addon-actions": "8.1.11",
         "@storybook/addon-controls": "8.1.11",
@@ -7274,22 +7274,24 @@
       }
     },
     "node_modules/@signalapp/mock-server": {
-      "version": "7.0.1",
+      "version": "7.1.3",
-      "resolved": "https://registry.npmjs.org/@signalapp/mock-server/-/mock-server-7.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/@signalapp/mock-server/-/mock-server-7.1.3.tgz",
-      "integrity": "sha512-iwH57apXyTHKjozaV1ZJW6nbVhFH3KlVOQYaJiO2bT3YgAGdYoJvHp8+MMIQ8OFYVGRo3g7wouqX/JT5HElAvw==",
+      "integrity": "sha512-Xvpeai+E0mhz6WHSycYuY31y5saCNJYX7ioDn1Q0LqUAOUKGVQjnWvdxeXLPKv8C06mbWn0lP16o9swClWVsmg==",
       "dev": true,
       "license": "AGPL-3.0-only",
       "dependencies": {
         "@indutny/parallel-prettier": "^3.0.0",
-        "@signalapp/libsignal-client": "^0.45.0",
+        "@signalapp/libsignal-client": "^0.58.2",
         "@tus/file-store": "^1.4.0",
         "@tus/server": "^1.7.0",
         "debug": "^4.3.2",
+        "is-plain-obj": "3.0.0",
         "long": "^4.0.0",
         "micro": "^9.3.4",
         "microrouter": "^3.1.3",
         "prettier": "^3.3.3",
         "protobufjs": "^7.2.4",
+        "type-fest": "^4.26.1",
         "url-pattern": "^1.0.3",
         "uuid": "^8.3.2",
         "ws": "^8.4.2",
@@ -7297,14 +7299,15 @@
       }
     },
     "node_modules/@signalapp/mock-server/node_modules/@signalapp/libsignal-client": {
-      "version": "0.45.1",
+      "version": "0.58.2",
-      "resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.45.1.tgz",
+      "resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.58.2.tgz",
-      "integrity": "sha512-jKNGLD8QQkLEopX7Fb5XG7LlIe559TgqfC1UCgUV9YW4pPpvM+RPbW4ndL1v8WO/Toff4nVXJXJV6kzYiK2lDA==",
+      "integrity": "sha512-3OF9fGmh7tz9JVfT9xTR4DWcm4HOpbQknO9k7Oj23uSsBSEcJYmYPGM3Rdm16C/z+evVgyvrLptPtjTzXXuNzA==",
       "dev": true,
       "hasInstallScript": true,
+      "license": "AGPL-3.0-only",
       "dependencies": {
-        "node-gyp-build": "^4.2.3",
+        "node-gyp-build": "^4.8.0",
-        "type-fest": "^3.5.0",
+        "type-fest": "^4.26.0",
         "uuid": "^8.3.0"
       }
     },
@@ -7325,6 +7328,19 @@
         }
       }
     },
+    "node_modules/@signalapp/mock-server/node_modules/is-plain-obj": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz",
+      "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/@signalapp/mock-server/node_modules/ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -7332,12 +7348,13 @@
       "dev": true
     },
     "node_modules/@signalapp/mock-server/node_modules/type-fest": {
-      "version": "3.13.1",
+      "version": "4.26.1",
-      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.13.1.tgz",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.26.1.tgz",
-      "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==",
+      "integrity": "sha512-yOGpmOAL7CkKe/91I5O3gPICmJNLJ1G4zFYVAsRHg7M64biSnPtRj0WNQt++bRkjYOqjWXrhnUw1utzmVErAdg==",
       "dev": true,
+      "license": "(MIT OR CC0-1.0)",
       "engines": {
-        "node": ">=14.16"
+        "node": ">=16"
       },
       "funding": {
         "url": "https://github.com/sponsors/sindresorhus"
@@ -7348,6 +7365,7 @@
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
       "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
       "dev": true,
+      "license": "MIT",
       "bin": {
         "uuid": "dist/bin/uuid"
       }

package.json

@@ -210,7 +210,7 @@
     "@indutny/parallel-prettier": "3.0.0",
     "@indutny/rezip-electron": "1.3.1",
     "@indutny/symbolicate-mac": "2.3.0",
-    "@signalapp/mock-server": "7.0.1",
+    "@signalapp/mock-server": "7.1.3",
     "@storybook/addon-a11y": "8.1.11",
     "@storybook/addon-actions": "8.1.11",
     "@storybook/addon-controls": "8.1.11",

ts/CI.ts

@@ -3,14 +3,13 @@
 
 import { format } from 'node:util';
 import { ipcRenderer } from 'electron';
-import { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
 
 import type { IPCResponse as ChallengeResponseType } from './challenge';
 import type { MessageAttributesType } from './model-types.d';
 import * as log from './logging/log';
 import { explodePromise } from './util/explodePromise';
 import { AccessType, ipcInvoke } from './sql/channels';
-import { backupsService, BackupType } from './services/backups';
+import { backupsService } from './services/backups';
 import { SECOND } from './util/durations';
 import { isSignalRoute } from './util/signalRoutes';
 import { strictAssert } from './util/assert';
@@ -19,7 +18,6 @@ type ResolveType = (data: unknown) => void;
 
 export type CIType = {
   deviceName: string;
-  backupData?: Uint8Array;
   getConversationId: (address: string | null) => string | null;
   getMessagesBySentAt(
     sentAt: number
@@ -36,18 +34,16 @@ export type CIType = {
     }
   ) => unknown;
   openSignalRoute(url: string): Promise<void>;
-  exportBackupToDisk(path: string): Promise<void>;
+  uploadBackup(): Promise<void>;
-  exportPlaintextBackupToDisk(path: string): Promise<void>;
   unlink: () => void;
   print: (...args: ReadonlyArray<unknown>) => void;
 };
 
 export type GetCIOptionsType = Readonly<{
   deviceName: string;
-  backupData?: Uint8Array;
 }>;
 
-export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
+export function getCI({ deviceName }: GetCIOptionsType): CIType {
   const eventListeners = new Map<string, Array<ResolveType>>();
   const completedEvents = new Map<string, Array<unknown>>();
 
@@ -66,8 +62,8 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
 
     if (!options?.ignorePastEvents) {
       const pendingCompleted = completedEvents.get(event) || [];
+      if (pendingCompleted.length) {
         const pending = pendingCompleted.shift();
-      if (pending) {
         log.info(`CI: resolving pending result for ${event}`, pending);
 
         if (pendingCompleted.length === 0) {
@@ -170,16 +166,8 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
     document.body.removeChild(a);
   }
 
-  async function exportBackupToDisk(path: string) {
+  async function uploadBackup() {
-    await backupsService.exportToDisk(path, BackupLevel.Media);
+    await backupsService.upload();
-  }
-
-  async function exportPlaintextBackupToDisk(path: string) {
-    await backupsService.exportToDisk(
-      path,
-      BackupLevel.Media,
-      BackupType.TestOnlyPlaintext
-    );
   }
 
   function unlink() {
@@ -192,7 +180,6 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
 
   return {
     deviceName,
-    backupData,
     getConversationId,
     getMessagesBySentAt,
     handleEvent,
@@ -200,8 +187,7 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
     solveChallenge,
     waitForEvent,
     openSignalRoute,
-    exportBackupToDisk,
+    uploadBackup,
-    exportPlaintextBackupToDisk,
     unlink,
     getPendingEventCount,
     print,

ts/background.ts

@@ -1598,8 +1598,10 @@ export async function startApp(): Promise<void> {
         },
       });
 
+      log.info('afterStart: backup downloaded, resolving');
       backupReady.resolve();
     } catch (error) {
+      log.error('afterStart: backup download failed, rejecting');
       backupReady.reject(error);
       throw error;
     }
@@ -1706,17 +1708,21 @@ export async function startApp(): Promise<void> {
 
     strictAssert(server !== undefined, 'WebAPI not connected');
 
+    try {
+      connectPromise = explodePromise();
+
       // Wait for backup to be downloaded
       try {
         await backupReady.promise;
       } catch (error) {
-      log.error('background: backup download failed, not reconnecting', error);
+        log.error(
+          'background: backup download failed, not reconnecting',
+          error
+        );
         return;
       }
       log.info('background: connect unblocked by backups');
 
-    try {
-      connectPromise = explodePromise();
       // Reset the flag and update it below if needed
       setIsInitialSync(false);
 

ts/services/allLoaders.ts

@@ -26,7 +26,8 @@ import {
   getInitialState as getStickersReduxState,
 } from '../types/Stickers';
 
-import type { ReduxInitData } from '../state/initializeRedux';
+import { type ReduxInitData } from '../state/initializeRedux';
+import { reinitializeRedux } from '../state/reinitializeRedux';
 
 export async function loadAll(): Promise<void> {
   await Promise.all([
@@ -41,6 +42,11 @@ export async function loadAll(): Promise<void> {
   ]);
 }
 
+export async function loadAllAndReinitializeRedux(): Promise<void> {
+  await loadAll();
+  reinitializeRedux(getParametersForRedux());
+}
+
 export function getParametersForRedux(): ReduxInitData {
   const { mainWindowStats, menuOptions, theme } = getUserDataForRedux();
 

ts/services/backups/api.ts

@@ -25,7 +25,6 @@ export class BackupAPI {
   constructor(private credentials: BackupCredentials) {}
 
   public async refresh(): Promise<void> {
-    // TODO: DESKTOP-6979
     await this.server.refreshBackup(
       await this.credentials.getHeadersForToday()
     );

ts/services/backups/import.ts

@@ -18,7 +18,7 @@ import {
 import * as log from '../../logging/log';
 import { GiftBadgeStates } from '../../components/conversation/Message';
 import { StorySendMode, MY_STORY_ID } from '../../types/Stories';
-import type { ServiceIdString, AciString } from '../../types/ServiceId';
+import type { ServiceIdString } from '../../types/ServiceId';
 import {
   fromAciObject,
   fromPniObject,
@@ -64,7 +64,6 @@ import {
 } from '../../util/zkgroup';
 import { incrementMessageCounter } from '../../util/incrementMessageCounter';
 import { isAciString } from '../../util/isAciString';
-import { createBatcher } from '../../util/batcher';
 import { PhoneNumberDiscoverability } from '../../util/phoneNumberDiscoverability';
 import { PhoneNumberSharingMode } from '../../util/phoneNumberSharingMode';
 import { bytesToUuid } from '../../util/uuidToBytes';
@@ -79,7 +78,6 @@ import type { AboutMe, LocalChatStyle } from './types';
 import { BackupType } from './types';
 import type { GroupV2ChangeDetailType } from '../../groups';
 import { queueAttachmentDownloads } from '../../util/queueAttachmentDownloads';
-import { drop } from '../../util/drop';
 import { isNotNil } from '../../util/isNotNil';
 import { isGroup } from '../../util/whatTypeOfConversation';
 import { rgbToHSL } from '../../util/rgbToHSL';
@@ -107,89 +105,23 @@ import type { CallLinkType } from '../../types/CallLink';
 import type { RawBodyRange } from '../../types/BodyRange';
 import { fromAdminKeyBytes } from '../../util/callLinks';
 import { getRoomIdFromRootKey } from '../../util/callLinksRingrtc';
-import { reinitializeRedux } from '../../state/reinitializeRedux';
+import { loadAllAndReinitializeRedux } from '../allLoaders';
-import { getParametersForRedux, loadAll } from '../allLoaders';
 import { resetBackupMediaDownloadProgress } from '../../util/backupMediaDownload';
 import { getEnvironment, isTestEnvironment } from '../../environment';
 
 const MAX_CONCURRENCY = 10;
 
+const CONVERSATION_OP_BATCH_SIZE = 10000;
+const SAVE_MESSAGE_BATCH_SIZE = 10000;
+
 // Keep 1000 recent messages in memory to speed up quote lookup.
 const RECENT_MESSAGES_CACHE_SIZE = 1000;
 
-type ConversationOpType = Readonly<{
-  isUpdate: boolean;
-  attributes: ConversationAttributesType;
-}>;
-
 type ChatItemParseResult = {
   message: Partial<MessageAttributesType>;
   additionalMessages: Array<Partial<MessageAttributesType>>;
 };
 
-async function processConversationOpBatch(
-  batch: ReadonlyArray<ConversationOpType>
-): Promise<void> {
-  // Note that we might have duplicates since we update attributes in-place
-  const saves = [
-    ...new Set(batch.filter(x => x.isUpdate === false).map(x => x.attributes)),
-  ];
-  const updates = [
-    ...new Set(batch.filter(x => x.isUpdate === true).map(x => x.attributes)),
-  ];
-
-  log.info(
-    `backups: running conversation op batch, saves=${saves.length} ` +
-      `updates=${updates.length}`
-  );
-
-  await DataWriter.saveConversations(saves);
-  await DataWriter.updateConversations(updates);
-}
-async function processMessagesBatch(
-  ourAci: AciString,
-  batch: ReadonlyArray<MessageAttributesType>
-): Promise<void> {
-  const ids = await DataWriter.saveMessages(batch, {
-    forceSave: true,
-    ourAci,
-  });
-  strictAssert(ids.length === batch.length, 'Should get same number of ids');
-
-  // TODO (DESKTOP-7402): consider re-saving after updating the pending state
-  for (const [index, rawAttributes] of batch.entries()) {
-    const attributes = {
-      ...rawAttributes,
-      id: ids[index],
-    };
-
-    const { editHistory } = attributes;
-
-    if (editHistory?.length) {
-      drop(
-        DataWriter.saveEditedMessages(
-          attributes,
-          ourAci,
-          editHistory.slice(0, -1).map(({ timestamp }) => ({
-            conversationId: attributes.conversationId,
-            messageId: attributes.id,
-
-            // Main message will track this
-            readStatus: ReadStatus.Read,
-            sentAt: timestamp,
-          }))
-        )
-      );
-    }
-
-    drop(
-      queueAttachmentDownloads(attributes, {
-        source: AttachmentDownloadSource.BACKUP_IMPORT,
-      })
-    );
-  }
-}
-
 function phoneToContactFormType(
   type: Backups.ContactAttachment.Phone.Type | null | undefined
 ): ContactFormType {
@@ -269,26 +201,11 @@ export class BackupImportStream extends Writable {
     number,
     ConversationAttributesType
   >();
-  private readonly conversationOpBatcher = createBatcher<{
+  private readonly conversationOpBatch = new Map<
-    isUpdate: boolean;
+    ConversationAttributesType,
-    attributes: ConversationAttributesType;
+    'save' | 'update'
-  }>({
+  >();
-    name: 'BackupImport.conversationOpBatcher',
+  private readonly saveMessageBatch = new Set<MessageAttributesType>();
-    wait: 0,
-    maxSize: 1000,
-    processBatch: processConversationOpBatch,
-  });
-  private readonly saveMessageBatcher = createBatcher<MessageAttributesType>({
-    name: 'BackupImport.saveMessageBatcher',
-    wait: 0,
-    maxSize: 1000,
-    processBatch: batch => {
-      const ourAci = this.ourConversation?.serviceId;
-      assertDev(isAciString(ourAci), 'Our conversation must have ACI');
-
-      return processMessagesBatch(ourAci, batch);
-    },
-  });
   private readonly stickerPacks = new Array<StickerPackPointerType>();
   private ourConversation?: ConversationAttributesType;
   private pinnedConversations = new Array<[number, string]>();
@@ -298,7 +215,7 @@ export class BackupImportStream extends Writable {
   private pendingGroupAvatars = new Map<string, string>();
   private recentMessages = new CircularMessageCache({
     size: RECENT_MESSAGES_CACHE_SIZE,
-    flush: () => this.saveMessageBatcher.flushAndWait(),
+    flush: () => this.flushMessages(),
   });
 
   private constructor(private readonly backupType: BackupType) {
@@ -360,8 +277,9 @@ export class BackupImportStream extends Writable {
   override async _final(done: (error?: Error) => void): Promise<void> {
     try {
       // Finish saving remaining conversations/messages
-      await this.conversationOpBatcher.flushAndWait();
+      await this.flushConversations();
-      await this.saveMessageBatcher.flushAndWait();
+      await this.flushMessages();
+      log.info(`${this.logId}: flushed messages and conversations`);
 
       // Store sticker packs and schedule downloads
       await createPacksFromBackup(this.stickerPacks);
@@ -408,8 +326,7 @@ export class BackupImportStream extends Writable {
           .map(([, id]) => id)
       );
 
-      await loadAll();
+      await loadAllAndReinitializeRedux();
-      reinitializeRedux(getParametersForRedux());
 
       await window.storage.put(
         'backupMediaDownloadTotalBytes',
@@ -429,11 +346,6 @@ export class BackupImportStream extends Writable {
     }
   }
 
-  public cleanup(): void {
-    this.conversationOpBatcher.unregister();
-    this.saveMessageBatcher.unregister();
-  }
-
   private async processFrame(
     frame: Backups.Frame,
     options: { aboutMe?: AboutMe }
@@ -487,7 +399,7 @@ export class BackupImportStream extends Writable {
         }
 
         if (convo !== this.ourConversation) {
-          this.saveConversation(convo);
+          await this.saveConversation(convo);
         }
 
         this.recipientIdToConvo.set(recipientId, convo);
@@ -516,17 +428,95 @@ export class BackupImportStream extends Writable {
     }
   }
 
-  private saveConversation(attributes: ConversationAttributesType): void {
+  private async saveConversation(
-    this.conversationOpBatcher.add({ isUpdate: false, attributes });
+    attributes: ConversationAttributesType
+  ): Promise<void> {
+    this.conversationOpBatch.set(attributes, 'save');
+    if (this.conversationOpBatch.size >= CONVERSATION_OP_BATCH_SIZE) {
+      return this.flushConversations();
+    }
   }
 
-  private updateConversation(attributes: ConversationAttributesType): void {
+  private async updateConversation(
-    this.conversationOpBatcher.add({ isUpdate: true, attributes });
+    attributes: ConversationAttributesType
+  ): Promise<void> {
+    if (!this.conversationOpBatch.has(attributes)) {
+      this.conversationOpBatch.set(attributes, 'update');
     }
 
-  private saveMessage(attributes: MessageAttributesType): void {
+    if (this.conversationOpBatch.size >= CONVERSATION_OP_BATCH_SIZE) {
+      return this.flushConversations();
+    }
+  }
+
+  private async saveMessage(attributes: MessageAttributesType): Promise<void> {
     this.recentMessages.push(attributes);
-    this.saveMessageBatcher.add(attributes);
+    this.saveMessageBatch.add(attributes);
+    if (this.saveMessageBatch.size >= SAVE_MESSAGE_BATCH_SIZE) {
+      return this.flushMessages();
+    }
+  }
+
+  private async flushConversations(): Promise<void> {
+    const saves = new Array<ConversationAttributesType>();
+    const updates = new Array<ConversationAttributesType>();
+    for (const [conversation, op] of this.conversationOpBatch) {
+      if (op === 'save') {
+        saves.push(conversation);
+      } else {
+        updates.push(conversation);
+      }
+    }
+    this.conversationOpBatch.clear();
+
+    // Queue writes at the same time to prevent races.
+    await Promise.all([
+      saves.length > 0
+        ? DataWriter.saveConversations(saves)
+        : Promise.resolve(),
+      updates.length > 0
+        ? DataWriter.updateConversations(updates)
+        : Promise.resolve(),
+    ]);
+  }
+
+  private async flushMessages(): Promise<void> {
+    const ourAci = this.ourConversation?.serviceId;
+    strictAssert(isAciString(ourAci), 'Must have our aci for messages');
+
+    const batch = Array.from(this.saveMessageBatch);
+    this.saveMessageBatch.clear();
+
+    await DataWriter.saveMessages(batch, {
+      forceSave: true,
+      ourAci,
+    });
+
+    // TODO (DESKTOP-7402): consider re-saving after updating the pending state
+    for (const attributes of batch) {
+      const { editHistory } = attributes;
+
+      if (editHistory?.length) {
+        // eslint-disable-next-line no-await-in-loop
+        await DataWriter.saveEditedMessages(
+          attributes,
+          ourAci,
+          editHistory.slice(0, -1).map(({ timestamp }) => ({
+            conversationId: attributes.conversationId,
+            messageId: attributes.id,
+
+            // Main message will track this
+            readStatus: ReadStatus.Read,
+            sentAt: timestamp,
+          }))
+        );
+      }
+
+      // eslint-disable-next-line no-await-in-loop
+      await queueAttachmentDownloads(attributes, {
+        source: AttachmentDownloadSource.BACKUP_IMPORT,
+      });
+    }
   }
 
   private async saveCallHistory(
@@ -749,7 +739,7 @@ export class BackupImportStream extends Writable {
       );
     }
 
-    this.updateConversation(me);
+    await this.updateConversation(me);
   }
 
   private async fromContact(
@@ -1175,7 +1165,7 @@ export class BackupImportStream extends Writable {
       conversation.autoBubbleColor = chatStyle.autoBubbleColor;
     }
 
-    this.updateConversation(conversation);
+    await this.updateConversation(conversation);
 
     if (chat.pinnedOrder != null) {
       this.pinnedConversations.push([chat.pinnedOrder, conversation.id]);
@@ -1333,8 +1323,10 @@ export class BackupImportStream extends Writable {
       isAciString(this.ourConversation.serviceId),
       `${logId}: Our conversation must have ACI`
     );
-    this.saveMessage(attributes);
+    await Promise.all([
-    additionalMessages.forEach(additional => this.saveMessage(additional));
+      this.saveMessage(attributes),
+      ...additionalMessages.map(additional => this.saveMessage(additional)),
+    ]);
 
     // TODO (DESKTOP-6964): We'll want to increment for more types here - stickers, etc.
     if (item.standardMessage) {
@@ -1344,7 +1336,7 @@ export class BackupImportStream extends Writable {
         chatConvo.messageCount = (chatConvo.messageCount ?? 0) + 1;
       }
     }
-    this.updateConversation(chatConvo);
+    await this.updateConversation(chatConvo);
   }
 
   private fromDirectionDetails(

ts/services/contactSync.ts

@@ -224,6 +224,7 @@ async function doContactSync({
 
   await window.storage.put('synced_at', Date.now());
   window.Whisper.events.trigger('contactSync:complete');
+  window.SignalCI?.handleEvent('contactSync', isFullSync);
 
   log.info(`${logId}: done`);
 }

ts/sql/Server.ts

@@ -1585,11 +1585,13 @@ function saveConversation(db: WritableDB, data: ConversationType): void {
     profileLastFetchedAt,
     type,
     serviceId,
+    expireTimerVersion,
   } = data;
 
   const membersList = getConversationMembersList(data);
 
-  db.prepare<Query>(
+  prepare(
+    db,
     `
     INSERT INTO conversations (
       id,
@@ -1606,7 +1608,8 @@ function saveConversation(db: WritableDB, data: ConversationType): void {
       profileName,
       profileFamilyName,
       profileFullName,
-      profileLastFetchedAt
+      profileLastFetchedAt,
+      expireTimerVersion
     ) values (
       $id,
       $json,
@@ -1622,7 +1625,8 @@ function saveConversation(db: WritableDB, data: ConversationType): void {
       $profileName,
       $profileFamilyName,
       $profileFullName,
-      $profileLastFetchedAt
+      $profileLastFetchedAt,
+      $expireTimerVersion
     );
     `
   ).run({
@@ -1643,6 +1647,7 @@ function saveConversation(db: WritableDB, data: ConversationType): void {
     profileFamilyName: profileFamilyName || null,
     profileFullName: combineNames(profileName, profileFamilyName) || null,
     profileLastFetchedAt: profileLastFetchedAt || null,
+    expireTimerVersion,
   });
 }
 
@@ -1673,7 +1678,8 @@ function updateConversation(db: WritableDB, data: ConversationType): void {
 
   const membersList = getConversationMembersList(data);
 
-  db.prepare(
+  prepare(
+    db,
     `
     UPDATE conversations SET
       json = $json,

ts/state/ducks/installer.ts

@@ -322,7 +322,6 @@ function startInstaller(): ThunkAction<
       dispatch(
         finishInstall({
           deviceName: SignalCI.deviceName,
-          backupFile: SignalCI.backupData,
         })
       );
     }

ts/test-both/helpers/generateBackup.ts

@@ -0,0 +1,219 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { Readable } from 'node:stream';
+import { createGzip } from 'node:zlib';
+import { createCipheriv, randomBytes } from 'node:crypto';
+import { Buffer } from 'node:buffer';
+import Long from 'long';
+
+import type { AciString } from '../../types/ServiceId';
+import { generateAci } from '../../types/ServiceId';
+import { CipherType } from '../../types/Crypto';
+import { appendPaddingStream } from '../../util/logPadding';
+import { prependStream } from '../../util/prependStream';
+import { appendMacStream } from '../../util/appendMacStream';
+import { toAciObject } from '../../util/ServiceId';
+import {
+  deriveBackupKey,
+  deriveBackupId,
+  deriveBackupKeyMaterial,
+} from '../../Crypto';
+import { BACKUP_VERSION } from '../../services/backups/constants';
+import { Backups } from '../../protobuf';
+
+export type BackupGeneratorConfigType = Readonly<{
+  aci: AciString;
+  profileKey: Buffer;
+  masterKey: Buffer;
+  conversations: number;
+  messages: number;
+}>;
+
+const IV_LENGTH = 16;
+
+export type GenerateBackupResultType = Readonly<{
+  backupId: Buffer;
+  stream: Readable;
+}>;
+
+export function generateBackup(
+  options: BackupGeneratorConfigType
+): GenerateBackupResultType {
+  const { aci, masterKey } = options;
+  const backupKey = deriveBackupKey(masterKey);
+  const aciBytes = toAciObject(aci).getServiceIdBinary();
+  const backupId = Buffer.from(deriveBackupId(backupKey, aciBytes));
+  const { aesKey, macKey } = deriveBackupKeyMaterial(backupKey, backupId);
+
+  const iv = randomBytes(IV_LENGTH);
+
+  const stream = Readable.from(createRecords(options))
+    .pipe(createGzip())
+    .pipe(appendPaddingStream())
+    .pipe(createCipheriv(CipherType.AES256CBC, aesKey, iv))
+    .pipe(prependStream(iv))
+    .pipe(appendMacStream(macKey));
+
+  return { backupId, stream };
+}
+
+function frame(data: Backups.IFrame): Buffer {
+  return Buffer.from(Backups.Frame.encodeDelimited(data).finish());
+}
+
+let now = Date.now();
+function getTimestamp(): Long {
+  now += 1;
+  return Long.fromNumber(now);
+}
+
+function* createRecords({
+  profileKey,
+  conversations,
+  messages,
+}: BackupGeneratorConfigType): Iterable<Buffer> {
+  yield Buffer.from(
+    Backups.BackupInfo.encodeDelimited({
+      version: Long.fromNumber(BACKUP_VERSION),
+      backupTimeMs: getTimestamp(),
+    }).finish()
+  );
+
+  // Account data
+  yield frame({
+    account: {
+      profileKey,
+      givenName: 'Backup',
+      familyName: 'Benchmark',
+      accountSettings: {
+        readReceipts: false,
+        sealedSenderIndicators: false,
+        typingIndicators: false,
+        linkPreviews: false,
+        notDiscoverableByPhoneNumber: false,
+        preferContactAvatars: false,
+        universalExpireTimerSeconds: 0,
+        preferredReactionEmoji: [],
+        displayBadgesOnProfile: true,
+        keepMutedChatsArchived: false,
+        hasSetMyStoriesPrivacy: true,
+        hasViewedOnboardingStory: true,
+        storiesDisabled: false,
+        hasSeenGroupStoryEducationSheet: true,
+        hasCompletedUsernameOnboarding: true,
+        phoneNumberSharingMode:
+          Backups.AccountData.PhoneNumberSharingMode.EVERYBODY,
+        defaultChatStyle: {
+          autoBubbleColor: {},
+          dimWallpaperInDarkMode: false,
+        },
+        customChatColors: [],
+      },
+    },
+  });
+
+  const selfId = Long.fromNumber(0);
+
+  yield frame({
+    recipient: {
+      id: selfId,
+      self: {},
+    },
+  });
+
+  const chats = new Array<{
+    id: Long;
+    aci: Buffer;
+  }>();
+
+  for (let i = 1; i <= conversations; i += 1) {
+    const id = Long.fromNumber(i);
+    const chatAci = toAciObject(generateAci()).getRawUuidBytes();
+
+    chats.push({
+      id,
+      aci: chatAci,
+    });
+
+    yield frame({
+      recipient: {
+        id,
+        contact: {
+          aci: chatAci,
+          blocked: false,
+          visibility: Backups.Contact.Visibility.VISIBLE,
+          registered: {},
+          profileKey: randomBytes(32),
+          profileSharing: true,
+          profileGivenName: `Contact ${i}`,
+          profileFamilyName: 'Generated',
+          hideStory: false,
+        },
+      },
+    });
+
+    yield frame({
+      chat: {
+        id,
+        recipientId: id,
+        archived: false,
+        pinnedOrder: 0,
+        expirationTimerMs: Long.fromNumber(0),
+        muteUntilMs: Long.fromNumber(0),
+        markedUnread: false,
+        dontNotifyForMentionsIfMuted: false,
+        style: {
+          autoBubbleColor: {},
+          dimWallpaperInDarkMode: false,
+        },
+        expireTimerVersion: 1,
+      },
+    });
+  }
+
+  for (let i = 0; i < messages; i += 1) {
+    const chat = chats[i % chats.length];
+
+    const isIncoming = i % 2 === 0;
+
+    const dateSent = getTimestamp();
+
+    yield frame({
+      chatItem: {
+        chatId: chat.id,
+        authorId: isIncoming ? chat.id : selfId,
+        dateSent,
+        revisions: [],
+        sms: false,
+
+        ...(isIncoming
+          ? {
+              incoming: {
+                dateReceived: getTimestamp(),
+                dateServerSent: getTimestamp(),
+                read: true,
+                sealedSender: true,
+              },
+            }
+          : {
+              outgoing: {
+                sendStatus: [
+                  {
+                    recipientId: chat.id,
+                    timestamp: dateSent,
+                    sent: { sealedSender: true },
+                  },
+                ],
+              },
+            }),
+
+        standardMessage: {
+          text: {
+            body: `Message ${i}`,
+          },
+        },
+      },
+    });
+  }
+}

ts/test-electron/backup/attachments_test.ts

@@ -31,7 +31,7 @@ import { isVoiceMessage, type AttachmentType } from '../../types/Attachment';
 import { strictAssert } from '../../util/assert';
 import { SignalService } from '../../protobuf';
 import { getRandomBytes } from '../../Crypto';
-import { loadAll } from '../../services/allLoaders';
+import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
 
 const CONTACT_A = generateAci();
 
@@ -65,7 +65,7 @@ describe('backup/attachments', () => {
       { systemGivenName: 'CONTACT_A', active_at: 1 }
     );
 
-    await loadAll();
+    await loadAllAndReinitializeRedux();
 
     sandbox = sinon.createSandbox();
     const getAbsoluteAttachmentPath = sandbox.stub(

ts/test-electron/backup/backup_groupv2_notifications_test.ts

@@ -23,7 +23,7 @@ import {
 } from './helpers';
 import { ReadStatus } from '../../messages/MessageReadStatus';
 import { SeenStatus } from '../../MessageSeenStatus';
-import { loadAll } from '../../services/allLoaders';
+import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
 
 // Note: this should be kept up to date with GroupV2Change.stories.tsx, to
 //   maintain the comprehensive set of GroupV2 notifications we need to handle
@@ -127,7 +127,7 @@ describe('backup/groupv2/notifications', () => {
       active_at: 1,
     });
 
-    await loadAll();
+    await loadAllAndReinitializeRedux();
   });
   afterEach(async () => {
     await DataWriter.removeAll();

ts/test-electron/backup/bubble_test.ts

@@ -21,7 +21,7 @@ import {
   symmetricRoundtripHarness,
   OUR_ACI,
 } from './helpers';
-import { loadAll } from '../../services/allLoaders';
+import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
 
 const CONTACT_A = generateAci();
 const CONTACT_B = generateAci();
@@ -68,7 +68,7 @@ describe('backup/bubble messages', () => {
       }
     );
 
-    await loadAll();
+    await loadAllAndReinitializeRedux();
   });
 
   it('roundtrips incoming edited message', async () => {

ts/test-electron/backup/calling_test.ts

@@ -29,7 +29,7 @@ import { fromAdminKeyBytes } from '../../util/callLinks';
 import { ReadStatus } from '../../messages/MessageReadStatus';
 import { SeenStatus } from '../../MessageSeenStatus';
 import { deriveGroupID, deriveGroupSecretParams } from '../../util/zkgroup';
-import { loadAll } from '../../services/allLoaders';
+import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
 
 const CONTACT_A = generateAci();
 const GROUP_MASTER_KEY = getRandomBytes(32);
@@ -82,7 +82,7 @@ describe('backup/calling', () => {
 
     await DataWriter.insertCallLink(callLink);
 
-    await loadAll();
+    await loadAllAndReinitializeRedux();
   });
   after(async () => {
     await DataWriter.removeAll();
@@ -105,7 +105,7 @@ describe('backup/calling', () => {
         endedTimestamp: null,
       };
       await DataWriter.saveCallHistory(callHistory);
-      await loadAll();
+      await loadAllAndReinitializeRedux();
 
       const messageUnseen: MessageAttributesType = {
         id: generateGuid(),
@@ -154,7 +154,7 @@ describe('backup/calling', () => {
         endedTimestamp: null,
       };
       await DataWriter.saveCallHistory(callHistory);
-      await loadAll();
+      await loadAllAndReinitializeRedux();
 
       const messageUnseen: MessageAttributesType = {
         id: generateGuid(),
@@ -245,7 +245,7 @@ describe('backup/calling', () => {
         endedTimestamp: null,
       };
       await DataWriter.saveCallHistory(callHistory);
-      await loadAll();
+      await loadAllAndReinitializeRedux();
 
       await symmetricRoundtripHarness([]);
 
@@ -271,7 +271,7 @@ describe('backup/calling', () => {
         endedTimestamp: null,
       };
       await DataWriter.saveCallHistory(callHistory);
-      await loadAll();
+      await loadAllAndReinitializeRedux();
 
       await symmetricRoundtripHarness([]);
 

ts/test-electron/backup/integration_test.ts

@@ -14,8 +14,10 @@ import {
 import { assert } from 'chai';
 
 import { clearData } from './helpers';
-import { loadAll } from '../../services/allLoaders';
+import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
 import { backupsService, BackupType } from '../../services/backups';
+import { initialize as initializeExpiringMessageService } from '../../services/expiringMessagesDeletion';
+import { singleProtoJobQueue } from '../../jobs/singleProtoJobQueue';
 import { DataWriter } from '../../sql/Client';
 
 const { BACKUP_INTEGRATION_DIR } = process.env;
@@ -39,9 +41,13 @@ class MemoryStream extends InputStream {
 }
 
 describe('backup/integration', () => {
+  before(async () => {
+    await initializeExpiringMessageService(singleProtoJobQueue);
+  });
+
   beforeEach(async () => {
     await clearData();
-    await loadAll();
+    await loadAllAndReinitializeRedux();
   });
 
   afterEach(async () => {

ts/test-electron/backup/non_bubble_test.ts

@@ -24,7 +24,7 @@ import {
   symmetricRoundtripHarness,
   OUR_ACI,
 } from './helpers';
-import { loadAll } from '../../services/allLoaders';
+import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
 
 const CONTACT_A = generateAci();
 const GROUP_ID = Bytes.toBase64(getRandomBytes(32));
@@ -57,7 +57,7 @@ describe('backup/non-bubble messages', () => {
       }
     );
 
-    await loadAll();
+    await loadAllAndReinitializeRedux();
   });
 
   it('roundtrips END_SESSION simple update', async () => {

ts/test-mock/backups/backups_test.ts

@@ -188,8 +188,7 @@ describe('backups', function (this: Mocha.Suite) {
       );
     }
 
-    const backupPath = bootstrap.getBackupPath('backup.bin');
+    await app.uploadBackup();
-    await app.exportBackupToDisk(backupPath);
 
     const comparator = await bootstrap.createScreenshotComparator(
       app,
@@ -247,9 +246,12 @@ describe('backups', function (this: Mocha.Suite) {
 
     // Restart
     await bootstrap.eraseStorage();
-    app = await bootstrap.link({
+    app = await bootstrap.link();
-      ciBackupPath: backupPath,
+    await app.waitForBackupImportComplete();
-    });
+
+    // Make sure that contact sync happens after backup import, otherwise the
+    // app won't show contacts as "system"
+    await app.waitForContactSync();
 
     await comparator(app);
   });

ts/test-mock/benchmarks/backup_bench.ts

@@ -0,0 +1,47 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+/* eslint-disable no-console */
+
+import { pipeline } from 'node:stream/promises';
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+
+import { Bootstrap } from './fixtures';
+import { generateBackup } from '../../test-both/helpers/generateBackup';
+
+Bootstrap.benchmark(async (bootstrap: Bootstrap): Promise<void> => {
+  const { phone, cdn3Path } = bootstrap;
+
+  const { backupId, stream: backupStream } = generateBackup({
+    aci: phone.device.aci,
+    profileKey: phone.profileKey.serialize(),
+    masterKey: phone.masterKey,
+    conversations: 1000,
+    messages: 60 * 1000,
+  });
+  const backupFolder = join(
+    cdn3Path,
+    'backups',
+    backupId.toString('base64url')
+  );
+  await mkdir(backupFolder, { recursive: true });
+  const fileStream = createWriteStream(join(backupFolder, 'backup'));
+  await pipeline(backupStream, fileStream);
+
+  const importStart = Date.now();
+
+  const app = await bootstrap.link();
+  await app.waitForBackupImportComplete();
+
+  const importEnd = Date.now();
+
+  const exportStart = Date.now();
+  await app.uploadBackup();
+  const exportEnd = Date.now();
+
+  console.log('run=%d info=%j', 0, {
+    importDuration: importEnd - importStart,
+    exportDuration: exportEnd - exportStart,
+  });
+});

ts/test-mock/bootstrap.ts

@@ -150,6 +150,7 @@ function sanitizePathComponent(component: string): string {
 //
 export class Bootstrap {
   public readonly server: Server;
+  public readonly cdn3Path: string;
 
   private readonly options: BootstrapInternalOptions;
   private privContacts?: ReadonlyArray<PrimaryDevice>;
@@ -158,8 +159,6 @@ export class Bootstrap {
   private privPhone?: PrimaryDevice;
   private privDesktop?: Device;
   private storagePath?: string;
-  private backupPath?: string;
-  private cdn3Path: string;
   private timestamp: number = Date.now() - durations.WEEK;
   private lastApp?: App;
   private readonly randomId = crypto.randomBytes(8).toString('hex');
@@ -238,9 +237,6 @@ export class Bootstrap {
     });
 
     this.storagePath = await fs.mkdtemp(path.join(os.tmpdir(), 'mock-signal-'));
-    this.backupPath = await fs.mkdtemp(
-      path.join(os.tmpdir(), 'mock-signal-backup-')
-    );
 
     debug('setting storage path=%j', this.storagePath);
   }
@@ -270,15 +266,6 @@ export class Bootstrap {
     return path.join(this.storagePath, 'ephemeral.json');
   }
 
-  public getBackupPath(fileName: string): string {
-    assert(
-      this.backupPath !== undefined,
-      'Bootstrap has to be initialized first, see: bootstrap.init()'
-    );
-
-    return path.join(this.backupPath, fileName);
-  }
-
   public eraseStorage(): Promise<void> {
     return this.resetAppStorage();
   }
@@ -289,7 +276,6 @@ export class Bootstrap {
       'Bootstrap has to be initialized first, see: bootstrap.init()'
     );
 
-    // Note that backupPath must remain unchanged!
     await fs.rm(this.storagePath, { recursive: true });
     this.storagePath = await fs.mkdtemp(path.join(os.tmpdir(), 'mock-signal-'));
   }
@@ -299,7 +285,7 @@ export class Bootstrap {
 
     await Promise.race([
       Promise.all([
-        ...[this.storagePath, this.backupPath, this.cdn3Path].map(tmpPath =>
+        ...[this.storagePath, this.cdn3Path].map(tmpPath =>
           tmpPath ? fs.rm(tmpPath, { recursive: true }) : Promise.resolve()
         ),
         this.server.close(),
@@ -354,11 +340,6 @@ export class Bootstrap {
       }
     }
 
-    if (extraConfig?.ciBackupPath) {
-      debug('waiting for backup import to complete');
-      await app.waitForBackupImportComplete();
-    }
-
     await this.phone.waitForSync(this.desktop);
     this.phone.resetSyncState(this.desktop);
 
@@ -384,7 +365,7 @@ export class Bootstrap {
 
     debug('starting the app');
 
-    const { port } = this.server.address();
+    const { port, family } = this.server.address();
 
     let startAttempts = 0;
     const MAX_ATTEMPTS = 4;
@@ -398,7 +379,7 @@ export class Bootstrap {
       }
 
       // eslint-disable-next-line no-await-in-loop
-      const config = await this.generateConfig(port, extraConfig);
+      const config = await this.generateConfig(port, family, extraConfig);
 
       const startedApp = new App({
         main: ELECTRON,
@@ -661,9 +642,12 @@ export class Bootstrap {
 
   private async generateConfig(
     port: number,
+    family: string,
     extraConfig?: Partial<RendererConfigType>
   ): Promise<string> {
-    const url = `https://127.0.0.1:${port}`;
+    const host = family === 'IPv6' ? '[::1]' : '127.0.0.1';
+
+    const url = `https://${host}:${port}`;
     return JSON.stringify({
       ...(await loadCertificates()),
 

ts/test-mock/playwright.ts

@@ -105,6 +105,10 @@ export class App extends EventEmitter {
     return this.waitForEvent('app-loaded');
   }
 
+  public async waitForContactSync(): Promise<void> {
+    return this.waitForEvent('contactSync');
+  }
+
   public async waitForBackupImportComplete(): Promise<void> {
     return this.waitForEvent('backupImportComplete');
   }
@@ -186,18 +190,9 @@ export class App extends EventEmitter {
     return window.evaluate(`window.SignalCI.getMessagesBySentAt(${timestamp})`);
   }
 
-  public async exportBackupToDisk(path: string): Promise<Uint8Array> {
+  public async uploadBackup(): Promise<void> {
     const window = await this.getWindow();
-    return window.evaluate(
+    await window.evaluate('window.SignalCI.uploadBackup()');
-      `window.SignalCI.exportBackupToDisk(${JSON.stringify(path)})`
-    );
-  }
-
-  public async exportPlaintextBackupToDisk(path: string): Promise<Uint8Array> {
-    const window = await this.getWindow();
-    return window.evaluate(
-      `window.SignalCI.exportPlaintextBackupToDisk(${JSON.stringify(path)})`
-    );
   }
 
   public async unlink(): Promise<void> {

ts/util/isBackupEnabled.ts

@@ -2,10 +2,11 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import * as RemoteConfig from '../RemoteConfig';
+import { isTestOrMockEnvironment } from '../environment';
 import { isStagingServer } from './isStagingServer';
 
 export function isBackupEnabled(): boolean {
-  if (isStagingServer()) {
+  if (isStagingServer() || isTestOrMockEnvironment()) {
     return true;
   }
   return Boolean(RemoteConfig.isEnabled('desktop.backup.credentialFetch'));

ts/windows/main/phase4-test.ts

@@ -5,8 +5,6 @@
 /* eslint-disable no-console */
 /* eslint-disable global-require */
 
-import fs from 'fs';
-
 const { config } = window.SignalContext;
 
 if (config.environment === 'test') {
@@ -16,14 +14,10 @@ if (config.environment === 'test') {
 
 if (config.ciMode) {
   console.log(
-    `Importing CI infrastructure; enabled in config, mode: ${config.ciMode}, ` +
+    `Importing CI infrastructure; enabled in config, mode: ${config.ciMode}`
-      `backupPath: ${config.ciBackupPath}`
   );
   const { getCI } = require('../../CI');
   window.SignalCI = getCI({
     deviceName: window.getTitle(),
-    backupData: config.ciBackupPath
-      ? fs.readFileSync(config.ciBackupPath)
-      : undefined,
   });
 }