Add backup comparator test harness

app/main.ts

@@ -2721,6 +2721,7 @@ ipc.on('get-config', async event => {
     dnsFallback: await getDNSFallback(),
     disableIPv6: DISABLE_IPV6,
     ciBackupPath: config.get<string | null>('ciBackupPath') || undefined,
+    ciIsPlaintextBackup: config.get<boolean>('ciIsPlaintextBackup'),
     nodeVersion: process.versions.node,
     hostname: os.hostname(),
     osRelease: os.release(),

config/default.json

@@ -18,6 +18,7 @@
   "updatesEnabled": false,
   "ciMode": false,
   "ciBackupPath": null,
+  "ciIsPlaintextBackup": false,
   "forcePreloadBundle": false,
   "openDevTools": false,
   "buildCreation": 0,

ts/CI.ts

@@ -8,7 +8,7 @@ import type { MessageAttributesType } from './model-types.d';
 import * as log from './logging/log';
 import { explodePromise } from './util/explodePromise';
 import { AccessType, ipcInvoke } from './sql/channels';
-import { backupsService } from './services/backups';
+import { backupsService, BackupType } from './services/backups';
 import { SECOND } from './util/durations';
 import { isSignalRoute } from './util/signalRoutes';
 import { strictAssert } from './util/assert';
@@ -18,6 +18,7 @@ type ResolveType = (data: unknown) => void;
 export type CIType = {
   deviceName: string;
   backupData?: Uint8Array;
+  isPlaintextBackup?: boolean;
   getConversationId: (address: string | null) => string | null;
   getMessagesBySentAt(
     sentAt: number
@@ -34,15 +35,21 @@ export type CIType = {
   ) => unknown;
   openSignalRoute(url: string): Promise<void>;
   exportBackupToDisk(path: string): Promise<void>;
+  exportPlaintextBackupToDisk(path: string): Promise<void>;
   unlink: () => void;
 };
 
 export type GetCIOptionsType = Readonly<{
   deviceName: string;
   backupData?: Uint8Array;
+  isPlaintextBackup?: boolean;
 }>;
 
-export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
+export function getCI({
+  deviceName,
+  backupData,
+  isPlaintextBackup,
+}: GetCIOptionsType): CIType {
   const eventListeners = new Map<string, Array<ResolveType>>();
   const completedEvents = new Map<string, Array<unknown>>();
 
@@ -164,6 +171,14 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
     await backupsService.exportToDisk(path);
   }
 
+  async function exportPlaintextBackupToDisk(path: string) {
+    await backupsService.exportToDisk(
+      path,
+      undefined,
+      BackupType.TestOnlyPlaintext
+    );
+  }
+
   function unlink() {
     window.Whisper.events.trigger('unlinkAndDisconnect');
   }
@@ -171,6 +186,7 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
   return {
     deviceName,
     backupData,
+    isPlaintextBackup,
     getConversationId,
     getMessagesBySentAt,
     handleEvent,
@@ -179,6 +195,7 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
     waitForEvent,
     openSignalRoute,
     exportBackupToDisk,
+    exportPlaintextBackupToDisk,
     unlink,
   };
 }

ts/services/backups/export.ts

@@ -278,6 +278,14 @@ export class BackupExportStream extends Readable {
       stats.conversations += 1;
     }
 
+    this.pushFrame({
+      recipient: {
+        id: Long.fromNumber(this.getNextRecipientId()),
+        releaseNotes: {},
+      },
+    });
+    await this.flush();
+
     const distributionLists =
       await DataReader.getAllStoryDistributionsWithMembers();
 
@@ -2401,7 +2409,11 @@ export class BackupExportStream extends Readable {
       const id = Long.fromNumber(result.length);
       this.customColorIdByUuid.set(uuid, id);
 
-      const start = hslToRGBInt(color.start.hue, color.start.saturation);
+      const start = hslToRGBInt(
+        color.start.hue,
+        color.start.saturation,
+        color.start.luminance
+      );
 
       if (color.end == null) {
         result.push({
@@ -2409,7 +2421,11 @@ export class BackupExportStream extends Readable {
           solid: start,
         });
       } else {
-        const end = hslToRGBInt(color.end.hue, color.end.saturation);
+        const end = hslToRGBInt(
+          color.end.hue,
+          color.end.saturation,
+          color.end.luminance
+        );
 
         result.push({
           id,
@@ -2562,8 +2578,8 @@ function checkServiceIdEquivalence(
   return leftConvo && rightConvo && leftConvo === rightConvo;
 }
 
-function hslToRGBInt(hue: number, saturation: number): number {
-  const { r, g, b } = hslToRGB(hue, saturation, 1);
+function hslToRGBInt(hue: number, saturation: number, luminance = 1): number {
+  const { r, g, b } = hslToRGB(hue, saturation, luminance);
   // eslint-disable-next-line no-bitwise
   return ((0xff << 24) | (r << 16) | (g << 8) | b) >>> 0;
 }

ts/services/backups/import.ts

@@ -14,7 +14,7 @@ import { DataWriter } from '../../sql/Client';
 import type { StoryDistributionWithMembersType } from '../../sql/Interface';
 import * as log from '../../logging/log';
 import { GiftBadgeStates } from '../../components/conversation/Message';
-import { StorySendMode } from '../../types/Stories';
+import { StorySendMode, MY_STORY_ID } from '../../types/Stories';
 import type { ServiceIdString, AciString } from '../../types/ServiceId';
 import {
   fromAciObject,
@@ -942,7 +942,7 @@ export class BackupImportStream extends Writable {
       'Missing distribution list id'
     );
 
-    const id = bytesToUuid(listItem.distributionId);
+    const id = bytesToUuid(listItem.distributionId) || MY_STORY_ID;
     strictAssert(isStoryDistributionId(id), 'Invalid distribution list id');
 
     const commonFields = {
@@ -2987,17 +2987,20 @@ export class BackupImportStream extends Writable {
   }
 }
 
-function rgbIntToHSL(intValue: number): { hue: number; saturation: number } {
-  const { h: hue, s: saturation } = rgbToHSL(
+function rgbIntToHSL(intValue: number): {
+  hue: number;
+  saturation: number;
+  luminance: number;
+} {
   // eslint-disable-next-line no-bitwise
-    (intValue >>> 16) & 0xff,
+  const r = (intValue >>> 16) & 0xff;
   // eslint-disable-next-line no-bitwise
-    (intValue >>> 8) & 0xff,
+  const g = (intValue >>> 8) & 0xff;
   // eslint-disable-next-line no-bitwise
-    intValue & 0xff
-  );
+  const b = intValue & 0xff;
+  const { h: hue, s: saturation, l: luminance } = rgbToHSL(r, g, b);
 
-  return { hue, saturation };
+  return { hue, saturation, luminance };
 }
 
 function fromGroupCallStateProto(

ts/services/backups/index.ts

@@ -24,25 +24,32 @@ import { prependStream } from '../../util/prependStream';
 import { appendMacStream } from '../../util/appendMacStream';
 import { getIvAndDecipher } from '../../util/getIvAndDecipher';
 import { getMacAndUpdateHmac } from '../../util/getMacAndUpdateHmac';
+import { missingCaseError } from '../../util/missingCaseError';
 import { HOUR } from '../../util/durations';
 import { CipherType, HashType } from '../../types/Crypto';
 import * as Errors from '../../types/errors';
 import { HTTPError } from '../../textsecure/Errors';
 import { constantTimeEqual } from '../../Crypto';
 import { measureSize } from '../../AttachmentCrypto';
+import { reinitializeRedux } from '../../state/reinitializeRedux';
+import { isTestOrMockEnvironment } from '../../environment';
 import { BackupExportStream } from './export';
 import { BackupImportStream } from './import';
 import { getKeyMaterial } from './crypto';
 import { BackupCredentials } from './credentials';
 import { BackupAPI, type DownloadOptionsType } from './api';
 import { validateBackup } from './validator';
-import { reinitializeRedux } from '../../state/reinitializeRedux';
 import { getParametersForRedux, loadAll } from '../allLoaders';
 
 const IV_LENGTH = 16;
 
 const BACKUP_REFRESH_INTERVAL = 24 * HOUR;
 
+export enum BackupType {
+  Ciphertext = 'Ciphertext',
+  TestOnlyPlaintext = 'TestOnlyPlaintext',
+}
+
 export class BackupsService {
   private isStarted = false;
   private isRunning = false;
@@ -108,11 +115,18 @@ export class BackupsService {
   // Test harness
   public async exportToDisk(
     path: string,
-    backupLevel: BackupLevel = BackupLevel.Messages
+    backupLevel: BackupLevel = BackupLevel.Messages,
+    backupType = BackupType.Ciphertext
   ): Promise<number> {
-    const size = await this.exportBackup(createWriteStream(path), backupLevel);
+    const size = await this.exportBackup(
+      createWriteStream(path),
+      backupLevel,
+      backupType
+    );
 
+    if (backupType === BackupType.Ciphertext) {
       await validateBackup(path, size);
+    }
 
     return size;
   }
@@ -184,13 +198,17 @@ export class BackupsService {
     }
   }
 
-  public async importBackup(createBackupStream: () => Readable): Promise<void> {
+  public async importBackup(
+    createBackupStream: () => Readable,
+    backupType = BackupType.Ciphertext
+  ): Promise<void> {
     strictAssert(!this.isRunning, 'BackupService is already running');
 
-    log.info('importBackup: starting...');
+    log.info(`importBackup: starting ${backupType}...`);
     this.isRunning = true;
 
     try {
+      if (backupType === BackupType.Ciphertext) {
         const { aesKey, macKey } = getKeyMaterial();
 
         // First pass - don't decrypt, only verify mac
@@ -231,6 +249,19 @@ export class BackupsService {
           constantTimeEqual(hmac.digest(), theirMac),
           'importBackup: Bad MAC, second pass'
         );
+      } else if (backupType === BackupType.TestOnlyPlaintext) {
+        strictAssert(
+          isTestOrMockEnvironment(),
+          'Plaintext backups can be imported only in test harness'
+        );
+        await pipeline(
+          createBackupStream(),
+          new DelimitedStream(),
+          new BackupImportStream()
+        );
+      } else {
+        throw missingCaseError(backupType);
+      }
 
       await this.resetStateAfterImport();
 
@@ -299,7 +330,8 @@ export class BackupsService {
 
   private async exportBackup(
     sink: Writable,
-    backupLevel: BackupLevel = BackupLevel.Messages
+    backupLevel: BackupLevel = BackupLevel.Messages,
+    backupType = BackupType.Ciphertext
   ): Promise<number> {
     strictAssert(!this.isRunning, 'BackupService is already running');
 
@@ -324,6 +356,7 @@ export class BackupsService {
 
       let totalBytes = 0;
 
+      if (backupType === BackupType.Ciphertext) {
         await pipeline(
           recordStream,
           createGzip(),
@@ -336,6 +369,15 @@ export class BackupsService {
           }),
           sink
         );
+      } else if (backupType === BackupType.TestOnlyPlaintext) {
+        strictAssert(
+          isTestOrMockEnvironment(),
+          'Plaintext backups can be exported only in test harness'
+        );
+        await pipeline(recordStream, sink);
+      } else {
+        throw missingCaseError(backupType);
+      }
 
       return totalBytes;
     } finally {

ts/services/backups/util/FileStream.ts

@@ -0,0 +1,61 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { type FileHandle, open } from 'node:fs/promises';
+import { Buffer } from 'node:buffer';
+import { InputStream } from '@signalapp/libsignal-client/dist/io';
+
+export class FileStream extends InputStream {
+  private file: FileHandle | undefined;
+  private position = 0;
+  private buffer = Buffer.alloc(16 * 1024);
+  private initPromise: Promise<unknown> | undefined;
+
+  constructor(private readonly filePath: string) {
+    super();
+  }
+
+  public async close(): Promise<void> {
+    await this.initPromise;
+    await this.file?.close();
+  }
+
+  // Only for comparator tests
+  public async size(): Promise<number> {
+    const file = await this.lazyOpen();
+    const { size } = await file.stat();
+    return size;
+  }
+
+  async read(amount: number): Promise<Buffer> {
+    const file = await this.lazyOpen();
+    if (this.buffer.length < amount) {
+      this.buffer = Buffer.alloc(amount);
+    }
+    const { bytesRead } = await file.read(
+      this.buffer,
+      0,
+      amount,
+      this.position
+    );
+    this.position += bytesRead;
+    return this.buffer.slice(0, bytesRead);
+  }
+
+  async skip(amount: number): Promise<void> {
+    this.position += amount;
+  }
+
+  private async lazyOpen(): Promise<FileHandle> {
+    await this.initPromise;
+
+    if (this.file) {
+      return this.file;
+    }
+
+    const filePromise = open(this.filePath);
+    this.initPromise = filePromise;
+    this.file = await filePromise;
+    return this.file;
+  }
+}

ts/services/backups/validator.ts

@@ -1,55 +1,12 @@
 // Copyright 2024 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
-import { type FileHandle, open } from 'node:fs/promises';
 import * as libsignal from '@signalapp/libsignal-client/dist/MessageBackup';
-import { InputStream } from '@signalapp/libsignal-client/dist/io';
 
 import { strictAssert } from '../../util/assert';
 import { toAciObject } from '../../util/ServiceId';
 import { isTestOrMockEnvironment } from '../../environment';
-
-class FileStream extends InputStream {
-  private file: FileHandle | undefined;
-  private position = 0;
-  private buffer = Buffer.alloc(16 * 1024);
-  private initPromise: Promise<unknown> | undefined;
-
-  constructor(private readonly filePath: string) {
-    super();
-  }
-
-  public async close(): Promise<void> {
-    await this.initPromise;
-    await this.file?.close();
-  }
-
-  async read(amount: number): Promise<Buffer> {
-    await this.initPromise;
-
-    if (!this.file) {
-      const filePromise = open(this.filePath);
-      this.initPromise = filePromise;
-      this.file = await filePromise;
-    }
-
-    if (this.buffer.length < amount) {
-      this.buffer = Buffer.alloc(amount);
-    }
-    const { bytesRead } = await this.file.read(
-      this.buffer,
-      0,
-      amount,
-      this.position
-    );
-    this.position += bytesRead;
-    return this.buffer.slice(0, bytesRead);
-  }
-
-  async skip(amount: number): Promise<void> {
-    this.position += amount;
-  }
-}
+import { FileStream } from './util/FileStream';
 
 export async function validateBackup(
   filePath: string,

ts/state/smart/InstallScreen.tsx

@@ -225,8 +225,13 @@ export const SmartInstallScreen = memo(function SmartInstallScreen() {
 
         let deviceName: string;
         let backupFileData: Uint8Array | undefined;
+        let isPlaintextBackup = false;
         if (window.SignalCI) {
-          ({ deviceName, backupData: backupFileData } = window.SignalCI);
+          ({
+            deviceName,
+            backupData: backupFileData,
+            isPlaintextBackup = false,
+          } = window.SignalCI);
         } else {
           deviceName = await chooseDeviceNamePromiseWrapperRef.current.promise;
           const backupFile =
@@ -264,6 +269,7 @@ export const SmartInstallScreen = memo(function SmartInstallScreen() {
         const data = provisioner.prepareLinkData({
           deviceName,
           backupFile: backupFileData,
+          isPlaintextBackup,
         });
         await accountManager.registerSecondDevice(data);
       } catch (error) {

ts/test-mock/backups/integration_test.ts

@@ -0,0 +1,86 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { assert } from 'chai';
+import { join } from 'node:path';
+import createDebug from 'debug';
+import fastGlob from 'fast-glob';
+import {
+  ComparableBackup,
+  Purpose,
+} from '@signalapp/libsignal-client/dist/MessageBackup';
+
+import * as durations from '../../util/durations';
+import { FileStream } from '../../services/backups/util/FileStream';
+import type { App } from '../playwright';
+import { Bootstrap } from '../bootstrap';
+
+export const debug = createDebug('mock:test:backups');
+
+const TEST_FOLDER = process.env.BACKUP_TEST_FOLDER;
+
+describe('backups/integration', async function (this: Mocha.Suite) {
+  this.timeout(100 * durations.MINUTE);
+
+  if (!TEST_FOLDER) {
+    return;
+  }
+
+  let bootstrap: Bootstrap;
+  let app: App | undefined;
+
+  beforeEach(async () => {
+    bootstrap = new Bootstrap();
+    await bootstrap.init();
+  });
+
+  afterEach(async function (this: Mocha.Context) {
+    if (!bootstrap) {
+      return;
+    }
+
+    await bootstrap.maybeSaveLogs(this.currentTest, app);
+    await app?.close();
+    await bootstrap.teardown();
+  });
+
+  const testFiles = fastGlob.sync(join(TEST_FOLDER, '*.binproto'), {
+    onlyFiles: true,
+  });
+  testFiles.forEach(fullPath => {
+    it(`passes ${fullPath}`, async () => {
+      app = await bootstrap.link({
+        ciBackupPath: fullPath,
+        ciIsPlaintextBackup: true,
+      });
+
+      const backupPath = bootstrap.getBackupPath('backup.bin');
+      await app.exportPlaintextBackupToDisk(backupPath);
+
+      await app.close();
+
+      const actualStream = new FileStream(backupPath);
+      const expectedStream = new FileStream(fullPath);
+      try {
+        const actual = await ComparableBackup.fromUnencrypted(
+          Purpose.RemoteBackup,
+          actualStream,
+          BigInt(await actualStream.size())
+        );
+        const expected = await ComparableBackup.fromUnencrypted(
+          Purpose.RemoteBackup,
+          expectedStream,
+          BigInt(await expectedStream.size())
+        );
+
+        assert.strictEqual(
+          actual.comparableString(),
+          expected.comparableString()
+        );
+      } finally {
+        await actualStream.close();
+        await expectedStream.close();
+      }
+    });
+  });
+});

ts/test-mock/playwright.ts

@@ -179,6 +179,13 @@ export class App extends EventEmitter {
     );
   }
 
+  public async exportPlaintextBackupToDisk(path: string): Promise<Uint8Array> {
+    const window = await this.getWindow();
+    return window.evaluate(
+      `window.SignalCI.exportPlaintextBackupToDisk(${JSON.stringify(path)})`
+    );
+  }
+
   public async unlink(): Promise<void> {
     const window = await this.getWindow();
     return window.evaluate('window.SignalCI.unlink()');

ts/textsecure/AccountManager.ts

@@ -25,7 +25,7 @@ import createTaskWithTimeout from './TaskWithTimeout';
 import * as Bytes from '../Bytes';
 import * as Errors from '../types/errors';
 import { senderCertificateService } from '../services/senderCertificate';
-import { backupsService } from '../services/backups';
+import { backupsService, BackupType } from '../services/backups';
 import {
   decryptDeviceName,
   deriveAccessKey,
@@ -123,7 +123,10 @@ type CreateAccountSharedOptionsType = Readonly<{
   pniKeyPair: KeyPairType;
   profileKey: Uint8Array;
   masterKey: Uint8Array;
+
+  // Test-only
   backupFile?: Uint8Array;
+  isPlaintextBackup?: boolean;
 }>;
 
 type CreatePrimaryDeviceOptionsType = Readonly<{
@@ -217,6 +220,7 @@ function signedPreKeyToUploadSignedPreKey({
 export type ConfirmNumberResultType = Readonly<{
   deviceName: string;
   backupFile: Uint8Array | undefined;
+  isPlaintextBackup: boolean;
 }>;
 
 export default class AccountManager extends EventTarget {
@@ -919,6 +923,7 @@ export default class AccountManager extends EventTarget {
       readReceipts,
       userAgent,
       backupFile,
+      isPlaintextBackup,
     } = options;
 
     const { storage } = window.textsecure;
@@ -963,7 +968,9 @@ export default class AccountManager extends EventTarget {
       }
       if (backupFile !== undefined) {
         log.warn(
-          'createAccount: Restoring from backup; deleting all previous data'
+          'createAccount: Restoring from ' +
+            `${isPlaintextBackup ? 'plaintext' : 'ciphertext'} backup; ` +
+            'deleting all previous data'
         );
       }
 
@@ -1222,7 +1229,10 @@ export default class AccountManager extends EventTarget {
     ]);
 
     if (backupFile !== undefined) {
-      await backupsService.importBackup(() => Readable.from([backupFile]));
+      await backupsService.importBackup(
+        () => Readable.from([backupFile]),
+        isPlaintextBackup ? BackupType.TestOnlyPlaintext : BackupType.Ciphertext
+      );
     }
   }
 

ts/textsecure/Provisioner.ts

@@ -67,6 +67,7 @@ type StateType = Readonly<
 export type PrepareLinkDataOptionsType = Readonly<{
   deviceName: string;
   backupFile?: Uint8Array;
+  isPlaintextBackup?: boolean;
 }>;
 
 export class Provisioner {
@@ -150,6 +151,7 @@ export class Provisioner {
   public prepareLinkData({
     deviceName,
     backupFile,
+    isPlaintextBackup,
   }: PrepareLinkDataOptionsType): CreateLinkedDeviceOptionsType {
     strictAssert(
       this.state.step === Step.ReadyToLink,
@@ -204,6 +206,7 @@ export class Provisioner {
       profileKey,
       deviceName,
       backupFile,
+      isPlaintextBackup,
       userAgent,
       ourAci,
       ourPni,

ts/types/Colors.ts

@@ -159,8 +159,8 @@ export const ContactNameColors = [
 export type ContactNameColorType = (typeof ContactNameColors)[number];
 
 export type CustomColorType = {
-  start: { hue: number; saturation: number };
-  end?: { hue: number; saturation: number };
+  start: { hue: number; saturation: number; luminance?: number };
+  end?: { hue: number; saturation: number; luminance?: number };
   deg?: number;
 };
 

ts/types/RendererConfig.ts

@@ -43,6 +43,7 @@ export const rendererConfigSchema = z.object({
   disableIPv6: z.boolean(),
   dnsFallback: DNSFallbackSchema,
   ciBackupPath: configOptionalStringSchema,
+  ciIsPlaintextBackup: z.boolean(),
   environment: environmentSchema,
   isMockTestEnvironment: z.boolean(),
   homePath: configRequiredStringSchema,

ts/util/getHSL.ts

@@ -46,14 +46,17 @@ export function getHSL(
   {
     hue,
     saturation,
+    luminance,
   }: {
     hue: number;
     saturation: number;
+    luminance?: number;
   },
   adjustedLightness = 0
 ): string {
-  return `hsl(${hue}, ${saturation}%, ${adjustLightnessValue(
-    calculateLightness(hue),
-    adjustedLightness
-  )}%)`;
+  return `hsl(${hue}, ${saturation}%, ${
+    luminance == null
+      ? adjustLightnessValue(calculateLightness(hue), adjustedLightness)
+      : luminance * 100
+  }%)`;
 }

ts/windows/main/phase4-test.ts

@@ -25,5 +25,6 @@ if (config.ciMode) {
     backupData: config.ciBackupPath
       ? fs.readFileSync(config.ciBackupPath)
       : undefined,
+    isPlaintextBackup: config.ciIsPlaintextBackup === true,
   });
 }