diff --git a/ts/CI.ts b/ts/CI.ts index d2d9792457..4166369f06 100644 --- a/ts/CI.ts +++ b/ts/CI.ts @@ -20,7 +20,6 @@ type ResolveType = (data: unknown) => void; export type CIType = { deviceName: string; backupData?: Uint8Array; - isBackupIntegration?: boolean; getConversationId: (address: string | null) => string | null; getMessagesBySentAt( sentAt: number @@ -46,14 +45,9 @@ export type CIType = { export type GetCIOptionsType = Readonly<{ deviceName: string; backupData?: Uint8Array; - isBackupIntegration?: boolean; }>; -export function getCI({ - deviceName, - backupData, - isBackupIntegration, -}: GetCIOptionsType): CIType { +export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType { const eventListeners = new Map>(); const completedEvents = new Map>(); @@ -199,7 +193,6 @@ export function getCI({ return { deviceName, backupData, - isBackupIntegration, getConversationId, getMessagesBySentAt, handleEvent, diff --git a/ts/ConversationController.ts b/ts/ConversationController.ts index 5d9f028de6..3604809e71 100644 --- a/ts/ConversationController.ts +++ b/ts/ConversationController.ts @@ -38,6 +38,7 @@ import { getTitleNoDefault } from './util/getTitle'; import * as StorageService from './services/storage'; import type { ConversationPropsForUnreadStats } from './util/countUnreadStats'; import { countAllConversationsUnreadStats } from './util/countUnreadStats'; +import { isTestOrMockEnvironment } from './environment'; type ConvoMatchType = | { @@ -183,7 +184,7 @@ export class ConversationController { // then we reset the state right away. this._conversations.on('add', (model: ConversationModel): void => { // Don't modify conversations in backup integration testing - if (window.SignalCI?.isBackupIntegration) { + if (isTestOrMockEnvironment()) { return; } model.startMuteTimer(); diff --git a/ts/background.ts b/ts/background.ts index ab7ecbfe98..cdc90911f1 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -2097,9 +2097,7 @@ export async function startApp(): Promise { storage, }); - if (!window.SignalCI?.isBackupIntegration) { - void routineProfileRefresher.start(); - } + void routineProfileRefresher.start(); } drop(usernameIntegrity.start()); @@ -2135,10 +2133,6 @@ export async function startApp(): Promise { async function onConfiguration(ev: ConfigurationEvent): Promise { ev.confirm(); - if (window.SignalCI?.isBackupIntegration) { - return; - } - const { configuration } = ev; const { readReceipts, @@ -3204,10 +3198,6 @@ export async function startApp(): Promise { async function onKeysSync(ev: KeysEvent) { ev.confirm(); - if (window.SignalCI?.isBackupIntegration) { - return; - } - const { masterKey } = ev; let { storageServiceKey } = ev; diff --git a/ts/jobs/AttachmentDownloadManager.ts b/ts/jobs/AttachmentDownloadManager.ts index e22a0e90e7..055661fcea 100644 --- a/ts/jobs/AttachmentDownloadManager.ts +++ b/ts/jobs/AttachmentDownloadManager.ts @@ -219,9 +219,6 @@ export class AttachmentDownloadManager extends JobManager { - if (window.SignalCI?.isBackupIntegration) { - return; - } await AttachmentDownloadManager.instance.start(); } diff --git a/ts/services/backups/export.ts b/ts/services/backups/export.ts index 4c5519509c..7ae48cb608 100644 --- a/ts/services/backups/export.ts +++ b/ts/services/backups/export.ts @@ -109,6 +109,7 @@ import { import { isAciString } from '../../util/isAciString'; import { hslToRGB } from '../../util/hslToRGB'; import type { AboutMe, LocalChatStyle } from './types'; +import { BackupType } from './types'; import { messageHasPaymentEvent } from '../../messages/helpers'; import { numberToAddressType, @@ -209,6 +210,10 @@ export class BackupExportStream extends Readable { // array. private customColorIdByUuid = new Map(); + constructor(private readonly backupType: BackupType) { + super(); + } + public run(backupLevel: BackupLevel): void { drop( (async () => { @@ -224,7 +229,7 @@ export class BackupExportStream extends Readable { // TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction await DataWriter.clearAllAttachmentBackupJobs(); - if (!window.SignalCI?.isBackupIntegration) { + if (this.backupType !== BackupType.TestOnlyPlaintext) { await Promise.all( this.attachmentBackupJobs.map(job => AttachmentBackupManager.addJobAndMaybeThumbnailJob(job) @@ -2180,7 +2185,7 @@ export class BackupExportStream extends Readable { // We don't download attachments during integration tests and thus have no // "iv" for an attachment and can't create a job - if (!window.SignalCI?.isBackupIntegration) { + if (this.backupType !== BackupType.TestOnlyPlaintext) { const backupJob = await maybeGetBackupJobForAttachmentAndFilePointer({ attachment: updatedAttachment ?? attachment, filePointer, diff --git a/ts/services/backups/import.ts b/ts/services/backups/import.ts index 5df1505c58..1d83da78bf 100644 --- a/ts/services/backups/import.ts +++ b/ts/services/backups/import.ts @@ -76,6 +76,7 @@ import { SeenStatus } from '../../MessageSeenStatus'; import * as Bytes from '../../Bytes'; import { BACKUP_VERSION, WALLPAPER_TO_BUBBLE_COLOR } from './constants'; import type { AboutMe, LocalChatStyle } from './types'; +import { BackupType } from './types'; import type { GroupV2ChangeDetailType } from '../../groups'; import { queueAttachmentDownloads } from '../../util/queueAttachmentDownloads'; import { drop } from '../../util/drop'; @@ -298,17 +299,19 @@ export class BackupImportStream extends Writable { flush: () => this.saveMessageBatcher.flushAndWait(), }); - private constructor() { + private constructor(private readonly backupType: BackupType) { super({ objectMode: true }); } - public static async create(): Promise { + public static async create( + backupType = BackupType.Ciphertext + ): Promise { await AttachmentDownloadManager.stop(); await DataWriter.removeAllBackupAttachmentDownloadJobs(); await window.storage.put('backupMediaDownloadCompletedBytes', 0); await window.storage.put('backupMediaDownloadTotalBytes', 0); - return new BackupImportStream(); + return new BackupImportStream(backupType); } override async _write( @@ -387,9 +390,10 @@ export class BackupImportStream extends Writable { await pMap( [...this.pendingGroupAvatars.entries()], async ([conversationId, newAvatarUrl]) => { - if (!window.SignalCI?.isBackupIntegration) { - await groupAvatarJobQueue.add({ conversationId, newAvatarUrl }); + if (this.backupType === BackupType.TestOnlyPlaintext) { + return; } + await groupAvatarJobQueue.add({ conversationId, newAvatarUrl }); }, { concurrency: MAX_CONCURRENCY } ); @@ -411,7 +415,7 @@ export class BackupImportStream extends Writable { await DataReader.getSizeOfPendingBackupAttachmentDownloadJobs() ); - if (!window.SignalCI?.isBackupIntegration) { + if (this.backupType !== BackupType.TestOnlyPlaintext) { await AttachmentDownloadManager.start(); } diff --git a/ts/services/backups/index.ts b/ts/services/backups/index.ts index 4aa7536c3b..fa2f58c7a2 100644 --- a/ts/services/backups/index.ts +++ b/ts/services/backups/index.ts @@ -38,16 +38,14 @@ import { getKeyMaterial } from './crypto'; import { BackupCredentials } from './credentials'; import { BackupAPI, type DownloadOptionsType } from './api'; import { validateBackup } from './validator'; +import { BackupType } from './types'; + +export { BackupType }; const IV_LENGTH = 16; const BACKUP_REFRESH_INTERVAL = 24 * HOUR; -export enum BackupType { - Ciphertext = 'Ciphertext', - TestOnlyPlaintext = 'TestOnlyPlaintext', -} - export class BackupsService { private isStarted = false; private isRunning = false; @@ -100,13 +98,14 @@ export class BackupsService { // Test harness public async exportBackupData( - backupLevel: BackupLevel = BackupLevel.Messages + backupLevel: BackupLevel = BackupLevel.Messages, + backupType = BackupType.Ciphertext ): Promise { const sink = new PassThrough(); const chunks = new Array(); sink.on('data', chunk => chunks.push(chunk)); - await this.exportBackup(sink, backupLevel); + await this.exportBackup(sink, backupLevel, backupType); return Bytes.concatenate(chunks); } @@ -246,7 +245,7 @@ export class BackupsService { this.isRunning = true; try { - const importStream = await BackupImportStream.create(); + const importStream = await BackupImportStream.create(backupType); if (backupType === BackupType.Ciphertext) { const { aesKey, macKey } = getKeyMaterial(); @@ -370,7 +369,12 @@ export class BackupsService { try { // TODO (DESKTOP-7168): Update mock-server to support this endpoint - if (!window.SignalCI) { + if (window.SignalCI || backupType === BackupType.TestOnlyPlaintext) { + strictAssert( + isTestOrMockEnvironment(), + 'Plaintext backups can be exported only in test harness' + ); + } else { // We first fetch the latest info on what's on the CDN, since this affects the // filePointers we will generate during export log.info('Fetching latest backup CDN metadata'); @@ -378,7 +382,7 @@ export class BackupsService { } const { aesKey, macKey } = getKeyMaterial(); - const recordStream = new BackupExportStream(); + const recordStream = new BackupExportStream(backupType); recordStream.run(backupLevel); diff --git a/ts/services/backups/types.d.ts b/ts/services/backups/types.ts similarity index 85% rename from ts/services/backups/types.d.ts rename to ts/services/backups/types.ts index db0fed0fa2..c66e4b710e 100644 --- a/ts/services/backups/types.d.ts +++ b/ts/services/backups/types.ts @@ -9,6 +9,11 @@ export type AboutMe = { pni?: PniString; }; +export enum BackupType { + Ciphertext = 'Ciphertext', + TestOnlyPlaintext = 'TestOnlyPlaintext', +} + export type LocalChatStyle = Readonly<{ wallpaperPhotoPointer: Uint8Array | undefined; wallpaperPreset: number | undefined; diff --git a/ts/services/backups/util/FileStream.ts b/ts/services/backups/util/FileStream.ts index 17898c1f67..f31e764897 100644 --- a/ts/services/backups/util/FileStream.ts +++ b/ts/services/backups/util/FileStream.ts @@ -20,13 +20,6 @@ export class FileStream extends InputStream { await this.file?.close(); } - // Only for comparator tests - public async size(): Promise { - const file = await this.lazyOpen(); - const { size } = await file.stat(); - return size; - } - async read(amount: number): Promise { const file = await this.lazyOpen(); if (this.buffer.length < amount) { diff --git a/ts/services/contactSync.ts b/ts/services/contactSync.ts index c8d1efdd35..ca9f6e2914 100644 --- a/ts/services/contactSync.ts +++ b/ts/services/contactSync.ts @@ -229,10 +229,6 @@ async function doContactSync({ } export async function onContactSync(ev: ContactSyncEvent): Promise { - if (window.SignalCI?.isBackupIntegration) { - return; - } - log.info( `onContactSync(sent=${ev.sentAt}, receivedAt=${ev.receivedAtCounter}): queueing sync` ); diff --git a/ts/state/ducks/installer.ts b/ts/state/ducks/installer.ts index 4f68234a72..ce7dfca472 100644 --- a/ts/state/ducks/installer.ts +++ b/ts/state/ducks/installer.ts @@ -308,7 +308,6 @@ function startInstaller(): ThunkAction< finishInstall({ deviceName: SignalCI.deviceName, backupFile: SignalCI.backupData, - isBackupIntegration: SignalCI.isBackupIntegration, }) ); } diff --git a/ts/test-electron/backup/helpers.ts b/ts/test-electron/backup/helpers.ts index f578ed21f9..d09f99077d 100644 --- a/ts/test-electron/backup/helpers.ts +++ b/ts/test-electron/backup/helpers.ts @@ -233,9 +233,9 @@ export async function asymmetricRoundtripHarness( } } -async function clearData() { +export async function clearData(): Promise { await DataWriter.removeAll(); - window.storage.reset(); + await window.storage.fetch(); window.ConversationController.reset(); await setupBasics(); @@ -255,7 +255,8 @@ export async function setupBasics(): Promise { window.Events = { ...window.Events, - getTypingIndicatorSetting: () => false, - getLinkPreviewSetting: () => false, + getTypingIndicatorSetting: () => + window.storage.get('typingIndicators', false), + getLinkPreviewSetting: () => window.storage.get('linkPreviews', false), }; } diff --git a/ts/test-electron/backup/integration_test.ts b/ts/test-electron/backup/integration_test.ts new file mode 100644 index 0000000000..db696a093f --- /dev/null +++ b/ts/test-electron/backup/integration_test.ts @@ -0,0 +1,99 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { readdirSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { basename, join } from 'node:path'; +import { Readable } from 'node:stream'; +import { BackupLevel } from '@signalapp/libsignal-client/zkgroup'; +import { InputStream } from '@signalapp/libsignal-client/dist/io'; +import { + ComparableBackup, + Purpose, +} from '@signalapp/libsignal-client/dist/MessageBackup'; +import { assert } from 'chai'; + +import { clearData } from './helpers'; +import { loadAll } from '../../services/allLoaders'; +import { backupsService, BackupType } from '../../services/backups'; +import { DataWriter } from '../../sql/Client'; + +const { BACKUP_INTEGRATION_DIR } = process.env; + +class MemoryStream extends InputStream { + private offset = 0; + + constructor(private readonly buffer: Buffer) { + super(); + } + + public override async read(amount: number): Promise { + const result = this.buffer.slice(this.offset, this.offset + amount); + this.offset += amount; + return result; + } + + public override async skip(amount: number): Promise { + this.offset += amount; + } +} + +describe('backup/integration', () => { + beforeEach(async () => { + await clearData(); + await loadAll(); + }); + + afterEach(async () => { + await DataWriter.removeAll(); + }); + + if (!BACKUP_INTEGRATION_DIR) { + return; + } + + const files = readdirSync(BACKUP_INTEGRATION_DIR) + .filter(file => file.endsWith('.binproto')) + .map(file => join(BACKUP_INTEGRATION_DIR, file)); + + for (const fullPath of files) { + it(basename(fullPath), async () => { + const expectedBuffer = await readFile(fullPath); + + await backupsService.importBackup( + () => Readable.from([expectedBuffer]), + BackupType.TestOnlyPlaintext + ); + + const exported = await backupsService.exportBackupData( + BackupLevel.Media, + BackupType.TestOnlyPlaintext + ); + + const actualStream = new MemoryStream(Buffer.from(exported)); + const expectedStream = new MemoryStream(expectedBuffer); + + const actual = await ComparableBackup.fromUnencrypted( + Purpose.RemoteBackup, + actualStream, + BigInt(exported.byteLength) + ); + const expected = await ComparableBackup.fromUnencrypted( + Purpose.RemoteBackup, + expectedStream, + BigInt(expectedBuffer.byteLength) + ); + + const actualString = actual.comparableString(); + const expectedString = expected.comparableString(); + + if (expectedString.includes('ReleaseChannelDonationRequest')) { + // Skip the unsupported tests + return; + } + + // We need "deep*" for fancy diffs + assert.deepStrictEqual(actualString, expectedString); + }); + } +}); diff --git a/ts/test-mock/backups/integration.ts b/ts/test-mock/backups/integration.ts deleted file mode 100644 index ee9125a428..0000000000 --- a/ts/test-mock/backups/integration.ts +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2023 Signal Messenger, LLC -// SPDX-License-Identifier: AGPL-3.0-only -/* eslint-disable no-console */ - -import { cpus } from 'node:os'; -import { inspect } from 'node:util'; -import { basename } from 'node:path'; -import { reporters } from 'mocha'; -import pMap from 'p-map'; -import logSymbols from 'log-symbols'; -import { - ComparableBackup, - Purpose, -} from '@signalapp/libsignal-client/dist/MessageBackup'; - -import { FileStream } from '../../services/backups/util/FileStream'; -import { drop } from '../../util/drop'; -import type { App } from '../playwright'; -import { Bootstrap } from '../bootstrap'; - -const WORKER_COUNT = process.env.WORKER_COUNT - ? parseInt(process.env.WORKER_COUNT, 10) - : Math.min(8, cpus().length); - -(reporters.base as unknown as { maxDiffSize: number }).maxDiffSize = Infinity; - -const testFiles = process.argv.slice(2); -let total = 0; -let passed = 0; -let failed = 0; - -function pass(): void { - process.stdout.write(`${logSymbols.success}`); - total += 1; - passed += 1; -} - -function fail(filePath: string, error: string): void { - total += 1; - failed += 1; - console.log(`\n${logSymbols.error} ${basename(filePath)}`); - console.error(error); -} - -async function runOne(filePath: string): Promise { - const bootstrap = new Bootstrap({ contactCount: 0 }); - let app: App | undefined; - try { - await bootstrap.init(); - - app = await bootstrap.link({ - ciBackupPath: filePath, - ciIsBackupIntegration: true, - }); - - const backupPath = bootstrap.getBackupPath('backup.bin'); - await app.exportPlaintextBackupToDisk(backupPath); - - await app.close(); - app = undefined; - - const actualStream = new FileStream(backupPath); - const expectedStream = new FileStream(filePath); - try { - const actual = await ComparableBackup.fromUnencrypted( - Purpose.RemoteBackup, - actualStream, - BigInt(await actualStream.size()) - ); - const expected = await ComparableBackup.fromUnencrypted( - Purpose.RemoteBackup, - expectedStream, - BigInt(await expectedStream.size()) - ); - - const actualString = actual.comparableString(); - const expectedString = expected.comparableString(); - - if (actualString === expectedString) { - pass(); - } else { - fail( - filePath, - reporters.base.generateDiff( - inspect(actualString, { depth: Infinity, sorted: true }), - inspect(expectedString, { depth: Infinity, sorted: true }) - ) - ); - - await bootstrap.saveLogs(app, basename(filePath)); - } - } finally { - await actualStream.close(); - await expectedStream.close(); - } - } catch (error) { - await bootstrap.saveLogs(app, basename(filePath)); - fail(filePath, error.stack); - } finally { - // No need to block on this - drop( - (async () => { - try { - await bootstrap.teardown(); - } catch (error) { - console.error(`Failed to teardown ${basename(filePath)}`, error); - } - })() - ); - } -} - -async function main(): Promise { - await pMap(testFiles, runOne, { concurrency: WORKER_COUNT }); - - console.log(`${passed}/${total} (${failed} failures)`); - if (failed !== 0) { - process.exit(0); - } -} - -main().catch(error => { - console.error(error); - process.exit(1); -}); diff --git a/ts/textsecure/AccountManager.ts b/ts/textsecure/AccountManager.ts index fa018e26a1..e6a13ff20b 100644 --- a/ts/textsecure/AccountManager.ts +++ b/ts/textsecure/AccountManager.ts @@ -25,7 +25,7 @@ import createTaskWithTimeout from './TaskWithTimeout'; import * as Bytes from '../Bytes'; import * as Errors from '../types/errors'; import { senderCertificateService } from '../services/senderCertificate'; -import { backupsService, BackupType } from '../services/backups'; +import { backupsService } from '../services/backups'; import { decryptDeviceName, deriveAccessKey, @@ -126,7 +126,6 @@ type CreateAccountSharedOptionsType = Readonly<{ // Test-only backupFile?: Uint8Array; - isBackupIntegration?: boolean; }>; type CreatePrimaryDeviceOptionsType = Readonly<{ @@ -220,7 +219,6 @@ function signedPreKeyToUploadSignedPreKey({ export type ConfirmNumberResultType = Readonly<{ deviceName: string; backupFile: Uint8Array | undefined; - isBackupIntegration: boolean; }>; export default class AccountManager extends EventTarget { @@ -923,7 +921,6 @@ export default class AccountManager extends EventTarget { readReceipts, userAgent, backupFile, - isBackupIntegration, } = options; const { storage } = window.textsecure; @@ -968,8 +965,7 @@ export default class AccountManager extends EventTarget { } if (backupFile !== undefined) { log.warn( - 'createAccount: Restoring from ' + - `${isBackupIntegration ? 'plaintext' : 'ciphertext'} backup; ` + + 'createAccount: Restoring from backup; ' + 'deleting all previous data' ); } @@ -1229,12 +1225,7 @@ export default class AccountManager extends EventTarget { ]); if (backupFile !== undefined) { - await backupsService.importBackup( - () => Readable.from([backupFile]), - isBackupIntegration - ? BackupType.TestOnlyPlaintext - : BackupType.Ciphertext - ); + await backupsService.importBackup(() => Readable.from([backupFile])); } } diff --git a/ts/textsecure/Provisioner.ts b/ts/textsecure/Provisioner.ts index 84d44d961f..ec8a99065c 100644 --- a/ts/textsecure/Provisioner.ts +++ b/ts/textsecure/Provisioner.ts @@ -69,7 +69,6 @@ type StateType = Readonly< export type PrepareLinkDataOptionsType = Readonly<{ deviceName: string; backupFile?: Uint8Array; - isBackupIntegration?: boolean; }>; export class Provisioner { @@ -153,7 +152,6 @@ export class Provisioner { public prepareLinkData({ deviceName, backupFile, - isBackupIntegration, }: PrepareLinkDataOptionsType): CreateLinkedDeviceOptionsType { strictAssert( this.state.step === Step.ReadyToLink, @@ -211,7 +209,6 @@ export class Provisioner { MAX_DEVICE_NAME_LENGTH ), backupFile, - isBackupIntegration, userAgent, ourAci, ourPni, diff --git a/ts/windows/main/phase4-test.ts b/ts/windows/main/phase4-test.ts index 51e5a31d4b..80c76323e9 100644 --- a/ts/windows/main/phase4-test.ts +++ b/ts/windows/main/phase4-test.ts @@ -25,6 +25,5 @@ if (config.ciMode) { backupData: config.ciBackupPath ? fs.readFileSync(config.ciBackupPath) : undefined, - isBackupIntegration: config.ciIsBackupIntegration === true, }); }