diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 6324c3823e89..ecf41cffb73b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -170,6 +170,20 @@ jobs: # DEBUG: 'mock:benchmarks' ARTIFACTS_DIR: artifacts/call-history-search + - name: Run backup benchmarks + run: | + set -o pipefail + rm -rf /tmp/mock + xvfb-run --auto-servernum node \ + ts/test-mock/benchmarks/backup_bench.js | \ + tee benchmark-backup.log + timeout-minutes: 10 + env: + NODE_ENV: production + ELECTRON_ENABLE_STACK_DUMPING: on + # DEBUG: 'mock:benchmarks' + ARTIFACTS_DIR: artifacts/backup-bench + - name: Upload benchmark logs on failure if: failure() uses: actions/upload-artifact@v4 @@ -200,5 +214,6 @@ jobs: node ./bin/publish.js ../benchmark-large-group-send.log desktop.ci.performance.largeGroupSend node ./bin/publish.js ../benchmark-convo-open.log desktop.ci.performance.convoOpen node ./bin/publish.js ../benchmark-call-history-search.log desktop.ci.performance.callHistorySearch + node ./bin/publish.js ../benchmark-backup.log desktop.ci.performance.backup env: DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} diff --git a/package-lock.json b/package-lock.json index 581da8d753e1..280c46725015 100644 --- a/package-lock.json +++ b/package-lock.json @@ -126,7 +126,7 @@ "@indutny/parallel-prettier": "3.0.0", "@indutny/rezip-electron": "1.3.1", "@indutny/symbolicate-mac": "2.3.0", - "@signalapp/mock-server": "7.0.1", + "@signalapp/mock-server": "7.1.3", "@storybook/addon-a11y": "8.1.11", "@storybook/addon-actions": "8.1.11", "@storybook/addon-controls": "8.1.11", @@ -7274,22 +7274,24 @@ } }, "node_modules/@signalapp/mock-server": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/@signalapp/mock-server/-/mock-server-7.0.1.tgz", - "integrity": "sha512-iwH57apXyTHKjozaV1ZJW6nbVhFH3KlVOQYaJiO2bT3YgAGdYoJvHp8+MMIQ8OFYVGRo3g7wouqX/JT5HElAvw==", + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@signalapp/mock-server/-/mock-server-7.1.3.tgz", + "integrity": "sha512-Xvpeai+E0mhz6WHSycYuY31y5saCNJYX7ioDn1Q0LqUAOUKGVQjnWvdxeXLPKv8C06mbWn0lP16o9swClWVsmg==", "dev": true, "license": "AGPL-3.0-only", "dependencies": { "@indutny/parallel-prettier": "^3.0.0", - "@signalapp/libsignal-client": "^0.45.0", + "@signalapp/libsignal-client": "^0.58.2", "@tus/file-store": "^1.4.0", "@tus/server": "^1.7.0", "debug": "^4.3.2", + "is-plain-obj": "3.0.0", "long": "^4.0.0", "micro": "^9.3.4", "microrouter": "^3.1.3", "prettier": "^3.3.3", "protobufjs": "^7.2.4", + "type-fest": "^4.26.1", "url-pattern": "^1.0.3", "uuid": "^8.3.2", "ws": "^8.4.2", @@ -7297,14 +7299,15 @@ } }, "node_modules/@signalapp/mock-server/node_modules/@signalapp/libsignal-client": { - "version": "0.45.1", - "resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.45.1.tgz", - "integrity": "sha512-jKNGLD8QQkLEopX7Fb5XG7LlIe559TgqfC1UCgUV9YW4pPpvM+RPbW4ndL1v8WO/Toff4nVXJXJV6kzYiK2lDA==", + "version": "0.58.2", + "resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.58.2.tgz", + "integrity": "sha512-3OF9fGmh7tz9JVfT9xTR4DWcm4HOpbQknO9k7Oj23uSsBSEcJYmYPGM3Rdm16C/z+evVgyvrLptPtjTzXXuNzA==", "dev": true, "hasInstallScript": true, + "license": "AGPL-3.0-only", "dependencies": { - "node-gyp-build": "^4.2.3", - "type-fest": "^3.5.0", + "node-gyp-build": "^4.8.0", + "type-fest": "^4.26.0", "uuid": "^8.3.0" } }, @@ -7325,6 +7328,19 @@ } } }, + "node_modules/@signalapp/mock-server/node_modules/is-plain-obj": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz", + "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@signalapp/mock-server/node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -7332,12 +7348,13 @@ "dev": true }, "node_modules/@signalapp/mock-server/node_modules/type-fest": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.13.1.tgz", - "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==", + "version": "4.26.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.26.1.tgz", + "integrity": "sha512-yOGpmOAL7CkKe/91I5O3gPICmJNLJ1G4zFYVAsRHg7M64biSnPtRj0WNQt++bRkjYOqjWXrhnUw1utzmVErAdg==", "dev": true, + "license": "(MIT OR CC0-1.0)", "engines": { - "node": ">=14.16" + "node": ">=16" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -7348,6 +7365,7 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", "dev": true, + "license": "MIT", "bin": { "uuid": "dist/bin/uuid" } diff --git a/package.json b/package.json index beb740e45462..0d9b42ac30ce 100644 --- a/package.json +++ b/package.json @@ -210,7 +210,7 @@ "@indutny/parallel-prettier": "3.0.0", "@indutny/rezip-electron": "1.3.1", "@indutny/symbolicate-mac": "2.3.0", - "@signalapp/mock-server": "7.0.1", + "@signalapp/mock-server": "7.1.3", "@storybook/addon-a11y": "8.1.11", "@storybook/addon-actions": "8.1.11", "@storybook/addon-controls": "8.1.11", diff --git a/ts/CI.ts b/ts/CI.ts index 4166369f064d..f6994bd75be9 100644 --- a/ts/CI.ts +++ b/ts/CI.ts @@ -3,14 +3,13 @@ import { format } from 'node:util'; import { ipcRenderer } from 'electron'; -import { BackupLevel } from '@signalapp/libsignal-client/zkgroup'; import type { IPCResponse as ChallengeResponseType } from './challenge'; import type { MessageAttributesType } from './model-types.d'; import * as log from './logging/log'; import { explodePromise } from './util/explodePromise'; import { AccessType, ipcInvoke } from './sql/channels'; -import { backupsService, BackupType } from './services/backups'; +import { backupsService } from './services/backups'; import { SECOND } from './util/durations'; import { isSignalRoute } from './util/signalRoutes'; import { strictAssert } from './util/assert'; @@ -19,7 +18,6 @@ type ResolveType = (data: unknown) => void; export type CIType = { deviceName: string; - backupData?: Uint8Array; getConversationId: (address: string | null) => string | null; getMessagesBySentAt( sentAt: number @@ -36,18 +34,16 @@ export type CIType = { } ) => unknown; openSignalRoute(url: string): Promise; - exportBackupToDisk(path: string): Promise; - exportPlaintextBackupToDisk(path: string): Promise; + uploadBackup(): Promise; unlink: () => void; print: (...args: ReadonlyArray) => void; }; export type GetCIOptionsType = Readonly<{ deviceName: string; - backupData?: Uint8Array; }>; -export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType { +export function getCI({ deviceName }: GetCIOptionsType): CIType { const eventListeners = new Map>(); const completedEvents = new Map>(); @@ -66,8 +62,8 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType { if (!options?.ignorePastEvents) { const pendingCompleted = completedEvents.get(event) || []; - const pending = pendingCompleted.shift(); - if (pending) { + if (pendingCompleted.length) { + const pending = pendingCompleted.shift(); log.info(`CI: resolving pending result for ${event}`, pending); if (pendingCompleted.length === 0) { @@ -170,16 +166,8 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType { document.body.removeChild(a); } - async function exportBackupToDisk(path: string) { - await backupsService.exportToDisk(path, BackupLevel.Media); - } - - async function exportPlaintextBackupToDisk(path: string) { - await backupsService.exportToDisk( - path, - BackupLevel.Media, - BackupType.TestOnlyPlaintext - ); + async function uploadBackup() { + await backupsService.upload(); } function unlink() { @@ -192,7 +180,6 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType { return { deviceName, - backupData, getConversationId, getMessagesBySentAt, handleEvent, @@ -200,8 +187,7 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType { solveChallenge, waitForEvent, openSignalRoute, - exportBackupToDisk, - exportPlaintextBackupToDisk, + uploadBackup, unlink, getPendingEventCount, print, diff --git a/ts/background.ts b/ts/background.ts index ca21ef11540d..1af60dc3004b 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -1598,8 +1598,10 @@ export async function startApp(): Promise { }, }); + log.info('afterStart: backup downloaded, resolving'); backupReady.resolve(); } catch (error) { + log.error('afterStart: backup download failed, rejecting'); backupReady.reject(error); throw error; } @@ -1706,17 +1708,21 @@ export async function startApp(): Promise { strictAssert(server !== undefined, 'WebAPI not connected'); - // Wait for backup to be downloaded - try { - await backupReady.promise; - } catch (error) { - log.error('background: backup download failed, not reconnecting', error); - return; - } - log.info('background: connect unblocked by backups'); - try { connectPromise = explodePromise(); + + // Wait for backup to be downloaded + try { + await backupReady.promise; + } catch (error) { + log.error( + 'background: backup download failed, not reconnecting', + error + ); + return; + } + log.info('background: connect unblocked by backups'); + // Reset the flag and update it below if needed setIsInitialSync(false); diff --git a/ts/services/allLoaders.ts b/ts/services/allLoaders.ts index a50341a85a36..19e76d8755e5 100644 --- a/ts/services/allLoaders.ts +++ b/ts/services/allLoaders.ts @@ -26,7 +26,8 @@ import { getInitialState as getStickersReduxState, } from '../types/Stickers'; -import type { ReduxInitData } from '../state/initializeRedux'; +import { type ReduxInitData } from '../state/initializeRedux'; +import { reinitializeRedux } from '../state/reinitializeRedux'; export async function loadAll(): Promise { await Promise.all([ @@ -41,6 +42,11 @@ export async function loadAll(): Promise { ]); } +export async function loadAllAndReinitializeRedux(): Promise { + await loadAll(); + reinitializeRedux(getParametersForRedux()); +} + export function getParametersForRedux(): ReduxInitData { const { mainWindowStats, menuOptions, theme } = getUserDataForRedux(); diff --git a/ts/services/backups/api.ts b/ts/services/backups/api.ts index e022c9a042d3..ee21cf882a40 100644 --- a/ts/services/backups/api.ts +++ b/ts/services/backups/api.ts @@ -25,7 +25,6 @@ export class BackupAPI { constructor(private credentials: BackupCredentials) {} public async refresh(): Promise { - // TODO: DESKTOP-6979 await this.server.refreshBackup( await this.credentials.getHeadersForToday() ); diff --git a/ts/services/backups/import.ts b/ts/services/backups/import.ts index c7381a059ab9..860d120892cf 100644 --- a/ts/services/backups/import.ts +++ b/ts/services/backups/import.ts @@ -18,7 +18,7 @@ import { import * as log from '../../logging/log'; import { GiftBadgeStates } from '../../components/conversation/Message'; import { StorySendMode, MY_STORY_ID } from '../../types/Stories'; -import type { ServiceIdString, AciString } from '../../types/ServiceId'; +import type { ServiceIdString } from '../../types/ServiceId'; import { fromAciObject, fromPniObject, @@ -64,7 +64,6 @@ import { } from '../../util/zkgroup'; import { incrementMessageCounter } from '../../util/incrementMessageCounter'; import { isAciString } from '../../util/isAciString'; -import { createBatcher } from '../../util/batcher'; import { PhoneNumberDiscoverability } from '../../util/phoneNumberDiscoverability'; import { PhoneNumberSharingMode } from '../../util/phoneNumberSharingMode'; import { bytesToUuid } from '../../util/uuidToBytes'; @@ -79,7 +78,6 @@ import type { AboutMe, LocalChatStyle } from './types'; import { BackupType } from './types'; import type { GroupV2ChangeDetailType } from '../../groups'; import { queueAttachmentDownloads } from '../../util/queueAttachmentDownloads'; -import { drop } from '../../util/drop'; import { isNotNil } from '../../util/isNotNil'; import { isGroup } from '../../util/whatTypeOfConversation'; import { rgbToHSL } from '../../util/rgbToHSL'; @@ -107,89 +105,23 @@ import type { CallLinkType } from '../../types/CallLink'; import type { RawBodyRange } from '../../types/BodyRange'; import { fromAdminKeyBytes } from '../../util/callLinks'; import { getRoomIdFromRootKey } from '../../util/callLinksRingrtc'; -import { reinitializeRedux } from '../../state/reinitializeRedux'; -import { getParametersForRedux, loadAll } from '../allLoaders'; +import { loadAllAndReinitializeRedux } from '../allLoaders'; import { resetBackupMediaDownloadProgress } from '../../util/backupMediaDownload'; import { getEnvironment, isTestEnvironment } from '../../environment'; const MAX_CONCURRENCY = 10; +const CONVERSATION_OP_BATCH_SIZE = 10000; +const SAVE_MESSAGE_BATCH_SIZE = 10000; + // Keep 1000 recent messages in memory to speed up quote lookup. const RECENT_MESSAGES_CACHE_SIZE = 1000; -type ConversationOpType = Readonly<{ - isUpdate: boolean; - attributes: ConversationAttributesType; -}>; - type ChatItemParseResult = { message: Partial; additionalMessages: Array>; }; -async function processConversationOpBatch( - batch: ReadonlyArray -): Promise { - // Note that we might have duplicates since we update attributes in-place - const saves = [ - ...new Set(batch.filter(x => x.isUpdate === false).map(x => x.attributes)), - ]; - const updates = [ - ...new Set(batch.filter(x => x.isUpdate === true).map(x => x.attributes)), - ]; - - log.info( - `backups: running conversation op batch, saves=${saves.length} ` + - `updates=${updates.length}` - ); - - await DataWriter.saveConversations(saves); - await DataWriter.updateConversations(updates); -} -async function processMessagesBatch( - ourAci: AciString, - batch: ReadonlyArray -): Promise { - const ids = await DataWriter.saveMessages(batch, { - forceSave: true, - ourAci, - }); - strictAssert(ids.length === batch.length, 'Should get same number of ids'); - - // TODO (DESKTOP-7402): consider re-saving after updating the pending state - for (const [index, rawAttributes] of batch.entries()) { - const attributes = { - ...rawAttributes, - id: ids[index], - }; - - const { editHistory } = attributes; - - if (editHistory?.length) { - drop( - DataWriter.saveEditedMessages( - attributes, - ourAci, - editHistory.slice(0, -1).map(({ timestamp }) => ({ - conversationId: attributes.conversationId, - messageId: attributes.id, - - // Main message will track this - readStatus: ReadStatus.Read, - sentAt: timestamp, - })) - ) - ); - } - - drop( - queueAttachmentDownloads(attributes, { - source: AttachmentDownloadSource.BACKUP_IMPORT, - }) - ); - } -} - function phoneToContactFormType( type: Backups.ContactAttachment.Phone.Type | null | undefined ): ContactFormType { @@ -269,26 +201,11 @@ export class BackupImportStream extends Writable { number, ConversationAttributesType >(); - private readonly conversationOpBatcher = createBatcher<{ - isUpdate: boolean; - attributes: ConversationAttributesType; - }>({ - name: 'BackupImport.conversationOpBatcher', - wait: 0, - maxSize: 1000, - processBatch: processConversationOpBatch, - }); - private readonly saveMessageBatcher = createBatcher({ - name: 'BackupImport.saveMessageBatcher', - wait: 0, - maxSize: 1000, - processBatch: batch => { - const ourAci = this.ourConversation?.serviceId; - assertDev(isAciString(ourAci), 'Our conversation must have ACI'); - - return processMessagesBatch(ourAci, batch); - }, - }); + private readonly conversationOpBatch = new Map< + ConversationAttributesType, + 'save' | 'update' + >(); + private readonly saveMessageBatch = new Set(); private readonly stickerPacks = new Array(); private ourConversation?: ConversationAttributesType; private pinnedConversations = new Array<[number, string]>(); @@ -298,7 +215,7 @@ export class BackupImportStream extends Writable { private pendingGroupAvatars = new Map(); private recentMessages = new CircularMessageCache({ size: RECENT_MESSAGES_CACHE_SIZE, - flush: () => this.saveMessageBatcher.flushAndWait(), + flush: () => this.flushMessages(), }); private constructor(private readonly backupType: BackupType) { @@ -360,8 +277,9 @@ export class BackupImportStream extends Writable { override async _final(done: (error?: Error) => void): Promise { try { // Finish saving remaining conversations/messages - await this.conversationOpBatcher.flushAndWait(); - await this.saveMessageBatcher.flushAndWait(); + await this.flushConversations(); + await this.flushMessages(); + log.info(`${this.logId}: flushed messages and conversations`); // Store sticker packs and schedule downloads await createPacksFromBackup(this.stickerPacks); @@ -408,8 +326,7 @@ export class BackupImportStream extends Writable { .map(([, id]) => id) ); - await loadAll(); - reinitializeRedux(getParametersForRedux()); + await loadAllAndReinitializeRedux(); await window.storage.put( 'backupMediaDownloadTotalBytes', @@ -429,11 +346,6 @@ export class BackupImportStream extends Writable { } } - public cleanup(): void { - this.conversationOpBatcher.unregister(); - this.saveMessageBatcher.unregister(); - } - private async processFrame( frame: Backups.Frame, options: { aboutMe?: AboutMe } @@ -487,7 +399,7 @@ export class BackupImportStream extends Writable { } if (convo !== this.ourConversation) { - this.saveConversation(convo); + await this.saveConversation(convo); } this.recipientIdToConvo.set(recipientId, convo); @@ -516,17 +428,95 @@ export class BackupImportStream extends Writable { } } - private saveConversation(attributes: ConversationAttributesType): void { - this.conversationOpBatcher.add({ isUpdate: false, attributes }); + private async saveConversation( + attributes: ConversationAttributesType + ): Promise { + this.conversationOpBatch.set(attributes, 'save'); + if (this.conversationOpBatch.size >= CONVERSATION_OP_BATCH_SIZE) { + return this.flushConversations(); + } } - private updateConversation(attributes: ConversationAttributesType): void { - this.conversationOpBatcher.add({ isUpdate: true, attributes }); + private async updateConversation( + attributes: ConversationAttributesType + ): Promise { + if (!this.conversationOpBatch.has(attributes)) { + this.conversationOpBatch.set(attributes, 'update'); + } + + if (this.conversationOpBatch.size >= CONVERSATION_OP_BATCH_SIZE) { + return this.flushConversations(); + } } - private saveMessage(attributes: MessageAttributesType): void { + private async saveMessage(attributes: MessageAttributesType): Promise { this.recentMessages.push(attributes); - this.saveMessageBatcher.add(attributes); + this.saveMessageBatch.add(attributes); + if (this.saveMessageBatch.size >= SAVE_MESSAGE_BATCH_SIZE) { + return this.flushMessages(); + } + } + + private async flushConversations(): Promise { + const saves = new Array(); + const updates = new Array(); + for (const [conversation, op] of this.conversationOpBatch) { + if (op === 'save') { + saves.push(conversation); + } else { + updates.push(conversation); + } + } + this.conversationOpBatch.clear(); + + // Queue writes at the same time to prevent races. + await Promise.all([ + saves.length > 0 + ? DataWriter.saveConversations(saves) + : Promise.resolve(), + updates.length > 0 + ? DataWriter.updateConversations(updates) + : Promise.resolve(), + ]); + } + + private async flushMessages(): Promise { + const ourAci = this.ourConversation?.serviceId; + strictAssert(isAciString(ourAci), 'Must have our aci for messages'); + + const batch = Array.from(this.saveMessageBatch); + this.saveMessageBatch.clear(); + + await DataWriter.saveMessages(batch, { + forceSave: true, + ourAci, + }); + + // TODO (DESKTOP-7402): consider re-saving after updating the pending state + for (const attributes of batch) { + const { editHistory } = attributes; + + if (editHistory?.length) { + // eslint-disable-next-line no-await-in-loop + await DataWriter.saveEditedMessages( + attributes, + ourAci, + editHistory.slice(0, -1).map(({ timestamp }) => ({ + conversationId: attributes.conversationId, + messageId: attributes.id, + + // Main message will track this + readStatus: ReadStatus.Read, + sentAt: timestamp, + })) + ); + } + + // eslint-disable-next-line no-await-in-loop + await queueAttachmentDownloads(attributes, { + source: AttachmentDownloadSource.BACKUP_IMPORT, + }); + } } private async saveCallHistory( @@ -749,7 +739,7 @@ export class BackupImportStream extends Writable { ); } - this.updateConversation(me); + await this.updateConversation(me); } private async fromContact( @@ -1175,7 +1165,7 @@ export class BackupImportStream extends Writable { conversation.autoBubbleColor = chatStyle.autoBubbleColor; } - this.updateConversation(conversation); + await this.updateConversation(conversation); if (chat.pinnedOrder != null) { this.pinnedConversations.push([chat.pinnedOrder, conversation.id]); @@ -1333,8 +1323,10 @@ export class BackupImportStream extends Writable { isAciString(this.ourConversation.serviceId), `${logId}: Our conversation must have ACI` ); - this.saveMessage(attributes); - additionalMessages.forEach(additional => this.saveMessage(additional)); + await Promise.all([ + this.saveMessage(attributes), + ...additionalMessages.map(additional => this.saveMessage(additional)), + ]); // TODO (DESKTOP-6964): We'll want to increment for more types here - stickers, etc. if (item.standardMessage) { @@ -1344,7 +1336,7 @@ export class BackupImportStream extends Writable { chatConvo.messageCount = (chatConvo.messageCount ?? 0) + 1; } } - this.updateConversation(chatConvo); + await this.updateConversation(chatConvo); } private fromDirectionDetails( diff --git a/ts/services/contactSync.ts b/ts/services/contactSync.ts index ca9f6e291434..715b590552f2 100644 --- a/ts/services/contactSync.ts +++ b/ts/services/contactSync.ts @@ -224,6 +224,7 @@ async function doContactSync({ await window.storage.put('synced_at', Date.now()); window.Whisper.events.trigger('contactSync:complete'); + window.SignalCI?.handleEvent('contactSync', isFullSync); log.info(`${logId}: done`); } diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 10212026ea6d..a712dcb5568a 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -1585,11 +1585,13 @@ function saveConversation(db: WritableDB, data: ConversationType): void { profileLastFetchedAt, type, serviceId, + expireTimerVersion, } = data; const membersList = getConversationMembersList(data); - db.prepare( + prepare( + db, ` INSERT INTO conversations ( id, @@ -1606,7 +1608,8 @@ function saveConversation(db: WritableDB, data: ConversationType): void { profileName, profileFamilyName, profileFullName, - profileLastFetchedAt + profileLastFetchedAt, + expireTimerVersion ) values ( $id, $json, @@ -1622,7 +1625,8 @@ function saveConversation(db: WritableDB, data: ConversationType): void { $profileName, $profileFamilyName, $profileFullName, - $profileLastFetchedAt + $profileLastFetchedAt, + $expireTimerVersion ); ` ).run({ @@ -1643,6 +1647,7 @@ function saveConversation(db: WritableDB, data: ConversationType): void { profileFamilyName: profileFamilyName || null, profileFullName: combineNames(profileName, profileFamilyName) || null, profileLastFetchedAt: profileLastFetchedAt || null, + expireTimerVersion, }); } @@ -1673,7 +1678,8 @@ function updateConversation(db: WritableDB, data: ConversationType): void { const membersList = getConversationMembersList(data); - db.prepare( + prepare( + db, ` UPDATE conversations SET json = $json, diff --git a/ts/state/ducks/installer.ts b/ts/state/ducks/installer.ts index 0b33489f9914..071248bfcc21 100644 --- a/ts/state/ducks/installer.ts +++ b/ts/state/ducks/installer.ts @@ -322,7 +322,6 @@ function startInstaller(): ThunkAction< dispatch( finishInstall({ deviceName: SignalCI.deviceName, - backupFile: SignalCI.backupData, }) ); } diff --git a/ts/test-both/helpers/generateBackup.ts b/ts/test-both/helpers/generateBackup.ts new file mode 100644 index 000000000000..ada584344db7 --- /dev/null +++ b/ts/test-both/helpers/generateBackup.ts @@ -0,0 +1,219 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { Readable } from 'node:stream'; +import { createGzip } from 'node:zlib'; +import { createCipheriv, randomBytes } from 'node:crypto'; +import { Buffer } from 'node:buffer'; +import Long from 'long'; + +import type { AciString } from '../../types/ServiceId'; +import { generateAci } from '../../types/ServiceId'; +import { CipherType } from '../../types/Crypto'; +import { appendPaddingStream } from '../../util/logPadding'; +import { prependStream } from '../../util/prependStream'; +import { appendMacStream } from '../../util/appendMacStream'; +import { toAciObject } from '../../util/ServiceId'; +import { + deriveBackupKey, + deriveBackupId, + deriveBackupKeyMaterial, +} from '../../Crypto'; +import { BACKUP_VERSION } from '../../services/backups/constants'; +import { Backups } from '../../protobuf'; + +export type BackupGeneratorConfigType = Readonly<{ + aci: AciString; + profileKey: Buffer; + masterKey: Buffer; + conversations: number; + messages: number; +}>; + +const IV_LENGTH = 16; + +export type GenerateBackupResultType = Readonly<{ + backupId: Buffer; + stream: Readable; +}>; + +export function generateBackup( + options: BackupGeneratorConfigType +): GenerateBackupResultType { + const { aci, masterKey } = options; + const backupKey = deriveBackupKey(masterKey); + const aciBytes = toAciObject(aci).getServiceIdBinary(); + const backupId = Buffer.from(deriveBackupId(backupKey, aciBytes)); + const { aesKey, macKey } = deriveBackupKeyMaterial(backupKey, backupId); + + const iv = randomBytes(IV_LENGTH); + + const stream = Readable.from(createRecords(options)) + .pipe(createGzip()) + .pipe(appendPaddingStream()) + .pipe(createCipheriv(CipherType.AES256CBC, aesKey, iv)) + .pipe(prependStream(iv)) + .pipe(appendMacStream(macKey)); + + return { backupId, stream }; +} + +function frame(data: Backups.IFrame): Buffer { + return Buffer.from(Backups.Frame.encodeDelimited(data).finish()); +} + +let now = Date.now(); +function getTimestamp(): Long { + now += 1; + return Long.fromNumber(now); +} + +function* createRecords({ + profileKey, + conversations, + messages, +}: BackupGeneratorConfigType): Iterable { + yield Buffer.from( + Backups.BackupInfo.encodeDelimited({ + version: Long.fromNumber(BACKUP_VERSION), + backupTimeMs: getTimestamp(), + }).finish() + ); + + // Account data + yield frame({ + account: { + profileKey, + givenName: 'Backup', + familyName: 'Benchmark', + accountSettings: { + readReceipts: false, + sealedSenderIndicators: false, + typingIndicators: false, + linkPreviews: false, + notDiscoverableByPhoneNumber: false, + preferContactAvatars: false, + universalExpireTimerSeconds: 0, + preferredReactionEmoji: [], + displayBadgesOnProfile: true, + keepMutedChatsArchived: false, + hasSetMyStoriesPrivacy: true, + hasViewedOnboardingStory: true, + storiesDisabled: false, + hasSeenGroupStoryEducationSheet: true, + hasCompletedUsernameOnboarding: true, + phoneNumberSharingMode: + Backups.AccountData.PhoneNumberSharingMode.EVERYBODY, + defaultChatStyle: { + autoBubbleColor: {}, + dimWallpaperInDarkMode: false, + }, + customChatColors: [], + }, + }, + }); + + const selfId = Long.fromNumber(0); + + yield frame({ + recipient: { + id: selfId, + self: {}, + }, + }); + + const chats = new Array<{ + id: Long; + aci: Buffer; + }>(); + + for (let i = 1; i <= conversations; i += 1) { + const id = Long.fromNumber(i); + const chatAci = toAciObject(generateAci()).getRawUuidBytes(); + + chats.push({ + id, + aci: chatAci, + }); + + yield frame({ + recipient: { + id, + contact: { + aci: chatAci, + blocked: false, + visibility: Backups.Contact.Visibility.VISIBLE, + registered: {}, + profileKey: randomBytes(32), + profileSharing: true, + profileGivenName: `Contact ${i}`, + profileFamilyName: 'Generated', + hideStory: false, + }, + }, + }); + + yield frame({ + chat: { + id, + recipientId: id, + archived: false, + pinnedOrder: 0, + expirationTimerMs: Long.fromNumber(0), + muteUntilMs: Long.fromNumber(0), + markedUnread: false, + dontNotifyForMentionsIfMuted: false, + style: { + autoBubbleColor: {}, + dimWallpaperInDarkMode: false, + }, + expireTimerVersion: 1, + }, + }); + } + + for (let i = 0; i < messages; i += 1) { + const chat = chats[i % chats.length]; + + const isIncoming = i % 2 === 0; + + const dateSent = getTimestamp(); + + yield frame({ + chatItem: { + chatId: chat.id, + authorId: isIncoming ? chat.id : selfId, + dateSent, + revisions: [], + sms: false, + + ...(isIncoming + ? { + incoming: { + dateReceived: getTimestamp(), + dateServerSent: getTimestamp(), + read: true, + sealedSender: true, + }, + } + : { + outgoing: { + sendStatus: [ + { + recipientId: chat.id, + timestamp: dateSent, + sent: { sealedSender: true }, + }, + ], + }, + }), + + standardMessage: { + text: { + body: `Message ${i}`, + }, + }, + }, + }); + } +} diff --git a/ts/test-electron/backup/attachments_test.ts b/ts/test-electron/backup/attachments_test.ts index c26b5c4504ae..04dab2f1ed0d 100644 --- a/ts/test-electron/backup/attachments_test.ts +++ b/ts/test-electron/backup/attachments_test.ts @@ -31,7 +31,7 @@ import { isVoiceMessage, type AttachmentType } from '../../types/Attachment'; import { strictAssert } from '../../util/assert'; import { SignalService } from '../../protobuf'; import { getRandomBytes } from '../../Crypto'; -import { loadAll } from '../../services/allLoaders'; +import { loadAllAndReinitializeRedux } from '../../services/allLoaders'; const CONTACT_A = generateAci(); @@ -65,7 +65,7 @@ describe('backup/attachments', () => { { systemGivenName: 'CONTACT_A', active_at: 1 } ); - await loadAll(); + await loadAllAndReinitializeRedux(); sandbox = sinon.createSandbox(); const getAbsoluteAttachmentPath = sandbox.stub( diff --git a/ts/test-electron/backup/backup_groupv2_notifications_test.ts b/ts/test-electron/backup/backup_groupv2_notifications_test.ts index c0bfbda4654c..216f52ce57db 100644 --- a/ts/test-electron/backup/backup_groupv2_notifications_test.ts +++ b/ts/test-electron/backup/backup_groupv2_notifications_test.ts @@ -23,7 +23,7 @@ import { } from './helpers'; import { ReadStatus } from '../../messages/MessageReadStatus'; import { SeenStatus } from '../../MessageSeenStatus'; -import { loadAll } from '../../services/allLoaders'; +import { loadAllAndReinitializeRedux } from '../../services/allLoaders'; // Note: this should be kept up to date with GroupV2Change.stories.tsx, to // maintain the comprehensive set of GroupV2 notifications we need to handle @@ -127,7 +127,7 @@ describe('backup/groupv2/notifications', () => { active_at: 1, }); - await loadAll(); + await loadAllAndReinitializeRedux(); }); afterEach(async () => { await DataWriter.removeAll(); diff --git a/ts/test-electron/backup/bubble_test.ts b/ts/test-electron/backup/bubble_test.ts index 429d5e9dc0d1..a87f6f59912d 100644 --- a/ts/test-electron/backup/bubble_test.ts +++ b/ts/test-electron/backup/bubble_test.ts @@ -21,7 +21,7 @@ import { symmetricRoundtripHarness, OUR_ACI, } from './helpers'; -import { loadAll } from '../../services/allLoaders'; +import { loadAllAndReinitializeRedux } from '../../services/allLoaders'; const CONTACT_A = generateAci(); const CONTACT_B = generateAci(); @@ -68,7 +68,7 @@ describe('backup/bubble messages', () => { } ); - await loadAll(); + await loadAllAndReinitializeRedux(); }); it('roundtrips incoming edited message', async () => { diff --git a/ts/test-electron/backup/calling_test.ts b/ts/test-electron/backup/calling_test.ts index a44b4d8deb3a..8431b7e53212 100644 --- a/ts/test-electron/backup/calling_test.ts +++ b/ts/test-electron/backup/calling_test.ts @@ -29,7 +29,7 @@ import { fromAdminKeyBytes } from '../../util/callLinks'; import { ReadStatus } from '../../messages/MessageReadStatus'; import { SeenStatus } from '../../MessageSeenStatus'; import { deriveGroupID, deriveGroupSecretParams } from '../../util/zkgroup'; -import { loadAll } from '../../services/allLoaders'; +import { loadAllAndReinitializeRedux } from '../../services/allLoaders'; const CONTACT_A = generateAci(); const GROUP_MASTER_KEY = getRandomBytes(32); @@ -82,7 +82,7 @@ describe('backup/calling', () => { await DataWriter.insertCallLink(callLink); - await loadAll(); + await loadAllAndReinitializeRedux(); }); after(async () => { await DataWriter.removeAll(); @@ -105,7 +105,7 @@ describe('backup/calling', () => { endedTimestamp: null, }; await DataWriter.saveCallHistory(callHistory); - await loadAll(); + await loadAllAndReinitializeRedux(); const messageUnseen: MessageAttributesType = { id: generateGuid(), @@ -154,7 +154,7 @@ describe('backup/calling', () => { endedTimestamp: null, }; await DataWriter.saveCallHistory(callHistory); - await loadAll(); + await loadAllAndReinitializeRedux(); const messageUnseen: MessageAttributesType = { id: generateGuid(), @@ -245,7 +245,7 @@ describe('backup/calling', () => { endedTimestamp: null, }; await DataWriter.saveCallHistory(callHistory); - await loadAll(); + await loadAllAndReinitializeRedux(); await symmetricRoundtripHarness([]); @@ -271,7 +271,7 @@ describe('backup/calling', () => { endedTimestamp: null, }; await DataWriter.saveCallHistory(callHistory); - await loadAll(); + await loadAllAndReinitializeRedux(); await symmetricRoundtripHarness([]); diff --git a/ts/test-electron/backup/integration_test.ts b/ts/test-electron/backup/integration_test.ts index 58e6bfaa0eb1..5c907fad2e8a 100644 --- a/ts/test-electron/backup/integration_test.ts +++ b/ts/test-electron/backup/integration_test.ts @@ -14,8 +14,10 @@ import { import { assert } from 'chai'; import { clearData } from './helpers'; -import { loadAll } from '../../services/allLoaders'; +import { loadAllAndReinitializeRedux } from '../../services/allLoaders'; import { backupsService, BackupType } from '../../services/backups'; +import { initialize as initializeExpiringMessageService } from '../../services/expiringMessagesDeletion'; +import { singleProtoJobQueue } from '../../jobs/singleProtoJobQueue'; import { DataWriter } from '../../sql/Client'; const { BACKUP_INTEGRATION_DIR } = process.env; @@ -39,9 +41,13 @@ class MemoryStream extends InputStream { } describe('backup/integration', () => { + before(async () => { + await initializeExpiringMessageService(singleProtoJobQueue); + }); + beforeEach(async () => { await clearData(); - await loadAll(); + await loadAllAndReinitializeRedux(); }); afterEach(async () => { diff --git a/ts/test-electron/backup/non_bubble_test.ts b/ts/test-electron/backup/non_bubble_test.ts index 9a8b0118b0eb..109162a61bfd 100644 --- a/ts/test-electron/backup/non_bubble_test.ts +++ b/ts/test-electron/backup/non_bubble_test.ts @@ -24,7 +24,7 @@ import { symmetricRoundtripHarness, OUR_ACI, } from './helpers'; -import { loadAll } from '../../services/allLoaders'; +import { loadAllAndReinitializeRedux } from '../../services/allLoaders'; const CONTACT_A = generateAci(); const GROUP_ID = Bytes.toBase64(getRandomBytes(32)); @@ -57,7 +57,7 @@ describe('backup/non-bubble messages', () => { } ); - await loadAll(); + await loadAllAndReinitializeRedux(); }); it('roundtrips END_SESSION simple update', async () => { diff --git a/ts/test-mock/backups/backups_test.ts b/ts/test-mock/backups/backups_test.ts index d123bebf703f..cc74b597db13 100644 --- a/ts/test-mock/backups/backups_test.ts +++ b/ts/test-mock/backups/backups_test.ts @@ -188,8 +188,7 @@ describe('backups', function (this: Mocha.Suite) { ); } - const backupPath = bootstrap.getBackupPath('backup.bin'); - await app.exportBackupToDisk(backupPath); + await app.uploadBackup(); const comparator = await bootstrap.createScreenshotComparator( app, @@ -247,9 +246,12 @@ describe('backups', function (this: Mocha.Suite) { // Restart await bootstrap.eraseStorage(); - app = await bootstrap.link({ - ciBackupPath: backupPath, - }); + app = await bootstrap.link(); + await app.waitForBackupImportComplete(); + + // Make sure that contact sync happens after backup import, otherwise the + // app won't show contacts as "system" + await app.waitForContactSync(); await comparator(app); }); diff --git a/ts/test-mock/benchmarks/backup_bench.ts b/ts/test-mock/benchmarks/backup_bench.ts new file mode 100644 index 000000000000..a8552dfb57b7 --- /dev/null +++ b/ts/test-mock/benchmarks/backup_bench.ts @@ -0,0 +1,47 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only +/* eslint-disable no-console */ + +import { pipeline } from 'node:stream/promises'; +import { createWriteStream } from 'node:fs'; +import { mkdir } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { Bootstrap } from './fixtures'; +import { generateBackup } from '../../test-both/helpers/generateBackup'; + +Bootstrap.benchmark(async (bootstrap: Bootstrap): Promise => { + const { phone, cdn3Path } = bootstrap; + + const { backupId, stream: backupStream } = generateBackup({ + aci: phone.device.aci, + profileKey: phone.profileKey.serialize(), + masterKey: phone.masterKey, + conversations: 1000, + messages: 60 * 1000, + }); + const backupFolder = join( + cdn3Path, + 'backups', + backupId.toString('base64url') + ); + await mkdir(backupFolder, { recursive: true }); + const fileStream = createWriteStream(join(backupFolder, 'backup')); + await pipeline(backupStream, fileStream); + + const importStart = Date.now(); + + const app = await bootstrap.link(); + await app.waitForBackupImportComplete(); + + const importEnd = Date.now(); + + const exportStart = Date.now(); + await app.uploadBackup(); + const exportEnd = Date.now(); + + console.log('run=%d info=%j', 0, { + importDuration: importEnd - importStart, + exportDuration: exportEnd - exportStart, + }); +}); diff --git a/ts/test-mock/bootstrap.ts b/ts/test-mock/bootstrap.ts index c70b2aff7eb3..4db76d5b05e9 100644 --- a/ts/test-mock/bootstrap.ts +++ b/ts/test-mock/bootstrap.ts @@ -150,6 +150,7 @@ function sanitizePathComponent(component: string): string { // export class Bootstrap { public readonly server: Server; + public readonly cdn3Path: string; private readonly options: BootstrapInternalOptions; private privContacts?: ReadonlyArray; @@ -158,8 +159,6 @@ export class Bootstrap { private privPhone?: PrimaryDevice; private privDesktop?: Device; private storagePath?: string; - private backupPath?: string; - private cdn3Path: string; private timestamp: number = Date.now() - durations.WEEK; private lastApp?: App; private readonly randomId = crypto.randomBytes(8).toString('hex'); @@ -238,9 +237,6 @@ export class Bootstrap { }); this.storagePath = await fs.mkdtemp(path.join(os.tmpdir(), 'mock-signal-')); - this.backupPath = await fs.mkdtemp( - path.join(os.tmpdir(), 'mock-signal-backup-') - ); debug('setting storage path=%j', this.storagePath); } @@ -270,15 +266,6 @@ export class Bootstrap { return path.join(this.storagePath, 'ephemeral.json'); } - public getBackupPath(fileName: string): string { - assert( - this.backupPath !== undefined, - 'Bootstrap has to be initialized first, see: bootstrap.init()' - ); - - return path.join(this.backupPath, fileName); - } - public eraseStorage(): Promise { return this.resetAppStorage(); } @@ -289,7 +276,6 @@ export class Bootstrap { 'Bootstrap has to be initialized first, see: bootstrap.init()' ); - // Note that backupPath must remain unchanged! await fs.rm(this.storagePath, { recursive: true }); this.storagePath = await fs.mkdtemp(path.join(os.tmpdir(), 'mock-signal-')); } @@ -299,7 +285,7 @@ export class Bootstrap { await Promise.race([ Promise.all([ - ...[this.storagePath, this.backupPath, this.cdn3Path].map(tmpPath => + ...[this.storagePath, this.cdn3Path].map(tmpPath => tmpPath ? fs.rm(tmpPath, { recursive: true }) : Promise.resolve() ), this.server.close(), @@ -354,11 +340,6 @@ export class Bootstrap { } } - if (extraConfig?.ciBackupPath) { - debug('waiting for backup import to complete'); - await app.waitForBackupImportComplete(); - } - await this.phone.waitForSync(this.desktop); this.phone.resetSyncState(this.desktop); @@ -384,7 +365,7 @@ export class Bootstrap { debug('starting the app'); - const { port } = this.server.address(); + const { port, family } = this.server.address(); let startAttempts = 0; const MAX_ATTEMPTS = 4; @@ -398,7 +379,7 @@ export class Bootstrap { } // eslint-disable-next-line no-await-in-loop - const config = await this.generateConfig(port, extraConfig); + const config = await this.generateConfig(port, family, extraConfig); const startedApp = new App({ main: ELECTRON, @@ -661,9 +642,12 @@ export class Bootstrap { private async generateConfig( port: number, + family: string, extraConfig?: Partial ): Promise { - const url = `https://127.0.0.1:${port}`; + const host = family === 'IPv6' ? '[::1]' : '127.0.0.1'; + + const url = `https://${host}:${port}`; return JSON.stringify({ ...(await loadCertificates()), diff --git a/ts/test-mock/playwright.ts b/ts/test-mock/playwright.ts index 127ce0b8861a..e2b09ab267dc 100644 --- a/ts/test-mock/playwright.ts +++ b/ts/test-mock/playwright.ts @@ -105,6 +105,10 @@ export class App extends EventEmitter { return this.waitForEvent('app-loaded'); } + public async waitForContactSync(): Promise { + return this.waitForEvent('contactSync'); + } + public async waitForBackupImportComplete(): Promise { return this.waitForEvent('backupImportComplete'); } @@ -186,18 +190,9 @@ export class App extends EventEmitter { return window.evaluate(`window.SignalCI.getMessagesBySentAt(${timestamp})`); } - public async exportBackupToDisk(path: string): Promise { + public async uploadBackup(): Promise { const window = await this.getWindow(); - return window.evaluate( - `window.SignalCI.exportBackupToDisk(${JSON.stringify(path)})` - ); - } - - public async exportPlaintextBackupToDisk(path: string): Promise { - const window = await this.getWindow(); - return window.evaluate( - `window.SignalCI.exportPlaintextBackupToDisk(${JSON.stringify(path)})` - ); + await window.evaluate('window.SignalCI.uploadBackup()'); } public async unlink(): Promise { diff --git a/ts/util/isBackupEnabled.ts b/ts/util/isBackupEnabled.ts index 212e85f4ed51..92704dd280a6 100644 --- a/ts/util/isBackupEnabled.ts +++ b/ts/util/isBackupEnabled.ts @@ -2,10 +2,11 @@ // SPDX-License-Identifier: AGPL-3.0-only import * as RemoteConfig from '../RemoteConfig'; +import { isTestOrMockEnvironment } from '../environment'; import { isStagingServer } from './isStagingServer'; export function isBackupEnabled(): boolean { - if (isStagingServer()) { + if (isStagingServer() || isTestOrMockEnvironment()) { return true; } return Boolean(RemoteConfig.isEnabled('desktop.backup.credentialFetch')); diff --git a/ts/windows/main/phase4-test.ts b/ts/windows/main/phase4-test.ts index 80c76323e957..cf38eea7bbc6 100644 --- a/ts/windows/main/phase4-test.ts +++ b/ts/windows/main/phase4-test.ts @@ -5,8 +5,6 @@ /* eslint-disable no-console */ /* eslint-disable global-require */ -import fs from 'fs'; - const { config } = window.SignalContext; if (config.environment === 'test') { @@ -16,14 +14,10 @@ if (config.environment === 'test') { if (config.ciMode) { console.log( - `Importing CI infrastructure; enabled in config, mode: ${config.ciMode}, ` + - `backupPath: ${config.ciBackupPath}` + `Importing CI infrastructure; enabled in config, mode: ${config.ciMode}` ); const { getCI } = require('../../CI'); window.SignalCI = getCI({ deviceName: window.getTitle(), - backupData: config.ciBackupPath - ? fs.readFileSync(config.ciBackupPath) - : undefined, }); }