Fully move backup integration test to mock server
This commit is contained in:
Fedor Indutny
GitHub
parent
12f28448b2
commit
bad065859c
24 changed files with 508 additions and 232 deletions
15
.github/workflows/benchmark.yml
vendored
15
.github/workflows/benchmark.yml
vendored
|
|
@ -170,6 +170,20 @@ jobs:
|
|||
# DEBUG: 'mock:benchmarks'
|
||||
ARTIFACTS_DIR: artifacts/call-history-search
|
||||
|
||||
- name: Run backup benchmarks
|
||||
run: |
|
||||
set -o pipefail
|
||||
rm -rf /tmp/mock
|
||||
xvfb-run --auto-servernum node \
|
||||
ts/test-mock/benchmarks/backup_bench.js | \
|
||||
tee benchmark-backup.log
|
||||
timeout-minutes: 10
|
||||
env:
|
||||
NODE_ENV: production
|
||||
ELECTRON_ENABLE_STACK_DUMPING: on
|
||||
# DEBUG: 'mock:benchmarks'
|
||||
ARTIFACTS_DIR: artifacts/backup-bench
|
||||
|
||||
- name: Upload benchmark logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
|
|
@ -200,5 +214,6 @@ jobs:
|
|||
node ./bin/publish.js ../benchmark-large-group-send.log desktop.ci.performance.largeGroupSend
|
||||
node ./bin/publish.js ../benchmark-convo-open.log desktop.ci.performance.convoOpen
|
||||
node ./bin/publish.js ../benchmark-call-history-search.log desktop.ci.performance.callHistorySearch
|
||||
node ./bin/publish.js ../benchmark-backup.log desktop.ci.performance.backup
|
||||
env:
|
||||
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
|
||||
|
|
|
|||
46
package-lock.json
generated
46
package-lock.json
generated
|
|
@ -126,7 +126,7 @@
|
|||
"@indutny/parallel-prettier": "3.0.0",
|
||||
"@indutny/rezip-electron": "1.3.1",
|
||||
"@indutny/symbolicate-mac": "2.3.0",
|
||||
"@signalapp/mock-server": "7.0.1",
|
||||
"@signalapp/mock-server": "7.1.3",
|
||||
"@storybook/addon-a11y": "8.1.11",
|
||||
"@storybook/addon-actions": "8.1.11",
|
||||
"@storybook/addon-controls": "8.1.11",
|
||||
|
|
@ -7274,22 +7274,24 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@signalapp/mock-server": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@signalapp/mock-server/-/mock-server-7.0.1.tgz",
|
||||
"integrity": "sha512-iwH57apXyTHKjozaV1ZJW6nbVhFH3KlVOQYaJiO2bT3YgAGdYoJvHp8+MMIQ8OFYVGRo3g7wouqX/JT5HElAvw==",
|
||||
"version": "7.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@signalapp/mock-server/-/mock-server-7.1.3.tgz",
|
||||
"integrity": "sha512-Xvpeai+E0mhz6WHSycYuY31y5saCNJYX7ioDn1Q0LqUAOUKGVQjnWvdxeXLPKv8C06mbWn0lP16o9swClWVsmg==",
|
||||
"dev": true,
|
||||
"license": "AGPL-3.0-only",
|
||||
"dependencies": {
|
||||
"@indutny/parallel-prettier": "^3.0.0",
|
||||
"@signalapp/libsignal-client": "^0.45.0",
|
||||
"@signalapp/libsignal-client": "^0.58.2",
|
||||
"@tus/file-store": "^1.4.0",
|
||||
"@tus/server": "^1.7.0",
|
||||
"debug": "^4.3.2",
|
||||
"is-plain-obj": "3.0.0",
|
||||
"long": "^4.0.0",
|
||||
"micro": "^9.3.4",
|
||||
"microrouter": "^3.1.3",
|
||||
"prettier": "^3.3.3",
|
||||
"protobufjs": "^7.2.4",
|
||||
"type-fest": "^4.26.1",
|
||||
"url-pattern": "^1.0.3",
|
||||
"uuid": "^8.3.2",
|
||||
"ws": "^8.4.2",
|
||||
|
|
@ -7297,14 +7299,15 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@signalapp/mock-server/node_modules/@signalapp/libsignal-client": {
|
||||
"version": "0.45.1",
|
||||
"resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.45.1.tgz",
|
||||
"integrity": "sha512-jKNGLD8QQkLEopX7Fb5XG7LlIe559TgqfC1UCgUV9YW4pPpvM+RPbW4ndL1v8WO/Toff4nVXJXJV6kzYiK2lDA==",
|
||||
"version": "0.58.2",
|
||||
"resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.58.2.tgz",
|
||||
"integrity": "sha512-3OF9fGmh7tz9JVfT9xTR4DWcm4HOpbQknO9k7Oj23uSsBSEcJYmYPGM3Rdm16C/z+evVgyvrLptPtjTzXXuNzA==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "AGPL-3.0-only",
|
||||
"dependencies": {
|
||||
"node-gyp-build": "^4.2.3",
|
||||
"type-fest": "^3.5.0",
|
||||
"node-gyp-build": "^4.8.0",
|
||||
"type-fest": "^4.26.0",
|
||||
"uuid": "^8.3.0"
|
||||
}
|
||||
},
|
||||
|
|
@ -7325,6 +7328,19 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@signalapp/mock-server/node_modules/is-plain-obj": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-3.0.0.tgz",
|
||||
"integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/@signalapp/mock-server/node_modules/ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
|
|
@ -7332,12 +7348,13 @@
|
|||
"dev": true
|
||||
},
|
||||
"node_modules/@signalapp/mock-server/node_modules/type-fest": {
|
||||
"version": "3.13.1",
|
||||
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.13.1.tgz",
|
||||
"integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==",
|
||||
"version": "4.26.1",
|
||||
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.26.1.tgz",
|
||||
"integrity": "sha512-yOGpmOAL7CkKe/91I5O3gPICmJNLJ1G4zFYVAsRHg7M64biSnPtRj0WNQt++bRkjYOqjWXrhnUw1utzmVErAdg==",
|
||||
"dev": true,
|
||||
"license": "(MIT OR CC0-1.0)",
|
||||
"engines": {
|
||||
"node": ">=14.16"
|
||||
"node": ">=16"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
|
|
@ -7348,6 +7365,7 @@
|
|||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
|
||||
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"uuid": "dist/bin/uuid"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -210,7 +210,7 @@
|
|||
"@indutny/parallel-prettier": "3.0.0",
|
||||
"@indutny/rezip-electron": "1.3.1",
|
||||
"@indutny/symbolicate-mac": "2.3.0",
|
||||
"@signalapp/mock-server": "7.0.1",
|
||||
"@signalapp/mock-server": "7.1.3",
|
||||
"@storybook/addon-a11y": "8.1.11",
|
||||
"@storybook/addon-actions": "8.1.11",
|
||||
"@storybook/addon-controls": "8.1.11",
|
||||
|
|
|
|||
28
ts/CI.ts
28
ts/CI.ts
|
|
@ -3,14 +3,13 @@
|
|||
|
||||
import { format } from 'node:util';
|
||||
import { ipcRenderer } from 'electron';
|
||||
import { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
|
||||
|
||||
import type { IPCResponse as ChallengeResponseType } from './challenge';
|
||||
import type { MessageAttributesType } from './model-types.d';
|
||||
import * as log from './logging/log';
|
||||
import { explodePromise } from './util/explodePromise';
|
||||
import { AccessType, ipcInvoke } from './sql/channels';
|
||||
import { backupsService, BackupType } from './services/backups';
|
||||
import { backupsService } from './services/backups';
|
||||
import { SECOND } from './util/durations';
|
||||
import { isSignalRoute } from './util/signalRoutes';
|
||||
import { strictAssert } from './util/assert';
|
||||
|
|
@ -19,7 +18,6 @@ type ResolveType = (data: unknown) => void;
|
|||
|
||||
export type CIType = {
|
||||
deviceName: string;
|
||||
backupData?: Uint8Array;
|
||||
getConversationId: (address: string | null) => string | null;
|
||||
getMessagesBySentAt(
|
||||
sentAt: number
|
||||
|
|
@ -36,18 +34,16 @@ export type CIType = {
|
|||
}
|
||||
) => unknown;
|
||||
openSignalRoute(url: string): Promise<void>;
|
||||
exportBackupToDisk(path: string): Promise<void>;
|
||||
exportPlaintextBackupToDisk(path: string): Promise<void>;
|
||||
uploadBackup(): Promise<void>;
|
||||
unlink: () => void;
|
||||
print: (...args: ReadonlyArray<unknown>) => void;
|
||||
};
|
||||
|
||||
export type GetCIOptionsType = Readonly<{
|
||||
deviceName: string;
|
||||
backupData?: Uint8Array;
|
||||
}>;
|
||||
|
||||
export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
|
||||
export function getCI({ deviceName }: GetCIOptionsType): CIType {
|
||||
const eventListeners = new Map<string, Array<ResolveType>>();
|
||||
const completedEvents = new Map<string, Array<unknown>>();
|
||||
|
||||
|
|
@ -66,8 +62,8 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
|
|||
|
||||
if (!options?.ignorePastEvents) {
|
||||
const pendingCompleted = completedEvents.get(event) || [];
|
||||
if (pendingCompleted.length) {
|
||||
const pending = pendingCompleted.shift();
|
||||
if (pending) {
|
||||
log.info(`CI: resolving pending result for ${event}`, pending);
|
||||
|
||||
if (pendingCompleted.length === 0) {
|
||||
|
|
@ -170,16 +166,8 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
|
|||
document.body.removeChild(a);
|
||||
}
|
||||
|
||||
async function exportBackupToDisk(path: string) {
|
||||
await backupsService.exportToDisk(path, BackupLevel.Media);
|
||||
}
|
||||
|
||||
async function exportPlaintextBackupToDisk(path: string) {
|
||||
await backupsService.exportToDisk(
|
||||
path,
|
||||
BackupLevel.Media,
|
||||
BackupType.TestOnlyPlaintext
|
||||
);
|
||||
async function uploadBackup() {
|
||||
await backupsService.upload();
|
||||
}
|
||||
|
||||
function unlink() {
|
||||
|
|
@ -192,7 +180,6 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
|
|||
|
||||
return {
|
||||
deviceName,
|
||||
backupData,
|
||||
getConversationId,
|
||||
getMessagesBySentAt,
|
||||
handleEvent,
|
||||
|
|
@ -200,8 +187,7 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
|
|||
solveChallenge,
|
||||
waitForEvent,
|
||||
openSignalRoute,
|
||||
exportBackupToDisk,
|
||||
exportPlaintextBackupToDisk,
|
||||
uploadBackup,
|
||||
unlink,
|
||||
getPendingEventCount,
|
||||
print,
|
||||
|
|
|
|||
|
|
@ -1598,8 +1598,10 @@ export async function startApp(): Promise<void> {
|
|||
},
|
||||
});
|
||||
|
||||
log.info('afterStart: backup downloaded, resolving');
|
||||
backupReady.resolve();
|
||||
} catch (error) {
|
||||
log.error('afterStart: backup download failed, rejecting');
|
||||
backupReady.reject(error);
|
||||
throw error;
|
||||
}
|
||||
|
|
@ -1706,17 +1708,21 @@ export async function startApp(): Promise<void> {
|
|||
|
||||
strictAssert(server !== undefined, 'WebAPI not connected');
|
||||
|
||||
try {
|
||||
connectPromise = explodePromise();
|
||||
|
||||
// Wait for backup to be downloaded
|
||||
try {
|
||||
await backupReady.promise;
|
||||
} catch (error) {
|
||||
log.error('background: backup download failed, not reconnecting', error);
|
||||
log.error(
|
||||
'background: backup download failed, not reconnecting',
|
||||
error
|
||||
);
|
||||
return;
|
||||
}
|
||||
log.info('background: connect unblocked by backups');
|
||||
|
||||
try {
|
||||
connectPromise = explodePromise();
|
||||
// Reset the flag and update it below if needed
|
||||
setIsInitialSync(false);
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,8 @@ import {
|
|||
getInitialState as getStickersReduxState,
|
||||
} from '../types/Stickers';
|
||||
|
||||
import type { ReduxInitData } from '../state/initializeRedux';
|
||||
import { type ReduxInitData } from '../state/initializeRedux';
|
||||
import { reinitializeRedux } from '../state/reinitializeRedux';
|
||||
|
||||
export async function loadAll(): Promise<void> {
|
||||
await Promise.all([
|
||||
|
|
@ -41,6 +42,11 @@ export async function loadAll(): Promise<void> {
|
|||
]);
|
||||
}
|
||||
|
||||
export async function loadAllAndReinitializeRedux(): Promise<void> {
|
||||
await loadAll();
|
||||
reinitializeRedux(getParametersForRedux());
|
||||
}
|
||||
|
||||
export function getParametersForRedux(): ReduxInitData {
|
||||
const { mainWindowStats, menuOptions, theme } = getUserDataForRedux();
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ export class BackupAPI {
|
|||
constructor(private credentials: BackupCredentials) {}
|
||||
|
||||
public async refresh(): Promise<void> {
|
||||
// TODO: DESKTOP-6979
|
||||
await this.server.refreshBackup(
|
||||
await this.credentials.getHeadersForToday()
|
||||
);
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import {
|
|||
import * as log from '../../logging/log';
|
||||
import { GiftBadgeStates } from '../../components/conversation/Message';
|
||||
import { StorySendMode, MY_STORY_ID } from '../../types/Stories';
|
||||
import type { ServiceIdString, AciString } from '../../types/ServiceId';
|
||||
import type { ServiceIdString } from '../../types/ServiceId';
|
||||
import {
|
||||
fromAciObject,
|
||||
fromPniObject,
|
||||
|
|
@ -64,7 +64,6 @@ import {
|
|||
} from '../../util/zkgroup';
|
||||
import { incrementMessageCounter } from '../../util/incrementMessageCounter';
|
||||
import { isAciString } from '../../util/isAciString';
|
||||
import { createBatcher } from '../../util/batcher';
|
||||
import { PhoneNumberDiscoverability } from '../../util/phoneNumberDiscoverability';
|
||||
import { PhoneNumberSharingMode } from '../../util/phoneNumberSharingMode';
|
||||
import { bytesToUuid } from '../../util/uuidToBytes';
|
||||
|
|
@ -79,7 +78,6 @@ import type { AboutMe, LocalChatStyle } from './types';
|
|||
import { BackupType } from './types';
|
||||
import type { GroupV2ChangeDetailType } from '../../groups';
|
||||
import { queueAttachmentDownloads } from '../../util/queueAttachmentDownloads';
|
||||
import { drop } from '../../util/drop';
|
||||
import { isNotNil } from '../../util/isNotNil';
|
||||
import { isGroup } from '../../util/whatTypeOfConversation';
|
||||
import { rgbToHSL } from '../../util/rgbToHSL';
|
||||
|
|
@ -107,89 +105,23 @@ import type { CallLinkType } from '../../types/CallLink';
|
|||
import type { RawBodyRange } from '../../types/BodyRange';
|
||||
import { fromAdminKeyBytes } from '../../util/callLinks';
|
||||
import { getRoomIdFromRootKey } from '../../util/callLinksRingrtc';
|
||||
import { reinitializeRedux } from '../../state/reinitializeRedux';
|
||||
import { getParametersForRedux, loadAll } from '../allLoaders';
|
||||
import { loadAllAndReinitializeRedux } from '../allLoaders';
|
||||
import { resetBackupMediaDownloadProgress } from '../../util/backupMediaDownload';
|
||||
import { getEnvironment, isTestEnvironment } from '../../environment';
|
||||
|
||||
const MAX_CONCURRENCY = 10;
|
||||
|
||||
const CONVERSATION_OP_BATCH_SIZE = 10000;
|
||||
const SAVE_MESSAGE_BATCH_SIZE = 10000;
|
||||
|
||||
// Keep 1000 recent messages in memory to speed up quote lookup.
|
||||
const RECENT_MESSAGES_CACHE_SIZE = 1000;
|
||||
|
||||
type ConversationOpType = Readonly<{
|
||||
isUpdate: boolean;
|
||||
attributes: ConversationAttributesType;
|
||||
}>;
|
||||
|
||||
type ChatItemParseResult = {
|
||||
message: Partial<MessageAttributesType>;
|
||||
additionalMessages: Array<Partial<MessageAttributesType>>;
|
||||
};
|
||||
|
||||
async function processConversationOpBatch(
|
||||
batch: ReadonlyArray<ConversationOpType>
|
||||
): Promise<void> {
|
||||
// Note that we might have duplicates since we update attributes in-place
|
||||
const saves = [
|
||||
...new Set(batch.filter(x => x.isUpdate === false).map(x => x.attributes)),
|
||||
];
|
||||
const updates = [
|
||||
...new Set(batch.filter(x => x.isUpdate === true).map(x => x.attributes)),
|
||||
];
|
||||
|
||||
log.info(
|
||||
`backups: running conversation op batch, saves=${saves.length} ` +
|
||||
`updates=${updates.length}`
|
||||
);
|
||||
|
||||
await DataWriter.saveConversations(saves);
|
||||
await DataWriter.updateConversations(updates);
|
||||
}
|
||||
async function processMessagesBatch(
|
||||
ourAci: AciString,
|
||||
batch: ReadonlyArray<MessageAttributesType>
|
||||
): Promise<void> {
|
||||
const ids = await DataWriter.saveMessages(batch, {
|
||||
forceSave: true,
|
||||
ourAci,
|
||||
});
|
||||
strictAssert(ids.length === batch.length, 'Should get same number of ids');
|
||||
|
||||
// TODO (DESKTOP-7402): consider re-saving after updating the pending state
|
||||
for (const [index, rawAttributes] of batch.entries()) {
|
||||
const attributes = {
|
||||
...rawAttributes,
|
||||
id: ids[index],
|
||||
};
|
||||
|
||||
const { editHistory } = attributes;
|
||||
|
||||
if (editHistory?.length) {
|
||||
drop(
|
||||
DataWriter.saveEditedMessages(
|
||||
attributes,
|
||||
ourAci,
|
||||
editHistory.slice(0, -1).map(({ timestamp }) => ({
|
||||
conversationId: attributes.conversationId,
|
||||
messageId: attributes.id,
|
||||
|
||||
// Main message will track this
|
||||
readStatus: ReadStatus.Read,
|
||||
sentAt: timestamp,
|
||||
}))
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
drop(
|
||||
queueAttachmentDownloads(attributes, {
|
||||
source: AttachmentDownloadSource.BACKUP_IMPORT,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function phoneToContactFormType(
|
||||
type: Backups.ContactAttachment.Phone.Type | null | undefined
|
||||
): ContactFormType {
|
||||
|
|
@ -269,26 +201,11 @@ export class BackupImportStream extends Writable {
|
|||
number,
|
||||
ConversationAttributesType
|
||||
>();
|
||||
private readonly conversationOpBatcher = createBatcher<{
|
||||
isUpdate: boolean;
|
||||
attributes: ConversationAttributesType;
|
||||
}>({
|
||||
name: 'BackupImport.conversationOpBatcher',
|
||||
wait: 0,
|
||||
maxSize: 1000,
|
||||
processBatch: processConversationOpBatch,
|
||||
});
|
||||
private readonly saveMessageBatcher = createBatcher<MessageAttributesType>({
|
||||
name: 'BackupImport.saveMessageBatcher',
|
||||
wait: 0,
|
||||
maxSize: 1000,
|
||||
processBatch: batch => {
|
||||
const ourAci = this.ourConversation?.serviceId;
|
||||
assertDev(isAciString(ourAci), 'Our conversation must have ACI');
|
||||
|
||||
return processMessagesBatch(ourAci, batch);
|
||||
},
|
||||
});
|
||||
private readonly conversationOpBatch = new Map<
|
||||
ConversationAttributesType,
|
||||
'save' | 'update'
|
||||
>();
|
||||
private readonly saveMessageBatch = new Set<MessageAttributesType>();
|
||||
private readonly stickerPacks = new Array<StickerPackPointerType>();
|
||||
private ourConversation?: ConversationAttributesType;
|
||||
private pinnedConversations = new Array<[number, string]>();
|
||||
|
|
@ -298,7 +215,7 @@ export class BackupImportStream extends Writable {
|
|||
private pendingGroupAvatars = new Map<string, string>();
|
||||
private recentMessages = new CircularMessageCache({
|
||||
size: RECENT_MESSAGES_CACHE_SIZE,
|
||||
flush: () => this.saveMessageBatcher.flushAndWait(),
|
||||
flush: () => this.flushMessages(),
|
||||
});
|
||||
|
||||
private constructor(private readonly backupType: BackupType) {
|
||||
|
|
@ -360,8 +277,9 @@ export class BackupImportStream extends Writable {
|
|||
override async _final(done: (error?: Error) => void): Promise<void> {
|
||||
try {
|
||||
// Finish saving remaining conversations/messages
|
||||
await this.conversationOpBatcher.flushAndWait();
|
||||
await this.saveMessageBatcher.flushAndWait();
|
||||
await this.flushConversations();
|
||||
await this.flushMessages();
|
||||
log.info(`${this.logId}: flushed messages and conversations`);
|
||||
|
||||
// Store sticker packs and schedule downloads
|
||||
await createPacksFromBackup(this.stickerPacks);
|
||||
|
|
@ -408,8 +326,7 @@ export class BackupImportStream extends Writable {
|
|||
.map(([, id]) => id)
|
||||
);
|
||||
|
||||
await loadAll();
|
||||
reinitializeRedux(getParametersForRedux());
|
||||
await loadAllAndReinitializeRedux();
|
||||
|
||||
await window.storage.put(
|
||||
'backupMediaDownloadTotalBytes',
|
||||
|
|
@ -429,11 +346,6 @@ export class BackupImportStream extends Writable {
|
|||
}
|
||||
}
|
||||
|
||||
public cleanup(): void {
|
||||
this.conversationOpBatcher.unregister();
|
||||
this.saveMessageBatcher.unregister();
|
||||
}
|
||||
|
||||
private async processFrame(
|
||||
frame: Backups.Frame,
|
||||
options: { aboutMe?: AboutMe }
|
||||
|
|
@ -487,7 +399,7 @@ export class BackupImportStream extends Writable {
|
|||
}
|
||||
|
||||
if (convo !== this.ourConversation) {
|
||||
this.saveConversation(convo);
|
||||
await this.saveConversation(convo);
|
||||
}
|
||||
|
||||
this.recipientIdToConvo.set(recipientId, convo);
|
||||
|
|
@ -516,17 +428,95 @@ export class BackupImportStream extends Writable {
|
|||
}
|
||||
}
|
||||
|
||||
private saveConversation(attributes: ConversationAttributesType): void {
|
||||
this.conversationOpBatcher.add({ isUpdate: false, attributes });
|
||||
private async saveConversation(
|
||||
attributes: ConversationAttributesType
|
||||
): Promise<void> {
|
||||
this.conversationOpBatch.set(attributes, 'save');
|
||||
if (this.conversationOpBatch.size >= CONVERSATION_OP_BATCH_SIZE) {
|
||||
return this.flushConversations();
|
||||
}
|
||||
}
|
||||
|
||||
private updateConversation(attributes: ConversationAttributesType): void {
|
||||
this.conversationOpBatcher.add({ isUpdate: true, attributes });
|
||||
private async updateConversation(
|
||||
attributes: ConversationAttributesType
|
||||
): Promise<void> {
|
||||
if (!this.conversationOpBatch.has(attributes)) {
|
||||
this.conversationOpBatch.set(attributes, 'update');
|
||||
}
|
||||
|
||||
private saveMessage(attributes: MessageAttributesType): void {
|
||||
if (this.conversationOpBatch.size >= CONVERSATION_OP_BATCH_SIZE) {
|
||||
return this.flushConversations();
|
||||
}
|
||||
}
|
||||
|
||||
private async saveMessage(attributes: MessageAttributesType): Promise<void> {
|
||||
this.recentMessages.push(attributes);
|
||||
this.saveMessageBatcher.add(attributes);
|
||||
this.saveMessageBatch.add(attributes);
|
||||
if (this.saveMessageBatch.size >= SAVE_MESSAGE_BATCH_SIZE) {
|
||||
return this.flushMessages();
|
||||
}
|
||||
}
|
||||
|
||||
private async flushConversations(): Promise<void> {
|
||||
const saves = new Array<ConversationAttributesType>();
|
||||
const updates = new Array<ConversationAttributesType>();
|
||||
for (const [conversation, op] of this.conversationOpBatch) {
|
||||
if (op === 'save') {
|
||||
saves.push(conversation);
|
||||
} else {
|
||||
updates.push(conversation);
|
||||
}
|
||||
}
|
||||
this.conversationOpBatch.clear();
|
||||
|
||||
// Queue writes at the same time to prevent races.
|
||||
await Promise.all([
|
||||
saves.length > 0
|
||||
? DataWriter.saveConversations(saves)
|
||||
: Promise.resolve(),
|
||||
updates.length > 0
|
||||
? DataWriter.updateConversations(updates)
|
||||
: Promise.resolve(),
|
||||
]);
|
||||
}
|
||||
|
||||
private async flushMessages(): Promise<void> {
|
||||
const ourAci = this.ourConversation?.serviceId;
|
||||
strictAssert(isAciString(ourAci), 'Must have our aci for messages');
|
||||
|
||||
const batch = Array.from(this.saveMessageBatch);
|
||||
this.saveMessageBatch.clear();
|
||||
|
||||
await DataWriter.saveMessages(batch, {
|
||||
forceSave: true,
|
||||
ourAci,
|
||||
});
|
||||
|
||||
// TODO (DESKTOP-7402): consider re-saving after updating the pending state
|
||||
for (const attributes of batch) {
|
||||
const { editHistory } = attributes;
|
||||
|
||||
if (editHistory?.length) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await DataWriter.saveEditedMessages(
|
||||
attributes,
|
||||
ourAci,
|
||||
editHistory.slice(0, -1).map(({ timestamp }) => ({
|
||||
conversationId: attributes.conversationId,
|
||||
messageId: attributes.id,
|
||||
|
||||
// Main message will track this
|
||||
readStatus: ReadStatus.Read,
|
||||
sentAt: timestamp,
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await queueAttachmentDownloads(attributes, {
|
||||
source: AttachmentDownloadSource.BACKUP_IMPORT,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async saveCallHistory(
|
||||
|
|
@ -749,7 +739,7 @@ export class BackupImportStream extends Writable {
|
|||
);
|
||||
}
|
||||
|
||||
this.updateConversation(me);
|
||||
await this.updateConversation(me);
|
||||
}
|
||||
|
||||
private async fromContact(
|
||||
|
|
@ -1175,7 +1165,7 @@ export class BackupImportStream extends Writable {
|
|||
conversation.autoBubbleColor = chatStyle.autoBubbleColor;
|
||||
}
|
||||
|
||||
this.updateConversation(conversation);
|
||||
await this.updateConversation(conversation);
|
||||
|
||||
if (chat.pinnedOrder != null) {
|
||||
this.pinnedConversations.push([chat.pinnedOrder, conversation.id]);
|
||||
|
|
@ -1333,8 +1323,10 @@ export class BackupImportStream extends Writable {
|
|||
isAciString(this.ourConversation.serviceId),
|
||||
`${logId}: Our conversation must have ACI`
|
||||
);
|
||||
this.saveMessage(attributes);
|
||||
additionalMessages.forEach(additional => this.saveMessage(additional));
|
||||
await Promise.all([
|
||||
this.saveMessage(attributes),
|
||||
...additionalMessages.map(additional => this.saveMessage(additional)),
|
||||
]);
|
||||
|
||||
// TODO (DESKTOP-6964): We'll want to increment for more types here - stickers, etc.
|
||||
if (item.standardMessage) {
|
||||
|
|
@ -1344,7 +1336,7 @@ export class BackupImportStream extends Writable {
|
|||
chatConvo.messageCount = (chatConvo.messageCount ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
this.updateConversation(chatConvo);
|
||||
await this.updateConversation(chatConvo);
|
||||
}
|
||||
|
||||
private fromDirectionDetails(
|
||||
|
|
|
|||
|
|
@ -224,6 +224,7 @@ async function doContactSync({
|
|||
|
||||
await window.storage.put('synced_at', Date.now());
|
||||
window.Whisper.events.trigger('contactSync:complete');
|
||||
window.SignalCI?.handleEvent('contactSync', isFullSync);
|
||||
|
||||
log.info(`${logId}: done`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1585,11 +1585,13 @@ function saveConversation(db: WritableDB, data: ConversationType): void {
|
|||
profileLastFetchedAt,
|
||||
type,
|
||||
serviceId,
|
||||
expireTimerVersion,
|
||||
} = data;
|
||||
|
||||
const membersList = getConversationMembersList(data);
|
||||
|
||||
db.prepare<Query>(
|
||||
prepare(
|
||||
db,
|
||||
`
|
||||
INSERT INTO conversations (
|
||||
id,
|
||||
|
|
@ -1606,7 +1608,8 @@ function saveConversation(db: WritableDB, data: ConversationType): void {
|
|||
profileName,
|
||||
profileFamilyName,
|
||||
profileFullName,
|
||||
profileLastFetchedAt
|
||||
profileLastFetchedAt,
|
||||
expireTimerVersion
|
||||
) values (
|
||||
$id,
|
||||
$json,
|
||||
|
|
@ -1622,7 +1625,8 @@ function saveConversation(db: WritableDB, data: ConversationType): void {
|
|||
$profileName,
|
||||
$profileFamilyName,
|
||||
$profileFullName,
|
||||
$profileLastFetchedAt
|
||||
$profileLastFetchedAt,
|
||||
$expireTimerVersion
|
||||
);
|
||||
`
|
||||
).run({
|
||||
|
|
@ -1643,6 +1647,7 @@ function saveConversation(db: WritableDB, data: ConversationType): void {
|
|||
profileFamilyName: profileFamilyName || null,
|
||||
profileFullName: combineNames(profileName, profileFamilyName) || null,
|
||||
profileLastFetchedAt: profileLastFetchedAt || null,
|
||||
expireTimerVersion,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -1673,7 +1678,8 @@ function updateConversation(db: WritableDB, data: ConversationType): void {
|
|||
|
||||
const membersList = getConversationMembersList(data);
|
||||
|
||||
db.prepare(
|
||||
prepare(
|
||||
db,
|
||||
`
|
||||
UPDATE conversations SET
|
||||
json = $json,
|
||||
|
|
|
|||
|
|
@ -322,7 +322,6 @@ function startInstaller(): ThunkAction<
|
|||
dispatch(
|
||||
finishInstall({
|
||||
deviceName: SignalCI.deviceName,
|
||||
backupFile: SignalCI.backupData,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
|
|
|||
219
ts/test-both/helpers/generateBackup.ts
Normal file
219
ts/test-both/helpers/generateBackup.ts
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
// Copyright 2024 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import { Readable } from 'node:stream';
|
||||
import { createGzip } from 'node:zlib';
|
||||
import { createCipheriv, randomBytes } from 'node:crypto';
|
||||
import { Buffer } from 'node:buffer';
|
||||
import Long from 'long';
|
||||
|
||||
import type { AciString } from '../../types/ServiceId';
|
||||
import { generateAci } from '../../types/ServiceId';
|
||||
import { CipherType } from '../../types/Crypto';
|
||||
import { appendPaddingStream } from '../../util/logPadding';
|
||||
import { prependStream } from '../../util/prependStream';
|
||||
import { appendMacStream } from '../../util/appendMacStream';
|
||||
import { toAciObject } from '../../util/ServiceId';
|
||||
import {
|
||||
deriveBackupKey,
|
||||
deriveBackupId,
|
||||
deriveBackupKeyMaterial,
|
||||
} from '../../Crypto';
|
||||
import { BACKUP_VERSION } from '../../services/backups/constants';
|
||||
import { Backups } from '../../protobuf';
|
||||
|
||||
export type BackupGeneratorConfigType = Readonly<{
|
||||
aci: AciString;
|
||||
profileKey: Buffer;
|
||||
masterKey: Buffer;
|
||||
conversations: number;
|
||||
messages: number;
|
||||
}>;
|
||||
|
||||
const IV_LENGTH = 16;
|
||||
|
||||
export type GenerateBackupResultType = Readonly<{
|
||||
backupId: Buffer;
|
||||
stream: Readable;
|
||||
}>;
|
||||
|
||||
export function generateBackup(
|
||||
options: BackupGeneratorConfigType
|
||||
): GenerateBackupResultType {
|
||||
const { aci, masterKey } = options;
|
||||
const backupKey = deriveBackupKey(masterKey);
|
||||
const aciBytes = toAciObject(aci).getServiceIdBinary();
|
||||
const backupId = Buffer.from(deriveBackupId(backupKey, aciBytes));
|
||||
const { aesKey, macKey } = deriveBackupKeyMaterial(backupKey, backupId);
|
||||
|
||||
const iv = randomBytes(IV_LENGTH);
|
||||
|
||||
const stream = Readable.from(createRecords(options))
|
||||
.pipe(createGzip())
|
||||
.pipe(appendPaddingStream())
|
||||
.pipe(createCipheriv(CipherType.AES256CBC, aesKey, iv))
|
||||
.pipe(prependStream(iv))
|
||||
.pipe(appendMacStream(macKey));
|
||||
|
||||
return { backupId, stream };
|
||||
}
|
||||
|
||||
function frame(data: Backups.IFrame): Buffer {
|
||||
return Buffer.from(Backups.Frame.encodeDelimited(data).finish());
|
||||
}
|
||||
|
||||
let now = Date.now();
|
||||
function getTimestamp(): Long {
|
||||
now += 1;
|
||||
return Long.fromNumber(now);
|
||||
}
|
||||
|
||||
function* createRecords({
|
||||
profileKey,
|
||||
conversations,
|
||||
messages,
|
||||
}: BackupGeneratorConfigType): Iterable<Buffer> {
|
||||
yield Buffer.from(
|
||||
Backups.BackupInfo.encodeDelimited({
|
||||
version: Long.fromNumber(BACKUP_VERSION),
|
||||
backupTimeMs: getTimestamp(),
|
||||
}).finish()
|
||||
);
|
||||
|
||||
// Account data
|
||||
yield frame({
|
||||
account: {
|
||||
profileKey,
|
||||
givenName: 'Backup',
|
||||
familyName: 'Benchmark',
|
||||
accountSettings: {
|
||||
readReceipts: false,
|
||||
sealedSenderIndicators: false,
|
||||
typingIndicators: false,
|
||||
linkPreviews: false,
|
||||
notDiscoverableByPhoneNumber: false,
|
||||
preferContactAvatars: false,
|
||||
universalExpireTimerSeconds: 0,
|
||||
preferredReactionEmoji: [],
|
||||
displayBadgesOnProfile: true,
|
||||
keepMutedChatsArchived: false,
|
||||
hasSetMyStoriesPrivacy: true,
|
||||
hasViewedOnboardingStory: true,
|
||||
storiesDisabled: false,
|
||||
hasSeenGroupStoryEducationSheet: true,
|
||||
hasCompletedUsernameOnboarding: true,
|
||||
phoneNumberSharingMode:
|
||||
Backups.AccountData.PhoneNumberSharingMode.EVERYBODY,
|
||||
defaultChatStyle: {
|
||||
autoBubbleColor: {},
|
||||
dimWallpaperInDarkMode: false,
|
||||
},
|
||||
customChatColors: [],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const selfId = Long.fromNumber(0);
|
||||
|
||||
yield frame({
|
||||
recipient: {
|
||||
id: selfId,
|
||||
self: {},
|
||||
},
|
||||
});
|
||||
|
||||
const chats = new Array<{
|
||||
id: Long;
|
||||
aci: Buffer;
|
||||
}>();
|
||||
|
||||
for (let i = 1; i <= conversations; i += 1) {
|
||||
const id = Long.fromNumber(i);
|
||||
const chatAci = toAciObject(generateAci()).getRawUuidBytes();
|
||||
|
||||
chats.push({
|
||||
id,
|
||||
aci: chatAci,
|
||||
});
|
||||
|
||||
yield frame({
|
||||
recipient: {
|
||||
id,
|
||||
contact: {
|
||||
aci: chatAci,
|
||||
blocked: false,
|
||||
visibility: Backups.Contact.Visibility.VISIBLE,
|
||||
registered: {},
|
||||
profileKey: randomBytes(32),
|
||||
profileSharing: true,
|
||||
profileGivenName: `Contact ${i}`,
|
||||
profileFamilyName: 'Generated',
|
||||
hideStory: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
yield frame({
|
||||
chat: {
|
||||
id,
|
||||
recipientId: id,
|
||||
archived: false,
|
||||
pinnedOrder: 0,
|
||||
expirationTimerMs: Long.fromNumber(0),
|
||||
muteUntilMs: Long.fromNumber(0),
|
||||
markedUnread: false,
|
||||
dontNotifyForMentionsIfMuted: false,
|
||||
style: {
|
||||
autoBubbleColor: {},
|
||||
dimWallpaperInDarkMode: false,
|
||||
},
|
||||
expireTimerVersion: 1,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
for (let i = 0; i < messages; i += 1) {
|
||||
const chat = chats[i % chats.length];
|
||||
|
||||
const isIncoming = i % 2 === 0;
|
||||
|
||||
const dateSent = getTimestamp();
|
||||
|
||||
yield frame({
|
||||
chatItem: {
|
||||
chatId: chat.id,
|
||||
authorId: isIncoming ? chat.id : selfId,
|
||||
dateSent,
|
||||
revisions: [],
|
||||
sms: false,
|
||||
|
||||
...(isIncoming
|
||||
? {
|
||||
incoming: {
|
||||
dateReceived: getTimestamp(),
|
||||
dateServerSent: getTimestamp(),
|
||||
read: true,
|
||||
sealedSender: true,
|
||||
},
|
||||
}
|
||||
: {
|
||||
outgoing: {
|
||||
sendStatus: [
|
||||
{
|
||||
recipientId: chat.id,
|
||||
timestamp: dateSent,
|
||||
sent: { sealedSender: true },
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
|
||||
standardMessage: {
|
||||
text: {
|
||||
body: `Message ${i}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -31,7 +31,7 @@ import { isVoiceMessage, type AttachmentType } from '../../types/Attachment';
|
|||
import { strictAssert } from '../../util/assert';
|
||||
import { SignalService } from '../../protobuf';
|
||||
import { getRandomBytes } from '../../Crypto';
|
||||
import { loadAll } from '../../services/allLoaders';
|
||||
import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
|
||||
|
||||
const CONTACT_A = generateAci();
|
||||
|
||||
|
|
@ -65,7 +65,7 @@ describe('backup/attachments', () => {
|
|||
{ systemGivenName: 'CONTACT_A', active_at: 1 }
|
||||
);
|
||||
|
||||
await loadAll();
|
||||
await loadAllAndReinitializeRedux();
|
||||
|
||||
sandbox = sinon.createSandbox();
|
||||
const getAbsoluteAttachmentPath = sandbox.stub(
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import {
|
|||
} from './helpers';
|
||||
import { ReadStatus } from '../../messages/MessageReadStatus';
|
||||
import { SeenStatus } from '../../MessageSeenStatus';
|
||||
import { loadAll } from '../../services/allLoaders';
|
||||
import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
|
||||
|
||||
// Note: this should be kept up to date with GroupV2Change.stories.tsx, to
|
||||
// maintain the comprehensive set of GroupV2 notifications we need to handle
|
||||
|
|
@ -127,7 +127,7 @@ describe('backup/groupv2/notifications', () => {
|
|||
active_at: 1,
|
||||
});
|
||||
|
||||
await loadAll();
|
||||
await loadAllAndReinitializeRedux();
|
||||
});
|
||||
afterEach(async () => {
|
||||
await DataWriter.removeAll();
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import {
|
|||
symmetricRoundtripHarness,
|
||||
OUR_ACI,
|
||||
} from './helpers';
|
||||
import { loadAll } from '../../services/allLoaders';
|
||||
import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
|
||||
|
||||
const CONTACT_A = generateAci();
|
||||
const CONTACT_B = generateAci();
|
||||
|
|
@ -68,7 +68,7 @@ describe('backup/bubble messages', () => {
|
|||
}
|
||||
);
|
||||
|
||||
await loadAll();
|
||||
await loadAllAndReinitializeRedux();
|
||||
});
|
||||
|
||||
it('roundtrips incoming edited message', async () => {
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import { fromAdminKeyBytes } from '../../util/callLinks';
|
|||
import { ReadStatus } from '../../messages/MessageReadStatus';
|
||||
import { SeenStatus } from '../../MessageSeenStatus';
|
||||
import { deriveGroupID, deriveGroupSecretParams } from '../../util/zkgroup';
|
||||
import { loadAll } from '../../services/allLoaders';
|
||||
import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
|
||||
|
||||
const CONTACT_A = generateAci();
|
||||
const GROUP_MASTER_KEY = getRandomBytes(32);
|
||||
|
|
@ -82,7 +82,7 @@ describe('backup/calling', () => {
|
|||
|
||||
await DataWriter.insertCallLink(callLink);
|
||||
|
||||
await loadAll();
|
||||
await loadAllAndReinitializeRedux();
|
||||
});
|
||||
after(async () => {
|
||||
await DataWriter.removeAll();
|
||||
|
|
@ -105,7 +105,7 @@ describe('backup/calling', () => {
|
|||
endedTimestamp: null,
|
||||
};
|
||||
await DataWriter.saveCallHistory(callHistory);
|
||||
await loadAll();
|
||||
await loadAllAndReinitializeRedux();
|
||||
|
||||
const messageUnseen: MessageAttributesType = {
|
||||
id: generateGuid(),
|
||||
|
|
@ -154,7 +154,7 @@ describe('backup/calling', () => {
|
|||
endedTimestamp: null,
|
||||
};
|
||||
await DataWriter.saveCallHistory(callHistory);
|
||||
await loadAll();
|
||||
await loadAllAndReinitializeRedux();
|
||||
|
||||
const messageUnseen: MessageAttributesType = {
|
||||
id: generateGuid(),
|
||||
|
|
@ -245,7 +245,7 @@ describe('backup/calling', () => {
|
|||
endedTimestamp: null,
|
||||
};
|
||||
await DataWriter.saveCallHistory(callHistory);
|
||||
await loadAll();
|
||||
await loadAllAndReinitializeRedux();
|
||||
|
||||
await symmetricRoundtripHarness([]);
|
||||
|
||||
|
|
@ -271,7 +271,7 @@ describe('backup/calling', () => {
|
|||
endedTimestamp: null,
|
||||
};
|
||||
await DataWriter.saveCallHistory(callHistory);
|
||||
await loadAll();
|
||||
await loadAllAndReinitializeRedux();
|
||||
|
||||
await symmetricRoundtripHarness([]);
|
||||
|
||||
|
|
|
|||
|
|
@ -14,8 +14,10 @@ import {
|
|||
import { assert } from 'chai';
|
||||
|
||||
import { clearData } from './helpers';
|
||||
import { loadAll } from '../../services/allLoaders';
|
||||
import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
|
||||
import { backupsService, BackupType } from '../../services/backups';
|
||||
import { initialize as initializeExpiringMessageService } from '../../services/expiringMessagesDeletion';
|
||||
import { singleProtoJobQueue } from '../../jobs/singleProtoJobQueue';
|
||||
import { DataWriter } from '../../sql/Client';
|
||||
|
||||
const { BACKUP_INTEGRATION_DIR } = process.env;
|
||||
|
|
@ -39,9 +41,13 @@ class MemoryStream extends InputStream {
|
|||
}
|
||||
|
||||
describe('backup/integration', () => {
|
||||
before(async () => {
|
||||
await initializeExpiringMessageService(singleProtoJobQueue);
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await clearData();
|
||||
await loadAll();
|
||||
await loadAllAndReinitializeRedux();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import {
|
|||
symmetricRoundtripHarness,
|
||||
OUR_ACI,
|
||||
} from './helpers';
|
||||
import { loadAll } from '../../services/allLoaders';
|
||||
import { loadAllAndReinitializeRedux } from '../../services/allLoaders';
|
||||
|
||||
const CONTACT_A = generateAci();
|
||||
const GROUP_ID = Bytes.toBase64(getRandomBytes(32));
|
||||
|
|
@ -57,7 +57,7 @@ describe('backup/non-bubble messages', () => {
|
|||
}
|
||||
);
|
||||
|
||||
await loadAll();
|
||||
await loadAllAndReinitializeRedux();
|
||||
});
|
||||
|
||||
it('roundtrips END_SESSION simple update', async () => {
|
||||
|
|
|
|||
|
|
@ -188,8 +188,7 @@ describe('backups', function (this: Mocha.Suite) {
|
|||
);
|
||||
}
|
||||
|
||||
const backupPath = bootstrap.getBackupPath('backup.bin');
|
||||
await app.exportBackupToDisk(backupPath);
|
||||
await app.uploadBackup();
|
||||
|
||||
const comparator = await bootstrap.createScreenshotComparator(
|
||||
app,
|
||||
|
|
@ -247,9 +246,12 @@ describe('backups', function (this: Mocha.Suite) {
|
|||
|
||||
// Restart
|
||||
await bootstrap.eraseStorage();
|
||||
app = await bootstrap.link({
|
||||
ciBackupPath: backupPath,
|
||||
});
|
||||
app = await bootstrap.link();
|
||||
await app.waitForBackupImportComplete();
|
||||
|
||||
// Make sure that contact sync happens after backup import, otherwise the
|
||||
// app won't show contacts as "system"
|
||||
await app.waitForContactSync();
|
||||
|
||||
await comparator(app);
|
||||
});
|
||||
|
|
|
|||
47
ts/test-mock/benchmarks/backup_bench.ts
Normal file
47
ts/test-mock/benchmarks/backup_bench.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
// Copyright 2024 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
/* eslint-disable no-console */
|
||||
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { createWriteStream } from 'node:fs';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { Bootstrap } from './fixtures';
|
||||
import { generateBackup } from '../../test-both/helpers/generateBackup';
|
||||
|
||||
Bootstrap.benchmark(async (bootstrap: Bootstrap): Promise<void> => {
|
||||
const { phone, cdn3Path } = bootstrap;
|
||||
|
||||
const { backupId, stream: backupStream } = generateBackup({
|
||||
aci: phone.device.aci,
|
||||
profileKey: phone.profileKey.serialize(),
|
||||
masterKey: phone.masterKey,
|
||||
conversations: 1000,
|
||||
messages: 60 * 1000,
|
||||
});
|
||||
const backupFolder = join(
|
||||
cdn3Path,
|
||||
'backups',
|
||||
backupId.toString('base64url')
|
||||
);
|
||||
await mkdir(backupFolder, { recursive: true });
|
||||
const fileStream = createWriteStream(join(backupFolder, 'backup'));
|
||||
await pipeline(backupStream, fileStream);
|
||||
|
||||
const importStart = Date.now();
|
||||
|
||||
const app = await bootstrap.link();
|
||||
await app.waitForBackupImportComplete();
|
||||
|
||||
const importEnd = Date.now();
|
||||
|
||||
const exportStart = Date.now();
|
||||
await app.uploadBackup();
|
||||
const exportEnd = Date.now();
|
||||
|
||||
console.log('run=%d info=%j', 0, {
|
||||
importDuration: importEnd - importStart,
|
||||
exportDuration: exportEnd - exportStart,
|
||||
});
|
||||
});
|
||||
|
|
@ -150,6 +150,7 @@ function sanitizePathComponent(component: string): string {
|
|||
//
|
||||
export class Bootstrap {
|
||||
public readonly server: Server;
|
||||
public readonly cdn3Path: string;
|
||||
|
||||
private readonly options: BootstrapInternalOptions;
|
||||
private privContacts?: ReadonlyArray<PrimaryDevice>;
|
||||
|
|
@ -158,8 +159,6 @@ export class Bootstrap {
|
|||
private privPhone?: PrimaryDevice;
|
||||
private privDesktop?: Device;
|
||||
private storagePath?: string;
|
||||
private backupPath?: string;
|
||||
private cdn3Path: string;
|
||||
private timestamp: number = Date.now() - durations.WEEK;
|
||||
private lastApp?: App;
|
||||
private readonly randomId = crypto.randomBytes(8).toString('hex');
|
||||
|
|
@ -238,9 +237,6 @@ export class Bootstrap {
|
|||
});
|
||||
|
||||
this.storagePath = await fs.mkdtemp(path.join(os.tmpdir(), 'mock-signal-'));
|
||||
this.backupPath = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), 'mock-signal-backup-')
|
||||
);
|
||||
|
||||
debug('setting storage path=%j', this.storagePath);
|
||||
}
|
||||
|
|
@ -270,15 +266,6 @@ export class Bootstrap {
|
|||
return path.join(this.storagePath, 'ephemeral.json');
|
||||
}
|
||||
|
||||
public getBackupPath(fileName: string): string {
|
||||
assert(
|
||||
this.backupPath !== undefined,
|
||||
'Bootstrap has to be initialized first, see: bootstrap.init()'
|
||||
);
|
||||
|
||||
return path.join(this.backupPath, fileName);
|
||||
}
|
||||
|
||||
public eraseStorage(): Promise<void> {
|
||||
return this.resetAppStorage();
|
||||
}
|
||||
|
|
@ -289,7 +276,6 @@ export class Bootstrap {
|
|||
'Bootstrap has to be initialized first, see: bootstrap.init()'
|
||||
);
|
||||
|
||||
// Note that backupPath must remain unchanged!
|
||||
await fs.rm(this.storagePath, { recursive: true });
|
||||
this.storagePath = await fs.mkdtemp(path.join(os.tmpdir(), 'mock-signal-'));
|
||||
}
|
||||
|
|
@ -299,7 +285,7 @@ export class Bootstrap {
|
|||
|
||||
await Promise.race([
|
||||
Promise.all([
|
||||
...[this.storagePath, this.backupPath, this.cdn3Path].map(tmpPath =>
|
||||
...[this.storagePath, this.cdn3Path].map(tmpPath =>
|
||||
tmpPath ? fs.rm(tmpPath, { recursive: true }) : Promise.resolve()
|
||||
),
|
||||
this.server.close(),
|
||||
|
|
@ -354,11 +340,6 @@ export class Bootstrap {
|
|||
}
|
||||
}
|
||||
|
||||
if (extraConfig?.ciBackupPath) {
|
||||
debug('waiting for backup import to complete');
|
||||
await app.waitForBackupImportComplete();
|
||||
}
|
||||
|
||||
await this.phone.waitForSync(this.desktop);
|
||||
this.phone.resetSyncState(this.desktop);
|
||||
|
||||
|
|
@ -384,7 +365,7 @@ export class Bootstrap {
|
|||
|
||||
debug('starting the app');
|
||||
|
||||
const { port } = this.server.address();
|
||||
const { port, family } = this.server.address();
|
||||
|
||||
let startAttempts = 0;
|
||||
const MAX_ATTEMPTS = 4;
|
||||
|
|
@ -398,7 +379,7 @@ export class Bootstrap {
|
|||
}
|
||||
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
const config = await this.generateConfig(port, extraConfig);
|
||||
const config = await this.generateConfig(port, family, extraConfig);
|
||||
|
||||
const startedApp = new App({
|
||||
main: ELECTRON,
|
||||
|
|
@ -661,9 +642,12 @@ export class Bootstrap {
|
|||
|
||||
private async generateConfig(
|
||||
port: number,
|
||||
family: string,
|
||||
extraConfig?: Partial<RendererConfigType>
|
||||
): Promise<string> {
|
||||
const url = `https://127.0.0.1:${port}`;
|
||||
const host = family === 'IPv6' ? '[::1]' : '127.0.0.1';
|
||||
|
||||
const url = `https://${host}:${port}`;
|
||||
return JSON.stringify({
|
||||
...(await loadCertificates()),
|
||||
|
||||
|
|
|
|||
|
|
@ -105,6 +105,10 @@ export class App extends EventEmitter {
|
|||
return this.waitForEvent('app-loaded');
|
||||
}
|
||||
|
||||
public async waitForContactSync(): Promise<void> {
|
||||
return this.waitForEvent('contactSync');
|
||||
}
|
||||
|
||||
public async waitForBackupImportComplete(): Promise<void> {
|
||||
return this.waitForEvent('backupImportComplete');
|
||||
}
|
||||
|
|
@ -186,18 +190,9 @@ export class App extends EventEmitter {
|
|||
return window.evaluate(`window.SignalCI.getMessagesBySentAt(${timestamp})`);
|
||||
}
|
||||
|
||||
public async exportBackupToDisk(path: string): Promise<Uint8Array> {
|
||||
public async uploadBackup(): Promise<void> {
|
||||
const window = await this.getWindow();
|
||||
return window.evaluate(
|
||||
`window.SignalCI.exportBackupToDisk(${JSON.stringify(path)})`
|
||||
);
|
||||
}
|
||||
|
||||
public async exportPlaintextBackupToDisk(path: string): Promise<Uint8Array> {
|
||||
const window = await this.getWindow();
|
||||
return window.evaluate(
|
||||
`window.SignalCI.exportPlaintextBackupToDisk(${JSON.stringify(path)})`
|
||||
);
|
||||
await window.evaluate('window.SignalCI.uploadBackup()');
|
||||
}
|
||||
|
||||
public async unlink(): Promise<void> {
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import * as RemoteConfig from '../RemoteConfig';
|
||||
import { isTestOrMockEnvironment } from '../environment';
|
||||
import { isStagingServer } from './isStagingServer';
|
||||
|
||||
export function isBackupEnabled(): boolean {
|
||||
if (isStagingServer()) {
|
||||
if (isStagingServer() || isTestOrMockEnvironment()) {
|
||||
return true;
|
||||
}
|
||||
return Boolean(RemoteConfig.isEnabled('desktop.backup.credentialFetch'));
|
||||
|
|
|
|||
|
|
@ -5,8 +5,6 @@
|
|||
/* eslint-disable no-console */
|
||||
/* eslint-disable global-require */
|
||||
|
||||
import fs from 'fs';
|
||||
|
||||
const { config } = window.SignalContext;
|
||||
|
||||
if (config.environment === 'test') {
|
||||
|
|
@ -16,14 +14,10 @@ if (config.environment === 'test') {
|
|||
|
||||
if (config.ciMode) {
|
||||
console.log(
|
||||
`Importing CI infrastructure; enabled in config, mode: ${config.ciMode}, ` +
|
||||
`backupPath: ${config.ciBackupPath}`
|
||||
`Importing CI infrastructure; enabled in config, mode: ${config.ciMode}`
|
||||
);
|
||||
const { getCI } = require('../../CI');
|
||||
window.SignalCI = getCI({
|
||||
deviceName: window.getTitle(),
|
||||
backupData: config.ciBackupPath
|
||||
? fs.readFileSync(config.ciBackupPath)
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue