Add backup comparator test harness

This commit is contained in:
Fedor Indutny 2024-09-03 10:18:15 -07:00 committed by GitHub
parent 98eb6dec68
commit 84f1d98020
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 339 additions and 124 deletions

View file

@ -2721,6 +2721,7 @@ ipc.on('get-config', async event => {
dnsFallback: await getDNSFallback(),
disableIPv6: DISABLE_IPV6,
ciBackupPath: config.get<string | null>('ciBackupPath') || undefined,
ciIsPlaintextBackup: config.get<boolean>('ciIsPlaintextBackup'),
nodeVersion: process.versions.node,
hostname: os.hostname(),
osRelease: os.release(),

View file

@ -18,6 +18,7 @@
"updatesEnabled": false,
"ciMode": false,
"ciBackupPath": null,
"ciIsPlaintextBackup": false,
"forcePreloadBundle": false,
"openDevTools": false,
"buildCreation": 0,

View file

@ -8,7 +8,7 @@ import type { MessageAttributesType } from './model-types.d';
import * as log from './logging/log';
import { explodePromise } from './util/explodePromise';
import { AccessType, ipcInvoke } from './sql/channels';
import { backupsService } from './services/backups';
import { backupsService, BackupType } from './services/backups';
import { SECOND } from './util/durations';
import { isSignalRoute } from './util/signalRoutes';
import { strictAssert } from './util/assert';
@ -18,6 +18,7 @@ type ResolveType = (data: unknown) => void;
export type CIType = {
deviceName: string;
backupData?: Uint8Array;
isPlaintextBackup?: boolean;
getConversationId: (address: string | null) => string | null;
getMessagesBySentAt(
sentAt: number
@ -34,15 +35,21 @@ export type CIType = {
) => unknown;
openSignalRoute(url: string): Promise<void>;
exportBackupToDisk(path: string): Promise<void>;
exportPlaintextBackupToDisk(path: string): Promise<void>;
unlink: () => void;
};
export type GetCIOptionsType = Readonly<{
deviceName: string;
backupData?: Uint8Array;
isPlaintextBackup?: boolean;
}>;
export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
export function getCI({
deviceName,
backupData,
isPlaintextBackup,
}: GetCIOptionsType): CIType {
const eventListeners = new Map<string, Array<ResolveType>>();
const completedEvents = new Map<string, Array<unknown>>();
@ -164,6 +171,14 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
await backupsService.exportToDisk(path);
}
async function exportPlaintextBackupToDisk(path: string) {
await backupsService.exportToDisk(
path,
undefined,
BackupType.TestOnlyPlaintext
);
}
function unlink() {
window.Whisper.events.trigger('unlinkAndDisconnect');
}
@ -171,6 +186,7 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
return {
deviceName,
backupData,
isPlaintextBackup,
getConversationId,
getMessagesBySentAt,
handleEvent,
@ -179,6 +195,7 @@ export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
waitForEvent,
openSignalRoute,
exportBackupToDisk,
exportPlaintextBackupToDisk,
unlink,
};
}

View file

@ -278,6 +278,14 @@ export class BackupExportStream extends Readable {
stats.conversations += 1;
}
this.pushFrame({
recipient: {
id: Long.fromNumber(this.getNextRecipientId()),
releaseNotes: {},
},
});
await this.flush();
const distributionLists =
await DataReader.getAllStoryDistributionsWithMembers();
@ -2401,7 +2409,11 @@ export class BackupExportStream extends Readable {
const id = Long.fromNumber(result.length);
this.customColorIdByUuid.set(uuid, id);
const start = hslToRGBInt(color.start.hue, color.start.saturation);
const start = hslToRGBInt(
color.start.hue,
color.start.saturation,
color.start.luminance
);
if (color.end == null) {
result.push({
@ -2409,7 +2421,11 @@ export class BackupExportStream extends Readable {
solid: start,
});
} else {
const end = hslToRGBInt(color.end.hue, color.end.saturation);
const end = hslToRGBInt(
color.end.hue,
color.end.saturation,
color.end.luminance
);
result.push({
id,
@ -2562,8 +2578,8 @@ function checkServiceIdEquivalence(
return leftConvo && rightConvo && leftConvo === rightConvo;
}
function hslToRGBInt(hue: number, saturation: number): number {
const { r, g, b } = hslToRGB(hue, saturation, 1);
function hslToRGBInt(hue: number, saturation: number, luminance = 1): number {
const { r, g, b } = hslToRGB(hue, saturation, luminance);
// eslint-disable-next-line no-bitwise
return ((0xff << 24) | (r << 16) | (g << 8) | b) >>> 0;
}

View file

@ -14,7 +14,7 @@ import { DataWriter } from '../../sql/Client';
import type { StoryDistributionWithMembersType } from '../../sql/Interface';
import * as log from '../../logging/log';
import { GiftBadgeStates } from '../../components/conversation/Message';
import { StorySendMode } from '../../types/Stories';
import { StorySendMode, MY_STORY_ID } from '../../types/Stories';
import type { ServiceIdString, AciString } from '../../types/ServiceId';
import {
fromAciObject,
@ -942,7 +942,7 @@ export class BackupImportStream extends Writable {
'Missing distribution list id'
);
const id = bytesToUuid(listItem.distributionId);
const id = bytesToUuid(listItem.distributionId) || MY_STORY_ID;
strictAssert(isStoryDistributionId(id), 'Invalid distribution list id');
const commonFields = {
@ -2987,17 +2987,20 @@ export class BackupImportStream extends Writable {
}
}
function rgbIntToHSL(intValue: number): { hue: number; saturation: number } {
const { h: hue, s: saturation } = rgbToHSL(
// eslint-disable-next-line no-bitwise
(intValue >>> 16) & 0xff,
// eslint-disable-next-line no-bitwise
(intValue >>> 8) & 0xff,
// eslint-disable-next-line no-bitwise
intValue & 0xff
);
function rgbIntToHSL(intValue: number): {
hue: number;
saturation: number;
luminance: number;
} {
// eslint-disable-next-line no-bitwise
const r = (intValue >>> 16) & 0xff;
// eslint-disable-next-line no-bitwise
const g = (intValue >>> 8) & 0xff;
// eslint-disable-next-line no-bitwise
const b = intValue & 0xff;
const { h: hue, s: saturation, l: luminance } = rgbToHSL(r, g, b);
return { hue, saturation };
return { hue, saturation, luminance };
}
function fromGroupCallStateProto(

View file

@ -24,25 +24,32 @@ import { prependStream } from '../../util/prependStream';
import { appendMacStream } from '../../util/appendMacStream';
import { getIvAndDecipher } from '../../util/getIvAndDecipher';
import { getMacAndUpdateHmac } from '../../util/getMacAndUpdateHmac';
import { missingCaseError } from '../../util/missingCaseError';
import { HOUR } from '../../util/durations';
import { CipherType, HashType } from '../../types/Crypto';
import * as Errors from '../../types/errors';
import { HTTPError } from '../../textsecure/Errors';
import { constantTimeEqual } from '../../Crypto';
import { measureSize } from '../../AttachmentCrypto';
import { reinitializeRedux } from '../../state/reinitializeRedux';
import { isTestOrMockEnvironment } from '../../environment';
import { BackupExportStream } from './export';
import { BackupImportStream } from './import';
import { getKeyMaterial } from './crypto';
import { BackupCredentials } from './credentials';
import { BackupAPI, type DownloadOptionsType } from './api';
import { validateBackup } from './validator';
import { reinitializeRedux } from '../../state/reinitializeRedux';
import { getParametersForRedux, loadAll } from '../allLoaders';
const IV_LENGTH = 16;
const BACKUP_REFRESH_INTERVAL = 24 * HOUR;
export enum BackupType {
Ciphertext = 'Ciphertext',
TestOnlyPlaintext = 'TestOnlyPlaintext',
}
export class BackupsService {
private isStarted = false;
private isRunning = false;
@ -108,11 +115,18 @@ export class BackupsService {
// Test harness
public async exportToDisk(
path: string,
backupLevel: BackupLevel = BackupLevel.Messages
backupLevel: BackupLevel = BackupLevel.Messages,
backupType = BackupType.Ciphertext
): Promise<number> {
const size = await this.exportBackup(createWriteStream(path), backupLevel);
const size = await this.exportBackup(
createWriteStream(path),
backupLevel,
backupType
);
await validateBackup(path, size);
if (backupType === BackupType.Ciphertext) {
await validateBackup(path, size);
}
return size;
}
@ -184,53 +198,70 @@ export class BackupsService {
}
}
public async importBackup(createBackupStream: () => Readable): Promise<void> {
public async importBackup(
createBackupStream: () => Readable,
backupType = BackupType.Ciphertext
): Promise<void> {
strictAssert(!this.isRunning, 'BackupService is already running');
log.info('importBackup: starting...');
log.info(`importBackup: starting ${backupType}...`);
this.isRunning = true;
try {
const { aesKey, macKey } = getKeyMaterial();
if (backupType === BackupType.Ciphertext) {
const { aesKey, macKey } = getKeyMaterial();
// First pass - don't decrypt, only verify mac
let hmac = createHmac(HashType.size256, macKey);
let theirMac: Uint8Array | undefined;
// First pass - don't decrypt, only verify mac
let hmac = createHmac(HashType.size256, macKey);
let theirMac: Uint8Array | undefined;
const sink = new PassThrough();
// Discard the data in the first pass
sink.resume();
const sink = new PassThrough();
// Discard the data in the first pass
sink.resume();
await pipeline(
createBackupStream(),
getMacAndUpdateHmac(hmac, theirMacValue => {
theirMac = theirMacValue;
}),
sink
);
await pipeline(
createBackupStream(),
getMacAndUpdateHmac(hmac, theirMacValue => {
theirMac = theirMacValue;
}),
sink
);
strictAssert(theirMac != null, 'importBackup: Missing MAC');
strictAssert(
constantTimeEqual(hmac.digest(), theirMac),
'importBackup: Bad MAC'
);
strictAssert(theirMac != null, 'importBackup: Missing MAC');
strictAssert(
constantTimeEqual(hmac.digest(), theirMac),
'importBackup: Bad MAC'
);
// Second pass - decrypt (but still check the mac at the end)
hmac = createHmac(HashType.size256, macKey);
// Second pass - decrypt (but still check the mac at the end)
hmac = createHmac(HashType.size256, macKey);
await pipeline(
createBackupStream(),
getMacAndUpdateHmac(hmac, noop),
getIvAndDecipher(aesKey),
createGunzip(),
new DelimitedStream(),
new BackupImportStream()
);
await pipeline(
createBackupStream(),
getMacAndUpdateHmac(hmac, noop),
getIvAndDecipher(aesKey),
createGunzip(),
new DelimitedStream(),
new BackupImportStream()
);
strictAssert(
constantTimeEqual(hmac.digest(), theirMac),
'importBackup: Bad MAC, second pass'
);
strictAssert(
constantTimeEqual(hmac.digest(), theirMac),
'importBackup: Bad MAC, second pass'
);
} else if (backupType === BackupType.TestOnlyPlaintext) {
strictAssert(
isTestOrMockEnvironment(),
'Plaintext backups can be imported only in test harness'
);
await pipeline(
createBackupStream(),
new DelimitedStream(),
new BackupImportStream()
);
} else {
throw missingCaseError(backupType);
}
await this.resetStateAfterImport();
@ -299,7 +330,8 @@ export class BackupsService {
private async exportBackup(
sink: Writable,
backupLevel: BackupLevel = BackupLevel.Messages
backupLevel: BackupLevel = BackupLevel.Messages,
backupType = BackupType.Ciphertext
): Promise<number> {
strictAssert(!this.isRunning, 'BackupService is already running');
@ -324,18 +356,28 @@ export class BackupsService {
let totalBytes = 0;
await pipeline(
recordStream,
createGzip(),
appendPaddingStream(),
createCipheriv(CipherType.AES256CBC, aesKey, iv),
prependStream(iv),
appendMacStream(macKey),
measureSize(size => {
totalBytes = size;
}),
sink
);
if (backupType === BackupType.Ciphertext) {
await pipeline(
recordStream,
createGzip(),
appendPaddingStream(),
createCipheriv(CipherType.AES256CBC, aesKey, iv),
prependStream(iv),
appendMacStream(macKey),
measureSize(size => {
totalBytes = size;
}),
sink
);
} else if (backupType === BackupType.TestOnlyPlaintext) {
strictAssert(
isTestOrMockEnvironment(),
'Plaintext backups can be exported only in test harness'
);
await pipeline(recordStream, sink);
} else {
throw missingCaseError(backupType);
}
return totalBytes;
} finally {

View file

@ -0,0 +1,61 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { type FileHandle, open } from 'node:fs/promises';
import { Buffer } from 'node:buffer';
import { InputStream } from '@signalapp/libsignal-client/dist/io';
export class FileStream extends InputStream {
private file: FileHandle | undefined;
private position = 0;
private buffer = Buffer.alloc(16 * 1024);
private initPromise: Promise<unknown> | undefined;
constructor(private readonly filePath: string) {
super();
}
public async close(): Promise<void> {
await this.initPromise;
await this.file?.close();
}
// Only for comparator tests
public async size(): Promise<number> {
const file = await this.lazyOpen();
const { size } = await file.stat();
return size;
}
async read(amount: number): Promise<Buffer> {
const file = await this.lazyOpen();
if (this.buffer.length < amount) {
this.buffer = Buffer.alloc(amount);
}
const { bytesRead } = await file.read(
this.buffer,
0,
amount,
this.position
);
this.position += bytesRead;
return this.buffer.slice(0, bytesRead);
}
async skip(amount: number): Promise<void> {
this.position += amount;
}
private async lazyOpen(): Promise<FileHandle> {
await this.initPromise;
if (this.file) {
return this.file;
}
const filePromise = open(this.filePath);
this.initPromise = filePromise;
this.file = await filePromise;
return this.file;
}
}

View file

@ -1,55 +1,12 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { type FileHandle, open } from 'node:fs/promises';
import * as libsignal from '@signalapp/libsignal-client/dist/MessageBackup';
import { InputStream } from '@signalapp/libsignal-client/dist/io';
import { strictAssert } from '../../util/assert';
import { toAciObject } from '../../util/ServiceId';
import { isTestOrMockEnvironment } from '../../environment';
class FileStream extends InputStream {
private file: FileHandle | undefined;
private position = 0;
private buffer = Buffer.alloc(16 * 1024);
private initPromise: Promise<unknown> | undefined;
constructor(private readonly filePath: string) {
super();
}
public async close(): Promise<void> {
await this.initPromise;
await this.file?.close();
}
async read(amount: number): Promise<Buffer> {
await this.initPromise;
if (!this.file) {
const filePromise = open(this.filePath);
this.initPromise = filePromise;
this.file = await filePromise;
}
if (this.buffer.length < amount) {
this.buffer = Buffer.alloc(amount);
}
const { bytesRead } = await this.file.read(
this.buffer,
0,
amount,
this.position
);
this.position += bytesRead;
return this.buffer.slice(0, bytesRead);
}
async skip(amount: number): Promise<void> {
this.position += amount;
}
}
import { FileStream } from './util/FileStream';
export async function validateBackup(
filePath: string,

View file

@ -225,8 +225,13 @@ export const SmartInstallScreen = memo(function SmartInstallScreen() {
let deviceName: string;
let backupFileData: Uint8Array | undefined;
let isPlaintextBackup = false;
if (window.SignalCI) {
({ deviceName, backupData: backupFileData } = window.SignalCI);
({
deviceName,
backupData: backupFileData,
isPlaintextBackup = false,
} = window.SignalCI);
} else {
deviceName = await chooseDeviceNamePromiseWrapperRef.current.promise;
const backupFile =
@ -264,6 +269,7 @@ export const SmartInstallScreen = memo(function SmartInstallScreen() {
const data = provisioner.prepareLinkData({
deviceName,
backupFile: backupFileData,
isPlaintextBackup,
});
await accountManager.registerSecondDevice(data);
} catch (error) {

View file

@ -0,0 +1,86 @@
// Copyright 2023 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { assert } from 'chai';
import { join } from 'node:path';
import createDebug from 'debug';
import fastGlob from 'fast-glob';
import {
ComparableBackup,
Purpose,
} from '@signalapp/libsignal-client/dist/MessageBackup';
import * as durations from '../../util/durations';
import { FileStream } from '../../services/backups/util/FileStream';
import type { App } from '../playwright';
import { Bootstrap } from '../bootstrap';
export const debug = createDebug('mock:test:backups');
const TEST_FOLDER = process.env.BACKUP_TEST_FOLDER;
describe('backups/integration', async function (this: Mocha.Suite) {
this.timeout(100 * durations.MINUTE);
if (!TEST_FOLDER) {
return;
}
let bootstrap: Bootstrap;
let app: App | undefined;
beforeEach(async () => {
bootstrap = new Bootstrap();
await bootstrap.init();
});
afterEach(async function (this: Mocha.Context) {
if (!bootstrap) {
return;
}
await bootstrap.maybeSaveLogs(this.currentTest, app);
await app?.close();
await bootstrap.teardown();
});
const testFiles = fastGlob.sync(join(TEST_FOLDER, '*.binproto'), {
onlyFiles: true,
});
testFiles.forEach(fullPath => {
it(`passes ${fullPath}`, async () => {
app = await bootstrap.link({
ciBackupPath: fullPath,
ciIsPlaintextBackup: true,
});
const backupPath = bootstrap.getBackupPath('backup.bin');
await app.exportPlaintextBackupToDisk(backupPath);
await app.close();
const actualStream = new FileStream(backupPath);
const expectedStream = new FileStream(fullPath);
try {
const actual = await ComparableBackup.fromUnencrypted(
Purpose.RemoteBackup,
actualStream,
BigInt(await actualStream.size())
);
const expected = await ComparableBackup.fromUnencrypted(
Purpose.RemoteBackup,
expectedStream,
BigInt(await expectedStream.size())
);
assert.strictEqual(
actual.comparableString(),
expected.comparableString()
);
} finally {
await actualStream.close();
await expectedStream.close();
}
});
});
});

View file

@ -179,6 +179,13 @@ export class App extends EventEmitter {
);
}
public async exportPlaintextBackupToDisk(path: string): Promise<Uint8Array> {
const window = await this.getWindow();
return window.evaluate(
`window.SignalCI.exportPlaintextBackupToDisk(${JSON.stringify(path)})`
);
}
public async unlink(): Promise<void> {
const window = await this.getWindow();
return window.evaluate('window.SignalCI.unlink()');

View file

@ -25,7 +25,7 @@ import createTaskWithTimeout from './TaskWithTimeout';
import * as Bytes from '../Bytes';
import * as Errors from '../types/errors';
import { senderCertificateService } from '../services/senderCertificate';
import { backupsService } from '../services/backups';
import { backupsService, BackupType } from '../services/backups';
import {
decryptDeviceName,
deriveAccessKey,
@ -123,7 +123,10 @@ type CreateAccountSharedOptionsType = Readonly<{
pniKeyPair: KeyPairType;
profileKey: Uint8Array;
masterKey: Uint8Array;
// Test-only
backupFile?: Uint8Array;
isPlaintextBackup?: boolean;
}>;
type CreatePrimaryDeviceOptionsType = Readonly<{
@ -217,6 +220,7 @@ function signedPreKeyToUploadSignedPreKey({
export type ConfirmNumberResultType = Readonly<{
deviceName: string;
backupFile: Uint8Array | undefined;
isPlaintextBackup: boolean;
}>;
export default class AccountManager extends EventTarget {
@ -919,6 +923,7 @@ export default class AccountManager extends EventTarget {
readReceipts,
userAgent,
backupFile,
isPlaintextBackup,
} = options;
const { storage } = window.textsecure;
@ -963,7 +968,9 @@ export default class AccountManager extends EventTarget {
}
if (backupFile !== undefined) {
log.warn(
'createAccount: Restoring from backup; deleting all previous data'
'createAccount: Restoring from ' +
`${isPlaintextBackup ? 'plaintext' : 'ciphertext'} backup; ` +
'deleting all previous data'
);
}
@ -1222,7 +1229,10 @@ export default class AccountManager extends EventTarget {
]);
if (backupFile !== undefined) {
await backupsService.importBackup(() => Readable.from([backupFile]));
await backupsService.importBackup(
() => Readable.from([backupFile]),
isPlaintextBackup ? BackupType.TestOnlyPlaintext : BackupType.Ciphertext
);
}
}

View file

@ -67,6 +67,7 @@ type StateType = Readonly<
export type PrepareLinkDataOptionsType = Readonly<{
deviceName: string;
backupFile?: Uint8Array;
isPlaintextBackup?: boolean;
}>;
export class Provisioner {
@ -150,6 +151,7 @@ export class Provisioner {
public prepareLinkData({
deviceName,
backupFile,
isPlaintextBackup,
}: PrepareLinkDataOptionsType): CreateLinkedDeviceOptionsType {
strictAssert(
this.state.step === Step.ReadyToLink,
@ -204,6 +206,7 @@ export class Provisioner {
profileKey,
deviceName,
backupFile,
isPlaintextBackup,
userAgent,
ourAci,
ourPni,

View file

@ -159,8 +159,8 @@ export const ContactNameColors = [
export type ContactNameColorType = (typeof ContactNameColors)[number];
export type CustomColorType = {
start: { hue: number; saturation: number };
end?: { hue: number; saturation: number };
start: { hue: number; saturation: number; luminance?: number };
end?: { hue: number; saturation: number; luminance?: number };
deg?: number;
};

View file

@ -43,6 +43,7 @@ export const rendererConfigSchema = z.object({
disableIPv6: z.boolean(),
dnsFallback: DNSFallbackSchema,
ciBackupPath: configOptionalStringSchema,
ciIsPlaintextBackup: z.boolean(),
environment: environmentSchema,
isMockTestEnvironment: z.boolean(),
homePath: configRequiredStringSchema,

View file

@ -46,14 +46,17 @@ export function getHSL(
{
hue,
saturation,
luminance,
}: {
hue: number;
saturation: number;
luminance?: number;
},
adjustedLightness = 0
): string {
return `hsl(${hue}, ${saturation}%, ${adjustLightnessValue(
calculateLightness(hue),
adjustedLightness
)}%)`;
return `hsl(${hue}, ${saturation}%, ${
luminance == null
? adjustLightnessValue(calculateLightness(hue), adjustedLightness)
: luminance * 100
}%)`;
}

View file

@ -25,5 +25,6 @@ if (config.ciMode) {
backupData: config.ciBackupPath
? fs.readFileSync(config.ciBackupPath)
: undefined,
isPlaintextBackup: config.ciIsPlaintextBackup === true,
});
}