Convert attachments to filePointers for backup export

This commit is contained in:
trevor-signal 2024-05-15 10:55:20 -04:00 committed by GitHub
parent ad94fef92d
commit 6f7545926a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 876 additions and 194 deletions

View file

@ -11,7 +11,7 @@ import {
randomBytes,
} from 'crypto';
import type { Decipher, Hash, Hmac } from 'crypto';
import { Transform } from 'stream';
import { PassThrough, Transform, type Writable } from 'stream';
import { pipeline } from 'stream/promises';
import { ensureFile } from 'fs-extra';
import * as log from './logging/log';
@ -45,7 +45,6 @@ export function _generateAttachmentIv(): Uint8Array {
}
export type EncryptedAttachmentV2 = {
path: string;
digest: Uint8Array;
plaintextHash: string;
};
@ -55,24 +54,55 @@ export type DecryptedAttachmentV2 = {
plaintextHash: string;
};
type EncryptAttachmentV2PropsType = {
keys: Readonly<Uint8Array>;
plaintextAbsolutePath: string;
dangerousTestOnlyIv?: Readonly<Uint8Array>;
dangerousTestOnlySkipPadding?: boolean;
};
export async function encryptAttachmentV2ToDisk(
args: EncryptAttachmentV2PropsType
): Promise<EncryptedAttachmentV2 & { path: string }> {
// Create random output file
const relativeTargetPath = getRelativePath(createName());
const absoluteTargetPath =
window.Signal.Migrations.getAbsoluteAttachmentPath(relativeTargetPath);
let writeFd;
let encryptResult: EncryptedAttachmentV2;
try {
await ensureFile(absoluteTargetPath);
writeFd = await open(absoluteTargetPath, 'w');
encryptResult = await encryptAttachmentV2({
...args,
sink: writeFd.createWriteStream(),
});
} catch (error) {
safeUnlinkSync(absoluteTargetPath);
throw error;
} finally {
await writeFd?.close();
}
return {
...encryptResult,
path: relativeTargetPath,
};
}
export async function encryptAttachmentV2({
keys,
plaintextAbsolutePath,
dangerousTestOnlyIv,
dangerousTestOnlySkipPadding = false,
}: {
keys: Readonly<Uint8Array>;
plaintextAbsolutePath: string;
dangerousTestOnlyIv?: Readonly<Uint8Array>;
dangerousTestOnlySkipPadding?: boolean;
sink,
}: EncryptAttachmentV2PropsType & {
sink?: Writable;
}): Promise<EncryptedAttachmentV2> {
const logId = 'encryptAttachmentV2';
// Create random output file
const relativeTargetPath = getRelativePath(createName());
const absoluteTargetPath =
window.Signal.Migrations.getAbsoluteAttachmentPath(relativeTargetPath);
const { aesKey, macKey } = splitKeys(keys);
if (dangerousTestOnlyIv && window.getEnvironment() !== Environment.Test) {
@ -92,19 +122,12 @@ export async function encryptAttachmentV2({
const digest = createHash(HashType.size256);
let readFd;
let writeFd;
try {
try {
readFd = await open(plaintextAbsolutePath, 'r');
} catch (cause) {
throw new Error(`${logId}: Read path doesn't exist`, { cause });
}
try {
await ensureFile(absoluteTargetPath);
writeFd = await open(absoluteTargetPath, 'w');
} catch (cause) {
throw new Error(`${logId}: Failed to create write path`, { cause });
}
await pipeline(
[
@ -115,7 +138,7 @@ export async function encryptAttachmentV2({
prependIv(iv),
appendMacStream(macKey),
peekAndUpdateHash(digest),
writeFd.createWriteStream(),
sink ?? new PassThrough().resume(),
].filter(isNotNil)
);
} catch (error) {
@ -123,10 +146,9 @@ export async function encryptAttachmentV2({
`${logId}: Failed to encrypt attachment`,
Errors.toLogFormat(error)
);
safeUnlinkSync(absoluteTargetPath);
throw error;
} finally {
await Promise.all([readFd?.close(), writeFd?.close()]);
await readFd?.close();
}
const ourPlaintextHash = plaintextHash.digest('hex');
@ -143,7 +165,6 @@ export async function encryptAttachmentV2({
);
return {
path: relativeTargetPath,
digest: ourDigest,
plaintextHash: ourPlaintextHash,
};

View file

@ -8,6 +8,9 @@ const bytes = globalThis.window?.SignalContext?.bytes || new Bytes();
export function fromBase64(value: string): Uint8Array {
return bytes.fromBase64(value);
}
export function fromBase64url(value: string): Uint8Array {
return bytes.fromBase64url(value);
}
export function fromHex(value: string): Uint8Array {
return bytes.fromHex(value);

View file

@ -8,6 +8,10 @@ export class Bytes {
return Buffer.from(value, 'base64');
}
public fromBase64url(value: string): Uint8Array {
return Buffer.from(value, 'base64url');
}
public fromHex(value: string): Uint8Array {
return Buffer.from(value, 'hex');
}

View file

@ -3,6 +3,7 @@
import Long from 'long';
import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client';
import type { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
import pMap from 'p-map';
import pTimeout from 'p-timeout';
import { Readable } from 'stream';
@ -82,6 +83,13 @@ import {
numberToAddressType,
numberToPhoneType,
} from '../../types/EmbeddedContact';
import {
isVoiceMessage,
type AttachmentType,
isGIF,
isDownloaded,
} from '../../types/Attachment';
import { convertAttachmentToFilePointer } from './util/filePointers';
const MAX_CONCURRENCY = 10;
@ -116,14 +124,13 @@ export class BackupExportStream extends Readable {
private nextRecipientId = 0;
private flushResolve: (() => void) | undefined;
public run(): void {
public run(backupLevel: BackupLevel): void {
drop(
(async () => {
log.info('BackupExportStream: starting...');
await Data.pauseWriteAccess();
try {
await this.unsafeRun();
await this.unsafeRun(backupLevel);
} catch (error) {
this.emit('error', error);
} finally {
@ -134,7 +141,7 @@ export class BackupExportStream extends Readable {
);
}
private async unsafeRun(): Promise<void> {
private async unsafeRun(backupLevel: BackupLevel): Promise<void> {
this.push(
Backups.BackupInfo.encodeDelimited({
version: Long.fromNumber(BACKUP_VERSION),
@ -279,7 +286,12 @@ export class BackupExportStream extends Readable {
// eslint-disable-next-line no-await-in-loop
const items = await pMap(
messages,
message => this.toChatItem(message, { aboutMe, callHistoryByCallId }),
message =>
this.toChatItem(message, {
aboutMe,
callHistoryByCallId,
backupLevel,
}),
{ concurrency: MAX_CONCURRENCY }
);
@ -308,6 +320,7 @@ export class BackupExportStream extends Readable {
}
await this.flush();
log.warn('backups: final stats', stats);
this.push(null);
@ -562,6 +575,7 @@ export class BackupExportStream extends Readable {
options: {
aboutMe: AboutMe;
callHistoryByCallId: Record<string, CallHistoryDetails>;
backupLevel: BackupLevel;
}
): Promise<Backups.IChatItem | undefined> {
const chatId = this.getRecipientId({ id: message.conversationId });
@ -629,6 +643,16 @@ export class BackupExportStream extends Readable {
// TODO (DESKTOP-6964): put incoming/outgoing fields below onto non-bubble messages
result.standardMessage = {
quote: await this.toQuote(message.quote),
attachments: message.attachments
? await Promise.all(
message.attachments.map(attachment => {
return this.processMessageAttachment({
attachment,
backupLevel: options.backupLevel,
});
})
)
: undefined,
text: {
// Note that we store full text on the message model so we have to
// trim it before serializing.
@ -1579,6 +1603,63 @@ export class BackupExportStream extends Readable {
}),
};
}
private getMessageAttachmentFlag(
attachment: AttachmentType
): Backups.MessageAttachment.Flag {
if (isVoiceMessage(attachment)) {
return Backups.MessageAttachment.Flag.VOICE_MESSAGE;
}
if (isGIF([attachment])) {
return Backups.MessageAttachment.Flag.GIF;
}
if (
attachment.flags &&
// eslint-disable-next-line no-bitwise
attachment.flags & SignalService.AttachmentPointer.Flags.BORDERLESS
) {
return Backups.MessageAttachment.Flag.BORDERLESS;
}
return Backups.MessageAttachment.Flag.NONE;
}
private async processMessageAttachment({
attachment,
backupLevel,
}: {
attachment: AttachmentType;
backupLevel: BackupLevel;
}): Promise<Backups.MessageAttachment> {
const filePointer = await this.processAttachment({
attachment,
backupLevel,
});
return new Backups.MessageAttachment({
pointer: filePointer,
flag: this.getMessageAttachmentFlag(attachment),
wasDownloaded: isDownloaded(attachment), // should always be true
});
}
private async processAttachment({
attachment,
backupLevel,
}: {
attachment: AttachmentType;
backupLevel: BackupLevel;
}): Promise<Backups.FilePointer> {
const filePointer = await convertAttachmentToFilePointer({
attachment,
backupLevel,
// TODO (DESKTOP-6983) -- Retrieve & save backup tier media list
getBackupTierInfo: () => ({
isInBackupTier: false,
}),
});
return filePointer;
}
}
function checkServiceIdEquivalence(

View file

@ -10,6 +10,7 @@ import { join } from 'path';
import { createGzip, createGunzip } from 'zlib';
import { createCipheriv, createHmac, randomBytes } from 'crypto';
import { noop } from 'lodash';
import { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
import * as log from '../../logging/log';
import * as Bytes from '../../Bytes';
@ -68,8 +69,11 @@ export class BackupsService {
const fileName = `backup-${randomBytes(32).toString('hex')}`;
const filePath = join(window.BasePaths.temp, fileName);
const backupLevel = await this.credentials.getBackupLevel();
log.info(`exportBackup: starting, backup level: ${backupLevel}...`);
try {
const fileSize = await this.exportToDisk(filePath);
const fileSize = await this.exportToDisk(filePath, backupLevel);
await this.api.upload(filePath, fileSize);
} finally {
@ -82,19 +86,24 @@ export class BackupsService {
}
// Test harness
public async exportBackupData(): Promise<Uint8Array> {
public async exportBackupData(
backupLevel: BackupLevel = BackupLevel.Messages
): Promise<Uint8Array> {
const sink = new PassThrough();
const chunks = new Array<Uint8Array>();
sink.on('data', chunk => chunks.push(chunk));
await this.exportBackup(sink);
await this.exportBackup(sink, backupLevel);
return Bytes.concatenate(chunks);
}
// Test harness
public async exportToDisk(path: string): Promise<number> {
const size = await this.exportBackup(createWriteStream(path));
public async exportToDisk(
path: string,
backupLevel: BackupLevel = BackupLevel.Messages
): Promise<number> {
const size = await this.exportBackup(createWriteStream(path), backupLevel);
await validateBackup(path, size);
@ -174,7 +183,10 @@ export class BackupsService {
}
}
private async exportBackup(sink: Writable): Promise<number> {
private async exportBackup(
sink: Writable,
backupLevel: BackupLevel = BackupLevel.Messages
): Promise<number> {
strictAssert(!this.isRunning, 'BackupService is already running');
log.info('exportBackup: starting...');
@ -184,7 +196,7 @@ export class BackupsService {
const { aesKey, macKey } = getKeyMaterial();
const recordStream = new BackupExportStream();
recordStream.run();
recordStream.run(backupLevel);
const iv = randomBytes(IV_LENGTH);

View file

@ -1,13 +1,28 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import Long from 'long';
import { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
import {
APPLICATION_OCTET_STREAM,
stringToMIMEType,
} from '../../../types/MIME';
import type { AttachmentType } from '../../../types/Attachment';
import type { Backups } from '../../../protobuf';
import {
type AttachmentType,
isDownloadableFromTransitTier,
isDownloadableFromBackupTier,
isDownloadedToLocalFile,
type AttachmentDownloadableFromTransitTier,
type AttachmentDownloadableFromBackupTier,
type DownloadedAttachment,
type AttachmentReadyForBackup,
} from '../../../types/Attachment';
import { Backups } from '../../../protobuf';
import * as Bytes from '../../../Bytes';
import { getTimestampFromLong } from '../../../util/timestampLongUtils';
import { getRandomBytes } from '../../../Crypto';
import { encryptAttachmentV2 } from '../../../AttachmentCrypto';
import { strictAssert } from '../../../util/assert';
export function convertFilePointerToAttachment(
filePointer: Backups.FilePointer
@ -94,3 +109,174 @@ export function convertFilePointerToAttachment(
throw new Error('convertFilePointerToAttachment: mising locator');
}
/**
* Some attachments saved on desktop do not include the key used to encrypt the file
* originally. This means that we need to encrypt the file in-memory now (at
* export-creation time) to calculate the digest which will be saved in the backup proto
* along with the new keys.
*/
async function fixupAttachmentForBackup(
attachment: DownloadedAttachment
): Promise<AttachmentReadyForBackup> {
const fixedUpAttachment = { ...attachment };
const keyToUse = attachment.key ?? Bytes.toBase64(getRandomBytes(64));
let digestToUse = attachment.key ? attachment.digest : undefined;
if (!digestToUse) {
// Delete current locators for the attachment; we can no longer use them and will need
// to fully re-encrypt and upload
delete fixedUpAttachment.cdnId;
delete fixedUpAttachment.cdnKey;
delete fixedUpAttachment.cdnNumber;
// encrypt this file in memory in order to calculate the digest
const { digest } = await encryptAttachmentV2({
keys: Bytes.fromBase64(keyToUse),
plaintextAbsolutePath: window.Signal.Migrations.getAbsoluteAttachmentPath(
attachment.path
),
});
digestToUse = Bytes.toBase64(digest);
// TODO (DESKTOP-6688): ensure that we update the message/attachment in DB with the
// new keys so that we don't try to re-upload it again on the next export
}
return {
...fixedUpAttachment,
key: keyToUse,
digest: digestToUse,
};
}
export async function convertAttachmentToFilePointer({
attachment,
backupLevel,
getBackupTierInfo,
}: {
attachment: AttachmentType;
backupLevel: BackupLevel;
getBackupTierInfo: (
mediaName: string
) => { isInBackupTier: true; cdnNumber: number } | { isInBackupTier: false };
}): Promise<Backups.FilePointer> {
const filePointerRootProps = new Backups.FilePointer({
contentType: attachment.contentType,
incrementalMac: attachment.incrementalMac
? Bytes.fromBase64(attachment.incrementalMac)
: undefined,
incrementalMacChunkSize: attachment.incrementalMacChunkSize,
fileName: attachment.fileName,
width: attachment.width,
height: attachment.height,
caption: attachment.caption,
blurHash: attachment.blurHash,
});
if (!isDownloadedToLocalFile(attachment)) {
// 1. If the attachment is undownloaded, we cannot trust its digest / mediaName. Thus,
// we only include a BackupLocator if this attachment already had one (e.g. we
// restored it from a backup and it had a BackupLocator then, which means we have at
// one point in the past verified the digest).
if (
isDownloadableFromBackupTier(attachment) &&
backupLevel === BackupLevel.Media
) {
return new Backups.FilePointer({
...filePointerRootProps,
backupLocator: getBackupLocator(attachment),
});
}
// 2. Otherwise, we only return the transit CDN info via AttachmentLocator
if (isDownloadableFromTransitTier(attachment)) {
return new Backups.FilePointer({
...filePointerRootProps,
attachmentLocator: getAttachmentLocator(attachment),
});
}
}
if (backupLevel !== BackupLevel.Media) {
if (isDownloadableFromTransitTier(attachment)) {
return new Backups.FilePointer({
...filePointerRootProps,
attachmentLocator: getAttachmentLocator(attachment),
});
}
return new Backups.FilePointer({
...filePointerRootProps,
invalidAttachmentLocator: getInvalidAttachmentLocator(),
});
}
if (!isDownloadedToLocalFile(attachment)) {
return new Backups.FilePointer({
...filePointerRootProps,
invalidAttachmentLocator: getInvalidAttachmentLocator(),
});
}
const attachmentForBackup = await fixupAttachmentForBackup(attachment);
const mediaName = getMediaNameForAttachment(attachmentForBackup);
const backupTierInfo = getBackupTierInfo(mediaName);
let cdnNumberInBackupTier: number | undefined;
if (backupTierInfo.isInBackupTier) {
cdnNumberInBackupTier = backupTierInfo.cdnNumber;
}
return new Backups.FilePointer({
...filePointerRootProps,
backupLocator: getBackupLocator({
...attachmentForBackup,
backupLocator: {
mediaName,
cdnNumber: cdnNumberInBackupTier,
},
}),
});
}
export function getMediaNameForAttachment(attachment: AttachmentType): string {
strictAssert(attachment.digest, 'Digest must be present');
return attachment.digest;
}
// mediaId is special in that it is encoded in base64url
export function getBytesFromMediaId(mediaId: string): Uint8Array {
return Bytes.fromBase64url(mediaId);
}
function getAttachmentLocator(
attachment: AttachmentDownloadableFromTransitTier
) {
return new Backups.FilePointer.AttachmentLocator({
cdnKey: attachment.cdnKey,
cdnNumber: attachment.cdnNumber,
uploadTimestamp: attachment.uploadTimestamp
? Long.fromNumber(attachment.uploadTimestamp)
: null,
digest: Bytes.fromBase64(attachment.digest),
key: Bytes.fromBase64(attachment.key),
size: attachment.size,
});
}
function getBackupLocator(attachment: AttachmentDownloadableFromBackupTier) {
return new Backups.FilePointer.BackupLocator({
mediaName: attachment.backupLocator.mediaName,
cdnNumber: attachment.backupLocator.cdnNumber,
digest: Bytes.fromBase64(attachment.digest),
key: Bytes.fromBase64(attachment.key),
size: attachment.size,
transitCdnKey: attachment.cdnKey,
transitCdnNumber: attachment.cdnNumber,
});
}
function getInvalidAttachmentLocator() {
return new Backups.FilePointer.InvalidAttachmentLocator();
}

View file

@ -40,7 +40,7 @@ import {
KEY_SET_LENGTH,
_generateAttachmentIv,
decryptAttachmentV2,
encryptAttachmentV2,
encryptAttachmentV2ToDisk,
getAesCbcCiphertextLength,
splitKeys,
} from '../AttachmentCrypto';
@ -607,7 +607,7 @@ describe('Crypto', () => {
let ciphertextPath;
try {
const encryptedAttachment = await encryptAttachmentV2({
const encryptedAttachment = await encryptAttachmentV2ToDisk({
keys,
plaintextAbsolutePath: FILE_PATH,
});
@ -655,7 +655,7 @@ describe('Crypto', () => {
let ciphertextPath;
try {
const encryptedAttachment = await encryptAttachmentV2({
const encryptedAttachment = await encryptAttachmentV2ToDisk({
keys,
plaintextAbsolutePath: sourcePath,
});
@ -700,7 +700,7 @@ describe('Crypto', () => {
let ciphertextPath;
try {
const encryptedAttachment = await encryptAttachmentV2({
const encryptedAttachment = await encryptAttachmentV2ToDisk({
keys,
plaintextAbsolutePath: FILE_PATH,
});
@ -753,7 +753,7 @@ describe('Crypto', () => {
});
const ciphertextV1 = encryptedAttachmentV1.ciphertext;
const encryptedAttachmentV2 = await encryptAttachmentV2({
const encryptedAttachmentV2 = await encryptAttachmentV2ToDisk({
keys,
plaintextAbsolutePath: FILE_PATH,
dangerousTestOnlyIv,
@ -788,7 +788,7 @@ describe('Crypto', () => {
let outerCiphertextPath;
let innerEncryptedAttachment;
try {
innerEncryptedAttachment = await encryptAttachmentV2({
innerEncryptedAttachment = await encryptAttachmentV2ToDisk({
keys: innerKeys,
plaintextAbsolutePath,
});
@ -797,7 +797,7 @@ describe('Crypto', () => {
innerEncryptedAttachment.path
);
const outerEncryptedAttachment = await encryptAttachmentV2({
const outerEncryptedAttachment = await encryptAttachmentV2ToDisk({
keys: outerKeys,
plaintextAbsolutePath: innerCiphertextPath,
// We (and the server!) don't pad the second layer

View file

@ -0,0 +1,461 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { assert } from 'chai';
import Long from 'long';
import { join } from 'path';
import * as sinon from 'sinon';
import { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
import { Backups } from '../../protobuf';
import {
convertAttachmentToFilePointer,
convertFilePointerToAttachment,
} from '../../services/backups/util/filePointers';
import { APPLICATION_OCTET_STREAM, IMAGE_PNG } from '../../types/MIME';
import * as Bytes from '../../Bytes';
import type { AttachmentType } from '../../types/Attachment';
import { strictAssert } from '../../util/assert';
describe('convertFilePointerToAttachment', () => {
it('processes filepointer with attachmentLocator', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
contentType: 'image/png',
width: 100,
height: 100,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
incrementalMac: Bytes.fromString('incrementalMac'),
incrementalMacChunkSize: 1000,
attachmentLocator: new Backups.FilePointer.AttachmentLocator({
size: 128,
cdnKey: 'cdnKey',
cdnNumber: 2,
key: Bytes.fromString('key'),
digest: Bytes.fromString('digest'),
uploadTimestamp: Long.fromNumber(1970),
}),
})
);
assert.deepStrictEqual(result, {
contentType: IMAGE_PNG,
width: 100,
height: 100,
size: 128,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
cdnKey: 'cdnKey',
cdnNumber: 2,
key: Bytes.toBase64(Bytes.fromString('key')),
digest: Bytes.toBase64(Bytes.fromString('digest')),
uploadTimestamp: 1970,
incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
incrementalMacChunkSize: 1000,
});
});
it('processes filepointer with backupLocator and missing fields', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
contentType: 'image/png',
width: 100,
height: 100,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
incrementalMac: Bytes.fromString('incrementalMac'),
incrementalMacChunkSize: 1000,
backupLocator: new Backups.FilePointer.BackupLocator({
mediaName: 'mediaName',
cdnNumber: 3,
size: 128,
key: Bytes.fromString('key'),
digest: Bytes.fromString('digest'),
transitCdnKey: 'transitCdnKey',
transitCdnNumber: 2,
}),
})
);
assert.deepStrictEqual(result, {
contentType: IMAGE_PNG,
width: 100,
height: 100,
size: 128,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
cdnKey: 'transitCdnKey',
cdnNumber: 2,
key: Bytes.toBase64(Bytes.fromString('key')),
digest: Bytes.toBase64(Bytes.fromString('digest')),
incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
incrementalMacChunkSize: 1000,
backupLocator: {
mediaName: 'mediaName',
cdnNumber: 3,
},
});
});
it('processes filepointer with invalidAttachmentLocator', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
contentType: 'image/png',
width: 100,
height: 100,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
incrementalMac: Bytes.fromString('incrementalMac'),
incrementalMacChunkSize: 1000,
invalidAttachmentLocator:
new Backups.FilePointer.InvalidAttachmentLocator(),
})
);
assert.deepStrictEqual(result, {
contentType: IMAGE_PNG,
width: 100,
height: 100,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
incrementalMacChunkSize: 1000,
size: 0,
error: true,
});
});
it('accepts missing / null fields and adds defaults to contentType and size', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
backupLocator: new Backups.FilePointer.BackupLocator(),
})
);
assert.deepStrictEqual(result, {
contentType: APPLICATION_OCTET_STREAM,
size: 0,
width: undefined,
height: undefined,
blurHash: undefined,
fileName: undefined,
caption: undefined,
cdnKey: undefined,
cdnNumber: undefined,
key: undefined,
digest: undefined,
incrementalMac: undefined,
incrementalMacChunkSize: undefined,
backupLocator: undefined,
});
});
});
function composeAttachment(
overrides: Partial<AttachmentType> = {}
): AttachmentType {
return {
size: 100,
contentType: IMAGE_PNG,
cdnKey: 'cdnKey',
cdnNumber: 2,
path: 'path/to/file.png',
key: 'key',
digest: 'digest',
width: 100,
height: 100,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
incrementalMac: 'incrementalMac',
incrementalMacChunkSize: 1000,
uploadTimestamp: 1234,
...overrides,
};
}
const defaultFilePointer = new Backups.FilePointer({
contentType: IMAGE_PNG,
width: 100,
height: 100,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
incrementalMac: Bytes.fromBase64('incrementalMac'),
incrementalMacChunkSize: 1000,
});
const defaultAttachmentLocator = new Backups.FilePointer.AttachmentLocator({
cdnKey: 'cdnKey',
cdnNumber: 2,
key: Bytes.fromBase64('key'),
digest: Bytes.fromBase64('digest'),
size: 100,
uploadTimestamp: Long.fromNumber(1234),
});
const defaultMediaName = 'digest';
const defaultBackupLocator = new Backups.FilePointer.BackupLocator({
mediaName: defaultMediaName,
cdnNumber: null,
key: Bytes.fromBase64('key'),
digest: Bytes.fromBase64('digest'),
size: 100,
transitCdnKey: 'cdnKey',
transitCdnNumber: 2,
});
const filePointerWithAttachmentLocator = new Backups.FilePointer({
...defaultFilePointer,
attachmentLocator: defaultAttachmentLocator,
});
const filePointerWithBackupLocator = new Backups.FilePointer({
...defaultFilePointer,
backupLocator: defaultBackupLocator,
});
const filePointerWithInvalidLocator = new Backups.FilePointer({
...defaultFilePointer,
invalidAttachmentLocator: new Backups.FilePointer.InvalidAttachmentLocator(),
});
async function testAttachmentToFilePointer(
attachment: AttachmentType,
filePointer: Backups.FilePointer,
options?: { backupLevel?: BackupLevel; backupCdnNumber?: number }
) {
async function _doTest(withBackupLevel: BackupLevel) {
assert.deepStrictEqual(
await convertAttachmentToFilePointer({
attachment,
backupLevel: withBackupLevel,
getBackupTierInfo: _mediaName => {
if (options?.backupCdnNumber != null) {
return { isInBackupTier: true, cdnNumber: options.backupCdnNumber };
}
return { isInBackupTier: false };
},
}),
filePointer
);
}
if (!options?.backupLevel) {
await _doTest(BackupLevel.Messages);
await _doTest(BackupLevel.Media);
} else {
await _doTest(options.backupLevel);
}
}
describe('convertAttachmentToFilePointer', () => {
describe('not downloaded locally', () => {
const undownloadedAttachment = composeAttachment({ path: undefined });
it('returns invalidAttachmentLocator if missing critical decryption info', async () => {
await testAttachmentToFilePointer(
{
...undownloadedAttachment,
key: undefined,
},
filePointerWithInvalidLocator
);
await testAttachmentToFilePointer(
{
...undownloadedAttachment,
digest: undefined,
},
filePointerWithInvalidLocator
);
});
describe('attachment does not have attachment.backupLocator', () => {
it('returns attachmentLocator, regardless of backupLevel or backup tier status', async () => {
await testAttachmentToFilePointer(
undownloadedAttachment,
filePointerWithAttachmentLocator,
{ backupCdnNumber: 3 }
);
});
it('returns invalidAttachmentLocator if missing critical locator info', async () => {
await testAttachmentToFilePointer(
{
...undownloadedAttachment,
cdnKey: undefined,
},
filePointerWithInvalidLocator
);
await testAttachmentToFilePointer(
{
...undownloadedAttachment,
cdnNumber: undefined,
},
filePointerWithInvalidLocator
);
});
});
describe('attachment has attachment.backupLocator', () => {
const undownloadedAttachmentWithBackupLocator = {
...undownloadedAttachment,
backupLocator: { mediaName: defaultMediaName },
};
it('returns backupLocator if backupLevel is Media', async () => {
await testAttachmentToFilePointer(
undownloadedAttachmentWithBackupLocator,
filePointerWithBackupLocator,
{ backupLevel: BackupLevel.Media }
);
});
it('returns backupLocator even if missing transit CDN info', async () => {
// Even if missing transit CDNKey
await testAttachmentToFilePointer(
{ ...undownloadedAttachmentWithBackupLocator, cdnKey: undefined },
new Backups.FilePointer({
...filePointerWithBackupLocator,
backupLocator: new Backups.FilePointer.BackupLocator({
...defaultBackupLocator,
transitCdnKey: undefined,
}),
}),
{ backupLevel: BackupLevel.Media }
);
});
it('returns attachmentLocator if backupLevel is Messages', async () => {
await testAttachmentToFilePointer(
undownloadedAttachmentWithBackupLocator,
filePointerWithAttachmentLocator,
{ backupLevel: BackupLevel.Messages }
);
});
});
});
describe('downloaded locally', () => {
const downloadedAttachment = composeAttachment();
describe('BackupLevel.Messages', () => {
it('returns attachmentLocator', async () => {
await testAttachmentToFilePointer(
downloadedAttachment,
filePointerWithAttachmentLocator,
{ backupLevel: BackupLevel.Messages }
);
});
it('returns invalidAttachmentLocator if missing critical locator info', async () => {
await testAttachmentToFilePointer(
{
...downloadedAttachment,
cdnKey: undefined,
},
filePointerWithInvalidLocator,
{ backupLevel: BackupLevel.Messages }
);
await testAttachmentToFilePointer(
{
...downloadedAttachment,
cdnNumber: undefined,
},
filePointerWithInvalidLocator,
{ backupLevel: BackupLevel.Messages }
);
});
it('returns invalidAttachmentLocator if missing critical decryption info', async () => {
await testAttachmentToFilePointer(
{
...downloadedAttachment,
key: undefined,
},
filePointerWithInvalidLocator,
{ backupLevel: BackupLevel.Messages }
);
await testAttachmentToFilePointer(
{
...downloadedAttachment,
digest: undefined,
},
filePointerWithInvalidLocator,
{ backupLevel: BackupLevel.Messages }
);
});
});
describe('BackupLevel.Media', () => {
describe('if missing critical decryption info', () => {
const FILE_PATH = join(__dirname, '../../../fixtures/ghost-kitty.mp4');
let sandbox: sinon.SinonSandbox;
beforeEach(() => {
sandbox = sinon.createSandbox();
sandbox
.stub(window.Signal.Migrations, 'getAbsoluteAttachmentPath')
.callsFake(relPath => {
if (relPath === downloadedAttachment.path) {
return FILE_PATH;
}
return relPath;
});
});
afterEach(() => {
sandbox.restore();
});
it('generates new key & digest and removes existing CDN info', async () => {
const result = await convertAttachmentToFilePointer({
attachment: {
...downloadedAttachment,
key: undefined,
},
backupLevel: BackupLevel.Media,
getBackupTierInfo: () => ({ isInBackupTier: false }),
});
const newKey = result.backupLocator?.key;
const newDigest = result.backupLocator?.digest;
strictAssert(newDigest, 'must create new digest');
assert.deepStrictEqual(
result,
new Backups.FilePointer({
...filePointerWithBackupLocator,
backupLocator: new Backups.FilePointer.BackupLocator({
...defaultBackupLocator,
key: newKey,
digest: newDigest,
mediaName: Bytes.toBase64(newDigest),
transitCdnKey: undefined,
transitCdnNumber: undefined,
}),
})
);
});
});
it('returns BackupLocator, with cdnNumber if in backup tier already', async () => {
await testAttachmentToFilePointer(
downloadedAttachment,
new Backups.FilePointer({
...filePointerWithBackupLocator,
backupLocator: new Backups.FilePointer.BackupLocator({
...defaultBackupLocator,
cdnNumber: 12,
}),
}),
{ backupLevel: BackupLevel.Media, backupCdnNumber: 12 }
);
});
it('returns BackupLocator, with empty cdnNumber if not in backup tier', async () => {
await testAttachmentToFilePointer(
downloadedAttachment,
filePointerWithBackupLocator,
{ backupLevel: BackupLevel.Media }
);
});
});
});
});

View file

@ -1,149 +0,0 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { assert } from 'chai';
import Long from 'long';
import { Backups } from '../../protobuf';
import { convertFilePointerToAttachment } from '../../services/backups/util/filePointers';
import { APPLICATION_OCTET_STREAM, IMAGE_PNG } from '../../types/MIME';
import * as Bytes from '../../Bytes';
describe('convertFilePointerToAttachment', () => {
it('processes filepointer with attachmentLocator', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
contentType: 'image/png',
width: 100,
height: 100,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
incrementalMac: Bytes.fromString('incrementalMac'),
incrementalMacChunkSize: 1000,
attachmentLocator: new Backups.FilePointer.AttachmentLocator({
size: 128,
cdnKey: 'cdnKey',
cdnNumber: 2,
key: Bytes.fromString('key'),
digest: Bytes.fromString('digest'),
uploadTimestamp: Long.fromNumber(1970),
}),
})
);
assert.deepStrictEqual(result, {
contentType: IMAGE_PNG,
width: 100,
height: 100,
size: 128,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
cdnKey: 'cdnKey',
cdnNumber: 2,
key: Bytes.toBase64(Bytes.fromString('key')),
digest: Bytes.toBase64(Bytes.fromString('digest')),
uploadTimestamp: 1970,
incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
incrementalMacChunkSize: 1000,
});
});
it('processes filepointer with backupLocator and missing fields', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
contentType: 'image/png',
width: 100,
height: 100,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
incrementalMac: Bytes.fromString('incrementalMac'),
incrementalMacChunkSize: 1000,
backupLocator: new Backups.FilePointer.BackupLocator({
mediaName: 'mediaName',
cdnNumber: 3,
size: 128,
key: Bytes.fromString('key'),
digest: Bytes.fromString('digest'),
transitCdnKey: 'transitCdnKey',
transitCdnNumber: 2,
}),
})
);
assert.deepStrictEqual(result, {
contentType: IMAGE_PNG,
width: 100,
height: 100,
size: 128,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
cdnKey: 'transitCdnKey',
cdnNumber: 2,
key: Bytes.toBase64(Bytes.fromString('key')),
digest: Bytes.toBase64(Bytes.fromString('digest')),
incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
incrementalMacChunkSize: 1000,
backupLocator: {
mediaName: 'mediaName',
cdnNumber: 3,
},
});
});
it('processes filepointer with invalidAttachmentLocator', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
contentType: 'image/png',
width: 100,
height: 100,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
incrementalMac: Bytes.fromString('incrementalMac'),
incrementalMacChunkSize: 1000,
invalidAttachmentLocator:
new Backups.FilePointer.InvalidAttachmentLocator(),
})
);
assert.deepStrictEqual(result, {
contentType: IMAGE_PNG,
width: 100,
height: 100,
blurHash: 'blurhash',
fileName: 'filename',
caption: 'caption',
incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
incrementalMacChunkSize: 1000,
size: 0,
error: true,
});
});
it('accepts missing / null fields and adds defaults to contentType and size', () => {
const result = convertFilePointerToAttachment(
new Backups.FilePointer({
backupLocator: new Backups.FilePointer.BackupLocator(),
})
);
assert.deepStrictEqual(result, {
contentType: APPLICATION_OCTET_STREAM,
size: 0,
width: undefined,
height: undefined,
blurHash: undefined,
fileName: undefined,
caption: undefined,
cdnKey: undefined,
cdnNumber: undefined,
key: undefined,
digest: undefined,
incrementalMac: undefined,
incrementalMacChunkSize: undefined,
backupLocator: undefined,
});
});
});

View file

@ -21,7 +21,7 @@ import {
isImageTypeSupported,
isVideoTypeSupported,
} from '../util/GoogleChrome';
import type { LocalizerType } from './Util';
import type { LocalizerType, WithRequiredProperties } from './Util';
import { ThemeType } from './Util';
import * as GoogleChrome from '../util/GoogleChrome';
import { ReadStatus } from '../messages/MessageReadStatus';
@ -994,3 +994,66 @@ export function getAttachmentSignature(attachment: AttachmentType): string {
strictAssert(attachment.digest, 'attachment missing digest');
return attachment.digest;
}
type RequiredPropertiesForDecryption = 'key' | 'digest';
type DecryptableAttachment = WithRequiredProperties<
AttachmentType,
RequiredPropertiesForDecryption
>;
export type AttachmentDownloadableFromTransitTier = WithRequiredProperties<
DecryptableAttachment,
'cdnKey' | 'cdnNumber'
>;
export type AttachmentDownloadableFromBackupTier = WithRequiredProperties<
DecryptableAttachment,
'backupLocator'
>;
export type DownloadedAttachment = WithRequiredProperties<
AttachmentType,
'path'
>;
export type AttachmentReadyForBackup = WithRequiredProperties<
DownloadedAttachment,
RequiredPropertiesForDecryption
>;
function isDecryptable(
attachment: AttachmentType
): attachment is DecryptableAttachment {
return Boolean(attachment.key) && Boolean(attachment.digest);
}
export function isDownloadableFromTransitTier(
attachment: AttachmentType
): attachment is AttachmentDownloadableFromTransitTier {
if (!isDecryptable(attachment)) {
return false;
}
if (attachment.cdnKey && attachment.cdnNumber) {
return true;
}
return false;
}
export function isDownloadableFromBackupTier(
attachment: AttachmentType
): attachment is AttachmentDownloadableFromBackupTier {
if (!attachment.key || !attachment.digest) {
return false;
}
if (attachment.backupLocator?.mediaName) {
return true;
}
return false;
}
export function isDownloadedToLocalFile(
attachment: AttachmentType
): attachment is DownloadedAttachment {
return Boolean(attachment.path);
}