diff --git a/ts/AttachmentCrypto.ts b/ts/AttachmentCrypto.ts index 3576ff4616c5..20035bdd574c 100644 --- a/ts/AttachmentCrypto.ts +++ b/ts/AttachmentCrypto.ts @@ -11,7 +11,7 @@ import { randomBytes, } from 'crypto'; import type { Decipher, Hash, Hmac } from 'crypto'; -import { Transform } from 'stream'; +import { PassThrough, Transform, type Writable } from 'stream'; import { pipeline } from 'stream/promises'; import { ensureFile } from 'fs-extra'; import * as log from './logging/log'; @@ -45,7 +45,6 @@ export function _generateAttachmentIv(): Uint8Array { } export type EncryptedAttachmentV2 = { - path: string; digest: Uint8Array; plaintextHash: string; }; @@ -55,24 +54,55 @@ export type DecryptedAttachmentV2 = { plaintextHash: string; }; +type EncryptAttachmentV2PropsType = { + keys: Readonly; + plaintextAbsolutePath: string; + dangerousTestOnlyIv?: Readonly; + dangerousTestOnlySkipPadding?: boolean; +}; + +export async function encryptAttachmentV2ToDisk( + args: EncryptAttachmentV2PropsType +): Promise { + // Create random output file + const relativeTargetPath = getRelativePath(createName()); + const absoluteTargetPath = + window.Signal.Migrations.getAbsoluteAttachmentPath(relativeTargetPath); + + let writeFd; + let encryptResult: EncryptedAttachmentV2; + + try { + await ensureFile(absoluteTargetPath); + writeFd = await open(absoluteTargetPath, 'w'); + encryptResult = await encryptAttachmentV2({ + ...args, + sink: writeFd.createWriteStream(), + }); + } catch (error) { + safeUnlinkSync(absoluteTargetPath); + throw error; + } finally { + await writeFd?.close(); + } + + return { + ...encryptResult, + path: relativeTargetPath, + }; +} + export async function encryptAttachmentV2({ keys, plaintextAbsolutePath, dangerousTestOnlyIv, dangerousTestOnlySkipPadding = false, -}: { - keys: Readonly; - plaintextAbsolutePath: string; - dangerousTestOnlyIv?: Readonly; - dangerousTestOnlySkipPadding?: boolean; + sink, +}: EncryptAttachmentV2PropsType & { + sink?: Writable; }): Promise { const logId = 'encryptAttachmentV2'; - // Create random output file - const relativeTargetPath = getRelativePath(createName()); - const absoluteTargetPath = - window.Signal.Migrations.getAbsoluteAttachmentPath(relativeTargetPath); - const { aesKey, macKey } = splitKeys(keys); if (dangerousTestOnlyIv && window.getEnvironment() !== Environment.Test) { @@ -92,19 +122,12 @@ export async function encryptAttachmentV2({ const digest = createHash(HashType.size256); let readFd; - let writeFd; try { try { readFd = await open(plaintextAbsolutePath, 'r'); } catch (cause) { throw new Error(`${logId}: Read path doesn't exist`, { cause }); } - try { - await ensureFile(absoluteTargetPath); - writeFd = await open(absoluteTargetPath, 'w'); - } catch (cause) { - throw new Error(`${logId}: Failed to create write path`, { cause }); - } await pipeline( [ @@ -115,7 +138,7 @@ export async function encryptAttachmentV2({ prependIv(iv), appendMacStream(macKey), peekAndUpdateHash(digest), - writeFd.createWriteStream(), + sink ?? new PassThrough().resume(), ].filter(isNotNil) ); } catch (error) { @@ -123,10 +146,9 @@ export async function encryptAttachmentV2({ `${logId}: Failed to encrypt attachment`, Errors.toLogFormat(error) ); - safeUnlinkSync(absoluteTargetPath); throw error; } finally { - await Promise.all([readFd?.close(), writeFd?.close()]); + await readFd?.close(); } const ourPlaintextHash = plaintextHash.digest('hex'); @@ -143,7 +165,6 @@ export async function encryptAttachmentV2({ ); return { - path: relativeTargetPath, digest: ourDigest, plaintextHash: ourPlaintextHash, }; diff --git a/ts/Bytes.ts b/ts/Bytes.ts index 6baff3ed2686..bf1678c1ecca 100644 --- a/ts/Bytes.ts +++ b/ts/Bytes.ts @@ -8,6 +8,9 @@ const bytes = globalThis.window?.SignalContext?.bytes || new Bytes(); export function fromBase64(value: string): Uint8Array { return bytes.fromBase64(value); } +export function fromBase64url(value: string): Uint8Array { + return bytes.fromBase64url(value); +} export function fromHex(value: string): Uint8Array { return bytes.fromHex(value); diff --git a/ts/context/Bytes.ts b/ts/context/Bytes.ts index c6177031a106..6d3e9ba209fb 100644 --- a/ts/context/Bytes.ts +++ b/ts/context/Bytes.ts @@ -8,6 +8,10 @@ export class Bytes { return Buffer.from(value, 'base64'); } + public fromBase64url(value: string): Uint8Array { + return Buffer.from(value, 'base64url'); + } + public fromHex(value: string): Uint8Array { return Buffer.from(value, 'hex'); } diff --git a/ts/services/backups/export.ts b/ts/services/backups/export.ts index 202c9d5ff5c2..87db9ddf60b6 100644 --- a/ts/services/backups/export.ts +++ b/ts/services/backups/export.ts @@ -3,6 +3,7 @@ import Long from 'long'; import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client'; +import type { BackupLevel } from '@signalapp/libsignal-client/zkgroup'; import pMap from 'p-map'; import pTimeout from 'p-timeout'; import { Readable } from 'stream'; @@ -82,6 +83,13 @@ import { numberToAddressType, numberToPhoneType, } from '../../types/EmbeddedContact'; +import { + isVoiceMessage, + type AttachmentType, + isGIF, + isDownloaded, +} from '../../types/Attachment'; +import { convertAttachmentToFilePointer } from './util/filePointers'; const MAX_CONCURRENCY = 10; @@ -116,14 +124,13 @@ export class BackupExportStream extends Readable { private nextRecipientId = 0; private flushResolve: (() => void) | undefined; - public run(): void { + public run(backupLevel: BackupLevel): void { drop( (async () => { log.info('BackupExportStream: starting...'); await Data.pauseWriteAccess(); - try { - await this.unsafeRun(); + await this.unsafeRun(backupLevel); } catch (error) { this.emit('error', error); } finally { @@ -134,7 +141,7 @@ export class BackupExportStream extends Readable { ); } - private async unsafeRun(): Promise { + private async unsafeRun(backupLevel: BackupLevel): Promise { this.push( Backups.BackupInfo.encodeDelimited({ version: Long.fromNumber(BACKUP_VERSION), @@ -279,7 +286,12 @@ export class BackupExportStream extends Readable { // eslint-disable-next-line no-await-in-loop const items = await pMap( messages, - message => this.toChatItem(message, { aboutMe, callHistoryByCallId }), + message => + this.toChatItem(message, { + aboutMe, + callHistoryByCallId, + backupLevel, + }), { concurrency: MAX_CONCURRENCY } ); @@ -308,6 +320,7 @@ export class BackupExportStream extends Readable { } await this.flush(); + log.warn('backups: final stats', stats); this.push(null); @@ -562,6 +575,7 @@ export class BackupExportStream extends Readable { options: { aboutMe: AboutMe; callHistoryByCallId: Record; + backupLevel: BackupLevel; } ): Promise { const chatId = this.getRecipientId({ id: message.conversationId }); @@ -629,6 +643,16 @@ export class BackupExportStream extends Readable { // TODO (DESKTOP-6964): put incoming/outgoing fields below onto non-bubble messages result.standardMessage = { quote: await this.toQuote(message.quote), + attachments: message.attachments + ? await Promise.all( + message.attachments.map(attachment => { + return this.processMessageAttachment({ + attachment, + backupLevel: options.backupLevel, + }); + }) + ) + : undefined, text: { // Note that we store full text on the message model so we have to // trim it before serializing. @@ -1579,6 +1603,63 @@ export class BackupExportStream extends Readable { }), }; } + + private getMessageAttachmentFlag( + attachment: AttachmentType + ): Backups.MessageAttachment.Flag { + if (isVoiceMessage(attachment)) { + return Backups.MessageAttachment.Flag.VOICE_MESSAGE; + } + if (isGIF([attachment])) { + return Backups.MessageAttachment.Flag.GIF; + } + if ( + attachment.flags && + // eslint-disable-next-line no-bitwise + attachment.flags & SignalService.AttachmentPointer.Flags.BORDERLESS + ) { + return Backups.MessageAttachment.Flag.BORDERLESS; + } + + return Backups.MessageAttachment.Flag.NONE; + } + + private async processMessageAttachment({ + attachment, + backupLevel, + }: { + attachment: AttachmentType; + backupLevel: BackupLevel; + }): Promise { + const filePointer = await this.processAttachment({ + attachment, + backupLevel, + }); + + return new Backups.MessageAttachment({ + pointer: filePointer, + flag: this.getMessageAttachmentFlag(attachment), + wasDownloaded: isDownloaded(attachment), // should always be true + }); + } + + private async processAttachment({ + attachment, + backupLevel, + }: { + attachment: AttachmentType; + backupLevel: BackupLevel; + }): Promise { + const filePointer = await convertAttachmentToFilePointer({ + attachment, + backupLevel, + // TODO (DESKTOP-6983) -- Retrieve & save backup tier media list + getBackupTierInfo: () => ({ + isInBackupTier: false, + }), + }); + return filePointer; + } } function checkServiceIdEquivalence( diff --git a/ts/services/backups/index.ts b/ts/services/backups/index.ts index 1e08a33e3fce..e37be99ff6ff 100644 --- a/ts/services/backups/index.ts +++ b/ts/services/backups/index.ts @@ -10,6 +10,7 @@ import { join } from 'path'; import { createGzip, createGunzip } from 'zlib'; import { createCipheriv, createHmac, randomBytes } from 'crypto'; import { noop } from 'lodash'; +import { BackupLevel } from '@signalapp/libsignal-client/zkgroup'; import * as log from '../../logging/log'; import * as Bytes from '../../Bytes'; @@ -68,8 +69,11 @@ export class BackupsService { const fileName = `backup-${randomBytes(32).toString('hex')}`; const filePath = join(window.BasePaths.temp, fileName); + const backupLevel = await this.credentials.getBackupLevel(); + log.info(`exportBackup: starting, backup level: ${backupLevel}...`); + try { - const fileSize = await this.exportToDisk(filePath); + const fileSize = await this.exportToDisk(filePath, backupLevel); await this.api.upload(filePath, fileSize); } finally { @@ -82,19 +86,24 @@ export class BackupsService { } // Test harness - public async exportBackupData(): Promise { + public async exportBackupData( + backupLevel: BackupLevel = BackupLevel.Messages + ): Promise { const sink = new PassThrough(); const chunks = new Array(); sink.on('data', chunk => chunks.push(chunk)); - await this.exportBackup(sink); + await this.exportBackup(sink, backupLevel); return Bytes.concatenate(chunks); } // Test harness - public async exportToDisk(path: string): Promise { - const size = await this.exportBackup(createWriteStream(path)); + public async exportToDisk( + path: string, + backupLevel: BackupLevel = BackupLevel.Messages + ): Promise { + const size = await this.exportBackup(createWriteStream(path), backupLevel); await validateBackup(path, size); @@ -174,7 +183,10 @@ export class BackupsService { } } - private async exportBackup(sink: Writable): Promise { + private async exportBackup( + sink: Writable, + backupLevel: BackupLevel = BackupLevel.Messages + ): Promise { strictAssert(!this.isRunning, 'BackupService is already running'); log.info('exportBackup: starting...'); @@ -184,7 +196,7 @@ export class BackupsService { const { aesKey, macKey } = getKeyMaterial(); const recordStream = new BackupExportStream(); - recordStream.run(); + recordStream.run(backupLevel); const iv = randomBytes(IV_LENGTH); diff --git a/ts/services/backups/util/filePointers.ts b/ts/services/backups/util/filePointers.ts index ca79f1006a67..a40b52e2e376 100644 --- a/ts/services/backups/util/filePointers.ts +++ b/ts/services/backups/util/filePointers.ts @@ -1,13 +1,28 @@ // Copyright 2024 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only +import Long from 'long'; +import { BackupLevel } from '@signalapp/libsignal-client/zkgroup'; + import { APPLICATION_OCTET_STREAM, stringToMIMEType, } from '../../../types/MIME'; -import type { AttachmentType } from '../../../types/Attachment'; -import type { Backups } from '../../../protobuf'; +import { + type AttachmentType, + isDownloadableFromTransitTier, + isDownloadableFromBackupTier, + isDownloadedToLocalFile, + type AttachmentDownloadableFromTransitTier, + type AttachmentDownloadableFromBackupTier, + type DownloadedAttachment, + type AttachmentReadyForBackup, +} from '../../../types/Attachment'; +import { Backups } from '../../../protobuf'; import * as Bytes from '../../../Bytes'; import { getTimestampFromLong } from '../../../util/timestampLongUtils'; +import { getRandomBytes } from '../../../Crypto'; +import { encryptAttachmentV2 } from '../../../AttachmentCrypto'; +import { strictAssert } from '../../../util/assert'; export function convertFilePointerToAttachment( filePointer: Backups.FilePointer @@ -94,3 +109,174 @@ export function convertFilePointerToAttachment( throw new Error('convertFilePointerToAttachment: mising locator'); } + +/** + * Some attachments saved on desktop do not include the key used to encrypt the file + * originally. This means that we need to encrypt the file in-memory now (at + * export-creation time) to calculate the digest which will be saved in the backup proto + * along with the new keys. + */ +async function fixupAttachmentForBackup( + attachment: DownloadedAttachment +): Promise { + const fixedUpAttachment = { ...attachment }; + const keyToUse = attachment.key ?? Bytes.toBase64(getRandomBytes(64)); + let digestToUse = attachment.key ? attachment.digest : undefined; + + if (!digestToUse) { + // Delete current locators for the attachment; we can no longer use them and will need + // to fully re-encrypt and upload + delete fixedUpAttachment.cdnId; + delete fixedUpAttachment.cdnKey; + delete fixedUpAttachment.cdnNumber; + + // encrypt this file in memory in order to calculate the digest + const { digest } = await encryptAttachmentV2({ + keys: Bytes.fromBase64(keyToUse), + plaintextAbsolutePath: window.Signal.Migrations.getAbsoluteAttachmentPath( + attachment.path + ), + }); + + digestToUse = Bytes.toBase64(digest); + + // TODO (DESKTOP-6688): ensure that we update the message/attachment in DB with the + // new keys so that we don't try to re-upload it again on the next export + } + + return { + ...fixedUpAttachment, + key: keyToUse, + digest: digestToUse, + }; +} + +export async function convertAttachmentToFilePointer({ + attachment, + backupLevel, + getBackupTierInfo, +}: { + attachment: AttachmentType; + backupLevel: BackupLevel; + getBackupTierInfo: ( + mediaName: string + ) => { isInBackupTier: true; cdnNumber: number } | { isInBackupTier: false }; +}): Promise { + const filePointerRootProps = new Backups.FilePointer({ + contentType: attachment.contentType, + incrementalMac: attachment.incrementalMac + ? Bytes.fromBase64(attachment.incrementalMac) + : undefined, + incrementalMacChunkSize: attachment.incrementalMacChunkSize, + fileName: attachment.fileName, + width: attachment.width, + height: attachment.height, + caption: attachment.caption, + blurHash: attachment.blurHash, + }); + + if (!isDownloadedToLocalFile(attachment)) { + // 1. If the attachment is undownloaded, we cannot trust its digest / mediaName. Thus, + // we only include a BackupLocator if this attachment already had one (e.g. we + // restored it from a backup and it had a BackupLocator then, which means we have at + // one point in the past verified the digest). + if ( + isDownloadableFromBackupTier(attachment) && + backupLevel === BackupLevel.Media + ) { + return new Backups.FilePointer({ + ...filePointerRootProps, + backupLocator: getBackupLocator(attachment), + }); + } + + // 2. Otherwise, we only return the transit CDN info via AttachmentLocator + if (isDownloadableFromTransitTier(attachment)) { + return new Backups.FilePointer({ + ...filePointerRootProps, + attachmentLocator: getAttachmentLocator(attachment), + }); + } + } + + if (backupLevel !== BackupLevel.Media) { + if (isDownloadableFromTransitTier(attachment)) { + return new Backups.FilePointer({ + ...filePointerRootProps, + attachmentLocator: getAttachmentLocator(attachment), + }); + } + return new Backups.FilePointer({ + ...filePointerRootProps, + invalidAttachmentLocator: getInvalidAttachmentLocator(), + }); + } + + if (!isDownloadedToLocalFile(attachment)) { + return new Backups.FilePointer({ + ...filePointerRootProps, + invalidAttachmentLocator: getInvalidAttachmentLocator(), + }); + } + + const attachmentForBackup = await fixupAttachmentForBackup(attachment); + const mediaName = getMediaNameForAttachment(attachmentForBackup); + + const backupTierInfo = getBackupTierInfo(mediaName); + let cdnNumberInBackupTier: number | undefined; + if (backupTierInfo.isInBackupTier) { + cdnNumberInBackupTier = backupTierInfo.cdnNumber; + } + + return new Backups.FilePointer({ + ...filePointerRootProps, + backupLocator: getBackupLocator({ + ...attachmentForBackup, + backupLocator: { + mediaName, + cdnNumber: cdnNumberInBackupTier, + }, + }), + }); +} + +export function getMediaNameForAttachment(attachment: AttachmentType): string { + strictAssert(attachment.digest, 'Digest must be present'); + return attachment.digest; +} + +// mediaId is special in that it is encoded in base64url +export function getBytesFromMediaId(mediaId: string): Uint8Array { + return Bytes.fromBase64url(mediaId); +} + +function getAttachmentLocator( + attachment: AttachmentDownloadableFromTransitTier +) { + return new Backups.FilePointer.AttachmentLocator({ + cdnKey: attachment.cdnKey, + cdnNumber: attachment.cdnNumber, + uploadTimestamp: attachment.uploadTimestamp + ? Long.fromNumber(attachment.uploadTimestamp) + : null, + digest: Bytes.fromBase64(attachment.digest), + key: Bytes.fromBase64(attachment.key), + size: attachment.size, + }); +} + +function getBackupLocator(attachment: AttachmentDownloadableFromBackupTier) { + return new Backups.FilePointer.BackupLocator({ + mediaName: attachment.backupLocator.mediaName, + cdnNumber: attachment.backupLocator.cdnNumber, + digest: Bytes.fromBase64(attachment.digest), + key: Bytes.fromBase64(attachment.key), + size: attachment.size, + transitCdnKey: attachment.cdnKey, + transitCdnNumber: attachment.cdnNumber, + }); +} + +function getInvalidAttachmentLocator() { + return new Backups.FilePointer.InvalidAttachmentLocator(); +} diff --git a/ts/test-electron/Crypto_test.ts b/ts/test-electron/Crypto_test.ts index 30952405a1b1..9585e47cbb4b 100644 --- a/ts/test-electron/Crypto_test.ts +++ b/ts/test-electron/Crypto_test.ts @@ -40,7 +40,7 @@ import { KEY_SET_LENGTH, _generateAttachmentIv, decryptAttachmentV2, - encryptAttachmentV2, + encryptAttachmentV2ToDisk, getAesCbcCiphertextLength, splitKeys, } from '../AttachmentCrypto'; @@ -607,7 +607,7 @@ describe('Crypto', () => { let ciphertextPath; try { - const encryptedAttachment = await encryptAttachmentV2({ + const encryptedAttachment = await encryptAttachmentV2ToDisk({ keys, plaintextAbsolutePath: FILE_PATH, }); @@ -655,7 +655,7 @@ describe('Crypto', () => { let ciphertextPath; try { - const encryptedAttachment = await encryptAttachmentV2({ + const encryptedAttachment = await encryptAttachmentV2ToDisk({ keys, plaintextAbsolutePath: sourcePath, }); @@ -700,7 +700,7 @@ describe('Crypto', () => { let ciphertextPath; try { - const encryptedAttachment = await encryptAttachmentV2({ + const encryptedAttachment = await encryptAttachmentV2ToDisk({ keys, plaintextAbsolutePath: FILE_PATH, }); @@ -753,7 +753,7 @@ describe('Crypto', () => { }); const ciphertextV1 = encryptedAttachmentV1.ciphertext; - const encryptedAttachmentV2 = await encryptAttachmentV2({ + const encryptedAttachmentV2 = await encryptAttachmentV2ToDisk({ keys, plaintextAbsolutePath: FILE_PATH, dangerousTestOnlyIv, @@ -788,7 +788,7 @@ describe('Crypto', () => { let outerCiphertextPath; let innerEncryptedAttachment; try { - innerEncryptedAttachment = await encryptAttachmentV2({ + innerEncryptedAttachment = await encryptAttachmentV2ToDisk({ keys: innerKeys, plaintextAbsolutePath, }); @@ -797,7 +797,7 @@ describe('Crypto', () => { innerEncryptedAttachment.path ); - const outerEncryptedAttachment = await encryptAttachmentV2({ + const outerEncryptedAttachment = await encryptAttachmentV2ToDisk({ keys: outerKeys, plaintextAbsolutePath: innerCiphertextPath, // We (and the server!) don't pad the second layer diff --git a/ts/test-electron/backup/filePointer_test.ts b/ts/test-electron/backup/filePointer_test.ts new file mode 100644 index 000000000000..0ee2df2e9904 --- /dev/null +++ b/ts/test-electron/backup/filePointer_test.ts @@ -0,0 +1,461 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only +import { assert } from 'chai'; +import Long from 'long'; +import { join } from 'path'; +import * as sinon from 'sinon'; +import { BackupLevel } from '@signalapp/libsignal-client/zkgroup'; +import { Backups } from '../../protobuf'; +import { + convertAttachmentToFilePointer, + convertFilePointerToAttachment, +} from '../../services/backups/util/filePointers'; +import { APPLICATION_OCTET_STREAM, IMAGE_PNG } from '../../types/MIME'; +import * as Bytes from '../../Bytes'; +import type { AttachmentType } from '../../types/Attachment'; +import { strictAssert } from '../../util/assert'; + +describe('convertFilePointerToAttachment', () => { + it('processes filepointer with attachmentLocator', () => { + const result = convertFilePointerToAttachment( + new Backups.FilePointer({ + contentType: 'image/png', + width: 100, + height: 100, + blurHash: 'blurhash', + fileName: 'filename', + caption: 'caption', + incrementalMac: Bytes.fromString('incrementalMac'), + incrementalMacChunkSize: 1000, + attachmentLocator: new Backups.FilePointer.AttachmentLocator({ + size: 128, + cdnKey: 'cdnKey', + cdnNumber: 2, + key: Bytes.fromString('key'), + digest: Bytes.fromString('digest'), + uploadTimestamp: Long.fromNumber(1970), + }), + }) + ); + + assert.deepStrictEqual(result, { + contentType: IMAGE_PNG, + width: 100, + height: 100, + size: 128, + blurHash: 'blurhash', + fileName: 'filename', + caption: 'caption', + cdnKey: 'cdnKey', + cdnNumber: 2, + key: Bytes.toBase64(Bytes.fromString('key')), + digest: Bytes.toBase64(Bytes.fromString('digest')), + uploadTimestamp: 1970, + incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')), + incrementalMacChunkSize: 1000, + }); + }); + + it('processes filepointer with backupLocator and missing fields', () => { + const result = convertFilePointerToAttachment( + new Backups.FilePointer({ + contentType: 'image/png', + width: 100, + height: 100, + blurHash: 'blurhash', + fileName: 'filename', + caption: 'caption', + incrementalMac: Bytes.fromString('incrementalMac'), + incrementalMacChunkSize: 1000, + backupLocator: new Backups.FilePointer.BackupLocator({ + mediaName: 'mediaName', + cdnNumber: 3, + size: 128, + key: Bytes.fromString('key'), + digest: Bytes.fromString('digest'), + transitCdnKey: 'transitCdnKey', + transitCdnNumber: 2, + }), + }) + ); + + assert.deepStrictEqual(result, { + contentType: IMAGE_PNG, + width: 100, + height: 100, + size: 128, + blurHash: 'blurhash', + fileName: 'filename', + caption: 'caption', + cdnKey: 'transitCdnKey', + cdnNumber: 2, + key: Bytes.toBase64(Bytes.fromString('key')), + digest: Bytes.toBase64(Bytes.fromString('digest')), + incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')), + incrementalMacChunkSize: 1000, + backupLocator: { + mediaName: 'mediaName', + cdnNumber: 3, + }, + }); + }); + + it('processes filepointer with invalidAttachmentLocator', () => { + const result = convertFilePointerToAttachment( + new Backups.FilePointer({ + contentType: 'image/png', + width: 100, + height: 100, + blurHash: 'blurhash', + fileName: 'filename', + caption: 'caption', + incrementalMac: Bytes.fromString('incrementalMac'), + incrementalMacChunkSize: 1000, + invalidAttachmentLocator: + new Backups.FilePointer.InvalidAttachmentLocator(), + }) + ); + + assert.deepStrictEqual(result, { + contentType: IMAGE_PNG, + width: 100, + height: 100, + blurHash: 'blurhash', + fileName: 'filename', + caption: 'caption', + incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')), + incrementalMacChunkSize: 1000, + size: 0, + error: true, + }); + }); + + it('accepts missing / null fields and adds defaults to contentType and size', () => { + const result = convertFilePointerToAttachment( + new Backups.FilePointer({ + backupLocator: new Backups.FilePointer.BackupLocator(), + }) + ); + + assert.deepStrictEqual(result, { + contentType: APPLICATION_OCTET_STREAM, + size: 0, + width: undefined, + height: undefined, + blurHash: undefined, + fileName: undefined, + caption: undefined, + cdnKey: undefined, + cdnNumber: undefined, + key: undefined, + digest: undefined, + incrementalMac: undefined, + incrementalMacChunkSize: undefined, + backupLocator: undefined, + }); + }); +}); + +function composeAttachment( + overrides: Partial = {} +): AttachmentType { + return { + size: 100, + contentType: IMAGE_PNG, + cdnKey: 'cdnKey', + cdnNumber: 2, + path: 'path/to/file.png', + key: 'key', + digest: 'digest', + width: 100, + height: 100, + blurHash: 'blurhash', + fileName: 'filename', + caption: 'caption', + incrementalMac: 'incrementalMac', + incrementalMacChunkSize: 1000, + uploadTimestamp: 1234, + ...overrides, + }; +} + +const defaultFilePointer = new Backups.FilePointer({ + contentType: IMAGE_PNG, + width: 100, + height: 100, + blurHash: 'blurhash', + fileName: 'filename', + caption: 'caption', + incrementalMac: Bytes.fromBase64('incrementalMac'), + incrementalMacChunkSize: 1000, +}); + +const defaultAttachmentLocator = new Backups.FilePointer.AttachmentLocator({ + cdnKey: 'cdnKey', + cdnNumber: 2, + key: Bytes.fromBase64('key'), + digest: Bytes.fromBase64('digest'), + size: 100, + uploadTimestamp: Long.fromNumber(1234), +}); + +const defaultMediaName = 'digest'; +const defaultBackupLocator = new Backups.FilePointer.BackupLocator({ + mediaName: defaultMediaName, + cdnNumber: null, + key: Bytes.fromBase64('key'), + digest: Bytes.fromBase64('digest'), + size: 100, + transitCdnKey: 'cdnKey', + transitCdnNumber: 2, +}); + +const filePointerWithAttachmentLocator = new Backups.FilePointer({ + ...defaultFilePointer, + attachmentLocator: defaultAttachmentLocator, +}); + +const filePointerWithBackupLocator = new Backups.FilePointer({ + ...defaultFilePointer, + backupLocator: defaultBackupLocator, +}); +const filePointerWithInvalidLocator = new Backups.FilePointer({ + ...defaultFilePointer, + invalidAttachmentLocator: new Backups.FilePointer.InvalidAttachmentLocator(), +}); + +async function testAttachmentToFilePointer( + attachment: AttachmentType, + filePointer: Backups.FilePointer, + options?: { backupLevel?: BackupLevel; backupCdnNumber?: number } +) { + async function _doTest(withBackupLevel: BackupLevel) { + assert.deepStrictEqual( + await convertAttachmentToFilePointer({ + attachment, + backupLevel: withBackupLevel, + getBackupTierInfo: _mediaName => { + if (options?.backupCdnNumber != null) { + return { isInBackupTier: true, cdnNumber: options.backupCdnNumber }; + } + return { isInBackupTier: false }; + }, + }), + filePointer + ); + } + + if (!options?.backupLevel) { + await _doTest(BackupLevel.Messages); + await _doTest(BackupLevel.Media); + } else { + await _doTest(options.backupLevel); + } +} + +describe('convertAttachmentToFilePointer', () => { + describe('not downloaded locally', () => { + const undownloadedAttachment = composeAttachment({ path: undefined }); + it('returns invalidAttachmentLocator if missing critical decryption info', async () => { + await testAttachmentToFilePointer( + { + ...undownloadedAttachment, + key: undefined, + }, + filePointerWithInvalidLocator + ); + await testAttachmentToFilePointer( + { + ...undownloadedAttachment, + digest: undefined, + }, + filePointerWithInvalidLocator + ); + }); + describe('attachment does not have attachment.backupLocator', () => { + it('returns attachmentLocator, regardless of backupLevel or backup tier status', async () => { + await testAttachmentToFilePointer( + undownloadedAttachment, + filePointerWithAttachmentLocator, + { backupCdnNumber: 3 } + ); + }); + + it('returns invalidAttachmentLocator if missing critical locator info', async () => { + await testAttachmentToFilePointer( + { + ...undownloadedAttachment, + cdnKey: undefined, + }, + filePointerWithInvalidLocator + ); + await testAttachmentToFilePointer( + { + ...undownloadedAttachment, + cdnNumber: undefined, + }, + filePointerWithInvalidLocator + ); + }); + }); + describe('attachment has attachment.backupLocator', () => { + const undownloadedAttachmentWithBackupLocator = { + ...undownloadedAttachment, + backupLocator: { mediaName: defaultMediaName }, + }; + + it('returns backupLocator if backupLevel is Media', async () => { + await testAttachmentToFilePointer( + undownloadedAttachmentWithBackupLocator, + filePointerWithBackupLocator, + { backupLevel: BackupLevel.Media } + ); + }); + + it('returns backupLocator even if missing transit CDN info', async () => { + // Even if missing transit CDNKey + await testAttachmentToFilePointer( + { ...undownloadedAttachmentWithBackupLocator, cdnKey: undefined }, + new Backups.FilePointer({ + ...filePointerWithBackupLocator, + backupLocator: new Backups.FilePointer.BackupLocator({ + ...defaultBackupLocator, + transitCdnKey: undefined, + }), + }), + { backupLevel: BackupLevel.Media } + ); + }); + + it('returns attachmentLocator if backupLevel is Messages', async () => { + await testAttachmentToFilePointer( + undownloadedAttachmentWithBackupLocator, + filePointerWithAttachmentLocator, + { backupLevel: BackupLevel.Messages } + ); + }); + }); + }); + describe('downloaded locally', () => { + const downloadedAttachment = composeAttachment(); + describe('BackupLevel.Messages', () => { + it('returns attachmentLocator', async () => { + await testAttachmentToFilePointer( + downloadedAttachment, + filePointerWithAttachmentLocator, + { backupLevel: BackupLevel.Messages } + ); + }); + it('returns invalidAttachmentLocator if missing critical locator info', async () => { + await testAttachmentToFilePointer( + { + ...downloadedAttachment, + cdnKey: undefined, + }, + filePointerWithInvalidLocator, + { backupLevel: BackupLevel.Messages } + ); + await testAttachmentToFilePointer( + { + ...downloadedAttachment, + cdnNumber: undefined, + }, + filePointerWithInvalidLocator, + { backupLevel: BackupLevel.Messages } + ); + }); + it('returns invalidAttachmentLocator if missing critical decryption info', async () => { + await testAttachmentToFilePointer( + { + ...downloadedAttachment, + key: undefined, + }, + filePointerWithInvalidLocator, + { backupLevel: BackupLevel.Messages } + ); + await testAttachmentToFilePointer( + { + ...downloadedAttachment, + digest: undefined, + }, + filePointerWithInvalidLocator, + { backupLevel: BackupLevel.Messages } + ); + }); + }); + describe('BackupLevel.Media', () => { + describe('if missing critical decryption info', () => { + const FILE_PATH = join(__dirname, '../../../fixtures/ghost-kitty.mp4'); + + let sandbox: sinon.SinonSandbox; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + sandbox + .stub(window.Signal.Migrations, 'getAbsoluteAttachmentPath') + .callsFake(relPath => { + if (relPath === downloadedAttachment.path) { + return FILE_PATH; + } + return relPath; + }); + }); + + afterEach(() => { + sandbox.restore(); + }); + + it('generates new key & digest and removes existing CDN info', async () => { + const result = await convertAttachmentToFilePointer({ + attachment: { + ...downloadedAttachment, + key: undefined, + }, + backupLevel: BackupLevel.Media, + getBackupTierInfo: () => ({ isInBackupTier: false }), + }); + const newKey = result.backupLocator?.key; + const newDigest = result.backupLocator?.digest; + + strictAssert(newDigest, 'must create new digest'); + assert.deepStrictEqual( + result, + new Backups.FilePointer({ + ...filePointerWithBackupLocator, + backupLocator: new Backups.FilePointer.BackupLocator({ + ...defaultBackupLocator, + key: newKey, + digest: newDigest, + mediaName: Bytes.toBase64(newDigest), + transitCdnKey: undefined, + transitCdnNumber: undefined, + }), + }) + ); + }); + }); + + it('returns BackupLocator, with cdnNumber if in backup tier already', async () => { + await testAttachmentToFilePointer( + downloadedAttachment, + new Backups.FilePointer({ + ...filePointerWithBackupLocator, + backupLocator: new Backups.FilePointer.BackupLocator({ + ...defaultBackupLocator, + cdnNumber: 12, + }), + }), + { backupLevel: BackupLevel.Media, backupCdnNumber: 12 } + ); + }); + + it('returns BackupLocator, with empty cdnNumber if not in backup tier', async () => { + await testAttachmentToFilePointer( + downloadedAttachment, + filePointerWithBackupLocator, + { backupLevel: BackupLevel.Media } + ); + }); + }); + }); +}); diff --git a/ts/test-node/backups/filePointer_test.ts b/ts/test-node/backups/filePointer_test.ts deleted file mode 100644 index c19c596564cb..000000000000 --- a/ts/test-node/backups/filePointer_test.ts +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2024 Signal Messenger, LLC -// SPDX-License-Identifier: AGPL-3.0-only -import { assert } from 'chai'; -import Long from 'long'; -import { Backups } from '../../protobuf'; -import { convertFilePointerToAttachment } from '../../services/backups/util/filePointers'; -import { APPLICATION_OCTET_STREAM, IMAGE_PNG } from '../../types/MIME'; -import * as Bytes from '../../Bytes'; - -describe('convertFilePointerToAttachment', () => { - it('processes filepointer with attachmentLocator', () => { - const result = convertFilePointerToAttachment( - new Backups.FilePointer({ - contentType: 'image/png', - width: 100, - height: 100, - blurHash: 'blurhash', - fileName: 'filename', - caption: 'caption', - incrementalMac: Bytes.fromString('incrementalMac'), - incrementalMacChunkSize: 1000, - attachmentLocator: new Backups.FilePointer.AttachmentLocator({ - size: 128, - cdnKey: 'cdnKey', - cdnNumber: 2, - key: Bytes.fromString('key'), - digest: Bytes.fromString('digest'), - uploadTimestamp: Long.fromNumber(1970), - }), - }) - ); - - assert.deepStrictEqual(result, { - contentType: IMAGE_PNG, - width: 100, - height: 100, - size: 128, - blurHash: 'blurhash', - fileName: 'filename', - caption: 'caption', - cdnKey: 'cdnKey', - cdnNumber: 2, - key: Bytes.toBase64(Bytes.fromString('key')), - digest: Bytes.toBase64(Bytes.fromString('digest')), - uploadTimestamp: 1970, - incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')), - incrementalMacChunkSize: 1000, - }); - }); - - it('processes filepointer with backupLocator and missing fields', () => { - const result = convertFilePointerToAttachment( - new Backups.FilePointer({ - contentType: 'image/png', - width: 100, - height: 100, - blurHash: 'blurhash', - fileName: 'filename', - caption: 'caption', - incrementalMac: Bytes.fromString('incrementalMac'), - incrementalMacChunkSize: 1000, - backupLocator: new Backups.FilePointer.BackupLocator({ - mediaName: 'mediaName', - cdnNumber: 3, - size: 128, - key: Bytes.fromString('key'), - digest: Bytes.fromString('digest'), - transitCdnKey: 'transitCdnKey', - transitCdnNumber: 2, - }), - }) - ); - - assert.deepStrictEqual(result, { - contentType: IMAGE_PNG, - width: 100, - height: 100, - size: 128, - blurHash: 'blurhash', - fileName: 'filename', - caption: 'caption', - cdnKey: 'transitCdnKey', - cdnNumber: 2, - key: Bytes.toBase64(Bytes.fromString('key')), - digest: Bytes.toBase64(Bytes.fromString('digest')), - incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')), - incrementalMacChunkSize: 1000, - backupLocator: { - mediaName: 'mediaName', - cdnNumber: 3, - }, - }); - }); - - it('processes filepointer with invalidAttachmentLocator', () => { - const result = convertFilePointerToAttachment( - new Backups.FilePointer({ - contentType: 'image/png', - width: 100, - height: 100, - blurHash: 'blurhash', - fileName: 'filename', - caption: 'caption', - incrementalMac: Bytes.fromString('incrementalMac'), - incrementalMacChunkSize: 1000, - invalidAttachmentLocator: - new Backups.FilePointer.InvalidAttachmentLocator(), - }) - ); - - assert.deepStrictEqual(result, { - contentType: IMAGE_PNG, - width: 100, - height: 100, - blurHash: 'blurhash', - fileName: 'filename', - caption: 'caption', - incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')), - incrementalMacChunkSize: 1000, - size: 0, - error: true, - }); - }); - - it('accepts missing / null fields and adds defaults to contentType and size', () => { - const result = convertFilePointerToAttachment( - new Backups.FilePointer({ - backupLocator: new Backups.FilePointer.BackupLocator(), - }) - ); - - assert.deepStrictEqual(result, { - contentType: APPLICATION_OCTET_STREAM, - size: 0, - width: undefined, - height: undefined, - blurHash: undefined, - fileName: undefined, - caption: undefined, - cdnKey: undefined, - cdnNumber: undefined, - key: undefined, - digest: undefined, - incrementalMac: undefined, - incrementalMacChunkSize: undefined, - backupLocator: undefined, - }); - }); -}); diff --git a/ts/types/Attachment.ts b/ts/types/Attachment.ts index 57702adf7a4f..c06696e2d64e 100644 --- a/ts/types/Attachment.ts +++ b/ts/types/Attachment.ts @@ -21,7 +21,7 @@ import { isImageTypeSupported, isVideoTypeSupported, } from '../util/GoogleChrome'; -import type { LocalizerType } from './Util'; +import type { LocalizerType, WithRequiredProperties } from './Util'; import { ThemeType } from './Util'; import * as GoogleChrome from '../util/GoogleChrome'; import { ReadStatus } from '../messages/MessageReadStatus'; @@ -994,3 +994,66 @@ export function getAttachmentSignature(attachment: AttachmentType): string { strictAssert(attachment.digest, 'attachment missing digest'); return attachment.digest; } + +type RequiredPropertiesForDecryption = 'key' | 'digest'; + +type DecryptableAttachment = WithRequiredProperties< + AttachmentType, + RequiredPropertiesForDecryption +>; + +export type AttachmentDownloadableFromTransitTier = WithRequiredProperties< + DecryptableAttachment, + 'cdnKey' | 'cdnNumber' +>; + +export type AttachmentDownloadableFromBackupTier = WithRequiredProperties< + DecryptableAttachment, + 'backupLocator' +>; + +export type DownloadedAttachment = WithRequiredProperties< + AttachmentType, + 'path' +>; + +export type AttachmentReadyForBackup = WithRequiredProperties< + DownloadedAttachment, + RequiredPropertiesForDecryption +>; + +function isDecryptable( + attachment: AttachmentType +): attachment is DecryptableAttachment { + return Boolean(attachment.key) && Boolean(attachment.digest); +} + +export function isDownloadableFromTransitTier( + attachment: AttachmentType +): attachment is AttachmentDownloadableFromTransitTier { + if (!isDecryptable(attachment)) { + return false; + } + if (attachment.cdnKey && attachment.cdnNumber) { + return true; + } + return false; +} + +export function isDownloadableFromBackupTier( + attachment: AttachmentType +): attachment is AttachmentDownloadableFromBackupTier { + if (!attachment.key || !attachment.digest) { + return false; + } + if (attachment.backupLocator?.mediaName) { + return true; + } + return false; +} + +export function isDownloadedToLocalFile( + attachment: AttachmentType +): attachment is DownloadedAttachment { + return Boolean(attachment.path); +}