Convert attachments to filePointers for backup export

ts/AttachmentCrypto.ts

@@ -11,7 +11,7 @@ import {
   randomBytes,
 } from 'crypto';
 import type { Decipher, Hash, Hmac } from 'crypto';
-import { Transform } from 'stream';
+import { PassThrough, Transform, type Writable } from 'stream';
 import { pipeline } from 'stream/promises';
 import { ensureFile } from 'fs-extra';
 import * as log from './logging/log';
@@ -45,7 +45,6 @@ export function _generateAttachmentIv(): Uint8Array {
 }
 
 export type EncryptedAttachmentV2 = {
-  path: string;
   digest: Uint8Array;
   plaintextHash: string;
 };
@@ -55,24 +54,55 @@ export type DecryptedAttachmentV2 = {
   plaintextHash: string;
 };
 
+type EncryptAttachmentV2PropsType = {
+  keys: Readonly<Uint8Array>;
+  plaintextAbsolutePath: string;
+  dangerousTestOnlyIv?: Readonly<Uint8Array>;
+  dangerousTestOnlySkipPadding?: boolean;
+};
+
+export async function encryptAttachmentV2ToDisk(
+  args: EncryptAttachmentV2PropsType
+): Promise<EncryptedAttachmentV2 & { path: string }> {
+  // Create random output file
+  const relativeTargetPath = getRelativePath(createName());
+  const absoluteTargetPath =
+    window.Signal.Migrations.getAbsoluteAttachmentPath(relativeTargetPath);
+
+  let writeFd;
+  let encryptResult: EncryptedAttachmentV2;
+
+  try {
+    await ensureFile(absoluteTargetPath);
+    writeFd = await open(absoluteTargetPath, 'w');
+    encryptResult = await encryptAttachmentV2({
+      ...args,
+      sink: writeFd.createWriteStream(),
+    });
+  } catch (error) {
+    safeUnlinkSync(absoluteTargetPath);
+    throw error;
+  } finally {
+    await writeFd?.close();
+  }
+
+  return {
+    ...encryptResult,
+    path: relativeTargetPath,
+  };
+}
+
 export async function encryptAttachmentV2({
   keys,
   plaintextAbsolutePath,
   dangerousTestOnlyIv,
   dangerousTestOnlySkipPadding = false,
-}: {
+  sink,
-  keys: Readonly<Uint8Array>;
+}: EncryptAttachmentV2PropsType & {
-  plaintextAbsolutePath: string;
+  sink?: Writable;
-  dangerousTestOnlyIv?: Readonly<Uint8Array>;
-  dangerousTestOnlySkipPadding?: boolean;
 }): Promise<EncryptedAttachmentV2> {
   const logId = 'encryptAttachmentV2';
 
-  // Create random output file
-  const relativeTargetPath = getRelativePath(createName());
-  const absoluteTargetPath =
-    window.Signal.Migrations.getAbsoluteAttachmentPath(relativeTargetPath);
-
   const { aesKey, macKey } = splitKeys(keys);
 
   if (dangerousTestOnlyIv && window.getEnvironment() !== Environment.Test) {
@@ -92,19 +122,12 @@ export async function encryptAttachmentV2({
   const digest = createHash(HashType.size256);
 
   let readFd;
-  let writeFd;
   try {
     try {
       readFd = await open(plaintextAbsolutePath, 'r');
     } catch (cause) {
       throw new Error(`${logId}: Read path doesn't exist`, { cause });
     }
-    try {
-      await ensureFile(absoluteTargetPath);
-      writeFd = await open(absoluteTargetPath, 'w');
-    } catch (cause) {
-      throw new Error(`${logId}: Failed to create write path`, { cause });
-    }
 
     await pipeline(
       [
@@ -115,7 +138,7 @@ export async function encryptAttachmentV2({
         prependIv(iv),
         appendMacStream(macKey),
         peekAndUpdateHash(digest),
-        writeFd.createWriteStream(),
+        sink ?? new PassThrough().resume(),
       ].filter(isNotNil)
     );
   } catch (error) {
@@ -123,10 +146,9 @@ export async function encryptAttachmentV2({
       `${logId}: Failed to encrypt attachment`,
       Errors.toLogFormat(error)
     );
-    safeUnlinkSync(absoluteTargetPath);
     throw error;
   } finally {
-    await Promise.all([readFd?.close(), writeFd?.close()]);
+    await readFd?.close();
   }
 
   const ourPlaintextHash = plaintextHash.digest('hex');
@@ -143,7 +165,6 @@ export async function encryptAttachmentV2({
   );
 
   return {
-    path: relativeTargetPath,
     digest: ourDigest,
     plaintextHash: ourPlaintextHash,
   };

ts/Bytes.ts

@@ -8,6 +8,9 @@ const bytes = globalThis.window?.SignalContext?.bytes || new Bytes();
 export function fromBase64(value: string): Uint8Array {
   return bytes.fromBase64(value);
 }
+export function fromBase64url(value: string): Uint8Array {
+  return bytes.fromBase64url(value);
+}
 
 export function fromHex(value: string): Uint8Array {
   return bytes.fromHex(value);

ts/context/Bytes.ts

@@ -8,6 +8,10 @@ export class Bytes {
     return Buffer.from(value, 'base64');
   }
 
+  public fromBase64url(value: string): Uint8Array {
+    return Buffer.from(value, 'base64url');
+  }
+
   public fromHex(value: string): Uint8Array {
     return Buffer.from(value, 'hex');
   }

ts/services/backups/export.ts

@@ -3,6 +3,7 @@
 
 import Long from 'long';
 import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client';
+import type { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
 import pMap from 'p-map';
 import pTimeout from 'p-timeout';
 import { Readable } from 'stream';
@@ -82,6 +83,13 @@ import {
   numberToAddressType,
   numberToPhoneType,
 } from '../../types/EmbeddedContact';
+import {
+  isVoiceMessage,
+  type AttachmentType,
+  isGIF,
+  isDownloaded,
+} from '../../types/Attachment';
+import { convertAttachmentToFilePointer } from './util/filePointers';
 
 const MAX_CONCURRENCY = 10;
 
@@ -116,14 +124,13 @@ export class BackupExportStream extends Readable {
   private nextRecipientId = 0;
   private flushResolve: (() => void) | undefined;
 
-  public run(): void {
+  public run(backupLevel: BackupLevel): void {
     drop(
       (async () => {
         log.info('BackupExportStream: starting...');
         await Data.pauseWriteAccess();
-
         try {
-          await this.unsafeRun();
+          await this.unsafeRun(backupLevel);
         } catch (error) {
           this.emit('error', error);
         } finally {
@@ -134,7 +141,7 @@ export class BackupExportStream extends Readable {
     );
   }
 
-  private async unsafeRun(): Promise<void> {
+  private async unsafeRun(backupLevel: BackupLevel): Promise<void> {
     this.push(
       Backups.BackupInfo.encodeDelimited({
         version: Long.fromNumber(BACKUP_VERSION),
@@ -279,7 +286,12 @@ export class BackupExportStream extends Readable {
         // eslint-disable-next-line no-await-in-loop
         const items = await pMap(
           messages,
-          message => this.toChatItem(message, { aboutMe, callHistoryByCallId }),
+          message =>
+            this.toChatItem(message, {
+              aboutMe,
+              callHistoryByCallId,
+              backupLevel,
+            }),
           { concurrency: MAX_CONCURRENCY }
         );
 
@@ -308,6 +320,7 @@ export class BackupExportStream extends Readable {
     }
 
     await this.flush();
+
     log.warn('backups: final stats', stats);
 
     this.push(null);
@@ -562,6 +575,7 @@ export class BackupExportStream extends Readable {
     options: {
       aboutMe: AboutMe;
       callHistoryByCallId: Record<string, CallHistoryDetails>;
+      backupLevel: BackupLevel;
     }
   ): Promise<Backups.IChatItem | undefined> {
     const chatId = this.getRecipientId({ id: message.conversationId });
@@ -629,6 +643,16 @@ export class BackupExportStream extends Readable {
     // TODO (DESKTOP-6964): put incoming/outgoing fields below onto non-bubble messages
     result.standardMessage = {
       quote: await this.toQuote(message.quote),
+      attachments: message.attachments
+        ? await Promise.all(
+            message.attachments.map(attachment => {
+              return this.processMessageAttachment({
+                attachment,
+                backupLevel: options.backupLevel,
+              });
+            })
+          )
+        : undefined,
       text: {
         // Note that we store full text on the message model so we have to
         // trim it before serializing.
@@ -1579,6 +1603,63 @@ export class BackupExportStream extends Readable {
           }),
     };
   }
+
+  private getMessageAttachmentFlag(
+    attachment: AttachmentType
+  ): Backups.MessageAttachment.Flag {
+    if (isVoiceMessage(attachment)) {
+      return Backups.MessageAttachment.Flag.VOICE_MESSAGE;
+    }
+    if (isGIF([attachment])) {
+      return Backups.MessageAttachment.Flag.GIF;
+    }
+    if (
+      attachment.flags &&
+      // eslint-disable-next-line no-bitwise
+      attachment.flags & SignalService.AttachmentPointer.Flags.BORDERLESS
+    ) {
+      return Backups.MessageAttachment.Flag.BORDERLESS;
+    }
+
+    return Backups.MessageAttachment.Flag.NONE;
+  }
+
+  private async processMessageAttachment({
+    attachment,
+    backupLevel,
+  }: {
+    attachment: AttachmentType;
+    backupLevel: BackupLevel;
+  }): Promise<Backups.MessageAttachment> {
+    const filePointer = await this.processAttachment({
+      attachment,
+      backupLevel,
+    });
+
+    return new Backups.MessageAttachment({
+      pointer: filePointer,
+      flag: this.getMessageAttachmentFlag(attachment),
+      wasDownloaded: isDownloaded(attachment), // should always be true
+    });
+  }
+
+  private async processAttachment({
+    attachment,
+    backupLevel,
+  }: {
+    attachment: AttachmentType;
+    backupLevel: BackupLevel;
+  }): Promise<Backups.FilePointer> {
+    const filePointer = await convertAttachmentToFilePointer({
+      attachment,
+      backupLevel,
+      // TODO (DESKTOP-6983) -- Retrieve & save backup tier media list
+      getBackupTierInfo: () => ({
+        isInBackupTier: false,
+      }),
+    });
+    return filePointer;
+  }
 }
 
 function checkServiceIdEquivalence(

ts/services/backups/index.ts

@@ -10,6 +10,7 @@ import { join } from 'path';
 import { createGzip, createGunzip } from 'zlib';
 import { createCipheriv, createHmac, randomBytes } from 'crypto';
 import { noop } from 'lodash';
+import { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
 
 import * as log from '../../logging/log';
 import * as Bytes from '../../Bytes';
@@ -68,8 +69,11 @@ export class BackupsService {
     const fileName = `backup-${randomBytes(32).toString('hex')}`;
     const filePath = join(window.BasePaths.temp, fileName);
 
+    const backupLevel = await this.credentials.getBackupLevel();
+    log.info(`exportBackup: starting, backup level: ${backupLevel}...`);
+
     try {
-      const fileSize = await this.exportToDisk(filePath);
+      const fileSize = await this.exportToDisk(filePath, backupLevel);
 
       await this.api.upload(filePath, fileSize);
     } finally {
@@ -82,19 +86,24 @@ export class BackupsService {
   }
 
   // Test harness
-  public async exportBackupData(): Promise<Uint8Array> {
+  public async exportBackupData(
+    backupLevel: BackupLevel = BackupLevel.Messages
+  ): Promise<Uint8Array> {
     const sink = new PassThrough();
 
     const chunks = new Array<Uint8Array>();
     sink.on('data', chunk => chunks.push(chunk));
-    await this.exportBackup(sink);
+    await this.exportBackup(sink, backupLevel);
 
     return Bytes.concatenate(chunks);
   }
 
   // Test harness
-  public async exportToDisk(path: string): Promise<number> {
+  public async exportToDisk(
-    const size = await this.exportBackup(createWriteStream(path));
+    path: string,
+    backupLevel: BackupLevel = BackupLevel.Messages
+  ): Promise<number> {
+    const size = await this.exportBackup(createWriteStream(path), backupLevel);
 
     await validateBackup(path, size);
 
@@ -174,7 +183,10 @@ export class BackupsService {
     }
   }
 
-  private async exportBackup(sink: Writable): Promise<number> {
+  private async exportBackup(
+    sink: Writable,
+    backupLevel: BackupLevel = BackupLevel.Messages
+  ): Promise<number> {
     strictAssert(!this.isRunning, 'BackupService is already running');
 
     log.info('exportBackup: starting...');
@@ -184,7 +196,7 @@ export class BackupsService {
       const { aesKey, macKey } = getKeyMaterial();
 
       const recordStream = new BackupExportStream();
-      recordStream.run();
+      recordStream.run(backupLevel);
 
       const iv = randomBytes(IV_LENGTH);
 

ts/services/backups/util/filePointers.ts

@@ -1,13 +1,28 @@
 // Copyright 2024 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
+import Long from 'long';
+import { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
+
 import {
   APPLICATION_OCTET_STREAM,
   stringToMIMEType,
 } from '../../../types/MIME';
-import type { AttachmentType } from '../../../types/Attachment';
+import {
-import type { Backups } from '../../../protobuf';
+  type AttachmentType,
+  isDownloadableFromTransitTier,
+  isDownloadableFromBackupTier,
+  isDownloadedToLocalFile,
+  type AttachmentDownloadableFromTransitTier,
+  type AttachmentDownloadableFromBackupTier,
+  type DownloadedAttachment,
+  type AttachmentReadyForBackup,
+} from '../../../types/Attachment';
+import { Backups } from '../../../protobuf';
 import * as Bytes from '../../../Bytes';
 import { getTimestampFromLong } from '../../../util/timestampLongUtils';
+import { getRandomBytes } from '../../../Crypto';
+import { encryptAttachmentV2 } from '../../../AttachmentCrypto';
+import { strictAssert } from '../../../util/assert';
 
 export function convertFilePointerToAttachment(
   filePointer: Backups.FilePointer
@@ -94,3 +109,174 @@ export function convertFilePointerToAttachment(
 
   throw new Error('convertFilePointerToAttachment: mising locator');
 }
+
+/**
+ * Some attachments saved on desktop do not include the key used to encrypt the file
+ * originally. This means that we need to encrypt the file in-memory now (at
+ * export-creation time) to calculate the digest which will be saved in the backup proto
+ * along with the new keys.
+ */
+async function fixupAttachmentForBackup(
+  attachment: DownloadedAttachment
+): Promise<AttachmentReadyForBackup> {
+  const fixedUpAttachment = { ...attachment };
+  const keyToUse = attachment.key ?? Bytes.toBase64(getRandomBytes(64));
+  let digestToUse = attachment.key ? attachment.digest : undefined;
+
+  if (!digestToUse) {
+    // Delete current locators for the attachment; we can no longer use them and will need
+    // to fully re-encrypt and upload
+    delete fixedUpAttachment.cdnId;
+    delete fixedUpAttachment.cdnKey;
+    delete fixedUpAttachment.cdnNumber;
+
+    // encrypt this file in memory in order to calculate the digest
+    const { digest } = await encryptAttachmentV2({
+      keys: Bytes.fromBase64(keyToUse),
+      plaintextAbsolutePath: window.Signal.Migrations.getAbsoluteAttachmentPath(
+        attachment.path
+      ),
+    });
+
+    digestToUse = Bytes.toBase64(digest);
+
+    // TODO (DESKTOP-6688): ensure that we update the message/attachment in DB with the
+    // new keys so that we don't try to re-upload it again on the next export
+  }
+
+  return {
+    ...fixedUpAttachment,
+    key: keyToUse,
+    digest: digestToUse,
+  };
+}
+
+export async function convertAttachmentToFilePointer({
+  attachment,
+  backupLevel,
+  getBackupTierInfo,
+}: {
+  attachment: AttachmentType;
+  backupLevel: BackupLevel;
+  getBackupTierInfo: (
+    mediaName: string
+  ) => { isInBackupTier: true; cdnNumber: number } | { isInBackupTier: false };
+}): Promise<Backups.FilePointer> {
+  const filePointerRootProps = new Backups.FilePointer({
+    contentType: attachment.contentType,
+    incrementalMac: attachment.incrementalMac
+      ? Bytes.fromBase64(attachment.incrementalMac)
+      : undefined,
+    incrementalMacChunkSize: attachment.incrementalMacChunkSize,
+    fileName: attachment.fileName,
+    width: attachment.width,
+    height: attachment.height,
+    caption: attachment.caption,
+    blurHash: attachment.blurHash,
+  });
+
+  if (!isDownloadedToLocalFile(attachment)) {
+    // 1. If the attachment is undownloaded, we cannot trust its digest / mediaName. Thus,
+    // we only include a BackupLocator if this attachment already had one (e.g. we
+    // restored it from a backup and it had a BackupLocator then, which means we have at
+    // one point in the past verified the digest).
+    if (
+      isDownloadableFromBackupTier(attachment) &&
+      backupLevel === BackupLevel.Media
+    ) {
+      return new Backups.FilePointer({
+        ...filePointerRootProps,
+        backupLocator: getBackupLocator(attachment),
+      });
+    }
+
+    // 2. Otherwise, we only return the transit CDN info via AttachmentLocator
+    if (isDownloadableFromTransitTier(attachment)) {
+      return new Backups.FilePointer({
+        ...filePointerRootProps,
+        attachmentLocator: getAttachmentLocator(attachment),
+      });
+    }
+  }
+
+  if (backupLevel !== BackupLevel.Media) {
+    if (isDownloadableFromTransitTier(attachment)) {
+      return new Backups.FilePointer({
+        ...filePointerRootProps,
+        attachmentLocator: getAttachmentLocator(attachment),
+      });
+    }
+    return new Backups.FilePointer({
+      ...filePointerRootProps,
+      invalidAttachmentLocator: getInvalidAttachmentLocator(),
+    });
+  }
+
+  if (!isDownloadedToLocalFile(attachment)) {
+    return new Backups.FilePointer({
+      ...filePointerRootProps,
+      invalidAttachmentLocator: getInvalidAttachmentLocator(),
+    });
+  }
+
+  const attachmentForBackup = await fixupAttachmentForBackup(attachment);
+  const mediaName = getMediaNameForAttachment(attachmentForBackup);
+
+  const backupTierInfo = getBackupTierInfo(mediaName);
+  let cdnNumberInBackupTier: number | undefined;
+  if (backupTierInfo.isInBackupTier) {
+    cdnNumberInBackupTier = backupTierInfo.cdnNumber;
+  }
+
+  return new Backups.FilePointer({
+    ...filePointerRootProps,
+    backupLocator: getBackupLocator({
+      ...attachmentForBackup,
+      backupLocator: {
+        mediaName,
+        cdnNumber: cdnNumberInBackupTier,
+      },
+    }),
+  });
+}
+
+export function getMediaNameForAttachment(attachment: AttachmentType): string {
+  strictAssert(attachment.digest, 'Digest must be present');
+  return attachment.digest;
+}
+
+// mediaId is special in that it is encoded in base64url
+export function getBytesFromMediaId(mediaId: string): Uint8Array {
+  return Bytes.fromBase64url(mediaId);
+}
+
+function getAttachmentLocator(
+  attachment: AttachmentDownloadableFromTransitTier
+) {
+  return new Backups.FilePointer.AttachmentLocator({
+    cdnKey: attachment.cdnKey,
+    cdnNumber: attachment.cdnNumber,
+    uploadTimestamp: attachment.uploadTimestamp
+      ? Long.fromNumber(attachment.uploadTimestamp)
+      : null,
+    digest: Bytes.fromBase64(attachment.digest),
+    key: Bytes.fromBase64(attachment.key),
+    size: attachment.size,
+  });
+}
+
+function getBackupLocator(attachment: AttachmentDownloadableFromBackupTier) {
+  return new Backups.FilePointer.BackupLocator({
+    mediaName: attachment.backupLocator.mediaName,
+    cdnNumber: attachment.backupLocator.cdnNumber,
+    digest: Bytes.fromBase64(attachment.digest),
+    key: Bytes.fromBase64(attachment.key),
+    size: attachment.size,
+    transitCdnKey: attachment.cdnKey,
+    transitCdnNumber: attachment.cdnNumber,
+  });
+}
+
+function getInvalidAttachmentLocator() {
+  return new Backups.FilePointer.InvalidAttachmentLocator();
+}

ts/test-electron/Crypto_test.ts

@@ -40,7 +40,7 @@ import {
   KEY_SET_LENGTH,
   _generateAttachmentIv,
   decryptAttachmentV2,
-  encryptAttachmentV2,
+  encryptAttachmentV2ToDisk,
   getAesCbcCiphertextLength,
   splitKeys,
 } from '../AttachmentCrypto';
@@ -607,7 +607,7 @@ describe('Crypto', () => {
       let ciphertextPath;
 
       try {
-        const encryptedAttachment = await encryptAttachmentV2({
+        const encryptedAttachment = await encryptAttachmentV2ToDisk({
           keys,
           plaintextAbsolutePath: FILE_PATH,
         });
@@ -655,7 +655,7 @@ describe('Crypto', () => {
       let ciphertextPath;
 
       try {
-        const encryptedAttachment = await encryptAttachmentV2({
+        const encryptedAttachment = await encryptAttachmentV2ToDisk({
           keys,
           plaintextAbsolutePath: sourcePath,
         });
@@ -700,7 +700,7 @@ describe('Crypto', () => {
       let ciphertextPath;
 
       try {
-        const encryptedAttachment = await encryptAttachmentV2({
+        const encryptedAttachment = await encryptAttachmentV2ToDisk({
           keys,
           plaintextAbsolutePath: FILE_PATH,
         });
@@ -753,7 +753,7 @@ describe('Crypto', () => {
         });
         const ciphertextV1 = encryptedAttachmentV1.ciphertext;
 
-        const encryptedAttachmentV2 = await encryptAttachmentV2({
+        const encryptedAttachmentV2 = await encryptAttachmentV2ToDisk({
           keys,
           plaintextAbsolutePath: FILE_PATH,
           dangerousTestOnlyIv,
@@ -788,7 +788,7 @@ describe('Crypto', () => {
         let outerCiphertextPath;
         let innerEncryptedAttachment;
         try {
-          innerEncryptedAttachment = await encryptAttachmentV2({
+          innerEncryptedAttachment = await encryptAttachmentV2ToDisk({
             keys: innerKeys,
             plaintextAbsolutePath,
           });
@@ -797,7 +797,7 @@ describe('Crypto', () => {
               innerEncryptedAttachment.path
             );
 
-          const outerEncryptedAttachment = await encryptAttachmentV2({
+          const outerEncryptedAttachment = await encryptAttachmentV2ToDisk({
             keys: outerKeys,
             plaintextAbsolutePath: innerCiphertextPath,
             // We (and the server!) don't pad the second layer

ts/test-electron/backup/filePointer_test.ts

@@ -0,0 +1,461 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+import { assert } from 'chai';
+import Long from 'long';
+import { join } from 'path';
+import * as sinon from 'sinon';
+import { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
+import { Backups } from '../../protobuf';
+import {
+  convertAttachmentToFilePointer,
+  convertFilePointerToAttachment,
+} from '../../services/backups/util/filePointers';
+import { APPLICATION_OCTET_STREAM, IMAGE_PNG } from '../../types/MIME';
+import * as Bytes from '../../Bytes';
+import type { AttachmentType } from '../../types/Attachment';
+import { strictAssert } from '../../util/assert';
+
+describe('convertFilePointerToAttachment', () => {
+  it('processes filepointer with attachmentLocator', () => {
+    const result = convertFilePointerToAttachment(
+      new Backups.FilePointer({
+        contentType: 'image/png',
+        width: 100,
+        height: 100,
+        blurHash: 'blurhash',
+        fileName: 'filename',
+        caption: 'caption',
+        incrementalMac: Bytes.fromString('incrementalMac'),
+        incrementalMacChunkSize: 1000,
+        attachmentLocator: new Backups.FilePointer.AttachmentLocator({
+          size: 128,
+          cdnKey: 'cdnKey',
+          cdnNumber: 2,
+          key: Bytes.fromString('key'),
+          digest: Bytes.fromString('digest'),
+          uploadTimestamp: Long.fromNumber(1970),
+        }),
+      })
+    );
+
+    assert.deepStrictEqual(result, {
+      contentType: IMAGE_PNG,
+      width: 100,
+      height: 100,
+      size: 128,
+      blurHash: 'blurhash',
+      fileName: 'filename',
+      caption: 'caption',
+      cdnKey: 'cdnKey',
+      cdnNumber: 2,
+      key: Bytes.toBase64(Bytes.fromString('key')),
+      digest: Bytes.toBase64(Bytes.fromString('digest')),
+      uploadTimestamp: 1970,
+      incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
+      incrementalMacChunkSize: 1000,
+    });
+  });
+
+  it('processes filepointer with backupLocator and missing fields', () => {
+    const result = convertFilePointerToAttachment(
+      new Backups.FilePointer({
+        contentType: 'image/png',
+        width: 100,
+        height: 100,
+        blurHash: 'blurhash',
+        fileName: 'filename',
+        caption: 'caption',
+        incrementalMac: Bytes.fromString('incrementalMac'),
+        incrementalMacChunkSize: 1000,
+        backupLocator: new Backups.FilePointer.BackupLocator({
+          mediaName: 'mediaName',
+          cdnNumber: 3,
+          size: 128,
+          key: Bytes.fromString('key'),
+          digest: Bytes.fromString('digest'),
+          transitCdnKey: 'transitCdnKey',
+          transitCdnNumber: 2,
+        }),
+      })
+    );
+
+    assert.deepStrictEqual(result, {
+      contentType: IMAGE_PNG,
+      width: 100,
+      height: 100,
+      size: 128,
+      blurHash: 'blurhash',
+      fileName: 'filename',
+      caption: 'caption',
+      cdnKey: 'transitCdnKey',
+      cdnNumber: 2,
+      key: Bytes.toBase64(Bytes.fromString('key')),
+      digest: Bytes.toBase64(Bytes.fromString('digest')),
+      incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
+      incrementalMacChunkSize: 1000,
+      backupLocator: {
+        mediaName: 'mediaName',
+        cdnNumber: 3,
+      },
+    });
+  });
+
+  it('processes filepointer with invalidAttachmentLocator', () => {
+    const result = convertFilePointerToAttachment(
+      new Backups.FilePointer({
+        contentType: 'image/png',
+        width: 100,
+        height: 100,
+        blurHash: 'blurhash',
+        fileName: 'filename',
+        caption: 'caption',
+        incrementalMac: Bytes.fromString('incrementalMac'),
+        incrementalMacChunkSize: 1000,
+        invalidAttachmentLocator:
+          new Backups.FilePointer.InvalidAttachmentLocator(),
+      })
+    );
+
+    assert.deepStrictEqual(result, {
+      contentType: IMAGE_PNG,
+      width: 100,
+      height: 100,
+      blurHash: 'blurhash',
+      fileName: 'filename',
+      caption: 'caption',
+      incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
+      incrementalMacChunkSize: 1000,
+      size: 0,
+      error: true,
+    });
+  });
+
+  it('accepts missing / null fields and adds defaults to contentType and size', () => {
+    const result = convertFilePointerToAttachment(
+      new Backups.FilePointer({
+        backupLocator: new Backups.FilePointer.BackupLocator(),
+      })
+    );
+
+    assert.deepStrictEqual(result, {
+      contentType: APPLICATION_OCTET_STREAM,
+      size: 0,
+      width: undefined,
+      height: undefined,
+      blurHash: undefined,
+      fileName: undefined,
+      caption: undefined,
+      cdnKey: undefined,
+      cdnNumber: undefined,
+      key: undefined,
+      digest: undefined,
+      incrementalMac: undefined,
+      incrementalMacChunkSize: undefined,
+      backupLocator: undefined,
+    });
+  });
+});
+
+function composeAttachment(
+  overrides: Partial<AttachmentType> = {}
+): AttachmentType {
+  return {
+    size: 100,
+    contentType: IMAGE_PNG,
+    cdnKey: 'cdnKey',
+    cdnNumber: 2,
+    path: 'path/to/file.png',
+    key: 'key',
+    digest: 'digest',
+    width: 100,
+    height: 100,
+    blurHash: 'blurhash',
+    fileName: 'filename',
+    caption: 'caption',
+    incrementalMac: 'incrementalMac',
+    incrementalMacChunkSize: 1000,
+    uploadTimestamp: 1234,
+    ...overrides,
+  };
+}
+
+const defaultFilePointer = new Backups.FilePointer({
+  contentType: IMAGE_PNG,
+  width: 100,
+  height: 100,
+  blurHash: 'blurhash',
+  fileName: 'filename',
+  caption: 'caption',
+  incrementalMac: Bytes.fromBase64('incrementalMac'),
+  incrementalMacChunkSize: 1000,
+});
+
+const defaultAttachmentLocator = new Backups.FilePointer.AttachmentLocator({
+  cdnKey: 'cdnKey',
+  cdnNumber: 2,
+  key: Bytes.fromBase64('key'),
+  digest: Bytes.fromBase64('digest'),
+  size: 100,
+  uploadTimestamp: Long.fromNumber(1234),
+});
+
+const defaultMediaName = 'digest';
+const defaultBackupLocator = new Backups.FilePointer.BackupLocator({
+  mediaName: defaultMediaName,
+  cdnNumber: null,
+  key: Bytes.fromBase64('key'),
+  digest: Bytes.fromBase64('digest'),
+  size: 100,
+  transitCdnKey: 'cdnKey',
+  transitCdnNumber: 2,
+});
+
+const filePointerWithAttachmentLocator = new Backups.FilePointer({
+  ...defaultFilePointer,
+  attachmentLocator: defaultAttachmentLocator,
+});
+
+const filePointerWithBackupLocator = new Backups.FilePointer({
+  ...defaultFilePointer,
+  backupLocator: defaultBackupLocator,
+});
+const filePointerWithInvalidLocator = new Backups.FilePointer({
+  ...defaultFilePointer,
+  invalidAttachmentLocator: new Backups.FilePointer.InvalidAttachmentLocator(),
+});
+
+async function testAttachmentToFilePointer(
+  attachment: AttachmentType,
+  filePointer: Backups.FilePointer,
+  options?: { backupLevel?: BackupLevel; backupCdnNumber?: number }
+) {
+  async function _doTest(withBackupLevel: BackupLevel) {
+    assert.deepStrictEqual(
+      await convertAttachmentToFilePointer({
+        attachment,
+        backupLevel: withBackupLevel,
+        getBackupTierInfo: _mediaName => {
+          if (options?.backupCdnNumber != null) {
+            return { isInBackupTier: true, cdnNumber: options.backupCdnNumber };
+          }
+          return { isInBackupTier: false };
+        },
+      }),
+      filePointer
+    );
+  }
+
+  if (!options?.backupLevel) {
+    await _doTest(BackupLevel.Messages);
+    await _doTest(BackupLevel.Media);
+  } else {
+    await _doTest(options.backupLevel);
+  }
+}
+
+describe('convertAttachmentToFilePointer', () => {
+  describe('not downloaded locally', () => {
+    const undownloadedAttachment = composeAttachment({ path: undefined });
+    it('returns invalidAttachmentLocator if missing critical decryption info', async () => {
+      await testAttachmentToFilePointer(
+        {
+          ...undownloadedAttachment,
+          key: undefined,
+        },
+        filePointerWithInvalidLocator
+      );
+      await testAttachmentToFilePointer(
+        {
+          ...undownloadedAttachment,
+          digest: undefined,
+        },
+        filePointerWithInvalidLocator
+      );
+    });
+    describe('attachment does not have attachment.backupLocator', () => {
+      it('returns attachmentLocator, regardless of backupLevel or backup tier status', async () => {
+        await testAttachmentToFilePointer(
+          undownloadedAttachment,
+          filePointerWithAttachmentLocator,
+          { backupCdnNumber: 3 }
+        );
+      });
+
+      it('returns invalidAttachmentLocator if missing critical locator info', async () => {
+        await testAttachmentToFilePointer(
+          {
+            ...undownloadedAttachment,
+            cdnKey: undefined,
+          },
+          filePointerWithInvalidLocator
+        );
+        await testAttachmentToFilePointer(
+          {
+            ...undownloadedAttachment,
+            cdnNumber: undefined,
+          },
+          filePointerWithInvalidLocator
+        );
+      });
+    });
+    describe('attachment has attachment.backupLocator', () => {
+      const undownloadedAttachmentWithBackupLocator = {
+        ...undownloadedAttachment,
+        backupLocator: { mediaName: defaultMediaName },
+      };
+
+      it('returns backupLocator if backupLevel is Media', async () => {
+        await testAttachmentToFilePointer(
+          undownloadedAttachmentWithBackupLocator,
+          filePointerWithBackupLocator,
+          { backupLevel: BackupLevel.Media }
+        );
+      });
+
+      it('returns backupLocator even if missing transit CDN info', async () => {
+        // Even if missing transit CDNKey
+        await testAttachmentToFilePointer(
+          { ...undownloadedAttachmentWithBackupLocator, cdnKey: undefined },
+          new Backups.FilePointer({
+            ...filePointerWithBackupLocator,
+            backupLocator: new Backups.FilePointer.BackupLocator({
+              ...defaultBackupLocator,
+              transitCdnKey: undefined,
+            }),
+          }),
+          { backupLevel: BackupLevel.Media }
+        );
+      });
+
+      it('returns attachmentLocator if backupLevel is Messages', async () => {
+        await testAttachmentToFilePointer(
+          undownloadedAttachmentWithBackupLocator,
+          filePointerWithAttachmentLocator,
+          { backupLevel: BackupLevel.Messages }
+        );
+      });
+    });
+  });
+  describe('downloaded locally', () => {
+    const downloadedAttachment = composeAttachment();
+    describe('BackupLevel.Messages', () => {
+      it('returns attachmentLocator', async () => {
+        await testAttachmentToFilePointer(
+          downloadedAttachment,
+          filePointerWithAttachmentLocator,
+          { backupLevel: BackupLevel.Messages }
+        );
+      });
+      it('returns invalidAttachmentLocator if missing critical locator info', async () => {
+        await testAttachmentToFilePointer(
+          {
+            ...downloadedAttachment,
+            cdnKey: undefined,
+          },
+          filePointerWithInvalidLocator,
+          { backupLevel: BackupLevel.Messages }
+        );
+        await testAttachmentToFilePointer(
+          {
+            ...downloadedAttachment,
+            cdnNumber: undefined,
+          },
+          filePointerWithInvalidLocator,
+          { backupLevel: BackupLevel.Messages }
+        );
+      });
+      it('returns invalidAttachmentLocator if missing critical decryption info', async () => {
+        await testAttachmentToFilePointer(
+          {
+            ...downloadedAttachment,
+            key: undefined,
+          },
+          filePointerWithInvalidLocator,
+          { backupLevel: BackupLevel.Messages }
+        );
+        await testAttachmentToFilePointer(
+          {
+            ...downloadedAttachment,
+            digest: undefined,
+          },
+          filePointerWithInvalidLocator,
+          { backupLevel: BackupLevel.Messages }
+        );
+      });
+    });
+    describe('BackupLevel.Media', () => {
+      describe('if missing critical decryption info', () => {
+        const FILE_PATH = join(__dirname, '../../../fixtures/ghost-kitty.mp4');
+
+        let sandbox: sinon.SinonSandbox;
+
+        beforeEach(() => {
+          sandbox = sinon.createSandbox();
+          sandbox
+            .stub(window.Signal.Migrations, 'getAbsoluteAttachmentPath')
+            .callsFake(relPath => {
+              if (relPath === downloadedAttachment.path) {
+                return FILE_PATH;
+              }
+              return relPath;
+            });
+        });
+
+        afterEach(() => {
+          sandbox.restore();
+        });
+
+        it('generates new key & digest and removes existing CDN info', async () => {
+          const result = await convertAttachmentToFilePointer({
+            attachment: {
+              ...downloadedAttachment,
+              key: undefined,
+            },
+            backupLevel: BackupLevel.Media,
+            getBackupTierInfo: () => ({ isInBackupTier: false }),
+          });
+          const newKey = result.backupLocator?.key;
+          const newDigest = result.backupLocator?.digest;
+
+          strictAssert(newDigest, 'must create new digest');
+          assert.deepStrictEqual(
+            result,
+            new Backups.FilePointer({
+              ...filePointerWithBackupLocator,
+              backupLocator: new Backups.FilePointer.BackupLocator({
+                ...defaultBackupLocator,
+                key: newKey,
+                digest: newDigest,
+                mediaName: Bytes.toBase64(newDigest),
+                transitCdnKey: undefined,
+                transitCdnNumber: undefined,
+              }),
+            })
+          );
+        });
+      });
+
+      it('returns BackupLocator, with cdnNumber if in backup tier already', async () => {
+        await testAttachmentToFilePointer(
+          downloadedAttachment,
+          new Backups.FilePointer({
+            ...filePointerWithBackupLocator,
+            backupLocator: new Backups.FilePointer.BackupLocator({
+              ...defaultBackupLocator,
+              cdnNumber: 12,
+            }),
+          }),
+          { backupLevel: BackupLevel.Media, backupCdnNumber: 12 }
+        );
+      });
+
+      it('returns BackupLocator, with empty cdnNumber if not in backup tier', async () => {
+        await testAttachmentToFilePointer(
+          downloadedAttachment,
+          filePointerWithBackupLocator,
+          { backupLevel: BackupLevel.Media }
+        );
+      });
+    });
+  });
+});

ts/test-node/backups/filePointer_test.ts

@@ -1,149 +0,0 @@
-// Copyright 2024 Signal Messenger, LLC
-// SPDX-License-Identifier: AGPL-3.0-only
-import { assert } from 'chai';
-import Long from 'long';
-import { Backups } from '../../protobuf';
-import { convertFilePointerToAttachment } from '../../services/backups/util/filePointers';
-import { APPLICATION_OCTET_STREAM, IMAGE_PNG } from '../../types/MIME';
-import * as Bytes from '../../Bytes';
-
-describe('convertFilePointerToAttachment', () => {
-  it('processes filepointer with attachmentLocator', () => {
-    const result = convertFilePointerToAttachment(
-      new Backups.FilePointer({
-        contentType: 'image/png',
-        width: 100,
-        height: 100,
-        blurHash: 'blurhash',
-        fileName: 'filename',
-        caption: 'caption',
-        incrementalMac: Bytes.fromString('incrementalMac'),
-        incrementalMacChunkSize: 1000,
-        attachmentLocator: new Backups.FilePointer.AttachmentLocator({
-          size: 128,
-          cdnKey: 'cdnKey',
-          cdnNumber: 2,
-          key: Bytes.fromString('key'),
-          digest: Bytes.fromString('digest'),
-          uploadTimestamp: Long.fromNumber(1970),
-        }),
-      })
-    );
-
-    assert.deepStrictEqual(result, {
-      contentType: IMAGE_PNG,
-      width: 100,
-      height: 100,
-      size: 128,
-      blurHash: 'blurhash',
-      fileName: 'filename',
-      caption: 'caption',
-      cdnKey: 'cdnKey',
-      cdnNumber: 2,
-      key: Bytes.toBase64(Bytes.fromString('key')),
-      digest: Bytes.toBase64(Bytes.fromString('digest')),
-      uploadTimestamp: 1970,
-      incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
-      incrementalMacChunkSize: 1000,
-    });
-  });
-
-  it('processes filepointer with backupLocator and missing fields', () => {
-    const result = convertFilePointerToAttachment(
-      new Backups.FilePointer({
-        contentType: 'image/png',
-        width: 100,
-        height: 100,
-        blurHash: 'blurhash',
-        fileName: 'filename',
-        caption: 'caption',
-        incrementalMac: Bytes.fromString('incrementalMac'),
-        incrementalMacChunkSize: 1000,
-        backupLocator: new Backups.FilePointer.BackupLocator({
-          mediaName: 'mediaName',
-          cdnNumber: 3,
-          size: 128,
-          key: Bytes.fromString('key'),
-          digest: Bytes.fromString('digest'),
-          transitCdnKey: 'transitCdnKey',
-          transitCdnNumber: 2,
-        }),
-      })
-    );
-
-    assert.deepStrictEqual(result, {
-      contentType: IMAGE_PNG,
-      width: 100,
-      height: 100,
-      size: 128,
-      blurHash: 'blurhash',
-      fileName: 'filename',
-      caption: 'caption',
-      cdnKey: 'transitCdnKey',
-      cdnNumber: 2,
-      key: Bytes.toBase64(Bytes.fromString('key')),
-      digest: Bytes.toBase64(Bytes.fromString('digest')),
-      incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
-      incrementalMacChunkSize: 1000,
-      backupLocator: {
-        mediaName: 'mediaName',
-        cdnNumber: 3,
-      },
-    });
-  });
-
-  it('processes filepointer with invalidAttachmentLocator', () => {
-    const result = convertFilePointerToAttachment(
-      new Backups.FilePointer({
-        contentType: 'image/png',
-        width: 100,
-        height: 100,
-        blurHash: 'blurhash',
-        fileName: 'filename',
-        caption: 'caption',
-        incrementalMac: Bytes.fromString('incrementalMac'),
-        incrementalMacChunkSize: 1000,
-        invalidAttachmentLocator:
-          new Backups.FilePointer.InvalidAttachmentLocator(),
-      })
-    );
-
-    assert.deepStrictEqual(result, {
-      contentType: IMAGE_PNG,
-      width: 100,
-      height: 100,
-      blurHash: 'blurhash',
-      fileName: 'filename',
-      caption: 'caption',
-      incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
-      incrementalMacChunkSize: 1000,
-      size: 0,
-      error: true,
-    });
-  });
-
-  it('accepts missing / null fields and adds defaults to contentType and size', () => {
-    const result = convertFilePointerToAttachment(
-      new Backups.FilePointer({
-        backupLocator: new Backups.FilePointer.BackupLocator(),
-      })
-    );
-
-    assert.deepStrictEqual(result, {
-      contentType: APPLICATION_OCTET_STREAM,
-      size: 0,
-      width: undefined,
-      height: undefined,
-      blurHash: undefined,
-      fileName: undefined,
-      caption: undefined,
-      cdnKey: undefined,
-      cdnNumber: undefined,
-      key: undefined,
-      digest: undefined,
-      incrementalMac: undefined,
-      incrementalMacChunkSize: undefined,
-      backupLocator: undefined,
-    });
-  });
-});

ts/types/Attachment.ts

@@ -21,7 +21,7 @@ import {
   isImageTypeSupported,
   isVideoTypeSupported,
 } from '../util/GoogleChrome';
-import type { LocalizerType } from './Util';
+import type { LocalizerType, WithRequiredProperties } from './Util';
 import { ThemeType } from './Util';
 import * as GoogleChrome from '../util/GoogleChrome';
 import { ReadStatus } from '../messages/MessageReadStatus';
@@ -994,3 +994,66 @@ export function getAttachmentSignature(attachment: AttachmentType): string {
   strictAssert(attachment.digest, 'attachment missing digest');
   return attachment.digest;
 }
+
+type RequiredPropertiesForDecryption = 'key' | 'digest';
+
+type DecryptableAttachment = WithRequiredProperties<
+  AttachmentType,
+  RequiredPropertiesForDecryption
+>;
+
+export type AttachmentDownloadableFromTransitTier = WithRequiredProperties<
+  DecryptableAttachment,
+  'cdnKey' | 'cdnNumber'
+>;
+
+export type AttachmentDownloadableFromBackupTier = WithRequiredProperties<
+  DecryptableAttachment,
+  'backupLocator'
+>;
+
+export type DownloadedAttachment = WithRequiredProperties<
+  AttachmentType,
+  'path'
+>;
+
+export type AttachmentReadyForBackup = WithRequiredProperties<
+  DownloadedAttachment,
+  RequiredPropertiesForDecryption
+>;
+
+function isDecryptable(
+  attachment: AttachmentType
+): attachment is DecryptableAttachment {
+  return Boolean(attachment.key) && Boolean(attachment.digest);
+}
+
+export function isDownloadableFromTransitTier(
+  attachment: AttachmentType
+): attachment is AttachmentDownloadableFromTransitTier {
+  if (!isDecryptable(attachment)) {
+    return false;
+  }
+  if (attachment.cdnKey && attachment.cdnNumber) {
+    return true;
+  }
+  return false;
+}
+
+export function isDownloadableFromBackupTier(
+  attachment: AttachmentType
+): attachment is AttachmentDownloadableFromBackupTier {
+  if (!attachment.key || !attachment.digest) {
+    return false;
+  }
+  if (attachment.backupLocator?.mediaName) {
+    return true;
+  }
+  return false;
+}
+
+export function isDownloadedToLocalFile(
+  attachment: AttachmentType
+): attachment is DownloadedAttachment {
+  return Boolean(attachment.path);
+}