Enable downloading attachments from backup CDN

protos/Backups.proto

@@ -12,6 +12,18 @@ message BackupInfo {
   uint64 backupTimeMs = 2;
 }
 
+// Frames must follow in the following ordering rules:
+//
+// 1. There is exactly one AccountData and it is the first frame.
+// 2. A frame referenced by ID must come before the referencing frame.
+//    e.g. a Recipient must come before any Chat referencing it.
+// 3. All ChatItems must appear in global Chat rendering order.
+//    (The order in which they were received by the client.)
+//
+// Recipients, Chats, Ad-hoc Calls, & StickerPacks can be in any order.
+// (But must respect rule 2.)
+// For example, Chats may all be together at the beginning,
+// or may each immediately precede its first ChatItem.
 message Frame {
   oneof item {
     AccountData account = 1;
@@ -364,10 +376,16 @@ message FilePointer {
   // References attachments in the backup (media) storage tier.
   message BackupLocator {
     string mediaName = 1;
-    uint32 cdnNumber = 2;
+    // If present, the cdn number of the succesful upload.
+    // If empty/0, may still have been uploaded, and clients
+    // can discover the cdn number via the list endpoint.
+    optional uint32 cdnNumber = 2;
     bytes key = 3;
     bytes digest = 4;
     uint32 size = 5;
+    // Fallback in case backup tier upload failed.
+    optional string transitCdnKey = 6;
+    optional uint32 transitCdnNumber = 7;
   }
 
   // References attachments in the transit storage tier.
@@ -519,9 +537,17 @@ message IndividualCallChatUpdate {
 }
 
 message GroupCallChatUpdate {
+  enum LocalUserJoined {
+    UNKNOWN = 0;
+    JOINED = 1;
+    DID_NOT_JOIN = 2;
+  }
+
   optional bytes startedCallAci = 1;
   uint64 startedCallTimestamp = 2;
   repeated bytes inCallAcis = 3;
+  uint64 endedCallTimestamp = 4; // 0 indicates we do not know
+  LocalUserJoined localUserJoined = 5;
 }
 
 message SimpleChatUpdate {
@@ -820,4 +846,4 @@ message StickerPack {
 message StickerPackSticker {
   string emoji = 1;
   uint32 id = 2;
-}
+}

ts/AttachmentCrypto.ts

@@ -26,6 +26,7 @@ import type { AttachmentType } from './types/Attachment';
 import type { ContextType } from './types/Message2';
 import { strictAssert } from './util/assert';
 import * as Errors from './types/errors';
+import { isNotNil } from './util/isNotNil';
 
 // This file was split from ts/Crypto.ts because it pulls things in from node, and
 //   too many things pull in Crypto.ts, so it broke storybook.
@@ -58,10 +59,12 @@ export async function encryptAttachmentV2({
   keys,
   plaintextAbsolutePath,
   dangerousTestOnlyIv,
+  dangerousTestOnlySkipPadding = false,
 }: {
   keys: Readonly<Uint8Array>;
   plaintextAbsolutePath: string;
   dangerousTestOnlyIv?: Readonly<Uint8Array>;
+  dangerousTestOnlySkipPadding?: boolean;
 }): Promise<EncryptedAttachmentV2> {
   const logId = 'encryptAttachmentV2';
 
@@ -75,6 +78,14 @@ export async function encryptAttachmentV2({
   if (dangerousTestOnlyIv && window.getEnvironment() !== Environment.Test) {
     throw new Error(`${logId}: Used dangerousTestOnlyIv outside tests!`);
   }
+  if (
+    dangerousTestOnlySkipPadding &&
+    window.getEnvironment() !== Environment.Test
+  ) {
+    throw new Error(
+      `${logId}: Used dangerousTestOnlySkipPadding outside tests!`
+    );
+  }
   const iv = dangerousTestOnlyIv || _generateAttachmentIv();
 
   const plaintextHash = createHash(HashType.size256);
@@ -96,14 +107,16 @@ export async function encryptAttachmentV2({
     }
 
     await pipeline(
-      readFd.createReadStream(),
+      [
-      peekAndUpdateHash(plaintextHash),
+        readFd.createReadStream(),
-      appendPaddingStream(),
+        peekAndUpdateHash(plaintextHash),
-      createCipheriv(CipherType.AES256CBC, aesKey, iv),
+        dangerousTestOnlySkipPadding ? undefined : appendPaddingStream(),
-      prependIv(iv),
+        createCipheriv(CipherType.AES256CBC, aesKey, iv),
-      appendMacStream(macKey),
+        prependIv(iv),
-      peekAndUpdateHash(digest),
+        appendMacStream(macKey),
-      writeFd.createWriteStream()
+        peekAndUpdateHash(digest),
+        writeFd.createWriteStream(),
+      ].filter(isNotNil)
     );
   } catch (error) {
     log.error(
@@ -136,32 +149,59 @@ export async function encryptAttachmentV2({
   };
 }
 
-export async function decryptAttachmentV2({
+type DecryptAttachmentOptionsType = Readonly<{
-  ciphertextPath,
-  id,
-  keys,
-  size,
-  theirDigest,
-}: {
   ciphertextPath: string;
-  id: string;
+  idForLogging: string;
-  keys: Readonly<Uint8Array>;
+  aesKey: Readonly<Uint8Array>;
+  macKey: Readonly<Uint8Array>;
   size: number;
   theirDigest: Readonly<Uint8Array>;
-}): Promise<DecryptedAttachmentV2> {
+  outerEncryption?: {
-  const logId = `decryptAttachmentV2(${id})`;
+    aesKey: Readonly<Uint8Array>;
+    macKey: Readonly<Uint8Array>;
+  };
+}>;
+
+export async function decryptAttachmentV2(
+  options: DecryptAttachmentOptionsType
+): Promise<DecryptedAttachmentV2> {
+  const {
+    idForLogging,
+    macKey,
+    aesKey,
+    ciphertextPath,
+    theirDigest,
+    outerEncryption,
+  } = options;
+
+  const logId = `decryptAttachmentV2(${idForLogging})`;
 
   // Create random output file
   const relativeTargetPath = getRelativePath(createName());
   const absoluteTargetPath =
     window.Signal.Migrations.getAbsoluteAttachmentPath(relativeTargetPath);
 
-  const { aesKey, macKey } = splitKeys(keys);
-
   const digest = createHash(HashType.size256);
   const hmac = createHmac(HashType.size256, macKey);
   const plaintextHash = createHash(HashType.size256);
-  let theirMac = null as Uint8Array | null; // TypeScript shenanigans
+  let theirMac: Uint8Array | undefined;
+
+  // When downloading from backup there is an outer encryption layer; in that case we
+  // need to decrypt the outer layer and check its MAC
+  let theirOuterMac: Uint8Array | undefined;
+  const outerHmac = outerEncryption
+    ? createHmac(HashType.size256, outerEncryption.macKey)
+    : undefined;
+
+  const maybeOuterEncryptionGetIvAndDecipher = outerEncryption
+    ? getIvAndDecipher(outerEncryption.aesKey)
+    : undefined;
+
+  const maybeOuterEncryptionGetMacAndUpdateMac = outerHmac
+    ? getMacAndUpdateHmac(outerHmac, theirOuterMacValue => {
+        theirOuterMac = theirOuterMacValue;
+      })
+    : undefined;
 
   let readFd;
   let writeFd;
@@ -179,15 +219,19 @@ export async function decryptAttachmentV2({
     }
 
     await pipeline(
-      readFd.createReadStream(),
+      [
-      peekAndUpdateHash(digest),
+        readFd.createReadStream(),
-      getMacAndUpdateHmac(hmac, theirMacValue => {
+        maybeOuterEncryptionGetMacAndUpdateMac,
-        theirMac = theirMacValue;
+        maybeOuterEncryptionGetIvAndDecipher,
-      }),
+        peekAndUpdateHash(digest),
-      getIvAndDecipher(aesKey),
+        getMacAndUpdateHmac(hmac, theirMacValue => {
-      trimPadding(size),
+          theirMac = theirMacValue;
-      peekAndUpdateHash(plaintextHash),
+        }),
-      writeFd.createWriteStream()
+        getIvAndDecipher(aesKey),
+        trimPadding(options.size),
+        peekAndUpdateHash(plaintextHash),
+        writeFd.createWriteStream(),
+      ].filter(isNotNil)
     );
   } catch (error) {
     log.error(
@@ -224,11 +268,29 @@ export async function decryptAttachmentV2({
   if (!constantTimeEqual(ourMac, theirMac)) {
     throw new Error(`${logId}: Bad MAC`);
   }
-
   if (!constantTimeEqual(ourDigest, theirDigest)) {
     throw new Error(`${logId}: Bad digest`);
   }
 
+  if (outerEncryption) {
+    strictAssert(outerHmac, 'outerHmac must exist');
+
+    const ourOuterMac = outerHmac.digest();
+    strictAssert(
+      ourOuterMac.byteLength === ATTACHMENT_MAC_LENGTH,
+      `${logId}: Failed to generate ourOuterMac!`
+    );
+    strictAssert(
+      theirOuterMac != null &&
+        theirOuterMac.byteLength === ATTACHMENT_MAC_LENGTH,
+      `${logId}: Failed to find theirOuterMac!`
+    );
+
+    if (!constantTimeEqual(ourOuterMac, theirOuterMac)) {
+      throw new Error(`${logId}: Bad outer encryption MAC`);
+    }
+  }
+
   return {
     path: relativeTargetPath,
     plaintextHash: ourPlaintextHash,
@@ -238,7 +300,12 @@ export async function decryptAttachmentV2({
 /**
  * Splits the keys into aes and mac keys.
  */
-function splitKeys(keys: Uint8Array) {
+
+type AttachmentEncryptionKeysType = {
+  aesKey: Uint8Array;
+  macKey: Uint8Array;
+};
+export function splitKeys(keys: Uint8Array): AttachmentEncryptionKeysType {
   strictAssert(
     keys.byteLength === KEY_SET_LENGTH,
     `attachment keys must be ${KEY_SET_LENGTH} bytes, got ${keys.byteLength}`
@@ -376,10 +443,20 @@ function trimPadding(size: number) {
   });
 }
 
-export function getAttachmentDownloadSize(size: number): number {
+export function getAttachmentCiphertextLength(plaintextLength: number): number {
+  const paddedPlaintextSize = logPadSize(plaintextLength);
+
   return (
-    // Multiply this by 1.05 to allow some variance
+    IV_LENGTH +
-    logPadSize(size) * 1.05 + IV_LENGTH + ATTACHMENT_MAC_LENGTH
+    getAesCbcCiphertextLength(paddedPlaintextSize) +
+    ATTACHMENT_MAC_LENGTH
+  );
+}
+
+export function getAesCbcCiphertextLength(plaintextLength: number): number {
+  const AES_CBC_BLOCK_SIZE = 16;
+  return (
+    (1 + Math.floor(plaintextLength / AES_CBC_BLOCK_SIZE)) * AES_CBC_BLOCK_SIZE
   );
 }
 

ts/Crypto.ts

@@ -208,10 +208,23 @@ export type BackupKeyMaterialType = Readonly<{
   aesKey: Uint8Array;
 }>;
 
+export type BackupMediaKeyMaterialType = Readonly<{
+  macKey: Uint8Array;
+  aesKey: Uint8Array;
+  iv: Uint8Array;
+}>;
+
 const BACKUP_AES_KEY_LEN = 32;
 const BACKUP_MAC_KEY_LEN = 32;
 const BACKUP_MATERIAL_INFO = '20231003_Signal_Backups_EncryptMessageBackup';
 
+const BACKUP_MEDIA_ID_INFO = '20231003_Signal_Backups_Media_ID';
+const BACKUP_MEDIA_ID_LEN = 15;
+const BACKUP_MEDIA_ENCRYPT_INFO = '20231003_Signal_Backups_Media_ID';
+const BACKUP_MEDIA_AES_KEY_LEN = 32;
+const BACKUP_MEDIA_MAC_KEY_LEN = 32;
+const BACKUP_MEDIA_IV_LEN = 16;
+
 export function deriveBackupKeyMaterial(
   backupKey: Uint8Array,
   backupId: Uint8Array
@@ -238,6 +251,56 @@ export function deriveBackupKeyMaterial(
   };
 }
 
+export function deriveMediaIdFromMediaName(
+  backupKey: Uint8Array,
+  mediaName: string
+): Uint8Array {
+  if (backupKey.byteLength !== BACKUP_KEY_LEN) {
+    throw new Error('deriveMediaIdFromMediaName: invalid backup key length');
+  }
+
+  if (!mediaName) {
+    throw new Error('deriveMediaIdFromMediaName: mediaName missing');
+  }
+
+  const hkdf = HKDF.new(3);
+  return hkdf.deriveSecrets(
+    BACKUP_MEDIA_ID_LEN,
+    Buffer.from(backupKey),
+    Buffer.from(BACKUP_MEDIA_ID_INFO),
+    Buffer.from(Bytes.fromBase64(mediaName))
+  );
+}
+
+export function deriveBackupMediaKeyMaterial(
+  backupKey: Uint8Array,
+  mediaId: Uint8Array
+): BackupMediaKeyMaterialType {
+  if (backupKey.byteLength !== BACKUP_KEY_LEN) {
+    throw new Error('deriveMediaIdFromMediaName: invalid backup key length');
+  }
+
+  if (!mediaId.length) {
+    throw new Error('deriveMediaIdFromMediaName: mediaId missing');
+  }
+
+  const hkdf = HKDF.new(3);
+  const material = hkdf.deriveSecrets(
+    BACKUP_MEDIA_MAC_KEY_LEN + BACKUP_MEDIA_AES_KEY_LEN + BACKUP_MEDIA_IV_LEN,
+    Buffer.from(backupKey),
+    Buffer.from(BACKUP_MEDIA_ENCRYPT_INFO),
+    Buffer.from(mediaId)
+  );
+
+  return {
+    macKey: material.subarray(0, BACKUP_MEDIA_MAC_KEY_LEN),
+    aesKey: material.subarray(
+      BACKUP_MEDIA_MAC_KEY_LEN,
+      BACKUP_MEDIA_MAC_KEY_LEN + BACKUP_MEDIA_AES_KEY_LEN
+    ),
+    iv: material.subarray(BACKUP_MEDIA_MAC_KEY_LEN + BACKUP_MEDIA_AES_KEY_LEN),
+  };
+}
 export function deriveStorageItemKey(
   storageServiceKey: Uint8Array,
   itemID: string

ts/services/backups/api.ts

@@ -13,6 +13,7 @@ import type {
 import type { BackupCredentials } from './credentials';
 
 export class BackupAPI {
+  private cachedBackupInfo: GetBackupInfoResponseType | undefined;
   constructor(private credentials: BackupCredentials) {}
 
   public async refresh(): Promise<void> {
@@ -23,9 +24,33 @@ export class BackupAPI {
   }
 
   public async getInfo(): Promise<GetBackupInfoResponseType> {
-    return this.server.getBackupInfo(
+    const backupInfo = await this.server.getBackupInfo(
       await this.credentials.getHeadersForToday()
     );
+    this.cachedBackupInfo = backupInfo;
+    return backupInfo;
+  }
+
+  private async getCachedInfo(): Promise<GetBackupInfoResponseType> {
+    if (this.cachedBackupInfo) {
+      return this.cachedBackupInfo;
+    }
+
+    return this.getInfo();
+  }
+
+  public async getMediaDir(): Promise<string> {
+    return (await this.getCachedInfo()).mediaDir;
+  }
+
+  public async getBackupDir(): Promise<string> {
+    return (await this.getCachedInfo())?.backupDir;
+  }
+
+  // Backup name will change whenever a new backup is created, so we don't want to cache
+  // it
+  public async getBackupName(): Promise<string> {
+    return (await this.getInfo()).backupName;
   }
 
   public async getUploadForm(): Promise<GetBackupUploadFormResponseType> {
@@ -63,6 +88,10 @@ export class BackupAPI {
     });
   }
 
+  public clearCache(): void {
+    this.cachedBackupInfo = undefined;
+  }
+
   private get server(): WebAPIType {
     const { server } = window.textsecure;
     strictAssert(server, 'server not available');

ts/services/backups/credentials.ts

@@ -6,16 +6,18 @@ import {
   BackupAuthCredential,
   BackupAuthCredentialRequestContext,
   BackupAuthCredentialResponse,
+  type BackupLevel,
   GenericServerPublicParams,
 } from '@signalapp/libsignal-client/zkgroup';
 
 import * as log from '../../logging/log';
 import { strictAssert } from '../../util/assert';
 import { drop } from '../../util/drop';
-import { toDayMillis } from '../../util/timestamp';
+import { isMoreRecentThan, toDayMillis } from '../../util/timestamp';
-import { DAY, DurationInSeconds } from '../../util/durations';
+import { DAY, DurationInSeconds, HOUR } from '../../util/durations';
 import { BackOff, FIBONACCI_TIMEOUTS } from '../../util/BackOff';
 import type {
+  BackupCdnReadCredentialType,
   BackupCredentialType,
   BackupPresentationHeadersType,
   BackupSignedPresentationType,
@@ -37,9 +39,15 @@ export function getAuthContext(): BackupAuthCredentialRequestContext {
 
 const FETCH_INTERVAL = 3 * DAY;
 
+//  Credentials should be good for 24 hours, but let's play it safe.
+const BACKUP_CDN_READ_CREDENTIALS_VALID_DURATION = 12 * HOUR;
+
 export class BackupCredentials {
   private activeFetch: ReturnType<typeof this.fetch> | undefined;
-
+  private cachedCdnReadCredentials: Record<
+    number,
+    BackupCdnReadCredentialType
+  > = {};
   private readonly fetchBackoff = new BackOff(FIBONACCI_TIMEOUTS);
 
   public start(): void {
@@ -112,15 +120,41 @@ export class BackupCredentials {
     return headers;
   }
 
-  public async getCDNCredentials(
+  public async getCDNReadCredentials(
     cdn: number
   ): Promise<GetBackupCDNCredentialsResponseType> {
     const { server } = window.textsecure;
     strictAssert(server, 'server not available');
 
+    // Backup CDN read credentials are short-lived; we'll just cache them in memory so
+    // that they get invalidated for any reason, we'll fetch new ones on app restart
+    const cachedCredentialsForThisCdn = this.cachedCdnReadCredentials[cdn];
+
+    if (
+      cachedCredentialsForThisCdn &&
+      isMoreRecentThan(
+        cachedCredentialsForThisCdn.retrievedAtMs,
+        BACKUP_CDN_READ_CREDENTIALS_VALID_DURATION
+      )
+    ) {
+      return cachedCredentialsForThisCdn.credentials;
+    }
+
     const headers = await this.getHeadersForToday();
 
-    return server.getBackupCDNCredentials({ headers, cdn });
+    const retrievedAtMs = Date.now();
+    const newCredentials = await server.getBackupCDNCredentials({
+      headers,
+      cdn,
+    });
+
+    this.cachedCdnReadCredentials[cdn] = {
+      credentials: newCredentials,
+      cdnNumber: cdn,
+      retrievedAtMs,
+    };
+
+    return newCredentials;
   }
 
   private scheduleFetch(): void {
@@ -281,8 +315,13 @@ export class BackupCredentials {
     return result;
   }
 
-  // Called when backup tier changes
+  public async getBackupLevel(): Promise<BackupLevel> {
-  public async clear(): Promise<void> {
+    return (await this.getForToday()).level;
+  }
+
+  // Called when backup tier changes or when userChanged event
+  public async clearCache(): Promise<void> {
+    this.cachedCdnReadCredentials = {};
     await window.storage.put('backupCredentials', []);
   }
 }

ts/services/backups/import.ts

@@ -40,7 +40,10 @@ import * as Bytes from '../../Bytes';
 import { BACKUP_VERSION } from './constants';
 import type { AboutMe } from './types';
 import type { GroupV2ChangeDetailType } from '../../groups';
+import { queueAttachmentDownloads } from '../../util/queueAttachmentDownloads';
+import { drop } from '../../util/drop';
 import { isNotNil } from '../../util/isNotNil';
+import { convertFilePointerToAttachment } from './util/filePointers';
 
 const MAX_CONCURRENCY = 10;
 
@@ -100,14 +103,18 @@ export class BackupImportStream extends Writable {
     name: 'BackupImport.saveMessageBatcher',
     wait: 0,
     maxSize: 1000,
-    processBatch: batch => {
+    processBatch: async batch => {
       const ourAci = this.ourConversation?.serviceId;
       assertDev(isAciString(ourAci), 'Our conversation must have ACI');
-      return Data.saveMessages(batch, {
+      await Data.saveMessages(batch, {
         forceSave: true,
         ourAci,
       });
-      // TODO (DESKTOP-6845): after we save messages, queue their attachment downloads
+
+      // TODO (DESKTOP-7402): consider re-saving after updating the pending state
+      for (const messageAttributes of batch) {
+        drop(queueAttachmentDownloads(messageAttributes));
+      }
     },
   });
   private ourConversation?: ConversationAttributesType;
@@ -722,7 +729,14 @@ export class BackupImportStream extends Writable {
   ): Partial<MessageAttributesType> {
     return {
       body: data.text?.body ?? '',
-      // TODO (DESKTOP-6845): add attachments
+      attachments: data.attachments
+        ?.map(attachment => {
+          if (!attachment.pointer) {
+            return null;
+          }
+          return convertFilePointerToAttachment(attachment.pointer);
+        })
+        .filter(isNotNil),
       reactions: data.reactions?.map(
         ({ emoji, authorId, sentTimestamp, receivedTimestamp }) => {
           strictAssert(emoji != null, 'reaction must have an emoji');

ts/services/backups/index.ts

@@ -4,7 +4,7 @@
 import { pipeline } from 'stream/promises';
 import { PassThrough } from 'stream';
 import type { Readable, Writable } from 'stream';
-import { createWriteStream } from 'fs';
+import { createReadStream, createWriteStream } from 'fs';
 import { createGzip, createGunzip } from 'zlib';
 import { createCipheriv, createHmac, randomBytes } from 'crypto';
 import { noop } from 'lodash';
@@ -54,6 +54,11 @@ export class BackupsService {
 
     drop(this.runPeriodicRefresh());
     this.credentials.start();
+
+    window.Whisper.events.on('userChanged', () => {
+      drop(this.credentials.clearCache());
+      this.api.clearCache();
+    });
   }
 
   public async exportBackup(sink: Writable): Promise<void> {
@@ -113,6 +118,10 @@ export class BackupsService {
     });
   }
 
+  public async importFromDisk(backupFile: string): Promise<void> {
+    return backupsService.importBackup(() => createReadStream(backupFile));
+  }
+
   public async importBackup(createBackupStream: () => Readable): Promise<void> {
     strictAssert(!this.isRunning, 'BackupService is already running');
 
@@ -175,7 +184,7 @@ export class BackupsService {
       await this.api.refresh();
       log.info('Backup: refreshed');
     } catch (error) {
-      log.error('Backup: periodic refresh failed', Errors.toLogFormat(error));
+      log.error('Backup: periodic refresh kufailed', Errors.toLogFormat(error));
     }
   }
 }

ts/services/backups/util/filePointers.ts

@@ -0,0 +1,96 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+import {
+  APPLICATION_OCTET_STREAM,
+  stringToMIMEType,
+} from '../../../types/MIME';
+import type { AttachmentType } from '../../../types/Attachment';
+import type { Backups } from '../../../protobuf';
+import * as Bytes from '../../../Bytes';
+import { getTimestampFromLong } from '../../../util/timestampLongUtils';
+
+export function convertFilePointerToAttachment(
+  filePointer: Backups.FilePointer
+): AttachmentType {
+  const {
+    contentType,
+    width,
+    height,
+    fileName,
+    caption,
+    blurHash,
+    incrementalMac,
+    incrementalMacChunkSize,
+    attachmentLocator,
+    backupLocator,
+    invalidAttachmentLocator,
+  } = filePointer;
+
+  const commonProps: Omit<AttachmentType, 'size'> = {
+    contentType: contentType
+      ? stringToMIMEType(contentType)
+      : APPLICATION_OCTET_STREAM,
+    width: width ?? undefined,
+    height: height ?? undefined,
+    fileName: fileName ?? undefined,
+    caption: caption ?? undefined,
+    blurHash: blurHash ?? undefined,
+    incrementalMac: incrementalMac?.length
+      ? Bytes.toBase64(incrementalMac)
+      : undefined,
+    incrementalMacChunkSize: incrementalMacChunkSize ?? undefined,
+  };
+
+  if (attachmentLocator) {
+    const { cdnKey, cdnNumber, key, digest, uploadTimestamp, size } =
+      attachmentLocator;
+    return {
+      ...commonProps,
+      size: size ?? 0,
+      cdnKey: cdnKey ?? undefined,
+      cdnNumber: cdnNumber ?? undefined,
+      key: key?.length ? Bytes.toBase64(key) : undefined,
+      digest: digest?.length ? Bytes.toBase64(digest) : undefined,
+      uploadTimestamp: uploadTimestamp
+        ? getTimestampFromLong(uploadTimestamp)
+        : undefined,
+    };
+  }
+
+  if (backupLocator) {
+    const {
+      mediaName,
+      cdnNumber,
+      key,
+      digest,
+      size,
+      transitCdnKey,
+      transitCdnNumber,
+    } = backupLocator;
+
+    return {
+      ...commonProps,
+      cdnKey: transitCdnKey ?? undefined,
+      cdnNumber: transitCdnNumber ?? undefined,
+      key: key?.length ? Bytes.toBase64(key) : undefined,
+      digest: digest?.length ? Bytes.toBase64(digest) : undefined,
+      size: size ?? 0,
+      backupLocator: mediaName
+        ? {
+            mediaName,
+            cdnNumber: cdnNumber ?? undefined,
+          }
+        : undefined,
+    };
+  }
+
+  if (invalidAttachmentLocator) {
+    return {
+      ...commonProps,
+      error: true,
+      size: 0,
+    };
+  }
+
+  throw new Error('convertFilePointerToAttachment: mising locator');
+}

ts/test-electron/Crypto_test.ts

@@ -5,7 +5,7 @@ import { assert } from 'chai';
 import { readFileSync, unlinkSync, writeFileSync } from 'fs';
 import { join } from 'path';
 
-import { randomBytes } from 'crypto';
+import { createCipheriv, randomBytes } from 'crypto';
 import * as log from '../logging/log';
 import * as Bytes from '../Bytes';
 import * as Curve from '../Curve';
@@ -34,12 +34,15 @@ import {
   encryptAttachment,
   decryptAttachmentV1,
   padAndEncryptAttachment,
+  CipherType,
 } from '../Crypto';
 import {
   KEY_SET_LENGTH,
   _generateAttachmentIv,
   decryptAttachmentV2,
   encryptAttachmentV2,
+  getAesCbcCiphertextLength,
+  splitKeys,
 } from '../AttachmentCrypto';
 import { createTempDir, deleteTempDir } from '../updater/common';
 import { uuidToBytes, bytesToUuid } from '../util/uuidToBytes';
@@ -576,8 +579,8 @@ describe('Crypto', () => {
 
         const decryptedAttachment = await decryptAttachmentV2({
           ciphertextPath,
-          id: 'test',
+          idForLogging: 'test',
-          keys,
+          ...splitKeys(keys),
           size: FILE_CONTENTS.byteLength,
           theirDigest: encryptedAttachment.digest,
         });
@@ -613,8 +616,8 @@ describe('Crypto', () => {
         );
         const decryptedAttachment = await decryptAttachmentV2({
           ciphertextPath,
-          id: 'test',
+          idForLogging: 'test',
-          keys,
+          ...splitKeys(keys),
           size: FILE_CONTENTS.byteLength,
           theirDigest: encryptedAttachment.digest,
         });
@@ -661,8 +664,8 @@ describe('Crypto', () => {
         );
         const decryptedAttachment = await decryptAttachmentV2({
           ciphertextPath,
-          id: 'test',
+          idForLogging: 'test',
-          keys,
+          ...splitKeys(keys),
           size: data.byteLength,
           theirDigest: encryptedAttachment.digest,
         });
@@ -770,5 +773,205 @@ describe('Crypto', () => {
         }
       }
     });
+
+    describe('decryptAttachmentV2 with outer layer of encryption', () => {
+      async function doubleEncrypt({
+        plaintextAbsolutePath,
+        innerKeys,
+        outerKeys,
+      }: {
+        plaintextAbsolutePath: string;
+        innerKeys: Uint8Array;
+        outerKeys: Uint8Array;
+      }) {
+        let innerCiphertextPath;
+        let outerCiphertextPath;
+        let innerEncryptedAttachment;
+        try {
+          innerEncryptedAttachment = await encryptAttachmentV2({
+            keys: innerKeys,
+            plaintextAbsolutePath,
+          });
+          innerCiphertextPath =
+            window.Signal.Migrations.getAbsoluteAttachmentPath(
+              innerEncryptedAttachment.path
+            );
+
+          const outerEncryptedAttachment = await encryptAttachmentV2({
+            keys: outerKeys,
+            plaintextAbsolutePath: innerCiphertextPath,
+            // We (and the server!) don't pad the second layer
+            dangerousTestOnlySkipPadding: true,
+          });
+
+          outerCiphertextPath =
+            window.Signal.Migrations.getAbsoluteAttachmentPath(
+              outerEncryptedAttachment.path
+            );
+        } finally {
+          if (innerCiphertextPath) {
+            unlinkSync(innerCiphertextPath);
+          }
+        }
+        return {
+          outerCiphertextPath,
+          innerEncryptedAttachment,
+        };
+      }
+
+      it('v2 roundtrips smaller file (all on disk)', async () => {
+        const outerKeys = generateAttachmentKeys();
+        const innerKeys = generateAttachmentKeys();
+        let plaintextPath;
+        let outerCiphertextPath;
+
+        try {
+          const encryptResult = await doubleEncrypt({
+            plaintextAbsolutePath: FILE_PATH,
+            innerKeys,
+            outerKeys,
+          });
+          outerCiphertextPath = encryptResult.outerCiphertextPath;
+
+          const decryptedAttachment = await decryptAttachmentV2({
+            ciphertextPath: outerCiphertextPath,
+            idForLogging: 'test',
+            ...splitKeys(innerKeys),
+            size: FILE_CONTENTS.byteLength,
+            theirDigest: encryptResult.innerEncryptedAttachment.digest,
+            outerEncryption: splitKeys(outerKeys),
+          });
+
+          plaintextPath = window.Signal.Migrations.getAbsoluteAttachmentPath(
+            decryptedAttachment.path
+          );
+          const plaintext = readFileSync(plaintextPath);
+          assert.isTrue(constantTimeEqual(FILE_CONTENTS, plaintext));
+          assert.strictEqual(
+            encryptResult.innerEncryptedAttachment.plaintextHash,
+            GHOST_KITTY_HASH
+          );
+          assert.strictEqual(
+            decryptedAttachment.plaintextHash,
+            encryptResult.innerEncryptedAttachment.plaintextHash
+          );
+        } finally {
+          if (plaintextPath) {
+            unlinkSync(plaintextPath);
+          }
+          if (outerCiphertextPath) {
+            unlinkSync(outerCiphertextPath);
+          }
+        }
+      });
+
+      it('v2 roundtrips random data (all on disk)', async () => {
+        const sourcePath = join(tempDir, 'random');
+        // Get sufficient large file to have more than 64kb of padding and
+        // trigger push back on the streams.
+        const data = getRandomBytes(5 * 1024 * 1024);
+
+        writeFileSync(sourcePath, data);
+
+        const outerKeys = generateAttachmentKeys();
+        const innerKeys = generateAttachmentKeys();
+        let plaintextPath;
+        let outerCiphertextPath;
+
+        try {
+          const encryptResult = await doubleEncrypt({
+            plaintextAbsolutePath: sourcePath,
+            innerKeys,
+            outerKeys,
+          });
+          outerCiphertextPath = encryptResult.outerCiphertextPath;
+
+          const decryptedAttachment = await decryptAttachmentV2({
+            ciphertextPath: outerCiphertextPath,
+            idForLogging: 'test',
+            ...splitKeys(innerKeys),
+            size: data.byteLength,
+            theirDigest: encryptResult.innerEncryptedAttachment.digest,
+            outerEncryption: splitKeys(outerKeys),
+          });
+          plaintextPath = window.Signal.Migrations.getAbsoluteAttachmentPath(
+            decryptedAttachment.path
+          );
+          const plaintext = readFileSync(plaintextPath);
+          assert.isTrue(constantTimeEqual(data, plaintext));
+        } finally {
+          if (sourcePath) {
+            unlinkSync(sourcePath);
+          }
+          if (plaintextPath) {
+            unlinkSync(plaintextPath);
+          }
+          if (outerCiphertextPath) {
+            unlinkSync(outerCiphertextPath);
+          }
+        }
+      });
+
+      it('v2 fails if outer encryption mac is wrong', async () => {
+        const sourcePath = join(tempDir, 'random');
+        // Get sufficient large file to have more than 64kb of padding and
+        // trigger push back on the streams.
+        const data = getRandomBytes(5 * 1024 * 1024);
+
+        writeFileSync(sourcePath, data);
+
+        const outerKeys = generateAttachmentKeys();
+        const innerKeys = generateAttachmentKeys();
+        let outerCiphertextPath;
+
+        try {
+          const encryptResult = await doubleEncrypt({
+            plaintextAbsolutePath: sourcePath,
+            innerKeys,
+            outerKeys,
+          });
+          outerCiphertextPath = encryptResult.outerCiphertextPath;
+
+          await assert.isRejected(
+            decryptAttachmentV2({
+              ciphertextPath: outerCiphertextPath,
+              idForLogging: 'test',
+              ...splitKeys(innerKeys),
+              size: data.byteLength,
+              theirDigest: encryptResult.innerEncryptedAttachment.digest,
+              outerEncryption: {
+                aesKey: splitKeys(outerKeys).aesKey,
+                macKey: splitKeys(innerKeys).macKey, // wrong mac!
+              },
+            }),
+            /Bad outer encryption MAC/
+          );
+        } finally {
+          if (sourcePath) {
+            unlinkSync(sourcePath);
+          }
+          if (outerCiphertextPath) {
+            unlinkSync(outerCiphertextPath);
+          }
+        }
+      });
+    });
+  });
+
+  describe('getAesCbcCiphertextLength', () => {
+    function encrypt(length: number) {
+      const cipher = createCipheriv(
+        CipherType.AES256CBC,
+        getRandomBytes(32),
+        getRandomBytes(16)
+      );
+      const encrypted = cipher.update(Buffer.alloc(length));
+      return Buffer.concat([encrypted, cipher.final()]);
+    }
+    it('calculates cipherTextLength correctly', () => {
+      for (let i = 0; i < 128; i += 1) {
+        assert.strictEqual(getAesCbcCiphertextLength(i), encrypt(i).length);
+      }
+    });
   });
 });

ts/test-electron/backup/backup_attachments_test.ts

@@ -1,102 +0,0 @@
-// Copyright 2024 Signal Messenger, LLC
-// SPDX-License-Identifier: AGPL-3.0-only
-
-import { assert } from 'chai';
-import * as sinon from 'sinon';
-
-import { getMediaNameForBackup } from '../../util/attachments/getMediaNameForBackup';
-import { IMAGE_PNG } from '../../types/MIME';
-import { sha256 } from '../../Crypto';
-import { DAY } from '../../util/durations';
-
-describe('getMediaNameForBackup', () => {
-  const TEST_HASH = sha256(Buffer.from('testattachmentdata'));
-  const TEST_HASH_BASE_64 =
-    // calculated as Buffer.from(TEST_HASH).toString('base64')
-    'ds5/U14lB2ziO90B7MldFTJUQdyw4qQ9y6Gnt9fmHL0=';
-
-  afterEach(function (this: Mocha.Context) {
-    sinon.restore();
-  });
-
-  it("should return base64 encoded plaintextHash if it's already been calculated", async () => {
-    assert.strictEqual(
-      await getMediaNameForBackup(
-        {
-          contentType: IMAGE_PNG,
-          size: 100,
-          plaintextHash: Buffer.from(TEST_HASH).toString('hex'),
-        },
-        'senderAci',
-        Date.now()
-      ),
-      TEST_HASH_BASE_64
-    );
-  });
-
-  it('should calculate hash from file on disk if plaintextHash has not yet been calculated', async () => {
-    const stubbedGetHashFromDisk = sinon
-      .stub()
-      .callsFake(async (_path: string) =>
-        Buffer.from(TEST_HASH).toString('hex')
-      );
-
-    const mediaName = await getMediaNameForBackup(
-      {
-        contentType: IMAGE_PNG,
-        size: 100,
-        path: 'path/to/file',
-      },
-      'senderAci',
-      Date.now(),
-      { getPlaintextHashForAttachmentOnDisk: stubbedGetHashFromDisk }
-    );
-
-    assert.strictEqual(stubbedGetHashFromDisk.callCount, 1);
-    assert.strictEqual(mediaName, TEST_HASH_BASE_64);
-  });
-
-  it('should return temporary identifier if attachment is undownloaded but in attachment tier', async () => {
-    const mediaName = await getMediaNameForBackup(
-      {
-        contentType: IMAGE_PNG,
-        size: 100,
-        cdnKey: 'cdnKey',
-      },
-      'senderAci',
-      Date.now()
-    );
-
-    assert.strictEqual(mediaName, 'senderAci_cdnKey');
-  });
-
-  it('should return temporary identifier if undownloaded attachment has temporary error', async () => {
-    const mediaName = await getMediaNameForBackup(
-      {
-        contentType: IMAGE_PNG,
-        size: 100,
-        cdnKey: 'cdnKey',
-        error: true,
-        key: 'attachmentkey',
-      },
-      'senderAci',
-      Date.now()
-    );
-
-    assert.strictEqual(mediaName, 'senderAci_cdnKey');
-  });
-
-  it('should return undefined if attachment is too old to be in attachment tier', async () => {
-    const mediaName = await getMediaNameForBackup(
-      {
-        contentType: IMAGE_PNG,
-        size: 100,
-        cdnKey: 'cdnKey',
-      },
-      'senderAci',
-      Date.now() - 31 * DAY
-    );
-
-    assert.strictEqual(mediaName, undefined);
-  });
-});

ts/test-node/backups/filePointer_test.ts

@@ -0,0 +1,149 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+import { assert } from 'chai';
+import Long from 'long';
+import { Backups } from '../../protobuf';
+import { convertFilePointerToAttachment } from '../../services/backups/util/filePointers';
+import { APPLICATION_OCTET_STREAM, IMAGE_PNG } from '../../types/MIME';
+import * as Bytes from '../../Bytes';
+
+describe('convertFilePointerToAttachment', () => {
+  it('processes filepointer with attachmentLocator', () => {
+    const result = convertFilePointerToAttachment(
+      new Backups.FilePointer({
+        contentType: 'image/png',
+        width: 100,
+        height: 100,
+        blurHash: 'blurhash',
+        fileName: 'filename',
+        caption: 'caption',
+        incrementalMac: Bytes.fromString('incrementalMac'),
+        incrementalMacChunkSize: 1000,
+        attachmentLocator: new Backups.FilePointer.AttachmentLocator({
+          size: 128,
+          cdnKey: 'cdnKey',
+          cdnNumber: 2,
+          key: Bytes.fromString('key'),
+          digest: Bytes.fromString('digest'),
+          uploadTimestamp: Long.fromNumber(1970),
+        }),
+      })
+    );
+
+    assert.deepStrictEqual(result, {
+      contentType: IMAGE_PNG,
+      width: 100,
+      height: 100,
+      size: 128,
+      blurHash: 'blurhash',
+      fileName: 'filename',
+      caption: 'caption',
+      cdnKey: 'cdnKey',
+      cdnNumber: 2,
+      key: Bytes.toBase64(Bytes.fromString('key')),
+      digest: Bytes.toBase64(Bytes.fromString('digest')),
+      uploadTimestamp: 1970,
+      incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
+      incrementalMacChunkSize: 1000,
+    });
+  });
+
+  it('processes filepointer with backupLocator and missing fields', () => {
+    const result = convertFilePointerToAttachment(
+      new Backups.FilePointer({
+        contentType: 'image/png',
+        width: 100,
+        height: 100,
+        blurHash: 'blurhash',
+        fileName: 'filename',
+        caption: 'caption',
+        incrementalMac: Bytes.fromString('incrementalMac'),
+        incrementalMacChunkSize: 1000,
+        backupLocator: new Backups.FilePointer.BackupLocator({
+          mediaName: 'mediaName',
+          cdnNumber: 3,
+          size: 128,
+          key: Bytes.fromString('key'),
+          digest: Bytes.fromString('digest'),
+          transitCdnKey: 'transitCdnKey',
+          transitCdnNumber: 2,
+        }),
+      })
+    );
+
+    assert.deepStrictEqual(result, {
+      contentType: IMAGE_PNG,
+      width: 100,
+      height: 100,
+      size: 128,
+      blurHash: 'blurhash',
+      fileName: 'filename',
+      caption: 'caption',
+      cdnKey: 'transitCdnKey',
+      cdnNumber: 2,
+      key: Bytes.toBase64(Bytes.fromString('key')),
+      digest: Bytes.toBase64(Bytes.fromString('digest')),
+      incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
+      incrementalMacChunkSize: 1000,
+      backupLocator: {
+        mediaName: 'mediaName',
+        cdnNumber: 3,
+      },
+    });
+  });
+
+  it('processes filepointer with invalidAttachmentLocator', () => {
+    const result = convertFilePointerToAttachment(
+      new Backups.FilePointer({
+        contentType: 'image/png',
+        width: 100,
+        height: 100,
+        blurHash: 'blurhash',
+        fileName: 'filename',
+        caption: 'caption',
+        incrementalMac: Bytes.fromString('incrementalMac'),
+        incrementalMacChunkSize: 1000,
+        invalidAttachmentLocator:
+          new Backups.FilePointer.InvalidAttachmentLocator(),
+      })
+    );
+
+    assert.deepStrictEqual(result, {
+      contentType: IMAGE_PNG,
+      width: 100,
+      height: 100,
+      blurHash: 'blurhash',
+      fileName: 'filename',
+      caption: 'caption',
+      incrementalMac: Bytes.toBase64(Bytes.fromString('incrementalMac')),
+      incrementalMacChunkSize: 1000,
+      size: 0,
+      error: true,
+    });
+  });
+
+  it('accepts missing / null fields and adds defaults to contentType and size', () => {
+    const result = convertFilePointerToAttachment(
+      new Backups.FilePointer({
+        backupLocator: new Backups.FilePointer.BackupLocator(),
+      })
+    );
+
+    assert.deepStrictEqual(result, {
+      contentType: APPLICATION_OCTET_STREAM,
+      size: 0,
+      width: undefined,
+      height: undefined,
+      blurHash: undefined,
+      fileName: undefined,
+      caption: undefined,
+      cdnKey: undefined,
+      cdnNumber: undefined,
+      key: undefined,
+      digest: undefined,
+      incrementalMac: undefined,
+      incrementalMacChunkSize: undefined,
+      backupLocator: undefined,
+    });
+  });
+});

ts/textsecure/MessageReceiver.ts

@@ -83,7 +83,7 @@ import {
 import { processSyncMessage } from './processSyncMessage';
 import type { EventHandler } from './EventTarget';
 import EventTarget from './EventTarget';
-import { downloadAttachmentV2 } from './downloadAttachment';
+import { downloadAttachment } from './downloadAttachment';
 import type { IncomingWebSocketRequest } from './WebsocketResources';
 import type { ContactDetailsWithAvatar } from './ContactsParser';
 import { parseContactsV2 } from './ContactsParser';
@@ -3764,7 +3764,7 @@ export default class MessageReceiver
     options?: { timeout?: number; disableRetries?: boolean }
   ): Promise<AttachmentType> {
     const cleaned = processAttachment(attachment);
-    return downloadAttachmentV2(this.server, cleaned, options);
+    return downloadAttachment(this.server, cleaned, options);
   }
 
   private async handleEndSession(

ts/textsecure/Types.d.ts

@@ -4,7 +4,7 @@
 import type { SignalService as Proto } from '../protobuf';
 import type { IncomingWebSocketRequest } from './WebsocketResources';
 import type { ServiceIdString, AciString, PniString } from '../types/ServiceId';
-import type { TextAttachmentType } from '../types/Attachment';
+import type { AttachmentType, TextAttachmentType } from '../types/Attachment';
 import type { GiftBadgeStates } from '../components/conversation/Message';
 import type { MIMEType } from '../types/MIME';
 import type { DurationInSeconds } from '../util/durations';
@@ -117,6 +117,7 @@ export type ProcessedAttachment = {
   blurHash?: string;
   cdnNumber?: number;
   textAttachment?: Omit<TextAttachmentType, 'preview'>;
+  backupLocator?: AttachmentType['backupLocator'];
 };
 
 export type ProcessedGroupV2Context = {

ts/textsecure/WebAPI.ts

@@ -23,10 +23,7 @@ import * as durations from '../util/durations';
 import type { ExplodePromiseResultType } from '../util/explodePromise';
 import { explodePromise } from '../util/explodePromise';
 import { getUserAgent } from '../util/getUserAgent';
-import {
+import { getTimeoutStream } from '../util/getStreamWithTimeout';
-  getTimeoutStream,
-  getStreamWithTimeout,
-} from '../util/getStreamWithTimeout';
 import { formatAcceptLanguageHeader } from '../util/userLanguages';
 import { toWebSafeBase64, fromWebSafeBase64 } from '../util/webSafeBase64';
 import { getBasicAuth } from '../util/getBasicAuth';
@@ -1154,22 +1151,25 @@ export type WebAPIType = {
     imageFiles: Array<string>
   ) => Promise<Array<Uint8Array>>;
   getArtAuth: () => Promise<ArtAuthType>;
-  getAttachment: (
+  getAttachmentFromBackupTier: (args: {
-    cdnKey: string,
+    mediaId: string;
-    cdnNumber?: number,
+    backupDir: string;
+    mediaDir: string;
+    cdnNumber: number;
+    headers: Record<string, string>;
     options?: {
       disableRetries?: boolean;
       timeout?: number;
-    }
+    };
-  ) => Promise<Uint8Array>;
+  }) => Promise<Readable>;
-  getAttachmentV2: (
+  getAttachment: (args: {
-    cdnKey: string,
+    cdnKey: string;
-    cdnNumber?: number,
+    cdnNumber?: number;
     options?: {
       disableRetries?: boolean;
       timeout?: number;
-    }
+    };
-  ) => Promise<Readable>;
+  }) => Promise<Readable>;
   getAvatar: (path: string) => Promise<Uint8Array>;
   getHasSubscription: (subscriberId: Uint8Array) => Promise<boolean>;
   getGroup: (options: GroupCredentialsType) => Promise<Proto.Group>;
@@ -1650,7 +1650,7 @@ export function initialize({
       getArtAuth,
       getArtProvisioningSocket,
       getAttachment,
-      getAttachmentV2,
+      getAttachmentFromBackupTier,
       getAvatar,
       getBackupCredentials,
       getBackupCDNCredentials,
@@ -3310,84 +3310,89 @@ export function initialize({
       return packId;
     }
 
-    async function getAttachment(
+    // Transit tier is the default place for normal (non-backup) attachments.
-      cdnKey: string,
+    // Called "transit" because it is transitory
-      cdnNumber?: number,
+    async function getAttachment({
+      cdnKey,
+      cdnNumber,
+      options,
+    }: {
+      cdnKey: string;
+      cdnNumber?: number;
       options?: {
         disableRetries?: boolean;
         timeout?: number;
-      }
+      };
-    ) {
+    }) {
-      const abortController = new AbortController();
+      return _getAttachment({
+        cdnPath: `/attachments/${cdnKey}`,
+        cdnNumber: cdnNumber ?? 0,
+        redactor: _createRedactor(cdnKey),
+        options,
+      });
+    }
 
-      const cdnUrl = isNumber(cdnNumber)
+    async function getAttachmentFromBackupTier({
-        ? cdnUrlObject[cdnNumber] ?? cdnUrlObject['0']
+      mediaId,
-        : cdnUrlObject['0'];
+      backupDir,
+      mediaDir,
+      cdnNumber,
+      headers,
+      options,
+    }: {
+      mediaId: string;
+      backupDir: string;
+      mediaDir: string;
+      cdnNumber: number;
+      headers: Record<string, string>;
+      options?: {
+        disableRetries?: boolean;
+        timeout?: number;
+      };
+    }) {
+      return _getAttachment({
+        cdnPath: `/backups/${backupDir}/${mediaDir}/${mediaId}`,
+        cdnNumber,
+        headers,
+        redactor: _createRedactor(backupDir, mediaDir, mediaId),
+        options,
+      });
+    }
+
+    async function _getAttachment({
+      cdnPath,
+      cdnNumber,
+      headers,
+      redactor,
+      options,
+    }: {
+      cdnPath: string;
+      cdnNumber: number;
+      headers?: Record<string, string>;
+      redactor: RedactUrl;
+      options?: {
+        disableRetries?: boolean;
+        timeout?: number;
+      };
+    }): Promise<Readable> {
+      const abortController = new AbortController();
+      const cdnUrl = cdnUrlObject[cdnNumber] ?? cdnUrlObject['0'];
       // This is going to the CDN, not the service, so we use _outerAjax
-      const stream = await _outerAjax(`${cdnUrl}/attachments/${cdnKey}`, {
+      const downloadStream = await _outerAjax(`${cdnUrl}${cdnPath}`, {
+        headers,
         certificateAuthority,
         disableRetries: options?.disableRetries,
         proxyUrl,
         responseType: 'stream',
         timeout: options?.timeout || 0,
         type: 'GET',
-        redactUrl: _createRedactor(cdnKey),
+        redactUrl: redactor,
         version,
         abortSignal: abortController.signal,
       });
 
-      const streamPromise = getStreamWithTimeout(stream, {
-        name: `getAttachment(${cdnKey})`,
-        timeout: GET_ATTACHMENT_CHUNK_TIMEOUT,
-        abortController,
-      });
-
-      // Add callback to central store that would reject a promise
-      const { promise: cancelPromise, reject } = explodePromise<Uint8Array>();
-      const inflightRequest = (error: Error) => {
-        reject(error);
-        abortController.abort();
-      };
-      registerInflightRequest(inflightRequest);
-
-      try {
-        return Promise.race([streamPromise, cancelPromise]);
-      } finally {
-        unregisterInFlightRequest(inflightRequest);
-      }
-    }
-
-    async function getAttachmentV2(
-      cdnKey: string,
-      cdnNumber?: number,
-      options?: {
-        disableRetries?: boolean;
-        timeout?: number;
-      }
-    ): Promise<Readable> {
-      const abortController = new AbortController();
-
-      const cdnUrl = isNumber(cdnNumber)
-        ? cdnUrlObject[cdnNumber] ?? cdnUrlObject['0']
-        : cdnUrlObject['0'];
-      // This is going to the CDN, not the service, so we use _outerAjax
-      const downloadStream = await _outerAjax(
-        `${cdnUrl}/attachments/${cdnKey}`,
-        {
-          certificateAuthority,
-          disableRetries: options?.disableRetries,
-          proxyUrl,
-          responseType: 'stream',
-          timeout: options?.timeout || 0,
-          type: 'GET',
-          redactUrl: _createRedactor(cdnKey),
-          version,
-          abortSignal: abortController.signal,
-        }
-      );
-
       const timeoutStream = getTimeoutStream({
-        name: `getAttachment(${cdnKey})`,
+        name: `getAttachment(${redactor(cdnPath)})`,
         timeout: GET_ATTACHMENT_CHUNK_TIMEOUT,
         abortController,
       });

ts/textsecure/downloadAttachment.ts

@@ -10,110 +10,139 @@ import { ensureFile } from 'fs-extra';
 import * as log from '../logging/log';
 import * as Errors from '../types/errors';
 import { strictAssert } from '../util/assert';
-import { dropNull } from '../util/dropNull';
+import { AttachmentSizeError, type AttachmentType } from '../types/Attachment';
-import {
-  AttachmentSizeError,
-  type AttachmentType,
-  type DownloadedAttachmentType,
-} from '../types/Attachment';
 import * as MIME from '../types/MIME';
 import * as Bytes from '../Bytes';
-import { getFirstBytes, decryptAttachmentV1 } from '../Crypto';
+import {
+  deriveMediaIdFromMediaName,
+  deriveBackupMediaKeyMaterial,
+  type BackupMediaKeyMaterialType,
+} from '../Crypto';
 import {
   decryptAttachmentV2,
-  getAttachmentDownloadSize,
+  getAttachmentCiphertextLength,
   safeUnlinkSync,
+  splitKeys,
 } from '../AttachmentCrypto';
 import type { ProcessedAttachment } from './Types.d';
 import type { WebAPIType } from './WebAPI';
 import { createName, getRelativePath } from '../windows/attachments';
-import { redactCdnKey } from '../util/privacy';
+import { MediaTier } from '../types/AttachmentDownload';
+import { getBackupKey } from '../services/backups/crypto';
+import { backupsService } from '../services/backups';
 
-export function getCdn(attachment: ProcessedAttachment): string {
+const DEFAULT_BACKUP_CDN_NUMBER = 3;
-  const { cdnId, cdnKey } = attachment;
+
-  const cdn = cdnId || cdnKey;
+export function getCdnKey(attachment: ProcessedAttachment): string {
-  strictAssert(cdn, 'Attachment was missing cdnId or cdnKey');
+  const cdnKey = attachment.cdnId || attachment.cdnKey;
-  return cdn;
+  strictAssert(cdnKey, 'Attachment was missing cdnId or cdnKey');
+  return cdnKey;
 }
 
-export async function downloadAttachmentV1(
+function getMediaIdBytes(attachment: ProcessedAttachment): Uint8Array {
-  server: WebAPIType,
+  const mediaName = attachment.backupLocator?.mediaName;
-  attachment: ProcessedAttachment,
+  strictAssert(mediaName, 'Attachment was missing mediaName');
-  options?: {
+  const backupKey = getBackupKey();
-    disableRetries?: boolean;
+  return deriveMediaIdFromMediaName(backupKey, mediaName);
-    timeout?: number;
+}
-  }
+
-): Promise<DownloadedAttachmentType> {
+function getMediaIdForBackupTier(attachment: ProcessedAttachment): string {
-  const { cdnNumber, key, digest, size, contentType } = attachment;
+  return Bytes.toBase64url(getMediaIdBytes(attachment));
-  const cdn = getCdn(attachment);
+}
-
+
-  const encrypted = await server.getAttachment(
+function getBackupMediaKeyMaterial(
-    cdn,
+  attachment: ProcessedAttachment
-    dropNull(cdnNumber),
+): BackupMediaKeyMaterialType {
-    options
+  const mediaId = getMediaIdBytes(attachment);
-  );
+  const backupKey = getBackupKey();
-
+  return deriveBackupMediaKeyMaterial(backupKey, mediaId);
-  strictAssert(digest, 'Failure: Ask sender to update Signal and resend.');
+}
-  strictAssert(key, 'attachment has no key');
+
-
+async function getCdnNumberForBackupTier(
-  const paddedData = decryptAttachmentV1(
+  attachment: ProcessedAttachment
-    encrypted,
+): Promise<number> {
-    Bytes.fromBase64(key),
+  strictAssert(
-    Bytes.fromBase64(digest)
+    attachment.backupLocator,
-  );
+    'Attachment was missing backupLocator'
-
+  );
-  if (!isNumber(size)) {
+  const backupCdnNumber = attachment.backupLocator.cdnNumber;
-    throw new Error(
+  // TODO (DESKTOP-6983): get backupNumber by querying for all media
-      `downloadAttachment: Size was not provided, actual size was ${paddedData.byteLength}`
+  return backupCdnNumber || DEFAULT_BACKUP_CDN_NUMBER;
-    );
+}
-  }
+
-
+export async function downloadAttachment(
-  const data = getFirstBytes(paddedData, size);
-
-  return {
-    ...attachment,
-    size,
-    contentType: contentType
-      ? MIME.stringToMIMEType(contentType)
-      : MIME.APPLICATION_OCTET_STREAM,
-    data,
-  };
-}
-
-export async function downloadAttachmentV2(
   server: WebAPIType,
   attachment: ProcessedAttachment,
   options?: {
     disableRetries?: boolean;
     timeout?: number;
+    onlyFromTransitTier?: boolean;
+    logPrefix?: string;
   }
 ): Promise<AttachmentType> {
-  const { cdnNumber, contentType, digest, key, size } = attachment;
+  const logId = `${options?.logPrefix}/downloadAttachmentV2`;
-  const cdn = getCdn(attachment);
+
-  const logId = `downloadAttachmentV2(${redactCdnKey(cdn)}:`;
+  const { digest, key, size, contentType } = attachment;
 
   strictAssert(digest, `${logId}: missing digest`);
   strictAssert(key, `${logId}: missing key`);
   strictAssert(isNumber(size), `${logId}: missing size`);
 
-  // TODO (DESKTOP-6845): download attachments differentially based on their
+  // TODO (DESKTOP-7043): allow downloading from transit tier even if there is a backup
-  // media tier (i.e. transit tier or backup tier)
+  // locator (as fallback)
-  const downloadStream = await server.getAttachmentV2(
+  const mediaTier = attachment.backupLocator
-    cdn,
+    ? MediaTier.BACKUP
-    dropNull(cdnNumber),
+    : MediaTier.STANDARD;
-    options
+
-  );
+  let downloadedPath: string;
+  if (mediaTier === MediaTier.STANDARD) {
+    const cdnKey = getCdnKey(attachment);
+    const { cdnNumber } = attachment;
+
+    const downloadStream = await server.getAttachment({
+      cdnKey,
+      cdnNumber,
+      options,
+    });
+    downloadedPath = await downloadToDisk({ downloadStream, size });
+  } else {
+    const mediaId = getMediaIdForBackupTier(attachment);
+    const cdnNumber = await getCdnNumberForBackupTier(attachment);
+    const cdnCredentials =
+      await backupsService.credentials.getCDNReadCredentials(cdnNumber);
+
+    const backupDir = await backupsService.api.getBackupDir();
+    const mediaDir = await backupsService.api.getMediaDir();
+
+    const downloadStream = await server.getAttachmentFromBackupTier({
+      mediaId,
+      backupDir,
+      mediaDir,
+      headers: cdnCredentials.headers,
+      cdnNumber,
+      options,
+    });
+    downloadedPath = await downloadToDisk({
+      downloadStream,
+      size: getAttachmentCiphertextLength(size),
+    });
+  }
 
-  const cipherTextRelativePath = await downloadToDisk({ downloadStream, size });
   const cipherTextAbsolutePath =
-    window.Signal.Migrations.getAbsoluteAttachmentPath(cipherTextRelativePath);
+    window.Signal.Migrations.getAbsoluteAttachmentPath(downloadedPath);
 
+  const { aesKey, macKey } = splitKeys(Bytes.fromBase64(key));
   const { path, plaintextHash } = await decryptAttachmentV2({
     ciphertextPath: cipherTextAbsolutePath,
-    id: cdn,
+    idForLogging: logId,
-    keys: Bytes.fromBase64(key),
+    aesKey,
+    macKey,
     size,
     theirDigest: Bytes.fromBase64(digest),
+    outerEncryption:
+      mediaTier === 'backup'
+        ? getBackupMediaKeyMaterial(attachment)
+        : undefined,
   });
 
   safeUnlinkSync(cipherTextAbsolutePath);
@@ -141,7 +170,7 @@ async function downloadToDisk({
     window.Signal.Migrations.getAbsoluteAttachmentPath(relativeTargetPath);
   await ensureFile(absoluteTargetPath);
   const writeStream = createWriteStream(absoluteTargetPath);
-  const targetSize = getAttachmentDownloadSize(size);
+  const targetSize = getAttachmentCiphertextLength(size);
 
   try {
     await pipeline(downloadStream, checkSize(targetSize), writeStream);
@@ -164,17 +193,27 @@ async function downloadToDisk({
 // A simple transform that throws if it sees more than maxBytes on the stream.
 function checkSize(expectedBytes: number) {
   let totalBytes = 0;
+
+  // TODO (DESKTOP-7046): remove size buffer
+  const maximumSizeBeforeError = expectedBytes * 1.05;
   return new Transform({
     transform(chunk, encoding, callback) {
       totalBytes += chunk.byteLength;
-      if (totalBytes > expectedBytes) {
+      if (totalBytes > maximumSizeBeforeError) {
         callback(
           new AttachmentSizeError(
-            `checkSize: Received ${totalBytes} bytes, max is ${expectedBytes}, `
+            `checkSize: Received ${totalBytes} bytes, max is ${maximumSizeBeforeError}`
           )
         );
         return;
       }
+
+      if (totalBytes > expectedBytes) {
+        log.warn(
+          `checkSize: Received ${totalBytes} bytes, expected ${expectedBytes}`
+        );
+      }
+
       this.push(chunk, encoding);
       callback();
     },

ts/types/Attachment.ts

@@ -78,6 +78,14 @@ export type AttachmentType = {
   textAttachment?: TextAttachmentType;
   wasTooBig?: boolean;
 
+  incrementalMac?: string;
+  incrementalMacChunkSize?: number;
+
+  backupLocator?: {
+    mediaName: string;
+    cdnNumber?: number;
+  };
+
   /** Legacy field. Used only for downloading old attachments */
   id?: number;
 

ts/types/AttachmentDownload.ts

@@ -4,6 +4,11 @@ import { z } from 'zod';
 import { MIMETypeSchema, type MIMEType } from './MIME';
 import type { AttachmentType } from './Attachment';
 
+export enum MediaTier {
+  STANDARD = 'standard',
+  BACKUP = 'backup',
+}
+
 export const attachmentDownloadTypeSchema = z.enum([
   'long-message',
   'attachment',

ts/types/Util.ts

@@ -89,3 +89,6 @@ export type JSONWithUnknownFields<Value> = Value extends Record<
   : Value extends Array<infer E>
   ? ReadonlyArray<JSONWithUnknownFields<E>>
   : Value;
+
+export type WithRequiredProperties<T, P extends keyof T> = Omit<T, P> &
+  Required<Pick<T, P>>;

ts/types/backups.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import type { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
+import type { GetBackupCDNCredentialsResponseType } from '../textsecure/WebAPI';
 
 export type BackupCredentialType = Readonly<{
   credential: string;
@@ -18,3 +19,9 @@ export type BackupSignedPresentationType = Readonly<{
   headers: BackupPresentationHeadersType;
   level: BackupLevel;
 }>;
+
+export type BackupCdnReadCredentialType = Readonly<{
+  credentials: Readonly<GetBackupCDNCredentialsResponseType>;
+  retrievedAtMs: number;
+  cdnNumber: number;
+}>;

ts/util/attachments/getMediaNameForBackup.ts

@@ -1,67 +0,0 @@
-// Copyright 2024 Signal Messenger, LLC
-// SPDX-License-Identifier: AGPL-3.0-only
-
-import { getPlaintextHashForAttachmentOnDisk } from '../../AttachmentCrypto';
-import type { AttachmentType } from '../../types/Attachment';
-import { DAY } from '../durations';
-import * as log from '../../logging/log';
-import { isOlderThan } from '../timestamp';
-import { getCdn } from '../../textsecure/downloadAttachment';
-import * as Bytes from '../../Bytes';
-
-const TIME_IN_ATTACHMENT_TIER = 30 * DAY;
-
-// We store the plaintext hash as a hex string, but the mediaName should be
-// the base64 encoded version.
-function convertHexStringToBase64(hexString: string): string {
-  return Bytes.toBase64(Bytes.fromHex(hexString));
-}
-
-type GetMediaNameDependenciesType = {
-  getPlaintextHashForAttachmentOnDisk: (
-    path: string
-  ) => Promise<string | undefined>;
-};
-
-export async function getMediaNameForBackup(
-  attachment: AttachmentType,
-  senderAci: string,
-  messageTimestamp: number,
-  // allow optional dependency injection for testing
-  dependencies: GetMediaNameDependenciesType = {
-    getPlaintextHashForAttachmentOnDisk,
-  }
-): Promise<string | undefined> {
-  if (attachment.plaintextHash) {
-    return convertHexStringToBase64(attachment.plaintextHash);
-  }
-
-  if (attachment.path) {
-    const hashFromFileOnDisk =
-      await dependencies.getPlaintextHashForAttachmentOnDisk(
-        window.Signal.Migrations.getAbsoluteAttachmentPath(attachment.path)
-      );
-    if (!hashFromFileOnDisk) {
-      log.error(
-        'getMediaNameForBackup: no hash from attachment on disk (maybe it is empty?)'
-      );
-      return;
-    }
-    return convertHexStringToBase64(hashFromFileOnDisk);
-  }
-
-  const cdnKey = getCdn(attachment);
-  if (!cdnKey) {
-    log.error('getMediaNameForBackup: attachment has no cdnKey');
-    return;
-  }
-
-  if (isOlderThan(messageTimestamp, TIME_IN_ATTACHMENT_TIER)) {
-    log.error(
-      "getMediaNameForBackup: attachment is not downloaded but is too old; it's no longer in attachment tier."
-    );
-    return;
-  }
-
-  return `${senderAci}_${cdnKey}`;
-}

ts/util/downloadAttachment.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import type { AttachmentType } from '../types/Attachment';
-import { downloadAttachmentV2 as doDownloadAttachment } from '../textsecure/downloadAttachment';
+import { downloadAttachment as doDownloadAttachment } from '../textsecure/downloadAttachment';
 
 export class AttachmentNotFoundOnCdnError extends Error {}
 export async function downloadAttachment(