mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Enable downloading attachments from backup CDN

Browse source
This commit is contained in:
trevor-signal 2024-05-02 13:11:34 -04:00 • committed by GitHub
parent 2964006b79
commit 1e8047cf73
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 989 additions and 385 deletions
Download patch file Download diff file Expand all files Collapse all files

4
ts/textsecure/MessageReceiver.ts
View file

@ -83,7 +83,7 @@ import {
import { processSyncMessage } from './processSyncMessage';
import type { EventHandler } from './EventTarget';
import EventTarget from './EventTarget';
import { downloadAttachmentV2 } from './downloadAttachment';
import { downloadAttachment } from './downloadAttachment';
import type { IncomingWebSocketRequest } from './WebsocketResources';
import type { ContactDetailsWithAvatar } from './ContactsParser';
import { parseContactsV2 } from './ContactsParser';
@ -3764,7 +3764,7 @@ export default class MessageReceiver
options?: { timeout?: number; disableRetries?: boolean }
): Promise<AttachmentType> {
const cleaned = processAttachment(attachment);
return downloadAttachmentV2(this.server, cleaned, options);
return downloadAttachment(this.server, cleaned, options);
}
private async handleEndSession(

3
ts/textsecure/Types.d.ts vendored
View file

@ -4,7 +4,7 @@
import type { SignalService as Proto } from '../protobuf';
import type { IncomingWebSocketRequest } from './WebsocketResources';
import type { ServiceIdString, AciString, PniString } from '../types/ServiceId';
import type { TextAttachmentType } from '../types/Attachment';
import type { AttachmentType, TextAttachmentType } from '../types/Attachment';
import type { GiftBadgeStates } from '../components/conversation/Message';
import type { MIMEType } from '../types/MIME';
import type { DurationInSeconds } from '../util/durations';
@ -117,6 +117,7 @@ export type ProcessedAttachment = {
blurHash?: string;
cdnNumber?: number;
textAttachment?: Omit<TextAttachmentType, 'preview'>;
backupLocator?: AttachmentType['backupLocator'];
};
export type ProcessedGroupV2Context = {

159
ts/textsecure/WebAPI.ts
View file

@ -23,10 +23,7 @@ import * as durations from '../util/durations';
import type { ExplodePromiseResultType } from '../util/explodePromise';
import { explodePromise } from '../util/explodePromise';
import { getUserAgent } from '../util/getUserAgent';
import {
getTimeoutStream,
getStreamWithTimeout,
} from '../util/getStreamWithTimeout';
import { getTimeoutStream } from '../util/getStreamWithTimeout';
import { formatAcceptLanguageHeader } from '../util/userLanguages';
import { toWebSafeBase64, fromWebSafeBase64 } from '../util/webSafeBase64';
import { getBasicAuth } from '../util/getBasicAuth';
@ -1154,22 +1151,25 @@ export type WebAPIType = {
imageFiles: Array<string>
) => Promise<Array<Uint8Array>>;
getArtAuth: () => Promise<ArtAuthType>;
getAttachment: (
cdnKey: string,
cdnNumber?: number,
getAttachmentFromBackupTier: (args: {
mediaId: string;
backupDir: string;
mediaDir: string;
cdnNumber: number;
headers: Record<string, string>;
options?: {
disableRetries?: boolean;
timeout?: number;
}
) => Promise<Uint8Array>;
getAttachmentV2: (
cdnKey: string,
cdnNumber?: number,
};
}) => Promise<Readable>;
getAttachment: (args: {
cdnKey: string;
cdnNumber?: number;
options?: {
disableRetries?: boolean;
timeout?: number;
}
) => Promise<Readable>;
};
}) => Promise<Readable>;
getAvatar: (path: string) => Promise<Uint8Array>;
getHasSubscription: (subscriberId: Uint8Array) => Promise<boolean>;
getGroup: (options: GroupCredentialsType) => Promise<Proto.Group>;
@ -1650,7 +1650,7 @@ export function initialize({
getArtAuth,
getArtProvisioningSocket,
getAttachment,
getAttachmentV2,
getAttachmentFromBackupTier,
getAvatar,
getBackupCredentials,
getBackupCDNCredentials,
@ -3310,84 +3310,89 @@ export function initialize({
return packId;
}
async function getAttachment(
cdnKey: string,
cdnNumber?: number,
// Transit tier is the default place for normal (non-backup) attachments.
// Called "transit" because it is transitory
async function getAttachment({
cdnKey,
cdnNumber,
options,
}: {
cdnKey: string;
cdnNumber?: number;
options?: {
disableRetries?: boolean;
timeout?: number;
}
) {
const abortController = new AbortController();
};
}) {
return _getAttachment({
cdnPath: `/attachments/${cdnKey}`,
cdnNumber: cdnNumber ?? 0,
redactor: _createRedactor(cdnKey),
options,
});
}
const cdnUrl = isNumber(cdnNumber)
? cdnUrlObject[cdnNumber] ?? cdnUrlObject['0']
: cdnUrlObject['0'];
async function getAttachmentFromBackupTier({
mediaId,
backupDir,
mediaDir,
cdnNumber,
headers,
options,
}: {
mediaId: string;
backupDir: string;
mediaDir: string;
cdnNumber: number;
headers: Record<string, string>;
options?: {
disableRetries?: boolean;
timeout?: number;
};
}) {
return _getAttachment({
cdnPath: `/backups/${backupDir}/${mediaDir}/${mediaId}`,
cdnNumber,
headers,
redactor: _createRedactor(backupDir, mediaDir, mediaId),
options,
});
}
async function _getAttachment({
cdnPath,
cdnNumber,
headers,
redactor,
options,
}: {
cdnPath: string;
cdnNumber: number;
headers?: Record<string, string>;
redactor: RedactUrl;
options?: {
disableRetries?: boolean;
timeout?: number;
};
}): Promise<Readable> {
const abortController = new AbortController();
const cdnUrl = cdnUrlObject[cdnNumber] ?? cdnUrlObject['0'];
// This is going to the CDN, not the service, so we use _outerAjax
const stream = await _outerAjax(`${cdnUrl}/attachments/${cdnKey}`, {
const downloadStream = await _outerAjax(`${cdnUrl}${cdnPath}`, {
headers,
certificateAuthority,
disableRetries: options?.disableRetries,
proxyUrl,
responseType: 'stream',
timeout: options?.timeout || 0,
type: 'GET',
redactUrl: _createRedactor(cdnKey),
redactUrl: redactor,
version,
abortSignal: abortController.signal,
});
const streamPromise = getStreamWithTimeout(stream, {
name: `getAttachment(${cdnKey})`,
timeout: GET_ATTACHMENT_CHUNK_TIMEOUT,
abortController,
});
// Add callback to central store that would reject a promise
const { promise: cancelPromise, reject } = explodePromise<Uint8Array>();
const inflightRequest = (error: Error) => {
reject(error);
abortController.abort();
};
registerInflightRequest(inflightRequest);
try {
return Promise.race([streamPromise, cancelPromise]);
} finally {
unregisterInFlightRequest(inflightRequest);
}
}
async function getAttachmentV2(
cdnKey: string,
cdnNumber?: number,
options?: {
disableRetries?: boolean;
timeout?: number;
}
): Promise<Readable> {
const abortController = new AbortController();
const cdnUrl = isNumber(cdnNumber)
? cdnUrlObject[cdnNumber] ?? cdnUrlObject['0']
: cdnUrlObject['0'];
// This is going to the CDN, not the service, so we use _outerAjax
const downloadStream = await _outerAjax(
`${cdnUrl}/attachments/${cdnKey}`,
{
certificateAuthority,
disableRetries: options?.disableRetries,
proxyUrl,
responseType: 'stream',
timeout: options?.timeout || 0,
type: 'GET',
redactUrl: _createRedactor(cdnKey),
version,
abortSignal: abortController.signal,
}
);
const timeoutStream = getTimeoutStream({
name: `getAttachment(${cdnKey})`,
name: `getAttachment(${redactor(cdnPath)})`,
timeout: GET_ATTACHMENT_CHUNK_TIMEOUT,
abortController,
});

191
ts/textsecure/downloadAttachment.ts
View file

@ -10,110 +10,139 @@ import { ensureFile } from 'fs-extra';
import * as log from '../logging/log';
import * as Errors from '../types/errors';
import { strictAssert } from '../util/assert';
import { dropNull } from '../util/dropNull';
import {
AttachmentSizeError,
type AttachmentType,
type DownloadedAttachmentType,
} from '../types/Attachment';
import { AttachmentSizeError, type AttachmentType } from '../types/Attachment';
import * as MIME from '../types/MIME';
import * as Bytes from '../Bytes';
import { getFirstBytes, decryptAttachmentV1 } from '../Crypto';
import {
deriveMediaIdFromMediaName,
deriveBackupMediaKeyMaterial,
type BackupMediaKeyMaterialType,
} from '../Crypto';
import {
decryptAttachmentV2,
getAttachmentDownloadSize,
getAttachmentCiphertextLength,
safeUnlinkSync,
splitKeys,
} from '../AttachmentCrypto';
import type { ProcessedAttachment } from './Types.d';
import type { WebAPIType } from './WebAPI';
import { createName, getRelativePath } from '../windows/attachments';
import { redactCdnKey } from '../util/privacy';
import { MediaTier } from '../types/AttachmentDownload';
import { getBackupKey } from '../services/backups/crypto';
import { backupsService } from '../services/backups';
export function getCdn(attachment: ProcessedAttachment): string {
const { cdnId, cdnKey } = attachment;
const cdn = cdnId || cdnKey;
strictAssert(cdn, 'Attachment was missing cdnId or cdnKey');
return cdn;
const DEFAULT_BACKUP_CDN_NUMBER = 3;
export function getCdnKey(attachment: ProcessedAttachment): string {
const cdnKey = attachment.cdnId || attachment.cdnKey;
strictAssert(cdnKey, 'Attachment was missing cdnId or cdnKey');
return cdnKey;
}
export async function downloadAttachmentV1(
server: WebAPIType,
attachment: ProcessedAttachment,
options?: {
disableRetries?: boolean;
timeout?: number;
}
): Promise<DownloadedAttachmentType> {
const { cdnNumber, key, digest, size, contentType } = attachment;
const cdn = getCdn(attachment);
const encrypted = await server.getAttachment(
cdn,
dropNull(cdnNumber),
options
);
strictAssert(digest, 'Failure: Ask sender to update Signal and resend.');
strictAssert(key, 'attachment has no key');
const paddedData = decryptAttachmentV1(
encrypted,
Bytes.fromBase64(key),
Bytes.fromBase64(digest)
);
if (!isNumber(size)) {
throw new Error(
`downloadAttachment: Size was not provided, actual size was ${paddedData.byteLength}`
);
}
const data = getFirstBytes(paddedData, size);
return {
...attachment,
size,
contentType: contentType
? MIME.stringToMIMEType(contentType)
: MIME.APPLICATION_OCTET_STREAM,
data,
};
}
export async function downloadAttachmentV2(
function getMediaIdBytes(attachment: ProcessedAttachment): Uint8Array {
const mediaName = attachment.backupLocator?.mediaName;
strictAssert(mediaName, 'Attachment was missing mediaName');
const backupKey = getBackupKey();
return deriveMediaIdFromMediaName(backupKey, mediaName);
}
function getMediaIdForBackupTier(attachment: ProcessedAttachment): string {
return Bytes.toBase64url(getMediaIdBytes(attachment));
}
function getBackupMediaKeyMaterial(
attachment: ProcessedAttachment
): BackupMediaKeyMaterialType {
const mediaId = getMediaIdBytes(attachment);
const backupKey = getBackupKey();
return deriveBackupMediaKeyMaterial(backupKey, mediaId);
}
async function getCdnNumberForBackupTier(
attachment: ProcessedAttachment
): Promise<number> {
strictAssert(
attachment.backupLocator,
'Attachment was missing backupLocator'
);
const backupCdnNumber = attachment.backupLocator.cdnNumber;
// TODO (DESKTOP-6983): get backupNumber by querying for all media
return backupCdnNumber || DEFAULT_BACKUP_CDN_NUMBER;
}
export async function downloadAttachment(
server: WebAPIType,
attachment: ProcessedAttachment,
options?: {
disableRetries?: boolean;
timeout?: number;
onlyFromTransitTier?: boolean;
logPrefix?: string;
}
): Promise<AttachmentType> {
const { cdnNumber, contentType, digest, key, size } = attachment;
const cdn = getCdn(attachment);
const logId = `downloadAttachmentV2(${redactCdnKey(cdn)}:`;
const logId = `${options?.logPrefix}/downloadAttachmentV2`;
const { digest, key, size, contentType } = attachment;
strictAssert(digest, `${logId}: missing digest`);
strictAssert(key, `${logId}: missing key`);
strictAssert(isNumber(size), `${logId}: missing size`);
// TODO (DESKTOP-6845): download attachments differentially based on their
// media tier (i.e. transit tier or backup tier)
const downloadStream = await server.getAttachmentV2(
cdn,
dropNull(cdnNumber),
options
);
// TODO (DESKTOP-7043): allow downloading from transit tier even if there is a backup
// locator (as fallback)
const mediaTier = attachment.backupLocator
? MediaTier.BACKUP
: MediaTier.STANDARD;
let downloadedPath: string;
if (mediaTier === MediaTier.STANDARD) {
const cdnKey = getCdnKey(attachment);
const { cdnNumber } = attachment;
const downloadStream = await server.getAttachment({
cdnKey,
cdnNumber,
options,
});
downloadedPath = await downloadToDisk({ downloadStream, size });
} else {
const mediaId = getMediaIdForBackupTier(attachment);
const cdnNumber = await getCdnNumberForBackupTier(attachment);
const cdnCredentials =
await backupsService.credentials.getCDNReadCredentials(cdnNumber);
const backupDir = await backupsService.api.getBackupDir();
const mediaDir = await backupsService.api.getMediaDir();
const downloadStream = await server.getAttachmentFromBackupTier({
mediaId,
backupDir,
mediaDir,
headers: cdnCredentials.headers,
cdnNumber,
options,
});
downloadedPath = await downloadToDisk({
downloadStream,
size: getAttachmentCiphertextLength(size),
});
}
const cipherTextRelativePath = await downloadToDisk({ downloadStream, size });
const cipherTextAbsolutePath =
window.Signal.Migrations.getAbsoluteAttachmentPath(cipherTextRelativePath);
window.Signal.Migrations.getAbsoluteAttachmentPath(downloadedPath);
const { aesKey, macKey } = splitKeys(Bytes.fromBase64(key));
const { path, plaintextHash } = await decryptAttachmentV2({
ciphertextPath: cipherTextAbsolutePath,
id: cdn,
keys: Bytes.fromBase64(key),
idForLogging: logId,
aesKey,
macKey,
size,
theirDigest: Bytes.fromBase64(digest),
outerEncryption:
mediaTier === 'backup'
? getBackupMediaKeyMaterial(attachment)
: undefined,
});
safeUnlinkSync(cipherTextAbsolutePath);
@ -141,7 +170,7 @@ async function downloadToDisk({
window.Signal.Migrations.getAbsoluteAttachmentPath(relativeTargetPath);
await ensureFile(absoluteTargetPath);
const writeStream = createWriteStream(absoluteTargetPath);
const targetSize = getAttachmentDownloadSize(size);
const targetSize = getAttachmentCiphertextLength(size);
try {
await pipeline(downloadStream, checkSize(targetSize), writeStream);
@ -164,17 +193,27 @@ async function downloadToDisk({
// A simple transform that throws if it sees more than maxBytes on the stream.
function checkSize(expectedBytes: number) {
let totalBytes = 0;
// TODO (DESKTOP-7046): remove size buffer
const maximumSizeBeforeError = expectedBytes * 1.05;
return new Transform({
transform(chunk, encoding, callback) {
totalBytes += chunk.byteLength;
if (totalBytes > expectedBytes) {
if (totalBytes > maximumSizeBeforeError) {
callback(
new AttachmentSizeError(
`checkSize: Received ${totalBytes} bytes, max is ${expectedBytes}, `
`checkSize: Received ${totalBytes} bytes, max is ${maximumSizeBeforeError}`
)
);
return;
}
if (totalBytes > expectedBytes) {
log.warn(
`checkSize: Received ${totalBytes} bytes, expected ${expectedBytes}`
);
}
this.push(chunk, encoding);
callback();
},