Implement API for backup upload

Co-authored-by: trevor-signal <131492920+trevor-signal@users.noreply.github.com>
This commit is contained in:
Fedor Indutny 2024-05-06 09:03:27 -07:00 committed by GitHub
parent 408444352f
commit de2def7119
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 109 additions and 5 deletions

View file

@ -1,6 +1,8 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import type { Readable } from 'stream';
import { strictAssert } from '../../util/assert';
import type {
WebAPIType,
@ -53,10 +55,11 @@ export class BackupAPI {
return (await this.getInfo()).backupName;
}
public async getUploadForm(): Promise<GetBackupUploadFormResponseType> {
return this.server.getBackupUploadForm(
await this.credentials.getHeadersForToday()
);
public async upload(stream: Readable): Promise<string> {
return this.server.uploadBackup({
headers: await this.credentials.getHeadersForToday(),
stream,
});
}
public async getMediaUploadForm(): Promise<GetBackupUploadFormResponseType> {

View file

@ -90,6 +90,12 @@ export class BackupsService {
}
}
public async upload(): Promise<void> {
const pipe = new PassThrough();
await Promise.all([this.api.upload(pipe), this.exportBackup(pipe)]);
}
// Test harness
public async exportBackupData(): Promise<Uint8Array> {
const sink = new PassThrough();

View file

@ -143,6 +143,7 @@ function _validateResponse(response: any, schema: any) {
const FIVE_MINUTES = 5 * durations.MINUTE;
const GET_ATTACHMENT_CHUNK_TIMEOUT = 10 * durations.SECOND;
const BACKUP_CDN_VERSION = 3;
type AgentCacheType = {
[name: string]: {
@ -171,7 +172,7 @@ type PromiseAjaxOptionsType = {
basicAuth?: string;
certificateAuthority?: string;
contentType?: string;
data?: Uint8Array | string;
data?: Uint8Array | Readable | string;
disableRetries?: boolean;
disableSessionResumption?: boolean;
headers?: HeaderListType;
@ -1013,6 +1014,11 @@ export type SetBackupSignatureKeyOptionsType = Readonly<{
backupIdPublicKey: Uint8Array;
}>;
export type UploadBackupOptionsType = Readonly<{
headers: BackupPresentationHeadersType;
stream: Readable;
}>;
export type BackupMediaItemType = Readonly<{
sourceAttachment: Readonly<{
cdn: number;
@ -1354,6 +1360,7 @@ export type WebAPIType = {
uploadAvatarRequestHeaders: UploadAvatarHeadersType,
avatarData: Uint8Array
) => Promise<string>;
uploadBackup: (options: UploadBackupOptionsType) => Promise<string>;
uploadGroupAvatar: (
avatarData: Uint8Array,
options: GroupCredentialsType
@ -1720,6 +1727,7 @@ export function initialize({
unregisterRequestHandler,
updateDeviceName,
uploadAvatar,
uploadBackup,
uploadGroupAvatar,
whoami,
};
@ -2733,6 +2741,93 @@ export function initialize({
return getBackupUploadFormResponseSchema.parse(res);
}
async function uploadBackup({ headers, stream }: UploadBackupOptionsType) {
const {
signedUploadLocation,
headers: uploadHeaders,
cdn,
key,
} = await getBackupUploadForm(headers);
strictAssert(
cdn === BACKUP_CDN_VERSION,
'uploadBackup: unexpected cdn version'
);
let size = 0n;
stream.pause();
stream.on('data', chunk => {
size += BigInt(chunk.length);
});
const uploadOptions = {
certificateAuthority,
proxyUrl,
timeout: 0,
type: 'POST' as const,
version,
headers: {
...uploadHeaders,
'Tus-Resumable': '1.0.0',
'Content-Type': 'application/offset+octet-stream',
'Upload-Defer-Length': '1',
},
redactUrl: () => {
const tmp = new URL(signedUploadLocation);
tmp.search = '';
tmp.pathname = '';
return `${tmp}[REDACTED]`;
},
data: stream,
responseType: 'byteswithdetails' as const,
};
let response: Response;
try {
({ response } = await _outerAjax(signedUploadLocation, uploadOptions));
} catch (e) {
// Another upload in progress, getting 409 should have aborted it.
if (e instanceof HTTPError && e.code === 409) {
log.warn('uploadBackup: aborting previous unfinished upload');
({ response } = await _outerAjax(
signedUploadLocation,
uploadOptions
));
} else {
throw e;
}
}
const uploadLocation = response.headers.get('location');
strictAssert(uploadLocation, 'backup response header has no location');
// Finish the upload by sending a PATCH with the stream length
// This is going to the CDN, not the service, so we use _outerAjax
await _outerAjax(uploadLocation, {
certificateAuthority,
proxyUrl,
timeout: 0,
type: 'PATCH',
version,
headers: {
...uploadHeaders,
'Tus-Resumable': '1.0.0',
'Content-Type': 'application/offset+octet-stream',
'Upload-Offset': String(size),
'Upload-Length': String(size),
},
redactUrl: () => {
const tmp = new URL(uploadLocation);
tmp.search = '';
tmp.pathname = '';
return `${tmp}[REDACTED]`;
},
});
return key;
}
async function refreshBackup(headers: BackupPresentationHeadersType) {
await _ajax({
call: 'backup',