mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Enable attachment backup uploading

Browse source
This commit is contained in:
trevor-signal 2024-05-29 19:46:43 -04:00 • committed by GitHub
parent 94a262b799
commit 4254356812
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 2054 additions and 534 deletions
Download patch file Download diff file Expand all files Collapse all files

8
ts/test-electron/Crypto_test.ts
View file

@ -5,7 +5,7 @@ import { assert } from 'chai';
import { readFileSync, unlinkSync, writeFileSync } from 'fs';
import { join } from 'path';
import { createCipheriv, randomBytes } from 'crypto';
import { createCipheriv } from 'crypto';
import * as log from '../logging/log';
import * as Bytes from '../Bytes';
import * as Curve from '../Curve';
@ -38,13 +38,13 @@ import {
} from '../Crypto';
import {
type HardcodedIVForEncryptionType,
KEY_SET_LENGTH,
_generateAttachmentIv,
decryptAttachmentV2,
encryptAttachmentV2ToDisk,
getAesCbcCiphertextLength,
getAttachmentCiphertextLength,
splitKeys,
generateAttachmentKeys,
} from '../AttachmentCrypto';
import { createTempDir, deleteTempDir } from '../updater/common';
import { uuidToBytes, bytesToUuid } from '../util/uuidToBytes';
@ -536,10 +536,6 @@ describe('Crypto', () => {
const FILE_HASH = sha256(FILE_CONTENTS);
let tempDir: string;
function generateAttachmentKeys(): Uint8Array {
return randomBytes(KEY_SET_LENGTH);
}
beforeEach(async () => {
tempDir = await createTempDir();
});

162
ts/test-electron/backup/filePointer_test.ts
View file

@ -7,13 +7,15 @@ import * as sinon from 'sinon';
import { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
import { Backups } from '../../protobuf';
import {
convertAttachmentToFilePointer,
getFilePointerForAttachment,
convertFilePointerToAttachment,
maybeGetBackupJobForAttachmentAndFilePointer,
} from '../../services/backups/util/filePointers';
import { APPLICATION_OCTET_STREAM, IMAGE_PNG } from '../../types/MIME';
import * as Bytes from '../../Bytes';
import type { AttachmentType } from '../../types/Attachment';
import { strictAssert } from '../../util/assert';
import type { GetBackupCdnInfoType } from '../../services/backups/util/mediaId';
describe('convertFilePointerToAttachment', () => {
it('processes filepointer with attachmentLocator', () => {
@ -167,6 +169,7 @@ function composeAttachment(
path: 'path/to/file.png',
key: 'key',
digest: 'digest',
iv: 'iv',
width: 100,
height: 100,
blurHash: 'blurhash',
@ -227,21 +230,30 @@ const filePointerWithInvalidLocator = new Backups.FilePointer({
async function testAttachmentToFilePointer(
attachment: AttachmentType,
filePointer: Backups.FilePointer,
options?: { backupLevel?: BackupLevel; backupCdnNumber?: number }
options?: {
backupLevel?: BackupLevel;
backupCdnNumber?: number;
updatedAttachment?: AttachmentType;
}
) {
async function _doTest(withBackupLevel: BackupLevel) {
assert.deepStrictEqual(
await convertAttachmentToFilePointer({
assert.deepEqual(
await getFilePointerForAttachment({
attachment,
backupLevel: withBackupLevel,
getBackupTierInfo: _mediaName => {
getBackupCdnInfo: async _mediaId => {
if (options?.backupCdnNumber != null) {
return { isInBackupTier: true, cdnNumber: options.backupCdnNumber };
}
return { isInBackupTier: false };
},
}),
filePointer
{
filePointer,
...(options?.updatedAttachment
? { updatedAttachment: options?.updatedAttachment }
: {}),
}
);
}
@ -253,7 +265,11 @@ async function testAttachmentToFilePointer(
}
}
describe('convertAttachmentToFilePointer', () => {
const notInBackupCdn: GetBackupCdnInfoType = async () => {
return { isInBackupTier: false };
};
describe('getFilePointerForAttachment', () => {
describe('not downloaded locally', () => {
const undownloadedAttachment = composeAttachment({ path: undefined });
it('returns invalidAttachmentLocator if missing critical decryption info', async () => {
@ -384,7 +400,7 @@ describe('convertAttachmentToFilePointer', () => {
});
});
describe('BackupLevel.Media', () => {
describe('if missing critical decryption info', () => {
describe('if missing critical decryption / encryption info', () => {
const FILE_PATH = join(__dirname, '../../../fixtures/ghost-kitty.mp4');
let sandbox: sinon.SinonSandbox;
@ -405,14 +421,14 @@ describe('convertAttachmentToFilePointer', () => {
sandbox.restore();
});
it('generates new key & digest and removes existing CDN info', async () => {
const result = await convertAttachmentToFilePointer({
it('if missing key, generates new key & digest and removes existing CDN info', async () => {
const { filePointer: result } = await getFilePointerForAttachment({
attachment: {
...downloadedAttachment,
key: undefined,
},
backupLevel: BackupLevel.Media,
getBackupTierInfo: () => ({ isInBackupTier: false }),
getBackupCdnInfo: notInBackupCdn,
});
const newKey = result.backupLocator?.key;
const newDigest = result.backupLocator?.digest;
@ -433,6 +449,53 @@ describe('convertAttachmentToFilePointer', () => {
})
);
});
it('if not on backup tier, and missing iv, regenerates encryption info', async () => {
const { filePointer: result } = await getFilePointerForAttachment({
attachment: {
...downloadedAttachment,
iv: undefined,
},
backupLevel: BackupLevel.Media,
getBackupCdnInfo: notInBackupCdn,
});
const newKey = result.backupLocator?.key;
const newDigest = result.backupLocator?.digest;
strictAssert(newDigest, 'must create new digest');
assert.deepStrictEqual(
result,
new Backups.FilePointer({
...filePointerWithBackupLocator,
backupLocator: new Backups.FilePointer.BackupLocator({
...defaultBackupLocator,
key: newKey,
digest: newDigest,
mediaName: Bytes.toBase64(newDigest),
transitCdnKey: undefined,
transitCdnNumber: undefined,
}),
})
);
});
it('if on backup tier, and not missing iv, does not regenerate encryption info', async () => {
await testAttachmentToFilePointer(
{
...downloadedAttachment,
iv: undefined,
},
new Backups.FilePointer({
...filePointerWithBackupLocator,
backupLocator: new Backups.FilePointer.BackupLocator({
...defaultBackupLocator,
cdnNumber: 12,
}),
}),
{ backupLevel: BackupLevel.Media, backupCdnNumber: 12 }
);
});
});
it('returns BackupLocator, with cdnNumber if in backup tier already', async () => {
@ -459,3 +522,80 @@ describe('convertAttachmentToFilePointer', () => {
});
});
});
describe('getBackupJobForAttachmentAndFilePointer', async () => {
const attachment = composeAttachment();
it('returns null if filePointer does not have backupLocator', async () => {
const { filePointer } = await getFilePointerForAttachment({
attachment,
backupLevel: BackupLevel.Messages,
getBackupCdnInfo: notInBackupCdn,
});
assert.strictEqual(
await maybeGetBackupJobForAttachmentAndFilePointer({
attachment,
filePointer,
messageReceivedAt: 100,
getBackupCdnInfo: notInBackupCdn,
}),
null
);
});
it('returns job if filePointer does have backupLocator', async () => {
const { filePointer, updatedAttachment } =
await getFilePointerForAttachment({
attachment,
backupLevel: BackupLevel.Media,
getBackupCdnInfo: notInBackupCdn,
});
const attachmentToUse = updatedAttachment ?? attachment;
assert.deepStrictEqual(
await maybeGetBackupJobForAttachmentAndFilePointer({
attachment: attachmentToUse,
filePointer,
messageReceivedAt: 100,
getBackupCdnInfo: notInBackupCdn,
}),
{
mediaName: 'digest',
receivedAt: 100,
type: 'standard',
data: {
path: 'path/to/file.png',
contentType: IMAGE_PNG,
keys: 'key',
digest: 'digest',
iv: 'iv',
size: 100,
transitCdnInfo: {
cdnKey: 'cdnKey',
cdnNumber: 2,
uploadTimestamp: 1234,
},
},
}
);
});
it('does not return job if already in backup tier', async () => {
const isInBackupTier = async () => ({
isInBackupTier: true,
cdnNumber: 42,
});
const { filePointer } = await getFilePointerForAttachment({
attachment,
backupLevel: BackupLevel.Media,
getBackupCdnInfo: isInBackupTier,
});
assert.deepStrictEqual(
await maybeGetBackupJobForAttachmentAndFilePointer({
attachment,
filePointer,
messageReceivedAt: 100,
getBackupCdnInfo: isInBackupTier,
}),
null
);
});
});

6
ts/test-electron/backup/helpers.ts
View file

@ -7,6 +7,7 @@ import { tmpdir } from 'os';
import { sortBy } from 'lodash';
import { createReadStream } from 'fs';
import { mkdtemp, rm } from 'fs/promises';
import * as sinon from 'sinon';
import type { MessageAttributesType } from '../../model-types';
import type {
@ -128,6 +129,10 @@ export async function asymmetricRoundtripHarness(
after: Array<MessageAttributesType>
): Promise<void> {
const outDir = await mkdtemp(path.join(tmpdir(), 'signal-temp-'));
const fetchAndSaveBackupCdnObjectMetadata = sinon.stub(
backupsService,
'fetchAndSaveBackupCdnObjectMetadata'
);
try {
const targetOutputFile = path.join(outDir, 'backup.bin');
@ -149,6 +154,7 @@ export async function asymmetricRoundtripHarness(
const actual = sortAndNormalize(messagesFromDatabase);
assert.deepEqual(expected, actual);
} finally {
fetchAndSaveBackupCdnObjectMetadata.restore();
await rm(outDir, { recursive: true });
}
}

264
ts/test-electron/services/AttachmentBackupManager_test.ts Normal file
View file

@ -0,0 +1,264 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import * as sinon from 'sinon';
import { assert } from 'chai';
import { join } from 'path';
import * as Bytes from '../../Bytes';
import {
AttachmentBackupManager,
FILE_NOT_FOUND_ON_TRANSIT_TIER_STATUS,
runAttachmentBackupJob,
} from '../../jobs/AttachmentBackupManager';
import type {
AttachmentBackupJobType,
CoreAttachmentBackupJobType,
} from '../../types/AttachmentBackup';
import dataInterface from '../../sql/Client';
import { getRandomBytes } from '../../Crypto';
import { VIDEO_MP4 } from '../../types/MIME';
const TRANSIT_CDN = 2;
const TRANSIT_CDN_FOR_NEW_UPLOAD = 42;
const BACKUP_CDN = 3;
describe('AttachmentBackupManager/JobManager', () => {
let backupManager: AttachmentBackupManager | undefined;
let runJob: sinon.SinonSpy;
let backupMediaBatch: sinon.SinonStub;
let backupsService = {};
let encryptAndUploadAttachment: sinon.SinonStub;
let getAbsoluteAttachmentPath: sinon.SinonStub;
let sandbox: sinon.SinonSandbox;
let isInCall: sinon.SinonStub;
function composeJob(
index: number,
overrides: Partial<CoreAttachmentBackupJobType['data']> = {}
): CoreAttachmentBackupJobType {
const mediaName = `mediaName${index}`;
return {
mediaName,
type: 'standard',
receivedAt: index,
data: {
path: 'ghost-kitty.mp4',
contentType: VIDEO_MP4,
keys: 'keys=',
iv: 'iv==',
digest: 'digest=',
transitCdnInfo: {
cdnKey: 'transitCdnKey',
cdnNumber: TRANSIT_CDN,
uploadTimestamp: Date.now(),
},
size: 128,
...overrides,
},
};
}
beforeEach(async () => {
await dataInterface.removeAll();
await window.storage.put('masterKey', Bytes.toBase64(getRandomBytes(32)));
sandbox = sinon.createSandbox();
isInCall = sandbox.stub().returns(false);
backupMediaBatch = sandbox
.stub()
.returns(Promise.resolve({ responses: [{ isSuccess: true, cdn: 3 }] }));
backupsService = {
credentials: {
getHeadersForToday: () => Promise.resolve({}),
},
getBackupCdnInfo: () => ({
isInBackupTier: false,
}),
};
encryptAndUploadAttachment = sinon.stub().returns({
cdnKey: 'newKeyOnTransitTier',
cdnNumber: TRANSIT_CDN_FOR_NEW_UPLOAD,
});
getAbsoluteAttachmentPath = sandbox.stub().callsFake(path => {
if (path === 'ghost-kitty.mp4') {
return join(__dirname, '../../../fixtures/ghost-kitty.mp4');
}
return getAbsoluteAttachmentPath.wrappedMethod(path);
});
runJob = sandbox.stub().callsFake((job: AttachmentBackupJobType) => {
return runAttachmentBackupJob(job, false, {
// @ts-expect-error incomplete stubbing
backupsService,
backupMediaBatch,
getAbsoluteAttachmentPath,
encryptAndUploadAttachment,
});
});
backupManager = new AttachmentBackupManager({
...AttachmentBackupManager.defaultParams,
shouldHoldOffOnStartingQueuedJobs: isInCall,
runJob,
});
});
afterEach(async () => {
sandbox.restore();
delete window.textsecure.server;
await backupManager?.stop();
});
async function addJobs(
num: number,
overrides: Partial<CoreAttachmentBackupJobType['data']> = {}
): Promise<Array<CoreAttachmentBackupJobType>> {
const jobs = new Array(num)
.fill(null)
.map((_, idx) => composeJob(idx, overrides));
for (const job of jobs) {
// eslint-disable-next-line no-await-in-loop
await backupManager?.addJob(job);
}
return jobs;
}
function waitForJobToBeStarted(
job: CoreAttachmentBackupJobType,
attempts: number = 0
) {
return backupManager?.waitForJobToBeStarted({ ...job, attempts });
}
function waitForJobToBeCompleted(
job: CoreAttachmentBackupJobType,
attempts: number = 0
) {
return backupManager?.waitForJobToBeCompleted({ ...job, attempts });
}
function assertRunJobCalledWith(jobs: Array<CoreAttachmentBackupJobType>) {
return assert.strictEqual(
JSON.stringify(runJob.getCalls().map(call => call.args[0].mediaName)),
JSON.stringify(jobs.map(job => job.mediaName))
);
}
async function getAllSavedJobs(): Promise<Array<AttachmentBackupJobType>> {
return dataInterface.getNextAttachmentBackupJobs({
limit: 1000,
timestamp: Infinity,
});
}
it('saves jobs, removes jobs, and runs 3 jobs at a time in descending receivedAt order', async () => {
const jobs = await addJobs(5);
// Confirm they are saved to DB
const allJobs = await getAllSavedJobs();
assert.strictEqual(allJobs.length, 5);
assert.strictEqual(
JSON.stringify(allJobs.map(job => job.mediaName)),
JSON.stringify([
'mediaName4',
'mediaName3',
'mediaName2',
'mediaName1',
'mediaName0',
])
);
await backupManager?.start();
await waitForJobToBeStarted(jobs[2]);
assert.strictEqual(runJob.callCount, 3);
assertRunJobCalledWith([jobs[4], jobs[3], jobs[2]]);
await waitForJobToBeStarted(jobs[0]);
assert.strictEqual(runJob.callCount, 5);
assertRunJobCalledWith([jobs[4], jobs[3], jobs[2], jobs[1], jobs[0]]);
await waitForJobToBeCompleted(jobs[0]);
assert.strictEqual((await getAllSavedJobs()).length, 0);
});
it('with transitCdnInfo, will copy to backup tier', async () => {
const [job] = await addJobs(1);
await backupManager?.start();
await waitForJobToBeCompleted(job);
assert.strictEqual(backupMediaBatch.callCount, 1);
assert.strictEqual(encryptAndUploadAttachment.callCount, 0);
assert.deepStrictEqual(
backupMediaBatch.getCall(0).args[0].items[0].sourceAttachment,
{ key: 'transitCdnKey', cdn: TRANSIT_CDN }
);
});
it('with transitCdnInfo, will upload to attachment tier if copy operation returns FileNotFoundOnTransitTier', async () => {
backupMediaBatch.onFirstCall().returns(
Promise.resolve({
responses: [
{ isSuccess: false, status: FILE_NOT_FOUND_ON_TRANSIT_TIER_STATUS },
],
})
);
backupMediaBatch.onSecondCall().returns(
Promise.resolve({
responses: [{ isSuccess: true, cdn: BACKUP_CDN }],
})
);
const [job] = await addJobs(1);
await backupManager?.start();
await waitForJobToBeCompleted(job);
assert.strictEqual(encryptAndUploadAttachment.callCount, 1);
assert.strictEqual(backupMediaBatch.callCount, 2);
assert.deepStrictEqual(
backupMediaBatch.getCall(0).args[0].items[0].sourceAttachment,
{ key: 'transitCdnKey', cdn: TRANSIT_CDN }
);
assert.deepStrictEqual(
backupMediaBatch.getCall(1).args[0].items[0].sourceAttachment,
{ key: 'newKeyOnTransitTier', cdn: TRANSIT_CDN_FOR_NEW_UPLOAD }
);
const allRemainingJobs = await getAllSavedJobs();
assert.strictEqual(allRemainingJobs.length, 0);
});
it('without transitCdnInfo, will upload then copy', async () => {
const [job] = await addJobs(1, { transitCdnInfo: undefined });
await backupManager?.start();
await waitForJobToBeCompleted(job);
assert.strictEqual(backupMediaBatch.callCount, 1);
assert.strictEqual(encryptAndUploadAttachment.callCount, 1);
// Job removed
const allRemainingJobs = await getAllSavedJobs();
assert.strictEqual(allRemainingJobs.length, 0);
});
it('without transitCdnInfo, will permanently remove job if file not found at path', async () => {
const [job] = await addJobs(1, { transitCdnInfo: undefined });
getAbsoluteAttachmentPath.returns('no/file/here');
await backupManager?.start();
await waitForJobToBeCompleted(job);
assert.strictEqual(backupMediaBatch.callCount, 0);
assert.strictEqual(encryptAndUploadAttachment.callCount, 0);
// Job removed
const allRemainingJobs = await getAllSavedJobs();
assert.strictEqual(allRemainingJobs.length, 0);
});
});

20
ts/test-electron/services/AttachmentDownloadManager_test.ts
View file

@ -17,7 +17,7 @@ import dataInterface from '../../sql/Client';
import { HOUR, MINUTE, SECOND } from '../../util/durations';
import { type AciString } from '../../types/ServiceId';
describe('AttachmentDownloadManager', () => {
describe('AttachmentDownloadManager/JobManager', () => {
let downloadManager: AttachmentDownloadManager | undefined;
let runJob: sinon.SinonStub;
let sandbox: sinon.SinonSandbox;
@ -58,10 +58,10 @@ describe('AttachmentDownloadManager', () => {
await dataInterface.removeAll();
sandbox = sinon.createSandbox();
clock = sinon.useFakeTimers();
clock = sandbox.useFakeTimers();
isInCall = sinon.stub().returns(false);
runJob = sinon.stub().callsFake(async () => {
isInCall = sandbox.stub().returns(false);
runJob = sandbox.stub().callsFake(async () => {
return new Promise<{ status: 'finished' | 'retry' }>(resolve => {
Promise.resolve().then(() => {
resolve({ status: 'finished' });
@ -71,7 +71,7 @@ describe('AttachmentDownloadManager', () => {
downloadManager = new AttachmentDownloadManager({
...AttachmentDownloadManager.defaultParams,
isInCall,
shouldHoldOffOnStartingQueuedJobs: isInCall,
runJob,
});
});
@ -287,7 +287,7 @@ describe('AttachmentDownloadManager', () => {
assert.strictEqual(retriedJob?.attempts, 1);
assert.isNumber(retriedJob?.retryAfter);
await advanceTime(30 * SECOND);
await advanceTime(60 * SECOND); // one tick
await job1Attempts[1].completed;
assert.strictEqual(runJob.callCount, 3);
@ -331,7 +331,7 @@ describe('AttachmentDownloadManager', () => {
await attempts[0].completed;
assert.strictEqual(runJob.callCount, 1);
await advanceTime(30 * SECOND);
await advanceTime(1 * MINUTE);
await attempts[1].completed;
assert.strictEqual(runJob.callCount, 2);
@ -340,12 +340,12 @@ describe('AttachmentDownloadManager', () => {
assert.strictEqual(runJob.callCount, 3);
// add the same job again and it should retry ASAP and reset attempts
attempts = getPromisesForAttempts(jobs[0], 4);
attempts = getPromisesForAttempts(jobs[0], 5);
await downloadManager?.addJob(jobs[0]);
await attempts[0].completed;
assert.strictEqual(runJob.callCount, 4);
await advanceTime(30 * SECOND);
await advanceTime(1 * MINUTE);
await attempts[1].completed;
assert.strictEqual(runJob.callCount, 5);
@ -358,7 +358,7 @@ describe('AttachmentDownloadManager', () => {
assert.strictEqual(runJob.callCount, 7);
await advanceTime(6 * HOUR);
await attempts[3].completed;
await attempts[4].completed;
assert.strictEqual(runJob.callCount, 8);
// Ensure it's been removed

9
ts/test-electron/state/ducks/stories_test.ts
View file

@ -937,7 +937,10 @@ describe('both/state/ducks/stories', () => {
},
],
};
await window.Signal.Data.saveMessage(messageAttributes, {
forceSave: true,
ourAci: generateAci(),
});
const rootState = getEmptyRootState();
const getState = () => ({
@ -1000,6 +1003,10 @@ describe('both/state/ducks/stories', () => {
preview: [preview],
};
await window.Signal.Data.saveMessage(messageAttributes, {
forceSave: true,
ourAci: generateAci(),
});
const rootState = getEmptyRootState();
const getState = () => ({