From 11e612f57b49d44cbea34e881b5fb4dc08977435 Mon Sep 17 00:00:00 2001 From: trevor-signal <131492920+trevor-signal@users.noreply.github.com> Date: Wed, 10 Sep 2025 16:16:46 -0400 Subject: [PATCH] Add media granularity to backup attachment download source --- ts/jobs/AttachmentDownloadManager.ts | 24 ++++++++----------- ts/services/backups/import.ts | 24 +++++++++++++++---- ts/sql/Client.ts | 11 ++++++--- ts/sql/Interface.ts | 5 +++- ts/sql/Server.ts | 6 ++++- .../AttachmentDownloadManager_test.ts | 17 +++++++------ .../attachment_download_backup_stats_test.ts | 18 +++++++------- ts/test-node/sql/migration_1200_test.ts | 2 +- ts/test-node/sql/migration_1420_test.ts | 2 +- ts/util/backupMediaDownload.ts | 8 +++---- ts/util/backupSubscriptionData.ts | 2 ++ ts/util/queueAttachmentDownloads.ts | 4 +++- 12 files changed, 74 insertions(+), 49 deletions(-) diff --git a/ts/jobs/AttachmentDownloadManager.ts b/ts/jobs/AttachmentDownloadManager.ts index ab02f4d7886..6cae5ffab34 100644 --- a/ts/jobs/AttachmentDownloadManager.ts +++ b/ts/jobs/AttachmentDownloadManager.ts @@ -253,7 +253,7 @@ export class AttachmentDownloadManager extends JobManager(); #frameErrorCount: number = 0; + #backupTier: BackupLevel | undefined; private constructor( private readonly backupType: BackupType, @@ -267,7 +271,8 @@ export class BackupImportStream extends Writable { localBackupSnapshotDir: string | undefined = undefined ): Promise { await AttachmentDownloadManager.stop(); - await resetBackupMediaDownloadProgress(); + await DataWriter.removeAllBackupAttachmentDownloadJobs(); + await resetBackupMediaDownloadStats(); return new BackupImportStream(backupType, localBackupSnapshotDir); } @@ -673,7 +678,9 @@ export class BackupImportStream extends Writable { const model = new MessageModel(attributes); attachmentDownloadJobPromises.push( queueAttachmentDownloads(model, { - source: AttachmentDownloadSource.BACKUP_IMPORT, + source: this.#isMediaEnabledBackup() + ? AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA + : AttachmentDownloadSource.BACKUP_IMPORT_NO_MEDIA, isManualDownload: false, }) ); @@ -835,6 +842,7 @@ export class BackupImportStream extends Writable { ); } + this.#backupTier = accountSettings?.backupTier?.toNumber(); await storage.put('backupTier', accountSettings?.backupTier?.toNumber()); const { PhoneNumberSharingMode: BackupMode } = Backups.AccountData; @@ -3774,6 +3782,14 @@ export class BackupImportStream extends Writable { return {}; } + + #isLocalBackup() { + return this.localBackupSnapshotDir != null; + } + + #isMediaEnabledBackup() { + return this.#isLocalBackup() || this.#backupTier === BackupLevel.Paid; + } } function rgbIntToDesktopHSL(intValue: number): { diff --git a/ts/sql/Client.ts b/ts/sql/Client.ts index cdc0fd83c25..6dba611ac63 100644 --- a/ts/sql/Client.ts +++ b/ts/sql/Client.ts @@ -833,7 +833,9 @@ async function saveAttachmentDownloadJob( job: AttachmentDownloadJobType ): Promise { await writableChannel.saveAttachmentDownloadJob(job); - if (job.originalSource === AttachmentDownloadSource.BACKUP_IMPORT) { + if ( + job.originalSource === AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA + ) { drop( throttledUpdateBackupMediaDownloadProgress( readableChannel.getBackupAttachmentDownloadProgress @@ -847,7 +849,8 @@ async function saveAttachmentDownloadJobs( await writableChannel.saveAttachmentDownloadJobs(jobs); if ( jobs.some( - job => job.originalSource === AttachmentDownloadSource.BACKUP_IMPORT + job => + job.originalSource === AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA ) ) { drop( @@ -862,7 +865,9 @@ async function removeAttachmentDownloadJob( job: AttachmentDownloadJobType ): Promise { await writableChannel.removeAttachmentDownloadJob(job); - if (job.originalSource === AttachmentDownloadSource.BACKUP_IMPORT) { + if ( + job.originalSource === AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA + ) { drop( throttledUpdateBackupMediaDownloadProgress( readableChannel.getBackupAttachmentDownloadProgress diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 16595b342a5..3c3f3c2bfbc 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -548,7 +548,10 @@ export type GetRecentStoryRepliesOptionsType = { }; export enum AttachmentDownloadSource { - BACKUP_IMPORT = 'backup_import', + // Imported when paid (media) backups were enabled, or from a local backup + BACKUP_IMPORT_WITH_MEDIA = 'backup_import', + // Imported when paid (media) backups were not enabled + BACKUP_IMPORT_NO_MEDIA = 'backup_import_no_media', STANDARD = 'standard', BACKFILL = 'backfill', } diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 6257503d71f..52c18cfe99b 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -5577,7 +5577,10 @@ function _getAttachmentDownloadJob( function removeAllBackupAttachmentDownloadJobs(db: WritableDB): void { const [query, params] = sql` DELETE FROM attachment_downloads - WHERE source = ${AttachmentDownloadSource.BACKUP_IMPORT};`; + WHERE + source = ${AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA} + OR + source = ${AttachmentDownloadSource.BACKUP_IMPORT_NO_MEDIA};`; db.prepare(query).run(params); } @@ -7739,6 +7742,7 @@ function removeAllConfiguration(db: WritableDB): void { db.exec( ` DELETE FROM attachment_backup_jobs; + DELETE FROM attachment_downloads; DELETE FROM backup_cdn_object_metadata; DELETE FROM groupSendCombinedEndorsement; DELETE FROM groupSendMemberEndorsement; diff --git a/ts/test-electron/services/AttachmentDownloadManager_test.ts b/ts/test-electron/services/AttachmentDownloadManager_test.ts index a3a77c46214..593623a22d9 100644 --- a/ts/test-electron/services/AttachmentDownloadManager_test.ts +++ b/ts/test-electron/services/AttachmentDownloadManager_test.ts @@ -336,7 +336,7 @@ describe('AttachmentDownloadManager/JobManager', () => { it('triggers onLowDiskSpace for backup import jobs', async () => { const jobs = await addJobs(1, _idx => ({ - source: AttachmentDownloadSource.BACKUP_IMPORT, + source: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, })); const jobAttempts = getPromisesForAttempts(jobs[0], 2); @@ -479,7 +479,7 @@ describe('AttachmentDownloadManager/JobManager', () => { const jobs = await addJobs(6, idx => ({ source: idx % 2 === 0 - ? AttachmentDownloadSource.BACKUP_IMPORT + ? AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA : AttachmentDownloadSource.STANDARD, })); // make one of the backup job messages visible to test that code path as well @@ -506,7 +506,7 @@ describe('AttachmentDownloadManager/JobManager', () => { it('retries backup job immediately if retryAfters are reset', async () => { strictAssert(downloadManager, 'must exist'); const jobs = await addJobs(1, { - source: AttachmentDownloadSource.BACKUP_IMPORT, + source: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, }); const jobAttempts = getPromisesForAttempts(jobs[0], 2); @@ -532,7 +532,7 @@ describe('AttachmentDownloadManager/JobManager', () => { strictAssert(downloadManager, 'must exist'); const job = ( await addJobs(1, { - source: AttachmentDownloadSource.BACKUP_IMPORT, + source: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, }) )[0]; const jobAttempts = getPromisesForAttempts(job, 3); @@ -567,11 +567,10 @@ describe('AttachmentDownloadManager/JobManager', () => { describe('will drop jobs from non-media backup imports that are old', () => { it('will not queue attachments older than 90 days (2 * message queue time)', async () => { - hasMediaBackups.returns(false); await addJobs( 1, { - source: AttachmentDownloadSource.BACKUP_IMPORT, + source: AttachmentDownloadSource.BACKUP_IMPORT_NO_MEDIA, }, { uploadTimestamp: Date.now() - 4 * MONTH } ); @@ -586,7 +585,7 @@ describe('AttachmentDownloadManager/JobManager', () => { await addJobs( 1, { - source: AttachmentDownloadSource.BACKUP_IMPORT, + source: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, }, { uploadTimestamp: Date.now() - 4 * MONTH } ); @@ -601,7 +600,7 @@ describe('AttachmentDownloadManager/JobManager', () => { await addJobs( 1, { - source: AttachmentDownloadSource.BACKUP_IMPORT, + source: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, }, { uploadTimestamp: Date.now() - 4 * MONTH, @@ -620,7 +619,7 @@ describe('AttachmentDownloadManager/JobManager', () => { await addJobs( 1, { - source: AttachmentDownloadSource.BACKUP_IMPORT, + source: AttachmentDownloadSource.BACKUP_IMPORT_NO_MEDIA, sentAt: Date.now() - 4 * MONTH, }, { uploadTimestamp: 0 } diff --git a/ts/test-electron/sql/attachment_download_backup_stats_test.ts b/ts/test-electron/sql/attachment_download_backup_stats_test.ts index 0cf6b8e7da2..435f1570ba1 100644 --- a/ts/test-electron/sql/attachment_download_backup_stats_test.ts +++ b/ts/test-electron/sql/attachment_download_backup_stats_test.ts @@ -47,8 +47,8 @@ describe('sql/AttachmentDownloadBackupStats', () => { const backupJob1 = createAttachmentDownloadJob(1, { messageId: 'message0', ciphertextSize: 1000, - originalSource: AttachmentDownloadSource.BACKUP_IMPORT, - source: AttachmentDownloadSource.BACKUP_IMPORT, + originalSource: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, + source: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, }); await DataWriter.saveAttachmentDownloadJob(backupJob1); assert.deepStrictEqual( @@ -59,8 +59,8 @@ describe('sql/AttachmentDownloadBackupStats', () => { const backupJob2 = createAttachmentDownloadJob(2, { messageId: 'message0', ciphertextSize: 2000, - originalSource: AttachmentDownloadSource.BACKUP_IMPORT, - source: AttachmentDownloadSource.BACKUP_IMPORT, + originalSource: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, + source: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, }); await DataWriter.saveAttachmentDownloadJob(backupJob2); assert.deepStrictEqual( @@ -128,8 +128,8 @@ describe('sql/AttachmentDownloadBackupStats', () => { } ); const backupJob = createAttachmentDownloadJob(0, { - originalSource: AttachmentDownloadSource.BACKUP_IMPORT, - source: AttachmentDownloadSource.BACKUP_IMPORT, + originalSource: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, + source: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, ciphertextSize: 128, }); await DataWriter.saveAttachmentDownloadJob(backupJob); @@ -167,8 +167,8 @@ describe('sql/AttachmentDownloadBackupStats', () => { } ); const backupJob = createAttachmentDownloadJob(0, { - originalSource: AttachmentDownloadSource.BACKUP_IMPORT, - source: AttachmentDownloadSource.BACKUP_IMPORT, + originalSource: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, + source: AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, ciphertextSize: 128, }); await DataWriter.saveAttachmentDownloadJob(backupJob); @@ -190,7 +190,7 @@ describe('sql/AttachmentDownloadBackupStats', () => { assert.strictEqual( savedJob?.originalSource, - AttachmentDownloadSource.BACKUP_IMPORT + AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA ); assert.strictEqual(savedJob?.source, AttachmentDownloadSource.STANDARD); diff --git a/ts/test-node/sql/migration_1200_test.ts b/ts/test-node/sql/migration_1200_test.ts index e9bd42df26a..90f9d1476c2 100644 --- a/ts/test-node/sql/migration_1200_test.ts +++ b/ts/test-node/sql/migration_1200_test.ts @@ -105,7 +105,7 @@ describe('SQL/updateToSchemaVersion1200', () => { source: i < NUM_STANDARD_JOBS ? AttachmentDownloadSource.STANDARD - : AttachmentDownloadSource.BACKUP_IMPORT, + : AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, }); } })(); diff --git a/ts/test-node/sql/migration_1420_test.ts b/ts/test-node/sql/migration_1420_test.ts index a7a7085a51b..2baad46bff0 100644 --- a/ts/test-node/sql/migration_1420_test.ts +++ b/ts/test-node/sql/migration_1420_test.ts @@ -87,7 +87,7 @@ describe('SQL/updateToSchemaVersion1410', () => { source: i < 5 ? AttachmentDownloadSource.STANDARD - : AttachmentDownloadSource.BACKUP_IMPORT, + : AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA, }); } })(); diff --git a/ts/util/backupMediaDownload.ts b/ts/util/backupMediaDownload.ts index 2c59ed5eb1f..b23e96da2fe 100644 --- a/ts/util/backupMediaDownload.ts +++ b/ts/util/backupMediaDownload.ts @@ -36,14 +36,12 @@ export async function resetBackupMediaDownloadItems(): Promise { export async function cancelBackupMediaDownload(): Promise { log.info('Canceling media download'); - await window.storage.put('backupMediaDownloadBannerDismissed', true); + await dismissBackupMediaDownloadBanner(); await DataWriter.removeAllBackupAttachmentDownloadJobs(); - await DataWriter.resetBackupAttachmentDownloadStats(); - await resetBackupMediaDownloadItems(); + await resetBackupMediaDownloadStats(); } -export async function resetBackupMediaDownloadProgress(): Promise { - await DataWriter.removeAllBackupAttachmentDownloadJobs(); +export async function resetBackupMediaDownloadStats(): Promise { await DataWriter.resetBackupAttachmentDownloadStats(); await resetBackupMediaDownloadItems(); } diff --git a/ts/util/backupSubscriptionData.ts b/ts/util/backupSubscriptionData.ts index 7e7b07fe393..7837e83b955 100644 --- a/ts/util/backupSubscriptionData.ts +++ b/ts/util/backupSubscriptionData.ts @@ -6,6 +6,7 @@ import type { Backups, SignalService } from '../protobuf'; import * as Bytes from '../Bytes'; import { drop } from './drop'; import { createLogger } from '../logging/log'; +import { resetBackupMediaDownloadStats } from './backupMediaDownload'; const log = createLogger('BackupSubscriptionData'); @@ -64,6 +65,7 @@ export async function saveBackupTier( await window.storage.put('backupTier', backupTier); if (backupTier !== previousBackupTier) { log.info('backup tier has changed', { previousBackupTier, backupTier }); + await resetBackupMediaDownloadStats(); drop(window.Signal.Services.backups.resetCachedData()); } } diff --git a/ts/util/queueAttachmentDownloads.ts b/ts/util/queueAttachmentDownloads.ts index 33ff36ab0fc..288a41c8638 100644 --- a/ts/util/queueAttachmentDownloads.ts +++ b/ts/util/queueAttachmentDownloads.ts @@ -61,7 +61,9 @@ export type MessageAttachmentsDownloadedType = { }; function getLogger(source: AttachmentDownloadSource) { - const verbose = source !== AttachmentDownloadSource.BACKUP_IMPORT; + const verbose = + source !== AttachmentDownloadSource.BACKUP_IMPORT_NO_MEDIA && + source !== AttachmentDownloadSource.BACKUP_IMPORT_WITH_MEDIA; const log = verbose ? defaultLogger : { ...defaultLogger, info: () => null }; return log; }