diff --git a/app/attachment_channel.ts b/app/attachment_channel.ts index 5be5093e6560..7979b81c7b96 100644 --- a/app/attachment_channel.ts +++ b/app/attachment_channel.ts @@ -18,13 +18,17 @@ import { import { RangeFinder, DefaultStorage } from '@indutny/range-finder'; import { getAllAttachments, + getAllDownloads, getAvatarsPath, getPath, getStickersPath, getTempPath, getDraftPath, + getDownloadsPath, deleteAll as deleteAllAttachments, deleteAllBadges, + deleteAllDownloads, + deleteStaleDownloads, getAllStickers, deleteAllStickers, getAllDraftAttachments, @@ -50,6 +54,8 @@ const ERASE_ATTACHMENTS_KEY = 'erase-attachments'; const ERASE_STICKERS_KEY = 'erase-stickers'; const ERASE_TEMP_KEY = 'erase-temp'; const ERASE_DRAFTS_KEY = 'erase-drafts'; +const ERASE_DOWNLOADS_KEY = 'erase-downloads'; +const CLEANUP_DOWNLOADS_KEY = 'cleanup-downloads'; const CLEANUP_ORPHANED_ATTACHMENTS_KEY = 'cleanup-orphaned-attachments'; const INTERACTIVITY_DELAY = 50; @@ -189,6 +195,7 @@ const dispositionSchema = z.enum([ type DeleteOrphanedAttachmentsOptionsType = Readonly<{ orphanedAttachments: Set; + orphanedDownloads: Set; sql: MainSQL; userDataPath: string; }>; @@ -235,8 +242,14 @@ async function cleanupOrphanedAttachments({ `${orphanedAttachments.size} attachments on disk` ); + const orphanedDownloads = new Set(await getAllDownloads(userDataPath)); + console.log( + 'cleanupOrphanedAttachments: found ' + + `${orphanedDownloads.size} downloads on disk` + ); + { - const attachments: ReadonlyArray = await sql.sqlRead( + const attachments: Array = await sql.sqlRead( 'getKnownConversationAttachments' ); @@ -258,6 +271,7 @@ async function cleanupOrphanedAttachments({ // are saved to disk, but not put into any message or conversation model yet. deleteOrphanedAttachments({ orphanedAttachments, + orphanedDownloads, sql, userDataPath, }); @@ -265,6 +279,7 @@ async function cleanupOrphanedAttachments({ function deleteOrphanedAttachments({ orphanedAttachments, + orphanedDownloads, sql, userDataPath, }: DeleteOrphanedAttachmentsOptionsType): void { @@ -273,17 +288,21 @@ function deleteOrphanedAttachments({ let cursor: MessageAttachmentsCursorType | undefined; let totalFound = 0; let totalMissing = 0; + let totalDownloadsFound = 0; + let totalDownloadsMissing = 0; try { do { let attachments: ReadonlyArray; + let downloads: ReadonlyArray; // eslint-disable-next-line no-await-in-loop - ({ attachments, cursor } = await sql.sqlRead( + ({ attachments, downloads, cursor } = await sql.sqlRead( 'getKnownMessageAttachments', cursor )); totalFound += attachments.length; + totalDownloadsFound += downloads.length; for (const known of attachments) { if (!orphanedAttachments.delete(known)) { @@ -291,6 +310,12 @@ function deleteOrphanedAttachments({ } } + for (const known of downloads) { + if (!orphanedDownloads.delete(known)) { + totalDownloadsMissing += 1; + } + } + if (cursor === undefined) { break; } @@ -316,6 +341,16 @@ function deleteOrphanedAttachments({ userDataPath, attachments: Array.from(orphanedAttachments), }); + + console.log( + `cleanupOrphanedAttachments: found ${totalDownloadsFound} downloads ` + + `(${totalDownloadsMissing} missing) ` + + `${orphanedDownloads.size} remain` + ); + await deleteAllDownloads({ + userDataPath, + downloads: Array.from(orphanedDownloads), + }); } async function runSafe() { @@ -341,6 +376,7 @@ let attachmentsDir: string | undefined; let stickersDir: string | undefined; let tempDir: string | undefined; let draftDir: string | undefined; +let downloadsDir: string | undefined; let avatarDataDir: string | undefined; export function initialize({ @@ -359,6 +395,7 @@ export function initialize({ stickersDir = getStickersPath(configDir); tempDir = getTempPath(configDir); draftDir = getDraftPath(configDir); + downloadsDir = getDownloadsPath(configDir); avatarDataDir = getAvatarsPath(configDir); ipcMain.handle(ERASE_TEMP_KEY, () => { @@ -377,6 +414,10 @@ export function initialize({ strictAssert(draftDir != null, 'not initialized'); rimraf.sync(draftDir); }); + ipcMain.handle(ERASE_DOWNLOADS_KEY, () => { + strictAssert(downloadsDir != null, 'not initialized'); + rimraf.sync(downloadsDir); + }); ipcMain.handle(CLEANUP_ORPHANED_ATTACHMENTS_KEY, async () => { const start = Date.now(); @@ -385,6 +426,13 @@ export function initialize({ console.log(`cleanupOrphanedAttachments: took ${duration}ms`); }); + ipcMain.handle(CLEANUP_DOWNLOADS_KEY, async () => { + const start = Date.now(); + await deleteStaleDownloads(configDir); + const duration = Date.now() - start; + console.log(`cleanupDownloads: took ${duration}ms`); + }); + protocol.handle('attachment', handleAttachmentRequest); } diff --git a/app/attachments.ts b/app/attachments.ts index 4dfb83104f12..f8859c63dfc2 100644 --- a/app/attachments.ts +++ b/app/attachments.ts @@ -2,18 +2,24 @@ // SPDX-License-Identifier: AGPL-3.0-only import { PassThrough } from 'node:stream'; +import { stat } from 'node:fs/promises'; import { join, relative, normalize } from 'path'; +import pMap from 'p-map'; import fastGlob from 'fast-glob'; import fse from 'fs-extra'; import { map, isString } from 'lodash'; import normalizePath from 'normalize-path'; import { isPathInside } from '../ts/util/isPathInside'; +import { DAY } from '../ts/util/durations'; +import { isOlderThan } from '../ts/util/timestamp'; +import { isNotNil } from '../ts/util/isNotNil'; import { generateKeys, decryptAttachmentV2ToSink, encryptAttachmentV2ToDisk, } from '../ts/AttachmentCrypto'; import type { LocalAttachmentV2Type } from '../ts/types/Attachment'; +import * as Errors from '../ts/types/errors'; const PATH = 'attachments.noindex'; const AVATAR_PATH = 'avatars.noindex'; @@ -22,9 +28,12 @@ const STICKER_PATH = 'stickers.noindex'; const TEMP_PATH = 'temp'; const UPDATE_CACHE_PATH = 'update-cache'; const DRAFT_PATH = 'drafts.noindex'; +const DOWNLOADS_PATH = 'downloads.noindex'; const CACHED_PATHS = new Map(); +const FS_CONCURRENCY = 100; + const createPathGetter = (subpath: string) => (userDataPath: string): string => { @@ -52,6 +61,7 @@ const createPathGetter = export const getAvatarsPath = createPathGetter(AVATAR_PATH); export const getBadgesPath = createPathGetter(BADGES_PATH); export const getDraftPath = createPathGetter(DRAFT_PATH); +export const getDownloadsPath = createPathGetter(DOWNLOADS_PATH); export const getPath = createPathGetter(PATH); export const getStickersPath = createPathGetter(STICKER_PATH); export const getTempPath = createPathGetter(TEMP_PATH); @@ -88,6 +98,16 @@ export const getAllAttachments = async ( return map(files, file => relative(dir, file)); }; +export const getAllDownloads = async ( + userDataPath: string +): Promise> => { + const dir = getDownloadsPath(userDataPath); + const pattern = normalizePath(join(dir, '**', '*')); + + const files = await fastGlob(pattern, { onlyFiles: true }); + return map(files, file => relative(dir, file)); +}; + const getAllBadgeImageFiles = async ( userDataPath: string ): Promise> => { @@ -123,6 +143,43 @@ export const clearTempPath = (userDataPath: string): Promise => { return fse.emptyDir(tempPath); }; +export const deleteStaleDownloads = async ( + userDataPath: string +): Promise => { + const dir = getDownloadsPath(userDataPath); + const files = await getAllDownloads(userDataPath); + + const result = await pMap( + files, + async file => { + try { + const { birthtimeMs } = await stat(join(dir, file)); + if (isOlderThan(birthtimeMs, DAY)) { + return file; + } + } catch (error) { + // No longer exists + if (error.code === 'ENOENT') { + return; + } + console.error( + 'deleteStaleDownloads: failed to get file stats', + Errors.toLogFormat(error) + ); + } + return undefined; + }, + { concurrency: FS_CONCURRENCY } + ); + + const stale = result.filter(isNotNil); + if (stale.length === 0) { + return; + } + console.log(`deleteStaleDownloads: found ${stale.length}`); + await deleteAllDownloads({ userDataPath, downloads: stale }); +}; + export const deleteAll = async ({ userDataPath, attachments, @@ -132,15 +189,25 @@ export const deleteAll = async ({ }): Promise => { const deleteFromDisk = createDeleter(getPath(userDataPath)); - for (let index = 0, max = attachments.length; index < max; index += 1) { - const file = attachments[index]; - // eslint-disable-next-line no-await-in-loop - await deleteFromDisk(file); - } + await pMap(attachments, deleteFromDisk, { concurrency: FS_CONCURRENCY }); console.log(`deleteAll: deleted ${attachments.length} files`); }; +export const deleteAllDownloads = async ({ + userDataPath, + downloads, +}: { + userDataPath: string; + downloads: ReadonlyArray; +}): Promise => { + const deleteFromDisk = createDeleter(getDownloadsPath(userDataPath)); + + await pMap(downloads, deleteFromDisk, { concurrency: FS_CONCURRENCY }); + + console.log(`deleteAllDownloads: deleted ${downloads.length} files`); +}; + export const deleteAllStickers = async ({ userDataPath, stickers, @@ -150,11 +217,7 @@ export const deleteAllStickers = async ({ }): Promise => { const deleteFromDisk = createDeleter(getStickersPath(userDataPath)); - for (let index = 0, max = stickers.length; index < max; index += 1) { - const file = stickers[index]; - // eslint-disable-next-line no-await-in-loop - await deleteFromDisk(file); - } + await pMap(stickers, deleteFromDisk, { concurrency: FS_CONCURRENCY }); console.log(`deleteAllStickers: deleted ${stickers.length} files`); }; @@ -189,11 +252,7 @@ export const deleteAllDraftAttachments = async ({ }): Promise => { const deleteFromDisk = createDeleter(getDraftPath(userDataPath)); - for (let index = 0, max = attachments.length; index < max; index += 1) { - const file = attachments[index]; - // eslint-disable-next-line no-await-in-loop - await deleteFromDisk(file); - } + await pMap(attachments, deleteFromDisk, { concurrency: FS_CONCURRENCY }); console.log(`deleteAllDraftAttachments: deleted ${attachments.length} files`); }; diff --git a/app/main.ts b/app/main.ts index fefd28821f0c..8e4326fcc759 100644 --- a/app/main.ts +++ b/app/main.ts @@ -2208,6 +2208,15 @@ app.on('ready', async () => { ); } + try { + await attachments.deleteStaleDownloads(userDataPath); + } catch (err) { + logger.error( + 'main/ready: Error deleting stale downloads:', + Errors.toLogFormat(err) + ); + } + // Initialize IPC channels before creating the window attachmentChannel.initialize({ diff --git a/app/protocol_filter.ts b/app/protocol_filter.ts index afb903aae64b..68dceea2ac4b 100644 --- a/app/protocol_filter.ts +++ b/app/protocol_filter.ts @@ -9,6 +9,7 @@ import { getAvatarsPath, getBadgesPath, getDraftPath, + getDownloadsPath, getPath, getStickersPath, getTempPath, @@ -61,6 +62,7 @@ function _createFileHandler({ getAvatarsPath(userDataPath), getBadgesPath(userDataPath), getDraftPath(userDataPath), + getDownloadsPath(userDataPath), getPath(userDataPath), getStickersPath(userDataPath), getTempPath(userDataPath), diff --git a/ts/AttachmentCrypto.ts b/ts/AttachmentCrypto.ts index 71afdede0617..6cc48625d8a7 100644 --- a/ts/AttachmentCrypto.ts +++ b/ts/AttachmentCrypto.ts @@ -1,8 +1,8 @@ // Copyright 2020 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only -import { unlinkSync, createReadStream, createWriteStream } from 'fs'; -import { open } from 'fs/promises'; +import { createReadStream, createWriteStream } from 'fs'; +import { open, unlink } from 'fs/promises'; import { createCipheriv, createHash, createHmac, randomBytes } from 'crypto'; import type { Hash } from 'crypto'; import { PassThrough, Transform, type Writable, Readable } from 'stream'; @@ -114,7 +114,7 @@ export async function encryptAttachmentV2ToDisk( sink: createWriteStream(absoluteTargetPath), }); } catch (error) { - safeUnlinkSync(absoluteTargetPath); + await safeUnlink(absoluteTargetPath); throw error; } @@ -307,7 +307,7 @@ export async function decryptAttachmentV2( `${logId}: Failed to decrypt attachment to disk`, Errors.toLogFormat(error) ); - safeUnlinkSync(absoluteTargetPath); + await safeUnlink(absoluteTargetPath); throw error; } finally { await writeFd?.close(); @@ -523,7 +523,7 @@ export async function decryptAndReencryptLocally( `${logId}: Failed to decrypt attachment`, Errors.toLogFormat(error) ); - safeUnlinkSync(absoluteTargetPath); + await safeUnlink(absoluteTargetPath); throw error; } finally { await writeFd?.close(); @@ -618,9 +618,9 @@ export function getPlaintextHashForInMemoryAttachment( * Unlinks a file without throwing an error if it doesn't exist. * Throws an error if it fails to unlink for any other reason. */ -export function safeUnlinkSync(filePath: string): void { +export async function safeUnlink(filePath: string): Promise { try { - unlinkSync(filePath); + await unlink(filePath); } catch (error) { // Ignore if file doesn't exist if (error.code !== 'ENOENT') { diff --git a/ts/background.ts b/ts/background.ts index dddf8c56e7f1..f32c6b3a1809 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -1104,6 +1104,10 @@ export async function startApp(): Promise { reportLongRunningTasks(); }, FIVE_MINUTES); + setInterval(() => { + drop(window.Events.cleanupDownloads()); + }, DAY); + let mainWindowStats = { isMaximized: false, isFullScreen: false, diff --git a/ts/jobs/AttachmentDownloadManager.ts b/ts/jobs/AttachmentDownloadManager.ts index 78d33305ea19..44f9e5ef97da 100644 --- a/ts/jobs/AttachmentDownloadManager.ts +++ b/ts/jobs/AttachmentDownloadManager.ts @@ -411,7 +411,7 @@ export async function runDownloadAttachmentJobInner({ const upgradedAttachment = await window.Signal.Migrations.processNewAttachment({ - ...omit(attachment, ['error', 'pending']), + ...omit(attachment, ['error', 'pending', 'downloadPath']), ...downloaded, }); diff --git a/ts/messageModifiers/DeletesForMe.ts b/ts/messageModifiers/DeletesForMe.ts index 4ecd5d8392ff..b905b570514d 100644 --- a/ts/messageModifiers/DeletesForMe.ts +++ b/ts/messageModifiers/DeletesForMe.ts @@ -107,6 +107,7 @@ export async function onDelete(item: DeleteForMeAttributesType): Promise { item.deleteAttachmentData, { deleteOnDisk: window.Signal.Migrations.deleteAttachmentData, + deleteDownloadOnDisk: window.Signal.Migrations.deleteDownloadData, logId, } ); diff --git a/ts/signal.ts b/ts/signal.ts index 644beaa8fec1..109878e98c7b 100644 --- a/ts/signal.ts +++ b/ts/signal.ts @@ -71,6 +71,7 @@ type MigrationsModuleType = { ) => Promise<{ path: string; size: number }>; deleteAttachmentData: (path: string) => Promise; deleteAvatar: (path: string) => Promise; + deleteDownloadData: (path: string) => Promise; deleteDraftFile: (path: string) => Promise; deleteExternalMessageFiles: ( attributes: MessageAttributesType @@ -81,6 +82,7 @@ type MigrationsModuleType = { getAbsoluteAttachmentPath: (path: string) => string; getAbsoluteAvatarPath: (src: string) => string; getAbsoluteBadgeImageFilePath: (path: string) => string; + getAbsoluteDownloadsPath: (path: string) => string; getAbsoluteDraftPath: (path: string) => string; getAbsoluteStickerPath: (path: string) => string; getAbsoluteTempPath: (path: string) => string; @@ -161,6 +163,7 @@ export function initializeMigrations({ createDoesExist, getAvatarsPath, getDraftPath, + getDownloadsPath, getPath, getStickersPath, getBadgesPath, @@ -260,6 +263,10 @@ export function initializeMigrations({ const deleteDraftFile = Attachments.createDeleter(draftPath); const readDraftData = createEncryptedReader(draftPath); + const downloadsPath = getDownloadsPath(userDataPath); + const getAbsoluteDownloadsPath = createAbsolutePathGetter(downloadsPath); + const deleteDownloadOnDisk = Attachments.createDeleter(downloadsPath); + const avatarsPath = getAvatarsPath(userDataPath); const readAvatarData = createEncryptedReader(avatarsPath); const getAbsoluteAvatarPath = createAbsolutePathGetter(avatarsPath); @@ -272,9 +279,13 @@ export function initializeMigrations({ copyIntoTempDirectory, deleteAttachmentData: deleteOnDisk, deleteAvatar, + deleteDownloadData: deleteDownloadOnDisk, deleteDraftFile, deleteExternalMessageFiles: MessageType.deleteAllExternalFiles({ - deleteAttachmentData: Type.deleteData(deleteOnDisk), + deleteAttachmentData: Type.deleteData({ + deleteOnDisk, + deleteDownloadOnDisk, + }), deleteOnDisk, }), deleteSticker, @@ -283,6 +294,7 @@ export function initializeMigrations({ getAbsoluteAttachmentPath, getAbsoluteAvatarPath, getAbsoluteBadgeImageFilePath, + getAbsoluteDownloadsPath, getAbsoluteDraftPath, getAbsoluteStickerPath, getAbsoluteTempPath, @@ -357,6 +369,7 @@ type StringGetterType = (basePath: string) => string; type AttachmentsModuleType = { getAvatarsPath: StringGetterType; getBadgesPath: StringGetterType; + getDownloadsPath: StringGetterType; getDraftPath: StringGetterType; getPath: StringGetterType; getStickersPath: StringGetterType; diff --git a/ts/sql/Client.ts b/ts/sql/Client.ts index 34b81a9d08bf..d189c510c85a 100644 --- a/ts/sql/Client.ts +++ b/ts/sql/Client.ts @@ -70,6 +70,7 @@ const ERASE_ATTACHMENTS_KEY = 'erase-attachments'; const ERASE_STICKERS_KEY = 'erase-stickers'; const ERASE_TEMP_KEY = 'erase-temp'; const ERASE_DRAFTS_KEY = 'erase-drafts'; +const ERASE_DOWNLOADS_KEY = 'erase-downloads'; const CLEANUP_ORPHANED_ATTACHMENTS_KEY = 'cleanup-orphaned-attachments'; const ENSURE_FILE_PERMISSIONS = 'ensure-file-permissions'; const PAUSE_WRITE_ACCESS = 'pause-sql-writes'; @@ -803,6 +804,7 @@ async function removeOtherData(): Promise { invokeWithTimeout(ERASE_ATTACHMENTS_KEY), invokeWithTimeout(ERASE_STICKERS_KEY), invokeWithTimeout(ERASE_TEMP_KEY), + invokeWithTimeout(ERASE_DOWNLOADS_KEY), invokeWithTimeout(ERASE_DRAFTS_KEY), ]); } diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index df391134d467..96910f768507 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -406,6 +406,7 @@ export type MessageAttachmentsCursorType = MessageCursorType & export type GetKnownMessageAttachmentsResultType = Readonly<{ cursor: MessageAttachmentsCursorType; attachments: ReadonlyArray; + downloads: ReadonlyArray; }>; export type PageMessagesCursorType = MessageCursorType & diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index e49924aa3295..fe91f43ed357 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -6442,9 +6442,13 @@ function getMessageServerGuidsForSpam( .all({ conversationId }); } -function getExternalFilesForMessage(message: MessageType): Array { +function getExternalFilesForMessage(message: MessageType): { + externalAttachments: Array; + externalDownloads: Array; +} { const { attachments, contact, quote, preview, sticker } = message; - const files: Array = []; + const externalAttachments: Array = []; + const externalDownloads: Array = []; forEach(attachments, attachment => { const { @@ -6452,21 +6456,28 @@ function getExternalFilesForMessage(message: MessageType): Array { thumbnail, screenshot, thumbnailFromBackup, + downloadPath, } = attachment; if (file) { - files.push(file); + externalAttachments.push(file); + } + + // downloadPath is relative to downloads folder and has to be tracked + // separately. + if (downloadPath) { + externalDownloads.push(downloadPath); } if (thumbnail && thumbnail.path) { - files.push(thumbnail.path); + externalAttachments.push(thumbnail.path); } if (screenshot && screenshot.path) { - files.push(screenshot.path); + externalAttachments.push(screenshot.path); } if (thumbnailFromBackup && thumbnailFromBackup.path) { - files.push(thumbnailFromBackup.path); + externalAttachments.push(thumbnailFromBackup.path); } }); @@ -6475,7 +6486,7 @@ function getExternalFilesForMessage(message: MessageType): Array { const { thumbnail } = attachment; if (thumbnail && thumbnail.path) { - files.push(thumbnail.path); + externalAttachments.push(thumbnail.path); } }); } @@ -6485,7 +6496,7 @@ function getExternalFilesForMessage(message: MessageType): Array { const { avatar } = item; if (avatar && avatar.avatar && avatar.avatar.path) { - files.push(avatar.avatar.path); + externalAttachments.push(avatar.avatar.path); } }); } @@ -6495,20 +6506,20 @@ function getExternalFilesForMessage(message: MessageType): Array { const { image } = item; if (image && image.path) { - files.push(image.path); + externalAttachments.push(image.path); } }); } if (sticker && sticker.data && sticker.data.path) { - files.push(sticker.data.path); + externalAttachments.push(sticker.data.path); if (sticker.data.thumbnail && sticker.data.thumbnail.path) { - files.push(sticker.data.thumbnail.path); + externalAttachments.push(sticker.data.thumbnail.path); } } - return files; + return { externalAttachments, externalDownloads }; } function getExternalFilesForConversation( @@ -6559,17 +6570,20 @@ function getKnownMessageAttachments( const innerCursor = cursor as MessageCursorType | undefined as | PageMessagesCursorType | undefined; - const result = new Set(); + const attachments = new Set(); + const downloads = new Set(); const { messages, cursor: newCursor } = pageMessages(db, innerCursor); - for (const message of messages) { - const externalFiles = getExternalFilesForMessage(message); - forEach(externalFiles, file => result.add(file)); + const { externalAttachments, externalDownloads } = + getExternalFilesForMessage(message); + externalAttachments.forEach(file => attachments.add(file)); + externalDownloads.forEach(file => downloads.add(file)); } return { - attachments: Array.from(result), + attachments: Array.from(attachments), + downloads: Array.from(downloads), cursor: newCursor as MessageCursorType as MessageAttachmentsCursorType, }; } diff --git a/ts/test-both/processDataMessage_test.ts b/ts/test-both/processDataMessage_test.ts index b644b27b0d45..8a39e4a8ac18 100644 --- a/ts/test-both/processDataMessage_test.ts +++ b/ts/test-both/processDataMessage_test.ts @@ -46,7 +46,10 @@ describe('processDataMessage', () => { timestamp: Long.fromNumber(TIMESTAMP), ...message, }, - TIMESTAMP + TIMESTAMP, + { + _createName: () => 'random-path', + } ); it('should process attachments', () => { @@ -54,7 +57,12 @@ describe('processDataMessage', () => { attachments: [UNPROCESSED_ATTACHMENT], }); - assert.deepStrictEqual(out.attachments, [PROCESSED_ATTACHMENT]); + assert.deepStrictEqual(out.attachments, [ + { + ...PROCESSED_ATTACHMENT, + downloadPath: 'random-path', + }, + ]); }); it('should process attachments with 0 cdnId', () => { @@ -71,6 +79,7 @@ describe('processDataMessage', () => { { ...PROCESSED_ATTACHMENT, cdnId: undefined, + downloadPath: 'random-path', }, ]); }); diff --git a/ts/textsecure/Types.d.ts b/ts/textsecure/Types.d.ts index 6fee83541b20..1ff3c3d7fae6 100644 --- a/ts/textsecure/Types.d.ts +++ b/ts/textsecure/Types.d.ts @@ -119,6 +119,7 @@ export type ProcessedAttachment = { cdnNumber?: number; textAttachment?: Omit; backupLocator?: AttachmentType['backupLocator']; + downloadPath?: string; }; export type ProcessedGroupV2Context = { diff --git a/ts/textsecure/WebAPI.ts b/ts/textsecure/WebAPI.ts index 7c0d2d1e2cd8..3e1431a33ccf 100644 --- a/ts/textsecure/WebAPI.ts +++ b/ts/textsecure/WebAPI.ts @@ -184,7 +184,8 @@ type PromiseAjaxOptionsType = { | 'jsonwithdetails' | 'bytes' | 'byteswithdetails' - | 'stream'; + | 'stream' + | 'streamwithdetails'; serverUrl?: string; stack?: string; timeout?: number; @@ -214,6 +215,11 @@ type BytesWithDetailsType = { contentType: string | null; response: Response; }; +type StreamWithDetailsType = { + stream: Readable; + contentType: string | null; + response: Response; +}; export const multiRecipient200ResponseSchema = z.object({ uuids404: z.array(serviceIdSchema).optional(), @@ -386,7 +392,10 @@ async function _promiseAjax( options.responseType === 'byteswithdetails' ) { result = await response.buffer(); - } else if (options.responseType === 'stream') { + } else if ( + options.responseType === 'stream' || + options.responseType === 'streamwithdetails' + ) { result = response.body; } else { result = await response.textConverted(); @@ -437,6 +446,24 @@ async function _promiseAjax( return result; } + if (options.responseType === 'streamwithdetails') { + log.info(logId, response.status, 'Streaming with details'); + response.body.on('error', e => { + log.info(logId, 'Errored while streaming:', e.message); + }); + response.body.on('end', () => { + log.info(logId, response.status, 'Streaming ended'); + }); + + const fullResult: StreamWithDetailsType = { + stream: result as Readable, + contentType: getContentType(response), + response, + }; + + return fullResult; + } + log.info(logId, response.status, 'Success'); if (options.responseType === 'byteswithdetails') { @@ -506,6 +533,10 @@ function _outerAjax( providedUrl: string | null, options: PromiseAjaxOptionsType & { responseType?: 'stream' } ): Promise; +function _outerAjax( + providedUrl: string | null, + options: PromiseAjaxOptionsType & { responseType: 'streamwithdetails' } +): Promise; function _outerAjax( providedUrl: string | null, options: PromiseAjaxOptionsType @@ -1215,6 +1246,7 @@ export type WebAPIType = { options?: { disableRetries?: boolean; timeout?: number; + downloadOffset?: number; }; }) => Promise; getAttachment: (args: { @@ -1223,6 +1255,7 @@ export type WebAPIType = { options?: { disableRetries?: boolean; timeout?: number; + downloadOffset?: number; }; }) => Promise; getAttachmentUploadForm: () => Promise; @@ -1789,6 +1822,9 @@ export function initialize({ function _ajax( param: AjaxOptionsType & { responseType: 'stream' } ): Promise; + function _ajax( + param: AjaxOptionsType & { responseType: 'streamwithdetails' } + ): Promise; function _ajax( param: AjaxOptionsType & { responseType: 'json' } ): Promise; @@ -3442,6 +3478,7 @@ export function initialize({ options?: { disableRetries?: boolean; timeout?: number; + downloadOffset?: number; }; }) { return _getAttachment({ @@ -3468,6 +3505,7 @@ export function initialize({ options?: { disableRetries?: boolean; timeout?: number; + downloadOffset?: number; }; }) { return _getAttachment({ @@ -3482,7 +3520,7 @@ export function initialize({ async function _getAttachment({ cdnPath, cdnNumber, - headers, + headers = {}, redactor, options, }: { @@ -3493,12 +3531,13 @@ export function initialize({ options?: { disableRetries?: boolean; timeout?: number; + downloadOffset?: number; }; }): Promise { const abortController = new AbortController(); const cdnUrl = cdnUrlObject[cdnNumber] ?? cdnUrlObject['0']; - let downloadStream: Readable | undefined; + let streamWithDetails: StreamWithDetailsType | undefined; const cancelRequest = () => { abortController.abort(); @@ -3508,23 +3547,38 @@ export function initialize({ // This is going to the CDN, not the service, so we use _outerAjax try { - downloadStream = await _outerAjax(`${cdnUrl}${cdnPath}`, { - headers, + const targetHeaders = { ...headers }; + if (options?.downloadOffset) { + targetHeaders.range = `bytes=${options.downloadOffset}-`; + } + streamWithDetails = await _outerAjax(`${cdnUrl}${cdnPath}`, { + headers: targetHeaders, certificateAuthority, disableRetries: options?.disableRetries, proxyUrl, - responseType: 'stream', + responseType: 'streamwithdetails', timeout: options?.timeout ?? DEFAULT_TIMEOUT, type: 'GET', redactUrl: redactor, version, abortSignal: abortController.signal, }); + + if (targetHeaders.range != null) { + strictAssert( + streamWithDetails.response.status === 206, + `Expected 206 status code for offset ${options?.downloadOffset}` + ); + strictAssert( + !streamWithDetails.contentType?.includes('multipart'), + `Expected non-multipart response for ${cdnUrl}${cdnPath}` + ); + } } finally { - if (!downloadStream) { + if (!streamWithDetails) { unregisterInFlightRequest(cancelRequest); } else { - downloadStream.on('close', () => { + streamWithDetails.stream.on('close', () => { unregisterInFlightRequest(cancelRequest); }); } @@ -3536,7 +3590,7 @@ export function initialize({ abortController, }); - const combinedStream = downloadStream + const combinedStream = streamWithDetails.stream // We do this manually; pipe() doesn't flow errors through the streams for us .on('error', (error: Error) => { timeoutStream.emit('error', error); diff --git a/ts/textsecure/downloadAttachment.ts b/ts/textsecure/downloadAttachment.ts index a677d3712614..69328db6c853 100644 --- a/ts/textsecure/downloadAttachment.ts +++ b/ts/textsecure/downloadAttachment.ts @@ -1,9 +1,10 @@ // Copyright 2020 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only -import { createWriteStream } from 'fs'; +import { createWriteStream } from 'node:fs'; +import { stat } from 'node:fs/promises'; import { isNumber } from 'lodash'; -import type { Readable } from 'stream'; +import type { Readable, Writable } from 'stream'; import { Transform } from 'stream'; import { pipeline } from 'stream/promises'; import { ensureFile } from 'fs-extra'; @@ -24,7 +25,7 @@ import { } from '../Crypto'; import { getAttachmentCiphertextLength, - safeUnlinkSync, + safeUnlink, splitKeys, type ReencryptedAttachmentV2, decryptAndReencryptLocally, @@ -124,6 +125,39 @@ export async function downloadAttachment( let downloadResult: Awaited>; + let { downloadPath } = attachment; + let downloadOffset = 0; + if (downloadPath) { + const absoluteDownloadPath = + window.Signal.Migrations.getAbsoluteAttachmentPath(downloadPath); + try { + ({ size: downloadOffset } = await stat(absoluteDownloadPath)); + } catch (error) { + if (error.code !== 'ENOENT') { + log.error( + 'downloadAttachment: Failed to get file size for previous download', + Errors.toLogFormat(error) + ); + try { + await safeUnlink(downloadPath); + } catch { + downloadPath = undefined; + } + } + } + } + + // Start over if we go over the size + if (downloadOffset >= size && downloadPath) { + log.warn('downloadAttachment: went over, retrying'); + await safeUnlink(downloadPath); + downloadOffset = 0; + } + + if (downloadOffset !== 0) { + log.info(`${logId}: resuming from ${downloadOffset}`); + } + if (mediaTier === MediaTier.STANDARD) { strictAssert( options.variant !== AttachmentVariant.ThumbnailFromBackup, @@ -135,9 +169,17 @@ export async function downloadAttachment( const downloadStream = await server.getAttachment({ cdnKey, cdnNumber, - options, + options: { + ...options, + downloadOffset, + }, + }); + downloadResult = await downloadToDisk({ + downloadStream, + size, + downloadPath, + downloadOffset, }); - downloadResult = await downloadToDisk({ downloadStream, size }); } else { const mediaId = options.variant === AttachmentVariant.ThumbnailFromBackup @@ -157,10 +199,15 @@ export async function downloadAttachment( mediaDir, headers: cdnCredentials.headers, cdnNumber, - options, + options: { + ...options, + downloadOffset, + }, }); downloadResult = await downloadToDisk({ downloadStream, + downloadPath, + downloadOffset, size: getAttachmentCiphertextLength( options.variant === AttachmentVariant.ThumbnailFromBackup ? // be generous, accept downloads of up to twice what we expect for thumbnail @@ -170,10 +217,7 @@ export async function downloadAttachment( }); } - const { relativePath: downloadedRelativePath, downloadSize } = downloadResult; - - const cipherTextAbsolutePath = - window.Signal.Migrations.getAbsoluteAttachmentPath(downloadedRelativePath); + const { absolutePath: cipherTextAbsolutePath, downloadSize } = downloadResult; try { switch (options.variant) { @@ -226,23 +270,42 @@ export async function downloadAttachment( } } } finally { - safeUnlinkSync(cipherTextAbsolutePath); + await safeUnlink(cipherTextAbsolutePath); } } async function downloadToDisk({ downloadStream, + downloadPath, + downloadOffset = 0, size, }: { downloadStream: Readable; + downloadPath?: string; + downloadOffset?: number; size: number; -}): Promise<{ relativePath: string; downloadSize: number }> { - const relativeTargetPath = getRelativePath(createName()); - const absoluteTargetPath = - window.Signal.Migrations.getAbsoluteAttachmentPath(relativeTargetPath); +}): Promise<{ absolutePath: string; downloadSize: number }> { + const absoluteTargetPath = downloadPath + ? window.Signal.Migrations.getAbsoluteDownloadsPath(downloadPath) + : window.Signal.Migrations.getAbsoluteAttachmentPath( + getRelativePath(createName()) + ); await ensureFile(absoluteTargetPath); - const writeStream = createWriteStream(absoluteTargetPath); - const targetSize = getAttachmentCiphertextLength(size); + let writeStream: Writable; + if (downloadPath) { + writeStream = createWriteStream(absoluteTargetPath, { + flags: 'a', + start: downloadOffset, + }); + } else { + strictAssert( + !downloadOffset, + 'Download cannot be resumed without downloadPath' + ); + writeStream = createWriteStream(absoluteTargetPath); + } + + const targetSize = getAttachmentCiphertextLength(size) - downloadOffset; let downloadSize = 0; try { @@ -255,19 +318,23 @@ async function downloadToDisk({ writeStream ); } catch (error) { - try { - safeUnlinkSync(absoluteTargetPath); - } catch (cleanupError) { - log.error( - 'downloadToDisk: Error while cleaning up', - Errors.toLogFormat(cleanupError) - ); + if (downloadPath) { + log.warn(`downloadToDisk: stopping at ${downloadSize}`); + } else { + try { + await safeUnlink(absoluteTargetPath); + } catch (cleanupError) { + log.error( + 'downloadToDisk: Error while cleaning up', + Errors.toLogFormat(cleanupError) + ); + } } throw error; } - return { relativePath: relativeTargetPath, downloadSize }; + return { absolutePath: absoluteTargetPath, downloadSize }; } // A simple transform that throws if it sees more than maxBytes on the stream. diff --git a/ts/textsecure/processDataMessage.ts b/ts/textsecure/processDataMessage.ts index ab63f59a7c10..a9304ad91a88 100644 --- a/ts/textsecure/processDataMessage.ts +++ b/ts/textsecure/processDataMessage.ts @@ -32,6 +32,7 @@ import { filterAndClean } from '../types/BodyRange'; import { isAciString } from '../util/isAciString'; import { normalizeAci } from '../util/normalizeAci'; import { bytesToUuid } from '../util/uuidToBytes'; +import { createName } from '../util/attachmentPath'; const FLAGS = Proto.DataMessage.Flags; export const ATTACHMENT_MAX = 32; @@ -284,7 +285,10 @@ export function processGiftBadge( export function processDataMessage( message: Proto.IDataMessage, - envelopeTimestamp: number + envelopeTimestamp: number, + + // Only for testing + { _createName: doCreateName = createName } = {} ): ProcessedDataMessage { /* eslint-disable no-bitwise */ @@ -309,7 +313,10 @@ export function processDataMessage( const result: ProcessedDataMessage = { body: dropNull(message.body), attachments: (message.attachments ?? []).map( - (attachment: Proto.IAttachmentPointer) => processAttachment(attachment) + (attachment: Proto.IAttachmentPointer) => ({ + ...processAttachment(attachment), + downloadPath: doCreateName(), + }) ), groupV2: processGroupV2Context(message.groupV2), flags: message.flags ?? 0, diff --git a/ts/types/Attachment.ts b/ts/types/Attachment.ts index 4244b79729d8..ae7cf0fc63bf 100644 --- a/ts/types/Attachment.ts +++ b/ts/types/Attachment.ts @@ -78,6 +78,7 @@ export type AttachmentType = { cdnNumber?: number; cdnId?: string; cdnKey?: string; + downloadPath?: string; key?: string; iv?: string; data?: Uint8Array; @@ -386,9 +387,13 @@ export function loadData( }; } -export function deleteData( - deleteOnDisk: (path: string) => Promise -): (attachment?: AttachmentType) => Promise { +export function deleteData({ + deleteOnDisk, + deleteDownloadOnDisk, +}: { + deleteOnDisk: (path: string) => Promise; + deleteDownloadOnDisk: (path: string) => Promise; +}): (attachment?: AttachmentType) => Promise { if (!isFunction(deleteOnDisk)) { throw new TypeError('deleteData: deleteOnDisk must be a function'); } @@ -398,12 +403,17 @@ export function deleteData( throw new TypeError('deleteData: attachment is not valid'); } - const { path, thumbnail, screenshot, thumbnailFromBackup } = attachment; + const { path, downloadPath, thumbnail, screenshot, thumbnailFromBackup } = + attachment; if (isString(path)) { await deleteOnDisk(path); } + if (isString(downloadPath)) { + await deleteDownloadOnDisk(downloadPath); + } + if (thumbnail && isString(thumbnail.path)) { await deleteOnDisk(thumbnail.path); } diff --git a/ts/util/createIPCEvents.ts b/ts/util/createIPCEvents.ts index ee0ffe5a371a..56a74821d8e5 100644 --- a/ts/util/createIPCEvents.ts +++ b/ts/util/createIPCEvents.ts @@ -104,6 +104,7 @@ export type IPCEventsCallbacksType = { }>; addCustomColor: (customColor: CustomColorType) => void; addDarkOverlay: () => void; + cleanupDownloads: () => Promise; deleteAllData: () => Promise; deleteAllMyStories: () => Promise; editCustomColor: (colorId: string, customColor: CustomColorType) => void; @@ -533,6 +534,10 @@ export function createIPCEvents( showKeyboardShortcuts: () => window.reduxActions.globalModals.showShortcutGuideModal(), + cleanupDownloads: async () => { + await ipcRenderer.invoke('cleanup-downloads'); + }, + deleteAllData: async () => { renderClearingDataView(); }, diff --git a/ts/util/deleteForMe.ts b/ts/util/deleteForMe.ts index 6b6c617760a5..910e494e9148 100644 --- a/ts/util/deleteForMe.ts +++ b/ts/util/deleteForMe.ts @@ -123,9 +123,11 @@ export async function deleteAttachmentFromMessage( }, { deleteOnDisk, + deleteDownloadOnDisk, logId, }: { deleteOnDisk: (path: string) => Promise; + deleteDownloadOnDisk: (path: string) => Promise; logId: string; } ): Promise { @@ -147,6 +149,7 @@ export async function deleteAttachmentFromMessage( return applyDeleteAttachmentFromMessage(message, deleteAttachmentData, { deleteOnDisk, + deleteDownloadOnDisk, logId, shouldSave: true, }); @@ -165,10 +168,12 @@ export async function applyDeleteAttachmentFromMessage( }, { deleteOnDisk, + deleteDownloadOnDisk, shouldSave, logId, }: { deleteOnDisk: (path: string) => Promise; + deleteDownloadOnDisk: (path: string) => Promise; shouldSave: boolean; logId: string; } @@ -206,7 +211,7 @@ export async function applyDeleteAttachmentFromMessage( if (shouldSave) { await saveMessage(message.attributes, { ourAci }); } - await deleteData(deleteOnDisk)(attachment); + await deleteData({ deleteOnDisk, deleteDownloadOnDisk })(attachment); return true; } diff --git a/ts/util/modifyTargetMessage.ts b/ts/util/modifyTargetMessage.ts index c880533e0add..09154ec749ac 100644 --- a/ts/util/modifyTargetMessage.ts +++ b/ts/util/modifyTargetMessage.ts @@ -92,6 +92,7 @@ export async function modifyTargetMessage( logId, shouldSave: false, deleteOnDisk: window.Signal.Migrations.deleteAttachmentData, + deleteDownloadOnDisk: window.Signal.Migrations.deleteDownloadData, } ); if (result) { diff --git a/ts/util/uploadAttachment.ts b/ts/util/uploadAttachment.ts index 2a1f826f8e23..3f74857e7ca6 100644 --- a/ts/util/uploadAttachment.ts +++ b/ts/util/uploadAttachment.ts @@ -15,7 +15,7 @@ import type { AttachmentUploadFormResponseType } from '../textsecure/WebAPI'; import { type EncryptedAttachmentV2, encryptAttachmentV2ToDisk, - safeUnlinkSync, + safeUnlink, type PlaintextSourceType, type HardcodedIVForEncryptionType, } from '../AttachmentCrypto'; @@ -117,7 +117,7 @@ export async function encryptAndUploadAttachment({ return { cdnKey: uploadForm.key, cdnNumber: uploadForm.cdn, encrypted }; } finally { if (absoluteCiphertextPath) { - safeUnlinkSync(absoluteCiphertextPath); + await safeUnlink(absoluteCiphertextPath); } } }