mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Resumable attachment downloads

Browse source
This commit is contained in:
Fedor Indutny 2024-08-19 13:05:35 -07:00 committed by GitHub
parent 2c92591b59
commit 38f532cdda
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 401 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
app/attachment_channel.ts
View file

@ -18,13 +18,17 @@ import {
import { RangeFinder, DefaultStorage } from '@indutny/range-finder';
import {
getAllAttachments,
getAllDownloads,
getAvatarsPath,
getPath,
getStickersPath,
getTempPath,
getDraftPath,
getDownloadsPath,
deleteAll as deleteAllAttachments,
deleteAllBadges,
deleteAllDownloads,
deleteStaleDownloads,
getAllStickers,
deleteAllStickers,
getAllDraftAttachments,
@ -50,6 +54,8 @@ const ERASE_ATTACHMENTS_KEY = 'erase-attachments';
const ERASE_STICKERS_KEY = 'erase-stickers';
const ERASE_TEMP_KEY = 'erase-temp';
const ERASE_DRAFTS_KEY = 'erase-drafts';
const ERASE_DOWNLOADS_KEY = 'erase-downloads';
const CLEANUP_DOWNLOADS_KEY = 'cleanup-downloads';
const CLEANUP_ORPHANED_ATTACHMENTS_KEY = 'cleanup-orphaned-attachments';
const INTERACTIVITY_DELAY = 50;
@ -189,6 +195,7 @@ const dispositionSchema = z.enum([
type DeleteOrphanedAttachmentsOptionsType = Readonly<{
orphanedAttachments: Set<string>;
orphanedDownloads: Set<string>;
sql: MainSQL;
userDataPath: string;
}>;
@ -235,8 +242,14 @@ async function cleanupOrphanedAttachments({
`${orphanedAttachments.size} attachments on disk`
);
const orphanedDownloads = new Set(await getAllDownloads(userDataPath));
console.log(
'cleanupOrphanedAttachments: found ' +
`${orphanedDownloads.size} downloads on disk`
);
{
const attachments: ReadonlyArray<string> = await sql.sqlRead(
const attachments: Array<string> = await sql.sqlRead(
'getKnownConversationAttachments'
);
@ -258,6 +271,7 @@ async function cleanupOrphanedAttachments({
// are saved to disk, but not put into any message or conversation model yet.
deleteOrphanedAttachments({
orphanedAttachments,
orphanedDownloads,
sql,
userDataPath,
});
@ -265,6 +279,7 @@ async function cleanupOrphanedAttachments({
function deleteOrphanedAttachments({
orphanedAttachments,
orphanedDownloads,
sql,
userDataPath,
}: DeleteOrphanedAttachmentsOptionsType): void {
@ -273,17 +288,21 @@ function deleteOrphanedAttachments({
let cursor: MessageAttachmentsCursorType | undefined;
let totalFound = 0;
let totalMissing = 0;
let totalDownloadsFound = 0;
let totalDownloadsMissing = 0;
try {
do {
let attachments: ReadonlyArray<string>;
let downloads: ReadonlyArray<string>;
// eslint-disable-next-line no-await-in-loop
({ attachments, cursor } = await sql.sqlRead(
({ attachments, downloads, cursor } = await sql.sqlRead(
'getKnownMessageAttachments',
cursor
));
totalFound += attachments.length;
totalDownloadsFound += downloads.length;
for (const known of attachments) {
if (!orphanedAttachments.delete(known)) {
@ -291,6 +310,12 @@ function deleteOrphanedAttachments({
}
}
for (const known of downloads) {
if (!orphanedDownloads.delete(known)) {
totalDownloadsMissing += 1;
}
}
if (cursor === undefined) {
break;
}
@ -316,6 +341,16 @@ function deleteOrphanedAttachments({
userDataPath,
attachments: Array.from(orphanedAttachments),
});
console.log(
`cleanupOrphanedAttachments: found ${totalDownloadsFound} downloads ` +
`(${totalDownloadsMissing} missing) ` +
`${orphanedDownloads.size} remain`
);
await deleteAllDownloads({
userDataPath,
downloads: Array.from(orphanedDownloads),
});
}
async function runSafe() {
@ -341,6 +376,7 @@ let attachmentsDir: string | undefined;
let stickersDir: string | undefined;
let tempDir: string | undefined;
let draftDir: string | undefined;
let downloadsDir: string | undefined;
let avatarDataDir: string | undefined;
export function initialize({
@ -359,6 +395,7 @@ export function initialize({
stickersDir = getStickersPath(configDir);
tempDir = getTempPath(configDir);
draftDir = getDraftPath(configDir);
downloadsDir = getDownloadsPath(configDir);
avatarDataDir = getAvatarsPath(configDir);
ipcMain.handle(ERASE_TEMP_KEY, () => {
@ -377,6 +414,10 @@ export function initialize({
strictAssert(draftDir != null, 'not initialized');
rimraf.sync(draftDir);
});
ipcMain.handle(ERASE_DOWNLOADS_KEY, () => {
strictAssert(downloadsDir != null, 'not initialized');
rimraf.sync(downloadsDir);
});
ipcMain.handle(CLEANUP_ORPHANED_ATTACHMENTS_KEY, async () => {
const start = Date.now();
@ -385,6 +426,13 @@ export function initialize({
console.log(`cleanupOrphanedAttachments: took ${duration}ms`);
});
ipcMain.handle(CLEANUP_DOWNLOADS_KEY, async () => {
const start = Date.now();
await deleteStaleDownloads(configDir);
const duration = Date.now() - start;
console.log(`cleanupDownloads: took ${duration}ms`);
});
protocol.handle('attachment', handleAttachmentRequest);
}

89
app/attachments.ts
View file

@ -2,18 +2,24 @@
// SPDX-License-Identifier: AGPL-3.0-only
import { PassThrough } from 'node:stream';
import { stat } from 'node:fs/promises';
import { join, relative, normalize } from 'path';
import pMap from 'p-map';
import fastGlob from 'fast-glob';
import fse from 'fs-extra';
import { map, isString } from 'lodash';
import normalizePath from 'normalize-path';
import { isPathInside } from '../ts/util/isPathInside';
import { DAY } from '../ts/util/durations';
import { isOlderThan } from '../ts/util/timestamp';
import { isNotNil } from '../ts/util/isNotNil';
import {
generateKeys,
decryptAttachmentV2ToSink,
encryptAttachmentV2ToDisk,
} from '../ts/AttachmentCrypto';
import type { LocalAttachmentV2Type } from '../ts/types/Attachment';
import * as Errors from '../ts/types/errors';
const PATH = 'attachments.noindex';
const AVATAR_PATH = 'avatars.noindex';
@ -22,9 +28,12 @@ const STICKER_PATH = 'stickers.noindex';
const TEMP_PATH = 'temp';
const UPDATE_CACHE_PATH = 'update-cache';
const DRAFT_PATH = 'drafts.noindex';
const DOWNLOADS_PATH = 'downloads.noindex';
const CACHED_PATHS = new Map<string, string>();
const FS_CONCURRENCY = 100;
const createPathGetter =
(subpath: string) =>
(userDataPath: string): string => {
@ -52,6 +61,7 @@ const createPathGetter =
export const getAvatarsPath = createPathGetter(AVATAR_PATH);
export const getBadgesPath = createPathGetter(BADGES_PATH);
export const getDraftPath = createPathGetter(DRAFT_PATH);
export const getDownloadsPath = createPathGetter(DOWNLOADS_PATH);
export const getPath = createPathGetter(PATH);
export const getStickersPath = createPathGetter(STICKER_PATH);
export const getTempPath = createPathGetter(TEMP_PATH);
@ -88,6 +98,16 @@ export const getAllAttachments = async (
return map(files, file => relative(dir, file));
};
export const getAllDownloads = async (
userDataPath: string
): Promise<ReadonlyArray<string>> => {
const dir = getDownloadsPath(userDataPath);
const pattern = normalizePath(join(dir, '**', '*'));
const files = await fastGlob(pattern, { onlyFiles: true });
return map(files, file => relative(dir, file));
};
const getAllBadgeImageFiles = async (
userDataPath: string
): Promise<ReadonlyArray<string>> => {
@ -123,6 +143,43 @@ export const clearTempPath = (userDataPath: string): Promise<void> => {
return fse.emptyDir(tempPath);
};
export const deleteStaleDownloads = async (
userDataPath: string
): Promise<void> => {
const dir = getDownloadsPath(userDataPath);
const files = await getAllDownloads(userDataPath);
const result = await pMap(
files,
async file => {
try {
const { birthtimeMs } = await stat(join(dir, file));
if (isOlderThan(birthtimeMs, DAY)) {
return file;
}
} catch (error) {
// No longer exists
if (error.code === 'ENOENT') {
return;
}
console.error(
'deleteStaleDownloads: failed to get file stats',
Errors.toLogFormat(error)
);
}
return undefined;
},
{ concurrency: FS_CONCURRENCY }
);
const stale = result.filter(isNotNil);
if (stale.length === 0) {
return;
}
console.log(`deleteStaleDownloads: found ${stale.length}`);
await deleteAllDownloads({ userDataPath, downloads: stale });
};
export const deleteAll = async ({
userDataPath,
attachments,
@ -132,15 +189,25 @@ export const deleteAll = async ({
}): Promise<void> => {
const deleteFromDisk = createDeleter(getPath(userDataPath));
for (let index = 0, max = attachments.length; index < max; index += 1) {
const file = attachments[index];
// eslint-disable-next-line no-await-in-loop
await deleteFromDisk(file);
}
await pMap(attachments, deleteFromDisk, { concurrency: FS_CONCURRENCY });
console.log(`deleteAll: deleted ${attachments.length} files`);
};
export const deleteAllDownloads = async ({
userDataPath,
downloads,
}: {
userDataPath: string;
downloads: ReadonlyArray<string>;
}): Promise<void> => {
const deleteFromDisk = createDeleter(getDownloadsPath(userDataPath));
await pMap(downloads, deleteFromDisk, { concurrency: FS_CONCURRENCY });
console.log(`deleteAllDownloads: deleted ${downloads.length} files`);
};
export const deleteAllStickers = async ({
userDataPath,
stickers,
@ -150,11 +217,7 @@ export const deleteAllStickers = async ({
}): Promise<void> => {
const deleteFromDisk = createDeleter(getStickersPath(userDataPath));
for (let index = 0, max = stickers.length; index < max; index += 1) {
const file = stickers[index];
// eslint-disable-next-line no-await-in-loop
await deleteFromDisk(file);
}
await pMap(stickers, deleteFromDisk, { concurrency: FS_CONCURRENCY });
console.log(`deleteAllStickers: deleted ${stickers.length} files`);
};
@ -189,11 +252,7 @@ export const deleteAllDraftAttachments = async ({
}): Promise<void> => {
const deleteFromDisk = createDeleter(getDraftPath(userDataPath));
for (let index = 0, max = attachments.length; index < max; index += 1) {
const file = attachments[index];
// eslint-disable-next-line no-await-in-loop
await deleteFromDisk(file);
}
await pMap(attachments, deleteFromDisk, { concurrency: FS_CONCURRENCY });
console.log(`deleteAllDraftAttachments: deleted ${attachments.length} files`);
};

9
app/main.ts
View file

@ -2208,6 +2208,15 @@ app.on('ready', async () => {
);
}
try {
await attachments.deleteStaleDownloads(userDataPath);
} catch (err) {
logger.error(
'main/ready: Error deleting stale downloads:',
Errors.toLogFormat(err)
);
}
// Initialize IPC channels before creating the window
attachmentChannel.initialize({

2
app/protocol_filter.ts
View file

@ -9,6 +9,7 @@ import {
getAvatarsPath,
getBadgesPath,
getDraftPath,
getDownloadsPath,
getPath,
getStickersPath,
getTempPath,
@ -61,6 +62,7 @@ function _createFileHandler({
getAvatarsPath(userDataPath),
getBadgesPath(userDataPath),
getDraftPath(userDataPath),
getDownloadsPath(userDataPath),
getPath(userDataPath),
getStickersPath(userDataPath),
getTempPath(userDataPath),

14
ts/AttachmentCrypto.ts
View file

@ -1,8 +1,8 @@
// Copyright 2020 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { unlinkSync, createReadStream, createWriteStream } from 'fs';
import { open } from 'fs/promises';
import { createReadStream, createWriteStream } from 'fs';
import { open, unlink } from 'fs/promises';
import { createCipheriv, createHash, createHmac, randomBytes } from 'crypto';
import type { Hash } from 'crypto';
import { PassThrough, Transform, type Writable, Readable } from 'stream';
@ -114,7 +114,7 @@ export async function encryptAttachmentV2ToDisk(
sink: createWriteStream(absoluteTargetPath),
});
} catch (error) {
safeUnlinkSync(absoluteTargetPath);
await safeUnlink(absoluteTargetPath);
throw error;
}
@ -307,7 +307,7 @@ export async function decryptAttachmentV2(
`${logId}: Failed to decrypt attachment to disk`,
Errors.toLogFormat(error)
);
safeUnlinkSync(absoluteTargetPath);
await safeUnlink(absoluteTargetPath);
throw error;
} finally {
await writeFd?.close();
@ -523,7 +523,7 @@ export async function decryptAndReencryptLocally(
`${logId}: Failed to decrypt attachment`,
Errors.toLogFormat(error)
);
safeUnlinkSync(absoluteTargetPath);
await safeUnlink(absoluteTargetPath);
throw error;
} finally {
await writeFd?.close();
@ -618,9 +618,9 @@ export function getPlaintextHashForInMemoryAttachment(
* Unlinks a file without throwing an error if it doesn't exist.
* Throws an error if it fails to unlink for any other reason.
*/
export function safeUnlinkSync(filePath: string): void {
export async function safeUnlink(filePath: string): Promise<void> {
try {
unlinkSync(filePath);
await unlink(filePath);
} catch (error) {
// Ignore if file doesn't exist
if (error.code !== 'ENOENT') {

4
ts/background.ts
View file

@ -1104,6 +1104,10 @@ export async function startApp(): Promise<void> {
reportLongRunningTasks();
}, FIVE_MINUTES);
setInterval(() => {
drop(window.Events.cleanupDownloads());
}, DAY);
let mainWindowStats = {
isMaximized: false,
isFullScreen: false,

2
ts/jobs/AttachmentDownloadManager.ts
View file

@ -411,7 +411,7 @@ export async function runDownloadAttachmentJobInner({
const upgradedAttachment =
await window.Signal.Migrations.processNewAttachment({
...omit(attachment, ['error', 'pending']),
...omit(attachment, ['error', 'pending', 'downloadPath']),
...downloaded,
});

1
ts/messageModifiers/DeletesForMe.ts
View file

@ -107,6 +107,7 @@ export async function onDelete(item: DeleteForMeAttributesType): Promise<void> {
item.deleteAttachmentData,
{
deleteOnDisk: window.Signal.Migrations.deleteAttachmentData,
deleteDownloadOnDisk: window.Signal.Migrations.deleteDownloadData,
logId,
}
);

15
ts/signal.ts
View file

@ -71,6 +71,7 @@ type MigrationsModuleType = {
) => Promise<{ path: string; size: number }>;
deleteAttachmentData: (path: string) => Promise<void>;
deleteAvatar: (path: string) => Promise<void>;
deleteDownloadData: (path: string) => Promise<void>;
deleteDraftFile: (path: string) => Promise<void>;
deleteExternalMessageFiles: (
attributes: MessageAttributesType
@ -81,6 +82,7 @@ type MigrationsModuleType = {
getAbsoluteAttachmentPath: (path: string) => string;
getAbsoluteAvatarPath: (src: string) => string;
getAbsoluteBadgeImageFilePath: (path: string) => string;
getAbsoluteDownloadsPath: (path: string) => string;
getAbsoluteDraftPath: (path: string) => string;
getAbsoluteStickerPath: (path: string) => string;
getAbsoluteTempPath: (path: string) => string;
@ -161,6 +163,7 @@ export function initializeMigrations({
createDoesExist,
getAvatarsPath,
getDraftPath,
getDownloadsPath,
getPath,
getStickersPath,
getBadgesPath,
@ -260,6 +263,10 @@ export function initializeMigrations({
const deleteDraftFile = Attachments.createDeleter(draftPath);
const readDraftData = createEncryptedReader(draftPath);
const downloadsPath = getDownloadsPath(userDataPath);
const getAbsoluteDownloadsPath = createAbsolutePathGetter(downloadsPath);
const deleteDownloadOnDisk = Attachments.createDeleter(downloadsPath);
const avatarsPath = getAvatarsPath(userDataPath);
const readAvatarData = createEncryptedReader(avatarsPath);
const getAbsoluteAvatarPath = createAbsolutePathGetter(avatarsPath);
@ -272,9 +279,13 @@ export function initializeMigrations({
copyIntoTempDirectory,
deleteAttachmentData: deleteOnDisk,
deleteAvatar,
deleteDownloadData: deleteDownloadOnDisk,
deleteDraftFile,
deleteExternalMessageFiles: MessageType.deleteAllExternalFiles({
deleteAttachmentData: Type.deleteData(deleteOnDisk),
deleteAttachmentData: Type.deleteData({
deleteOnDisk,
deleteDownloadOnDisk,
}),
deleteOnDisk,
}),
deleteSticker,
@ -283,6 +294,7 @@ export function initializeMigrations({
getAbsoluteAttachmentPath,
getAbsoluteAvatarPath,
getAbsoluteBadgeImageFilePath,
getAbsoluteDownloadsPath,
getAbsoluteDraftPath,
getAbsoluteStickerPath,
getAbsoluteTempPath,
@ -357,6 +369,7 @@ type StringGetterType = (basePath: string) => string;
type AttachmentsModuleType = {
getAvatarsPath: StringGetterType;
getBadgesPath: StringGetterType;
getDownloadsPath: StringGetterType;
getDraftPath: StringGetterType;
getPath: StringGetterType;
getStickersPath: StringGetterType;

2
ts/sql/Client.ts
View file

@ -70,6 +70,7 @@ const ERASE_ATTACHMENTS_KEY = 'erase-attachments';
const ERASE_STICKERS_KEY = 'erase-stickers';
const ERASE_TEMP_KEY = 'erase-temp';
const ERASE_DRAFTS_KEY = 'erase-drafts';
const ERASE_DOWNLOADS_KEY = 'erase-downloads';
const CLEANUP_ORPHANED_ATTACHMENTS_KEY = 'cleanup-orphaned-attachments';
const ENSURE_FILE_PERMISSIONS = 'ensure-file-permissions';
const PAUSE_WRITE_ACCESS = 'pause-sql-writes';
@ -803,6 +804,7 @@ async function removeOtherData(): Promise<void> {
invokeWithTimeout(ERASE_ATTACHMENTS_KEY),
invokeWithTimeout(ERASE_STICKERS_KEY),
invokeWithTimeout(ERASE_TEMP_KEY),
invokeWithTimeout(ERASE_DOWNLOADS_KEY),
invokeWithTimeout(ERASE_DRAFTS_KEY),
]);
}

1
ts/sql/Interface.ts
View file

@ -406,6 +406,7 @@ export type MessageAttachmentsCursorType = MessageCursorType &
export type GetKnownMessageAttachmentsResultType = Readonly<{
cursor: MessageAttachmentsCursorType;
attachments: ReadonlyArray<string>;
downloads: ReadonlyArray<string>;
}>;
export type PageMessagesCursorType = MessageCursorType &

48
ts/sql/Server.ts
View file

@ -6442,9 +6442,13 @@ function getMessageServerGuidsForSpam(
.all({ conversationId });
}
function getExternalFilesForMessage(message: MessageType): Array<string> {
function getExternalFilesForMessage(message: MessageType): {
externalAttachments: Array<string>;
externalDownloads: Array<string>;
} {
const { attachments, contact, quote, preview, sticker } = message;
const files: Array<string> = [];
const externalAttachments: Array<string> = [];
const externalDownloads: Array<string> = [];
forEach(attachments, attachment => {
const {
@ -6452,21 +6456,28 @@ function getExternalFilesForMessage(message: MessageType): Array<string> {
thumbnail,
screenshot,
thumbnailFromBackup,
downloadPath,
} = attachment;
if (file) {
files.push(file);
externalAttachments.push(file);
}
// downloadPath is relative to downloads folder and has to be tracked
// separately.
if (downloadPath) {
externalDownloads.push(downloadPath);
}
if (thumbnail && thumbnail.path) {
files.push(thumbnail.path);
externalAttachments.push(thumbnail.path);
}
if (screenshot && screenshot.path) {
files.push(screenshot.path);
externalAttachments.push(screenshot.path);
}
if (thumbnailFromBackup && thumbnailFromBackup.path) {
files.push(thumbnailFromBackup.path);
externalAttachments.push(thumbnailFromBackup.path);
}
});
@ -6475,7 +6486,7 @@ function getExternalFilesForMessage(message: MessageType): Array<string> {
const { thumbnail } = attachment;
if (thumbnail && thumbnail.path) {
files.push(thumbnail.path);
externalAttachments.push(thumbnail.path);
}
});
}
@ -6485,7 +6496,7 @@ function getExternalFilesForMessage(message: MessageType): Array<string> {
const { avatar } = item;
if (avatar && avatar.avatar && avatar.avatar.path) {
files.push(avatar.avatar.path);
externalAttachments.push(avatar.avatar.path);
}
});
}
@ -6495,20 +6506,20 @@ function getExternalFilesForMessage(message: MessageType): Array<string> {
const { image } = item;
if (image && image.path) {
files.push(image.path);
externalAttachments.push(image.path);
}
});
}
if (sticker && sticker.data && sticker.data.path) {
files.push(sticker.data.path);
externalAttachments.push(sticker.data.path);
if (sticker.data.thumbnail && sticker.data.thumbnail.path) {
files.push(sticker.data.thumbnail.path);
externalAttachments.push(sticker.data.thumbnail.path);
}
}
return files;
return { externalAttachments, externalDownloads };
}
function getExternalFilesForConversation(
@ -6559,17 +6570,20 @@ function getKnownMessageAttachments(
const innerCursor = cursor as MessageCursorType | undefined as
| PageMessagesCursorType
| undefined;
const result = new Set<string>();
const attachments = new Set<string>();
const downloads = new Set<string>();
const { messages, cursor: newCursor } = pageMessages(db, innerCursor);
for (const message of messages) {
const externalFiles = getExternalFilesForMessage(message);
forEach(externalFiles, file => result.add(file));
const { externalAttachments, externalDownloads } =
getExternalFilesForMessage(message);
externalAttachments.forEach(file => attachments.add(file));
externalDownloads.forEach(file => downloads.add(file));
}
return {
attachments: Array.from(result),
attachments: Array.from(attachments),
downloads: Array.from(downloads),
cursor: newCursor as MessageCursorType as MessageAttachmentsCursorType,
};
}

13
ts/test-both/processDataMessage_test.ts
View file

@ -46,7 +46,10 @@ describe('processDataMessage', () => {
timestamp: Long.fromNumber(TIMESTAMP),
...message,
},
TIMESTAMP
TIMESTAMP,
{
_createName: () => 'random-path',
}
);
it('should process attachments', () => {
@ -54,7 +57,12 @@ describe('processDataMessage', () => {
attachments: [UNPROCESSED_ATTACHMENT],
});
assert.deepStrictEqual(out.attachments, [PROCESSED_ATTACHMENT]);
assert.deepStrictEqual(out.attachments, [
{
...PROCESSED_ATTACHMENT,
downloadPath: 'random-path',
},
]);
});
it('should process attachments with 0 cdnId', () => {
@ -71,6 +79,7 @@ describe('processDataMessage', () => {
{
...PROCESSED_ATTACHMENT,
cdnId: undefined,
downloadPath: 'random-path',
},
]);
});

1
ts/textsecure/Types.d.ts vendored
View file

@ -119,6 +119,7 @@ export type ProcessedAttachment = {
cdnNumber?: number;
textAttachment?: Omit<TextAttachmentType, 'preview'>;
backupLocator?: AttachmentType['backupLocator'];
downloadPath?: string;
};
export type ProcessedGroupV2Context = {

74
ts/textsecure/WebAPI.ts
View file

@ -184,7 +184,8 @@ type PromiseAjaxOptionsType = {
| 'jsonwithdetails'
| 'bytes'
| 'byteswithdetails'
| 'stream';
| 'stream'
| 'streamwithdetails';
serverUrl?: string;
stack?: string;
timeout?: number;
@ -214,6 +215,11 @@ type BytesWithDetailsType = {
contentType: string | null;
response: Response;
};
type StreamWithDetailsType = {
stream: Readable;
contentType: string | null;
response: Response;
};
export const multiRecipient200ResponseSchema = z.object({
uuids404: z.array(serviceIdSchema).optional(),
@ -386,7 +392,10 @@ async function _promiseAjax(
options.responseType === 'byteswithdetails'
) {
result = await response.buffer();
} else if (options.responseType === 'stream') {
} else if (
options.responseType === 'stream' ||
options.responseType === 'streamwithdetails'
) {
result = response.body;
} else {
result = await response.textConverted();
@ -437,6 +446,24 @@ async function _promiseAjax(
return result;
}
if (options.responseType === 'streamwithdetails') {
log.info(logId, response.status, 'Streaming with details');
response.body.on('error', e => {
log.info(logId, 'Errored while streaming:', e.message);
});
response.body.on('end', () => {
log.info(logId, response.status, 'Streaming ended');
});
const fullResult: StreamWithDetailsType = {
stream: result as Readable,
contentType: getContentType(response),
response,
};
return fullResult;
}
log.info(logId, response.status, 'Success');
if (options.responseType === 'byteswithdetails') {
@ -506,6 +533,10 @@ function _outerAjax(
providedUrl: string | null,
options: PromiseAjaxOptionsType & { responseType?: 'stream' }
): Promise<Readable>;
function _outerAjax(
providedUrl: string | null,
options: PromiseAjaxOptionsType & { responseType: 'streamwithdetails' }
): Promise<StreamWithDetailsType>;
function _outerAjax(
providedUrl: string | null,
options: PromiseAjaxOptionsType
@ -1215,6 +1246,7 @@ export type WebAPIType = {
options?: {
disableRetries?: boolean;
timeout?: number;
downloadOffset?: number;
};
}) => Promise<Readable>;
getAttachment: (args: {
@ -1223,6 +1255,7 @@ export type WebAPIType = {
options?: {
disableRetries?: boolean;
timeout?: number;
downloadOffset?: number;
};
}) => Promise<Readable>;
getAttachmentUploadForm: () => Promise<AttachmentUploadFormResponseType>;
@ -1789,6 +1822,9 @@ export function initialize({
function _ajax(
param: AjaxOptionsType & { responseType: 'stream' }
): Promise<Readable>;
function _ajax(
param: AjaxOptionsType & { responseType: 'streamwithdetails' }
): Promise<StreamWithDetailsType>;
function _ajax(
param: AjaxOptionsType & { responseType: 'json' }
): Promise<unknown>;
@ -3442,6 +3478,7 @@ export function initialize({
options?: {
disableRetries?: boolean;
timeout?: number;
downloadOffset?: number;
};
}) {
return _getAttachment({
@ -3468,6 +3505,7 @@ export function initialize({
options?: {
disableRetries?: boolean;
timeout?: number;
downloadOffset?: number;
};
}) {
return _getAttachment({
@ -3482,7 +3520,7 @@ export function initialize({
async function _getAttachment({
cdnPath,
cdnNumber,
headers,
headers = {},
redactor,
options,
}: {
@ -3493,12 +3531,13 @@ export function initialize({
options?: {
disableRetries?: boolean;
timeout?: number;
downloadOffset?: number;
};
}): Promise<Readable> {
const abortController = new AbortController();
const cdnUrl = cdnUrlObject[cdnNumber] ?? cdnUrlObject['0'];
let downloadStream: Readable | undefined;
let streamWithDetails: StreamWithDetailsType | undefined;
const cancelRequest = () => {
abortController.abort();
@ -3508,23 +3547,38 @@ export function initialize({
// This is going to the CDN, not the service, so we use _outerAjax
try {
downloadStream = await _outerAjax(`${cdnUrl}${cdnPath}`, {
headers,
const targetHeaders = { ...headers };
if (options?.downloadOffset) {
targetHeaders.range = `bytes=${options.downloadOffset}-`;
}
streamWithDetails = await _outerAjax(`${cdnUrl}${cdnPath}`, {
headers: targetHeaders,
certificateAuthority,
disableRetries: options?.disableRetries,
proxyUrl,
responseType: 'stream',
responseType: 'streamwithdetails',
timeout: options?.timeout ?? DEFAULT_TIMEOUT,
type: 'GET',
redactUrl: redactor,
version,
abortSignal: abortController.signal,
});
if (targetHeaders.range != null) {
strictAssert(
streamWithDetails.response.status === 206,
`Expected 206 status code for offset ${options?.downloadOffset}`
);
strictAssert(
!streamWithDetails.contentType?.includes('multipart'),
`Expected non-multipart response for ${cdnUrl}${cdnPath}`
);
}
} finally {
if (!downloadStream) {
if (!streamWithDetails) {
unregisterInFlightRequest(cancelRequest);
} else {
downloadStream.on('close', () => {
streamWithDetails.stream.on('close', () => {
unregisterInFlightRequest(cancelRequest);
});
}
@ -3536,7 +3590,7 @@ export function initialize({
abortController,
});
const combinedStream = downloadStream
const combinedStream = streamWithDetails.stream
// We do this manually; pipe() doesn't flow errors through the streams for us
.on('error', (error: Error) => {
timeoutStream.emit('error', error);

117
ts/textsecure/downloadAttachment.ts
View file

@ -1,9 +1,10 @@
// Copyright 2020 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { createWriteStream } from 'fs';
import { createWriteStream } from 'node:fs';
import { stat } from 'node:fs/promises';
import { isNumber } from 'lodash';
import type { Readable } from 'stream';
import type { Readable, Writable } from 'stream';
import { Transform } from 'stream';
import { pipeline } from 'stream/promises';
import { ensureFile } from 'fs-extra';
@ -24,7 +25,7 @@ import {
} from '../Crypto';
import {
getAttachmentCiphertextLength,
safeUnlinkSync,
safeUnlink,
splitKeys,
type ReencryptedAttachmentV2,
decryptAndReencryptLocally,
@ -124,6 +125,39 @@ export async function downloadAttachment(
let downloadResult: Awaited<ReturnType<typeof downloadToDisk>>;
let { downloadPath } = attachment;
let downloadOffset = 0;
if (downloadPath) {
const absoluteDownloadPath =
window.Signal.Migrations.getAbsoluteAttachmentPath(downloadPath);
try {
({ size: downloadOffset } = await stat(absoluteDownloadPath));
} catch (error) {
if (error.code !== 'ENOENT') {
log.error(
'downloadAttachment: Failed to get file size for previous download',
Errors.toLogFormat(error)
);
try {
await safeUnlink(downloadPath);
} catch {
downloadPath = undefined;
}
}
}
}
// Start over if we go over the size
if (downloadOffset >= size && downloadPath) {
log.warn('downloadAttachment: went over, retrying');
await safeUnlink(downloadPath);
downloadOffset = 0;
}
if (downloadOffset !== 0) {
log.info(`${logId}: resuming from ${downloadOffset}`);
}
if (mediaTier === MediaTier.STANDARD) {
strictAssert(
options.variant !== AttachmentVariant.ThumbnailFromBackup,
@ -135,9 +169,17 @@ export async function downloadAttachment(
const downloadStream = await server.getAttachment({
cdnKey,
cdnNumber,
options,
options: {
...options,
downloadOffset,
},
});
downloadResult = await downloadToDisk({
downloadStream,
size,
downloadPath,
downloadOffset,
});
downloadResult = await downloadToDisk({ downloadStream, size });
} else {
const mediaId =
options.variant === AttachmentVariant.ThumbnailFromBackup
@ -157,10 +199,15 @@ export async function downloadAttachment(
mediaDir,
headers: cdnCredentials.headers,
cdnNumber,
options,
options: {
...options,
downloadOffset,
},
});
downloadResult = await downloadToDisk({
downloadStream,
downloadPath,
downloadOffset,
size: getAttachmentCiphertextLength(
options.variant === AttachmentVariant.ThumbnailFromBackup
? // be generous, accept downloads of up to twice what we expect for thumbnail
@ -170,10 +217,7 @@ export async function downloadAttachment(
});
}
const { relativePath: downloadedRelativePath, downloadSize } = downloadResult;
const cipherTextAbsolutePath =
window.Signal.Migrations.getAbsoluteAttachmentPath(downloadedRelativePath);
const { absolutePath: cipherTextAbsolutePath, downloadSize } = downloadResult;
try {
switch (options.variant) {
@ -226,23 +270,42 @@ export async function downloadAttachment(
}
}
} finally {
safeUnlinkSync(cipherTextAbsolutePath);
await safeUnlink(cipherTextAbsolutePath);
}
}
async function downloadToDisk({
downloadStream,
downloadPath,
downloadOffset = 0,
size,
}: {
downloadStream: Readable;
downloadPath?: string;
downloadOffset?: number;
size: number;
}): Promise<{ relativePath: string; downloadSize: number }> {
const relativeTargetPath = getRelativePath(createName());
const absoluteTargetPath =
window.Signal.Migrations.getAbsoluteAttachmentPath(relativeTargetPath);
}): Promise<{ absolutePath: string; downloadSize: number }> {
const absoluteTargetPath = downloadPath
? window.Signal.Migrations.getAbsoluteDownloadsPath(downloadPath)
: window.Signal.Migrations.getAbsoluteAttachmentPath(
getRelativePath(createName())
);
await ensureFile(absoluteTargetPath);
const writeStream = createWriteStream(absoluteTargetPath);
const targetSize = getAttachmentCiphertextLength(size);
let writeStream: Writable;
if (downloadPath) {
writeStream = createWriteStream(absoluteTargetPath, {
flags: 'a',
start: downloadOffset,
});
} else {
strictAssert(
!downloadOffset,
'Download cannot be resumed without downloadPath'
);
writeStream = createWriteStream(absoluteTargetPath);
}
const targetSize = getAttachmentCiphertextLength(size) - downloadOffset;
let downloadSize = 0;
try {
@ -255,19 +318,23 @@ async function downloadToDisk({
writeStream
);
} catch (error) {
try {
safeUnlinkSync(absoluteTargetPath);
} catch (cleanupError) {
log.error(
'downloadToDisk: Error while cleaning up',
Errors.toLogFormat(cleanupError)
);
if (downloadPath) {
log.warn(`downloadToDisk: stopping at ${downloadSize}`);
} else {
try {
await safeUnlink(absoluteTargetPath);
} catch (cleanupError) {
log.error(
'downloadToDisk: Error while cleaning up',
Errors.toLogFormat(cleanupError)
);
}
}
throw error;
}
return { relativePath: relativeTargetPath, downloadSize };
return { absolutePath: absoluteTargetPath, downloadSize };
}
// A simple transform that throws if it sees more than maxBytes on the stream.

11
ts/textsecure/processDataMessage.ts
View file

@ -32,6 +32,7 @@ import { filterAndClean } from '../types/BodyRange';
import { isAciString } from '../util/isAciString';
import { normalizeAci } from '../util/normalizeAci';
import { bytesToUuid } from '../util/uuidToBytes';
import { createName } from '../util/attachmentPath';
const FLAGS = Proto.DataMessage.Flags;
export const ATTACHMENT_MAX = 32;
@ -284,7 +285,10 @@ export function processGiftBadge(
export function processDataMessage(
message: Proto.IDataMessage,
envelopeTimestamp: number
envelopeTimestamp: number,
// Only for testing
{ _createName: doCreateName = createName } = {}
): ProcessedDataMessage {
/* eslint-disable no-bitwise */
@ -309,7 +313,10 @@ export function processDataMessage(
const result: ProcessedDataMessage = {
body: dropNull(message.body),
attachments: (message.attachments ?? []).map(
(attachment: Proto.IAttachmentPointer) => processAttachment(attachment)
(attachment: Proto.IAttachmentPointer) => ({
...processAttachment(attachment),
downloadPath: doCreateName(),
})
),
groupV2: processGroupV2Context(message.groupV2),
flags: message.flags ?? 0,

18
ts/types/Attachment.ts
View file

@ -78,6 +78,7 @@ export type AttachmentType = {
cdnNumber?: number;
cdnId?: string;
cdnKey?: string;
downloadPath?: string;
key?: string;
iv?: string;
data?: Uint8Array;
@ -386,9 +387,13 @@ export function loadData(
};
}
export function deleteData(
deleteOnDisk: (path: string) => Promise<void>
): (attachment?: AttachmentType) => Promise<void> {
export function deleteData({
deleteOnDisk,
deleteDownloadOnDisk,
}: {
deleteOnDisk: (path: string) => Promise<void>;
deleteDownloadOnDisk: (path: string) => Promise<void>;
}): (attachment?: AttachmentType) => Promise<void> {
if (!isFunction(deleteOnDisk)) {
throw new TypeError('deleteData: deleteOnDisk must be a function');
}
@ -398,12 +403,17 @@ export function deleteData(
throw new TypeError('deleteData: attachment is not valid');
}
const { path, thumbnail, screenshot, thumbnailFromBackup } = attachment;
const { path, downloadPath, thumbnail, screenshot, thumbnailFromBackup } =
attachment;
if (isString(path)) {
await deleteOnDisk(path);
}
if (isString(downloadPath)) {
await deleteDownloadOnDisk(downloadPath);
}
if (thumbnail && isString(thumbnail.path)) {
await deleteOnDisk(thumbnail.path);
}

5
ts/util/createIPCEvents.ts
View file

@ -104,6 +104,7 @@ export type IPCEventsCallbacksType = {
}>;
addCustomColor: (customColor: CustomColorType) => void;
addDarkOverlay: () => void;
cleanupDownloads: () => Promise<void>;
deleteAllData: () => Promise<void>;
deleteAllMyStories: () => Promise<void>;
editCustomColor: (colorId: string, customColor: CustomColorType) => void;
@ -533,6 +534,10 @@ export function createIPCEvents(
showKeyboardShortcuts: () =>
window.reduxActions.globalModals.showShortcutGuideModal(),
cleanupDownloads: async () => {
await ipcRenderer.invoke('cleanup-downloads');
},
deleteAllData: async () => {
renderClearingDataView();
},

7
ts/util/deleteForMe.ts
View file

@ -123,9 +123,11 @@ export async function deleteAttachmentFromMessage(
},
{
deleteOnDisk,
deleteDownloadOnDisk,
logId,
}: {
deleteOnDisk: (path: string) => Promise<void>;
deleteDownloadOnDisk: (path: string) => Promise<void>;
logId: string;
}
): Promise<boolean> {
@ -147,6 +149,7 @@ export async function deleteAttachmentFromMessage(
return applyDeleteAttachmentFromMessage(message, deleteAttachmentData, {
deleteOnDisk,
deleteDownloadOnDisk,
logId,
shouldSave: true,
});
@ -165,10 +168,12 @@ export async function applyDeleteAttachmentFromMessage(
},
{
deleteOnDisk,
deleteDownloadOnDisk,
shouldSave,
logId,
}: {
deleteOnDisk: (path: string) => Promise<void>;
deleteDownloadOnDisk: (path: string) => Promise<void>;
shouldSave: boolean;
logId: string;
}
@ -206,7 +211,7 @@ export async function applyDeleteAttachmentFromMessage(
if (shouldSave) {
await saveMessage(message.attributes, { ourAci });
}
await deleteData(deleteOnDisk)(attachment);
await deleteData({ deleteOnDisk, deleteDownloadOnDisk })(attachment);
return true;
}

1
ts/util/modifyTargetMessage.ts
View file

@ -92,6 +92,7 @@ export async function modifyTargetMessage(
logId,
shouldSave: false,
deleteOnDisk: window.Signal.Migrations.deleteAttachmentData,
deleteDownloadOnDisk: window.Signal.Migrations.deleteDownloadData,
}
);
if (result) {

4
ts/util/uploadAttachment.ts
View file

@ -15,7 +15,7 @@ import type { AttachmentUploadFormResponseType } from '../textsecure/WebAPI';
import {
type EncryptedAttachmentV2,
encryptAttachmentV2ToDisk,
safeUnlinkSync,
safeUnlink,
type PlaintextSourceType,
type HardcodedIVForEncryptionType,
} from '../AttachmentCrypto';
@ -117,7 +117,7 @@ export async function encryptAndUploadAttachment({
return { cdnKey: uploadForm.key, cdnNumber: uploadForm.cdn, encrypted };
} finally {
if (absoluteCiphertextPath) {
safeUnlinkSync(absoluteCiphertextPath);
await safeUnlink(absoluteCiphertextPath);
}
}
}