mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Add a backup media download progress bar

Browse source
This commit is contained in:
trevor-signal 2024-09-03 18:00:51 -04:00 • committed by GitHub
parent 84f1d98020
commit 501f27127f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 640 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

10
ts/components/BackupImportScreen.tsx
View file

@ -7,6 +7,7 @@ import type { LocalizerType } from '../types/Util';
import { formatFileSize } from '../util/formatFileSize';
import { TitlebarDragArea } from './TitlebarDragArea';
import { InstallScreenSignalLogo } from './installScreen/InstallScreenSignalLogo';
import { ProgressBar } from './ProgressBar';
// We can't always use destructuring assignment because of the complexity of this props
// type.
@ -36,12 +37,7 @@ export function BackupImportScreen({
progress = (
<>
<div className="BackupImportScreen__progressbar">
<div
className="BackupImportScreen__progressbar__fill"
style={{ transform: `translateX(${(percentage - 1) * 100}%)` }}
/>
</div>
<ProgressBar fractionComplete={percentage} />
<div className="BackupImportScreen__progressbar-hint">
{i18n('icu:BackupImportScreen__progressbar-hint', {
currentSize: formatFileSize(currentBytes),
@ -54,7 +50,7 @@ export function BackupImportScreen({
} else {
progress = (
<>
<div className="BackupImportScreen__progressbar" />
<ProgressBar fractionComplete={0} />
<div className="BackupImportScreen__progressbar-hint BackupImportScreen__progressbar-hint--hidden">
{i18n('icu:BackupImportScreen__progressbar-hint', {
currentSize: '',

27
ts/components/BackupMediaDownloadProgress.stories.tsx Normal file
View file

@ -0,0 +1,27 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import React, { type ComponentProps } from 'react';
import type { Meta, StoryFn } from '@storybook/react';
import { setupI18n } from '../util/setupI18n';
import enMessages from '../../_locales/en/messages.json';
import { BackupMediaDownloadProgressBanner } from './BackupMediaDownloadProgress';
const i18n = setupI18n('en', enMessages);
type PropsType = ComponentProps<typeof BackupMediaDownloadProgressBanner>;
export default {
title: 'Components/BackupMediaDownloadProgress',
} satisfies Meta<PropsType>;
// eslint-disable-next-line react/function-component-definition
const Template: StoryFn<PropsType> = (args: PropsType) => (
<BackupMediaDownloadProgressBanner {...args} i18n={i18n} />
);
export const InProgress = Template.bind({});
InProgress.args = {
downloadedBytes: 92048023,
totalBytes: 1024102532,
};

48
ts/components/BackupMediaDownloadProgress.tsx Normal file
View file

@ -0,0 +1,48 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import React from 'react';
import type { LocalizerType } from '../types/Util';
import { formatFileSize } from '../util/formatFileSize';
import { ProgressBar } from './ProgressBar';
export type PropsType = Readonly<{
i18n: LocalizerType;
downloadedBytes: number;
totalBytes: number;
}>;
export function BackupMediaDownloadProgressBanner({
i18n,
downloadedBytes,
totalBytes,
}: PropsType): JSX.Element | null {
if (totalBytes === 0) {
return null;
}
const fractionComplete = Math.max(
0,
Math.min(1, downloadedBytes / totalBytes)
);
return (
<div className="BackupMediaDownloadProgressBanner">
<div className="BackupMediaDownloadProgressBanner__icon" />
<div className="BackupMediaDownloadProgressBanner__content">
<div className="BackupMediaDownloadProgressBanner__title">
{i18n('icu:BackupMediaDownloadProgress__title')}
</div>
<ProgressBar fractionComplete={fractionComplete} />
<div className="BackupMediaDownloadProgressBanner__progressbar-hint">
{i18n('icu:BackupMediaDownloadProgress__progressbar-hint', {
currentSize: formatFileSize(downloadedBytes),
totalSize: formatFileSize(totalBytes),
fractionComplete,
})}
</div>
</div>
</div>
);
}

1
ts/components/LeftPane.stories.tsx
View file

@ -137,6 +137,7 @@ const useProps = (overrideProps: OverridePropsType = {}): PropsType => {
unreadMentionsCount: 0,
markedUnread: false,
},
backupMediaDownloadProgress: { totalBytes: 0, downloadedBytes: 0 },
clearConversationSearch: action('clearConversationSearch'),
clearGroupCreationError: action('clearGroupCreationError'),
clearSearch: action('clearSearch'),

39
ts/components/LeftPane.tsx
View file

@ -1,7 +1,13 @@
// Copyright 2019 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import React, { useEffect, useCallback, useMemo, useRef } from 'react';
import React, {
useEffect,
useCallback,
useMemo,
useRef,
useState,
} from 'react';
import classNames from 'classnames';
import { isNumber } from 'lodash';
@ -56,8 +62,10 @@ import {
import { ContextMenu } from './ContextMenu';
import { EditState as ProfileEditorEditState } from './ProfileEditor';
import type { UnreadStats } from '../util/countUnreadStats';
import { BackupMediaDownloadProgressBanner } from './BackupMediaDownloadProgress';
export type PropsType = {
backupMediaDownloadProgress: { totalBytes: number; downloadedBytes: number };
otherTabsUnreadStats: UnreadStats;
hasExpiredDialog: boolean;
hasFailedStorySends: boolean;
@ -173,6 +181,7 @@ export type PropsType = {
} & LookupConversationWithoutServiceIdActionsType;
export function LeftPane({
backupMediaDownloadProgress,
otherTabsUnreadStats,
blockConversation,
challengeStatus,
@ -634,6 +643,34 @@ export function LeftPane({
dialogs.push({ key: 'banner', dialog: maybeBanner });
}
// We'll show the backup media download progress banner if the download is currently or
// was ongoing at some point during the lifecycle of this component
const [
hasMediaBackupDownloadBeenOngoing,
setHasMediaBackupDownloadBeenOngoing,
] = useState(false);
const isMediaBackupDownloadOngoing =
backupMediaDownloadProgress?.totalBytes > 0 &&
backupMediaDownloadProgress.downloadedBytes <
backupMediaDownloadProgress.totalBytes;
if (isMediaBackupDownloadOngoing && !hasMediaBackupDownloadBeenOngoing) {
setHasMediaBackupDownloadBeenOngoing(true);
}
if (hasMediaBackupDownloadBeenOngoing) {
dialogs.push({
key: 'backupMediaDownload',
dialog: (
<BackupMediaDownloadProgressBanner
i18n={i18n}
{...backupMediaDownloadProgress}
/>
),
});
}
const hideHeader =
modeSpecificProps.mode === LeftPaneMode.Archive ||
modeSpecificProps.mode === LeftPaneMode.Compose ||

21
ts/components/ProgressBar.tsx Normal file
View file

@ -0,0 +1,21 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import React from 'react';
export function ProgressBar({
fractionComplete,
}: {
fractionComplete: number;
}): JSX.Element {
return (
<div className="ProgressBar">
<div
className="ProgressBar__fill"
style={{
marginInlineEnd: `${(1 - fractionComplete) * 100}%`,
}}
/>
</div>
);
}

55
ts/jobs/AttachmentDownloadManager.ts
View file

@ -42,6 +42,9 @@ import {
isVideoTypeSupported,
} from '../util/GoogleChrome';
import type { MIMEType } from '../types/MIME';
import type { AttachmentDownloadSource } from '../sql/Interface';
import { drop } from '../util/drop';
import { getAttachmentCiphertextLength } from '../AttachmentCrypto';
export enum AttachmentDownloadUrgency {
IMMEDIATE = 'immediate',
@ -55,6 +58,7 @@ export type NewAttachmentDownloadJobType = {
receivedAt: number;
sentAt: number;
attachmentType: AttachmentDownloadJobTypeType;
source: AttachmentDownloadSource;
urgency?: AttachmentDownloadUrgency;
};
@ -69,6 +73,10 @@ const DEFAULT_RETRY_CONFIG = {
maxBackoffTime: 6 * durations.HOUR,
},
};
const BACKUP_RETRY_CONFIG = {
...DEFAULT_RETRY_CONFIG,
maxAttempts: Infinity,
};
type AttachmentDownloadManagerParamsType = Omit<
JobManagerParamsType<CoreAttachmentDownloadJobType>,
'getNextJobs' | 'runJob'
@ -117,7 +125,10 @@ export class AttachmentDownloadManager extends JobManager<CoreAttachmentDownload
},
getJobId,
getJobIdForLogging,
getRetryConfig: () => DEFAULT_RETRY_CONFIG,
getRetryConfig: job =>
job.attachment.backupLocator?.mediaName
? BACKUP_RETRY_CONFIG
: DEFAULT_RETRY_CONFIG,
maxConcurrentJobs: MAX_CONCURRENT_JOBS,
};
@ -157,6 +168,7 @@ export class AttachmentDownloadManager extends JobManager<CoreAttachmentDownload
attachmentType,
receivedAt,
sentAt,
source,
urgency = AttachmentDownloadUrgency.STANDARD,
} = newJobData;
const parseResult = coreAttachmentDownloadJobSchema.safeParse({
@ -167,7 +179,9 @@ export class AttachmentDownloadManager extends JobManager<CoreAttachmentDownload
digest: attachment.digest,
contentType: attachment.contentType,
size: attachment.size,
ciphertextSize: getAttachmentCiphertextLength(attachment.size),
attachment,
source,
});
if (!parseResult.success) {
@ -251,13 +265,24 @@ async function runDownloadAttachmentJob({
dependencies,
});
if (result.onlyAttemptedBackupThumbnail) {
if (result.downloadedVariant === AttachmentVariant.ThumbnailFromBackup) {
return {
status: 'finished',
newJob: { ...job, attachment: result.attachmentWithThumbnail },
};
}
if (job.attachment.backupLocator?.mediaName) {
const currentDownloadedSize =
window.storage.get('backupAttachmentsSuccessfullyDownloadedSize') ?? 0;
drop(
window.storage.put(
'backupAttachmentsSuccessfullyDownloadedSize',
currentDownloadedSize + job.ciphertextSize
)
);
}
return {
status: 'finished',
};
@ -319,11 +344,9 @@ async function runDownloadAttachmentJob({
}
type DownloadAttachmentResultType =
| { downloadedVariant: AttachmentVariant.Default }
| {
onlyAttemptedBackupThumbnail: false;
}
| {
onlyAttemptedBackupThumbnail: true;
downloadedVariant: AttachmentVariant.ThumbnailFromBackup;
attachmentWithThumbnail: AttachmentType;
};
@ -386,7 +409,7 @@ export async function runDownloadAttachmentJobInner({
type: attachmentType,
});
return {
onlyAttemptedBackupThumbnail: true,
downloadedVariant: AttachmentVariant.ThumbnailFromBackup,
attachmentWithThumbnail,
};
} catch (e) {
@ -418,7 +441,7 @@ export async function runDownloadAttachmentJobInner({
await addAttachmentToMessage(messageId, upgradedAttachment, logId, {
type: attachmentType,
});
return { onlyAttemptedBackupThumbnail: false };
return { downloadedVariant: AttachmentVariant.Default };
} catch (error) {
if (
!job.attachment.thumbnailFromBackup &&
@ -430,20 +453,24 @@ export async function runDownloadAttachmentJobInner({
Errors.toLogFormat(error)
);
try {
const attachmentWithThumbnail = await downloadBackupThumbnail({
attachment,
dependencies,
});
const attachmentWithThumbnail = omit(
await downloadBackupThumbnail({
attachment,
dependencies,
}),
'pending'
);
await addAttachmentToMessage(
messageId,
omit(attachmentWithThumbnail, 'pending'),
attachmentWithThumbnail,
logId,
{
type: attachmentType,
}
);
return {
onlyAttemptedBackupThumbnail: false,
downloadedVariant: AttachmentVariant.ThumbnailFromBackup,
attachmentWithThumbnail,
};
} catch (thumbnailError) {
log.error(

2
ts/messageModifiers/ViewSyncs.ts
View file

@ -127,7 +127,7 @@ export async function onSync(sync: ViewSyncAttributesType): Promise<void> {
if (!attachments?.every(isDownloaded)) {
const updatedFields = await queueAttachmentDownloads(
message.attributes,
AttachmentDownloadUrgency.STANDARD
{ urgency: AttachmentDownloadUrgency.STANDARD }
);
if (updatedFields) {
message.set(updatedFields);

2
ts/models/messages.ts
View file

@ -1344,7 +1344,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
async queueAttachmentDownloads(
urgency?: AttachmentDownloadUrgency
): Promise<boolean> {
const value = await queueAttachmentDownloads(this.attributes, urgency);
const value = await queueAttachmentDownloads(this.attributes, { urgency });
if (!value) {
return false;
}

11
ts/services/backups/import.ts
View file

@ -11,7 +11,10 @@ import { CallLinkRootKey } from '@signalapp/ringrtc';
import { Backups, SignalService } from '../../protobuf';
import { DataWriter } from '../../sql/Client';
import type { StoryDistributionWithMembersType } from '../../sql/Interface';
import {
AttachmentDownloadSource,
type StoryDistributionWithMembersType,
} from '../../sql/Interface';
import * as log from '../../logging/log';
import { GiftBadgeStates } from '../../components/conversation/Message';
import { StorySendMode, MY_STORY_ID } from '../../types/Stories';
@ -169,7 +172,11 @@ async function processMessagesBatch(
);
}
drop(queueAttachmentDownloads(attributes));
drop(
queueAttachmentDownloads(attributes, {
source: AttachmentDownloadSource.BACKUP_IMPORT,
})
);
}
}

17
ts/services/backups/index.ts
View file

@ -40,6 +40,7 @@ import { BackupCredentials } from './credentials';
import { BackupAPI, type DownloadOptionsType } from './api';
import { validateBackup } from './validator';
import { getParametersForRedux, loadAll } from '../allLoaders';
import { AttachmentDownloadManager } from '../../jobs/AttachmentDownloadManager';
const IV_LENGTH = 16;
@ -236,6 +237,8 @@ export class BackupsService {
// Second pass - decrypt (but still check the mac at the end)
hmac = createHmac(HashType.size256, macKey);
await this.prepareForImport();
await pipeline(
createBackupStream(),
getMacAndUpdateHmac(hmac, noop),
@ -278,11 +281,25 @@ export class BackupsService {
}
}
public async prepareForImport(): Promise<void> {
await AttachmentDownloadManager.stop();
await DataWriter.removeAllBackupAttachmentDownloadJobs();
await window.storage.put('backupAttachmentsSuccessfullyDownloadedSize', 0);
await window.storage.put('backupAttachmentsTotalSizeToDownload', 0);
}
public async resetStateAfterImport(): Promise<void> {
window.ConversationController.reset();
await window.ConversationController.load();
await loadAll();
reinitializeRedux(getParametersForRedux());
await window.storage.put(
'backupAttachmentsTotalSizeToDownload',
await DataReader.getSizeOfPendingBackupAttachmentDownloadJobs()
);
await AttachmentDownloadManager.start();
}
public async fetchAndSaveBackupCdnObjectMetadata(): Promise<void> {

7
ts/sql/Interface.ts
View file

@ -446,6 +446,11 @@ export type GetRecentStoryRepliesOptionsType = {
sentAt?: number;
};
export enum AttachmentDownloadSource {
BACKUP_IMPORT = 'backup_import',
STANDARD = 'standard',
}
type ReadableInterface = {
close: () => void;
@ -642,6 +647,7 @@ type ReadableInterface = {
getMaxMessageCounter(): number | undefined;
getStatisticsForLogging(): Record<string, string>;
getSizeOfPendingBackupAttachmentDownloadJobs(): number;
};
type WritableInterface = {
@ -840,6 +846,7 @@ type WritableInterface = {
saveAttachmentDownloadJob: (job: AttachmentDownloadJobType) => void;
resetAttachmentDownloadActive: () => void;
removeAttachmentDownloadJob: (job: AttachmentDownloadJobType) => void;
removeAllBackupAttachmentDownloadJobs: () => void;
getNextAttachmentBackupJobs: (options: {
limit: number;

29
ts/sql/Server.ts
View file

@ -145,6 +145,7 @@ import type {
StoredKyberPreKeyType,
BackupCdnMediaObjectType,
} from './Interface';
import { AttachmentDownloadSource } from './Interface';
import { SeenStatus } from '../MessageSeenStatus';
import {
SNIPPET_LEFT_PLACEHOLDER,
@ -200,6 +201,7 @@ import {
attachmentBackupJobSchema,
} from '../types/AttachmentBackup';
import { redactGenericText } from '../util/privacy';
import { getAttachmentCiphertextLength } from '../AttachmentCrypto';
type ConversationRow = Readonly<{
json: string;
@ -340,6 +342,7 @@ export const DataReader: ServerReadableInterface = {
getStatisticsForLogging,
getBackupCdnObjectMetadata,
getSizeOfPendingBackupAttachmentDownloadJobs,
// Server-only
getKnownMessageAttachments,
@ -463,6 +466,7 @@ export const DataWriter: ServerWritableInterface = {
saveAttachmentDownloadJob,
resetAttachmentDownloadActive,
removeAttachmentDownloadJob,
removeAllBackupAttachmentDownloadJobs,
getNextAttachmentBackupJobs,
saveAttachmentBackupJob,
@ -4733,6 +4737,20 @@ function getAttachmentDownloadJob(
return db.prepare(query).get(params);
}
function removeAllBackupAttachmentDownloadJobs(db: WritableDB): void {
const [query, params] = sql`
DELETE FROM attachment_downloads
WHERE source = ${AttachmentDownloadSource.BACKUP_IMPORT};`;
db.prepare(query).run(params);
}
function getSizeOfPendingBackupAttachmentDownloadJobs(db: ReadableDB): number {
const [query, params] = sql`
SELECT SUM(ciphertextSize) FROM attachment_downloads
WHERE source = ${AttachmentDownloadSource.BACKUP_IMPORT};`;
return db.prepare(query).pluck().get(params);
}
function getNextAttachmentDownloadJobs(
db: WritableDB,
{
@ -4799,6 +4817,9 @@ function getNextAttachmentDownloadJobs(
...row,
active: Boolean(row.active),
attachment: jsonToObject(row.attachmentJson),
ciphertextSize:
row.ciphertextSize ||
getAttachmentCiphertextLength(row.attachment.size),
});
} catch (error) {
logger.error(
@ -4839,7 +4860,9 @@ function saveAttachmentDownloadJob(
attempts,
retryAfter,
lastAttemptTimestamp,
attachmentJson
attachmentJson,
ciphertextSize,
source
) VALUES (
${job.messageId},
${job.attachmentType},
@ -4852,7 +4875,9 @@ function saveAttachmentDownloadJob(
${job.attempts},
${job.retryAfter},
${job.lastAttemptTimestamp},
${objectToJSON(job.attachment)}
${objectToJSON(job.attachment)},
${job.ciphertextSize},
${job.source}
);
`;
db.prepare(query).run(params);

4
ts/sql/migrations/1040-undownloaded-backed-up-media.ts
View file

@ -11,6 +11,7 @@ import {
} from '../../types/AttachmentDownload';
import type { AttachmentType } from '../../types/Attachment';
import { jsonToObject, objectToJSON, sql } from '../util';
import { AttachmentDownloadSource } from '../Interface';
export const version = 1040;
@ -133,6 +134,9 @@ export function updateToSchemaVersion1040(
attempts: existingJobData.attempts ?? 0,
retryAfter: null,
lastAttemptTimestamp: null,
// adding due to changes in the schema
source: AttachmentDownloadSource.STANDARD,
ciphertextSize: 0,
};
const parsed = attachmentDownloadJobSchema.parse(updatedJob);

37
ts/sql/migrations/1180-add-attachment-download-source.ts Normal file
View file

@ -0,0 +1,37 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import type { Database } from '@signalapp/better-sqlite3';
import type { LoggerType } from '../../types/Logging';
import { AttachmentDownloadSource } from '../Interface';
export const version = 1180;
export function updateToSchemaVersion1180(
currentVersion: number,
db: Database,
logger: LoggerType
): void {
if (currentVersion >= 1180) {
return;
}
db.transaction(() => {
db.exec(`
ALTER TABLE attachment_downloads
ADD COLUMN source TEXT NOT NULL DEFAULT ${AttachmentDownloadSource.STANDARD};
ALTER TABLE attachment_downloads
-- this default value will be overridden by getNextAttachmentDownloadJobs
ADD COLUMN ciphertextSize INTEGER NOT NULL DEFAULT 0;
`);
db.exec(`
CREATE INDEX attachment_downloads_source_ciphertextSize
ON attachment_downloads (
source, ciphertextSize
);
`);
db.pragma('user_version = 1180');
})();
logger.info('updateToSchemaVersion1180: success!');
}

6
ts/sql/migrations/index.ts
View file

@ -93,10 +93,11 @@ import { updateToSchemaVersion1130 } from './1130-isStory-index';
import { updateToSchemaVersion1140 } from './1140-call-links-deleted-column';
import { updateToSchemaVersion1150 } from './1150-expire-timer-version';
import { updateToSchemaVersion1160 } from './1160-optimize-calls-unread-count';
import { updateToSchemaVersion1170 } from './1170-update-call-history-unread-index';
import {
updateToSchemaVersion1170,
updateToSchemaVersion1180,
version as MAX_VERSION,
} from './1170-update-call-history-unread-index';
} from './1180-add-attachment-download-source';
function updateToSchemaVersion1(
currentVersion: number,
@ -2058,6 +2059,7 @@ export const SCHEMA_VERSIONS = [
updateToSchemaVersion1150,
updateToSchemaVersion1160,
updateToSchemaVersion1170,
updateToSchemaVersion1180,
];
export class DBVersionFromFutureError extends Error {

8
ts/state/selectors/items.ts
View file

@ -248,3 +248,11 @@ export const getLocalDeleteWarningShown = createSelector(
(state: ItemsStateType): boolean =>
Boolean(state.localDeleteWarningShown ?? false)
);
export const getBackupMediaDownloadProgress = createSelector(
getItems,
(state: ItemsStateType): { totalBytes: number; downloadedBytes: number } => ({
totalBytes: state.backupAttachmentsTotalSizeToDownload ?? 0,
downloadedBytes: state.backupAttachmentsSuccessfullyDownloadedSize ?? 0,
})
);

7
ts/state/smart/LeftPane.tsx
View file

@ -58,6 +58,7 @@ import {
import { getCrashReportCount } from '../selectors/crashReports';
import { hasExpired } from '../selectors/expiration';
import {
getBackupMediaDownloadProgress,
getNavTabsCollapsed,
getPreferredLeftPaneWidth,
getUsernameCorrupted,
@ -115,6 +116,7 @@ function renderUpdateDialog(
): JSX.Element {
return <SmartUpdateDialog {...props} />;
}
function renderCaptchaDialog({ onSkip }: { onSkip(): void }): JSX.Element {
return <SmartCaptchaDialog onSkip={onSkip} />;
}
@ -289,7 +291,9 @@ export const SmartLeftPane = memo(function SmartLeftPane({
const theme = useSelector(getTheme);
const usernameCorrupted = useSelector(getUsernameCorrupted);
const usernameLinkCorrupted = useSelector(getUsernameLinkCorrupted);
const backupMediaDownloadProgress = useSelector(
getBackupMediaDownloadProgress
);
const {
blockConversation,
clearGroupCreationError,
@ -360,6 +364,7 @@ export const SmartLeftPane = memo(function SmartLeftPane({
return (
<LeftPane
backupMediaDownloadProgress={backupMediaDownloadProgress}
blockConversation={blockConversation}
challengeStatus={challengeStatus}
clearConversationSearch={clearConversationSearch}

21
ts/test-electron/services/AttachmentDownloadManager_test.ts
View file

@ -20,6 +20,8 @@ import { MINUTE } from '../../util/durations';
import { type AciString } from '../../types/ServiceId';
import { type AttachmentType, AttachmentVariant } from '../../types/Attachment';
import { strictAssert } from '../../util/assert';
import { AttachmentDownloadSource } from '../../sql/Interface';
import { getAttachmentCiphertextLength } from '../../AttachmentCrypto';
function composeJob({
messageId,
@ -38,11 +40,13 @@ function composeJob({
attachmentType: 'attachment',
digest,
size,
ciphertextSize: getAttachmentCiphertextLength(size),
contentType,
active: false,
attempts: 0,
retryAfter: null,
lastAttemptTimestamp: null,
source: AttachmentDownloadSource.STANDARD,
attachment: {
contentType,
size,
@ -114,8 +118,8 @@ describe('AttachmentDownloadManager/JobManager', () => {
}
);
await downloadManager?.addJob({
...job,
urgency,
...job,
});
}
async function addJobs(
@ -419,7 +423,7 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => {
dependencies: { downloadAttachment },
});
assert.strictEqual(result.onlyAttemptedBackupThumbnail, false);
assert.strictEqual(result.downloadedVariant, AttachmentVariant.Default);
assert.strictEqual(downloadAttachment.callCount, 1);
assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], {
attachment: job.attachment,
@ -444,8 +448,8 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => {
});
strictAssert(
result.onlyAttemptedBackupThumbnail === true,
'only attempted backup thumbnail'
result.downloadedVariant === AttachmentVariant.ThumbnailFromBackup,
'downloaded thumbnail'
);
assert.deepStrictEqual(
omit(result.attachmentWithThumbnail, 'thumbnailFromBackup'),
@ -485,7 +489,7 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => {
isForCurrentlyVisibleMessage: true,
dependencies: { downloadAttachment },
});
assert.strictEqual(result.onlyAttemptedBackupThumbnail, false);
assert.strictEqual(result.downloadedVariant, AttachmentVariant.Default);
assert.strictEqual(downloadAttachment.callCount, 1);
assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], {
attachment: job.attachment,
@ -544,7 +548,7 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => {
isForCurrentlyVisibleMessage: false,
dependencies: { downloadAttachment },
});
assert.strictEqual(result.onlyAttemptedBackupThumbnail, false);
assert.strictEqual(result.downloadedVariant, AttachmentVariant.Default);
assert.strictEqual(downloadAttachment.callCount, 1);
assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], {
attachment: job.attachment,
@ -578,7 +582,10 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => {
isForCurrentlyVisibleMessage: false,
dependencies: { downloadAttachment },
});
assert.strictEqual(result.onlyAttemptedBackupThumbnail, false);
assert.strictEqual(
result.downloadedVariant,
AttachmentVariant.ThumbnailFromBackup
);
assert.strictEqual(downloadAttachment.callCount, 2);
assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], {
attachment: job.attachment,

33
ts/test-node/sql/migration_1040_test.ts
View file

@ -27,7 +27,7 @@ function getAttachmentDownloadJobs(db: ReadableDB) {
type UnflattenedAttachmentDownloadJobType = Omit<
AttachmentDownloadJobType,
'digest' | 'contentType' | 'size'
'digest' | 'contentType' | 'size' | 'source' | 'ciphertextSize'
>;
function insertNewJob(
db: WritableDB,
@ -304,24 +304,25 @@ describe('SQL/updateToSchemaVersion1040', () => {
});
it('respects foreign key constraint on messageId', () => {
const job: AttachmentDownloadJobType = {
messageId: 'message1',
attachmentType: 'attachment',
attachment: {
const job: Omit<AttachmentDownloadJobType, 'source' | 'ciphertextSize'> =
{
messageId: 'message1',
attachmentType: 'attachment',
attachment: {
digest: 'digest1',
contentType: IMAGE_JPEG,
size: 128,
},
receivedAt: 1970,
digest: 'digest1',
contentType: IMAGE_JPEG,
size: 128,
},
receivedAt: 1970,
digest: 'digest1',
contentType: IMAGE_JPEG,
size: 128,
sentAt: 2070,
active: false,
retryAfter: null,
attempts: 0,
lastAttemptTimestamp: null,
};
sentAt: 2070,
active: false,
retryAfter: null,
attempts: 0,
lastAttemptTimestamp: null,
};
// throws if we don't add the message first
assert.throws(() => insertNewJob(db, job, false));
insertNewJob(db, job, true);

147
ts/test-node/sql/migration_1180_test.ts Normal file
View file

@ -0,0 +1,147 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { assert } from 'chai';
import { omit } from 'lodash';
import type { WritableDB } from '../../sql/Interface';
import { createDB, updateToVersion } from './helpers';
import type { AttachmentDownloadJobType } from '../../types/AttachmentDownload';
import { jsonToObject, objectToJSON, sql } from '../../sql/util';
import { IMAGE_BMP } from '../../types/MIME';
function insertOldJob(
db: WritableDB,
job: Omit<AttachmentDownloadJobType, 'source' | 'ciphertextSize'>,
addMessageFirst: boolean = true
): void {
if (addMessageFirst) {
try {
db.prepare('INSERT INTO messages (id) VALUES ($id)').run({
id: job.messageId,
});
} catch (e) {
// pass; message has already been inserted
}
}
const [query, params] = sql`
INSERT INTO attachment_downloads
(
messageId,
attachmentType,
attachmentJson,
digest,
contentType,
size,
receivedAt,
sentAt,
active,
attempts,
retryAfter,
lastAttemptTimestamp
)
VALUES
(
${job.messageId},
${job.attachmentType},
${objectToJSON(job.attachment)},
${job.digest},
${job.contentType},
${job.size},
${job.receivedAt},
${job.sentAt},
${job.active ? 1 : 0},
${job.attempts},
${job.retryAfter},
${job.lastAttemptTimestamp}
);
`;
db.prepare(query).run(params);
}
function getAttachmentDownloadJobs(db: WritableDB) {
const [query] = sql`
SELECT * FROM attachment_downloads ORDER BY receivedAt DESC;
`;
return db
.prepare(query)
.all()
.map(job => ({
...omit(job, 'attachmentJson'),
active: job.active === 1,
attachment: jsonToObject(job.attachmentJson),
}));
}
describe('SQL/updateToSchemaVersion1180', () => {
let db: WritableDB;
beforeEach(() => {
db = createDB();
updateToVersion(db, 1170);
});
afterEach(() => {
db.close();
});
it('adds source column with default standard to any existing jobs', async () => {
const job: Omit<AttachmentDownloadJobType, 'source' | 'ciphertextSize'> = {
messageId: '123',
digest: 'digest',
attachmentType: 'attachment',
attachment: { size: 128, contentType: IMAGE_BMP },
size: 128,
contentType: IMAGE_BMP,
receivedAt: 120,
sentAt: 120,
active: false,
attempts: 0,
retryAfter: null,
lastAttemptTimestamp: null,
};
insertOldJob(db, job);
updateToVersion(db, 1180);
assert.deepEqual(getAttachmentDownloadJobs(db), [
{ ...job, source: 'standard', ciphertextSize: 0 },
]);
});
it('uses convering index for summing all pending backup jobs', async () => {
updateToVersion(db, 1180);
const details = db
.prepare(
`
EXPLAIN QUERY PLAN
SELECT SUM(ciphertextSize) FROM attachment_downloads
WHERE source = 'backup_import';
`
)
.all()
.map(step => step.detail)
.join(', ');
assert.strictEqual(
details,
'SEARCH attachment_downloads USING COVERING INDEX attachment_downloads_source_ciphertextSize (source=?)'
);
});
it('uses index for deleting all backup jobs', async () => {
updateToVersion(db, 1180);
const details = db
.prepare(
`
EXPLAIN QUERY PLAN
DELETE FROM attachment_downloads
WHERE source = 'backup_import';
`
)
.all()
.map(step => step.detail)
.join(', ');
assert.strictEqual(
details,
'SEARCH attachment_downloads USING COVERING INDEX attachment_downloads_source_ciphertextSize (source=?)'
);
});
});

5
ts/types/AttachmentDownload.ts
View file

@ -7,6 +7,7 @@ import {
type JobManagerJobType,
jobManagerJobSchema,
} from '../jobs/JobManager';
import { AttachmentDownloadSource } from '../sql/Interface';
export enum MediaTier {
STANDARD = 'standard',
@ -35,6 +36,8 @@ export type CoreAttachmentDownloadJobType = {
digest: string;
contentType: MIMEType;
size: number;
ciphertextSize: number;
source: AttachmentDownloadSource;
};
export type AttachmentDownloadJobType = CoreAttachmentDownloadJobType &
@ -51,7 +54,9 @@ export const coreAttachmentDownloadJobSchema = z.object({
digest: z.string(),
contentType: MIMETypeSchema,
size: z.number(),
ciphertextSize: z.number(),
messageIdForLogging: z.string().optional(),
source: z.nativeEnum(AttachmentDownloadSource),
});
export const attachmentDownloadJobSchema = coreAttachmentDownloadJobSchema.and(

2
ts/types/Storage.d.ts vendored
View file

@ -141,6 +141,8 @@ export type StorageAccessType = {
callLinkAuthCredentials: ReadonlyArray<GroupCredentialType>;
backupCredentials: ReadonlyArray<BackupCredentialType>;
backupCredentialsLastRequestTime: number;
backupAttachmentsSuccessfullyDownloadedSize: number;
backupAttachmentsTotalSizeToDownload: number;
setBackupSignatureKey: boolean;
lastReceivedAtCounter: number;
preferredReactionEmoji: ReadonlyArray<string>;

26
ts/util/queueAttachmentDownloads.ts
View file

@ -32,6 +32,7 @@ import {
AttachmentDownloadManager,
AttachmentDownloadUrgency,
} from '../jobs/AttachmentDownloadManager';
import { AttachmentDownloadSource } from '../sql/Interface';
export type MessageAttachmentsDownloadedType = {
bodyAttachment?: AttachmentType;
@ -62,7 +63,13 @@ function getAttachmentSignatureSafe(
// count then you'll also have to modify ./hasAttachmentsDownloads
export async function queueAttachmentDownloads(
message: MessageAttributesType,
urgency: AttachmentDownloadUrgency = AttachmentDownloadUrgency.STANDARD
{
urgency = AttachmentDownloadUrgency.STANDARD,
source = AttachmentDownloadSource.STANDARD,
}: {
urgency?: AttachmentDownloadUrgency;
source?: AttachmentDownloadSource;
} = {}
): Promise<MessageAttachmentsDownloadedType | undefined> {
const attachmentsToQueue = message.attachments || [];
const messageId = message.id;
@ -109,6 +116,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
}
@ -126,6 +134,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
}
);
count += attachmentsCount;
@ -144,6 +153,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
count += previewCount;
@ -162,6 +172,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
count += thumbnailCount;
@ -194,6 +205,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
}),
},
};
@ -230,6 +242,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
} else {
log.error(`${idLog}: Sticker data was missing`);
@ -267,6 +280,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
count += editAttachmentsCount;
if (editAttachmentsCount !== 0) {
@ -285,6 +299,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
count += editPreviewCount;
if (editPreviewCount !== 0) {
@ -328,6 +343,7 @@ async function queueNormalAttachments({
receivedAt,
sentAt,
urgency,
source,
}: {
idLog: string;
messageId: string;
@ -336,6 +352,7 @@ async function queueNormalAttachments({
receivedAt: number;
sentAt: number;
urgency: AttachmentDownloadUrgency;
source: AttachmentDownloadSource;
}): Promise<{
attachments: Array<AttachmentType>;
count: number;
@ -393,6 +410,7 @@ async function queueNormalAttachments({
receivedAt,
sentAt,
urgency,
source,
});
})
);
@ -426,6 +444,7 @@ async function queuePreviews({
receivedAt,
sentAt,
urgency,
source,
}: {
idLog: string;
messageId: string;
@ -434,6 +453,7 @@ async function queuePreviews({
receivedAt: number;
sentAt: number;
urgency: AttachmentDownloadUrgency;
source: AttachmentDownloadSource;
}): Promise<{ preview: Array<LinkPreviewType>; count: number }> {
// Similar to queueNormalAttachments' logic for detecting same attachments
// except here we also pick by link preview URL.
@ -485,6 +505,7 @@ async function queuePreviews({
receivedAt,
sentAt,
urgency,
source,
}),
};
})
@ -518,6 +539,7 @@ async function queueQuoteAttachments({
receivedAt,
sentAt,
urgency,
source,
}: {
idLog: string;
messageId: string;
@ -526,6 +548,7 @@ async function queueQuoteAttachments({
receivedAt: number;
sentAt: number;
urgency: AttachmentDownloadUrgency;
source: AttachmentDownloadSource;
}): Promise<{ quote?: QuotedMessageType; count: number }> {
let count = 0;
if (!quote) {
@ -600,6 +623,7 @@ async function queueQuoteAttachments({
receivedAt,
sentAt,
urgency,
source,
}),
};
})