Add a backup media download progress bar

This commit is contained in:
trevor-signal 2024-09-03 18:00:51 -04:00 committed by GitHub
parent 84f1d98020
commit 501f27127f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
31 changed files with 640 additions and 78 deletions

View file

@ -4687,6 +4687,15 @@
"messageformat": "This may take a few minutes depending on the size of your backup",
"description": "Description at the bottom of backup import screen"
},
"icu:BackupMediaDownloadProgress__title": {
"messageformat": "Restoring media",
"description": "Label above a progress bar showing media (attachment) download progress after restoring from backup"
},
"icu:BackupMediaDownloadProgress__progressbar-hint": {
"messageformat": "{currentSize} of {totalSize} ({fractionComplete, number, percent})",
"description": "Hint under the progressbar showing media (attachment) download progress after restoring from backup"
},
"icu:CompositionArea--expand": {
"messageformat": "Expand",
"description": "Aria label for expanding composition area"

View file

@ -0,0 +1,4 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M10.0002 2.50016C5.85803 2.50016 2.50016 5.85803 2.50016 10.0002C2.50016 11.8841 3.19389 13.6047 4.34133 14.9225L4.65934 14.6045C5.05886 14.205 5.74134 14.392 5.88136 14.9394L6.51385 17.4118C6.65114 17.9485 6.16341 18.4363 5.62672 18.299L3.15428 17.6665C2.60691 17.5265 2.41988 16.844 2.81939 16.4445L3.16053 16.1033C1.71395 14.4831 0.833496 12.3438 0.833496 10.0002C0.833496 4.93755 4.93755 0.833496 10.0002 0.833496C15.0628 0.833496 19.1668 4.93755 19.1668 10.0002C19.1668 15.0628 15.0628 19.1668 10.0002 19.1668C9.53992 19.1668 9.16683 18.7937 9.16683 18.3335C9.16683 17.8733 9.53992 17.5002 10.0002 17.5002C14.1423 17.5002 17.5002 14.1423 17.5002 10.0002C17.5002 5.85803 14.1423 2.50016 10.0002 2.50016Z" fill="#2C6BED"/>
<path d="M8.84768 4.3577C8.8609 3.96086 9.18644 3.646 9.5835 3.646C9.98056 3.646 10.3061 3.96086 10.3193 4.3577L10.4914 9.52006L13.9824 9.67874C14.377 9.69668 14.6877 10.0218 14.6877 10.4168C14.6877 10.8118 14.377 11.137 13.9824 11.1549L9.65504 11.3516C9.63134 11.3534 9.60747 11.3543 9.5835 11.3543C9.06573 11.3543 8.646 10.9346 8.646 10.4168C8.646 10.3992 8.64648 10.3817 8.64746 10.3643L8.84768 4.3577Z" fill="#2C6BED"/>
</svg>

After

Width:  |  Height:  |  Size: 1.2 KiB

View file

@ -79,6 +79,7 @@ $color-ultramarine-icon: #3a76f0;
$color-ultramarine-light: #6191f3;
$color-ultramarine-dawn: #406ec9;
$color-ultramarine-pastel: #abc4f8;
$color-ultramarine-pale: #d2dffb;
$color-ultramarine: #2c6bed;
$color-link: #315ff4;

View file

@ -19,30 +19,8 @@
margin-block: 0 20px;
}
.BackupImportScreen__progressbar {
overflow: hidden;
.BackupImportScreen .ProgressBar {
margin-block-end: 14px;
background: rgba($color-ultramarine, 0.2);
height: 5px;
border-radius: 2px;
}
.BackupImportScreen__progressbar__fill {
background-color: $color-ultramarine;
border-radius: 2px;
display: block;
height: 100%;
width: 100%;
&:dir(ltr) {
/* stylelint-disable-next-line declaration-property-value-disallowed-list */
transform: translateX(-100%);
}
&:dir(rtl) {
/* stylelint-disable-next-line declaration-property-value-disallowed-list */
transform: translateX(100%);
}
transition: transform 500ms ease-out;
}
.BackupImportScreen__progressbar-hint {

View file

@ -0,0 +1,96 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
.BackupMediaDownloadProgressBanner {
@include font-body-2;
border-radius: 10px;
display: flex;
gap: 12px;
padding: 12px;
padding-inline-end: 16px;
margin-inline: 10px;
user-select: none;
@include light-theme {
background-color: $color-white;
border: 1px solid $color-gray-20;
}
@include dark-theme {
background: $color-gray-75;
border: 1px solid $color-gray-60;
}
}
.BackupMediaDownloadProgressBanner__icon {
background: rgba($color-ultramarine, 0.2);
width: 30px;
height: 30px;
padding: 6px;
border-radius: 50%;
@include dark-theme {
background: $color-gray-60;
}
&::after {
content: '';
display: inline-block;
width: 18px;
height: 18px;
@include light-theme {
@include color-svg(
'../images/icons/v3/backup/backup-bold.svg',
$color-ultramarine
);
}
@include dark-theme {
@include color-svg(
'../images/icons/v3/backup/backup-bold.svg',
$color-ultramarine-pale
);
}
}
}
.BackupMediaDownloadProgressBanner__content {
display: flex;
flex-direction: column;
flex: 1;
gap: 7px;
}
.BackupMediaDownloadProgressBanner .Progressbar {
overflow: hidden;
background: rgba($color-ultramarine, 0.2);
height: 5px;
border-radius: 2px;
}
.BackupMediaDownloadProgressBanner__progressbar__fill {
background-color: $color-ultramarine;
border-radius: 2px;
display: block;
height: 100%;
width: 100%;
&:dir(ltr) {
/* stylelint-disable-next-line declaration-property-value-disallowed-list */
transform: translateX(-100%);
}
&:dir(rtl) {
/* stylelint-disable-next-line declaration-property-value-disallowed-list */
transform: translateX(100%);
}
transition: transform 500ms ease-out;
}
.BackupMediaDownloadProgressBanner__progressbar-hint {
@include font-caption;
@include light-theme {
color: rgba($color-gray-60, 0.8);
}
@include dark-theme {
color: $color-gray-25;
}
}

View file

@ -0,0 +1,17 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
.ProgressBar {
overflow: hidden;
background: rgba($color-ultramarine, 0.2);
height: 5px;
border-radius: 2px;
}
.ProgressBar__fill {
background-color: $color-ultramarine;
border-radius: 2px;
display: block;
height: 100%;
transition: margin 500ms ease-out;
}

View file

@ -33,6 +33,7 @@
@import './components/AvatarPreview.scss';
@import './components/AvatarTextEditor.scss';
@import './components/BackupImportScreen.scss';
@import './components/BackupMediaDownloadProgress.scss';
@import './components/BadgeCarouselIndex.scss';
@import './components/BadgeDialog.scss';
@import './components/BadgeSustainerInstructionsDialog.scss';
@ -137,6 +138,7 @@
@import './components/PlaybackRateButton.scss';
@import './components/Preferences.scss';
@import './components/ProfileEditor.scss';
@import './components/ProgressBar.scss';
@import './components/Quote.scss';
@import './components/ReactionPickerPicker.scss';
@import './components/RecordingComposer.scss';

View file

@ -7,6 +7,7 @@ import type { LocalizerType } from '../types/Util';
import { formatFileSize } from '../util/formatFileSize';
import { TitlebarDragArea } from './TitlebarDragArea';
import { InstallScreenSignalLogo } from './installScreen/InstallScreenSignalLogo';
import { ProgressBar } from './ProgressBar';
// We can't always use destructuring assignment because of the complexity of this props
// type.
@ -36,12 +37,7 @@ export function BackupImportScreen({
progress = (
<>
<div className="BackupImportScreen__progressbar">
<div
className="BackupImportScreen__progressbar__fill"
style={{ transform: `translateX(${(percentage - 1) * 100}%)` }}
/>
</div>
<ProgressBar fractionComplete={percentage} />
<div className="BackupImportScreen__progressbar-hint">
{i18n('icu:BackupImportScreen__progressbar-hint', {
currentSize: formatFileSize(currentBytes),
@ -54,7 +50,7 @@ export function BackupImportScreen({
} else {
progress = (
<>
<div className="BackupImportScreen__progressbar" />
<ProgressBar fractionComplete={0} />
<div className="BackupImportScreen__progressbar-hint BackupImportScreen__progressbar-hint--hidden">
{i18n('icu:BackupImportScreen__progressbar-hint', {
currentSize: '',

View file

@ -0,0 +1,27 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import React, { type ComponentProps } from 'react';
import type { Meta, StoryFn } from '@storybook/react';
import { setupI18n } from '../util/setupI18n';
import enMessages from '../../_locales/en/messages.json';
import { BackupMediaDownloadProgressBanner } from './BackupMediaDownloadProgress';
const i18n = setupI18n('en', enMessages);
type PropsType = ComponentProps<typeof BackupMediaDownloadProgressBanner>;
export default {
title: 'Components/BackupMediaDownloadProgress',
} satisfies Meta<PropsType>;
// eslint-disable-next-line react/function-component-definition
const Template: StoryFn<PropsType> = (args: PropsType) => (
<BackupMediaDownloadProgressBanner {...args} i18n={i18n} />
);
export const InProgress = Template.bind({});
InProgress.args = {
downloadedBytes: 92048023,
totalBytes: 1024102532,
};

View file

@ -0,0 +1,48 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import React from 'react';
import type { LocalizerType } from '../types/Util';
import { formatFileSize } from '../util/formatFileSize';
import { ProgressBar } from './ProgressBar';
export type PropsType = Readonly<{
i18n: LocalizerType;
downloadedBytes: number;
totalBytes: number;
}>;
export function BackupMediaDownloadProgressBanner({
i18n,
downloadedBytes,
totalBytes,
}: PropsType): JSX.Element | null {
if (totalBytes === 0) {
return null;
}
const fractionComplete = Math.max(
0,
Math.min(1, downloadedBytes / totalBytes)
);
return (
<div className="BackupMediaDownloadProgressBanner">
<div className="BackupMediaDownloadProgressBanner__icon" />
<div className="BackupMediaDownloadProgressBanner__content">
<div className="BackupMediaDownloadProgressBanner__title">
{i18n('icu:BackupMediaDownloadProgress__title')}
</div>
<ProgressBar fractionComplete={fractionComplete} />
<div className="BackupMediaDownloadProgressBanner__progressbar-hint">
{i18n('icu:BackupMediaDownloadProgress__progressbar-hint', {
currentSize: formatFileSize(downloadedBytes),
totalSize: formatFileSize(totalBytes),
fractionComplete,
})}
</div>
</div>
</div>
);
}

View file

@ -137,6 +137,7 @@ const useProps = (overrideProps: OverridePropsType = {}): PropsType => {
unreadMentionsCount: 0,
markedUnread: false,
},
backupMediaDownloadProgress: { totalBytes: 0, downloadedBytes: 0 },
clearConversationSearch: action('clearConversationSearch'),
clearGroupCreationError: action('clearGroupCreationError'),
clearSearch: action('clearSearch'),

View file

@ -1,7 +1,13 @@
// Copyright 2019 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import React, { useEffect, useCallback, useMemo, useRef } from 'react';
import React, {
useEffect,
useCallback,
useMemo,
useRef,
useState,
} from 'react';
import classNames from 'classnames';
import { isNumber } from 'lodash';
@ -56,8 +62,10 @@ import {
import { ContextMenu } from './ContextMenu';
import { EditState as ProfileEditorEditState } from './ProfileEditor';
import type { UnreadStats } from '../util/countUnreadStats';
import { BackupMediaDownloadProgressBanner } from './BackupMediaDownloadProgress';
export type PropsType = {
backupMediaDownloadProgress: { totalBytes: number; downloadedBytes: number };
otherTabsUnreadStats: UnreadStats;
hasExpiredDialog: boolean;
hasFailedStorySends: boolean;
@ -173,6 +181,7 @@ export type PropsType = {
} & LookupConversationWithoutServiceIdActionsType;
export function LeftPane({
backupMediaDownloadProgress,
otherTabsUnreadStats,
blockConversation,
challengeStatus,
@ -634,6 +643,34 @@ export function LeftPane({
dialogs.push({ key: 'banner', dialog: maybeBanner });
}
// We'll show the backup media download progress banner if the download is currently or
// was ongoing at some point during the lifecycle of this component
const [
hasMediaBackupDownloadBeenOngoing,
setHasMediaBackupDownloadBeenOngoing,
] = useState(false);
const isMediaBackupDownloadOngoing =
backupMediaDownloadProgress?.totalBytes > 0 &&
backupMediaDownloadProgress.downloadedBytes <
backupMediaDownloadProgress.totalBytes;
if (isMediaBackupDownloadOngoing && !hasMediaBackupDownloadBeenOngoing) {
setHasMediaBackupDownloadBeenOngoing(true);
}
if (hasMediaBackupDownloadBeenOngoing) {
dialogs.push({
key: 'backupMediaDownload',
dialog: (
<BackupMediaDownloadProgressBanner
i18n={i18n}
{...backupMediaDownloadProgress}
/>
),
});
}
const hideHeader =
modeSpecificProps.mode === LeftPaneMode.Archive ||
modeSpecificProps.mode === LeftPaneMode.Compose ||

View file

@ -0,0 +1,21 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import React from 'react';
export function ProgressBar({
fractionComplete,
}: {
fractionComplete: number;
}): JSX.Element {
return (
<div className="ProgressBar">
<div
className="ProgressBar__fill"
style={{
marginInlineEnd: `${(1 - fractionComplete) * 100}%`,
}}
/>
</div>
);
}

View file

@ -42,6 +42,9 @@ import {
isVideoTypeSupported,
} from '../util/GoogleChrome';
import type { MIMEType } from '../types/MIME';
import type { AttachmentDownloadSource } from '../sql/Interface';
import { drop } from '../util/drop';
import { getAttachmentCiphertextLength } from '../AttachmentCrypto';
export enum AttachmentDownloadUrgency {
IMMEDIATE = 'immediate',
@ -55,6 +58,7 @@ export type NewAttachmentDownloadJobType = {
receivedAt: number;
sentAt: number;
attachmentType: AttachmentDownloadJobTypeType;
source: AttachmentDownloadSource;
urgency?: AttachmentDownloadUrgency;
};
@ -69,6 +73,10 @@ const DEFAULT_RETRY_CONFIG = {
maxBackoffTime: 6 * durations.HOUR,
},
};
const BACKUP_RETRY_CONFIG = {
...DEFAULT_RETRY_CONFIG,
maxAttempts: Infinity,
};
type AttachmentDownloadManagerParamsType = Omit<
JobManagerParamsType<CoreAttachmentDownloadJobType>,
'getNextJobs' | 'runJob'
@ -117,7 +125,10 @@ export class AttachmentDownloadManager extends JobManager<CoreAttachmentDownload
},
getJobId,
getJobIdForLogging,
getRetryConfig: () => DEFAULT_RETRY_CONFIG,
getRetryConfig: job =>
job.attachment.backupLocator?.mediaName
? BACKUP_RETRY_CONFIG
: DEFAULT_RETRY_CONFIG,
maxConcurrentJobs: MAX_CONCURRENT_JOBS,
};
@ -157,6 +168,7 @@ export class AttachmentDownloadManager extends JobManager<CoreAttachmentDownload
attachmentType,
receivedAt,
sentAt,
source,
urgency = AttachmentDownloadUrgency.STANDARD,
} = newJobData;
const parseResult = coreAttachmentDownloadJobSchema.safeParse({
@ -167,7 +179,9 @@ export class AttachmentDownloadManager extends JobManager<CoreAttachmentDownload
digest: attachment.digest,
contentType: attachment.contentType,
size: attachment.size,
ciphertextSize: getAttachmentCiphertextLength(attachment.size),
attachment,
source,
});
if (!parseResult.success) {
@ -251,13 +265,24 @@ async function runDownloadAttachmentJob({
dependencies,
});
if (result.onlyAttemptedBackupThumbnail) {
if (result.downloadedVariant === AttachmentVariant.ThumbnailFromBackup) {
return {
status: 'finished',
newJob: { ...job, attachment: result.attachmentWithThumbnail },
};
}
if (job.attachment.backupLocator?.mediaName) {
const currentDownloadedSize =
window.storage.get('backupAttachmentsSuccessfullyDownloadedSize') ?? 0;
drop(
window.storage.put(
'backupAttachmentsSuccessfullyDownloadedSize',
currentDownloadedSize + job.ciphertextSize
)
);
}
return {
status: 'finished',
};
@ -319,11 +344,9 @@ async function runDownloadAttachmentJob({
}
type DownloadAttachmentResultType =
| { downloadedVariant: AttachmentVariant.Default }
| {
onlyAttemptedBackupThumbnail: false;
}
| {
onlyAttemptedBackupThumbnail: true;
downloadedVariant: AttachmentVariant.ThumbnailFromBackup;
attachmentWithThumbnail: AttachmentType;
};
@ -386,7 +409,7 @@ export async function runDownloadAttachmentJobInner({
type: attachmentType,
});
return {
onlyAttemptedBackupThumbnail: true,
downloadedVariant: AttachmentVariant.ThumbnailFromBackup,
attachmentWithThumbnail,
};
} catch (e) {
@ -418,7 +441,7 @@ export async function runDownloadAttachmentJobInner({
await addAttachmentToMessage(messageId, upgradedAttachment, logId, {
type: attachmentType,
});
return { onlyAttemptedBackupThumbnail: false };
return { downloadedVariant: AttachmentVariant.Default };
} catch (error) {
if (
!job.attachment.thumbnailFromBackup &&
@ -430,20 +453,24 @@ export async function runDownloadAttachmentJobInner({
Errors.toLogFormat(error)
);
try {
const attachmentWithThumbnail = await downloadBackupThumbnail({
attachment,
dependencies,
});
const attachmentWithThumbnail = omit(
await downloadBackupThumbnail({
attachment,
dependencies,
}),
'pending'
);
await addAttachmentToMessage(
messageId,
omit(attachmentWithThumbnail, 'pending'),
attachmentWithThumbnail,
logId,
{
type: attachmentType,
}
);
return {
onlyAttemptedBackupThumbnail: false,
downloadedVariant: AttachmentVariant.ThumbnailFromBackup,
attachmentWithThumbnail,
};
} catch (thumbnailError) {
log.error(

View file

@ -127,7 +127,7 @@ export async function onSync(sync: ViewSyncAttributesType): Promise<void> {
if (!attachments?.every(isDownloaded)) {
const updatedFields = await queueAttachmentDownloads(
message.attributes,
AttachmentDownloadUrgency.STANDARD
{ urgency: AttachmentDownloadUrgency.STANDARD }
);
if (updatedFields) {
message.set(updatedFields);

View file

@ -1344,7 +1344,7 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
async queueAttachmentDownloads(
urgency?: AttachmentDownloadUrgency
): Promise<boolean> {
const value = await queueAttachmentDownloads(this.attributes, urgency);
const value = await queueAttachmentDownloads(this.attributes, { urgency });
if (!value) {
return false;
}

View file

@ -11,7 +11,10 @@ import { CallLinkRootKey } from '@signalapp/ringrtc';
import { Backups, SignalService } from '../../protobuf';
import { DataWriter } from '../../sql/Client';
import type { StoryDistributionWithMembersType } from '../../sql/Interface';
import {
AttachmentDownloadSource,
type StoryDistributionWithMembersType,
} from '../../sql/Interface';
import * as log from '../../logging/log';
import { GiftBadgeStates } from '../../components/conversation/Message';
import { StorySendMode, MY_STORY_ID } from '../../types/Stories';
@ -169,7 +172,11 @@ async function processMessagesBatch(
);
}
drop(queueAttachmentDownloads(attributes));
drop(
queueAttachmentDownloads(attributes, {
source: AttachmentDownloadSource.BACKUP_IMPORT,
})
);
}
}

View file

@ -40,6 +40,7 @@ import { BackupCredentials } from './credentials';
import { BackupAPI, type DownloadOptionsType } from './api';
import { validateBackup } from './validator';
import { getParametersForRedux, loadAll } from '../allLoaders';
import { AttachmentDownloadManager } from '../../jobs/AttachmentDownloadManager';
const IV_LENGTH = 16;
@ -236,6 +237,8 @@ export class BackupsService {
// Second pass - decrypt (but still check the mac at the end)
hmac = createHmac(HashType.size256, macKey);
await this.prepareForImport();
await pipeline(
createBackupStream(),
getMacAndUpdateHmac(hmac, noop),
@ -278,11 +281,25 @@ export class BackupsService {
}
}
public async prepareForImport(): Promise<void> {
await AttachmentDownloadManager.stop();
await DataWriter.removeAllBackupAttachmentDownloadJobs();
await window.storage.put('backupAttachmentsSuccessfullyDownloadedSize', 0);
await window.storage.put('backupAttachmentsTotalSizeToDownload', 0);
}
public async resetStateAfterImport(): Promise<void> {
window.ConversationController.reset();
await window.ConversationController.load();
await loadAll();
reinitializeRedux(getParametersForRedux());
await window.storage.put(
'backupAttachmentsTotalSizeToDownload',
await DataReader.getSizeOfPendingBackupAttachmentDownloadJobs()
);
await AttachmentDownloadManager.start();
}
public async fetchAndSaveBackupCdnObjectMetadata(): Promise<void> {

View file

@ -446,6 +446,11 @@ export type GetRecentStoryRepliesOptionsType = {
sentAt?: number;
};
export enum AttachmentDownloadSource {
BACKUP_IMPORT = 'backup_import',
STANDARD = 'standard',
}
type ReadableInterface = {
close: () => void;
@ -642,6 +647,7 @@ type ReadableInterface = {
getMaxMessageCounter(): number | undefined;
getStatisticsForLogging(): Record<string, string>;
getSizeOfPendingBackupAttachmentDownloadJobs(): number;
};
type WritableInterface = {
@ -840,6 +846,7 @@ type WritableInterface = {
saveAttachmentDownloadJob: (job: AttachmentDownloadJobType) => void;
resetAttachmentDownloadActive: () => void;
removeAttachmentDownloadJob: (job: AttachmentDownloadJobType) => void;
removeAllBackupAttachmentDownloadJobs: () => void;
getNextAttachmentBackupJobs: (options: {
limit: number;

View file

@ -145,6 +145,7 @@ import type {
StoredKyberPreKeyType,
BackupCdnMediaObjectType,
} from './Interface';
import { AttachmentDownloadSource } from './Interface';
import { SeenStatus } from '../MessageSeenStatus';
import {
SNIPPET_LEFT_PLACEHOLDER,
@ -200,6 +201,7 @@ import {
attachmentBackupJobSchema,
} from '../types/AttachmentBackup';
import { redactGenericText } from '../util/privacy';
import { getAttachmentCiphertextLength } from '../AttachmentCrypto';
type ConversationRow = Readonly<{
json: string;
@ -340,6 +342,7 @@ export const DataReader: ServerReadableInterface = {
getStatisticsForLogging,
getBackupCdnObjectMetadata,
getSizeOfPendingBackupAttachmentDownloadJobs,
// Server-only
getKnownMessageAttachments,
@ -463,6 +466,7 @@ export const DataWriter: ServerWritableInterface = {
saveAttachmentDownloadJob,
resetAttachmentDownloadActive,
removeAttachmentDownloadJob,
removeAllBackupAttachmentDownloadJobs,
getNextAttachmentBackupJobs,
saveAttachmentBackupJob,
@ -4733,6 +4737,20 @@ function getAttachmentDownloadJob(
return db.prepare(query).get(params);
}
function removeAllBackupAttachmentDownloadJobs(db: WritableDB): void {
const [query, params] = sql`
DELETE FROM attachment_downloads
WHERE source = ${AttachmentDownloadSource.BACKUP_IMPORT};`;
db.prepare(query).run(params);
}
function getSizeOfPendingBackupAttachmentDownloadJobs(db: ReadableDB): number {
const [query, params] = sql`
SELECT SUM(ciphertextSize) FROM attachment_downloads
WHERE source = ${AttachmentDownloadSource.BACKUP_IMPORT};`;
return db.prepare(query).pluck().get(params);
}
function getNextAttachmentDownloadJobs(
db: WritableDB,
{
@ -4799,6 +4817,9 @@ function getNextAttachmentDownloadJobs(
...row,
active: Boolean(row.active),
attachment: jsonToObject(row.attachmentJson),
ciphertextSize:
row.ciphertextSize ||
getAttachmentCiphertextLength(row.attachment.size),
});
} catch (error) {
logger.error(
@ -4839,7 +4860,9 @@ function saveAttachmentDownloadJob(
attempts,
retryAfter,
lastAttemptTimestamp,
attachmentJson
attachmentJson,
ciphertextSize,
source
) VALUES (
${job.messageId},
${job.attachmentType},
@ -4852,7 +4875,9 @@ function saveAttachmentDownloadJob(
${job.attempts},
${job.retryAfter},
${job.lastAttemptTimestamp},
${objectToJSON(job.attachment)}
${objectToJSON(job.attachment)},
${job.ciphertextSize},
${job.source}
);
`;
db.prepare(query).run(params);

View file

@ -11,6 +11,7 @@ import {
} from '../../types/AttachmentDownload';
import type { AttachmentType } from '../../types/Attachment';
import { jsonToObject, objectToJSON, sql } from '../util';
import { AttachmentDownloadSource } from '../Interface';
export const version = 1040;
@ -133,6 +134,9 @@ export function updateToSchemaVersion1040(
attempts: existingJobData.attempts ?? 0,
retryAfter: null,
lastAttemptTimestamp: null,
// adding due to changes in the schema
source: AttachmentDownloadSource.STANDARD,
ciphertextSize: 0,
};
const parsed = attachmentDownloadJobSchema.parse(updatedJob);

View file

@ -0,0 +1,37 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import type { Database } from '@signalapp/better-sqlite3';
import type { LoggerType } from '../../types/Logging';
import { AttachmentDownloadSource } from '../Interface';
export const version = 1180;
export function updateToSchemaVersion1180(
currentVersion: number,
db: Database,
logger: LoggerType
): void {
if (currentVersion >= 1180) {
return;
}
db.transaction(() => {
db.exec(`
ALTER TABLE attachment_downloads
ADD COLUMN source TEXT NOT NULL DEFAULT ${AttachmentDownloadSource.STANDARD};
ALTER TABLE attachment_downloads
-- this default value will be overridden by getNextAttachmentDownloadJobs
ADD COLUMN ciphertextSize INTEGER NOT NULL DEFAULT 0;
`);
db.exec(`
CREATE INDEX attachment_downloads_source_ciphertextSize
ON attachment_downloads (
source, ciphertextSize
);
`);
db.pragma('user_version = 1180');
})();
logger.info('updateToSchemaVersion1180: success!');
}

View file

@ -93,10 +93,11 @@ import { updateToSchemaVersion1130 } from './1130-isStory-index';
import { updateToSchemaVersion1140 } from './1140-call-links-deleted-column';
import { updateToSchemaVersion1150 } from './1150-expire-timer-version';
import { updateToSchemaVersion1160 } from './1160-optimize-calls-unread-count';
import { updateToSchemaVersion1170 } from './1170-update-call-history-unread-index';
import {
updateToSchemaVersion1170,
updateToSchemaVersion1180,
version as MAX_VERSION,
} from './1170-update-call-history-unread-index';
} from './1180-add-attachment-download-source';
function updateToSchemaVersion1(
currentVersion: number,
@ -2058,6 +2059,7 @@ export const SCHEMA_VERSIONS = [
updateToSchemaVersion1150,
updateToSchemaVersion1160,
updateToSchemaVersion1170,
updateToSchemaVersion1180,
];
export class DBVersionFromFutureError extends Error {

View file

@ -248,3 +248,11 @@ export const getLocalDeleteWarningShown = createSelector(
(state: ItemsStateType): boolean =>
Boolean(state.localDeleteWarningShown ?? false)
);
export const getBackupMediaDownloadProgress = createSelector(
getItems,
(state: ItemsStateType): { totalBytes: number; downloadedBytes: number } => ({
totalBytes: state.backupAttachmentsTotalSizeToDownload ?? 0,
downloadedBytes: state.backupAttachmentsSuccessfullyDownloadedSize ?? 0,
})
);

View file

@ -58,6 +58,7 @@ import {
import { getCrashReportCount } from '../selectors/crashReports';
import { hasExpired } from '../selectors/expiration';
import {
getBackupMediaDownloadProgress,
getNavTabsCollapsed,
getPreferredLeftPaneWidth,
getUsernameCorrupted,
@ -115,6 +116,7 @@ function renderUpdateDialog(
): JSX.Element {
return <SmartUpdateDialog {...props} />;
}
function renderCaptchaDialog({ onSkip }: { onSkip(): void }): JSX.Element {
return <SmartCaptchaDialog onSkip={onSkip} />;
}
@ -289,7 +291,9 @@ export const SmartLeftPane = memo(function SmartLeftPane({
const theme = useSelector(getTheme);
const usernameCorrupted = useSelector(getUsernameCorrupted);
const usernameLinkCorrupted = useSelector(getUsernameLinkCorrupted);
const backupMediaDownloadProgress = useSelector(
getBackupMediaDownloadProgress
);
const {
blockConversation,
clearGroupCreationError,
@ -360,6 +364,7 @@ export const SmartLeftPane = memo(function SmartLeftPane({
return (
<LeftPane
backupMediaDownloadProgress={backupMediaDownloadProgress}
blockConversation={blockConversation}
challengeStatus={challengeStatus}
clearConversationSearch={clearConversationSearch}

View file

@ -20,6 +20,8 @@ import { MINUTE } from '../../util/durations';
import { type AciString } from '../../types/ServiceId';
import { type AttachmentType, AttachmentVariant } from '../../types/Attachment';
import { strictAssert } from '../../util/assert';
import { AttachmentDownloadSource } from '../../sql/Interface';
import { getAttachmentCiphertextLength } from '../../AttachmentCrypto';
function composeJob({
messageId,
@ -38,11 +40,13 @@ function composeJob({
attachmentType: 'attachment',
digest,
size,
ciphertextSize: getAttachmentCiphertextLength(size),
contentType,
active: false,
attempts: 0,
retryAfter: null,
lastAttemptTimestamp: null,
source: AttachmentDownloadSource.STANDARD,
attachment: {
contentType,
size,
@ -114,8 +118,8 @@ describe('AttachmentDownloadManager/JobManager', () => {
}
);
await downloadManager?.addJob({
...job,
urgency,
...job,
});
}
async function addJobs(
@ -419,7 +423,7 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => {
dependencies: { downloadAttachment },
});
assert.strictEqual(result.onlyAttemptedBackupThumbnail, false);
assert.strictEqual(result.downloadedVariant, AttachmentVariant.Default);
assert.strictEqual(downloadAttachment.callCount, 1);
assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], {
attachment: job.attachment,
@ -444,8 +448,8 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => {
});
strictAssert(
result.onlyAttemptedBackupThumbnail === true,
'only attempted backup thumbnail'
result.downloadedVariant === AttachmentVariant.ThumbnailFromBackup,
'downloaded thumbnail'
);
assert.deepStrictEqual(
omit(result.attachmentWithThumbnail, 'thumbnailFromBackup'),
@ -485,7 +489,7 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => {
isForCurrentlyVisibleMessage: true,
dependencies: { downloadAttachment },
});
assert.strictEqual(result.onlyAttemptedBackupThumbnail, false);
assert.strictEqual(result.downloadedVariant, AttachmentVariant.Default);
assert.strictEqual(downloadAttachment.callCount, 1);
assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], {
attachment: job.attachment,
@ -544,7 +548,7 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => {
isForCurrentlyVisibleMessage: false,
dependencies: { downloadAttachment },
});
assert.strictEqual(result.onlyAttemptedBackupThumbnail, false);
assert.strictEqual(result.downloadedVariant, AttachmentVariant.Default);
assert.strictEqual(downloadAttachment.callCount, 1);
assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], {
attachment: job.attachment,
@ -578,7 +582,10 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => {
isForCurrentlyVisibleMessage: false,
dependencies: { downloadAttachment },
});
assert.strictEqual(result.onlyAttemptedBackupThumbnail, false);
assert.strictEqual(
result.downloadedVariant,
AttachmentVariant.ThumbnailFromBackup
);
assert.strictEqual(downloadAttachment.callCount, 2);
assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], {
attachment: job.attachment,

View file

@ -27,7 +27,7 @@ function getAttachmentDownloadJobs(db: ReadableDB) {
type UnflattenedAttachmentDownloadJobType = Omit<
AttachmentDownloadJobType,
'digest' | 'contentType' | 'size'
'digest' | 'contentType' | 'size' | 'source' | 'ciphertextSize'
>;
function insertNewJob(
db: WritableDB,
@ -304,24 +304,25 @@ describe('SQL/updateToSchemaVersion1040', () => {
});
it('respects foreign key constraint on messageId', () => {
const job: AttachmentDownloadJobType = {
messageId: 'message1',
attachmentType: 'attachment',
attachment: {
const job: Omit<AttachmentDownloadJobType, 'source' | 'ciphertextSize'> =
{
messageId: 'message1',
attachmentType: 'attachment',
attachment: {
digest: 'digest1',
contentType: IMAGE_JPEG,
size: 128,
},
receivedAt: 1970,
digest: 'digest1',
contentType: IMAGE_JPEG,
size: 128,
},
receivedAt: 1970,
digest: 'digest1',
contentType: IMAGE_JPEG,
size: 128,
sentAt: 2070,
active: false,
retryAfter: null,
attempts: 0,
lastAttemptTimestamp: null,
};
sentAt: 2070,
active: false,
retryAfter: null,
attempts: 0,
lastAttemptTimestamp: null,
};
// throws if we don't add the message first
assert.throws(() => insertNewJob(db, job, false));
insertNewJob(db, job, true);

View file

@ -0,0 +1,147 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { assert } from 'chai';
import { omit } from 'lodash';
import type { WritableDB } from '../../sql/Interface';
import { createDB, updateToVersion } from './helpers';
import type { AttachmentDownloadJobType } from '../../types/AttachmentDownload';
import { jsonToObject, objectToJSON, sql } from '../../sql/util';
import { IMAGE_BMP } from '../../types/MIME';
function insertOldJob(
db: WritableDB,
job: Omit<AttachmentDownloadJobType, 'source' | 'ciphertextSize'>,
addMessageFirst: boolean = true
): void {
if (addMessageFirst) {
try {
db.prepare('INSERT INTO messages (id) VALUES ($id)').run({
id: job.messageId,
});
} catch (e) {
// pass; message has already been inserted
}
}
const [query, params] = sql`
INSERT INTO attachment_downloads
(
messageId,
attachmentType,
attachmentJson,
digest,
contentType,
size,
receivedAt,
sentAt,
active,
attempts,
retryAfter,
lastAttemptTimestamp
)
VALUES
(
${job.messageId},
${job.attachmentType},
${objectToJSON(job.attachment)},
${job.digest},
${job.contentType},
${job.size},
${job.receivedAt},
${job.sentAt},
${job.active ? 1 : 0},
${job.attempts},
${job.retryAfter},
${job.lastAttemptTimestamp}
);
`;
db.prepare(query).run(params);
}
function getAttachmentDownloadJobs(db: WritableDB) {
const [query] = sql`
SELECT * FROM attachment_downloads ORDER BY receivedAt DESC;
`;
return db
.prepare(query)
.all()
.map(job => ({
...omit(job, 'attachmentJson'),
active: job.active === 1,
attachment: jsonToObject(job.attachmentJson),
}));
}
describe('SQL/updateToSchemaVersion1180', () => {
let db: WritableDB;
beforeEach(() => {
db = createDB();
updateToVersion(db, 1170);
});
afterEach(() => {
db.close();
});
it('adds source column with default standard to any existing jobs', async () => {
const job: Omit<AttachmentDownloadJobType, 'source' | 'ciphertextSize'> = {
messageId: '123',
digest: 'digest',
attachmentType: 'attachment',
attachment: { size: 128, contentType: IMAGE_BMP },
size: 128,
contentType: IMAGE_BMP,
receivedAt: 120,
sentAt: 120,
active: false,
attempts: 0,
retryAfter: null,
lastAttemptTimestamp: null,
};
insertOldJob(db, job);
updateToVersion(db, 1180);
assert.deepEqual(getAttachmentDownloadJobs(db), [
{ ...job, source: 'standard', ciphertextSize: 0 },
]);
});
it('uses convering index for summing all pending backup jobs', async () => {
updateToVersion(db, 1180);
const details = db
.prepare(
`
EXPLAIN QUERY PLAN
SELECT SUM(ciphertextSize) FROM attachment_downloads
WHERE source = 'backup_import';
`
)
.all()
.map(step => step.detail)
.join(', ');
assert.strictEqual(
details,
'SEARCH attachment_downloads USING COVERING INDEX attachment_downloads_source_ciphertextSize (source=?)'
);
});
it('uses index for deleting all backup jobs', async () => {
updateToVersion(db, 1180);
const details = db
.prepare(
`
EXPLAIN QUERY PLAN
DELETE FROM attachment_downloads
WHERE source = 'backup_import';
`
)
.all()
.map(step => step.detail)
.join(', ');
assert.strictEqual(
details,
'SEARCH attachment_downloads USING COVERING INDEX attachment_downloads_source_ciphertextSize (source=?)'
);
});
});

View file

@ -7,6 +7,7 @@ import {
type JobManagerJobType,
jobManagerJobSchema,
} from '../jobs/JobManager';
import { AttachmentDownloadSource } from '../sql/Interface';
export enum MediaTier {
STANDARD = 'standard',
@ -35,6 +36,8 @@ export type CoreAttachmentDownloadJobType = {
digest: string;
contentType: MIMEType;
size: number;
ciphertextSize: number;
source: AttachmentDownloadSource;
};
export type AttachmentDownloadJobType = CoreAttachmentDownloadJobType &
@ -51,7 +54,9 @@ export const coreAttachmentDownloadJobSchema = z.object({
digest: z.string(),
contentType: MIMETypeSchema,
size: z.number(),
ciphertextSize: z.number(),
messageIdForLogging: z.string().optional(),
source: z.nativeEnum(AttachmentDownloadSource),
});
export const attachmentDownloadJobSchema = coreAttachmentDownloadJobSchema.and(

View file

@ -141,6 +141,8 @@ export type StorageAccessType = {
callLinkAuthCredentials: ReadonlyArray<GroupCredentialType>;
backupCredentials: ReadonlyArray<BackupCredentialType>;
backupCredentialsLastRequestTime: number;
backupAttachmentsSuccessfullyDownloadedSize: number;
backupAttachmentsTotalSizeToDownload: number;
setBackupSignatureKey: boolean;
lastReceivedAtCounter: number;
preferredReactionEmoji: ReadonlyArray<string>;

View file

@ -32,6 +32,7 @@ import {
AttachmentDownloadManager,
AttachmentDownloadUrgency,
} from '../jobs/AttachmentDownloadManager';
import { AttachmentDownloadSource } from '../sql/Interface';
export type MessageAttachmentsDownloadedType = {
bodyAttachment?: AttachmentType;
@ -62,7 +63,13 @@ function getAttachmentSignatureSafe(
// count then you'll also have to modify ./hasAttachmentsDownloads
export async function queueAttachmentDownloads(
message: MessageAttributesType,
urgency: AttachmentDownloadUrgency = AttachmentDownloadUrgency.STANDARD
{
urgency = AttachmentDownloadUrgency.STANDARD,
source = AttachmentDownloadSource.STANDARD,
}: {
urgency?: AttachmentDownloadUrgency;
source?: AttachmentDownloadSource;
} = {}
): Promise<MessageAttachmentsDownloadedType | undefined> {
const attachmentsToQueue = message.attachments || [];
const messageId = message.id;
@ -109,6 +116,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
}
@ -126,6 +134,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
}
);
count += attachmentsCount;
@ -144,6 +153,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
count += previewCount;
@ -162,6 +172,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
count += thumbnailCount;
@ -194,6 +205,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
}),
},
};
@ -230,6 +242,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
} else {
log.error(`${idLog}: Sticker data was missing`);
@ -267,6 +280,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
count += editAttachmentsCount;
if (editAttachmentsCount !== 0) {
@ -285,6 +299,7 @@ export async function queueAttachmentDownloads(
receivedAt: message.received_at,
sentAt: message.sent_at,
urgency,
source,
});
count += editPreviewCount;
if (editPreviewCount !== 0) {
@ -328,6 +343,7 @@ async function queueNormalAttachments({
receivedAt,
sentAt,
urgency,
source,
}: {
idLog: string;
messageId: string;
@ -336,6 +352,7 @@ async function queueNormalAttachments({
receivedAt: number;
sentAt: number;
urgency: AttachmentDownloadUrgency;
source: AttachmentDownloadSource;
}): Promise<{
attachments: Array<AttachmentType>;
count: number;
@ -393,6 +410,7 @@ async function queueNormalAttachments({
receivedAt,
sentAt,
urgency,
source,
});
})
);
@ -426,6 +444,7 @@ async function queuePreviews({
receivedAt,
sentAt,
urgency,
source,
}: {
idLog: string;
messageId: string;
@ -434,6 +453,7 @@ async function queuePreviews({
receivedAt: number;
sentAt: number;
urgency: AttachmentDownloadUrgency;
source: AttachmentDownloadSource;
}): Promise<{ preview: Array<LinkPreviewType>; count: number }> {
// Similar to queueNormalAttachments' logic for detecting same attachments
// except here we also pick by link preview URL.
@ -485,6 +505,7 @@ async function queuePreviews({
receivedAt,
sentAt,
urgency,
source,
}),
};
})
@ -518,6 +539,7 @@ async function queueQuoteAttachments({
receivedAt,
sentAt,
urgency,
source,
}: {
idLog: string;
messageId: string;
@ -526,6 +548,7 @@ async function queueQuoteAttachments({
receivedAt: number;
sentAt: number;
urgency: AttachmentDownloadUrgency;
source: AttachmentDownloadSource;
}): Promise<{ quote?: QuotedMessageType; count: number }> {
let count = 0;
if (!quote) {
@ -600,6 +623,7 @@ async function queueQuoteAttachments({
receivedAt,
sentAt,
urgency,
source,
}),
};
})