diff --git a/_locales/en/messages.json b/_locales/en/messages.json index 83b263e359c7..e956a808b3c9 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -4687,6 +4687,15 @@ "messageformat": "This may take a few minutes depending on the size of your backup", "description": "Description at the bottom of backup import screen" }, + + "icu:BackupMediaDownloadProgress__title": { + "messageformat": "Restoring media", + "description": "Label above a progress bar showing media (attachment) download progress after restoring from backup" + }, + "icu:BackupMediaDownloadProgress__progressbar-hint": { + "messageformat": "{currentSize} of {totalSize} ({fractionComplete, number, percent})", + "description": "Hint under the progressbar showing media (attachment) download progress after restoring from backup" + }, "icu:CompositionArea--expand": { "messageformat": "Expand", "description": "Aria label for expanding composition area" diff --git a/images/icons/v3/backup/backup-bold.svg b/images/icons/v3/backup/backup-bold.svg new file mode 100644 index 000000000000..0ae7195b0bfa --- /dev/null +++ b/images/icons/v3/backup/backup-bold.svg @@ -0,0 +1,4 @@ + + + + diff --git a/stylesheets/_variables.scss b/stylesheets/_variables.scss index 58313ace18b0..b177c04a7864 100644 --- a/stylesheets/_variables.scss +++ b/stylesheets/_variables.scss @@ -79,6 +79,7 @@ $color-ultramarine-icon: #3a76f0; $color-ultramarine-light: #6191f3; $color-ultramarine-dawn: #406ec9; $color-ultramarine-pastel: #abc4f8; +$color-ultramarine-pale: #d2dffb; $color-ultramarine: #2c6bed; $color-link: #315ff4; diff --git a/stylesheets/components/BackupImportScreen.scss b/stylesheets/components/BackupImportScreen.scss index 1fe6dcbac8cb..d627c92b7772 100644 --- a/stylesheets/components/BackupImportScreen.scss +++ b/stylesheets/components/BackupImportScreen.scss @@ -19,30 +19,8 @@ margin-block: 0 20px; } -.BackupImportScreen__progressbar { - overflow: hidden; +.BackupImportScreen .ProgressBar { margin-block-end: 14px; - - background: rgba($color-ultramarine, 0.2); - height: 5px; - border-radius: 2px; -} - -.BackupImportScreen__progressbar__fill { - background-color: $color-ultramarine; - border-radius: 2px; - display: block; - height: 100%; - width: 100%; - &:dir(ltr) { - /* stylelint-disable-next-line declaration-property-value-disallowed-list */ - transform: translateX(-100%); - } - &:dir(rtl) { - /* stylelint-disable-next-line declaration-property-value-disallowed-list */ - transform: translateX(100%); - } - transition: transform 500ms ease-out; } .BackupImportScreen__progressbar-hint { diff --git a/stylesheets/components/BackupMediaDownloadProgress.scss b/stylesheets/components/BackupMediaDownloadProgress.scss new file mode 100644 index 000000000000..1752fd2b229b --- /dev/null +++ b/stylesheets/components/BackupMediaDownloadProgress.scss @@ -0,0 +1,96 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +.BackupMediaDownloadProgressBanner { + @include font-body-2; + + border-radius: 10px; + display: flex; + gap: 12px; + padding: 12px; + padding-inline-end: 16px; + margin-inline: 10px; + user-select: none; + + @include light-theme { + background-color: $color-white; + border: 1px solid $color-gray-20; + } + @include dark-theme { + background: $color-gray-75; + border: 1px solid $color-gray-60; + } +} + +.BackupMediaDownloadProgressBanner__icon { + background: rgba($color-ultramarine, 0.2); + width: 30px; + height: 30px; + padding: 6px; + border-radius: 50%; + @include dark-theme { + background: $color-gray-60; + } + &::after { + content: ''; + display: inline-block; + width: 18px; + height: 18px; + @include light-theme { + @include color-svg( + '../images/icons/v3/backup/backup-bold.svg', + $color-ultramarine + ); + } + @include dark-theme { + @include color-svg( + '../images/icons/v3/backup/backup-bold.svg', + $color-ultramarine-pale + ); + } + } +} + +.BackupMediaDownloadProgressBanner__content { + display: flex; + flex-direction: column; + flex: 1; + gap: 7px; +} + +.BackupMediaDownloadProgressBanner .Progressbar { + overflow: hidden; + + background: rgba($color-ultramarine, 0.2); + height: 5px; + border-radius: 2px; +} + +.BackupMediaDownloadProgressBanner__progressbar__fill { + background-color: $color-ultramarine; + border-radius: 2px; + display: block; + height: 100%; + width: 100%; + &:dir(ltr) { + /* stylelint-disable-next-line declaration-property-value-disallowed-list */ + transform: translateX(-100%); + } + &:dir(rtl) { + /* stylelint-disable-next-line declaration-property-value-disallowed-list */ + transform: translateX(100%); + } + transition: transform 500ms ease-out; +} + +.BackupMediaDownloadProgressBanner__progressbar-hint { + @include font-caption; + + @include light-theme { + color: rgba($color-gray-60, 0.8); + } + + @include dark-theme { + color: $color-gray-25; + } +} diff --git a/stylesheets/components/ProgressBar.scss b/stylesheets/components/ProgressBar.scss new file mode 100644 index 000000000000..a46b9dc6af11 --- /dev/null +++ b/stylesheets/components/ProgressBar.scss @@ -0,0 +1,17 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +.ProgressBar { + overflow: hidden; + background: rgba($color-ultramarine, 0.2); + height: 5px; + border-radius: 2px; +} + +.ProgressBar__fill { + background-color: $color-ultramarine; + border-radius: 2px; + display: block; + height: 100%; + transition: margin 500ms ease-out; +} diff --git a/stylesheets/manifest.scss b/stylesheets/manifest.scss index 7078f84dc225..5d8033b5a256 100644 --- a/stylesheets/manifest.scss +++ b/stylesheets/manifest.scss @@ -33,6 +33,7 @@ @import './components/AvatarPreview.scss'; @import './components/AvatarTextEditor.scss'; @import './components/BackupImportScreen.scss'; +@import './components/BackupMediaDownloadProgress.scss'; @import './components/BadgeCarouselIndex.scss'; @import './components/BadgeDialog.scss'; @import './components/BadgeSustainerInstructionsDialog.scss'; @@ -137,6 +138,7 @@ @import './components/PlaybackRateButton.scss'; @import './components/Preferences.scss'; @import './components/ProfileEditor.scss'; +@import './components/ProgressBar.scss'; @import './components/Quote.scss'; @import './components/ReactionPickerPicker.scss'; @import './components/RecordingComposer.scss'; diff --git a/ts/components/BackupImportScreen.tsx b/ts/components/BackupImportScreen.tsx index 13eef852ec39..25a4f1ccd039 100644 --- a/ts/components/BackupImportScreen.tsx +++ b/ts/components/BackupImportScreen.tsx @@ -7,6 +7,7 @@ import type { LocalizerType } from '../types/Util'; import { formatFileSize } from '../util/formatFileSize'; import { TitlebarDragArea } from './TitlebarDragArea'; import { InstallScreenSignalLogo } from './installScreen/InstallScreenSignalLogo'; +import { ProgressBar } from './ProgressBar'; // We can't always use destructuring assignment because of the complexity of this props // type. @@ -36,12 +37,7 @@ export function BackupImportScreen({ progress = ( <> -
-
-
+
{i18n('icu:BackupImportScreen__progressbar-hint', { currentSize: formatFileSize(currentBytes), @@ -54,7 +50,7 @@ export function BackupImportScreen({ } else { progress = ( <> -
+
{i18n('icu:BackupImportScreen__progressbar-hint', { currentSize: '', diff --git a/ts/components/BackupMediaDownloadProgress.stories.tsx b/ts/components/BackupMediaDownloadProgress.stories.tsx new file mode 100644 index 000000000000..8421849915f4 --- /dev/null +++ b/ts/components/BackupMediaDownloadProgress.stories.tsx @@ -0,0 +1,27 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import React, { type ComponentProps } from 'react'; +import type { Meta, StoryFn } from '@storybook/react'; +import { setupI18n } from '../util/setupI18n'; +import enMessages from '../../_locales/en/messages.json'; +import { BackupMediaDownloadProgressBanner } from './BackupMediaDownloadProgress'; + +const i18n = setupI18n('en', enMessages); + +type PropsType = ComponentProps; + +export default { + title: 'Components/BackupMediaDownloadProgress', +} satisfies Meta; + +// eslint-disable-next-line react/function-component-definition +const Template: StoryFn = (args: PropsType) => ( + +); + +export const InProgress = Template.bind({}); +InProgress.args = { + downloadedBytes: 92048023, + totalBytes: 1024102532, +}; diff --git a/ts/components/BackupMediaDownloadProgress.tsx b/ts/components/BackupMediaDownloadProgress.tsx new file mode 100644 index 000000000000..06fd9e26b71c --- /dev/null +++ b/ts/components/BackupMediaDownloadProgress.tsx @@ -0,0 +1,48 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import React from 'react'; + +import type { LocalizerType } from '../types/Util'; +import { formatFileSize } from '../util/formatFileSize'; +import { ProgressBar } from './ProgressBar'; + +export type PropsType = Readonly<{ + i18n: LocalizerType; + downloadedBytes: number; + totalBytes: number; +}>; + +export function BackupMediaDownloadProgressBanner({ + i18n, + downloadedBytes, + totalBytes, +}: PropsType): JSX.Element | null { + if (totalBytes === 0) { + return null; + } + + const fractionComplete = Math.max( + 0, + Math.min(1, downloadedBytes / totalBytes) + ); + + return ( +
+
+
+
+ {i18n('icu:BackupMediaDownloadProgress__title')} +
+ +
+ {i18n('icu:BackupMediaDownloadProgress__progressbar-hint', { + currentSize: formatFileSize(downloadedBytes), + totalSize: formatFileSize(totalBytes), + fractionComplete, + })} +
+
+
+ ); +} diff --git a/ts/components/LeftPane.stories.tsx b/ts/components/LeftPane.stories.tsx index 0b76a1067419..8d7f6f8e0b5b 100644 --- a/ts/components/LeftPane.stories.tsx +++ b/ts/components/LeftPane.stories.tsx @@ -137,6 +137,7 @@ const useProps = (overrideProps: OverridePropsType = {}): PropsType => { unreadMentionsCount: 0, markedUnread: false, }, + backupMediaDownloadProgress: { totalBytes: 0, downloadedBytes: 0 }, clearConversationSearch: action('clearConversationSearch'), clearGroupCreationError: action('clearGroupCreationError'), clearSearch: action('clearSearch'), diff --git a/ts/components/LeftPane.tsx b/ts/components/LeftPane.tsx index 2af0470a33f2..0f9ce5ce6ca1 100644 --- a/ts/components/LeftPane.tsx +++ b/ts/components/LeftPane.tsx @@ -1,7 +1,13 @@ // Copyright 2019 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only -import React, { useEffect, useCallback, useMemo, useRef } from 'react'; +import React, { + useEffect, + useCallback, + useMemo, + useRef, + useState, +} from 'react'; import classNames from 'classnames'; import { isNumber } from 'lodash'; @@ -56,8 +62,10 @@ import { import { ContextMenu } from './ContextMenu'; import { EditState as ProfileEditorEditState } from './ProfileEditor'; import type { UnreadStats } from '../util/countUnreadStats'; +import { BackupMediaDownloadProgressBanner } from './BackupMediaDownloadProgress'; export type PropsType = { + backupMediaDownloadProgress: { totalBytes: number; downloadedBytes: number }; otherTabsUnreadStats: UnreadStats; hasExpiredDialog: boolean; hasFailedStorySends: boolean; @@ -173,6 +181,7 @@ export type PropsType = { } & LookupConversationWithoutServiceIdActionsType; export function LeftPane({ + backupMediaDownloadProgress, otherTabsUnreadStats, blockConversation, challengeStatus, @@ -634,6 +643,34 @@ export function LeftPane({ dialogs.push({ key: 'banner', dialog: maybeBanner }); } + // We'll show the backup media download progress banner if the download is currently or + // was ongoing at some point during the lifecycle of this component + const [ + hasMediaBackupDownloadBeenOngoing, + setHasMediaBackupDownloadBeenOngoing, + ] = useState(false); + + const isMediaBackupDownloadOngoing = + backupMediaDownloadProgress?.totalBytes > 0 && + backupMediaDownloadProgress.downloadedBytes < + backupMediaDownloadProgress.totalBytes; + + if (isMediaBackupDownloadOngoing && !hasMediaBackupDownloadBeenOngoing) { + setHasMediaBackupDownloadBeenOngoing(true); + } + + if (hasMediaBackupDownloadBeenOngoing) { + dialogs.push({ + key: 'backupMediaDownload', + dialog: ( + + ), + }); + } + const hideHeader = modeSpecificProps.mode === LeftPaneMode.Archive || modeSpecificProps.mode === LeftPaneMode.Compose || diff --git a/ts/components/ProgressBar.tsx b/ts/components/ProgressBar.tsx new file mode 100644 index 000000000000..5c1e28c74d53 --- /dev/null +++ b/ts/components/ProgressBar.tsx @@ -0,0 +1,21 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import React from 'react'; + +export function ProgressBar({ + fractionComplete, +}: { + fractionComplete: number; +}): JSX.Element { + return ( +
+
+
+ ); +} diff --git a/ts/jobs/AttachmentDownloadManager.ts b/ts/jobs/AttachmentDownloadManager.ts index 44f9e5ef97da..dd98d0946618 100644 --- a/ts/jobs/AttachmentDownloadManager.ts +++ b/ts/jobs/AttachmentDownloadManager.ts @@ -42,6 +42,9 @@ import { isVideoTypeSupported, } from '../util/GoogleChrome'; import type { MIMEType } from '../types/MIME'; +import type { AttachmentDownloadSource } from '../sql/Interface'; +import { drop } from '../util/drop'; +import { getAttachmentCiphertextLength } from '../AttachmentCrypto'; export enum AttachmentDownloadUrgency { IMMEDIATE = 'immediate', @@ -55,6 +58,7 @@ export type NewAttachmentDownloadJobType = { receivedAt: number; sentAt: number; attachmentType: AttachmentDownloadJobTypeType; + source: AttachmentDownloadSource; urgency?: AttachmentDownloadUrgency; }; @@ -69,6 +73,10 @@ const DEFAULT_RETRY_CONFIG = { maxBackoffTime: 6 * durations.HOUR, }, }; +const BACKUP_RETRY_CONFIG = { + ...DEFAULT_RETRY_CONFIG, + maxAttempts: Infinity, +}; type AttachmentDownloadManagerParamsType = Omit< JobManagerParamsType, 'getNextJobs' | 'runJob' @@ -117,7 +125,10 @@ export class AttachmentDownloadManager extends JobManager DEFAULT_RETRY_CONFIG, + getRetryConfig: job => + job.attachment.backupLocator?.mediaName + ? BACKUP_RETRY_CONFIG + : DEFAULT_RETRY_CONFIG, maxConcurrentJobs: MAX_CONCURRENT_JOBS, }; @@ -157,6 +168,7 @@ export class AttachmentDownloadManager extends JobManager { if (!attachments?.every(isDownloaded)) { const updatedFields = await queueAttachmentDownloads( message.attributes, - AttachmentDownloadUrgency.STANDARD + { urgency: AttachmentDownloadUrgency.STANDARD } ); if (updatedFields) { message.set(updatedFields); diff --git a/ts/models/messages.ts b/ts/models/messages.ts index f006bc10ef70..51ce86ef0b0b 100644 --- a/ts/models/messages.ts +++ b/ts/models/messages.ts @@ -1344,7 +1344,7 @@ export class MessageModel extends window.Backbone.Model { async queueAttachmentDownloads( urgency?: AttachmentDownloadUrgency ): Promise { - const value = await queueAttachmentDownloads(this.attributes, urgency); + const value = await queueAttachmentDownloads(this.attributes, { urgency }); if (!value) { return false; } diff --git a/ts/services/backups/import.ts b/ts/services/backups/import.ts index 7eff273a2177..64efd5c165ff 100644 --- a/ts/services/backups/import.ts +++ b/ts/services/backups/import.ts @@ -11,7 +11,10 @@ import { CallLinkRootKey } from '@signalapp/ringrtc'; import { Backups, SignalService } from '../../protobuf'; import { DataWriter } from '../../sql/Client'; -import type { StoryDistributionWithMembersType } from '../../sql/Interface'; +import { + AttachmentDownloadSource, + type StoryDistributionWithMembersType, +} from '../../sql/Interface'; import * as log from '../../logging/log'; import { GiftBadgeStates } from '../../components/conversation/Message'; import { StorySendMode, MY_STORY_ID } from '../../types/Stories'; @@ -169,7 +172,11 @@ async function processMessagesBatch( ); } - drop(queueAttachmentDownloads(attributes)); + drop( + queueAttachmentDownloads(attributes, { + source: AttachmentDownloadSource.BACKUP_IMPORT, + }) + ); } } diff --git a/ts/services/backups/index.ts b/ts/services/backups/index.ts index 3b7774c6e7d9..831308fce01a 100644 --- a/ts/services/backups/index.ts +++ b/ts/services/backups/index.ts @@ -40,6 +40,7 @@ import { BackupCredentials } from './credentials'; import { BackupAPI, type DownloadOptionsType } from './api'; import { validateBackup } from './validator'; import { getParametersForRedux, loadAll } from '../allLoaders'; +import { AttachmentDownloadManager } from '../../jobs/AttachmentDownloadManager'; const IV_LENGTH = 16; @@ -236,6 +237,8 @@ export class BackupsService { // Second pass - decrypt (but still check the mac at the end) hmac = createHmac(HashType.size256, macKey); + await this.prepareForImport(); + await pipeline( createBackupStream(), getMacAndUpdateHmac(hmac, noop), @@ -278,11 +281,25 @@ export class BackupsService { } } + public async prepareForImport(): Promise { + await AttachmentDownloadManager.stop(); + await DataWriter.removeAllBackupAttachmentDownloadJobs(); + await window.storage.put('backupAttachmentsSuccessfullyDownloadedSize', 0); + await window.storage.put('backupAttachmentsTotalSizeToDownload', 0); + } + public async resetStateAfterImport(): Promise { window.ConversationController.reset(); await window.ConversationController.load(); await loadAll(); reinitializeRedux(getParametersForRedux()); + + await window.storage.put( + 'backupAttachmentsTotalSizeToDownload', + await DataReader.getSizeOfPendingBackupAttachmentDownloadJobs() + ); + + await AttachmentDownloadManager.start(); } public async fetchAndSaveBackupCdnObjectMetadata(): Promise { diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 326801b75faf..8e572bf3f3df 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -446,6 +446,11 @@ export type GetRecentStoryRepliesOptionsType = { sentAt?: number; }; +export enum AttachmentDownloadSource { + BACKUP_IMPORT = 'backup_import', + STANDARD = 'standard', +} + type ReadableInterface = { close: () => void; @@ -642,6 +647,7 @@ type ReadableInterface = { getMaxMessageCounter(): number | undefined; getStatisticsForLogging(): Record; + getSizeOfPendingBackupAttachmentDownloadJobs(): number; }; type WritableInterface = { @@ -840,6 +846,7 @@ type WritableInterface = { saveAttachmentDownloadJob: (job: AttachmentDownloadJobType) => void; resetAttachmentDownloadActive: () => void; removeAttachmentDownloadJob: (job: AttachmentDownloadJobType) => void; + removeAllBackupAttachmentDownloadJobs: () => void; getNextAttachmentBackupJobs: (options: { limit: number; diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 27269f48970b..cc77d083abf1 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -145,6 +145,7 @@ import type { StoredKyberPreKeyType, BackupCdnMediaObjectType, } from './Interface'; +import { AttachmentDownloadSource } from './Interface'; import { SeenStatus } from '../MessageSeenStatus'; import { SNIPPET_LEFT_PLACEHOLDER, @@ -200,6 +201,7 @@ import { attachmentBackupJobSchema, } from '../types/AttachmentBackup'; import { redactGenericText } from '../util/privacy'; +import { getAttachmentCiphertextLength } from '../AttachmentCrypto'; type ConversationRow = Readonly<{ json: string; @@ -340,6 +342,7 @@ export const DataReader: ServerReadableInterface = { getStatisticsForLogging, getBackupCdnObjectMetadata, + getSizeOfPendingBackupAttachmentDownloadJobs, // Server-only getKnownMessageAttachments, @@ -463,6 +466,7 @@ export const DataWriter: ServerWritableInterface = { saveAttachmentDownloadJob, resetAttachmentDownloadActive, removeAttachmentDownloadJob, + removeAllBackupAttachmentDownloadJobs, getNextAttachmentBackupJobs, saveAttachmentBackupJob, @@ -4733,6 +4737,20 @@ function getAttachmentDownloadJob( return db.prepare(query).get(params); } +function removeAllBackupAttachmentDownloadJobs(db: WritableDB): void { + const [query, params] = sql` + DELETE FROM attachment_downloads + WHERE source = ${AttachmentDownloadSource.BACKUP_IMPORT};`; + db.prepare(query).run(params); +} + +function getSizeOfPendingBackupAttachmentDownloadJobs(db: ReadableDB): number { + const [query, params] = sql` + SELECT SUM(ciphertextSize) FROM attachment_downloads + WHERE source = ${AttachmentDownloadSource.BACKUP_IMPORT};`; + return db.prepare(query).pluck().get(params); +} + function getNextAttachmentDownloadJobs( db: WritableDB, { @@ -4799,6 +4817,9 @@ function getNextAttachmentDownloadJobs( ...row, active: Boolean(row.active), attachment: jsonToObject(row.attachmentJson), + ciphertextSize: + row.ciphertextSize || + getAttachmentCiphertextLength(row.attachment.size), }); } catch (error) { logger.error( @@ -4839,7 +4860,9 @@ function saveAttachmentDownloadJob( attempts, retryAfter, lastAttemptTimestamp, - attachmentJson + attachmentJson, + ciphertextSize, + source ) VALUES ( ${job.messageId}, ${job.attachmentType}, @@ -4852,7 +4875,9 @@ function saveAttachmentDownloadJob( ${job.attempts}, ${job.retryAfter}, ${job.lastAttemptTimestamp}, - ${objectToJSON(job.attachment)} + ${objectToJSON(job.attachment)}, + ${job.ciphertextSize}, + ${job.source} ); `; db.prepare(query).run(params); diff --git a/ts/sql/migrations/1040-undownloaded-backed-up-media.ts b/ts/sql/migrations/1040-undownloaded-backed-up-media.ts index 5bad63672d8e..d8062bc7de16 100644 --- a/ts/sql/migrations/1040-undownloaded-backed-up-media.ts +++ b/ts/sql/migrations/1040-undownloaded-backed-up-media.ts @@ -11,6 +11,7 @@ import { } from '../../types/AttachmentDownload'; import type { AttachmentType } from '../../types/Attachment'; import { jsonToObject, objectToJSON, sql } from '../util'; +import { AttachmentDownloadSource } from '../Interface'; export const version = 1040; @@ -133,6 +134,9 @@ export function updateToSchemaVersion1040( attempts: existingJobData.attempts ?? 0, retryAfter: null, lastAttemptTimestamp: null, + // adding due to changes in the schema + source: AttachmentDownloadSource.STANDARD, + ciphertextSize: 0, }; const parsed = attachmentDownloadJobSchema.parse(updatedJob); diff --git a/ts/sql/migrations/1180-add-attachment-download-source.ts b/ts/sql/migrations/1180-add-attachment-download-source.ts new file mode 100644 index 000000000000..42f593bcdd80 --- /dev/null +++ b/ts/sql/migrations/1180-add-attachment-download-source.ts @@ -0,0 +1,37 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only +import type { Database } from '@signalapp/better-sqlite3'; +import type { LoggerType } from '../../types/Logging'; +import { AttachmentDownloadSource } from '../Interface'; + +export const version = 1180; +export function updateToSchemaVersion1180( + currentVersion: number, + db: Database, + logger: LoggerType +): void { + if (currentVersion >= 1180) { + return; + } + + db.transaction(() => { + db.exec(` + ALTER TABLE attachment_downloads + ADD COLUMN source TEXT NOT NULL DEFAULT ${AttachmentDownloadSource.STANDARD}; + + ALTER TABLE attachment_downloads + -- this default value will be overridden by getNextAttachmentDownloadJobs + ADD COLUMN ciphertextSize INTEGER NOT NULL DEFAULT 0; + `); + + db.exec(` + CREATE INDEX attachment_downloads_source_ciphertextSize + ON attachment_downloads ( + source, ciphertextSize + ); + `); + + db.pragma('user_version = 1180'); + })(); + logger.info('updateToSchemaVersion1180: success!'); +} diff --git a/ts/sql/migrations/index.ts b/ts/sql/migrations/index.ts index 41de8a230345..cd3506bf71fc 100644 --- a/ts/sql/migrations/index.ts +++ b/ts/sql/migrations/index.ts @@ -93,10 +93,11 @@ import { updateToSchemaVersion1130 } from './1130-isStory-index'; import { updateToSchemaVersion1140 } from './1140-call-links-deleted-column'; import { updateToSchemaVersion1150 } from './1150-expire-timer-version'; import { updateToSchemaVersion1160 } from './1160-optimize-calls-unread-count'; +import { updateToSchemaVersion1170 } from './1170-update-call-history-unread-index'; import { - updateToSchemaVersion1170, + updateToSchemaVersion1180, version as MAX_VERSION, -} from './1170-update-call-history-unread-index'; +} from './1180-add-attachment-download-source'; function updateToSchemaVersion1( currentVersion: number, @@ -2058,6 +2059,7 @@ export const SCHEMA_VERSIONS = [ updateToSchemaVersion1150, updateToSchemaVersion1160, updateToSchemaVersion1170, + updateToSchemaVersion1180, ]; export class DBVersionFromFutureError extends Error { diff --git a/ts/state/selectors/items.ts b/ts/state/selectors/items.ts index a3a001a6ea84..922fe6f72989 100644 --- a/ts/state/selectors/items.ts +++ b/ts/state/selectors/items.ts @@ -248,3 +248,11 @@ export const getLocalDeleteWarningShown = createSelector( (state: ItemsStateType): boolean => Boolean(state.localDeleteWarningShown ?? false) ); + +export const getBackupMediaDownloadProgress = createSelector( + getItems, + (state: ItemsStateType): { totalBytes: number; downloadedBytes: number } => ({ + totalBytes: state.backupAttachmentsTotalSizeToDownload ?? 0, + downloadedBytes: state.backupAttachmentsSuccessfullyDownloadedSize ?? 0, + }) +); diff --git a/ts/state/smart/LeftPane.tsx b/ts/state/smart/LeftPane.tsx index 17afc136e6e2..ea51a99753c1 100644 --- a/ts/state/smart/LeftPane.tsx +++ b/ts/state/smart/LeftPane.tsx @@ -58,6 +58,7 @@ import { import { getCrashReportCount } from '../selectors/crashReports'; import { hasExpired } from '../selectors/expiration'; import { + getBackupMediaDownloadProgress, getNavTabsCollapsed, getPreferredLeftPaneWidth, getUsernameCorrupted, @@ -115,6 +116,7 @@ function renderUpdateDialog( ): JSX.Element { return ; } + function renderCaptchaDialog({ onSkip }: { onSkip(): void }): JSX.Element { return ; } @@ -289,7 +291,9 @@ export const SmartLeftPane = memo(function SmartLeftPane({ const theme = useSelector(getTheme); const usernameCorrupted = useSelector(getUsernameCorrupted); const usernameLinkCorrupted = useSelector(getUsernameLinkCorrupted); - + const backupMediaDownloadProgress = useSelector( + getBackupMediaDownloadProgress + ); const { blockConversation, clearGroupCreationError, @@ -360,6 +364,7 @@ export const SmartLeftPane = memo(function SmartLeftPane({ return ( { } ); await downloadManager?.addJob({ - ...job, urgency, + ...job, }); } async function addJobs( @@ -419,7 +423,7 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => { dependencies: { downloadAttachment }, }); - assert.strictEqual(result.onlyAttemptedBackupThumbnail, false); + assert.strictEqual(result.downloadedVariant, AttachmentVariant.Default); assert.strictEqual(downloadAttachment.callCount, 1); assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], { attachment: job.attachment, @@ -444,8 +448,8 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => { }); strictAssert( - result.onlyAttemptedBackupThumbnail === true, - 'only attempted backup thumbnail' + result.downloadedVariant === AttachmentVariant.ThumbnailFromBackup, + 'downloaded thumbnail' ); assert.deepStrictEqual( omit(result.attachmentWithThumbnail, 'thumbnailFromBackup'), @@ -485,7 +489,7 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => { isForCurrentlyVisibleMessage: true, dependencies: { downloadAttachment }, }); - assert.strictEqual(result.onlyAttemptedBackupThumbnail, false); + assert.strictEqual(result.downloadedVariant, AttachmentVariant.Default); assert.strictEqual(downloadAttachment.callCount, 1); assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], { attachment: job.attachment, @@ -544,7 +548,7 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => { isForCurrentlyVisibleMessage: false, dependencies: { downloadAttachment }, }); - assert.strictEqual(result.onlyAttemptedBackupThumbnail, false); + assert.strictEqual(result.downloadedVariant, AttachmentVariant.Default); assert.strictEqual(downloadAttachment.callCount, 1); assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], { attachment: job.attachment, @@ -578,7 +582,10 @@ describe('AttachmentDownloadManager/runDownloadAttachmentJob', () => { isForCurrentlyVisibleMessage: false, dependencies: { downloadAttachment }, }); - assert.strictEqual(result.onlyAttemptedBackupThumbnail, false); + assert.strictEqual( + result.downloadedVariant, + AttachmentVariant.ThumbnailFromBackup + ); assert.strictEqual(downloadAttachment.callCount, 2); assert.deepStrictEqual(downloadAttachment.getCall(0).args[0], { attachment: job.attachment, diff --git a/ts/test-node/sql/migration_1040_test.ts b/ts/test-node/sql/migration_1040_test.ts index 7d207f0b759c..d8137109887f 100644 --- a/ts/test-node/sql/migration_1040_test.ts +++ b/ts/test-node/sql/migration_1040_test.ts @@ -27,7 +27,7 @@ function getAttachmentDownloadJobs(db: ReadableDB) { type UnflattenedAttachmentDownloadJobType = Omit< AttachmentDownloadJobType, - 'digest' | 'contentType' | 'size' + 'digest' | 'contentType' | 'size' | 'source' | 'ciphertextSize' >; function insertNewJob( db: WritableDB, @@ -304,24 +304,25 @@ describe('SQL/updateToSchemaVersion1040', () => { }); it('respects foreign key constraint on messageId', () => { - const job: AttachmentDownloadJobType = { - messageId: 'message1', - attachmentType: 'attachment', - attachment: { + const job: Omit = + { + messageId: 'message1', + attachmentType: 'attachment', + attachment: { + digest: 'digest1', + contentType: IMAGE_JPEG, + size: 128, + }, + receivedAt: 1970, digest: 'digest1', contentType: IMAGE_JPEG, size: 128, - }, - receivedAt: 1970, - digest: 'digest1', - contentType: IMAGE_JPEG, - size: 128, - sentAt: 2070, - active: false, - retryAfter: null, - attempts: 0, - lastAttemptTimestamp: null, - }; + sentAt: 2070, + active: false, + retryAfter: null, + attempts: 0, + lastAttemptTimestamp: null, + }; // throws if we don't add the message first assert.throws(() => insertNewJob(db, job, false)); insertNewJob(db, job, true); diff --git a/ts/test-node/sql/migration_1180_test.ts b/ts/test-node/sql/migration_1180_test.ts new file mode 100644 index 000000000000..cb4bc5608e9c --- /dev/null +++ b/ts/test-node/sql/migration_1180_test.ts @@ -0,0 +1,147 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { assert } from 'chai'; +import { omit } from 'lodash'; +import type { WritableDB } from '../../sql/Interface'; +import { createDB, updateToVersion } from './helpers'; +import type { AttachmentDownloadJobType } from '../../types/AttachmentDownload'; +import { jsonToObject, objectToJSON, sql } from '../../sql/util'; +import { IMAGE_BMP } from '../../types/MIME'; + +function insertOldJob( + db: WritableDB, + job: Omit, + addMessageFirst: boolean = true +): void { + if (addMessageFirst) { + try { + db.prepare('INSERT INTO messages (id) VALUES ($id)').run({ + id: job.messageId, + }); + } catch (e) { + // pass; message has already been inserted + } + } + const [query, params] = sql` + INSERT INTO attachment_downloads + ( + messageId, + attachmentType, + attachmentJson, + digest, + contentType, + size, + receivedAt, + sentAt, + active, + attempts, + retryAfter, + lastAttemptTimestamp + ) + VALUES + ( + ${job.messageId}, + ${job.attachmentType}, + ${objectToJSON(job.attachment)}, + ${job.digest}, + ${job.contentType}, + ${job.size}, + ${job.receivedAt}, + ${job.sentAt}, + ${job.active ? 1 : 0}, + ${job.attempts}, + ${job.retryAfter}, + ${job.lastAttemptTimestamp} + ); + `; + + db.prepare(query).run(params); +} + +function getAttachmentDownloadJobs(db: WritableDB) { + const [query] = sql` + SELECT * FROM attachment_downloads ORDER BY receivedAt DESC; + `; + + return db + .prepare(query) + .all() + .map(job => ({ + ...omit(job, 'attachmentJson'), + active: job.active === 1, + attachment: jsonToObject(job.attachmentJson), + })); +} + +describe('SQL/updateToSchemaVersion1180', () => { + let db: WritableDB; + beforeEach(() => { + db = createDB(); + updateToVersion(db, 1170); + }); + + afterEach(() => { + db.close(); + }); + + it('adds source column with default standard to any existing jobs', async () => { + const job: Omit = { + messageId: '123', + digest: 'digest', + attachmentType: 'attachment', + attachment: { size: 128, contentType: IMAGE_BMP }, + size: 128, + contentType: IMAGE_BMP, + receivedAt: 120, + sentAt: 120, + active: false, + attempts: 0, + retryAfter: null, + lastAttemptTimestamp: null, + }; + insertOldJob(db, job); + updateToVersion(db, 1180); + assert.deepEqual(getAttachmentDownloadJobs(db), [ + { ...job, source: 'standard', ciphertextSize: 0 }, + ]); + }); + it('uses convering index for summing all pending backup jobs', async () => { + updateToVersion(db, 1180); + const details = db + .prepare( + ` + EXPLAIN QUERY PLAN + SELECT SUM(ciphertextSize) FROM attachment_downloads + WHERE source = 'backup_import'; + ` + ) + .all() + .map(step => step.detail) + .join(', '); + + assert.strictEqual( + details, + 'SEARCH attachment_downloads USING COVERING INDEX attachment_downloads_source_ciphertextSize (source=?)' + ); + }); + it('uses index for deleting all backup jobs', async () => { + updateToVersion(db, 1180); + const details = db + .prepare( + ` + EXPLAIN QUERY PLAN + DELETE FROM attachment_downloads + WHERE source = 'backup_import'; + ` + ) + .all() + .map(step => step.detail) + .join(', '); + + assert.strictEqual( + details, + 'SEARCH attachment_downloads USING COVERING INDEX attachment_downloads_source_ciphertextSize (source=?)' + ); + }); +}); diff --git a/ts/types/AttachmentDownload.ts b/ts/types/AttachmentDownload.ts index a0906454f2a2..242f9089cec3 100644 --- a/ts/types/AttachmentDownload.ts +++ b/ts/types/AttachmentDownload.ts @@ -7,6 +7,7 @@ import { type JobManagerJobType, jobManagerJobSchema, } from '../jobs/JobManager'; +import { AttachmentDownloadSource } from '../sql/Interface'; export enum MediaTier { STANDARD = 'standard', @@ -35,6 +36,8 @@ export type CoreAttachmentDownloadJobType = { digest: string; contentType: MIMEType; size: number; + ciphertextSize: number; + source: AttachmentDownloadSource; }; export type AttachmentDownloadJobType = CoreAttachmentDownloadJobType & @@ -51,7 +54,9 @@ export const coreAttachmentDownloadJobSchema = z.object({ digest: z.string(), contentType: MIMETypeSchema, size: z.number(), + ciphertextSize: z.number(), messageIdForLogging: z.string().optional(), + source: z.nativeEnum(AttachmentDownloadSource), }); export const attachmentDownloadJobSchema = coreAttachmentDownloadJobSchema.and( diff --git a/ts/types/Storage.d.ts b/ts/types/Storage.d.ts index 4bc1c9cc2037..44544eba02ee 100644 --- a/ts/types/Storage.d.ts +++ b/ts/types/Storage.d.ts @@ -141,6 +141,8 @@ export type StorageAccessType = { callLinkAuthCredentials: ReadonlyArray; backupCredentials: ReadonlyArray; backupCredentialsLastRequestTime: number; + backupAttachmentsSuccessfullyDownloadedSize: number; + backupAttachmentsTotalSizeToDownload: number; setBackupSignatureKey: boolean; lastReceivedAtCounter: number; preferredReactionEmoji: ReadonlyArray; diff --git a/ts/util/queueAttachmentDownloads.ts b/ts/util/queueAttachmentDownloads.ts index 462f91ea9770..7e327ffdf5d8 100644 --- a/ts/util/queueAttachmentDownloads.ts +++ b/ts/util/queueAttachmentDownloads.ts @@ -32,6 +32,7 @@ import { AttachmentDownloadManager, AttachmentDownloadUrgency, } from '../jobs/AttachmentDownloadManager'; +import { AttachmentDownloadSource } from '../sql/Interface'; export type MessageAttachmentsDownloadedType = { bodyAttachment?: AttachmentType; @@ -62,7 +63,13 @@ function getAttachmentSignatureSafe( // count then you'll also have to modify ./hasAttachmentsDownloads export async function queueAttachmentDownloads( message: MessageAttributesType, - urgency: AttachmentDownloadUrgency = AttachmentDownloadUrgency.STANDARD + { + urgency = AttachmentDownloadUrgency.STANDARD, + source = AttachmentDownloadSource.STANDARD, + }: { + urgency?: AttachmentDownloadUrgency; + source?: AttachmentDownloadSource; + } = {} ): Promise { const attachmentsToQueue = message.attachments || []; const messageId = message.id; @@ -109,6 +116,7 @@ export async function queueAttachmentDownloads( receivedAt: message.received_at, sentAt: message.sent_at, urgency, + source, }); } @@ -126,6 +134,7 @@ export async function queueAttachmentDownloads( receivedAt: message.received_at, sentAt: message.sent_at, urgency, + source, } ); count += attachmentsCount; @@ -144,6 +153,7 @@ export async function queueAttachmentDownloads( receivedAt: message.received_at, sentAt: message.sent_at, urgency, + source, }); count += previewCount; @@ -162,6 +172,7 @@ export async function queueAttachmentDownloads( receivedAt: message.received_at, sentAt: message.sent_at, urgency, + source, }); count += thumbnailCount; @@ -194,6 +205,7 @@ export async function queueAttachmentDownloads( receivedAt: message.received_at, sentAt: message.sent_at, urgency, + source, }), }, }; @@ -230,6 +242,7 @@ export async function queueAttachmentDownloads( receivedAt: message.received_at, sentAt: message.sent_at, urgency, + source, }); } else { log.error(`${idLog}: Sticker data was missing`); @@ -267,6 +280,7 @@ export async function queueAttachmentDownloads( receivedAt: message.received_at, sentAt: message.sent_at, urgency, + source, }); count += editAttachmentsCount; if (editAttachmentsCount !== 0) { @@ -285,6 +299,7 @@ export async function queueAttachmentDownloads( receivedAt: message.received_at, sentAt: message.sent_at, urgency, + source, }); count += editPreviewCount; if (editPreviewCount !== 0) { @@ -328,6 +343,7 @@ async function queueNormalAttachments({ receivedAt, sentAt, urgency, + source, }: { idLog: string; messageId: string; @@ -336,6 +352,7 @@ async function queueNormalAttachments({ receivedAt: number; sentAt: number; urgency: AttachmentDownloadUrgency; + source: AttachmentDownloadSource; }): Promise<{ attachments: Array; count: number; @@ -393,6 +410,7 @@ async function queueNormalAttachments({ receivedAt, sentAt, urgency, + source, }); }) ); @@ -426,6 +444,7 @@ async function queuePreviews({ receivedAt, sentAt, urgency, + source, }: { idLog: string; messageId: string; @@ -434,6 +453,7 @@ async function queuePreviews({ receivedAt: number; sentAt: number; urgency: AttachmentDownloadUrgency; + source: AttachmentDownloadSource; }): Promise<{ preview: Array; count: number }> { // Similar to queueNormalAttachments' logic for detecting same attachments // except here we also pick by link preview URL. @@ -485,6 +505,7 @@ async function queuePreviews({ receivedAt, sentAt, urgency, + source, }), }; }) @@ -518,6 +539,7 @@ async function queueQuoteAttachments({ receivedAt, sentAt, urgency, + source, }: { idLog: string; messageId: string; @@ -526,6 +548,7 @@ async function queueQuoteAttachments({ receivedAt: number; sentAt: number; urgency: AttachmentDownloadUrgency; + source: AttachmentDownloadSource; }): Promise<{ quote?: QuotedMessageType; count: number }> { let count = 0; if (!quote) { @@ -600,6 +623,7 @@ async function queueQuoteAttachments({ receivedAt, sentAt, urgency, + source, }), }; })