mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions 1

Basic support for local encrypted backups

Browse source
This commit is contained in:
ayumi-signal 2025-05-12 14:15:11 -07:00 • committed by GitHub
parent 2df601b135
commit a2c74c3a8b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
37 changed files with 1782 additions and 176 deletions
Download patch file Download diff file Expand all files Collapse all files

20
_locales/en/messages.json
View file

@ -6652,6 +6652,26 @@
"messageformat": "Internal",
"description": "Button to switch the settings view to control internal configuration"
},
"icu:Preferences__internal__local-backups": {
"messageformat": "Local backups",
"description": "Text header for internal local backup tools"
},
"icu:Preferences__internal__export-local-backup": {
"messageformat": "Export…",
"description": "Button to run internal local backup export tool"
},
"icu:Preferences__internal__export-local-backup--description": {
"messageformat": "Export local encrypted backup to a folder and validate it",
"description": "Description of the internal local backup export tool"
},
"icu:Preferences__internal__import-local-backup": {
"messageformat": "Import…",
"description": "Button to run internal local backup import tool"
},
"icu:Preferences__internal__import-local-backup--description": {
"messageformat": "Stage a local encrypted backup for import on link",
"description": "Description of the internal local backup export tool"
},
"icu:Preferences__internal__validate-backup--description": {
"messageformat": "Export encrypted backup to memory and run validation suite on it",
"description": "Description of the internal backup validation tool"

63
app/main.ts
View file

@ -3110,31 +3110,50 @@ ipc.handle('show-save-dialog', async (_event, { defaultPath }) => {
return { canceled: false, filePath: finalFilePath };
});
ipc.handle('show-save-multi-dialog', async _event => {
if (!mainWindow) {
getLogger().warn('show-save-multi-dialog: no main window');
ipc.handle(
'show-open-folder-dialog',
async (
_event,
{ useMainWindow }: { useMainWindow: boolean } = { useMainWindow: false }
) => {
let canceled: boolean;
let selectedDirPaths: ReadonlyArray<string>;
return { canceled: true };
}
const { canceled, filePaths: selectedDirPaths } = await dialog.showOpenDialog(
mainWindow,
{
defaultPath: app.getPath('downloads'),
properties: ['openDirectory', 'createDirectory'],
if (useMainWindow) {
if (!mainWindow) {
getLogger().warn('show-open-folder-dialog: no main window');
return { canceled: true };
}
({ canceled, filePaths: selectedDirPaths } = await dialog.showOpenDialog(
mainWindow,
{
defaultPath: app.getPath('downloads'),
properties: ['openDirectory', 'createDirectory'],
}
));
} else {
({ canceled, filePaths: selectedDirPaths } = await dialog.showOpenDialog({
defaultPath: app.getPath('downloads'),
properties: ['openDirectory', 'createDirectory'],
}));
}
);
if (canceled || selectedDirPaths.length === 0) {
return { canceled: true };
if (canceled || selectedDirPaths.length === 0) {
return { canceled: true };
}
if (selectedDirPaths.length > 1) {
getLogger().warn(
'show-open-folder-dialog: multiple directories selected'
);
return { canceled: true };
}
return { canceled: false, dirPath: selectedDirPaths[0] };
}
if (selectedDirPaths.length > 1) {
getLogger().warn('show-save-multi-dialog: multiple directories selected');
return { canceled: true };
}
return { canceled: false, dirPath: selectedDirPaths[0] };
});
);
ipc.handle('executeMenuRole', async ({ sender }, untypedRole) => {
const role = untypedRole as MenuItemConstructorOptions['role'];

19
protos/Backups.proto
View file

@ -725,11 +725,30 @@ message FilePointer {
message InvalidAttachmentLocator {
}
// References attachments in a local encrypted backup.
// Importers should first attempt to read the file from the local backup,
// and on failure fallback to backup and transit cdn if possible.
message LocalLocator {
string mediaName = 1;
// Separate key used to encrypt this file for the local backup.
// Generally required. Missing field indicates attachment was not
// available locally when the backup was generated, but remote
// backup or transit info was available.
optional bytes localKey = 2;
bytes remoteKey = 3;
bytes remoteDigest = 4;
uint32 size = 5;
optional uint32 backupCdnNumber = 6;
optional string transitCdnKey = 7;
optional uint32 transitCdnNumber = 8;
}
// If unset, importers should consider it to be an InvalidAttachmentLocator without throwing an error.
oneof locator {
BackupLocator backupLocator = 1;
AttachmentLocator attachmentLocator = 2;
InvalidAttachmentLocator invalidAttachmentLocator = 3;
LocalLocator localLocator = 12;
}
optional string contentType = 4;

24
protos/LocalBackup.proto Normal file
View file

@ -0,0 +1,24 @@
syntax = "proto3";
package signal.backup.local;
option java_package = "org.thoughtcrime.securesms.backup.v2.local.proto";
option swift_prefix = "LocalBackupProto_";
message Metadata {
message EncryptedBackupId {
bytes iv = 1; // 12 bytes, randomly generated
bytes encryptedId = 2; // AES-256-CTR, key = local backup metadata key, message = backup ID bytes
// local backup metadata key = hkdf(input: K_B, info: UTF8("20241011_SIGNAL_LOCAL_BACKUP_METADATA_KEY"), length: 32)
// No hash of the ID; if it's decrypted incorrectly, the main backup will fail to decrypt anyway.
}
uint32 version = 1;
EncryptedBackupId backupId = 2; // used to decrypt the backup file knowing only the Account Entropy Pool
}
message FilesFrame {
oneof item {
string mediaName = 1;
}
}

18
ts/CI.ts
View file

@ -43,6 +43,8 @@ export type CIType = {
) => unknown;
openSignalRoute(url: string): Promise<void>;
migrateAllMessages(): Promise<void>;
exportLocalBackup(backupsBaseDir: string): Promise<string>;
stageLocalBackupForImport(snapshotDir: string): Promise<void>;
uploadBackup(): Promise<void>;
unlink: () => void;
print: (...args: ReadonlyArray<unknown>) => void;
@ -193,6 +195,20 @@ export function getCI({
document.body.removeChild(a);
}
async function exportLocalBackup(backupsBaseDir: string): Promise<string> {
const { snapshotDir } =
await backupsService.exportLocalBackup(backupsBaseDir);
return snapshotDir;
}
async function stageLocalBackupForImport(snapshotDir: string): Promise<void> {
const { error } =
await backupsService.stageLocalBackupForImport(snapshotDir);
if (error) {
throw error;
}
}
async function uploadBackup() {
await backupsService.upload();
await AttachmentBackupManager.waitForIdle();
@ -237,6 +253,8 @@ export function getCI({
waitForEvent,
openSignalRoute,
migrateAllMessages,
exportLocalBackup,
stageLocalBackupForImport,
uploadBackup,
unlink,
getPendingEventCount,

32
ts/background.ts
View file

@ -214,6 +214,7 @@ import { MessageModel } from './models/messages';
import { waitForEvent } from './shims/events';
import { sendSyncRequests } from './textsecure/syncRequests';
import { handleServerAlerts } from './util/handleServerAlerts';
import { isLocalBackupsEnabled } from './util/isLocalBackupsEnabled';
export function isOverHourIntoPast(timestamp: number): boolean {
return isNumber(timestamp) && isOlderThan(timestamp, HOUR);
@ -1758,7 +1759,7 @@ export async function startApp(): Promise<void> {
hasSentSyncRequests = true;
}
// 4. Download (or resume download) of link & sync backup
// 4. Download (or resume download) of link & sync backup or local backup
const { wasBackupImported } = await maybeDownloadAndImportBackup();
log.info(logId, {
wasBackupImported,
@ -1836,20 +1837,29 @@ export async function startApp(): Promise<void> {
wasBackupImported: boolean;
}> {
const backupDownloadPath = window.storage.get('backupDownloadPath');
if (backupDownloadPath) {
const isLocalBackupAvailable =
backupsService.isLocalBackupStaged() && isLocalBackupsEnabled();
if (isLocalBackupAvailable || backupDownloadPath) {
tapToViewMessagesDeletionService.pause();
// Download backup before enabling request handler and storage service
try {
const { wasBackupImported } = await backupsService.downloadAndImport({
onProgress: (backupStep, currentBytes, totalBytes) => {
window.reduxActions.installer.updateBackupImportProgress({
backupStep,
currentBytes,
totalBytes,
});
},
});
let wasBackupImported = false;
if (isLocalBackupAvailable) {
await backupsService.importLocalBackup();
wasBackupImported = true;
} else {
({ wasBackupImported } = await backupsService.downloadAndImport({
onProgress: (backupStep, currentBytes, totalBytes) => {
window.reduxActions.installer.updateBackupImportProgress({
backupStep,
currentBytes,
totalBytes,
});
},
}));
}
log.info('afterAppStart: backup download attempt completed, resolving');
backupReady.resolve({ wasBackupImported });

51
ts/components/Preferences.stories.tsx
View file

@ -43,6 +43,28 @@ const availableSpeakers = [
},
];
const validateBackupResult = {
totalBytes: 100,
duration: 10000,
stats: {
adHocCalls: 1,
callLinks: 2,
conversations: 3,
chats: 4,
distributionLists: 5,
messages: 6,
notificationProfiles: 2,
skippedMessages: 7,
stickerPacks: 8,
fixedDirectMessages: 9,
},
};
const exportLocalBackupResult = {
...validateBackupResult,
snapshotDir: '/home/signaluser/SignalBackups/signal-backup-1745618069169',
};
export default {
title: 'Components/Preferences',
component: Preferences,
@ -138,6 +160,18 @@ export default {
doDeleteAllData: action('doDeleteAllData'),
doneRendering: action('doneRendering'),
editCustomColor: action('editCustomColor'),
exportLocalBackup: async () => {
return {
result: exportLocalBackupResult,
};
},
importLocalBackup: async () => {
return {
success: true,
error: undefined,
snapshotDir: exportLocalBackupResult.snapshotDir,
};
},
makeSyncRequest: action('makeSyncRequest'),
onAudioNotificationsChange: action('onAudioNotificationsChange'),
onAutoConvertEmojiChange: action('onAutoConvertEmojiChange'),
@ -192,22 +226,7 @@ export default {
),
validateBackup: async () => {
return {
result: {
totalBytes: 100,
duration: 10000,
stats: {
adHocCalls: 1,
callLinks: 2,
conversations: 3,
chats: 4,
distributionLists: 5,
messages: 6,
notificationProfiles: 2,
skippedMessages: 7,
stickerPacks: 8,
fixedDirectMessages: 9,
},
},
result: validateBackupResult,
};
},
} satisfies PropsType,

12
ts/components/Preferences.tsx
View file

@ -74,6 +74,7 @@ import {
import { PreferencesBackups } from './PreferencesBackups';
import { PreferencesInternal } from './PreferencesInternal';
import { FunEmojiLocalizationProvider } from './fun/FunEmojiLocalizationProvider';
import type { ValidateLocalBackupStructureResultType } from '../services/backups/util/localBackup';
type CheckboxChangeHandlerType = (value: boolean) => unknown;
type SelectChangeHandlerType<T = string | number> = (value: T) => unknown;
@ -157,9 +158,11 @@ type PropsFunctionType = {
doDeleteAllData: () => unknown;
doneRendering: () => unknown;
editCustomColor: (colorId: string, color: CustomColorType) => unknown;
exportLocalBackup: () => Promise<BackupValidationResultType>;
getConversationsWithCustomColor: (
colorId: string
) => Promise<Array<ConversationType>>;
importLocalBackup: () => Promise<ValidateLocalBackupStructureResultType>;
makeSyncRequest: () => unknown;
refreshCloudBackupStatus: () => void;
refreshBackupSubscriptionStatus: () => void;
@ -286,6 +289,7 @@ export function Preferences({
doneRendering,
editCustomColor,
emojiSkinToneDefault,
exportLocalBackup,
getConversationsWithCustomColor,
hasAudioNotifications,
hasAutoConvertEmoji,
@ -311,6 +315,7 @@ export function Preferences({
hasTextFormatting,
hasTypingIndicators,
i18n,
importLocalBackup,
initialPage = Page.General,
initialSpellCheckSetting,
isAutoDownloadUpdatesSupported,
@ -1736,7 +1741,12 @@ export function Preferences({
);
} else if (page === Page.Internal) {
settings = (
<PreferencesInternal i18n={i18n} validateBackup={validateBackup} />
<PreferencesInternal
i18n={i18n}
exportLocalBackup={exportLocalBackup}
importLocalBackup={importLocalBackup}
validateBackup={validateBackup}
/>
);
}

171
ts/components/PreferencesInternal.tsx
View file

@ -7,17 +7,30 @@ import { toLogFormat } from '../types/errors';
import { formatFileSize } from '../util/formatFileSize';
import { SECOND } from '../util/durations';
import type { ValidationResultType as BackupValidationResultType } from '../services/backups';
import type { ValidateLocalBackupStructureResultType } from '../services/backups/util/localBackup';
import { SettingsRow, SettingsControl } from './PreferencesUtil';
import { Button, ButtonVariant } from './Button';
import { Spinner } from './Spinner';
export function PreferencesInternal({
i18n,
exportLocalBackup: doExportLocalBackup,
importLocalBackup: doImportLocalBackup,
validateBackup: doValidateBackup,
}: {
i18n: LocalizerType;
exportLocalBackup: () => Promise<BackupValidationResultType>;
importLocalBackup: () => Promise<ValidateLocalBackupStructureResultType>;
validateBackup: () => Promise<BackupValidationResultType>;
}): JSX.Element {
const [isExportPending, setIsExportPending] = useState(false);
const [exportResult, setExportResult] = useState<
BackupValidationResultType | undefined
>();
const [importResult, setImportResult] = useState<
ValidateLocalBackupStructureResultType | undefined
>();
const [isValidationPending, setIsValidationPending] = useState(false);
const [validationResult, setValidationResult] = useState<
BackupValidationResultType | undefined
@ -35,34 +48,110 @@ export function PreferencesInternal({
}
}, [doValidateBackup]);
let validationElem: JSX.Element | undefined;
if (validationResult != null) {
if ('result' in validationResult) {
const {
result: { totalBytes, stats, duration },
} = validationResult;
const renderValidationResult = useCallback(
(
backupResult: BackupValidationResultType | undefined
): JSX.Element | undefined => {
if (backupResult == null) {
return;
}
validationElem = (
<div className="Preferences--internal--validate-backup--result">
<p>File size: {formatFileSize(totalBytes)}</p>
<p>Duration: {Math.round(duration / SECOND)}s</p>
<pre>
<code>{JSON.stringify(stats, null, 2)}</code>
</pre>
</div>
);
} else {
const { error } = validationResult;
if ('result' in backupResult) {
const {
result: { totalBytes, stats, duration },
} = backupResult;
validationElem = (
let snapshotDirEl: JSX.Element | undefined;
if ('snapshotDir' in backupResult.result) {
snapshotDirEl = (
<p>
Backup path:
<pre>
<code>{backupResult.result.snapshotDir}</code>
</pre>
</p>
);
}
return (
<div className="Preferences--internal--validate-backup--result">
{snapshotDirEl}
<p>Main file size: {formatFileSize(totalBytes)}</p>
<p>Duration: {Math.round(duration / SECOND)}s</p>
<pre>
<code>{JSON.stringify(stats, null, 2)}</code>
</pre>
</div>
);
}
const { error } = backupResult;
return (
<div className="Preferences--internal--validate-backup--error">
<pre>
<code>{error}</code>
</pre>
</div>
);
},
[]
);
const exportLocalBackup = useCallback(async () => {
setIsExportPending(true);
setExportResult(undefined);
try {
setExportResult(await doExportLocalBackup());
} catch (error) {
setExportResult({ error: toLogFormat(error) });
} finally {
setIsExportPending(false);
}
}
}, [doExportLocalBackup]);
const importLocalBackup = useCallback(async () => {
setImportResult(undefined);
try {
setImportResult(await doImportLocalBackup());
} catch (error) {
setImportResult({
success: false,
error: toLogFormat(error),
snapshotDir: undefined,
});
}
}, [doImportLocalBackup]);
const renderImportResult = useCallback(
(
didImportResult: ValidateLocalBackupStructureResultType | undefined
): JSX.Element | undefined => {
if (didImportResult == null) {
return;
}
const { success, error, snapshotDir } = didImportResult;
if (success) {
return (
<div className="Preferences--internal--validate-backup--result">
<pre>
<code>{`Staged: ${snapshotDir}\n\nPlease link to finish import.`}</code>
</pre>
</div>
);
}
return (
<div className="Preferences--internal--validate-backup--error">
<pre>
<code>{`Failed: ${error}`}</code>
</pre>
</div>
);
},
[]
);
return (
<>
@ -93,7 +182,49 @@ export function PreferencesInternal({
}
/>
{validationElem}
{renderValidationResult(validationResult)}
</SettingsRow>
<SettingsRow
className="Preferences--internal--backups"
title={i18n('icu:Preferences__internal__local-backups')}
>
<SettingsControl
left={i18n(
'icu:Preferences__internal__export-local-backup--description'
)}
right={
<Button
variant={ButtonVariant.Secondary}
onClick={exportLocalBackup}
disabled={isExportPending}
>
{isExportPending ? (
<Spinner size="22px" svgSize="small" />
) : (
i18n('icu:Preferences__internal__export-local-backup')
)}
</Button>
}
/>
{renderValidationResult(exportResult)}
<SettingsControl
left={i18n(
'icu:Preferences__internal__import-local-backup--description'
)}
right={
<Button
variant={ButtonVariant.Secondary}
onClick={importLocalBackup}
>
{i18n('icu:Preferences__internal__import-local-backup')}
</Button>
}
/>
{renderImportResult(importResult)}
</SettingsRow>
</>
);

20
ts/jobs/AttachmentDownloadManager.ts
View file

@ -24,6 +24,7 @@ import {
AttachmentVariant,
AttachmentPermanentlyUndownloadableError,
mightBeOnBackupTier,
mightBeInLocalBackup,
} from '../types/Attachment';
import { type ReadonlyMessageAttributesType } from '../model-types.d';
import { getMessageById } from '../messages/getMessageById';
@ -285,9 +286,10 @@ export class AttachmentDownloadManager extends JobManager<CoreAttachmentDownload
// try to download from the transit tier (or it's an invalid attachment, etc.). We
// may need to extend the attachment_downloads table in the future to better
// differentiate source vs. location.
source: mightBeOnBackupTier(attachment)
? source
: AttachmentDownloadSource.STANDARD,
source:
mightBeOnBackupTier(attachment) || mightBeInLocalBackup(attachment)
? source
: AttachmentDownloadSource.STANDARD,
});
if (!parseResult.success) {
@ -462,7 +464,10 @@ async function runDownloadAttachmentJob({
};
}
if (mightBeOnBackupTier(job.attachment)) {
if (
mightBeOnBackupTier(job.attachment) ||
mightBeInLocalBackup(job.attachment)
) {
const currentDownloadedSize =
window.storage.get('backupMediaDownloadCompletedBytes') ?? 0;
drop(
@ -615,7 +620,8 @@ export async function runDownloadAttachmentJobInner({
isForCurrentlyVisibleMessage &&
mightHaveThumbnailOnBackupTier(job.attachment) &&
// TODO (DESKTOP-7204): check if thumbnail exists on attachment, not on job
!job.attachment.thumbnailFromBackup;
!job.attachment.thumbnailFromBackup &&
!mightBeInLocalBackup(attachment);
if (preferBackupThumbnail) {
logId += '.preferringBackupThumbnail';
@ -811,7 +817,9 @@ async function downloadBackupThumbnail({
}: {
attachment: AttachmentType;
abortSignal: AbortSignal;
dependencies: { downloadAttachment: typeof downloadAttachmentUtil };
dependencies: {
downloadAttachment: typeof downloadAttachmentUtil;
};
}): Promise<AttachmentType> {
const downloadedThumbnail = await dependencies.downloadAttachment({
attachment,

279
ts/jobs/AttachmentLocalBackupManager.ts Normal file
View file

@ -0,0 +1,279 @@
// Copyright 2025 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
/* eslint-disable max-classes-per-file */
import { existsSync } from 'node:fs';
import { PassThrough } from 'node:stream';
import { constants as FS_CONSTANTS, copyFile, mkdir } from 'fs/promises';
import * as durations from '../util/durations';
import * as log from '../logging/log';
import * as Errors from '../types/errors';
import { redactGenericText } from '../util/privacy';
import {
JobManager,
type JobManagerParamsType,
type JobManagerJobResultType,
} from './JobManager';
import { type BackupsService, backupsService } from '../services/backups';
import { decryptAttachmentV2ToSink } from '../AttachmentCrypto';
import {
type AttachmentLocalBackupJobType,
type CoreAttachmentLocalBackupJobType,
} from '../types/AttachmentBackup';
import { isInCall as isInCallSelector } from '../state/selectors/calling';
import { encryptAndUploadAttachment } from '../util/uploadAttachment';
import type { WebAPIType } from '../textsecure/WebAPI';
import type { LocallySavedAttachment } from '../types/Attachment';
import {
getLocalBackupDirectoryForMediaName,
getLocalBackupPathForMediaName,
} from '../services/backups/util/localBackup';
const MAX_CONCURRENT_JOBS = 3;
const RETRY_CONFIG = {
maxAttempts: 3,
backoffConfig: {
// 1 minute, 5 minutes, 25 minutes, every hour
multiplier: 3,
firstBackoffs: [10 * durations.SECOND],
maxBackoffTime: durations.MINUTE,
},
};
export class AttachmentLocalBackupManager extends JobManager<CoreAttachmentLocalBackupJobType> {
static #instance: AttachmentLocalBackupManager | undefined;
readonly #jobsByMediaName = new Map<string, AttachmentLocalBackupJobType>();
static defaultParams: JobManagerParamsType<CoreAttachmentLocalBackupJobType> =
{
markAllJobsInactive: AttachmentLocalBackupManager.markAllJobsInactive,
saveJob: AttachmentLocalBackupManager.saveJob,
removeJob: AttachmentLocalBackupManager.removeJob,
getNextJobs: AttachmentLocalBackupManager.getNextJobs,
runJob: runAttachmentBackupJob,
shouldHoldOffOnStartingQueuedJobs: () => {
const reduxState = window.reduxStore?.getState();
if (reduxState) {
return isInCallSelector(reduxState);
}
return false;
},
getJobId,
getJobIdForLogging,
getRetryConfig: () => RETRY_CONFIG,
maxConcurrentJobs: MAX_CONCURRENT_JOBS,
};
override logPrefix = 'AttachmentLocalBackupManager';
static get instance(): AttachmentLocalBackupManager {
if (!AttachmentLocalBackupManager.#instance) {
AttachmentLocalBackupManager.#instance = new AttachmentLocalBackupManager(
AttachmentLocalBackupManager.defaultParams
);
}
return AttachmentLocalBackupManager.#instance;
}
static get jobs(): Map<string, AttachmentLocalBackupJobType> {
return AttachmentLocalBackupManager.instance.#jobsByMediaName;
}
static async start(): Promise<void> {
log.info('AttachmentLocalBackupManager/starting');
await AttachmentLocalBackupManager.instance.start();
}
static async stop(): Promise<void> {
log.info('AttachmentLocalBackupManager/stopping');
return AttachmentLocalBackupManager.#instance?.stop();
}
static async addJob(newJob: CoreAttachmentLocalBackupJobType): Promise<void> {
return AttachmentLocalBackupManager.instance.addJob(newJob);
}
static async waitForIdle(): Promise<void> {
return AttachmentLocalBackupManager.instance.waitForIdle();
}
static async markAllJobsInactive(): Promise<void> {
for (const [mediaName, job] of AttachmentLocalBackupManager.jobs) {
AttachmentLocalBackupManager.jobs.set(mediaName, {
...job,
active: false,
});
}
}
static async saveJob(job: AttachmentLocalBackupJobType): Promise<void> {
AttachmentLocalBackupManager.jobs.set(job.mediaName, job);
}
static async removeJob(
job: Pick<AttachmentLocalBackupJobType, 'mediaName'>
): Promise<void> {
AttachmentLocalBackupManager.jobs.delete(job.mediaName);
}
static clearAllJobs(): void {
AttachmentLocalBackupManager.jobs.clear();
}
static async getNextJobs({
limit,
timestamp,
}: {
limit: number;
timestamp: number;
}): Promise<Array<AttachmentLocalBackupJobType>> {
let countRemaining = limit;
const nextJobs: Array<AttachmentLocalBackupJobType> = [];
for (const job of AttachmentLocalBackupManager.jobs.values()) {
if (job.active || (job.retryAfter && job.retryAfter > timestamp)) {
continue;
}
nextJobs.push(job);
countRemaining -= 1;
if (countRemaining <= 0) {
break;
}
}
return nextJobs;
}
}
function getJobId(job: CoreAttachmentLocalBackupJobType): string {
return job.mediaName;
}
function getJobIdForLogging(job: CoreAttachmentLocalBackupJobType): string {
return `${redactGenericText(job.mediaName)}.${job.type}`;
}
/**
* Backup-specific methods
*/
class AttachmentPermanentlyMissingError extends Error {}
type RunAttachmentBackupJobDependenciesType = {
getAbsoluteAttachmentPath: typeof window.Signal.Migrations.getAbsoluteAttachmentPath;
backupMediaBatch?: WebAPIType['backupMediaBatch'];
backupsService: BackupsService;
encryptAndUploadAttachment: typeof encryptAndUploadAttachment;
decryptAttachmentV2ToSink: typeof decryptAttachmentV2ToSink;
};
export async function runAttachmentBackupJob(
job: AttachmentLocalBackupJobType,
_options: {
isLastAttempt: boolean;
abortSignal: AbortSignal;
},
dependencies: RunAttachmentBackupJobDependenciesType = {
getAbsoluteAttachmentPath:
window.Signal.Migrations.getAbsoluteAttachmentPath,
backupsService,
backupMediaBatch: window.textsecure.server?.backupMediaBatch,
encryptAndUploadAttachment,
decryptAttachmentV2ToSink,
}
): Promise<JobManagerJobResultType<CoreAttachmentLocalBackupJobType>> {
const jobIdForLogging = getJobIdForLogging(job);
const logId = `AttachmentLocalBackupManager/runAttachmentBackupJob/${jobIdForLogging}`;
try {
await runAttachmentBackupJobInner(job, dependencies);
return { status: 'finished' };
} catch (error) {
log.error(
`${logId}: Failed to backup attachment, attempt ${job.attempts}`,
Errors.toLogFormat(error)
);
if (error instanceof AttachmentPermanentlyMissingError) {
log.error(`${logId}: Attachment unable to be found, giving up on job`);
return { status: 'finished' };
}
return { status: 'retry' };
}
}
async function runAttachmentBackupJobInner(
job: AttachmentLocalBackupJobType,
dependencies: RunAttachmentBackupJobDependenciesType
): Promise<void> {
const jobIdForLogging = getJobIdForLogging(job);
const logId = `AttachmentLocalBackupManager.runAttachmentBackupJobInner(${jobIdForLogging})`;
log.info(`${logId}: starting`);
const { backupsBaseDir, mediaName } = job;
const { contentType, digest, iv, keys, localKey, path, size } = job.data;
if (!path) {
throw new AttachmentPermanentlyMissingError('No path property');
}
const absolutePath = dependencies.getAbsoluteAttachmentPath(path);
if (!existsSync(absolutePath)) {
throw new AttachmentPermanentlyMissingError('No file at provided path');
}
if (!localKey) {
throw new Error('No localKey property, required for test decryption');
}
const localBackupFileDir = getLocalBackupDirectoryForMediaName({
backupsBaseDir,
mediaName,
});
await mkdir(localBackupFileDir, { recursive: true });
const localBackupFilePath = getLocalBackupPathForMediaName({
backupsBaseDir,
mediaName,
});
const attachment: LocallySavedAttachment = {
path,
iv,
key: keys,
localKey,
digest,
contentType,
size,
};
// TODO: Add check in local FS to prevent double backup
// File is already encrypted with localKey, so we just have to copy it to the backup dir
const attachmentPath = window.Signal.Migrations.getAbsoluteAttachmentPath(
attachment.path
);
// Set COPYFILE_FICLONE for Copy on Write (OS dependent, gracefully falls back to copy)
await copyFile(
attachmentPath,
localBackupFilePath,
FS_CONSTANTS.COPYFILE_FICLONE
);
// TODO: Optimize this check -- it can be expensive to test decrypt on every export
log.info(`${logId}: Verifying file restored from local backup`);
const sink = new PassThrough();
sink.resume();
await decryptAttachmentV2ToSink(
{
ciphertextPath: localBackupFilePath,
idForLogging: 'attachments/readAndDecryptDataFromDisk',
keysBase64: localKey,
size,
type: 'local',
},
sink
);
}

2
ts/main/settingsChannel.ts
View file

@ -68,6 +68,8 @@ export class SettingsChannel extends EventEmitter {
this.#installCallback('syncRequest');
this.#installCallback('setEmojiSkinToneDefault');
this.#installCallback('getEmojiSkinToneDefault');
this.#installCallback('exportLocalBackup');
this.#installCallback('importLocalBackup');
this.#installCallback('validateBackup');
// Backups

4
ts/services/backups/constants.ts
View file

@ -6,6 +6,10 @@ import type { ConversationColorType } from '../../types/Colors';
export const BACKUP_VERSION = 1;
export const LOCAL_BACKUP_VERSION = 1;
export const LOCAL_BACKUP_BACKUP_ID_IV_LENGTH = 16;
const { WallpaperPreset } = Backups.ChatStyle;
// See https://github.com/signalapp/Signal-Android-Private/blob/4a41e9f9a1ed0aba7cae0e0dc4dbcac50fddc469/app/src/main/java/org/thoughtcrime/securesms/conversation/colors/ChatColorsMapper.kt#L32

9
ts/services/backups/crypto.ts
View file

@ -125,3 +125,12 @@ export function deriveBackupThumbnailTransitKeyMaterial(
),
};
}
export function getBackupId(): Uint8Array {
const aci = window.storage.user.getCheckedAci();
return getBackupKey().deriveBackupId(toAciObject(aci));
}
export function getLocalBackupMetadataKey(): Uint8Array {
return getBackupKey().deriveLocalBackupMetadataKey();
}

153
ts/services/backups/export.ts
View file

@ -4,6 +4,7 @@
import Long from 'long';
import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client';
import type { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
import { dirname } from 'path';
import pMap from 'p-map';
import pTimeout from 'p-timeout';
import { Readable } from 'stream';
@ -125,11 +126,13 @@ import {
} from '../../types/Attachment';
import {
getFilePointerForAttachment,
getLocalBackupFilePointerForAttachment,
maybeGetBackupJobForAttachmentAndFilePointer,
} from './util/filePointers';
import { getBackupMediaRootKey } from './crypto';
import type { CoreAttachmentBackupJobType } from '../../types/AttachmentBackup';
import { AttachmentBackupManager } from '../../jobs/AttachmentBackupManager';
import { AttachmentLocalBackupManager } from '../../jobs/AttachmentLocalBackupManager';
import { getBackupCdnInfo } from './util/mediaId';
import { calculateExpirationTimestamp } from '../../util/expirationTimer';
import { ReadStatus } from '../../messages/MessageReadStatus';
@ -177,6 +180,7 @@ type ToChatItemOptionsType = Readonly<{
aboutMe: AboutMe;
callHistoryByCallId: Record<string, CallHistoryDetails>;
backupLevel: BackupLevel;
isLocalBackup: boolean;
}>;
type NonBubbleOptionsType = Pick<
@ -227,6 +231,7 @@ export class BackupExportStream extends Readable {
readonly #serviceIdToRecipientId = new Map<string, number>();
readonly #e164ToRecipientId = new Map<string, number>();
readonly #roomIdToRecipientId = new Map<string, number>();
readonly #mediaNamesToFilePointers = new Map<string, Backups.FilePointer>();
readonly #stats: StatsType = {
adHocCalls: 0,
callLinks: 0,
@ -253,43 +258,82 @@ export class BackupExportStream extends Readable {
super();
}
public run(backupLevel: BackupLevel): void {
public run(
backupLevel: BackupLevel,
localBackupSnapshotDir: string | undefined = undefined
): void {
const localBackupsBaseDir = localBackupSnapshotDir
? dirname(localBackupSnapshotDir)
: undefined;
const isLocalBackup = localBackupsBaseDir != null;
drop(
(async () => {
log.info('BackupExportStream: starting...');
drop(AttachmentBackupManager.stop());
drop(AttachmentLocalBackupManager.stop());
log.info('BackupExportStream: message migration starting...');
await migrateAllMessages();
await pauseWriteAccess();
try {
await this.#unsafeRun(backupLevel);
await this.#unsafeRun(backupLevel, isLocalBackup);
} catch (error) {
this.emit('error', error);
} finally {
await resumeWriteAccess();
// TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction
await DataWriter.clearAllAttachmentBackupJobs();
if (this.backupType !== BackupType.TestOnlyPlaintext) {
await Promise.all(
this.#attachmentBackupJobs.map(job =>
AttachmentBackupManager.addJobAndMaybeThumbnailJob(job)
)
if (isLocalBackup) {
log.info(
`BackupExportStream: Adding ${this.#attachmentBackupJobs.length} jobs for AttachmentLocalBackupManager`
);
drop(AttachmentBackupManager.start());
AttachmentLocalBackupManager.clearAllJobs();
await Promise.all(
this.#attachmentBackupJobs.map(job => {
if (job.type === 'thumbnail') {
log.error(
"BackupExportStream: Can't backup thumbnails to local backup, skipping"
);
return Promise.resolve();
}
return AttachmentLocalBackupManager.addJob({
...job,
backupsBaseDir: localBackupsBaseDir,
});
})
);
drop(AttachmentLocalBackupManager.start());
} else {
// TODO (DESKTOP-7344): Clear & add backup jobs in a single transaction
await DataWriter.clearAllAttachmentBackupJobs();
if (this.backupType !== BackupType.TestOnlyPlaintext) {
await Promise.all(
this.#attachmentBackupJobs.map(job =>
AttachmentBackupManager.addJobAndMaybeThumbnailJob(job)
)
);
drop(AttachmentBackupManager.start());
}
}
log.info('BackupExportStream: finished');
}
})()
);
}
public getMediaNamesIterator(): MapIterator<string> {
return this.#mediaNamesToFilePointers.keys();
}
public getStats(): Readonly<StatsType> {
return this.#stats;
}
async #unsafeRun(backupLevel: BackupLevel): Promise<void> {
async #unsafeRun(
backupLevel: BackupLevel,
isLocalBackup: boolean
): Promise<void> {
this.#ourConversation =
window.ConversationController.getOurConversationOrThrow().attributes;
this.push(
@ -663,6 +707,7 @@ export class BackupExportStream extends Readable {
aboutMe,
callHistoryByCallId,
backupLevel,
isLocalBackup,
}),
{ concurrency: MAX_CONCURRENCY }
);
@ -1093,7 +1138,12 @@ export class BackupExportStream extends Readable {
async #toChatItem(
message: MessageAttributesType,
{ aboutMe, callHistoryByCallId, backupLevel }: ToChatItemOptionsType
{
aboutMe,
callHistoryByCallId,
backupLevel,
isLocalBackup,
}: ToChatItemOptionsType
): Promise<Backups.IChatItem | undefined> {
const conversation = window.ConversationController.get(
message.conversationId
@ -1253,6 +1303,7 @@ export class BackupExportStream extends Readable {
result.viewOnceMessage = await this.#toViewOnceMessage({
message,
backupLevel,
isLocalBackup,
});
} else if (message.deletedForEveryone) {
result.remoteDeletedMessage = {};
@ -1314,6 +1365,7 @@ export class BackupExportStream extends Readable {
? await this.#processAttachment({
attachment: contactDetails.avatar.avatar,
backupLevel,
isLocalBackup,
messageReceivedAt: message.received_at,
})
: undefined,
@ -1335,6 +1387,7 @@ export class BackupExportStream extends Readable {
? await this.#processAttachment({
attachment: sticker.data,
backupLevel,
isLocalBackup,
messageReceivedAt: message.received_at,
})
: undefined;
@ -1378,23 +1431,27 @@ export class BackupExportStream extends Readable {
result.directStoryReplyMessage = await this.#toDirectStoryReplyMessage({
message,
backupLevel,
isLocalBackup,
});
result.revisions = await this.#toChatItemRevisions(
result,
message,
backupLevel
backupLevel,
isLocalBackup
);
} else {
result.standardMessage = await this.#toStandardMessage({
message,
backupLevel,
isLocalBackup,
});
result.revisions = await this.#toChatItemRevisions(
result,
message,
backupLevel
backupLevel,
isLocalBackup
);
}
@ -2297,9 +2354,11 @@ export class BackupExportStream extends Readable {
async #toQuote({
message,
backupLevel,
isLocalBackup,
}: {
message: Pick<MessageAttributesType, 'quote' | 'received_at' | 'body'>;
backupLevel: BackupLevel;
isLocalBackup: boolean;
}): Promise<Backups.IQuote | null> {
const { quote } = message;
if (!quote) {
@ -2359,6 +2418,7 @@ export class BackupExportStream extends Readable {
attachment: attachment.thumbnail,
backupLevel,
message,
isLocalBackup,
})
: undefined,
};
@ -2417,16 +2477,19 @@ export class BackupExportStream extends Readable {
attachment,
backupLevel,
message,
isLocalBackup,
}: {
attachment: AttachmentType;
backupLevel: BackupLevel;
message: Pick<MessageAttributesType, 'quote' | 'received_at' | 'body'>;
isLocalBackup: boolean;
}): Promise<Backups.MessageAttachment> {
const { clientUuid } = attachment;
const filePointer = await this.#processAttachment({
attachment,
backupLevel,
messageReceivedAt: message.received_at,
isLocalBackup,
});
return new Backups.MessageAttachment({
@ -2440,18 +2503,51 @@ export class BackupExportStream extends Readable {
async #processAttachment({
attachment,
backupLevel,
isLocalBackup,
messageReceivedAt,
}: {
attachment: AttachmentType;
backupLevel: BackupLevel;
isLocalBackup: boolean;
messageReceivedAt: number;
}): Promise<Backups.FilePointer> {
const { filePointer, updatedAttachment } =
await getFilePointerForAttachment({
attachment,
backupLevel,
getBackupCdnInfo,
});
// We need to always get updatedAttachment in case the attachment wasn't reencryptable
// to the original digest. In that case mediaName will be based on updatedAttachment.
const { filePointer, updatedAttachment } = isLocalBackup
? await getLocalBackupFilePointerForAttachment({
attachment,
backupLevel,
getBackupCdnInfo,
})
: await getFilePointerForAttachment({
attachment,
backupLevel,
getBackupCdnInfo,
});
if (isLocalBackup && filePointer.localLocator) {
// Duplicate attachment check. Local backups can only contain 1 file per mediaName,
// so if we see a duplicate mediaName then we must reuse the previous FilePointer.
const { mediaName } = filePointer.localLocator;
strictAssert(
mediaName,
'FilePointer.LocalLocator must contain mediaName'
);
const existingFilePointer = this.#mediaNamesToFilePointers.get(mediaName);
if (existingFilePointer) {
strictAssert(
existingFilePointer.localLocator,
'Local backup existing mediaName FilePointer must contain LocalLocator'
);
strictAssert(
existingFilePointer.localLocator.size === attachment.size,
'Local backup existing mediaName FilePointer size must match attachment'
);
return existingFilePointer;
}
this.#mediaNamesToFilePointers.set(mediaName, filePointer);
}
if (updatedAttachment) {
// TODO (DESKTOP-6688): ensure that we update the message/attachment in DB with the
@ -2639,6 +2735,7 @@ export class BackupExportStream extends Readable {
async #toStandardMessage({
message,
backupLevel,
isLocalBackup,
}: {
message: Pick<
MessageAttributesType,
@ -2652,11 +2749,13 @@ export class BackupExportStream extends Readable {
| 'received_at'
>;
backupLevel: BackupLevel;
isLocalBackup: boolean;
}): Promise<Backups.IStandardMessage> {
return {
quote: await this.#toQuote({
message,
backupLevel,
isLocalBackup,
}),
attachments: message.attachments?.length
? await Promise.all(
@ -2665,6 +2764,7 @@ export class BackupExportStream extends Readable {
attachment,
backupLevel,
message,
isLocalBackup,
});
})
)
@ -2673,6 +2773,7 @@ export class BackupExportStream extends Readable {
? await this.#processAttachment({
attachment: message.bodyAttachment,
backupLevel,
isLocalBackup,
messageReceivedAt: message.received_at,
})
: undefined,
@ -2697,6 +2798,7 @@ export class BackupExportStream extends Readable {
? await this.#processAttachment({
attachment: preview.image,
backupLevel,
isLocalBackup,
messageReceivedAt: message.received_at,
})
: undefined,
@ -2711,6 +2813,7 @@ export class BackupExportStream extends Readable {
async #toDirectStoryReplyMessage({
message,
backupLevel,
isLocalBackup,
}: {
message: Pick<
MessageAttributesType,
@ -2722,6 +2825,7 @@ export class BackupExportStream extends Readable {
| 'reactions'
>;
backupLevel: BackupLevel;
isLocalBackup: boolean;
}): Promise<Backups.IDirectStoryReplyMessage> {
const result = new Backups.DirectStoryReplyMessage({
reactions: this.#getMessageReactions(message),
@ -2735,6 +2839,7 @@ export class BackupExportStream extends Readable {
? await this.#processAttachment({
attachment: message.bodyAttachment,
backupLevel,
isLocalBackup,
messageReceivedAt: message.received_at,
})
: undefined,
@ -2755,12 +2860,14 @@ export class BackupExportStream extends Readable {
async #toViewOnceMessage({
message,
backupLevel,
isLocalBackup,
}: {
message: Pick<
MessageAttributesType,
'attachments' | 'received_at' | 'reactions'
>;
backupLevel: BackupLevel;
isLocalBackup: boolean;
}): Promise<Backups.IViewOnceMessage> {
const attachment = message.attachments?.at(0);
return {
@ -2771,6 +2878,7 @@ export class BackupExportStream extends Readable {
attachment,
backupLevel,
message,
isLocalBackup,
}),
reactions: this.#getMessageReactions(message),
};
@ -2779,7 +2887,8 @@ export class BackupExportStream extends Readable {
async #toChatItemRevisions(
parent: Backups.IChatItem,
message: MessageAttributesType,
backupLevel: BackupLevel
backupLevel: BackupLevel,
isLocalBackup: boolean
): Promise<Array<Backups.IChatItem> | undefined> {
const { editHistory } = message;
if (editHistory == null) {
@ -2818,11 +2927,13 @@ export class BackupExportStream extends Readable {
await this.#toDirectStoryReplyMessage({
message: history,
backupLevel,
isLocalBackup,
});
} else {
result.standardMessage = await this.#toStandardMessage({
message: history,
backupLevel,
isLocalBackup,
});
}
return result;

67
ts/services/backups/import.ts
View file

@ -244,18 +244,22 @@ export class BackupImportStream extends Writable {
#pendingGroupAvatars = new Map<string, string>();
#frameErrorCount: number = 0;
private constructor(private readonly backupType: BackupType) {
private constructor(
private readonly backupType: BackupType,
private readonly localBackupSnapshotDir: string | undefined
) {
super({ objectMode: true });
}
public static async create(
backupType = BackupType.Ciphertext
backupType = BackupType.Ciphertext,
localBackupSnapshotDir: string | undefined = undefined
): Promise<BackupImportStream> {
await AttachmentDownloadManager.stop();
await DataWriter.removeAllBackupAttachmentDownloadJobs();
await resetBackupMediaDownloadProgress();
return new BackupImportStream(backupType);
return new BackupImportStream(backupType, localBackupSnapshotDir);
}
override async _write(
@ -1854,11 +1858,19 @@ export class BackupImportStream extends Writable {
bodyRanges: this.#fromBodyRanges(data.text),
})),
bodyAttachment: data.longText
? convertFilePointerToAttachment(data.longText)
? convertFilePointerToAttachment(
data.longText,
this.#getFilePointerOptions()
)
: undefined,
attachments: data.attachments?.length
? data.attachments
.map(convertBackupMessageAttachmentToAttachment)
.map(attachment =>
convertBackupMessageAttachmentToAttachment(
attachment,
this.#getFilePointerOptions()
)
)
.filter(isNotNil)
: undefined,
preview: data.linkPreview?.length
@ -1902,7 +1914,10 @@ export class BackupImportStream extends Writable {
description: dropNull(preview.description),
date: getCheckedTimestampOrUndefinedFromLong(preview.date),
image: preview.image
? convertFilePointerToAttachment(preview.image)
? convertFilePointerToAttachment(
preview.image,
this.#getFilePointerOptions()
)
: undefined,
};
})
@ -1917,7 +1932,10 @@ export class BackupImportStream extends Writable {
...(attachment
? {
attachments: [
convertBackupMessageAttachmentToAttachment(attachment),
convertBackupMessageAttachmentToAttachment(
attachment,
this.#getFilePointerOptions()
),
].filter(isNotNil),
}
: {
@ -1948,7 +1966,10 @@ export class BackupImportStream extends Writable {
result.body = textReply.text?.body ?? undefined;
result.bodyRanges = this.#fromBodyRanges(textReply.text);
result.bodyAttachment = textReply.longText
? convertFilePointerToAttachment(textReply.longText)
? convertFilePointerToAttachment(
textReply.longText,
this.#getFilePointerOptions()
)
: undefined;
} else if (emoji) {
result.storyReaction = {
@ -1978,7 +1999,10 @@ export class BackupImportStream extends Writable {
body: textReply.text?.body ?? undefined,
bodyRanges: this.#fromBodyRanges(textReply.text),
bodyAttachment: textReply.longText
? convertFilePointerToAttachment(textReply.longText)
? convertFilePointerToAttachment(
textReply.longText,
this.#getFilePointerOptions()
)
: undefined,
};
}
@ -2101,7 +2125,10 @@ export class BackupImportStream extends Writable {
? stringToMIMEType(contentType)
: APPLICATION_OCTET_STREAM,
thumbnail: thumbnail?.pointer
? convertFilePointerToAttachment(thumbnail.pointer)
? convertFilePointerToAttachment(
thumbnail.pointer,
this.#getFilePointerOptions()
)
: undefined,
};
}) ?? [],
@ -2263,7 +2290,10 @@ export class BackupImportStream extends Writable {
organization: organization || undefined,
avatar: avatar
? {
avatar: convertFilePointerToAttachment(avatar),
avatar: convertFilePointerToAttachment(
avatar,
this.#getFilePointerOptions()
),
isProfile: false,
}
: undefined,
@ -2310,7 +2340,12 @@ export class BackupImportStream extends Writable {
packId: Bytes.toHex(packId),
packKey: Bytes.toBase64(packKey),
stickerId,
data: data ? convertFilePointerToAttachment(data) : undefined,
data: data
? convertFilePointerToAttachment(
data,
this.#getFilePointerOptions()
)
: undefined,
},
reactions: this.#fromReactions(chatItem.stickerMessage.reactions),
},
@ -3691,6 +3726,14 @@ export class BackupImportStream extends Writable {
autoBubbleColor,
};
}
#getFilePointerOptions() {
if (this.localBackupSnapshotDir != null) {
return { localBackupSnapshotDir: this.localBackupSnapshotDir };
}
return {};
}
}
function rgbIntToDesktopHSL(intValue: number): {

237
ts/services/backups/index.ts
View file

@ -5,15 +5,16 @@ import { pipeline } from 'stream/promises';
import { PassThrough } from 'stream';
import type { Readable, Writable } from 'stream';
import { createReadStream, createWriteStream } from 'fs';
import { unlink, stat } from 'fs/promises';
import { mkdir, stat, unlink } from 'fs/promises';
import { ensureFile } from 'fs-extra';
import { join } from 'path';
import { createGzip, createGunzip } from 'zlib';
import { createCipheriv, createHmac, randomBytes } from 'crypto';
import { noop } from 'lodash';
import { isEqual, noop } from 'lodash';
import { BackupLevel } from '@signalapp/libsignal-client/zkgroup';
import { BackupKey } from '@signalapp/libsignal-client/dist/AccountKeys';
import { throttle } from 'lodash/fp';
import { ipcRenderer } from 'electron';
import { DataReader, DataWriter } from '../../sql/Client';
import * as log from '../../logging/log';
@ -49,7 +50,11 @@ import { isTestOrMockEnvironment } from '../../environment';
import { runStorageServiceSyncJob } from '../storage';
import { BackupExportStream, type StatsType } from './export';
import { BackupImportStream } from './import';
import { getKeyMaterial } from './crypto';
import {
getBackupId,
getKeyMaterial,
getLocalBackupMetadataKey,
} from './crypto';
import { BackupCredentials } from './credentials';
import { BackupAPI } from './api';
import { validateBackup, ValidationType } from './validator';
@ -66,6 +71,16 @@ import { MemoryStream } from './util/MemoryStream';
import { ToastType } from '../../types/Toast';
import { isAdhoc, isNightly } from '../../util/version';
import { getMessageQueueTime } from '../../util/getMessageQueueTime';
import { isLocalBackupsEnabled } from '../../util/isLocalBackupsEnabled';
import type { ValidateLocalBackupStructureResultType } from './util/localBackup';
import {
writeLocalBackupMetadata,
verifyLocalBackupMetadata,
writeLocalBackupFilesList,
readLocalBackupFilesList,
validateLocalBackupStructure,
} from './util/localBackup';
import { AttachmentLocalBackupManager } from '../../jobs/AttachmentLocalBackupManager';
export { BackupType };
@ -94,6 +109,7 @@ type DoDownloadOptionsType = Readonly<{
export type ImportOptionsType = Readonly<{
backupType?: BackupType;
localBackupSnapshotDir?: string;
ephemeralKey?: Uint8Array;
onProgress?: (currentBytes: number, totalBytes: number) => void;
}>;
@ -104,9 +120,13 @@ export type ExportResultType = Readonly<{
stats: Readonly<StatsType>;
}>;
export type LocalBackupExportResultType = ExportResultType & {
snapshotDir: string;
};
export type ValidationResultType = Readonly<
| {
result: ExportResultType;
result: ExportResultType | LocalBackupExportResultType;
}
| {
error: string;
@ -123,6 +143,8 @@ export class BackupsService {
| ExplodePromiseResultType<RetryBackupImportValue>
| undefined;
#localBackupSnapshotDir: string | undefined;
public readonly credentials = new BackupCredentials();
public readonly api = new BackupAPI(this.credentials);
public readonly throttledFetchCloudBackupStatus = throttle(
@ -263,23 +285,7 @@ export class BackupsService {
}
public async upload(): Promise<void> {
// Make sure we are up-to-date on storage service
{
const { promise: storageService, resolve } = explodePromise<void>();
window.Whisper.events.once('storageService:syncComplete', resolve);
runStorageServiceSyncJob({ reason: 'backups.upload' });
await storageService;
}
// Clear message queue
await window.waitForEmptyEventQueue();
// Make sure all batches are flushed
await Promise.all([
window.waitForAllBatchers(),
window.flushAllWaitBatchers(),
]);
await this.#waitForEmptyQueues('backups.upload');
const fileName = `backup-${randomBytes(32).toString('hex')}`;
const filePath = join(window.BasePaths.temp, fileName);
@ -290,9 +296,9 @@ export class BackupsService {
log.info(`exportBackup: starting, backup level: ${backupLevel}...`);
try {
const fileSize = await this.exportToDisk(filePath, backupLevel);
const { totalBytes } = await this.exportToDisk(filePath, backupLevel);
await this.api.upload(filePath, fileSize);
await this.api.upload(filePath, totalBytes);
} finally {
try {
await unlink(filePath);
@ -302,6 +308,95 @@ export class BackupsService {
}
}
public async exportLocalBackup(
backupsBaseDir: string | undefined = undefined,
backupLevel: BackupLevel = BackupLevel.Free
): Promise<LocalBackupExportResultType> {
strictAssert(isLocalBackupsEnabled(), 'Local backups must be enabled');
await this.#waitForEmptyQueues('backups.exportLocalBackup');
const baseDir =
backupsBaseDir ??
join(window.SignalContext.getPath('userData'), 'SignalBackups');
const snapshotDir = join(baseDir, `signal-backup-${new Date().getTime()}`);
await mkdir(snapshotDir, { recursive: true });
const mainProtoPath = join(snapshotDir, 'main');
log.info('exportLocalBackup: starting');
const exportResult = await this.exportToDisk(
mainProtoPath,
backupLevel,
BackupType.Ciphertext,
snapshotDir
);
log.info('exportLocalBackup: writing metadata');
const metadataArgs = {
snapshotDir,
backupId: getBackupId(),
metadataKey: getLocalBackupMetadataKey(),
};
await writeLocalBackupMetadata(metadataArgs);
await verifyLocalBackupMetadata(metadataArgs);
log.info(
'exportLocalBackup: waiting for AttachmentLocalBackupManager to finish'
);
await AttachmentLocalBackupManager.waitForIdle();
log.info(`exportLocalBackup: exported to disk: ${snapshotDir}`);
return { ...exportResult, snapshotDir };
}
public async stageLocalBackupForImport(
snapshotDir: string
): Promise<ValidateLocalBackupStructureResultType> {
const result = await validateLocalBackupStructure(snapshotDir);
const { success, error } = result;
if (success) {
this.#localBackupSnapshotDir = snapshotDir;
log.info(
`stageLocalBackupForImport: Staged ${snapshotDir} for import. Please link to perform import.`
);
} else {
this.#localBackupSnapshotDir = undefined;
log.info(
`stageLocalBackupForImport: Invalid snapshot ${snapshotDir}. Error: ${error}.`
);
}
return result;
}
public isLocalBackupStaged(): boolean {
return Boolean(this.#localBackupSnapshotDir);
}
public async importLocalBackup(): Promise<void> {
strictAssert(
this.#localBackupSnapshotDir,
'importLocalBackup: Staged backup is required, use stageLocalBackupForImport()'
);
log.info(`importLocalBackup: Importing ${this.#localBackupSnapshotDir}`);
const backupFile = join(this.#localBackupSnapshotDir, 'main');
await this.importFromDisk(backupFile, {
localBackupSnapshotDir: this.#localBackupSnapshotDir,
});
await verifyLocalBackupMetadata({
snapshotDir: this.#localBackupSnapshotDir,
backupId: getBackupId(),
metadataKey: getLocalBackupMetadataKey(),
});
this.#localBackupSnapshotDir = undefined;
log.info('importLocalBackup: Done');
}
// Test harness
public async exportBackupData(
backupLevel: BackupLevel = BackupLevel.Free,
@ -319,29 +414,63 @@ export class BackupsService {
};
}
// Test harness
public async exportToDisk(
path: string,
backupLevel: BackupLevel = BackupLevel.Free,
backupType = BackupType.Ciphertext
): Promise<number> {
const { totalBytes } = await this.#exportBackup(
backupType = BackupType.Ciphertext,
localBackupSnapshotDir: string | undefined = undefined
): Promise<ExportResultType> {
const exportResult = await this.#exportBackup(
createWriteStream(path),
backupLevel,
backupType
backupType,
localBackupSnapshotDir
);
if (backupType === BackupType.Ciphertext) {
await validateBackup(
() => new FileStream(path),
totalBytes,
exportResult.totalBytes,
isTestOrMockEnvironment()
? ValidationType.Internal
: ValidationType.Export
);
}
return totalBytes;
return exportResult;
}
public async _internalExportLocalBackup(
backupLevel: BackupLevel = BackupLevel.Free
): Promise<ValidationResultType> {
try {
const { canceled, dirPath: backupsBaseDir } = await ipcRenderer.invoke(
'show-open-folder-dialog'
);
if (canceled || !backupsBaseDir) {
return { error: 'Backups directory not selected' };
}
const result = await this.exportLocalBackup(backupsBaseDir, backupLevel);
return { result };
} catch (error) {
return { error: Errors.toLogFormat(error) };
}
}
public async _internalStageLocalBackupForImport(): Promise<ValidateLocalBackupStructureResultType> {
const { canceled, dirPath: snapshotDir } = await ipcRenderer.invoke(
'show-open-folder-dialog'
);
if (canceled || !snapshotDir) {
return {
success: false,
error: 'File dialog canceled',
snapshotDir: undefined,
};
}
return this.stageLocalBackupForImport(snapshotDir);
}
// Test harness
@ -417,6 +546,7 @@ export class BackupsService {
backupType = BackupType.Ciphertext,
ephemeralKey,
onProgress,
localBackupSnapshotDir = undefined,
}: ImportOptionsType = {}
): Promise<void> {
strictAssert(!this.#isRunning, 'BackupService is already running');
@ -438,7 +568,10 @@ export class BackupsService {
window.ConversationController.setReadOnly(true);
const importStream = await BackupImportStream.create(backupType);
const importStream = await BackupImportStream.create(
backupType,
localBackupSnapshotDir
);
if (backupType === BackupType.Ciphertext) {
const { aesKey, macKey } = getKeyMaterial(
ephemeralKey ? new BackupKey(Buffer.from(ephemeralKey)) : undefined
@ -761,7 +894,8 @@ export class BackupsService {
async #exportBackup(
sink: Writable,
backupLevel: BackupLevel = BackupLevel.Free,
backupType = BackupType.Ciphertext
backupType = BackupType.Ciphertext,
localBackupSnapshotDir: string | undefined = undefined
): Promise<ExportResultType> {
strictAssert(!this.#isRunning, 'BackupService is already running');
@ -786,7 +920,7 @@ export class BackupsService {
const { aesKey, macKey } = getKeyMaterial();
const recordStream = new BackupExportStream(backupType);
recordStream.run(backupLevel);
recordStream.run(backupLevel, localBackupSnapshotDir);
const iv = randomBytes(IV_LENGTH);
@ -817,6 +951,21 @@ export class BackupsService {
throw missingCaseError(backupType);
}
if (localBackupSnapshotDir) {
log.info('exportBackup: writing local backup files list');
const filesWritten = await writeLocalBackupFilesList({
snapshotDir: localBackupSnapshotDir,
mediaNamesIterator: recordStream.getMediaNamesIterator(),
});
const filesRead = await readLocalBackupFilesList(
localBackupSnapshotDir
);
strictAssert(
isEqual(filesWritten, filesRead),
'exportBackup: Local backup files proto must match files written'
);
}
const duration = Date.now() - start;
return { totalBytes, stats: recordStream.getStats(), duration };
} finally {
@ -866,6 +1015,28 @@ export class BackupsService {
window.reduxActions.installer.startInstaller();
}
async #waitForEmptyQueues(
reason: 'backups.upload' | 'backups.exportLocalBackup'
) {
// Make sure we are up-to-date on storage service
{
const { promise: storageService, resolve } = explodePromise<void>();
window.Whisper.events.once('storageService:syncComplete', resolve);
runStorageServiceSyncJob({ reason });
await storageService;
}
// Clear message queue
await window.waitForEmptyEventQueue();
// Make sure all batches are flushed
await Promise.all([
window.waitForAllBatchers(),
window.flushAllWaitBatchers(),
]);
}
public isImportRunning(): boolean {
return this.#isRunning === 'import';
}

230
ts/services/backups/util/filePointers.ts
View file

@ -40,11 +40,17 @@ import { bytesToUuid } from '../../../util/uuidToBytes';
import { createName } from '../../../util/attachmentPath';
import { ensureAttachmentIsReencryptable } from '../../../util/ensureAttachmentIsReencryptable';
import type { ReencryptionInfo } from '../../../AttachmentCrypto';
import { getAttachmentLocalBackupPathFromSnapshotDir } from './localBackup';
type ConvertFilePointerToAttachmentOptions = {
// Only for testing
_createName: (suffix?: string) => string;
localBackupSnapshotDir: string | undefined;
};
export function convertFilePointerToAttachment(
filePointer: Backups.FilePointer,
// Only for testing
{ _createName: doCreateName = createName } = {}
options: Partial<ConvertFilePointerToAttachmentOptions> = {}
): AttachmentType {
const {
contentType,
@ -58,7 +64,9 @@ export function convertFilePointerToAttachment(
attachmentLocator,
backupLocator,
invalidAttachmentLocator,
localLocator,
} = filePointer;
const doCreateName = options._createName ?? createName;
const commonProps: Omit<AttachmentType, 'size'> = {
contentType: contentType
@ -122,6 +130,57 @@ export function convertFilePointerToAttachment(
};
}
if (localLocator) {
const {
mediaName,
localKey,
backupCdnNumber,
remoteKey: key,
remoteDigest: digest,
size,
transitCdnKey,
transitCdnNumber,
} = localLocator;
const { localBackupSnapshotDir } = options;
strictAssert(
localBackupSnapshotDir,
'localBackupSnapshotDir is required for filePointer.localLocator'
);
if (mediaName == null) {
log.error(
'convertFilePointerToAttachment: filePointer.localLocator missing mediaName!'
);
return {
...omit(commonProps, 'downloadPath'),
error: true,
size: 0,
};
}
const localBackupPath = getAttachmentLocalBackupPathFromSnapshotDir(
mediaName,
localBackupSnapshotDir
);
return {
...commonProps,
cdnKey: transitCdnKey ?? undefined,
cdnNumber: transitCdnNumber ?? undefined,
key: key?.length ? Bytes.toBase64(key) : undefined,
digest: digest?.length ? Bytes.toBase64(digest) : undefined,
size: size ?? 0,
localBackupPath,
localKey: localKey?.length ? Bytes.toBase64(localKey) : undefined,
backupLocator: backupCdnNumber
? {
mediaName,
cdnNumber: backupCdnNumber,
}
: undefined,
};
}
if (!invalidAttachmentLocator) {
log.error('convertFilePointerToAttachment: filePointer had no locator');
}
@ -134,7 +193,8 @@ export function convertFilePointerToAttachment(
}
export function convertBackupMessageAttachmentToAttachment(
messageAttachment: Backups.IMessageAttachment
messageAttachment: Backups.IMessageAttachment,
options: Partial<ConvertFilePointerToAttachmentOptions> = {}
): AttachmentType | null {
const { clientUuid } = messageAttachment;
@ -142,7 +202,7 @@ export function convertBackupMessageAttachmentToAttachment(
return null;
}
const result = {
...convertFilePointerToAttachment(messageAttachment.pointer),
...convertFilePointerToAttachment(messageAttachment.pointer, options),
clientUuid: clientUuid ? bytesToUuid(clientUuid) : undefined,
};
@ -372,6 +432,99 @@ export async function getFilePointerForAttachment({
};
}
// Given a remote backup FilePointer, return a FilePointer referencing a local backup
export async function getLocalBackupFilePointerForAttachment({
attachment,
backupLevel,
getBackupCdnInfo,
}: {
attachment: Readonly<AttachmentType>;
backupLevel: BackupLevel;
getBackupCdnInfo: GetBackupCdnInfoType;
}): Promise<{
filePointer: Backups.FilePointer;
updatedAttachment?: AttachmentType;
}> {
const { filePointer: remoteFilePointer, updatedAttachment } =
await getFilePointerForAttachment({
attachment,
backupLevel,
getBackupCdnInfo,
});
if (attachment.localKey == null) {
return { filePointer: remoteFilePointer, updatedAttachment };
}
strictAssert(
attachment.localKey != null,
'getLocalBackupFilePointerForAttachment: attachment must have localKey'
);
if (remoteFilePointer.backupLocator) {
const { backupLocator } = remoteFilePointer;
const { mediaName } = backupLocator;
strictAssert(
mediaName,
'getLocalBackupFilePointerForAttachment: BackupLocator must have mediaName'
);
const localLocator = new Backups.FilePointer.LocalLocator({
mediaName,
localKey: Bytes.fromBase64(attachment.localKey),
remoteKey: backupLocator.key,
remoteDigest: backupLocator.digest,
size: backupLocator.size,
backupCdnNumber: backupLocator.cdnNumber,
transitCdnKey: backupLocator.transitCdnKey,
transitCdnNumber: backupLocator.transitCdnNumber,
});
return {
filePointer: {
...omit(remoteFilePointer, 'backupLocator'),
localLocator,
},
updatedAttachment,
};
}
if (remoteFilePointer.attachmentLocator) {
const { attachmentLocator } = remoteFilePointer;
const { digest } = attachmentLocator;
strictAssert(
digest,
'getLocalBackupFilePointerForAttachment: AttachmentLocator must have digest'
);
const mediaName = getMediaNameFromDigest(Bytes.toBase64(digest));
strictAssert(
mediaName,
'getLocalBackupFilePointerForAttachment: mediaName must be derivable from AttachmentLocator'
);
const localLocator = new Backups.FilePointer.LocalLocator({
mediaName,
localKey: Bytes.fromBase64(attachment.localKey),
remoteKey: attachmentLocator.key,
remoteDigest: attachmentLocator.digest,
size: attachmentLocator.size,
backupCdnNumber: undefined,
transitCdnKey: attachmentLocator.cdnKey,
transitCdnNumber: attachmentLocator.cdnNumber,
});
return {
filePointer: {
...omit(remoteFilePointer, 'attachmentLocator'),
localLocator,
},
updatedAttachment,
};
}
return { filePointer: remoteFilePointer, updatedAttachment };
}
function getAttachmentLocator(
attachment: AttachmentDownloadableFromTransitTier
) {
@ -419,21 +572,33 @@ export async function maybeGetBackupJobForAttachmentAndFilePointer({
getBackupCdnInfo: GetBackupCdnInfoType;
messageReceivedAt: number;
}): Promise<CoreAttachmentBackupJobType | null> {
if (!filePointer.backupLocator) {
let locator:
| Backups.FilePointer.IBackupLocator
| Backups.FilePointer.ILocalLocator;
if (filePointer.backupLocator) {
locator = filePointer.backupLocator;
} else if (filePointer.localLocator) {
locator = filePointer.localLocator;
} else {
return null;
}
const { mediaName } = filePointer.backupLocator;
const { mediaName } = locator;
strictAssert(mediaName, 'mediaName must exist');
const { isInBackupTier } = await getBackupCdnInfo(
getMediaIdFromMediaName(mediaName).string
);
if (isInBackupTier) {
return null;
if (filePointer.backupLocator) {
const { isInBackupTier } = await getBackupCdnInfo(
getMediaIdFromMediaName(mediaName).string
);
if (isInBackupTier) {
return null;
}
}
// TODO: For local backups we don't want to double back up the same file, so
// we could check for the same file here and if it's found then return early.
// Also ok to skip downstream when the job runs.
strictAssert(
isAttachmentLocallySaved(attachment),
'Attachment must be saved locally for it to be backed up'
@ -455,19 +620,38 @@ export async function maybeGetBackupJobForAttachmentAndFilePointer({
encryptionInfo = attachment.reencryptionInfo;
}
strictAssert(
filePointer.backupLocator.digest,
'digest must exist on backupLocator'
);
strictAssert(
encryptionInfo.digest === Bytes.toBase64(filePointer.backupLocator.digest),
'digest on job and backupLocator must match'
);
if (filePointer.backupLocator) {
strictAssert(
filePointer.backupLocator.digest,
'digest must exist on backupLocator'
);
strictAssert(
encryptionInfo.digest ===
Bytes.toBase64(filePointer.backupLocator.digest),
'digest on job and backupLocator must match'
);
} else if (filePointer.localLocator) {
strictAssert(
filePointer.localLocator.remoteDigest,
'digest must exist on backupLocator'
);
strictAssert(
encryptionInfo.digest ===
Bytes.toBase64(filePointer.localLocator.remoteDigest),
'digest on job and localLocator must match'
);
}
const { path, contentType, size, uploadTimestamp, version, localKey } =
attachment;
const { path, contentType, size, uploadTimestamp, version } = attachment;
const { transitCdnKey, transitCdnNumber } = filePointer.backupLocator;
const { transitCdnKey, transitCdnNumber } = locator;
let localKey: string | undefined;
if (filePointer.localLocator && filePointer.localLocator.localKey != null) {
localKey = Bytes.toBase64(filePointer.localLocator.localKey);
} else {
localKey = attachment.localKey;
}
return {
mediaName,

296
ts/services/backups/util/localBackup.ts Normal file
View file

@ -0,0 +1,296 @@
// Copyright 2025 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { randomBytes } from 'crypto';
import { dirname, join } from 'path';
import { readFile, stat, writeFile } from 'fs/promises';
import { createReadStream, createWriteStream } from 'fs';
import { Transform } from 'stream';
import { pipeline } from 'stream/promises';
import * as log from '../../../logging/log';
import * as Bytes from '../../../Bytes';
import * as Errors from '../../../types/errors';
import { Signal } from '../../../protobuf';
import protobuf from '../../../protobuf/wrap';
import { strictAssert } from '../../../util/assert';
import { decryptAesCtr, encryptAesCtr } from '../../../Crypto';
import type { LocalBackupMetadataVerificationType } from '../../../types/backups';
import {
LOCAL_BACKUP_VERSION,
LOCAL_BACKUP_BACKUP_ID_IV_LENGTH,
} from '../constants';
import { explodePromise } from '../../../util/explodePromise';
const { Reader } = protobuf;
export function getLocalBackupDirectoryForMediaName({
backupsBaseDir,
mediaName,
}: {
backupsBaseDir: string;
mediaName: string;
}): string {
if (mediaName.length < 2) {
throw new Error('Invalid mediaName input');
}
return join(backupsBaseDir, 'files', mediaName.substring(0, 2));
}
export function getLocalBackupPathForMediaName({
backupsBaseDir,
mediaName,
}: {
backupsBaseDir: string;
mediaName: string;
}): string {
return join(
getLocalBackupDirectoryForMediaName({ backupsBaseDir, mediaName }),
mediaName
);
}
/**
* Given a target local backup import e.g. /etc/SignalBackups/signal-backup-1743119037066
* and an attachment, return the attachment's file path within the local backup
* e.g. /etc/SignalBackups/files/a1/[a1bcdef...]
*
* @param {string} snapshotDir - Timestamped local backup directory
*/
export function getAttachmentLocalBackupPathFromSnapshotDir(
mediaName: string,
snapshotDir: string
): string {
return join(
dirname(snapshotDir),
'files',
mediaName.substring(0, 2),
mediaName
);
}
export async function writeLocalBackupMetadata({
snapshotDir,
backupId,
metadataKey,
}: LocalBackupMetadataVerificationType): Promise<void> {
const iv = randomBytes(LOCAL_BACKUP_BACKUP_ID_IV_LENGTH);
const encryptedId = encryptAesCtr(metadataKey, backupId, iv);
const metadataSerialized = Signal.backup.local.Metadata.encode({
backupId: new Signal.backup.local.Metadata.EncryptedBackupId({
iv,
encryptedId,
}),
version: LOCAL_BACKUP_VERSION,
}).finish();
const metadataPath = join(snapshotDir, 'metadata');
await writeFile(metadataPath, metadataSerialized);
}
export async function verifyLocalBackupMetadata({
snapshotDir,
backupId,
metadataKey,
}: LocalBackupMetadataVerificationType): Promise<boolean> {
const metadataPath = join(snapshotDir, 'metadata');
const metadataSerialized = await readFile(metadataPath);
const metadata = Signal.backup.local.Metadata.decode(metadataSerialized);
strictAssert(
metadata.version === LOCAL_BACKUP_VERSION,
'verifyLocalBackupMetadata: Local backup version must match'
);
strictAssert(
metadata.backupId,
'verifyLocalBackupMetadata: Must have backupId'
);
const { iv, encryptedId } = metadata.backupId;
strictAssert(iv, 'verifyLocalBackupMetadata: Must have backupId.iv');
strictAssert(
encryptedId,
'verifyLocalBackupMetadata: Must have backupId.encryptedId'
);
const localBackupBackupId = decryptAesCtr(metadataKey, encryptedId, iv);
strictAssert(
Bytes.areEqual(backupId, localBackupBackupId),
'verifyLocalBackupMetadata: backupId must match the local backup backupId'
);
return true;
}
export async function writeLocalBackupFilesList({
snapshotDir,
mediaNamesIterator,
}: {
snapshotDir: string;
mediaNamesIterator: MapIterator<string>;
}): Promise<ReadonlyArray<string>> {
const { promise, resolve, reject } = explodePromise<ReadonlyArray<string>>();
const filesListPath = join(snapshotDir, 'files');
const writeStream = createWriteStream(filesListPath);
writeStream.on('error', error => {
reject(error);
});
const files: Array<string> = [];
for (const mediaName of mediaNamesIterator) {
const data = Signal.backup.local.FilesFrame.encodeDelimited({
mediaName,
}).finish();
if (!writeStream.write(data)) {
// eslint-disable-next-line no-await-in-loop
await new Promise(resolveStream =>
writeStream.once('drain', resolveStream)
);
}
files.push(mediaName);
}
writeStream.end(() => {
resolve(files);
});
await promise;
return files;
}
export async function readLocalBackupFilesList(
snapshotDir: string
): Promise<ReadonlyArray<string>> {
const filesListPath = join(snapshotDir, 'files');
const readStream = createReadStream(filesListPath);
const parseFilesTransform = new ParseFilesListTransform();
try {
await pipeline(readStream, parseFilesTransform);
} catch (error) {
try {
readStream.close();
} catch (closeError) {
log.error(
'readLocalBackupFilesList: Error when closing readStream',
Errors.toLogFormat(closeError)
);
}
throw error;
}
readStream.close();
return parseFilesTransform.mediaNames;
}
export class ParseFilesListTransform extends Transform {
public mediaNames: Array<string> = [];
public activeFile: Signal.backup.local.FilesFrame | undefined;
#unused: Uint8Array | undefined;
override async _transform(
chunk: Buffer | undefined,
_encoding: string,
done: (error?: Error) => void
): Promise<void> {
if (!chunk || chunk.byteLength === 0) {
done();
return;
}
try {
let data = chunk;
if (this.#unused) {
data = Buffer.concat([this.#unused, data]);
this.#unused = undefined;
}
const reader = Reader.create(data);
while (reader.pos < reader.len) {
const startPos = reader.pos;
if (!this.activeFile) {
try {
this.activeFile =
Signal.backup.local.FilesFrame.decodeDelimited(reader);
} catch (err) {
// We get a RangeError if there wasn't enough data to read the next record.
if (err instanceof RangeError) {
// Note: A failed decodeDelimited() does in fact update reader.pos, so we
// must reset to startPos
this.#unused = data.subarray(startPos);
done();
return;
}
// Something deeper has gone wrong; the proto is malformed or something
done(err);
return;
}
}
if (!this.activeFile) {
done(
new Error(
'ParseFilesListTransform: No active file after successful decode!'
)
);
return;
}
if (this.activeFile.mediaName) {
this.mediaNames.push(this.activeFile.mediaName);
} else {
log.warn(
'ParseFilesListTransform: Active file had empty mediaName, ignoring'
);
}
this.activeFile = undefined;
}
} catch (error) {
done(error);
return;
}
done();
}
}
export type ValidateLocalBackupStructureResultType =
| { success: true; error: undefined; snapshotDir: string | undefined }
| { success: false; error: string; snapshotDir: string | undefined };
export async function validateLocalBackupStructure(
snapshotDir: string
): Promise<ValidateLocalBackupStructureResultType> {
try {
await stat(snapshotDir);
} catch (error) {
return {
success: false,
error: 'Snapshot directory does not exist',
snapshotDir,
};
}
for (const file of ['main', 'metadata', 'files']) {
try {
// eslint-disable-next-line no-await-in-loop
await stat(join(snapshotDir, 'main'));
} catch (error) {
return {
success: false,
error: `Snapshot directory does not contain ${file} file`,
snapshotDir,
};
}
}
return { success: true, error: undefined, snapshotDir };
}

2
ts/state/ducks/conversations.ts
View file

@ -4088,7 +4088,7 @@ const showSaveMultiDialog = (): Promise<{
canceled: boolean;
dirPath?: string;
}> => {
return ipcRenderer.invoke('show-save-multi-dialog');
return ipcRenderer.invoke('show-open-folder-dialog', { useMainWindow: true });
};
export type SaveAttachmentsActionCreatorType = ReadonlyDeep<

6
ts/test-electron/util/downloadAttachment_test.ts
View file

@ -50,6 +50,7 @@ describe('utils/downloadAttachment', () => {
abortSignal: abortController.signal,
},
dependencies: {
downloadAttachmentFromLocalBackup: stubDownload,
downloadAttachmentFromServer: stubDownload,
},
});
@ -86,6 +87,7 @@ describe('utils/downloadAttachment', () => {
abortSignal: abortController.signal,
},
dependencies: {
downloadAttachmentFromLocalBackup: stubDownload,
downloadAttachmentFromServer: stubDownload,
},
}),
@ -123,6 +125,7 @@ describe('utils/downloadAttachment', () => {
abortSignal: abortController.signal,
},
dependencies: {
downloadAttachmentFromLocalBackup: stubDownload,
downloadAttachmentFromServer: stubDownload,
},
});
@ -161,6 +164,7 @@ describe('utils/downloadAttachment', () => {
abortSignal: abortController.signal,
},
dependencies: {
downloadAttachmentFromLocalBackup: stubDownload,
downloadAttachmentFromServer: stubDownload,
},
});
@ -210,6 +214,7 @@ describe('utils/downloadAttachment', () => {
abortSignal: abortController.signal,
},
dependencies: {
downloadAttachmentFromLocalBackup: stubDownload,
downloadAttachmentFromServer: stubDownload,
},
});
@ -260,6 +265,7 @@ describe('utils/downloadAttachment', () => {
abortSignal: abortController.signal,
},
dependencies: {
downloadAttachmentFromLocalBackup: stubDownload,
downloadAttachmentFromServer: stubDownload,
},
}),

46
ts/test-mock/backups/backups_test.ts
View file

@ -1,12 +1,15 @@
// Copyright 2023 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import fs from 'fs/promises';
import { randomBytes } from 'node:crypto';
import { join } from 'node:path';
import os from 'os';
import { readFile } from 'node:fs/promises';
import createDebug from 'debug';
import Long from 'long';
import { Proto, StorageState } from '@signalapp/mock-server';
import { assert } from 'chai';
import { expect } from 'playwright/test';
import { generateStoryDistributionId } from '../../types/StoryDistributionId';
@ -17,7 +20,7 @@ import { IMAGE_JPEG } from '../../types/MIME';
import { uuidToBytes } from '../../util/uuidToBytes';
import * as durations from '../../util/durations';
import type { App } from '../playwright';
import { Bootstrap } from '../bootstrap';
import { Bootstrap, type LinkOptionsType } from '../bootstrap';
import {
getMessageInTimelineByTimestamp,
sendTextMessage,
@ -60,7 +63,11 @@ describe('backups', function (this: Mocha.Suite) {
await bootstrap.teardown();
});
it('exports and imports regular backup', async function () {
async function generateTestDataThenRestoreBackup(
thisVal: Mocha.Context,
exportBackupFn: () => void,
getBootstrapLinkParams: () => LinkOptionsType
) {
let state = StorageState.getEmpty();
const { phone, contacts } = bootstrap;
@ -237,7 +244,7 @@ describe('backups', function (this: Mocha.Suite) {
.waitFor();
}
await app.uploadBackup();
await exportBackupFn();
const comparator = await bootstrap.createScreenshotComparator(
app,
@ -288,7 +295,7 @@ describe('backups', function (this: Mocha.Suite) {
await snapshot('story privacy');
},
this.test
thisVal.test
);
await app.close();
@ -296,7 +303,7 @@ describe('backups', function (this: Mocha.Suite) {
// Restart
await bootstrap.eraseStorage();
await server.removeAllCDNAttachments();
app = await bootstrap.link();
app = await bootstrap.link(getBootstrapLinkParams());
await app.waitForBackupImportComplete();
// Make sure that contact sync happens after backup import, otherwise the
@ -312,6 +319,35 @@ describe('backups', function (this: Mocha.Suite) {
}
await comparator(app);
}
it('exports and imports local backup', async function () {
let snapshotDir: string;
await generateTestDataThenRestoreBackup(
this,
async () => {
const backupsBaseDir = await fs.mkdtemp(
join(os.tmpdir(), 'SignalBackups')
);
snapshotDir = await app.exportLocalBackup(backupsBaseDir);
assert.exists(
snapshotDir,
'Local backup export should return backup dir'
);
},
() => ({ localBackup: snapshotDir })
);
});
it('exports and imports regular backup', async function () {
await generateTestDataThenRestoreBackup(
this,
async () => {
await app.uploadBackup();
},
() => ({})
);
});
it('imports ephemeral backup', async function () {

6
ts/test-mock/bootstrap.ts
View file

@ -132,6 +132,7 @@ export type EphemeralBackupType = Readonly<
export type LinkOptionsType = Readonly<{
extraConfig?: Partial<RendererConfigType>;
ephemeralBackup?: EphemeralBackupType;
localBackup?: string;
}>;
type BootstrapInternalOptions = BootstrapOptions &
@ -376,6 +377,7 @@ export class Bootstrap {
public async link({
extraConfig,
ephemeralBackup,
localBackup,
}: LinkOptionsType = {}): Promise<App> {
debug('linking');
@ -383,6 +385,10 @@ export class Bootstrap {
const window = await app.getWindow();
if (localBackup != null) {
await app.stageLocalBackupForImport(localBackup);
}
debug('looking for QR code or relink button');
const qrCode = window.locator(
'.module-InstallScreenQrCodeNotScannedStep__qr-code__code'

14
ts/test-mock/playwright.ts
View file

@ -207,6 +207,20 @@ export class App extends EventEmitter {
return window.evaluate(`window.SignalCI.getMessagesBySentAt(${timestamp})`);
}
public async exportLocalBackup(backupsBaseDir: string): Promise<string> {
const window = await this.getWindow();
return window.evaluate(
`window.SignalCI.exportLocalBackup('${backupsBaseDir}')`
);
}
public async stageLocalBackupForImport(snapshotDir: string): Promise<void> {
const window = await this.getWindow();
return window.evaluate(
`window.SignalCI.stageLocalBackupForImport('${snapshotDir}')`
);
}
public async uploadBackup(): Promise<void> {
const window = await this.getWindow();
await window.evaluate('window.SignalCI.uploadBackup()');

2
ts/textsecure/AccountManager.ts
View file

@ -1209,6 +1209,8 @@ export default class AccountManager extends EventTarget {
}
if (accountEntropyPool) {
await storage.put('accountEntropyPool', accountEntropyPool);
} else {
log.warn('createAccount: accountEntropyPool was missing!');
}
let derivedMasterKey = masterKey;
if (derivedMasterKey == null) {

2
ts/textsecure/downloadAttachment.ts
View file

@ -55,7 +55,7 @@ export function getCdnKey(attachment: ProcessedAttachment): string {
return cdnKey;
}
function getBackupMediaOuterEncryptionKeyMaterial(
export function getBackupMediaOuterEncryptionKeyMaterial(
attachment: AttachmentType
): BackupMediaKeyMaterialType {
const mediaId = getMediaIdForAttachment(attachment);

7
ts/types/Attachment.ts
View file

@ -109,6 +109,7 @@ export type AttachmentType = {
mediaName: string;
cdnNumber?: number;
};
localBackupPath?: string;
// See app/attachment_channel.ts
version?: 1 | 2;
@ -1281,6 +1282,12 @@ export function mightBeOnBackupTier(
return Boolean(attachment.backupLocator?.mediaName);
}
export function mightBeInLocalBackup(
attachment: Pick<AttachmentType, 'localBackupPath' | 'localKey'>
): boolean {
return Boolean(attachment.localBackupPath && attachment.localKey);
}
export function isDownloadableFromTransitTier(
attachment: AttachmentType
): attachment is AttachmentDownloadableFromTransitTier {

8
ts/types/AttachmentBackup.ts
View file

@ -45,6 +45,14 @@ export type ThumbnailAttachmentBackupJobType = {
};
};
export type CoreAttachmentLocalBackupJobType =
StandardAttachmentBackupJobType & {
backupsBaseDir: string;
};
export type AttachmentLocalBackupJobType = CoreAttachmentLocalBackupJobType &
JobManagerJobType;
const standardBackupJobDataSchema = z.object({
type: z.literal('standard'),
mediaName: z.string(),

6
ts/types/backups.ts
View file

@ -61,3 +61,9 @@ export type BackupsSubscriptionType =
cost?: SubscriptionCostType;
}
);
export type LocalBackupMetadataVerificationType = {
snapshotDir: string;
backupId: Uint8Array;
metadataKey: Uint8Array;
};

10
ts/util/createIPCEvents.ts
View file

@ -69,7 +69,8 @@ import type {
BackupStatusType,
} from '../types/backups';
import { isBackupFeatureEnabled } from './isBackupEnabled';
import * as RemoteConfig from '../RemoteConfig';
import { isSettingsInternalEnabled } from './isSettingsInternalEnabled';
import type { ValidateLocalBackupStructureResultType } from '../services/backups/util/localBackup';
type SentMediaQualityType = 'standard' | 'high';
type NotificationSettingType = 'message' | 'name' | 'count' | 'off';
@ -164,6 +165,8 @@ export type IPCEventsCallbacksType = {
unknownSignalLink: () => void;
getCustomColors: () => Record<string, CustomColorType>;
syncRequest: () => Promise<void>;
exportLocalBackup: () => Promise<BackupValidationResultType>;
importLocalBackup: () => Promise<ValidateLocalBackupStructureResultType>;
validateBackup: () => Promise<BackupValidationResultType>;
setGlobalDefaultConversationColor: (
color: ConversationColorType,
@ -542,7 +545,7 @@ export function createIPCEvents(
},
isPrimary: () => window.textsecure.storage.user.getDeviceId() === 1,
isInternalUser: () => RemoteConfig.isEnabled('desktop.internalUser'),
isInternalUser: () => isSettingsInternalEnabled(),
syncRequest: async () => {
const contactSyncComplete = waitForEvent(
'contactSync:complete',
@ -552,6 +555,9 @@ export function createIPCEvents(
return contactSyncComplete;
},
// Only for internal use
exportLocalBackup: () => backupsService._internalExportLocalBackup(),
importLocalBackup: () =>
backupsService._internalStageLocalBackupForImport(),
validateBackup: () => backupsService._internalValidate(),
getLastSyncTime: () => window.storage.get('synced_at'),
setLastSyncTime: value => window.storage.put('synced_at', value),

29
ts/util/downloadAttachment.ts
View file

@ -7,8 +7,10 @@ import {
AttachmentVariant,
AttachmentPermanentlyUndownloadableError,
getAttachmentIdForLogging,
mightBeInLocalBackup,
} from '../types/Attachment';
import { downloadAttachment as doDownloadAttachment } from '../textsecure/downloadAttachment';
import { downloadAttachmentFromLocalBackup as doDownloadAttachmentFromLocalBackup } from './downloadAttachmentFromLocalBackup';
import { MediaTier } from '../types/AttachmentDownload';
import * as log from '../logging/log';
import { HTTPError } from '../textsecure/Errors';
@ -18,7 +20,10 @@ import type { ReencryptedAttachmentV2 } from '../AttachmentCrypto';
export async function downloadAttachment({
attachment,
options: { variant = AttachmentVariant.Default, onSizeUpdate, abortSignal },
dependencies = { downloadAttachmentFromServer: doDownloadAttachment },
dependencies = {
downloadAttachmentFromServer: doDownloadAttachment,
downloadAttachmentFromLocalBackup: doDownloadAttachmentFromLocalBackup,
},
}: {
attachment: AttachmentType;
options: {
@ -26,7 +31,10 @@ export async function downloadAttachment({
onSizeUpdate: (totalBytes: number) => void;
abortSignal: AbortSignal;
};
dependencies?: { downloadAttachmentFromServer: typeof doDownloadAttachment };
dependencies?: {
downloadAttachmentFromServer: typeof doDownloadAttachment;
downloadAttachmentFromLocalBackup: typeof doDownloadAttachmentFromLocalBackup;
};
}): Promise<ReencryptedAttachmentV2> {
const attachmentId = getAttachmentIdForLogging(attachment);
const variantForLogging =
@ -51,6 +59,23 @@ export async function downloadAttachment({
};
}
if (mightBeInLocalBackup(attachment)) {
log.info(`${logId}: Downloading attachment from local backup`);
try {
const result =
await dependencies.downloadAttachmentFromLocalBackup(attachment);
onSizeUpdate(attachment.size);
return result;
} catch (error) {
// We also just log this error instead of throwing, since we want to still try to
// find it on the backup then transit tiers.
log.error(
`${logId}: error when downloading from local backup; will try backup and transit tier`,
toLogFormat(error)
);
}
}
if (mightBeOnBackupTier(migratedAttachment)) {
try {
return await dependencies.downloadAttachmentFromServer(

61
ts/util/downloadAttachmentFromLocalBackup.ts Normal file
View file

@ -0,0 +1,61 @@
// Copyright 2025 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { isNumber } from 'lodash';
import {
type AttachmentType,
getAttachmentIdForLogging,
} from '../types/Attachment';
import * as log from '../logging/log';
import { toLogFormat } from '../types/errors';
import {
decryptAndReencryptLocally,
type ReencryptedAttachmentV2,
} from '../AttachmentCrypto';
import { strictAssert } from './assert';
export class AttachmentPermanentlyUndownloadableError extends Error {}
export async function downloadAttachmentFromLocalBackup(
attachment: AttachmentType
): Promise<ReencryptedAttachmentV2> {
const attachmentId = getAttachmentIdForLogging(attachment);
const dataId = `${attachmentId}`;
const logId = `downloadAttachmentFromLocalBackup(${dataId})`;
try {
return await doDownloadFromLocalBackup(attachment, { logId });
} catch (error) {
log.error(
`${logId}: error when copying from local backup`,
toLogFormat(error)
);
throw new AttachmentPermanentlyUndownloadableError();
}
}
async function doDownloadFromLocalBackup(
attachment: AttachmentType,
{
logId,
}: {
logId: string;
}
): Promise<ReencryptedAttachmentV2> {
const { digest, localBackupPath, localKey, size } = attachment;
strictAssert(digest, `${logId}: missing digest`);
strictAssert(localKey, `${logId}: missing localKey`);
strictAssert(localBackupPath, `${logId}: missing localBackupPath`);
strictAssert(isNumber(size), `${logId}: missing size`);
return decryptAndReencryptLocally({
type: 'local',
ciphertextPath: localBackupPath,
idForLogging: logId,
keysBase64: localKey,
size,
getAbsoluteAttachmentPath:
window.Signal.Migrations.getAbsoluteAttachmentPath,
});
}

24
ts/util/isLocalBackupsEnabled.ts Normal file
View file

@ -0,0 +1,24 @@
// Copyright 2025 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import * as RemoteConfig from '../RemoteConfig';
import { isTestOrMockEnvironment } from '../environment';
import { isStagingServer } from './isStagingServer';
import { isNightly } from './version';
export function isLocalBackupsEnabled(): boolean {
if (isStagingServer() || isTestOrMockEnvironment()) {
return true;
}
if (RemoteConfig.isEnabled('desktop.internalUser')) {
return true;
}
const version = window.getVersion?.();
if (version != null) {
return isNightly(version);
}
return false;
}

18
ts/util/isSettingsInternalEnabled.ts Normal file
View file

@ -0,0 +1,18 @@
// Copyright 2025 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import * as RemoteConfig from '../RemoteConfig';
import { isNightly } from './version';
export function isSettingsInternalEnabled(): boolean {
if (RemoteConfig.isEnabled('desktop.internalUser')) {
return true;
}
const version = window.getVersion?.();
if (version != null) {
return isNightly(version);
}
return false;
}

2
ts/windows/preload.ts
View file

@ -51,6 +51,8 @@ installCallback('isInternalUser');
installCallback('syncRequest');
installCallback('getEmojiSkinToneDefault');
installCallback('setEmojiSkinToneDefault');
installCallback('exportLocalBackup');
installCallback('importLocalBackup');
installCallback('validateBackup');
installSetting('alwaysRelayCalls');

4
ts/windows/settings/app.tsx
View file

@ -43,6 +43,7 @@ SettingsWindowProps.onRender(
doDeleteAllData,
doneRendering,
editCustomColor,
exportLocalBackup,
getConversationsWithCustomColor,
hasAudioNotifications,
hasAutoConvertEmoji,
@ -67,6 +68,7 @@ SettingsWindowProps.onRender(
hasStoriesDisabled,
hasTextFormatting,
hasTypingIndicators,
importLocalBackup,
initialSpellCheckSetting,
isAutoDownloadUpdatesSupported,
isAutoLaunchSupported,
@ -152,6 +154,7 @@ SettingsWindowProps.onRender(
defaultConversationColor={defaultConversationColor}
deviceName={deviceName}
emojiSkinToneDefault={emojiSkinToneDefault}
exportLocalBackup={exportLocalBackup}
phoneNumber={phoneNumber}
doDeleteAllData={doDeleteAllData}
doneRendering={doneRendering}
@ -181,6 +184,7 @@ SettingsWindowProps.onRender(
hasTextFormatting={hasTextFormatting}
hasTypingIndicators={hasTypingIndicators}
i18n={i18n}
importLocalBackup={importLocalBackup}
initialSpellCheckSetting={initialSpellCheckSetting}
isAutoDownloadUpdatesSupported={isAutoDownloadUpdatesSupported}
isAutoLaunchSupported={isAutoLaunchSupported}

4
ts/windows/settings/preload.ts
View file

@ -100,6 +100,8 @@ const ipcGetEmojiSkinToneDefault = createCallback('getEmojiSkinToneDefault');
const ipcIsSyncNotSupported = createCallback('isPrimary');
const ipcIsInternalUser = createCallback('isInternalUser');
const ipcMakeSyncRequest = createCallback('syncRequest');
const ipcExportLocalBackup = createCallback('exportLocalBackup');
const ipcImportLocalBackup = createCallback('importLocalBackup');
const ipcValidateBackup = createCallback('validateBackup');
const ipcDeleteAllMyStories = createCallback('deleteAllMyStories');
const ipcRefreshCloudBackupStatus = createCallback('refreshCloudBackupStatus');
@ -369,6 +371,8 @@ async function renderPreferences() {
resetAllChatColors: ipcResetAllChatColors,
resetDefaultChatColor: ipcResetDefaultChatColor,
setGlobalDefaultConversationColor: ipcSetGlobalDefaultConversationColor,
exportLocalBackup: ipcExportLocalBackup,
importLocalBackup: ipcImportLocalBackup,
validateBackup: ipcValidateBackup,
// Limited support features
isAutoDownloadUpdatesSupported: Settings.isAutoDownloadUpdatesSupported(