Enable attachment backup uploading
This commit is contained in:
parent
94a262b799
commit
4254356812
27 changed files with 2054 additions and 534 deletions
|
@ -33,6 +33,7 @@ import type { CallLinkStateType, CallLinkType } from '../types/CallLink';
|
|||
import type { AttachmentDownloadJobType } from '../types/AttachmentDownload';
|
||||
import type { GroupSendEndorsementsData } from '../types/GroupSendEndorsements';
|
||||
import type { SyncTaskType } from '../util/syncTasks';
|
||||
import type { AttachmentBackupJobType } from '../types/AttachmentBackup';
|
||||
|
||||
export type AdjacentMessagesByConversationOptionsType = Readonly<{
|
||||
conversationId: string;
|
||||
|
@ -425,6 +426,12 @@ export type EditedMessageType = Readonly<{
|
|||
readStatus: MessageType['readStatus'];
|
||||
}>;
|
||||
|
||||
export type BackupCdnMediaObjectType = {
|
||||
mediaId: string;
|
||||
cdnNumber: number;
|
||||
sizeOnBackupCdn: number;
|
||||
};
|
||||
|
||||
export type DataInterface = {
|
||||
close: () => Promise<void>;
|
||||
pauseWriteAccess(): Promise<void>;
|
||||
|
@ -761,6 +768,22 @@ export type DataInterface = {
|
|||
job: AttachmentDownloadJobType
|
||||
) => Promise<void>;
|
||||
|
||||
getNextAttachmentBackupJobs: (options: {
|
||||
limit: number;
|
||||
timestamp?: number;
|
||||
}) => Promise<Array<AttachmentBackupJobType>>;
|
||||
saveAttachmentBackupJob: (job: AttachmentBackupJobType) => Promise<void>;
|
||||
markAllAttachmentBackupJobsInactive: () => Promise<void>;
|
||||
removeAttachmentBackupJob: (job: AttachmentBackupJobType) => Promise<void>;
|
||||
|
||||
clearAllBackupCdnObjectMetadata: () => Promise<void>;
|
||||
saveBackupCdnObjectMetadata: (
|
||||
mediaObjects: Array<BackupCdnMediaObjectType>
|
||||
) => Promise<void>;
|
||||
getBackupCdnObjectMetadata: (
|
||||
mediaId: string
|
||||
) => Promise<BackupCdnMediaObjectType | undefined>;
|
||||
|
||||
createOrUpdateStickerPack: (pack: StickerPackType) => Promise<void>;
|
||||
updateStickerPackStatus: (
|
||||
id: string,
|
||||
|
|
170
ts/sql/Server.ts
170
ts/sql/Server.ts
|
@ -144,6 +144,7 @@ import type {
|
|||
UnprocessedUpdateType,
|
||||
GetNearbyMessageFromDeletedSetOptionsType,
|
||||
StoredKyberPreKeyType,
|
||||
BackupCdnMediaObjectType,
|
||||
} from './Interface';
|
||||
import { SeenStatus } from '../MessageSeenStatus';
|
||||
import {
|
||||
|
@ -187,6 +188,11 @@ import {
|
|||
import { MAX_SYNC_TASK_ATTEMPTS } from '../util/syncTasks.types';
|
||||
import type { SyncTaskType } from '../util/syncTasks';
|
||||
import { isMoreRecentThan } from '../util/timestamp';
|
||||
import {
|
||||
type AttachmentBackupJobType,
|
||||
attachmentBackupJobSchema,
|
||||
} from '../types/AttachmentBackup';
|
||||
import { redactGenericText } from '../util/privacy';
|
||||
|
||||
type ConversationRow = Readonly<{
|
||||
json: string;
|
||||
|
@ -384,6 +390,15 @@ const dataInterface: ServerInterface = {
|
|||
resetAttachmentDownloadActive,
|
||||
removeAttachmentDownloadJob,
|
||||
|
||||
getNextAttachmentBackupJobs,
|
||||
saveAttachmentBackupJob,
|
||||
markAllAttachmentBackupJobsInactive,
|
||||
removeAttachmentBackupJob,
|
||||
|
||||
clearAllBackupCdnObjectMetadata,
|
||||
saveBackupCdnObjectMetadata,
|
||||
getBackupCdnObjectMetadata,
|
||||
|
||||
createOrUpdateStickerPack,
|
||||
updateStickerPackStatus,
|
||||
updateStickerPackInfo,
|
||||
|
@ -4843,6 +4858,157 @@ async function removeAttachmentDownloadJob(
|
|||
return removeAttachmentDownloadJobSync(db, job);
|
||||
}
|
||||
|
||||
// Backup Attachments
|
||||
|
||||
async function markAllAttachmentBackupJobsInactive(): Promise<void> {
|
||||
const db = await getWritableInstance();
|
||||
db.prepare<EmptyQuery>(
|
||||
`
|
||||
UPDATE attachment_backup_jobs
|
||||
SET active = 0;
|
||||
`
|
||||
).run();
|
||||
}
|
||||
|
||||
async function saveAttachmentBackupJob(
|
||||
job: AttachmentBackupJobType
|
||||
): Promise<void> {
|
||||
const db = await getWritableInstance();
|
||||
|
||||
const [query, params] = sql`
|
||||
INSERT OR REPLACE INTO attachment_backup_jobs (
|
||||
active,
|
||||
attempts,
|
||||
data,
|
||||
lastAttemptTimestamp,
|
||||
mediaName,
|
||||
receivedAt,
|
||||
retryAfter,
|
||||
type
|
||||
) VALUES (
|
||||
${job.active ? 1 : 0},
|
||||
${job.attempts},
|
||||
${objectToJSON(job.data)},
|
||||
${job.lastAttemptTimestamp},
|
||||
${job.mediaName},
|
||||
${job.receivedAt},
|
||||
${job.retryAfter},
|
||||
${job.type}
|
||||
);
|
||||
`;
|
||||
db.prepare(query).run(params);
|
||||
}
|
||||
|
||||
async function getNextAttachmentBackupJobs({
|
||||
limit,
|
||||
timestamp = Date.now(),
|
||||
}: {
|
||||
limit: number;
|
||||
timestamp?: number;
|
||||
}): Promise<Array<AttachmentBackupJobType>> {
|
||||
const db = await getWritableInstance();
|
||||
|
||||
const [query, params] = sql`
|
||||
SELECT * FROM attachment_backup_jobs
|
||||
WHERE
|
||||
active = 0
|
||||
AND
|
||||
(retryAfter is NULL OR retryAfter <= ${timestamp})
|
||||
ORDER BY receivedAt DESC
|
||||
LIMIT ${limit}
|
||||
`;
|
||||
const rows = db.prepare(query).all(params);
|
||||
return rows
|
||||
.map(row => {
|
||||
const parseResult = attachmentBackupJobSchema.safeParse({
|
||||
...row,
|
||||
active: Boolean(row.active),
|
||||
data: jsonToObject(row.data),
|
||||
});
|
||||
if (!parseResult.success) {
|
||||
const redactedMediaName = redactGenericText(row.mediaName);
|
||||
logger.error(
|
||||
`getNextAttachmentBackupJobs: invalid data, removing. mediaName: ${redactedMediaName}`,
|
||||
Errors.toLogFormat(parseResult.error)
|
||||
);
|
||||
removeAttachmentBackupJobSync(db, { mediaName: row.mediaName });
|
||||
return null;
|
||||
}
|
||||
return parseResult.data;
|
||||
})
|
||||
.filter(isNotNil);
|
||||
}
|
||||
|
||||
async function removeAttachmentBackupJob(
|
||||
job: Pick<AttachmentBackupJobType, 'mediaName'>
|
||||
): Promise<void> {
|
||||
const db = await getWritableInstance();
|
||||
return removeAttachmentBackupJobSync(db, job);
|
||||
}
|
||||
|
||||
function removeAttachmentBackupJobSync(
|
||||
db: Database,
|
||||
job: Pick<AttachmentBackupJobType, 'mediaName'>
|
||||
): void {
|
||||
const [query, params] = sql`
|
||||
DELETE FROM attachment_backup_jobs
|
||||
WHERE
|
||||
mediaName = ${job.mediaName};
|
||||
`;
|
||||
|
||||
db.prepare(query).run(params);
|
||||
}
|
||||
|
||||
// Attachments on backup CDN
|
||||
async function clearAllBackupCdnObjectMetadata(): Promise<void> {
|
||||
const db = await getWritableInstance();
|
||||
db.prepare('DELETE FROM backup_cdn_object_metadata;').run();
|
||||
}
|
||||
|
||||
function saveBackupCdnObjectMetadataSync(
|
||||
db: Database,
|
||||
storedMediaObject: BackupCdnMediaObjectType
|
||||
) {
|
||||
const { mediaId, cdnNumber, sizeOnBackupCdn } = storedMediaObject;
|
||||
const [query, params] = sql`
|
||||
INSERT OR REPLACE INTO backup_cdn_object_metadata
|
||||
(
|
||||
mediaId,
|
||||
cdnNumber,
|
||||
sizeOnBackupCdn
|
||||
) VALUES (
|
||||
${mediaId},
|
||||
${cdnNumber},
|
||||
${sizeOnBackupCdn}
|
||||
);
|
||||
`;
|
||||
|
||||
db.prepare(query).run(params);
|
||||
}
|
||||
|
||||
async function saveBackupCdnObjectMetadata(
|
||||
storedMediaObjects: Array<BackupCdnMediaObjectType>
|
||||
): Promise<void> {
|
||||
const db = await getWritableInstance();
|
||||
db.transaction(() => {
|
||||
for (const obj of storedMediaObjects) {
|
||||
saveBackupCdnObjectMetadataSync(db, obj);
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
async function getBackupCdnObjectMetadata(
|
||||
mediaId: string
|
||||
): Promise<BackupCdnMediaObjectType | undefined> {
|
||||
const db = getReadonlyInstance();
|
||||
const [
|
||||
query,
|
||||
params,
|
||||
] = sql`SELECT * from backup_cdn_object_metadata WHERE mediaId = ${mediaId}`;
|
||||
|
||||
return db.prepare(query).get(params);
|
||||
}
|
||||
|
||||
// Stickers
|
||||
|
||||
async function createOrUpdateStickerPack(pack: StickerPackType): Promise<void> {
|
||||
|
@ -6140,6 +6306,8 @@ async function removeAll(): Promise<void> {
|
|||
DROP TRIGGER messages_on_delete;
|
||||
|
||||
DELETE FROM attachment_downloads;
|
||||
DELETE FROM attachment_backup_jobs;
|
||||
DELETE FROM backup_cdn_object_metadata;
|
||||
DELETE FROM badgeImageFiles;
|
||||
DELETE FROM badges;
|
||||
DELETE FROM callLinks;
|
||||
|
@ -6200,6 +6368,8 @@ async function removeAllConfiguration(): Promise<void> {
|
|||
db.transaction(() => {
|
||||
db.exec(
|
||||
`
|
||||
DELETE FROM attachment_backup_jobs;
|
||||
DELETE FROM backup_cdn_object_metadata;
|
||||
DELETE FROM groupSendCombinedEndorsement;
|
||||
DELETE FROM groupSendMemberEndorsement;
|
||||
DELETE FROM identityKeys;
|
||||
|
|
55
ts/sql/migrations/1070-attachment-backup.ts
Normal file
55
ts/sql/migrations/1070-attachment-backup.ts
Normal file
|
@ -0,0 +1,55 @@
|
|||
// Copyright 2024 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import type { Database } from '@signalapp/better-sqlite3';
|
||||
|
||||
import type { LoggerType } from '../../types/Logging';
|
||||
|
||||
export const version = 1070;
|
||||
|
||||
export function updateToSchemaVersion1070(
|
||||
currentVersion: number,
|
||||
db: Database,
|
||||
logger: LoggerType
|
||||
): void {
|
||||
if (currentVersion >= 1070) {
|
||||
return;
|
||||
}
|
||||
|
||||
db.transaction(() => {
|
||||
db.exec(`
|
||||
CREATE TABLE attachment_backup_jobs (
|
||||
mediaName TEXT NOT NULL PRIMARY KEY,
|
||||
type TEXT NOT NULL,
|
||||
data TEXT NOT NULL,
|
||||
receivedAt INTEGER NOT NULL,
|
||||
|
||||
-- job manager fields
|
||||
attempts INTEGER NOT NULL,
|
||||
active INTEGER NOT NULL,
|
||||
retryAfter INTEGER,
|
||||
lastAttemptTimestamp INTEGER
|
||||
) STRICT;
|
||||
|
||||
CREATE INDEX attachment_backup_jobs_receivedAt
|
||||
ON attachment_backup_jobs (
|
||||
receivedAt
|
||||
);
|
||||
|
||||
CREATE INDEX attachment_backup_jobs_type_receivedAt
|
||||
ON attachment_backup_jobs (
|
||||
type, receivedAt
|
||||
);
|
||||
|
||||
CREATE TABLE backup_cdn_object_metadata (
|
||||
mediaId TEXT NOT NULL PRIMARY KEY,
|
||||
cdnNumber INTEGER NOT NULL,
|
||||
sizeOnBackupCdn INTEGER
|
||||
) STRICT;
|
||||
`);
|
||||
})();
|
||||
|
||||
db.pragma('user_version = 1070');
|
||||
|
||||
logger.info('updateToSchemaVersion1070: success!');
|
||||
}
|
|
@ -81,10 +81,11 @@ import { updateToSchemaVersion1020 } from './1020-self-merges';
|
|||
import { updateToSchemaVersion1030 } from './1030-unblock-event';
|
||||
import { updateToSchemaVersion1040 } from './1040-undownloaded-backed-up-media';
|
||||
import { updateToSchemaVersion1050 } from './1050-group-send-endorsements';
|
||||
import { updateToSchemaVersion1060 } from './1060-addressable-messages-and-sync-tasks';
|
||||
import {
|
||||
updateToSchemaVersion1060,
|
||||
updateToSchemaVersion1070,
|
||||
version as MAX_VERSION,
|
||||
} from './1060-addressable-messages-and-sync-tasks';
|
||||
} from './1070-attachment-backup';
|
||||
|
||||
function updateToSchemaVersion1(
|
||||
currentVersion: number,
|
||||
|
@ -2034,6 +2035,7 @@ export const SCHEMA_VERSIONS = [
|
|||
updateToSchemaVersion1040,
|
||||
updateToSchemaVersion1050,
|
||||
updateToSchemaVersion1060,
|
||||
updateToSchemaVersion1070,
|
||||
];
|
||||
|
||||
export class DBVersionFromFutureError extends Error {
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue