Enable attachment backup uploading

This commit is contained in:
trevor-signal 2024-05-29 19:46:43 -04:00 committed by GitHub
parent 94a262b799
commit 4254356812
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 2054 additions and 534 deletions

View file

@ -33,6 +33,7 @@ import type { CallLinkStateType, CallLinkType } from '../types/CallLink';
import type { AttachmentDownloadJobType } from '../types/AttachmentDownload';
import type { GroupSendEndorsementsData } from '../types/GroupSendEndorsements';
import type { SyncTaskType } from '../util/syncTasks';
import type { AttachmentBackupJobType } from '../types/AttachmentBackup';
export type AdjacentMessagesByConversationOptionsType = Readonly<{
conversationId: string;
@ -425,6 +426,12 @@ export type EditedMessageType = Readonly<{
readStatus: MessageType['readStatus'];
}>;
export type BackupCdnMediaObjectType = {
mediaId: string;
cdnNumber: number;
sizeOnBackupCdn: number;
};
export type DataInterface = {
close: () => Promise<void>;
pauseWriteAccess(): Promise<void>;
@ -761,6 +768,22 @@ export type DataInterface = {
job: AttachmentDownloadJobType
) => Promise<void>;
getNextAttachmentBackupJobs: (options: {
limit: number;
timestamp?: number;
}) => Promise<Array<AttachmentBackupJobType>>;
saveAttachmentBackupJob: (job: AttachmentBackupJobType) => Promise<void>;
markAllAttachmentBackupJobsInactive: () => Promise<void>;
removeAttachmentBackupJob: (job: AttachmentBackupJobType) => Promise<void>;
clearAllBackupCdnObjectMetadata: () => Promise<void>;
saveBackupCdnObjectMetadata: (
mediaObjects: Array<BackupCdnMediaObjectType>
) => Promise<void>;
getBackupCdnObjectMetadata: (
mediaId: string
) => Promise<BackupCdnMediaObjectType | undefined>;
createOrUpdateStickerPack: (pack: StickerPackType) => Promise<void>;
updateStickerPackStatus: (
id: string,

View file

@ -144,6 +144,7 @@ import type {
UnprocessedUpdateType,
GetNearbyMessageFromDeletedSetOptionsType,
StoredKyberPreKeyType,
BackupCdnMediaObjectType,
} from './Interface';
import { SeenStatus } from '../MessageSeenStatus';
import {
@ -187,6 +188,11 @@ import {
import { MAX_SYNC_TASK_ATTEMPTS } from '../util/syncTasks.types';
import type { SyncTaskType } from '../util/syncTasks';
import { isMoreRecentThan } from '../util/timestamp';
import {
type AttachmentBackupJobType,
attachmentBackupJobSchema,
} from '../types/AttachmentBackup';
import { redactGenericText } from '../util/privacy';
type ConversationRow = Readonly<{
json: string;
@ -384,6 +390,15 @@ const dataInterface: ServerInterface = {
resetAttachmentDownloadActive,
removeAttachmentDownloadJob,
getNextAttachmentBackupJobs,
saveAttachmentBackupJob,
markAllAttachmentBackupJobsInactive,
removeAttachmentBackupJob,
clearAllBackupCdnObjectMetadata,
saveBackupCdnObjectMetadata,
getBackupCdnObjectMetadata,
createOrUpdateStickerPack,
updateStickerPackStatus,
updateStickerPackInfo,
@ -4843,6 +4858,157 @@ async function removeAttachmentDownloadJob(
return removeAttachmentDownloadJobSync(db, job);
}
// Backup Attachments
async function markAllAttachmentBackupJobsInactive(): Promise<void> {
const db = await getWritableInstance();
db.prepare<EmptyQuery>(
`
UPDATE attachment_backup_jobs
SET active = 0;
`
).run();
}
async function saveAttachmentBackupJob(
job: AttachmentBackupJobType
): Promise<void> {
const db = await getWritableInstance();
const [query, params] = sql`
INSERT OR REPLACE INTO attachment_backup_jobs (
active,
attempts,
data,
lastAttemptTimestamp,
mediaName,
receivedAt,
retryAfter,
type
) VALUES (
${job.active ? 1 : 0},
${job.attempts},
${objectToJSON(job.data)},
${job.lastAttemptTimestamp},
${job.mediaName},
${job.receivedAt},
${job.retryAfter},
${job.type}
);
`;
db.prepare(query).run(params);
}
async function getNextAttachmentBackupJobs({
limit,
timestamp = Date.now(),
}: {
limit: number;
timestamp?: number;
}): Promise<Array<AttachmentBackupJobType>> {
const db = await getWritableInstance();
const [query, params] = sql`
SELECT * FROM attachment_backup_jobs
WHERE
active = 0
AND
(retryAfter is NULL OR retryAfter <= ${timestamp})
ORDER BY receivedAt DESC
LIMIT ${limit}
`;
const rows = db.prepare(query).all(params);
return rows
.map(row => {
const parseResult = attachmentBackupJobSchema.safeParse({
...row,
active: Boolean(row.active),
data: jsonToObject(row.data),
});
if (!parseResult.success) {
const redactedMediaName = redactGenericText(row.mediaName);
logger.error(
`getNextAttachmentBackupJobs: invalid data, removing. mediaName: ${redactedMediaName}`,
Errors.toLogFormat(parseResult.error)
);
removeAttachmentBackupJobSync(db, { mediaName: row.mediaName });
return null;
}
return parseResult.data;
})
.filter(isNotNil);
}
async function removeAttachmentBackupJob(
job: Pick<AttachmentBackupJobType, 'mediaName'>
): Promise<void> {
const db = await getWritableInstance();
return removeAttachmentBackupJobSync(db, job);
}
function removeAttachmentBackupJobSync(
db: Database,
job: Pick<AttachmentBackupJobType, 'mediaName'>
): void {
const [query, params] = sql`
DELETE FROM attachment_backup_jobs
WHERE
mediaName = ${job.mediaName};
`;
db.prepare(query).run(params);
}
// Attachments on backup CDN
async function clearAllBackupCdnObjectMetadata(): Promise<void> {
const db = await getWritableInstance();
db.prepare('DELETE FROM backup_cdn_object_metadata;').run();
}
function saveBackupCdnObjectMetadataSync(
db: Database,
storedMediaObject: BackupCdnMediaObjectType
) {
const { mediaId, cdnNumber, sizeOnBackupCdn } = storedMediaObject;
const [query, params] = sql`
INSERT OR REPLACE INTO backup_cdn_object_metadata
(
mediaId,
cdnNumber,
sizeOnBackupCdn
) VALUES (
${mediaId},
${cdnNumber},
${sizeOnBackupCdn}
);
`;
db.prepare(query).run(params);
}
async function saveBackupCdnObjectMetadata(
storedMediaObjects: Array<BackupCdnMediaObjectType>
): Promise<void> {
const db = await getWritableInstance();
db.transaction(() => {
for (const obj of storedMediaObjects) {
saveBackupCdnObjectMetadataSync(db, obj);
}
})();
}
async function getBackupCdnObjectMetadata(
mediaId: string
): Promise<BackupCdnMediaObjectType | undefined> {
const db = getReadonlyInstance();
const [
query,
params,
] = sql`SELECT * from backup_cdn_object_metadata WHERE mediaId = ${mediaId}`;
return db.prepare(query).get(params);
}
// Stickers
async function createOrUpdateStickerPack(pack: StickerPackType): Promise<void> {
@ -6140,6 +6306,8 @@ async function removeAll(): Promise<void> {
DROP TRIGGER messages_on_delete;
DELETE FROM attachment_downloads;
DELETE FROM attachment_backup_jobs;
DELETE FROM backup_cdn_object_metadata;
DELETE FROM badgeImageFiles;
DELETE FROM badges;
DELETE FROM callLinks;
@ -6200,6 +6368,8 @@ async function removeAllConfiguration(): Promise<void> {
db.transaction(() => {
db.exec(
`
DELETE FROM attachment_backup_jobs;
DELETE FROM backup_cdn_object_metadata;
DELETE FROM groupSendCombinedEndorsement;
DELETE FROM groupSendMemberEndorsement;
DELETE FROM identityKeys;

View file

@ -0,0 +1,55 @@
// Copyright 2024 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import type { Database } from '@signalapp/better-sqlite3';
import type { LoggerType } from '../../types/Logging';
export const version = 1070;
export function updateToSchemaVersion1070(
currentVersion: number,
db: Database,
logger: LoggerType
): void {
if (currentVersion >= 1070) {
return;
}
db.transaction(() => {
db.exec(`
CREATE TABLE attachment_backup_jobs (
mediaName TEXT NOT NULL PRIMARY KEY,
type TEXT NOT NULL,
data TEXT NOT NULL,
receivedAt INTEGER NOT NULL,
-- job manager fields
attempts INTEGER NOT NULL,
active INTEGER NOT NULL,
retryAfter INTEGER,
lastAttemptTimestamp INTEGER
) STRICT;
CREATE INDEX attachment_backup_jobs_receivedAt
ON attachment_backup_jobs (
receivedAt
);
CREATE INDEX attachment_backup_jobs_type_receivedAt
ON attachment_backup_jobs (
type, receivedAt
);
CREATE TABLE backup_cdn_object_metadata (
mediaId TEXT NOT NULL PRIMARY KEY,
cdnNumber INTEGER NOT NULL,
sizeOnBackupCdn INTEGER
) STRICT;
`);
})();
db.pragma('user_version = 1070');
logger.info('updateToSchemaVersion1070: success!');
}

View file

@ -81,10 +81,11 @@ import { updateToSchemaVersion1020 } from './1020-self-merges';
import { updateToSchemaVersion1030 } from './1030-unblock-event';
import { updateToSchemaVersion1040 } from './1040-undownloaded-backed-up-media';
import { updateToSchemaVersion1050 } from './1050-group-send-endorsements';
import { updateToSchemaVersion1060 } from './1060-addressable-messages-and-sync-tasks';
import {
updateToSchemaVersion1060,
updateToSchemaVersion1070,
version as MAX_VERSION,
} from './1060-addressable-messages-and-sync-tasks';
} from './1070-attachment-backup';
function updateToSchemaVersion1(
currentVersion: number,
@ -2034,6 +2035,7 @@ export const SCHEMA_VERSIONS = [
updateToSchemaVersion1040,
updateToSchemaVersion1050,
updateToSchemaVersion1060,
updateToSchemaVersion1070,
];
export class DBVersionFromFutureError extends Error {