Enable attachment backup uploading

ts/sql/Interface.ts

@@ -33,6 +33,7 @@ import type { CallLinkStateType, CallLinkType } from '../types/CallLink';
 import type { AttachmentDownloadJobType } from '../types/AttachmentDownload';
 import type { GroupSendEndorsementsData } from '../types/GroupSendEndorsements';
 import type { SyncTaskType } from '../util/syncTasks';
+import type { AttachmentBackupJobType } from '../types/AttachmentBackup';
 
 export type AdjacentMessagesByConversationOptionsType = Readonly<{
   conversationId: string;
@@ -425,6 +426,12 @@ export type EditedMessageType = Readonly<{
   readStatus: MessageType['readStatus'];
 }>;
 
+export type BackupCdnMediaObjectType = {
+  mediaId: string;
+  cdnNumber: number;
+  sizeOnBackupCdn: number;
+};
+
 export type DataInterface = {
   close: () => Promise<void>;
   pauseWriteAccess(): Promise<void>;
@@ -761,6 +768,22 @@ export type DataInterface = {
     job: AttachmentDownloadJobType
   ) => Promise<void>;
 
+  getNextAttachmentBackupJobs: (options: {
+    limit: number;
+    timestamp?: number;
+  }) => Promise<Array<AttachmentBackupJobType>>;
+  saveAttachmentBackupJob: (job: AttachmentBackupJobType) => Promise<void>;
+  markAllAttachmentBackupJobsInactive: () => Promise<void>;
+  removeAttachmentBackupJob: (job: AttachmentBackupJobType) => Promise<void>;
+
+  clearAllBackupCdnObjectMetadata: () => Promise<void>;
+  saveBackupCdnObjectMetadata: (
+    mediaObjects: Array<BackupCdnMediaObjectType>
+  ) => Promise<void>;
+  getBackupCdnObjectMetadata: (
+    mediaId: string
+  ) => Promise<BackupCdnMediaObjectType | undefined>;
+
   createOrUpdateStickerPack: (pack: StickerPackType) => Promise<void>;
   updateStickerPackStatus: (
     id: string,

ts/sql/Server.ts

@@ -144,6 +144,7 @@ import type {
   UnprocessedUpdateType,
   GetNearbyMessageFromDeletedSetOptionsType,
   StoredKyberPreKeyType,
+  BackupCdnMediaObjectType,
 } from './Interface';
 import { SeenStatus } from '../MessageSeenStatus';
 import {
@@ -187,6 +188,11 @@ import {
 import { MAX_SYNC_TASK_ATTEMPTS } from '../util/syncTasks.types';
 import type { SyncTaskType } from '../util/syncTasks';
 import { isMoreRecentThan } from '../util/timestamp';
+import {
+  type AttachmentBackupJobType,
+  attachmentBackupJobSchema,
+} from '../types/AttachmentBackup';
+import { redactGenericText } from '../util/privacy';
 
 type ConversationRow = Readonly<{
   json: string;
@@ -384,6 +390,15 @@ const dataInterface: ServerInterface = {
   resetAttachmentDownloadActive,
   removeAttachmentDownloadJob,
 
+  getNextAttachmentBackupJobs,
+  saveAttachmentBackupJob,
+  markAllAttachmentBackupJobsInactive,
+  removeAttachmentBackupJob,
+
+  clearAllBackupCdnObjectMetadata,
+  saveBackupCdnObjectMetadata,
+  getBackupCdnObjectMetadata,
+
   createOrUpdateStickerPack,
   updateStickerPackStatus,
   updateStickerPackInfo,
@@ -4843,6 +4858,157 @@ async function removeAttachmentDownloadJob(
   return removeAttachmentDownloadJobSync(db, job);
 }
 
+// Backup Attachments
+
+async function markAllAttachmentBackupJobsInactive(): Promise<void> {
+  const db = await getWritableInstance();
+  db.prepare<EmptyQuery>(
+    `
+    UPDATE attachment_backup_jobs
+    SET active = 0;
+    `
+  ).run();
+}
+
+async function saveAttachmentBackupJob(
+  job: AttachmentBackupJobType
+): Promise<void> {
+  const db = await getWritableInstance();
+
+  const [query, params] = sql`
+    INSERT OR REPLACE INTO attachment_backup_jobs (
+      active,
+      attempts,
+      data,
+      lastAttemptTimestamp,
+      mediaName, 
+      receivedAt,
+      retryAfter,
+      type
+    ) VALUES (
+      ${job.active ? 1 : 0},
+      ${job.attempts},
+      ${objectToJSON(job.data)},
+      ${job.lastAttemptTimestamp},
+      ${job.mediaName},
+      ${job.receivedAt},
+      ${job.retryAfter},
+      ${job.type}
+    );
+  `;
+  db.prepare(query).run(params);
+}
+
+async function getNextAttachmentBackupJobs({
+  limit,
+  timestamp = Date.now(),
+}: {
+  limit: number;
+  timestamp?: number;
+}): Promise<Array<AttachmentBackupJobType>> {
+  const db = await getWritableInstance();
+
+  const [query, params] = sql`
+    SELECT * FROM attachment_backup_jobs
+    WHERE
+      active = 0
+    AND
+      (retryAfter is NULL OR retryAfter <= ${timestamp})
+    ORDER BY receivedAt DESC
+    LIMIT ${limit}
+  `;
+  const rows = db.prepare(query).all(params);
+  return rows
+    .map(row => {
+      const parseResult = attachmentBackupJobSchema.safeParse({
+        ...row,
+        active: Boolean(row.active),
+        data: jsonToObject(row.data),
+      });
+      if (!parseResult.success) {
+        const redactedMediaName = redactGenericText(row.mediaName);
+        logger.error(
+          `getNextAttachmentBackupJobs: invalid data, removing. mediaName: ${redactedMediaName}`,
+          Errors.toLogFormat(parseResult.error)
+        );
+        removeAttachmentBackupJobSync(db, { mediaName: row.mediaName });
+        return null;
+      }
+      return parseResult.data;
+    })
+    .filter(isNotNil);
+}
+
+async function removeAttachmentBackupJob(
+  job: Pick<AttachmentBackupJobType, 'mediaName'>
+): Promise<void> {
+  const db = await getWritableInstance();
+  return removeAttachmentBackupJobSync(db, job);
+}
+
+function removeAttachmentBackupJobSync(
+  db: Database,
+  job: Pick<AttachmentBackupJobType, 'mediaName'>
+): void {
+  const [query, params] = sql`
+  DELETE FROM attachment_backup_jobs
+  WHERE 
+    mediaName = ${job.mediaName};
+`;
+
+  db.prepare(query).run(params);
+}
+
+// Attachments on backup CDN
+async function clearAllBackupCdnObjectMetadata(): Promise<void> {
+  const db = await getWritableInstance();
+  db.prepare('DELETE FROM backup_cdn_object_metadata;').run();
+}
+
+function saveBackupCdnObjectMetadataSync(
+  db: Database,
+  storedMediaObject: BackupCdnMediaObjectType
+) {
+  const { mediaId, cdnNumber, sizeOnBackupCdn } = storedMediaObject;
+  const [query, params] = sql`
+    INSERT OR REPLACE INTO backup_cdn_object_metadata
+    (
+      mediaId,
+      cdnNumber,
+      sizeOnBackupCdn
+    ) VALUES (
+      ${mediaId},
+      ${cdnNumber},
+      ${sizeOnBackupCdn}
+    );
+  `;
+
+  db.prepare(query).run(params);
+}
+
+async function saveBackupCdnObjectMetadata(
+  storedMediaObjects: Array<BackupCdnMediaObjectType>
+): Promise<void> {
+  const db = await getWritableInstance();
+  db.transaction(() => {
+    for (const obj of storedMediaObjects) {
+      saveBackupCdnObjectMetadataSync(db, obj);
+    }
+  })();
+}
+
+async function getBackupCdnObjectMetadata(
+  mediaId: string
+): Promise<BackupCdnMediaObjectType | undefined> {
+  const db = getReadonlyInstance();
+  const [
+    query,
+    params,
+  ] = sql`SELECT * from backup_cdn_object_metadata WHERE mediaId = ${mediaId}`;
+
+  return db.prepare(query).get(params);
+}
+
 // Stickers
 
 async function createOrUpdateStickerPack(pack: StickerPackType): Promise<void> {
@@ -6140,6 +6306,8 @@ async function removeAll(): Promise<void> {
       DROP   TRIGGER messages_on_delete;
 
       DELETE FROM attachment_downloads;
+      DELETE FROM attachment_backup_jobs;
+      DELETE FROM backup_cdn_object_metadata;
       DELETE FROM badgeImageFiles;
       DELETE FROM badges;
       DELETE FROM callLinks;
@@ -6200,6 +6368,8 @@ async function removeAllConfiguration(): Promise<void> {
   db.transaction(() => {
     db.exec(
       `
+      DELETE FROM attachment_backup_jobs;
+      DELETE FROM backup_cdn_object_metadata;
       DELETE FROM groupSendCombinedEndorsement;
       DELETE FROM groupSendMemberEndorsement;
       DELETE FROM identityKeys;

ts/sql/migrations/1070-attachment-backup.ts

@@ -0,0 +1,55 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { Database } from '@signalapp/better-sqlite3';
+
+import type { LoggerType } from '../../types/Logging';
+
+export const version = 1070;
+
+export function updateToSchemaVersion1070(
+  currentVersion: number,
+  db: Database,
+  logger: LoggerType
+): void {
+  if (currentVersion >= 1070) {
+    return;
+  }
+
+  db.transaction(() => {
+    db.exec(`
+      CREATE TABLE attachment_backup_jobs (
+        mediaName TEXT NOT NULL PRIMARY KEY,
+        type TEXT NOT NULL,
+        data TEXT NOT NULL,
+        receivedAt INTEGER NOT NULL,
+
+        -- job manager fields
+        attempts INTEGER NOT NULL,
+        active INTEGER NOT NULL,
+        retryAfter INTEGER,
+        lastAttemptTimestamp INTEGER
+      ) STRICT;
+
+      CREATE INDEX attachment_backup_jobs_receivedAt
+        ON attachment_backup_jobs (
+          receivedAt
+      );
+
+      CREATE INDEX attachment_backup_jobs_type_receivedAt
+        ON attachment_backup_jobs (
+          type, receivedAt
+      ); 
+
+      CREATE TABLE backup_cdn_object_metadata (
+        mediaId TEXT NOT NULL PRIMARY KEY,
+        cdnNumber INTEGER NOT NULL,
+        sizeOnBackupCdn INTEGER
+      ) STRICT;
+    `);
+  })();
+
+  db.pragma('user_version = 1070');
+
+  logger.info('updateToSchemaVersion1070: success!');
+}

ts/sql/migrations/index.ts

@@ -81,10 +81,11 @@ import { updateToSchemaVersion1020 } from './1020-self-merges';
 import { updateToSchemaVersion1030 } from './1030-unblock-event';
 import { updateToSchemaVersion1040 } from './1040-undownloaded-backed-up-media';
 import { updateToSchemaVersion1050 } from './1050-group-send-endorsements';
+import { updateToSchemaVersion1060 } from './1060-addressable-messages-and-sync-tasks';
 import {
-  updateToSchemaVersion1060,
+  updateToSchemaVersion1070,
   version as MAX_VERSION,
-} from './1060-addressable-messages-and-sync-tasks';
+} from './1070-attachment-backup';
 
 function updateToSchemaVersion1(
   currentVersion: number,
@@ -2034,6 +2035,7 @@ export const SCHEMA_VERSIONS = [
   updateToSchemaVersion1040,
   updateToSchemaVersion1050,
   updateToSchemaVersion1060,
+  updateToSchemaVersion1070,
 ];
 
 export class DBVersionFromFutureError extends Error {