Move receipts and view/read syncs to new syncTasks system

ts/sql/Client.ts

@@ -2,13 +2,11 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import { ipcRenderer as ipc } from 'electron';
-import PQueue from 'p-queue';
-import { batch } from 'react-redux';
 
 import { has, get, groupBy, isTypedArray, last, map, omit } from 'lodash';
 
 import { deleteExternalFiles } from '../types/Conversation';
-import { expiringMessagesDeletionService } from '../services/expiringMessagesDeletion';
+import { update as updateExpiringMessagesService } from '../services/expiringMessagesDeletion';
 import { tapToViewMessagesDeletionService } from '../services/tapToViewMessagesDeletionService';
 import * as Bytes from '../Bytes';
 import { createBatcher } from '../util/batcher';
@@ -24,12 +22,7 @@ import * as Errors from '../types/errors';
 
 import type { StoredJob } from '../jobs/types';
 import { formatJobForInsert } from '../jobs/formatJobForInsert';
-import {
-  cleanupMessage,
-  cleanupMessageFromMemory,
-  deleteMessageData,
-} from '../util/cleanup';
-import { drop } from '../util/drop';
+import { cleanupMessages } from '../util/cleanup';
 import { ipcInvoke, doShutdown } from './channels';
 
 import type {
@@ -60,12 +53,12 @@ import type {
   KyberPreKeyType,
   StoredKyberPreKeyType,
 } from './Interface';
-import { MINUTE } from '../util/durations';
 import { getMessageIdForLogging } from '../util/idForLogging';
 import type { MessageAttributesType } from '../model-types';
 import { incrementMessageCounter } from '../util/incrementMessageCounter';
 import { generateSnippetAroundMention } from '../util/search';
 import type { AttachmentDownloadJobType } from '../types/AttachmentDownload';
+import type { SingleProtoJobQueue } from '../jobs/singleProtoJobQueue';
 
 const ERASE_SQL_KEY = 'erase-sql-key';
 const ERASE_ATTACHMENTS_KEY = 'erase-attachments';
@@ -104,6 +97,8 @@ const exclusiveInterface: ClientExclusiveInterface = {
   removeConversation,
 
   searchMessages,
+  removeMessage,
+  removeMessages,
 
   getRecentStoryReplies,
   getOlderMessagesByConversation,
@@ -125,8 +120,6 @@ const exclusiveInterface: ClientExclusiveInterface = {
 type ClientOverridesType = ClientExclusiveInterface &
   Pick<
     ServerInterface,
-    | 'removeMessage'
-    | 'removeMessages'
     | 'saveAttachmentDownloadJob'
     | 'saveMessage'
     | 'saveMessages'
@@ -142,8 +135,6 @@ const channels: ServerInterface = new Proxy({} as ServerInterface, {
 
 const clientExclusiveOverrides: ClientOverridesType = {
   ...exclusiveInterface,
-  removeMessage,
-  removeMessages,
   saveAttachmentDownloadJob,
   saveMessage,
   saveMessages,
@@ -562,7 +553,7 @@ async function saveMessage(
 
   softAssert(isValidUuid(id), 'saveMessage: messageId is not a UUID');
 
-  void expiringMessagesDeletionService.update();
+  void updateExpiringMessagesService();
   void tapToViewMessagesDeletionService.update();
 
   return id;
@@ -577,26 +568,39 @@ async function saveMessages(
     options
   );
 
-  void expiringMessagesDeletionService.update();
+  void updateExpiringMessagesService();
   void tapToViewMessagesDeletionService.update();
 
   return result;
 }
 
-async function removeMessage(id: string): Promise<void> {
+async function removeMessage(
+  id: string,
+  options: {
+    singleProtoJobQueue: SingleProtoJobQueue;
+    fromSync?: boolean;
+  }
+): Promise<void> {
   const message = await channels.getMessageById(id);
 
   // Note: It's important to have a fully database-hydrated model to delete here because
   //   it needs to delete all associated on-disk files along with the database delete.
   if (message) {
     await channels.removeMessage(id);
-    await cleanupMessage(message);
+    await cleanupMessages([message], {
+      ...options,
+      markCallHistoryDeleted: dataInterface.markCallHistoryDeleted,
+    });
   }
 }
 
 export async function deleteAndCleanup(
   messages: Array<MessageAttributesType>,
-  logId: string
+  logId: string,
+  options: {
+    fromSync?: boolean;
+    singleProtoJobQueue: SingleProtoJobQueue;
+  }
 ): Promise<void> {
   const ids = messages.map(message => message.id);
 
@@ -604,37 +608,26 @@ export async function deleteAndCleanup(
   await channels.removeMessages(ids);
 
   log.info(`deleteAndCleanup/${logId}: Cleanup for ${ids.length} messages...`);
-  await _cleanupMessages(messages);
+  await cleanupMessages(messages, {
+    ...options,
+    markCallHistoryDeleted: dataInterface.markCallHistoryDeleted,
+  });
 
   log.info(`deleteAndCleanup/${logId}: Complete`);
 }
 
-async function _cleanupMessages(
-  messages: ReadonlyArray<MessageAttributesType>
-): Promise<void> {
-  // First, remove messages from memory, so we can batch the updates in redux
-  batch(() => {
-    messages.forEach(message => cleanupMessageFromMemory(message));
-  });
-
-  // Then, handle any asynchronous actions (e.g. deleting data from disk)
-  const queue = new PQueue({ concurrency: 3, timeout: MINUTE * 30 });
-  drop(
-    queue.addAll(
-      messages.map(
-        (message: MessageAttributesType) => async () =>
-          deleteMessageData(message)
-      )
-    )
-  );
-  await queue.onIdle();
-}
-
 async function removeMessages(
-  messageIds: ReadonlyArray<string>
+  messageIds: ReadonlyArray<string>,
+  options: {
+    fromSync?: boolean;
+    singleProtoJobQueue: SingleProtoJobQueue;
+  }
 ): Promise<void> {
   const messages = await channels.getMessagesById(messageIds);
-  await _cleanupMessages(messages);
+  await cleanupMessages(messages, {
+    ...options,
+    markCallHistoryDeleted: dataInterface.markCallHistoryDeleted,
+  });
   await channels.removeMessages(messageIds);
 }
 
@@ -686,9 +679,13 @@ async function removeMessagesInConversation(
   {
     logId,
     receivedAt,
+    singleProtoJobQueue,
+    fromSync,
   }: {
+    fromSync?: boolean;
     logId: string;
     receivedAt?: number;
+    singleProtoJobQueue: SingleProtoJobQueue;
   }
 ): Promise<void> {
   let messages;
@@ -713,7 +710,7 @@ async function removeMessagesInConversation(
     }
 
     // eslint-disable-next-line no-await-in-loop
-    await deleteAndCleanup(messages, logId);
+    await deleteAndCleanup(messages, logId, { fromSync, singleProtoJobQueue });
   } while (messages.length > 0);
 }
 

ts/sql/Interface.ts

@@ -34,6 +34,7 @@ import type { AttachmentDownloadJobType } from '../types/AttachmentDownload';
 import type { GroupSendEndorsementsData } from '../types/GroupSendEndorsements';
 import type { SyncTaskType } from '../util/syncTasks';
 import type { AttachmentBackupJobType } from '../types/AttachmentBackup';
+import type { SingleProtoJobQueue } from '../jobs/singleProtoJobQueue';
 
 export type AdjacentMessagesByConversationOptionsType = Readonly<{
   conversationId: string;
@@ -557,8 +558,6 @@ export type DataInterface = {
     arrayOfMessages: ReadonlyArray<MessageType>,
     options: { forceSave?: boolean; ourAci: AciString }
   ) => Promise<Array<string>>;
-  removeMessage: (id: string) => Promise<void>;
-  removeMessages: (ids: ReadonlyArray<string>) => Promise<void>;
   pageMessages: (
     cursor?: PageMessagesCursorType
   ) => Promise<PageMessagesResultType>;
@@ -667,6 +666,7 @@ export type DataInterface = {
     conversationId: string;
   }): Promise<MessageType | undefined>;
   getAllCallHistory: () => Promise<ReadonlyArray<CallHistoryDetails>>;
+  markCallHistoryDeleted: (callId: string) => Promise<void>;
   clearCallHistory: (beforeTimestamp: number) => Promise<Array<string>>;
   cleanupCallHistoryMessages: () => Promise<void>;
   getCallHistoryUnreadCount(): Promise<number>;
@@ -929,6 +929,8 @@ export type ServerInterface = DataInterface & {
     options?: { limit?: number };
     contactServiceIdsMatchingQuery?: Array<ServiceIdString>;
   }) => Promise<Array<ServerSearchResultMessageType>>;
+  removeMessage: (id: string) => Promise<void>;
+  removeMessages: (ids: ReadonlyArray<string>) => Promise<void>;
 
   getRecentStoryReplies(
     storyId: string,
@@ -1022,6 +1024,20 @@ export type ClientExclusiveInterface = {
   removeConversation: (id: string) => Promise<void>;
   flushUpdateConversationBatcher: () => Promise<void>;
 
+  removeMessage: (
+    id: string,
+    options: {
+      fromSync?: boolean;
+      singleProtoJobQueue: SingleProtoJobQueue;
+    }
+  ) => Promise<void>;
+  removeMessages: (
+    ids: ReadonlyArray<string>,
+    options: {
+      fromSync?: boolean;
+      singleProtoJobQueue: SingleProtoJobQueue;
+    }
+  ) => Promise<void>;
   searchMessages: ({
     query,
     conversationId,
@@ -1084,8 +1100,10 @@ export type ClientExclusiveInterface = {
   removeMessagesInConversation: (
     conversationId: string,
     options: {
+      fromSync?: boolean;
       logId: string;
       receivedAt?: number;
+      singleProtoJobQueue: SingleProtoJobQueue;
     }
   ) => Promise<void>;
   removeOtherData: () => Promise<void>;

ts/sql/Server.ts

@@ -347,6 +347,7 @@ const dataInterface: ServerInterface = {
   getLastConversationMessage,
   getAllCallHistory,
   clearCallHistory,
+  markCallHistoryDeleted,
   cleanupCallHistoryMessages,
   getCallHistoryUnreadCount,
   markCallHistoryRead,
@@ -3635,6 +3636,19 @@ async function clearCallHistory(
   })();
 }
 
+async function markCallHistoryDeleted(callId: string): Promise<void> {
+  const db = await getWritableInstance();
+  const [query, params] = sql`
+    UPDATE callsHistory
+    SET
+      status = ${DirectCallStatus.Deleted},
+      timestamp = ${Date.now()}
+    WHERE callId = ${callId};
+  `;
+
+  db.prepare(query).run(params);
+}
+
 async function cleanupCallHistoryMessages(): Promise<void> {
   const db = await getWritableInstance();
   return db