Interactive cleanup of orphaned attachments

app/SystemTraySettingCache.ts

@@ -62,7 +62,7 @@ export class SystemTraySettingCache {
 
       const value =
         fastValue ??
-        (await this.sql.sqlCall('getItemById', ['system-tray-setting']))?.value;
+        (await this.sql.sqlCall('getItemById', 'system-tray-setting'))?.value;
 
       if (value !== undefined) {
         result = parseSystemTraySetting(value);

app/attachment_channel.ts

@@ -4,11 +4,22 @@
 import { ipcMain } from 'electron';
 import * as rimraf from 'rimraf';
 import {
+  getAllAttachments,
   getPath,
   getStickersPath,
   getTempPath,
   getDraftPath,
+  deleteAll as deleteAllAttachments,
+  deleteAllBadges,
+  getAllStickers,
+  deleteAllStickers,
+  getAllDraftAttachments,
+  deleteAllDraftAttachments,
 } from './attachments';
+import type { MainSQL } from '../ts/sql/main';
+import type { MessageAttachmentsCursorType } from '../ts/sql/Interface';
+import * as Errors from '../ts/types/errors';
+import { sleep } from '../ts/util/sleep';
 
 let initialized = false;
 
@@ -18,12 +29,140 @@ const ERASE_TEMP_KEY = 'erase-temp';
 const ERASE_DRAFTS_KEY = 'erase-drafts';
 const CLEANUP_ORPHANED_ATTACHMENTS_KEY = 'cleanup-orphaned-attachments';
 
+const INTERACTIVITY_DELAY = 50;
+
+type DeleteOrphanedAttachmentsOptionsType = Readonly<{
+  orphanedAttachments: Set<string>;
+  sql: MainSQL;
+  userDataPath: string;
+}>;
+
+type CleanupOrphanedAttachmentsOptionsType = Readonly<{
+  sql: MainSQL;
+  userDataPath: string;
+}>;
+
+async function cleanupOrphanedAttachments({
+  sql,
+  userDataPath,
+}: CleanupOrphanedAttachmentsOptionsType): Promise<void> {
+  await deleteAllBadges({
+    userDataPath,
+    pathsToKeep: await sql.sqlCall('getAllBadgeImageFileLocalPaths'),
+  });
+
+  const allStickers = await getAllStickers(userDataPath);
+  const orphanedStickers = await sql.sqlCall(
+    'removeKnownStickers',
+    allStickers
+  );
+  await deleteAllStickers({
+    userDataPath,
+    stickers: orphanedStickers,
+  });
+
+  const allDraftAttachments = await getAllDraftAttachments(userDataPath);
+  const orphanedDraftAttachments = await sql.sqlCall(
+    'removeKnownDraftAttachments',
+    allDraftAttachments
+  );
+  await deleteAllDraftAttachments({
+    userDataPath,
+    attachments: orphanedDraftAttachments,
+  });
+
+  // Delete orphaned attachments from conversations and messages.
+
+  const orphanedAttachments = new Set(await getAllAttachments(userDataPath));
+
+  {
+    const attachments: ReadonlyArray<string> = await sql.sqlCall(
+      'getKnownConversationAttachments'
+    );
+
+    for (const known of attachments) {
+      orphanedAttachments.delete(known);
+    }
+  }
+
+  // This call is intentionally not awaited. We block the app while running
+  // all fetches above to ensure that there are no in-flight attachments that
+  // are saved to disk, but not put into any message or conversation model yet.
+  deleteOrphanedAttachments({
+    orphanedAttachments,
+    sql,
+    userDataPath,
+  });
+}
+
+function deleteOrphanedAttachments({
+  orphanedAttachments,
+  sql,
+  userDataPath,
+}: DeleteOrphanedAttachmentsOptionsType): void {
+  // This function *can* throw.
+  async function runWithPossibleException(): Promise<void> {
+    let cursor: MessageAttachmentsCursorType | undefined;
+    try {
+      do {
+        let attachments: ReadonlyArray<string>;
+
+        // eslint-disable-next-line no-await-in-loop
+        ({ attachments, cursor } = await sql.sqlCall(
+          'getKnownMessageAttachments',
+          cursor
+        ));
+
+        for (const known of attachments) {
+          orphanedAttachments.delete(known);
+        }
+
+        if (cursor === undefined) {
+          break;
+        }
+
+        // Let other SQL calls come through. There are hundreds of thousands of
+        // messages in the database and it might take time to go through them all.
+        // eslint-disable-next-line no-await-in-loop
+        await sleep(INTERACTIVITY_DELAY);
+      } while (cursor !== undefined && !cursor.done);
+    } finally {
+      if (cursor !== undefined) {
+        await sql.sqlCall('finishGetKnownMessageAttachments', cursor);
+      }
+    }
+
+    await deleteAllAttachments({
+      userDataPath,
+      attachments: Array.from(orphanedAttachments),
+    });
+  }
+
+  async function runSafe() {
+    const start = Date.now();
+    try {
+      await runWithPossibleException();
+    } catch (error) {
+      console.error(
+        'deleteOrphanedAttachments: error',
+        Errors.toLogFormat(error)
+      );
+    } finally {
+      const duration = Date.now() - start;
+      console.log(`deleteOrphanedAttachments: took ${duration}ms`);
+    }
+  }
+
+  // Intentionally not awaiting
+  runSafe();
+}
+
 export function initialize({
   configDir,
-  cleanupOrphanedAttachments,
+  sql,
 }: {
   configDir: string;
-  cleanupOrphanedAttachments: () => Promise<void>;
+  sql: MainSQL;
 }): void {
   if (initialized) {
     throw new Error('initialze: Already initialized!');
@@ -35,58 +174,15 @@ export function initialize({
   const tempDir = getTempPath(configDir);
   const draftDir = getDraftPath(configDir);
 
-  ipcMain.on(ERASE_TEMP_KEY, event => {
+  ipcMain.handle(ERASE_TEMP_KEY, () => rimraf.sync(tempDir));
-    try {
+  ipcMain.handle(ERASE_ATTACHMENTS_KEY, () => rimraf.sync(attachmentsDir));
-      rimraf.sync(tempDir);
+  ipcMain.handle(ERASE_STICKERS_KEY, () => rimraf.sync(stickersDir));
-      event.sender.send(`${ERASE_TEMP_KEY}-done`);
+  ipcMain.handle(ERASE_DRAFTS_KEY, () => rimraf.sync(draftDir));
-    } catch (error) {
-      const errorForDisplay = error && error.stack ? error.stack : error;
-      console.log(`erase temp error: ${errorForDisplay}`);
-      event.sender.send(`${ERASE_TEMP_KEY}-done`, error);
-    }
-  });
 
-  ipcMain.on(ERASE_ATTACHMENTS_KEY, event => {
+  ipcMain.handle(CLEANUP_ORPHANED_ATTACHMENTS_KEY, async () => {
-    try {
+    const start = Date.now();
-      rimraf.sync(attachmentsDir);
+    await cleanupOrphanedAttachments({ sql, userDataPath: configDir });
-      event.sender.send(`${ERASE_ATTACHMENTS_KEY}-done`);
+    const duration = Date.now() - start;
-    } catch (error) {
+    console.log(`cleanupOrphanedAttachments: took ${duration}ms`);
-      const errorForDisplay = error && error.stack ? error.stack : error;
-      console.log(`erase attachments error: ${errorForDisplay}`);
-      event.sender.send(`${ERASE_ATTACHMENTS_KEY}-done`, error);
-    }
-  });
-
-  ipcMain.on(ERASE_STICKERS_KEY, event => {
-    try {
-      rimraf.sync(stickersDir);
-      event.sender.send(`${ERASE_STICKERS_KEY}-done`);
-    } catch (error) {
-      const errorForDisplay = error && error.stack ? error.stack : error;
-      console.log(`erase stickers error: ${errorForDisplay}`);
-      event.sender.send(`${ERASE_STICKERS_KEY}-done`, error);
-    }
-  });
-
-  ipcMain.on(ERASE_DRAFTS_KEY, event => {
-    try {
-      rimraf.sync(draftDir);
-      event.sender.send(`${ERASE_DRAFTS_KEY}-done`);
-    } catch (error) {
-      const errorForDisplay = error && error.stack ? error.stack : error;
-      console.log(`erase drafts error: ${errorForDisplay}`);
-      event.sender.send(`${ERASE_DRAFTS_KEY}-done`, error);
-    }
-  });
-
-  ipcMain.on(CLEANUP_ORPHANED_ATTACHMENTS_KEY, async event => {
-    try {
-      await cleanupOrphanedAttachments();
-      event.sender.send(`${CLEANUP_ORPHANED_ATTACHMENTS_KEY}-done`);
-    } catch (error) {
-      const errorForDisplay = error && error.stack ? error.stack : error;
-      console.log(`cleanup orphaned attachments error: ${errorForDisplay}`);
-      event.sender.send(`${CLEANUP_ORPHANED_ATTACHMENTS_KEY}-done`, error);
-    }
   });
 }

app/main.ts

@@ -224,17 +224,17 @@ let sqlInitTimeEnd = 0;
 const sql = new MainSQL();
 const heicConverter = getHeicConverter();
 
-async function getSpellCheckSetting() {
+async function getSpellCheckSetting(): Promise<boolean> {
   const fastValue = ephemeralConfig.get('spell-check');
-  if (fastValue !== undefined) {
+  if (typeof fastValue === 'boolean') {
     getLogger().info('got fast spellcheck setting', fastValue);
     return fastValue;
   }
 
-  const json = await sql.sqlCall('getItemById', ['spell-check']);
+  const json = await sql.sqlCall('getItemById', 'spell-check');
 
   // Default to `true` if setting doesn't exist yet
-  const slowValue = json ? json.value : true;
+  const slowValue = typeof json?.value === 'boolean' ? json.value : true;
 
   ephemeralConfig.set('spell-check', slowValue);
 
@@ -260,7 +260,7 @@ async function getThemeSetting({
     return 'system';
   }
 
-  const json = await sql.sqlCall('getItemById', ['theme-setting']);
+  const json = await sql.sqlCall('getItemById', 'theme-setting');
 
   // Default to `system` if setting doesn't exist or is invalid
   const setting: unknown = json?.value;
@@ -928,7 +928,7 @@ async function createWindow() {
 }
 
 // Renderer asks if we are done with the database
-ipc.on('database-ready', async event => {
+ipc.handle('database-ready', async () => {
   if (!sqlInitPromise) {
     getLogger().error('database-ready requested, but sqlInitPromise is falsey');
     return;
@@ -944,7 +944,6 @@ ipc.on('database-ready', async event => {
   }
 
   getLogger().info('sending `database-ready`');
-  event.sender.send('database-ready');
 });
 
 ipc.on('show-window', () => {
@@ -1259,8 +1258,8 @@ async function showSettingsWindow() {
 
 async function getIsLinked() {
   try {
-    const number = await sql.sqlCall('getItemById', ['number_id']);
+    const number = await sql.sqlCall('getItemById', 'number_id');
-    const password = await sql.sqlCall('getItemById', ['password']);
+    const password = await sql.sqlCall('getItemById', 'password');
     return Boolean(number && password);
   } catch (e) {
     return false;
@@ -1651,12 +1650,10 @@ app.on('ready', async () => {
 
     // Update both stores
     ephemeralConfig.set('system-tray-setting', newValue);
-    await sql.sqlCall('createOrUpdateItem', [
+    await sql.sqlCall('createOrUpdateItem', {
-      {
+      id: 'system-tray-setting',
-        id: 'system-tray-setting',
+      value: newValue,
-        value: newValue,
+    });
-      },
-    ]);
 
     if (OS.isWindows()) {
       getLogger().info('app.ready: enabling open at login');
@@ -1806,8 +1803,8 @@ app.on('ready', async () => {
   // Initialize IPC channels before creating the window
 
   attachmentChannel.initialize({
+    sql,
     configDir: userDataPath,
-    cleanupOrphanedAttachments,
   });
   sqlChannels.initialize(sql);
   PowerChannel.initialize({
@@ -1835,10 +1832,10 @@ app.on('ready', async () => {
 
   try {
     const IDB_KEY = 'indexeddb-delete-needed';
-    const item = await sql.sqlCall('getItemById', [IDB_KEY]);
+    const item = await sql.sqlCall('getItemById', IDB_KEY);
     if (item && item.value) {
-      await sql.sqlCall('removeIndexedDBFiles', []);
+      await sql.sqlCall('removeIndexedDBFiles');
-      await sql.sqlCall('removeItemById', [IDB_KEY]);
+      await sql.sqlCall('removeItemById', IDB_KEY);
     }
   } catch (err) {
     getLogger().error(
@@ -1847,43 +1844,6 @@ app.on('ready', async () => {
     );
   }
 
-  async function cleanupOrphanedAttachments() {
-    const allAttachments = await attachments.getAllAttachments(userDataPath);
-    const orphanedAttachments = await sql.sqlCall('removeKnownAttachments', [
-      allAttachments,
-    ]);
-    await attachments.deleteAll({
-      userDataPath,
-      attachments: orphanedAttachments,
-    });
-
-    await attachments.deleteAllBadges({
-      userDataPath,
-      pathsToKeep: await sql.sqlCall('getAllBadgeImageFileLocalPaths', []),
-    });
-
-    const allStickers = await attachments.getAllStickers(userDataPath);
-    const orphanedStickers = await sql.sqlCall('removeKnownStickers', [
-      allStickers,
-    ]);
-    await attachments.deleteAllStickers({
-      userDataPath,
-      stickers: orphanedStickers,
-    });
-
-    const allDraftAttachments = await attachments.getAllDraftAttachments(
-      userDataPath
-    );
-    const orphanedDraftAttachments = await sql.sqlCall(
-      'removeKnownDraftAttachments',
-      [allDraftAttachments]
-    );
-    await attachments.deleteAllDraftAttachments({
-      userDataPath,
-      attachments: orphanedDraftAttachments,
-    });
-  }
-
   ready = true;
 
   setupMenu();
@@ -2320,10 +2280,7 @@ ipc.on('install-sticker-pack', (_event, packId, packKeyHex) => {
   }
 });
 
-ipc.on('ensure-file-permissions', async event => {
+ipc.handle('ensure-file-permissions', () => ensureFilePermissions());
-  await ensureFilePermissions();
-  event.reply('ensure-file-permissions-done');
-});
 
 /**
  * Ensure files in the user's data directory have the proper permissions.

app/sql_channel.ts

@@ -3,21 +3,18 @@
 
 import { ipcMain } from 'electron';
 
+import type { MainSQL } from '../ts/sql/main';
 import { remove as removeUserConfig } from './user_config';
 import { remove as removeEphemeralConfig } from './ephemeral_config';
 
-type SQLType = {
+let sql: Pick<MainSQL, 'sqlCall'> | undefined;
-  sqlCall(callName: string, args: ReadonlyArray<unknown>): unknown;
-};
-
-let sql: SQLType | undefined;
 
 let initialized = false;
 
 const SQL_CHANNEL_KEY = 'sql-channel';
 const ERASE_SQL_KEY = 'erase-sql-key';
 
-export function initialize(mainSQL: SQLType): void {
+export function initialize(mainSQL: Pick<MainSQL, 'sqlCall'>): void {
   if (initialized) {
     throw new Error('sqlChannels: already initialized!');
   }
@@ -25,33 +22,15 @@ export function initialize(mainSQL: SQLType): void {
 
   sql = mainSQL;
 
-  ipcMain.on(SQL_CHANNEL_KEY, async (event, jobId, callName, ...args) => {
+  ipcMain.handle(SQL_CHANNEL_KEY, (_event, callName, ...args) => {
-    try {
+    if (!sql) {
-      if (!sql) {
+      throw new Error(`${SQL_CHANNEL_KEY}: Not yet initialized!`);
-        throw new Error(`${SQL_CHANNEL_KEY}: Not yet initialized!`);
-      }
-      const result = await sql.sqlCall(callName, args);
-      event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, null, result);
-    } catch (error) {
-      const errorForDisplay = error && error.stack ? error.stack : error;
-      console.log(
-        `sql channel error with call ${callName}: ${errorForDisplay}`
-      );
-      if (!event.sender.isDestroyed()) {
-        event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, errorForDisplay);
-      }
     }
+    return sql.sqlCall(callName, ...args);
   });
 
-  ipcMain.on(ERASE_SQL_KEY, async event => {
+  ipcMain.handle(ERASE_SQL_KEY, () => {
-    try {
+    removeUserConfig();
-      removeUserConfig();
+    removeEphemeralConfig();
-      removeEphemeralConfig();
-      event.sender.send(`${ERASE_SQL_KEY}-done`);
-    } catch (error) {
-      const errorForDisplay = error && error.stack ? error.stack : error;
-      console.log(`sql-erase error: ${errorForDisplay}`);
-      event.sender.send(`${ERASE_SQL_KEY}-done`, error);
-    }
   });
 }

ts/background.ts

@@ -849,16 +849,7 @@ export async function startApp(): Promise<void> {
     );
 
     if (newVersion) {
-      // We've received reports that this update can take longer than two minutes, so we
+      await window.Signal.Data.cleanupOrphanedAttachments();
-      //   allow it to continue and just move on in that timeout case.
-      try {
-        await window.Signal.Data.cleanupOrphanedAttachments();
-      } catch (error) {
-        log.error(
-          'background: Failed to cleanup orphaned attachments:',
-          error && error.stack ? error.stack : error
-        );
-      }
 
       // Don't block on the following operation
       window.Signal.Data.ensureFilePermissions();

ts/sql/Interface.ts

@@ -344,6 +344,17 @@ export type GetConversationRangeCenteredOnMessageResultType<Message> =
     metrics: ConversationMetricsType;
   }>;
 
+export type MessageAttachmentsCursorType = Readonly<{
+  done: boolean;
+  runId: string;
+  count: number;
+}>;
+
+export type GetKnownMessageAttachmentsResultType = Readonly<{
+  cursor: MessageAttachmentsCursorType;
+  attachments: ReadonlyArray<string>;
+}>;
+
 export type DataInterface = {
   close: () => Promise<void>;
   removeDB: () => Promise<void>;
@@ -777,17 +788,24 @@ export type ServerInterface = DataInterface & {
     key: string;
   }) => Promise<void>;
 
-  removeKnownAttachments: (
+  getKnownMessageAttachments: (
-    allAttachments: Array<string>
+    cursor?: MessageAttachmentsCursorType
+  ) => Promise<GetKnownMessageAttachmentsResultType>;
+  finishGetKnownMessageAttachments: (
+    cursor: MessageAttachmentsCursorType
+  ) => Promise<void>;
+  getKnownConversationAttachments: () => Promise<Array<string>>;
+  removeKnownStickers: (
+    allStickers: ReadonlyArray<string>
   ) => Promise<Array<string>>;
-  removeKnownStickers: (allStickers: Array<string>) => Promise<Array<string>>;
   removeKnownDraftAttachments: (
-    allStickers: Array<string>
+    allStickers: ReadonlyArray<string>
   ) => Promise<Array<string>>;
   getAllBadgeImageFileLocalPaths: () => Promise<Set<string>>;
 };
 
-export type ClientInterface = DataInterface & {
+// Differing signature on client/server
+export type ClientExclusiveInterface = {
   // Differing signature on client/server
 
   updateConversation: (data: ConversationType) => void;
@@ -870,21 +888,10 @@ export type ClientInterface = DataInterface & {
   cleanupOrphanedAttachments: () => Promise<void>;
   ensureFilePermissions: () => Promise<void>;
 
-  _jobs: { [id: string]: ClientJobType };
-
   // To decide whether to use IPC to use the database in the main process or
   //   use the db already running in the renderer.
   goBackToMainProcess: () => Promise<void>;
   startInRendererProcess: (isTesting?: boolean) => Promise<void>;
 };
 
-export type ClientJobType = {
+export type ClientInterface = DataInterface & ClientExclusiveInterface;
-  fnName: string;
-  start: number;
-  resolve?: (value: unknown) => void;
-  reject?: (error: Error) => void;
-
-  // Only in DEBUG mode
-  complete?: boolean;
-  args?: ReadonlyArray<unknown>;
-};

ts/sql/Server.ts

@@ -6,6 +6,7 @@
 import { join } from 'path';
 import mkdirp from 'mkdirp';
 import rimraf from 'rimraf';
+import { randomBytes } from 'crypto';
 import type { Database, Statement } from 'better-sqlite3';
 import SQL from 'better-sqlite3';
 import pProps from 'p-props';
@@ -64,7 +65,6 @@ import {
   getById,
   bulkAdd,
   createOrUpdate,
-  TableIterator,
   setUserVersion,
   getUserVersion,
   getSchemaVersion,
@@ -80,6 +80,7 @@ import type {
   DeleteSentProtoRecipientResultType,
   EmojiType,
   GetConversationRangeCenteredOnMessageResultType,
+  GetKnownMessageAttachmentsResultType,
   GetUnreadByConversationAndMarkReadResultType,
   IdentityKeyIdType,
   StoredIdentityKeyType,
@@ -87,6 +88,7 @@ import type {
   ItemKeyType,
   StoredItemType,
   ConversationMessageStatsType,
+  MessageAttachmentsCursorType,
   MessageMetricsType,
   MessageType,
   MessageTypeUnhydrated,
@@ -344,7 +346,9 @@ const dataInterface: ServerInterface = {
   initialize,
   initializeRenderer,
 
-  removeKnownAttachments,
+  getKnownMessageAttachments,
+  finishGetKnownMessageAttachments,
+  getKnownConversationAttachments,
   removeKnownStickers,
   removeKnownDraftAttachments,
   getAllBadgeImageFileLocalPaths,
@@ -924,7 +928,7 @@ async function insertSentProto(
       `
     );
 
-    for (const messageId of messageIds) {
+    for (const messageId of new Set(messageIds)) {
       messageStatement.run({
         id,
         messageId,
@@ -4512,6 +4516,11 @@ async function _deleteAllStoryDistributions(): Promise<void> {
 async function createNewStoryDistribution(
   distribution: StoryDistributionWithMembersType
 ): Promise<void> {
+  strictAssert(
+    distribution.name,
+    'Distribution list does not have a valid name'
+  );
+
   const db = getInstance();
 
   db.transaction(() => {
@@ -4613,6 +4622,18 @@ function modifyStoryDistributionSync(
   db: Database,
   payload: StoryDistributionForDatabase
 ): void {
+  if (payload.deletedAtTimestamp) {
+    strictAssert(
+      !payload.name,
+      'Attempt to delete distribution list but still has a name'
+    );
+  } else {
+    strictAssert(
+      payload.name,
+      'Cannot clear distribution list name without deletedAtTimestamp set'
+    );
+  }
+
   prepare(
     db,
     `
@@ -5079,39 +5100,129 @@ function getExternalDraftFilesForConversation(
   return files;
 }
 
-async function removeKnownAttachments(
+async function getKnownMessageAttachments(
-  allAttachments: Array<string>
+  cursor?: MessageAttachmentsCursorType
-): Promise<Array<string>> {
+): Promise<GetKnownMessageAttachmentsResultType> {
   const db = getInstance();
-  const lookup: Dictionary<boolean> = fromPairs(
+  const result = new Set<string>();
-    map(allAttachments, file => [file, true])
+  const chunkSize = 1000;
-  );
-  const chunkSize = 500;
 
-  const total = getMessageCountSync();
+  return db.transaction(() => {
-  logger.info(
+    let count = cursor?.count ?? 0;
-    `removeKnownAttachments: About to iterate through ${total} messages`
-  );
 
-  let count = 0;
+    strictAssert(
+      !cursor?.done,
+      'getKnownMessageAttachments: iteration cannot be restarted'
+    );
 
-  for (const message of new TableIterator<MessageType>(db, 'messages')) {
+    let runId: string;
-    const externalFiles = getExternalFilesForMessage(message);
+    if (cursor === undefined) {
-    forEach(externalFiles, file => {
+      runId = randomBytes(8).toString('hex');
-      delete lookup[file];
+
-    });
+      const total = getMessageCountSync();
-    count += 1;
+      logger.info(
+        `getKnownMessageAttachments(${runId}): ` +
+          `Starting iteration through ${total} messages`
+      );
+
+      db.exec(
+        `
+        CREATE TEMP TABLE tmp_${runId}_updated_messages
+          (rowid INTEGER PRIMARY KEY ASC);
+
+        INSERT INTO tmp_${runId}_updated_messages (rowid)
+        SELECT rowid FROM messages;
+
+        CREATE TEMP TRIGGER tmp_${runId}_message_updates
+        UPDATE OF json ON messages
+        BEGIN
+          INSERT OR IGNORE INTO tmp_${runId}_updated_messages (rowid)
+          VALUES (NEW.rowid);
+        END;
+
+        CREATE TEMP TRIGGER tmp_${runId}_message_inserts
+        AFTER INSERT ON messages
+        BEGIN
+          INSERT OR IGNORE INTO tmp_${runId}_updated_messages (rowid)
+          VALUES (NEW.rowid);
+        END;
+        `
+      );
+    } else {
+      ({ runId } = cursor);
+    }
+
+    const rowids: Array<number> = db
+      .prepare<Query>(
+        `
+      DELETE FROM tmp_${runId}_updated_messages
+      RETURNING rowid
+      LIMIT $chunkSize;
+      `
+      )
+      .pluck()
+      .all({ chunkSize });
+
+    const messages = batchMultiVarQuery(
+      db,
+      rowids,
+      (batch: Array<number>): Array<MessageType> => {
+        const query = db.prepare<ArrayQuery>(
+          `SELECT json FROM messages WHERE rowid IN (${Array(batch.length)
+            .fill('?')
+            .join(',')});`
+        );
+        const rows: JSONRows = query.all(batch);
+        return rows.map(row => jsonToObject(row.json));
+      }
+    );
+
+    for (const message of messages) {
+      const externalFiles = getExternalFilesForMessage(message);
+      forEach(externalFiles, file => result.add(file));
+      count += 1;
+    }
+
+    const done = messages.length < chunkSize;
+    return {
+      attachments: Array.from(result),
+      cursor: { runId, count, done },
+    };
+  })();
+}
+
+async function finishGetKnownMessageAttachments({
+  runId,
+  count,
+  done,
+}: MessageAttachmentsCursorType): Promise<void> {
+  const db = getInstance();
+
+  const logId = `finishGetKnownMessageAttachments(${runId})`;
+  if (!done) {
+    logger.warn(`${logId}: iteration not finished`);
   }
 
-  logger.info(`removeKnownAttachments: Done processing ${count} messages`);
+  logger.info(`${logId}: reached the end after processing ${count} messages`);
+  db.exec(`
+    DROP TABLE tmp_${runId}_updated_messages;
+    DROP TRIGGER tmp_${runId}_message_updates;
+    DROP TRIGGER tmp_${runId}_message_inserts;
+  `);
+}
+
+async function getKnownConversationAttachments(): Promise<Array<string>> {
+  const db = getInstance();
+  const result = new Set<string>();
+  const chunkSize = 500;
 
   let complete = false;
-  count = 0;
   let id = '';
 
   const conversationTotal = await getConversationCount();
   logger.info(
-    `removeKnownAttachments: About to iterate through ${conversationTotal} conversations`
+    'getKnownConversationAttachments: About to iterate through ' +
+      `${conversationTotal}`
   );
 
   const fetchConversations = db.prepare<Query>(
@@ -5134,9 +5245,7 @@ async function removeKnownAttachments(
     );
     conversations.forEach(conversation => {
       const externalFiles = getExternalFilesForConversation(conversation);
-      externalFiles.forEach(file => {
+      externalFiles.forEach(file => result.add(file));
-        delete lookup[file];
-      });
     });
 
     const lastMessage: ConversationType | undefined = last(conversations);
@@ -5144,16 +5253,15 @@ async function removeKnownAttachments(
       ({ id } = lastMessage);
     }
     complete = conversations.length < chunkSize;
-    count += conversations.length;
   }
 
-  logger.info(`removeKnownAttachments: Done processing ${count} conversations`);
+  logger.info('getKnownConversationAttachments: Done processing');
 
-  return Object.keys(lookup);
+  return Array.from(result);
 }
 
 async function removeKnownStickers(
-  allStickers: Array<string>
+  allStickers: ReadonlyArray<string>
 ): Promise<Array<string>> {
   const db = getInstance();
   const lookup: Dictionary<boolean> = fromPairs(
@@ -5204,7 +5312,7 @@ async function removeKnownStickers(
 }
 
 async function removeKnownDraftAttachments(
-  allStickers: Array<string>
+  allStickers: ReadonlyArray<string>
 ): Promise<Array<string>> {
   const db = getInstance();
   const lookup: Dictionary<boolean> = fromPairs(

ts/sql/main.ts

@@ -10,6 +10,7 @@ import { strictAssert } from '../util/assert';
 import { explodePromise } from '../util/explodePromise';
 import type { LoggerType } from '../types/Logging';
 import { isCorruptionError } from './errors';
+import type DB from './Server';
 
 const MIN_TRACE_DURATION = 40;
 
@@ -32,9 +33,8 @@ export type WorkerRequest = Readonly<
     }
   | {
       type: 'sqlCall';
-      method: string;
+      method: keyof typeof DB;
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      args: ReadonlyArray<unknown>;
-      args: ReadonlyArray<any>;
     }
 >;
 
@@ -164,8 +164,10 @@ export class MainSQL {
     await this.send({ type: 'removeDB' });
   }
 
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  public async sqlCall<Method extends keyof typeof DB>(
-  public async sqlCall(method: string, args: ReadonlyArray<any>): Promise<any> {
+    method: Method,
+    ...args: Parameters<typeof DB[Method]>
+  ): Promise<ReturnType<typeof DB[Method]>> {
     if (this.onReady) {
       await this.onReady;
     }
@@ -175,8 +177,7 @@ export class MainSQL {
     }
 
     type SqlCallResult = Readonly<{
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      result: ReturnType<typeof DB[Method]>;
-      result: any;
       duration: number;
     }>;