Periodically optimize FTS table

package.json

@@ -90,7 +90,7 @@
     "@indutny/sneequals": "4.0.0",
     "@popperjs/core": "2.11.6",
     "@react-spring/web": "9.5.5",
-    "@signalapp/better-sqlite3": "8.4.1",
+    "@signalapp/better-sqlite3": "8.4.2",
     "@signalapp/libsignal-client": "0.22.0",
     "@signalapp/ringrtc": "2.24.0",
     "@types/fabric": "4.5.3",

ts/background.ts

@@ -56,6 +56,7 @@ import { senderCertificateService } from './services/senderCertificate';
 import { GROUP_CREDENTIALS_KEY } from './services/groupCredentialFetcher';
 import * as KeyboardLayout from './services/keyboardLayout';
 import * as StorageService from './services/storage';
+import { optimizeFTS } from './services/ftsOptimizer';
 import { RoutineProfileRefresher } from './routineProfileRefresh';
 import { isOlderThan, toDayMillis } from './util/timestamp';
 import { isValidReactionEmoji } from './reactions/isValidReactionEmoji';
@@ -874,6 +875,8 @@ export async function startApp(): Promise<void> {
     if (newVersion) {
       await window.Signal.Data.cleanupOrphanedAttachments();
 
+      optimizeFTS();
+
       // Don't block on the following operation
       void window.Signal.Data.ensureFilePermissions();
     }

ts/models/conversations.ts

@@ -90,6 +90,7 @@ import { isNotNil } from '../util/isNotNil';
 import { dropNull } from '../util/dropNull';
 import { notificationService } from '../services/notifications';
 import { storageServiceUploadJob } from '../services/storage';
+import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
 import { getSendOptions } from '../util/getSendOptions';
 import { isConversationAccepted } from '../util/isConversationAccepted';
 import {
@@ -5120,6 +5121,8 @@ export class ConversationModel extends window.Backbone
     await window.Signal.Data.removeAllMessagesInConversation(this.id, {
       logId: this.idForLogging(),
     });
+
+    scheduleOptimizeFTS();
   }
 
   getTitle(options?: { isShort?: boolean }): string {

ts/models/messages.ts

@@ -79,6 +79,7 @@ import { migrateLegacyReadStatus } from '../messages/migrateLegacyReadStatus';
 import { migrateLegacySendAttributes } from '../messages/migrateLegacySendAttributes';
 import { getOwn } from '../util/getOwn';
 import { markRead, markViewed } from '../services/MessageUpdater';
+import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
 import { isMessageUnread } from '../util/isMessageUnread';
 import {
   isDirectConversation,
@@ -1192,6 +1193,8 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
     }
 
     await window.Signal.Data.deleteSentProtoByMessageId(this.id);
+
+    scheduleOptimizeFTS();
   }
 
   override isEmpty(): boolean {

ts/services/expiringMessagesDeletion.ts

@@ -8,6 +8,7 @@ import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary';
 import { sleep } from '../util/sleep';
 import { SECOND } from '../util/durations';
 import * as Errors from '../types/errors';
+import { scheduleOptimizeFTS } from './ftsOptimizer';
 
 class ExpiringMessagesDeletionService {
   public update: typeof this.checkExpiringMessages;
@@ -64,6 +65,10 @@ class ExpiringMessagesDeletionService {
           conversation.decrementMessageCount();
         }
       });
+
+      if (messages.length > 0) {
+        scheduleOptimizeFTS();
+      }
     } catch (error) {
       window.SignalContext.log.error(
         'destroyExpiredMessages: Error deleting expired messages',

ts/services/ftsOptimizer.ts

@@ -0,0 +1,63 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { debounce } from 'lodash';
+
+import { SECOND } from '../util/durations';
+import { sleep } from '../util/sleep';
+import { drop } from '../util/drop';
+import dataInterface from '../sql/Client';
+import type { FTSOptimizationStateType } from '../sql/Interface';
+import * as log from '../logging/log';
+
+const INTERACTIVITY_DELAY_MS = 50;
+
+class FTSOptimizer {
+  private isRunning = false;
+
+  public async run(): Promise<void> {
+    if (this.isRunning) {
+      return;
+    }
+    this.isRunning = true;
+
+    log.info('ftsOptimizer: starting');
+
+    let state: FTSOptimizationStateType | undefined;
+
+    const start = Date.now();
+
+    try {
+      do {
+        if (state !== undefined) {
+          // eslint-disable-next-line no-await-in-loop
+          await sleep(INTERACTIVITY_DELAY_MS);
+        }
+
+        // eslint-disable-next-line no-await-in-loop
+        state = await dataInterface.optimizeFTS(state);
+      } while (!state?.done);
+    } finally {
+      this.isRunning = false;
+    }
+
+    const duration = Date.now() - start;
+
+    if (!state) {
+      log.warn('ftsOptimizer: no final state');
+      return;
+    }
+
+    log.info(`ftsOptimizer: took ${duration}ms and ${state.steps} steps`);
+  }
+}
+
+const optimizer = new FTSOptimizer();
+
+export const optimizeFTS = (): void => {
+  drop(optimizer.run());
+};
+
+export const scheduleOptimizeFTS = debounce(optimizeFTS, SECOND, {
+  maxWait: 5 * SECOND,
+});

ts/sql/Interface.ts

@@ -361,6 +361,12 @@ export type GetAllStoriesResultType = ReadonlyArray<
   }
 >;
 
+export type FTSOptimizationStateType = Readonly<{
+  changes: number;
+  steps: number;
+  done?: boolean;
+}>;
+
 export type DataInterface = {
   close: () => Promise<void>;
   removeDB: () => Promise<void>;
@@ -710,6 +716,10 @@ export type DataInterface = {
   getMaxMessageCounter(): Promise<number | undefined>;
 
   getStatisticsForLogging(): Promise<Record<string, string>>;
+
+  optimizeFTS: (
+    state?: FTSOptimizationStateType
+  ) => Promise<FTSOptimizationStateType | undefined>;
 };
 
 export type ServerInterface = DataInterface & {

ts/sql/Server.ts

@@ -78,6 +78,7 @@ import type {
   DeleteSentProtoRecipientOptionsType,
   DeleteSentProtoRecipientResultType,
   EmojiType,
+  FTSOptimizationStateType,
   GetAllStoriesResultType,
   GetConversationRangeCenteredOnMessageResultType,
   GetKnownMessageAttachmentsResultType,
@@ -342,6 +343,8 @@ const dataInterface: ServerInterface = {
 
   getStatisticsForLogging,
 
+  optimizeFTS,
+
   // Server-only
 
   initialize,
@@ -5440,6 +5443,47 @@ async function removeKnownDraftAttachments(
   return Object.keys(lookup);
 }
 
+// Default value of 'automerge'.
+// See: https://www.sqlite.org/fts5.html#the_automerge_configuration_option
+const OPTIMIZE_FTS_PAGE_COUNT = 4;
+
+// This query is incremental. It gets the `state` from the return value of
+// previous `optimizeFTS` call. When `state.done` is `true` - optimization is
+// complete.
+async function optimizeFTS(
+  state?: FTSOptimizationStateType
+): Promise<FTSOptimizationStateType | undefined> {
+  // See https://www.sqlite.org/fts5.html#the_merge_command
+  let pageCount = OPTIMIZE_FTS_PAGE_COUNT;
+  if (state === undefined) {
+    pageCount = -pageCount;
+  }
+
+  const db = getInstance();
+  const { changes } = prepare(
+    db,
+    `
+      INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount);
+    `
+  ).run({ pageCount });
+
+  if (state === undefined) {
+    return {
+      changes,
+      steps: 1,
+    };
+  }
+
+  const { changes: prevChanges, steps } = state;
+
+  if (Math.abs(changes - prevChanges) < 2) {
+    return { changes, steps, done: true };
+  }
+
+  // More work is needed.
+  return { changes, steps: steps + 1 };
+}
+
 async function getJobsInQueue(queueType: string): Promise<Array<StoredJob>> {
   const db = getInstance();
   return getJobsInQueueSync(db, queueType);

yarn.lock

@@ -2154,10 +2154,10 @@
     "@react-spring/shared" "~9.5.5"
     "@react-spring/types" "~9.5.5"
 
-"@signalapp/better-sqlite3@8.4.1":
-  version "8.4.1"
-  resolved "https://registry.yarnpkg.com/@signalapp/better-sqlite3/-/better-sqlite3-8.4.1.tgz#34bae41b90db72ce03cf07d1f21af38b7b7df568"
-  integrity sha512-oBODIoowPukVGvS6UUrSvn2hM2TS29N8ED13tM5hJi6D6XDUomR1jsdiaUxT+W8oS8yPj0XjDogzMxQ1PfDDWg==
+"@signalapp/better-sqlite3@8.4.2":
+  version "8.4.2"
+  resolved "https://registry.yarnpkg.com/@signalapp/better-sqlite3/-/better-sqlite3-8.4.2.tgz#5354b6f50cad4784e74c69f2e550572fd4bb4887"
+  integrity sha512-txik3OAVLyHJN+DHnSxxO4UCxU00e30HBr6DQsD1XPx37aHXz3IrUtlCDwzPTCkBOu3rMFYxfSg11Ki1CpLRJA==
   dependencies:
     bindings "^1.5.0"
     tar "^6.1.0"