Reenable FTS optimization

ts/background.ts

@@ -53,6 +53,7 @@ import { senderCertificateService } from './services/senderCertificate';
 import { GROUP_CREDENTIALS_KEY } from './services/groupCredentialFetcher';
 import * as KeyboardLayout from './services/keyboardLayout';
 import * as StorageService from './services/storage';
+import { optimizeFTS } from './services/ftsOptimizer';
 import { RoutineProfileRefresher } from './routineProfileRefresh';
 import { isOlderThan, toDayMillis } from './util/timestamp';
 import { isValidReactionEmoji } from './reactions/isValidReactionEmoji';
@@ -982,6 +983,8 @@ export async function startApp(): Promise<void> {
     if (newVersion) {
       await window.Signal.Data.cleanupOrphanedAttachments();
 
+      optimizeFTS();
+
       drop(window.Signal.Data.ensureFilePermissions());
     }
 

ts/models/conversations.ts

@@ -87,6 +87,7 @@ import {
   notificationService,
 } from '../services/notifications';
 import { storageServiceUploadJob } from '../services/storage';
+import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
 import { getSendOptions } from '../util/getSendOptions';
 import { isConversationAccepted } from '../util/isConversationAccepted';
 import {
@@ -4759,6 +4760,8 @@ export class ConversationModel extends window.Backbone
     await window.Signal.Data.removeAllMessagesInConversation(this.id, {
       logId: this.idForLogging(),
     });
+
+    scheduleOptimizeFTS();
   }
 
   getTitle(options?: { isShort?: boolean }): string {

ts/models/messages.ts

@@ -69,6 +69,7 @@ import { migrateLegacyReadStatus } from '../messages/migrateLegacyReadStatus';
 import { migrateLegacySendAttributes } from '../messages/migrateLegacySendAttributes';
 import { getOwn } from '../util/getOwn';
 import { markRead, markViewed } from '../services/MessageUpdater';
+import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
 import {
   isDirectConversation,
   isGroup,
@@ -1156,6 +1157,8 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
     }
 
     await window.Signal.Data.deleteSentProtoByMessageId(this.id);
+
+    scheduleOptimizeFTS();
   }
 
   override isEmpty(): boolean {

ts/services/expiringMessagesDeletion.ts

@@ -8,6 +8,7 @@ import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary';
 import { sleep } from '../util/sleep';
 import { SECOND } from '../util/durations';
 import * as Errors from '../types/errors';
+import { scheduleOptimizeFTS } from './ftsOptimizer';
 
 class ExpiringMessagesDeletionService {
   public update: typeof this.checkExpiringMessages;
@@ -56,6 +57,10 @@ class ExpiringMessagesDeletionService {
         message.trigger('expired');
         window.reduxActions.conversations.messageExpired(message.id);
       });
+
+      if (messages.length > 0) {
+        scheduleOptimizeFTS();
+      }
     } catch (error) {
       window.SignalContext.log.error(
         'destroyExpiredMessages: Error deleting expired messages',

ts/services/ftsOptimizer.ts

@@ -0,0 +1,63 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { debounce } from 'lodash';
+
+import { SECOND } from '../util/durations';
+import { sleep } from '../util/sleep';
+import { drop } from '../util/drop';
+import dataInterface from '../sql/Client';
+import type { FTSOptimizationStateType } from '../sql/Interface';
+import * as log from '../logging/log';
+
+const INTERACTIVITY_DELAY_MS = 50;
+
+class FTSOptimizer {
+  private isRunning = false;
+
+  public async run(): Promise<void> {
+    if (this.isRunning) {
+      return;
+    }
+    this.isRunning = true;
+
+    log.info('ftsOptimizer: starting');
+
+    let state: FTSOptimizationStateType | undefined;
+
+    const start = Date.now();
+
+    try {
+      do {
+        if (state !== undefined) {
+          // eslint-disable-next-line no-await-in-loop
+          await sleep(INTERACTIVITY_DELAY_MS);
+        }
+
+        // eslint-disable-next-line no-await-in-loop
+        state = await dataInterface.optimizeFTS(state);
+      } while (!state?.done);
+    } finally {
+      this.isRunning = false;
+    }
+
+    const duration = Date.now() - start;
+
+    if (!state) {
+      log.warn('ftsOptimizer: no final state');
+      return;
+    }
+
+    log.info(`ftsOptimizer: took ${duration}ms and ${state.steps} steps`);
+  }
+}
+
+const optimizer = new FTSOptimizer();
+
+export const optimizeFTS = (): void => {
+  drop(optimizer.run());
+};
+
+export const scheduleOptimizeFTS = debounce(optimizeFTS, SECOND, {
+  maxWait: 5 * SECOND,
+});

ts/sql/Interface.ts

@@ -411,6 +411,11 @@ export type GetAllStoriesResultType = ReadonlyArray<
   }
 >;
 
+export type FTSOptimizationStateType = Readonly<{
+  steps: number;
+  done?: boolean;
+}>;
+
 export type EditedMessageType = Readonly<{
   conversationId: string;
   messageId: string;
@@ -818,6 +823,10 @@ export type DataInterface = {
   getMaxMessageCounter(): Promise<number | undefined>;
 
   getStatisticsForLogging(): Promise<Record<string, string>>;
+
+  optimizeFTS: (
+    state?: FTSOptimizationStateType
+  ) => Promise<FTSOptimizationStateType | undefined>;
 };
 
 export type ServerInterface = DataInterface & {

ts/sql/Server.ts

@@ -93,6 +93,7 @@ import type {
   DeleteSentProtoRecipientResultType,
   EditedMessageType,
   EmojiType,
+  FTSOptimizationStateType,
   GetAllStoriesResultType,
   GetConversationRangeCenteredOnMessageResultType,
   GetKnownMessageAttachmentsResultType,
@@ -402,6 +403,8 @@ const dataInterface: ServerInterface = {
 
   getStatisticsForLogging,
 
+  optimizeFTS,
+
   // Server-only
 
   initialize,
@@ -2219,6 +2222,7 @@ async function _removeAllMessages(): Promise<void> {
   const db = getInstance();
   db.exec(`
     DELETE FROM messages;
+    INSERT INTO messages_fts(messages_fts) VALUES('optimize');
   `);
 }
 
@@ -5544,6 +5548,8 @@ async function removeAll(): Promise<void> {
       DELETE FROM unprocessed;
       DELETE FROM uninstalled_sticker_packs;
 
+      INSERT INTO messages_fts(messages_fts) VALUES('optimize');
+
       --- Re-create the messages delete trigger
       --- See migration 45
       CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN
@@ -6127,6 +6133,48 @@ async function removeKnownDraftAttachments(
   return Object.keys(lookup);
 }
 
+const OPTIMIZE_FTS_PAGE_COUNT = 64;
+
+// This query is incremental. It gets the `state` from the return value of
+// previous `optimizeFTS` call. When `state.done` is `true` - optimization is
+// complete.
+async function optimizeFTS(
+  state?: FTSOptimizationStateType
+): Promise<FTSOptimizationStateType | undefined> {
+  // See https://www.sqlite.org/fts5.html#the_merge_command
+  let pageCount = OPTIMIZE_FTS_PAGE_COUNT;
+  if (state === undefined) {
+    pageCount = -pageCount;
+  }
+  const db = getInstance();
+  const getChanges = prepare(db, 'SELECT total_changes() as changes;', {
+    pluck: true,
+  });
+
+  const changeDifference = db.transaction(() => {
+    const before: number = getChanges.get({});
+
+    prepare(
+      db,
+      `
+        INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount);
+      `
+    ).run({ pageCount });
+
+    const after: number = getChanges.get({});
+
+    return after - before;
+  })();
+
+  const nextSteps = (state?.steps ?? 0) + 1;
+
+  // From documentation:
+  // "If the difference is less than 2, then the 'merge' command was a no-op"
+  const done = changeDifference < 2;
+
+  return { steps: nextSteps, done };
+}
+
 async function getJobsInQueue(queueType: string): Promise<Array<StoredJob>> {
   const db = getInstance();
   return getJobsInQueueSync(db, queueType);

ts/sql/migrations/930-fts5-secure-delete.ts

@@ -17,12 +17,8 @@ export function updateToSchemaVersion930(
   }
 
   db.transaction(() => {
-    db.exec(`
-      INSERT INTO messages_fts
-      (messages_fts, rank)
-      VALUES
-      ('secure-delete', 1);
-    `);
+    // This was a migration that enabled 'secure-delete' in FTS
+
     db.pragma('user_version = 930');
   })();
 

ts/sql/migrations/940-fts5-revert.ts

@@ -0,0 +1,55 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { Database } from '@signalapp/better-sqlite3';
+
+import type { LoggerType } from '../../types/Logging';
+
+export const version = 940;
+
+export function updateToSchemaVersion940(
+  currentVersion: number,
+  db: Database,
+  logger: LoggerType
+): void {
+  if (currentVersion >= 940) {
+    return;
+  }
+
+  db.transaction(() => {
+    const wasEnabled =
+      db
+        .prepare(
+          `
+            SELECT v FROM messages_fts_config WHERE k is 'secure-delete';
+          `
+        )
+        .pluck()
+        .get() === 1;
+
+    if (wasEnabled) {
+      logger.info('updateToSchemaVersion940: rebuilding fts5 index');
+      db.exec(`
+        --- Disable 'secure-delete'
+        INSERT INTO messages_fts
+        (messages_fts, rank)
+        VALUES
+        ('secure-delete', 0);
+
+        --- Rebuild the index to fix the corruption
+        INSERT INTO messages_fts
+        (messages_fts)
+        VALUES
+        ('rebuild');
+      `);
+    } else {
+      logger.info(
+        'updateToSchemaVersion940: secure delete was not enabled, skipping'
+      );
+    }
+
+    db.pragma('user_version = 940');
+  })();
+
+  logger.info('updateToSchemaVersion940: success!');
+}

ts/sql/migrations/index.ts

@@ -67,10 +67,11 @@ import updateToSchemaVersion89 from './89-call-history';
 import updateToSchemaVersion90 from './90-delete-story-reply-screenshot';
 import updateToSchemaVersion91 from './91-clean-keys';
 import { updateToSchemaVersion920 } from './920-clean-more-keys';
+import { updateToSchemaVersion930 } from './930-fts5-secure-delete';
 import {
   version as MAX_VERSION,
-  updateToSchemaVersion930,
-} from './930-fts5-secure-delete';
+  updateToSchemaVersion940,
+} from './940-fts5-revert';
 
 function updateToSchemaVersion1(
   currentVersion: number,
@@ -2012,6 +2013,7 @@ export const SCHEMA_VERSIONS = [
   // From here forward, all migrations should be multiples of 10
   updateToSchemaVersion920,
   updateToSchemaVersion930,
+  updateToSchemaVersion940,
 ];
 
 export function updateSchema(db: Database, logger: LoggerType): void {