Reenable FTS optimization

ts/sql/Interface.ts

@@ -411,6 +411,11 @@ export type GetAllStoriesResultType = ReadonlyArray<
   }
 >;
 
+export type FTSOptimizationStateType = Readonly<{
+  steps: number;
+  done?: boolean;
+}>;
+
 export type EditedMessageType = Readonly<{
   conversationId: string;
   messageId: string;
@@ -818,6 +823,10 @@ export type DataInterface = {
   getMaxMessageCounter(): Promise<number | undefined>;
 
   getStatisticsForLogging(): Promise<Record<string, string>>;
+
+  optimizeFTS: (
+    state?: FTSOptimizationStateType
+  ) => Promise<FTSOptimizationStateType | undefined>;
 };
 
 export type ServerInterface = DataInterface & {

ts/sql/Server.ts

@@ -93,6 +93,7 @@ import type {
   DeleteSentProtoRecipientResultType,
   EditedMessageType,
   EmojiType,
+  FTSOptimizationStateType,
   GetAllStoriesResultType,
   GetConversationRangeCenteredOnMessageResultType,
   GetKnownMessageAttachmentsResultType,
@@ -402,6 +403,8 @@ const dataInterface: ServerInterface = {
 
   getStatisticsForLogging,
 
+  optimizeFTS,
+
   // Server-only
 
   initialize,
@@ -2219,6 +2222,7 @@ async function _removeAllMessages(): Promise<void> {
   const db = getInstance();
   db.exec(`
     DELETE FROM messages;
+    INSERT INTO messages_fts(messages_fts) VALUES('optimize');
   `);
 }
 
@@ -5544,6 +5548,8 @@ async function removeAll(): Promise<void> {
       DELETE FROM unprocessed;
       DELETE FROM uninstalled_sticker_packs;
 
+      INSERT INTO messages_fts(messages_fts) VALUES('optimize');
+
       --- Re-create the messages delete trigger
       --- See migration 45
       CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN
@@ -6127,6 +6133,48 @@ async function removeKnownDraftAttachments(
   return Object.keys(lookup);
 }
 
+const OPTIMIZE_FTS_PAGE_COUNT = 64;
+
+// This query is incremental. It gets the `state` from the return value of
+// previous `optimizeFTS` call. When `state.done` is `true` - optimization is
+// complete.
+async function optimizeFTS(
+  state?: FTSOptimizationStateType
+): Promise<FTSOptimizationStateType | undefined> {
+  // See https://www.sqlite.org/fts5.html#the_merge_command
+  let pageCount = OPTIMIZE_FTS_PAGE_COUNT;
+  if (state === undefined) {
+    pageCount = -pageCount;
+  }
+  const db = getInstance();
+  const getChanges = prepare(db, 'SELECT total_changes() as changes;', {
+    pluck: true,
+  });
+
+  const changeDifference = db.transaction(() => {
+    const before: number = getChanges.get({});
+
+    prepare(
+      db,
+      `
+        INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount);
+      `
+    ).run({ pageCount });
+
+    const after: number = getChanges.get({});
+
+    return after - before;
+  })();
+
+  const nextSteps = (state?.steps ?? 0) + 1;
+
+  // From documentation:
+  // "If the difference is less than 2, then the 'merge' command was a no-op"
+  const done = changeDifference < 2;
+
+  return { steps: nextSteps, done };
+}
+
 async function getJobsInQueue(queueType: string): Promise<Array<StoredJob>> {
   const db = getInstance();
   return getJobsInQueueSync(db, queueType);

ts/sql/migrations/930-fts5-secure-delete.ts

@@ -17,12 +17,8 @@ export function updateToSchemaVersion930(
   }
 
   db.transaction(() => {
-    db.exec(`
-      INSERT INTO messages_fts
-      (messages_fts, rank)
-      VALUES
-      ('secure-delete', 1);
-    `);
+    // This was a migration that enabled 'secure-delete' in FTS
+
     db.pragma('user_version = 930');
   })();
 

ts/sql/migrations/940-fts5-revert.ts

@@ -0,0 +1,55 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { Database } from '@signalapp/better-sqlite3';
+
+import type { LoggerType } from '../../types/Logging';
+
+export const version = 940;
+
+export function updateToSchemaVersion940(
+  currentVersion: number,
+  db: Database,
+  logger: LoggerType
+): void {
+  if (currentVersion >= 940) {
+    return;
+  }
+
+  db.transaction(() => {
+    const wasEnabled =
+      db
+        .prepare(
+          `
+            SELECT v FROM messages_fts_config WHERE k is 'secure-delete';
+          `
+        )
+        .pluck()
+        .get() === 1;
+
+    if (wasEnabled) {
+      logger.info('updateToSchemaVersion940: rebuilding fts5 index');
+      db.exec(`
+        --- Disable 'secure-delete'
+        INSERT INTO messages_fts
+        (messages_fts, rank)
+        VALUES
+        ('secure-delete', 0);
+
+        --- Rebuild the index to fix the corruption
+        INSERT INTO messages_fts
+        (messages_fts)
+        VALUES
+        ('rebuild');
+      `);
+    } else {
+      logger.info(
+        'updateToSchemaVersion940: secure delete was not enabled, skipping'
+      );
+    }
+
+    db.pragma('user_version = 940');
+  })();
+
+  logger.info('updateToSchemaVersion940: success!');
+}

ts/sql/migrations/index.ts

@@ -67,10 +67,11 @@ import updateToSchemaVersion89 from './89-call-history';
 import updateToSchemaVersion90 from './90-delete-story-reply-screenshot';
 import updateToSchemaVersion91 from './91-clean-keys';
 import { updateToSchemaVersion920 } from './920-clean-more-keys';
+import { updateToSchemaVersion930 } from './930-fts5-secure-delete';
 import {
   version as MAX_VERSION,
-  updateToSchemaVersion930,
-} from './930-fts5-secure-delete';
+  updateToSchemaVersion940,
+} from './940-fts5-revert';
 
 function updateToSchemaVersion1(
   currentVersion: number,
@@ -2012,6 +2013,7 @@ export const SCHEMA_VERSIONS = [
   // From here forward, all migrations should be multiples of 10
   updateToSchemaVersion920,
   updateToSchemaVersion930,
+  updateToSchemaVersion940,
 ];
 
 export function updateSchema(db: Database, logger: LoggerType): void {