Use FTS5 optimizer in production

app/main.ts

@@ -1590,6 +1590,7 @@ async function initializeSQL(
     // `sql.sqlCall` will throw an uninitialized error instead of waiting for
     // init to finish.
     await sql.initialize({
+      appVersion: app.getVersion(),
       configDir: userDataPath,
       key,
       logger: getLogger(),

ts/background.ts

@@ -52,6 +52,7 @@ import { senderCertificateService } from './services/senderCertificate';
 import { GROUP_CREDENTIALS_KEY } from './services/groupCredentialFetcher';
 import * as KeyboardLayout from './services/keyboardLayout';
 import * as StorageService from './services/storage';
+import { optimizeFTS } from './services/ftsOptimizer';
 import { RoutineProfileRefresher } from './routineProfileRefresh';
 import { isOlderThan, toDayMillis } from './util/timestamp';
 import { isValidReactionEmoji } from './reactions/isValidReactionEmoji';
@@ -980,6 +981,8 @@ export async function startApp(): Promise<void> {
     if (newVersion) {
       await window.Signal.Data.cleanupOrphanedAttachments();
 
+      optimizeFTS();
+
       drop(window.Signal.Data.ensureFilePermissions());
     }
 

ts/models/conversations.ts

@@ -87,6 +87,7 @@ import {
   notificationService,
 } from '../services/notifications';
 import { storageServiceUploadJob } from '../services/storage';
+import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
 import { getSendOptions } from '../util/getSendOptions';
 import { isConversationAccepted } from '../util/isConversationAccepted';
 import {
@@ -4806,6 +4807,8 @@ export class ConversationModel extends window.Backbone
     await window.Signal.Data.removeAllMessagesInConversation(this.id, {
       logId: this.idForLogging(),
     });
+
+    scheduleOptimizeFTS();
   }
 
   getTitle(options?: { isShort?: boolean }): string {

ts/models/messages.ts

@@ -65,6 +65,7 @@ import { migrateLegacyReadStatus } from '../messages/migrateLegacyReadStatus';
 import { migrateLegacySendAttributes } from '../messages/migrateLegacySendAttributes';
 import { getOwn } from '../util/getOwn';
 import { markRead, markViewed } from '../services/MessageUpdater';
+import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
 import {
   isDirectConversation,
   isGroup,
@@ -605,6 +606,8 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
     }
 
     await window.Signal.Data.deleteSentProtoByMessageId(this.id);
+
+    scheduleOptimizeFTS();
   }
 
   override isEmpty(): boolean {

ts/services/expiringMessagesDeletion.ts

@@ -9,6 +9,7 @@ import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary';
 import { sleep } from '../util/sleep';
 import { SECOND } from '../util/durations';
 import * as Errors from '../types/errors';
+import { scheduleOptimizeFTS } from './ftsOptimizer';
 
 class ExpiringMessagesDeletionService {
   public update: typeof this.checkExpiringMessages;
@@ -55,6 +56,10 @@ class ExpiringMessagesDeletionService {
           window.reduxActions.conversations.messageExpired(message.id);
         });
       });
+
+      if (messages.length > 0) {
+        scheduleOptimizeFTS();
+      }
     } catch (error) {
       window.SignalContext.log.error(
         'destroyExpiredMessages: Error deleting expired messages',

ts/services/ftsOptimizer.ts

@@ -0,0 +1,69 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { debounce } from 'lodash';
+
+import { SECOND } from '../util/durations';
+import { sleep } from '../util/sleep';
+import { drop } from '../util/drop';
+import { isProduction } from '../util/version';
+import dataInterface from '../sql/Client';
+import type { FTSOptimizationStateType } from '../sql/Interface';
+import * as log from '../logging/log';
+
+const INTERACTIVITY_DELAY_MS = 50;
+
+class FTSOptimizer {
+  private isRunning = false;
+
+  public async run(): Promise<void> {
+    if (!isProduction(window.getVersion())) {
+      log.info('ftsOptimizer: not running when not in production');
+      return;
+    }
+
+    if (this.isRunning) {
+      return;
+    }
+    this.isRunning = true;
+
+    log.info('ftsOptimizer: starting');
+
+    let state: FTSOptimizationStateType | undefined;
+
+    const start = Date.now();
+
+    try {
+      do {
+        if (state !== undefined) {
+          // eslint-disable-next-line no-await-in-loop
+          await sleep(INTERACTIVITY_DELAY_MS);
+        }
+
+        // eslint-disable-next-line no-await-in-loop
+        state = await dataInterface.optimizeFTS(state);
+      } while (!state?.done);
+    } finally {
+      this.isRunning = false;
+    }
+
+    const duration = Date.now() - start;
+
+    if (!state) {
+      log.warn('ftsOptimizer: no final state');
+      return;
+    }
+
+    log.info(`ftsOptimizer: took ${duration}ms and ${state.steps} steps`);
+  }
+}
+
+const optimizer = new FTSOptimizer();
+
+export const optimizeFTS = (): void => {
+  drop(optimizer.run());
+};
+
+export const scheduleOptimizeFTS = debounce(optimizeFTS, SECOND, {
+  maxWait: 5 * SECOND,
+});

ts/sql/Interface.ts

@@ -411,6 +411,11 @@ export type GetAllStoriesResultType = ReadonlyArray<
   }
 >;
 
+export type FTSOptimizationStateType = Readonly<{
+  steps: number;
+  done?: boolean;
+}>;
+
 export type EditedMessageType = Readonly<{
   conversationId: string;
   messageId: string;
@@ -819,6 +824,10 @@ export type DataInterface = {
   getMaxMessageCounter(): Promise<number | undefined>;
 
   getStatisticsForLogging(): Promise<Record<string, string>>;
+
+  optimizeFTS: (
+    state?: FTSOptimizationStateType
+  ) => Promise<FTSOptimizationStateType | undefined>;
 };
 
 export type ServerInterface = DataInterface & {
@@ -892,6 +901,7 @@ export type ServerInterface = DataInterface & {
   // Server-only
 
   initialize: (options: {
+    appVersion: string;
     configDir: string;
     key: string;
     logger: LoggerType;

ts/sql/Server.ts

@@ -93,6 +93,7 @@ import type {
   DeleteSentProtoRecipientResultType,
   EditedMessageType,
   EmojiType,
+  FTSOptimizationStateType,
   GetAllStoriesResultType,
   GetConversationRangeCenteredOnMessageResultType,
   GetKnownMessageAttachmentsResultType,
@@ -403,6 +404,8 @@ const dataInterface: ServerInterface = {
 
   getStatisticsForLogging,
 
+  optimizeFTS,
+
   // Server-only
 
   initialize,
@@ -574,10 +577,12 @@ SQL.setLogHandler((code, value) => {
 });
 
 async function initialize({
+  appVersion,
   configDir,
   key,
   logger: suppliedLogger,
 }: {
+  appVersion: string;
   configDir: string;
   key: string;
   logger: LoggerType;
@@ -614,7 +619,7 @@ async function initialize({
     // For profiling use:
     // db.pragma('cipher_profile=\'sqlcipher.log\'');
 
-    updateSchema(writable, logger);
+    updateSchema(writable, logger, appVersion);
 
     readonly = openAndSetUpSQLCipher(databaseFilePath, { key, readonly: true });
 
@@ -2274,6 +2279,7 @@ async function _removeAllMessages(): Promise<void> {
   const db = await getWritableInstance();
   db.exec(`
     DELETE FROM messages;
+    INSERT INTO messages_fts(messages_fts) VALUES('optimize');
   `);
 }
 
@@ -5656,6 +5662,8 @@ async function removeAll(): Promise<void> {
       DELETE FROM unprocessed;
       DELETE FROM uninstalled_sticker_packs;
 
+      INSERT INTO messages_fts(messages_fts) VALUES('optimize');
+
       --- Re-create the messages delete trigger
       --- See migration 45
       CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN
@@ -6239,6 +6247,48 @@ async function removeKnownDraftAttachments(
   return Object.keys(lookup);
 }
 
+const OPTIMIZE_FTS_PAGE_COUNT = 64;
+
+// This query is incremental. It gets the `state` from the return value of
+// previous `optimizeFTS` call. When `state.done` is `true` - optimization is
+// complete.
+async function optimizeFTS(
+  state?: FTSOptimizationStateType
+): Promise<FTSOptimizationStateType | undefined> {
+  // See https://www.sqlite.org/fts5.html#the_merge_command
+  let pageCount = OPTIMIZE_FTS_PAGE_COUNT;
+  if (state === undefined) {
+    pageCount = -pageCount;
+  }
+  const db = await getWritableInstance();
+  const getChanges = prepare(db, 'SELECT total_changes() as changes;', {
+    pluck: true,
+  });
+
+  const changeDifference = db.transaction(() => {
+    const before: number = getChanges.get({});
+
+    prepare(
+      db,
+      `
+        INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount);
+      `
+    ).run({ pageCount });
+
+    const after: number = getChanges.get({});
+
+    return after - before;
+  })();
+
+  const nextSteps = (state?.steps ?? 0) + 1;
+
+  // From documentation:
+  // "If the difference is less than 2, then the 'merge' command was a no-op"
+  const done = changeDifference < 2;
+
+  return { steps: nextSteps, done };
+}
+
 async function getJobsInQueue(queueType: string): Promise<Array<StoredJob>> {
   const db = getReadonlyInstance();
   return getJobsInQueueSync(db, queueType);

ts/sql/main.ts

@@ -15,6 +15,7 @@ import type DB from './Server';
 const MIN_TRACE_DURATION = 40;
 
 export type InitializeOptions = Readonly<{
+  appVersion: string;
   configDir: string;
   key: string;
   logger: LoggerType;
@@ -127,6 +128,7 @@ export class MainSQL {
   }
 
   public async initialize({
+    appVersion,
     configDir,
     key,
     logger,
@@ -141,7 +143,7 @@ export class MainSQL {
 
     this.onReady = this.send({
       type: 'init',
-      options: { configDir, key },
+      options: { appVersion, configDir, key },
     });
 
     await this.onReady;

ts/sql/migrations/940-fts5-revert.ts

@@ -17,37 +17,7 @@ export function updateToSchemaVersion940(
   }
 
   db.transaction(() => {
-    const wasEnabled =
-      db
-        .prepare(
-          `
-            SELECT v FROM messages_fts_config WHERE k is 'secure-delete';
-          `
-        )
-        .pluck()
-        .get() === 1;
-
-    if (wasEnabled) {
-      logger.info('updateToSchemaVersion940: rebuilding fts5 index');
-      db.exec(`
-        --- Disable 'secure-delete'
-        INSERT INTO messages_fts
-        (messages_fts, rank)
-        VALUES
-        ('secure-delete', 0);
-
-        --- Rebuild the index to fix the corruption
-        INSERT INTO messages_fts
-        (messages_fts)
-        VALUES
-        ('rebuild');
-      `);
-    } else {
-      logger.info(
-        'updateToSchemaVersion940: secure delete was not enabled, skipping'
-      );
-    }
-
+    // This was a migration that disabled secure-delete and rebuilt the index
     db.pragma('user_version = 940');
   })();
 

ts/sql/migrations/950-fts5-secure-delete.ts

@@ -17,14 +17,7 @@ export function updateToSchemaVersion950(
   }
 
   db.transaction(() => {
-    db.exec(`
-      --- Enable 'secure-delete'
-      INSERT INTO messages_fts
-      (messages_fts, rank)
-      VALUES
-      ('secure-delete', 1);
-    `);
-
+    // This was a migration that enable secure-delete
     db.pragma('user_version = 950');
   })();
 

ts/sql/migrations/index.ts

@@ -6,6 +6,7 @@ import { keyBy } from 'lodash';
 import { v4 as generateUuid } from 'uuid';
 
 import type { LoggerType } from '../../types/Logging';
+import { isProduction } from '../../util/version';
 import {
   getSchemaVersion,
   getUserVersion,
@@ -2019,7 +2020,53 @@ export class DBVersionFromFutureError extends Error {
   override name = 'DBVersionFromFutureError';
 }
 
-export function updateSchema(db: Database, logger: LoggerType): void {
+export function lazyFTS5SecureDelete(
+  db: Database,
+  logger: LoggerType,
+  enabled: boolean
+): void {
+  const isEnabled =
+    db
+      .prepare(
+        `
+          SELECT v FROM messages_fts_config WHERE k is 'secure-delete';
+        `
+      )
+      .pluck()
+      .get() === 1;
+
+  if (isEnabled && !enabled) {
+    logger.info('lazyFTS5SecureDelete: disabling, rebuilding fts5 index');
+    db.exec(`
+      -- Disable secure-delete
+      INSERT INTO messages_fts
+      (messages_fts, rank)
+      VALUES
+      ('secure-delete', 0);
+
+      --- Rebuild the index to fix the corruption
+      INSERT INTO messages_fts
+      (messages_fts)
+      VALUES
+      ('rebuild');
+    `);
+  } else if (!isEnabled && enabled) {
+    logger.info('lazyFTS5SecureDelete: enabling');
+    db.exec(`
+      -- Enable secure-delete
+      INSERT INTO messages_fts
+      (messages_fts, rank)
+      VALUES
+      ('secure-delete', 1);
+    `);
+  }
+}
+
+export function updateSchema(
+  db: Database,
+  logger: LoggerType,
+  appVersion: string
+): void {
   const sqliteVersion = getSQLiteVersion(db);
   const sqlcipherVersion = getSQLCipherVersion(db);
   const startingVersion = getUserVersion(db);
@@ -2047,6 +2094,8 @@ export function updateSchema(db: Database, logger: LoggerType): void {
     runSchemaUpdate(startingVersion, db, logger);
   }
 
+  lazyFTS5SecureDelete(db, logger, !isProduction(appVersion));
+
   if (startingVersion !== MAX_VERSION) {
     const start = Date.now();
     db.pragma('optimize');