Use FTS5 optimizer in production

ts/sql/Interface.ts

@@ -411,6 +411,11 @@ export type GetAllStoriesResultType = ReadonlyArray<
   }
 >;
 
+export type FTSOptimizationStateType = Readonly<{
+  steps: number;
+  done?: boolean;
+}>;
+
 export type EditedMessageType = Readonly<{
   conversationId: string;
   messageId: string;
@@ -819,6 +824,10 @@ export type DataInterface = {
   getMaxMessageCounter(): Promise<number | undefined>;
 
   getStatisticsForLogging(): Promise<Record<string, string>>;
+
+  optimizeFTS: (
+    state?: FTSOptimizationStateType
+  ) => Promise<FTSOptimizationStateType | undefined>;
 };
 
 export type ServerInterface = DataInterface & {
@@ -892,6 +901,7 @@ export type ServerInterface = DataInterface & {
   // Server-only
 
   initialize: (options: {
+    appVersion: string;
     configDir: string;
     key: string;
     logger: LoggerType;

ts/sql/Server.ts

@@ -93,6 +93,7 @@ import type {
   DeleteSentProtoRecipientResultType,
   EditedMessageType,
   EmojiType,
+  FTSOptimizationStateType,
   GetAllStoriesResultType,
   GetConversationRangeCenteredOnMessageResultType,
   GetKnownMessageAttachmentsResultType,
@@ -403,6 +404,8 @@ const dataInterface: ServerInterface = {
 
   getStatisticsForLogging,
 
+  optimizeFTS,
+
   // Server-only
 
   initialize,
@@ -574,10 +577,12 @@ SQL.setLogHandler((code, value) => {
 });
 
 async function initialize({
+  appVersion,
   configDir,
   key,
   logger: suppliedLogger,
 }: {
+  appVersion: string;
   configDir: string;
   key: string;
   logger: LoggerType;
@@ -614,7 +619,7 @@ async function initialize({
     // For profiling use:
     // db.pragma('cipher_profile=\'sqlcipher.log\'');
 
-    updateSchema(writable, logger);
+    updateSchema(writable, logger, appVersion);
 
     readonly = openAndSetUpSQLCipher(databaseFilePath, { key, readonly: true });
 
@@ -2274,6 +2279,7 @@ async function _removeAllMessages(): Promise<void> {
   const db = await getWritableInstance();
   db.exec(`
     DELETE FROM messages;
+    INSERT INTO messages_fts(messages_fts) VALUES('optimize');
   `);
 }
 
@@ -5656,6 +5662,8 @@ async function removeAll(): Promise<void> {
       DELETE FROM unprocessed;
       DELETE FROM uninstalled_sticker_packs;
 
+      INSERT INTO messages_fts(messages_fts) VALUES('optimize');
+
       --- Re-create the messages delete trigger
       --- See migration 45
       CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN
@@ -6239,6 +6247,48 @@ async function removeKnownDraftAttachments(
   return Object.keys(lookup);
 }
 
+const OPTIMIZE_FTS_PAGE_COUNT = 64;
+
+// This query is incremental. It gets the `state` from the return value of
+// previous `optimizeFTS` call. When `state.done` is `true` - optimization is
+// complete.
+async function optimizeFTS(
+  state?: FTSOptimizationStateType
+): Promise<FTSOptimizationStateType | undefined> {
+  // See https://www.sqlite.org/fts5.html#the_merge_command
+  let pageCount = OPTIMIZE_FTS_PAGE_COUNT;
+  if (state === undefined) {
+    pageCount = -pageCount;
+  }
+  const db = await getWritableInstance();
+  const getChanges = prepare(db, 'SELECT total_changes() as changes;', {
+    pluck: true,
+  });
+
+  const changeDifference = db.transaction(() => {
+    const before: number = getChanges.get({});
+
+    prepare(
+      db,
+      `
+        INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount);
+      `
+    ).run({ pageCount });
+
+    const after: number = getChanges.get({});
+
+    return after - before;
+  })();
+
+  const nextSteps = (state?.steps ?? 0) + 1;
+
+  // From documentation:
+  // "If the difference is less than 2, then the 'merge' command was a no-op"
+  const done = changeDifference < 2;
+
+  return { steps: nextSteps, done };
+}
+
 async function getJobsInQueue(queueType: string): Promise<Array<StoredJob>> {
   const db = getReadonlyInstance();
   return getJobsInQueueSync(db, queueType);

ts/sql/main.ts

@@ -15,6 +15,7 @@ import type DB from './Server';
 const MIN_TRACE_DURATION = 40;
 
 export type InitializeOptions = Readonly<{
+  appVersion: string;
   configDir: string;
   key: string;
   logger: LoggerType;
@@ -127,6 +128,7 @@ export class MainSQL {
   }
 
   public async initialize({
+    appVersion,
     configDir,
     key,
     logger,
@@ -141,7 +143,7 @@ export class MainSQL {
 
     this.onReady = this.send({
       type: 'init',
-      options: { configDir, key },
+      options: { appVersion, configDir, key },
     });
 
     await this.onReady;

ts/sql/migrations/940-fts5-revert.ts

@@ -17,37 +17,7 @@ export function updateToSchemaVersion940(
   }
 
   db.transaction(() => {
-    const wasEnabled =
-      db
-        .prepare(
-          `
-            SELECT v FROM messages_fts_config WHERE k is 'secure-delete';
-          `
-        )
-        .pluck()
-        .get() === 1;
-
-    if (wasEnabled) {
-      logger.info('updateToSchemaVersion940: rebuilding fts5 index');
-      db.exec(`
-        --- Disable 'secure-delete'
-        INSERT INTO messages_fts
-        (messages_fts, rank)
-        VALUES
-        ('secure-delete', 0);
-
-        --- Rebuild the index to fix the corruption
-        INSERT INTO messages_fts
-        (messages_fts)
-        VALUES
-        ('rebuild');
-      `);
-    } else {
-      logger.info(
-        'updateToSchemaVersion940: secure delete was not enabled, skipping'
-      );
-    }
-
+    // This was a migration that disabled secure-delete and rebuilt the index
     db.pragma('user_version = 940');
   })();
 

ts/sql/migrations/950-fts5-secure-delete.ts

@@ -17,14 +17,7 @@ export function updateToSchemaVersion950(
   }
 
   db.transaction(() => {
-    db.exec(`
-      --- Enable 'secure-delete'
-      INSERT INTO messages_fts
-      (messages_fts, rank)
-      VALUES
-      ('secure-delete', 1);
-    `);
-
+    // This was a migration that enable secure-delete
     db.pragma('user_version = 950');
   })();
 

ts/sql/migrations/index.ts

@@ -6,6 +6,7 @@ import { keyBy } from 'lodash';
 import { v4 as generateUuid } from 'uuid';
 
 import type { LoggerType } from '../../types/Logging';
+import { isProduction } from '../../util/version';
 import {
   getSchemaVersion,
   getUserVersion,
@@ -2019,7 +2020,53 @@ export class DBVersionFromFutureError extends Error {
   override name = 'DBVersionFromFutureError';
 }
 
-export function updateSchema(db: Database, logger: LoggerType): void {
+export function lazyFTS5SecureDelete(
+  db: Database,
+  logger: LoggerType,
+  enabled: boolean
+): void {
+  const isEnabled =
+    db
+      .prepare(
+        `
+          SELECT v FROM messages_fts_config WHERE k is 'secure-delete';
+        `
+      )
+      .pluck()
+      .get() === 1;
+
+  if (isEnabled && !enabled) {
+    logger.info('lazyFTS5SecureDelete: disabling, rebuilding fts5 index');
+    db.exec(`
+      -- Disable secure-delete
+      INSERT INTO messages_fts
+      (messages_fts, rank)
+      VALUES
+      ('secure-delete', 0);
+
+      --- Rebuild the index to fix the corruption
+      INSERT INTO messages_fts
+      (messages_fts)
+      VALUES
+      ('rebuild');
+    `);
+  } else if (!isEnabled && enabled) {
+    logger.info('lazyFTS5SecureDelete: enabling');
+    db.exec(`
+      -- Enable secure-delete
+      INSERT INTO messages_fts
+      (messages_fts, rank)
+      VALUES
+      ('secure-delete', 1);
+    `);
+  }
+}
+
+export function updateSchema(
+  db: Database,
+  logger: LoggerType,
+  appVersion: string
+): void {
   const sqliteVersion = getSQLiteVersion(db);
   const sqlcipherVersion = getSQLCipherVersion(db);
   const startingVersion = getUserVersion(db);
@@ -2047,6 +2094,8 @@ export function updateSchema(db: Database, logger: LoggerType): void {
     runSchemaUpdate(startingVersion, db, logger);
   }
 
+  lazyFTS5SecureDelete(db, logger, !isProduction(appVersion));
+
   if (startingVersion !== MAX_VERSION) {
     const start = Date.now();
     db.pragma('optimize');