Update better-sqlite3 and remove FTS optimization

package.json

@@ -92,7 +92,7 @@
     "@popperjs/core": "2.11.6",
     "@react-aria/utils": "3.16.0",
     "@react-spring/web": "9.5.5",
-    "@signalapp/better-sqlite3": "8.4.3",
+    "@signalapp/better-sqlite3": "8.5.1",
     "@signalapp/libsignal-client": "0.31.0",
     "@signalapp/ringrtc": "2.31.2",
     "@signalapp/windows-dummy-keystroke": "1.0.0",

ts/background.ts

@@ -53,7 +53,6 @@ import { senderCertificateService } from './services/senderCertificate';
 import { GROUP_CREDENTIALS_KEY } from './services/groupCredentialFetcher';
 import * as KeyboardLayout from './services/keyboardLayout';
 import * as StorageService from './services/storage';
-import { optimizeFTS } from './services/ftsOptimizer';
 import { RoutineProfileRefresher } from './routineProfileRefresh';
 import { isOlderThan, toDayMillis } from './util/timestamp';
 import { isValidReactionEmoji } from './reactions/isValidReactionEmoji';
@@ -983,8 +982,6 @@ export async function startApp(): Promise<void> {
     if (newVersion) {
       await window.Signal.Data.cleanupOrphanedAttachments();
 
-      optimizeFTS();
-
       drop(window.Signal.Data.ensureFilePermissions());
     }
 

ts/models/conversations.ts

@@ -87,7 +87,6 @@ import {
   notificationService,
 } from '../services/notifications';
 import { storageServiceUploadJob } from '../services/storage';
-import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
 import { getSendOptions } from '../util/getSendOptions';
 import { isConversationAccepted } from '../util/isConversationAccepted';
 import {
@@ -4760,8 +4759,6 @@ export class ConversationModel extends window.Backbone
     await window.Signal.Data.removeAllMessagesInConversation(this.id, {
       logId: this.idForLogging(),
     });
-
-    scheduleOptimizeFTS();
   }
 
   getTitle(options?: { isShort?: boolean }): string {

ts/models/messages.ts

@@ -69,7 +69,6 @@ import { migrateLegacyReadStatus } from '../messages/migrateLegacyReadStatus';
 import { migrateLegacySendAttributes } from '../messages/migrateLegacySendAttributes';
 import { getOwn } from '../util/getOwn';
 import { markRead, markViewed } from '../services/MessageUpdater';
-import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
 import {
   isDirectConversation,
   isGroup,
@@ -1157,8 +1156,6 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
     }
 
     await window.Signal.Data.deleteSentProtoByMessageId(this.id);
-
-    scheduleOptimizeFTS();
   }
 
   override isEmpty(): boolean {

ts/services/expiringMessagesDeletion.ts

@@ -8,7 +8,6 @@ import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary';
 import { sleep } from '../util/sleep';
 import { SECOND } from '../util/durations';
 import * as Errors from '../types/errors';
-import { scheduleOptimizeFTS } from './ftsOptimizer';
 
 class ExpiringMessagesDeletionService {
   public update: typeof this.checkExpiringMessages;
@@ -57,10 +56,6 @@ class ExpiringMessagesDeletionService {
         message.trigger('expired');
         window.reduxActions.conversations.messageExpired(message.id);
       });
-
-      if (messages.length > 0) {
-        scheduleOptimizeFTS();
-      }
     } catch (error) {
       window.SignalContext.log.error(
         'destroyExpiredMessages: Error deleting expired messages',

ts/services/ftsOptimizer.ts

@@ -1,63 +0,0 @@
-// Copyright 2023 Signal Messenger, LLC
-// SPDX-License-Identifier: AGPL-3.0-only
-
-import { debounce } from 'lodash';
-
-import { SECOND } from '../util/durations';
-import { sleep } from '../util/sleep';
-import { drop } from '../util/drop';
-import dataInterface from '../sql/Client';
-import type { FTSOptimizationStateType } from '../sql/Interface';
-import * as log from '../logging/log';
-
-const INTERACTIVITY_DELAY_MS = 50;
-
-class FTSOptimizer {
-  private isRunning = false;
-
-  public async run(): Promise<void> {
-    if (this.isRunning) {
-      return;
-    }
-    this.isRunning = true;
-
-    log.info('ftsOptimizer: starting');
-
-    let state: FTSOptimizationStateType | undefined;
-
-    const start = Date.now();
-
-    try {
-      do {
-        if (state !== undefined) {
-          // eslint-disable-next-line no-await-in-loop
-          await sleep(INTERACTIVITY_DELAY_MS);
-        }
-
-        // eslint-disable-next-line no-await-in-loop
-        state = await dataInterface.optimizeFTS(state);
-      } while (!state?.done);
-    } finally {
-      this.isRunning = false;
-    }
-
-    const duration = Date.now() - start;
-
-    if (!state) {
-      log.warn('ftsOptimizer: no final state');
-      return;
-    }
-
-    log.info(`ftsOptimizer: took ${duration}ms and ${state.steps} steps`);
-  }
-}
-
-const optimizer = new FTSOptimizer();
-
-export const optimizeFTS = (): void => {
-  drop(optimizer.run());
-};
-
-export const scheduleOptimizeFTS = debounce(optimizeFTS, SECOND, {
-  maxWait: 5 * SECOND,
-});

ts/sql/Interface.ts

@@ -411,11 +411,6 @@ export type GetAllStoriesResultType = ReadonlyArray<
   }
 >;
 
-export type FTSOptimizationStateType = Readonly<{
-  steps: number;
-  done?: boolean;
-}>;
-
 export type EditedMessageType = Readonly<{
   conversationId: string;
   messageId: string;
@@ -823,10 +818,6 @@ export type DataInterface = {
   getMaxMessageCounter(): Promise<number | undefined>;
 
   getStatisticsForLogging(): Promise<Record<string, string>>;
-
-  optimizeFTS: (
-    state?: FTSOptimizationStateType
-  ) => Promise<FTSOptimizationStateType | undefined>;
 };
 
 export type ServerInterface = DataInterface & {

ts/sql/Server.ts

@@ -93,7 +93,6 @@ import type {
   DeleteSentProtoRecipientResultType,
   EditedMessageType,
   EmojiType,
-  FTSOptimizationStateType,
   GetAllStoriesResultType,
   GetConversationRangeCenteredOnMessageResultType,
   GetKnownMessageAttachmentsResultType,
@@ -403,8 +402,6 @@ const dataInterface: ServerInterface = {
 
   getStatisticsForLogging,
 
-  optimizeFTS,
-
   // Server-only
 
   initialize,
@@ -2222,7 +2219,6 @@ async function _removeAllMessages(): Promise<void> {
   const db = getInstance();
   db.exec(`
     DELETE FROM messages;
-    INSERT INTO messages_fts(messages_fts) VALUES('optimize');
   `);
 }
 
@@ -5548,8 +5544,6 @@ async function removeAll(): Promise<void> {
       DELETE FROM unprocessed;
       DELETE FROM uninstalled_sticker_packs;
 
-      INSERT INTO messages_fts(messages_fts) VALUES('optimize');
-
       --- Re-create the messages delete trigger
       --- See migration 45
       CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN
@@ -6133,48 +6127,6 @@ async function removeKnownDraftAttachments(
   return Object.keys(lookup);
 }
 
-const OPTIMIZE_FTS_PAGE_COUNT = 64;
-
-// This query is incremental. It gets the `state` from the return value of
-// previous `optimizeFTS` call. When `state.done` is `true` - optimization is
-// complete.
-async function optimizeFTS(
-  state?: FTSOptimizationStateType
-): Promise<FTSOptimizationStateType | undefined> {
-  // See https://www.sqlite.org/fts5.html#the_merge_command
-  let pageCount = OPTIMIZE_FTS_PAGE_COUNT;
-  if (state === undefined) {
-    pageCount = -pageCount;
-  }
-  const db = getInstance();
-  const getChanges = prepare(db, 'SELECT total_changes() as changes;', {
-    pluck: true,
-  });
-
-  const changeDifference = db.transaction(() => {
-    const before: number = getChanges.get({});
-
-    prepare(
-      db,
-      `
-        INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount);
-      `
-    ).run({ pageCount });
-
-    const after: number = getChanges.get({});
-
-    return after - before;
-  })();
-
-  const nextSteps = (state?.steps ?? 0) + 1;
-
-  // From documentation:
-  // "If the difference is less than 2, then the 'merge' command was a no-op"
-  const done = changeDifference < 2;
-
-  return { steps: nextSteps, done };
-}
-
 async function getJobsInQueue(queueType: string): Promise<Array<StoredJob>> {
   const db = getInstance();
   return getJobsInQueueSync(db, queueType);

ts/sql/migrations/930-fts5-secure-delete.ts

@@ -0,0 +1,30 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { Database } from '@signalapp/better-sqlite3';
+
+import type { LoggerType } from '../../types/Logging';
+
+export const version = 930;
+
+export function updateToSchemaVersion930(
+  currentVersion: number,
+  db: Database,
+  logger: LoggerType
+): void {
+  if (currentVersion >= 930) {
+    return;
+  }
+
+  db.transaction(() => {
+    db.exec(`
+      INSERT INTO messages_fts
+      (messages_fts, rank)
+      VALUES
+      ('secure-delete', 1);
+    `);
+    db.pragma('user_version = 930');
+  })();
+
+  logger.info('updateToSchemaVersion930: success!');
+}

ts/sql/migrations/index.ts

@@ -66,10 +66,11 @@ import updateToSchemaVersion88 from './88-service-ids';
 import updateToSchemaVersion89 from './89-call-history';
 import updateToSchemaVersion90 from './90-delete-story-reply-screenshot';
 import updateToSchemaVersion91 from './91-clean-keys';
+import { updateToSchemaVersion920 } from './920-clean-more-keys';
 import {
   version as MAX_VERSION,
-  updateToSchemaVersion920,
+  updateToSchemaVersion930,
-} from './920-clean-more-keys';
+} from './930-fts5-secure-delete';
 
 function updateToSchemaVersion1(
   currentVersion: number,
@@ -2010,6 +2011,7 @@ export const SCHEMA_VERSIONS = [
   updateToSchemaVersion91,
   // From here forward, all migrations should be multiples of 10
   updateToSchemaVersion920,
+  updateToSchemaVersion930,
 ];
 
 export function updateSchema(db: Database, logger: LoggerType): void {

yarn.lock

@@ -3355,10 +3355,10 @@
     "@react-types/overlays" "^3.7.1"
     "@react-types/shared" "^3.18.0"
 
-"@signalapp/better-sqlite3@8.4.3":
+"@signalapp/better-sqlite3@8.5.1":
-  version "8.4.3"
+  version "8.5.1"
-  resolved "https://registry.yarnpkg.com/@signalapp/better-sqlite3/-/better-sqlite3-8.4.3.tgz#7ffa8d03d2a12543247936bfb7b9f74cdbc6fe9b"
+  resolved "https://registry.yarnpkg.com/@signalapp/better-sqlite3/-/better-sqlite3-8.5.1.tgz#b2bcf69d29f4d01cfecf2ad940ecd138f32470b3"
-  integrity sha512-6HaN8a90fWHBPIIZRRmSG7wc3BPczLx3Mb9MJ8wsatYqf8C1+2NdFae5AzII8Oe9YE9SSHzCjdPd8ST1m9+Qag==
+  integrity sha512-s9NDufkbTAuKShFmEg7eOew4cprdLI7mn5ISQFKgcFSiA/TG3wgLJW5wUff9S2vednIV9utyqRJZXrwcSAEQUg==
   dependencies:
     bindings "^1.5.0"
     tar "^6.1.0"