Use Signal tokenizer in FTS5

ts/sql/migrations/75-signal-tokenizer.ts

@@ -0,0 +1,44 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { Database } from '@signalapp/better-sqlite3';
+
+import type { LoggerType } from '../../types/Logging';
+
+export default function updateToSchemaVersion75(
+  currentVersion: number,
+  db: Database,
+  logger: LoggerType
+): void {
+  if (currentVersion >= 75) {
+    return;
+  }
+
+  db.transaction(() => {
+    db.exec(
+      `
+      -- Create FTS table with custom tokenizer from
+      -- @signalapp/better-sqlite3.
+
+      DROP TABLE messages_fts;
+
+      CREATE VIRTUAL TABLE messages_fts USING fts5(
+        body,
+        tokenize = 'signal_tokenizer'
+      );
+
+      -- Reindex messages
+      -- Based on messages_on_insert trigger from migrations/45-stories.ts
+
+      INSERT INTO messages_fts (rowid, body)
+      SELECT rowid, body
+      FROM messages
+      WHERE isViewOnce IS NOT 1 AND storyId IS NULL;
+      `
+    );
+
+    db.pragma('user_version = 75');
+  })();
+
+  logger.info('updateToSchemaVersion75: success!');
+}

ts/sql/migrations/index.ts

@@ -50,6 +50,7 @@ import updateToSchemaVersion71 from './71-merge-notifications';
 import updateToSchemaVersion72 from './72-optimize-call-id-message-lookup';
 import updateToSchemaVersion73 from './73-remove-phone-number-discovery';
 import updateToSchemaVersion74 from './74-optimize-convo-open';
+import updateToSchemaVersion75 from './75-signal-tokenizer';
 
 function updateToSchemaVersion1(
   currentVersion: number,
@@ -1969,6 +1970,7 @@ export const SCHEMA_VERSIONS = [
   updateToSchemaVersion72,
   updateToSchemaVersion73,
   updateToSchemaVersion74,
+  updateToSchemaVersion75,
 ];
 
 export function updateSchema(db: Database, logger: LoggerType): void {