Use Signal tokenizer in FTS5

package.json

@@ -89,7 +89,7 @@
     "@indutny/sneequals": "4.0.0",
     "@popperjs/core": "2.11.6",
     "@react-spring/web": "9.5.5",
-    "@signalapp/better-sqlite3": "8.1.1",
+    "@signalapp/better-sqlite3": "8.3.1",
     "@signalapp/libsignal-client": "0.21.1",
     "@signalapp/ringrtc": "2.23.0",
     "@types/fabric": "4.5.3",

ts/scripts/test-electron.ts

@@ -13,15 +13,23 @@ const ELECTRON = join(
   process.platform === 'win32' ? 'electron.cmd' : 'electron'
 );
 
-const stdout = execFileSync(ELECTRON, [ROOT_DIR], {
-  cwd: ROOT_DIR,
-  env: {
-    ...process.env,
-    NODE_ENV: 'test',
-    TEST_QUIT_ON_COMPLETE: 'on',
-  },
-  encoding: 'utf8',
-});
+let stdout: string;
+try {
+  stdout = execFileSync(ELECTRON, [ROOT_DIR], {
+    cwd: ROOT_DIR,
+    env: {
+      ...process.env,
+      NODE_ENV: 'test',
+      TEST_QUIT_ON_COMPLETE: 'on',
+    },
+    encoding: 'utf8',
+  });
+} catch (error) {
+  console.error('Status', error.status);
+  console.error(error.output[0] ?? '');
+  console.error(error.output[1] ?? '');
+  process.exit(1);
+}
 
 const match = stdout.match(/ci:test-electron:done=(.*)?\n/);
 

ts/sql/Server.ts

@@ -2125,7 +2125,10 @@ async function _getAllMessages(): Promise<Array<MessageType>> {
 }
 async function _removeAllMessages(): Promise<void> {
   const db = getInstance();
-  db.prepare<EmptyQuery>('DELETE from messages;').run();
+  db.exec(`
+    DELETE FROM messages;
+    INSERT INTO messages_fts(messages_fts) VALUES('optimize');
+  `);
 }
 
 async function getAllMessageIds(): Promise<Array<string>> {
@@ -4881,6 +4884,8 @@ async function removeAll(): Promise<void> {
       DELETE FROM storyReads;
       DELETE FROM unprocessed;
       DELETE FROM uninstalled_sticker_packs;
+
+      INSERT INTO messages_fts(messages_fts) VALUES('optimize');
     `);
   })();
 }

ts/sql/migrations/75-signal-tokenizer.ts

@@ -0,0 +1,44 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { Database } from '@signalapp/better-sqlite3';
+
+import type { LoggerType } from '../../types/Logging';
+
+export default function updateToSchemaVersion75(
+  currentVersion: number,
+  db: Database,
+  logger: LoggerType
+): void {
+  if (currentVersion >= 75) {
+    return;
+  }
+
+  db.transaction(() => {
+    db.exec(
+      `
+      -- Create FTS table with custom tokenizer from
+      -- @signalapp/better-sqlite3.
+
+      DROP TABLE messages_fts;
+
+      CREATE VIRTUAL TABLE messages_fts USING fts5(
+        body,
+        tokenize = 'signal_tokenizer'
+      );
+
+      -- Reindex messages
+      -- Based on messages_on_insert trigger from migrations/45-stories.ts
+
+      INSERT INTO messages_fts (rowid, body)
+      SELECT rowid, body
+      FROM messages
+      WHERE isViewOnce IS NOT 1 AND storyId IS NULL;
+      `
+    );
+
+    db.pragma('user_version = 75');
+  })();
+
+  logger.info('updateToSchemaVersion75: success!');
+}

ts/sql/migrations/index.ts

@@ -50,6 +50,7 @@ import updateToSchemaVersion71 from './71-merge-notifications';
 import updateToSchemaVersion72 from './72-optimize-call-id-message-lookup';
 import updateToSchemaVersion73 from './73-remove-phone-number-discovery';
 import updateToSchemaVersion74 from './74-optimize-convo-open';
+import updateToSchemaVersion75 from './75-signal-tokenizer';
 
 function updateToSchemaVersion1(
   currentVersion: number,
@@ -1969,6 +1970,7 @@ export const SCHEMA_VERSIONS = [
   updateToSchemaVersion72,
   updateToSchemaVersion73,
   updateToSchemaVersion74,
+  updateToSchemaVersion75,
 ];
 
 export function updateSchema(db: Database, logger: LoggerType): void {

yarn.lock

@@ -2154,10 +2154,10 @@
     "@react-spring/shared" "~9.5.5"
     "@react-spring/types" "~9.5.5"
 
-"@signalapp/better-sqlite3@8.1.1":
-  version "8.1.1"
-  resolved "https://registry.yarnpkg.com/@signalapp/better-sqlite3/-/better-sqlite3-8.1.1.tgz#b39b9138bc79992d5e9707b246a67a1b88a18c49"
-  integrity sha512-k4njS/Vsr12Eow7G7TNBjJKSPU971DZQWGY4dfISdOBobAAiewstOwlqF3ZhpJznL6M3sri14MUM0iV1aZR15g==
+"@signalapp/better-sqlite3@8.3.1":
+  version "8.3.1"
+  resolved "https://registry.yarnpkg.com/@signalapp/better-sqlite3/-/better-sqlite3-8.3.1.tgz#51ac07e7d6a231725097877a2b0910000122cd01"
+  integrity sha512-i563EnQvOwUm9zFuX6O0KdAL3T/P17BT8lALrx0WrgQurGn3MCwsuNne6YpSSl9X3+JW+P4SN2tMfA0a1FGWzQ==
   dependencies:
     bindings "^1.5.0"
     tar "^6.1.0"