Use signal_tokenizer for search query

ts/sql/Server.ts

@@ -1749,6 +1749,11 @@ async function searchMessages({
 
   const db = getUnsafeWritableInstance('only temp table use');
 
+  const normalizedQuery = db
+    .signalTokenize(query)
+    .map(token => `"${token.replace(/"/g, '""')}"*`)
+    .join(' ');
+
   // sqlite queries with a join on a virtual table (like FTS5) are de-optimized
   // and can't use indices for ordering results. Instead an in-memory index of
   // the join rows is sorted on the fly, and this becomes substantially
@@ -1778,7 +1783,7 @@ async function searchMessages({
         WHERE
           messages_fts.body MATCH $query;
       `
-    ).run({ query });
+    ).run({ query: normalizedQuery });
 
     if (conversationId === undefined) {
       db.prepare<Query>(
@@ -1829,7 +1834,7 @@ async function searchMessages({
       INNER JOIN messages
         ON messages.rowid = tmp_filtered_results.rowid
       WHERE
-        messages_fts.body MATCH ${query}
+        messages_fts.body MATCH ${normalizedQuery}
       ORDER BY messages.received_at DESC, messages.sent_at DESC
       LIMIT ${limit}
     `;

ts/state/ducks/search.ts

@@ -6,7 +6,6 @@ import { debounce, omit, reject } from 'lodash';
 
 import type { ReadonlyDeep } from 'type-fest';
 import type { StateType as RootStateType } from '../reducer';
-import { cleanSearchTerm } from '../../util/cleanSearchTerm';
 import { filterAndSortConversationsByRecent } from '../../util/filterAndSortConversations';
 import type {
   ClientSearchResultMessageType,
@@ -294,21 +293,20 @@ async function queryMessages({
   contactServiceIdsMatchingQuery?: Array<ServiceIdString>;
 }): Promise<Array<ClientSearchResultMessageType>> {
   try {
-    const normalized = cleanSearchTerm(query);
-    if (normalized.length === 0) {
+    if (query.length === 0) {
       return [];
     }
 
     if (searchConversationId) {
       return dataSearchMessages({
-        query: normalized,
+        query,
         conversationId: searchConversationId,
         contactServiceIdsMatchingQuery,
       });
     }
 
     return dataSearchMessages({
-      query: normalized,
+      query,
       contactServiceIdsMatchingQuery,
     });
   } catch (e) {

ts/test-both/util/cleanSearchTerm_test.ts

@@ -1,13 +0,0 @@
-// Copyright 2021 Signal Messenger, LLC
-// SPDX-License-Identifier: AGPL-3.0-only
-
-import { assert } from 'chai';
-import { cleanSearchTerm } from '../../util/cleanSearchTerm';
-
-describe('cleanSearchTerm', () => {
-  it('should remove \\ from a search term', () => {
-    const searchTerm = '\\search\\term';
-    const sanitizedSearchTerm = cleanSearchTerm(searchTerm);
-    assert.strictEqual(sanitizedSearchTerm, 'search* term*');
-  });
-});

ts/util/cleanSearchTerm.ts

@@ -1,27 +0,0 @@
-// Copyright 2019 Signal Messenger, LLC
-// SPDX-License-Identifier: AGPL-3.0-only
-
-export function cleanSearchTerm(searchTerm: string): string {
-  const lowercase = searchTerm.toLowerCase();
-  const withoutSpecialCharacters = lowercase.replace(
-    /([-!"#$%&'()*+,./\\:;<=>?@[\]^_`{|}~])/g,
-    ' '
-  );
-  const whiteSpaceNormalized = withoutSpecialCharacters.replace(/\s+/g, ' ');
-  const byToken = whiteSpaceNormalized.split(' ');
-  const withoutSpecialTokens = byToken.filter(
-    token =>
-      token &&
-      token !== 'and' &&
-      token !== 'or' &&
-      token !== 'not' &&
-      token !== ')' &&
-      token !== '(' &&
-      token !== '+' &&
-      token !== ',' &&
-      token !== 'near'
-  );
-  const withWildcards = withoutSpecialTokens.map(token => `${token}*`);
-
-  return withWildcards.join(' ').trim();
-}