From b6ed789197125d45383473721fb508c5fdc64c50 Mon Sep 17 00:00:00 2001 From: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com> Date: Sun, 3 Sep 2023 01:03:19 +0200 Subject: [PATCH] Reenable FTS optimization --- ts/background.ts | 3 + ts/models/conversations.ts | 3 + ts/models/messages.ts | 3 + ts/services/expiringMessagesDeletion.ts | 5 ++ ts/services/ftsOptimizer.ts | 63 +++++++++++++++++++++ ts/sql/Interface.ts | 9 +++ ts/sql/Server.ts | 48 ++++++++++++++++ ts/sql/migrations/930-fts5-secure-delete.ts | 8 +-- ts/sql/migrations/940-fts5-revert.ts | 55 ++++++++++++++++++ ts/sql/migrations/index.ts | 6 +- 10 files changed, 195 insertions(+), 8 deletions(-) create mode 100644 ts/services/ftsOptimizer.ts create mode 100644 ts/sql/migrations/940-fts5-revert.ts diff --git a/ts/background.ts b/ts/background.ts index d81c7e012dfc..ddbc0bafc299 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -53,6 +53,7 @@ import { senderCertificateService } from './services/senderCertificate'; import { GROUP_CREDENTIALS_KEY } from './services/groupCredentialFetcher'; import * as KeyboardLayout from './services/keyboardLayout'; import * as StorageService from './services/storage'; +import { optimizeFTS } from './services/ftsOptimizer'; import { RoutineProfileRefresher } from './routineProfileRefresh'; import { isOlderThan, toDayMillis } from './util/timestamp'; import { isValidReactionEmoji } from './reactions/isValidReactionEmoji'; @@ -982,6 +983,8 @@ export async function startApp(): Promise { if (newVersion) { await window.Signal.Data.cleanupOrphanedAttachments(); + optimizeFTS(); + drop(window.Signal.Data.ensureFilePermissions()); } diff --git a/ts/models/conversations.ts b/ts/models/conversations.ts index ad720a835e6f..6eaf279cbae1 100644 --- a/ts/models/conversations.ts +++ b/ts/models/conversations.ts @@ -87,6 +87,7 @@ import { notificationService, } from '../services/notifications'; import { storageServiceUploadJob } from '../services/storage'; +import { scheduleOptimizeFTS } from '../services/ftsOptimizer'; import { getSendOptions } from '../util/getSendOptions'; import { isConversationAccepted } from '../util/isConversationAccepted'; import { @@ -4759,6 +4760,8 @@ export class ConversationModel extends window.Backbone await window.Signal.Data.removeAllMessagesInConversation(this.id, { logId: this.idForLogging(), }); + + scheduleOptimizeFTS(); } getTitle(options?: { isShort?: boolean }): string { diff --git a/ts/models/messages.ts b/ts/models/messages.ts index 31a82b75c061..89812fb0937f 100644 --- a/ts/models/messages.ts +++ b/ts/models/messages.ts @@ -69,6 +69,7 @@ import { migrateLegacyReadStatus } from '../messages/migrateLegacyReadStatus'; import { migrateLegacySendAttributes } from '../messages/migrateLegacySendAttributes'; import { getOwn } from '../util/getOwn'; import { markRead, markViewed } from '../services/MessageUpdater'; +import { scheduleOptimizeFTS } from '../services/ftsOptimizer'; import { isDirectConversation, isGroup, @@ -1156,6 +1157,8 @@ export class MessageModel extends window.Backbone.Model { } await window.Signal.Data.deleteSentProtoByMessageId(this.id); + + scheduleOptimizeFTS(); } override isEmpty(): boolean { diff --git a/ts/services/expiringMessagesDeletion.ts b/ts/services/expiringMessagesDeletion.ts index 2a538b5cd9f6..ceee06d72a79 100644 --- a/ts/services/expiringMessagesDeletion.ts +++ b/ts/services/expiringMessagesDeletion.ts @@ -8,6 +8,7 @@ import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary'; import { sleep } from '../util/sleep'; import { SECOND } from '../util/durations'; import * as Errors from '../types/errors'; +import { scheduleOptimizeFTS } from './ftsOptimizer'; class ExpiringMessagesDeletionService { public update: typeof this.checkExpiringMessages; @@ -56,6 +57,10 @@ class ExpiringMessagesDeletionService { message.trigger('expired'); window.reduxActions.conversations.messageExpired(message.id); }); + + if (messages.length > 0) { + scheduleOptimizeFTS(); + } } catch (error) { window.SignalContext.log.error( 'destroyExpiredMessages: Error deleting expired messages', diff --git a/ts/services/ftsOptimizer.ts b/ts/services/ftsOptimizer.ts new file mode 100644 index 000000000000..3add63450581 --- /dev/null +++ b/ts/services/ftsOptimizer.ts @@ -0,0 +1,63 @@ +// Copyright 2023 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { debounce } from 'lodash'; + +import { SECOND } from '../util/durations'; +import { sleep } from '../util/sleep'; +import { drop } from '../util/drop'; +import dataInterface from '../sql/Client'; +import type { FTSOptimizationStateType } from '../sql/Interface'; +import * as log from '../logging/log'; + +const INTERACTIVITY_DELAY_MS = 50; + +class FTSOptimizer { + private isRunning = false; + + public async run(): Promise { + if (this.isRunning) { + return; + } + this.isRunning = true; + + log.info('ftsOptimizer: starting'); + + let state: FTSOptimizationStateType | undefined; + + const start = Date.now(); + + try { + do { + if (state !== undefined) { + // eslint-disable-next-line no-await-in-loop + await sleep(INTERACTIVITY_DELAY_MS); + } + + // eslint-disable-next-line no-await-in-loop + state = await dataInterface.optimizeFTS(state); + } while (!state?.done); + } finally { + this.isRunning = false; + } + + const duration = Date.now() - start; + + if (!state) { + log.warn('ftsOptimizer: no final state'); + return; + } + + log.info(`ftsOptimizer: took ${duration}ms and ${state.steps} steps`); + } +} + +const optimizer = new FTSOptimizer(); + +export const optimizeFTS = (): void => { + drop(optimizer.run()); +}; + +export const scheduleOptimizeFTS = debounce(optimizeFTS, SECOND, { + maxWait: 5 * SECOND, +}); diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 9fc9c5befa6c..5819730eca5b 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -411,6 +411,11 @@ export type GetAllStoriesResultType = ReadonlyArray< } >; +export type FTSOptimizationStateType = Readonly<{ + steps: number; + done?: boolean; +}>; + export type EditedMessageType = Readonly<{ conversationId: string; messageId: string; @@ -818,6 +823,10 @@ export type DataInterface = { getMaxMessageCounter(): Promise; getStatisticsForLogging(): Promise>; + + optimizeFTS: ( + state?: FTSOptimizationStateType + ) => Promise; }; export type ServerInterface = DataInterface & { diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 138a61721f20..fd414de34f82 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -93,6 +93,7 @@ import type { DeleteSentProtoRecipientResultType, EditedMessageType, EmojiType, + FTSOptimizationStateType, GetAllStoriesResultType, GetConversationRangeCenteredOnMessageResultType, GetKnownMessageAttachmentsResultType, @@ -402,6 +403,8 @@ const dataInterface: ServerInterface = { getStatisticsForLogging, + optimizeFTS, + // Server-only initialize, @@ -2219,6 +2222,7 @@ async function _removeAllMessages(): Promise { const db = getInstance(); db.exec(` DELETE FROM messages; + INSERT INTO messages_fts(messages_fts) VALUES('optimize'); `); } @@ -5544,6 +5548,8 @@ async function removeAll(): Promise { DELETE FROM unprocessed; DELETE FROM uninstalled_sticker_packs; + INSERT INTO messages_fts(messages_fts) VALUES('optimize'); + --- Re-create the messages delete trigger --- See migration 45 CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN @@ -6127,6 +6133,48 @@ async function removeKnownDraftAttachments( return Object.keys(lookup); } +const OPTIMIZE_FTS_PAGE_COUNT = 64; + +// This query is incremental. It gets the `state` from the return value of +// previous `optimizeFTS` call. When `state.done` is `true` - optimization is +// complete. +async function optimizeFTS( + state?: FTSOptimizationStateType +): Promise { + // See https://www.sqlite.org/fts5.html#the_merge_command + let pageCount = OPTIMIZE_FTS_PAGE_COUNT; + if (state === undefined) { + pageCount = -pageCount; + } + const db = getInstance(); + const getChanges = prepare(db, 'SELECT total_changes() as changes;', { + pluck: true, + }); + + const changeDifference = db.transaction(() => { + const before: number = getChanges.get({}); + + prepare( + db, + ` + INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount); + ` + ).run({ pageCount }); + + const after: number = getChanges.get({}); + + return after - before; + })(); + + const nextSteps = (state?.steps ?? 0) + 1; + + // From documentation: + // "If the difference is less than 2, then the 'merge' command was a no-op" + const done = changeDifference < 2; + + return { steps: nextSteps, done }; +} + async function getJobsInQueue(queueType: string): Promise> { const db = getInstance(); return getJobsInQueueSync(db, queueType); diff --git a/ts/sql/migrations/930-fts5-secure-delete.ts b/ts/sql/migrations/930-fts5-secure-delete.ts index 983c7b5bcd0e..90fccb9d3347 100644 --- a/ts/sql/migrations/930-fts5-secure-delete.ts +++ b/ts/sql/migrations/930-fts5-secure-delete.ts @@ -17,12 +17,8 @@ export function updateToSchemaVersion930( } db.transaction(() => { - db.exec(` - INSERT INTO messages_fts - (messages_fts, rank) - VALUES - ('secure-delete', 1); - `); + // This was a migration that enabled 'secure-delete' in FTS + db.pragma('user_version = 930'); })(); diff --git a/ts/sql/migrations/940-fts5-revert.ts b/ts/sql/migrations/940-fts5-revert.ts new file mode 100644 index 000000000000..45792bfb49d8 --- /dev/null +++ b/ts/sql/migrations/940-fts5-revert.ts @@ -0,0 +1,55 @@ +// Copyright 2023 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import type { Database } from '@signalapp/better-sqlite3'; + +import type { LoggerType } from '../../types/Logging'; + +export const version = 940; + +export function updateToSchemaVersion940( + currentVersion: number, + db: Database, + logger: LoggerType +): void { + if (currentVersion >= 940) { + return; + } + + db.transaction(() => { + const wasEnabled = + db + .prepare( + ` + SELECT v FROM messages_fts_config WHERE k is 'secure-delete'; + ` + ) + .pluck() + .get() === 1; + + if (wasEnabled) { + logger.info('updateToSchemaVersion940: rebuilding fts5 index'); + db.exec(` + --- Disable 'secure-delete' + INSERT INTO messages_fts + (messages_fts, rank) + VALUES + ('secure-delete', 0); + + --- Rebuild the index to fix the corruption + INSERT INTO messages_fts + (messages_fts) + VALUES + ('rebuild'); + `); + } else { + logger.info( + 'updateToSchemaVersion940: secure delete was not enabled, skipping' + ); + } + + db.pragma('user_version = 940'); + })(); + + logger.info('updateToSchemaVersion940: success!'); +} diff --git a/ts/sql/migrations/index.ts b/ts/sql/migrations/index.ts index b867128e3cea..c7be006ae826 100644 --- a/ts/sql/migrations/index.ts +++ b/ts/sql/migrations/index.ts @@ -67,10 +67,11 @@ import updateToSchemaVersion89 from './89-call-history'; import updateToSchemaVersion90 from './90-delete-story-reply-screenshot'; import updateToSchemaVersion91 from './91-clean-keys'; import { updateToSchemaVersion920 } from './920-clean-more-keys'; +import { updateToSchemaVersion930 } from './930-fts5-secure-delete'; import { version as MAX_VERSION, - updateToSchemaVersion930, -} from './930-fts5-secure-delete'; + updateToSchemaVersion940, +} from './940-fts5-revert'; function updateToSchemaVersion1( currentVersion: number, @@ -2012,6 +2013,7 @@ export const SCHEMA_VERSIONS = [ // From here forward, all migrations should be multiples of 10 updateToSchemaVersion920, updateToSchemaVersion930, + updateToSchemaVersion940, ]; export function updateSchema(db: Database, logger: LoggerType): void {