diff --git a/app/main.ts b/app/main.ts index 75dc4c1db..e46822a85 100644 --- a/app/main.ts +++ b/app/main.ts @@ -1590,6 +1590,7 @@ async function initializeSQL( // `sql.sqlCall` will throw an uninitialized error instead of waiting for // init to finish. await sql.initialize({ + appVersion: app.getVersion(), configDir: userDataPath, key, logger: getLogger(), diff --git a/ts/background.ts b/ts/background.ts index 7c9295ee8..add167359 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -52,6 +52,7 @@ import { senderCertificateService } from './services/senderCertificate'; import { GROUP_CREDENTIALS_KEY } from './services/groupCredentialFetcher'; import * as KeyboardLayout from './services/keyboardLayout'; import * as StorageService from './services/storage'; +import { optimizeFTS } from './services/ftsOptimizer'; import { RoutineProfileRefresher } from './routineProfileRefresh'; import { isOlderThan, toDayMillis } from './util/timestamp'; import { isValidReactionEmoji } from './reactions/isValidReactionEmoji'; @@ -980,6 +981,8 @@ export async function startApp(): Promise { if (newVersion) { await window.Signal.Data.cleanupOrphanedAttachments(); + optimizeFTS(); + drop(window.Signal.Data.ensureFilePermissions()); } diff --git a/ts/models/conversations.ts b/ts/models/conversations.ts index 0cff6f36c..50b15b83b 100644 --- a/ts/models/conversations.ts +++ b/ts/models/conversations.ts @@ -87,6 +87,7 @@ import { notificationService, } from '../services/notifications'; import { storageServiceUploadJob } from '../services/storage'; +import { scheduleOptimizeFTS } from '../services/ftsOptimizer'; import { getSendOptions } from '../util/getSendOptions'; import { isConversationAccepted } from '../util/isConversationAccepted'; import { @@ -4806,6 +4807,8 @@ export class ConversationModel extends window.Backbone await window.Signal.Data.removeAllMessagesInConversation(this.id, { logId: this.idForLogging(), }); + + scheduleOptimizeFTS(); } getTitle(options?: { isShort?: boolean }): string { diff --git a/ts/models/messages.ts b/ts/models/messages.ts index fbda908b9..db90d45a1 100644 --- a/ts/models/messages.ts +++ b/ts/models/messages.ts @@ -65,6 +65,7 @@ import { migrateLegacyReadStatus } from '../messages/migrateLegacyReadStatus'; import { migrateLegacySendAttributes } from '../messages/migrateLegacySendAttributes'; import { getOwn } from '../util/getOwn'; import { markRead, markViewed } from '../services/MessageUpdater'; +import { scheduleOptimizeFTS } from '../services/ftsOptimizer'; import { isDirectConversation, isGroup, @@ -605,6 +606,8 @@ export class MessageModel extends window.Backbone.Model { } await window.Signal.Data.deleteSentProtoByMessageId(this.id); + + scheduleOptimizeFTS(); } override isEmpty(): boolean { diff --git a/ts/services/expiringMessagesDeletion.ts b/ts/services/expiringMessagesDeletion.ts index dcb943041..98b03158e 100644 --- a/ts/services/expiringMessagesDeletion.ts +++ b/ts/services/expiringMessagesDeletion.ts @@ -9,6 +9,7 @@ import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary'; import { sleep } from '../util/sleep'; import { SECOND } from '../util/durations'; import * as Errors from '../types/errors'; +import { scheduleOptimizeFTS } from './ftsOptimizer'; class ExpiringMessagesDeletionService { public update: typeof this.checkExpiringMessages; @@ -55,6 +56,10 @@ class ExpiringMessagesDeletionService { window.reduxActions.conversations.messageExpired(message.id); }); }); + + if (messages.length > 0) { + scheduleOptimizeFTS(); + } } catch (error) { window.SignalContext.log.error( 'destroyExpiredMessages: Error deleting expired messages', diff --git a/ts/services/ftsOptimizer.ts b/ts/services/ftsOptimizer.ts new file mode 100644 index 000000000..dafadf084 --- /dev/null +++ b/ts/services/ftsOptimizer.ts @@ -0,0 +1,69 @@ +// Copyright 2023 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { debounce } from 'lodash'; + +import { SECOND } from '../util/durations'; +import { sleep } from '../util/sleep'; +import { drop } from '../util/drop'; +import { isProduction } from '../util/version'; +import dataInterface from '../sql/Client'; +import type { FTSOptimizationStateType } from '../sql/Interface'; +import * as log from '../logging/log'; + +const INTERACTIVITY_DELAY_MS = 50; + +class FTSOptimizer { + private isRunning = false; + + public async run(): Promise { + if (!isProduction(window.getVersion())) { + log.info('ftsOptimizer: not running when not in production'); + return; + } + + if (this.isRunning) { + return; + } + this.isRunning = true; + + log.info('ftsOptimizer: starting'); + + let state: FTSOptimizationStateType | undefined; + + const start = Date.now(); + + try { + do { + if (state !== undefined) { + // eslint-disable-next-line no-await-in-loop + await sleep(INTERACTIVITY_DELAY_MS); + } + + // eslint-disable-next-line no-await-in-loop + state = await dataInterface.optimizeFTS(state); + } while (!state?.done); + } finally { + this.isRunning = false; + } + + const duration = Date.now() - start; + + if (!state) { + log.warn('ftsOptimizer: no final state'); + return; + } + + log.info(`ftsOptimizer: took ${duration}ms and ${state.steps} steps`); + } +} + +const optimizer = new FTSOptimizer(); + +export const optimizeFTS = (): void => { + drop(optimizer.run()); +}; + +export const scheduleOptimizeFTS = debounce(optimizeFTS, SECOND, { + maxWait: 5 * SECOND, +}); diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index b4775a337..48c1497fb 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -411,6 +411,11 @@ export type GetAllStoriesResultType = ReadonlyArray< } >; +export type FTSOptimizationStateType = Readonly<{ + steps: number; + done?: boolean; +}>; + export type EditedMessageType = Readonly<{ conversationId: string; messageId: string; @@ -819,6 +824,10 @@ export type DataInterface = { getMaxMessageCounter(): Promise; getStatisticsForLogging(): Promise>; + + optimizeFTS: ( + state?: FTSOptimizationStateType + ) => Promise; }; export type ServerInterface = DataInterface & { @@ -892,6 +901,7 @@ export type ServerInterface = DataInterface & { // Server-only initialize: (options: { + appVersion: string; configDir: string; key: string; logger: LoggerType; diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 033c7bfcf..4f106a9b5 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -93,6 +93,7 @@ import type { DeleteSentProtoRecipientResultType, EditedMessageType, EmojiType, + FTSOptimizationStateType, GetAllStoriesResultType, GetConversationRangeCenteredOnMessageResultType, GetKnownMessageAttachmentsResultType, @@ -403,6 +404,8 @@ const dataInterface: ServerInterface = { getStatisticsForLogging, + optimizeFTS, + // Server-only initialize, @@ -574,10 +577,12 @@ SQL.setLogHandler((code, value) => { }); async function initialize({ + appVersion, configDir, key, logger: suppliedLogger, }: { + appVersion: string; configDir: string; key: string; logger: LoggerType; @@ -614,7 +619,7 @@ async function initialize({ // For profiling use: // db.pragma('cipher_profile=\'sqlcipher.log\''); - updateSchema(writable, logger); + updateSchema(writable, logger, appVersion); readonly = openAndSetUpSQLCipher(databaseFilePath, { key, readonly: true }); @@ -2274,6 +2279,7 @@ async function _removeAllMessages(): Promise { const db = await getWritableInstance(); db.exec(` DELETE FROM messages; + INSERT INTO messages_fts(messages_fts) VALUES('optimize'); `); } @@ -5656,6 +5662,8 @@ async function removeAll(): Promise { DELETE FROM unprocessed; DELETE FROM uninstalled_sticker_packs; + INSERT INTO messages_fts(messages_fts) VALUES('optimize'); + --- Re-create the messages delete trigger --- See migration 45 CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN @@ -6239,6 +6247,48 @@ async function removeKnownDraftAttachments( return Object.keys(lookup); } +const OPTIMIZE_FTS_PAGE_COUNT = 64; + +// This query is incremental. It gets the `state` from the return value of +// previous `optimizeFTS` call. When `state.done` is `true` - optimization is +// complete. +async function optimizeFTS( + state?: FTSOptimizationStateType +): Promise { + // See https://www.sqlite.org/fts5.html#the_merge_command + let pageCount = OPTIMIZE_FTS_PAGE_COUNT; + if (state === undefined) { + pageCount = -pageCount; + } + const db = await getWritableInstance(); + const getChanges = prepare(db, 'SELECT total_changes() as changes;', { + pluck: true, + }); + + const changeDifference = db.transaction(() => { + const before: number = getChanges.get({}); + + prepare( + db, + ` + INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount); + ` + ).run({ pageCount }); + + const after: number = getChanges.get({}); + + return after - before; + })(); + + const nextSteps = (state?.steps ?? 0) + 1; + + // From documentation: + // "If the difference is less than 2, then the 'merge' command was a no-op" + const done = changeDifference < 2; + + return { steps: nextSteps, done }; +} + async function getJobsInQueue(queueType: string): Promise> { const db = getReadonlyInstance(); return getJobsInQueueSync(db, queueType); diff --git a/ts/sql/main.ts b/ts/sql/main.ts index c5f7dfc06..5662b6f8b 100644 --- a/ts/sql/main.ts +++ b/ts/sql/main.ts @@ -15,6 +15,7 @@ import type DB from './Server'; const MIN_TRACE_DURATION = 40; export type InitializeOptions = Readonly<{ + appVersion: string; configDir: string; key: string; logger: LoggerType; @@ -127,6 +128,7 @@ export class MainSQL { } public async initialize({ + appVersion, configDir, key, logger, @@ -141,7 +143,7 @@ export class MainSQL { this.onReady = this.send({ type: 'init', - options: { configDir, key }, + options: { appVersion, configDir, key }, }); await this.onReady; diff --git a/ts/sql/migrations/940-fts5-revert.ts b/ts/sql/migrations/940-fts5-revert.ts index 45792bfb4..ce1e7d0d8 100644 --- a/ts/sql/migrations/940-fts5-revert.ts +++ b/ts/sql/migrations/940-fts5-revert.ts @@ -17,37 +17,7 @@ export function updateToSchemaVersion940( } db.transaction(() => { - const wasEnabled = - db - .prepare( - ` - SELECT v FROM messages_fts_config WHERE k is 'secure-delete'; - ` - ) - .pluck() - .get() === 1; - - if (wasEnabled) { - logger.info('updateToSchemaVersion940: rebuilding fts5 index'); - db.exec(` - --- Disable 'secure-delete' - INSERT INTO messages_fts - (messages_fts, rank) - VALUES - ('secure-delete', 0); - - --- Rebuild the index to fix the corruption - INSERT INTO messages_fts - (messages_fts) - VALUES - ('rebuild'); - `); - } else { - logger.info( - 'updateToSchemaVersion940: secure delete was not enabled, skipping' - ); - } - + // This was a migration that disabled secure-delete and rebuilt the index db.pragma('user_version = 940'); })(); diff --git a/ts/sql/migrations/950-fts5-secure-delete.ts b/ts/sql/migrations/950-fts5-secure-delete.ts index 85bddba01..7f3569e80 100644 --- a/ts/sql/migrations/950-fts5-secure-delete.ts +++ b/ts/sql/migrations/950-fts5-secure-delete.ts @@ -17,14 +17,7 @@ export function updateToSchemaVersion950( } db.transaction(() => { - db.exec(` - --- Enable 'secure-delete' - INSERT INTO messages_fts - (messages_fts, rank) - VALUES - ('secure-delete', 1); - `); - + // This was a migration that enable secure-delete db.pragma('user_version = 950'); })(); diff --git a/ts/sql/migrations/index.ts b/ts/sql/migrations/index.ts index f2d0b1284..923e4ce34 100644 --- a/ts/sql/migrations/index.ts +++ b/ts/sql/migrations/index.ts @@ -6,6 +6,7 @@ import { keyBy } from 'lodash'; import { v4 as generateUuid } from 'uuid'; import type { LoggerType } from '../../types/Logging'; +import { isProduction } from '../../util/version'; import { getSchemaVersion, getUserVersion, @@ -2019,7 +2020,53 @@ export class DBVersionFromFutureError extends Error { override name = 'DBVersionFromFutureError'; } -export function updateSchema(db: Database, logger: LoggerType): void { +export function lazyFTS5SecureDelete( + db: Database, + logger: LoggerType, + enabled: boolean +): void { + const isEnabled = + db + .prepare( + ` + SELECT v FROM messages_fts_config WHERE k is 'secure-delete'; + ` + ) + .pluck() + .get() === 1; + + if (isEnabled && !enabled) { + logger.info('lazyFTS5SecureDelete: disabling, rebuilding fts5 index'); + db.exec(` + -- Disable secure-delete + INSERT INTO messages_fts + (messages_fts, rank) + VALUES + ('secure-delete', 0); + + --- Rebuild the index to fix the corruption + INSERT INTO messages_fts + (messages_fts) + VALUES + ('rebuild'); + `); + } else if (!isEnabled && enabled) { + logger.info('lazyFTS5SecureDelete: enabling'); + db.exec(` + -- Enable secure-delete + INSERT INTO messages_fts + (messages_fts, rank) + VALUES + ('secure-delete', 1); + `); + } +} + +export function updateSchema( + db: Database, + logger: LoggerType, + appVersion: string +): void { const sqliteVersion = getSQLiteVersion(db); const sqlcipherVersion = getSQLCipherVersion(db); const startingVersion = getUserVersion(db); @@ -2047,6 +2094,8 @@ export function updateSchema(db: Database, logger: LoggerType): void { runSchemaUpdate(startingVersion, db, logger); } + lazyFTS5SecureDelete(db, logger, !isProduction(appVersion)); + if (startingVersion !== MAX_VERSION) { const start = Date.now(); db.pragma('optimize');