diff --git a/ts/background.ts b/ts/background.ts index 1fc429d960..7151209a78 100644 --- a/ts/background.ts +++ b/ts/background.ts @@ -55,6 +55,7 @@ import { senderCertificateService } from './services/senderCertificate'; import { GROUP_CREDENTIALS_KEY } from './services/groupCredentialFetcher'; import * as KeyboardLayout from './services/keyboardLayout'; import * as StorageService from './services/storage'; +import { optimizeFTS } from './services/ftsOptimizer'; import { RoutineProfileRefresher } from './routineProfileRefresh'; import { isOlderThan, toDayMillis } from './util/timestamp'; import { isValidReactionEmoji } from './reactions/isValidReactionEmoji'; @@ -862,6 +863,8 @@ export async function startApp(): Promise { if (newVersion) { await window.Signal.Data.cleanupOrphanedAttachments(); + optimizeFTS(); + // Don't block on the following operation void window.Signal.Data.ensureFilePermissions(); } diff --git a/ts/models/conversations.ts b/ts/models/conversations.ts index 785f27c98d..f28ca5f030 100644 --- a/ts/models/conversations.ts +++ b/ts/models/conversations.ts @@ -90,6 +90,7 @@ import { isNotNil } from '../util/isNotNil'; import { dropNull } from '../util/dropNull'; import { notificationService } from '../services/notifications'; import { storageServiceUploadJob } from '../services/storage'; +import { scheduleOptimizeFTS } from '../services/ftsOptimizer'; import { getSendOptions } from '../util/getSendOptions'; import { isConversationAccepted } from '../util/isConversationAccepted'; import { @@ -5039,6 +5040,8 @@ export class ConversationModel extends window.Backbone await window.Signal.Data.removeAllMessagesInConversation(this.id, { logId: this.idForLogging(), }); + + scheduleOptimizeFTS(); } getTitle(options?: { isShort?: boolean }): string { diff --git a/ts/models/messages.ts b/ts/models/messages.ts index 1a43d6ac95..8970709517 100644 --- a/ts/models/messages.ts +++ b/ts/models/messages.ts @@ -79,6 +79,7 @@ import { migrateLegacyReadStatus } from '../messages/migrateLegacyReadStatus'; import { migrateLegacySendAttributes } from '../messages/migrateLegacySendAttributes'; import { getOwn } from '../util/getOwn'; import { markRead, markViewed } from '../services/MessageUpdater'; +import { scheduleOptimizeFTS } from '../services/ftsOptimizer'; import { isMessageUnread } from '../util/isMessageUnread'; import { isDirectConversation, @@ -1184,6 +1185,8 @@ export class MessageModel extends window.Backbone.Model { } await window.Signal.Data.deleteSentProtoByMessageId(this.id); + + scheduleOptimizeFTS(); } override isEmpty(): boolean { diff --git a/ts/services/expiringMessagesDeletion.ts b/ts/services/expiringMessagesDeletion.ts index 1b078a2839..20d369199d 100644 --- a/ts/services/expiringMessagesDeletion.ts +++ b/ts/services/expiringMessagesDeletion.ts @@ -8,6 +8,7 @@ import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary'; import { sleep } from '../util/sleep'; import { SECOND } from '../util/durations'; import * as Errors from '../types/errors'; +import { scheduleOptimizeFTS } from './ftsOptimizer'; class ExpiringMessagesDeletionService { public update: typeof this.checkExpiringMessages; @@ -64,6 +65,10 @@ class ExpiringMessagesDeletionService { conversation.decrementMessageCount(); } }); + + if (messages.length > 0) { + scheduleOptimizeFTS(); + } } catch (error) { window.SignalContext.log.error( 'destroyExpiredMessages: Error deleting expired messages', diff --git a/ts/services/ftsOptimizer.ts b/ts/services/ftsOptimizer.ts new file mode 100644 index 0000000000..3add634505 --- /dev/null +++ b/ts/services/ftsOptimizer.ts @@ -0,0 +1,63 @@ +// Copyright 2023 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import { debounce } from 'lodash'; + +import { SECOND } from '../util/durations'; +import { sleep } from '../util/sleep'; +import { drop } from '../util/drop'; +import dataInterface from '../sql/Client'; +import type { FTSOptimizationStateType } from '../sql/Interface'; +import * as log from '../logging/log'; + +const INTERACTIVITY_DELAY_MS = 50; + +class FTSOptimizer { + private isRunning = false; + + public async run(): Promise { + if (this.isRunning) { + return; + } + this.isRunning = true; + + log.info('ftsOptimizer: starting'); + + let state: FTSOptimizationStateType | undefined; + + const start = Date.now(); + + try { + do { + if (state !== undefined) { + // eslint-disable-next-line no-await-in-loop + await sleep(INTERACTIVITY_DELAY_MS); + } + + // eslint-disable-next-line no-await-in-loop + state = await dataInterface.optimizeFTS(state); + } while (!state?.done); + } finally { + this.isRunning = false; + } + + const duration = Date.now() - start; + + if (!state) { + log.warn('ftsOptimizer: no final state'); + return; + } + + log.info(`ftsOptimizer: took ${duration}ms and ${state.steps} steps`); + } +} + +const optimizer = new FTSOptimizer(); + +export const optimizeFTS = (): void => { + drop(optimizer.run()); +}; + +export const scheduleOptimizeFTS = debounce(optimizeFTS, SECOND, { + maxWait: 5 * SECOND, +}); diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index f896174d97..2dbcd9140c 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -360,6 +360,12 @@ export type GetAllStoriesResultType = ReadonlyArray< } >; +export type FTSOptimizationStateType = Readonly<{ + changes: number; + steps: number; + done?: boolean; +}>; + export type DataInterface = { close: () => Promise; removeDB: () => Promise; @@ -704,6 +710,10 @@ export type DataInterface = { getMaxMessageCounter(): Promise; getStatisticsForLogging(): Promise>; + + optimizeFTS: ( + state?: FTSOptimizationStateType + ) => Promise; }; export type ServerInterface = DataInterface & { diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 4fb6edbe81..537a30eed8 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -78,6 +78,7 @@ import type { DeleteSentProtoRecipientOptionsType, DeleteSentProtoRecipientResultType, EmojiType, + FTSOptimizationStateType, GetAllStoriesResultType, GetConversationRangeCenteredOnMessageResultType, GetKnownMessageAttachmentsResultType, @@ -341,6 +342,8 @@ const dataInterface: ServerInterface = { getStatisticsForLogging, + optimizeFTS, + // Server-only initialize, @@ -5405,6 +5408,47 @@ async function removeKnownDraftAttachments( return Object.keys(lookup); } +// Default value of 'automerge'. +// See: https://www.sqlite.org/fts5.html#the_automerge_configuration_option +const OPTIMIZE_FTS_PAGE_COUNT = 4; + +// This query is incremental. It gets the `state` from the return value of +// previous `optimizeFTS` call. When `state.done` is `true` - optimization is +// complete. +async function optimizeFTS( + state?: FTSOptimizationStateType +): Promise { + // See https://www.sqlite.org/fts5.html#the_merge_command + let pageCount = OPTIMIZE_FTS_PAGE_COUNT; + if (state === undefined) { + pageCount = -pageCount; + } + + const db = getInstance(); + const { changes } = prepare( + db, + ` + INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount); + ` + ).run({ pageCount }); + + if (state === undefined) { + return { + changes, + steps: 1, + }; + } + + const { changes: prevChanges, steps } = state; + + if (Math.abs(changes - prevChanges) < 2) { + return { changes, steps, done: true }; + } + + // More work is needed. + return { changes, steps: steps + 1 }; +} + async function getJobsInQueue(queueType: string): Promise> { const db = getInstance(); return getJobsInQueueSync(db, queueType);