Use FTS5 optimizer in production
This commit is contained in:
parent
f5c18cfb51
commit
e124730cb0
12 changed files with 200 additions and 42 deletions
|
@ -1590,6 +1590,7 @@ async function initializeSQL(
|
|||
// `sql.sqlCall` will throw an uninitialized error instead of waiting for
|
||||
// init to finish.
|
||||
await sql.initialize({
|
||||
appVersion: app.getVersion(),
|
||||
configDir: userDataPath,
|
||||
key,
|
||||
logger: getLogger(),
|
||||
|
|
|
@ -52,6 +52,7 @@ import { senderCertificateService } from './services/senderCertificate';
|
|||
import { GROUP_CREDENTIALS_KEY } from './services/groupCredentialFetcher';
|
||||
import * as KeyboardLayout from './services/keyboardLayout';
|
||||
import * as StorageService from './services/storage';
|
||||
import { optimizeFTS } from './services/ftsOptimizer';
|
||||
import { RoutineProfileRefresher } from './routineProfileRefresh';
|
||||
import { isOlderThan, toDayMillis } from './util/timestamp';
|
||||
import { isValidReactionEmoji } from './reactions/isValidReactionEmoji';
|
||||
|
@ -980,6 +981,8 @@ export async function startApp(): Promise<void> {
|
|||
if (newVersion) {
|
||||
await window.Signal.Data.cleanupOrphanedAttachments();
|
||||
|
||||
optimizeFTS();
|
||||
|
||||
drop(window.Signal.Data.ensureFilePermissions());
|
||||
}
|
||||
|
||||
|
|
|
@ -87,6 +87,7 @@ import {
|
|||
notificationService,
|
||||
} from '../services/notifications';
|
||||
import { storageServiceUploadJob } from '../services/storage';
|
||||
import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
|
||||
import { getSendOptions } from '../util/getSendOptions';
|
||||
import { isConversationAccepted } from '../util/isConversationAccepted';
|
||||
import {
|
||||
|
@ -4806,6 +4807,8 @@ export class ConversationModel extends window.Backbone
|
|||
await window.Signal.Data.removeAllMessagesInConversation(this.id, {
|
||||
logId: this.idForLogging(),
|
||||
});
|
||||
|
||||
scheduleOptimizeFTS();
|
||||
}
|
||||
|
||||
getTitle(options?: { isShort?: boolean }): string {
|
||||
|
|
|
@ -65,6 +65,7 @@ import { migrateLegacyReadStatus } from '../messages/migrateLegacyReadStatus';
|
|||
import { migrateLegacySendAttributes } from '../messages/migrateLegacySendAttributes';
|
||||
import { getOwn } from '../util/getOwn';
|
||||
import { markRead, markViewed } from '../services/MessageUpdater';
|
||||
import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
|
||||
import {
|
||||
isDirectConversation,
|
||||
isGroup,
|
||||
|
@ -605,6 +606,8 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
|
|||
}
|
||||
|
||||
await window.Signal.Data.deleteSentProtoByMessageId(this.id);
|
||||
|
||||
scheduleOptimizeFTS();
|
||||
}
|
||||
|
||||
override isEmpty(): boolean {
|
||||
|
|
|
@ -9,6 +9,7 @@ import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary';
|
|||
import { sleep } from '../util/sleep';
|
||||
import { SECOND } from '../util/durations';
|
||||
import * as Errors from '../types/errors';
|
||||
import { scheduleOptimizeFTS } from './ftsOptimizer';
|
||||
|
||||
class ExpiringMessagesDeletionService {
|
||||
public update: typeof this.checkExpiringMessages;
|
||||
|
@ -55,6 +56,10 @@ class ExpiringMessagesDeletionService {
|
|||
window.reduxActions.conversations.messageExpired(message.id);
|
||||
});
|
||||
});
|
||||
|
||||
if (messages.length > 0) {
|
||||
scheduleOptimizeFTS();
|
||||
}
|
||||
} catch (error) {
|
||||
window.SignalContext.log.error(
|
||||
'destroyExpiredMessages: Error deleting expired messages',
|
||||
|
|
69
ts/services/ftsOptimizer.ts
Normal file
69
ts/services/ftsOptimizer.ts
Normal file
|
@ -0,0 +1,69 @@
|
|||
// Copyright 2023 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
import { debounce } from 'lodash';
|
||||
|
||||
import { SECOND } from '../util/durations';
|
||||
import { sleep } from '../util/sleep';
|
||||
import { drop } from '../util/drop';
|
||||
import { isProduction } from '../util/version';
|
||||
import dataInterface from '../sql/Client';
|
||||
import type { FTSOptimizationStateType } from '../sql/Interface';
|
||||
import * as log from '../logging/log';
|
||||
|
||||
const INTERACTIVITY_DELAY_MS = 50;
|
||||
|
||||
class FTSOptimizer {
|
||||
private isRunning = false;
|
||||
|
||||
public async run(): Promise<void> {
|
||||
if (!isProduction(window.getVersion())) {
|
||||
log.info('ftsOptimizer: not running when not in production');
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.isRunning) {
|
||||
return;
|
||||
}
|
||||
this.isRunning = true;
|
||||
|
||||
log.info('ftsOptimizer: starting');
|
||||
|
||||
let state: FTSOptimizationStateType | undefined;
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
try {
|
||||
do {
|
||||
if (state !== undefined) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await sleep(INTERACTIVITY_DELAY_MS);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
state = await dataInterface.optimizeFTS(state);
|
||||
} while (!state?.done);
|
||||
} finally {
|
||||
this.isRunning = false;
|
||||
}
|
||||
|
||||
const duration = Date.now() - start;
|
||||
|
||||
if (!state) {
|
||||
log.warn('ftsOptimizer: no final state');
|
||||
return;
|
||||
}
|
||||
|
||||
log.info(`ftsOptimizer: took ${duration}ms and ${state.steps} steps`);
|
||||
}
|
||||
}
|
||||
|
||||
const optimizer = new FTSOptimizer();
|
||||
|
||||
export const optimizeFTS = (): void => {
|
||||
drop(optimizer.run());
|
||||
};
|
||||
|
||||
export const scheduleOptimizeFTS = debounce(optimizeFTS, SECOND, {
|
||||
maxWait: 5 * SECOND,
|
||||
});
|
|
@ -411,6 +411,11 @@ export type GetAllStoriesResultType = ReadonlyArray<
|
|||
}
|
||||
>;
|
||||
|
||||
export type FTSOptimizationStateType = Readonly<{
|
||||
steps: number;
|
||||
done?: boolean;
|
||||
}>;
|
||||
|
||||
export type EditedMessageType = Readonly<{
|
||||
conversationId: string;
|
||||
messageId: string;
|
||||
|
@ -819,6 +824,10 @@ export type DataInterface = {
|
|||
getMaxMessageCounter(): Promise<number | undefined>;
|
||||
|
||||
getStatisticsForLogging(): Promise<Record<string, string>>;
|
||||
|
||||
optimizeFTS: (
|
||||
state?: FTSOptimizationStateType
|
||||
) => Promise<FTSOptimizationStateType | undefined>;
|
||||
};
|
||||
|
||||
export type ServerInterface = DataInterface & {
|
||||
|
@ -892,6 +901,7 @@ export type ServerInterface = DataInterface & {
|
|||
// Server-only
|
||||
|
||||
initialize: (options: {
|
||||
appVersion: string;
|
||||
configDir: string;
|
||||
key: string;
|
||||
logger: LoggerType;
|
||||
|
|
|
@ -93,6 +93,7 @@ import type {
|
|||
DeleteSentProtoRecipientResultType,
|
||||
EditedMessageType,
|
||||
EmojiType,
|
||||
FTSOptimizationStateType,
|
||||
GetAllStoriesResultType,
|
||||
GetConversationRangeCenteredOnMessageResultType,
|
||||
GetKnownMessageAttachmentsResultType,
|
||||
|
@ -403,6 +404,8 @@ const dataInterface: ServerInterface = {
|
|||
|
||||
getStatisticsForLogging,
|
||||
|
||||
optimizeFTS,
|
||||
|
||||
// Server-only
|
||||
|
||||
initialize,
|
||||
|
@ -574,10 +577,12 @@ SQL.setLogHandler((code, value) => {
|
|||
});
|
||||
|
||||
async function initialize({
|
||||
appVersion,
|
||||
configDir,
|
||||
key,
|
||||
logger: suppliedLogger,
|
||||
}: {
|
||||
appVersion: string;
|
||||
configDir: string;
|
||||
key: string;
|
||||
logger: LoggerType;
|
||||
|
@ -614,7 +619,7 @@ async function initialize({
|
|||
// For profiling use:
|
||||
// db.pragma('cipher_profile=\'sqlcipher.log\'');
|
||||
|
||||
updateSchema(writable, logger);
|
||||
updateSchema(writable, logger, appVersion);
|
||||
|
||||
readonly = openAndSetUpSQLCipher(databaseFilePath, { key, readonly: true });
|
||||
|
||||
|
@ -2274,6 +2279,7 @@ async function _removeAllMessages(): Promise<void> {
|
|||
const db = await getWritableInstance();
|
||||
db.exec(`
|
||||
DELETE FROM messages;
|
||||
INSERT INTO messages_fts(messages_fts) VALUES('optimize');
|
||||
`);
|
||||
}
|
||||
|
||||
|
@ -5656,6 +5662,8 @@ async function removeAll(): Promise<void> {
|
|||
DELETE FROM unprocessed;
|
||||
DELETE FROM uninstalled_sticker_packs;
|
||||
|
||||
INSERT INTO messages_fts(messages_fts) VALUES('optimize');
|
||||
|
||||
--- Re-create the messages delete trigger
|
||||
--- See migration 45
|
||||
CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN
|
||||
|
@ -6239,6 +6247,48 @@ async function removeKnownDraftAttachments(
|
|||
return Object.keys(lookup);
|
||||
}
|
||||
|
||||
const OPTIMIZE_FTS_PAGE_COUNT = 64;
|
||||
|
||||
// This query is incremental. It gets the `state` from the return value of
|
||||
// previous `optimizeFTS` call. When `state.done` is `true` - optimization is
|
||||
// complete.
|
||||
async function optimizeFTS(
|
||||
state?: FTSOptimizationStateType
|
||||
): Promise<FTSOptimizationStateType | undefined> {
|
||||
// See https://www.sqlite.org/fts5.html#the_merge_command
|
||||
let pageCount = OPTIMIZE_FTS_PAGE_COUNT;
|
||||
if (state === undefined) {
|
||||
pageCount = -pageCount;
|
||||
}
|
||||
const db = await getWritableInstance();
|
||||
const getChanges = prepare(db, 'SELECT total_changes() as changes;', {
|
||||
pluck: true,
|
||||
});
|
||||
|
||||
const changeDifference = db.transaction(() => {
|
||||
const before: number = getChanges.get({});
|
||||
|
||||
prepare(
|
||||
db,
|
||||
`
|
||||
INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount);
|
||||
`
|
||||
).run({ pageCount });
|
||||
|
||||
const after: number = getChanges.get({});
|
||||
|
||||
return after - before;
|
||||
})();
|
||||
|
||||
const nextSteps = (state?.steps ?? 0) + 1;
|
||||
|
||||
// From documentation:
|
||||
// "If the difference is less than 2, then the 'merge' command was a no-op"
|
||||
const done = changeDifference < 2;
|
||||
|
||||
return { steps: nextSteps, done };
|
||||
}
|
||||
|
||||
async function getJobsInQueue(queueType: string): Promise<Array<StoredJob>> {
|
||||
const db = getReadonlyInstance();
|
||||
return getJobsInQueueSync(db, queueType);
|
||||
|
|
|
@ -15,6 +15,7 @@ import type DB from './Server';
|
|||
const MIN_TRACE_DURATION = 40;
|
||||
|
||||
export type InitializeOptions = Readonly<{
|
||||
appVersion: string;
|
||||
configDir: string;
|
||||
key: string;
|
||||
logger: LoggerType;
|
||||
|
@ -127,6 +128,7 @@ export class MainSQL {
|
|||
}
|
||||
|
||||
public async initialize({
|
||||
appVersion,
|
||||
configDir,
|
||||
key,
|
||||
logger,
|
||||
|
@ -141,7 +143,7 @@ export class MainSQL {
|
|||
|
||||
this.onReady = this.send({
|
||||
type: 'init',
|
||||
options: { configDir, key },
|
||||
options: { appVersion, configDir, key },
|
||||
});
|
||||
|
||||
await this.onReady;
|
||||
|
|
|
@ -17,37 +17,7 @@ export function updateToSchemaVersion940(
|
|||
}
|
||||
|
||||
db.transaction(() => {
|
||||
const wasEnabled =
|
||||
db
|
||||
.prepare(
|
||||
`
|
||||
SELECT v FROM messages_fts_config WHERE k is 'secure-delete';
|
||||
`
|
||||
)
|
||||
.pluck()
|
||||
.get() === 1;
|
||||
|
||||
if (wasEnabled) {
|
||||
logger.info('updateToSchemaVersion940: rebuilding fts5 index');
|
||||
db.exec(`
|
||||
--- Disable 'secure-delete'
|
||||
INSERT INTO messages_fts
|
||||
(messages_fts, rank)
|
||||
VALUES
|
||||
('secure-delete', 0);
|
||||
|
||||
--- Rebuild the index to fix the corruption
|
||||
INSERT INTO messages_fts
|
||||
(messages_fts)
|
||||
VALUES
|
||||
('rebuild');
|
||||
`);
|
||||
} else {
|
||||
logger.info(
|
||||
'updateToSchemaVersion940: secure delete was not enabled, skipping'
|
||||
);
|
||||
}
|
||||
|
||||
// This was a migration that disabled secure-delete and rebuilt the index
|
||||
db.pragma('user_version = 940');
|
||||
})();
|
||||
|
||||
|
|
|
@ -17,14 +17,7 @@ export function updateToSchemaVersion950(
|
|||
}
|
||||
|
||||
db.transaction(() => {
|
||||
db.exec(`
|
||||
--- Enable 'secure-delete'
|
||||
INSERT INTO messages_fts
|
||||
(messages_fts, rank)
|
||||
VALUES
|
||||
('secure-delete', 1);
|
||||
`);
|
||||
|
||||
// This was a migration that enable secure-delete
|
||||
db.pragma('user_version = 950');
|
||||
})();
|
||||
|
||||
|
|
|
@ -6,6 +6,7 @@ import { keyBy } from 'lodash';
|
|||
import { v4 as generateUuid } from 'uuid';
|
||||
|
||||
import type { LoggerType } from '../../types/Logging';
|
||||
import { isProduction } from '../../util/version';
|
||||
import {
|
||||
getSchemaVersion,
|
||||
getUserVersion,
|
||||
|
@ -2019,7 +2020,53 @@ export class DBVersionFromFutureError extends Error {
|
|||
override name = 'DBVersionFromFutureError';
|
||||
}
|
||||
|
||||
export function updateSchema(db: Database, logger: LoggerType): void {
|
||||
export function lazyFTS5SecureDelete(
|
||||
db: Database,
|
||||
logger: LoggerType,
|
||||
enabled: boolean
|
||||
): void {
|
||||
const isEnabled =
|
||||
db
|
||||
.prepare(
|
||||
`
|
||||
SELECT v FROM messages_fts_config WHERE k is 'secure-delete';
|
||||
`
|
||||
)
|
||||
.pluck()
|
||||
.get() === 1;
|
||||
|
||||
if (isEnabled && !enabled) {
|
||||
logger.info('lazyFTS5SecureDelete: disabling, rebuilding fts5 index');
|
||||
db.exec(`
|
||||
-- Disable secure-delete
|
||||
INSERT INTO messages_fts
|
||||
(messages_fts, rank)
|
||||
VALUES
|
||||
('secure-delete', 0);
|
||||
|
||||
--- Rebuild the index to fix the corruption
|
||||
INSERT INTO messages_fts
|
||||
(messages_fts)
|
||||
VALUES
|
||||
('rebuild');
|
||||
`);
|
||||
} else if (!isEnabled && enabled) {
|
||||
logger.info('lazyFTS5SecureDelete: enabling');
|
||||
db.exec(`
|
||||
-- Enable secure-delete
|
||||
INSERT INTO messages_fts
|
||||
(messages_fts, rank)
|
||||
VALUES
|
||||
('secure-delete', 1);
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
export function updateSchema(
|
||||
db: Database,
|
||||
logger: LoggerType,
|
||||
appVersion: string
|
||||
): void {
|
||||
const sqliteVersion = getSQLiteVersion(db);
|
||||
const sqlcipherVersion = getSQLCipherVersion(db);
|
||||
const startingVersion = getUserVersion(db);
|
||||
|
@ -2047,6 +2094,8 @@ export function updateSchema(db: Database, logger: LoggerType): void {
|
|||
runSchemaUpdate(startingVersion, db, logger);
|
||||
}
|
||||
|
||||
lazyFTS5SecureDelete(db, logger, !isProduction(appVersion));
|
||||
|
||||
if (startingVersion !== MAX_VERSION) {
|
||||
const start = Date.now();
|
||||
db.pragma('optimize');
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue