mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Update better-sqlite3 and remove FTS optimization

Browse source
This commit is contained in:
Fedor Indutny 2023-09-01 00:33:31 +02:00 committed by GitHub
parent 77b799c7b0
commit f3eee779a0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 39 additions and 141 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -92,7 +92,7 @@
"@popperjs/core": "2.11.6",
"@react-aria/utils": "3.16.0",
"@react-spring/web": "9.5.5",
"@signalapp/better-sqlite3": "8.4.3",
"@signalapp/better-sqlite3": "8.5.1",
"@signalapp/libsignal-client": "0.31.0",
"@signalapp/ringrtc": "2.31.2",
"@signalapp/windows-dummy-keystroke": "1.0.0",

3
ts/background.ts
View file

@ -53,7 +53,6 @@ import { senderCertificateService } from './services/senderCertificate';
import { GROUP_CREDENTIALS_KEY } from './services/groupCredentialFetcher';
import * as KeyboardLayout from './services/keyboardLayout';
import * as StorageService from './services/storage';
import { optimizeFTS } from './services/ftsOptimizer';
import { RoutineProfileRefresher } from './routineProfileRefresh';
import { isOlderThan, toDayMillis } from './util/timestamp';
import { isValidReactionEmoji } from './reactions/isValidReactionEmoji';
@ -983,8 +982,6 @@ export async function startApp(): Promise<void> {
if (newVersion) {
await window.Signal.Data.cleanupOrphanedAttachments();
optimizeFTS();
drop(window.Signal.Data.ensureFilePermissions());
}

3
ts/models/conversations.ts
View file

@ -87,7 +87,6 @@ import {
notificationService,
} from '../services/notifications';
import { storageServiceUploadJob } from '../services/storage';
import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
import { getSendOptions } from '../util/getSendOptions';
import { isConversationAccepted } from '../util/isConversationAccepted';
import {
@ -4760,8 +4759,6 @@ export class ConversationModel extends window.Backbone
await window.Signal.Data.removeAllMessagesInConversation(this.id, {
logId: this.idForLogging(),
});
scheduleOptimizeFTS();
}
getTitle(options?: { isShort?: boolean }): string {

3
ts/models/messages.ts
View file

@ -69,7 +69,6 @@ import { migrateLegacyReadStatus } from '../messages/migrateLegacyReadStatus';
import { migrateLegacySendAttributes } from '../messages/migrateLegacySendAttributes';
import { getOwn } from '../util/getOwn';
import { markRead, markViewed } from '../services/MessageUpdater';
import { scheduleOptimizeFTS } from '../services/ftsOptimizer';
import {
isDirectConversation,
isGroup,
@ -1157,8 +1156,6 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
}
await window.Signal.Data.deleteSentProtoByMessageId(this.id);
scheduleOptimizeFTS();
}
override isEmpty(): boolean {

5
ts/services/expiringMessagesDeletion.ts
View file

@ -8,7 +8,6 @@ import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary';
import { sleep } from '../util/sleep';
import { SECOND } from '../util/durations';
import * as Errors from '../types/errors';
import { scheduleOptimizeFTS } from './ftsOptimizer';
class ExpiringMessagesDeletionService {
public update: typeof this.checkExpiringMessages;
@ -57,10 +56,6 @@ class ExpiringMessagesDeletionService {
message.trigger('expired');
window.reduxActions.conversations.messageExpired(message.id);
});
if (messages.length > 0) {
scheduleOptimizeFTS();
}
} catch (error) {
window.SignalContext.log.error(
'destroyExpiredMessages: Error deleting expired messages',

63
ts/services/ftsOptimizer.ts
View file

@ -1,63 +0,0 @@
// Copyright 2023 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { debounce } from 'lodash';
import { SECOND } from '../util/durations';
import { sleep } from '../util/sleep';
import { drop } from '../util/drop';
import dataInterface from '../sql/Client';
import type { FTSOptimizationStateType } from '../sql/Interface';
import * as log from '../logging/log';
const INTERACTIVITY_DELAY_MS = 50;
class FTSOptimizer {
private isRunning = false;
public async run(): Promise<void> {
if (this.isRunning) {
return;
}
this.isRunning = true;
log.info('ftsOptimizer: starting');
let state: FTSOptimizationStateType | undefined;
const start = Date.now();
try {
do {
if (state !== undefined) {
// eslint-disable-next-line no-await-in-loop
await sleep(INTERACTIVITY_DELAY_MS);
}
// eslint-disable-next-line no-await-in-loop
state = await dataInterface.optimizeFTS(state);
} while (!state?.done);
} finally {
this.isRunning = false;
}
const duration = Date.now() - start;
if (!state) {
log.warn('ftsOptimizer: no final state');
return;
}
log.info(`ftsOptimizer: took ${duration}ms and ${state.steps} steps`);
}
}
const optimizer = new FTSOptimizer();
export const optimizeFTS = (): void => {
drop(optimizer.run());
};
export const scheduleOptimizeFTS = debounce(optimizeFTS, SECOND, {
maxWait: 5 * SECOND,
});

9
ts/sql/Interface.ts
View file

@ -411,11 +411,6 @@ export type GetAllStoriesResultType = ReadonlyArray<
}
>;
export type FTSOptimizationStateType = Readonly<{
steps: number;
done?: boolean;
}>;
export type EditedMessageType = Readonly<{
conversationId: string;
messageId: string;
@ -823,10 +818,6 @@ export type DataInterface = {
getMaxMessageCounter(): Promise<number | undefined>;
getStatisticsForLogging(): Promise<Record<string, string>>;
optimizeFTS: (
state?: FTSOptimizationStateType
) => Promise<FTSOptimizationStateType | undefined>;
};
export type ServerInterface = DataInterface & {

48
ts/sql/Server.ts
View file

@ -93,7 +93,6 @@ import type {
DeleteSentProtoRecipientResultType,
EditedMessageType,
EmojiType,
FTSOptimizationStateType,
GetAllStoriesResultType,
GetConversationRangeCenteredOnMessageResultType,
GetKnownMessageAttachmentsResultType,
@ -403,8 +402,6 @@ const dataInterface: ServerInterface = {
getStatisticsForLogging,
optimizeFTS,
// Server-only
initialize,
@ -2222,7 +2219,6 @@ async function _removeAllMessages(): Promise<void> {
const db = getInstance();
db.exec(`
DELETE FROM messages;
INSERT INTO messages_fts(messages_fts) VALUES('optimize');
`);
}
@ -5548,8 +5544,6 @@ async function removeAll(): Promise<void> {
DELETE FROM unprocessed;
DELETE FROM uninstalled_sticker_packs;
INSERT INTO messages_fts(messages_fts) VALUES('optimize');
--- Re-create the messages delete trigger
--- See migration 45
CREATE TRIGGER messages_on_delete AFTER DELETE ON messages BEGIN
@ -6133,48 +6127,6 @@ async function removeKnownDraftAttachments(
return Object.keys(lookup);
}
const OPTIMIZE_FTS_PAGE_COUNT = 64;
// This query is incremental. It gets the `state` from the return value of
// previous `optimizeFTS` call. When `state.done` is `true` - optimization is
// complete.
async function optimizeFTS(
state?: FTSOptimizationStateType
): Promise<FTSOptimizationStateType | undefined> {
// See https://www.sqlite.org/fts5.html#the_merge_command
let pageCount = OPTIMIZE_FTS_PAGE_COUNT;
if (state === undefined) {
pageCount = -pageCount;
}
const db = getInstance();
const getChanges = prepare(db, 'SELECT total_changes() as changes;', {
pluck: true,
});
const changeDifference = db.transaction(() => {
const before: number = getChanges.get({});
prepare(
db,
`
INSERT INTO messages_fts(messages_fts, rank) VALUES ('merge', $pageCount);
`
).run({ pageCount });
const after: number = getChanges.get({});
return after - before;
})();
const nextSteps = (state?.steps ?? 0) + 1;
// From documentation:
// "If the difference is less than 2, then the 'merge' command was a no-op"
const done = changeDifference < 2;
return { steps: nextSteps, done };
}
async function getJobsInQueue(queueType: string): Promise<Array<StoredJob>> {
const db = getInstance();
return getJobsInQueueSync(db, queueType);

30
ts/sql/migrations/930-fts5-secure-delete.ts Normal file
View file

@ -0,0 +1,30 @@
// Copyright 2023 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import type { Database } from '@signalapp/better-sqlite3';
import type { LoggerType } from '../../types/Logging';
export const version = 930;
export function updateToSchemaVersion930(
currentVersion: number,
db: Database,
logger: LoggerType
): void {
if (currentVersion >= 930) {
return;
}
db.transaction(() => {
db.exec(`
INSERT INTO messages_fts
(messages_fts, rank)
VALUES
('secure-delete', 1);
`);
db.pragma('user_version = 930');
})();
logger.info('updateToSchemaVersion930: success!');
}

6
ts/sql/migrations/index.ts
View file

@ -66,10 +66,11 @@ import updateToSchemaVersion88 from './88-service-ids';
import updateToSchemaVersion89 from './89-call-history';
import updateToSchemaVersion90 from './90-delete-story-reply-screenshot';
import updateToSchemaVersion91 from './91-clean-keys';
import { updateToSchemaVersion920 } from './920-clean-more-keys';
import {
version as MAX_VERSION,
updateToSchemaVersion920,
} from './920-clean-more-keys';
updateToSchemaVersion930,
} from './930-fts5-secure-delete';
function updateToSchemaVersion1(
currentVersion: number,
@ -2010,6 +2011,7 @@ export const SCHEMA_VERSIONS = [
updateToSchemaVersion91,
// From here forward, all migrations should be multiples of 10
updateToSchemaVersion920,
updateToSchemaVersion930,
];
export function updateSchema(db: Database, logger: LoggerType): void {

8
yarn.lock
View file

@ -3355,10 +3355,10 @@
"@react-types/overlays" "^3.7.1"
"@react-types/shared" "^3.18.0"
"@signalapp/better-sqlite3@8.4.3":
version "8.4.3"
resolved "https://registry.yarnpkg.com/@signalapp/better-sqlite3/-/better-sqlite3-8.4.3.tgz#7ffa8d03d2a12543247936bfb7b9f74cdbc6fe9b"
integrity sha512-6HaN8a90fWHBPIIZRRmSG7wc3BPczLx3Mb9MJ8wsatYqf8C1+2NdFae5AzII8Oe9YE9SSHzCjdPd8ST1m9+Qag==
"@signalapp/better-sqlite3@8.5.1":
version "8.5.1"
resolved "https://registry.yarnpkg.com/@signalapp/better-sqlite3/-/better-sqlite3-8.5.1.tgz#b2bcf69d29f4d01cfecf2ad940ecd138f32470b3"
integrity sha512-s9NDufkbTAuKShFmEg7eOew4cprdLI7mn5ISQFKgcFSiA/TG3wgLJW5wUff9S2vednIV9utyqRJZXrwcSAEQUg==
dependencies:
bindings "^1.5.0"
tar "^6.1.0"