Improve bulk message deletion speed

This commit is contained in:
trevor-signal 2023-09-07 16:07:07 -04:00 committed by GitHub
parent c8c10d2d76
commit 7ca8f4c763
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 32 additions and 16 deletions

View file

@ -1,6 +1,7 @@
// Copyright 2016 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import { batch } from 'react-redux';
import { debounce } from 'lodash';
import type { MessageModel } from '../models/messages';
@ -31,7 +32,6 @@ class ExpiringMessagesDeletionService {
const messageIds: Array<string> = [];
const inMemoryMessages: Array<MessageModel> = [];
const messageCleanup: Array<Promise<void>> = [];
messages.forEach(dbMessage => {
const message = window.MessageController.register(
@ -40,22 +40,20 @@ class ExpiringMessagesDeletionService {
);
messageIds.push(message.id);
inMemoryMessages.push(message);
messageCleanup.push(message.cleanup());
});
// We delete after the trigger to allow the conversation time to process
// the expiration before the message is removed from the database.
await window.Signal.Data.removeMessages(messageIds);
await Promise.all(messageCleanup);
inMemoryMessages.forEach(message => {
window.SignalContext.log.info('Message expired', {
sentAt: message.get('sent_at'),
batch(() => {
inMemoryMessages.forEach(message => {
window.SignalContext.log.info('Message expired', {
sentAt: message.get('sent_at'),
});
// We do this to update the UI, if this message is being displayed somewhere
message.trigger('expired');
window.reduxActions.conversations.messageExpired(message.id);
});
// We do this to update the UI, if this message is being displayed somewhere
message.trigger('expired');
window.reduxActions.conversations.messageExpired(message.id);
});
if (messages.length > 0) {

View file

@ -3,6 +3,7 @@
import { ipcRenderer as ipc } from 'electron';
import PQueue from 'p-queue';
import { batch } from 'react-redux';
import { has, get, groupBy, isTypedArray, last, map, omit } from 'lodash';
@ -23,7 +24,11 @@ import * as Errors from '../types/errors';
import type { StoredJob } from '../jobs/types';
import { formatJobForInsert } from '../jobs/formatJobForInsert';
import { cleanupMessage } from '../util/cleanup';
import {
cleanupMessage,
cleanupMessageFromMemory,
deleteMessageData,
} from '../util/cleanup';
import { drop } from '../util/drop';
import { ipcInvoke, doShutdown } from './channels';
@ -590,11 +595,18 @@ async function removeMessage(id: string): Promise<void> {
async function _cleanupMessages(
messages: ReadonlyArray<MessageAttributesType>
): Promise<void> {
// First, remove messages from memory, so we can batch the updates in redux
batch(() => {
messages.forEach(message => cleanupMessageFromMemory(message));
});
// Then, handle any asynchronous actions (e.g. deleting data from disk)
const queue = new PQueue({ concurrency: 3, timeout: MINUTE * 30 });
drop(
queue.addAll(
messages.map(
(message: MessageAttributesType) => async () => cleanupMessage(message)
(message: MessageAttributesType) => async () =>
deleteMessageData(message)
)
)
);

View file

@ -10,6 +10,14 @@ import * as log from '../logging/log';
export async function cleanupMessage(
message: MessageAttributesType
): Promise<void> {
cleanupMessageFromMemory(message);
await deleteMessageData(message);
}
/** Removes a message from redux caches & backbone, but does NOT delete files on disk,
* story replies, edit histories, attachments, etc. Should ONLY be called in conjunction
* with deleteMessageData. */
export function cleanupMessageFromMemory(message: MessageAttributesType): void {
const { id, conversationId } = message;
window.reduxActions?.conversations.messageDeleted(id, conversationId);
@ -18,8 +26,6 @@ export async function cleanupMessage(
parentConversation?.debouncedUpdateLastMessage();
window.MessageController.unregister(id);
await deleteMessageData(message);
}
async function cleanupStoryReplies(