Improve backup export benchmark measurement

This commit is contained in:
Fedor Indutny 2024-11-13 10:47:58 -08:00 committed by GitHub
parent 2e886235fa
commit 45e9c07125
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 67 additions and 41 deletions

View file

@ -3,9 +3,11 @@
import { isFunction, isNumber } from 'lodash';
import pMap from 'p-map';
import PQueue from 'p-queue';
import { CURRENT_SCHEMA_VERSION } from '../types/Message2';
import { isNotNil } from '../util/isNotNil';
import { MINUTE } from '../util/durations';
import type { MessageAttributesType } from '../model-types.d';
import type { AciString } from '../types/ServiceId';
import * as Errors from '../types/errors';
@ -13,10 +15,30 @@ import { DataReader, DataWriter } from '../sql/Client';
const MAX_CONCURRENCY = 5;
// Don't migrate batches concurrently
const migrationQueue = new PQueue({
concurrency: 1,
timeout: MINUTE * 30,
});
type BatchResultType = Readonly<{
done: boolean;
numProcessed: number;
numSucceeded?: number;
numFailedSave?: number;
numFailedUpgrade?: number;
fetchDuration?: number;
upgradeDuration?: number;
saveDuration?: number;
totalDuration?: number;
}>;
/**
* Ensures that messages in database are at the right schema.
*
* @internal
*/
export async function migrateMessageData({
export async function _migrateMessageData({
numMessagesPerBatch,
upgradeMessageSchema,
getMessagesNeedingUpgrade,
@ -41,17 +63,7 @@ export async function migrateMessageData({
messageIds: ReadonlyArray<string>
) => Promise<void>;
maxVersion?: number;
}>): Promise<{
done: boolean;
numProcessed: number;
numSucceeded?: number;
numFailedSave?: number;
numFailedUpgrade?: number;
fetchDuration?: number;
upgradeDuration?: number;
saveDuration?: number;
totalDuration?: number;
}> {
}>): Promise<BatchResultType> {
if (!isNumber(numMessagesPerBatch)) {
throw new TypeError("'numMessagesPerBatch' is required");
}
@ -148,13 +160,33 @@ export async function migrateBatchOfMessages({
numMessagesPerBatch,
}: {
numMessagesPerBatch: number;
}): ReturnType<typeof migrateMessageData> {
return migrateMessageData({
numMessagesPerBatch,
upgradeMessageSchema: window.Signal.Migrations.upgradeMessageSchema,
getMessagesNeedingUpgrade: DataReader.getMessagesNeedingUpgrade,
saveMessagesIndividually: DataWriter.saveMessagesIndividually,
incrementMessagesMigrationAttempts:
DataWriter.incrementMessagesMigrationAttempts,
});
}): ReturnType<typeof _migrateMessageData> {
return migrationQueue.add(() =>
_migrateMessageData({
numMessagesPerBatch,
upgradeMessageSchema: window.Signal.Migrations.upgradeMessageSchema,
getMessagesNeedingUpgrade: DataReader.getMessagesNeedingUpgrade,
saveMessagesIndividually: DataWriter.saveMessagesIndividually,
incrementMessagesMigrationAttempts:
DataWriter.incrementMessagesMigrationAttempts,
})
);
}
export async function migrateAllMessages(): Promise<void> {
const { log } = window.SignalContext;
let batch: BatchResultType | undefined;
let total = 0;
while (!batch?.done) {
// eslint-disable-next-line no-await-in-loop
batch = await migrateBatchOfMessages({
numMessagesPerBatch: 1000,
});
total += batch.numProcessed;
log.info(`migrateAllMessages: Migrated batch of ${batch.numProcessed}`);
}
log.info(
`migrateAllMessages: message migration complete; ${total} messages migrated`
);
}