Improve backup export benchmark measurement
This commit is contained in:
parent
2e886235fa
commit
45e9c07125
7 changed files with 67 additions and 41 deletions
|
@ -3,9 +3,11 @@
|
|||
|
||||
import { isFunction, isNumber } from 'lodash';
|
||||
import pMap from 'p-map';
|
||||
import PQueue from 'p-queue';
|
||||
|
||||
import { CURRENT_SCHEMA_VERSION } from '../types/Message2';
|
||||
import { isNotNil } from '../util/isNotNil';
|
||||
import { MINUTE } from '../util/durations';
|
||||
import type { MessageAttributesType } from '../model-types.d';
|
||||
import type { AciString } from '../types/ServiceId';
|
||||
import * as Errors from '../types/errors';
|
||||
|
@ -13,10 +15,30 @@ import { DataReader, DataWriter } from '../sql/Client';
|
|||
|
||||
const MAX_CONCURRENCY = 5;
|
||||
|
||||
// Don't migrate batches concurrently
|
||||
const migrationQueue = new PQueue({
|
||||
concurrency: 1,
|
||||
timeout: MINUTE * 30,
|
||||
});
|
||||
|
||||
type BatchResultType = Readonly<{
|
||||
done: boolean;
|
||||
numProcessed: number;
|
||||
numSucceeded?: number;
|
||||
numFailedSave?: number;
|
||||
numFailedUpgrade?: number;
|
||||
fetchDuration?: number;
|
||||
upgradeDuration?: number;
|
||||
saveDuration?: number;
|
||||
totalDuration?: number;
|
||||
}>;
|
||||
|
||||
/**
|
||||
* Ensures that messages in database are at the right schema.
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
export async function migrateMessageData({
|
||||
export async function _migrateMessageData({
|
||||
numMessagesPerBatch,
|
||||
upgradeMessageSchema,
|
||||
getMessagesNeedingUpgrade,
|
||||
|
@ -41,17 +63,7 @@ export async function migrateMessageData({
|
|||
messageIds: ReadonlyArray<string>
|
||||
) => Promise<void>;
|
||||
maxVersion?: number;
|
||||
}>): Promise<{
|
||||
done: boolean;
|
||||
numProcessed: number;
|
||||
numSucceeded?: number;
|
||||
numFailedSave?: number;
|
||||
numFailedUpgrade?: number;
|
||||
fetchDuration?: number;
|
||||
upgradeDuration?: number;
|
||||
saveDuration?: number;
|
||||
totalDuration?: number;
|
||||
}> {
|
||||
}>): Promise<BatchResultType> {
|
||||
if (!isNumber(numMessagesPerBatch)) {
|
||||
throw new TypeError("'numMessagesPerBatch' is required");
|
||||
}
|
||||
|
@ -148,13 +160,33 @@ export async function migrateBatchOfMessages({
|
|||
numMessagesPerBatch,
|
||||
}: {
|
||||
numMessagesPerBatch: number;
|
||||
}): ReturnType<typeof migrateMessageData> {
|
||||
return migrateMessageData({
|
||||
numMessagesPerBatch,
|
||||
upgradeMessageSchema: window.Signal.Migrations.upgradeMessageSchema,
|
||||
getMessagesNeedingUpgrade: DataReader.getMessagesNeedingUpgrade,
|
||||
saveMessagesIndividually: DataWriter.saveMessagesIndividually,
|
||||
incrementMessagesMigrationAttempts:
|
||||
DataWriter.incrementMessagesMigrationAttempts,
|
||||
});
|
||||
}): ReturnType<typeof _migrateMessageData> {
|
||||
return migrationQueue.add(() =>
|
||||
_migrateMessageData({
|
||||
numMessagesPerBatch,
|
||||
upgradeMessageSchema: window.Signal.Migrations.upgradeMessageSchema,
|
||||
getMessagesNeedingUpgrade: DataReader.getMessagesNeedingUpgrade,
|
||||
saveMessagesIndividually: DataWriter.saveMessagesIndividually,
|
||||
incrementMessagesMigrationAttempts:
|
||||
DataWriter.incrementMessagesMigrationAttempts,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
export async function migrateAllMessages(): Promise<void> {
|
||||
const { log } = window.SignalContext;
|
||||
|
||||
let batch: BatchResultType | undefined;
|
||||
let total = 0;
|
||||
while (!batch?.done) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
batch = await migrateBatchOfMessages({
|
||||
numMessagesPerBatch: 1000,
|
||||
});
|
||||
total += batch.numProcessed;
|
||||
log.info(`migrateAllMessages: Migrated batch of ${batch.numProcessed}`);
|
||||
}
|
||||
log.info(
|
||||
`migrateAllMessages: message migration complete; ${total} messages migrated`
|
||||
);
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue