Optimize initial storage service fetch

This commit is contained in:
Fedor Indutny 2022-03-09 10:22:34 -08:00 committed by GitHub
parent cc51cdccc7
commit a72cf075ef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 172 additions and 36 deletions

View file

@ -50,8 +50,11 @@ import type {
type IManifestRecordIdentifier = Proto.ManifestRecord.IIdentifier;
const { eraseStorageServiceStateFromConversations, updateConversation } =
dataInterface;
const {
eraseStorageServiceStateFromConversations,
updateConversation,
updateConversations,
} = dataInterface;
const uploadBucket: Array<number> = [];
@ -738,6 +741,8 @@ type MergedRecordType = UnknownRecord & {
shouldDrop: boolean;
hasError: boolean;
isUnsupported: boolean;
updatedConversations: ReadonlyArray<ConversationModel>;
needProfileFetch: ReadonlyArray<ConversationModel>;
};
async function mergeRecord(
@ -751,6 +756,8 @@ async function mergeRecord(
let mergeResult: MergeResultType = { hasConflict: false, details: [] };
let isUnsupported = false;
let hasError = false;
let updatedConversations = new Array<ConversationModel>();
const needProfileFetch = new Array<ConversationModel>();
try {
if (itemType === ITEM_TYPE.UNKNOWN) {
@ -797,6 +804,15 @@ async function mergeRecord(
const oldID = mergeResult.oldStorageID
? redactStorageID(mergeResult.oldStorageID, mergeResult.oldStorageVersion)
: '?';
updatedConversations = [
...updatedConversations,
...(mergeResult.updatedConversations ?? []),
];
if (mergeResult.needsProfileFetch) {
strictAssert(mergeResult.conversation, 'needsProfileFetch, but no convo');
needProfileFetch.push(mergeResult.conversation);
}
log.info(
`storageService.merge(${redactedID}): merged item type=${itemType} ` +
`oldID=${oldID} ` +
@ -821,6 +837,8 @@ async function mergeRecord(
isUnsupported,
itemType,
storageID,
updatedConversations,
needProfileFetch,
};
}
@ -1110,7 +1128,7 @@ async function processRemoteRecords(
...(await pMap(
prunedStorageItems,
(item: MergeableItemType) => mergeRecord(storageVersion, item),
{ concurrency: 5 }
{ concurrency: 32 }
)),
// Merge Account records last since it contains the pinned conversations
@ -1124,6 +1142,29 @@ async function processRemoteRecords(
`processed records=${mergedRecords.length}`
);
const updatedConversations = mergedRecords
.map(record => record.updatedConversations)
.flat()
.map(convo => convo.attributes);
await updateConversations(updatedConversations);
log.info(
`storageService.process(${storageVersion}): ` +
`updated conversations=${updatedConversations.length}`
);
const needProfileFetch = mergedRecords
.map(record => record.needProfileFetch)
.flat();
log.info(
`storageService.process(${storageVersion}): ` +
`kicking off profile fetches=${needProfileFetch.length}`
);
// Intentionally not awaiting
pMap(needProfileFetch, convo => convo.getProfiles(), { concurrency: 3 });
// Collect full map of previously and currently unknown records
const unknownRecords: Map<string, UnknownRecord> = new Map();