Removed hard limit on unprocessed messages in cache
This commit is contained in:
parent
1381e8df5d
commit
e51f582bfb
6 changed files with 130 additions and 44 deletions
|
@ -121,6 +121,7 @@ import { TEXT_ATTACHMENT } from '../types/MIME';
|
|||
import type { SendTypesType } from '../util/handleMessageSend';
|
||||
import { getStoriesBlocked } from '../util/stories';
|
||||
import { isNotNil } from '../util/isNotNil';
|
||||
import { chunk } from '../util/iterables';
|
||||
|
||||
const GROUPV1_ID_LENGTH = 16;
|
||||
const GROUPV2_ID_LENGTH = 32;
|
||||
|
@ -444,7 +445,9 @@ export default class MessageReceiver
|
|||
createTaskWithTimeout(
|
||||
async () => this.queueAllCached(),
|
||||
'incomingQueue/queueAllCached',
|
||||
TASK_WITH_TIMEOUT_OPTIONS
|
||||
{
|
||||
timeout: 10 * durations.MINUTE,
|
||||
}
|
||||
)
|
||||
)
|
||||
);
|
||||
|
@ -789,12 +792,14 @@ export default class MessageReceiver
|
|||
return;
|
||||
}
|
||||
|
||||
const items = await this.getAllFromCache();
|
||||
const max = items.length;
|
||||
for (let i = 0; i < max; i += 1) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await this.queueCached(items[i]);
|
||||
for await (const batch of this.getAllFromCache()) {
|
||||
const max = batch.length;
|
||||
for (let i = 0; i < max; i += 1) {
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
await this.queueCached(batch[i]);
|
||||
}
|
||||
}
|
||||
log.info('MessageReceiver.queueAllCached - finished');
|
||||
}
|
||||
|
||||
private async queueCached(item: UnprocessedType): Promise<void> {
|
||||
|
@ -928,23 +933,20 @@ export default class MessageReceiver
|
|||
}
|
||||
}
|
||||
|
||||
private async getAllFromCache(): Promise<Array<UnprocessedType>> {
|
||||
private async *getAllFromCache(): AsyncIterable<Array<UnprocessedType>> {
|
||||
log.info('getAllFromCache');
|
||||
const count = await this.storage.protocol.getUnprocessedCount();
|
||||
|
||||
if (count > 1500) {
|
||||
await this.storage.protocol.removeAllUnprocessed();
|
||||
log.warn(
|
||||
`There were ${count} messages in cache. Deleted all instead of reprocessing`
|
||||
const ids = await this.storage.protocol.getAllUnprocessedIds();
|
||||
|
||||
log.info(`getAllFromCache - ${ids.length} unprocessed`);
|
||||
|
||||
for (const batch of chunk(ids, 1000)) {
|
||||
log.info(`getAllFromCache - yielding batch of ${batch.length}`);
|
||||
yield this.storage.protocol.getUnprocessedByIdsAndIncrementAttempts(
|
||||
batch
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
const items =
|
||||
await this.storage.protocol.getAllUnprocessedAndIncrementAttempts();
|
||||
log.info('getAllFromCache loaded', items.length, 'saved envelopes');
|
||||
|
||||
return items;
|
||||
log.info(`getAllFromCache - done retrieving ${ids.length} unprocessed`);
|
||||
}
|
||||
|
||||
private async decryptAndCacheBatch(
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue