Migrate unprocessed table to BLOBs
This commit is contained in:
Fedor Indutny
GitHub
parent
06aa2f6ce4
commit
4b6ef3a1ed
10 changed files with 492 additions and 253 deletions
|
|
@ -368,32 +368,24 @@ export type StickerPackType = InstalledStickerPackType &
|
|||
export type UnprocessedType = {
|
||||
id: string;
|
||||
timestamp: number;
|
||||
receivedAtCounter: number | null;
|
||||
version: number;
|
||||
receivedAtCounter: number;
|
||||
attempts: number;
|
||||
envelope?: string;
|
||||
type: number;
|
||||
isEncrypted: boolean;
|
||||
content: Uint8Array;
|
||||
|
||||
messageAgeSec?: number;
|
||||
source?: string;
|
||||
sourceServiceId?: ServiceIdString;
|
||||
sourceDevice?: number;
|
||||
destinationServiceId?: ServiceIdString;
|
||||
updatedPni?: PniString;
|
||||
serverGuid?: string;
|
||||
serverTimestamp?: number;
|
||||
decrypted?: string;
|
||||
urgent?: boolean;
|
||||
story?: boolean;
|
||||
reportingToken?: string;
|
||||
};
|
||||
|
||||
export type UnprocessedUpdateType = {
|
||||
source?: string;
|
||||
sourceServiceId?: ServiceIdString;
|
||||
sourceDevice?: number;
|
||||
serverGuid?: string;
|
||||
serverTimestamp?: number;
|
||||
decrypted?: string;
|
||||
messageAgeSec: number;
|
||||
source: string | undefined;
|
||||
sourceServiceId: ServiceIdString | undefined;
|
||||
sourceDevice: number | undefined;
|
||||
destinationServiceId: ServiceIdString;
|
||||
updatedPni: PniString | undefined;
|
||||
serverGuid: string;
|
||||
serverTimestamp: number;
|
||||
urgent: boolean;
|
||||
story: boolean;
|
||||
reportingToken: Uint8Array | undefined;
|
||||
groupId: string | undefined;
|
||||
};
|
||||
|
||||
export type ConversationMessageStatsType = {
|
||||
|
|
@ -901,10 +893,6 @@ type WritableInterface = {
|
|||
getUnprocessedByIdsAndIncrementAttempts: (
|
||||
ids: ReadonlyArray<string>
|
||||
) => Array<UnprocessedType>;
|
||||
updateUnprocessedWithData: (id: string, data: UnprocessedUpdateType) => void;
|
||||
updateUnprocessedsWithData: (
|
||||
array: Array<{ id: string; data: UnprocessedUpdateType }>
|
||||
) => void;
|
||||
removeUnprocessed: (id: string | Array<string>) => void;
|
||||
|
||||
/** only for testing */
|
||||
|
|
|
|||
109
ts/sql/Server.ts
109
ts/sql/Server.ts
|
|
@ -174,7 +174,6 @@ import type {
|
|||
StoryReadType,
|
||||
UninstalledStickerPackType,
|
||||
UnprocessedType,
|
||||
UnprocessedUpdateType,
|
||||
WritableDB,
|
||||
} from './Interface';
|
||||
import { AttachmentDownloadSource, MESSAGE_COLUMNS } from './Interface';
|
||||
|
|
@ -474,8 +473,6 @@ export const DataWriter: ServerWritableInterface = {
|
|||
|
||||
getUnprocessedByIdsAndIncrementAttempts,
|
||||
getAllUnprocessedIds,
|
||||
updateUnprocessedWithData,
|
||||
updateUnprocessedsWithData,
|
||||
removeUnprocessed,
|
||||
removeAllUnprocessed,
|
||||
|
||||
|
|
@ -4632,17 +4629,23 @@ function saveUnprocessed(db: WritableDB, data: UnprocessedType): string {
|
|||
id,
|
||||
timestamp,
|
||||
receivedAtCounter,
|
||||
version,
|
||||
attempts,
|
||||
envelope,
|
||||
type,
|
||||
isEncrypted,
|
||||
content,
|
||||
|
||||
messageAgeSec,
|
||||
source,
|
||||
sourceServiceId,
|
||||
sourceDevice,
|
||||
destinationServiceId,
|
||||
updatedPni,
|
||||
serverGuid,
|
||||
serverTimestamp,
|
||||
decrypted,
|
||||
urgent,
|
||||
story,
|
||||
reportingToken,
|
||||
groupId,
|
||||
} = data;
|
||||
if (!id) {
|
||||
throw new Error('saveUnprocessed: id was falsey');
|
||||
|
|
@ -4655,102 +4658,72 @@ function saveUnprocessed(db: WritableDB, data: UnprocessedType): string {
|
|||
id,
|
||||
timestamp,
|
||||
receivedAtCounter,
|
||||
version,
|
||||
attempts,
|
||||
envelope,
|
||||
type,
|
||||
isEncrypted,
|
||||
content,
|
||||
|
||||
messageAgeSec,
|
||||
source,
|
||||
sourceServiceId,
|
||||
sourceDevice,
|
||||
destinationServiceId,
|
||||
updatedPni,
|
||||
serverGuid,
|
||||
serverTimestamp,
|
||||
decrypted,
|
||||
urgent,
|
||||
story
|
||||
story,
|
||||
reportingToken,
|
||||
groupId
|
||||
) values (
|
||||
$id,
|
||||
$timestamp,
|
||||
$receivedAtCounter,
|
||||
$version,
|
||||
$attempts,
|
||||
$envelope,
|
||||
$type,
|
||||
$isEncrypted,
|
||||
$content,
|
||||
|
||||
$messageAgeSec,
|
||||
$source,
|
||||
$sourceServiceId,
|
||||
$sourceDevice,
|
||||
$destinationServiceId,
|
||||
$updatedPni,
|
||||
$serverGuid,
|
||||
$serverTimestamp,
|
||||
$decrypted,
|
||||
$urgent,
|
||||
$story
|
||||
$story,
|
||||
$reportingToken,
|
||||
$groupId
|
||||
);
|
||||
`
|
||||
).run({
|
||||
id,
|
||||
timestamp,
|
||||
receivedAtCounter: receivedAtCounter ?? null,
|
||||
version,
|
||||
attempts,
|
||||
envelope: envelope || null,
|
||||
type,
|
||||
isEncrypted: isEncrypted ? 1 : 0,
|
||||
content,
|
||||
|
||||
messageAgeSec,
|
||||
source: source || null,
|
||||
sourceServiceId: sourceServiceId || null,
|
||||
sourceDevice: sourceDevice || null,
|
||||
serverGuid: serverGuid || null,
|
||||
serverTimestamp: serverTimestamp || null,
|
||||
decrypted: decrypted || null,
|
||||
destinationServiceId,
|
||||
updatedPni: updatedPni || null,
|
||||
serverGuid,
|
||||
serverTimestamp,
|
||||
urgent: urgent || !isBoolean(urgent) ? 1 : 0,
|
||||
story: story ? 1 : 0,
|
||||
reportingToken: reportingToken || null,
|
||||
groupId: groupId || null,
|
||||
});
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
function updateUnprocessedWithData(
|
||||
db: WritableDB,
|
||||
id: string,
|
||||
data: UnprocessedUpdateType
|
||||
): void {
|
||||
const {
|
||||
source,
|
||||
sourceServiceId,
|
||||
sourceDevice,
|
||||
serverGuid,
|
||||
serverTimestamp,
|
||||
decrypted,
|
||||
} = data;
|
||||
|
||||
prepare(
|
||||
db,
|
||||
`
|
||||
UPDATE unprocessed SET
|
||||
source = $source,
|
||||
sourceServiceId = $sourceServiceId,
|
||||
sourceDevice = $sourceDevice,
|
||||
serverGuid = $serverGuid,
|
||||
serverTimestamp = $serverTimestamp,
|
||||
decrypted = $decrypted
|
||||
WHERE id = $id;
|
||||
`
|
||||
).run({
|
||||
id,
|
||||
source: source || null,
|
||||
sourceServiceId: sourceServiceId || null,
|
||||
sourceDevice: sourceDevice || null,
|
||||
serverGuid: serverGuid || null,
|
||||
serverTimestamp: serverTimestamp || null,
|
||||
decrypted: decrypted || null,
|
||||
});
|
||||
}
|
||||
|
||||
function updateUnprocessedsWithData(
|
||||
db: WritableDB,
|
||||
arrayOfUnprocessed: Array<{ id: string; data: UnprocessedUpdateType }>
|
||||
): void {
|
||||
db.transaction(() => {
|
||||
for (const { id, data } of arrayOfUnprocessed) {
|
||||
updateUnprocessedWithData(db, id, data);
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
function getUnprocessedById(
|
||||
db: ReadableDB,
|
||||
id: string
|
||||
|
|
@ -4778,7 +4751,7 @@ function getAllUnprocessedIds(db: WritableDB): Array<string> {
|
|||
const { changes: deletedStaleCount } = db
|
||||
.prepare<Query>('DELETE FROM unprocessed WHERE timestamp < $monthAgo')
|
||||
.run({
|
||||
monthAgo: Date.now() - durations.MONTH,
|
||||
monthAgo: Date.now() - 45 * durations.DAY,
|
||||
});
|
||||
|
||||
if (deletedStaleCount !== 0) {
|
||||
|
|
|
|||
176
ts/sql/migrations/1280-blob-unprocessed.ts
Normal file
176
ts/sql/migrations/1280-blob-unprocessed.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
// Copyright 2025 Signal Messenger, LLC
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
import { v7 as getGuid } from 'uuid';
|
||||
|
||||
import type { LoggerType } from '../../types/Logging';
|
||||
import {
|
||||
normalizePni,
|
||||
normalizeServiceId,
|
||||
toTaggedPni,
|
||||
isUntaggedPniString,
|
||||
} from '../../types/ServiceId';
|
||||
import { SignalService as Proto } from '../../protobuf';
|
||||
import { sql } from '../util';
|
||||
import type { WritableDB } from '../Interface';
|
||||
import { getOurUuid } from './41-uuid-keys';
|
||||
|
||||
export const version = 1280;
|
||||
|
||||
export function updateToSchemaVersion1280(
|
||||
currentVersion: number,
|
||||
db: WritableDB,
|
||||
logger: LoggerType
|
||||
): void {
|
||||
if (currentVersion >= 1280) {
|
||||
return;
|
||||
}
|
||||
|
||||
db.transaction(() => {
|
||||
const ourAci = getOurUuid(db);
|
||||
|
||||
let rows = db.prepare('SELECT * FROM unprocessed').all();
|
||||
|
||||
const [query] = sql`
|
||||
DROP TABLE unprocessed;
|
||||
|
||||
CREATE TABLE unprocessed(
|
||||
id TEXT NOT NULL PRIMARY KEY ASC,
|
||||
type INTEGER NOT NULL,
|
||||
timestamp INTEGER NOT NULL,
|
||||
attempts INTEGER NOT NULL,
|
||||
receivedAtCounter INTEGER NOT NULL,
|
||||
urgent INTEGER NOT NULL,
|
||||
story INTEGER NOT NULL,
|
||||
serverGuid TEXT NOT NULL,
|
||||
serverTimestamp INTEGER NOT NULL,
|
||||
isEncrypted INTEGER NOT NULL,
|
||||
content BLOB NOT NULL,
|
||||
messageAgeSec INTEGER NOT NULL,
|
||||
destinationServiceId TEXT NOT NULL,
|
||||
|
||||
-- Not present for 1:1 messages and not sealed messages
|
||||
groupId TEXT,
|
||||
|
||||
-- Not present for sealed envelopes
|
||||
reportingToken BLOB,
|
||||
source TEXT,
|
||||
sourceServiceId TEXT,
|
||||
sourceDevice TEXT,
|
||||
|
||||
-- Present only for PNP change number
|
||||
updatedPni TEXT
|
||||
) STRICT;
|
||||
|
||||
CREATE INDEX unprocessed_timestamp ON unprocessed
|
||||
(timestamp);
|
||||
|
||||
CREATE INDEX unprocessed_byReceivedAtCounter ON unprocessed
|
||||
(receivedAtCounter);
|
||||
`;
|
||||
db.exec(query);
|
||||
|
||||
const insertStmt = db.prepare(`
|
||||
INSERT INTO unprocessed
|
||||
(id, type, timestamp, attempts, receivedAtCounter, urgent, story,
|
||||
serverGuid, serverTimestamp, isEncrypted, content, source,
|
||||
messageAgeSec, sourceServiceId, sourceDevice,
|
||||
destinationServiceId, reportingToken)
|
||||
VALUES
|
||||
($id, $type, $timestamp, $attempts, $receivedAtCounter, $urgent, $story,
|
||||
$serverGuid, $serverTimestamp, $isEncrypted, $content, $source,
|
||||
$messageAgeSec, $sourceServiceId, $sourceDevice,
|
||||
$destinationServiceId, $reportingToken);
|
||||
`);
|
||||
|
||||
let oldEnvelopes = 0;
|
||||
|
||||
if (!ourAci) {
|
||||
if (rows.length) {
|
||||
logger.warn(
|
||||
`updateToSchemaVersion1280: no aci, dropping ${rows.length} envelopes`
|
||||
);
|
||||
rows = [];
|
||||
}
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
const {
|
||||
id,
|
||||
envelope,
|
||||
decrypted,
|
||||
timestamp,
|
||||
attempts,
|
||||
version: envelopeVersion,
|
||||
receivedAtCounter,
|
||||
urgent,
|
||||
story,
|
||||
serverGuid,
|
||||
serverTimestamp,
|
||||
...rest
|
||||
} = row;
|
||||
|
||||
// Skip old and/or invalid rows
|
||||
if (envelopeVersion !== 2 || !envelope) {
|
||||
oldEnvelopes += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const decoded = Proto.Envelope.decode(Buffer.from(envelope, 'base64'));
|
||||
if (!decoded.content) {
|
||||
throw new Error('Missing envelope content');
|
||||
}
|
||||
|
||||
const content = decrypted
|
||||
? Buffer.from(decrypted, 'base64')
|
||||
: decoded.content;
|
||||
|
||||
insertStmt.run({
|
||||
...rest,
|
||||
id,
|
||||
type: decoded.type ?? Proto.Envelope.Type.UNKNOWN,
|
||||
content,
|
||||
isEncrypted: decrypted ? 0 : 1,
|
||||
timestamp: timestamp || Date.now(),
|
||||
attempts: attempts || 0,
|
||||
receivedAtCounter: receivedAtCounter || 0,
|
||||
urgent: urgent ? 1 : 0,
|
||||
story: story ? 1 : 0,
|
||||
serverGuid: serverGuid || getGuid(),
|
||||
serverTimestamp: serverTimestamp || 0,
|
||||
destinationServiceId: normalizeServiceId(
|
||||
decoded.destinationServiceId || ourAci,
|
||||
'Envelope.destinationServiceId'
|
||||
),
|
||||
updatedPni: isUntaggedPniString(decoded.updatedPni)
|
||||
? normalizePni(
|
||||
toTaggedPni(decoded.updatedPni),
|
||||
'Envelope.updatedPni'
|
||||
)
|
||||
: undefined,
|
||||
// Sadly not captured previously
|
||||
messageAgeSec: 0,
|
||||
reportingToken: decoded.reportingToken?.length
|
||||
? decoded.reportingToken
|
||||
: null,
|
||||
});
|
||||
} catch (error) {
|
||||
logger.warn(
|
||||
'updateToSchemaVersion1280: failed to migrate unprocessed',
|
||||
id,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (oldEnvelopes !== 0) {
|
||||
logger.warn(
|
||||
`updateToSchemaVersion1280: dropped ${oldEnvelopes} envelopes`
|
||||
);
|
||||
}
|
||||
|
||||
db.pragma('user_version = 1280');
|
||||
})();
|
||||
|
||||
logger.info('updateToSchemaVersion1280: success!');
|
||||
}
|
||||
|
|
@ -103,10 +103,11 @@ import { updateToSchemaVersion1230 } from './1230-call-links-admin-key-index';
|
|||
import { updateToSchemaVersion1240 } from './1240-defunct-call-links-table';
|
||||
import { updateToSchemaVersion1250 } from './1250-defunct-call-links-storage';
|
||||
import { updateToSchemaVersion1260 } from './1260-sync-tasks-rowid';
|
||||
import { updateToSchemaVersion1270 } from './1270-normalize-messages';
|
||||
import {
|
||||
updateToSchemaVersion1270,
|
||||
updateToSchemaVersion1280,
|
||||
version as MAX_VERSION,
|
||||
} from './1270-normalize-messages';
|
||||
} from './1280-blob-unprocessed';
|
||||
import { DataWriter } from '../Server';
|
||||
|
||||
function updateToSchemaVersion1(
|
||||
|
|
@ -2080,6 +2081,7 @@ export const SCHEMA_VERSIONS = [
|
|||
updateToSchemaVersion1250,
|
||||
updateToSchemaVersion1260,
|
||||
updateToSchemaVersion1270,
|
||||
updateToSchemaVersion1280,
|
||||
];
|
||||
|
||||
export class DBVersionFromFutureError extends Error {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue