From 9fab74e867011bc93c40a3462934a3ad98487a6e Mon Sep 17 00:00:00 2001 From: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com> Date: Tue, 28 Jan 2025 13:57:19 -0800 Subject: [PATCH] Resolve sticker pack references after import --- ts/models/conversations.ts | 7 +- ts/sql/Interface.ts | 15 ++- ts/sql/Server.ts | 53 +++++--- ts/sql/migrations/1300-sticker-pack-refs.ts | 35 ++++++ ts/sql/migrations/index.ts | 7 +- ts/types/Stickers.ts | 129 +++++++++++++++++--- ts/util/queueAttachmentDownloads.ts | 10 +- 7 files changed, 216 insertions(+), 40 deletions(-) create mode 100644 ts/sql/migrations/1300-sticker-pack-refs.ts diff --git a/ts/models/conversations.ts b/ts/models/conversations.ts index e12b9b4de..e15701e88 100644 --- a/ts/models/conversations.ts +++ b/ts/models/conversations.ts @@ -4155,7 +4155,12 @@ export class ConversationModel extends window.Backbone await this.#beforeAddSingleMessage(model.attributes); if (sticker) { - await addStickerPackReference(model.id, sticker.packId); + await addStickerPackReference({ + messageId: model.id, + packId: sticker.packId, + stickerId: sticker.stickerId, + isUnresolved: false, + }); } this.beforeMessageSend({ diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index ff411e95b..feb24105e 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -365,6 +365,13 @@ export type StickerPackType = InstalledStickerPackType & title: string; }>; +export type StickerPackRefType = Readonly<{ + packId: string; + messageId: string; + stickerId: number; + isUnresolved: boolean; +}>; + export type UnprocessedType = { id: string; timestamp: number; @@ -940,12 +947,14 @@ type WritableInterface = { stickerId: number, lastUsed: number ) => void; - addStickerPackReference: (messageId: string, packId: string) => void; + addStickerPackReference: (ref: StickerPackRefType) => void; deleteStickerPackReference: ( - messageId: string, - packId: string + ref: Pick ) => ReadonlyArray | undefined; deleteStickerPack: (packId: string) => Array; + getUnresolvedStickerPackReferences: ( + packId: string + ) => Array; addUninstalledStickerPack: (pack: UninstalledStickerPackType) => void; addUninstalledStickerPacks: ( pack: ReadonlyArray diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 648f1e0d7..374728bbe 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -160,6 +160,7 @@ import type { SignedPreKeyIdType, StickerPackInfoType, StickerPackStatusType, + StickerPackRefType, StickerPackType, StickerType, StoredAllItemsType, @@ -503,6 +504,7 @@ export const DataWriter: ServerWritableInterface = { addStickerPackReference, deleteStickerPackReference, deleteStickerPack, + getUnresolvedStickerPackReferences, addUninstalledStickerPack, addUninstalledStickerPacks, removeUninstalledStickerPack, @@ -5570,8 +5572,7 @@ function updateStickerLastUsed( } function addStickerPackReference( db: WritableDB, - messageId: string, - packId: string + { messageId, packId, stickerId, isUnresolved }: StickerPackRefType ): void { if (!messageId) { throw new Error( @@ -5584,37 +5585,32 @@ function addStickerPackReference( ); } - db.prepare( + prepare( + db, ` INSERT OR REPLACE INTO sticker_references ( messageId, - packId + packId, + stickerId, + isUnresolved ) values ( $messageId, - $packId + $packId, + $stickerId, + $isUnresolved ) ` ).run({ messageId, packId, + stickerId, + isUnresolved: isUnresolved ? 1 : 0, }); } function deleteStickerPackReference( db: WritableDB, - messageId: string, - packId: string + { messageId, packId }: Pick ): ReadonlyArray | undefined { - if (!messageId) { - throw new Error( - 'addStickerPackReference: Provided data did not have a truthy messageId' - ); - } - if (!packId) { - throw new Error( - 'addStickerPackReference: Provided data did not have a truthy packId' - ); - } - return db.transaction(() => { // We use an immediate transaction here to immediately acquire an exclusive lock, // which would normally only happen when we did our first write. @@ -5690,6 +5686,27 @@ function deleteStickerPackReference( return (stickerPathRows || []).map(row => row.path); })(); } +function getUnresolvedStickerPackReferences( + db: WritableDB, + packId: string +): Array { + return db.transaction(() => { + const [query, params] = sql` + UPDATE sticker_references + SET isUnresolved = 0 + WHERE packId IS ${packId} AND isUnresolved IS 1 + RETURNING messageId, stickerId; + `; + const rows = db.prepare(query).all(params); + + return rows.map(({ messageId, stickerId }) => ({ + messageId, + packId, + stickerId, + isUnresolved: true, + })); + })(); +} function deleteStickerPack(db: WritableDB, packId: string): Array { if (!packId) { diff --git a/ts/sql/migrations/1300-sticker-pack-refs.ts b/ts/sql/migrations/1300-sticker-pack-refs.ts new file mode 100644 index 000000000..7867fe28c --- /dev/null +++ b/ts/sql/migrations/1300-sticker-pack-refs.ts @@ -0,0 +1,35 @@ +// Copyright 2025 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only +import type { LoggerType } from '../../types/Logging'; +import { sql } from '../util'; +import type { WritableDB } from '../Interface'; + +export const version = 1300; + +export function updateToSchemaVersion1300( + currentVersion: number, + db: WritableDB, + logger: LoggerType +): void { + if (currentVersion >= 1300) { + return; + } + + db.transaction(() => { + const [query] = sql` + ALTER TABLE sticker_references + ADD COLUMN stickerId INTEGER NOT NULL DEFAULT -1; + ALTER TABLE sticker_references + ADD COLUMN isUnresolved INTEGER NOT NULL DEFAULT 0; + + CREATE INDEX unresolved_sticker_refs + ON sticker_references (packId, stickerId) + WHERE isUnresolved IS 1; + `; + db.exec(query); + + db.pragma('user_version = 1300'); + })(); + + logger.info('updateToSchemaVersion1300: success!'); +} diff --git a/ts/sql/migrations/index.ts b/ts/sql/migrations/index.ts index e700b502d..7e3acce1f 100644 --- a/ts/sql/migrations/index.ts +++ b/ts/sql/migrations/index.ts @@ -105,10 +105,11 @@ import { updateToSchemaVersion1250 } from './1250-defunct-call-links-storage'; import { updateToSchemaVersion1260 } from './1260-sync-tasks-rowid'; import { updateToSchemaVersion1270 } from './1270-normalize-messages'; import { updateToSchemaVersion1280 } from './1280-blob-unprocessed'; +import { updateToSchemaVersion1290 } from './1290-int-unprocessed-source-device'; import { - updateToSchemaVersion1290, + updateToSchemaVersion1300, version as MAX_VERSION, -} from './1290-int-unprocessed-source-device'; +} from './1300-sticker-pack-refs'; import { DataWriter } from '../Server'; function updateToSchemaVersion1( @@ -2084,6 +2085,8 @@ export const SCHEMA_VERSIONS = [ updateToSchemaVersion1270, updateToSchemaVersion1280, updateToSchemaVersion1290, + + updateToSchemaVersion1300, ]; export class DBVersionFromFutureError extends Error { diff --git a/ts/types/Stickers.ts b/ts/types/Stickers.ts index 2540c6d32..3d9736125 100644 --- a/ts/types/Stickers.ts +++ b/ts/types/Stickers.ts @@ -1,7 +1,7 @@ // Copyright 2019 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only -import { isNumber, reject, groupBy, values } from 'lodash'; +import { isNumber, reject, groupBy, values, chunk } from 'lodash'; import pMap from 'p-map'; import Queue from 'p-queue'; @@ -9,6 +9,7 @@ import { strictAssert } from '../util/assert'; import { dropNull } from '../util/dropNull'; import { makeLookup } from '../util/makeLookup'; import { maybeParseUrl } from '../util/url'; +import { getMessagesById } from '../messages/getMessagesById'; import * as Bytes from '../Bytes'; import * as Errors from './errors'; import { deriveStickerPackKey, decryptAttachmentV1 } from '../Crypto'; @@ -71,6 +72,10 @@ export type StickerPackPointerType = Readonly<{ export const STICKERPACK_ID_BYTE_LEN = 16; export const STICKERPACK_KEY_BYTE_LEN = 32; +// Number of messages loaded and saved at the same time when resolving sticker +// pack references. +const RESOLVE_REFERENCES_BATCH_SIZE = 1000; + const BLESSED_PACKS: Record = { '9acc9e8aba563d26a4994e69263e3b25': { key: 'Wm3/OUjCjvubeq+T7MN1xp/DFueAd+0mhnoU0QoPahI=', @@ -428,7 +433,11 @@ async function downloadSticker( export async function savePackMetadata( packId: string, packKey: string, - { messageId }: { messageId?: string } = {} + { + messageId, + stickerId, + isUnresolved, + }: { messageId: string; stickerId: number; isUnresolved: boolean } ): Promise { const existing = getStickerPack(packId); if (existing) { @@ -447,7 +456,12 @@ export async function savePackMetadata( await DataWriter.createOrUpdateStickerPack(pack); if (messageId) { - await DataWriter.addStickerPackReference(messageId, packId); + await DataWriter.addStickerPackReference({ + messageId, + packId, + stickerId, + isUnresolved, + }); } } @@ -620,7 +634,6 @@ export async function downloadEphemeralPack( } export type DownloadStickerPackOptions = Readonly<{ - messageId?: string; fromSync?: boolean; fromStorageService?: boolean; fromBackup?: boolean; @@ -651,7 +664,6 @@ async function doDownloadStickerPack( packKey: string, { finalStatus = 'downloaded', - messageId, fromSync = false, fromStorageService = false, fromBackup = false, @@ -782,10 +794,6 @@ async function doDownloadStickerPack( }; await DataWriter.createOrUpdateStickerPack(pack); stickerPackAdded(pack); - - if (messageId) { - await DataWriter.addStickerPackReference(messageId, packId); - } } catch (error) { log.error( `Error downloading manifest for sticker pack ${redactPackId(packId)}:`, @@ -854,10 +862,8 @@ async function doDownloadStickerPack( // don't overwrite that status. const existingStatus = getStickerPackStatus(packId); if (existingStatus === 'installed') { - return; - } - - if (finalStatus === 'installed') { + // No-op + } else if (finalStatus === 'installed') { await installStickerPack(packId, packKey, { fromSync, fromStorageService, @@ -870,6 +876,8 @@ async function doDownloadStickerPack( status: finalStatus, }); } + + drop(safeResolveReferences(packId)); } catch (error) { log.error( `Error downloading stickers for sticker pack ${redactPackId(packId)}:`, @@ -891,6 +899,96 @@ async function doDownloadStickerPack( } } +async function safeResolveReferences(packId: string): Promise { + try { + await resolveReferences(packId); + } catch (error) { + const logId = `Stickers.resolveReferences(${redactPackId(packId)})`; + log.error(`${logId}: failed`, Errors.toLogFormat(error)); + } +} + +async function resolveReferences(packId: string): Promise { + const refs = await DataWriter.getUnresolvedStickerPackReferences(packId); + if (refs.length === 0) { + return; + } + + const logId = `Stickers.resolveReferences(${redactPackId(packId)})`; + log.info(`${logId}: resolving ${refs.length}`); + + const stickerIdToMessageIds = new Map>(); + for (const { stickerId, messageId } of refs) { + let list = stickerIdToMessageIds.get(stickerId); + if (list == null) { + list = []; + stickerIdToMessageIds.set(stickerId, list); + } + + list.push(messageId); + } + + await pMap( + Array.from(stickerIdToMessageIds.entries()), + ([stickerId, messageIds]) => + pMap( + chunk(messageIds, RESOLVE_REFERENCES_BATCH_SIZE), + async batch => { + let attachments: Array; + try { + attachments = await pMap( + messageIds, + () => copyStickerToAttachments(packId, stickerId), + { concurrency: 3 } + ); + } catch (error) { + log.error( + `${logId}: failed to copy sticker ${stickerId}`, + Errors.toLogFormat(error) + ); + return; + } + const messages = await getMessagesById(batch); + + const saves = new Array>(); + for (const [index, message] of messages.entries()) { + const data = attachments[index]; + strictAssert(data != null, 'Missing copied data'); + + const { sticker, sent_at: sentAt } = message.attributes; + if (!sticker) { + log.info(`${logId}: ${sentAt} has no sticker`); + continue; + } + + if (sticker?.data?.path) { + log.info(`${logId}: ${sentAt} already downloaded`); + continue; + } + + if (sticker.packId !== packId || sticker.stickerId !== stickerId) { + log.info(`${logId}: ${sentAt} has different sticker`); + continue; + } + + message.set({ + sticker: { + ...sticker, + data, + }, + }); + + saves.push(window.MessageCache.saveMessage(message)); + } + + await Promise.all(saves); + }, + { concurrency: 1 } + ), + { concurrency: 3 } + ); +} + export function getStickerPack(packId: string): StickerPackType | undefined { const state = window.reduxStore.getState(); const { stickers } = state; @@ -988,7 +1086,10 @@ export async function deletePackReference( // This call uses locking to prevent race conditions with other reference removals, // or an incoming message creating a new message->pack reference - const paths = await DataWriter.deleteStickerPackReference(messageId, packId); + const paths = await DataWriter.deleteStickerPackReference({ + messageId, + packId, + }); // If we don't get a list of paths back, then the sticker pack was not deleted if (!paths) { diff --git a/ts/util/queueAttachmentDownloads.ts b/ts/util/queueAttachmentDownloads.ts index 1258345a6..48ea46867 100644 --- a/ts/util/queueAttachmentDownloads.ts +++ b/ts/util/queueAttachmentDownloads.ts @@ -314,11 +314,17 @@ export async function queueAttachmentDownloads( log.error(`${idLog}: Sticker data was missing`); } } + const stickerRef = { + messageId, + packId, + stickerId, + isUnresolved: sticker.data?.error === true, + }; if (!status) { // Save the packId/packKey for future download/install - void savePackMetadata(packId, packKey, { messageId }); + void savePackMetadata(packId, packKey, stickerRef); } else { - await DataWriter.addStickerPackReference(messageId, packId); + await DataWriter.addStickerPackReference(stickerRef); } if (!data) {