Resolve sticker pack references after import

ts/models/conversations.ts

@@ -4155,7 +4155,12 @@ export class ConversationModel extends window.Backbone
     await this.#beforeAddSingleMessage(model.attributes);
 
     if (sticker) {
-      await addStickerPackReference(model.id, sticker.packId);
+      await addStickerPackReference({
+        messageId: model.id,
+        packId: sticker.packId,
+        stickerId: sticker.stickerId,
+        isUnresolved: false,
+      });
     }
 
     this.beforeMessageSend({

ts/sql/Interface.ts

@@ -365,6 +365,13 @@ export type StickerPackType = InstalledStickerPackType &
     title: string;
   }>;
 
+export type StickerPackRefType = Readonly<{
+  packId: string;
+  messageId: string;
+  stickerId: number;
+  isUnresolved: boolean;
+}>;
+
 export type UnprocessedType = {
   id: string;
   timestamp: number;
@@ -940,12 +947,14 @@ type WritableInterface = {
     stickerId: number,
     lastUsed: number
   ) => void;
-  addStickerPackReference: (messageId: string, packId: string) => void;
+  addStickerPackReference: (ref: StickerPackRefType) => void;
   deleteStickerPackReference: (
-    messageId: string,
-    packId: string
+    ref: Pick<StickerPackRefType, 'messageId' | 'packId'>
   ) => ReadonlyArray<string> | undefined;
   deleteStickerPack: (packId: string) => Array<string>;
+  getUnresolvedStickerPackReferences: (
+    packId: string
+  ) => Array<StickerPackRefType>;
   addUninstalledStickerPack: (pack: UninstalledStickerPackType) => void;
   addUninstalledStickerPacks: (
     pack: ReadonlyArray<UninstalledStickerPackType>

ts/sql/Server.ts

@@ -160,6 +160,7 @@ import type {
   SignedPreKeyIdType,
   StickerPackInfoType,
   StickerPackStatusType,
+  StickerPackRefType,
   StickerPackType,
   StickerType,
   StoredAllItemsType,
@@ -503,6 +504,7 @@ export const DataWriter: ServerWritableInterface = {
   addStickerPackReference,
   deleteStickerPackReference,
   deleteStickerPack,
+  getUnresolvedStickerPackReferences,
   addUninstalledStickerPack,
   addUninstalledStickerPacks,
   removeUninstalledStickerPack,
@@ -5570,8 +5572,7 @@ function updateStickerLastUsed(
 }
 function addStickerPackReference(
   db: WritableDB,
-  messageId: string,
-  packId: string
+  { messageId, packId, stickerId, isUnresolved }: StickerPackRefType
 ): void {
   if (!messageId) {
     throw new Error(
@@ -5584,37 +5585,32 @@ function addStickerPackReference(
     );
   }
 
-  db.prepare<Query>(
+  prepare(
+    db,
     `
     INSERT OR REPLACE INTO sticker_references (
       messageId,
-      packId
+      packId,
+      stickerId,
+      isUnresolved
     ) values (
       $messageId,
-      $packId
+      $packId,
+      $stickerId,
+      $isUnresolved
     )
     `
   ).run({
     messageId,
     packId,
+    stickerId,
+    isUnresolved: isUnresolved ? 1 : 0,
   });
 }
 function deleteStickerPackReference(
   db: WritableDB,
-  messageId: string,
-  packId: string
+  { messageId, packId }: Pick<StickerPackRefType, 'messageId' | 'packId'>
 ): ReadonlyArray<string> | undefined {
-  if (!messageId) {
-    throw new Error(
-      'addStickerPackReference: Provided data did not have a truthy messageId'
-    );
-  }
-  if (!packId) {
-    throw new Error(
-      'addStickerPackReference: Provided data did not have a truthy packId'
-    );
-  }
-
   return db.transaction(() => {
     // We use an immediate transaction here to immediately acquire an exclusive lock,
     //   which would normally only happen when we did our first write.
@@ -5690,6 +5686,27 @@ function deleteStickerPackReference(
     return (stickerPathRows || []).map(row => row.path);
   })();
 }
+function getUnresolvedStickerPackReferences(
+  db: WritableDB,
+  packId: string
+): Array<StickerPackRefType> {
+  return db.transaction(() => {
+    const [query, params] = sql`
+      UPDATE sticker_references
+      SET isUnresolved = 0
+      WHERE packId IS ${packId} AND isUnresolved IS 1
+      RETURNING messageId, stickerId;
+    `;
+    const rows = db.prepare(query).all(params);
+
+    return rows.map(({ messageId, stickerId }) => ({
+      messageId,
+      packId,
+      stickerId,
+      isUnresolved: true,
+    }));
+  })();
+}
 
 function deleteStickerPack(db: WritableDB, packId: string): Array<string> {
   if (!packId) {

ts/sql/migrations/1300-sticker-pack-refs.ts

@@ -0,0 +1,35 @@
+// Copyright 2025 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+import type { LoggerType } from '../../types/Logging';
+import { sql } from '../util';
+import type { WritableDB } from '../Interface';
+
+export const version = 1300;
+
+export function updateToSchemaVersion1300(
+  currentVersion: number,
+  db: WritableDB,
+  logger: LoggerType
+): void {
+  if (currentVersion >= 1300) {
+    return;
+  }
+
+  db.transaction(() => {
+    const [query] = sql`
+      ALTER TABLE sticker_references
+        ADD COLUMN stickerId INTEGER NOT NULL DEFAULT -1;
+      ALTER TABLE sticker_references
+        ADD COLUMN isUnresolved INTEGER NOT NULL DEFAULT 0;
+
+      CREATE INDEX unresolved_sticker_refs
+      ON sticker_references (packId, stickerId)
+      WHERE isUnresolved IS 1;
+    `;
+    db.exec(query);
+
+    db.pragma('user_version = 1300');
+  })();
+
+  logger.info('updateToSchemaVersion1300: success!');
+}

ts/sql/migrations/index.ts

@@ -105,10 +105,11 @@ import { updateToSchemaVersion1250 } from './1250-defunct-call-links-storage';
 import { updateToSchemaVersion1260 } from './1260-sync-tasks-rowid';
 import { updateToSchemaVersion1270 } from './1270-normalize-messages';
 import { updateToSchemaVersion1280 } from './1280-blob-unprocessed';
+import { updateToSchemaVersion1290 } from './1290-int-unprocessed-source-device';
 import {
-  updateToSchemaVersion1290,
+  updateToSchemaVersion1300,
   version as MAX_VERSION,
-} from './1290-int-unprocessed-source-device';
+} from './1300-sticker-pack-refs';
 import { DataWriter } from '../Server';
 
 function updateToSchemaVersion1(
@@ -2084,6 +2085,8 @@ export const SCHEMA_VERSIONS = [
   updateToSchemaVersion1270,
   updateToSchemaVersion1280,
   updateToSchemaVersion1290,
+
+  updateToSchemaVersion1300,
 ];
 
 export class DBVersionFromFutureError extends Error {

ts/types/Stickers.ts

@@ -1,7 +1,7 @@
 // Copyright 2019 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
-import { isNumber, reject, groupBy, values } from 'lodash';
+import { isNumber, reject, groupBy, values, chunk } from 'lodash';
 import pMap from 'p-map';
 import Queue from 'p-queue';
 
@@ -9,6 +9,7 @@ import { strictAssert } from '../util/assert';
 import { dropNull } from '../util/dropNull';
 import { makeLookup } from '../util/makeLookup';
 import { maybeParseUrl } from '../util/url';
+import { getMessagesById } from '../messages/getMessagesById';
 import * as Bytes from '../Bytes';
 import * as Errors from './errors';
 import { deriveStickerPackKey, decryptAttachmentV1 } from '../Crypto';
@@ -71,6 +72,10 @@ export type StickerPackPointerType = Readonly<{
 export const STICKERPACK_ID_BYTE_LEN = 16;
 export const STICKERPACK_KEY_BYTE_LEN = 32;
 
+// Number of messages loaded and saved at the same time when resolving sticker
+// pack references.
+const RESOLVE_REFERENCES_BATCH_SIZE = 1000;
+
 const BLESSED_PACKS: Record<string, BlessedType> = {
   '9acc9e8aba563d26a4994e69263e3b25': {
     key: 'Wm3/OUjCjvubeq+T7MN1xp/DFueAd+0mhnoU0QoPahI=',
@@ -428,7 +433,11 @@ async function downloadSticker(
 export async function savePackMetadata(
   packId: string,
   packKey: string,
-  { messageId }: { messageId?: string } = {}
+  {
+    messageId,
+    stickerId,
+    isUnresolved,
+  }: { messageId: string; stickerId: number; isUnresolved: boolean }
 ): Promise<void> {
   const existing = getStickerPack(packId);
   if (existing) {
@@ -447,7 +456,12 @@ export async function savePackMetadata(
 
   await DataWriter.createOrUpdateStickerPack(pack);
   if (messageId) {
-    await DataWriter.addStickerPackReference(messageId, packId);
+    await DataWriter.addStickerPackReference({
+      messageId,
+      packId,
+      stickerId,
+      isUnresolved,
+    });
   }
 }
 
@@ -620,7 +634,6 @@ export async function downloadEphemeralPack(
 }
 
 export type DownloadStickerPackOptions = Readonly<{
-  messageId?: string;
   fromSync?: boolean;
   fromStorageService?: boolean;
   fromBackup?: boolean;
@@ -651,7 +664,6 @@ async function doDownloadStickerPack(
   packKey: string,
   {
     finalStatus = 'downloaded',
-    messageId,
     fromSync = false,
     fromStorageService = false,
     fromBackup = false,
@@ -782,10 +794,6 @@ async function doDownloadStickerPack(
     };
     await DataWriter.createOrUpdateStickerPack(pack);
     stickerPackAdded(pack);
-
-    if (messageId) {
-      await DataWriter.addStickerPackReference(messageId, packId);
-    }
   } catch (error) {
     log.error(
       `Error downloading manifest for sticker pack ${redactPackId(packId)}:`,
@@ -854,10 +862,8 @@ async function doDownloadStickerPack(
     //   don't overwrite that status.
     const existingStatus = getStickerPackStatus(packId);
     if (existingStatus === 'installed') {
-      return;
-    }
-
-    if (finalStatus === 'installed') {
+      // No-op
+    } else if (finalStatus === 'installed') {
       await installStickerPack(packId, packKey, {
         fromSync,
         fromStorageService,
@@ -870,6 +876,8 @@ async function doDownloadStickerPack(
         status: finalStatus,
       });
     }
+
+    drop(safeResolveReferences(packId));
   } catch (error) {
     log.error(
       `Error downloading stickers for sticker pack ${redactPackId(packId)}:`,
@@ -891,6 +899,96 @@ async function doDownloadStickerPack(
   }
 }
 
+async function safeResolveReferences(packId: string): Promise<void> {
+  try {
+    await resolveReferences(packId);
+  } catch (error) {
+    const logId = `Stickers.resolveReferences(${redactPackId(packId)})`;
+    log.error(`${logId}: failed`, Errors.toLogFormat(error));
+  }
+}
+
+async function resolveReferences(packId: string): Promise<void> {
+  const refs = await DataWriter.getUnresolvedStickerPackReferences(packId);
+  if (refs.length === 0) {
+    return;
+  }
+
+  const logId = `Stickers.resolveReferences(${redactPackId(packId)})`;
+  log.info(`${logId}: resolving ${refs.length}`);
+
+  const stickerIdToMessageIds = new Map<number, Array<string>>();
+  for (const { stickerId, messageId } of refs) {
+    let list = stickerIdToMessageIds.get(stickerId);
+    if (list == null) {
+      list = [];
+      stickerIdToMessageIds.set(stickerId, list);
+    }
+
+    list.push(messageId);
+  }
+
+  await pMap(
+    Array.from(stickerIdToMessageIds.entries()),
+    ([stickerId, messageIds]) =>
+      pMap(
+        chunk(messageIds, RESOLVE_REFERENCES_BATCH_SIZE),
+        async batch => {
+          let attachments: Array<AttachmentType>;
+          try {
+            attachments = await pMap(
+              messageIds,
+              () => copyStickerToAttachments(packId, stickerId),
+              { concurrency: 3 }
+            );
+          } catch (error) {
+            log.error(
+              `${logId}: failed to copy sticker ${stickerId}`,
+              Errors.toLogFormat(error)
+            );
+            return;
+          }
+          const messages = await getMessagesById(batch);
+
+          const saves = new Array<Promise<unknown>>();
+          for (const [index, message] of messages.entries()) {
+            const data = attachments[index];
+            strictAssert(data != null, 'Missing copied data');
+
+            const { sticker, sent_at: sentAt } = message.attributes;
+            if (!sticker) {
+              log.info(`${logId}: ${sentAt} has no sticker`);
+              continue;
+            }
+
+            if (sticker?.data?.path) {
+              log.info(`${logId}: ${sentAt} already downloaded`);
+              continue;
+            }
+
+            if (sticker.packId !== packId || sticker.stickerId !== stickerId) {
+              log.info(`${logId}: ${sentAt} has different sticker`);
+              continue;
+            }
+
+            message.set({
+              sticker: {
+                ...sticker,
+                data,
+              },
+            });
+
+            saves.push(window.MessageCache.saveMessage(message));
+          }
+
+          await Promise.all(saves);
+        },
+        { concurrency: 1 }
+      ),
+    { concurrency: 3 }
+  );
+}
+
 export function getStickerPack(packId: string): StickerPackType | undefined {
   const state = window.reduxStore.getState();
   const { stickers } = state;
@@ -988,7 +1086,10 @@ export async function deletePackReference(
 
   // This call uses locking to prevent race conditions with other reference removals,
   //   or an incoming message creating a new message->pack reference
-  const paths = await DataWriter.deleteStickerPackReference(messageId, packId);
+  const paths = await DataWriter.deleteStickerPackReference({
+    messageId,
+    packId,
+  });
 
   // If we don't get a list of paths back, then the sticker pack was not deleted
   if (!paths) {

ts/util/queueAttachmentDownloads.ts

@@ -314,11 +314,17 @@ export async function queueAttachmentDownloads(
         log.error(`${idLog}: Sticker data was missing`);
       }
     }
+    const stickerRef = {
+      messageId,
+      packId,
+      stickerId,
+      isUnresolved: sticker.data?.error === true,
+    };
     if (!status) {
       // Save the packId/packKey for future download/install
-      void savePackMetadata(packId, packKey, { messageId });
+      void savePackMetadata(packId, packKey, stickerRef);
     } else {
-      await DataWriter.addStickerPackReference(messageId, packId);
+      await DataWriter.addStickerPackReference(stickerRef);
     }
 
     if (!data) {