Import/export group state

Co-authored-by: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com>

ts/groups.ts

@@ -91,6 +91,7 @@ import {
   conversationJobQueue,
   conversationQueueJobEnum,
 } from './jobs/conversationJobQueue';
+import { groupAvatarJobQueue } from './jobs/groupAvatarJobQueue';
 import { ReadStatus } from './messages/MessageReadStatus';
 import { SeenStatus } from './MessageSeenStatus';
 import { incrementMessageCounter } from './util/incrementMessageCounter';
@@ -3239,8 +3240,13 @@ async function updateGroup(
     await appendChangeMessages(conversation, changeMessagesToSave);
   }
 
+  const { avatar: newAvatar, ...restOfAttributes } = newAttributes;
+  const hasAvatarChanged =
+    'avatar' in newAttributes &&
+    newAvatar?.url !== conversation.get('avatar')?.url;
+
   conversation.set({
-    ...newAttributes,
+    ...restOfAttributes,
     active_at: activeAt,
   });
 
@@ -3250,6 +3256,13 @@ async function updateGroup(
 
   // Save these most recent updates to conversation
   await updateConversation(conversation.attributes);
+
+  if (hasAvatarChanged) {
+    await groupAvatarJobQueue.add({
+      conversationId: conversation.id,
+      newAvatarUrl: newAvatar?.url,
+    });
+  }
 }
 
 // Exported for testing
@@ -3685,12 +3698,11 @@ async function updateGroupViaPreJoinInfo({
       },
     ],
     revision: dropNull(preJoinInfo.version),
+    avatar: preJoinInfo.avatar ? { url: preJoinInfo.avatar } : undefined,
 
     temporaryMemberCount: preJoinInfo.memberCount || 1,
   };
 
-  await applyNewAvatar(dropNull(preJoinInfo.avatar), newAttributes, logId);
-
   return {
     newAttributes,
     groupChangeMessages: extractDiffs({
@@ -5247,7 +5259,7 @@ async function applyGroupChange({
   // modifyAvatar?: GroupChange.Actions.ModifyAvatarAction;
   if (actions.modifyAvatar) {
     const { avatar } = actions.modifyAvatar;
-    await applyNewAvatar(dropNull(avatar), result, logId);
+    result.avatar = avatar ? { url: avatar } : undefined;
   }
 
   // modifyDisappearingMessagesTimer?:
@@ -5519,46 +5531,60 @@ export async function decryptGroupAvatar(
 }
 
 // Overwriting result.avatar as part of functionality
-/* eslint-disable no-param-reassign */
 export async function applyNewAvatar(
-  newAvatar: string | undefined,
+  newAvatarUrl: string | undefined,
-  result: Pick<ConversationAttributesType, 'avatar' | 'secretParams'>,
+  attributes: Readonly<
+    Pick<ConversationAttributesType, 'avatar' | 'secretParams'>
+  >,
   logId: string
-): Promise<void> {
+): Promise<Pick<ConversationAttributesType, 'avatar'>> {
+  const result: Pick<ConversationAttributesType, 'avatar'> = {};
   try {
     // Avatar has been dropped
-    if (!newAvatar && result.avatar) {
+    if (!newAvatarUrl && attributes.avatar) {
-      await window.Signal.Migrations.deleteAttachmentData(result.avatar.path);
+      if (attributes.avatar.path) {
+        await window.Signal.Migrations.deleteAttachmentData(
+          attributes.avatar.path
+        );
+      }
       result.avatar = undefined;
     }
 
     // Group has avatar; has it changed?
-    if (newAvatar && (!result.avatar || result.avatar.url !== newAvatar)) {
+    if (
-      if (!result.secretParams) {
+      newAvatarUrl &&
+      (!attributes.avatar || attributes.avatar.url !== newAvatarUrl)
+    ) {
+      if (!attributes.secretParams) {
         throw new Error('applyNewAvatar: group was missing secretParams!');
       }
 
-      const data = await decryptGroupAvatar(newAvatar, result.secretParams);
+      const data = await decryptGroupAvatar(
+        newAvatarUrl,
+        attributes.secretParams
+      );
       const hash = computeHash(data);
 
-      if (result.avatar?.hash === hash) {
+      if (attributes.avatar?.hash === hash) {
         log.info(
           `applyNewAvatar/${logId}: Hash is the same, but url was different. Saving new url.`
         );
         result.avatar = {
-          ...result.avatar,
+          ...attributes.avatar,
-          url: newAvatar,
+          url: newAvatarUrl,
         };
-        return;
+        return result;
       }
 
-      if (result.avatar) {
+      if (attributes.avatar?.path) {
-        await window.Signal.Migrations.deleteAttachmentData(result.avatar.path);
+        await window.Signal.Migrations.deleteAttachmentData(
+          attributes.avatar.path
+        );
       }
 
       const path = await window.Signal.Migrations.writeNewAttachmentData(data);
       result.avatar = {
-        url: newAvatar,
+        url: newAvatarUrl,
         path,
         hash,
       };
@@ -5573,8 +5599,8 @@ export async function applyNewAvatar(
     }
     result.avatar = undefined;
   }
+  return result;
 }
-/* eslint-enable no-param-reassign */
 
 function profileKeyHasChanged(
   userId: ServiceIdString,
@@ -5654,7 +5680,11 @@ async function applyGroupState({
   }
 
   // avatar
-  await applyNewAvatar(dropNull(groupState.avatar), result, logId);
+  result.avatar = groupState.avatar
+    ? {
+        url: groupState.avatar,
+      }
+    : undefined;
 
   // disappearingMessagesTimer
   // Note: during decryption, disappearingMessageTimer becomes a GroupAttributeBlob

ts/groups/joinViaLink.ts

@@ -393,14 +393,15 @@ export async function joinViaLink(value: string): Promise<void> {
         loading: true,
       };
 
-      const attributes: Pick<
+      let attributes: Pick<
         ConversationAttributesType,
         'avatar' | 'secretParams'
       > = {
         avatar: null,
         secretParams,
       };
-      await applyNewAvatar(result.avatar, attributes, logId);
+      const patch = await applyNewAvatar(result.avatar, attributes, logId);
+      attributes = { ...attributes, ...patch };
 
       if (attributes.avatar && attributes.avatar.path) {
         localAvatar = {

ts/jobs/groupAvatarJobQueue.ts

@@ -0,0 +1,59 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import * as z from 'zod';
+import type { LoggerType } from '../types/Logging';
+import { applyNewAvatar } from '../groups';
+import { isGroupV2 } from '../util/whatTypeOfConversation';
+import Data from '../sql/Client';
+
+import type { JOB_STATUS } from './JobQueue';
+import { JobQueue } from './JobQueue';
+import { jobQueueDatabaseStore } from './JobQueueDatabaseStore';
+
+const groupAvatarJobDataSchema = z.object({
+  conversationId: z.string(),
+  newAvatarUrl: z.string().optional(),
+});
+
+export type GroupAvatarJobData = z.infer<typeof groupAvatarJobDataSchema>;
+
+export class GroupAvatarJobQueue extends JobQueue<GroupAvatarJobData> {
+  protected parseData(data: unknown): GroupAvatarJobData {
+    return groupAvatarJobDataSchema.parse(data);
+  }
+
+  protected async run(
+    { data }: Readonly<{ data: GroupAvatarJobData; timestamp: number }>,
+    { attempt, log }: Readonly<{ attempt: number; log: LoggerType }>
+  ): Promise<typeof JOB_STATUS.NEEDS_RETRY | undefined> {
+    const { conversationId, newAvatarUrl } = data;
+    const logId = `groupAvatarJobQueue(${conversationId}, attempt=${attempt})`;
+
+    const convo = window.ConversationController.get(conversationId);
+    if (!convo) {
+      log.warn(`${logId}: dropping ${conversationId}, not found`);
+      return undefined;
+    }
+
+    const { attributes } = convo;
+    if (!isGroupV2(attributes)) {
+      log.warn(`${logId}: dropping ${conversationId}, not a group`);
+      return undefined;
+    }
+
+    // Generate correct attributes patch
+    const patch = await applyNewAvatar(newAvatarUrl, attributes, logId);
+
+    convo.set(patch);
+    await Data.updateConversation(convo.attributes);
+
+    return undefined;
+  }
+}
+
+export const groupAvatarJobQueue = new GroupAvatarJobQueue({
+  store: jobQueueDatabaseStore,
+  queueType: 'groupAvatar',
+  maxAttempts: 25,
+});

ts/jobs/initializeAllJobQueues.ts

@@ -5,6 +5,7 @@ import type { WebAPIType } from '../textsecure/WebAPI';
 import { drop } from '../util/drop';
 
 import { conversationJobQueue } from './conversationJobQueue';
+import { groupAvatarJobQueue } from './groupAvatarJobQueue';
 import { readSyncJobQueue } from './readSyncJobQueue';
 import { removeStorageKeyJobQueue } from './removeStorageKeyJobQueue';
 import { reportSpamJobQueue } from './reportSpamJobQueue';
@@ -25,6 +26,9 @@ export function initializeAllJobQueues({
   // General conversation send queue
   drop(conversationJobQueue.streamJobs());
 
+  // Group avatar download after backup import
+  drop(groupAvatarJobQueue.streamJobs());
+
   // Single proto send queue, used for a variety of one-off simple messages
   drop(singleProtoJobQueue.streamJobs());
 
@@ -41,6 +45,7 @@ export function initializeAllJobQueues({
 export async function shutdownAllJobQueues(): Promise<void> {
   await Promise.allSettled([
     conversationJobQueue.shutdown(),
+    groupAvatarJobQueue.shutdown(),
     singleProtoJobQueue.shutdown(),
     readSyncJobQueue.shutdown(),
     viewSyncJobQueue.shutdown(),

ts/model-types.d.ts

@@ -509,7 +509,7 @@ export type LegacyMigrationPendingMemberType = {
 };
 
 export type GroupV2PendingMemberType = {
-  addedByUserId?: AciString;
+  addedByUserId: AciString;
   serviceId: ServiceIdString;
   timestamp: number;
   role: MemberRoleEnum;

ts/services/backups/export.ts

@@ -76,6 +76,7 @@ import {
 import * as Bytes from '../../Bytes';
 import { canBeSynced as canPreferredReactionEmojiBeSynced } from '../../reactions/preferredReactionEmoji';
 import { SendStatus } from '../../messages/MessageSendState';
+import { deriveGroupFields } from '../../groups';
 import { BACKUP_VERSION } from './constants';
 import { getMessageIdForLogging } from '../../util/idForLogging';
 import { getCallsHistoryForRedux } from '../callHistoryLoader';
@@ -112,6 +113,8 @@ const FLUSH_TIMEOUT = 30 * MINUTE;
 // Threshold for reporting slow flushes
 const REPORTING_THRESHOLD = SECOND;
 
+const ZERO_PROFILE_KEY = new Uint8Array(32);
+
 type GetRecipientIdOptionsType =
   | Readonly<{
       serviceId: ServiceIdString;
@@ -672,11 +675,92 @@ export class BackupExportStream extends Readable {
           break;
       }
 
+      const masterKey = Bytes.fromBase64(convo.masterKey);
+
+      let publicKey;
+      if (convo.publicParams) {
+        publicKey = Bytes.fromBase64(convo.publicParams);
+      } else {
+        ({ publicParams: publicKey } = deriveGroupFields(masterKey));
+      }
+
       res.group = {
-        masterKey: Bytes.fromBase64(convo.masterKey),
+        masterKey,
         whitelisted: convo.profileSharing,
         hideStory: convo.hideStory === true,
         storySendMode,
+        snapshot: {
+          publicKey,
+          title: {
+            title: convo.name ?? '',
+          },
+          description: {
+            descriptionText: convo.description ?? '',
+          },
+          avatarUrl: convo.avatar?.url,
+          disappearingMessagesTimer:
+            convo.expireTimer != null
+              ? {
+                  disappearingMessagesDuration: DurationInSeconds.toSeconds(
+                    convo.expireTimer
+                  ),
+                }
+              : null,
+          accessControl: convo.accessControl,
+          version: convo.revision || 0,
+          members: convo.membersV2?.map(member => {
+            const memberConvo = window.ConversationController.get(member.aci);
+            strictAssert(memberConvo, 'Missing GV2 member');
+
+            const { profileKey } = memberConvo.attributes;
+
+            return {
+              userId: this.aciToBytes(member.aci),
+              role: member.role,
+              profileKey: profileKey
+                ? Bytes.fromBase64(profileKey)
+                : ZERO_PROFILE_KEY,
+              joinedAtVersion: member.joinedAtVersion,
+            };
+          }),
+          membersPendingProfileKey: convo.pendingMembersV2?.map(member => {
+            return {
+              member: {
+                userId: this.serviceIdToBytes(member.serviceId),
+                role: member.role,
+                profileKey: ZERO_PROFILE_KEY,
+                joinedAtVersion: 0,
+              },
+              addedByUserId: this.aciToBytes(member.addedByUserId),
+              timestamp: getSafeLongFromTimestamp(member.timestamp),
+            };
+          }),
+          membersPendingAdminApproval: convo.pendingAdminApprovalV2?.map(
+            member => {
+              const memberConvo = window.ConversationController.get(member.aci);
+              strictAssert(memberConvo, 'Missing GV2 member pending approval');
+
+              const { profileKey } = memberConvo.attributes;
+              return {
+                userId: this.aciToBytes(member.aci),
+                profileKey: profileKey
+                  ? Bytes.fromBase64(profileKey)
+                  : ZERO_PROFILE_KEY,
+                timestamp: getSafeLongFromTimestamp(member.timestamp),
+              };
+            }
+          ),
+          membersBanned: convo.bannedMembersV2?.map(member => {
+            return {
+              userId: this.serviceIdToBytes(member.serviceId),
+              timestamp: getSafeLongFromTimestamp(member.timestamp),
+            };
+          }),
+          inviteLinkPassword: convo.groupInviteLinkPassword
+            ? Bytes.fromBase64(convo.groupInviteLinkPassword)
+            : null,
+          announcementsOnly: convo.announcementsOnly === true,
+        },
       };
     } else {
       return undefined;

ts/services/backups/import.ts

@@ -1,7 +1,7 @@
 // Copyright 2023 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
-import { Aci, Pni } from '@signalapp/libsignal-client';
+import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client';
 import { ReceiptCredentialPresentation } from '@signalapp/libsignal-client/zkgroup';
 import { v4 as generateUuid } from 'uuid';
 import pMap from 'p-map';
@@ -15,7 +15,11 @@ import * as log from '../../logging/log';
 import { GiftBadgeStates } from '../../components/conversation/Message';
 import { StorySendMode } from '../../types/Stories';
 import type { ServiceIdString, AciString } from '../../types/ServiceId';
-import { fromAciObject, fromPniObject } from '../../types/ServiceId';
+import {
+  fromAciObject,
+  fromPniObject,
+  fromServiceIdObject,
+} from '../../types/ServiceId';
 import { isStoryDistributionId } from '../../types/StoryDistributionId';
 import * as Errors from '../../types/errors';
 import { PaymentEventKind } from '../../types/Payment';
@@ -63,7 +67,7 @@ import type { GroupV2ChangeDetailType } from '../../groups';
 import { queueAttachmentDownloads } from '../../util/queueAttachmentDownloads';
 import { drop } from '../../util/drop';
 import { isNotNil } from '../../util/isNotNil';
-import { isGroup } from '../../util/whatTypeOfConversation';
+import { isGroup, isGroupV2 } from '../../util/whatTypeOfConversation';
 import {
   convertBackupMessageAttachmentToAttachment,
   convertFilePointerToAttachment,
@@ -71,6 +75,7 @@ import {
 import { filterAndClean } from '../../types/BodyRange';
 import { APPLICATION_OCTET_STREAM, stringToMIMEType } from '../../types/MIME';
 import { copyFromQuotedMessage } from '../../messages/copyQuote';
+import { groupAvatarJobQueue } from '../../jobs/groupAvatarJobQueue';
 
 const MAX_CONCURRENCY = 10;
 
@@ -306,16 +311,28 @@ export class BackupImportStream extends Writable {
       window.storage.reset();
       await window.storage.fetch();
 
+      const allConversations = window.ConversationController.getAll();
+
       // Update last message in every active conversation now that we have
       // them loaded into memory.
       await pMap(
-        window.ConversationController.getAll().filter(convo => {
+        allConversations.filter(convo => {
           return convo.get('active_at') || convo.get('isPinned');
         }),
         convo => convo.updateLastMessage(),
         { concurrency: MAX_CONCURRENCY }
       );
 
+      // Schedule group avatar download.
+      await pMap(
+        allConversations.filter(({ attributes: convo }) => {
+          const { avatar } = convo;
+          return isGroupV2(convo) && avatar?.url && !avatar.path;
+        }),
+        convo => groupAvatarJobQueue.add({ conversationId: convo.id }),
+        { concurrency: MAX_CONCURRENCY }
+      );
+
       await window.storage.put(
         'pinnedConversationIds',
         this.pinnedConversations
@@ -675,30 +692,153 @@ export class BackupImportStream extends Writable {
   private async fromGroup(
     group: Backups.IGroup
   ): Promise<ConversationAttributesType> {
-    strictAssert(group.masterKey != null, 'fromGroup: missing masterKey');
+    const { masterKey, snapshot } = group;
+    strictAssert(masterKey != null, 'fromGroup: missing masterKey');
+    strictAssert(snapshot != null, 'fromGroup: missing snapshot');
 
-    const secretParams = deriveGroupSecretParams(group.masterKey);
+    const secretParams = deriveGroupSecretParams(masterKey);
     const publicParams = deriveGroupPublicParams(secretParams);
     const groupId = Bytes.toBase64(deriveGroupID(secretParams));
 
+    const {
+      title,
+      description,
+      avatarUrl,
+      disappearingMessagesTimer,
+      accessControl,
+      version,
+      members,
+      membersPendingProfileKey,
+      membersPendingAdminApproval,
+      membersBanned,
+      inviteLinkPassword,
+      announcementsOnly,
+    } = snapshot;
+
+    const expirationTimerS =
+      disappearingMessagesTimer?.disappearingMessagesDuration;
+
+    let storySendMode: StorySendMode | undefined;
+    switch (group.storySendMode) {
+      case Backups.Group.StorySendMode.ENABLED:
+        storySendMode = StorySendMode.Always;
+        break;
+      case Backups.Group.StorySendMode.DISABLED:
+        storySendMode = StorySendMode.Never;
+        break;
+      default:
+        storySendMode = undefined;
+        break;
+    }
     const attrs: ConversationAttributesType = {
       id: generateUuid(),
       type: 'group',
       version: 2,
       groupVersion: 2,
-      masterKey: Bytes.toBase64(group.masterKey),
+      masterKey: Bytes.toBase64(masterKey),
       groupId,
       secretParams: Bytes.toBase64(secretParams),
       publicParams: Bytes.toBase64(publicParams),
       profileSharing: group.whitelisted === true,
       hideStory: group.hideStory === true,
-    };
+      storySendMode,
 
-    if (group.storySendMode === Backups.Group.StorySendMode.ENABLED) {
+      // Snapshot
-      attrs.storySendMode = StorySendMode.Always;
+      name: dropNull(title?.title),
-    } else if (group.storySendMode === Backups.Group.StorySendMode.DISABLED) {
+      description: dropNull(description?.descriptionText),
-      attrs.storySendMode = StorySendMode.Never;
+      avatar: avatarUrl
+        ? {
+            url: avatarUrl,
+            path: '',
           }
+        : undefined,
+      expireTimer: expirationTimerS
+        ? DurationInSeconds.fromSeconds(expirationTimerS)
+        : undefined,
+      accessControl: accessControl
+        ? {
+            attributes:
+              dropNull(accessControl.attributes) ??
+              SignalService.AccessControl.AccessRequired.UNKNOWN,
+            members:
+              dropNull(accessControl.members) ??
+              SignalService.AccessControl.AccessRequired.UNKNOWN,
+            addFromInviteLink:
+              dropNull(accessControl.addFromInviteLink) ??
+              SignalService.AccessControl.AccessRequired.UNKNOWN,
+          }
+        : undefined,
+      membersV2: members?.map(({ userId, role, joinedAtVersion }) => {
+        strictAssert(Bytes.isNotEmpty(userId), 'Empty gv2 member userId');
+
+        // Note that we deliberately ignore profile key since it has to be
+        // in the Contact frame
+
+        return {
+          aci: fromAciObject(Aci.fromUuidBytes(userId)),
+          role: dropNull(role) ?? SignalService.Member.Role.UNKNOWN,
+          joinedAtVersion: dropNull(joinedAtVersion) ?? 0,
+        };
+      }),
+      pendingMembersV2: membersPendingProfileKey?.map(
+        ({ member, addedByUserId, timestamp }) => {
+          strictAssert(member != null, 'Missing gv2 pending member');
+          strictAssert(
+            Bytes.isNotEmpty(addedByUserId),
+            'Empty gv2 pending member addedByUserId'
+          );
+
+          // profileKey is not available for pending members.
+          const { userId, role } = member;
+
+          strictAssert(Bytes.isNotEmpty(userId), 'Empty gv2 member userId');
+
+          const serviceId = fromServiceIdObject(
+            ServiceId.parseFromServiceIdBinary(Buffer.from(userId))
+          );
+
+          return {
+            serviceId,
+            role: dropNull(role) ?? SignalService.Member.Role.UNKNOWN,
+            addedByUserId: fromAciObject(Aci.fromUuidBytes(addedByUserId)),
+            timestamp: timestamp != null ? getTimestampFromLong(timestamp) : 0,
+          };
+        }
+      ),
+      pendingAdminApprovalV2: membersPendingAdminApproval?.map(
+        ({ userId, timestamp }) => {
+          strictAssert(Bytes.isNotEmpty(userId), 'Empty gv2 member userId');
+
+          // Note that we deliberately ignore profile key since it has to be
+          // in the Contact frame
+
+          return {
+            aci: fromAciObject(Aci.fromUuidBytes(userId)),
+            timestamp: timestamp != null ? getTimestampFromLong(timestamp) : 0,
+          };
+        }
+      ),
+      bannedMembersV2: membersBanned?.map(({ userId, timestamp }) => {
+        strictAssert(Bytes.isNotEmpty(userId), 'Empty gv2 member userId');
+
+        // Note that we deliberately ignore profile key since it has to be
+        // in the Contact frame
+
+        const serviceId = fromServiceIdObject(
+          ServiceId.parseFromServiceIdBinary(Buffer.from(userId))
+        );
+
+        return {
+          serviceId,
+          timestamp: timestamp != null ? getTimestampFromLong(timestamp) : 0,
+        };
+      }),
+      revision: dropNull(version),
+      groupInviteLinkPassword: Bytes.isNotEmpty(inviteLinkPassword)
+        ? Bytes.toBase64(inviteLinkPassword)
+        : undefined,
+      announcementsOnly: dropNull(announcementsOnly),
+    };
 
     return attrs;
   }

ts/types/Avatar.ts

@@ -34,10 +34,18 @@ export const GroupAvatarIcons = [
   'surfboard',
 ] as const;
 
-export type ContactAvatarType = {
+export type ContactAvatarType =
+  | {
+      // Downloaded avatar
       path: string;
       url?: string;
       hash?: string;
+    }
+  | {
+      // Not-yet downloaded avatar
+      path?: string;
+      url: string;
+      hash?: string;
     };
 
 type GroupAvatarIconType = typeof GroupAvatarIcons[number];

ts/types/Conversation.ts

@@ -57,7 +57,7 @@ function buildAvatarUpdater({ field }: { field: 'avatar' | 'profileAvatar' }) {
     }
 
     const { hash, path } = oldAvatar;
-    const exists = await doesAttachmentExist(path);
+    const exists = path && (await doesAttachmentExist(path));
     if (!exists) {
       window.SignalContext.log.warn(
         `Conversation.buildAvatarUpdater: attachment ${path} did not exist`
@@ -66,7 +66,9 @@ function buildAvatarUpdater({ field }: { field: 'avatar' | 'profileAvatar' }) {
 
     if (exists) {
       if (newAvatar && hash && hash === newAvatar.hash) {
+        if (newAvatar.path) {
           await deleteAttachmentData(newAvatar.path);
+        }
         return conversation;
       }
       if (data && hash && hash === newHash) {
@@ -74,7 +76,9 @@ function buildAvatarUpdater({ field }: { field: 'avatar' | 'profileAvatar' }) {
       }
     }
 
+    if (path) {
       await deleteAttachmentData(path);
+    }
 
     if (newAvatar) {
       return {