From 5f310b4074f2c31213270e2ae0e64e53441b2c0e Mon Sep 17 00:00:00 2001 From: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com> Date: Mon, 24 Jun 2024 11:38:59 -0700 Subject: [PATCH] Import/export group state --- ts/groups.ts | 74 +++++++++---- ts/groups/joinViaLink.ts | 5 +- ts/jobs/groupAvatarJobQueue.ts | 59 +++++++++++ ts/jobs/initializeAllJobQueues.ts | 5 + ts/model-types.d.ts | 2 +- ts/services/backups/export.ts | 86 +++++++++++++++- ts/services/backups/import.ts | 166 +++++++++++++++++++++++++++--- ts/types/Avatar.ts | 18 +++- ts/types/Conversation.ts | 10 +- 9 files changed, 378 insertions(+), 47 deletions(-) create mode 100644 ts/jobs/groupAvatarJobQueue.ts diff --git a/ts/groups.ts b/ts/groups.ts index b62acb7911..221ae32464 100644 --- a/ts/groups.ts +++ b/ts/groups.ts @@ -91,6 +91,7 @@ import { conversationJobQueue, conversationQueueJobEnum, } from './jobs/conversationJobQueue'; +import { groupAvatarJobQueue } from './jobs/groupAvatarJobQueue'; import { ReadStatus } from './messages/MessageReadStatus'; import { SeenStatus } from './MessageSeenStatus'; import { incrementMessageCounter } from './util/incrementMessageCounter'; @@ -3239,8 +3240,13 @@ async function updateGroup( await appendChangeMessages(conversation, changeMessagesToSave); } + const { avatar: newAvatar, ...restOfAttributes } = newAttributes; + const hasAvatarChanged = + 'avatar' in newAttributes && + newAvatar?.url !== conversation.get('avatar')?.url; + conversation.set({ - ...newAttributes, + ...restOfAttributes, active_at: activeAt, }); @@ -3250,6 +3256,13 @@ async function updateGroup( // Save these most recent updates to conversation await updateConversation(conversation.attributes); + + if (hasAvatarChanged) { + await groupAvatarJobQueue.add({ + conversationId: conversation.id, + newAvatarUrl: newAvatar?.url, + }); + } } // Exported for testing @@ -3685,12 +3698,11 @@ async function updateGroupViaPreJoinInfo({ }, ], revision: dropNull(preJoinInfo.version), + avatar: preJoinInfo.avatar ? { url: preJoinInfo.avatar } : undefined, temporaryMemberCount: preJoinInfo.memberCount || 1, }; - await applyNewAvatar(dropNull(preJoinInfo.avatar), newAttributes, logId); - return { newAttributes, groupChangeMessages: extractDiffs({ @@ -5247,7 +5259,7 @@ async function applyGroupChange({ // modifyAvatar?: GroupChange.Actions.ModifyAvatarAction; if (actions.modifyAvatar) { const { avatar } = actions.modifyAvatar; - await applyNewAvatar(dropNull(avatar), result, logId); + result.avatar = avatar ? { url: avatar } : undefined; } // modifyDisappearingMessagesTimer?: @@ -5519,46 +5531,60 @@ export async function decryptGroupAvatar( } // Overwriting result.avatar as part of functionality -/* eslint-disable no-param-reassign */ export async function applyNewAvatar( - newAvatar: string | undefined, - result: Pick, + newAvatarUrl: string | undefined, + attributes: Readonly< + Pick + >, logId: string -): Promise { +): Promise> { + const result: Pick = {}; try { // Avatar has been dropped - if (!newAvatar && result.avatar) { - await window.Signal.Migrations.deleteAttachmentData(result.avatar.path); + if (!newAvatarUrl && attributes.avatar) { + if (attributes.avatar.path) { + await window.Signal.Migrations.deleteAttachmentData( + attributes.avatar.path + ); + } result.avatar = undefined; } // Group has avatar; has it changed? - if (newAvatar && (!result.avatar || result.avatar.url !== newAvatar)) { - if (!result.secretParams) { + if ( + newAvatarUrl && + (!attributes.avatar || attributes.avatar.url !== newAvatarUrl) + ) { + if (!attributes.secretParams) { throw new Error('applyNewAvatar: group was missing secretParams!'); } - const data = await decryptGroupAvatar(newAvatar, result.secretParams); + const data = await decryptGroupAvatar( + newAvatarUrl, + attributes.secretParams + ); const hash = computeHash(data); - if (result.avatar?.hash === hash) { + if (attributes.avatar?.hash === hash) { log.info( `applyNewAvatar/${logId}: Hash is the same, but url was different. Saving new url.` ); result.avatar = { - ...result.avatar, - url: newAvatar, + ...attributes.avatar, + url: newAvatarUrl, }; - return; + return result; } - if (result.avatar) { - await window.Signal.Migrations.deleteAttachmentData(result.avatar.path); + if (attributes.avatar?.path) { + await window.Signal.Migrations.deleteAttachmentData( + attributes.avatar.path + ); } const path = await window.Signal.Migrations.writeNewAttachmentData(data); result.avatar = { - url: newAvatar, + url: newAvatarUrl, path, hash, }; @@ -5573,8 +5599,8 @@ export async function applyNewAvatar( } result.avatar = undefined; } + return result; } -/* eslint-enable no-param-reassign */ function profileKeyHasChanged( userId: ServiceIdString, @@ -5654,7 +5680,11 @@ async function applyGroupState({ } // avatar - await applyNewAvatar(dropNull(groupState.avatar), result, logId); + result.avatar = groupState.avatar + ? { + url: groupState.avatar, + } + : undefined; // disappearingMessagesTimer // Note: during decryption, disappearingMessageTimer becomes a GroupAttributeBlob diff --git a/ts/groups/joinViaLink.ts b/ts/groups/joinViaLink.ts index 88cfc5200a..99886de4c5 100644 --- a/ts/groups/joinViaLink.ts +++ b/ts/groups/joinViaLink.ts @@ -393,14 +393,15 @@ export async function joinViaLink(value: string): Promise { loading: true, }; - const attributes: Pick< + let attributes: Pick< ConversationAttributesType, 'avatar' | 'secretParams' > = { avatar: null, secretParams, }; - await applyNewAvatar(result.avatar, attributes, logId); + const patch = await applyNewAvatar(result.avatar, attributes, logId); + attributes = { ...attributes, ...patch }; if (attributes.avatar && attributes.avatar.path) { localAvatar = { diff --git a/ts/jobs/groupAvatarJobQueue.ts b/ts/jobs/groupAvatarJobQueue.ts new file mode 100644 index 0000000000..cfa2ea39a1 --- /dev/null +++ b/ts/jobs/groupAvatarJobQueue.ts @@ -0,0 +1,59 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import * as z from 'zod'; +import type { LoggerType } from '../types/Logging'; +import { applyNewAvatar } from '../groups'; +import { isGroupV2 } from '../util/whatTypeOfConversation'; +import Data from '../sql/Client'; + +import type { JOB_STATUS } from './JobQueue'; +import { JobQueue } from './JobQueue'; +import { jobQueueDatabaseStore } from './JobQueueDatabaseStore'; + +const groupAvatarJobDataSchema = z.object({ + conversationId: z.string(), + newAvatarUrl: z.string().optional(), +}); + +export type GroupAvatarJobData = z.infer; + +export class GroupAvatarJobQueue extends JobQueue { + protected parseData(data: unknown): GroupAvatarJobData { + return groupAvatarJobDataSchema.parse(data); + } + + protected async run( + { data }: Readonly<{ data: GroupAvatarJobData; timestamp: number }>, + { attempt, log }: Readonly<{ attempt: number; log: LoggerType }> + ): Promise { + const { conversationId, newAvatarUrl } = data; + const logId = `groupAvatarJobQueue(${conversationId}, attempt=${attempt})`; + + const convo = window.ConversationController.get(conversationId); + if (!convo) { + log.warn(`${logId}: dropping ${conversationId}, not found`); + return undefined; + } + + const { attributes } = convo; + if (!isGroupV2(attributes)) { + log.warn(`${logId}: dropping ${conversationId}, not a group`); + return undefined; + } + + // Generate correct attributes patch + const patch = await applyNewAvatar(newAvatarUrl, attributes, logId); + + convo.set(patch); + await Data.updateConversation(convo.attributes); + + return undefined; + } +} + +export const groupAvatarJobQueue = new GroupAvatarJobQueue({ + store: jobQueueDatabaseStore, + queueType: 'groupAvatar', + maxAttempts: 25, +}); diff --git a/ts/jobs/initializeAllJobQueues.ts b/ts/jobs/initializeAllJobQueues.ts index de3cf907e3..9ce92c28b5 100644 --- a/ts/jobs/initializeAllJobQueues.ts +++ b/ts/jobs/initializeAllJobQueues.ts @@ -5,6 +5,7 @@ import type { WebAPIType } from '../textsecure/WebAPI'; import { drop } from '../util/drop'; import { conversationJobQueue } from './conversationJobQueue'; +import { groupAvatarJobQueue } from './groupAvatarJobQueue'; import { readSyncJobQueue } from './readSyncJobQueue'; import { removeStorageKeyJobQueue } from './removeStorageKeyJobQueue'; import { reportSpamJobQueue } from './reportSpamJobQueue'; @@ -25,6 +26,9 @@ export function initializeAllJobQueues({ // General conversation send queue drop(conversationJobQueue.streamJobs()); + // Group avatar download after backup import + drop(groupAvatarJobQueue.streamJobs()); + // Single proto send queue, used for a variety of one-off simple messages drop(singleProtoJobQueue.streamJobs()); @@ -41,6 +45,7 @@ export function initializeAllJobQueues({ export async function shutdownAllJobQueues(): Promise { await Promise.allSettled([ conversationJobQueue.shutdown(), + groupAvatarJobQueue.shutdown(), singleProtoJobQueue.shutdown(), readSyncJobQueue.shutdown(), viewSyncJobQueue.shutdown(), diff --git a/ts/model-types.d.ts b/ts/model-types.d.ts index 723ebf5df3..966a87bc54 100644 --- a/ts/model-types.d.ts +++ b/ts/model-types.d.ts @@ -509,7 +509,7 @@ export type LegacyMigrationPendingMemberType = { }; export type GroupV2PendingMemberType = { - addedByUserId?: AciString; + addedByUserId: AciString; serviceId: ServiceIdString; timestamp: number; role: MemberRoleEnum; diff --git a/ts/services/backups/export.ts b/ts/services/backups/export.ts index 1cc8f32d5c..74faf15a02 100644 --- a/ts/services/backups/export.ts +++ b/ts/services/backups/export.ts @@ -76,6 +76,7 @@ import { import * as Bytes from '../../Bytes'; import { canBeSynced as canPreferredReactionEmojiBeSynced } from '../../reactions/preferredReactionEmoji'; import { SendStatus } from '../../messages/MessageSendState'; +import { deriveGroupFields } from '../../groups'; import { BACKUP_VERSION } from './constants'; import { getMessageIdForLogging } from '../../util/idForLogging'; import { getCallsHistoryForRedux } from '../callHistoryLoader'; @@ -112,6 +113,8 @@ const FLUSH_TIMEOUT = 30 * MINUTE; // Threshold for reporting slow flushes const REPORTING_THRESHOLD = SECOND; +const ZERO_PROFILE_KEY = new Uint8Array(32); + type GetRecipientIdOptionsType = | Readonly<{ serviceId: ServiceIdString; @@ -672,11 +675,92 @@ export class BackupExportStream extends Readable { break; } + const masterKey = Bytes.fromBase64(convo.masterKey); + + let publicKey; + if (convo.publicParams) { + publicKey = Bytes.fromBase64(convo.publicParams); + } else { + ({ publicParams: publicKey } = deriveGroupFields(masterKey)); + } + res.group = { - masterKey: Bytes.fromBase64(convo.masterKey), + masterKey, whitelisted: convo.profileSharing, hideStory: convo.hideStory === true, storySendMode, + snapshot: { + publicKey, + title: { + title: convo.name ?? '', + }, + description: { + descriptionText: convo.description ?? '', + }, + avatarUrl: convo.avatar?.url, + disappearingMessagesTimer: + convo.expireTimer != null + ? { + disappearingMessagesDuration: DurationInSeconds.toSeconds( + convo.expireTimer + ), + } + : null, + accessControl: convo.accessControl, + version: convo.revision || 0, + members: convo.membersV2?.map(member => { + const memberConvo = window.ConversationController.get(member.aci); + strictAssert(memberConvo, 'Missing GV2 member'); + + const { profileKey } = memberConvo.attributes; + + return { + userId: this.aciToBytes(member.aci), + role: member.role, + profileKey: profileKey + ? Bytes.fromBase64(profileKey) + : ZERO_PROFILE_KEY, + joinedAtVersion: member.joinedAtVersion, + }; + }), + membersPendingProfileKey: convo.pendingMembersV2?.map(member => { + return { + member: { + userId: this.serviceIdToBytes(member.serviceId), + role: member.role, + profileKey: ZERO_PROFILE_KEY, + joinedAtVersion: 0, + }, + addedByUserId: this.aciToBytes(member.addedByUserId), + timestamp: getSafeLongFromTimestamp(member.timestamp), + }; + }), + membersPendingAdminApproval: convo.pendingAdminApprovalV2?.map( + member => { + const memberConvo = window.ConversationController.get(member.aci); + strictAssert(memberConvo, 'Missing GV2 member pending approval'); + + const { profileKey } = memberConvo.attributes; + return { + userId: this.aciToBytes(member.aci), + profileKey: profileKey + ? Bytes.fromBase64(profileKey) + : ZERO_PROFILE_KEY, + timestamp: getSafeLongFromTimestamp(member.timestamp), + }; + } + ), + membersBanned: convo.bannedMembersV2?.map(member => { + return { + userId: this.serviceIdToBytes(member.serviceId), + timestamp: getSafeLongFromTimestamp(member.timestamp), + }; + }), + inviteLinkPassword: convo.groupInviteLinkPassword + ? Bytes.fromBase64(convo.groupInviteLinkPassword) + : null, + announcementsOnly: convo.announcementsOnly === true, + }, }; } else { return undefined; diff --git a/ts/services/backups/import.ts b/ts/services/backups/import.ts index 72b6553353..9e456a1839 100644 --- a/ts/services/backups/import.ts +++ b/ts/services/backups/import.ts @@ -1,7 +1,7 @@ // Copyright 2023 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only -import { Aci, Pni } from '@signalapp/libsignal-client'; +import { Aci, Pni, ServiceId } from '@signalapp/libsignal-client'; import { ReceiptCredentialPresentation } from '@signalapp/libsignal-client/zkgroup'; import { v4 as generateUuid } from 'uuid'; import pMap from 'p-map'; @@ -15,7 +15,11 @@ import * as log from '../../logging/log'; import { GiftBadgeStates } from '../../components/conversation/Message'; import { StorySendMode } from '../../types/Stories'; import type { ServiceIdString, AciString } from '../../types/ServiceId'; -import { fromAciObject, fromPniObject } from '../../types/ServiceId'; +import { + fromAciObject, + fromPniObject, + fromServiceIdObject, +} from '../../types/ServiceId'; import { isStoryDistributionId } from '../../types/StoryDistributionId'; import * as Errors from '../../types/errors'; import { PaymentEventKind } from '../../types/Payment'; @@ -63,7 +67,7 @@ import type { GroupV2ChangeDetailType } from '../../groups'; import { queueAttachmentDownloads } from '../../util/queueAttachmentDownloads'; import { drop } from '../../util/drop'; import { isNotNil } from '../../util/isNotNil'; -import { isGroup } from '../../util/whatTypeOfConversation'; +import { isGroup, isGroupV2 } from '../../util/whatTypeOfConversation'; import { convertBackupMessageAttachmentToAttachment, convertFilePointerToAttachment, @@ -71,6 +75,7 @@ import { import { filterAndClean } from '../../types/BodyRange'; import { APPLICATION_OCTET_STREAM, stringToMIMEType } from '../../types/MIME'; import { copyFromQuotedMessage } from '../../messages/copyQuote'; +import { groupAvatarJobQueue } from '../../jobs/groupAvatarJobQueue'; const MAX_CONCURRENCY = 10; @@ -306,16 +311,28 @@ export class BackupImportStream extends Writable { window.storage.reset(); await window.storage.fetch(); + const allConversations = window.ConversationController.getAll(); + // Update last message in every active conversation now that we have // them loaded into memory. await pMap( - window.ConversationController.getAll().filter(convo => { + allConversations.filter(convo => { return convo.get('active_at') || convo.get('isPinned'); }), convo => convo.updateLastMessage(), { concurrency: MAX_CONCURRENCY } ); + // Schedule group avatar download. + await pMap( + allConversations.filter(({ attributes: convo }) => { + const { avatar } = convo; + return isGroupV2(convo) && avatar?.url && !avatar.path; + }), + convo => groupAvatarJobQueue.add({ conversationId: convo.id }), + { concurrency: MAX_CONCURRENCY } + ); + await window.storage.put( 'pinnedConversationIds', this.pinnedConversations @@ -675,30 +692,153 @@ export class BackupImportStream extends Writable { private async fromGroup( group: Backups.IGroup ): Promise { - strictAssert(group.masterKey != null, 'fromGroup: missing masterKey'); + const { masterKey, snapshot } = group; + strictAssert(masterKey != null, 'fromGroup: missing masterKey'); + strictAssert(snapshot != null, 'fromGroup: missing snapshot'); - const secretParams = deriveGroupSecretParams(group.masterKey); + const secretParams = deriveGroupSecretParams(masterKey); const publicParams = deriveGroupPublicParams(secretParams); const groupId = Bytes.toBase64(deriveGroupID(secretParams)); + const { + title, + description, + avatarUrl, + disappearingMessagesTimer, + accessControl, + version, + members, + membersPendingProfileKey, + membersPendingAdminApproval, + membersBanned, + inviteLinkPassword, + announcementsOnly, + } = snapshot; + + const expirationTimerS = + disappearingMessagesTimer?.disappearingMessagesDuration; + + let storySendMode: StorySendMode | undefined; + switch (group.storySendMode) { + case Backups.Group.StorySendMode.ENABLED: + storySendMode = StorySendMode.Always; + break; + case Backups.Group.StorySendMode.DISABLED: + storySendMode = StorySendMode.Never; + break; + default: + storySendMode = undefined; + break; + } const attrs: ConversationAttributesType = { id: generateUuid(), type: 'group', version: 2, groupVersion: 2, - masterKey: Bytes.toBase64(group.masterKey), + masterKey: Bytes.toBase64(masterKey), groupId, secretParams: Bytes.toBase64(secretParams), publicParams: Bytes.toBase64(publicParams), profileSharing: group.whitelisted === true, hideStory: group.hideStory === true, - }; + storySendMode, - if (group.storySendMode === Backups.Group.StorySendMode.ENABLED) { - attrs.storySendMode = StorySendMode.Always; - } else if (group.storySendMode === Backups.Group.StorySendMode.DISABLED) { - attrs.storySendMode = StorySendMode.Never; - } + // Snapshot + name: dropNull(title?.title), + description: dropNull(description?.descriptionText), + avatar: avatarUrl + ? { + url: avatarUrl, + path: '', + } + : undefined, + expireTimer: expirationTimerS + ? DurationInSeconds.fromSeconds(expirationTimerS) + : undefined, + accessControl: accessControl + ? { + attributes: + dropNull(accessControl.attributes) ?? + SignalService.AccessControl.AccessRequired.UNKNOWN, + members: + dropNull(accessControl.members) ?? + SignalService.AccessControl.AccessRequired.UNKNOWN, + addFromInviteLink: + dropNull(accessControl.addFromInviteLink) ?? + SignalService.AccessControl.AccessRequired.UNKNOWN, + } + : undefined, + membersV2: members?.map(({ userId, role, joinedAtVersion }) => { + strictAssert(Bytes.isNotEmpty(userId), 'Empty gv2 member userId'); + + // Note that we deliberately ignore profile key since it has to be + // in the Contact frame + + return { + aci: fromAciObject(Aci.fromUuidBytes(userId)), + role: dropNull(role) ?? SignalService.Member.Role.UNKNOWN, + joinedAtVersion: dropNull(joinedAtVersion) ?? 0, + }; + }), + pendingMembersV2: membersPendingProfileKey?.map( + ({ member, addedByUserId, timestamp }) => { + strictAssert(member != null, 'Missing gv2 pending member'); + strictAssert( + Bytes.isNotEmpty(addedByUserId), + 'Empty gv2 pending member addedByUserId' + ); + + // profileKey is not available for pending members. + const { userId, role } = member; + + strictAssert(Bytes.isNotEmpty(userId), 'Empty gv2 member userId'); + + const serviceId = fromServiceIdObject( + ServiceId.parseFromServiceIdBinary(Buffer.from(userId)) + ); + + return { + serviceId, + role: dropNull(role) ?? SignalService.Member.Role.UNKNOWN, + addedByUserId: fromAciObject(Aci.fromUuidBytes(addedByUserId)), + timestamp: timestamp != null ? getTimestampFromLong(timestamp) : 0, + }; + } + ), + pendingAdminApprovalV2: membersPendingAdminApproval?.map( + ({ userId, timestamp }) => { + strictAssert(Bytes.isNotEmpty(userId), 'Empty gv2 member userId'); + + // Note that we deliberately ignore profile key since it has to be + // in the Contact frame + + return { + aci: fromAciObject(Aci.fromUuidBytes(userId)), + timestamp: timestamp != null ? getTimestampFromLong(timestamp) : 0, + }; + } + ), + bannedMembersV2: membersBanned?.map(({ userId, timestamp }) => { + strictAssert(Bytes.isNotEmpty(userId), 'Empty gv2 member userId'); + + // Note that we deliberately ignore profile key since it has to be + // in the Contact frame + + const serviceId = fromServiceIdObject( + ServiceId.parseFromServiceIdBinary(Buffer.from(userId)) + ); + + return { + serviceId, + timestamp: timestamp != null ? getTimestampFromLong(timestamp) : 0, + }; + }), + revision: dropNull(version), + groupInviteLinkPassword: Bytes.isNotEmpty(inviteLinkPassword) + ? Bytes.toBase64(inviteLinkPassword) + : undefined, + announcementsOnly: dropNull(announcementsOnly), + }; return attrs; } diff --git a/ts/types/Avatar.ts b/ts/types/Avatar.ts index 9d5ae58dca..ca048c5222 100644 --- a/ts/types/Avatar.ts +++ b/ts/types/Avatar.ts @@ -34,11 +34,19 @@ export const GroupAvatarIcons = [ 'surfboard', ] as const; -export type ContactAvatarType = { - path: string; - url?: string; - hash?: string; -}; +export type ContactAvatarType = + | { + // Downloaded avatar + path: string; + url?: string; + hash?: string; + } + | { + // Not-yet downloaded avatar + path?: string; + url: string; + hash?: string; + }; type GroupAvatarIconType = typeof GroupAvatarIcons[number]; diff --git a/ts/types/Conversation.ts b/ts/types/Conversation.ts index 5aeb387229..e49e646cbe 100644 --- a/ts/types/Conversation.ts +++ b/ts/types/Conversation.ts @@ -57,7 +57,7 @@ function buildAvatarUpdater({ field }: { field: 'avatar' | 'profileAvatar' }) { } const { hash, path } = oldAvatar; - const exists = await doesAttachmentExist(path); + const exists = path && (await doesAttachmentExist(path)); if (!exists) { window.SignalContext.log.warn( `Conversation.buildAvatarUpdater: attachment ${path} did not exist` @@ -66,7 +66,9 @@ function buildAvatarUpdater({ field }: { field: 'avatar' | 'profileAvatar' }) { if (exists) { if (newAvatar && hash && hash === newAvatar.hash) { - await deleteAttachmentData(newAvatar.path); + if (newAvatar.path) { + await deleteAttachmentData(newAvatar.path); + } return conversation; } if (data && hash && hash === newHash) { @@ -74,7 +76,9 @@ function buildAvatarUpdater({ field }: { field: 'avatar' | 'profileAvatar' }) { } } - await deleteAttachmentData(path); + if (path) { + await deleteAttachmentData(path); + } if (newAvatar) { return {