From 301f7a505ad0a4a8a1afcc1040e6a63fa87256df Mon Sep 17 00:00:00 2001 From: trevor-signal <131492920+trevor-signal@users.noreply.github.com> Date: Mon, 19 Aug 2024 20:47:02 -0400 Subject: [PATCH] Update backup import/export with new SendStatus, FilePointer, and GroupSnapshot updates --- ACKNOWLEDGMENTS.md | 67 ++++++++++++- package-lock.json | 8 +- package.json | 2 +- protos/Backups.proto | 64 +++++++++---- ts/services/backups/export.ts | 92 +++++++++--------- ts/services/backups/import.ts | 100 +++++++++++--------- ts/services/backups/util/filePointers.ts | 4 +- ts/test-electron/backup/bubble_test.ts | 17 +++- ts/test-electron/backup/filePointer_test.ts | 4 +- 9 files changed, 239 insertions(+), 119 deletions(-) diff --git a/ACKNOWLEDGMENTS.md b/ACKNOWLEDGMENTS.md index 61dae7112223..65f61dfd8cfb 100644 --- a/ACKNOWLEDGMENTS.md +++ b/ACKNOWLEDGMENTS.md @@ -4733,7 +4733,7 @@ For more information on this, and how to apply and follow the GNU AGPL, see ``` -## attest 0.1.0, libsignal-ffi 0.55.0, libsignal-jni 0.55.0, libsignal-jni-testing 0.55.0, libsignal-node 0.55.0, signal-neon-futures 0.1.0, signal-neon-futures-tests 0.1.0, libsignal-bridge 0.1.0, libsignal-bridge-macros 0.1.0, libsignal-bridge-testing 0.1.0, libsignal-bridge-types 0.1.0, libsignal-core 0.1.0, signal-crypto 0.1.0, device-transfer 0.1.0, signal-media 0.1.0, libsignal-message-backup 0.1.0, libsignal-message-backup-macros 0.1.0, libsignal-net 0.1.0, signal-pin 0.1.0, poksho 0.7.0, libsignal-protocol 0.1.0, libsignal-svr3 0.1.0, usernames 0.1.0, zkcredential 0.1.0, zkgroup 0.9.0 +## attest 0.1.0, libsignal-ffi 0.55.1, libsignal-jni 0.55.1, libsignal-jni-testing 0.55.1, libsignal-node 0.55.1, signal-neon-futures 0.1.0, signal-neon-futures-tests 0.1.0, libsignal-bridge 0.1.0, libsignal-bridge-macros 0.1.0, libsignal-bridge-testing 0.1.0, libsignal-bridge-types 0.1.0, libsignal-core 0.1.0, signal-crypto 0.1.0, device-transfer 0.1.0, libsignal-keytrans 0.0.1, signal-media 0.1.0, libsignal-message-backup 0.1.0, libsignal-message-backup-macros 0.1.0, libsignal-net 0.1.0, signal-pin 0.1.0, poksho 0.7.0, libsignal-protocol 0.1.0, libsignal-svr3 0.1.0, usernames 0.1.0, zkcredential 0.1.0, zkgroup 0.9.0 ``` GNU AFFERO GENERAL PUBLIC LICENSE @@ -6176,6 +6176,40 @@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ``` +## ed25519-dalek 2.1.1 + +``` +Copyright (c) 2017-2019 isis agora lovecruft. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + ## x25519-dalek 2.0.1 ``` @@ -8574,6 +8608,37 @@ DEALINGS IN THE SOFTWARE. ``` +## ed25519 2.2.3, signature 2.2.0 + +``` +Copyright (c) 2018-2023 RustCrypto Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +``` + ## try-lock 0.2.5 ``` diff --git a/package-lock.json b/package-lock.json index d19737fb215b..9567493087f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "@react-aria/utils": "3.16.0", "@react-spring/web": "9.5.5", "@signalapp/better-sqlite3": "8.7.1", - "@signalapp/libsignal-client": "0.55.0", + "@signalapp/libsignal-client": "0.55.1", "@signalapp/ringrtc": "2.46.1", "@signalapp/windows-dummy-keystroke": "1.0.0", "@types/fabric": "4.5.3", @@ -7223,9 +7223,9 @@ } }, "node_modules/@signalapp/libsignal-client": { - "version": "0.55.0", - "resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.55.0.tgz", - "integrity": "sha512-SwsBLUlHsUU6ae6cWsvok2maaRqIoACi0LgW0uGtTrNDeSUdja3j2Dnt/M5InQ+LaU5DQg8xMyUq8KKRp2RmpQ==", + "version": "0.55.1", + "resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.55.1.tgz", + "integrity": "sha512-qa2sztxNy5QyXYg9Z8xH9zdYikwNORyWr/95HnLAdzf4YFGsee/8JS74L+2kAn55lE7CVD+EVpgXJYFFw2Gu/w==", "hasInstallScript": true, "dependencies": { "node-gyp-build": "^4.2.3", diff --git a/package.json b/package.json index 99d747026d7c..77b3c35a2551 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,7 @@ "@react-aria/utils": "3.16.0", "@react-spring/web": "9.5.5", "@signalapp/better-sqlite3": "8.7.1", - "@signalapp/libsignal-client": "0.55.0", + "@signalapp/libsignal-client": "0.55.1", "@signalapp/ringrtc": "2.46.1", "@signalapp/windows-dummy-keystroke": "1.0.0", "@types/fabric": "4.5.3", diff --git a/protos/Backups.proto b/protos/Backups.proto index 6093a585e307..a3972ad98c3d 100644 --- a/protos/Backups.proto +++ b/protos/Backups.proto @@ -1,6 +1,5 @@ // Copyright 2024 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only - syntax = "proto3"; package signalbackups; @@ -160,7 +159,7 @@ message Group { // We would use Groups.proto if we could, but we want a plaintext version to improve export readability. // For documentation, defer to Groups.proto. The only name change is Group -> GroupSnapshot to avoid the naming conflict. message GroupSnapshot { - bytes publicKey = 1; + reserved /*publicKey*/ 1; // The field is deprecated in the context of static group state GroupAttributeBlob title = 2; GroupAttributeBlob description = 11; string avatarUrl = 3; @@ -346,23 +345,49 @@ message ChatItem { } message SendStatus { - enum Status { - UNKNOWN = 0; - FAILED = 1; - PENDING = 2; - SENT = 3; - DELIVERED = 4; - READ = 5; - VIEWED = 6; - SKIPPED = 7; // e.g. user in group was blocked, so we skipped sending to them + message Pending {} + + message Sent { + bool sealedSender = 1; + } + + message Delivered { + bool sealedSender = 1; + } + + message Read { + bool sealedSender = 1; + } + + message Viewed { + bool sealedSender = 1; + } + + // e.g. user in group was blocked, so we skipped sending to them + message Skipped {} + + message Failed { + enum FailureReason { + UNKNOWN = 0; // A valid value -- could indicate a crash or lack of information + NETWORK = 1; + IDENTITY_KEY_MISMATCH = 2; + } + + FailureReason reason = 1; } uint64 recipientId = 1; - Status deliveryStatus = 2; - bool networkFailure = 3; - bool identityKeyMismatch = 4; - bool sealedSender = 5; - uint64 lastStatusUpdateTimestamp = 6; // the time the status was last updated -- if from a receipt, it should be the sentTime of the receipt + uint64 timestamp = 2; // the time the status was last updated -- if from a receipt, it should be the sentTime of the receipt + + oneof deliveryStatus { + Pending pending = 3; + Sent sent = 4; + Delivered delivered = 5; + Read read = 6; + Viewed viewed = 7; + Skipped skipped = 8; + Failed failed = 9; + } } message Text { @@ -568,7 +593,7 @@ message FilePointer { optional uint32 cdnNumber = 2; bytes key = 3; bytes digest = 4; - uint64 size = 5; + uint32 size = 5; // Fallback in case backup tier upload failed. optional string transitCdnKey = 6; optional uint32 transitCdnNumber = 7; @@ -655,8 +680,11 @@ message Reaction { string emoji = 1; uint64 authorId = 2; uint64 sentTimestamp = 3; + // Optional because some clients may not track this data optional uint64 receivedTimestamp = 4; - uint64 sortOrder = 5; // A higher sort order means that a reaction is more recent + // A higher sort order means that a reaction is more recent. Some clients may export this as + // incrementing numbers (e.g. 1, 2, 3), others as timestamps. + uint64 sortOrder = 5; } message ChatUpdateMessage { diff --git a/ts/services/backups/export.ts b/ts/services/backups/export.ts index 638f72ec6520..78d8d54fe827 100644 --- a/ts/services/backups/export.ts +++ b/ts/services/backups/export.ts @@ -86,7 +86,6 @@ import { import * as Bytes from '../../Bytes'; import { canBeSynced as canPreferredReactionEmojiBeSynced } from '../../reactions/preferredReactionEmoji'; import { SendStatus } from '../../messages/MessageSendState'; -import { deriveGroupFields } from '../../groups'; import { BACKUP_VERSION } from './constants'; import { getMessageIdForLogging } from '../../util/idForLogging'; import { getCallsHistoryForRedux } from '../callHistoryLoader'; @@ -811,20 +810,12 @@ export class BackupExportStream extends Readable { const masterKey = Bytes.fromBase64(convo.masterKey); - let publicKey; - if (convo.publicParams) { - publicKey = Bytes.fromBase64(convo.publicParams); - } else { - ({ publicParams: publicKey } = deriveGroupFields(masterKey)); - } - res.group = { masterKey, whitelisted: convo.profileSharing, hideStory: convo.hideStory === true, storySendMode, snapshot: { - publicKey, title: { title: convo.name ?? '', }, @@ -2204,8 +2195,6 @@ export class BackupExportStream extends Readable { 'sendStateByConversationId' | 'unidentifiedDeliveries' | 'errors' > ): Backups.ChatItem.IOutgoingMessageDetails { - const BackupSendStatus = Backups.SendStatus.Status; - const sealedSenderServiceIds = new Set(unidentifiedDeliveries); const errorMap = new Map( errors.map(({ serviceId, name }) => { @@ -2213,65 +2202,78 @@ export class BackupExportStream extends Readable { }) ); - const sendStatus = new Array(); + const sendStatuses = new Array(); for (const [id, entry] of Object.entries(sendStateByConversationId)) { const target = window.ConversationController.get(id); if (!target) { log.warn(`backups: no send target for a message ${sentAt}`); continue; } + const { serviceId } = target.attributes; + const recipientId = this.getOrPushPrivateRecipient(target.attributes); + const timestamp = + entry.updatedAt != null + ? getSafeLongFromTimestamp(entry.updatedAt) + : null; + + const sendStatus = new Backups.SendStatus({ recipientId, timestamp }); + + const sealedSender = serviceId + ? sealedSenderServiceIds.has(serviceId) + : false; - let deliveryStatus: Backups.SendStatus.Status; switch (entry.status) { case SendStatus.Pending: - deliveryStatus = BackupSendStatus.PENDING; + sendStatus.pending = new Backups.SendStatus.Pending(); break; case SendStatus.Sent: - deliveryStatus = BackupSendStatus.SENT; + sendStatus.sent = new Backups.SendStatus.Sent({ + sealedSender, + }); break; case SendStatus.Delivered: - deliveryStatus = BackupSendStatus.DELIVERED; + sendStatus.delivered = new Backups.SendStatus.Delivered({ + sealedSender, + }); break; case SendStatus.Read: - deliveryStatus = BackupSendStatus.READ; + sendStatus.read = new Backups.SendStatus.Read({ + sealedSender, + }); break; case SendStatus.Viewed: - deliveryStatus = BackupSendStatus.VIEWED; + sendStatus.viewed = new Backups.SendStatus.Viewed({ + sealedSender, + }); break; - case SendStatus.Failed: - deliveryStatus = BackupSendStatus.FAILED; + case SendStatus.Failed: { + sendStatus.failed = new Backups.SendStatus.Failed(); + if (!serviceId) { + break; + } + const errorName = errorMap.get(serviceId); + if (!errorName) { + break; + } + + const identityKeyMismatch = errorName === 'OutgoingIdentityKeyError'; + if (identityKeyMismatch) { + sendStatus.failed.reason = + Backups.SendStatus.Failed.FailureReason.IDENTITY_KEY_MISMATCH; + } else { + sendStatus.failed.reason = + Backups.SendStatus.Failed.FailureReason.NETWORK; + } break; + } default: throw missingCaseError(entry.status); } - const { serviceId } = target.attributes; - let networkFailure = false; - let identityKeyMismatch = false; - let sealedSender = false; - if (serviceId) { - const errorName = errorMap.get(serviceId); - if (errorName !== undefined) { - identityKeyMismatch = errorName === 'OutgoingIdentityKeyError'; - networkFailure = !identityKeyMismatch; - } - sealedSender = sealedSenderServiceIds.has(serviceId); - } - - sendStatus.push({ - recipientId: this.getOrPushPrivateRecipient(target.attributes), - lastStatusUpdateTimestamp: - entry.updatedAt != null - ? getSafeLongFromTimestamp(entry.updatedAt) - : null, - deliveryStatus, - networkFailure, - identityKeyMismatch, - sealedSender, - }); + sendStatuses.push(sendStatus); } return { - sendStatus, + sendStatus: sendStatuses, }; } diff --git a/ts/services/backups/import.ts b/ts/services/backups/import.ts index c79ebd626489..55dd9d188d90 100644 --- a/ts/services/backups/import.ts +++ b/ts/services/backups/import.ts @@ -1278,8 +1278,6 @@ export class BackupImportStream extends Writable { if (outgoing) { const sendStateByConversationId: SendStateByConversationId = {}; - const BackupSendStatus = Backups.SendStatus.Status; - const unidentifiedDeliveries = new Array(); const errors = new Array(); for (const status of outgoing.sendStatus ?? []) { @@ -1295,57 +1293,71 @@ export class BackupImportStream extends Writable { 'status target conversation not found' ); - let sendStatus: SendStatus; - switch (status.deliveryStatus) { - case BackupSendStatus.PENDING: - sendStatus = SendStatus.Pending; - break; - case BackupSendStatus.SENT: - sendStatus = SendStatus.Sent; - break; - case BackupSendStatus.DELIVERED: - sendStatus = SendStatus.Delivered; - break; - case BackupSendStatus.READ: - sendStatus = SendStatus.Read; - break; - case BackupSendStatus.VIEWED: - sendStatus = SendStatus.Viewed; - break; - case BackupSendStatus.FAILED: - default: - sendStatus = SendStatus.Failed; - break; + // Desktop does not keep track of users we did not attempt to send to + if (status.skipped) { + continue; } + const { serviceId } = target; - if (target.serviceId) { - if (status.sealedSender) { - unidentifiedDeliveries.push(target.serviceId); + let sendStatus: SendStatus; + if (status.pending) { + sendStatus = SendStatus.Pending; + } else if (status.sent) { + sendStatus = SendStatus.Sent; + if (serviceId && status.sent.sealedSender) { + unidentifiedDeliveries.push(serviceId); } - - if (status.identityKeyMismatch) { - errors.push({ - serviceId: target.serviceId, - name: 'OutgoingIdentityKeyError', - // See: ts/textsecure/Errors - message: `The identity of ${target.serviceId} has changed.`, - }); - } else if (status.networkFailure) { - errors.push({ - serviceId: target.serviceId, - name: 'OutgoingMessageError', - // See: ts/textsecure/Errors - message: 'no http error', - }); + } else if (status.delivered) { + sendStatus = SendStatus.Delivered; + if (serviceId && status.delivered.sealedSender) { + unidentifiedDeliveries.push(serviceId); } + } else if (status.read) { + sendStatus = SendStatus.Read; + if (serviceId && status.read.sealedSender) { + unidentifiedDeliveries.push(serviceId); + } + } else if (status.viewed) { + sendStatus = SendStatus.Viewed; + if (serviceId && status.viewed.sealedSender) { + unidentifiedDeliveries.push(serviceId); + } + } else if (status.failed) { + sendStatus = SendStatus.Failed; + strictAssert( + status.failed.reason != null, + 'Failure reason must exist' + ); + switch (status.failed.reason) { + case Backups.SendStatus.Failed.FailureReason.IDENTITY_KEY_MISMATCH: + errors.push({ + serviceId, + name: 'OutgoingIdentityKeyError', + // See: ts/textsecure/Errors + message: `The identity of ${serviceId} has changed.`, + }); + break; + case Backups.SendStatus.Failed.FailureReason.NETWORK: + case Backups.SendStatus.Failed.FailureReason.UNKNOWN: + errors.push({ + serviceId, + name: 'OutgoingMessageError', + // See: ts/textsecure/Errors + message: 'no http error', + }); + break; + default: + throw missingCaseError(status.failed.reason); + } + } else { + throw new Error(`Unknown sendStatus received: ${status}`); } sendStateByConversationId[target.id] = { status: sendStatus, updatedAt: - status.lastStatusUpdateTimestamp != null && - !status.lastStatusUpdateTimestamp.isZero() - ? getTimestampFromLong(status.lastStatusUpdateTimestamp) + status.timestamp != null && !status.timestamp.isZero() + ? getTimestampFromLong(status.timestamp) : undefined, }; } diff --git a/ts/services/backups/util/filePointers.ts b/ts/services/backups/util/filePointers.ts index 97e883985252..a85787779117 100644 --- a/ts/services/backups/util/filePointers.ts +++ b/ts/services/backups/util/filePointers.ts @@ -105,7 +105,7 @@ export function convertFilePointerToAttachment( cdnNumber: transitCdnNumber ?? undefined, key: key?.length ? Bytes.toBase64(key) : undefined, digest: digest?.length ? Bytes.toBase64(digest) : undefined, - size: size?.toNumber() ?? 0, + size: size ?? 0, backupLocator: mediaName ? { mediaName, @@ -401,7 +401,7 @@ function getBackupLocator(attachment: AttachmentDownloadableFromBackupTier) { cdnNumber: attachment.backupLocator.cdnNumber, digest: Bytes.fromBase64(attachment.digest), key: Bytes.fromBase64(attachment.key), - size: Long.fromNumber(attachment.size), + size: attachment.size, transitCdnKey: attachment.cdnKey, transitCdnNumber: attachment.cdnNumber, }); diff --git a/ts/test-electron/backup/bubble_test.ts b/ts/test-electron/backup/bubble_test.ts index 94c7ca264cb9..103bfc2f37a9 100644 --- a/ts/test-electron/backup/bubble_test.ts +++ b/ts/test-electron/backup/bubble_test.ts @@ -24,6 +24,7 @@ import { } from './helpers'; const CONTACT_A = generateAci(); +const CONTACT_B = generateAci(); const GV1_ID = Bytes.toBinary(getRandomBytes(ID_V1_LENGTH)); const BADGE_RECEIPT = @@ -37,6 +38,7 @@ const BADGE_RECEIPT = describe('backup/bubble messages', () => { let contactA: ConversationModel; + let contactB: ConversationModel; let gv1: ConversationModel; beforeEach(async () => { @@ -51,6 +53,11 @@ describe('backup/bubble messages', () => { 'private', { systemGivenName: 'CONTACT_A' } ); + contactB = await window.ConversationController.getOrCreateAndWait( + CONTACT_B, + 'private', + { systemGivenName: 'CONTACT_B' } + ); gv1 = await window.ConversationController.getOrCreateAndWait( GV1_ID, @@ -346,12 +353,15 @@ describe('backup/bubble messages', () => { [contactA.id]: { status: SendStatus.Delivered, }, + [contactB.id]: { + status: SendStatus.Failed, + }, }, errors: [ { - serviceId: CONTACT_A, + serviceId: CONTACT_B, name: 'OutgoingIdentityKeyError', - message: `The identity of ${CONTACT_A} has changed.`, + message: `The identity of ${CONTACT_B} has changed.`, }, ], timestamp: 3, @@ -367,6 +377,9 @@ describe('backup/bubble messages', () => { sourceServiceId: OUR_ACI, sendStateByConversationId: { [contactA.id]: { + status: SendStatus.Failed, + }, + [contactB.id]: { status: SendStatus.Delivered, }, }, diff --git a/ts/test-electron/backup/filePointer_test.ts b/ts/test-electron/backup/filePointer_test.ts index 85158fc20ba9..2e8d8c8a7b3a 100644 --- a/ts/test-electron/backup/filePointer_test.ts +++ b/ts/test-electron/backup/filePointer_test.ts @@ -75,7 +75,7 @@ describe('convertFilePointerToAttachment', () => { backupLocator: new Backups.FilePointer.BackupLocator({ mediaName: 'mediaName', cdnNumber: 3, - size: Long.fromNumber(128), + size: 128, key: Bytes.fromString('key'), digest: Bytes.fromString('digest'), transitCdnKey: 'transitCdnKey', @@ -215,7 +215,7 @@ const defaultBackupLocator = new Backups.FilePointer.BackupLocator({ cdnNumber: null, key: Bytes.fromBase64('key'), digest: defaultDigest, - size: Long.fromNumber(100), + size: 100, transitCdnKey: 'cdnKey', transitCdnNumber: 2, });