Update backup import/export with new SendStatus, FilePointer, and GroupSnapshot updates

ACKNOWLEDGMENTS.md

@@ -4733,7 +4733,7 @@ For more information on this, and how to apply and follow the GNU AGPL, see
 
 ```
 
-## attest 0.1.0, libsignal-ffi 0.55.0, libsignal-jni 0.55.0, libsignal-jni-testing 0.55.0, libsignal-node 0.55.0, signal-neon-futures 0.1.0, signal-neon-futures-tests 0.1.0, libsignal-bridge 0.1.0, libsignal-bridge-macros 0.1.0, libsignal-bridge-testing 0.1.0, libsignal-bridge-types 0.1.0, libsignal-core 0.1.0, signal-crypto 0.1.0, device-transfer 0.1.0, signal-media 0.1.0, libsignal-message-backup 0.1.0, libsignal-message-backup-macros 0.1.0, libsignal-net 0.1.0, signal-pin 0.1.0, poksho 0.7.0, libsignal-protocol 0.1.0, libsignal-svr3 0.1.0, usernames 0.1.0, zkcredential 0.1.0, zkgroup 0.9.0
+## attest 0.1.0, libsignal-ffi 0.55.1, libsignal-jni 0.55.1, libsignal-jni-testing 0.55.1, libsignal-node 0.55.1, signal-neon-futures 0.1.0, signal-neon-futures-tests 0.1.0, libsignal-bridge 0.1.0, libsignal-bridge-macros 0.1.0, libsignal-bridge-testing 0.1.0, libsignal-bridge-types 0.1.0, libsignal-core 0.1.0, signal-crypto 0.1.0, device-transfer 0.1.0, libsignal-keytrans 0.0.1, signal-media 0.1.0, libsignal-message-backup 0.1.0, libsignal-message-backup-macros 0.1.0, libsignal-net 0.1.0, signal-pin 0.1.0, poksho 0.7.0, libsignal-protocol 0.1.0, libsignal-svr3 0.1.0, usernames 0.1.0, zkcredential 0.1.0, zkgroup 0.9.0
 
 ```
 GNU AFFERO GENERAL PUBLIC LICENSE
@@ -6176,6 +6176,40 @@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ```
 
+## ed25519-dalek 2.1.1
+
+```
+Copyright (c) 2017-2019 isis agora lovecruft. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+```
+
 ## x25519-dalek 2.0.1
 
 ```
@@ -8574,6 +8608,37 @@ DEALINGS IN THE SOFTWARE.
 
 ```
 
+## ed25519 2.2.3, signature 2.2.0
+
+```
+Copyright (c) 2018-2023 RustCrypto Developers
+
+Permission is hereby granted, free of charge, to any
+person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without
+limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software
+is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+```
+
 ## try-lock 0.2.5
 
 ```

package-lock.json

@@ -22,7 +22,7 @@
         "@react-aria/utils": "3.16.0",
         "@react-spring/web": "9.5.5",
         "@signalapp/better-sqlite3": "8.7.1",
-        "@signalapp/libsignal-client": "0.55.0",
+        "@signalapp/libsignal-client": "0.55.1",
         "@signalapp/ringrtc": "2.46.1",
         "@signalapp/windows-dummy-keystroke": "1.0.0",
         "@types/fabric": "4.5.3",
@@ -7223,9 +7223,9 @@
       }
     },
     "node_modules/@signalapp/libsignal-client": {
-      "version": "0.55.0",
+      "version": "0.55.1",
-      "resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.55.0.tgz",
+      "resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.55.1.tgz",
-      "integrity": "sha512-SwsBLUlHsUU6ae6cWsvok2maaRqIoACi0LgW0uGtTrNDeSUdja3j2Dnt/M5InQ+LaU5DQg8xMyUq8KKRp2RmpQ==",
+      "integrity": "sha512-qa2sztxNy5QyXYg9Z8xH9zdYikwNORyWr/95HnLAdzf4YFGsee/8JS74L+2kAn55lE7CVD+EVpgXJYFFw2Gu/w==",
       "hasInstallScript": true,
       "dependencies": {
         "node-gyp-build": "^4.2.3",

package.json

@@ -105,7 +105,7 @@
     "@react-aria/utils": "3.16.0",
     "@react-spring/web": "9.5.5",
     "@signalapp/better-sqlite3": "8.7.1",
-    "@signalapp/libsignal-client": "0.55.0",
+    "@signalapp/libsignal-client": "0.55.1",
     "@signalapp/ringrtc": "2.46.1",
     "@signalapp/windows-dummy-keystroke": "1.0.0",
     "@types/fabric": "4.5.3",

protos/Backups.proto

@@ -1,6 +1,5 @@
 // Copyright 2024 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
-
 syntax = "proto3";
 
 package signalbackups;
@@ -160,7 +159,7 @@ message Group {
   // We would use Groups.proto if we could, but we want a plaintext version to improve export readability.
   // For documentation, defer to Groups.proto. The only name change is Group -> GroupSnapshot to avoid the naming conflict.
   message GroupSnapshot {
-    bytes publicKey = 1;
+    reserved /*publicKey*/ 1;  // The field is deprecated in the context of static group state
     GroupAttributeBlob title = 2;
     GroupAttributeBlob description = 11;
     string avatarUrl = 3;
@@ -346,23 +345,49 @@ message ChatItem {
 }
 
 message SendStatus {
-  enum Status {
+  message Pending {}
-    UNKNOWN = 0;
+
-    FAILED = 1;
+  message Sent {
-    PENDING = 2;
+    bool sealedSender = 1;
-    SENT = 3;
+  }
-    DELIVERED = 4;
+
-    READ = 5;
+  message Delivered {
-    VIEWED = 6;
+    bool sealedSender = 1;
-    SKIPPED = 7; // e.g. user in group was blocked, so we skipped sending to them
+  }
+
+  message Read {
+    bool sealedSender = 1;
+  }
+
+  message Viewed {
+    bool sealedSender = 1;
+  }
+
+  // e.g. user in group was blocked, so we skipped sending to them
+  message Skipped {}
+
+  message Failed {
+    enum FailureReason {
+      UNKNOWN = 0; // A valid value -- could indicate a crash or lack of information
+      NETWORK = 1;
+      IDENTITY_KEY_MISMATCH = 2;
+    }
+
+    FailureReason reason = 1;
   }
 
   uint64 recipientId = 1;
-  Status deliveryStatus = 2;
+  uint64 timestamp = 2; // the time the status was last updated -- if from a receipt, it should be the sentTime of the receipt
-  bool networkFailure = 3;
+
-  bool identityKeyMismatch = 4;
+  oneof deliveryStatus {
-  bool sealedSender = 5;
+    Pending pending = 3;
-  uint64 lastStatusUpdateTimestamp = 6; // the time the status was last updated -- if from a receipt, it should be the sentTime of the receipt
+    Sent sent = 4;
+    Delivered delivered = 5;
+    Read read = 6;
+    Viewed viewed = 7;
+    Skipped skipped = 8;
+    Failed failed = 9;
+  }
 }
 
 message Text {
@@ -568,7 +593,7 @@ message FilePointer {
     optional uint32 cdnNumber = 2;
     bytes key = 3;
     bytes digest = 4;
-    uint64 size = 5;
+    uint32 size = 5;
     // Fallback in case backup tier upload failed.
     optional string transitCdnKey = 6;
     optional uint32 transitCdnNumber = 7;
@@ -655,8 +680,11 @@ message Reaction {
   string emoji = 1;
   uint64 authorId = 2;
   uint64 sentTimestamp = 3;
+  // Optional because some clients may not track this data
   optional uint64 receivedTimestamp = 4;
-  uint64 sortOrder = 5; // A higher sort order means that a reaction is more recent
+  // A higher sort order means that a reaction is more recent. Some clients may export this as
+  // incrementing numbers (e.g. 1, 2, 3), others as timestamps.
+  uint64 sortOrder = 5;
 }
 
 message ChatUpdateMessage {

ts/services/backups/export.ts

@@ -86,7 +86,6 @@ import {
 import * as Bytes from '../../Bytes';
 import { canBeSynced as canPreferredReactionEmojiBeSynced } from '../../reactions/preferredReactionEmoji';
 import { SendStatus } from '../../messages/MessageSendState';
-import { deriveGroupFields } from '../../groups';
 import { BACKUP_VERSION } from './constants';
 import { getMessageIdForLogging } from '../../util/idForLogging';
 import { getCallsHistoryForRedux } from '../callHistoryLoader';
@@ -811,20 +810,12 @@ export class BackupExportStream extends Readable {
 
       const masterKey = Bytes.fromBase64(convo.masterKey);
 
-      let publicKey;
-      if (convo.publicParams) {
-        publicKey = Bytes.fromBase64(convo.publicParams);
-      } else {
-        ({ publicParams: publicKey } = deriveGroupFields(masterKey));
-      }
-
       res.group = {
         masterKey,
         whitelisted: convo.profileSharing,
         hideStory: convo.hideStory === true,
         storySendMode,
         snapshot: {
-          publicKey,
           title: {
             title: convo.name ?? '',
           },
@@ -2204,8 +2195,6 @@ export class BackupExportStream extends Readable {
       'sendStateByConversationId' | 'unidentifiedDeliveries' | 'errors'
     >
   ): Backups.ChatItem.IOutgoingMessageDetails {
-    const BackupSendStatus = Backups.SendStatus.Status;
-
     const sealedSenderServiceIds = new Set(unidentifiedDeliveries);
     const errorMap = new Map(
       errors.map(({ serviceId, name }) => {
@@ -2213,65 +2202,78 @@ export class BackupExportStream extends Readable {
       })
     );
 
-    const sendStatus = new Array<Backups.ISendStatus>();
+    const sendStatuses = new Array<Backups.ISendStatus>();
     for (const [id, entry] of Object.entries(sendStateByConversationId)) {
       const target = window.ConversationController.get(id);
       if (!target) {
         log.warn(`backups: no send target for a message ${sentAt}`);
         continue;
       }
+      const { serviceId } = target.attributes;
+      const recipientId = this.getOrPushPrivateRecipient(target.attributes);
+      const timestamp =
+        entry.updatedAt != null
+          ? getSafeLongFromTimestamp(entry.updatedAt)
+          : null;
+
+      const sendStatus = new Backups.SendStatus({ recipientId, timestamp });
+
+      const sealedSender = serviceId
+        ? sealedSenderServiceIds.has(serviceId)
+        : false;
 
-      let deliveryStatus: Backups.SendStatus.Status;
       switch (entry.status) {
         case SendStatus.Pending:
-          deliveryStatus = BackupSendStatus.PENDING;
+          sendStatus.pending = new Backups.SendStatus.Pending();
           break;
         case SendStatus.Sent:
-          deliveryStatus = BackupSendStatus.SENT;
+          sendStatus.sent = new Backups.SendStatus.Sent({
+            sealedSender,
+          });
           break;
         case SendStatus.Delivered:
-          deliveryStatus = BackupSendStatus.DELIVERED;
+          sendStatus.delivered = new Backups.SendStatus.Delivered({
+            sealedSender,
+          });
           break;
         case SendStatus.Read:
-          deliveryStatus = BackupSendStatus.READ;
+          sendStatus.read = new Backups.SendStatus.Read({
+            sealedSender,
+          });
           break;
         case SendStatus.Viewed:
-          deliveryStatus = BackupSendStatus.VIEWED;
+          sendStatus.viewed = new Backups.SendStatus.Viewed({
+            sealedSender,
+          });
           break;
-        case SendStatus.Failed:
+        case SendStatus.Failed: {
-          deliveryStatus = BackupSendStatus.FAILED;
+          sendStatus.failed = new Backups.SendStatus.Failed();
+          if (!serviceId) {
             break;
+          }
+          const errorName = errorMap.get(serviceId);
+          if (!errorName) {
+            break;
+          }
+
+          const identityKeyMismatch = errorName === 'OutgoingIdentityKeyError';
+          if (identityKeyMismatch) {
+            sendStatus.failed.reason =
+              Backups.SendStatus.Failed.FailureReason.IDENTITY_KEY_MISMATCH;
+          } else {
+            sendStatus.failed.reason =
+              Backups.SendStatus.Failed.FailureReason.NETWORK;
+          }
+          break;
+        }
         default:
           throw missingCaseError(entry.status);
       }
 
-      const { serviceId } = target.attributes;
+      sendStatuses.push(sendStatus);
-      let networkFailure = false;
-      let identityKeyMismatch = false;
-      let sealedSender = false;
-      if (serviceId) {
-        const errorName = errorMap.get(serviceId);
-        if (errorName !== undefined) {
-          identityKeyMismatch = errorName === 'OutgoingIdentityKeyError';
-          networkFailure = !identityKeyMismatch;
-        }
-        sealedSender = sealedSenderServiceIds.has(serviceId);
-      }
-
-      sendStatus.push({
-        recipientId: this.getOrPushPrivateRecipient(target.attributes),
-        lastStatusUpdateTimestamp:
-          entry.updatedAt != null
-            ? getSafeLongFromTimestamp(entry.updatedAt)
-            : null,
-        deliveryStatus,
-        networkFailure,
-        identityKeyMismatch,
-        sealedSender,
-      });
     }
     return {
-      sendStatus,
+      sendStatus: sendStatuses,
     };
   }
 

ts/services/backups/import.ts

@@ -1278,8 +1278,6 @@ export class BackupImportStream extends Writable {
     if (outgoing) {
       const sendStateByConversationId: SendStateByConversationId = {};
 
-      const BackupSendStatus = Backups.SendStatus.Status;
-
       const unidentifiedDeliveries = new Array<ServiceIdString>();
       const errors = new Array<CustomError>();
       for (const status of outgoing.sendStatus ?? []) {
@@ -1295,57 +1293,71 @@ export class BackupImportStream extends Writable {
           'status target conversation not found'
         );
 
+        // Desktop does not keep track of users we did not attempt to send to
+        if (status.skipped) {
+          continue;
+        }
+        const { serviceId } = target;
+
         let sendStatus: SendStatus;
-        switch (status.deliveryStatus) {
+        if (status.pending) {
-          case BackupSendStatus.PENDING:
           sendStatus = SendStatus.Pending;
-            break;
+        } else if (status.sent) {
-          case BackupSendStatus.SENT:
           sendStatus = SendStatus.Sent;
-            break;
+          if (serviceId && status.sent.sealedSender) {
-          case BackupSendStatus.DELIVERED:
+            unidentifiedDeliveries.push(serviceId);
+          }
+        } else if (status.delivered) {
           sendStatus = SendStatus.Delivered;
-            break;
+          if (serviceId && status.delivered.sealedSender) {
-          case BackupSendStatus.READ:
+            unidentifiedDeliveries.push(serviceId);
+          }
+        } else if (status.read) {
           sendStatus = SendStatus.Read;
-            break;
+          if (serviceId && status.read.sealedSender) {
-          case BackupSendStatus.VIEWED:
+            unidentifiedDeliveries.push(serviceId);
+          }
+        } else if (status.viewed) {
           sendStatus = SendStatus.Viewed;
-            break;
+          if (serviceId && status.viewed.sealedSender) {
-          case BackupSendStatus.FAILED:
+            unidentifiedDeliveries.push(serviceId);
-          default:
+          }
+        } else if (status.failed) {
           sendStatus = SendStatus.Failed;
-            break;
+          strictAssert(
-        }
+            status.failed.reason != null,
-
+            'Failure reason must exist'
-        if (target.serviceId) {
+          );
-          if (status.sealedSender) {
+          switch (status.failed.reason) {
-            unidentifiedDeliveries.push(target.serviceId);
+            case Backups.SendStatus.Failed.FailureReason.IDENTITY_KEY_MISMATCH:
-          }
-
-          if (status.identityKeyMismatch) {
               errors.push({
-              serviceId: target.serviceId,
+                serviceId,
                 name: 'OutgoingIdentityKeyError',
                 // See: ts/textsecure/Errors
-              message: `The identity of ${target.serviceId} has changed.`,
+                message: `The identity of ${serviceId} has changed.`,
               });
-          } else if (status.networkFailure) {
+              break;
+            case Backups.SendStatus.Failed.FailureReason.NETWORK:
+            case Backups.SendStatus.Failed.FailureReason.UNKNOWN:
               errors.push({
-              serviceId: target.serviceId,
+                serviceId,
                 name: 'OutgoingMessageError',
                 // See: ts/textsecure/Errors
                 message: 'no http error',
               });
+              break;
+            default:
+              throw missingCaseError(status.failed.reason);
           }
+        } else {
+          throw new Error(`Unknown sendStatus received: ${status}`);
         }
 
         sendStateByConversationId[target.id] = {
           status: sendStatus,
           updatedAt:
-            status.lastStatusUpdateTimestamp != null &&
+            status.timestamp != null && !status.timestamp.isZero()
-            !status.lastStatusUpdateTimestamp.isZero()
+              ? getTimestampFromLong(status.timestamp)
-              ? getTimestampFromLong(status.lastStatusUpdateTimestamp)
               : undefined,
         };
       }

ts/services/backups/util/filePointers.ts

@@ -105,7 +105,7 @@ export function convertFilePointerToAttachment(
       cdnNumber: transitCdnNumber ?? undefined,
       key: key?.length ? Bytes.toBase64(key) : undefined,
       digest: digest?.length ? Bytes.toBase64(digest) : undefined,
-      size: size?.toNumber() ?? 0,
+      size: size ?? 0,
       backupLocator: mediaName
         ? {
             mediaName,
@@ -401,7 +401,7 @@ function getBackupLocator(attachment: AttachmentDownloadableFromBackupTier) {
     cdnNumber: attachment.backupLocator.cdnNumber,
     digest: Bytes.fromBase64(attachment.digest),
     key: Bytes.fromBase64(attachment.key),
-    size: Long.fromNumber(attachment.size),
+    size: attachment.size,
     transitCdnKey: attachment.cdnKey,
     transitCdnNumber: attachment.cdnNumber,
   });

ts/test-electron/backup/bubble_test.ts

@@ -24,6 +24,7 @@ import {
 } from './helpers';
 
 const CONTACT_A = generateAci();
+const CONTACT_B = generateAci();
 const GV1_ID = Bytes.toBinary(getRandomBytes(ID_V1_LENGTH));
 
 const BADGE_RECEIPT =
@@ -37,6 +38,7 @@ const BADGE_RECEIPT =
 
 describe('backup/bubble messages', () => {
   let contactA: ConversationModel;
+  let contactB: ConversationModel;
   let gv1: ConversationModel;
 
   beforeEach(async () => {
@@ -51,6 +53,11 @@ describe('backup/bubble messages', () => {
       'private',
       { systemGivenName: 'CONTACT_A' }
     );
+    contactB = await window.ConversationController.getOrCreateAndWait(
+      CONTACT_B,
+      'private',
+      { systemGivenName: 'CONTACT_B' }
+    );
 
     gv1 = await window.ConversationController.getOrCreateAndWait(
       GV1_ID,
@@ -346,12 +353,15 @@ describe('backup/bubble messages', () => {
           [contactA.id]: {
             status: SendStatus.Delivered,
           },
+          [contactB.id]: {
+            status: SendStatus.Failed,
+          },
         },
         errors: [
           {
-            serviceId: CONTACT_A,
+            serviceId: CONTACT_B,
             name: 'OutgoingIdentityKeyError',
-            message: `The identity of ${CONTACT_A} has changed.`,
+            message: `The identity of ${CONTACT_B} has changed.`,
           },
         ],
         timestamp: 3,
@@ -367,6 +377,9 @@ describe('backup/bubble messages', () => {
         sourceServiceId: OUR_ACI,
         sendStateByConversationId: {
           [contactA.id]: {
+            status: SendStatus.Failed,
+          },
+          [contactB.id]: {
             status: SendStatus.Delivered,
           },
         },

ts/test-electron/backup/filePointer_test.ts

@@ -75,7 +75,7 @@ describe('convertFilePointerToAttachment', () => {
         backupLocator: new Backups.FilePointer.BackupLocator({
           mediaName: 'mediaName',
           cdnNumber: 3,
-          size: Long.fromNumber(128),
+          size: 128,
           key: Bytes.fromString('key'),
           digest: Bytes.fromString('digest'),
           transitCdnKey: 'transitCdnKey',
@@ -215,7 +215,7 @@ const defaultBackupLocator = new Backups.FilePointer.BackupLocator({
   cdnNumber: null,
   key: Bytes.fromBase64('key'),
   digest: defaultDigest,
-  size: Long.fromNumber(100),
+  size: 100,
   transitCdnKey: 'cdnKey',
   transitCdnNumber: 2,
 });