Backups: update to latest integration tests

.github/workflows/ci.yml

@@ -194,7 +194,7 @@ jobs:
       uses: actions/checkout@v4
       with:
         repository: 'signalapp/Signal-Message-Backup-Tests'
-        ref: 'fcd73d838997294dc2d27b536cd1112ab33c4ee0'
+        ref: 'a920df75ba02e011f6c56c59c6bb20571162a961'
         path: 'backup-integration-tests'
 
     - run: xvfb-run --auto-servernum npm run test-electron

ACKNOWLEDGMENTS.md

@@ -5476,7 +5476,7 @@ limitations under the License.
 
 ```
 
-## boring 4.9.0
+## boring 4.13.0
 
 ```
 Copyright 2011-2017 Google Inc.
@@ -5945,7 +5945,7 @@ express Statement of Purpose.
    CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 ```
 
-## boring-sys 4.9.0
+## boring-sys 4.13.0
 
 ```
 /* Copyright (c) 2015, Google Inc.
@@ -6316,7 +6316,7 @@ DEALINGS IN THE SOFTWARE.
 
 ```
 
-## boring-sys 4.9.0
+## boring-sys 4.13.0
 
 ```
 Copyright (c) 2014 Alex Crichton
@@ -6985,7 +6985,7 @@ DEALINGS IN THE SOFTWARE.
 
 ```
 
-## boring-sys 4.9.0
+## boring-sys 4.13.0
 
 ```
 Copyright (c) 2015-2016 the fiat-crypto authors (see
@@ -7524,7 +7524,7 @@ SOFTWARE.
 
 ```
 
-## tokio-boring 4.9.0
+## tokio-boring 4.13.0
 
 ```
 Copyright (c) 2016 Tokio contributors
@@ -9221,6 +9221,31 @@ SOFTWARE.
 
 ```
 
+## openssl-macros 0.1.1
+
+```
+Copyright (c) 2022 Steven Fackler
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+```
+
 ## inout 0.1.3
 
 ```
@@ -11102,7 +11127,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 ```
 
-## boring-sys 4.9.0, ring 0.17.8
+## boring-sys 4.13.0, ring 0.17.8
 
 ```
 /* ====================================================================

package-lock.json

@@ -22,7 +22,7 @@
         "@react-aria/utils": "3.25.3",
         "@react-spring/web": "9.7.5",
         "@signalapp/better-sqlite3": "9.0.8",
-        "@signalapp/libsignal-client": "0.63.0",
+        "@signalapp/libsignal-client": "0.64.1",
         "@signalapp/ringrtc": "2.49.1",
         "@types/fabric": "4.5.3",
         "backbone": "1.6.0",
@@ -6007,11 +6007,10 @@
       }
     },
     "node_modules/@signalapp/libsignal-client": {
-      "version": "0.63.0",
+      "version": "0.64.1",
-      "resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.63.0.tgz",
+      "resolved": "https://registry.npmjs.org/@signalapp/libsignal-client/-/libsignal-client-0.64.1.tgz",
-      "integrity": "sha512-tLzVj9NoFNk8cD05QLdlm+VzgA7Eq91K8EALiSKAAJjt4rlsXT3mouBTITnWpOgTvk7YTdAFN4K74SyUVEeYgQ==",
+      "integrity": "sha512-ex45KXdmPtTW9q4DkF/VM/K0XnuuXzsqFFkanEeUE07kEzjGfx1x9tVhQ5h2+lokVjEVLM1Oq6lVysCpk3iwcg==",
       "hasInstallScript": true,
-      "license": "AGPL-3.0-only",
       "dependencies": {
         "node-gyp-build": "^4.8.0",
         "type-fest": "^4.26.0",

package.json

@@ -108,7 +108,7 @@
     "@react-aria/utils": "3.25.3",
     "@react-spring/web": "9.7.5",
     "@signalapp/better-sqlite3": "9.0.8",
-    "@signalapp/libsignal-client": "0.63.0",
+    "@signalapp/libsignal-client": "0.64.1",
     "@signalapp/ringrtc": "2.49.1",
     "@types/fabric": "4.5.3",
     "backbone": "1.6.0",

protos/Backups.proto

@@ -10,6 +10,8 @@ message BackupInfo {
   uint64 version = 1;
   uint64 backupTimeMs = 2;
   bytes mediaRootBackupKey = 3; // 32-byte random value generated when the backup is uploaded for the first time.
+  string currentAppVersion = 4;
+  string firstAppVersion = 5;
 }
 
 // Frames must follow in the following ordering rules:
@@ -19,9 +21,13 @@ message BackupInfo {
 //    e.g. a Recipient must come before any Chat referencing it.
 // 3. All ChatItems must appear in global Chat rendering order.
 //    (The order in which they were received by the client.)
+// 4. ChatFolders must appear in render order (e.g., left to right for 
+//    LTR locales), but can appear anywhere relative to other frames respecting
+//    rule 2 (after Recipients and Chats).
+//
+// Recipients, Chats, StickerPacks, AdHocCalls, and NotificationProfiles
+// can be in any order. (But must respect rule 2.)
 //
-// Recipients, Chats, StickerPacks, and AdHocCalls can be in any order.
-// (But must respect rule 2.)
 // For example, Chats may all be together at the beginning,
 // or may each immediately precede its first ChatItem.
 message Frame {
@@ -32,6 +38,8 @@ message Frame {
     ChatItem chatItem = 4;
     StickerPack stickerPack = 5;
     AdHocCall adHocCall = 6;
+    NotificationProfile notificationProfile = 7;
+    ChatFolder chatFolder = 8;
   }
 }
 
@@ -87,6 +95,17 @@ message AccountData {
     bool manuallyCancelled = 3;
   }
 
+  message IAPSubscriberData {
+    bytes subscriberId = 1;
+
+    oneof iapSubscriptionId {
+      // Identifies an Android Play Store IAP subscription.
+      string purchaseToken = 2;
+      // Identifies an iOS App Store IAP subscription.
+      uint64 originalTransactionId = 3;
+    }
+  }
+
   bytes profileKey = 1;
   optional string username = 2;
   UsernameLink usernameLink = 3;
@@ -94,8 +113,9 @@ message AccountData {
   string familyName = 5;
   string avatarUrlPath = 6;
   SubscriberData donationSubscriberData = 7;
-  SubscriberData backupsSubscriberData = 8;
+  reserved /*backupsSubscriberData*/ 8; // A deprecated format
   AccountSettings accountSettings = 9;
+  IAPSubscriberData backupsSubscriberData = 10;
 }
 
 message Recipient {
@@ -249,7 +269,7 @@ message Chat {
   bool archived = 3;
   optional uint32 pinnedOrder = 4; // will be displayed in ascending order
   optional uint64 expirationTimerMs = 5;
-  optional uint64 muteUntilMs = 6; // UINT64_MAX (2^63 - 1) = "always muted".
+  optional uint64 muteUntilMs = 6; // INT64_MAX (2^63 - 1) = "always muted".
   bool markedUnread = 7;
   bool dontNotifyForMentionsIfMuted = 8;
   ChatStyle style = 9;
@@ -275,7 +295,7 @@ message CallLink {
   optional bytes adminKey = 2; // Only present if the user is an admin
   string name = 3;
   Restrictions restrictions = 4;
-  optional uint64 expirationMs = 5;
+  uint64 expirationMs = 5;
 }
 
 message AdHocCall {
@@ -1172,3 +1192,48 @@ message ChatStyle {
 
   bool dimWallpaperInDarkMode = 7;
 }
+
+message NotificationProfile {
+  enum DayOfWeek {
+    UNKNOWN = 0;
+    MONDAY = 1;
+    TUESDAY = 2;
+    WEDNESDAY = 3;
+    THURSDAY = 4;
+    FRIDAY = 5;
+    SATURDAY = 6;
+    SUNDAY = 7;
+  }
+
+  string name = 1;
+  optional string emoji = 2;
+  fixed32 color = 3; // 0xAARRGGBB
+  uint64 createdAtMs = 4;
+  bool allowAllCalls = 5;
+  bool allowAllMentions = 6;
+  repeated uint64 allowedMembers = 7; // generated recipient id for allowed groups and contacts
+  bool scheduleEnabled = 8;
+  uint32 scheduleStartTime = 9; // 24-hour clock int, 0000-2359 (e.g., 15, 900, 1130, 2345)
+  uint32 scheduleEndTime = 10; // 24-hour clock int, 0000-2359 (e.g., 15, 900, 1130, 2345)
+  repeated DayOfWeek scheduleDaysEnabled = 11;
+}
+
+message ChatFolder {
+  // Represents the default "All chats" folder record vs all other custom folders
+  enum FolderType {
+    UNKNOWN = 0;
+    ALL = 1;
+    CUSTOM = 2;
+  }
+
+  string name = 1;
+  bool showOnlyUnread = 2;
+  bool showMutedChats = 3;
+  // Folder includes all 1:1 chats, unless excluded
+  bool includeAllIndividualChats = 4;
+  // Folder includes all group chats, unless excluded
+  bool includeAllGroupChats = 5;
+  FolderType folderType = 6;
+  repeated uint64 includedRecipientIds = 7; // generated recipient id of groups, contacts, and/or note to self
+  repeated uint64 excludedRecipientIds = 8; // generated recipient id of groups, contacts, and/or note to self
+}

protos/SignalStorage.proto

@@ -176,6 +176,17 @@ message AccountRecord {
     optional Color color = 3; // color of the QR code itself
   }
 
+  message IAPSubscriberData {
+    optional bytes subscriberId = 1;
+
+    oneof iapSubscriptionId {
+      // Identifies an Android Play Store IAP subscription.
+      string purchaseToken = 2;
+      // Identifies an iOS App Store IAP subscription.
+      uint64 originalTransactionId = 3;
+    }
+  }
+
   optional bytes profileKey = 1;
   optional string givenName = 2;
   optional string familyName = 3;
@@ -210,9 +221,14 @@ message AccountRecord {
   optional string username = 33;
   optional bool hasCompletedUsernameOnboarding = 34;
   optional UsernameLink usernameLink = 35;
-  optional bytes backupsSubscriberId = 36;
+  reserved /*backupsSubscriberId*/ 36;
-  optional string backupsSubscriberCurrencyCode = 37;
+  reserved /*backupsSubscriberCurrencyCode*/ 37;
-  optional bool backupsSubscriptionManuallyCancelled = 38;
+  reserved /*backupsSubscriptionManuallyCancelled*/ 38;
+  // Set to true after backups are enabled and one is uploaded.
+  optional bool hasBackup = 39;
+  // See zkgroup for integer particular values
+  optional uint64 backupTier = 40;
+  optional IAPSubscriberData backupSubscriberData = 41;
 }
 
 message StoryDistributionListRecord {

ts/services/backups/export.ts

@@ -139,6 +139,7 @@ import { getRoomIdFromRootKey } from '../../util/callLinksRingrtc';
 import { SeenStatus } from '../../MessageSeenStatus';
 import { migrateAllMessages } from '../../messages/migrateMessageData';
 import { trimBody } from '../../util/longAttachment';
+import { generateBackupsSubscriberData } from '../../util/backupSubscriptionData';
 
 const MAX_CONCURRENCY = 10;
 
@@ -258,6 +259,8 @@ export class BackupExportStream extends Readable {
         version: Long.fromNumber(BACKUP_VERSION),
         backupTimeMs: this.backupTimeMs,
         mediaRootBackupKey: getBackupMediaRootKey().serialize(),
+        firstAppVersion: window.storage.get('restoredBackupFirstAppVersion'),
+        currentAppVersion: `Desktop ${window.getVersion()}`,
       }).finish()
     );
 
@@ -660,7 +663,8 @@ export class BackupExportStream extends Readable {
     const usernameLink = storage.get('usernameLink');
 
     const subscriberId = storage.get('subscriberId');
-    const backupsSubscriberId = storage.get('backupsSubscriberId');
+
+    const backupsSubscriberData = generateBackupsSubscriberData();
 
     return {
       profileKey: storage.get('profileKey'),
@@ -676,16 +680,7 @@ export class BackupExportStream extends Readable {
       givenName: me.get('profileName'),
       familyName: me.get('profileFamilyName'),
       avatarUrlPath: storage.get('avatarUrl'),
-      backupsSubscriberData: Bytes.isNotEmpty(backupsSubscriberId)
+      backupsSubscriberData,
-        ? {
-            subscriberId: backupsSubscriberId,
-            currencyCode: storage.get('backupsSubscriberCurrencyCode'),
-            manuallyCancelled: storage.get(
-              'backupsSubscriptionManuallyCancelled',
-              false
-            ),
-          }
-        : null,
       donationSubscriberData: Bytes.isNotEmpty(subscriberId)
         ? {
             subscriberId,

ts/services/backups/import.ts

@@ -122,6 +122,7 @@ import { hasAttachmentDownloads } from '../../util/hasAttachmentDownloads';
 import { isNightly } from '../../util/version';
 import { ToastType } from '../../types/Toast';
 import { isConversationAccepted } from '../../util/isConversationAccepted';
+import { saveBackupsSubscriberData } from '../../util/backupSubscriptionData';
 
 const MAX_CONCURRENCY = 10;
 
@@ -262,6 +263,11 @@ export class BackupImportStream extends Writable {
           throw new Error('Missing mediaRootBackupKey');
         }
 
+        await window.storage.put(
+          'restoredBackupFirstAppVersion',
+          info.firstAppVersion
+        );
+
         const theirKey = info.mediaRootBackupKey;
         const ourKey = getBackupMediaRootKey().serialize();
         if (!constantTimeEqual(theirKey, ourKey)) {
@@ -658,22 +664,8 @@ export class BackupImportStream extends Writable {
         );
       }
     }
-    if (backupsSubscriberData != null) {
+
-      const { subscriberId, currencyCode, manuallyCancelled } =
+    await saveBackupsSubscriberData(backupsSubscriberData);
-        backupsSubscriberData;
-      if (Bytes.isNotEmpty(subscriberId)) {
-        await storage.put('backupsSubscriberId', subscriberId);
-      }
-      if (currencyCode != null) {
-        await storage.put('backupsSubscriberCurrencyCode', currencyCode);
-      }
-      if (manuallyCancelled != null) {
-        await storage.put(
-          'backupsSubscriptionManuallyCancelled',
-          manuallyCancelled
-        );
-      }
-    }
 
     await storage.put(
       'read-receipt-setting',
@@ -1213,6 +1205,7 @@ export class BackupImportStream extends Writable {
     if (conversation.active_at == null) {
       conversation.active_at = Math.max(chat.id.toNumber(), 1);
     }
+
     conversation.isArchived = chat.archived === true;
     conversation.isPinned = (chat.pinnedOrder || 0) !== 0;
 

ts/services/storageRecordOps.ts

@@ -80,6 +80,10 @@ import { fromAdminKeyBytes, toAdminKeyBytes } from '../util/callLinks';
 import { isOlderThan } from '../util/timestamp';
 import { getMessageQueueTime } from '../util/getMessageQueueTime';
 import { callLinkRefreshJobQueue } from '../jobs/callLinkRefreshJobQueue';
+import {
+  generateBackupsSubscriberData,
+  saveBackupsSubscriberData,
+} from '../util/backupSubscriptionData';
 
 const MY_STORY_BYTES = uuidToBytes(MY_STORY_ID);
 
@@ -406,9 +410,7 @@ export function toAccountRecord(
   if (Bytes.isNotEmpty(subscriberId)) {
     accountRecord.subscriberId = subscriberId;
   }
-  const subscriberCurrencyCode = window.storage.get(
+  const subscriberCurrencyCode = window.storage.get('subscriberCurrencyCode');
-    'backupsSubscriberCurrencyCode'
-  );
   if (typeof subscriberCurrencyCode === 'string') {
     accountRecord.subscriberCurrencyCode = subscriberCurrencyCode;
   }
@@ -419,23 +421,9 @@ export function toAccountRecord(
     accountRecord.donorSubscriptionManuallyCancelled =
       donorSubscriptionManuallyCancelled;
   }
-  const backupsSubscriberId = window.storage.get('backupsSubscriberId');
+
-  if (Bytes.isNotEmpty(backupsSubscriberId)) {
+  accountRecord.backupSubscriberData = generateBackupsSubscriberData();
-    accountRecord.backupsSubscriberId = backupsSubscriberId;
+
-  }
-  const backupsSubscriberCurrencyCode = window.storage.get(
-    'backupsSubscriberCurrencyCode'
-  );
-  if (typeof backupsSubscriberCurrencyCode === 'string') {
-    accountRecord.backupsSubscriberCurrencyCode = backupsSubscriberCurrencyCode;
-  }
-  const backupsSubscriptionManuallyCancelled = window.storage.get(
-    'backupsSubscriptionManuallyCancelled'
-  );
-  if (typeof backupsSubscriptionManuallyCancelled === 'boolean') {
-    accountRecord.backupsSubscriptionManuallyCancelled =
-      backupsSubscriptionManuallyCancelled;
-  }
   const displayBadgesOnProfile = window.storage.get('displayBadgesOnProfile');
   if (displayBadgesOnProfile !== undefined) {
     accountRecord.displayBadgesOnProfile = displayBadgesOnProfile;
@@ -1327,9 +1315,7 @@ export async function mergeAccountRecord(
     subscriberId,
     subscriberCurrencyCode,
     donorSubscriptionManuallyCancelled,
-    backupsSubscriberId,
+    backupSubscriberData,
-    backupsSubscriberCurrencyCode,
-    backupsSubscriptionManuallyCancelled,
     displayBadgesOnProfile,
     keepMutedChatsArchived,
     hasCompletedUsernameOnboarding,
@@ -1548,21 +1534,9 @@ export async function mergeAccountRecord(
       donorSubscriptionManuallyCancelled
     );
   }
-  if (Bytes.isNotEmpty(backupsSubscriberId)) {
+
-    await window.storage.put('backupsSubscriberId', backupsSubscriberId);
+  await saveBackupsSubscriberData(backupSubscriberData);
-  }
+
-  if (typeof backupsSubscriberCurrencyCode === 'string') {
-    await window.storage.put(
-      'backupsSubscriberCurrencyCode',
-      backupsSubscriberCurrencyCode
-    );
-  }
-  if (backupsSubscriptionManuallyCancelled != null) {
-    await window.storage.put(
-      'backupsSubscriptionManuallyCancelled',
-      backupsSubscriptionManuallyCancelled
-    );
-  }
   await window.storage.put(
     'displayBadgesOnProfile',
     Boolean(displayBadgesOnProfile)

ts/test-electron/backup/integration_test.ts

@@ -60,6 +60,12 @@ describe('backup/integration', () => {
 
   const files = readdirSync(BACKUP_INTEGRATION_DIR)
     .filter(file => file.endsWith('.binproto'))
+    .filter(
+      file =>
+        // TODO (DESKTOP-8025)
+        !file.startsWith('chat_folder_') &&
+        !file.startsWith('notification_profile_')
+    )
     .map(file => join(BACKUP_INTEGRATION_DIR, file));
 
   if (files.length === 0) {

ts/types/Storage.d.ts

@@ -175,8 +175,8 @@ export type StorageAccessType = {
   subscriberCurrencyCode: string;
   donorSubscriptionManuallyCancelled: boolean;
   backupsSubscriberId: Uint8Array;
-  backupsSubscriberCurrencyCode: string;
+  backupsSubscriberPurchaseToken: string;
-  backupsSubscriptionManuallyCancelled: boolean;
+  backupsSubscriberOriginalTransactionId: string;
   displayBadgesOnProfile: boolean;
   keepMutedChatsArchived: boolean;
   usernameLastIntegrityCheck: number;
@@ -209,6 +209,9 @@ export type StorageAccessType = {
   // If true Desktop message history was restored from backup
   isRestoredFromBackup: boolean;
 
+  // The `firstAppVersion` present on an BackupInfo from an imported backup.
+  restoredBackupFirstAppVersion: string;
+
   // Deprecated
   'challenge:retry-message-ids': never;
   nextSignedKeyRotationTime: number;

ts/util/backupSubscriptionData.ts

@@ -0,0 +1,75 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import Long from 'long';
+import type { Backups, SignalService } from '../protobuf';
+import * as Bytes from '../Bytes';
+
+// These two proto messages (Backups.AccountData.IIAPSubscriberData &&
+// SignalService.AccountRecord.IIAPSubscriberData) should remain in sync. If they drift,
+// we'll need separate logic for each
+export async function saveBackupsSubscriberData(
+  backupsSubscriberData:
+    | Backups.AccountData.IIAPSubscriberData
+    | SignalService.AccountRecord.IIAPSubscriberData
+    | null
+    | undefined
+): Promise<void> {
+  if (backupsSubscriberData == null) {
+    await window.storage.remove('backupsSubscriberId');
+    await window.storage.remove('backupsSubscriberPurchaseToken');
+    await window.storage.remove('backupsSubscriberOriginalTransactionId');
+    return;
+  }
+
+  const { subscriberId, purchaseToken, originalTransactionId } =
+    backupsSubscriberData;
+
+  if (Bytes.isNotEmpty(subscriberId)) {
+    await window.storage.put('backupsSubscriberId', subscriberId);
+  } else {
+    await window.storage.remove('backupsSubscriberId');
+  }
+
+  if (purchaseToken) {
+    await window.storage.put('backupsSubscriberPurchaseToken', purchaseToken);
+  } else {
+    await window.storage.remove('backupsSubscriberPurchaseToken');
+  }
+
+  if (originalTransactionId) {
+    await window.storage.put(
+      'backupsSubscriberOriginalTransactionId',
+      originalTransactionId.toString()
+    );
+  } else {
+    await window.storage.remove('backupsSubscriberOriginalTransactionId');
+  }
+}
+
+export function generateBackupsSubscriberData(): Backups.AccountData.IIAPSubscriberData | null {
+  const backupsSubscriberId = window.storage.get('backupsSubscriberId');
+
+  if (Bytes.isEmpty(backupsSubscriberId)) {
+    return null;
+  }
+
+  const backupsSubscriberData: Backups.AccountData.IIAPSubscriberData = {
+    subscriberId: backupsSubscriberId,
+  };
+  const purchaseToken = window.storage.get('backupsSubscriberPurchaseToken');
+  if (purchaseToken) {
+    backupsSubscriberData.purchaseToken = purchaseToken;
+  } else {
+    const originalTransactionId = window.storage.get(
+      'backupsSubscriberOriginalTransactionId'
+    );
+    if (originalTransactionId) {
+      backupsSubscriberData.originalTransactionId = Long.fromString(
+        originalTransactionId
+      );
+    }
+  }
+
+  return backupsSubscriberData;
+}