Add preliminary message backup harness

app/main.ts

@@ -2481,6 +2481,7 @@ ipc.on('get-config', async event => {
     ciMode,
     // Should be already computed and cached at this point
     dnsFallback: await getDNSFallback(),
+    ciBackupPath: config.get<string | null>('ciBackupPath') || undefined,
     nodeVersion: process.versions.node,
     hostname: os.hostname(),
     osRelease: os.release(),

config/default.json

@@ -18,6 +18,7 @@
   "registrationChallengeUrl": "https://signalcaptchas.org/staging/registration/generate.html",
   "updatesEnabled": false,
   "ciMode": false,
+  "ciBackupPath": null,
   "forcePreloadBundle": false,
   "openDevTools": false,
   "buildCreation": 0,

package.json

@@ -252,6 +252,8 @@
     "@types/node-fetch": "2.6.2",
     "@types/normalize-path": "3.0.0",
     "@types/pify": "3.0.2",
+    "@types/pixelmatch": "5.2.6",
+    "@types/pngjs": "6.0.4",
     "@types/quill": "1.3.10",
     "@types/react": "17.0.45",
     "@types/react-dom": "17.0.17",
@@ -310,7 +312,9 @@
     "nyc": "11.4.1",
     "p-limit": "3.1.0",
     "patch-package": "8.0.0",
+    "pixelmatch": "5.3.0",
     "playwright": "1.41.0-alpha-jan-9-2024",
+    "pngjs": "7.0.0",
     "prettier": "2.8.0",
     "protobufjs-cli": "1.1.1",
     "resolve-url-loader": "5.0.0",

protos/Backups.proto

@@ -0,0 +1,823 @@
+// Copyright 2024 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+syntax = "proto3";
+
+package signalbackups;
+
+option java_package = "org.thoughtcrime.securesms.backup.v2.proto";
+
+message BackupInfo {
+  uint64 version = 1;
+  uint64 backupTimeMs = 2;
+}
+
+message Frame {
+  oneof item {
+    AccountData account = 1;
+    Recipient recipient = 2;
+    Chat chat = 3;
+    ChatItem chatItem = 4;
+    Call call = 5;
+    StickerPack stickerPack = 6;
+  }
+}
+
+message AccountData {
+  enum PhoneNumberSharingMode {
+    UNKNOWN = 0;
+    EVERYBODY = 1;
+    NOBODY = 2;
+  }
+  message UsernameLink {
+    enum Color {
+      UNKNOWN = 0;
+      BLUE = 1;
+      WHITE = 2;
+      GREY = 3;
+      OLIVE = 4;
+      GREEN = 5;
+      ORANGE = 6;
+      PINK = 7;
+      PURPLE = 8;
+    }
+
+    bytes entropy = 1;  // 32 bytes of entropy used for encryption
+    bytes serverId = 2; // 16 bytes of encoded UUID provided by the server
+    Color color = 3;
+  }
+
+  message AccountSettings {
+    bool readReceipts = 1;
+    bool sealedSenderIndicators = 2;
+    bool typingIndicators = 3;
+    bool linkPreviews = 4;
+    bool notDiscoverableByPhoneNumber = 5;
+    bool preferContactAvatars = 6;
+    uint32 universalExpireTimer = 7; // 0 means no universal expire timer.
+    repeated string preferredReactionEmoji = 8;
+    bool displayBadgesOnProfile = 9;
+    bool keepMutedChatsArchived = 10;
+    bool hasSetMyStoriesPrivacy = 11;
+    bool hasViewedOnboardingStory = 12;
+    bool storiesDisabled = 13;
+    optional bool storyViewReceiptsEnabled = 14;
+    bool hasSeenGroupStoryEducationSheet = 15;
+    bool hasCompletedUsernameOnboarding = 16;
+    PhoneNumberSharingMode phoneNumberSharingMode = 17;
+  }
+
+  bytes profileKey = 1;
+  optional string username = 2;
+  UsernameLink usernameLink = 3;
+  string givenName = 4;
+  string familyName = 5;
+  string avatarUrlPath = 6;
+  bytes subscriberId = 7;
+  string subscriberCurrencyCode = 8;
+  bool subscriptionManuallyCancelled = 9;
+  AccountSettings accountSettings = 10;
+}
+
+message Recipient {
+  uint64 id = 1; // generated id for reference only within this file
+  oneof destination {
+    Contact contact = 2;
+    Group group = 3;
+    DistributionList distributionList = 4;
+    Self self = 5;
+    ReleaseNotes releaseNotes = 6;
+  }
+}
+
+message Contact {
+  enum Registered {
+    UNKNOWN = 0;
+    REGISTERED = 1;
+    NOT_REGISTERED = 2;
+  }
+
+  optional bytes aci = 1; // should be 16 bytes
+  optional bytes pni = 2; // should be 16 bytes
+  optional string username = 3;
+  optional uint64 e164 = 4;
+  bool blocked = 5;
+  bool hidden = 6;
+  Registered registered = 7;
+  uint64 unregisteredTimestamp = 8;
+  optional bytes profileKey = 9;
+  bool profileSharing = 10;
+  optional string profileGivenName = 11;
+  optional string profileFamilyName = 12;
+  bool hideStory = 13;
+}
+
+message Group {
+  enum StorySendMode {
+    DEFAULT = 0;
+    DISABLED = 1;
+    ENABLED = 2;
+  }
+
+  bytes masterKey = 1;
+  bool whitelisted = 2;
+  bool hideStory = 3;
+  StorySendMode storySendMode = 4;
+  string name = 5;
+}
+
+message Self {}
+
+message ReleaseNotes {}
+
+message Chat {
+  uint64 id = 1; // generated id for reference only within this file
+  uint64 recipientId = 2;
+  bool archived = 3;
+  uint32 pinnedOrder = 4; // 0 = unpinned, otherwise chat is considered pinned and will be displayed in ascending order
+  uint64 expirationTimerMs = 5; // 0 = no expire timer.
+  uint64 muteUntilMs = 6;
+  bool markedUnread = 7;
+  bool dontNotifyForMentionsIfMuted = 8;
+  FilePointer wallpaper = 9;
+}
+
+message DistributionList {
+  enum PrivacyMode {
+    UNKNOWN = 0;
+    ONLY_WITH = 1;
+    ALL_EXCEPT = 2;
+    ALL = 3;
+  }
+
+  string name = 1;
+  bytes distributionId = 2; // distribution list ids are uuids
+  bool allowReplies = 3;
+  uint64 deletionTimestamp = 4;
+  PrivacyMode privacyMode = 5;
+  repeated uint64 memberRecipientIds = 6; // generated recipient id
+}
+
+message Identity {
+  bytes serviceId = 1;
+  bytes identityKey = 2;
+  uint64 timestamp = 3;
+  bool firstUse = 4;
+  bool verified = 5;
+  bool nonblockingApproval = 6;
+}
+
+message Call {
+  enum Type {
+    UNKNOWN_TYPE = 0;
+    AUDIO_CALL = 1;
+    VIDEO_CALL = 2;
+    GROUP_CALL = 3;
+    AD_HOC_CALL = 4;
+  }
+
+  enum State {
+    UNKNOWN_EVENT = 0;
+    COMPLETED = 1; // A call that was successfully completed or was accepted and in-progress at the time of the backup.
+    DECLINED_BY_USER = 2; // An incoming call that was manually declined by the user.
+    DECLINED_BY_NOTIFICATION_PROFILE = 3; // An incoming call that was automatically declined by an active notification profile.
+    MISSED = 4; // An incoming call that either expired, was cancelled by the sender, or was auto-rejected due to already being in a different call.
+  }
+
+  uint64 callId = 1;
+  uint64 conversationRecipientId = 2;
+  Type type = 3;
+  bool outgoing = 4;
+  uint64 timestamp = 5;
+  optional uint64 ringerRecipientId = 6;
+  State state = 7;
+}
+
+message ChatItem {
+  message IncomingMessageDetails {
+    uint64 dateReceived = 1;
+    uint64 dateServerSent = 2;
+    bool read = 3;
+    bool sealedSender = 4;
+  }
+
+  message OutgoingMessageDetails {
+    repeated SendStatus sendStatus = 1;
+  }
+
+  message DirectionlessMessageDetails {
+  }
+
+  uint64 chatId = 1;   // conversation id
+  uint64 authorId = 2; // recipient id
+  uint64 dateSent = 3;
+  optional uint64 expireStartDate = 4; // timestamp of when expiration timer started ticking down
+  optional uint64 expiresInMs = 5; // how long timer of message is (ms)
+  repeated ChatItem revisions = 6; // ordered from oldest to newest
+  bool sms = 7;
+
+  oneof directionalDetails {
+    IncomingMessageDetails incoming = 8;
+    OutgoingMessageDetails outgoing = 9;
+    DirectionlessMessageDetails directionless = 10;
+  }
+
+  oneof item {
+    StandardMessage standardMessage = 11;
+    ContactMessage contactMessage = 12;
+    StickerMessage stickerMessage = 13;
+    RemoteDeletedMessage remoteDeletedMessage = 14;
+    ChatUpdateMessage updateMessage = 15;
+  }
+}
+
+message SendStatus {
+  enum Status {
+    UNKNOWN = 0;
+    FAILED = 1;
+    PENDING = 2;
+    SENT = 3;
+    DELIVERED = 4;
+    READ = 5;
+    VIEWED = 6;
+    SKIPPED = 7; // e.g. user in group was blocked, so we skipped sending to them
+  }
+
+  uint64 recipientId = 1;
+  Status deliveryStatus = 2;
+  bool networkFailure = 3;
+  bool identityKeyMismatch = 4;
+  bool sealedSender = 5;
+  uint64 lastStatusUpdateTimestamp = 6; // the time the status was last updated -- if from a receipt, it should be the sentTime of the receipt
+}
+
+message Text {
+  string body = 1;
+  repeated BodyRange bodyRanges = 2;
+}
+
+message StandardMessage {
+  optional Quote quote = 1;
+  optional Text text = 2;
+  repeated MessageAttachment attachments = 3;
+  repeated LinkPreview linkPreview = 4;
+  optional FilePointer longText = 5;
+  repeated Reaction reactions = 6;
+}
+
+message ContactMessage {
+  repeated ContactAttachment contact = 1;
+  repeated Reaction reactions = 2;
+}
+
+message ContactAttachment {
+  message Name {
+    optional string givenName = 1;
+    optional string familyName = 2;
+    optional string prefix = 3;
+    optional string suffix = 4;
+    optional string middleName = 5;
+    optional string displayName = 6;
+  }
+
+  message Phone {
+    enum Type {
+      UNKNOWN = 0;
+      HOME = 1;
+      MOBILE = 2;
+      WORK = 3;
+      CUSTOM = 4;
+    }
+
+    optional string value = 1;
+    optional Type type = 2;
+    optional string label = 3;
+  }
+
+  message Email {
+    enum Type {
+      UNKNOWN = 0;
+      HOME = 1;
+      MOBILE = 2;
+      WORK = 3;
+      CUSTOM = 4;
+    }
+
+    optional string value = 1;
+    optional Type type = 2;
+    optional string label = 3;
+  }
+
+  message PostalAddress {
+    enum Type {
+      UNKNOWN = 0;
+      HOME = 1;
+      WORK = 2;
+      CUSTOM = 3;
+    }
+
+    optional Type type = 1;
+    optional string label = 2;
+    optional string street = 3;
+    optional string pobox = 4;
+    optional string neighborhood = 5;
+    optional string city = 6;
+    optional string region = 7;
+    optional string postcode = 8;
+    optional string country = 9;
+  }
+
+  optional Name name = 1;
+  repeated Phone number = 3;
+  repeated Email email = 4;
+  repeated PostalAddress address = 5;
+  optional string avatarUrlPath = 6;
+  optional string organization = 7;
+}
+
+message DocumentMessage {
+  Text text = 1;
+  FilePointer document = 2;
+  repeated Reaction reactions = 3;
+}
+
+message StickerMessage {
+  Sticker sticker = 1;
+  repeated Reaction reactions = 2;
+}
+
+// Tombstone for remote delete
+message RemoteDeletedMessage {}
+
+message Sticker {
+  bytes packId = 1;
+  bytes packKey = 2;
+  uint32 stickerId = 3;
+  optional string emoji = 4;
+  // Stickers are uploaded to be sent as attachments; we also
+  // back them up as normal attachments when they are in messages.
+  // DO NOT treat this as the definitive source of a sticker in
+  // an installed StickerPack that shares the same packId.
+  FilePointer data = 5;
+}
+
+message LinkPreview {
+  string url = 1;
+  optional string title = 2;
+  optional FilePointer image = 3;
+  optional string description = 4;
+  optional uint64 date = 5;
+}
+
+// A FilePointer on a message that has additional
+// metadata that applies only to message attachments.
+message MessageAttachment {
+  // Similar to SignalService.AttachmentPointer.Flags,
+  // but explicitly mutually exclusive. Note the different raw values
+  // (non-zero starting values are not supported in proto3.)
+  enum Flag {
+    NONE = 0;
+    VOICE_MESSAGE = 1;
+    BORDERLESS = 2;
+    GIF = 3;
+  }
+
+  FilePointer pointer = 1;
+  Flag flag = 2;
+}
+
+message FilePointer {
+  // References attachments in the backup (media) storage tier.
+  message BackupLocator {
+    string mediaName = 1;
+    uint32 cdnNumber = 2;
+  }
+
+  // References attachments in the transit storage tier.
+  // May be downloaded or not when the backup is generated;
+  // primarily for free-tier users who cannot copy the
+  // attachments to the backup (media) storage tier.
+  message AttachmentLocator {
+    string cdnKey = 1;
+    uint32 cdnNumber = 2;
+    uint64 uploadTimestamp = 3;
+  }
+
+  // An attachment that was copied from the transit storage tier
+  // to the backup (media) storage tier up without being downloaded.
+  // Its MediaName should be generated as “{sender_aci}_{cdn_attachment_key}”,
+  // but should eventually transition to a BackupLocator with mediaName
+  // being the content hash once it is downloaded.
+  message UndownloadedBackupLocator {
+    bytes senderAci = 1;
+    string cdnKey = 2;
+    uint32 cdnNumber = 3;
+  }
+
+  oneof locator {
+    BackupLocator backupLocator = 1;
+    AttachmentLocator attachmentLocator= 2;
+    UndownloadedBackupLocator undownloadedBackupLocator = 3;
+  }
+
+  optional bytes key = 5;
+  optional string contentType = 6;
+  // Size of fullsize decrypted media blob in bytes.
+  // Can be ignored if unset/unavailable.
+  optional uint32 size = 7;
+  optional bytes incrementalMac = 8;
+  optional uint32 incrementalMacChunkSize = 9;
+  optional string fileName = 10;
+  optional uint32 width = 11;
+  optional uint32 height = 12;
+  optional string caption = 13;
+  optional string blurHash = 14;
+}
+
+message Quote {
+  enum Type {
+    UNKNOWN = 0;
+    NORMAL = 1;
+    GIFTBADGE = 2;
+  }
+
+  message QuotedAttachment {
+    optional string contentType = 1;
+    optional string fileName = 2;
+    optional MessageAttachment thumbnail = 3;
+  }
+
+  optional uint64 targetSentTimestamp = 1; // null if the target message could not be found at time of quote insert
+  uint64 authorId = 2;
+  optional string text = 3;
+  repeated QuotedAttachment attachments = 4;
+  repeated BodyRange bodyRanges = 5;
+  Type type = 6;
+}
+
+message BodyRange {
+  enum Style {
+    NONE = 0;
+    BOLD = 1;
+    ITALIC = 2;
+    SPOILER = 3;
+    STRIKETHROUGH = 4;
+    MONOSPACE = 5;
+  }
+
+  optional uint32 start = 1;
+  optional uint32 length = 2;
+
+  oneof associatedValue {
+    bytes mentionAci = 3;
+    Style style = 4;
+  }
+}
+
+message Reaction {
+  string emoji = 1;
+  uint64 authorId = 2;
+  uint64 sentTimestamp = 3;
+  optional uint64 receivedTimestamp = 4;
+  uint64 sortOrder = 5; // A higher sort order means that a reaction is more recent
+}
+
+message ChatUpdateMessage {
+  oneof update {
+    SimpleChatUpdate simpleUpdate = 1;
+    GroupChangeChatUpdate groupChange = 2;
+    ExpirationTimerChatUpdate expirationTimerChange = 3;
+    ProfileChangeChatUpdate profileChange = 4;
+    ThreadMergeChatUpdate threadMerge = 5;
+    SessionSwitchoverChatUpdate sessionSwitchover = 6;
+    CallChatUpdate callingMessage = 7;
+  }
+}
+
+message CallChatUpdate{
+  oneof call {
+    uint64 callId = 1; // maps to id of Call from call log
+    IndividualCallChatUpdate callMessage = 2;
+    GroupCallChatUpdate groupCall = 3;
+  }
+}
+
+message IndividualCallChatUpdate {
+  enum Type {
+    UNKNOWN = 0;
+    INCOMING_AUDIO_CALL = 1;
+    INCOMING_VIDEO_CALL = 2;
+    OUTGOING_AUDIO_CALL = 3;
+    OUTGOING_VIDEO_CALL = 4;
+    MISSED_INCOMING_AUDIO_CALL = 5;
+    MISSED_INCOMING_VIDEO_CALL = 6;
+    UNANSWERED_OUTGOING_AUDIO_CALL = 7;
+    UNANSWERED_OUTGOING_VIDEO_CALL = 8;
+  }
+
+  Type type = 1;
+}
+
+message GroupCallChatUpdate {
+  optional bytes startedCallAci = 1;
+  uint64 startedCallTimestamp = 2;
+  repeated bytes inCallAcis = 3;
+}
+
+message SimpleChatUpdate {
+  enum Type {
+    UNKNOWN = 0;
+    JOINED_SIGNAL = 1;
+    IDENTITY_UPDATE = 2;
+    IDENTITY_VERIFIED = 3;
+    IDENTITY_DEFAULT = 4; // marking as unverified
+    CHANGE_NUMBER = 5;
+    BOOST_REQUEST = 6;
+    END_SESSION = 7;
+    CHAT_SESSION_REFRESH = 8;
+    BAD_DECRYPT = 9;
+    PAYMENTS_ACTIVATED = 10;
+    PAYMENT_ACTIVATION_REQUEST = 11;
+  }
+
+  Type type = 1;
+}
+
+// For 1:1 chat updates only.
+// For group thread updates use GroupExpirationTimerUpdate.
+message ExpirationTimerChatUpdate {
+  uint32 expiresInMs = 1; // 0 means the expiration timer was disabled
+}
+
+message ProfileChangeChatUpdate {
+  string previousName = 1;
+  string newName = 2;
+}
+
+message ThreadMergeChatUpdate {
+  uint64 previousE164 = 1;
+}
+
+message SessionSwitchoverChatUpdate {
+  uint64 e164 = 1;
+}
+
+message GroupChangeChatUpdate {
+  message Update {
+    // Note: group expiration timer changes are represented as ExpirationTimerChatUpdate.
+    oneof update {
+      GenericGroupUpdate genericGroupUpdate = 1;
+      GroupCreationUpdate groupCreationUpdate = 2;
+      GroupNameUpdate groupNameUpdate = 3;
+      GroupAvatarUpdate groupAvatarUpdate = 4;
+      GroupDescriptionUpdate groupDescriptionUpdate = 5;
+      GroupMembershipAccessLevelChangeUpdate groupMembershipAccessLevelChangeUpdate = 6;
+      GroupAttributesAccessLevelChangeUpdate groupAttributesAccessLevelChangeUpdate = 7;
+      GroupAnnouncementOnlyChangeUpdate groupAnnouncementOnlyChangeUpdate = 8;
+      GroupAdminStatusUpdate groupAdminStatusUpdate = 9;
+      GroupMemberLeftUpdate groupMemberLeftUpdate = 10;
+      GroupMemberRemovedUpdate groupMemberRemovedUpdate = 11;
+      SelfInvitedToGroupUpdate selfInvitedToGroupUpdate = 12;
+      SelfInvitedOtherUserToGroupUpdate selfInvitedOtherUserToGroupUpdate = 13;
+      GroupUnknownInviteeUpdate groupUnknownInviteeUpdate = 14;
+      GroupInvitationAcceptedUpdate groupInvitationAcceptedUpdate = 15;
+      GroupInvitationDeclinedUpdate groupInvitationDeclinedUpdate = 16;
+      GroupMemberJoinedUpdate groupMemberJoinedUpdate = 17;
+      GroupMemberAddedUpdate groupMemberAddedUpdate = 18;
+      GroupSelfInvitationRevokedUpdate groupSelfInvitationRevokedUpdate = 19;
+      GroupInvitationRevokedUpdate groupInvitationRevokedUpdate = 20;
+      GroupJoinRequestUpdate groupJoinRequestUpdate = 21;
+      GroupJoinRequestApprovalUpdate groupJoinRequestApprovalUpdate = 22;
+      GroupJoinRequestCanceledUpdate groupJoinRequestCanceledUpdate = 23;
+      GroupInviteLinkResetUpdate groupInviteLinkResetUpdate = 24;
+      GroupInviteLinkEnabledUpdate groupInviteLinkEnabledUpdate = 25;
+      GroupInviteLinkAdminApprovalUpdate groupInviteLinkAdminApprovalUpdate = 26;
+      GroupInviteLinkDisabledUpdate groupInviteLinkDisabledUpdate = 27;
+      GroupMemberJoinedByLinkUpdate groupMemberJoinedByLinkUpdate = 28;
+      GroupV2MigrationUpdate groupV2MigrationUpdate = 29;
+      GroupV2MigrationSelfInvitedUpdate groupV2MigrationSelfInvitedUpdate = 30;
+      GroupV2MigrationInvitedMembersUpdate groupV2MigrationInvitedMembersUpdate = 31;
+      GroupV2MigrationDroppedMembersUpdate groupV2MigrationDroppedMembersUpdate = 32;
+      GroupSequenceOfRequestsAndCancelsUpdate groupSequenceOfRequestsAndCancelsUpdate = 33;
+      GroupExpirationTimerUpdate groupExpirationTimerUpdate = 34;
+    }
+  }
+
+  // Must be one or more; all updates batched together came from
+  // a single batched group state update.
+  repeated Update updates = 1;
+}
+
+message GenericGroupUpdate {
+  optional bytes updaterAci = 1;
+}
+
+message GroupCreationUpdate {
+  optional bytes updaterAci = 1;
+}
+
+message GroupNameUpdate {
+  optional bytes updaterAci = 1;
+  // Null value means the group name was removed.
+  optional string newGroupName = 2;
+}
+
+message GroupAvatarUpdate {
+  optional bytes updaterAci = 1;
+  bool wasRemoved = 2;
+}
+
+message GroupDescriptionUpdate {
+  optional bytes updaterAci = 1;
+  // Null value means the group description was removed.
+  optional string newDescription = 2;
+}
+
+enum GroupV2AccessLevel {
+  UNKNOWN = 0;
+  ANY = 1;
+  MEMBER = 2;
+  ADMINISTRATOR = 3;
+  UNSATISFIABLE = 4;
+}
+
+message GroupMembershipAccessLevelChangeUpdate {
+  optional bytes updaterAci = 1;
+  GroupV2AccessLevel accessLevel = 2;
+}
+
+message GroupAttributesAccessLevelChangeUpdate {
+  optional bytes updaterAci = 1;
+  GroupV2AccessLevel accessLevel = 2;
+}
+
+message GroupAnnouncementOnlyChangeUpdate {
+  optional bytes updaterAci = 1;
+  bool isAnnouncementOnly = 2;
+}
+
+message GroupAdminStatusUpdate {
+  optional bytes updaterAci = 1;
+  // The aci who had admin status granted or revoked.
+  bytes memberAci = 2;
+  bool wasAdminStatusGranted = 3;
+}
+
+message GroupMemberLeftUpdate {
+  bytes aci = 1;
+}
+
+message GroupMemberRemovedUpdate {
+  optional bytes removerAci = 1;
+  bytes removedAci = 2;
+}
+
+message SelfInvitedToGroupUpdate {
+  optional bytes inviterAci = 1;
+}
+
+message SelfInvitedOtherUserToGroupUpdate {
+  // If no invitee id available, use GroupUnknownInviteeUpdate
+  bytes inviteeServiceId = 1;
+}
+
+message GroupUnknownInviteeUpdate {
+  // Can be the self user.
+  optional bytes inviterAci = 1;
+  uint32 inviteeCount = 2;
+}
+
+message GroupInvitationAcceptedUpdate {
+  optional bytes inviterAci = 1;
+  bytes newMemberAci = 2;
+}
+
+message GroupInvitationDeclinedUpdate {
+  optional bytes inviterAci = 1;
+  // Note: if invited by pni, just set inviteeAci to nil.
+  optional bytes inviteeAci = 2;
+}
+
+message GroupMemberJoinedUpdate {
+  bytes newMemberAci = 1;
+}
+
+message GroupMemberAddedUpdate {
+  optional bytes updaterAci = 1;
+  bytes newMemberAci = 2;
+  bool hadOpenInvitation = 3;
+  // If hadOpenInvitation is true, optionally include aci of the inviter.
+  optional bytes inviterAci = 4;
+}
+
+// An invitation to self was revoked.
+message GroupSelfInvitationRevokedUpdate {
+  optional bytes revokerAci = 1;
+}
+
+// These invitees should never be the local user.
+// Use GroupSelfInvitationRevokedUpdate in those cases.
+// The inviter or updater can be the local user.
+message GroupInvitationRevokedUpdate {
+  message Invitee {
+    optional bytes inviterAci = 1;
+    // Prefer to use aci over pni. No need to set
+    // pni if aci is set. Both can be missing.
+    optional bytes inviteeAci = 2;
+    optional bytes inviteePni = 3;
+  }
+
+  // The member that revoked the invite(s), not the inviter!
+  // Assumed to be an admin (at the time, may no longer be an
+  // admin or even a member).
+  optional bytes updaterAci = 1;
+  repeated Invitee invitees = 2;
+}
+
+message GroupJoinRequestUpdate {
+  bytes requestorAci = 1;
+}
+
+message GroupJoinRequestApprovalUpdate {
+  bytes requestorAci = 1;
+  // The aci that approved or rejected the request.
+  optional bytes updaterAci = 2;
+  bool wasApproved = 3;
+}
+
+message GroupJoinRequestCanceledUpdate {
+  bytes requestorAci = 1;
+}
+
+// A single requestor has requested to join and cancelled
+// their request repeatedly with no other updates in between.
+// The last action encompassed by this update is always a
+// cancellation; if there was another open request immediately
+// after, it will be a separate GroupJoinRequestUpdate, either
+// in the same frame or in a subsequent frame.
+message GroupSequenceOfRequestsAndCancelsUpdate {
+  bytes requestorAci = 1;
+  uint32 count = 2;
+}
+
+message GroupInviteLinkResetUpdate {
+  optional bytes updaterAci = 1;
+}
+
+message GroupInviteLinkEnabledUpdate {
+  optional bytes updaterAci = 1;
+  bool linkRequiresAdminApproval = 2;
+}
+
+message GroupInviteLinkAdminApprovalUpdate {
+  optional bytes updaterAci = 1;
+  bool linkRequiresAdminApproval = 2;
+}
+
+message GroupInviteLinkDisabledUpdate {
+  optional bytes updaterAci = 1;
+}
+
+message GroupMemberJoinedByLinkUpdate {
+  bytes newMemberAci = 1;
+}
+
+// A gv1->gv2 migration occurred.
+message GroupV2MigrationUpdate  {}
+
+// Another user migrated gv1->gv2 but was unable to add
+// the local user and invited them instead.
+message GroupV2MigrationSelfInvitedUpdate {}
+
+// The local user migrated gv1->gv2 but was unable to
+// add some members and invited them instead.
+// (Happens if we don't have the invitee's profile key)
+message GroupV2MigrationInvitedMembersUpdate {
+  uint32 invitedMembersCount = 1;
+}
+
+// The local user migrated gv1->gv2 but was unable to
+// add or invite some members and dropped them instead.
+// (Happens for e164 members where we don't have an aci).
+message GroupV2MigrationDroppedMembersUpdate {
+  uint32 droppedMembersCount = 1;
+}
+
+// For 1:1 timer updates, use ExpirationTimerChatUpdate.
+message GroupExpirationTimerUpdate {
+  uint32 expiresInMs = 1; // 0 means the expiration timer was disabled
+  optional bytes updaterAci = 2;
+}
+
+message StickerPack {
+  bytes packId = 1;
+  bytes packKey = 2;
+  string title = 3;
+  string author = 4;
+  repeated StickerPackSticker stickers = 5; // First one should be cover sticker.
+}
+
+message StickerPackSticker {
+  string emoji = 1;
+  uint32 id = 2;
+}

protos/SignalStorage.proto

@@ -174,7 +174,7 @@ message AccountRecord {
   optional bool readReceipts = 6;
   optional bool sealedSenderIndicators = 7;
   optional bool typingIndicators = 8;
-  optional bool proxiedLinkPreviews = 9;
+  reserved 9; // proxiedLinkPreviews
   optional bool noteToSelfMarkedUnread = 10;
   optional bool linkPreviews = 11;
   optional PhoneNumberSharingMode phoneNumberSharingMode = 12;

ts/CI.ts

@@ -8,6 +8,7 @@ import type { MessageAttributesType } from './model-types.d';
 import * as log from './logging/log';
 import { explodePromise } from './util/explodePromise';
 import { ipcInvoke } from './sql/channels';
+import { backupsService } from './services/backups';
 import { SECOND } from './util/durations';
 import { isSignalRoute } from './util/signalRoutes';
 import { strictAssert } from './util/assert';
@@ -16,6 +17,7 @@ type ResolveType = (data: unknown) => void;
 
 export type CIType = {
   deviceName: string;
+  backupData?: Uint8Array;
   getConversationId: (address: string | null) => string | null;
   getMessagesBySentAt(
     sentAt: number
@@ -31,9 +33,15 @@ export type CIType = {
     }
   ) => unknown;
   openSignalRoute(url: string): Promise<void>;
+  exportBackupToDisk(path: string): Promise<void>;
 };
 
-export function getCI(deviceName: string): CIType {
+export type GetCIOptionsType = Readonly<{
+  deviceName: string;
+  backupData?: Uint8Array;
+}>;
+
+export function getCI({ deviceName, backupData }: GetCIOptionsType): CIType {
   const eventListeners = new Map<string, Array<ResolveType>>();
   const completedEvents = new Map<string, Array<unknown>>();
 
@@ -150,8 +158,13 @@ export function getCI(deviceName: string): CIType {
     document.body.removeChild(a);
   }
 
+  async function exportBackupToDisk(path: string) {
+    await backupsService.exportToDisk(path);
+  }
+
   return {
     deviceName,
+    backupData,
     getConversationId,
     getMessagesBySentAt,
     handleEvent,
@@ -159,5 +172,6 @@ export function getCI(deviceName: string): CIType {
     solveChallenge,
     waitForEvent,
     openSignalRoute,
+    exportBackupToDisk,
   };
 }

ts/ConversationController.ts

@@ -4,6 +4,7 @@
 import { debounce, pick, uniq, without } from 'lodash';
 import PQueue from 'p-queue';
 import { v4 as generateUuid } from 'uuid';
+import { batch as batchDispatch } from 'react-redux';
 
 import type {
   ConversationModelCollectionType,
@@ -831,7 +832,7 @@ export class ConversationController {
   // Note: `doCombineConversations` is directly used within this function since both
   //   run on `_combineConversationsQueue` queue and we don't want deadlocks.
   private async doCheckForConflicts(): Promise<void> {
-    log.info('checkForConflicts: starting...');
+    log.info('ConversationController.checkForConflicts: starting...');
     const byServiceId = Object.create(null);
     const byE164 = Object.create(null);
     const byGroupV2Id = Object.create(null);
@@ -1420,13 +1421,17 @@ export class ConversationController {
       );
       await queue.onIdle();
 
-      // Hydrate the final set of conversations
-      this._conversations.add(
-        collection.filter(conversation => !conversation.isTemporary)
-      );
-
+      // It is alright to call it first because the 'add'/'update' events are
+      // triggered after updating the collection.
       this._initialFetchComplete = true;
 
+      // Hydrate the final set of conversations
+      batchDispatch(() => {
+        this._conversations.add(
+          collection.filter(conversation => !conversation.isTemporary)
+        );
+      });
+
       await Promise.all(
         this._conversations.map(async conversation => {
           try {
@@ -1466,7 +1471,10 @@ export class ConversationController {
           }
         })
       );
-      log.info('ConversationController: done with initial fetch');
+      log.info(
+        'ConversationController: done with initial fetch, ' +
+          `got ${this._conversations.length} conversations`
+      );
     } catch (error) {
       log.error(
         'ConversationController: initial fetch failed',

ts/components/installScreen/InstallScreenChoosingDeviceNameStep.stories.tsx

@@ -22,6 +22,7 @@ function Wrapper() {
     <InstallScreenChoosingDeviceNameStep
       i18n={i18n}
       deviceName={deviceName}
+      setBackupFile={action('setBackupFile')}
       setDeviceName={setDeviceName}
       onSubmit={action('onSubmit')}
     />

ts/components/installScreen/InstallScreenChoosingDeviceNameStep.tsx

@@ -6,6 +6,7 @@ import React, { useRef } from 'react';
 
 import type { LocalizerType } from '../../types/Util';
 import { normalizeDeviceName } from '../../util/normalizeDeviceName';
+import { getEnvironment, Environment } from '../../environment';
 
 import { Button, ButtonVariant } from '../Button';
 import { TitlebarDragArea } from '../TitlebarDragArea';
@@ -20,6 +21,7 @@ export type PropsType = {
   deviceName: string;
   i18n: LocalizerType;
   onSubmit: () => void;
+  setBackupFile: (file: File) => void;
   setDeviceName: (value: string) => void;
 };
 
@@ -27,6 +29,7 @@ export function InstallScreenChoosingDeviceNameStep({
   deviceName,
   i18n,
   onSubmit,
+  setBackupFile,
   setDeviceName,
 }: Readonly<PropsType>): ReactElement {
   const hasFocusedRef = useRef<boolean>(false);
@@ -42,6 +45,26 @@ export function InstallScreenChoosingDeviceNameStep({
     normalizedName.length > 0 &&
     normalizedName.length <= MAX_DEVICE_NAME_LENGTH;
 
+  let maybeBackupInput: JSX.Element | undefined;
+  if (getEnvironment() !== Environment.Production) {
+    maybeBackupInput = (
+      <label className="module-InstallScreenChoosingDeviceNameStep__input">
+        {/* Since this is only for testing - we don't require translation */}
+        Backup file:
+        <input
+          type="file"
+          accept=".bin"
+          onChange={event => {
+            const file = event.target.files && event.target.files[0];
+            if (file) {
+              setBackupFile(file);
+            }
+          }}
+        />
+      </label>
+    );
+  }
+
   return (
     <form
       className="module-InstallScreenChoosingDeviceNameStep"
@@ -62,6 +85,8 @@ export function InstallScreenChoosingDeviceNameStep({
           <h2>{i18n('icu:Install__choose-device-name__description')}</h2>
         </div>
         <div className="module-InstallScreenChoosingDeviceNameStep__inputs">
+          {maybeBackupInput}
+
           <input
             className="module-InstallScreenChoosingDeviceNameStep__input"
             id="deviceName"

ts/jobs/helpers/sendNormalMessage.ts

@@ -32,6 +32,7 @@ import type {
 } from '../../types/Attachment';
 import { copyCdnFields } from '../../util/attachments';
 import { LONG_MESSAGE } from '../../types/MIME';
+import { LONG_ATTACHMENT_LIMIT } from '../../types/Message';
 import type { RawBodyRange } from '../../types/BodyRange';
 import type {
   EmbeddedContactWithHydratedAvatar,
@@ -60,7 +61,6 @@ import {
 } from '../../util/editHelpers';
 import { getMessageSentTimestamp } from '../../util/getMessageSentTimestamp';
 
-const LONG_ATTACHMENT_LIMIT = 2048;
 const MAX_CONCURRENT_ATTACHMENT_UPLOADS = 5;
 
 export async function sendNormalMessage(

ts/models/conversations.ts

@@ -4970,7 +4970,7 @@ export class ConversationModel extends window.Backbone
   }
 
   getColor(): AvatarColorType {
-    return migrateColor(this.get('color'));
+    return migrateColor(this.getServiceId(), this.get('color'));
   }
 
   getConversationColor(): ConversationColorType | undefined {

ts/models/messages.ts

@@ -76,13 +76,11 @@ import {
   hasErrors,
   isCallHistory,
   isChatSessionRefreshed,
-  isContactRemovedNotification,
   isDeliveryIssue,
   isEndSession,
   isExpirationTimerUpdate,
   isGiftBadge,
   isGroupUpdate,
-  isGroupV1Migration,
   isGroupV2Change,
   isIncoming,
   isKeyChange,
@@ -273,29 +271,6 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
     return Number(this.get('received_at_ms') || this.get('received_at'));
   }
 
-  isNormalBubble(): boolean {
-    const { attributes } = this;
-
-    return (
-      !isCallHistory(attributes) &&
-      !isChatSessionRefreshed(attributes) &&
-      !isContactRemovedNotification(attributes) &&
-      !isConversationMerge(attributes) &&
-      !isEndSession(attributes) &&
-      !isExpirationTimerUpdate(attributes) &&
-      !isGroupUpdate(attributes) &&
-      !isGroupV1Migration(attributes) &&
-      !isGroupV2Change(attributes) &&
-      !isKeyChange(attributes) &&
-      !isPhoneNumberDiscovery(attributes) &&
-      !isTitleTransitionNotification(attributes) &&
-      !isProfileChange(attributes) &&
-      !isUniversalTimerNotification(attributes) &&
-      !isUnsupportedMessage(attributes) &&
-      !isVerifiedChange(attributes)
-    );
-  }
-
   async hydrateStoryContext(
     inMemoryMessage?: MessageAttributesType,
     {

ts/protobuf/index.ts

@@ -3,6 +3,10 @@
 
 import './wrap';
 
-import { signalservice as SignalService, signal as Signal } from './compiled';
+import {
+  signal as Signal,
+  signalbackups as Backups,
+  signalservice as SignalService,
+} from './compiled';
 
-export { SignalService, Signal };
+export { Backups, SignalService, Signal };

ts/services/backups/constants.ts

@@ -0,0 +1,4 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+export const BACKUP_VERSION = 1;

ts/services/backups/export.ts

@@ -0,0 +1,709 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import Long from 'long';
+import { Aci, Pni } from '@signalapp/libsignal-client';
+import pMap from 'p-map';
+import pTimeout from 'p-timeout';
+import { Readable } from 'stream';
+
+import { Backups } from '../../protobuf';
+import Data from '../../sql/Client';
+import type { PageMessagesCursorType } from '../../sql/Interface';
+import * as log from '../../logging/log';
+import { StorySendMode, MY_STORY_ID } from '../../types/Stories';
+import type { ServiceIdString } from '../../types/ServiceId';
+import type { RawBodyRange } from '../../types/BodyRange';
+import { LONG_ATTACHMENT_LIMIT } from '../../types/Message';
+import type {
+  ConversationAttributesType,
+  MessageAttributesType,
+  QuotedAttachment,
+  QuotedMessageType,
+} from '../../model-types.d';
+import { drop } from '../../util/drop';
+import { explodePromise } from '../../util/explodePromise';
+import {
+  isDirectConversation,
+  isGroupV2,
+  isMe,
+} from '../../util/whatTypeOfConversation';
+import { isConversationUnregistered } from '../../util/isConversationUnregistered';
+import { uuidToBytes } from '../../util/uuidToBytes';
+import { assertDev, strictAssert } from '../../util/assert';
+import { getSafeLongFromTimestamp } from '../../util/timestampLongUtils';
+import { MINUTE, SECOND, DurationInSeconds } from '../../util/durations';
+import {
+  PhoneNumberDiscoverability,
+  parsePhoneNumberDiscoverability,
+} from '../../util/phoneNumberDiscoverability';
+import {
+  PhoneNumberSharingMode,
+  parsePhoneNumberSharingMode,
+} from '../../util/phoneNumberSharingMode';
+import { missingCaseError } from '../../util/missingCaseError';
+import { isNormalBubble } from '../../state/selectors/message';
+import * as Bytes from '../../Bytes';
+import { canBeSynced as canPreferredReactionEmojiBeSynced } from '../../reactions/preferredReactionEmoji';
+import { SendStatus } from '../../messages/MessageSendState';
+import { BACKUP_VERSION } from './constants';
+
+const MAX_CONCURRENCY = 10;
+
+// We want a very generous timeout to make sure that we always resume write
+// access to the database.
+const FLUSH_TIMEOUT = 30 * MINUTE;
+
+// Threshold for reporting slow flushes
+const REPORTING_THRESHOLD = SECOND;
+
+type GetRecipientIdOptionsType =
+  | Readonly<{
+      serviceId: ServiceIdString;
+      id?: string;
+      e164?: string;
+    }>
+  | Readonly<{
+      serviceId?: ServiceIdString;
+      id: string;
+      e164?: string;
+    }>
+  | Readonly<{
+      serviceId?: ServiceIdString;
+      id?: string;
+      e164: string;
+    }>;
+
+export class BackupExportStream extends Readable {
+  private readonly convoIdToRecipientId = new Map<string, number>();
+  private buffers = new Array<Uint8Array>();
+  private nextRecipientId = 0;
+  private flushResolve: (() => void) | undefined;
+
+  public run(): void {
+    drop(
+      (async () => {
+        log.info('BackupExportStream: starting...');
+        await Data.pauseWriteAccess();
+
+        try {
+          await this.unsafeRun();
+        } catch (error) {
+          this.emit('error', error);
+        } finally {
+          await Data.resumeWriteAccess();
+          log.info('BackupExportStream: finished');
+        }
+      })()
+    );
+  }
+
+  private async unsafeRun(): Promise<void> {
+    this.push(
+      Backups.BackupInfo.encodeDelimited({
+        version: Long.fromNumber(BACKUP_VERSION),
+        backupTimeMs: getSafeLongFromTimestamp(Date.now()),
+      }).finish()
+    );
+
+    this.pushFrame({
+      account: await this.toAccountData(),
+    });
+    await this.flush();
+
+    const stats = {
+      conversations: 0,
+      chats: 0,
+      distributionLists: 0,
+      messages: 0,
+      skippedMessages: 0,
+    };
+
+    for (const { attributes } of window.ConversationController.getAll()) {
+      const recipientId = this.getRecipientId({
+        id: attributes.id,
+        serviceId: attributes.serviceId,
+        e164: attributes.e164,
+      });
+
+      const recipient = this.toRecipient(recipientId, attributes);
+      if (recipient === undefined) {
+        // Can't be backed up.
+        continue;
+      }
+
+      this.pushFrame({
+        recipient,
+      });
+
+      // eslint-disable-next-line no-await-in-loop
+      await this.flush();
+      stats.conversations += 1;
+    }
+
+    const distributionLists = await Data.getAllStoryDistributionsWithMembers();
+
+    for (const list of distributionLists) {
+      const { PrivacyMode } = Backups.DistributionList;
+
+      let privacyMode: Backups.DistributionList.PrivacyMode;
+      if (list.id === MY_STORY_ID) {
+        if (list.isBlockList) {
+          if (!list.members.length) {
+            privacyMode = PrivacyMode.ALL;
+          } else {
+            privacyMode = PrivacyMode.ALL_EXCEPT;
+          }
+        } else {
+          privacyMode = PrivacyMode.ONLY_WITH;
+        }
+      } else {
+        privacyMode = PrivacyMode.ONLY_WITH;
+      }
+
+      this.pushFrame({
+        recipient: {
+          id: this.getDistributionListRecipientId(),
+          distributionList: {
+            name: list.name,
+            distributionId: uuidToBytes(list.id),
+            allowReplies: list.allowsReplies,
+            deletionTimestamp: list.deletedAtTimestamp
+              ? Long.fromNumber(list.deletedAtTimestamp)
+              : null,
+            privacyMode,
+            memberRecipientIds: list.members.map(serviceId =>
+              this.getOrPushPrivateRecipient({ serviceId })
+            ),
+          },
+        },
+      });
+
+      // eslint-disable-next-line no-await-in-loop
+      await this.flush();
+      stats.distributionLists += 1;
+    }
+
+    for (const { attributes } of window.ConversationController.getAll()) {
+      const recipientId = this.getRecipientId(attributes);
+
+      this.pushFrame({
+        chat: {
+          // We don't have to use separate identifiers
+          id: recipientId,
+          recipientId,
+
+          archived: attributes.isArchived === true,
+          pinnedOrder: attributes.isPinned === true ? 1 : null,
+          expirationTimerMs:
+            attributes.expireTimer != null
+              ? Long.fromNumber(
+                  DurationInSeconds.toMillis(attributes.expireTimer)
+                )
+              : null,
+          muteUntilMs: getSafeLongFromTimestamp(attributes.muteExpiresAt),
+          markedUnread: attributes.markedUnread === true,
+          dontNotifyForMentionsIfMuted:
+            attributes.dontNotifyForMentionsIfMuted === true,
+        },
+      });
+
+      // eslint-disable-next-line no-await-in-loop
+      await this.flush();
+      stats.chats += 1;
+    }
+
+    let cursor: PageMessagesCursorType | undefined;
+
+    try {
+      while (!cursor?.done) {
+        // eslint-disable-next-line no-await-in-loop
+        const { messages, cursor: newCursor } = await Data.pageMessages(cursor);
+
+        // eslint-disable-next-line no-await-in-loop
+        const items = await pMap(
+          messages,
+          message => this.toChatItem(message),
+          { concurrency: MAX_CONCURRENCY }
+        );
+
+        for (const chatItem of items) {
+          if (chatItem === undefined) {
+            stats.skippedMessages += 1;
+            // Can't be backed up.
+            continue;
+          }
+
+          this.pushFrame({
+            chatItem,
+          });
+
+          // eslint-disable-next-line no-await-in-loop
+          await this.flush();
+          stats.messages += 1;
+        }
+
+        cursor = newCursor;
+      }
+    } finally {
+      if (cursor !== undefined) {
+        await Data.finishPageMessages(cursor);
+      }
+    }
+
+    await this.flush();
+    log.warn('backups: final stats', stats);
+
+    this.push(null);
+  }
+
+  private pushBuffer(buffer: Uint8Array): void {
+    this.buffers.push(buffer);
+  }
+
+  private pushFrame(frame: Backups.IFrame): void {
+    this.pushBuffer(Backups.Frame.encodeDelimited(frame).finish());
+  }
+
+  private async flush(): Promise<void> {
+    const chunk = Bytes.concatenate(this.buffers);
+    this.buffers = [];
+
+    // Below watermark, no pausing required
+    if (this.push(chunk)) {
+      return;
+    }
+
+    const { promise, resolve } = explodePromise<void>();
+    strictAssert(this.flushResolve === undefined, 'flush already pending');
+    this.flushResolve = resolve;
+
+    const start = Date.now();
+    log.info('backups: flush paused due to pushback');
+    try {
+      await pTimeout(promise, FLUSH_TIMEOUT);
+    } finally {
+      const duration = Date.now() - start;
+      if (duration > REPORTING_THRESHOLD) {
+        log.info(`backups: flush resumed after ${duration}ms`);
+      }
+      this.flushResolve = undefined;
+    }
+  }
+
+  override _read(): void {
+    this.flushResolve?.();
+  }
+
+  private async toAccountData(): Promise<Backups.IAccountData> {
+    const { storage } = window;
+
+    const me = window.ConversationController.getOurConversationOrThrow();
+
+    const rawPreferredReactionEmoji = window.storage.get(
+      'preferredReactionEmoji'
+    );
+
+    let preferredReactionEmoji: Array<string> | undefined;
+    if (canPreferredReactionEmojiBeSynced(rawPreferredReactionEmoji)) {
+      preferredReactionEmoji = rawPreferredReactionEmoji;
+    }
+
+    const PHONE_NUMBER_SHARING_MODE_ENUM =
+      Backups.AccountData.PhoneNumberSharingMode;
+    const rawPhoneNumberSharingMode = parsePhoneNumberSharingMode(
+      storage.get('phoneNumberSharingMode')
+    );
+    let phoneNumberSharingMode: Backups.AccountData.PhoneNumberSharingMode;
+    switch (rawPhoneNumberSharingMode) {
+      case PhoneNumberSharingMode.Everybody:
+        phoneNumberSharingMode = PHONE_NUMBER_SHARING_MODE_ENUM.EVERYBODY;
+        break;
+      case PhoneNumberSharingMode.ContactsOnly:
+      case PhoneNumberSharingMode.Nobody:
+        phoneNumberSharingMode = PHONE_NUMBER_SHARING_MODE_ENUM.NOBODY;
+        break;
+      default:
+        throw missingCaseError(rawPhoneNumberSharingMode);
+    }
+
+    return {
+      profileKey: storage.get('profileKey'),
+      username: me.get('username'),
+      usernameLink: {
+        ...(storage.get('usernameLink') ?? {}),
+
+        // Same numeric value, no conversion needed
+        color: storage.get('usernameLinkColor'),
+      },
+      givenName: me.get('profileName'),
+      familyName: me.get('profileFamilyName'),
+      avatarUrlPath: storage.get('avatarUrl'),
+      subscriberId: storage.get('subscriberId'),
+      subscriberCurrencyCode: storage.get('subscriberCurrencyCode'),
+      accountSettings: {
+        readReceipts: storage.get('read-receipt-setting'),
+        sealedSenderIndicators: storage.get('sealedSenderIndicators'),
+        typingIndicators: window.Events.getTypingIndicatorSetting(),
+        linkPreviews: window.Events.getLinkPreviewSetting(),
+        notDiscoverableByPhoneNumber:
+          parsePhoneNumberDiscoverability(
+            storage.get('phoneNumberDiscoverability')
+          ) === PhoneNumberDiscoverability.NotDiscoverable,
+        preferContactAvatars: storage.get('preferContactAvatars'),
+        universalExpireTimer: storage.get('universalExpireTimer'),
+        preferredReactionEmoji,
+        displayBadgesOnProfile: storage.get('displayBadgesOnProfile'),
+        keepMutedChatsArchived: storage.get('keepMutedChatsArchived'),
+        hasSetMyStoriesPrivacy: storage.get('hasSetMyStoriesPrivacy'),
+        hasViewedOnboardingStory: storage.get('hasViewedOnboardingStory'),
+        storiesDisabled: storage.get('hasStoriesDisabled'),
+        storyViewReceiptsEnabled: storage.get('storyViewReceiptsEnabled'),
+        hasCompletedUsernameOnboarding: storage.get(
+          'hasCompletedUsernameOnboarding'
+        ),
+        phoneNumberSharingMode,
+      },
+    };
+  }
+
+  private getRecipientIdentifier({
+    id,
+    serviceId,
+    e164,
+  }: GetRecipientIdOptionsType): string {
+    const identifier = serviceId ?? e164 ?? id;
+    assertDev(identifier, 'Identifier cannot be blank');
+    return identifier;
+  }
+
+  private getRecipientId(options: GetRecipientIdOptionsType): Long {
+    const identifier = this.getRecipientIdentifier(options);
+
+    const existing = this.convoIdToRecipientId.get(identifier);
+    if (existing !== undefined) {
+      return Long.fromNumber(existing);
+    }
+
+    const { id, serviceId, e164 } = options;
+
+    const recipientId = this.nextRecipientId;
+    this.nextRecipientId += 1;
+
+    if (id !== undefined) {
+      this.convoIdToRecipientId.set(id, recipientId);
+    }
+    if (serviceId !== undefined) {
+      this.convoIdToRecipientId.set(serviceId, recipientId);
+    }
+    if (e164 !== undefined) {
+      this.convoIdToRecipientId.set(e164, recipientId);
+    }
+    const result = Long.fromNumber(recipientId);
+
+    return result;
+  }
+
+  private getOrPushPrivateRecipient(options: GetRecipientIdOptionsType): Long {
+    const identifier = this.getRecipientIdentifier(options);
+    const needsPush = !this.convoIdToRecipientId.has(identifier);
+    const result = this.getRecipientId(options);
+
+    if (needsPush) {
+      const { serviceId, e164 } = options;
+      this.pushFrame({
+        recipient: this.toRecipient(result, {
+          type: 'private',
+          serviceId,
+          e164,
+        }),
+      });
+    }
+
+    return result;
+  }
+
+  private getDistributionListRecipientId(): Long {
+    const recipientId = this.nextRecipientId;
+    this.nextRecipientId += 1;
+
+    return Long.fromNumber(recipientId);
+  }
+
+  private toRecipient(
+    recipientId: Long,
+    convo: Omit<ConversationAttributesType, 'id' | 'version'>
+  ): Backups.IRecipient | undefined {
+    const res: Backups.IRecipient = {
+      id: recipientId,
+    };
+
+    if (isMe(convo)) {
+      res.self = {};
+    } else if (isDirectConversation(convo)) {
+      const { Registered } = Backups.Contact;
+      res.contact = {
+        aci:
+          convo.serviceId && convo.serviceId !== convo.pni
+            ? Aci.parseFromServiceIdString(convo.serviceId).getRawUuidBytes()
+            : null,
+        pni: convo.pni
+          ? Pni.parseFromServiceIdString(convo.pni).getRawUuidBytes()
+          : null,
+        username: convo.username,
+        e164: convo.e164 ? Long.fromString(convo.e164) : null,
+        blocked: convo.serviceId
+          ? window.storage.blocked.isServiceIdBlocked(convo.serviceId)
+          : null,
+        hidden: convo.removalStage !== undefined,
+        registered: isConversationUnregistered(convo)
+          ? Registered.NOT_REGISTERED
+          : Registered.REGISTERED,
+        unregisteredTimestamp: convo.firstUnregisteredAt
+          ? Long.fromNumber(convo.firstUnregisteredAt)
+          : null,
+        profileKey: convo.profileKey
+          ? Bytes.fromBase64(convo.profileKey)
+          : null,
+        profileSharing: convo.profileSharing,
+        profileGivenName: convo.profileName,
+        profileFamilyName: convo.profileFamilyName,
+        hideStory: convo.hideStory === true,
+      };
+    } else if (isGroupV2(convo) && convo.masterKey) {
+      let storySendMode: Backups.Group.StorySendMode;
+      switch (convo.storySendMode) {
+        case StorySendMode.Always:
+          storySendMode = Backups.Group.StorySendMode.ENABLED;
+          break;
+        case StorySendMode.Never:
+          storySendMode = Backups.Group.StorySendMode.DISABLED;
+          break;
+        default:
+          storySendMode = Backups.Group.StorySendMode.DEFAULT;
+          break;
+      }
+
+      res.group = {
+        masterKey: Bytes.fromBase64(convo.masterKey),
+        whitelisted: convo.profileSharing,
+        hideStory: convo.hideStory === true,
+        storySendMode,
+      };
+    } else {
+      return undefined;
+    }
+
+    return res;
+  }
+
+  private async toChatItem(
+    message: MessageAttributesType
+  ): Promise<Backups.IChatItem | undefined> {
+    if (!isNormalBubble(message)) {
+      return undefined;
+    }
+
+    const chatId = this.getRecipientId({ id: message.conversationId });
+    if (chatId === undefined) {
+      log.warn('backups: message chat not found');
+      return undefined;
+    }
+
+    let authorId: Long;
+
+    const isOutgoing = message.type === 'outgoing';
+
+    if (isOutgoing) {
+      const ourAci = window.storage.user.getCheckedAci();
+
+      authorId = this.getOrPushPrivateRecipient({
+        serviceId: ourAci,
+      });
+      // Pacify typescript
+    } else if (message.sourceServiceId) {
+      authorId = this.getOrPushPrivateRecipient({
+        serviceId: message.sourceServiceId,
+        e164: message.source,
+      });
+    } else if (message.source) {
+      authorId = this.getOrPushPrivateRecipient({
+        serviceId: message.sourceServiceId,
+        e164: message.source,
+      });
+    } else {
+      return undefined;
+    }
+
+    const result: Backups.IChatItem = {
+      chatId,
+      authorId,
+      dateSent: getSafeLongFromTimestamp(message.sent_at),
+      expireStartDate:
+        message.expirationStartTimestamp != null
+          ? getSafeLongFromTimestamp(message.expirationStartTimestamp)
+          : null,
+      expiresInMs:
+        message.expireTimer != null
+          ? Long.fromNumber(DurationInSeconds.toMillis(message.expireTimer))
+          : null,
+      revisions: [],
+      sms: false,
+      standardMessage: {
+        quote: await this.toQuote(message.quote),
+        text: {
+          // Note that we store full text on the message model so we have to
+          // trim it before serializing.
+          body: message.body?.slice(0, LONG_ATTACHMENT_LIMIT),
+          bodyRanges: message.bodyRanges?.map(range => this.toBodyRange(range)),
+        },
+
+        linkPreview: message.preview?.map(preview => {
+          return {
+            url: preview.url,
+            title: preview.title,
+            description: preview.description,
+            date: getSafeLongFromTimestamp(preview.date),
+          };
+        }),
+        reactions: message.reactions?.map(reaction => {
+          return {
+            emoji: reaction.emoji,
+            authorId: this.getOrPushPrivateRecipient({
+              id: reaction.fromId,
+            }),
+            sentTimestamp: getSafeLongFromTimestamp(reaction.timestamp),
+            receivedTimestamp: getSafeLongFromTimestamp(
+              reaction.receivedAtDate ?? reaction.timestamp
+            ),
+          };
+        }),
+      },
+    };
+
+    if (isOutgoing) {
+      const BackupSendStatus = Backups.SendStatus.Status;
+
+      const sendStatus = new Array<Backups.ISendStatus>();
+      const { sendStateByConversationId = {} } = message;
+      for (const [id, entry] of Object.entries(sendStateByConversationId)) {
+        const target = window.ConversationController.get(id);
+        strictAssert(target != null, 'Send target not found');
+
+        let deliveryStatus: Backups.SendStatus.Status;
+        switch (entry.status) {
+          case SendStatus.Pending:
+            deliveryStatus = BackupSendStatus.PENDING;
+            break;
+          case SendStatus.Sent:
+            deliveryStatus = BackupSendStatus.SENT;
+            break;
+          case SendStatus.Delivered:
+            deliveryStatus = BackupSendStatus.DELIVERED;
+            break;
+          case SendStatus.Read:
+            deliveryStatus = BackupSendStatus.READ;
+            break;
+          case SendStatus.Viewed:
+            deliveryStatus = BackupSendStatus.VIEWED;
+            break;
+          case SendStatus.Failed:
+            deliveryStatus = BackupSendStatus.FAILED;
+            break;
+          default:
+            throw missingCaseError(entry.status);
+        }
+
+        sendStatus.push({
+          recipientId: this.getOrPushPrivateRecipient(target.attributes),
+          lastStatusUpdateTimestamp:
+            entry.updatedAt != null
+              ? getSafeLongFromTimestamp(entry.updatedAt)
+              : null,
+          deliveryStatus,
+        });
+      }
+      result.outgoing = {
+        sendStatus,
+      };
+    } else {
+      result.incoming = {
+        dateReceived:
+          message.received_at_ms != null
+            ? getSafeLongFromTimestamp(message.received_at_ms)
+            : null,
+        dateServerSent:
+          message.serverTimestamp != null
+            ? getSafeLongFromTimestamp(message.serverTimestamp)
+            : null,
+        read: Boolean(message.readAt),
+      };
+    }
+
+    return result;
+  }
+
+  private async toQuote(
+    quote?: QuotedMessageType
+  ): Promise<Backups.IQuote | null> {
+    if (!quote) {
+      return null;
+    }
+
+    const quotedMessage = await Data.getMessageById(quote.messageId);
+
+    let authorId: Long;
+    if (quote.authorAci) {
+      authorId = this.getOrPushPrivateRecipient({
+        serviceId: quote.authorAci,
+        e164: quote.author,
+      });
+    } else if (quote.author) {
+      authorId = this.getOrPushPrivateRecipient({
+        serviceId: quote.authorAci,
+        e164: quote.author,
+      });
+    } else {
+      log.warn('backups: quote has no author id');
+      return null;
+    }
+
+    return {
+      targetSentTimestamp:
+        quotedMessage && !quote.referencedMessageNotFound
+          ? Long.fromNumber(quotedMessage.sent_at)
+          : null,
+      authorId,
+      text: quote.text,
+      attachments: quote.attachments.map((attachment: QuotedAttachment) => {
+        return {
+          contentType: attachment.contentType,
+          fileName: attachment.fileName,
+          thumbnail: null,
+        };
+      }),
+      bodyRanges: quote.bodyRanges?.map(range => this.toBodyRange(range)),
+      type: quote.isGiftBadge
+        ? Backups.Quote.Type.GIFTBADGE
+        : Backups.Quote.Type.NORMAL,
+    };
+  }
+
+  private toBodyRange(range: RawBodyRange): Backups.IBodyRange {
+    return {
+      start: range.start,
+      length: range.length,
+
+      ...('mentionAci' in range
+        ? {
+            mentionAci: Aci.parseFromServiceIdString(
+              range.mentionAci
+            ).getRawUuidBytes(),
+          }
+        : {
+            // Numeric values are compatible between backup and message protos
+            style: range.style,
+          }),
+    };
+  }
+}

ts/services/backups/import.ts

@@ -0,0 +1,520 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { Aci, Pni } from '@signalapp/libsignal-client';
+import { v4 as generateUuid } from 'uuid';
+import pMap from 'p-map';
+import { Writable } from 'stream';
+
+import { Backups } from '../../protobuf';
+import Data from '../../sql/Client';
+import * as log from '../../logging/log';
+import { StorySendMode } from '../../types/Stories';
+import { fromAciObject, fromPniObject } from '../../types/ServiceId';
+import * as Errors from '../../types/errors';
+import type {
+  ConversationAttributesType,
+  MessageAttributesType,
+} from '../../model-types.d';
+import { assertDev, strictAssert } from '../../util/assert';
+import { getTimestampFromLong } from '../../util/timestampLongUtils';
+import { DurationInSeconds } from '../../util/durations';
+import { dropNull } from '../../util/dropNull';
+import {
+  deriveGroupID,
+  deriveGroupSecretParams,
+  deriveGroupPublicParams,
+} from '../../util/zkgroup';
+import { incrementMessageCounter } from '../../util/incrementMessageCounter';
+import { isAciString } from '../../util/isAciString';
+import { createBatcher } from '../../util/batcher';
+import { ReadStatus } from '../../messages/MessageReadStatus';
+import { SendStatus } from '../../messages/MessageSendState';
+import type { SendStateByConversationId } from '../../messages/MessageSendState';
+import { SeenStatus } from '../../MessageSeenStatus';
+import * as Bytes from '../../Bytes';
+import { BACKUP_VERSION } from './constants';
+
+const MAX_CONCURRENCY = 10;
+
+type ConversationOpType = Readonly<{
+  isUpdate: boolean;
+  attributes: ConversationAttributesType;
+}>;
+
+async function processConversationOpBatch(
+  batch: ReadonlyArray<ConversationOpType>
+): Promise<void> {
+  // Note that we might have duplicates since we update attributes in-place
+  const saves = [
+    ...new Set(batch.filter(x => x.isUpdate === false).map(x => x.attributes)),
+  ];
+  const updates = [
+    ...new Set(batch.filter(x => x.isUpdate === true).map(x => x.attributes)),
+  ];
+
+  log.info(
+    `backups: running conversation op batch, saves=${saves.length} ` +
+      `updates=${updates.length}`
+  );
+
+  await Data.saveConversations(saves);
+  await Data.updateConversations(updates);
+}
+
+export class BackupImportStream extends Writable {
+  private parsedBackupInfo = false;
+  private logId = 'BackupImportStream(unknown)';
+
+  private readonly recipientIdToConvo = new Map<
+    number,
+    ConversationAttributesType
+  >();
+  private readonly chatIdToConvo = new Map<
+    number,
+    ConversationAttributesType
+  >();
+  private readonly conversationOpBatcher = createBatcher<{
+    isUpdate: boolean;
+    attributes: ConversationAttributesType;
+  }>({
+    name: 'BackupImport.conversationOpBatcher',
+    wait: 0,
+    maxSize: 1000,
+    processBatch: processConversationOpBatch,
+  });
+  private readonly saveMessageBatcher = createBatcher<MessageAttributesType>({
+    name: 'BackupImport.saveMessageBatcher',
+    wait: 0,
+    maxSize: 1000,
+    processBatch: batch => {
+      const ourAci = this.ourConversation?.serviceId;
+      assertDev(isAciString(ourAci), 'Our conversation must have ACI');
+      return Data.saveMessages(batch, {
+        forceSave: true,
+        ourAci,
+      });
+    },
+  });
+  private ourConversation?: ConversationAttributesType;
+
+  constructor() {
+    super({ objectMode: true });
+  }
+
+  override async _write(
+    data: Buffer,
+    _enc: BufferEncoding,
+    done: (error?: Error) => void
+  ): Promise<void> {
+    try {
+      if (!this.parsedBackupInfo) {
+        const info = Backups.BackupInfo.decode(data);
+        this.parsedBackupInfo = true;
+
+        this.logId = `BackupImport.run(${info.backupTimeMs})`;
+
+        log.info(`${this.logId}: got BackupInfo`);
+
+        if (info.version?.toNumber() !== BACKUP_VERSION) {
+          throw new Error(`Unsupported backup version: ${info.version}`);
+        }
+      } else {
+        const frame = Backups.Frame.decode(data);
+
+        await this.processFrame(frame);
+      }
+      done();
+    } catch (error) {
+      const entryType = this.parsedBackupInfo ? 'frame' : 'info';
+      log.error(`${this.logId}: failed to process ${entryType}`);
+      done(error);
+    }
+  }
+
+  override async _final(done: (error?: Error) => void): Promise<void> {
+    try {
+      // Finish saving remaining conversations/messages
+      await this.conversationOpBatcher.flushAndWait();
+      await this.saveMessageBatcher.flushAndWait();
+
+      // Reset and reload conversations and storage again
+      window.ConversationController.reset();
+
+      await window.ConversationController.load();
+      await window.ConversationController.checkForConflicts();
+
+      window.storage.reset();
+      await window.storage.fetch();
+
+      // Update last message in every active conversation now that we have
+      // them loaded into memory.
+      await pMap(
+        window.ConversationController.getAll().filter(convo => {
+          return convo.get('active_at') || convo.get('isPinned');
+        }),
+        convo => convo.updateLastMessage(),
+        { concurrency: MAX_CONCURRENCY }
+      );
+
+      done();
+    } catch (error) {
+      done(error);
+    }
+  }
+
+  public cleanup(): void {
+    this.conversationOpBatcher.unregister();
+    this.saveMessageBatcher.unregister();
+  }
+
+  private async processFrame(frame: Backups.Frame): Promise<void> {
+    if (frame.account) {
+      await this.fromAccount(frame.account);
+
+      // We run this outside of try catch below because failure to restore
+      // the account data is fatal.
+      return;
+    }
+
+    try {
+      if (frame.recipient) {
+        const { recipient } = frame;
+        strictAssert(recipient.id != null, 'Recipient must have an id');
+        let convo: ConversationAttributesType;
+        if (recipient.contact) {
+          convo = await this.fromContact(recipient.contact);
+        } else if (recipient.self) {
+          strictAssert(this.ourConversation != null, 'Missing account data');
+          convo = this.ourConversation;
+        } else if (recipient.group) {
+          convo = await this.fromGroup(recipient.group);
+        } else {
+          log.warn(`${this.logId}: unsupported recipient item`);
+          return;
+        }
+
+        if (convo !== this.ourConversation) {
+          this.saveConversation(convo);
+        }
+
+        this.recipientIdToConvo.set(recipient.id.toNumber(), convo);
+      } else if (frame.chat) {
+        await this.fromChat(frame.chat);
+      } else if (frame.chatItem) {
+        await this.fromChatItem(frame.chatItem);
+      } else {
+        log.warn(`${this.logId}: unsupported frame item ${frame.item}`);
+      }
+    } catch (error) {
+      log.error(
+        `${this.logId}: failed to process a frame ${frame.item}, ` +
+          `${Errors.toLogFormat(error)}`
+      );
+    }
+  }
+
+  private saveConversation(attributes: ConversationAttributesType): void {
+    this.conversationOpBatcher.add({ isUpdate: false, attributes });
+  }
+
+  private updateConversation(attributes: ConversationAttributesType): void {
+    this.conversationOpBatcher.add({ isUpdate: true, attributes });
+  }
+
+  private saveMessage(attributes: MessageAttributesType): void {
+    this.saveMessageBatcher.add(attributes);
+  }
+
+  private async fromAccount(_account: Backups.IAccountData): Promise<void> {
+    strictAssert(this.ourConversation === undefined, 'Duplicate AccountData');
+    this.ourConversation =
+      window.ConversationController.getOurConversationOrThrow().attributes;
+  }
+
+  private async fromContact(
+    contact: Backups.IContact
+  ): Promise<ConversationAttributesType> {
+    strictAssert(
+      contact.aci != null || contact.pni != null || contact.e164 != null,
+      'fromContact: either aci, pni, or e164 must be present'
+    );
+
+    const aci = contact.aci
+      ? fromAciObject(Aci.fromUuidBytes(contact.aci))
+      : undefined;
+    const pni = contact.pni
+      ? fromPniObject(Pni.fromUuidBytes(contact.pni))
+      : undefined;
+    const e164 = contact.e164 ? `+${contact.e164}` : undefined;
+
+    const attrs: ConversationAttributesType = {
+      id: generateUuid(),
+      type: 'private',
+      version: 2,
+      serviceId: aci ?? pni,
+      pni,
+      e164,
+      removalStage: contact.hidden ? 'messageRequest' : undefined,
+      profileKey: contact.profileKey
+        ? Bytes.toBase64(contact.profileKey)
+        : undefined,
+      profileSharing: contact.profileSharing === true,
+      profileName: dropNull(contact.profileGivenName),
+      profileFamilyName: dropNull(contact.profileFamilyName),
+      hideStory: contact.hideStory === true,
+    };
+
+    if (contact.registered === Backups.Contact.Registered.NOT_REGISTERED) {
+      const timestamp = contact.unregisteredTimestamp?.toNumber() ?? Date.now();
+      attrs.discoveredUnregisteredAt = timestamp;
+      attrs.firstUnregisteredAt = timestamp;
+    }
+
+    if (contact.blocked) {
+      const serviceId = aci || pni;
+      if (serviceId) {
+        await window.storage.blocked.addBlockedServiceId(serviceId);
+      }
+      if (e164) {
+        await window.storage.blocked.addBlockedNumber(e164);
+      }
+    }
+
+    return attrs;
+  }
+
+  private async fromGroup(
+    group: Backups.IGroup
+  ): Promise<ConversationAttributesType> {
+    strictAssert(group.masterKey != null, 'fromGroup: missing masterKey');
+
+    const secretParams = deriveGroupSecretParams(group.masterKey);
+    const publicParams = deriveGroupPublicParams(secretParams);
+    const groupId = Bytes.toBase64(deriveGroupID(secretParams));
+
+    const attrs: ConversationAttributesType = {
+      id: generateUuid(),
+      type: 'group',
+      version: 2,
+      groupVersion: 2,
+      masterKey: Bytes.toBase64(group.masterKey),
+      groupId,
+      secretParams: Bytes.toBase64(secretParams),
+      publicParams: Bytes.toBase64(publicParams),
+      profileSharing: group.whitelisted === true,
+      hideStory: group.hideStory === true,
+    };
+
+    if (group.storySendMode === Backups.Group.StorySendMode.ENABLED) {
+      attrs.storySendMode = StorySendMode.Always;
+    } else if (group.storySendMode === Backups.Group.StorySendMode.DISABLED) {
+      attrs.storySendMode = StorySendMode.Never;
+    }
+
+    return attrs;
+  }
+
+  private async fromChat(chat: Backups.IChat): Promise<void> {
+    strictAssert(chat.id != null, 'chat must have an id');
+    strictAssert(chat.recipientId != null, 'chat must have a recipientId');
+
+    const conversation = this.recipientIdToConvo.get(
+      chat.recipientId.toNumber()
+    );
+    strictAssert(conversation !== undefined, 'unknown conversation');
+
+    this.chatIdToConvo.set(chat.id.toNumber(), conversation);
+
+    conversation.isArchived = chat.archived === true;
+    conversation.isPinned = chat.pinnedOrder != null;
+
+    conversation.expireTimer = chat.expirationTimerMs
+      ? DurationInSeconds.fromMillis(chat.expirationTimerMs.toNumber())
+      : undefined;
+    conversation.muteExpiresAt = chat.muteUntilMs
+      ? getTimestampFromLong(chat.muteUntilMs)
+      : undefined;
+    conversation.markedUnread = chat.markedUnread === true;
+    conversation.dontNotifyForMentionsIfMuted =
+      chat.dontNotifyForMentionsIfMuted === true;
+
+    this.updateConversation(conversation);
+
+    if (chat.pinnedOrder != null) {
+      const pinnedConversationIds = new Set(
+        window.storage.get('pinnedConversationIds', new Array<string>())
+      );
+
+      pinnedConversationIds.add(conversation.id);
+
+      await window.storage.put('pinnedConversationIds', [
+        ...pinnedConversationIds,
+      ]);
+    }
+  }
+
+  private async fromChatItem(item: Backups.IChatItem): Promise<void> {
+    strictAssert(this.ourConversation != null, 'AccountData missing');
+
+    strictAssert(item.chatId != null, 'chatItem must have a chatId');
+    strictAssert(item.authorId != null, 'chatItem must have a authorId');
+    strictAssert(item.dateSent != null, 'chatItem must have a dateSent');
+
+    const chatConvo = this.chatIdToConvo.get(item.chatId.toNumber());
+    strictAssert(chatConvo !== undefined, 'chat conversation not found');
+
+    const authorConvo = this.recipientIdToConvo.get(item.authorId.toNumber());
+    strictAssert(authorConvo !== undefined, 'author conversation not found');
+
+    const isOutgoing = this.ourConversation.id === authorConvo.id;
+
+    let attributes: MessageAttributesType = {
+      id: generateUuid(),
+      canReplyToStory: false,
+      conversationId: chatConvo.id,
+      received_at: incrementMessageCounter(),
+      sent_at: item.dateSent.toNumber(),
+      source: authorConvo.e164,
+      sourceServiceId: authorConvo.serviceId,
+      timestamp: item.dateSent.toNumber(),
+      type: isOutgoing ? 'outgoing' : 'incoming',
+      unidentifiedDeliveryReceived: false,
+      expirationStartTimestamp: item.expireStartDate
+        ? getTimestampFromLong(item.expireStartDate)
+        : undefined,
+      expireTimer: item.expiresInMs
+        ? DurationInSeconds.fromMillis(item.expiresInMs.toNumber())
+        : undefined,
+    };
+
+    if (isOutgoing) {
+      const { outgoing } = item;
+      strictAssert(outgoing, 'outgoing message must have outgoing field');
+
+      const sendStateByConversationId: SendStateByConversationId = {};
+
+      const BackupSendStatus = Backups.SendStatus.Status;
+
+      for (const status of outgoing.sendStatus ?? []) {
+        strictAssert(
+          status.recipientId,
+          'sendStatus recipient must have an id'
+        );
+        const target = this.recipientIdToConvo.get(
+          status.recipientId.toNumber()
+        );
+        strictAssert(
+          target !== undefined,
+          'status target conversation not found'
+        );
+
+        let sendStatus: SendStatus;
+        switch (status.deliveryStatus) {
+          case BackupSendStatus.PENDING:
+            sendStatus = SendStatus.Pending;
+            break;
+          case BackupSendStatus.SENT:
+            sendStatus = SendStatus.Sent;
+            break;
+          case BackupSendStatus.DELIVERED:
+            sendStatus = SendStatus.Delivered;
+            break;
+          case BackupSendStatus.READ:
+            sendStatus = SendStatus.Read;
+            break;
+          case BackupSendStatus.VIEWED:
+            sendStatus = SendStatus.Viewed;
+            break;
+          case BackupSendStatus.FAILED:
+          default:
+            sendStatus = SendStatus.Failed;
+            break;
+        }
+
+        sendStateByConversationId[target.id] = {
+          status: sendStatus,
+          updatedAt:
+            status.lastStatusUpdateTimestamp != null
+              ? getTimestampFromLong(status.lastStatusUpdateTimestamp)
+              : undefined,
+        };
+      }
+
+      attributes.sendStateByConversationId = sendStateByConversationId;
+      chatConvo.active_at = attributes.sent_at;
+    } else {
+      const { incoming } = item;
+      strictAssert(incoming, 'incoming message must have incoming field');
+      attributes.received_at_ms =
+        incoming.dateReceived?.toNumber() ?? Date.now();
+
+      if (incoming.read) {
+        attributes.readStatus = ReadStatus.Read;
+        attributes.seenStatus = SeenStatus.Seen;
+      } else {
+        attributes.readStatus = ReadStatus.Unread;
+        attributes.seenStatus = SeenStatus.Unseen;
+        chatConvo.unreadCount = (chatConvo.unreadCount ?? 0) + 1;
+      }
+
+      chatConvo.active_at = attributes.received_at_ms;
+    }
+
+    if (item.standardMessage) {
+      attributes = {
+        ...attributes,
+        ...this.fromStandardMessage(item.standardMessage),
+      };
+    }
+
+    assertDev(
+      isAciString(this.ourConversation.serviceId),
+      'Our conversation must have ACI'
+    );
+    this.saveMessage(attributes);
+
+    if (isOutgoing) {
+      chatConvo.sentMessageCount = (chatConvo.sentMessageCount ?? 0) + 1;
+    } else {
+      chatConvo.messageCount = (chatConvo.messageCount ?? 0) + 1;
+    }
+    this.updateConversation(chatConvo);
+  }
+
+  private fromStandardMessage(
+    data: Backups.IStandardMessage
+  ): Partial<MessageAttributesType> {
+    return {
+      body: data.text?.body ?? '',
+      reactions: data.reactions?.map(
+        ({ emoji, authorId, sentTimestamp, receivedTimestamp }) => {
+          strictAssert(emoji != null, 'reaction must have an emoji');
+          strictAssert(authorId != null, 'reaction must have authorId');
+          strictAssert(
+            sentTimestamp != null,
+            'reaction must have a sentTimestamp'
+          );
+          strictAssert(
+            receivedTimestamp != null,
+            'reaction must have a receivedTimestamp'
+          );
+
+          const authorConvo = this.recipientIdToConvo.get(authorId.toNumber());
+          strictAssert(
+            authorConvo !== undefined,
+            'author conversation not found'
+          );
+
+          return {
+            emoji,
+            fromId: authorConvo.id,
+            targetTimestamp: getTimestampFromLong(sentTimestamp),
+            receivedAtDate: getTimestampFromLong(receivedTimestamp),
+            timestamp: getTimestampFromLong(sentTimestamp),
+          };
+        }
+      ),
+    };
+  }
+}

ts/services/backups/index.ts

@@ -0,0 +1,95 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { pipeline } from 'stream/promises';
+import { Readable } from 'stream';
+import { createWriteStream } from 'fs';
+
+import * as log from '../../logging/log';
+import * as Bytes from '../../Bytes';
+import { DelimitedStream } from '../../util/DelimitedStream';
+import * as Errors from '../../types/errors';
+import { BackupExportStream } from './export';
+import { BackupImportStream } from './import';
+
+export class BackupsService {
+  private isRunning = false;
+
+  public exportBackup(): Readable {
+    if (this.isRunning) {
+      throw new Error('BackupService is already running');
+    }
+
+    log.info('exportBackup: starting...');
+    this.isRunning = true;
+
+    const stream = new BackupExportStream();
+    const cleanup = () => {
+      // Don't fire twice
+      stream.removeListener('end', cleanup);
+      stream.removeListener('error', cleanup);
+
+      log.info('exportBackup: finished...');
+      this.isRunning = false;
+    };
+
+    stream.once('end', cleanup);
+    stream.once('error', cleanup);
+
+    stream.run();
+
+    return stream;
+  }
+
+  // Test harness
+  public async exportBackupData(): Promise<Uint8Array> {
+    const chunks = new Array<Uint8Array>();
+    for await (const chunk of this.exportBackup()) {
+      chunks.push(chunk);
+    }
+
+    return Bytes.concatenate(chunks);
+  }
+
+  // Test harness
+  public async exportToDisk(path: string): Promise<void> {
+    await pipeline(this.exportBackup(), createWriteStream(path));
+  }
+
+  // Test harness
+  public async exportWithDialog(): Promise<void> {
+    const data = await this.exportBackupData();
+
+    const { saveAttachmentToDisk } = window.Signal.Migrations;
+
+    await saveAttachmentToDisk({
+      name: 'backup.bin',
+      data,
+    });
+  }
+
+  public async importBackup(backup: Uint8Array): Promise<void> {
+    if (this.isRunning) {
+      throw new Error('BackupService is already running');
+    }
+
+    log.info('importBackup: starting...');
+    this.isRunning = true;
+
+    try {
+      await pipeline(
+        Readable.from(backup),
+        new DelimitedStream(),
+        new BackupImportStream()
+      );
+      log.info('importBackup: finished...');
+    } catch (error) {
+      log.info(`importBackup: failed, error: ${Errors.toLogFormat(error)}`);
+      throw error;
+    } finally {
+      this.isRunning = false;
+    }
+  }
+}
+
+export const backupsService = new BackupsService();

ts/signal.ts

@@ -31,6 +31,7 @@ import { initializeNetworkObserver } from './services/networkObserver';
 import { initializeUpdateListener } from './services/updateListener';
 import { calling } from './services/calling';
 import * as storage from './services/storage';
+import { backupsService } from './services/backups';
 
 import type { LoggerType } from './types/Logging';
 import type {
@@ -370,6 +371,7 @@ export const setup = (options: {
   };
 
   const Services = {
+    backups: backupsService,
     calling,
     initializeGroupCredentialFetcher,
     initializeNetworkObserver,

ts/sql/Interface.ts

@@ -395,17 +395,32 @@ export type GetConversationRangeCenteredOnMessageResultType<Message> =
     metrics: ConversationMetricsType;
   }>;
 
-export type MessageAttachmentsCursorType = Readonly<{
+export type MessageCursorType = Readonly<{
   done: boolean;
   runId: string;
   count: number;
 }>;
 
+export type MessageAttachmentsCursorType = MessageCursorType &
+  Readonly<{
+    __message_attachments_cursor: never;
+  }>;
+
 export type GetKnownMessageAttachmentsResultType = Readonly<{
   cursor: MessageAttachmentsCursorType;
   attachments: ReadonlyArray<string>;
 }>;
 
+export type PageMessagesCursorType = MessageCursorType &
+  Readonly<{
+    __page_messages_cursor: never;
+  }>;
+
+export type PageMessagesResultType = Readonly<{
+  cursor: PageMessagesCursorType;
+  messages: ReadonlyArray<MessageAttributesType>;
+}>;
+
 export type GetAllStoriesResultType = ReadonlyArray<
   MessageType & {
     hasReplies: boolean;
@@ -427,6 +442,9 @@ export type EditedMessageType = Readonly<{
 
 export type DataInterface = {
   close: () => Promise<void>;
+  pauseWriteAccess(): Promise<void>;
+  resumeWriteAccess(): Promise<void>;
+
   removeDB: () => Promise<void>;
   removeIndexedDBFiles: () => Promise<void>;
 
@@ -541,6 +559,10 @@ export type DataInterface = {
   ) => Promise<void>;
   removeMessage: (id: string) => Promise<void>;
   removeMessages: (ids: ReadonlyArray<string>) => Promise<void>;
+  pageMessages: (
+    cursor?: PageMessagesCursorType
+  ) => Promise<PageMessagesResultType>;
+  finishPageMessages: (cursor: PageMessagesCursorType) => Promise<void>;
   getTotalUnreadForConversation: (
     conversationId: string,
     options: {

ts/sql/Server.ts

@@ -10,6 +10,7 @@ import { randomBytes } from 'crypto';
 import type { Database, Statement } from '@signalapp/better-sqlite3';
 import SQL from '@signalapp/better-sqlite3';
 import pProps from 'p-props';
+import pTimeout from 'p-timeout';
 import { v4 as generateUuid } from 'uuid';
 import { z } from 'zod';
 
@@ -48,6 +49,7 @@ import { isNormalNumber } from '../util/isNormalNumber';
 import { isNotNil } from '../util/isNotNil';
 import { parseIntOrThrow } from '../util/parseIntOrThrow';
 import * as durations from '../util/durations';
+import { explodePromise } from '../util/explodePromise';
 import { formatCountForLogging } from '../logging/formatCountForLogging';
 import type { ConversationColorType, CustomColorType } from '../types/Colors';
 import type { BadgeType, BadgeImageType } from '../badges/types';
@@ -106,9 +108,12 @@ import type {
   StoredItemType,
   ConversationMessageStatsType,
   MessageAttachmentsCursorType,
+  MessageCursorType,
   MessageMetricsType,
   MessageType,
   MessageTypeUnhydrated,
+  PageMessagesCursorType,
+  PageMessagesResultType,
   PreKeyIdType,
   ReactionResultType,
   StoredPreKeyType,
@@ -184,6 +189,8 @@ type StickerRow = Readonly<{
 // https://github.com/microsoft/TypeScript/issues/420
 const dataInterface: ServerInterface = {
   close,
+  pauseWriteAccess,
+  resumeWriteAccess,
   removeDB,
   removeIndexedDBFiles,
 
@@ -417,6 +424,8 @@ const dataInterface: ServerInterface = {
 
   getKnownMessageAttachments,
   finishGetKnownMessageAttachments,
+  pageMessages,
+  finishPageMessages,
   getKnownConversationAttachments,
   removeKnownStickers,
   removeKnownDraftAttachments,
@@ -571,6 +580,8 @@ function openAndSetUpSQLCipher(
   return db;
 }
 
+let pausedWriteQueue: Array<() => void> | undefined;
+
 let globalWritableInstance: Database | undefined;
 let globalReadonlyInstance: Database | undefined;
 let logger = consoleLogger;
@@ -653,6 +664,33 @@ async function close(): Promise<void> {
   globalWritableInstance = undefined;
 }
 
+async function pauseWriteAccess(): Promise<void> {
+  strictAssert(
+    pausedWriteQueue === undefined,
+    'Database writes are already paused'
+  );
+  pausedWriteQueue = [];
+
+  logger.warn('pauseWriteAccess: pausing write access');
+}
+
+async function resumeWriteAccess(): Promise<void> {
+  strictAssert(
+    pausedWriteQueue !== undefined,
+    'Database writes are not paused'
+  );
+  const queue = pausedWriteQueue;
+  pausedWriteQueue = undefined;
+
+  logger.warn(
+    `resumeWriteAccess: resuming write access, queue.length=${queue.length}`
+  );
+
+  for (const resumeOperation of queue) {
+    resumeOperation();
+  }
+}
+
 async function removeDB(): Promise<void> {
   if (globalReadonlyInstance) {
     try {
@@ -702,7 +740,15 @@ function getReadonlyInstance(): Database {
   return globalReadonlyInstance;
 }
 
+const WRITABLE_INSTANCE_MAX_WAIT = 5 * durations.MINUTE;
+
 async function getWritableInstance(): Promise<Database> {
+  if (pausedWriteQueue) {
+    const { promise, resolve } = explodePromise<void>();
+    pausedWriteQueue.push(resolve);
+    await pTimeout(promise, WRITABLE_INSTANCE_MAX_WAIT);
+  }
+
   if (!globalWritableInstance) {
     throw new Error('getWritableInstance: globalWritableInstance not set!');
   }
@@ -6086,17 +6132,42 @@ function getExternalDraftFilesForConversation(
 async function getKnownMessageAttachments(
   cursor?: MessageAttachmentsCursorType
 ): Promise<GetKnownMessageAttachmentsResultType> {
-  const db = await getWritableInstance();
+  const innerCursor = cursor as MessageCursorType | undefined as
+    | PageMessagesCursorType
+    | undefined;
   const result = new Set<string>();
+
+  const { messages, cursor: newCursor } = await pageMessages(innerCursor);
+
+  for (const message of messages) {
+    const externalFiles = getExternalFilesForMessage(message);
+    forEach(externalFiles, file => result.add(file));
+  }
+
+  return {
+    attachments: Array.from(result),
+    cursor: newCursor as MessageCursorType as MessageAttachmentsCursorType,
+  };
+}
+
+async function finishGetKnownMessageAttachments(
+  cursor: MessageAttachmentsCursorType
+): Promise<void> {
+  const innerCursor = cursor as MessageCursorType as PageMessagesCursorType;
+
+  await finishPageMessages(innerCursor);
+}
+
+async function pageMessages(
+  cursor?: PageMessagesCursorType
+): Promise<PageMessagesResultType> {
+  const db = getUnsafeWritableInstance('only temp table use');
   const chunkSize = 1000;
 
   return db.transaction(() => {
     let count = cursor?.count ?? 0;
 
-    strictAssert(
-      !cursor?.done,
-      'getKnownMessageAttachments: iteration cannot be restarted'
-    );
+    strictAssert(!cursor?.done, 'pageMessages: iteration cannot be restarted');
 
     let runId: string;
     if (cursor === undefined) {
@@ -6104,7 +6175,7 @@ async function getKnownMessageAttachments(
 
       const total = getMessageCountSync();
       logger.info(
-        `getKnownMessageAttachments(${runId}): ` +
+        `pageMessages(${runId}): ` +
           `Starting iteration through ${total} messages`
       );
 
@@ -6114,7 +6185,7 @@ async function getKnownMessageAttachments(
           (rowid INTEGER PRIMARY KEY ASC);
 
         INSERT INTO tmp_${runId}_updated_messages (rowid)
-        SELECT rowid FROM messages;
+        SELECT rowid FROM messages ORDER BY rowid ASC;
 
         CREATE TEMP TRIGGER tmp_${runId}_message_updates
         UPDATE OF json ON messages
@@ -6140,6 +6211,7 @@ async function getKnownMessageAttachments(
         `
       DELETE FROM tmp_${runId}_updated_messages
       RETURNING rowid
+      ORDER BY rowid ASC
       LIMIT $chunkSize;
       `
       )
@@ -6160,28 +6232,25 @@ async function getKnownMessageAttachments(
       }
     );
 
-    for (const message of messages) {
-      const externalFiles = getExternalFilesForMessage(message);
-      forEach(externalFiles, file => result.add(file));
-      count += 1;
-    }
-
+    count += messages.length;
     const done = rowids.length < chunkSize;
+    const newCursor: MessageCursorType = { runId, count, done };
+
     return {
-      attachments: Array.from(result),
-      cursor: { runId, count, done },
+      messages,
+      cursor: newCursor as PageMessagesCursorType,
     };
   })();
 }
 
-async function finishGetKnownMessageAttachments({
+async function finishPageMessages({
   runId,
   count,
   done,
-}: MessageAttachmentsCursorType): Promise<void> {
-  const db = await getWritableInstance();
+}: PageMessagesCursorType): Promise<void> {
+  const db = getUnsafeWritableInstance('only temp table use');
 
-  const logId = `finishGetKnownMessageAttachments(${runId})`;
+  const logId = `finishPageMessages(${runId})`;
   if (!done) {
     logger.warn(`${logId}: iteration not finished`);
   }

ts/state/ducks/composer.ts

@@ -70,7 +70,7 @@ import { resolveDraftAttachmentOnDisk } from '../../util/resolveDraftAttachmentO
 import { shouldShowInvalidMessageToast } from '../../util/shouldShowInvalidMessageToast';
 import { writeDraftAttachment } from '../../util/writeDraftAttachment';
 import { __DEPRECATED$getMessageById } from '../../messages/getMessageById';
-import { canReply } from '../selectors/message';
+import { canReply, isNormalBubble } from '../selectors/message';
 import { getContactId } from '../../messages/helpers';
 import { getConversationSelector } from '../selectors/conversations';
 import { enqueueReactionForSend } from '../../reactions/enqueueReactionForSend';
@@ -780,7 +780,7 @@ export function setQuoteByMessageId(
       return;
     }
 
-    if (message && !message.isNormalBubble()) {
+    if (message && !isNormalBubble(message.attributes)) {
       return;
     }
 

ts/state/selectors/message.ts

@@ -989,6 +989,27 @@ export function getPropsForBubble(
   };
 }
 
+export function isNormalBubble(message: MessageWithUIFieldsType): boolean {
+  return (
+    !isCallHistory(message) &&
+    !isChatSessionRefreshed(message) &&
+    !isContactRemovedNotification(message) &&
+    !isConversationMerge(message) &&
+    !isEndSession(message) &&
+    !isExpirationTimerUpdate(message) &&
+    !isGroupUpdate(message) &&
+    !isGroupV1Migration(message) &&
+    !isGroupV2Change(message) &&
+    !isKeyChange(message) &&
+    !isPhoneNumberDiscovery(message) &&
+    !isTitleTransitionNotification(message) &&
+    !isProfileChange(message) &&
+    !isUniversalTimerNotification(message) &&
+    !isUnsupportedMessage(message) &&
+    !isVerifiedChange(message)
+  );
+}
+
 function getPropsForPaymentEvent(
   message: MessageAttributesWithPaymentEvent,
   { conversationSelector }: GetPropsForBubbleOptions

ts/state/smart/InstallScreen.tsx

@@ -25,6 +25,7 @@ import { MAX_DEVICE_NAME_LENGTH } from '../../components/installScreen/InstallSc
 import { WidthBreakpoint } from '../../components/_util';
 import { HTTPError } from '../../textsecure/Errors';
 import { isRecord } from '../../util/isRecord';
+import type { ConfirmNumberResultType } from '../../textsecure/AccountManager';
 import * as Errors from '../../types/errors';
 import { normalizeDeviceName } from '../../util/normalizeDeviceName';
 import OS from '../../util/os/osMain';
@@ -32,6 +33,7 @@ import { SECOND } from '../../util/durations';
 import { BackOff } from '../../util/BackOff';
 import { drop } from '../../util/drop';
 import { SmartToastManager } from './ToastManager';
+import { fileToBytes } from '../../util/fileToBytes';
 
 type PropsType = ComponentProps<typeof InstallScreen>;
 
@@ -47,6 +49,7 @@ type StateType =
   | {
       step: InstallScreenStep.ChoosingDeviceName;
       deviceName: string;
+      backupFile?: File;
     }
   | {
       step: InstallScreenStep.LinkInProgress;
@@ -92,6 +95,9 @@ export const SmartInstallScreen = memo(function SmartInstallScreen() {
   const hasExpired = useSelector(hasExpiredSelector);
 
   const chooseDeviceNamePromiseWrapperRef = useRef(explodePromise<string>());
+  const chooseBackupFilePromiseWrapperRef = useRef(
+    explodePromise<File | undefined>()
+  );
 
   const [state, setState] = useState<StateType>(INITIAL_STATE);
   const [retryCounter, setRetryCounter] = useState(0);
@@ -146,6 +152,21 @@ export const SmartInstallScreen = memo(function SmartInstallScreen() {
     [setState]
   );
 
+  const setBackupFile = useCallback(
+    (backupFile: File) => {
+      setState(currentState => {
+        if (currentState.step !== InstallScreenStep.ChoosingDeviceName) {
+          return currentState;
+        }
+        return {
+          ...currentState,
+          backupFile,
+        };
+      });
+    },
+    [setState]
+  );
+
   const onSubmitDeviceName = useCallback(() => {
     if (state.step !== InstallScreenStep.ChoosingDeviceName) {
       return;
@@ -161,6 +182,7 @@ export const SmartInstallScreen = memo(function SmartInstallScreen() {
       deviceName = i18n('icu:Install__choose-device-name__placeholder');
     }
     chooseDeviceNamePromiseWrapperRef.current.resolve(deviceName);
+    chooseBackupFilePromiseWrapperRef.current.resolve(state.backupFile);
 
     setState({ step: InstallScreenStep.LinkInProgress });
   }, [state, i18n]);
@@ -180,19 +202,23 @@ export const SmartInstallScreen = memo(function SmartInstallScreen() {
       setProvisioningUrl(value);
     };
 
-    const confirmNumber = async (): Promise<string> => {
+    const confirmNumber = async (): Promise<ConfirmNumberResultType> => {
       if (hasCleanedUp) {
         throw new Error('Cannot confirm number; the component was unmounted');
       }
       onQrCodeScanned();
 
+      let deviceName: string;
+      let backupFileData: Uint8Array | undefined;
       if (window.SignalCI) {
-        chooseDeviceNamePromiseWrapperRef.current.resolve(
-          window.SignalCI.deviceName
-        );
-      }
+        ({ deviceName, backupData: backupFileData } = window.SignalCI);
+      } else {
+        deviceName = await chooseDeviceNamePromiseWrapperRef.current.promise;
+        const backupFile = await chooseBackupFilePromiseWrapperRef.current
+          .promise;
 
-      const result = await chooseDeviceNamePromiseWrapperRef.current.promise;
+        backupFileData = backupFile ? await fileToBytes(backupFile) : undefined;
+      }
 
       if (hasCleanedUp) {
         throw new Error('Cannot confirm number; the component was unmounted');
@@ -217,7 +243,7 @@ export const SmartInstallScreen = memo(function SmartInstallScreen() {
         throw new Error('Cannot confirm number; the component was unmounted');
       }
 
-      return result;
+      return { deviceName, backupFile: backupFileData };
     };
 
     async function getQRCode(): Promise<void> {
@@ -314,6 +340,7 @@ export const SmartInstallScreen = memo(function SmartInstallScreen() {
           i18n,
           deviceName: state.deviceName,
           setDeviceName,
+          setBackupFile,
           onSubmit: onSubmitDeviceName,
         },
       };

ts/test-mock/backups/backups_test.ts

@@ -0,0 +1,116 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import createDebug from 'debug';
+import Long from 'long';
+
+import * as durations from '../../util/durations';
+import type { App } from '../playwright';
+import { Bootstrap } from '../bootstrap';
+
+export const debug = createDebug('mock:test:backups');
+
+describe('backups', function (this: Mocha.Suite) {
+  this.timeout(100 * durations.MINUTE);
+
+  let bootstrap: Bootstrap;
+  let app: App;
+
+  beforeEach(async () => {
+    bootstrap = new Bootstrap();
+    await bootstrap.init();
+    app = await bootstrap.link();
+  });
+
+  afterEach(async function (this: Mocha.Context) {
+    if (!bootstrap) {
+      return;
+    }
+
+    await bootstrap.maybeSaveLogs(this.currentTest, app);
+    await app.close();
+    await bootstrap.teardown();
+  });
+
+  it('exports and imports backup', async function () {
+    const { contacts, phone, desktop, server } = bootstrap;
+    const [friend] = contacts;
+
+    for (let i = 0; i < 5; i += 1) {
+      const theirTimestamp = bootstrap.getTimestamp();
+
+      // eslint-disable-next-line no-await-in-loop
+      await friend.sendText(desktop, `msg ${i}`, {
+        timestamp: theirTimestamp,
+      });
+
+      const ourTimestamp = bootstrap.getTimestamp();
+
+      // eslint-disable-next-line no-await-in-loop
+      await server.send(
+        desktop,
+        // eslint-disable-next-line no-await-in-loop
+        await phone.encryptSyncSent(desktop, `respond ${i}`, {
+          timestamp: ourTimestamp,
+          destinationServiceId: friend.device.aci,
+        })
+      );
+
+      const reactionTimestamp = bootstrap.getTimestamp();
+
+      // eslint-disable-next-line no-await-in-loop
+      await friend.sendRaw(
+        desktop,
+        {
+          dataMessage: {
+            timestamp: Long.fromNumber(reactionTimestamp),
+            reaction: {
+              emoji: '👍',
+              targetAuthorAci: desktop.aci,
+              targetTimestamp: Long.fromNumber(ourTimestamp),
+            },
+          },
+        },
+        {
+          timestamp: reactionTimestamp,
+        }
+      );
+    }
+
+    const comparator = await bootstrap.createScreenshotComparator(
+      app,
+      async (window, snapshot) => {
+        const leftPane = window.locator('#LeftPane');
+        const contactElem = leftPane.locator(
+          `[data-testid="${friend.toContact().aci}"] >> "respond 4"`
+        );
+
+        debug('Waiting for messages to come through');
+        await contactElem.waitFor();
+
+        await snapshot('main screen');
+
+        debug('Going into the conversation');
+        await contactElem.click();
+        await window
+          .locator('.ConversationView .module-message >> "respond 4"')
+          .waitFor();
+
+        await snapshot('conversation');
+      },
+      this.test
+    );
+
+    const backupPath = bootstrap.getBackupPath('backup.bin');
+    await app.exportBackupToDisk(backupPath);
+    await app.close();
+
+    // Restart
+    await bootstrap.unlink();
+    app = await bootstrap.link({
+      ciBackupPath: backupPath,
+    });
+
+    await comparator(app);
+  });
+});

ts/test-mock/bootstrap.ts

@@ -3,11 +3,15 @@
 
 import assert from 'assert';
 import fs from 'fs/promises';
+import crypto from 'crypto';
 import path from 'path';
 import os from 'os';
 import createDebug from 'debug';
 import pTimeout from 'p-timeout';
 import normalizePath from 'normalize-path';
+import pixelmatch from 'pixelmatch';
+import { PNG } from 'pngjs';
+import type { Page } from 'playwright';
 
 import type { Device, PrimaryDevice } from '@signalapp/mock-server';
 import {
@@ -18,6 +22,7 @@ import {
 import { MAX_READ_KEYS as MAX_STORAGE_READ_KEYS } from '../services/storageConstants';
 import * as durations from '../util/durations';
 import { drop } from '../util/drop';
+import type { RendererConfigType } from '../types/RendererConfig';
 import { App } from './playwright';
 import { CONTACT_COUNT } from './benchmarks/fixtures';
 
@@ -93,7 +98,6 @@ for (const suffix of CONTACT_SUFFIXES) {
 const MAX_CONTACTS = CONTACT_NAMES.length;
 
 export type BootstrapOptions = Readonly<{
-  extraConfig?: Record<string, unknown>;
   benchmark?: boolean;
 
   linkedDevices?: number;
@@ -104,7 +108,7 @@ export type BootstrapOptions = Readonly<{
   contactPreKeyCount?: number;
 }>;
 
-type BootstrapInternalOptions = Pick<BootstrapOptions, 'extraConfig'> &
+type BootstrapInternalOptions = BootstrapOptions &
   Readonly<{
     benchmark: boolean;
     linkedDevices: number;
@@ -114,6 +118,10 @@ type BootstrapInternalOptions = Pick<BootstrapOptions, 'extraConfig'> &
     contactNames: ReadonlyArray<string>;
   }>;
 
+function sanitizePathComponent(component: string): string {
+  return normalizePath(component.replace(/[^a-z]+/gi, '-'));
+}
+
 //
 // Bootstrap is a class that prepares mock server and desktop for running
 // tests/benchmarks.
@@ -149,8 +157,10 @@ export class Bootstrap {
   private privPhone?: PrimaryDevice;
   private privDesktop?: Device;
   private storagePath?: string;
+  private backupPath?: string;
   private timestamp: number = Date.now() - durations.WEEK;
   private lastApp?: App;
+  private readonly randomId = crypto.randomBytes(8).toString('hex');
 
   constructor(options: BootstrapOptions = {}) {
     this.server = new Server({
@@ -224,6 +234,9 @@ export class Bootstrap {
     });
 
     this.storagePath = await fs.mkdtemp(path.join(os.tmpdir(), 'mock-signal-'));
+    this.backupPath = await fs.mkdtemp(
+      path.join(os.tmpdir(), 'mock-signal-backup-')
+    );
 
     debug('setting storage path=%j', this.storagePath);
   }
@@ -244,6 +257,26 @@ export class Bootstrap {
     return path.join(this.storagePath, 'logs');
   }
 
+  public getBackupPath(fileName: string): string {
+    assert(
+      this.backupPath !== undefined,
+      'Bootstrap has to be initialized first, see: bootstrap.init()'
+    );
+
+    return path.join(this.backupPath, fileName);
+  }
+
+  public async unlink(): Promise<void> {
+    assert(
+      this.storagePath !== undefined,
+      'Bootstrap has to be initialized first, see: bootstrap.init()'
+    );
+
+    // Note that backupPath must remain unchanged!
+    await fs.rm(this.storagePath, { recursive: true });
+    this.storagePath = await fs.mkdtemp(path.join(os.tmpdir(), 'mock-signal-'));
+  }
+
   public async teardown(): Promise<void> {
     debug('tearing down');
 
@@ -252,6 +285,9 @@ export class Bootstrap {
         this.storagePath
           ? fs.rm(this.storagePath, { recursive: true })
           : Promise.resolve(),
+        this.backupPath
+          ? fs.rm(this.backupPath, { recursive: true })
+          : Promise.resolve(),
         this.server.close(),
         this.lastApp?.close(),
       ]),
@@ -259,10 +295,10 @@ export class Bootstrap {
     ]);
   }
 
-  public async link(): Promise<App> {
+  public async link(extraConfig?: Partial<RendererConfigType>): Promise<App> {
     debug('linking');
 
-    const app = await this.startApp();
+    const app = await this.startApp(extraConfig);
 
     const provision = await this.server.waitForProvision();
 
@@ -302,7 +338,9 @@ export class Bootstrap {
     await app.close();
   }
 
-  public async startApp(): Promise<App> {
+  public async startApp(
+    extraConfig?: Partial<RendererConfigType>
+  ): Promise<App> {
     assert(
       this.storagePath !== undefined,
       'Bootstrap has to be initialized first, see: bootstrap.init()'
@@ -311,10 +349,10 @@ export class Bootstrap {
     debug('starting the app');
 
     const { port } = this.server.address();
-    const config = await this.generateConfig(port);
+    const config = await this.generateConfig(port, extraConfig);
 
     let startAttempts = 0;
-    const MAX_ATTEMPTS = 5;
+    const MAX_ATTEMPTS = 4;
     let app: App | undefined;
     while (!app) {
       startAttempts += 1;
@@ -360,7 +398,7 @@ export class Bootstrap {
   }
 
   public async maybeSaveLogs(
-    test?: Mocha.Test,
+    test?: Mocha.Runnable,
     app: App | undefined = this.lastApp
   ): Promise<void> {
     const { FORCE_ARTIFACT_SAVE } = process.env;
@@ -371,29 +409,18 @@ export class Bootstrap {
 
   public async saveLogs(
     app: App | undefined = this.lastApp,
-    pathPrefix?: string
+    testName?: string
   ): Promise<void> {
-    const { ARTIFACTS_DIR } = process.env;
-    if (!ARTIFACTS_DIR) {
-      // eslint-disable-next-line no-console
-      console.error('Not saving logs. Please set ARTIFACTS_DIR env variable');
+    const outDir = await this.getArtifactsDir(testName);
+    if (outDir == null) {
       return;
     }
 
-    await fs.mkdir(ARTIFACTS_DIR, { recursive: true });
-
-    const normalizedPrefix = pathPrefix
-      ? `-${normalizePath(pathPrefix.replace(/[^a-z]+/gi, '-'))}-`
-      : '';
-    const outDir = await fs.mkdtemp(
-      path.join(ARTIFACTS_DIR, `logs-${normalizedPrefix}`)
-    );
-
     // eslint-disable-next-line no-console
     console.error(`Saving logs to ${outDir}`);
 
     const { logsDir } = this;
-    await fs.rename(logsDir, outDir);
+    await fs.rename(logsDir, path.join(outDir, 'logs'));
 
     const page = await app?.getWindow();
     if (process.env.TRACING) {
@@ -408,6 +435,77 @@ export class Bootstrap {
     }
   }
 
+  public async createScreenshotComparator(
+    app: App,
+    callback: (
+      page: Page,
+      snapshot: (name: string) => Promise<void>
+    ) => Promise<void>,
+    test?: Mocha.Runnable
+  ): Promise<(app: App) => Promise<void>> {
+    const snapshots = new Array<{ name: string; data: Buffer }>();
+
+    const window = await app.getWindow();
+    await callback(window, async (name: string) => {
+      debug('creating screenshot');
+      snapshots.push({ name, data: await window.screenshot() });
+    });
+
+    let index = 0;
+
+    return async (anotherApp: App): Promise<void> => {
+      const anotherWindow = await anotherApp.getWindow();
+      await callback(anotherWindow, async (name: string) => {
+        index += 1;
+
+        const before = snapshots.shift();
+        assert(before != null, 'No previous snapshot');
+        assert.strictEqual(before.name, name, 'Wrong snapshot order');
+
+        const after = await anotherWindow.screenshot();
+
+        const beforePng = PNG.sync.read(before.data);
+        const afterPng = PNG.sync.read(after);
+
+        const { width, height } = beforePng;
+        const diffPng = new PNG({ width, height });
+
+        const numPixels = pixelmatch(
+          beforePng.data,
+          afterPng.data,
+          diffPng.data,
+          width,
+          height,
+          {}
+        );
+
+        if (numPixels === 0) {
+          debug('no screenshot difference');
+          return;
+        }
+
+        debug('screenshot difference', numPixels);
+
+        const outDir = await this.getArtifactsDir(test?.fullTitle());
+        if (outDir != null) {
+          debug('saving screenshots and diff');
+          const prefix = `${index}-${sanitizePathComponent(name)}`;
+          await fs.writeFile(
+            path.join(outDir, `${prefix}-before.png`),
+            before.data
+          );
+          await fs.writeFile(path.join(outDir, `${prefix}-after.png`), after);
+          await fs.writeFile(
+            path.join(outDir, `${prefix}-diff.png`),
+            PNG.sync.write(diffPng)
+          );
+        }
+
+        assert.strictEqual(numPixels, 0, 'Expected no pixels to be different');
+      });
+    };
+  }
+
   //
   // Getters
   //
@@ -463,6 +561,28 @@ export class Bootstrap {
   // Private
   //
 
+  private async getArtifactsDir(
+    testName?: string
+  ): Promise<string | undefined> {
+    const { ARTIFACTS_DIR } = process.env;
+    if (!ARTIFACTS_DIR) {
+      // eslint-disable-next-line no-console
+      console.error(
+        'Not saving artifacts. Please set ARTIFACTS_DIR env variable'
+      );
+      return undefined;
+    }
+
+    const normalizedPath = testName
+      ? `${this.randomId}-${sanitizePathComponent(testName)}`
+      : this.randomId;
+
+    const outDir = path.join(ARTIFACTS_DIR, normalizedPath);
+    await fs.mkdir(outDir, { recursive: true });
+
+    return outDir;
+  }
+
   private static async runBenchmark(
     fn: (bootstrap: Bootstrap) => Promise<void>,
     timeout: number
@@ -486,7 +606,10 @@ export class Bootstrap {
     }
   }
 
-  private async generateConfig(port: number): Promise<string> {
+  private async generateConfig(
+    port: number,
+    extraConfig?: Partial<RendererConfigType>
+  ): Promise<string> {
     const url = `https://127.0.0.1:${port}`;
     return JSON.stringify({
       ...(await loadCertificates()),
@@ -510,7 +633,7 @@ export class Bootstrap {
       directoryCDSIMRENCLAVE:
         '51133fecb3fa18aaf0c8f64cb763656d3272d9faaacdb26ae7df082e414fb142',
 
-      ...this.options.extraConfig,
+      ...extraConfig,
     });
   }
 }

ts/test-mock/playwright.ts

@@ -168,6 +168,13 @@ export class App extends EventEmitter {
     );
   }
 
+  public async exportBackupToDisk(path: string): Promise<Uint8Array> {
+    const window = await this.getWindow();
+    return window.evaluate(
+      `window.SignalCI.exportBackupToDisk(${JSON.stringify(path)})`
+    );
+  }
+
   // EventEmitter types
 
   public override on(type: 'close', callback: () => void): this;

ts/test-node/util/DelimitedStream_test.ts

@@ -0,0 +1,132 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { assert } from 'chai';
+import { Readable, Writable } from 'stream';
+import { pipeline } from 'stream/promises';
+import { BufferWriter } from 'protobufjs';
+
+import { DelimitedStream } from '../../util/DelimitedStream';
+
+describe('DelimitedStream', () => {
+  function collect(out: Array<string>): Writable {
+    return new Writable({
+      write(data, _enc, callback) {
+        out.push(data.toString());
+        callback(null);
+      },
+    });
+  }
+
+  async function strideTest(
+    data: Uint8Array,
+    result: ReadonlyArray<string>
+  ): Promise<void> {
+    // Just to keep reasonable run times
+    const decrease = Math.max(1, Math.round(data.length / 256));
+
+    for (let stride = data.length; stride > 0; stride -= decrease) {
+      const out = new Array<string>();
+
+      // eslint-disable-next-line no-await-in-loop
+      await pipeline(
+        Readable.from(
+          (function* () {
+            for (let offset = 0; offset < data.length; offset += stride) {
+              yield data.slice(offset, offset + stride);
+            }
+          })()
+        ),
+        new DelimitedStream(),
+        collect(out)
+      );
+
+      assert.deepStrictEqual(out, result, `Stride: ${stride}`);
+    }
+  }
+
+  it('should parse single-byte delimited data', async () => {
+    const w = new BufferWriter();
+    w.string('a');
+    w.string('bc');
+
+    await strideTest(w.finish(), ['a', 'bc']);
+  });
+
+  it('should parse two-byte delimited data', async () => {
+    const w = new BufferWriter();
+    w.string('a'.repeat(129));
+    w.string('b'.repeat(154));
+
+    await strideTest(w.finish(), ['a'.repeat(129), 'b'.repeat(154)]);
+  });
+
+  it('should parse three-byte delimited data', async () => {
+    const w = new BufferWriter();
+    w.string('a'.repeat(32000));
+    w.string('b'.repeat(32500));
+
+    await strideTest(w.finish(), ['a'.repeat(32000), 'b'.repeat(32500)]);
+  });
+
+  it('should parse mixed delimited data', async () => {
+    const w = new BufferWriter();
+    w.string('a');
+    w.string('b'.repeat(129));
+    w.string('c'.repeat(32000));
+    w.string('d'.repeat(32));
+    w.string('e'.repeat(415));
+    w.string('f'.repeat(33321));
+
+    await strideTest(w.finish(), [
+      'a',
+      'b'.repeat(129),
+      'c'.repeat(32000),
+      'd'.repeat(32),
+      'e'.repeat(415),
+      'f'.repeat(33321),
+    ]);
+  });
+
+  it('should error on incomplete prefix', async () => {
+    const w = new BufferWriter();
+    w.string('a'.repeat(32000));
+
+    const out = new Array<string>();
+    await assert.isRejected(
+      pipeline(
+        Readable.from(w.finish().slice(0, 1)),
+        new DelimitedStream(),
+        collect(out)
+      ),
+      'Unfinished prefix'
+    );
+  });
+
+  it('should error on incomplete data', async () => {
+    const w = new BufferWriter();
+    w.string('a'.repeat(32000));
+
+    const out = new Array<string>();
+    await assert.isRejected(
+      pipeline(
+        Readable.from(w.finish().slice(0, 10)),
+        new DelimitedStream(),
+        collect(out)
+      ),
+      'Unfinished data'
+    );
+  });
+
+  it('should error on prefix overflow', async () => {
+    const out = new Array<string>();
+    await assert.isRejected(
+      pipeline(
+        Readable.from(Buffer.from([0xff, 0xff, 0xff, 0xff, 0xff])),
+        new DelimitedStream(),
+        collect(out)
+      ),
+      'Delimiter encoding overflow'
+    );
+  });
+});

ts/textsecure/AccountManager.ts

@@ -26,6 +26,7 @@ import createTaskWithTimeout from './TaskWithTimeout';
 import * as Bytes from '../Bytes';
 import * as Errors from '../types/errors';
 import { senderCertificateService } from '../services/senderCertificate';
+import { backupsService } from '../services/backups';
 import {
   deriveAccessKey,
   generateRegistrationId,
@@ -123,6 +124,7 @@ type CreateAccountSharedOptionsType = Readonly<{
   pniKeyPair: KeyPairType;
   profileKey: Uint8Array;
   masterKey: Uint8Array | undefined;
+  backupFile?: Uint8Array;
 }>;
 
 type CreatePrimaryDeviceOptionsType = Readonly<{
@@ -213,6 +215,11 @@ function signedPreKeyToUploadSignedPreKey({
   };
 }
 
+export type ConfirmNumberResultType = Readonly<{
+  deviceName: string;
+  backupFile: Uint8Array | undefined;
+}>;
+
 export default class AccountManager extends EventTarget {
   pending: Promise<void>;
 
@@ -339,7 +346,7 @@ export default class AccountManager extends EventTarget {
 
   async registerSecondDevice(
     setProvisioningUrl: (url: string) => void,
-    confirmNumber: (number?: string) => Promise<string>
+    confirmNumber: (number?: string) => Promise<ConfirmNumberResultType>
   ): Promise<void> {
     const provisioningCipher = new ProvisioningCipher();
     const pubKey = await provisioningCipher.getPublicKey();
@@ -407,7 +414,9 @@ export default class AccountManager extends EventTarget {
     const provisionMessage = await provisioningCipher.decrypt(envelope);
 
     await this.queueTask(async () => {
-      const deviceName = await confirmNumber(provisionMessage.number);
+      const { deviceName, backupFile } = await confirmNumber(
+        provisionMessage.number
+      );
       if (typeof deviceName !== 'string' || deviceName.length === 0) {
         throw new Error(
           'AccountManager.registerSecondDevice: Invalid device name'
@@ -443,6 +452,7 @@ export default class AccountManager extends EventTarget {
           pniKeyPair: provisionMessage.pniKeyPair,
           profileKey: provisionMessage.profileKey,
           deviceName,
+          backupFile,
           userAgent: provisionMessage.userAgent,
           ourAci,
           ourPni,
@@ -1018,6 +1028,7 @@ export default class AccountManager extends EventTarget {
       masterKey,
       readReceipts,
       userAgent,
+      backupFile,
     } = options;
 
     const { storage } = window.textsecure;
@@ -1049,7 +1060,7 @@ export default class AccountManager extends EventTarget {
     const numberChanged =
       !previousACI && previousNumber && previousNumber !== number;
 
-    if (uuidChanged || numberChanged) {
+    if (uuidChanged || numberChanged || backupFile !== undefined) {
       if (uuidChanged) {
         log.warn(
           'createAccount: New uuid is different from old uuid; deleting all previous data'
@@ -1060,6 +1071,11 @@ export default class AccountManager extends EventTarget {
           'createAccount: New number is different from old number; deleting all previous data'
         );
       }
+      if (backupFile !== undefined) {
+        log.warn(
+          'createAccount: Restoring from backup; deleting all previous data'
+        );
+      }
 
       try {
         await storage.protocol.removeAllData();
@@ -1200,17 +1216,13 @@ export default class AccountManager extends EventTarget {
     // This needs to be done very early, because it changes how things are saved in the
     //   database. Your identity, for example, in the saveIdentityWithAttributes call
     //   below.
-    const { conversation } = window.ConversationController.maybeMergeContacts({
+    window.ConversationController.maybeMergeContacts({
       aci: ourAci,
       pni: ourPni,
       e164: number,
       reason: 'createAccount',
     });
 
-    if (!conversation) {
-      throw new Error('registrationDone: no conversation!');
-    }
-
     const identityAttrs = {
       firstUse: true,
       timestamp: Date.now(),
@@ -1317,6 +1329,10 @@ export default class AccountManager extends EventTarget {
       uploadKeys(ServiceIdKind.ACI),
       uploadKeys(ServiceIdKind.PNI),
     ]);
+
+    if (backupFile !== undefined) {
+      await backupsService.importBackup(backupFile);
+    }
   }
 
   // Exposed only for testing

ts/types/Message.ts

@@ -6,6 +6,8 @@ import type { AttachmentType } from './Attachment';
 import type { EmbeddedContactType } from './EmbeddedContact';
 import type { IndexableBoolean, IndexablePresence } from './IndexedDB';
 
+export const LONG_ATTACHMENT_LIMIT = 2048;
+
 export function getMentionsRegex(): RegExp {
   return /\uFFFC/g;
 }

ts/types/RendererConfig.ts

@@ -42,6 +42,7 @@ export const rendererConfigSchema = z.object({
   crashDumpsPath: configRequiredStringSchema,
   ciMode: z.enum(['full', 'benchmark']).or(z.literal(false)),
   dnsFallback: DNSFallbackSchema,
+  ciBackupPath: configOptionalStringSchema,
   environment: environmentSchema,
   homePath: configRequiredStringSchema,
   hostname: configRequiredStringSchema,

ts/util/DelimitedStream.ts

@@ -0,0 +1,91 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { Transform } from 'stream';
+
+import { missingCaseError } from './missingCaseError';
+
+enum State {
+  Prefix = 'Prefix',
+  Data = 'Data',
+}
+
+export class DelimitedStream extends Transform {
+  private state = State.Prefix;
+  private prefixValue = 0;
+  private prefixSize = 0;
+  private parts = new Array<Buffer>();
+
+  constructor() {
+    super({ readableObjectMode: true });
+  }
+
+  override _transform(
+    chunk: Buffer,
+    _encoding: BufferEncoding,
+    done: (error?: Error) => void
+  ): void {
+    let offset = 0;
+    while (offset < chunk.length) {
+      if (this.state === State.Prefix) {
+        const b = chunk[offset];
+        offset += 1;
+
+        // See: https://protobuf.dev/programming-guides/encoding/
+        // eslint-disable-next-line no-bitwise
+        const isLast = (b & 0x80) === 0;
+        // eslint-disable-next-line no-bitwise
+        const value = b & 0x7f;
+
+        // eslint-disable-next-line no-bitwise
+        this.prefixValue |= value << (7 * this.prefixSize);
+        this.prefixSize += 1;
+
+        // Check that we didn't go over 32bits. Node.js buffers can never
+        // be larger than 2gb anyway!
+        if (this.prefixSize > 4) {
+          done(new Error('Delimiter encoding overflow'));
+          return;
+        }
+
+        if (isLast) {
+          this.state = State.Data;
+        }
+      } else if (this.state === State.Data) {
+        const toTake = Math.min(this.prefixValue, chunk.length - offset);
+        const part = chunk.slice(offset, offset + toTake);
+        offset += toTake;
+        this.prefixValue -= toTake;
+
+        this.parts.push(part);
+
+        if (this.prefixValue <= 0) {
+          this.state = State.Prefix;
+          this.prefixSize = 0;
+          this.prefixValue = 0;
+
+          const whole = Buffer.concat(this.parts);
+          this.parts = [];
+          this.push(whole);
+        }
+      } else {
+        throw missingCaseError(this.state);
+      }
+    }
+    done();
+  }
+
+  override _flush(done: (error?: Error) => void): void {
+    if (this.state !== State.Prefix) {
+      done(new Error('Unfinished data'));
+      return;
+    }
+
+    if (this.prefixSize !== 0) {
+      done(new Error('Unfinished prefix'));
+      return;
+    }
+
+    done();
+  }
+}

ts/util/getConversation.ts

@@ -88,7 +88,7 @@ export function getConversation(model: ConversationModel): ConversationType {
   const ourAci = window.textsecure.storage.user.getAci();
   const ourPni = window.textsecure.storage.user.getPni();
 
-  const color = migrateColor(attributes.color);
+  const color = migrateColor(attributes.serviceId, attributes.color);
 
   const { draftTimestamp, draftEditMessage, timestamp } = attributes;
   const draftPreview = getDraftPreview(attributes);

ts/util/lint/exceptions.json

@@ -2872,20 +2872,6 @@
     "updated": "2023-11-14T23:29:51.425Z",
     "reasonDetail": "To render the reaction picker in the CallScreen"
   },
-  {
-    "rule": "React-useRef",
-    "path": "ts/components/CallScreen.tsx",
-    "line": "  const reactButtonRef = React.useRef<null | HTMLDivElement>(null);",
-    "reasonCategory": "usageTrusted",
-    "updated": "2024-01-16T22:59:06.336Z"
-  },
-  {
-    "rule": "React-useRef",
-    "path": "ts/components/CallScreen.tsx",
-    "line": "  const reactionPickerContainerRef = React.useRef<null | HTMLDivElement>(null);",
-    "reasonCategory": "usageTrusted",
-    "updated": "2024-01-16T22:59:06.336Z"
-  },
   {
     "rule": "React-useRef",
     "path": "ts/components/CallScreen.tsx",
@@ -2902,6 +2888,20 @@
     "updated": "2024-01-06T00:59:20.678Z",
     "reasonDetail": "Recent reactions shown for reactions burst"
   },
+  {
+    "rule": "React-useRef",
+    "path": "ts/components/CallScreen.tsx",
+    "line": "  const reactButtonRef = React.useRef<null | HTMLDivElement>(null);",
+    "reasonCategory": "usageTrusted",
+    "updated": "2024-01-16T22:59:06.336Z"
+  },
+  {
+    "rule": "React-useRef",
+    "path": "ts/components/CallScreen.tsx",
+    "line": "  const reactionPickerContainerRef = React.useRef<null | HTMLDivElement>(null);",
+    "reasonCategory": "usageTrusted",
+    "updated": "2024-01-16T22:59:06.336Z"
+  },
   {
     "rule": "React-useRef",
     "path": "ts/components/CallingLobby.tsx",
@@ -3865,6 +3865,13 @@
     "reasonCategory": "usageTrusted",
     "updated": "2023-08-20T22:14:52.008Z"
   },
+  {
+    "rule": "React-useRef",
+    "path": "ts/state/smart/InstallScreen.tsx",
+    "line": "  const chooseBackupFilePromiseWrapperRef = useRef(",
+    "reasonCategory": "testCode",
+    "updated": "2023-11-16T23:39:21.322Z"
+  },
   {
     "rule": "React-useRef",
     "path": "ts/state/smart/InstallScreen.tsx",

ts/util/migrateColor.ts

@@ -2,18 +2,28 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import { sample } from 'lodash';
+
 import { AvatarColors } from '../types/Colors';
 import type { ConversationAttributesType } from '../model-types';
 import type { AvatarColorType, CustomColorType } from '../types/Colors';
+import type { ServiceIdString } from '../types/ServiceId';
 
 const NEW_COLOR_NAMES = new Set(AvatarColors);
 
-export function migrateColor(color?: string): AvatarColorType {
+export function migrateColor(
+  serviceId?: ServiceIdString,
+  color?: string
+): AvatarColorType {
   if (color && NEW_COLOR_NAMES.has(color)) {
     return color;
   }
 
-  return sample(AvatarColors) || AvatarColors[0];
+  if (!serviceId) {
+    return sample(AvatarColors) || AvatarColors[0];
+  }
+
+  const index = (parseInt(serviceId.slice(-4), 16) || 0) % AvatarColors.length;
+  return AvatarColors[index];
 }
 
 export function getCustomColorData(conversation: ConversationAttributesType): {

ts/util/whatTypeOfConversation.ts

@@ -23,7 +23,9 @@ export function isDirectConversation(
   );
 }
 
-export function isMe(conversationAttrs: ConversationAttributesType): boolean {
+export function isMe(
+  conversationAttrs: Pick<ConversationAttributesType, 'e164' | 'serviceId'>
+): boolean {
   const { e164, serviceId } = conversationAttrs;
   const ourNumber = window.textsecure.storage.user.getNumber();
   const ourAci = window.textsecure.storage.user.getAci();
@@ -76,7 +78,10 @@ export function isGroupV2(
 }
 
 export function typeofConversation(
-  conversationAttrs: ConversationAttributesType
+  conversationAttrs: Pick<
+    ConversationAttributesType,
+    'type' | 'e164' | 'serviceId' | 'groupId' | 'groupVersion'
+  >
 ): ConversationTypes | undefined {
   if (isMe(conversationAttrs)) {
     return ConversationTypes.Me;

ts/window.d.ts

@@ -20,6 +20,7 @@ import type AccountManager from './textsecure/AccountManager';
 import type { WebAPIConnectType } from './textsecure/WebAPI';
 import type { CallingClass } from './services/calling';
 import type * as StorageService from './services/storage';
+import type { BackupsService } from './services/backups';
 import type * as Groups from './groups';
 import type * as Crypto from './Crypto';
 import type * as Curve from './Curve';
@@ -141,6 +142,7 @@ export type SignalCoreType = {
   ScreenShareWindowProps?: ScreenShareWindowPropsType;
   Services: {
     calling: CallingClass;
+    backups: BackupsService;
     initializeGroupCredentialFetcher: () => Promise<void>;
     initializeNetworkObserver: (network: ReduxActions['network']) => void;
     initializeUpdateListener: (updates: ReduxActions['updates']) => void;

ts/windows/main/phase4-test.ts

@@ -5,6 +5,8 @@
 /* eslint-disable no-console */
 /* eslint-disable global-require */
 
+import fs from 'fs';
+
 const { config } = window.SignalContext;
 
 if (config.environment === 'test') {
@@ -14,8 +16,14 @@ if (config.environment === 'test') {
 
 if (config.ciMode) {
   console.log(
-    `Importing CI infrastructure; enabled in config, mode: ${config.ciMode}`
+    `Importing CI infrastructure; enabled in config, mode: ${config.ciMode}, ` +
+      `backupPath: ${config.ciBackupPath}`
   );
   const { getCI } = require('../../CI');
-  window.SignalCI = getCI(window.getTitle());
+  window.SignalCI = getCI({
+    deviceName: window.getTitle(),
+    backupData: config.ciBackupPath
+      ? fs.readFileSync(config.ciBackupPath)
+      : undefined,
+  });
 }

yarn.lock

@@ -5556,6 +5556,20 @@
   resolved "https://registry.yarnpkg.com/@types/pify/-/pify-3.0.2.tgz#1bc75dac43e31dba981c37e0a08edddc1b49cd39"
   integrity sha512-a5AKF1/9pCU3HGMkesgY6LsBdXHUY3WU+I2qgpU0J+I8XuJA1aFr59eS84/HP0+dxsyBSNbt+4yGI2adUpHwSg==
 
+"@types/pixelmatch@5.2.6":
+  version "5.2.6"
+  resolved "https://registry.yarnpkg.com/@types/pixelmatch/-/pixelmatch-5.2.6.tgz#fba6de304ac958495f27d85989f5c6bb7499a686"
+  integrity sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg==
+  dependencies:
+    "@types/node" "*"
+
+"@types/pngjs@6.0.4":
+  version "6.0.4"
+  resolved "https://registry.yarnpkg.com/@types/pngjs/-/pngjs-6.0.4.tgz#9a457aebabd944efde1a773a0fa1d74933e8021b"
+  integrity sha512-atAK9xLKOnxiuArxcHovmnOUUGBZOQ3f0vCf43FnoKs6XnqiambT1kkJWmdo71IR+BoXSh+CueeFR0GfH3dTlQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/prettier@^2.1.5":
   version "2.7.3"
   resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.3.tgz#3e51a17e291d01d17d3fc61422015a933af7a08f"
@@ -15915,6 +15929,13 @@ pirates@^4.0.5:
   resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.5.tgz#feec352ea5c3268fb23a37c702ab1699f35a5f3b"
   integrity sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==
 
+pixelmatch@5.3.0:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-5.3.0.tgz#5e5321a7abedfb7962d60dbf345deda87cb9560a"
+  integrity sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q==
+  dependencies:
+    pngjs "^6.0.0"
+
 pkg-dir@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-1.0.0.tgz#7a4b508a8d5bb2d629d447056ff4e9c9314cf3d4"
@@ -15994,11 +16015,21 @@ plist@^3.0.5:
     base64-js "^1.5.1"
     xmlbuilder "^15.1.1"
 
+pngjs@7.0.0:
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-7.0.0.tgz#a8b7446020ebbc6ac739db6c5415a65d17090e26"
+  integrity sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==
+
 pngjs@^3.4.0:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f"
   integrity sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==
 
+pngjs@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-6.0.0.tgz#ca9e5d2aa48db0228a52c419c3308e87720da821"
+  integrity sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==
+
 polished@^4.2.2:
   version "4.2.2"
   resolved "https://registry.yarnpkg.com/polished/-/polished-4.2.2.tgz#2529bb7c3198945373c52e34618c8fe7b1aa84d1"