Storage Service: Write

js/background.js

@@ -549,7 +549,7 @@
         window.isBeforeVersion(lastVersion, 'v1.35.0-beta.11') &&
         window.isAfterVersion(lastVersion, 'v1.35.0-beta.1')
       ) {
-        await window.Signal.Util.eraseAllStorageServiceState();
+        await window.Signal.Services.eraseAllStorageServiceState();
       }
 
       // This one should always be last - it could restart the app
@@ -2818,7 +2818,7 @@
         break;
       case FETCH_LATEST_ENUM.STORAGE_MANIFEST:
         window.log.info('onFetchLatestSync: fetching latest manifest');
-        await window.Signal.Util.runStorageServiceSyncJob();
+        await window.Signal.Services.runStorageServiceSyncJob();
         break;
       default:
         window.log.info(
@@ -2832,6 +2832,11 @@
 
     const { storageServiceKey } = ev;
 
+    if (storageServiceKey === null) {
+      window.log.info('onKeysSync: deleting storageKey');
+      storage.remove('storageKey');
+    }
+
     if (storageServiceKey) {
       window.log.info('onKeysSync: received keys');
       const storageServiceKeyBase64 = window.Signal.Crypto.arrayBufferToBase64(
@@ -2839,7 +2844,7 @@
       );
       storage.put('storageKey', storageServiceKeyBase64);
 
-      await window.Signal.Util.runStorageServiceSyncJob();
+      await window.Signal.Services.runStorageServiceSyncJob();
     }
   }
 

js/models/conversations.js

@@ -25,7 +25,7 @@
     UNRESTRICTED: 3,
   };
 
-  const { Util } = window.Signal;
+  const { Services, Util } = window.Signal;
   const { Contact, Message } = window.Signal.Types;
   const {
     deleteAttachmentData,
@@ -234,48 +234,84 @@
       return false;
     },
 
-    block() {
+    block({ viaStorageServiceSync = false } = {}) {
+      let blocked = false;
+      const isBlocked = this.isBlocked();
+
       const uuid = this.get('uuid');
       if (uuid) {
         window.storage.addBlockedUuid(uuid);
+        blocked = true;
       }
 
       const e164 = this.get('e164');
       if (e164) {
         window.storage.addBlockedNumber(e164);
+        blocked = true;
       }
 
       const groupId = this.get('groupId');
       if (groupId) {
         window.storage.addBlockedGroup(groupId);
+        blocked = true;
+      }
+
+      if (!viaStorageServiceSync && !isBlocked && blocked) {
+        this.captureChange();
       }
     },
 
-    unblock() {
+    unblock({ viaStorageServiceSync = false } = {}) {
+      let unblocked = false;
+      const isBlocked = this.isBlocked();
+
       const uuid = this.get('uuid');
       if (uuid) {
         window.storage.removeBlockedUuid(uuid);
+        unblocked = true;
       }
 
       const e164 = this.get('e164');
       if (e164) {
         window.storage.removeBlockedNumber(e164);
+        unblocked = true;
       }
 
       const groupId = this.get('groupId');
       if (groupId) {
         window.storage.removeBlockedGroup(groupId);
+        unblocked = true;
       }
 
-      return false;
+      if (!viaStorageServiceSync && isBlocked && unblocked) {
+        this.captureChange();
+      }
+
+      return unblocked;
     },
 
-    enableProfileSharing() {
+    enableProfileSharing({ viaStorageServiceSync = false } = {}) {
+      const before = this.get('profileSharing');
+
       this.set({ profileSharing: true });
+
+      const after = this.get('profileSharing');
+
+      if (!viaStorageServiceSync && Boolean(before) !== Boolean(after)) {
+        this.captureChange();
+      }
     },
 
-    disableProfileSharing() {
+    disableProfileSharing({ viaStorageServiceSync = false } = {}) {
+      const before = this.get('profileSharing');
+
       this.set({ profileSharing: false });
+
+      const after = this.get('profileSharing');
+
+      if (!viaStorageServiceSync && Boolean(before) !== Boolean(after)) {
+        this.captureChange();
+      }
     },
 
     hasDraft() {
@@ -662,7 +698,10 @@
       } while (messages.length > 0);
     },
 
-    async applyMessageRequestResponse(response, { fromSync = false } = {}) {
+    async applyMessageRequestResponse(
+      response,
+      { fromSync = false, viaStorageServiceSync = false } = {}
+    ) {
       // Apply message request response locally
       this.set({
         messageRequestResponseType: response,
@@ -670,8 +709,8 @@
       window.Signal.Data.updateConversation(this.attributes);
 
       if (response === this.messageRequestEnum.ACCEPT) {
-        this.unblock();
+        this.unblock({ viaStorageServiceSync });
-        this.enableProfileSharing();
+        this.enableProfileSharing({ viaStorageServiceSync });
 
         if (!fromSync) {
           this.sendProfileKeyUpdate();
@@ -680,13 +719,13 @@
         }
       } else if (response === this.messageRequestEnum.BLOCK) {
         // Block locally, other devices should block upon receiving the sync message
-        this.block();
+        this.block({ viaStorageServiceSync });
-        this.disableProfileSharing();
+        this.disableProfileSharing({ viaStorageServiceSync });
       } else if (response === this.messageRequestEnum.DELETE) {
         // Delete messages locally, other devices should delete upon receiving
         // the sync message
         this.destroyMessages();
-        this.disableProfileSharing();
+        this.disableProfileSharing({ viaStorageServiceSync });
         this.updateLastMessage();
         if (!fromSync) {
           this.trigger('unload', 'deleted from message request');
@@ -695,10 +734,10 @@
         // Delete messages locally, other devices should delete upon receiving
         // the sync message
         this.destroyMessages();
-        this.disableProfileSharing();
+        this.disableProfileSharing({ viaStorageServiceSync });
         this.updateLastMessage();
         // Block locally, other devices should block upon receiving the sync message
-        this.block();
+        this.block({ viaStorageServiceSync });
         // Leave group if this was a local action
         if (!fromSync) {
           this.leaveGroup();
@@ -780,6 +819,7 @@
     async _setVerified(verified, providedOptions) {
       const options = providedOptions || {};
       _.defaults(options, {
+        viaStorageServiceSync: false,
         viaSyncMessage: false,
         viaContactSync: false,
         key: null,
@@ -814,6 +854,14 @@
       this.set({ verified });
       window.Signal.Data.updateConversation(this.attributes);
 
+      if (
+        !options.viaStorageServiceSync &&
+        !keyChange &&
+        beginningVerified !== verified
+      ) {
+        this.captureChange();
+      }
+
       // Three situations result in a verification notice in the conversation:
       //   1) The message came from an explicit verification in another client (not
       //      a contact sync)
@@ -1982,9 +2030,17 @@
       window.Signal.Data.updateConversation(this.attributes);
     },
 
-    async setArchived(isArchived) {
+    setArchived(isArchived) {
+      const before = this.get('isArchived');
+
       this.set({ isArchived });
       window.Signal.Data.updateConversation(this.attributes);
+
+      const after = this.get('isArchived');
+
+      if (Boolean(before) !== Boolean(after)) {
+        this.captureChange();
+      }
     },
 
     async updateExpirationTimer(
@@ -2573,7 +2629,7 @@
       }
 
       try {
-        await c.setProfileName(profile.name);
+        await c.setEncryptedProfileName(profile.name);
       } catch (error) {
         window.log.warn(
           'getProfile decryption failure:',
@@ -2598,7 +2654,7 @@
 
       window.Signal.Data.updateConversation(c.attributes);
     },
-    async setProfileName(encryptedName) {
+    async setEncryptedProfileName(encryptedName) {
       if (!encryptedName) {
         return;
       }
@@ -2648,6 +2704,10 @@
         return;
       }
 
+      if (this.isMe()) {
+        window.storage.put('avatarUrl', avatarPath);
+      }
+
       const avatar = await textsecure.messaging.getAvatar(avatarPath);
       const key = this.get('profileKey');
       if (!key) {
@@ -2675,7 +2735,7 @@
         this.set(newAttributes);
       }
     },
-    async setProfileKey(profileKey) {
+    async setProfileKey(profileKey, { viaStorageServiceSync = false } = {}) {
       // profileKey is a string so we can compare it directly
       if (this.get('profileKey') !== profileKey) {
         window.log.info(
@@ -2689,6 +2749,10 @@
           sealedSender: SEALED_SENDER.UNKNOWN,
         });
 
+        if (!viaStorageServiceSync) {
+          this.captureChange();
+        }
+
         await Promise.all([
           this.deriveAccessKeyIfNeeded(),
           this.deriveProfileKeyVersionIfNeeded(),
@@ -2883,6 +2947,25 @@
       return null;
     },
 
+    // Set of items to captureChanges on:
+    // [-] uuid
+    // [-] e164
+    // [X] profileKey
+    // [-] identityKey
+    // [X] verified!
+    // [-] profileName
+    // [-] profileFamilyName
+    // [X] blocked
+    // [X] whitelisted
+    // [X] archived
+    captureChange() {
+      this.set({ needsStorageServiceSync: true });
+
+      this.queueJob(() => {
+        Services.storageServiceUploadJob();
+      });
+    },
+
     async notify(message, reaction) {
       if (this.get('muteExpiresAt') && Date.now() < this.get('muteExpiresAt')) {
         return;

js/modules/signal.js

@@ -116,6 +116,12 @@ const {
 } = require('../../ts/services/updateListener');
 const { notify } = require('../../ts/services/notify');
 const { calling } = require('../../ts/services/calling');
+const {
+  eraseAllStorageServiceState,
+  handleUnknownRecords,
+  runStorageServiceSyncJob,
+  storageServiceUploadJob,
+} = require('../../ts/services/storage');
 
 function initializeMigrations({
   userDataPath,
@@ -324,10 +330,14 @@ exports.setup = (options = {}) => {
   };
 
   const Services = {
+    calling,
+    eraseAllStorageServiceState,
+    handleUnknownRecords,
     initializeNetworkObserver,
     initializeUpdateListener,
     notify,
-    calling,
+    runStorageServiceSyncJob,
+    storageServiceUploadJob,
   };
 
   const State = {

ts/model-types.d.ts

@@ -58,18 +58,27 @@ type ConversationAttributesType = {
   isArchived?: boolean;
   lastMessage?: string;
   members?: Array<string>;
+  needsStorageServiceSync?: boolean;
   needsVerification?: boolean;
   profileFamilyName?: string | null;
   profileKey?: string | null;
   profileName?: string | null;
   profileSharing: boolean;
   storageID?: string;
+  storageUnknownFields: string;
   type: ConversationTypeType;
   unreadCount?: number;
   verified?: number;
   version: number;
 };
 
+type VerificationOptions = {
+  key?: null | ArrayBuffer;
+  viaContactSync?: boolean;
+  viaStorageServiceSync?: boolean;
+  viaSyncMessage?: boolean;
+};
+
 export declare class ConversationModelType extends Backbone.Model<
   ConversationAttributesType
 > {
@@ -81,11 +90,12 @@ export declare class ConversationModelType extends Backbone.Model<
   addCallHistory(details: CallHistoryDetailsType): void;
   applyMessageRequestResponse(
     response: number,
-    options?: { fromSync: boolean }
+    options?: { fromSync: boolean; viaStorageServiceSync?: boolean }
   ): void;
   cleanup(): Promise<void>;
-  disableProfileSharing(): void;
+  disableProfileSharing(options?: { viaStorageServiceSync?: boolean }): void;
   dropProfileKey(): Promise<void>;
+  enableProfileSharing(options?: { viaStorageServiceSync?: boolean }): void;
   generateProps(): void;
   getAccepted(): boolean;
   getAvatarPath(): string | undefined;
@@ -99,11 +109,23 @@ export declare class ConversationModelType extends Backbone.Model<
   getTitle(): string;
   idForLogging(): string;
   isFromOrAddedByTrustedContact(): boolean;
+  isBlocked(): boolean;
+  isMe(): boolean;
+  isPrivate(): boolean;
   isVerified(): boolean;
   safeGetVerified(): Promise<number>;
-  setProfileKey(profileKey?: string | null): Promise<void>;
+  setArchived(isArchived: boolean): void;
+  setProfileKey(
+    profileKey?: string | null,
+    options?: { viaStorageServiceSync?: boolean }
+  ): Promise<void>;
+  setProfileAvatar(avatarPath: string): Promise<void>;
+  setUnverified(options: VerificationOptions): Promise<TaskResultType>;
+  setVerified(options: VerificationOptions): Promise<TaskResultType>;
+  setVerifiedDefault(options: VerificationOptions): Promise<TaskResultType>;
   toggleVerified(): Promise<TaskResultType>;
-  unblock(): boolean | undefined;
+  block(options?: { viaStorageServiceSync?: boolean }): void;
+  unblock(options?: { viaStorageServiceSync?: boolean }): boolean;
   updateE164: (e164?: string) => void;
   updateLastMessage: () => Promise<void>;
   updateUuid: (uuid?: string) => void;

ts/services/storage.ts

@@ -0,0 +1,707 @@
+/* tslint:disable no-backbone-get-set-outside-model */
+import _ from 'lodash';
+import pMap from 'p-map';
+
+import Crypto from '../textsecure/Crypto';
+import dataInterface from '../sql/Client';
+import {
+  arrayBufferToBase64,
+  base64ToArrayBuffer,
+  deriveStorageItemKey,
+  deriveStorageManifestKey,
+} from '../Crypto';
+import {
+  ManifestRecordClass,
+  ManifestRecordIdentifierClass,
+  StorageItemClass,
+  StorageManifestClass,
+  StorageRecordClass,
+} from '../textsecure.d';
+import { ConversationModelType } from '../model-types.d';
+import {
+  mergeAccountRecord,
+  mergeContactRecord,
+  mergeGroupV1Record,
+  toAccountRecord,
+  toContactRecord,
+  toGroupV1Record,
+} from './storageRecordOps';
+
+const {
+  eraseStorageServiceStateFromConversations,
+  updateConversation,
+} = dataInterface;
+
+let consecutiveConflicts = 0;
+
+type UnknownRecord = {
+  itemType: number;
+  storageID: string;
+};
+
+function createWriteOperation(
+  storageManifest: StorageManifestClass,
+  newItems: Array<StorageItemClass>,
+  deleteKeys: Array<ArrayBuffer>,
+  clearAll = false
+) {
+  const writeOperation = new window.textsecure.protobuf.WriteOperation();
+  writeOperation.manifest = storageManifest;
+  writeOperation.insertItem = newItems;
+  writeOperation.deleteKey = deleteKeys;
+  writeOperation.clearAll = clearAll;
+
+  return writeOperation;
+}
+
+async function encryptRecord(
+  storageID: string | undefined,
+  storageRecord: StorageRecordClass
+): Promise<StorageItemClass> {
+  const storageItem = new window.textsecure.protobuf.StorageItem();
+
+  const storageKeyBuffer = storageID
+    ? base64ToArrayBuffer(String(storageID))
+    : generateStorageID();
+
+  const storageKeyBase64 = window.storage.get('storageKey');
+  const storageKey = base64ToArrayBuffer(storageKeyBase64);
+  const storageItemKey = await deriveStorageItemKey(
+    storageKey,
+    arrayBufferToBase64(storageKeyBuffer)
+  );
+
+  const encryptedRecord = await Crypto.encryptProfile(
+    storageRecord.toArrayBuffer(),
+    storageItemKey
+  );
+
+  storageItem.key = storageKeyBuffer;
+  storageItem.value = encryptedRecord;
+
+  return storageItem;
+}
+
+function generateStorageID(): ArrayBuffer {
+  return Crypto.getRandomBytes(16);
+}
+
+type GeneratedManifestType = {
+  conversationsToUpdate: Array<{
+    conversation: ConversationModelType;
+    storageID: string | undefined;
+  }>;
+  deleteKeys: Array<ArrayBuffer>;
+  newItems: Set<StorageItemClass>;
+  storageManifest: StorageManifestClass;
+};
+
+/* tslint:disable-next-line max-func-body-length */
+async function generateManifest(
+  version: number,
+  isNewManifest = false
+): Promise<GeneratedManifestType> {
+  window.log.info(
+    `storageService.generateManifest: generating manifest for version ${version}. Is new? ${isNewManifest}`
+  );
+
+  const ITEM_TYPE = window.textsecure.protobuf.ManifestRecord.Identifier.Type;
+
+  const conversationsToUpdate = [];
+  const deleteKeys = [];
+  const manifestRecordKeys: Set<ManifestRecordIdentifierClass> = new Set();
+  const newItems: Set<StorageItemClass> = new Set();
+
+  const conversations = window.getConversations();
+  for (let i = 0; i < conversations.length; i += 1) {
+    const conversation = conversations.models[i];
+    const identifier = new window.textsecure.protobuf.ManifestRecord.Identifier();
+
+    let storageRecord;
+    if (conversation.isMe()) {
+      storageRecord = new window.textsecure.protobuf.StorageRecord();
+      // eslint-disable-next-line no-await-in-loop
+      storageRecord.account = await toAccountRecord(conversation);
+      identifier.type = ITEM_TYPE.ACCOUNT;
+    } else if (conversation.isPrivate()) {
+      storageRecord = new window.textsecure.protobuf.StorageRecord();
+      // eslint-disable-next-line no-await-in-loop
+      storageRecord.contact = await toContactRecord(conversation);
+      identifier.type = ITEM_TYPE.CONTACT;
+    } else {
+      storageRecord = new window.textsecure.protobuf.StorageRecord();
+      // eslint-disable-next-line no-await-in-loop
+      storageRecord.groupV1 = await toGroupV1Record(conversation);
+      identifier.type = ITEM_TYPE.GROUPV1;
+    }
+
+    if (storageRecord) {
+      const isNewItem =
+        isNewManifest || Boolean(conversation.get('needsStorageServiceSync'));
+
+      const storageID = isNewItem
+        ? arrayBufferToBase64(generateStorageID())
+        : conversation.get('storageID');
+
+      // eslint-disable-next-line no-await-in-loop
+      const storageItem = await encryptRecord(storageID, storageRecord);
+      identifier.raw = storageItem.key;
+
+      // When a client needs to update a given record it should create it
+      // under a new key and delete the existing key.
+      if (isNewItem) {
+        newItems.add(storageItem);
+
+        const oldStorageID = conversation.get('storageID');
+        if (oldStorageID) {
+          deleteKeys.push(base64ToArrayBuffer(oldStorageID));
+        }
+
+        conversationsToUpdate.push({
+          conversation,
+          storageID,
+        });
+      }
+
+      manifestRecordKeys.add(identifier);
+    }
+  }
+
+  const unknownRecordsArray =
+    window.storage.get('storage-service-unknown-records') || [];
+
+  window.log.info(
+    `storageService.generateManifest: adding ${unknownRecordsArray.length} unknown records`
+  );
+
+  // When updating the manifest, ensure all "unknown" keys are added to the
+  // new manifest, so we don't inadvertently delete something we don't understand
+  unknownRecordsArray.forEach((record: UnknownRecord) => {
+    const identifier = new window.textsecure.protobuf.ManifestRecord.Identifier();
+    identifier.type = record.itemType;
+    identifier.raw = base64ToArrayBuffer(record.storageID);
+
+    manifestRecordKeys.add(identifier);
+  });
+
+  const manifestRecord = new window.textsecure.protobuf.ManifestRecord();
+  manifestRecord.version = version;
+  manifestRecord.keys = Array.from(manifestRecordKeys);
+
+  const storageKeyBase64 = window.storage.get('storageKey');
+  const storageKey = base64ToArrayBuffer(storageKeyBase64);
+  const storageManifestKey = await deriveStorageManifestKey(
+    storageKey,
+    version
+  );
+  const encryptedManifest = await Crypto.encryptProfile(
+    manifestRecord.toArrayBuffer(),
+    storageManifestKey
+  );
+
+  const storageManifest = new window.textsecure.protobuf.StorageManifest();
+  storageManifest.version = version;
+  storageManifest.value = encryptedManifest;
+
+  return {
+    conversationsToUpdate,
+    deleteKeys,
+    newItems,
+    storageManifest,
+  };
+}
+
+async function uploadManifest(
+  version: number,
+  {
+    conversationsToUpdate,
+    deleteKeys,
+    newItems,
+    storageManifest,
+  }: GeneratedManifestType
+): Promise<void> {
+  if (!window.textsecure.messaging) {
+    throw new Error('storageService.uploadManifest: We are offline!');
+  }
+
+  const credentials = window.storage.get('storageCredentials');
+  try {
+    window.log.info(
+      `storageService.uploadManifest: inserting ${newItems.size} items, deleting ${deleteKeys.length} keys`
+    );
+
+    const writeOperation = createWriteOperation(
+      storageManifest,
+      Array.from(newItems),
+      deleteKeys
+    );
+
+    window.log.info('storageService.uploadManifest: uploading...');
+    await window.textsecure.messaging.modifyStorageRecords(
+      writeOperation.toArrayBuffer(),
+      {
+        credentials,
+      }
+    );
+
+    window.log.info(
+      `storageService.uploadManifest: upload done, updating ${conversationsToUpdate.length} conversation(s) with new storageIDs`
+    );
+
+    // update conversations with the new storageID
+    conversationsToUpdate.forEach(({ conversation, storageID }) => {
+      conversation.set({
+        needsStorageServiceSync: false,
+        storageID,
+      });
+      updateConversation(conversation.attributes);
+    });
+  } catch (err) {
+    window.log.error(
+      `storageService.uploadManifest: failed! ${
+        err && err.stack ? err.stack : String(err)
+      }`
+    );
+
+    if (err.code === 409) {
+      window.log.info(
+        `storageService.uploadManifest: Conflict found, running sync job times(${consecutiveConflicts})`
+      );
+
+      if (consecutiveConflicts > 3) {
+        window.log.error(
+          'storageService.uploadManifest: Exceeded maximum consecutive conflicts'
+        );
+        return;
+      }
+      consecutiveConflicts += 1;
+
+      throw err;
+    }
+
+    throw err;
+  }
+
+  window.log.info(
+    'storageService.uploadManifest: setting new manifestVersion',
+    version
+  );
+  window.storage.put('manifestVersion', version);
+  consecutiveConflicts = 0;
+  await window.textsecure.messaging.sendFetchManifestSyncMessage();
+}
+
+async function stopStorageServiceSync() {
+  window.log.info('storageService.stopStorageServiceSync');
+
+  await window.storage.remove('storageKey');
+
+  if (!window.textsecure.messaging) {
+    throw new Error('storageService.stopStorageServiceSync: We are offline!');
+  }
+
+  await window.textsecure.messaging.sendRequestKeySyncMessage();
+}
+
+async function createNewManifest() {
+  window.log.info('storageService.createNewManifest: creating new manifest');
+
+  const version = window.storage.get('manifestVersion') || 0;
+
+  const {
+    conversationsToUpdate,
+    newItems,
+    storageManifest,
+  } = await generateManifest(version, true);
+
+  await uploadManifest(version, {
+    conversationsToUpdate,
+    // we have created a new manifest, there should be no keys to delete
+    deleteKeys: [],
+    newItems,
+    storageManifest,
+  });
+}
+
+async function decryptManifest(
+  encryptedManifest: StorageManifestClass
+): Promise<ManifestRecordClass> {
+  const { version, value } = encryptedManifest;
+
+  const storageKeyBase64 = window.storage.get('storageKey');
+  const storageKey = base64ToArrayBuffer(storageKeyBase64);
+  const storageManifestKey = await deriveStorageManifestKey(
+    storageKey,
+    typeof version === 'number' ? version : version.toNumber()
+  );
+
+  const decryptedManifest = await Crypto.decryptProfile(
+    typeof value.toArrayBuffer === 'function' ? value.toArrayBuffer() : value,
+    storageManifestKey
+  );
+
+  return window.textsecure.protobuf.ManifestRecord.decode(decryptedManifest);
+}
+
+async function fetchManifest(
+  manifestVersion: string
+): Promise<ManifestRecordClass | undefined> {
+  window.log.info('storageService.fetchManifest');
+
+  if (!window.textsecure.messaging) {
+    throw new Error('storageService.fetchManifest: We are offline!');
+  }
+
+  try {
+    const credentials = await window.textsecure.messaging.getStorageCredentials();
+    window.storage.put('storageCredentials', credentials);
+
+    const manifestBinary = await window.textsecure.messaging.getStorageManifest(
+      {
+        credentials,
+        greaterThanVersion: manifestVersion,
+      }
+    );
+    const encryptedManifest = window.textsecure.protobuf.StorageManifest.decode(
+      manifestBinary
+    );
+
+    // if we don't get a value we're assuming that there's no newer manifest
+    if (!encryptedManifest.value || !encryptedManifest.version) {
+      window.log.info('storageService.fetchManifest: nothing changed');
+      return;
+    }
+
+    try {
+      // eslint-disable-next-line consistent-return
+      return decryptManifest(encryptedManifest);
+    } catch (err) {
+      await stopStorageServiceSync();
+      return;
+    }
+  } catch (err) {
+    window.log.error(
+      `storageService.fetchManifest: failed! ${
+        err && err.stack ? err.stack : String(err)
+      }`
+    );
+
+    if (err.code === 404) {
+      await createNewManifest();
+      return;
+    } else if (err.code === 204) {
+      // noNewerManifest we're ok
+      return;
+    }
+
+    throw err;
+  }
+}
+
+type MergeableItemType = {
+  itemType: number;
+  storageID: string;
+  storageRecord: StorageRecordClass;
+};
+
+type MergedRecordType = UnknownRecord & {
+  hasConflict: boolean;
+  isUnsupported: boolean;
+};
+
+async function mergeRecord(
+  itemToMerge: MergeableItemType
+): Promise<MergedRecordType> {
+  const { itemType, storageID, storageRecord } = itemToMerge;
+
+  const ITEM_TYPE = window.textsecure.protobuf.ManifestRecord.Identifier.Type;
+
+  let hasConflict = false;
+  let isUnsupported = false;
+
+  try {
+    if (itemType === ITEM_TYPE.UNKNOWN) {
+      window.log.info(
+        'storageService.mergeRecord: Unknown item type',
+        storageID
+      );
+    } else if (itemType === ITEM_TYPE.CONTACT && storageRecord.contact) {
+      hasConflict = await mergeContactRecord(storageID, storageRecord.contact);
+    } else if (itemType === ITEM_TYPE.GROUPV1 && storageRecord.groupV1) {
+      hasConflict = await mergeGroupV1Record(storageID, storageRecord.groupV1);
+    } else if (itemType === ITEM_TYPE.ACCOUNT && storageRecord.account) {
+      hasConflict = await mergeAccountRecord(storageID, storageRecord.account);
+    } else {
+      isUnsupported = true;
+      window.log.info(
+        `storageService.mergeRecord: Unknown record: ${itemType}::${storageID}`
+      );
+    }
+  } catch (err) {
+    window.log.error(
+      `storageService.mergeRecord: merging record failed ${storageID}`
+    );
+  }
+
+  return {
+    hasConflict,
+    isUnsupported,
+    itemType,
+    storageID,
+  };
+}
+
+/* tslint:disable-next-line max-func-body-length */
+async function processManifest(
+  manifest: ManifestRecordClass
+): Promise<boolean> {
+  const storageKeyBase64 = window.storage.get('storageKey');
+  const storageKey = base64ToArrayBuffer(storageKeyBase64);
+
+  if (!window.textsecure.messaging) {
+    throw new Error('storageService.processManifest: We are offline!');
+  }
+
+  const remoteKeysTypeMap = new Map();
+  manifest.keys.forEach((identifier: ManifestRecordIdentifierClass) => {
+    remoteKeysTypeMap.set(
+      arrayBufferToBase64(identifier.raw.toArrayBuffer()),
+      identifier.type
+    );
+  });
+
+  const localKeys = window
+    .getConversations()
+    .map((conversation: ConversationModelType) => conversation.get('storageID'))
+    .filter(Boolean);
+
+  const unknownRecordsArray =
+    window.storage.get('storage-service-unknown-records') || [];
+
+  unknownRecordsArray.forEach((record: UnknownRecord) => {
+    localKeys.push(record.storageID);
+  });
+
+  window.log.info(
+    `storageService.processManifest: localKeys.length ${localKeys.length}`
+  );
+
+  const remoteKeys = Array.from(remoteKeysTypeMap.keys());
+
+  const remoteOnly = remoteKeys.filter(
+    (key: string) => !localKeys.includes(key)
+  );
+
+  window.log.info(
+    `storageService.processManifest: remoteOnly.length ${remoteOnly.length}`
+  );
+
+  const readOperation = new window.textsecure.protobuf.ReadOperation();
+  readOperation.readKey = remoteOnly.map(base64ToArrayBuffer);
+
+  const credentials = window.storage.get('storageCredentials');
+  const storageItemsBuffer = await window.textsecure.messaging.getStorageRecords(
+    readOperation.toArrayBuffer(),
+    {
+      credentials,
+    }
+  );
+
+  const storageItems = window.textsecure.protobuf.StorageItems.decode(
+    storageItemsBuffer
+  );
+
+  if (!storageItems.items) {
+    window.log.info(
+      'storageService.processManifest: No storage items retrieved'
+    );
+    return false;
+  }
+
+  const decryptedStorageItems = await pMap(
+    storageItems.items,
+    async (storageRecordWrapper: StorageItemClass) => {
+      const { key, value: storageItemCiphertext } = storageRecordWrapper;
+
+      if (!key || !storageItemCiphertext) {
+        await stopStorageServiceSync();
+        throw new Error(
+          'storageService.processManifest: Missing key and/or Ciphertext'
+        );
+      }
+
+      const base64ItemID = arrayBufferToBase64(key.toArrayBuffer());
+
+      const storageItemKey = await deriveStorageItemKey(
+        storageKey,
+        base64ItemID
+      );
+
+      let storageItemPlaintext;
+      try {
+        storageItemPlaintext = await Crypto.decryptProfile(
+          storageItemCiphertext.toArrayBuffer(),
+          storageItemKey
+        );
+      } catch (err) {
+        await stopStorageServiceSync();
+        throw err;
+      }
+
+      const storageRecord = window.textsecure.protobuf.StorageRecord.decode(
+        storageItemPlaintext
+      );
+
+      return {
+        itemType: remoteKeysTypeMap.get(base64ItemID),
+        storageID: base64ItemID,
+        storageRecord,
+      };
+    },
+    { concurrency: 50 }
+  );
+
+  try {
+    const mergedRecords = await pMap(decryptedStorageItems, mergeRecord, {
+      concurrency: 5,
+    });
+
+    const unknownRecords: Map<string, UnknownRecord> = new Map();
+    unknownRecordsArray.forEach((record: UnknownRecord) => {
+      unknownRecords.set(record.storageID, record);
+    });
+
+    const hasConflict = mergedRecords.some((mergedRecord: MergedRecordType) => {
+      if (mergedRecord.isUnsupported) {
+        unknownRecords.set(mergedRecord.storageID, {
+          itemType: mergedRecord.itemType,
+          storageID: mergedRecord.storageID,
+        });
+      }
+      return mergedRecord.hasConflict;
+    });
+
+    window.storage.put(
+      'storage-service-unknown-records',
+      Array.from(unknownRecords.values())
+    );
+
+    if (hasConflict) {
+      window.log.info(
+        'storageService.processManifest: Conflict found, uploading changes'
+      );
+
+      return true;
+    } else {
+      consecutiveConflicts = 0;
+    }
+  } catch (err) {
+    window.log.error(
+      `storageService.processManifest: failed! ${
+        err && err.stack ? err.stack : String(err)
+      }`
+    );
+  }
+
+  return false;
+}
+
+// Exported functions
+
+export async function runStorageServiceSyncJob(): Promise<void> {
+  if (!window.storage.get('storageKey')) {
+    throw new Error(
+      'storageService.runStorageServiceSyncJob: Cannot start; no storage key!'
+    );
+  }
+
+  window.log.info('storageService.runStorageServiceSyncJob: starting...');
+
+  try {
+    const localManifestVersion = window.storage.get('manifestVersion') || 0;
+    const manifest = await fetchManifest(localManifestVersion);
+
+    // Guarding against no manifests being returned, everything should be ok
+    if (!manifest) {
+      window.log.info(
+        'storageService.runStorageServiceSyncJob: no manifest, returning early'
+      );
+      return;
+    }
+
+    const version = manifest.version.toNumber();
+
+    window.log.info(
+      `storageService.runStorageServiceSyncJob: manifest versions - previous: ${localManifestVersion}, current: ${version}`
+    );
+
+    const hasConflicts = await processManifest(manifest);
+    if (hasConflicts) {
+      await storageServiceUploadJob();
+    }
+
+    window.storage.put('manifestVersion', version);
+  } catch (err) {
+    window.log.error(
+      `storageService.runStorageServiceSyncJob: error processing manifest ${
+        err && err.stack ? err.stack : String(err)
+      }`
+    );
+  }
+
+  window.log.info('storageService.runStorageServiceSyncJob: complete');
+}
+
+// Note: this function must be called at startup once we handle unknown records
+// of a certain type. This way once the runStorageServiceSyncJob function runs
+// it'll pick up the new storage IDs and process them accordingly.
+export function handleUnknownRecords(itemType: number): void {
+  const unknownRecordsArray =
+    window.storage.get('storage-service-unknown-records') || [];
+  const newUnknownRecords = unknownRecordsArray.filter(
+    (record: UnknownRecord) => record.itemType !== itemType
+  );
+  window.storage.put('storage-service-unknown-records', newUnknownRecords);
+}
+
+// Note: this function is meant to be called before ConversationController is hydrated.
+//   It goes directly to the database, so in-memory conversations will be out of date.
+export async function eraseAllStorageServiceState(): Promise<void> {
+  window.log.info('storageService.eraseAllStorageServiceState: starting...');
+  await Promise.all([
+    window.storage.remove('manifestVersion'),
+    window.storage.remove('storage-service-unknown-records'),
+    window.storage.remove('storageCredentials'),
+  ]);
+  await eraseStorageServiceStateFromConversations();
+  window.log.info('storageService.eraseAllStorageServiceState: complete');
+}
+
+async function nondebouncedStorageServiceUploadJob(): Promise<void> {
+  if (!window.storage.get('storageKey')) {
+    throw new Error(
+      'storageService.storageServiceUploadJob: Cannot start; no storage key!'
+    );
+  }
+
+  const localManifestVersion = window.storage.get('manifestVersion') || 0;
+  const version = Number(localManifestVersion) + 1;
+
+  window.log.info(
+    'storageService.storageServiceUploadJob: will update to manifest version',
+    version
+  );
+
+  try {
+    await uploadManifest(version, await generateManifest(version));
+  } catch (err) {
+    if (err.code === 409) {
+      await runStorageServiceSyncJob();
+    }
+  }
+}
+
+export const storageServiceUploadJob = _.debounce(
+  nondebouncedStorageServiceUploadJob,
+  500
+);

ts/services/storageRecordOps.ts

@@ -0,0 +1,403 @@
+/* tslint:disable no-backbone-get-set-outside-model */
+import _ from 'lodash';
+
+import {
+  arrayBufferToBase64,
+  base64ToArrayBuffer,
+  fromEncodedBinaryToArrayBuffer,
+} from '../Crypto';
+import dataInterface from '../sql/Client';
+import {
+  AccountRecordClass,
+  ContactRecordClass,
+  GroupV1RecordClass,
+} from '../textsecure.d';
+import { ConversationModelType } from '../model-types.d';
+
+const { updateConversation } = dataInterface;
+
+type RecordClass = AccountRecordClass | ContactRecordClass | GroupV1RecordClass;
+
+function toRecordVerified(verified: number): number {
+  const VERIFIED_ENUM = window.textsecure.storage.protocol.VerifiedStatus;
+  const STATE_ENUM = window.textsecure.protobuf.ContactRecord.IdentityState;
+
+  switch (verified) {
+    case VERIFIED_ENUM.VERIFIED:
+      return STATE_ENUM.VERIFIED;
+    case VERIFIED_ENUM.UNVERIFIED:
+      return STATE_ENUM.UNVERIFIED;
+    default:
+      return STATE_ENUM.DEFAULT;
+  }
+}
+
+function addUnknownFields(
+  record: RecordClass,
+  conversation: ConversationModelType
+): void {
+  if (record.__unknownFields) {
+    conversation.set({
+      storageUnknownFields: arrayBufferToBase64(record.__unknownFields),
+    });
+  }
+}
+
+function applyUnknownFields(
+  record: RecordClass,
+  conversation: ConversationModelType
+): void {
+  if (conversation.get('storageUnknownFields')) {
+    // eslint-disable-next-line no-param-reassign
+    record.__unknownFields = base64ToArrayBuffer(
+      conversation.get('storageUnknownFields')
+    );
+  }
+}
+
+export async function toContactRecord(
+  conversation: ConversationModelType
+): Promise<ContactRecordClass> {
+  const contactRecord = new window.textsecure.protobuf.ContactRecord();
+  if (conversation.get('uuid')) {
+    contactRecord.serviceUuid = conversation.get('uuid');
+  }
+  if (conversation.get('e164')) {
+    contactRecord.serviceE164 = conversation.get('e164');
+  }
+  if (conversation.get('profileKey')) {
+    contactRecord.profileKey = base64ToArrayBuffer(
+      String(conversation.get('profileKey'))
+    );
+  }
+  const identityKey = await window.textsecure.storage.protocol.loadIdentityKey(
+    conversation.id
+  );
+  if (identityKey) {
+    contactRecord.identityKey = identityKey;
+  }
+  if (conversation.get('verified')) {
+    contactRecord.identityState = toRecordVerified(
+      Number(conversation.get('verified'))
+    );
+  }
+  if (conversation.get('profileName')) {
+    contactRecord.givenName = conversation.get('profileName');
+  }
+  if (conversation.get('profileFamilyName')) {
+    contactRecord.familyName = conversation.get('profileFamilyName');
+  }
+  contactRecord.blocked = conversation.isBlocked();
+  contactRecord.whitelisted = Boolean(conversation.get('profileSharing'));
+  contactRecord.archived = Boolean(conversation.get('isArchived'));
+
+  applyUnknownFields(contactRecord, conversation);
+
+  return contactRecord;
+}
+
+export async function toAccountRecord(
+  conversation: ConversationModelType
+): Promise<AccountRecordClass> {
+  const accountRecord = new window.textsecure.protobuf.AccountRecord();
+
+  if (conversation.get('profileKey')) {
+    accountRecord.profileKey = base64ToArrayBuffer(
+      String(conversation.get('profileKey'))
+    );
+  }
+  if (conversation.get('profileName')) {
+    accountRecord.givenName = conversation.get('profileName') || '';
+  }
+  if (conversation.get('profileFamilyName')) {
+    accountRecord.familyName = conversation.get('profileFamilyName') || '';
+  }
+  accountRecord.avatarUrl = window.storage.get('avatarUrl') || '';
+  accountRecord.noteToSelfArchived = Boolean(conversation.get('isArchived'));
+  accountRecord.readReceipts = Boolean(
+    window.storage.get('read-receipt-setting')
+  );
+  accountRecord.sealedSenderIndicators = Boolean(
+    window.storage.get('sealedSenderIndicators')
+  );
+  accountRecord.typingIndicators = Boolean(
+    window.storage.get('typingIndicators')
+  );
+  accountRecord.linkPreviews = Boolean(window.storage.get('linkPreviews'));
+
+  applyUnknownFields(accountRecord, conversation);
+
+  return accountRecord;
+}
+
+export async function toGroupV1Record(
+  conversation: ConversationModelType
+): Promise<GroupV1RecordClass> {
+  const groupV1Record = new window.textsecure.protobuf.GroupV1Record();
+
+  groupV1Record.id = fromEncodedBinaryToArrayBuffer(
+    String(conversation.get('groupId'))
+  );
+  groupV1Record.blocked = conversation.isBlocked();
+  groupV1Record.whitelisted = Boolean(conversation.get('profileSharing'));
+  groupV1Record.archived = Boolean(conversation.get('isArchived'));
+
+  applyUnknownFields(groupV1Record, conversation);
+
+  return groupV1Record;
+}
+
+type MessageRequestCapableRecord = ContactRecordClass | GroupV1RecordClass;
+
+function applyMessageRequestState(
+  record: MessageRequestCapableRecord,
+  conversation: ConversationModelType
+): void {
+  if (record.blocked) {
+    conversation.applyMessageRequestResponse(
+      conversation.messageRequestEnum.BLOCK,
+      { fromSync: true, viaStorageServiceSync: true }
+    );
+  } else if (record.whitelisted) {
+    // unblocking is also handled by this function which is why the next
+    // condition is part of the else-if and not separate
+    conversation.applyMessageRequestResponse(
+      conversation.messageRequestEnum.ACCEPT,
+      { fromSync: true, viaStorageServiceSync: true }
+    );
+  } else if (!record.blocked) {
+    // if the condition above failed the state could still be blocked=false
+    // in which case we should unblock the conversation
+    conversation.unblock({ viaStorageServiceSync: true });
+  }
+
+  if (!record.whitelisted) {
+    conversation.disableProfileSharing({ viaStorageServiceSync: true });
+  }
+}
+
+function doesRecordHavePendingChanges(
+  mergedRecord: RecordClass,
+  serviceRecord: RecordClass,
+  conversation: ConversationModelType
+): boolean {
+  const shouldSync = Boolean(conversation.get('needsStorageServiceSync'));
+
+  const hasConflict = !_.isEqual(mergedRecord, serviceRecord);
+
+  if (shouldSync && !hasConflict) {
+    conversation.set({ needsStorageServiceSync: false });
+  }
+
+  return shouldSync && hasConflict;
+}
+
+export async function mergeGroupV1Record(
+  storageID: string,
+  groupV1Record: GroupV1RecordClass
+): Promise<boolean> {
+  window.log.info(`storageService.mergeGroupV1Record: merging ${storageID}`);
+
+  if (!groupV1Record.id) {
+    window.log.info(
+      `storageService.mergeGroupV1Record: no ID for ${storageID}`
+    );
+    return false;
+  }
+
+  const groupId = groupV1Record.id.toBinary();
+
+  // We do a get here because we don't get enough information from just this source to
+  //   be able to do the right thing with this group. So we'll update the local group
+  //   record if we have one; otherwise we'll just drop this update.
+  const conversation = window.ConversationController.get(groupId);
+  if (!conversation) {
+    window.log.warn(
+      `storageService.mergeGroupV1Record: No conversation for group(${groupId})`
+    );
+    return false;
+  }
+
+  conversation.set({
+    isArchived: Boolean(groupV1Record.archived),
+    storageID,
+  });
+
+  applyMessageRequestState(groupV1Record, conversation);
+
+  addUnknownFields(groupV1Record, conversation);
+
+  const hasPendingChanges = doesRecordHavePendingChanges(
+    await toGroupV1Record(conversation),
+    groupV1Record,
+    conversation
+  );
+
+  updateConversation(conversation.attributes);
+
+  window.log.info(`storageService.mergeGroupV1Record: merged ${storageID}`);
+
+  return hasPendingChanges;
+}
+
+export async function mergeContactRecord(
+  storageID: string,
+  contactRecord: ContactRecordClass
+): Promise<boolean> {
+  window.log.info(`storageService.mergeContactRecord: merging ${storageID}`);
+
+  window.normalizeUuids(
+    contactRecord,
+    ['serviceUuid'],
+    'storageService.mergeContactRecord'
+  );
+
+  const e164 = contactRecord.serviceE164 || undefined;
+  const uuid = contactRecord.serviceUuid || undefined;
+
+  const id = window.ConversationController.ensureContactIds({
+    e164,
+    uuid,
+    highTrust: true,
+  });
+
+  if (!id) {
+    window.log.info(
+      `storageService.mergeContactRecord: no ID for ${storageID}`
+    );
+    return false;
+  }
+
+  const conversation = await window.ConversationController.getOrCreateAndWait(
+    id,
+    'private'
+  );
+
+  if (contactRecord.profileKey) {
+    await conversation.setProfileKey(
+      arrayBufferToBase64(contactRecord.profileKey.toArrayBuffer()),
+      { viaStorageServiceSync: true }
+    );
+  }
+
+  const verified = await conversation.safeGetVerified();
+  const storageServiceVerified = contactRecord.identityState || 0;
+  if (verified !== storageServiceVerified) {
+    const verifiedOptions = { viaStorageServiceSync: true };
+    const STATE_ENUM = window.textsecure.protobuf.ContactRecord.IdentityState;
+
+    switch (storageServiceVerified) {
+      case STATE_ENUM.VERIFIED:
+        await conversation.setVerified(verifiedOptions);
+        break;
+      case STATE_ENUM.UNVERIFIED:
+        await conversation.setUnverified(verifiedOptions);
+        break;
+      default:
+        await conversation.setVerifiedDefault(verifiedOptions);
+    }
+  }
+
+  applyMessageRequestState(contactRecord, conversation);
+
+  addUnknownFields(contactRecord, conversation);
+
+  conversation.set({
+    isArchived: Boolean(contactRecord.archived),
+    storageID,
+  });
+
+  const hasPendingChanges = doesRecordHavePendingChanges(
+    await toContactRecord(conversation),
+    contactRecord,
+    conversation
+  );
+
+  updateConversation(conversation.attributes);
+
+  window.log.info(`storageService.mergeContactRecord: merged ${storageID}`);
+
+  return hasPendingChanges;
+}
+
+export async function mergeAccountRecord(
+  storageID: string,
+  accountRecord: AccountRecordClass
+): Promise<boolean> {
+  window.log.info(`storageService.mergeAccountRecord: merging ${storageID}`);
+
+  const {
+    avatarUrl,
+    linkPreviews,
+    noteToSelfArchived,
+    profileKey,
+    readReceipts,
+    sealedSenderIndicators,
+    typingIndicators,
+  } = accountRecord;
+
+  window.storage.put('read-receipt-setting', readReceipts);
+
+  if (typeof sealedSenderIndicators === 'boolean') {
+    window.storage.put('sealedSenderIndicators', sealedSenderIndicators);
+  }
+
+  if (typeof typingIndicators === 'boolean') {
+    window.storage.put('typingIndicators', typingIndicators);
+  }
+
+  if (typeof linkPreviews === 'boolean') {
+    window.storage.put('linkPreviews', linkPreviews);
+  }
+
+  if (profileKey) {
+    window.storage.put('profileKey', profileKey.toArrayBuffer());
+  }
+
+  window.log.info(
+    `storageService.mergeAccountRecord: merged settings ${storageID}`
+  );
+
+  const ourID = window.ConversationController.getOurConversationId();
+
+  if (!ourID) {
+    return false;
+  }
+
+  const conversation = await window.ConversationController.getOrCreateAndWait(
+    ourID,
+    'private'
+  );
+
+  addUnknownFields(accountRecord, conversation);
+
+  conversation.set({
+    isArchived: Boolean(noteToSelfArchived),
+    storageID,
+  });
+
+  if (accountRecord.profileKey) {
+    await conversation.setProfileKey(
+      arrayBufferToBase64(accountRecord.profileKey.toArrayBuffer())
+    );
+  }
+
+  if (avatarUrl) {
+    await conversation.setProfileAvatar(avatarUrl);
+    window.storage.put('avatarUrl', avatarUrl);
+  }
+
+  const hasPendingChanges = doesRecordHavePendingChanges(
+    await toAccountRecord(conversation),
+    accountRecord,
+    conversation
+  );
+
+  updateConversation(conversation.attributes);
+
+  window.log.info(
+    `storageService.mergeAccountRecord: merged profile ${storageID}`
+  );
+
+  return hasPendingChanges;
+}

ts/sql/Client.ts

@@ -129,7 +129,7 @@ const dataInterface: ClientInterface = {
   updateConversations,
   removeConversation,
 
-  eraseStorageIdFromConversations,
+  eraseStorageServiceStateFromConversations,
   getAllConversations,
   getAllConversationIds,
   getAllPrivateConversations,
@@ -773,8 +773,8 @@ async function _removeConversations(ids: Array<string>) {
   await channels.removeConversation(ids);
 }
 
-async function eraseStorageIdFromConversations() {
+async function eraseStorageServiceStateFromConversations() {
-  await channels.eraseStorageIdFromConversations();
+  await channels.eraseStorageServiceStateFromConversations();
 }
 
 async function getAllConversations({

ts/sql/Interface.ts

@@ -67,7 +67,7 @@ export interface DataInterface {
   removeAllSessions: () => Promise<void>;
   getAllSessions: () => Promise<Array<SessionType>>;
 
-  eraseStorageIdFromConversations: () => Promise<void>;
+  eraseStorageServiceStateFromConversations: () => Promise<void>;
   getConversationCount: () => Promise<number>;
   saveConversation: (data: ConversationType) => Promise<void>;
   saveConversations: (array: Array<ConversationType>) => Promise<void>;

ts/sql/Server.ts

@@ -104,7 +104,7 @@ const dataInterface: ServerInterface = {
   updateConversation,
   updateConversations,
   removeConversation,
-  eraseStorageIdFromConversations,
+  eraseStorageServiceStateFromConversations,
   getAllConversations,
   getAllConversationIds,
   getAllPrivateConversations,
@@ -2243,12 +2243,12 @@ async function getConversationById(id: string) {
   return jsonToObject(row.json);
 }
 
-async function eraseStorageIdFromConversations() {
+async function eraseStorageServiceStateFromConversations() {
   const db = getInstance();
 
   await db.run(
     `UPDATE conversations SET
-      json = json_remove(json, '$.storageID');
+      json = json_remove(json, '$.storageID', '$.needsStorageServiceSync', '$.unknownFields');
     `
   );
 }

ts/textsecure.d.ts

@@ -151,11 +151,12 @@ type StorageServiceProtobufTypes = {
   GroupV1Record: typeof GroupV1RecordClass;
   GroupV2Record: typeof GroupV2RecordClass;
   ManifestRecord: typeof ManifestRecordClass;
-  ReadOperation: typeof ReadOperation;
+  ReadOperation: typeof ReadOperationClass;
   StorageItem: typeof StorageItemClass;
   StorageItems: typeof StorageItemsClass;
-  StorageManifest: typeof StorageManifest;
+  StorageManifest: typeof StorageManifestClass;
   StorageRecord: typeof StorageRecordClass;
+  WriteOperation: typeof WriteOperationClass;
 };
 
 type ProtobufCollectionType = StorageServiceProtobufTypes & {
@@ -490,23 +491,23 @@ declare enum ManifestType {
   ACCOUNT,
 }
 
-type ManifestRecordIdentifier = {
+export declare class ManifestRecordIdentifierClass {
+  static Type: typeof ManifestType;
   raw: ProtoBinaryType;
   type: ManifestType;
-};
+  toArrayBuffer: () => ArrayBuffer;
+}
 
 export declare class ManifestRecordClass {
   static decode: (
     data: ArrayBuffer | ByteBufferClass,
     encoding?: string
   ) => ManifestRecordClass;
-
+  toArrayBuffer: () => ArrayBuffer;
-  static Identifier: {
+  static Identifier: typeof ManifestRecordIdentifierClass;
-    Type: typeof ManifestType;
-  };
 
   version: ProtoBigNumberType;
-  keys: ManifestRecordIdentifier[];
+  keys: ManifestRecordIdentifierClass[];
 }
 
 export declare class NullMessageClass {
@@ -579,14 +580,14 @@ export declare namespace ReceiptMessageClass {
 
 // Storage Service related types
 
-declare class StorageManifest {
+export declare class StorageManifestClass {
   static decode: (
     data: ArrayBuffer | ByteBufferClass,
     encoding?: string
-  ) => StorageManifest;
+  ) => StorageManifestClass;
 
   version?: ProtoBigNumberType | null;
-  value?: ByteBufferClass | null;
+  value?: ProtoBinaryType;
 }
 
 export declare class StorageRecordClass {
@@ -594,6 +595,7 @@ export declare class StorageRecordClass {
     data: ArrayBuffer | ByteBufferClass,
     encoding?: string
   ) => StorageRecordClass;
+  toArrayBuffer: () => ArrayBuffer;
 
   contact?: ContactRecordClass | null;
   groupV1?: GroupV1RecordClass | null;
@@ -607,8 +609,8 @@ export declare class StorageItemClass {
     encoding?: string
   ) => StorageItemClass;
 
-  key?: ByteBufferClass | null;
+  key?: ProtoBinaryType;
-  value?: ByteBufferClass | null;
+  value?: ProtoBinaryType;
 }
 
 export declare class StorageItemsClass {
@@ -633,11 +635,12 @@ export declare class ContactRecordClass {
     data: ArrayBuffer | ByteBufferClass,
     encoding?: string
   ) => ContactRecordClass;
+  toArrayBuffer: () => ArrayBuffer;
 
   serviceUuid?: string | null;
   serviceE164?: string | null;
-  profileKey?: ByteBufferClass | null;
+  profileKey?: ProtoBinaryType;
-  identityKey?: ByteBufferClass | null;
+  identityKey?: ProtoBinaryType;
   identityState?: ContactRecordIdentityState | null;
   givenName?: string | null;
   familyName?: string | null;
@@ -645,6 +648,8 @@ export declare class ContactRecordClass {
   blocked?: boolean | null;
   whitelisted?: boolean | null;
   archived?: boolean | null;
+
+  __unknownFields?: ArrayBuffer;
 }
 
 export declare class GroupV1RecordClass {
@@ -652,11 +657,14 @@ export declare class GroupV1RecordClass {
     data: ArrayBuffer | ByteBufferClass,
     encoding?: string
   ) => GroupV1RecordClass;
+  toArrayBuffer: () => ArrayBuffer;
 
-  id?: ByteBufferClass | null;
+  id?: ProtoBinaryType;
   blocked?: boolean | null;
   whitelisted?: boolean | null;
   archived?: boolean | null;
+
+  __unknownFields?: ArrayBuffer;
 }
 
 export declare class GroupV2RecordClass {
@@ -664,11 +672,14 @@ export declare class GroupV2RecordClass {
     data: ArrayBuffer | ByteBufferClass,
     encoding?: string
   ) => GroupV2RecordClass;
+  toArrayBuffer: () => ArrayBuffer;
 
   masterKey?: ByteBufferClass | null;
   blocked?: boolean | null;
   whitelisted?: boolean | null;
   archived?: boolean | null;
+
+  __unknownFields?: ArrayBuffer;
 }
 
 export declare class AccountRecordClass {
@@ -676,8 +687,9 @@ export declare class AccountRecordClass {
     data: ArrayBuffer | ByteBufferClass,
     encoding?: string
   ) => AccountRecordClass;
+  toArrayBuffer: () => ArrayBuffer;
 
-  profileKey?: ByteBufferClass | null;
+  profileKey?: ProtoBinaryType;
   givenName?: string | null;
   familyName?: string | null;
   avatarUrl?: string | null;
@@ -686,18 +698,33 @@ export declare class AccountRecordClass {
   sealedSenderIndicators?: boolean | null;
   typingIndicators?: boolean | null;
   linkPreviews?: boolean | null;
+
+  __unknownFields?: ArrayBuffer;
 }
 
-declare class ReadOperation {
+declare class ReadOperationClass {
   static decode: (
     data: ArrayBuffer | ByteBufferClass,
     encoding?: string
-  ) => ReadOperation;
+  ) => ReadOperationClass;
 
   readKey: ArrayBuffer[] | ByteBufferClass[];
   toArrayBuffer: () => ArrayBuffer;
 }
 
+declare class WriteOperationClass {
+  static decode: (
+    data: ArrayBuffer | ByteBufferClass,
+    encoding?: string
+  ) => WriteOperationClass;
+  toArrayBuffer: () => ArrayBuffer;
+
+  manifest: StorageManifestClass;
+  insertItem: StorageItemClass[];
+  deleteKey: ArrayBuffer[] | ByteBufferClass[];
+  clearAll: boolean;
+}
+
 export declare class SyncMessageClass {
   static decode: (
     data: ArrayBuffer | ByteBufferClass,
@@ -770,6 +797,11 @@ export declare namespace SyncMessageClass {
     timestamp?: ProtoBinaryType;
   }
   class FetchLatest {
+    static Type: {
+      UNKNOWN: number;
+      LOCAL_PROFILE: number;
+      STORAGE_MANIFEST: number;
+    };
     type?: number;
   }
   class Keys {

ts/textsecure/SendMessage.ts

@@ -823,7 +823,35 @@ export default class MessageSender {
     return Promise.resolve();
   }
 
-  async sendRequestKeySyncMessage(options: SendOptionsType) {
+  async sendFetchManifestSyncMessage(options?: SendOptionsType) {
+    const myUuid = window.textsecure.storage.user.getUuid();
+    const myNumber = window.textsecure.storage.user.getNumber();
+    const myDevice = window.textsecure.storage.user.getDeviceId();
+
+    if (myDevice === 1 || myDevice === '1') {
+      return;
+    }
+
+    const fetchLatest = new window.textsecure.protobuf.SyncMessage.FetchLatest();
+    fetchLatest.type =
+      window.textsecure.protobuf.SyncMessage.FetchLatest.Type.STORAGE_MANIFEST;
+
+    const syncMessage = this.createSyncMessage();
+    syncMessage.fetchLatest = fetchLatest;
+    const contentMessage = new window.textsecure.protobuf.Content();
+    contentMessage.syncMessage = syncMessage;
+
+    const silent = true;
+    await this.sendIndividualProto(
+      myUuid || myNumber,
+      contentMessage,
+      Date.now(),
+      silent,
+      options
+    );
+  }
+
+  async sendRequestKeySyncMessage(options?: SendOptionsType) {
     const myUuid = window.textsecure.storage.user.getUuid();
     const myNumber = window.textsecure.storage.user.getNumber();
     const myDevice = window.textsecure.storage.user.getDeviceId();
@@ -1694,4 +1722,11 @@ export default class MessageSender {
   ): Promise<ArrayBuffer> {
     return this.server.getStorageRecords(data, options);
   }
+
+  async modifyStorageRecords(
+    data: ArrayBuffer,
+    options: StorageServiceCallOptionsType
+  ): Promise<ArrayBuffer> {
+    return this.server.modifyStorageRecords(data, options);
+  }
 }

ts/textsecure/WebAPI.ts

@@ -701,6 +701,7 @@ export type WebAPIType = {
     targetUrl: string,
     options?: ProxiedRequestOptionsType
   ) => Promise<any>;
+  modifyStorageRecords: MessageSender['modifyStorageRecords'];
   putAttachment: (encryptedBin: ArrayBuffer) => Promise<any>;
   registerCapabilities: (capabilities: any) => Promise<void>;
   putStickers: (
@@ -860,6 +861,7 @@ export function initialize({
       getStorageRecords,
       getUuidsForE164s,
       makeProxiedRequest,
+      modifyStorageRecords,
       putAttachment,
       registerCapabilities,
       putStickers,
@@ -1012,6 +1014,25 @@ export function initialize({
       });
     }
 
+    async function modifyStorageRecords(
+      data: ArrayBuffer,
+      options: StorageServiceCallOptionsType = {}
+    ): Promise<ArrayBuffer> {
+      const { credentials } = options;
+
+      return _ajax({
+        call: 'storageModify',
+        contentType: 'application/x-protobuf',
+        data,
+        host: storageUrl,
+        httpType: 'PUT',
+        // If we run into a conflict, the current manifest is returned -
+        //   it will will be an ArrayBuffer at the response key on the Error
+        responseType: 'arraybuffer',
+        ...credentials,
+      });
+    }
+
     async function registerSupportForUnauthenticatedDelivery() {
       return _ajax({
         call: 'supportUnauthenticatedDelivery',

ts/util/index.ts

@@ -16,10 +16,6 @@ import { isFileDangerous } from './isFileDangerous';
 import { makeLookup } from './makeLookup';
 import { migrateColor } from './migrateColor';
 import { missingCaseError } from './missingCaseError';
-import {
-  eraseAllStorageServiceState,
-  runStorageServiceSyncJob,
-} from './storageService';
 import * as zkgroup from './zkgroup';
 
 export {
@@ -29,7 +25,6 @@ export {
   createWaitBatcher,
   deleteForEveryone,
   downloadAttachment,
-  eraseAllStorageServiceState,
   generateSecurityNumber,
   getSafetyNumberPlaceholder,
   getStringForProfileChange,
@@ -40,6 +35,5 @@ export {
   migrateColor,
   missingCaseError,
   Registration,
-  runStorageServiceSyncJob,
   zkgroup,
 };

ts/util/lint/exceptions.json

@@ -12952,7 +12952,7 @@
     "rule": "jQuery-wrap(",
     "path": "ts/textsecure/WebAPI.js",
     "line": "            const byteBuffer = window.dcodeIO.ByteBuffer.wrap(quote, 'binary', window.dcodeIO.ByteBuffer.LITTLE_ENDIAN);",
-    "lineNumber": 1049,
+    "lineNumber": 1057,
     "reasonCategory": "falseMatch",
     "updated": "2020-09-08T23:07:22.682Z"
   },
@@ -12960,7 +12960,7 @@
     "rule": "jQuery-wrap(",
     "path": "ts/textsecure/WebAPI.ts",
     "line": "      const byteBuffer = window.dcodeIO.ByteBuffer.wrap(",
-    "lineNumber": 1748,
+    "lineNumber": 1769,
     "reasonCategory": "falseMatch",
     "updated": "2020-09-08T23:07:22.682Z"
   }

ts/util/storageService.ts

@@ -1,478 +0,0 @@
-/* tslint:disable no-backbone-get-set-outside-model */
-import _ from 'lodash';
-import PQueue from 'p-queue';
-
-import Crypto from '../textsecure/Crypto';
-import {
-  arrayBufferToBase64,
-  base64ToArrayBuffer,
-  constantTimeEqual,
-  deriveStorageItemKey,
-  deriveStorageManifestKey,
-} from '../Crypto';
-import dataInterface from '../sql/Client';
-const { eraseStorageIdFromConversations, updateConversation } = dataInterface;
-import {
-  AccountRecordClass,
-  ContactRecordClass,
-  GroupV1RecordClass,
-  ManifestRecordClass,
-  StorageItemClass,
-} from '../textsecure.d';
-import { ConversationModelType } from '../model-types.d';
-
-function fromRecordVerified(verified: number): number {
-  const VERIFIED_ENUM = window.textsecure.storage.protocol.VerifiedStatus;
-  const STATE_ENUM = window.textsecure.protobuf.ContactRecord.IdentityState;
-
-  switch (verified) {
-    case STATE_ENUM.VERIFIED:
-      return VERIFIED_ENUM.VERIFIED;
-    case STATE_ENUM.UNVERIFIED:
-      return VERIFIED_ENUM.UNVERIFIED;
-    default:
-      return VERIFIED_ENUM.DEFAULT;
-  }
-}
-
-async function fetchManifest(manifestVersion: string) {
-  window.log.info('storageService.fetchManifest');
-
-  if (!window.textsecure.messaging) {
-    throw new Error('fetchManifest: We are offline!');
-  }
-
-  try {
-    const credentials = await window.textsecure.messaging.getStorageCredentials();
-    window.storage.put('storageCredentials', credentials);
-
-    const manifestBinary = await window.textsecure.messaging.getStorageManifest(
-      {
-        credentials,
-        greaterThanVersion: manifestVersion,
-      }
-    );
-    const encryptedManifest = window.textsecure.protobuf.StorageManifest.decode(
-      manifestBinary
-    );
-
-    // If we don't get a value we're assuming we're receiving a 204
-    // it would be nice to get an actual e.code 204 and check against that.
-    if (!encryptedManifest.value || !encryptedManifest.version) {
-      window.log.info('storageService.fetchManifest: nothing changed');
-      return;
-    }
-
-    const storageKeyBase64 = window.storage.get('storageKey');
-    const storageKey = base64ToArrayBuffer(storageKeyBase64);
-    const storageManifestKey = await deriveStorageManifestKey(
-      storageKey,
-      encryptedManifest.version.toNumber()
-    );
-
-    const decryptedManifest = await Crypto.decryptProfile(
-      encryptedManifest.value.toArrayBuffer(),
-      storageManifestKey
-    );
-
-    return window.textsecure.protobuf.ManifestRecord.decode(decryptedManifest);
-  } catch (err) {
-    window.log.error(`storageService.fetchManifest: ${err}`);
-
-    if (err.code === 404) {
-      // No manifest exists, we create one
-      return { version: 0, keys: [] };
-    } else if (err.code === 204) {
-      // noNewerManifest we're ok
-      return;
-    }
-
-    throw err;
-  }
-}
-
-type MessageRequestCapableRecord = ContactRecordClass | GroupV1RecordClass;
-
-function applyMessageRequestState(
-  record: MessageRequestCapableRecord,
-  conversation: ConversationModelType
-): void {
-  if (record.blocked) {
-    conversation.applyMessageRequestResponse(
-      conversation.messageRequestEnum.BLOCK,
-      { fromSync: true }
-    );
-  } else if (record.whitelisted) {
-    // unblocking is also handled by this function which is why the next
-    // condition is part of the else-if and not separate
-    conversation.applyMessageRequestResponse(
-      conversation.messageRequestEnum.ACCEPT,
-      { fromSync: true }
-    );
-  } else if (!record.blocked) {
-    // if the condition above failed the state could still be blocked=false
-    // in which case we should unblock the conversation
-    conversation.unblock();
-  }
-
-  if (!record.whitelisted) {
-    conversation.disableProfileSharing();
-  }
-}
-
-async function mergeGroupV1Record(
-  storageID: string,
-  groupV1Record: GroupV1RecordClass
-): Promise<void> {
-  window.log.info(`storageService.mergeGroupV1Record: merging ${storageID}`);
-
-  if (!groupV1Record.id) {
-    window.log.info(
-      `storageService.mergeGroupV1Record: no ID for ${storageID}`
-    );
-    return;
-  }
-
-  const groupId = groupV1Record.id.toBinary();
-
-  // We do a get here because we don't get enough information from just this source to
-  //   be able to do the right thing with this group. So we'll update the local group
-  //   record if we have one; otherwise we'll just drop this update.
-  const conversation = window.ConversationController.get(groupId);
-  if (!conversation) {
-    window.log.warn(
-      `storageService.mergeGroupV1Record: No conversation for group(${groupId})`
-    );
-    return;
-  }
-
-  conversation.set({
-    isArchived: Boolean(groupV1Record.archived),
-    storageID,
-  });
-
-  applyMessageRequestState(groupV1Record, conversation);
-
-  updateConversation(conversation.attributes);
-
-  window.log.info(`storageService.mergeGroupV1Record: merged ${storageID}`);
-}
-
-async function mergeContactRecord(
-  storageID: string,
-  contactRecord: ContactRecordClass
-): Promise<void> {
-  window.log.info(`storageService.mergeContactRecord: merging ${storageID}`);
-
-  window.normalizeUuids(
-    contactRecord,
-    ['serviceUuid'],
-    'storageService.mergeContactRecord'
-  );
-
-  const e164 = contactRecord.serviceE164 || undefined;
-  const uuid = contactRecord.serviceUuid || undefined;
-
-  const id = window.ConversationController.ensureContactIds({
-    e164,
-    uuid,
-    highTrust: true,
-  });
-
-  if (!id) {
-    window.log.info(
-      `storageService.mergeContactRecord: no ID for ${storageID}`
-    );
-    return;
-  }
-
-  const conversation = await window.ConversationController.getOrCreateAndWait(
-    id,
-    'private'
-  );
-
-  const verified = contactRecord.identityState
-    ? fromRecordVerified(contactRecord.identityState)
-    : window.textsecure.storage.protocol.VerifiedStatus.DEFAULT;
-
-  conversation.set({
-    isArchived: Boolean(contactRecord.archived),
-    storageID,
-    verified,
-  });
-
-  if (contactRecord.profileKey) {
-    await conversation.setProfileKey(
-      arrayBufferToBase64(contactRecord.profileKey.toArrayBuffer())
-    );
-  } else {
-    await conversation.dropProfileKey();
-  }
-
-  applyMessageRequestState(contactRecord, conversation);
-
-  const identityKey = await window.textsecure.storage.protocol.loadIdentityKey(
-    conversation.id
-  );
-
-  const identityKeyChanged =
-    identityKey && contactRecord.identityKey
-      ? !constantTimeEqual(
-          identityKey,
-          contactRecord.identityKey.toArrayBuffer()
-        )
-      : false;
-
-  if (identityKeyChanged && contactRecord.identityKey) {
-    await window.textsecure.storage.protocol.processVerifiedMessage(
-      conversation.id,
-      verified,
-      contactRecord.identityKey.toArrayBuffer()
-    );
-  } else if (conversation.get('verified')) {
-    await window.textsecure.storage.protocol.setVerified(
-      conversation.id,
-      verified
-    );
-  }
-
-  updateConversation(conversation.attributes);
-
-  window.log.info(`storageService.mergeContactRecord: merged ${storageID}`);
-}
-
-async function mergeAccountRecord(
-  storageID: string,
-  accountRecord: AccountRecordClass
-): Promise<void> {
-  window.log.info(`storageService.mergeAccountRecord: merging ${storageID}`);
-
-  const {
-    profileKey,
-    linkPreviews,
-    readReceipts,
-    sealedSenderIndicators,
-    typingIndicators,
-  } = accountRecord;
-
-  window.storage.put('read-receipt-setting', readReceipts);
-
-  if (typeof sealedSenderIndicators === 'boolean') {
-    window.storage.put('sealedSenderIndicators', sealedSenderIndicators);
-  }
-
-  if (typeof typingIndicators === 'boolean') {
-    window.storage.put('typingIndicators', typingIndicators);
-  }
-
-  if (typeof linkPreviews === 'boolean') {
-    window.storage.put('linkPreviews', linkPreviews);
-  }
-
-  if (profileKey) {
-    window.storage.put('profileKey', profileKey.toArrayBuffer());
-  }
-
-  window.log.info(
-    `storageService.mergeAccountRecord: merged settings ${storageID}`
-  );
-
-  const ourID = window.ConversationController.getOurConversationId();
-
-  if (!ourID) {
-    return;
-  }
-
-  const conversation = await window.ConversationController.getOrCreateAndWait(
-    ourID,
-    'private'
-  );
-
-  conversation.set({
-    storageID,
-  });
-
-  if (accountRecord.profileKey) {
-    await conversation.setProfileKey(
-      arrayBufferToBase64(accountRecord.profileKey.toArrayBuffer())
-    );
-  } else {
-    await conversation.dropProfileKey();
-  }
-
-  updateConversation(conversation.attributes);
-
-  window.log.info(
-    `storageService.mergeAccountRecord: merged profile ${storageID}`
-  );
-}
-
-// tslint:disable-next-line max-func-body-length
-async function processManifest(
-  manifest: ManifestRecordClass
-): Promise<boolean> {
-  const credentials = window.storage.get('storageCredentials');
-  const storageKeyBase64 = window.storage.get('storageKey');
-  const storageKey = base64ToArrayBuffer(storageKeyBase64);
-
-  if (!window.textsecure.messaging) {
-    throw new Error('processManifest: We are offline!');
-  }
-
-  const remoteKeysTypeMap = new Map();
-  manifest.keys.forEach(key => {
-    remoteKeysTypeMap.set(
-      arrayBufferToBase64(key.raw.toArrayBuffer()),
-      key.type
-    );
-  });
-
-  const localKeys = window
-    .getConversations()
-    .map((conversation: ConversationModelType) => conversation.get('storageID'))
-    .filter(Boolean);
-
-  window.log.info(
-    `storageService.processManifest localKeys.length ${localKeys.length}`
-  );
-
-  const remoteKeys = Array.from(remoteKeysTypeMap.keys());
-
-  const remoteOnly = remoteKeys.filter(
-    (key: string) => !localKeys.includes(key)
-  );
-
-  window.log.info(
-    `storageService.processManifest remoteOnly.length ${remoteOnly.length}`
-  );
-
-  const readOperation = new window.textsecure.protobuf.ReadOperation();
-  readOperation.readKey = remoteOnly.map(base64ToArrayBuffer);
-
-  const storageItemsBuffer = await window.textsecure.messaging.getStorageRecords(
-    readOperation.toArrayBuffer(),
-    {
-      credentials,
-    }
-  );
-
-  const storageItems = window.textsecure.protobuf.StorageItems.decode(
-    storageItemsBuffer
-  );
-
-  if (!storageItems.items) {
-    return false;
-  }
-
-  const queue = new PQueue({ concurrency: 4 });
-
-  const mergedItems = storageItems.items.map(
-    (storageRecordWrapper: StorageItemClass) => async () => {
-      const { key, value: storageItemCiphertext } = storageRecordWrapper;
-
-      if (!key || !storageItemCiphertext) {
-        return;
-      }
-
-      const base64ItemID = arrayBufferToBase64(key.toArrayBuffer());
-
-      const storageItemKey = await deriveStorageItemKey(
-        storageKey,
-        base64ItemID
-      );
-
-      const storageItemPlaintext = await Crypto.decryptProfile(
-        storageItemCiphertext.toArrayBuffer(),
-        storageItemKey
-      );
-      const storageRecord = window.textsecure.protobuf.StorageRecord.decode(
-        storageItemPlaintext
-      );
-
-      const itemType = remoteKeysTypeMap.get(base64ItemID);
-
-      const ITEM_TYPE =
-        window.textsecure.protobuf.ManifestRecord.Identifier.Type;
-
-      try {
-        if (itemType === ITEM_TYPE.UNKNOWN) {
-          window.log.info('storageService.processManifest: Unknown item type');
-        } else if (itemType === ITEM_TYPE.CONTACT && storageRecord.contact) {
-          await mergeContactRecord(base64ItemID, storageRecord.contact);
-        } else if (itemType === ITEM_TYPE.GROUPV1 && storageRecord.groupV1) {
-          await mergeGroupV1Record(base64ItemID, storageRecord.groupV1);
-        } else if (itemType === ITEM_TYPE.GROUPV2 && storageRecord.groupV2) {
-          window.log.info(
-            'storageService.processManifest: Skipping GroupV2 item'
-          );
-        } else if (itemType === ITEM_TYPE.ACCOUNT && storageRecord.account) {
-          await mergeAccountRecord(base64ItemID, storageRecord.account);
-        }
-      } catch (err) {
-        window.log.error(
-          `storageService.processManifest: merging record failed ${base64ItemID}`
-        );
-      }
-    }
-  );
-
-  try {
-    await queue.addAll(mergedItems);
-    await queue.onEmpty();
-    return true;
-  } catch (err) {
-    window.log.error('storageService.processManifest: merging failed');
-    return false;
-  }
-}
-
-export async function runStorageServiceSyncJob() {
-  if (!window.storage.get('storageKey')) {
-    throw new Error('runStorageServiceSyncJob: Cannot start; no storage key!');
-  }
-
-  window.log.info('runStorageServiceSyncJob: starting...');
-
-  const localManifestVersion = window.storage.get('manifestVersion') || 0;
-
-  let manifest;
-  try {
-    manifest = await fetchManifest(localManifestVersion);
-
-    // Guarding against no manifests being returned, everything should be ok
-    if (!manifest) {
-      window.log.info('runStorageServiceSyncJob: no manifest, returning early');
-      return;
-    }
-  } catch (err) {
-    // We are supposed to retry here if it's a retryable error
-    window.log.error(
-      `storageService.runStorageServiceSyncJob: failed! ${
-        err && err.stack ? err.stack : String(err)
-      }`
-    );
-    return;
-  }
-
-  const version = manifest.version.toNumber();
-
-  window.log.info(
-    `runStorageServiceSyncJob: manifest versions - previous: ${localManifestVersion}, current: ${version}`
-  );
-
-  const shouldUpdateVersion = await processManifest(manifest);
-
-  if (shouldUpdateVersion) {
-    window.storage.put('manifestVersion', version);
-  }
-  window.log.info('runStorageServiceSyncJob: complete');
-}
-
-// Note: this function is meant to be called before ConversationController is hydrated.
-//   It goes directly to the database, so in-memory conversations will be out of date.
-export async function eraseAllStorageServiceState() {
-  window.log.info('eraseAllStorageServiceState: starting...');
-  await window.storage.remove('manifestVersion');
-  await eraseStorageIdFromConversations();
-  window.log.info('eraseAllStorageServiceState: complete');
-}