Move conversations to SQLCipher

app/sql.js

@@ -15,6 +15,18 @@ module.exports = {
   close,
   removeDB,
 
+  getConversationCount,
+  saveConversation,
+  saveConversations,
+  getConversationById,
+  updateConversation,
+  removeConversation,
+  getAllConversations,
+  getAllConversationIds,
+  getAllPrivateConversations,
+  getAllGroupsInvolvingId,
+  searchConversations,
+
   getMessageCount,
   saveMessage,
   saveMessages,
@@ -22,6 +34,7 @@ module.exports = {
   getUnreadByConversation,
   getMessageBySender,
   getMessageById,
+  getAllMessages,
   getAllMessageIds,
   getMessagesBySentAt,
   getExpiredMessages,
@@ -270,10 +283,47 @@ async function updateToSchemaVersion3(currentVersion, instance) {
   console.log('updateToSchemaVersion3: success!');
 }
 
+async function updateToSchemaVersion4(currentVersion, instance) {
+  if (currentVersion >= 4) {
+    return;
+  }
+
+  console.log('updateToSchemaVersion4: starting...');
+
+  await instance.run('BEGIN TRANSACTION;');
+
+  await instance.run(
+    `CREATE TABLE conversations(
+      id STRING PRIMARY KEY ASC,
+      json TEXT,
+
+      active_at INTEGER,
+      type STRING,
+      members TEXT,
+      name TEXT,
+      profileName TEXT
+    );`
+  );
+
+  await instance.run(`CREATE INDEX conversations_active ON conversations (
+      active_at
+    ) WHERE active_at IS NOT NULL;`);
+
+  await instance.run(`CREATE INDEX conversations_type ON conversations (
+      type
+    ) WHERE type IS NOT NULL;`);
+
+  await instance.run('PRAGMA schema_version = 4;');
+  await instance.run('COMMIT TRANSACTION;');
+
+  console.log('updateToSchemaVersion4: success!');
+}
+
 const SCHEMA_VERSIONS = [
   updateToSchemaVersion1,
   updateToSchemaVersion2,
   updateToSchemaVersion3,
+  updateToSchemaVersion4,
 ];
 
 async function updateSchema(instance) {
@@ -348,6 +398,190 @@ async function removeDB() {
   rimraf.sync(filePath);
 }
 
+async function getConversationCount() {
+  const row = await db.get('SELECT count(*) from conversations;');
+
+  if (!row) {
+    throw new Error('getMessageCount: Unable to get count of conversations');
+  }
+
+  return row['count(*)'];
+}
+
+async function saveConversation(data) {
+  // eslint-disable-next-line camelcase
+  const { id, active_at, type, members, name, profileName } = data;
+
+  await db.run(
+    `INSERT INTO conversations (
+    id,
+    json,
+
+    active_at,
+    type,
+    members,
+    name,
+    profileName
+  ) values (
+    $id,
+    $json,
+
+    $active_at,
+    $type,
+    $members,
+    $name,
+    $profileName
+  );`,
+    {
+      $id: id,
+      $json: objectToJSON(data),
+
+      $active_at: active_at,
+      $type: type,
+      $members: members ? members.join(' ') : null,
+      $name: name,
+      $profileName: profileName,
+    }
+  );
+}
+
+async function saveConversations(arrayOfConversations) {
+  let promise;
+
+  db.serialize(() => {
+    promise = Promise.all([
+      db.run('BEGIN TRANSACTION;'),
+      ...map(arrayOfConversations, conversation =>
+        saveConversation(conversation)
+      ),
+      db.run('COMMIT TRANSACTION;'),
+    ]);
+  });
+
+  await promise;
+}
+
+async function updateConversation(data) {
+  // eslint-disable-next-line camelcase
+  const { id, active_at, type, members, name, profileName } = data;
+
+  await db.run(
+    `UPDATE conversations SET
+    json = $json,
+
+    active_at = $active_at,
+    type = $type,
+    members = $members,
+    name = $name,
+    profileName = $profileName
+  WHERE id = $id;`,
+    {
+      $id: id,
+      $json: objectToJSON(data),
+
+      $active_at: active_at,
+      $type: type,
+      $members: members ? members.join(' ') : null,
+      $name: name,
+      $profileName: profileName,
+    }
+  );
+}
+
+async function removeConversation(id) {
+  if (!Array.isArray(id)) {
+    await db.run('DELETE FROM conversations WHERE id = $id;', { $id: id });
+    return;
+  }
+
+  if (!id.length) {
+    throw new Error('removeConversation: No ids to delete!');
+  }
+
+  // Our node interface doesn't seem to allow you to replace one single ? with an array
+  await db.run(
+    `DELETE FROM conversations WHERE id IN ( ${id
+      .map(() => '?')
+      .join(', ')} );`,
+    id
+  );
+}
+
+async function getConversationById(id) {
+  const row = await db.get('SELECT * FROM conversations WHERE id = $id;', {
+    $id: id,
+  });
+
+  if (!row) {
+    return null;
+  }
+
+  return jsonToObject(row.json);
+}
+
+async function getAllConversations() {
+  const rows = await db.all('SELECT json FROM conversations ORDER BY id ASC;');
+  return map(rows, row => jsonToObject(row.json));
+}
+
+async function getAllConversationIds() {
+  const rows = await db.all('SELECT id FROM conversations ORDER BY id ASC;');
+  return map(rows, row => row.id);
+}
+
+async function getAllPrivateConversations() {
+  const rows = await db.all(
+    `SELECT json FROM conversations WHERE
+      type = 'private'
+     ORDER BY id ASC;`
+  );
+
+  if (!rows) {
+    return null;
+  }
+
+  return map(rows, row => jsonToObject(row.json));
+}
+
+async function getAllGroupsInvolvingId(id) {
+  const rows = await db.all(
+    `SELECT json FROM conversations WHERE
+      type = 'group' AND
+      members LIKE $id
+     ORDER BY id ASC;`,
+    {
+      $id: `%${id}%`,
+    }
+  );
+
+  if (!rows) {
+    return null;
+  }
+
+  return map(rows, row => jsonToObject(row.json));
+}
+
+async function searchConversations(query) {
+  const rows = await db.all(
+    `SELECT json FROM conversations WHERE
+      id LIKE $id OR
+      name LIKE $name OR
+      profileName LIKE $profileName
+     ORDER BY id ASC;`,
+    {
+      $id: `%${query}%`,
+      $name: `%${query}%`,
+      $profileName: `%${query}%`,
+    }
+  );
+
+  if (!rows) {
+    return null;
+  }
+
+  return map(rows, row => jsonToObject(row.json));
+}
+
 async function getMessageCount() {
   const row = await db.get('SELECT count(*) from messages;');
 
@@ -522,6 +756,11 @@ async function getMessageById(id) {
   return jsonToObject(row.json);
 }
 
+async function getAllMessages() {
+  const rows = await db.all('SELECT json FROM messages ORDER BY id ASC;');
+  return map(rows, row => jsonToObject(row.json));
+}
+
 async function getAllMessageIds() {
   const rows = await db.all('SELECT id FROM messages ORDER BY id ASC;');
   return map(rows, row => row.id);
@@ -764,6 +1003,7 @@ async function removeAll() {
       db.run('BEGIN TRANSACTION;'),
       db.run('DELETE FROM messages;'),
       db.run('DELETE FROM unprocessed;'),
+      db.run('DELETE from conversations;'),
       db.run('COMMIT TRANSACTION;'),
     ]);
   });
@@ -874,6 +1114,21 @@ function getExternalFilesForMessage(message) {
   return files;
 }
 
+function getExternalFilesForConversation(conversation) {
+  const { avatar, profileAvatar } = conversation;
+  const files = [];
+
+  if (avatar && avatar.path) {
+    files.push(avatar.path);
+  }
+
+  if (profileAvatar && profileAvatar.path) {
+    files.push(profileAvatar.path);
+  }
+
+  return files;
+}
+
 async function removeKnownAttachments(allAttachments) {
   const lookup = fromPairs(map(allAttachments, file => [file, true]));
   const chunkSize = 50;
@@ -918,5 +1173,47 @@ async function removeKnownAttachments(allAttachments) {
 
   console.log(`removeKnownAttachments: Done processing ${count} messages`);
 
+  complete = false;
+  count = 0;
+  // Though conversations.id is a string, this ensures that, when coerced, this
+  //   value is still a string but it's smaller than every other string.
+  id = 0;
+
+  const conversationTotal = await getConversationCount();
+  console.log(
+    `removeKnownAttachments: About to iterate through ${conversationTotal} conversations`
+  );
+
+  while (!complete) {
+    // eslint-disable-next-line no-await-in-loop
+    const rows = await db.all(
+      `SELECT json FROM conversations
+       WHERE id > $id
+       ORDER BY id ASC
+       LIMIT $chunkSize;`,
+      {
+        $id: id,
+        $chunkSize: chunkSize,
+      }
+    );
+
+    const conversations = map(rows, row => jsonToObject(row.json));
+    forEach(conversations, conversation => {
+      const externalFiles = getExternalFilesForConversation(conversation);
+      forEach(externalFiles, file => {
+        delete lookup[file];
+      });
+    });
+
+    const lastMessage = last(conversations);
+    if (lastMessage) {
+      ({ id } = lastMessage);
+    }
+    complete = conversations.length < chunkSize;
+    count += conversations.length;
+  }
+
+  console.log(`removeKnownAttachments: Done processing ${count} conversations`);
+
   return Object.keys(lookup);
 }

js/background.js

@@ -1,13 +1,13 @@
 /* global Backbone: false */
 /* global $: false */
 
+/* global dcodeIO: false */
 /* global ConversationController: false */
 /* global getAccountManager: false */
 /* global Signal: false */
 /* global storage: false */
 /* global textsecure: false */
 /* global Whisper: false */
-/* global wrapDeferred: false */
 /* global _: false */
 
 // eslint-disable-next-line func-names
@@ -125,8 +125,16 @@
 
   const { IdleDetector, MessageDataMigrator } = Signal.Workflow;
   const { Errors, Message } = window.Signal.Types;
-  const { upgradeMessageSchema } = window.Signal.Migrations;
+  const {
-  const { Migrations0DatabaseWithAttachmentData } = window.Signal.Migrations;
+    upgradeMessageSchema,
+    writeNewAttachmentData,
+    deleteAttachmentData,
+    getCurrentVersion,
+  } = window.Signal.Migrations;
+  const {
+    Migrations0DatabaseWithAttachmentData,
+    Migrations1DatabaseWithoutAttachmentData,
+  } = window.Signal.Migrations;
   const { Views } = window.Signal;
 
   // Implicitly used in `indexeddb-backbonejs-adapter`:
@@ -183,6 +191,9 @@
     logger: window.log,
   });
 
+  const latestDBVersion2 = await getCurrentVersion();
+  Whisper.Database.migrations[0].version = latestDBVersion2;
+
   window.log.info('Storage fetch');
   storage.fetch();
 
@@ -337,9 +348,18 @@
     await upgradeMessages();
 
     const db = await Whisper.Database.open();
-    const totalMessages = await MessageDataMigrator.getNumMessages({
+    let totalMessages;
-      connection: db,
+    try {
-    });
+      totalMessages = await MessageDataMigrator.getNumMessages({
+        connection: db,
+      });
+    } catch (error) {
+      window.log.error(
+        'background.getNumMessages error:',
+        error && error.stack ? error.stack : error
+      );
+      totalMessages = 0;
+    }
 
     function showMigrationStatus(current) {
       const status = `${current}/${totalMessages}`;
@@ -350,23 +370,41 @@
 
     if (totalMessages) {
       window.log.info(`About to migrate ${totalMessages} messages`);
-
       showMigrationStatus(0);
-      await window.Signal.migrateToSQL({
+    } else {
-        db,
+      window.log.info('About to migrate non-messages');
-        clearStores: Whisper.Database.clearStores,
-        handleDOMException: Whisper.Database.handleDOMException,
-        arrayBufferToString:
-          textsecure.MessageReceiver.arrayBufferToStringBase64,
-        countCallback: count => {
-          window.log.info(`Migration: ${count} messages complete`);
-          showMigrationStatus(count);
-        },
-      });
     }
 
+    await window.Signal.migrateToSQL({
+      db,
+      clearStores: Whisper.Database.clearStores,
+      handleDOMException: Whisper.Database.handleDOMException,
+      arrayBufferToString: textsecure.MessageReceiver.arrayBufferToStringBase64,
+      countCallback: count => {
+        window.log.info(`Migration: ${count} messages complete`);
+        showMigrationStatus(count);
+      },
+      writeNewAttachmentData,
+    });
+
+    db.close();
+
     Views.Initialization.setMessage(window.i18n('optimizingApplication'));
 
+    window.log.info('Running cleanup IndexedDB migrations...');
+    await Whisper.Database.close();
+
+    // Now we clean up IndexedDB database after extracting data from it
+    await Migrations1DatabaseWithoutAttachmentData.run({
+      Backbone,
+      logger: window.log,
+    });
+
+    const latestDBVersion = _.last(
+      Migrations1DatabaseWithoutAttachmentData.migrations
+    ).version;
+    Whisper.Database.migrations[0].version = latestDBVersion;
+
     window.log.info('Cleanup: starting...');
     const messagesForCleanup = await window.Signal.Data.getOutgoingWithoutExpiresAt(
       {
@@ -844,7 +882,10 @@
       }
 
       if (details.profileKey) {
-        conversation.set({ profileKey: details.profileKey });
+        const profileKey = dcodeIO.ByteBuffer.wrap(details.profileKey).toString(
+          'base64'
+        );
+        conversation.set({ profileKey });
       }
 
       if (typeof details.blocked !== 'undefined') {
@@ -855,14 +896,29 @@
         }
       }
 
-      await wrapDeferred(
+      conversation.set({
-        conversation.save({
+        name: details.name,
-          name: details.name,
+        color: details.color,
-          avatar: details.avatar,
+        active_at: activeAt,
-          color: details.color,
+      });
-          active_at: activeAt,
+
-        })
+      // Update the conversation avatar only if new avatar exists and hash differs
-      );
+      const { avatar } = details;
+      if (avatar && avatar.data) {
+        const newAttributes = await window.Signal.Types.Conversation.maybeUpdateAvatar(
+          conversation.attributes,
+          avatar.data,
+          {
+            writeNewAttachmentData,
+            deleteAttachmentData,
+          }
+        );
+        conversation.set(newAttributes);
+      }
+
+      await window.Signal.Data.updateConversation(id, conversation.attributes, {
+        Conversation: Whisper.Conversation,
+      });
       const { expireTimer } = details;
       const isValidExpireTimer = typeof expireTimer === 'number';
       if (isValidExpireTimer) {
@@ -901,12 +957,13 @@
       id,
       'group'
     );
+
     const updates = {
       name: details.name,
       members: details.members,
-      avatar: details.avatar,
       type: 'group',
     };
+
     if (details.active) {
       const activeAt = conversation.get('active_at');
 
@@ -926,7 +983,25 @@
       storage.removeBlockedGroup(id);
     }
 
-    await wrapDeferred(conversation.save(updates));
+    conversation.set(updates);
+
+    // Update the conversation avatar only if new avatar exists and hash differs
+    const { avatar } = details;
+    if (avatar && avatar.data) {
+      const newAttributes = await window.Signal.Types.Conversation.maybeUpdateAvatar(
+        conversation.attributes,
+        avatar.data,
+        {
+          writeNewAttachmentData,
+          deleteAttachmentData,
+        }
+      );
+      conversation.set(newAttributes);
+    }
+
+    await window.Signal.Data.updateConversation(id, conversation.attributes, {
+      Conversation: Whisper.Conversation,
+    });
     const { expireTimer } = details;
     const isValidExpireTimer = typeof expireTimer === 'number';
     if (!isValidExpireTimer) {
@@ -1077,12 +1152,15 @@
     confirm,
     messageDescriptor,
   }) {
-    const profileKey = data.message.profileKey.toArrayBuffer();
+    const profileKey = data.message.profileKey.toString('base64');
     const sender = await ConversationController.getOrCreateAndWait(
       messageDescriptor.id,
       'private'
     );
+
+    // Will do the save for us
     await sender.setProfileKey(profileKey);
+
     return confirm();
   }
 
@@ -1097,11 +1175,17 @@
     confirm,
     messageDescriptor,
   }) {
+    const { id, type } = messageDescriptor;
     const conversation = await ConversationController.getOrCreateAndWait(
-      messageDescriptor.id,
+      id,
-      messageDescriptor.type
+      type
     );
-    await wrapDeferred(conversation.save({ profileSharing: true }));
+
+    conversation.set({ profileSharing: true });
+    await window.Signal.Data.updateConversation(id, conversation.attributes, {
+      Conversation: Whisper.Conversation,
+    });
+
     return confirm();
   }
 
@@ -1174,6 +1258,7 @@
       Whisper.Registration.remove();
 
       const NUMBER_ID_KEY = 'number_id';
+      const VERSION_KEY = 'version';
       const LAST_PROCESSED_INDEX_KEY = 'attachmentMigration_lastProcessedIndex';
       const IS_MIGRATION_COMPLETE_KEY = 'attachmentMigration_isComplete';
 
@@ -1203,6 +1288,7 @@
           LAST_PROCESSED_INDEX_KEY,
           lastProcessedIndex || null
         );
+        textsecure.storage.put(VERSION_KEY, window.getVersion());
 
         window.log.info('Successfully cleared local configuration');
       } catch (eraseError) {
@@ -1262,7 +1348,9 @@
         ev.confirm();
       }
 
-      await wrapDeferred(conversation.save());
+      await window.Signal.Data.updateConversation(id, conversation.attributes, {
+        Conversation: Whisper.Conversation,
+      });
     }
 
     throw error;

js/conversation_controller.js

@@ -1,4 +1,4 @@
-/* global _, Whisper, Backbone, storage, wrapDeferred */
+/* global _, Whisper, Backbone, storage */
 
 /* eslint-disable more/no-then */
 
@@ -131,8 +131,10 @@
       conversation = conversations.add({
         id,
         type,
+        version: 2,
       });
-      conversation.initialPromise = new Promise((resolve, reject) => {
+
+      const create = async () => {
         if (!conversation.isValid()) {
           const validationError = conversation.validationError || {};
           window.log.error(
@@ -141,19 +143,28 @@
             validationError.stack
           );
 
-          return resolve(conversation);
+          return conversation;
         }
 
-        const deferred = conversation.save();
+        try {
-        if (!deferred) {
+          await window.Signal.Data.saveConversation(conversation.attributes, {
-          window.log.error('Conversation save failed! ', id, type);
+            Conversation: Whisper.Conversation,
-          return reject(new Error('getOrCreate: Conversation save failed'));
+          });
+        } catch (error) {
+          window.log.error(
+            'Conversation save failed! ',
+            id,
+            type,
+            'Error:',
+            error && error.stack ? error.stack : error
+          );
+          throw error;
         }
 
-        return deferred.then(() => {
+        return conversation;
-          resolve(conversation);
+      };
-        }, reject);
+
-      });
+      conversation.initialPromise = create();
 
       return conversation;
     },
@@ -170,11 +181,11 @@
         );
       });
     },
-    getAllGroupsInvolvingId(id) {
+    async getAllGroupsInvolvingId(id) {
-      const groups = new Whisper.GroupCollection();
+      const groups = await window.Signal.Data.getAllGroupsInvolvingId(id, {
-      return groups
+        ConversationCollection: Whisper.ConversationCollection,
-        .fetchGroups(id)
+      });
-        .then(() => groups.map(group => conversations.add(group)));
+      return groups.map(group => conversations.add(group));
     },
     loadPromise() {
       return this._initialPromise;
@@ -193,7 +204,12 @@
 
       const load = async () => {
         try {
-          await wrapDeferred(conversations.fetch());
+          const collection = await window.Signal.Data.getAllConversations({
+            ConversationCollection: Whisper.ConversationCollection,
+          });
+
+          conversations.add(collection.models);
+
           this._initialFetchComplete = true;
           await Promise.all(
             conversations.map(conversation => conversation.updateLastMessage())

js/database.js

@@ -97,12 +97,14 @@
 
   Whisper.Database.clear = async () => {
     const db = await Whisper.Database.open();
-    return clearStores(db);
+    await clearStores(db);
+    db.close();
   };
 
   Whisper.Database.clearStores = async storeNames => {
     const db = await Whisper.Database.open();
-    return clearStores(db, storeNames);
+    await clearStores(db, storeNames);
+    db.close();
   };
 
   Whisper.Database.close = () => window.wrapDeferred(Backbone.sync('closeall'));

js/delivery_receipts.js

@@ -38,8 +38,9 @@
         return message;
       }
 
-      const groups = new Whisper.GroupCollection();
+      const groups = await window.Signal.Data.getAllGroupsInvolvingId(source, {
-      await groups.fetchGroups(source);
+        ConversationCollection: Whisper.ConversationCollection,
+      });
 
       const ids = groups.pluck('id');
       ids.push(source);

js/keychange_listener.js

@@ -14,18 +14,17 @@
         throw new Error('KeyChangeListener requires a SignalProtocolStore');
       }
 
-      signalProtocolStore.on('keychange', id => {
+      signalProtocolStore.on('keychange', async id => {
-        ConversationController.getOrCreateAndWait(id, 'private').then(
+        const conversation = await ConversationController.getOrCreateAndWait(
-          conversation => {
+          id,
-            conversation.addKeyChange(id);
+          'private'
-
-            ConversationController.getAllGroupsInvolvingId(id).then(groups => {
-              _.forEach(groups, group => {
-                group.addKeyChange(id);
-              });
-            });
-          }
         );
+        conversation.addKeyChange(id);
+
+        const groups = await ConversationController.getAllGroupsInvolvingId(id);
+        _.forEach(groups, group => {
+          group.addKeyChange(id);
+        });
       });
     },
   };

js/models/conversations.js

@@ -8,7 +8,6 @@
 /* global storage: false */
 /* global textsecure: false */
 /* global Whisper: false */
-/* global wrapDeferred: false */
 
 /* eslint-disable more/no-then */
 
@@ -30,6 +29,8 @@
     upgradeMessageSchema,
     loadAttachmentData,
     getAbsoluteAttachmentPath,
+    writeNewAttachmentData,
+    deleteAttachmentData,
   } = window.Signal.Migrations;
 
   // TODO: Factor out private and group subclasses of Conversation
@@ -52,23 +53,6 @@
     'blue_grey',
   ];
 
-  function constantTimeEqualArrayBuffers(ab1, ab2) {
-    if (!(ab1 instanceof ArrayBuffer && ab2 instanceof ArrayBuffer)) {
-      return false;
-    }
-    if (ab1.byteLength !== ab2.byteLength) {
-      return false;
-    }
-    let result = 0;
-    const ta1 = new Uint8Array(ab1);
-    const ta2 = new Uint8Array(ab2);
-    for (let i = 0; i < ab1.byteLength; i += 1) {
-      // eslint-disable-next-line no-bitwise
-      result |= ta1[i] ^ ta2[i];
-    }
-    return result === 0;
-  }
-
   Whisper.Conversation = Backbone.Model.extend({
     database: Whisper.Database,
     storeName: 'conversations',
@@ -130,10 +114,7 @@
       );
 
       this.on('newmessage', this.updateLastMessage);
-      this.on('change:avatar', this.updateAvatarUrl);
-      this.on('change:profileAvatar', this.updateAvatarUrl);
       this.on('change:profileKey', this.onChangeProfileKey);
-      this.on('destroy', this.revokeAvatarUrl);
 
       // Listening for out-of-band data updates
       this.on('delivered', this.updateAndMerge);
@@ -240,30 +221,31 @@
         () => textsecure.storage.protocol.VerifiedStatus.DEFAULT
       );
     },
-    updateVerified() {
+    async updateVerified() {
       if (this.isPrivate()) {
-        return Promise.all([this.safeGetVerified(), this.initialPromise]).then(
+        await this.initialPromise;
-          results => {
+        const verified = await this.safeGetVerified();
-            const trust = results[0];
-            // we don't return here because we don't need to wait for this to finish
-            this.save({ verified: trust });
-          }
-        );
-      }
-      const promise = this.fetchContacts();
 
-      return promise
+        // we don't await here because we don't need to wait for this to finish
-        .then(() =>
+        window.Signal.Data.updateConversation(
-          Promise.all(
+          this.id,
-            this.contactCollection.map(contact => {
+          { verified },
-              if (!contact.isMe()) {
+          { Conversation: Whisper.Conversation }
-                return contact.updateVerified();
+        );
-              }
+
-              return Promise.resolve();
+        return;
-            })
+      }
-          )
+
-        )
+      await this.fetchContacts();
-        .then(this.onMemberVerifiedChange.bind(this));
+      await Promise.all(
+        this.contactCollection.map(async contact => {
+          if (!contact.isMe()) {
+            await contact.updateVerified();
+          }
+        })
+      );
+
+      this.onMemberVerifiedChange();
     },
     setVerifiedDefault(options) {
       const { DEFAULT } = this.verifiedEnum;
@@ -277,7 +259,7 @@
       const { UNVERIFIED } = this.verifiedEnum;
       return this.queueJob(() => this._setVerified(UNVERIFIED, options));
     },
-    _setVerified(verified, providedOptions) {
+    async _setVerified(verified, providedOptions) {
       const options = providedOptions || {};
       _.defaults(options, {
         viaSyncMessage: false,
@@ -295,50 +277,47 @@
       }
 
       const beginningVerified = this.get('verified');
-      let promise;
+      let keyChange;
       if (options.viaSyncMessage) {
         // handle the incoming key from the sync messages - need different
         // behavior if that key doesn't match the current key
-        promise = textsecure.storage.protocol.processVerifiedMessage(
+        keyChange = await textsecure.storage.protocol.processVerifiedMessage(
           this.id,
           verified,
           options.key
         );
       } else {
-        promise = textsecure.storage.protocol.setVerified(this.id, verified);
+        keyChange = await textsecure.storage.protocol.setVerified(
+          this.id,
+          verified
+        );
       }
 
-      let keychange;
+      this.set({ verified });
-      return promise
+      await window.Signal.Data.updateConversation(this.id, this.attributes, {
-        .then(updatedKey => {
+        Conversation: Whisper.Conversation,
-          keychange = updatedKey;
+      });
-          return new Promise(resolve =>
+
-            this.save({ verified }).always(resolve)
+      // Three situations result in a verification notice in the conversation:
-          );
+      //   1) The message came from an explicit verification in another client (not
-        })
+      //      a contact sync)
-        .then(() => {
+      //   2) The verification value received by the contact sync is different
-          // Three situations result in a verification notice in the conversation:
+      //      from what we have on record (and it's not a transition to UNVERIFIED)
-          //   1) The message came from an explicit verification in another client (not
+      //   3) Our local verification status is VERIFIED and it hasn't changed,
-          //      a contact sync)
+      //      but the key did change (Key1/VERIFIED to Key2/VERIFIED - but we don't
-          //   2) The verification value received by the contact sync is different
+      //      want to show DEFAULT->DEFAULT or UNVERIFIED->UNVERIFIED)
-          //      from what we have on record (and it's not a transition to UNVERIFIED)
+      if (
-          //   3) Our local verification status is VERIFIED and it hasn't changed,
+        !options.viaContactSync ||
-          //      but the key did change (Key1/VERIFIED to Key2/VERIFIED - but we don't
+        (beginningVerified !== verified && verified !== UNVERIFIED) ||
-          //      want to show DEFAULT->DEFAULT or UNVERIFIED->UNVERIFIED)
+        (keyChange && verified === VERIFIED)
-          if (
+      ) {
-            !options.viaContactSync ||
+        await this.addVerifiedChange(this.id, verified === VERIFIED, {
-            (beginningVerified !== verified && verified !== UNVERIFIED) ||
+          local: !options.viaSyncMessage,
-            (keychange && verified === VERIFIED)
-          ) {
-            this.addVerifiedChange(this.id, verified === VERIFIED, {
-              local: !options.viaSyncMessage,
-            });
-          }
-          if (!options.viaSyncMessage) {
-            return this.sendVerifySyncMessage(this.id, verified);
-          }
-          return Promise.resolve();
         });
+      }
+      if (!options.viaSyncMessage) {
+        await this.sendVerifySyncMessage(this.id, verified);
+      }
     },
     sendVerifySyncMessage(number, state) {
       const promise = textsecure.storage.protocol.loadIdentityKey(number);
@@ -346,42 +325,6 @@
         textsecure.messaging.syncVerification(number, state, key)
       );
     },
-    getIdentityKeys() {
-      const lookup = {};
-
-      if (this.isPrivate()) {
-        return textsecure.storage.protocol
-          .loadIdentityKey(this.id)
-          .then(key => {
-            lookup[this.id] = key;
-            return lookup;
-          })
-          .catch(error => {
-            window.log.error(
-              'getIdentityKeys error for conversation',
-              this.idForLogging(),
-              error && error.stack ? error.stack : error
-            );
-            return lookup;
-          });
-      }
-      const promises = this.contactCollection.map(contact =>
-        textsecure.storage.protocol.loadIdentityKey(contact.id).then(
-          key => {
-            lookup[contact.id] = key;
-          },
-          error => {
-            window.log.error(
-              'getIdentityKeys error for group member',
-              contact.idForLogging(),
-              error && error.stack ? error.stack : error
-            );
-          }
-        )
-      );
-
-      return Promise.all(promises).then(() => lookup);
-    },
     isVerified() {
       if (this.isPrivate()) {
         return this.get('verified') === this.verifiedEnum.VERIFIED;
@@ -583,9 +526,9 @@
       );
 
       if (this.isPrivate()) {
-        ConversationController.getAllGroupsInvolvingId(id).then(groups => {
+        ConversationController.getAllGroupsInvolvingId(this.id).then(groups => {
           _.forEach(groups, group => {
-            group.addVerifiedChange(id, verified, options);
+            group.addVerifiedChange(this.id, verified, options);
           });
         });
       }
@@ -641,8 +584,6 @@
         return error;
       }
 
-      this.updateTokens();
-
       return null;
     },
 
@@ -661,29 +602,6 @@
       return null;
     },
 
-    updateTokens() {
-      let tokens = [];
-      const name = this.get('name');
-      if (typeof name === 'string') {
-        tokens.push(name.toLowerCase());
-        tokens = tokens.concat(
-          name
-            .trim()
-            .toLowerCase()
-            .split(/[\s\-_()+]+/)
-        );
-      }
-      if (this.isPrivate()) {
-        const regionCode = storage.get('regionCode');
-        const number = libphonenumber.util.parseNumber(this.id, regionCode);
-        tokens.push(
-          number.nationalNumber,
-          number.countryCode + number.nationalNumber
-        );
-      }
-      this.set({ tokens });
-    },
-
     queueJob(callback) {
       const previous = this.pending || Promise.resolve();
 
@@ -785,10 +703,13 @@
         this.lastMessage = message.getNotificationText();
         this.lastMessageStatus = 'sending';
 
-        this.save({
+        this.set({
           active_at: now,
           timestamp: now,
         });
+        await window.Signal.Data.updateConversation(this.id, this.attributes, {
+          Conversation: Whisper.Conversation,
+        });
 
         if (this.isPrivate()) {
           message.set({ destination });
@@ -808,7 +729,7 @@
             return error;
           });
           await message.saveErrors(errors);
-          return;
+          return null;
         }
 
         const conversationType = this.get('type');
@@ -828,7 +749,8 @@
         const attachmentsWithData = await Promise.all(
           messageWithSchema.attachments.map(loadAttachmentData)
         );
-        message.send(
+
+        return message.send(
           sendFunction(
             destination,
             body,
@@ -880,10 +802,15 @@
       hasChanged = hasChanged || lastMessageStatus !== this.lastMessageStatus;
       this.lastMessageStatus = lastMessageStatus;
 
+      // Because we're no longer using Backbone-integrated saves, we need to manually
+      //   clear the changed fields here so our hasChanged() check below is useful.
+      this.changed = {};
       this.set(lastMessageUpdate);
 
       if (this.hasChanged()) {
-        this.save();
+        await window.Signal.Data.updateConversation(this.id, this.attributes, {
+          Conversation: Whisper.Conversation,
+        });
       } else if (hasChanged) {
         this.trigger('change');
       }
@@ -907,7 +834,7 @@
         this.get('expireTimer') === expireTimer ||
         (!expireTimer && !this.get('expireTimer'))
       ) {
-        return Promise.resolve();
+        return null;
       }
 
       window.log.info("Update conversation 'expireTimer'", {
@@ -922,7 +849,10 @@
       //   to be above the message that initiated that change, hence the subtraction.
       const timestamp = (receivedAt || Date.now()) - 1;
 
-      await wrapDeferred(this.save({ expireTimer }));
+      this.set({ expireTimer });
+      await window.Signal.Data.updateConversation(this.id, this.attributes, {
+        Conversation: Whisper.Conversation,
+      });
 
       const message = this.messageCollection.add({
         // Even though this isn't reflected to the user, we want to place the last seen
@@ -1041,7 +971,11 @@
     async leaveGroup() {
       const now = Date.now();
       if (this.get('type') === 'group') {
-        this.save({ left: true });
+        this.set({ left: true });
+        await window.Signal.Data.updateConversation(this.id, this.attributes, {
+          Conversation: Whisper.Conversation,
+        });
+
         const message = this.messageCollection.add({
           group_update: { left: 'You' },
           conversationId: this.id,
@@ -1059,7 +993,7 @@
       }
     },
 
-    markRead(newestUnreadDate, providedOptions) {
+    async markRead(newestUnreadDate, providedOptions) {
       const options = providedOptions || {};
       _.defaults(options, { sendReadReceipts: true });
 
@@ -1070,15 +1004,13 @@
         })
       );
 
-      return this.getUnread().then(providedUnreadMessages => {
+      let unreadMessages = await this.getUnread();
-        let unreadMessages = providedUnreadMessages;
+      const oldUnread = unreadMessages.filter(
+        message => message.get('received_at') <= newestUnreadDate
+      );
 
-        const promises = [];
+      let read = await Promise.all(
-        const oldUnread = unreadMessages.filter(
+        _.map(oldUnread, async providedM => {
-          message => message.get('received_at') <= newestUnreadDate
-        );
-
-        let read = _.map(oldUnread, providedM => {
           let m = providedM;
 
           if (this.messageCollection.get(m.id)) {
@@ -1089,48 +1021,47 @@
                 'it was not in messageCollection.'
             );
           }
-          promises.push(m.markRead(options.readAt));
+
+          await m.markRead(options.readAt);
           const errors = m.get('errors');
           return {
             sender: m.get('source'),
             timestamp: m.get('sent_at'),
             hasErrors: Boolean(errors && errors.length),
           };
-        });
+        })
+      );
 
-        // Some messages we're marking read are local notifications with no sender
+      // Some messages we're marking read are local notifications with no sender
-        read = _.filter(read, m => Boolean(m.sender));
+      read = _.filter(read, m => Boolean(m.sender));
-        unreadMessages = unreadMessages.filter(m => Boolean(m.isIncoming()));
+      unreadMessages = unreadMessages.filter(m => Boolean(m.isIncoming()));
 
-        const unreadCount = unreadMessages.length - read.length;
+      const unreadCount = unreadMessages.length - read.length;
-        const promise = new Promise((resolve, reject) => {
+      this.set({ unreadCount });
-          this.save({ unreadCount }).then(resolve, reject);
+      await window.Signal.Data.updateConversation(this.id, this.attributes, {
-        });
+        Conversation: Whisper.Conversation,
-        promises.push(promise);
-
-        // If a message has errors, we don't want to send anything out about it.
-        //   read syncs - let's wait for a client that really understands the message
-        //      to mark it read. we'll mark our local error read locally, though.
-        //   read receipts - here we can run into infinite loops, where each time the
-        //      conversation is viewed, another error message shows up for the contact
-        read = read.filter(item => !item.hasErrors);
-
-        if (read.length && options.sendReadReceipts) {
-          window.log.info('Sending', read.length, 'read receipts');
-          promises.push(textsecure.messaging.syncReadMessages(read));
-
-          if (storage.get('read-receipt-setting')) {
-            _.each(_.groupBy(read, 'sender'), (receipts, sender) => {
-              const timestamps = _.map(receipts, 'timestamp');
-              promises.push(
-                textsecure.messaging.sendReadReceipts(sender, timestamps)
-              );
-            });
-          }
-        }
-
-        return Promise.all(promises);
       });
+
+      // If a message has errors, we don't want to send anything out about it.
+      //   read syncs - let's wait for a client that really understands the message
+      //      to mark it read. we'll mark our local error read locally, though.
+      //   read receipts - here we can run into infinite loops, where each time the
+      //      conversation is viewed, another error message shows up for the contact
+      read = read.filter(item => !item.hasErrors);
+
+      if (read.length && options.sendReadReceipts) {
+        window.log.info('Sending', read.length, 'read receipts');
+        await textsecure.messaging.syncReadMessages(read);
+
+        if (storage.get('read-receipt-setting')) {
+          await Promise.all(
+            _.map(_.groupBy(read, 'sender'), async (receipts, sender) => {
+              const timestamps = _.map(receipts, 'timestamp');
+              await textsecure.messaging.sendReadReceipts(sender, timestamps);
+            })
+          );
+        }
+      }
     },
 
     onChangeProfileKey() {
@@ -1150,128 +1081,132 @@
       return Promise.all(_.map(ids, this.getProfile));
     },
 
-    getProfile(id) {
+    async getProfile(id) {
       if (!textsecure.messaging) {
-        const message =
+        throw new Error(
-          'Conversation.getProfile: textsecure.messaging not available';
+          'Conversation.getProfile: textsecure.messaging not available'
-        return Promise.reject(new Error(message));
+        );
-      }
-
-      return textsecure.messaging
-        .getProfile(id)
-        .then(profile => {
-          const identityKey = dcodeIO.ByteBuffer.wrap(
-            profile.identityKey,
-            'base64'
-          ).toArrayBuffer();
-
-          return textsecure.storage.protocol
-            .saveIdentity(`${id}.1`, identityKey, false)
-            .then(changed => {
-              if (changed) {
-                // save identity will close all sessions except for .1, so we
-                // must close that one manually.
-                const address = new libsignal.SignalProtocolAddress(id, 1);
-                window.log.info('closing session for', address.toString());
-                const sessionCipher = new libsignal.SessionCipher(
-                  textsecure.storage.protocol,
-                  address
-                );
-                return sessionCipher.closeOpenSessionForDevice();
-              }
-              return Promise.resolve();
-            })
-            .then(() => {
-              const c = ConversationController.get(id);
-              return Promise.all([
-                c.setProfileName(profile.name),
-                c.setProfileAvatar(profile.avatar),
-              ]).then(
-                // success
-                () =>
-                  new Promise((resolve, reject) => {
-                    c.save().then(resolve, reject);
-                  }),
-                // fail
-                e => {
-                  if (e.name === 'ProfileDecryptError') {
-                    // probably the profile key has changed.
-                    window.log.error(
-                      'decryptProfile error:',
-                      id,
-                      profile,
-                      e && e.stack ? e.stack : e
-                    );
-                  }
-                }
-              );
-            });
-        })
-        .catch(error => {
-          window.log.error(
-            'getProfile error:',
-            error && error.stack ? error.stack : error
-          );
-        });
-    },
-    setProfileName(encryptedName) {
-      const key = this.get('profileKey');
-      if (!key) {
-        return Promise.resolve();
       }
 
       try {
-        // decode
+        const profile = await textsecure.messaging.getProfile(id);
-        const data = dcodeIO.ByteBuffer.wrap(
+        const identityKey = dcodeIO.ByteBuffer.wrap(
-          encryptedName,
+          profile.identityKey,
           'base64'
         ).toArrayBuffer();
 
-        // decrypt
+        const changed = await textsecure.storage.protocol.saveIdentity(
-        return textsecure.crypto
+          `${id}.1`,
-          .decryptProfileName(data, key)
+          identityKey,
-          .then(decrypted => {
+          false
-            // encode
+        );
-            const name = dcodeIO.ByteBuffer.wrap(decrypted).toString('utf8');
+        if (changed) {
+          // save identity will close all sessions except for .1, so we
+          // must close that one manually.
+          const address = new libsignal.SignalProtocolAddress(id, 1);
+          window.log.info('closing session for', address.toString());
+          const sessionCipher = new libsignal.SessionCipher(
+            textsecure.storage.protocol,
+            address
+          );
+          await sessionCipher.closeOpenSessionForDevice();
+        }
 
-            // set
+        try {
-            this.set({ profileName: name });
+          const c = ConversationController.get(id);
-          });
+
-      } catch (e) {
+          // Because we're no longer using Backbone-integrated saves, we need to manually
-        return Promise.reject(e);
+          //   clear the changed fields here so our hasChanged() check is useful.
+          c.changed = {};
+          await c.setProfileName(profile.name);
+          await c.setProfileAvatar(profile.avatar);
+
+          if (c.hasChanged()) {
+            await window.Signal.Data.updateConversation(id, c.attributes, {
+              Conversation: Whisper.Conversation,
+            });
+          }
+        } catch (e) {
+          if (e.name === 'ProfileDecryptError') {
+            // probably the profile key has changed.
+            window.log.error(
+              'decryptProfile error:',
+              id,
+              e && e.stack ? e.stack : e
+            );
+          }
+        }
+      } catch (error) {
+        window.log.error(
+          'getProfile error:',
+          error && error.stack ? error.stack : error
+        );
       }
     },
-    setProfileAvatar(avatarPath) {
+    async setProfileName(encryptedName) {
+      const key = this.get('profileKey');
+      if (!key) {
+        return;
+      }
+
+      // decode
+      const keyBuffer = dcodeIO.ByteBuffer.wrap(key, 'base64').toArrayBuffer();
+      const data = dcodeIO.ByteBuffer.wrap(
+        encryptedName,
+        'base64'
+      ).toArrayBuffer();
+
+      // decrypt
+      const decrypted = await textsecure.crypto.decryptProfileName(
+        data,
+        keyBuffer
+      );
+
+      // encode
+      const name = dcodeIO.ByteBuffer.wrap(decrypted).toString('utf8');
+
+      // set
+      this.set({ profileName: name });
+    },
+    async setProfileAvatar(avatarPath) {
       if (!avatarPath) {
-        return Promise.resolve();
+        return;
       }
 
-      return textsecure.messaging.getAvatar(avatarPath).then(avatar => {
+      const avatar = await textsecure.messaging.getAvatar(avatarPath);
-        const key = this.get('profileKey');
+      const key = this.get('profileKey');
-        if (!key) {
+      if (!key) {
-          return Promise.resolve();
+        return;
-        }
+      }
-        // decrypt
+      const keyBuffer = dcodeIO.ByteBuffer.wrap(key, 'base64').toArrayBuffer();
-        return textsecure.crypto.decryptProfile(avatar, key).then(decrypted => {
+
-          // set
+      // decrypt
-          this.set({
+      const decrypted = await textsecure.crypto.decryptProfile(
-            profileAvatar: {
+        avatar,
-              data: decrypted,
+        keyBuffer
-              contentType: 'image/jpeg',
+      );
-              size: decrypted.byteLength,
+
-            },
+      // update the conversation avatar only if hash differs
-          });
+      if (decrypted) {
-        });
+        const newAttributes = await window.Signal.Types.Conversation.maybeUpdateProfileAvatar(
-      });
+          this.attributes,
+          decrypted,
+          {
+            writeNewAttachmentData,
+            deleteAttachmentData,
+          }
+        );
+        this.set(newAttributes);
+      }
     },
-    setProfileKey(key) {
+    async setProfileKey(profileKey) {
-      return new Promise((resolve, reject) => {
+      // profileKey is now being saved as a string
-        if (!constantTimeEqualArrayBuffers(this.get('profileKey'), key)) {
+      if (this.get('profileKey') !== profileKey) {
-          this.save({ profileKey: key }).then(resolve, reject);
+        this.set({ profileKey });
-        } else {
+        await window.Signal.Data.updateConversation(this.id, this.attributes, {
-          resolve();
+          Conversation: Whisper.Conversation,
-        }
+        });
-      });
+      }
     },
 
     async upgradeMessages(messages) {
@@ -1358,11 +1293,14 @@
 
       this.messageCollection.reset([]);
 
-      this.save({
+      this.set({
         lastMessage: null,
         timestamp: null,
         active_at: null,
       });
+      await window.Signal.Data.updateConversation(this.id, this.attributes, {
+        Conversation: Whisper.Conversation,
+      });
     },
 
     getName() {
@@ -1431,41 +1369,17 @@
       return this.get('type') === 'private';
     },
 
-    revokeAvatarUrl() {
-      if (this.avatarUrl) {
-        URL.revokeObjectURL(this.avatarUrl);
-        this.avatarUrl = null;
-      }
-    },
-
-    updateAvatarUrl(silent) {
-      this.revokeAvatarUrl();
-      const avatar = this.get('avatar') || this.get('profileAvatar');
-      if (avatar) {
-        this.avatarUrl = URL.createObjectURL(
-          new Blob([avatar.data], { type: avatar.contentType })
-        );
-      } else {
-        this.avatarUrl = null;
-      }
-      if (!silent) {
-        this.trigger('change');
-      }
-    },
     getColor() {
       const { migrateColor } = Util;
       return migrateColor(this.get('color'));
     },
     getAvatar() {
-      if (this.avatarUrl === undefined) {
-        this.updateAvatarUrl(true);
-      }
-
       const title = this.get('name');
       const color = this.getColor();
+      const avatar = this.get('avatar') || this.get('profileAvatar');
 
-      if (this.avatarUrl) {
+      if (avatar && avatar.path) {
-        return { url: this.avatarUrl, color };
+        return { url: getAbsoluteAttachmentPath(avatar.path), color };
       } else if (this.isPrivate()) {
         return {
           color,
@@ -1519,24 +1433,6 @@
         })
       );
     },
-    hashCode() {
-      if (this.hash === undefined) {
-        const string = this.getTitle() || '';
-        if (string.length === 0) {
-          return 0;
-        }
-        let hash = 0;
-        for (let i = 0; i < string.length; i += 1) {
-          // eslint-disable-next-line no-bitwise
-          hash = (hash << 5) - hash + string.charCodeAt(i);
-          // eslint-disable-next-line no-bitwise
-          hash &= hash; // Convert to 32bit integer
-        }
-
-        this.hash = hash;
-      }
-      return this.hash;
-    },
   });
 
   Whisper.ConversationCollection = Backbone.Collection.extend({
@@ -1548,72 +1444,32 @@
       return -m.get('timestamp');
     },
 
-    destroyAll() {
+    async destroyAll() {
-      return Promise.all(
+      await Promise.all(
-        this.models.map(conversation => wrapDeferred(conversation.destroy()))
+        this.models.map(conversation =>
+          window.Signal.Data.removeConversation(conversation.id, {
+            Conversation: Whisper.Conversation,
+          })
+        )
       );
+      this.reset([]);
     },
 
-    search(providedQuery) {
+    async search(providedQuery) {
       let query = providedQuery.trim().toLowerCase();
-      if (query.length > 0) {
+      query = query.replace(/[+-.()]*/g, '');
-        query = query.replace(/[-.()]*/g, '').replace(/^\+(\d*)$/, '$1');
+
-        const lastCharCode = query.charCodeAt(query.length - 1);
+      if (query.length === 0) {
-        const nextChar = String.fromCharCode(lastCharCode + 1);
+        return;
-        const upper = query.slice(0, -1) + nextChar;
-        return new Promise(resolve => {
-          this.fetch({
-            index: {
-              name: 'search', // 'search' index on tokens array
-              lower: query,
-              upper,
-              excludeUpper: true,
-            },
-          }).always(resolve);
-        });
       }
-      return Promise.resolve();
-    },
 
-    fetchAlphabetical() {
+      const collection = await window.Signal.Data.searchConversations(query, {
-      return new Promise(resolve => {
+        ConversationCollection: Whisper.ConversationCollection,
-        this.fetch({
-          index: {
-            name: 'search', // 'search' index on tokens array
-          },
-          limit: 100,
-        }).always(resolve);
       });
-    },
 
-    fetchGroups(number) {
+      this.reset(collection.models);
-      return new Promise(resolve => {
-        this.fetch({
-          index: {
-            name: 'group',
-            only: number,
-          },
-        }).always(resolve);
-      });
     },
   });
 
   Whisper.Conversation.COLORS = COLORS.concat(['grey', 'default']).join(' ');
-
-  // Special collection for fetching all the groups a certain number appears in
-  Whisper.GroupCollection = Backbone.Collection.extend({
-    database: Whisper.Database,
-    storeName: 'conversations',
-    model: Whisper.Conversation,
-    fetchGroups(number) {
-      return new Promise(resolve => {
-        this.fetch({
-          index: {
-            name: 'group',
-            only: number,
-          },
-        }).always(resolve);
-      });
-    },
-  });
 })();

js/models/messages.js

@@ -8,7 +8,6 @@
 /* global Signal: false */
 /* global textsecure: false */
 /* global Whisper: false */
-/* global wrapDeferred: false */
 
 /* eslint-disable more/no-then */
 
@@ -1212,7 +1211,7 @@
           }
 
           if (dataMessage.profileKey) {
-            const profileKey = dataMessage.profileKey.toArrayBuffer();
+            const profileKey = dataMessage.profileKey.toString('base64');
             if (source === textsecure.storage.user.getNumber()) {
               conversation.set({ profileSharing: true });
             } else if (conversation.isPrivate()) {
@@ -1231,15 +1230,18 @@
           });
           message.set({ id });
 
-          await wrapDeferred(conversation.save());
+          await window.Signal.Data.updateConversation(
+            conversationId,
+            conversation.attributes,
+            { Conversation: Whisper.Conversation }
+          );
 
           conversation.trigger('newmessage', message);
 
           try {
-            // We fetch() here because, between the message.save() above and
+            // We go to the database here because, between the message save above and
-            // the previous line's trigger() call, we might have marked all
+            // the previous line's trigger() call, we might have marked all messages
-            // messages unread in the database. This message might already
+            // unread in the database. This message might already be read!
-            // be read!
             const fetched = await window.Signal.Data.getMessageById(
               message.get('id'),
               {

js/modules/backup.js

@@ -224,7 +224,49 @@ function eliminateClientConfigInBackup(data, targetPath) {
   }
 }
 
-function importFromJsonString(db, jsonString, targetPath, options) {
+async function importConversationsFromJSON(conversations, options) {
+  const { writeNewAttachmentData } = window.Signal.Migrations;
+  const { conversationLookup } = options;
+
+  let count = 0;
+  let skipCount = 0;
+
+  for (let i = 0, max = conversations.length; i < max; i += 1) {
+    const toAdd = unstringify(conversations[i]);
+    const haveConversationAlready =
+      conversationLookup[getConversationKey(toAdd)];
+
+    if (haveConversationAlready) {
+      skipCount += 1;
+      count += 1;
+      // eslint-disable-next-line no-continue
+      continue;
+    }
+
+    count += 1;
+    // eslint-disable-next-line no-await-in-loop
+    const migrated = await window.Signal.Types.Conversation.migrateConversation(
+      toAdd,
+      {
+        writeNewAttachmentData,
+      }
+    );
+    // eslint-disable-next-line no-await-in-loop
+    await window.Signal.Data.saveConversation(migrated, {
+      Conversation: Whisper.Conversation,
+    });
+  }
+
+  window.log.info(
+    'Done importing conversations:',
+    'Total count:',
+    count,
+    'Skipped:',
+    skipCount
+  );
+}
+
+async function importFromJsonString(db, jsonString, targetPath, options) {
   options = options || {};
   _.defaults(options, {
     forceLightImport: false,
@@ -232,12 +274,12 @@ function importFromJsonString(db, jsonString, targetPath, options) {
     groupLookup: {},
   });
 
-  const { conversationLookup, groupLookup } = options;
+  const { groupLookup } = options;
   const result = {
     fullImport: true,
   };
 
-  return new Promise((resolve, reject) => {
+  return new Promise(async (resolve, reject) => {
     const importObject = JSON.parse(jsonString);
     delete importObject.debug;
 
@@ -273,7 +315,25 @@ function importFromJsonString(db, jsonString, targetPath, options) {
       finished = true;
     };
 
-    const transaction = db.transaction(storeNames, 'readwrite');
+    // Special-case conversations key here, going to SQLCipher
+    const { conversations } = importObject;
+    const remainingStoreNames = _.without(
+      storeNames,
+      'conversations',
+      'unprocessed'
+    );
+    try {
+      await importConversationsFromJSON(conversations, options);
+    } catch (error) {
+      reject(error);
+    }
+
+    // Because the 'are we done?' check below looks at the keys remaining in importObject
+    delete importObject.conversations;
+    delete importObject.unprocessed;
+
+    // The rest go to IndexedDB
+    const transaction = db.transaction(remainingStoreNames, 'readwrite');
     transaction.onerror = () => {
       Whisper.Database.handleDOMException(
         'importFromJsonString transaction error',
@@ -283,7 +343,7 @@ function importFromJsonString(db, jsonString, targetPath, options) {
     };
     transaction.oncomplete = finish.bind(null, 'transaction complete');
 
-    _.each(storeNames, storeName => {
+    _.each(remainingStoreNames, storeName => {
       window.log.info('Importing items for store', storeName);
 
       if (!importObject[storeName].length) {
@@ -315,13 +375,10 @@ function importFromJsonString(db, jsonString, targetPath, options) {
       _.each(importObject[storeName], toAdd => {
         toAdd = unstringify(toAdd);
 
-        const haveConversationAlready =
-          storeName === 'conversations' &&
-          conversationLookup[getConversationKey(toAdd)];
         const haveGroupAlready =
           storeName === 'groups' && groupLookup[getGroupKey(toAdd)];
 
-        if (haveConversationAlready || haveGroupAlready) {
+        if (haveGroupAlready) {
           skipCount += 1;
           count += 1;
           return;
@@ -1137,20 +1194,17 @@ function getMessageKey(message) {
   const sourceDevice = message.sourceDevice || 1;
   return `${source}.${sourceDevice} ${message.timestamp}`;
 }
-async function loadMessagesLookup(db) {
+async function loadMessagesLookup() {
-  const array = await window.Signal.Data.getAllMessageIds({
+  const array = await window.Signal.Data.getAllMessageIds();
-    db,
+  return fromPairs(map(array, item => [getMessageKey(item), true]));
-    getMessageKey,
-    handleDOMException: Whisper.Database.handleDOMException,
-  });
-  return fromPairs(map(array, item => [item, true]));
 }
 
 function getConversationKey(conversation) {
   return conversation.id;
 }
-function loadConversationLookup(db) {
+async function loadConversationLookup() {
-  return assembleLookup(db, 'conversations', getConversationKey);
+  const array = await window.Signal.Data.getAllConversationIds();
+  return fromPairs(map(array, item => [getConversationKey(item), true]));
 }
 
 function getGroupKey(group) {

js/modules/data.js

@@ -1,7 +1,8 @@
 /* global window, setTimeout */
 
 const electron = require('electron');
-const { forEach, isFunction, isObject } = require('lodash');
+
+const { forEach, isFunction, isObject, merge } = require('lodash');
 
 const { deferredToPromise } = require('./deferred_to_promise');
 const MessageType = require('./types/message');
@@ -37,6 +38,20 @@ module.exports = {
   close,
   removeDB,
 
+  getConversationCount,
+  saveConversation,
+  saveConversations,
+  getConversationById,
+  updateConversation,
+  removeConversation,
+  _removeConversations,
+
+  getAllConversations,
+  getAllConversationIds,
+  getAllPrivateConversations,
+  getAllGroupsInvolvingId,
+  searchConversations,
+
   getMessageCount,
   saveMessage,
   saveLegacyMessage,
@@ -49,6 +64,7 @@ module.exports = {
 
   getMessageBySender,
   getMessageById,
+  getAllMessages,
   getAllMessageIds,
   getMessagesBySentAt,
   getExpiredMessages,
@@ -222,6 +238,86 @@ async function removeDB() {
   await channels.removeDB();
 }
 
+async function getConversationCount() {
+  return channels.getConversationCount();
+}
+
+async function saveConversation(data) {
+  await channels.saveConversation(data);
+}
+
+async function saveConversations(data) {
+  await channels.saveConversations(data);
+}
+
+async function getConversationById(id, { Conversation }) {
+  const data = await channels.getConversationById(id);
+  return new Conversation(data);
+}
+
+async function updateConversation(id, data, { Conversation }) {
+  const existing = await getConversationById(id, { Conversation });
+  if (!existing) {
+    throw new Error(`Conversation ${id} does not exist!`);
+  }
+
+  const merged = merge({}, existing.attributes, data);
+  await channels.updateConversation(merged);
+}
+
+async function removeConversation(id, { Conversation }) {
+  const existing = await getConversationById(id, { Conversation });
+
+  // Note: It's important to have a fully database-hydrated model to delete here because
+  //   it needs to delete all associated on-disk files along with the database delete.
+  if (existing) {
+    await channels.removeConversation(id);
+    await existing.cleanup();
+  }
+}
+
+// Note: this method will not clean up external files, just delete from SQL
+async function _removeConversations(ids) {
+  await channels.removeConversation(ids);
+}
+
+async function getAllConversations({ ConversationCollection }) {
+  const conversations = await channels.getAllConversations();
+
+  const collection = new ConversationCollection();
+  collection.add(conversations);
+  return collection;
+}
+
+async function getAllConversationIds() {
+  const ids = await channels.getAllConversationIds();
+  return ids;
+}
+
+async function getAllPrivateConversations({ ConversationCollection }) {
+  const conversations = await channels.getAllPrivateConversations();
+
+  const collection = new ConversationCollection();
+  collection.add(conversations);
+  return collection;
+}
+
+async function getAllGroupsInvolvingId(id, { ConversationCollection }) {
+  const conversations = await channels.getAllGroupsInvolvingId(id);
+
+  const collection = new ConversationCollection();
+  collection.add(conversations);
+  return collection;
+}
+
+async function searchConversations(query, { ConversationCollection }) {
+  const conversations = await channels.searchConversations(query);
+
+  const collection = new ConversationCollection();
+  collection.add(conversations);
+  return collection;
+}
+
 async function getMessageCount() {
   return channels.getMessageCount();
 }
@@ -267,6 +363,12 @@ async function getMessageById(id, { Message }) {
   return new Message(message);
 }
 
+// For testing only
+async function getAllMessages({ MessageCollection }) {
+  const messages = await channels.getAllMessages();
+  return new MessageCollection(messages);
+}
+
 async function getAllMessageIds() {
   const ids = await channels.getAllMessageIds();
   return ids;

js/modules/debug.js

@@ -16,7 +16,6 @@ const {
 
 const Attachments = require('../../app/attachments');
 const Message = require('./types/message');
-const { deferredToPromise } = require('./deferred_to_promise');
 const { sleep } = require('./sleep');
 
 // See: https://en.wikipedia.org/wiki/Fictitious_telephone_number#North_American_Numbering_Plan
@@ -50,9 +49,12 @@ exports.createConversation = async ({
     active_at: Date.now(),
     unread: numMessages,
   });
-  await deferredToPromise(conversation.save());
-
   const conversationId = conversation.get('id');
+  await Signal.Data.updateConversation(
+    conversationId,
+    conversation.attributes,
+    { Conversation: Whisper.Conversation }
+  );
 
   await Promise.all(
     range(0, numMessages).map(async index => {

js/modules/messages_data_migrator.js

@@ -4,7 +4,7 @@
 // IndexedDB access. This includes avoiding usage of `storage` module which uses
 // Backbone under the hood.
 
-/* global IDBKeyRange */
+/* global IDBKeyRange, window */
 
 const { isFunction, isNumber, isObject, isString, last } = require('lodash');
 
@@ -47,13 +47,25 @@ exports.processNext = async ({
   const startTime = Date.now();
 
   const fetchStartTime = Date.now();
-  const messagesRequiringSchemaUpgrade = await getMessagesNeedingUpgrade(
+  let messagesRequiringSchemaUpgrade;
-    numMessagesPerBatch,
+  try {
-    {
+    messagesRequiringSchemaUpgrade = await getMessagesNeedingUpgrade(
-      maxVersion,
+      numMessagesPerBatch,
-      MessageCollection: BackboneMessageCollection,
+      {
-    }
+        maxVersion,
-  );
+        MessageCollection: BackboneMessageCollection,
+      }
+    );
+  } catch (error) {
+    window.log.error(
+      'processNext error:',
+      error && error.stack ? error.stack : error
+    );
+    return {
+      done: true,
+      numProcessed: 0,
+    };
+  }
   const fetchDuration = Date.now() - fetchStartTime;
 
   const upgradeStartTime = Date.now();
@@ -263,13 +275,26 @@ const _processBatch = async ({
   );
 
   const fetchUnprocessedMessagesStartTime = Date.now();
-  const unprocessedMessages = await _dangerouslyFetchMessagesRequiringSchemaUpgradeWithoutIndex(
+  let unprocessedMessages;
-    {
+  try {
-      connection,
+    unprocessedMessages = await _dangerouslyFetchMessagesRequiringSchemaUpgradeWithoutIndex(
-      count: numMessagesPerBatch,
+      {
-      lastIndex: lastProcessedIndex,
+        connection,
-    }
+        count: numMessagesPerBatch,
-  );
+        lastIndex: lastProcessedIndex,
+      }
+    );
+  } catch (error) {
+    window.log.error(
+      '_processBatch error:',
+      error && error.stack ? error.stack : error
+    );
+    await settings.markAttachmentMigrationComplete(connection);
+    await settings.deleteAttachmentMigrationLastProcessedIndex(connection);
+    return {
+      done: true,
+    };
+  }
   const fetchDuration = Date.now() - fetchUnprocessedMessagesStartTime;
 
   const upgradeStartTime = Date.now();

js/modules/migrate_to_sql.js

@@ -6,6 +6,8 @@ const {
   _removeMessages,
   saveUnprocesseds,
   removeUnprocessed,
+  saveConversations,
+  _removeConversations,
 } = require('./data');
 const {
   getMessageExportLastIndex,
@@ -15,6 +17,7 @@ const {
   getUnprocessedExportLastIndex,
   setUnprocessedExportLastIndex,
 } = require('./settings');
+const { migrateConversation } = require('./types/conversation');
 
 module.exports = {
   migrateToSQL,
@@ -26,6 +29,7 @@ async function migrateToSQL({
   handleDOMException,
   countCallback,
   arrayBufferToString,
+  writeNewAttachmentData,
 }) {
   if (!db) {
     throw new Error('Need db for IndexedDB connection!');
@@ -74,6 +78,11 @@ async function migrateToSQL({
     }
   }
   window.log.info('migrateToSQL: migrate of messages complete');
+  try {
+    await clearStores(['messages']);
+  } catch (error) {
+    window.log.warn('Failed to clear messages store');
+  }
 
   lastIndex = await getUnprocessedExportLastIndex(db);
   complete = false;
@@ -116,8 +125,43 @@ async function migrateToSQL({
     await setUnprocessedExportLastIndex(db, lastIndex);
   }
   window.log.info('migrateToSQL: migrate of unprocessed complete');
+  try {
+    await clearStores(['unprocessed']);
+  } catch (error) {
+    window.log.warn('Failed to clear unprocessed store');
+  }
 
-  await clearStores(['messages', 'unprocessed']);
+  complete = false;
+  while (!complete) {
+    // eslint-disable-next-line no-await-in-loop
+    const status = await migrateStoreToSQLite({
+      db,
+      // eslint-disable-next-line no-loop-func
+      save: async array => {
+        const conversations = await Promise.all(
+          map(array, async conversation =>
+            migrateConversation(conversation, { writeNewAttachmentData })
+          )
+        );
+
+        saveConversations(conversations);
+      },
+      remove: _removeConversations,
+      storeName: 'conversations',
+      handleDOMException,
+      lastIndex,
+      // Because we're doing real-time moves to the filesystem, minimize parallelism
+      batchSize: 5,
+    });
+
+    ({ complete, lastIndex } = status);
+  }
+  window.log.info('migrateToSQL: migrate of conversations complete');
+  try {
+    await clearStores(['conversations']);
+  } catch (error) {
+    window.log.warn('Failed to clear conversations store');
+  }
 
   window.log.info('migrateToSQL: complete');
 }

js/modules/migrations/get_placeholder_migrations.js

@@ -1,15 +1,13 @@
+/* global window, Whisper */
+
 const Migrations0DatabaseWithAttachmentData = require('./migrations_0_database_with_attachment_data');
-const Migrations1DatabaseWithoutAttachmentData = require('./migrations_1_database_without_attachment_data');
 
 exports.getPlaceholderMigrations = () => {
   const last0MigrationVersion = Migrations0DatabaseWithAttachmentData.getLatestVersion();
-  const last1MigrationVersion = Migrations1DatabaseWithoutAttachmentData.getLatestVersion();
-
-  const lastMigrationVersion = last1MigrationVersion || last0MigrationVersion;
 
   return [
     {
-      version: lastMigrationVersion,
+      version: last0MigrationVersion,
       migrate() {
         throw new Error(
           'Unexpected invocation of placeholder migration!' +
@@ -20,3 +18,18 @@ exports.getPlaceholderMigrations = () => {
     },
   ];
 };
+
+exports.getCurrentVersion = () =>
+  new Promise((resolve, reject) => {
+    const request = window.indexedDB.open(Whisper.Database.id);
+
+    request.onerror = reject;
+    request.onupgradeneeded = reject;
+
+    request.onsuccess = () => {
+      const db = request.result;
+      const { version } = db;
+
+      return resolve(version);
+    };
+  });

js/modules/migrations/migrations_1_database_without_attachment_data.js

@@ -1,22 +1,38 @@
+/* global window */
+
 const { last } = require('lodash');
 
 const db = require('../database');
 const settings = require('../settings');
 const { runMigrations } = require('./run_migrations');
 
-// IMPORTANT: Add new migrations that need to traverse entire database, e.g.
+// These are cleanup migrations, to be run after migration to SQLCipher
-// messages store, below. Whenever we need this, we need to force attachment
+exports.migrations = [
-// migration on startup:
+  {
-const migrations = [
+    version: 19,
-  // {
+    migrate(transaction, next) {
-  //   version: 0,
+      window.log.info('Migration 19');
-  //   migrate(transaction, next) {
+      window.log.info(
-  //     next();
+        'Removing messages, unprocessed, and conversations object stores'
-  //   },
+      );
-  // },
+
+      // This should be run after things are migrated to SQLCipher
+      transaction.db.deleteObjectStore('messages');
+      transaction.db.deleteObjectStore('unprocessed');
+      transaction.db.deleteObjectStore('conversations');
+
+      next();
+    },
+  },
 ];
 
-exports.run = async ({ Backbone, database, logger } = {}) => {
+exports.run = async ({ Backbone, logger } = {}) => {
+  const database = {
+    id: 'signal',
+    nolog: true,
+    migrations: exports.migrations,
+  };
+
   const { canRun } = await exports.getStatus({ database });
   if (!canRun) {
     throw new Error(
@@ -24,7 +40,11 @@ exports.run = async ({ Backbone, database, logger } = {}) => {
     );
   }
 
-  await runMigrations({ Backbone, database, logger });
+  await runMigrations({
+    Backbone,
+    logger,
+    database,
+  });
 };
 
 exports.getStatus = async ({ database } = {}) => {
@@ -32,7 +52,7 @@ exports.getStatus = async ({ database } = {}) => {
   const isAttachmentMigrationComplete = await settings.isAttachmentMigrationComplete(
     connection
   );
-  const hasMigrations = migrations.length > 0;
+  const hasMigrations = exports.migrations.length > 0;
 
   const canRun = isAttachmentMigrationComplete && hasMigrations;
   return {
@@ -43,7 +63,7 @@ exports.getStatus = async ({ database } = {}) => {
 };
 
 exports.getLatestVersion = () => {
-  const lastMigration = last(migrations);
+  const lastMigration = last(exports.migrations);
   if (!lastMigration) {
     return null;
   }

js/modules/signal.js

@@ -58,6 +58,7 @@ const {
 // Migrations
 const {
   getPlaceholderMigrations,
+  getCurrentVersion,
 } = require('./migrations/get_placeholder_migrations');
 
 const Migrations0DatabaseWithAttachmentData = require('./migrations/migrations_0_database_with_attachment_data');
@@ -67,7 +68,7 @@ const Migrations1DatabaseWithoutAttachmentData = require('./migrations/migration
 const AttachmentType = require('./types/attachment');
 const VisualAttachment = require('./types/visual_attachment');
 const Contact = require('../../ts/types/Contact');
-const Conversation = require('../../ts/types/Conversation');
+const Conversation = require('./types/conversation');
 const Errors = require('./types/errors');
 const MediaGalleryMessage = require('../../ts/components/conversation/media-gallery/types/Message');
 const MessageType = require('./types/message');
@@ -123,11 +124,14 @@ function initializeMigrations({
     }),
     getAbsoluteAttachmentPath,
     getPlaceholderMigrations,
+    getCurrentVersion,
     loadAttachmentData,
     loadQuoteData,
     loadMessage: MessageType.createAttachmentLoader(loadAttachmentData),
     Migrations0DatabaseWithAttachmentData,
     Migrations1DatabaseWithoutAttachmentData,
+    writeNewAttachmentData: createWriterForNew(attachmentsPath),
+    deleteAttachmentData: deleteOnDisk,
     upgradeMessageSchema: (message, options = {}) => {
       const { maxVersion } = options;
 

js/modules/types/conversation.js

@@ -0,0 +1,133 @@
+/* global dcodeIO, crypto */
+
+const { isFunction, isNumber } = require('lodash');
+const { createLastMessageUpdate } = require('../../../ts/types/Conversation');
+
+async function computeHash(arraybuffer) {
+  const hash = await crypto.subtle.digest({ name: 'SHA-512' }, arraybuffer);
+  return arrayBufferToBase64(hash);
+}
+
+function arrayBufferToBase64(arraybuffer) {
+  return dcodeIO.ByteBuffer.wrap(arraybuffer).toString('base64');
+}
+
+function base64ToArrayBuffer(base64) {
+  return dcodeIO.ByteBuffer.wrap(base64, 'base64').toArrayBuffer();
+}
+
+function buildAvatarUpdater({ field }) {
+  return async (conversation, data, options = {}) => {
+    if (!conversation) {
+      return conversation;
+    }
+
+    const avatar = conversation[field];
+    const { writeNewAttachmentData, deleteAttachmentData } = options;
+    if (!isFunction(writeNewAttachmentData)) {
+      throw new Error(
+        'Conversation.buildAvatarUpdater: writeNewAttachmentData must be a function'
+      );
+    }
+    if (!isFunction(deleteAttachmentData)) {
+      throw new Error(
+        'Conversation.buildAvatarUpdater: deleteAttachmentData must be a function'
+      );
+    }
+
+    const newHash = await computeHash(data);
+
+    if (!avatar || !avatar.hash) {
+      return {
+        ...conversation,
+        avatar: {
+          hash: newHash,
+          path: await writeNewAttachmentData(data),
+        },
+      };
+    }
+
+    const { hash, path } = avatar;
+
+    if (hash === newHash) {
+      return conversation;
+    }
+
+    await deleteAttachmentData(path);
+
+    return {
+      ...conversation,
+      avatar: {
+        hash: newHash,
+        path: await writeNewAttachmentData(data),
+      },
+    };
+  };
+}
+
+const maybeUpdateAvatar = buildAvatarUpdater({ field: 'avatar' });
+const maybeUpdateProfileAvatar = buildAvatarUpdater({
+  field: 'profileAvatar',
+});
+
+async function upgradeToVersion2(conversation, options) {
+  if (conversation.version >= 2) {
+    return conversation;
+  }
+
+  const { writeNewAttachmentData } = options;
+  if (!isFunction(writeNewAttachmentData)) {
+    throw new Error(
+      'Conversation.upgradeToVersion2: writeNewAttachmentData must be a function'
+    );
+  }
+
+  let { avatar, profileAvatar, profileKey } = conversation;
+
+  if (avatar && avatar.data) {
+    avatar = {
+      hash: await computeHash(avatar.data),
+      path: await writeNewAttachmentData(avatar.data),
+    };
+  }
+
+  if (profileAvatar && profileAvatar.data) {
+    profileAvatar = {
+      hash: await computeHash(profileAvatar.data),
+      path: await writeNewAttachmentData(profileAvatar.data),
+    };
+  }
+
+  if (profileKey && profileKey.byteLength) {
+    profileKey = arrayBufferToBase64(profileKey);
+  }
+
+  return {
+    ...conversation,
+    version: 2,
+    avatar,
+    profileAvatar,
+    profileKey,
+  };
+}
+
+async function migrateConversation(conversation, options = {}) {
+  if (!conversation) {
+    return conversation;
+  }
+  if (!isNumber(conversation.version)) {
+    // eslint-disable-next-line no-param-reassign
+    conversation.version = 1;
+  }
+
+  return upgradeToVersion2(conversation, options);
+}
+
+module.exports = {
+  migrateConversation,
+  maybeUpdateAvatar,
+  maybeUpdateProfileAvatar,
+  createLastMessageUpdate,
+  arrayBufferToBase64,
+  base64ToArrayBuffer,
+};

js/read_receipts.js

@@ -40,15 +40,15 @@
         return message;
       }
 
-      const groups = new Whisper.GroupCollection();
+      const groups = await window.Signal.Data.getAllGroupsInvolvingId(reader, {
-      return groups.fetchGroups(reader).then(() => {
+        ConversationCollection: Whisper.ConversationCollection,
-        const ids = groups.pluck('id');
-        ids.push(reader);
-        return messages.find(
-          item =>
-            item.isOutgoing() && _.contains(ids, item.get('conversationId'))
-        );
       });
+      const ids = groups.pluck('id');
+      ids.push(reader);
+
+      return messages.find(
+        item => item.isOutgoing() && _.contains(ids, item.get('conversationId'))
+      );
     },
     async onReceipt(receipt) {
       try {

js/signal_protocol_store.js

@@ -981,7 +981,6 @@
         'sessions',
         'signedPreKeys',
         'preKeys',
-        'unprocessed',
       ]);
 
       await window.Signal.Data.removeAllUnprocessed();

js/views/conversation_search_view.js

@@ -134,24 +134,8 @@
       this.hideHints();
       this.new_contact_view.$el.hide();
       this.$input.val('').focus();
-      if (this.showAllContacts) {
+      this.typeahead_view.collection.reset([]);
-        // NOTE: Temporarily allow `then` until we convert the entire file
+      this.trigger('hide');
-        // to `async` / `await`:
-        // eslint-disable-next-line more/no-then
-        this.typeahead.fetchAlphabetical().then(() => {
-          if (this.typeahead.length > 0) {
-            this.typeahead_view.collection.reset(
-              this.typeahead.filter(isSearchable)
-            );
-          } else {
-            this.showHints();
-          }
-        });
-        this.trigger('show');
-      } else {
-        this.typeahead_view.collection.reset([]);
-        this.trigger('hide');
-      }
     },
 
     showHints() {

js/views/new_group_update_view.js

@@ -57,32 +57,43 @@
         avatar: this.model.getAvatar(),
       };
     },
-    send() {
+    async send() {
-      return this.avatarInput.getThumbnail().then(avatarFile => {
+      // When we turn this view on again, need to handle avatars in the new way
-        const now = Date.now();
-        const attrs = {
-          timestamp: now,
-          active_at: now,
-          name: this.$('.name').val(),
-          members: _.union(
-            this.model.get('members'),
-            this.recipients_view.recipients.pluck('id')
-          ),
-        };
-        if (avatarFile) {
-          attrs.avatar = avatarFile;
-        }
-        this.model.set(attrs);
-        const groupUpdate = this.model.changed;
-        this.model.save();
 
-        if (groupUpdate.avatar) {
+      // const avatarFile = await this.avatarInput.getThumbnail();
-          this.model.trigger('change:avatar');
+      const now = Date.now();
-        }
+      const attrs = {
+        timestamp: now,
+        active_at: now,
+        name: this.$('.name').val(),
+        members: _.union(
+          this.model.get('members'),
+          this.recipients_view.recipients.pluck('id')
+        ),
+      };
 
-        this.model.updateGroup(groupUpdate);
+      // if (avatarFile) {
-        this.goBack();
+      //   attrs.avatar = avatarFile;
-      });
+      // }
+
+      // Because we're no longer using Backbone-integrated saves, we need to manually
+      //   clear the changed fields here so model.changed is accurate.
+      this.model.changed = {};
+      this.model.set(attrs);
+      const groupUpdate = this.model.changed;
+
+      await window.Signal.Data.updateConversation(
+        this.model.id,
+        this.model.attributes,
+        { Conversation: Whisper.Conversation }
+      );
+
+      if (groupUpdate.avatar) {
+        this.model.trigger('change:avatar');
+      }
+
+      this.model.updateGroup(groupUpdate);
+      this.goBack();
     },
   });
 })();

js/views/recipients_input_view.js

@@ -16,8 +16,12 @@
     database: Whisper.Database,
     storeName: 'conversations',
     model: Whisper.Conversation,
-    fetchContacts() {
+    async fetchContacts() {
-      return this.fetch({ reset: true, conditions: { type: 'private' } });
+      const models = window.Signal.Data.getAllPrivateConversations({
+        ConversationCollection: Whisper.ConversationCollection,
+      });
+
+      this.reset(models);
     },
   });
 

test/backup_test.js

@@ -655,8 +655,9 @@ describe('Backup', () => {
           verified: 0,
         };
         console.log({ conversation });
-        const conversationModel = new Whisper.Conversation(conversation);
+        await window.Signal.Data.saveConversation(conversation, {
-        await window.wrapDeferred(conversationModel.save());
+          Conversation: Whisper.Conversation,
+        });
 
         console.log(
           'Backup test: Ensure that all attachments were saved to disk'
@@ -698,8 +699,9 @@ describe('Backup', () => {
         assert.deepEqual(attachmentFiles, recreatedAttachmentFiles);
 
         console.log('Backup test: Check messages');
-        const messageCollection = new Whisper.MessageCollection();
+        const messageCollection = await window.Signal.Data.getAllMessages({
-        await window.wrapDeferred(messageCollection.fetch());
+          MessageCollection: Whisper.MessageCollection,
+        });
         assert.strictEqual(messageCollection.length, MESSAGE_COUNT);
         const messageFromDB = removeId(messageCollection.at(0).attributes);
         const expectedMessage = omitUndefinedKeys(message);
@@ -725,8 +727,11 @@ describe('Backup', () => {
         );
 
         console.log('Backup test: Check conversations');
-        const conversationCollection = new Whisper.ConversationCollection();
+        const conversationCollection = await window.Signal.Data.getAllConversations(
-        await window.wrapDeferred(conversationCollection.fetch());
+          {
+            ConversationCollection: Whisper.ConversationCollection,
+          }
+        );
         assert.strictEqual(conversationCollection.length, CONVERSATION_COUNT);
 
         const conversationFromDB = conversationCollection.at(0).attributes;

test/fixtures.js

@@ -232,7 +232,9 @@ Whisper.Fixtures = function() {
   conversationCollection.saveAll = function() {
     return Promise.all(
       this.map(async (convo) => {
-        await wrapDeferred(convo.save());
+        await window.Signal.Data.saveConversation(convo.attributes, {
+          Conversation: Whisper.Conversation,
+        });
 
         await Promise.all(
           convo.messageCollection.map(async (message) => {

test/fixtures_test.js

@@ -1,11 +1,23 @@
 'use strict';
 
 describe('Fixtures', function() {
-  before(function() {
+  before(async function() {
     // NetworkStatusView checks this method every five seconds while showing
     window.getSocketStatus = function() {
       return WebSocket.OPEN;
     };
+
+    await clearDatabase();
+    await textsecure.storage.user.setNumberAndDeviceId(
+      '+17015552000',
+      2,
+      'testDevice'
+    );
+
+    await ConversationController.getOrCreateAndWait(
+      textsecure.storage.user.getNumber(),
+      'private'
+    );
   });
 
   it('renders', async () => {

test/keychange_listener_test.js

@@ -20,23 +20,25 @@ describe('KeyChangeListener', function() {
 
   describe('When we have a conversation with this contact', function() {
     let convo;
-    before(function() {
+    before(async function() {
       convo = ConversationController.dangerouslyCreateAndAdd({
         id: phoneNumberWithKeyChange,
         type: 'private',
       });
-      return convo.save();
+      await window.Signal.Data.saveConversation(convo.attributes, {
+        Conversation: Whisper.Conversation,
+      });
     });
 
-    after(function() {
+    after(async function() {
-      convo.destroyMessages();
+      await convo.destroyMessages();
-      return convo.destroy();
+      await window.Signal.Data.saveConversation(convo.id);
     });
 
     it('generates a key change notice in the private conversation with this contact', function(done) {
-      convo.on('newmessage', async function() {
+      convo.once('newmessage', async () => {
         await convo.fetchMessages();
-        var message = convo.messageCollection.at(0);
+        const message = convo.messageCollection.at(0);
         assert.strictEqual(message.get('type'), 'keychange');
         done();
       });
@@ -46,23 +48,26 @@ describe('KeyChangeListener', function() {
 
   describe('When we have a group with this contact', function() {
     let convo;
-    before(function() {
+    before(async function() {
+      console.log('Creating group with contact', phoneNumberWithKeyChange);
       convo = ConversationController.dangerouslyCreateAndAdd({
         id: 'groupId',
         type: 'group',
         members: [phoneNumberWithKeyChange],
       });
-      return convo.save();
+      await window.Signal.Data.saveConversation(convo.attributes, {
+        Conversation: Whisper.Conversation,
+      });
     });
-    after(function() {
+    after(async function() {
-      convo.destroyMessages();
+      await convo.destroyMessages();
-      return convo.destroy();
+      await window.Signal.Data.saveConversation(convo.id);
     });
 
     it('generates a key change notice in the group conversation with this contact', function(done) {
-      convo.on('newmessage', async function() {
+      convo.once('newmessage', async () => {
         await convo.fetchMessages();
-        var message = convo.messageCollection.at(0);
+        const message = convo.messageCollection.at(0);
         assert.strictEqual(message.get('type'), 'keychange');
         done();
       });

test/models/conversations_test.js

@@ -17,50 +17,6 @@
     before(clearDatabase);
     after(clearDatabase);
 
-    it('adds without saving', function(done) {
-      var convos = new Whisper.ConversationCollection();
-      convos.add(conversation_attributes);
-      assert.notEqual(convos.length, 0);
-
-      var convos = new Whisper.ConversationCollection();
-      convos.fetch().then(function() {
-        assert.strictEqual(convos.length, 0);
-        done();
-      });
-    });
-
-    it('saves asynchronously', function(done) {
-      new Whisper.ConversationCollection()
-        .add(conversation_attributes)
-        .save()
-        .then(done);
-    });
-
-    it('fetches persistent convos', async () => {
-      var convos = new Whisper.ConversationCollection();
-      assert.strictEqual(convos.length, 0);
-
-      await wrapDeferred(convos.fetch());
-
-      var m = convos.at(0).attributes;
-      _.each(conversation_attributes, function(val, key) {
-        assert.deepEqual(m[key], val);
-      });
-    });
-
-    it('destroys persistent convos', function(done) {
-      var convos = new Whisper.ConversationCollection();
-      convos.fetch().then(function() {
-        convos.destroyAll().then(function() {
-          var convos = new Whisper.ConversationCollection();
-          convos.fetch().then(function() {
-            assert.strictEqual(convos.length, 0);
-            done();
-          });
-        });
-      });
-    });
-
     it('should be ordered newest to oldest', function() {
       var conversations = new Whisper.ConversationCollection();
       // Timestamps
@@ -85,7 +41,9 @@
     var attributes = { type: 'private', id: '+18085555555' };
     before(async () => {
       var convo = new Whisper.ConversationCollection().add(attributes);
-      await wrapDeferred(convo.save());
+      await window.Signal.Data.saveConversation(convo.attributes, {
+        Conversation: Whisper.Conversation,
+      });
 
       var message = convo.messageCollection.add({
         body: 'hello world',
@@ -123,32 +81,28 @@
       assert.strictEqual(convo.contactCollection.at('2').get('name'), 'C');
     });
 
-    it('contains its own messages', function(done) {
+    it('contains its own messages', async function() {
       var convo = new Whisper.ConversationCollection().add({
         id: '+18085555555',
       });
-      convo.fetchMessages().then(function() {
+      await convo.fetchMessages();
-        assert.notEqual(convo.messageCollection.length, 0);
+      assert.notEqual(convo.messageCollection.length, 0);
-        done();
-      });
     });
 
-    it('contains only its own messages', function(done) {
+    it('contains only its own messages', async function() {
       var convo = new Whisper.ConversationCollection().add({
         id: '+18085556666',
       });
-      convo.fetchMessages().then(function() {
+      await convo.fetchMessages();
-        assert.strictEqual(convo.messageCollection.length, 0);
+      assert.strictEqual(convo.messageCollection.length, 0);
-        done();
-      });
     });
 
-    it('adds conversation to message collection upon leaving group', function() {
+    it('adds conversation to message collection upon leaving group', async function() {
       var convo = new Whisper.ConversationCollection().add({
         type: 'group',
         id: 'a random string',
       });
-      convo.leaveGroup();
+      await convo.leaveGroup();
       assert.notEqual(convo.messageCollection.length, 0);
     });
 
@@ -180,12 +134,6 @@
       assert.property(avatar, 'color');
     });
 
-    it('revokes the avatar URL', function() {
-      var convo = new Whisper.ConversationCollection().add(attributes);
-      convo.revokeAvatarUrl();
-      assert.notOk(convo.avatarUrl);
-    });
-
     describe('phone number parsing', function() {
       after(function() {
         storage.remove('regionCode');
@@ -228,57 +176,51 @@
   describe('Conversation search', function() {
     let convo;
 
-    beforeEach(function(done) {
+    beforeEach(async function() {
       convo = new Whisper.ConversationCollection().add({
         id: '+14155555555',
         type: 'private',
         name: 'John Doe',
       });
-      convo.save().then(done);
+      await window.Signal.Data.saveConversation(convo.attributes, {
+        Conversation: Whisper.Conversation,
+      });
     });
+
     afterEach(clearDatabase);
 
-    function testSearch(queries, done) {
+    async function testSearch(queries) {
-      return Promise.all(
+      await Promise.all(
-        queries.map(function(query) {
+        queries.map(async function(query) {
           var collection = new Whisper.ConversationCollection();
-          return collection
+          await collection.search(query);
-            .search(query)
+
-            .then(function() {
+          assert.isDefined(
-              assert.isDefined(
+            collection.get(convo.id),
-                collection.get(convo.id),
+            'no result for "' + query + '"'
-                'no result for "' + query + '"'
+          );
-              );
-            })
-            .catch(done);
         })
-      ).then(function() {
-        done();
-      });
-    }
-    it('matches by partial phone number', function(done) {
-      testSearch(
-        [
-          '1',
-          '4',
-          '+1',
-          '415',
-          '4155',
-          '4155555555',
-          '14155555555',
-          '+14155555555',
-        ],
-        done
       );
+    }
+    it('matches by partial phone number', function() {
+      return testSearch([
+        '1',
+        '4',
+        '+1',
+        '415',
+        '4155',
+        '4155555555',
+        '14155555555',
+        '+14155555555',
+      ]);
     });
-    it('matches by name', function(done) {
+    it('matches by name', function() {
-      testSearch(['John', 'Doe', 'john', 'doe', 'John Doe', 'john doe'], done);
+      return testSearch(['John', 'Doe', 'john', 'doe', 'John Doe', 'john doe']);
     });
-    it('does not match +', function() {
+    it('does not match +', async function() {
       var collection = new Whisper.ConversationCollection();
-      return collection.search('+').then(function() {
+      await collection.search('+');
-        assert.isUndefined(collection.get(convo.id), 'got result for "+"');
+      assert.isUndefined(collection.get(convo.id), 'got result for "+"');
-      });
     });
   });
 })();

test/views/conversation_search_view_test.js

@@ -28,14 +28,16 @@ describe('ConversationSearchView', function() {
 
     before(() => {
       convo = new Whisper.ConversationCollection().add({
-        id: 'a-left-group',
+        id: '1-search-view',
         name: 'i left this group',
         members: [],
         type: 'group',
         left: true,
       });
 
-      return wrapDeferred(convo.save());
+      return window.Signal.Data.saveConversation(convo.attributes, {
+        Conversation: Whisper.Conversation,
+      });
     });
     describe('with no messages', function() {
       var input;
@@ -60,65 +62,20 @@ describe('ConversationSearchView', function() {
     describe('with messages', function() {
       var input;
       var view;
-      before(function(done) {
+      before(async function() {
         input = $('<input>');
         view = new Whisper.ConversationSearchView({ input: input }).render();
-        convo.save({ lastMessage: 'asdf' }).then(function() {
+        convo.set({ id: '2-search-view', lastMessage: 'asdf' });
-          view.$input.val('left');
-          view.filterContacts();
-          view.typeahead_view.collection.on('reset', function() {
-            done();
-          });
-        });
-      });
-      it('should surface left groups with messages', function() {
-        assert.isDefined(
-          view.typeahead_view.collection.get(convo.id),
-          'got left group'
-        );
-      });
-    });
-  });
-  describe('Showing all contacts', function() {
-    let input;
-    let view;
-    let convo;
 
-    before(() => {
+        await window.Signal.Data.saveConversation(convo.attributes, {
-      input = $('<input>');
+          Conversation: Whisper.Conversation,
-      view = new Whisper.ConversationSearchView({ input: input }).render();
-      view.showAllContacts = true;
-      convo = new Whisper.ConversationCollection().add({
-        id: 'a-left-group',
-        name: 'i left this group',
-        members: [],
-        type: 'group',
-        left: true,
-      });
-
-      return wrapDeferred(convo.save());
-    });
-    describe('with no messages', function() {
-      before(function(done) {
-        view.resetTypeahead();
-        view.typeahead_view.collection.once('reset', function() {
-          done();
         });
-      });
+
-      it('should not surface left groups with no messages', function() {
+        view.$input.val('left');
-        assert.isUndefined(
+        view.filterContacts();
-          view.typeahead_view.collection.get(convo.id),
+
-          'got left group'
+        return new Promise(resolve => {
-        );
+          view.typeahead_view.collection.on('reset', resolve);
-      });
-    });
-    describe('with messages', function() {
-      before(done => {
-        wrapDeferred(convo.save({ lastMessage: 'asdf' })).then(function() {
-          view.resetTypeahead();
-          view.typeahead_view.collection.once('reset', function() {
-            done();
-          });
         });
       });
       it('should surface left groups with messages', function() {

test/views/inbox_view_test.js

@@ -2,7 +2,19 @@ describe('InboxView', function() {
   let inboxView;
   let conversation;
 
-  before(() => {
+  before(async () => {
+    try {
+      await ConversationController.load();
+    } catch (error) {
+      console.log(
+        'InboxView before:',
+        error && error.stack ? error.stack : error
+      );
+    }
+    await ConversationController.getOrCreateAndWait(
+      textsecure.storage.user.getNumber(),
+      'private'
+    );
     inboxView = new Whisper.InboxView({
       model: {},
       window: window,

ts/types/backbone/Collection.ts

@@ -1,8 +0,0 @@
-import { Model } from './Model';
-
-export interface Collection<T> {
-  models: Array<Model<T>>;
-  // tslint:disable-next-line no-misused-new
-  new (): Collection<T>;
-  fetch(options: object): JQuery.Deferred<any>;
-}

ts/types/backbone/Model.ts

@@ -1,3 +0,0 @@
-export interface Model<T> {
-  toJSON(): T;
-}