Move all remaining stores to SQLCipher

_locales/en/messages.json

@@ -798,11 +798,6 @@
     "message":
       "Are you sure? Clicking 'delete' will permanently remove this message from this device only."
   },
-  "unidentifiedDelivery": {
-    "message": "Unidentified Delivery",
-    "description":
-      "Label shown on the message detail screen for messages sent or received with Unidentified Delivery enabled"
-  },
   "deleteThisMessage": {
     "message": "Delete this message"
   },

app/sql.js

@@ -15,6 +15,47 @@ module.exports = {
   close,
   removeDB,
 
+  createOrUpdateGroup,
+  getGroupById,
+  getAllGroupIds,
+  bulkAddGroups,
+  removeGroupById,
+  removeAllGroups,
+
+  createOrUpdateIdentityKey,
+  getIdentityKeyById,
+  bulkAddIdentityKeys,
+  removeIdentityKeyById,
+  removeAllIdentityKeys,
+
+  createOrUpdatePreKey,
+  getPreKeyById,
+  bulkAddPreKeys,
+  removePreKeyById,
+  removeAllPreKeys,
+
+  createOrUpdateSignedPreKey,
+  getSignedPreKeyById,
+  getAllSignedPreKeys,
+  bulkAddSignedPreKeys,
+  removeSignedPreKeyById,
+  removeAllSignedPreKeys,
+
+  createOrUpdateItem,
+  getItemById,
+  getAllItems,
+  bulkAddItems,
+  removeItemById,
+  removeAllItems,
+
+  createOrUpdateSession,
+  getSessionById,
+  getSessionsByNumber,
+  bulkAddSessions,
+  removeSessionById,
+  removeSessionsByNumber,
+  removeAllSessions,
+
   getConversationCount,
   saveConversation,
   saveConversations,
@@ -51,6 +92,7 @@ module.exports = {
   removeAllUnprocessed,
 
   removeAll,
+  removeAllConfiguration,
 
   getMessagesNeedingUpgrade,
   getMessagesWithVisualMediaAttachments,
@@ -320,12 +362,72 @@ async function updateToSchemaVersion4(currentVersion, instance) {
   console.log('updateToSchemaVersion4: success!');
 }
 
+async function updateToSchemaVersion6(currentVersion, instance) {
+  if (currentVersion >= 6) {
+    return;
+  }
+  console.log('updateToSchemaVersion6: starting...');
+  await instance.run('BEGIN TRANSACTION;');
+
+  // key-value, ids are strings, one extra column
+  await instance.run(
+    `CREATE TABLE sessions(
+      id STRING PRIMARY KEY ASC,
+      number STRING,
+      json TEXT
+    );`
+  );
+
+  await instance.run(`CREATE INDEX sessions_number ON sessions (
+    number
+  ) WHERE number IS NOT NULL;`);
+
+  // key-value, ids are strings
+  await instance.run(
+    `CREATE TABLE groups(
+      id STRING PRIMARY KEY ASC,
+      json TEXT
+    );`
+  );
+  await instance.run(
+    `CREATE TABLE identityKeys(
+      id STRING PRIMARY KEY ASC,
+      json TEXT
+    );`
+  );
+  await instance.run(
+    `CREATE TABLE items(
+      id STRING PRIMARY KEY ASC,
+      json TEXT
+    );`
+  );
+
+  // key-value, ids are integers
+  await instance.run(
+    `CREATE TABLE preKeys(
+      id INTEGER PRIMARY KEY ASC,
+      json TEXT
+    );`
+  );
+  await instance.run(
+    `CREATE TABLE signedPreKeys(
+      id INTEGER PRIMARY KEY ASC,
+      json TEXT
+    );`
+  );
+
+  await instance.run('PRAGMA schema_version = 6;');
+  await instance.run('COMMIT TRANSACTION;');
+  console.log('updateToSchemaVersion6: success!');
+}
+
 const SCHEMA_VERSIONS = [
   updateToSchemaVersion1,
   updateToSchemaVersion2,
   updateToSchemaVersion3,
   updateToSchemaVersion4,
   // version 5 was dropped
+  updateToSchemaVersion6,
 ];
 
 async function updateSchema(instance) {
@@ -400,6 +502,228 @@ async function removeDB() {
   rimraf.sync(filePath);
 }
 
+const GROUPS_TABLE = 'groups';
+async function createOrUpdateGroup(data) {
+  return createOrUpdate(GROUPS_TABLE, data);
+}
+async function getGroupById(id) {
+  return getById(GROUPS_TABLE, id);
+}
+async function getAllGroupIds() {
+  const rows = await db.all('SELECT id FROM groups ORDER BY id ASC;');
+  return map(rows, row => row.id);
+}
+async function bulkAddGroups(array) {
+  return bulkAdd(GROUPS_TABLE, array);
+}
+async function removeGroupById(id) {
+  return removeById(GROUPS_TABLE, id);
+}
+async function removeAllGroups() {
+  return removeAllFromTable(GROUPS_TABLE);
+}
+
+const IDENTITY_KEYS_TABLE = 'identityKeys';
+async function createOrUpdateIdentityKey(data) {
+  return createOrUpdate(IDENTITY_KEYS_TABLE, data);
+}
+async function getIdentityKeyById(id) {
+  return getById(IDENTITY_KEYS_TABLE, id);
+}
+async function bulkAddIdentityKeys(array) {
+  return bulkAdd(IDENTITY_KEYS_TABLE, array);
+}
+async function removeIdentityKeyById(id) {
+  return removeById(IDENTITY_KEYS_TABLE, id);
+}
+async function removeAllIdentityKeys() {
+  return removeAllFromTable(IDENTITY_KEYS_TABLE);
+}
+
+const PRE_KEYS_TABLE = 'preKeys';
+async function createOrUpdatePreKey(data) {
+  return createOrUpdate(PRE_KEYS_TABLE, data);
+}
+async function getPreKeyById(id) {
+  return getById(PRE_KEYS_TABLE, id);
+}
+async function bulkAddPreKeys(array) {
+  return bulkAdd(PRE_KEYS_TABLE, array);
+}
+async function removePreKeyById(id) {
+  return removeById(PRE_KEYS_TABLE, id);
+}
+async function removeAllPreKeys() {
+  return removeAllFromTable(PRE_KEYS_TABLE);
+}
+
+const SIGNED_PRE_KEYS_TABLE = 'signedPreKeys';
+async function createOrUpdateSignedPreKey(data) {
+  return createOrUpdate(SIGNED_PRE_KEYS_TABLE, data);
+}
+async function getSignedPreKeyById(id) {
+  return getById(SIGNED_PRE_KEYS_TABLE, id);
+}
+async function getAllSignedPreKeys() {
+  const rows = await db.all('SELECT json FROM signedPreKeys ORDER BY id ASC;');
+  return map(rows, row => jsonToObject(row.json));
+}
+async function bulkAddSignedPreKeys(array) {
+  return bulkAdd(SIGNED_PRE_KEYS_TABLE, array);
+}
+async function removeSignedPreKeyById(id) {
+  return removeById(SIGNED_PRE_KEYS_TABLE, id);
+}
+async function removeAllSignedPreKeys() {
+  return removeAllFromTable(SIGNED_PRE_KEYS_TABLE);
+}
+
+const ITEMS_TABLE = 'items';
+async function createOrUpdateItem(data) {
+  return createOrUpdate(ITEMS_TABLE, data);
+}
+async function getItemById(id) {
+  return getById(ITEMS_TABLE, id);
+}
+async function getAllItems() {
+  const rows = await db.all('SELECT json FROM items ORDER BY id ASC;');
+  return map(rows, row => jsonToObject(row.json));
+}
+async function bulkAddItems(array) {
+  return bulkAdd(ITEMS_TABLE, array);
+}
+async function removeItemById(id) {
+  return removeById(ITEMS_TABLE, id);
+}
+async function removeAllItems() {
+  return removeAllFromTable(ITEMS_TABLE);
+}
+
+const SESSIONS_TABLE = 'sessions';
+async function createOrUpdateSession(data) {
+  const { id, number } = data;
+  if (!id) {
+    throw new Error(
+      'createOrUpdateSession: Provided data did not have a truthy id'
+    );
+  }
+  if (!number) {
+    throw new Error(
+      'createOrUpdateSession: Provided data did not have a truthy number'
+    );
+  }
+
+  await db.run(
+    `INSERT OR REPLACE INTO sessions (
+      id,
+      number,
+      json
+    ) values (
+      $id,
+      $number,
+      $json
+    )`,
+    {
+      $id: id,
+      $number: number,
+      $json: objectToJSON(data),
+    }
+  );
+}
+async function getSessionById(id) {
+  return getById(SESSIONS_TABLE, id);
+}
+async function getSessionsByNumber(number) {
+  const rows = await db.all('SELECT * FROM sessions WHERE number = $number;', {
+    $number: number,
+  });
+  return map(rows, row => jsonToObject(row.json));
+}
+async function bulkAddSessions(array) {
+  return bulkAdd(SESSIONS_TABLE, array);
+}
+async function removeSessionById(id) {
+  return removeById(SESSIONS_TABLE, id);
+}
+async function removeSessionsByNumber(number) {
+  await db.run('DELETE FROM sessions WHERE number = $number;', {
+    $number: number,
+  });
+}
+async function removeAllSessions() {
+  return removeAllFromTable(SESSIONS_TABLE);
+}
+
+async function createOrUpdate(table, data) {
+  const { id } = data;
+  if (!id) {
+    throw new Error('createOrUpdate: Provided data did not have a truthy id');
+  }
+
+  await db.run(
+    `INSERT OR REPLACE INTO ${table} (
+      id,
+      json
+    ) values (
+      $id,
+      $json
+    )`,
+    {
+      $id: id,
+      $json: objectToJSON(data),
+    }
+  );
+}
+
+async function bulkAdd(table, array) {
+  let promise;
+
+  db.serialize(() => {
+    promise = Promise.all([
+      db.run('BEGIN TRANSACTION;'),
+      ...map(array, data => createOrUpdate(table, data)),
+      db.run('COMMIT TRANSACTION;'),
+    ]);
+  });
+
+  await promise;
+}
+
+async function getById(table, id) {
+  const row = await db.get(`SELECT * FROM ${table} WHERE id = $id;`, {
+    $id: id,
+  });
+
+  if (!row) {
+    return null;
+  }
+
+  return jsonToObject(row.json);
+}
+
+async function removeById(table, id) {
+  if (!Array.isArray(id)) {
+    await db.run(`DELETE FROM ${table} WHERE id = $id;`, { $id: id });
+    return;
+  }
+
+  if (!id.length) {
+    throw new Error('removeById: No ids to delete!');
+  }
+
+  // Our node interface doesn't seem to allow you to replace one single ? with an array
+  await db.run(
+    `DELETE FROM ${table} WHERE id IN ( ${id.map(() => '?').join(', ')} );`,
+    id
+  );
+}
+
+async function removeAllFromTable(table) {
+  await db.run(`DELETE FROM ${table};`);
+}
+
+// Conversations
+
 async function getConversationCount() {
   const row = await db.get('SELECT count(*) from conversations;');
 
@@ -1007,15 +1331,42 @@ async function removeAllUnprocessed() {
   await db.run('DELETE FROM unprocessed;');
 }
 
+// All data in database
 async function removeAll() {
   let promise;
 
   db.serialize(() => {
     promise = Promise.all([
       db.run('BEGIN TRANSACTION;'),
+      db.run('DELETE FROM conversations;'),
+      db.run('DELETE FROM groups;'),
+      db.run('DELETE FROM identityKeys;'),
+      db.run('DELETE FROM items;'),
       db.run('DELETE FROM messages;'),
+      db.run('DELETE FROM preKeys;'),
+      db.run('DELETE FROM sessions;'),
+      db.run('DELETE FROM signedPreKeys;'),
+      db.run('DELETE FROM unprocessed;'),
+      db.run('COMMIT TRANSACTION;'),
+    ]);
+  });
+
+  await promise;
+}
+
+// Anything that isn't user-visible data
+async function removeAllConfiguration() {
+  let promise;
+
+  db.serialize(() => {
+    promise = Promise.all([
+      db.run('BEGIN TRANSACTION;'),
+      db.run('DELETE FROM identityKeys;'),
+      db.run('DELETE FROM items;'),
+      db.run('DELETE FROM preKeys;'),
+      db.run('DELETE FROM sessions;'),
+      db.run('DELETE FROM signedPreKeys;'),
       db.run('DELETE FROM unprocessed;'),
-      db.run('DELETE from conversations;'),
       db.run('COMMIT TRANSACTION;'),
     ]);
   });

background.html

@@ -590,6 +590,7 @@
   <script type='text/javascript' src='js/reliable_trigger.js'></script>
   <script type='text/javascript' src='js/database.js'></script>
   <script type='text/javascript' src='js/storage.js'></script>
+  <script type='text/javascript' src='js/legacy_storage.js'></script>
   <script type='text/javascript' src='js/signal_protocol_store.js'></script>
   <script type='text/javascript' src='js/libtextsecure.js'></script>
 

js/background.js

@@ -125,16 +125,18 @@
   window.setImmediate = window.nodeSetImmediate;
 
   const { IdleDetector, MessageDataMigrator } = Signal.Workflow;
+  const {
+    mandatoryMessageUpgrade,
+    migrateAllToSQLCipher,
+    removeDatabase,
+    runMigrations,
+    doesDatabaseExist,
+  } = Signal.IndexedDB;
   const { Errors, Message } = window.Signal.Types;
   const {
     upgradeMessageSchema,
     writeNewAttachmentData,
     deleteAttachmentData,
-    getCurrentVersion,
-  } = window.Signal.Migrations;
-  const {
-    Migrations0DatabaseWithAttachmentData,
-    Migrations1DatabaseWithoutAttachmentData,
   } = window.Signal.Migrations;
   const { Views } = window.Signal;
 
@@ -184,16 +186,13 @@
   };
 
   const cancelInitializationMessage = Views.Initialization.setMessage();
+
+  const isIndexedDBPresent = await doesDatabaseExist();
+  if (isIndexedDBPresent) {
+    window.installStorage(window.legacyStorage);
     window.log.info('Start IndexedDB migrations');
-
+    await runMigrations();
-  window.log.info('Run migrations on database with attachment data');
+  }
-  await Migrations0DatabaseWithAttachmentData.run({
-    Backbone,
-    logger: window.log,
-  });
-
-  const latestDBVersion2 = await getCurrentVersion();
-  Whisper.Database.migrations[0].version = latestDBVersion2;
 
   window.log.info('Storage fetch');
   storage.fetch();
@@ -294,121 +293,17 @@
       );
     }
 
-    const MINIMUM_VERSION = 7;
+    if (isIndexedDBPresent) {
-    async function upgradeMessages() {
+      await mandatoryMessageUpgrade({ upgradeMessageSchema });
-      const NUM_MESSAGES_PER_BATCH = 10;
+      await migrateAllToSQLCipher({ writeNewAttachmentData, Views });
-      window.log.info(
+      await removeDatabase();
-        'upgradeMessages: Mandatory message schema upgrade started.',
-        `Target version: ${MINIMUM_VERSION}`
-      );
 
-      let isMigrationWithoutIndexComplete = false;
+      window.installStorage(window.newStorage);
-      while (!isMigrationWithoutIndexComplete) {
+      await window.storage.fetch();
-        const database = Migrations0DatabaseWithAttachmentData.getDatabase();
-        // eslint-disable-next-line no-await-in-loop
-        const batchWithoutIndex = await MessageDataMigrator.processNextBatchWithoutIndex(
-          {
-            databaseName: database.name,
-            minDatabaseVersion: database.version,
-            numMessagesPerBatch: NUM_MESSAGES_PER_BATCH,
-            upgradeMessageSchema,
-            maxVersion: MINIMUM_VERSION,
-            BackboneMessage: Whisper.Message,
-            saveMessage: window.Signal.Data.saveLegacyMessage,
     }
-        );
-        window.log.info(
-          'upgradeMessages: upgrade without index',
-          batchWithoutIndex
-        );
-        isMigrationWithoutIndexComplete = batchWithoutIndex.done;
-      }
-      window.log.info('upgradeMessages: upgrade without index complete!');
-
-      let isMigrationWithIndexComplete = false;
-      while (!isMigrationWithIndexComplete) {
-        // eslint-disable-next-line no-await-in-loop
-        const batchWithIndex = await MessageDataMigrator.processNext({
-          BackboneMessage: Whisper.Message,
-          BackboneMessageCollection: Whisper.MessageCollection,
-          numMessagesPerBatch: NUM_MESSAGES_PER_BATCH,
-          upgradeMessageSchema,
-          getMessagesNeedingUpgrade:
-            window.Signal.Data.getLegacyMessagesNeedingUpgrade,
-          saveMessage: window.Signal.Data.saveLegacyMessage,
-          maxVersion: MINIMUM_VERSION,
-        });
-        window.log.info('upgradeMessages: upgrade with index', batchWithIndex);
-        isMigrationWithIndexComplete = batchWithIndex.done;
-      }
-      window.log.info('upgradeMessages: upgrade with index complete!');
-
-      window.log.info('upgradeMessages: Message schema upgrade complete');
-    }
-
-    await upgradeMessages();
-
-    const db = await Whisper.Database.open();
-    let totalMessages;
-    try {
-      totalMessages = await MessageDataMigrator.getNumMessages({
-        connection: db,
-      });
-    } catch (error) {
-      window.log.error(
-        'background.getNumMessages error:',
-        error && error.stack ? error.stack : error
-      );
-      totalMessages = 0;
-    }
-
-    function showMigrationStatus(current) {
-      const status = `${current}/${totalMessages}`;
-      Views.Initialization.setMessage(
-        window.i18n('migratingToSQLCipher', [status])
-      );
-    }
-
-    if (totalMessages) {
-      window.log.info(`About to migrate ${totalMessages} messages`);
-      showMigrationStatus(0);
-    } else {
-      window.log.info('About to migrate non-messages');
-    }
-
-    await window.Signal.migrateToSQL({
-      db,
-      clearStores: Whisper.Database.clearStores,
-      handleDOMException: Whisper.Database.handleDOMException,
-      arrayBufferToString: textsecure.MessageReceiver.arrayBufferToStringBase64,
-      countCallback: count => {
-        window.log.info(`Migration: ${count} messages complete`);
-        showMigrationStatus(count);
-      },
-      writeNewAttachmentData,
-    });
-
-    db.close();
 
     Views.Initialization.setMessage(window.i18n('optimizingApplication'));
 
-    window.log.info('Running cleanup IndexedDB migrations...');
-    // Close all previous connections to the database first
-    await Whisper.Database.close();
-
-    // Now we clean up IndexedDB database after extracting data from it
-    await Migrations1DatabaseWithoutAttachmentData.run({
-      Backbone,
-      logger: window.log,
-    });
-
-    await Whisper.Database.close();
-
-    const latestDBVersion = _.last(
-      Migrations1DatabaseWithoutAttachmentData.migrations
-    ).version;
-    Whisper.Database.migrations[0].version = latestDBVersion;
-
     window.log.info('Cleanup: starting...');
     const messagesForCleanup = await window.Signal.Data.getOutgoingWithoutExpiresAt(
       {

js/legacy_storage.js

@@ -0,0 +1,92 @@
+/* global Backbone, Whisper */
+
+/* eslint-disable more/no-then */
+
+// eslint-disable-next-line func-names
+(function() {
+  'use strict';
+
+  window.Whisper = window.Whisper || {};
+  const Item = Backbone.Model.extend({
+    database: Whisper.Database,
+    storeName: 'items',
+  });
+  const ItemCollection = Backbone.Collection.extend({
+    model: Item,
+    storeName: 'items',
+    database: Whisper.Database,
+  });
+
+  let ready = false;
+  const items = new ItemCollection();
+  items.on('reset', () => {
+    ready = true;
+  });
+  window.legacyStorage = {
+    /** ***************************
+     *** Base Storage Routines ***
+     **************************** */
+    put(key, value) {
+      if (value === undefined) {
+        throw new Error('Tried to store undefined');
+      }
+      if (!ready) {
+        window.log.warn(
+          'Called storage.put before storage is ready. key:',
+          key
+        );
+      }
+      const item = items.add({ id: key, value }, { merge: true });
+      return new Promise((resolve, reject) => {
+        item.save().then(resolve, reject);
+      });
+    },
+
+    get(key, defaultValue) {
+      const item = items.get(`${key}`);
+      if (!item) {
+        return defaultValue;
+      }
+      return item.get('value');
+    },
+
+    remove(key) {
+      const item = items.get(`${key}`);
+      if (item) {
+        items.remove(item);
+        return new Promise((resolve, reject) => {
+          item.destroy().then(resolve, reject);
+        });
+      }
+      return Promise.resolve();
+    },
+
+    onready(callback) {
+      if (ready) {
+        callback();
+      } else {
+        items.on('reset', callback);
+      }
+    },
+
+    fetch() {
+      return new Promise((resolve, reject) => {
+        items
+          .fetch({ reset: true })
+          .fail(() =>
+            reject(
+              new Error(
+                'Failed to fetch from storage.' +
+                  ' This may be due to an unexpected database version.'
+              )
+            )
+          )
+          .always(resolve);
+      });
+    },
+
+    reset() {
+      items.reset();
+    },
+  };
+})();

js/models/conversations.js

@@ -47,7 +47,6 @@
   ];
 
   Whisper.Conversation = Backbone.Model.extend({
-    database: Whisper.Database,
     storeName: 'conversations',
     defaults() {
       return {
@@ -1665,8 +1664,6 @@
   });
 
   Whisper.ConversationCollection = Backbone.Collection.extend({
-    database: Whisper.Database,
-    storeName: 'conversations',
     model: Whisper.Conversation,
 
     comparator(m) {

js/models/messages.js

@@ -65,9 +65,6 @@
   window.hasSignalAccount = number => window.AccountCache[number];
 
   window.Whisper.Message = Backbone.Model.extend({
-    // Keeping this for legacy upgrade pre-migrate to SQLCipher
-    database: Whisper.Database,
-    storeName: 'messages',
     initialize(attributes) {
       if (_.isObject(attributes)) {
         this.set(
@@ -1418,9 +1415,6 @@
 
   Whisper.MessageCollection = Backbone.Collection.extend({
     model: Whisper.Message,
-    // Keeping this for legacy upgrade pre-migrate to SQLCipher
-    database: Whisper.Database,
-    storeName: 'messages',
     comparator(left, right) {
       if (left.get('received_at') === right.get('received_at')) {
         return (left.get('sent_at') || 0) - (right.get('sent_at') || 0);

js/modules/backup.js

@@ -207,10 +207,10 @@ function exportContactsAndGroups(db, fileWriter) {
   });
 }
 
-async function importNonMessages(db, parent, options) {
+async function importNonMessages(parent, options) {
   const file = 'db.json';
   const string = await readFileAsText(parent, file);
-  return importFromJsonString(db, string, path.join(parent, file), options);
+  return importFromJsonString(string, path.join(parent, file), options);
 }
 
 function eliminateClientConfigInBackup(data, targetPath) {
@@ -265,7 +265,7 @@ async function importConversationsFromJSON(conversations, options) {
   );
 }
 
-async function importFromJsonString(db, jsonString, targetPath, options) {
+async function importFromJsonString(jsonString, targetPath, options) {
   options = options || {};
   _.defaults(options, {
     forceLightImport: false,
@@ -278,7 +278,6 @@ async function importFromJsonString(db, jsonString, targetPath, options) {
     fullImport: true,
   };
 
-  return new Promise(async (resolve, reject) => {
   const importObject = JSON.parse(jsonString);
   delete importObject.debug;
 
@@ -305,15 +304,6 @@ async function importFromJsonString(db, jsonString, targetPath, options) {
   const storeNames = _.keys(importObject);
   window.log.info('Importing to these stores:', storeNames.join(', '));
 
-    let finished = false;
-    const finish = via => {
-      window.log.info('non-messages import done via', via);
-      if (finished) {
-        resolve(result);
-      }
-      finished = true;
-    };
-
   // Special-case conversations key here, going to SQLCipher
   const { conversations } = importObject;
   const remainingStoreNames = _.without(
@@ -321,93 +311,63 @@ async function importFromJsonString(db, jsonString, targetPath, options) {
     'conversations',
     'unprocessed'
   );
-    try {
   await importConversationsFromJSON(conversations, options);
-    } catch (error) {
-      reject(error);
-    }
 
-    // Because the 'are we done?' check below looks at the keys remaining in importObject
+  const SAVE_FUNCTIONS = {
-    delete importObject.conversations;
+    groups: window.Signal.Data.createOrUpdateGroup,
-    delete importObject.unprocessed;
+    identityKeys: window.Signal.Data.createOrUpdateIdentityKey,
-
+    items: window.Signal.Data.createOrUpdateItem,
-    // The rest go to IndexedDB
+    preKeys: window.Signal.Data.createOrUpdatePreKey,
-    const transaction = db.transaction(remainingStoreNames, 'readwrite');
+    sessions: window.Signal.Data.createOrUpdateSession,
-    transaction.onerror = () => {
+    signedPreKeys: window.Signal.Data.createOrUpdateSignedPreKey,
-      Whisper.Database.handleDOMException(
-        'importFromJsonString transaction error',
-        transaction.error,
-        reject
-      );
-    };
-    transaction.oncomplete = finish.bind(null, 'transaction complete');
-
-    _.each(remainingStoreNames, storeName => {
-      const items = importObject[storeName];
-
-      window.log.info('Importing items for store', storeName);
-
-      let count = 0;
-      let skipCount = 0;
-
-      const finishStore = () => {
-        // added all objects for this store
-        delete importObject[storeName];
-        window.log.info(
-          'Done importing to store',
-          storeName,
-          'Total count:',
-          count,
-          'Skipped:',
-          skipCount
-        );
-        if (_.keys(importObject).length === 0) {
-          // added all object stores
-          window.log.info('DB import complete');
-          finish('puts scheduled');
-        }
   };
 
-      if (!items || !items.length) {
+  await Promise.all(
-        finishStore();
+    _.map(remainingStoreNames, async storeName => {
+      const save = SAVE_FUNCTIONS[storeName];
+      if (!_.isFunction(save)) {
+        throw new Error(
+          `importFromJsonString: Didn't have save function for store ${storeName}`
+        );
+      }
+
+      window.log.info(`Importing items for store ${storeName}`);
+      const toImport = importObject[storeName];
+
+      if (!toImport || !toImport.length) {
+        window.log.info(`No items in ${storeName} store`);
         return;
       }
 
-      _.each(items, toAdd => {
+      let skipCount = 0;
-        toAdd = unstringify(toAdd);
+
+      for (let i = 0, max = toImport.length; i < max; i += 1) {
+        const toAdd = unstringify(toImport[i]);
 
         const haveGroupAlready =
           storeName === 'groups' && groupLookup[getGroupKey(toAdd)];
 
         if (haveGroupAlready) {
           skipCount += 1;
-          count += 1;
+        } else {
-          return;
+          // eslint-disable-next-line no-await-in-loop
+          await save(toAdd);
+        }
       }
 
-        const request = transaction.objectStore(storeName).put(toAdd, toAdd.id);
+      window.log.info(
-        request.onsuccess = () => {
+        'Done importing to store',
-          count += 1;
+        storeName,
-          if (count + skipCount >= items.length) {
+        'Total count:',
-            finishStore();
+        toImport.length,
-          }
+        'Skipped:',
-        };
+        skipCount
-        request.onerror = () => {
+      );
-          Whisper.Database.handleDOMException(
+    })
-            `importFromJsonString request error (store: ${storeName})`,
-            request.error,
-            reject
   );
-        };
-      });
 
-      // We have to check here, because we may have skipped every item, resulting
+  window.log.info('DB import complete');
-      //   in no onsuccess callback at all.
+  return result;
-      if (skipCount === count) {
-        finishStore();
-      }
-    });
-  });
 }
 
 function createDirectory(parent, name) {
@@ -1043,11 +1003,11 @@ async function loadAttachments(dir, getName, options) {
   // TODO: Handle video screenshots, and image/video thumbnails
 }
 
-function saveMessage(db, message) {
+function saveMessage(message) {
-  return saveAllMessages(db, [message]);
+  return saveAllMessages([message]);
 }
 
-async function saveAllMessages(db, rawMessages) {
+async function saveAllMessages(rawMessages) {
   if (rawMessages.length === 0) {
     return;
   }
@@ -1085,7 +1045,7 @@ async function saveAllMessages(db, rawMessages) {
 //   message, save it, and only then do we move on to the next message. Thus, every
 //   message with attachments needs to be removed from our overall message save with the
 //   filter() call.
-async function importConversation(db, dir, options) {
+async function importConversation(dir, options) {
   options = options || {};
   _.defaults(options, { messageLookup: {} });
 
@@ -1141,7 +1101,7 @@ async function importConversation(db, dir, options) {
           message,
           key,
         });
-        return saveMessage(db, message);
+        return saveMessage(message);
       };
 
       // eslint-disable-next-line more/no-then
@@ -1153,7 +1113,7 @@ async function importConversation(db, dir, options) {
     return true;
   });
 
-  await saveAllMessages(db, messages);
+  await saveAllMessages(messages);
 
   await promiseChain;
   window.log.info(
@@ -1166,7 +1126,7 @@ async function importConversation(db, dir, options) {
   );
 }
 
-async function importConversations(db, dir, options) {
+async function importConversations(dir, options) {
   const contents = await getDirContents(dir);
   let promiseChain = Promise.resolve();
 
@@ -1175,8 +1135,7 @@ async function importConversations(db, dir, options) {
       return;
     }
 
-    const loadConversation = () =>
+    const loadConversation = () => importConversation(conversationDir, options);
-      importConversation(db, conversationDir, options);
 
     // eslint-disable-next-line more/no-then
     promiseChain = promiseChain.then(loadConversation);
@@ -1211,46 +1170,9 @@ async function loadConversationLookup() {
 function getGroupKey(group) {
   return group.id;
 }
-function loadGroupsLookup(db) {
+async function loadGroupsLookup() {
-  return assembleLookup(db, 'groups', getGroupKey);
+  const array = await window.Signal.Data.getAllGroupIds();
-}
+  return fromPairs(map(array, item => [getGroupKey(item), true]));
-
-function assembleLookup(db, storeName, keyFunction) {
-  const lookup = Object.create(null);
-
-  return new Promise((resolve, reject) => {
-    const transaction = db.transaction(storeName, 'readwrite');
-    transaction.onerror = () => {
-      Whisper.Database.handleDOMException(
-        `assembleLookup(${storeName}) transaction error`,
-        transaction.error,
-        reject
-      );
-    };
-    transaction.oncomplete = () => {
-      // not really very useful - fires at unexpected times
-    };
-
-    const store = transaction.objectStore(storeName);
-    const request = store.openCursor();
-    request.onerror = () => {
-      Whisper.Database.handleDOMException(
-        `assembleLookup(${storeName}) request error`,
-        request.error,
-        reject
-      );
-    };
-    request.onsuccess = event => {
-      const cursor = event.target.result;
-      if (cursor && cursor.value) {
-        lookup[keyFunction(cursor.value)] = true;
-        cursor.continue();
-      } else {
-        window.log.info(`Done creating ${storeName} lookup`);
-        resolve(lookup);
-      }
-    };
-  });
 }
 
 function getDirectoryForExport() {
@@ -1383,11 +1305,10 @@ async function importFromDirectory(directory, options) {
   options = options || {};
 
   try {
-    const db = await Whisper.Database.open();
     const lookups = await Promise.all([
-      loadMessagesLookup(db),
+      loadMessagesLookup(),
-      loadConversationLookup(db),
+      loadConversationLookup(),
-      loadGroupsLookup(db),
+      loadGroupsLookup(),
     ]);
     const [messageLookup, conversationLookup, groupLookup] = lookups;
     options = Object.assign({}, options, {
@@ -1422,8 +1343,8 @@ async function importFromDirectory(directory, options) {
         options = Object.assign({}, options, {
           attachmentsDir,
         });
-        const result = await importNonMessages(db, stagingDir, options);
+        const result = await importNonMessages(stagingDir, options);
-        await importConversations(db, stagingDir, Object.assign({}, options));
+        await importConversations(stagingDir, Object.assign({}, options));
 
         window.log.info('Done importing from backup!');
         return result;
@@ -1437,8 +1358,8 @@ async function importFromDirectory(directory, options) {
       }
     }
 
-    const result = await importNonMessages(db, directory, options);
+    const result = await importNonMessages(directory, options);
-    await importConversations(db, directory, options);
+    await importConversations(directory, options);
 
     window.log.info('Done importing!');
     return result;

js/modules/data.js

@@ -1,10 +1,19 @@
-/* global window, setTimeout */
+/* global window, setTimeout, IDBKeyRange */
 
 const electron = require('electron');
 
-const { forEach, isFunction, isObject, merge } = require('lodash');
+const {
+  cloneDeep,
+  forEach,
+  get,
+  isFunction,
+  isObject,
+  map,
+  merge,
+  set,
+} = require('lodash');
 
-const { deferredToPromise } = require('./deferred_to_promise');
+const { base64ToArrayBuffer, arrayBufferToBase64 } = require('./crypto');
 const MessageType = require('./types/message');
 
 const { ipcRenderer } = electron;
@@ -13,11 +22,6 @@ const { ipcRenderer } = electron;
 //   any warnings that might be sent to the console in that case.
 ipcRenderer.setMaxListeners(0);
 
-// calls to search for when finding functions to convert:
-//   .fetch(
-//   .save(
-//   .destroy(
-
 const DATABASE_UPDATE_TIMEOUT = 2 * 60 * 1000; // two minutes
 
 const SQL_CHANNEL_KEY = 'sql-channel';
@@ -38,6 +42,47 @@ module.exports = {
   close,
   removeDB,
 
+  createOrUpdateGroup,
+  getGroupById,
+  getAllGroupIds,
+  bulkAddGroups,
+  removeGroupById,
+  removeAllGroups,
+
+  createOrUpdateIdentityKey,
+  getIdentityKeyById,
+  bulkAddIdentityKeys,
+  removeIdentityKeyById,
+  removeAllIdentityKeys,
+
+  createOrUpdatePreKey,
+  getPreKeyById,
+  bulkAddPreKeys,
+  removePreKeyById,
+  removeAllPreKeys,
+
+  createOrUpdateSignedPreKey,
+  getSignedPreKeyById,
+  getAllSignedPreKeys,
+  bulkAddSignedPreKeys,
+  removeSignedPreKeyById,
+  removeAllSignedPreKeys,
+
+  createOrUpdateItem,
+  getItemById,
+  getAllItems,
+  bulkAddItems,
+  removeItemById,
+  removeAllItems,
+
+  createOrUpdateSession,
+  getSessionById,
+  getSessionsByNumber,
+  bulkAddSessions,
+  removeSessionById,
+  removeSessionsByNumber,
+  removeAllSessions,
+
   getConversationCount,
   saveConversation,
   saveConversations,
@@ -81,6 +126,8 @@ module.exports = {
   removeAllUnprocessed,
 
   removeAll,
+  removeAllConfiguration,
+
   removeOtherData,
   cleanupOrphanedAttachments,
 
@@ -229,6 +276,36 @@ forEach(module.exports, fn => {
   }
 });
 
+function keysToArrayBuffer(keys, data) {
+  const updated = cloneDeep(data);
+  for (let i = 0, max = keys.length; i < max; i += 1) {
+    const key = keys[i];
+    const value = get(data, key);
+
+    if (value) {
+      set(updated, key, base64ToArrayBuffer(value));
+    }
+  }
+
+  return updated;
+}
+
+function keysFromArrayBuffer(keys, data) {
+  const updated = cloneDeep(data);
+  for (let i = 0, max = keys.length; i < max; i += 1) {
+    const key = keys[i];
+    const value = get(data, key);
+
+    if (value) {
+      set(updated, key, arrayBufferToBase64(value));
+    }
+  }
+
+  return updated;
+}
+
+// Top-level calls
+
 // Note: will need to restart the app after calling this, to set up afresh
 async function close() {
   await channels.close();
@@ -239,6 +316,182 @@ async function removeDB() {
   await channels.removeDB();
 }
 
+// Groups
+
+async function createOrUpdateGroup(data) {
+  await channels.createOrUpdateGroup(data);
+}
+async function getGroupById(id) {
+  const group = await channels.getGroupById(id);
+  return group;
+}
+async function getAllGroupIds() {
+  const ids = await channels.getAllGroupIds();
+  return ids;
+}
+async function bulkAddGroups(array) {
+  await channels.bulkAddGroups(array);
+}
+async function removeGroupById(id) {
+  await channels.removeGroupById(id);
+}
+async function removeAllGroups() {
+  await channels.removeAllGroups();
+}
+
+// Identity Keys
+
+const IDENTITY_KEY_KEYS = ['publicKey'];
+async function createOrUpdateIdentityKey(data) {
+  const updated = keysFromArrayBuffer(IDENTITY_KEY_KEYS, data);
+  await channels.createOrUpdateIdentityKey(updated);
+}
+async function getIdentityKeyById(id) {
+  const data = await channels.getIdentityKeyById(id);
+  return keysToArrayBuffer(IDENTITY_KEY_KEYS, data);
+}
+async function bulkAddIdentityKeys(array) {
+  const updated = map(array, data =>
+    keysFromArrayBuffer(IDENTITY_KEY_KEYS, data)
+  );
+  await channels.bulkAddIdentityKeys(updated);
+}
+async function removeIdentityKeyById(id) {
+  await channels.removeIdentityKeyById(id);
+}
+async function removeAllIdentityKeys() {
+  await channels.removeAllIdentityKeys();
+}
+
+// Pre Keys
+
+async function createOrUpdatePreKey(data) {
+  const updated = keysFromArrayBuffer(PRE_KEY_KEYS, data);
+  await channels.createOrUpdatePreKey(updated);
+}
+async function getPreKeyById(id) {
+  const data = await channels.getPreKeyById(id);
+  return keysToArrayBuffer(PRE_KEY_KEYS, data);
+}
+async function bulkAddPreKeys(array) {
+  const updated = map(array, data => keysFromArrayBuffer(PRE_KEY_KEYS, data));
+  await channels.bulkAddPreKeys(updated);
+}
+async function removePreKeyById(id) {
+  await channels.removePreKeyById(id);
+}
+async function removeAllPreKeys() {
+  await channels.removeAllPreKeys();
+}
+
+// Signed Pre Keys
+
+const PRE_KEY_KEYS = ['privateKey', 'publicKey'];
+async function createOrUpdateSignedPreKey(data) {
+  const updated = keysFromArrayBuffer(PRE_KEY_KEYS, data);
+  await channels.createOrUpdateSignedPreKey(updated);
+}
+async function getSignedPreKeyById(id) {
+  const data = await channels.getSignedPreKeyById(id);
+  return keysToArrayBuffer(PRE_KEY_KEYS, data);
+}
+async function getAllSignedPreKeys() {
+  const keys = await channels.getAllSignedPreKeys();
+  return keys;
+}
+async function bulkAddSignedPreKeys(array) {
+  const updated = map(array, data => keysFromArrayBuffer(PRE_KEY_KEYS, data));
+  await channels.bulkAddSignedPreKeys(updated);
+}
+async function removeSignedPreKeyById(id) {
+  await channels.removeSignedPreKeyById(id);
+}
+async function removeAllSignedPreKeys() {
+  await channels.removeAllSignedPreKeys();
+}
+
+// Items
+
+const ITEM_KEYS = {
+  identityKey: ['value.pubKey', 'value.privKey'],
+  senderCertificate: [
+    'value.certificate',
+    'value.signature',
+    'value.serialized',
+  ],
+  signaling_key: ['value'],
+  profileKey: ['value'],
+};
+async function createOrUpdateItem(data) {
+  const { id } = data;
+  if (!id) {
+    throw new Error(
+      'createOrUpdateItem: Provided data did not have a truthy id'
+    );
+  }
+
+  const keys = ITEM_KEYS[id];
+  const updated = Array.isArray(keys) ? keysFromArrayBuffer(keys, data) : data;
+
+  await channels.createOrUpdateItem(updated);
+}
+async function getItemById(id) {
+  const keys = ITEM_KEYS[id];
+  const data = await channels.getItemById(id);
+
+  return Array.isArray(keys) ? keysToArrayBuffer(keys, data) : data;
+}
+async function getAllItems() {
+  const items = await channels.getAllItems();
+  return map(items, item => {
+    const { id } = item;
+    const keys = ITEM_KEYS[id];
+    return Array.isArray(keys) ? keysToArrayBuffer(keys, item) : item;
+  });
+}
+async function bulkAddItems(array) {
+  const updated = map(array, data => {
+    const { id } = data;
+    const keys = ITEM_KEYS[id];
+    return Array.isArray(keys) ? keysFromArrayBuffer(keys, data) : data;
+  });
+  await channels.bulkAddItems(updated);
+}
+async function removeItemById(id) {
+  await channels.removeItemById(id);
+}
+async function removeAllItems() {
+  await channels.removeAllItems();
+}
+
+// Sessions
+
+async function createOrUpdateSession(data) {
+  await channels.createOrUpdateSession(data);
+}
+async function getSessionById(id) {
+  const session = await channels.getSessionById(id);
+  return session;
+}
+async function getSessionsByNumber(number) {
+  const sessions = await channels.getSessionsByNumber(number);
+  return sessions;
+}
+async function bulkAddSessions(array) {
+  await channels.bulkAddSessions(array);
+}
+async function removeSessionById(id) {
+  await channels.removeSessionById(id);
+}
+async function removeSessionsByNumber(number) {
+  await channels.removeSessionsByNumber(number);
+}
+async function removeAllSessions(id) {
+  await channels.removeAllSessions(id);
+}
+
+// Conversation
+
 async function getConversationCount() {
   return channels.getConversationCount();
 }
@@ -319,6 +572,8 @@ async function searchConversations(query, { ConversationCollection }) {
   return collection;
 }
 
+// Message
+
 async function getMessageCount() {
   return channels.getMessageCount();
 }
@@ -329,10 +584,41 @@ async function saveMessage(data, { forceSave, Message } = {}) {
   return id;
 }
 
-async function saveLegacyMessage(data, { Message }) {
+async function saveLegacyMessage(data) {
-  const message = new Message(data);
+  const db = await window.Whisper.Database.open();
-  await deferredToPromise(message.save());
+  try {
-  return message.id;
+    await new Promise((resolve, reject) => {
+      const transaction = db.transaction('messages', 'readwrite');
+
+      transaction.onerror = () => {
+        window.Whisper.Database.handleDOMException(
+          'saveLegacyMessage transaction error',
+          transaction.error,
+          reject
+        );
+      };
+      transaction.oncomplete = resolve;
+
+      const store = transaction.objectStore('messages');
+
+      if (!data.id) {
+        // eslint-disable-next-line no-param-reassign
+        data.id = window.getGuid();
+      }
+
+      const request = store.put(data, data.id);
+      request.onsuccess = resolve;
+      request.onerror = () => {
+        window.Whisper.Database.handleDOMException(
+          'saveLegacyMessage request error',
+          request.error,
+          reject
+        );
+      };
+    });
+  } finally {
+    db.close();
+  }
 }
 
 async function saveMessages(arrayOfMessages, { forceSave } = {}) {
@@ -459,6 +745,8 @@ async function getNextExpiringMessage({ MessageCollection }) {
   return new MessageCollection(messages);
 }
 
+// Unprocessed
+
 async function getUnprocessedCount() {
   return channels.getUnprocessedCount();
 }
@@ -495,10 +783,16 @@ async function removeAllUnprocessed() {
   await channels.removeAllUnprocessed();
 }
 
+// Other
+
 async function removeAll() {
   await channels.removeAll();
 }
 
+async function removeAllConfiguration() {
+  await channels.removeAllConfiguration();
+}
+
 async function cleanupOrphanedAttachments() {
   await callChannel(CLEANUP_ORPHANED_ATTACHMENTS_KEY);
 }
@@ -529,28 +823,61 @@ async function callChannel(name) {
   });
 }
 
-// Functions below here return JSON
+// Functions below here return plain JSON instead of Backbone Models
 
 async function getLegacyMessagesNeedingUpgrade(
   limit,
-  { MessageCollection, maxVersion = MessageType.CURRENT_SCHEMA_VERSION }
+  { maxVersion = MessageType.CURRENT_SCHEMA_VERSION }
 ) {
-  const messages = new MessageCollection();
+  const db = await window.Whisper.Database.open();
+  try {
+    await new Promise((resolve, reject) => {
+      const transaction = db.transaction('messages', 'readonly');
+      const messages = [];
 
-  await deferredToPromise(
+      transaction.onerror = () => {
-    messages.fetch({
+        window.Whisper.Database.handleDOMException(
-      limit,
+          'getLegacyMessagesNeedingUpgrade transaction error',
-      index: {
+          transaction.error,
-        name: 'schemaVersion',
+          reject
-        upper: maxVersion,
-        excludeUpper: true,
-        order: 'desc',
-      },
-    })
         );
+      };
+      transaction.oncomplete = () => {
+        resolve(messages);
+      };
 
-  const models = messages.models || [];
+      const store = transaction.objectStore('messages');
-  return models.map(model => model.toJSON());
+      const index = store.index('schemaVersion');
+      const range = IDBKeyRange.upperBound(maxVersion, true);
+
+      const request = index.openCursor(range);
+      let count = 0;
+
+      request.onsuccess = event => {
+        const cursor = event.target.result;
+
+        if (cursor) {
+          count += 1;
+          messages.push(cursor.value);
+
+          if (count >= limit) {
+            return;
+          }
+
+          cursor.continue();
+        }
+      };
+      request.onerror = () => {
+        window.Whisper.Database.handleDOMException(
+          'getLegacyMessagesNeedingUpgrade request error',
+          request.error,
+          reject
+        );
+      };
+    });
+  } finally {
+    db.close();
+  }
 }
 
 async function getMessagesNeedingUpgrade(

js/modules/indexeddb.js

@@ -0,0 +1,168 @@
+/* global window, Whisper, textsecure */
+
+const { isFunction } = require('lodash');
+
+const MessageDataMigrator = require('./messages_data_migrator');
+const {
+  run,
+  getLatestVersion,
+  getDatabase,
+} = require('./migrations/migrations');
+
+const MESSAGE_MINIMUM_VERSION = 7;
+
+module.exports = {
+  doesDatabaseExist,
+  mandatoryMessageUpgrade,
+  MESSAGE_MINIMUM_VERSION,
+  migrateAllToSQLCipher,
+  removeDatabase,
+  runMigrations,
+};
+
+async function runMigrations() {
+  window.log.info('Run migrations on database with attachment data');
+  await run({
+    Backbone: window.Backbone,
+    logger: window.log,
+  });
+
+  Whisper.Database.migrations[0].version = getLatestVersion();
+}
+
+async function mandatoryMessageUpgrade({ upgradeMessageSchema } = {}) {
+  if (!isFunction(upgradeMessageSchema)) {
+    throw new Error(
+      'mandatoryMessageUpgrade: upgradeMessageSchema must be a function!'
+    );
+  }
+
+  const NUM_MESSAGES_PER_BATCH = 10;
+  window.log.info(
+    'upgradeMessages: Mandatory message schema upgrade started.',
+    `Target version: ${MESSAGE_MINIMUM_VERSION}`
+  );
+
+  let isMigrationWithoutIndexComplete = false;
+  while (!isMigrationWithoutIndexComplete) {
+    const database = getDatabase();
+    // eslint-disable-next-line no-await-in-loop
+    const batchWithoutIndex = await MessageDataMigrator.processNextBatchWithoutIndex(
+      {
+        databaseName: database.name,
+        minDatabaseVersion: database.version,
+        numMessagesPerBatch: NUM_MESSAGES_PER_BATCH,
+        upgradeMessageSchema,
+        maxVersion: MESSAGE_MINIMUM_VERSION,
+        BackboneMessage: Whisper.Message,
+        saveMessage: window.Signal.Data.saveLegacyMessage,
+      }
+    );
+    window.log.info(
+      'upgradeMessages: upgrade without index',
+      batchWithoutIndex
+    );
+    isMigrationWithoutIndexComplete = batchWithoutIndex.done;
+  }
+  window.log.info('upgradeMessages: upgrade without index complete!');
+
+  let isMigrationWithIndexComplete = false;
+  while (!isMigrationWithIndexComplete) {
+    // eslint-disable-next-line no-await-in-loop
+    const batchWithIndex = await MessageDataMigrator.processNext({
+      BackboneMessage: Whisper.Message,
+      BackboneMessageCollection: Whisper.MessageCollection,
+      numMessagesPerBatch: NUM_MESSAGES_PER_BATCH,
+      upgradeMessageSchema,
+      getMessagesNeedingUpgrade:
+        window.Signal.Data.getLegacyMessagesNeedingUpgrade,
+      saveMessage: window.Signal.Data.saveLegacyMessage,
+      maxVersion: MESSAGE_MINIMUM_VERSION,
+    });
+    window.log.info('upgradeMessages: upgrade with index', batchWithIndex);
+    isMigrationWithIndexComplete = batchWithIndex.done;
+  }
+  window.log.info('upgradeMessages: upgrade with index complete!');
+
+  window.log.info('upgradeMessages: Message schema upgrade complete');
+}
+
+async function migrateAllToSQLCipher({ writeNewAttachmentData, Views } = {}) {
+  if (!isFunction(writeNewAttachmentData)) {
+    throw new Error(
+      'migrateAllToSQLCipher: writeNewAttachmentData must be a function'
+    );
+  }
+  if (!Views) {
+    throw new Error('migrateAllToSQLCipher: Views must be provided!');
+  }
+
+  let totalMessages;
+  const db = await Whisper.Database.open();
+
+  function showMigrationStatus(current) {
+    const status = `${current}/${totalMessages}`;
+    Views.Initialization.setMessage(
+      window.i18n('migratingToSQLCipher', [status])
+    );
+  }
+
+  try {
+    totalMessages = await MessageDataMigrator.getNumMessages({
+      connection: db,
+    });
+  } catch (error) {
+    window.log.error(
+      'background.getNumMessages error:',
+      error && error.stack ? error.stack : error
+    );
+    totalMessages = 0;
+  }
+
+  if (totalMessages) {
+    window.log.info(`About to migrate ${totalMessages} messages`);
+    showMigrationStatus(0);
+  } else {
+    window.log.info('About to migrate non-messages');
+  }
+
+  await window.Signal.migrateToSQL({
+    db,
+    clearStores: Whisper.Database.clearStores,
+    handleDOMException: Whisper.Database.handleDOMException,
+    arrayBufferToString: textsecure.MessageReceiver.arrayBufferToStringBase64,
+    countCallback: count => {
+      window.log.info(`Migration: ${count} messages complete`);
+      showMigrationStatus(count);
+    },
+    writeNewAttachmentData,
+  });
+
+  db.close();
+}
+
+async function doesDatabaseExist() {
+  return new Promise((resolve, reject) => {
+    const { id } = Whisper.Database;
+    const req = window.indexedDB.open(id);
+
+    let existed = true;
+
+    req.onerror = reject;
+    req.onsuccess = () => {
+      req.result.close();
+      resolve(existed);
+    };
+    req.onupgradeneeded = () => {
+      if (req.result.version === 1) {
+        existed = false;
+        window.indexedDB.deleteDatabase(id);
+      }
+    };
+  });
+}
+
+function removeDatabase() {
+  window.log.info(`Deleting IndexedDB database '${Whisper.Database.id}'`);
+  window.indexedDB.deleteDatabase(Whisper.Database.id);
+}

js/modules/migrate_to_sql.js

@@ -2,10 +2,26 @@
 
 const { includes, isFunction, isString, last, map } = require('lodash');
 const {
+  bulkAddGroups,
+  bulkAddSessions,
+  bulkAddIdentityKeys,
+  bulkAddPreKeys,
+  bulkAddSignedPreKeys,
+  bulkAddItems,
+
+  removeGroupById,
+  removeSessionById,
+  removeIdentityKeyById,
+  removePreKeyById,
+  removeSignedPreKeyById,
+  removeItemById,
+
   saveMessages,
   _removeMessages,
+
   saveUnprocesseds,
   removeUnprocessed,
+
   saveConversations,
   _removeConversations,
 } = require('./data');
@@ -132,6 +148,8 @@ async function migrateToSQL({
   }
 
   complete = false;
+  lastIndex = null;
+
   while (!complete) {
     // eslint-disable-next-line no-await-in-loop
     const status = await migrateStoreToSQLite({
@@ -163,6 +181,153 @@ async function migrateToSQL({
     window.log.warn('Failed to clear conversations store');
   }
 
+  complete = false;
+  lastIndex = null;
+
+  while (!complete) {
+    // eslint-disable-next-line no-await-in-loop
+    const status = await migrateStoreToSQLite({
+      db,
+      // eslint-disable-next-line no-loop-func
+      save: bulkAddGroups,
+      remove: removeGroupById,
+      storeName: 'groups',
+      handleDOMException,
+      lastIndex,
+      batchSize: 10,
+    });
+
+    ({ complete, lastIndex } = status);
+  }
+  window.log.info('migrateToSQL: migrate of groups complete');
+  try {
+    await clearStores(['groups']);
+  } catch (error) {
+    window.log.warn('Failed to clear groups store');
+  }
+
+  complete = false;
+  lastIndex = null;
+
+  while (!complete) {
+    // eslint-disable-next-line no-await-in-loop
+    const status = await migrateStoreToSQLite({
+      db,
+      // eslint-disable-next-line no-loop-func
+      save: bulkAddSessions,
+      remove: removeSessionById,
+      storeName: 'sessions',
+      handleDOMException,
+      lastIndex,
+      batchSize: 10,
+    });
+
+    ({ complete, lastIndex } = status);
+  }
+  window.log.info('migrateToSQL: migrate of sessions complete');
+  try {
+    await clearStores(['sessions']);
+  } catch (error) {
+    window.log.warn('Failed to clear sessions store');
+  }
+
+  complete = false;
+  lastIndex = null;
+
+  while (!complete) {
+    // eslint-disable-next-line no-await-in-loop
+    const status = await migrateStoreToSQLite({
+      db,
+      // eslint-disable-next-line no-loop-func
+      save: bulkAddIdentityKeys,
+      remove: removeIdentityKeyById,
+      storeName: 'identityKeys',
+      handleDOMException,
+      lastIndex,
+      batchSize: 10,
+    });
+
+    ({ complete, lastIndex } = status);
+  }
+  window.log.info('migrateToSQL: migrate of identityKeys complete');
+  try {
+    await clearStores(['identityKeys']);
+  } catch (error) {
+    window.log.warn('Failed to clear identityKeys store');
+  }
+
+  complete = false;
+  lastIndex = null;
+
+  while (!complete) {
+    // eslint-disable-next-line no-await-in-loop
+    const status = await migrateStoreToSQLite({
+      db,
+      // eslint-disable-next-line no-loop-func
+      save: bulkAddPreKeys,
+      remove: removePreKeyById,
+      storeName: 'preKeys',
+      handleDOMException,
+      lastIndex,
+      batchSize: 10,
+    });
+
+    ({ complete, lastIndex } = status);
+  }
+  window.log.info('migrateToSQL: migrate of preKeys complete');
+  try {
+    await clearStores(['preKeys']);
+  } catch (error) {
+    window.log.warn('Failed to clear preKeys store');
+  }
+
+  complete = false;
+  lastIndex = null;
+
+  while (!complete) {
+    // eslint-disable-next-line no-await-in-loop
+    const status = await migrateStoreToSQLite({
+      db,
+      // eslint-disable-next-line no-loop-func
+      save: bulkAddSignedPreKeys,
+      remove: removeSignedPreKeyById,
+      storeName: 'signedPreKeys',
+      handleDOMException,
+      lastIndex,
+      batchSize: 10,
+    });
+
+    ({ complete, lastIndex } = status);
+  }
+  window.log.info('migrateToSQL: migrate of signedPreKeys complete');
+  try {
+    await clearStores(['signedPreKeys']);
+  } catch (error) {
+    window.log.warn('Failed to clear signedPreKeys store');
+  }
+
+  complete = false;
+  lastIndex = null;
+
+  while (!complete) {
+    // eslint-disable-next-line no-await-in-loop
+    const status = await migrateStoreToSQLite({
+      db,
+      // eslint-disable-next-line no-loop-func
+      save: bulkAddItems,
+      remove: removeItemById,
+      storeName: 'items',
+      handleDOMException,
+      lastIndex,
+      batchSize: 10,
+    });
+
+    ({ complete, lastIndex } = status);
+  }
+  window.log.info('migrateToSQL: migrate of items complete');
+  // Note: we don't clear the items store because it contains important metadata which,
+  //   if this process fails, will be crucial to going through this process again.
+
   window.log.info('migrateToSQL: complete');
 }
 

js/modules/migrations/get_placeholder_migrations.js

@@ -1,13 +1,13 @@
 /* global window, Whisper */
 
-const Migrations0DatabaseWithAttachmentData = require('./migrations_0_database_with_attachment_data');
+const Migrations = require('./migrations');
 
 exports.getPlaceholderMigrations = () => {
-  const last0MigrationVersion = Migrations0DatabaseWithAttachmentData.getLatestVersion();
+  const version = Migrations.getLatestVersion();
 
   return [
     {
-      version: last0MigrationVersion,
+      version,
       migrate() {
         throw new Error(
           'Unexpected invocation of placeholder migration!' +

js/modules/migrations/migrations.js

@@ -170,8 +170,19 @@ const migrations = [
     migrate(transaction, next) {
       window.log.info('Migration 19');
 
+      // Empty because we don't want to cause incompatibility with beta users who have
+      //   already run migration 19 when it was object store removal.
+
+      next();
+    },
+  },
+  {
+    version: 20,
+    migrate(transaction, next) {
+      window.log.info('Migration 20');
+
       // Empty because we don't want to cause incompatibility with users who have already
-      //   run migration 19 when it was the object store removal.
+      //   run migration 20 when it was object store removal.
 
       next();
     },

js/modules/migrations/migrations_1_database_without_attachment_data.js

@@ -1,84 +0,0 @@
-/* global window */
-
-const { last, includes } = require('lodash');
-
-const { open } = require('../database');
-const settings = require('../settings');
-const { runMigrations } = require('./run_migrations');
-
-// These are cleanup migrations, to be run after migration to SQLCipher
-exports.migrations = [
-  {
-    version: 20,
-    migrate(transaction, next) {
-      window.log.info('Migration 20');
-
-      const { db } = transaction;
-
-      // This should be run after things are migrated to SQLCipher
-
-      // We check for existence first, because this removal was present in v1.17.0.beta.1,
-      //  but reverted in v1.17.0-beta.3
-
-      if (includes(db.objectStoreNames, 'messages')) {
-        window.log.info('Removing messages store');
-        db.deleteObjectStore('messages');
-      }
-      if (includes(db.objectStoreNames, 'unprocessed')) {
-        window.log.info('Removing unprocessed store');
-        db.deleteObjectStore('unprocessed');
-      }
-      if (includes(db.objectStoreNames, 'conversations')) {
-        window.log.info('Removing conversations store');
-        db.deleteObjectStore('conversations');
-      }
-
-      next();
-    },
-  },
-];
-
-exports.run = async ({ Backbone, logger } = {}) => {
-  const database = {
-    id: 'signal',
-    nolog: true,
-    migrations: exports.migrations,
-  };
-
-  const { canRun } = await exports.getStatus({ database });
-  if (!canRun) {
-    throw new Error(
-      'Cannot run migrations on database without attachment data'
-    );
-  }
-
-  await runMigrations({
-    Backbone,
-    logger,
-    database,
-  });
-};
-
-exports.getStatus = async ({ database } = {}) => {
-  const connection = await open(database.id, database.version);
-  const isAttachmentMigrationComplete = await settings.isAttachmentMigrationComplete(
-    connection
-  );
-  const hasMigrations = exports.migrations.length > 0;
-
-  const canRun = isAttachmentMigrationComplete && hasMigrations;
-  return {
-    isAttachmentMigrationComplete,
-    hasMigrations,
-    canRun,
-  };
-};
-
-exports.getLatestVersion = () => {
-  const lastMigration = last(exports.migrations);
-  if (!lastMigration) {
-    return null;
-  }
-
-  return lastMigration.version;
-};

js/modules/migrations/run_migrations.js

@@ -52,7 +52,10 @@ exports.runMigrations = async ({ Backbone, database, logger } = {}) => {
     storeName: 'items',
   }))();
 
+  // Note: this legacy migration technique is required to bring old clients with
+  //   data in IndexedDB forward into the new world of SQLCipher only.
   await deferredToPromise(migrationCollection.fetch({ limit: 1 }));
+
   logger.info('Close database connection');
   await closeDatabaseConnection({ Backbone });
 };

js/modules/signal.js

@@ -5,6 +5,7 @@ const Crypto = require('./crypto');
 const Data = require('./data');
 const Database = require('./database');
 const Emoji = require('../../ts/util/emoji');
+const IndexedDB = require('./indexeddb');
 const Notifications = require('../../ts/notifications');
 const OS = require('../../ts/OS');
 const Settings = require('./settings');
@@ -63,9 +64,7 @@ const {
   getPlaceholderMigrations,
   getCurrentVersion,
 } = require('./migrations/get_placeholder_migrations');
-
+const { run } = require('./migrations/migrations');
-const Migrations0DatabaseWithAttachmentData = require('./migrations/migrations_0_database_with_attachment_data');
-const Migrations1DatabaseWithoutAttachmentData = require('./migrations/migrations_1_database_without_attachment_data');
 
 // Types
 const AttachmentType = require('./types/attachment');
@@ -132,8 +131,7 @@ function initializeMigrations({
     loadAttachmentData,
     loadQuoteData,
     loadMessage: MessageType.createAttachmentLoader(loadAttachmentData),
-    Migrations0DatabaseWithAttachmentData,
+    run,
-    Migrations1DatabaseWithoutAttachmentData,
     upgradeMessageSchema: (message, options = {}) => {
       const { maxVersion } = options;
 
@@ -225,6 +223,7 @@ exports.setup = (options = {}) => {
     Data,
     Database,
     Emoji,
+    IndexedDB,
     Migrations,
     Notifications,
     OS,

js/storage.js

@@ -1,5 +1,3 @@
-/* global Backbone, Whisper */
-
 /* eslint-disable more/no-then */
 
 // eslint-disable-next-line func-names
@@ -7,89 +5,103 @@
   'use strict';
 
   window.Whisper = window.Whisper || {};
-  const Item = Backbone.Model.extend({
-    database: Whisper.Database,
-    storeName: 'items',
-  });
-  const ItemCollection = Backbone.Collection.extend({
-    model: Item,
-    storeName: 'items',
-    database: Whisper.Database,
-  });
 
   let ready = false;
-  const items = new ItemCollection();
+  let items;
-  items.on('reset', () => {
+  let callbacks = [];
-    ready = true;
+
-  });
+  reset();
-  window.storage = {
+
-    /** ***************************
+  async function put(key, value) {
-     *** Base Storage Routines ***
-     **************************** */
-    put(key, value) {
     if (value === undefined) {
       throw new Error('Tried to store undefined');
     }
     if (!ready) {
-        window.log.warn(
+      window.log.warn('Called storage.put before storage is ready. key:', key);
-          'Called storage.put before storage is ready. key:',
-          key
-        );
     }
-      const item = items.add({ id: key, value }, { merge: true });
-      return new Promise((resolve, reject) => {
-        item.save().then(resolve, reject);
-      });
-    },
 
-    get(key, defaultValue) {
+    const data = { id: key, value };
-      const item = items.get(`${key}`);
+
+    items[key] = data;
+    await window.Signal.Data.createOrUpdateItem(data);
+  }
+
+  function get(key, defaultValue) {
+    if (!ready) {
+      window.log.warn('Called storage.get before storage is ready. key:', key);
+    }
+
+    const item = items[key];
     if (!item) {
       return defaultValue;
     }
-      return item.get('value');
-    },
 
-    remove(key) {
+    return item.value;
-      const item = items.get(`${key}`);
-      if (item) {
-        items.remove(item);
-        return new Promise((resolve, reject) => {
-          item.destroy().then(resolve, reject);
-        });
   }
-      return Promise.resolve();
-    },
 
-    onready(callback) {
+  async function remove(key) {
+    if (!ready) {
+      window.log.warn('Called storage.get before storage is ready. key:', key);
+    }
+
+    delete items[key];
+    await window.Signal.Data.removeItemById(key);
+  }
+
+  function onready(callback) {
     if (ready) {
       callback();
     } else {
-        items.on('reset', callback);
+      callbacks.push(callback);
+    }
   }
-    },
 
-    fetch() {
+  function callListeners() {
-      return new Promise((resolve, reject) => {
+    if (ready) {
-        items
+      callbacks.forEach(callback => callback());
-          .fetch({ reset: true })
+      callbacks = [];
-          .fail(() =>
+    }
-            reject(
+  }
-              new Error(
-                'Failed to fetch from storage.' +
-                  ' This may be due to an unexpected database version.'
-              )
-            )
-          )
-          .always(resolve);
-      });
-    },
 
-    reset() {
+  async function fetch() {
-      items.reset();
+    this.reset();
-    },
+    const array = await window.Signal.Data.getAllItems();
+
+    for (let i = 0, max = array.length; i < max; i += 1) {
+      const item = array[i];
+      const { id } = item;
+      items[id] = item;
+    }
+
+    ready = true;
+    callListeners();
+  }
+
+  function reset() {
+    ready = false;
+    items = Object.create(null);
+  }
+
+  const storage = {
+    fetch,
+    put,
+    get,
+    remove,
+    onready,
+    reset,
   };
+
+  // Keep a reference to this storage system, since there are scenarios where
+  //   we need to replace it with the legacy storage system for a while.
+  window.newStorage = storage;
+
   window.textsecure = window.textsecure || {};
   window.textsecure.storage = window.textsecure.storage || {};
-  window.textsecure.storage.impl = window.storage;
+
+  window.installStorage = newStorage => {
+    window.storage = newStorage;
+    window.textsecure.storage.impl = newStorage;
+  };
+
+  window.installStorage(storage);
 })();

js/views/clear_data_view.js

@@ -8,7 +8,6 @@
   'use strict';
 
   window.Whisper = window.Whisper || {};
-  const { Database } = window.Whisper;
   const { Logs } = window.Signal;
 
   const CLEAR_DATA_STEPS = {
@@ -33,26 +32,12 @@
       this.step = CLEAR_DATA_STEPS.DELETING;
       this.render();
 
-      try {
+      await this.clearAllData();
-        await Database.clear();
-        await Database.close();
-        window.log.info(
-          'All database connections closed. Starting database drop.'
-        );
-        await Database.drop();
-      } catch (error) {
-        window.log.error(
-          'Something went wrong deleting IndexedDB data then dropping database.'
-        );
-      }
-
-      this.clearAllData();
     },
     async clearAllData() {
       try {
         await Logs.deleteAll();
 
-        // SQLCipher
         await window.Signal.Data.removeAll();
         await window.Signal.Data.close();
         await window.Signal.Data.removeDB();

js/views/import_view.js

@@ -38,10 +38,7 @@
       return storage.put(IMPORT_LOCATION, location);
     },
     reset() {
-      return Promise.all([
+      return window.Signal.Data.removeAll();
-        Whisper.Database.clear(),
-        window.Signal.Data.removeAll(),
-      ]);
     },
   };
 

js/views/recipients_input_view.js

@@ -13,8 +13,6 @@
       'national_number',
       'international_number',
     ],
-    database: Whisper.Database,
-    storeName: 'conversations',
     model: Whisper.Conversation,
     async fetchContacts() {
       const models = window.Signal.Data.getAllPrivateConversations({

libtextsecure/storage/groups.js

@@ -73,7 +73,9 @@
 
     getNumbers(groupId) {
       return textsecure.storage.protocol.getGroup(groupId).then(group => {
-        if (group === undefined) return undefined;
+        if (!group) {
+          return undefined;
+        }
 
         return group.numbers;
       });

test/_test.js

@@ -72,16 +72,8 @@ function deleteDatabase() {
 before(async () => {
   await deleteDatabase();
   await window.Signal.Data.removeAll();
-
-  await Signal.Migrations.Migrations0DatabaseWithAttachmentData.run({
-    Backbone,
-    databaseName: Whisper.Database.id,
-    logger: window.log,
-  });
 });
 
 async function clearDatabase() {
-  const db = await Whisper.Database.open();
-  await Whisper.Database.clear();
   await window.Signal.Data.removeAll();
 }

ts/util/lint/exceptions.json

@@ -164,7 +164,7 @@
     "rule": "jQuery-$(",
     "path": "js/background.js",
     "line": "        if ($('.dark-overlay').length) {",
-    "lineNumber": 265,
+    "lineNumber": 264,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T21:59:32.770Z",
     "reasonDetail": "Protected from arbitrary input"
@@ -173,7 +173,7 @@
     "rule": "jQuery-$(",
     "path": "js/background.js",
     "line": "        $(document.body).prepend('<div class=\"dark-overlay\"></div>');",
-    "lineNumber": 268,
+    "lineNumber": 267,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T21:59:32.770Z",
     "reasonDetail": "Protected from arbitrary input"
@@ -182,7 +182,7 @@
     "rule": "jQuery-prepend(",
     "path": "js/background.js",
     "line": "        $(document.body).prepend('<div class=\"dark-overlay\"></div>');",
-    "lineNumber": 268,
+    "lineNumber": 267,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T18:13:29.628Z",
     "reasonDetail": "Hard-coded value"
@@ -191,7 +191,7 @@
     "rule": "jQuery-$(",
     "path": "js/background.js",
     "line": "        $('.dark-overlay').on('click', () => $('.dark-overlay').remove());",
-    "lineNumber": 269,
+    "lineNumber": 268,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T21:59:32.770Z",
     "reasonDetail": "Protected from arbitrary input"
@@ -200,7 +200,7 @@
     "rule": "jQuery-$(",
     "path": "js/background.js",
     "line": "      removeDarkOverlay: () => $('.dark-overlay').remove(),",
-    "lineNumber": 271,
+    "lineNumber": 270,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T21:59:32.770Z",
     "reasonDetail": "Protected from arbitrary input"
@@ -209,7 +209,7 @@
     "rule": "jQuery-$(",
     "path": "js/background.js",
     "line": "        $('body').append(clearDataView.el);",
-    "lineNumber": 274,
+    "lineNumber": 273,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T21:59:32.770Z",
     "reasonDetail": "Protected from arbitrary input"
@@ -218,7 +218,7 @@
     "rule": "jQuery-append(",
     "path": "js/background.js",
     "line": "        $('body').append(clearDataView.el);",
-    "lineNumber": 274,
+    "lineNumber": 273,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T18:13:29.628Z",
     "reasonDetail": "Interacting with already-existing DOM nodes"
@@ -227,7 +227,7 @@
     "rule": "jQuery-load(",
     "path": "js/background.js",
     "line": "      await ConversationController.load();",
-    "lineNumber": 509,
+    "lineNumber": 404,
     "reasonCategory": "falseMatch",
     "updated": "2018-10-02T21:00:44.007Z"
   },
@@ -235,7 +235,7 @@
     "rule": "jQuery-$(",
     "path": "js/background.js",
     "line": "      el: $('body'),",
-    "lineNumber": 572,
+    "lineNumber": 467,
     "reasonCategory": "usageTrusted",
     "updated": "2018-10-16T23:47:48.006Z",
     "reasonDetail": "Protected from arbitrary input"
@@ -244,7 +244,7 @@
     "rule": "jQuery-wrap(",
     "path": "js/background.js",
     "line": "        wrap(",
-    "lineNumber": 830,
+    "lineNumber": 725,
     "reasonCategory": "falseMatch",
     "updated": "2018-10-18T22:23:00.485Z"
   },
@@ -252,7 +252,7 @@
     "rule": "jQuery-wrap(",
     "path": "js/background.js",
     "line": "      await wrap(",
-    "lineNumber": 1320,
+    "lineNumber": 1215,
     "reasonCategory": "falseMatch",
     "updated": "2018-10-26T22:43:23.229Z"
   },
@@ -303,7 +303,7 @@
     "rule": "jQuery-wrap(",
     "path": "js/models/messages.js",
     "line": "        this.send(wrap(promise));",
-    "lineNumber": 794,
+    "lineNumber": 791,
     "reasonCategory": "falseMatch",
     "updated": "2018-10-05T23:12:28.961Z"
   },
@@ -311,7 +311,7 @@
     "rule": "jQuery-wrap(",
     "path": "js/models/messages.js",
     "line": "        return wrap(",
-    "lineNumber": 996,
+    "lineNumber": 993,
     "reasonCategory": "falseMatch",
     "updated": "2018-10-05T23:12:28.961Z"
   },
@@ -445,7 +445,7 @@
     "rule": "jQuery-load(",
     "path": "js/signal_protocol_store.js",
     "line": "      await ConversationController.load();",
-    "lineNumber": 972,
+    "lineNumber": 848,
     "reasonCategory": "falseMatch",
     "updated": "2018-09-15T00:38:04.183Z"
   },
@@ -1315,7 +1315,7 @@
     "rule": "jQuery-load(",
     "path": "js/views/import_view.js",
     "line": "      return ConversationController.load()",
-    "lineNumber": 179,
+    "lineNumber": 176,
     "reasonCategory": "falseMatch",
     "updated": "2018-09-15T00:38:04.183Z"
   },
@@ -1844,7 +1844,7 @@
     "rule": "jQuery-$(",
     "path": "js/views/recipients_input_view.js",
     "line": "      this.$input = this.$('input.search');",
-    "lineNumber": 71,
+    "lineNumber": 69,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T21:59:32.770Z",
     "reasonDetail": "Protected from arbitrary input"
@@ -1853,7 +1853,7 @@
     "rule": "jQuery-$(",
     "path": "js/views/recipients_input_view.js",
     "line": "      this.$new_contact = this.$('.new-contact');",
-    "lineNumber": 72,
+    "lineNumber": 70,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T21:59:32.770Z",
     "reasonDetail": "Protected from arbitrary input"
@@ -1862,7 +1862,7 @@
     "rule": "jQuery-$(",
     "path": "js/views/recipients_input_view.js",
     "line": "        el: this.$('.recipients'),",
-    "lineNumber": 82,
+    "lineNumber": 80,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T21:59:32.770Z",
     "reasonDetail": "Protected from arbitrary input"
@@ -1871,7 +1871,7 @@
     "rule": "jQuery-$(",
     "path": "js/views/recipients_input_view.js",
     "line": "      this.$('.contacts').append(this.typeahead_view.el);",
-    "lineNumber": 97,
+    "lineNumber": 95,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T21:59:32.770Z",
     "reasonDetail": "Protected from arbitrary input"
@@ -1880,7 +1880,7 @@
     "rule": "jQuery-append(",
     "path": "js/views/recipients_input_view.js",
     "line": "      this.$('.contacts').append(this.typeahead_view.el);",
-    "lineNumber": 97,
+    "lineNumber": 95,
     "reasonCategory": "usageTrusted",
     "updated": "2018-09-19T18:13:29.628Z",
     "reasonDetail": "Interacting with already-existing DOM nodes"