Split database migrations into pre- and post-attachment migration

- Run light-weight migrations before attachment migration.
- Run regular migrations after attachments have been moved to disk.

js/modules/migrations/17/index.js

@@ -1,55 +0,0 @@
-const Message = require('../../types/message');
-
-
-exports.run = async (transaction) => {
-  const messagesStore = transaction.objectStore('messages');
-
-  console.log('Initialize messages schema version');
-  const numUpgradedMessages = await _initializeMessageSchemaVersion(messagesStore);
-  console.log('Complete messages schema version initialization', { numUpgradedMessages });
-
-  console.log('Create index from attachment schema version to attachment');
-  messagesStore.createIndex('schemaVersion', 'schemaVersion', { unique: false });
-};
-
-const _initializeMessageSchemaVersion = messagesStore =>
-  new Promise((resolve, reject) => {
-    const messagePutOperations = [];
-
-    const cursorRequest = messagesStore.openCursor();
-    cursorRequest.onsuccess = async (event) => {
-      const cursor = event.target.result;
-      const hasMoreData = Boolean(cursor);
-      if (!hasMoreData) {
-        await Promise.all(messagePutOperations);
-        return resolve(messagePutOperations.length);
-      }
-
-      const message = cursor.value;
-      const messageWithSchemaVersion = Message.initializeSchemaVersion(message);
-      messagePutOperations.push(putItem(
-        messagesStore,
-        messageWithSchemaVersion,
-        messageWithSchemaVersion.id
-      ));
-
-      return cursor.continue();
-    };
-
-    cursorRequest.onerror = event =>
-      reject(event.target.error);
-  });
-
-//    putItem :: IDBObjectStore -> Item -> Key -> Promise Item
-const putItem = (store, item, key) =>
-  new Promise((resolve, reject) => {
-    try {
-      const request = store.put(item, key);
-      request.onsuccess = event =>
-        resolve(event.target.result);
-      request.onerror = event =>
-        reject(event.target.error);
-    } catch (error) {
-      reject(error);
-    }
-  });

js/modules/migrations/migrations_0_database_with_attachment_data.js

@@ -0,0 +1,166 @@
+const isFunction = require('lodash/isFunction');
+const isObject = require('lodash/isObject');
+
+
+// IMPORTANT: The migrations below are run on a database that may be very large
+// due to attachments being directly stored inside the database. Please avoid
+// any expensive operations, e.g. modifying all messages / attachments, etc., as
+// it may cause out-of-memory errors for users with long histories:
+// https://github.com/signalapp/Signal-Desktop/issues/2163
+const migrations = [
+  {
+    version: '12.0',
+    migrate(transaction, next) {
+      console.log('Migration 12');
+      console.log('creating object stores');
+      const messages = transaction.db.createObjectStore('messages');
+      messages.createIndex('conversation', ['conversationId', 'received_at'], {
+        unique: false,
+      });
+      messages.createIndex('receipt', 'sent_at', { unique: false });
+      messages.createIndex('unread', ['conversationId', 'unread'], { unique: false });
+      messages.createIndex('expires_at', 'expires_at', { unique: false });
+
+      const conversations = transaction.db.createObjectStore('conversations');
+      conversations.createIndex('inbox', 'active_at', { unique: false });
+      conversations.createIndex('group', 'members', {
+        unique: false,
+        multiEntry: true,
+      });
+      conversations.createIndex('type', 'type', {
+        unique: false,
+      });
+      conversations.createIndex('search', 'tokens', {
+        unique: false,
+        multiEntry: true,
+      });
+
+      transaction.db.createObjectStore('groups');
+
+      transaction.db.createObjectStore('sessions');
+      transaction.db.createObjectStore('identityKeys');
+      transaction.db.createObjectStore('preKeys');
+      transaction.db.createObjectStore('signedPreKeys');
+      transaction.db.createObjectStore('items');
+
+      console.log('creating debug log');
+      transaction.db.createObjectStore('debug');
+
+      next();
+    },
+  },
+  {
+    version: '13.0',
+    migrate(transaction, next) {
+      console.log('Migration 13');
+      console.log('Adding fields to identity keys');
+      const identityKeys = transaction.objectStore('identityKeys');
+      const request = identityKeys.openCursor();
+      const promises = [];
+      request.onsuccess = (event) => {
+        const cursor = event.target.result;
+        if (cursor) {
+          const attributes = cursor.value;
+          attributes.timestamp = 0;
+          attributes.firstUse = false;
+          attributes.nonblockingApproval = false;
+          attributes.verified = 0;
+          promises.push(new Promise(((resolve, reject) => {
+            const putRequest = identityKeys.put(attributes, attributes.id);
+            putRequest.onsuccess = resolve;
+            putRequest.onerror = (e) => {
+              console.log(e);
+              reject(e);
+            };
+          })));
+          cursor.continue();
+        } else {
+          // no more results
+          // eslint-disable-next-line more/no-then
+          Promise.all(promises).then(() => {
+            next();
+          });
+        }
+      };
+      request.onerror = (event) => {
+        console.log(event);
+      };
+    },
+  },
+  {
+    version: '14.0',
+    migrate(transaction, next) {
+      console.log('Migration 14');
+      console.log('Adding unprocessed message store');
+      const unprocessed = transaction.db.createObjectStore('unprocessed');
+      unprocessed.createIndex('received', 'timestamp', { unique: false });
+      next();
+    },
+  },
+  {
+    version: '15.0',
+    migrate(transaction, next) {
+      console.log('Migration 15');
+      console.log('Adding messages index for de-duplication');
+      const messages = transaction.objectStore('messages');
+      messages.createIndex('unique', ['source', 'sourceDevice', 'sent_at'], {
+        unique: true,
+      });
+      next();
+    },
+  },
+  {
+    version: '16.0',
+    migrate(transaction, next) {
+      console.log('Migration 16');
+      console.log('Dropping log table, since we now log to disk');
+      transaction.db.deleteObjectStore('debug');
+      next();
+    },
+  },
+  {
+    version: 17,
+    async migrate(transaction, next) {
+      console.log('Migration 17');
+      console.log('Start migration to database version 17');
+
+      const start = Date.now();
+
+      const messagesStore = transaction.objectStore('messages');
+      console.log('Create index from attachment schema version to attachment');
+      messagesStore.createIndex('schemaVersion', 'schemaVersion', { unique: false });
+
+      const duration = Date.now() - start;
+
+      console.log(
+        'Complete migration to database version 17.',
+        `Duration: ${duration}ms`
+      );
+      next();
+    },
+  },
+];
+
+const database = {
+  id: 'signal',
+  nolog: true,
+  migrations,
+};
+
+exports.run = ({ Backbone } = {}) => {
+  if (!isObject(Backbone) || !isObject(Backbone.Collection) ||
+      !isFunction(Backbone.Collection.extend)) {
+    throw new TypeError('"Backbone" is required');
+  }
+
+  const migrationCollection = new (Backbone.Collection.extend({
+    database,
+    storeName: 'conversations',
+  }))();
+
+  return new Promise((resolve) => {
+    // NOTE: This `then` refers to a jQuery `Deferred`:
+    // eslint-disable-next-line more/no-then
+    migrationCollection.fetch().then(() => resolve());
+  });
+};

js/modules/migrations/migrations_1_database_without_attachment_data.js

@@ -0,0 +1,9 @@
+exports.migrations = [
+  {
+    version: 18,
+    async migrate(transaction, next) {
+      console.log('Migration 18');
+      next();
+    },
+  },
+];