Support Attachments On File System During Export + Import (#2212)

- [x] Load attachments into memory before export.
- [x] Save attachments to correct place in user profile upon import.
- [x] Upgrade message schema upon import.
- [x] Support both encrypted imports as well as unencrypted Chrome imports.

**Tests**

- [x] Export / import small database (25 messages) with some attachments already
      on disk (`schemaVersion >= 3`) and some not (`schemaVersion < 3`).
- [x] Import Chrome web app import (found and fixed issue with missing
      `schemaVersion`).
- [x] Export / import of real-world Signal Beta data (before: `IndexedDB`
      ~372MB / after: `IndexedDB` ~5.7MB / `attachments.noindex` ~234MB)
 
**Test commands:**

```javascript
// Export
let key = new Uint8Array([1, 3, 4, 5, 6, 7, 8, 11, 23, 34, 1, 34, 3, 5, 45, 45, 1, 3, 4, 5, 6, 7, 8, 11, 23, 34, 1, 34, 3, 5, 45, 45]);
Signal.Backup.exportToDirectory('/Users/<user>/Desktop/Signal-import-export-test', {key});

// Import
let key = new Uint8Array([1, 3, 4, 5, 6, 7, 8, 11, 23, 34, 1, 34, 3, 5, 45, 45, 1, 3, 4, 5, 6, 7, 8, 11, 23, 34, 1, 34, 3, 5, 45, 45]);
Signal.Backup.importFromDirectory('/Users/<user>/Desktop/Signal-import-export-test', {key});

// Import from Chrome
Signal.Backup.importFromDirectory('/Users/<user>/Desktop/Signal Export 2018 Apr 3 at 11.00.54 pm');
```

app/attachments.js

@@ -11,7 +11,7 @@ const PATH = 'attachments.noindex';
 //      getPath :: AbsolutePath -> AbsolutePath
 exports.getPath = (userDataPath) => {
   if (!isString(userDataPath)) {
-    throw new TypeError('`userDataPath` must be a string');
+    throw new TypeError('"userDataPath" must be a string');
   }
   return path.join(userDataPath, PATH);
 };
@@ -19,7 +19,7 @@ exports.getPath = (userDataPath) => {
 //      ensureDirectory :: AbsolutePath -> IO Unit
 exports.ensureDirectory = async (userDataPath) => {
   if (!isString(userDataPath)) {
-    throw new TypeError('`userDataPath` must be a string');
+    throw new TypeError('"userDataPath" must be a string');
   }
   await fse.ensureDir(exports.getPath(userDataPath));
 };
@@ -29,12 +29,12 @@ exports.ensureDirectory = async (userDataPath) => {
 //                      IO (Promise ArrayBuffer)
 exports.createReader = (root) => {
   if (!isString(root)) {
-    throw new TypeError('`root` must be a path');
+    throw new TypeError('"root" must be a path');
   }
 
   return async (relativePath) => {
     if (!isString(relativePath)) {
-      throw new TypeError('`relativePath` must be a string');
+      throw new TypeError('"relativePath" must be a string');
     }
 
     const absolutePath = path.join(root, relativePath);
@@ -43,22 +43,46 @@ exports.createReader = (root) => {
   };
 };
 
-//      createWriter :: AttachmentsPath ->
+//      createWriterForNew :: AttachmentsPath ->
-//                      ArrayBuffer ->
+//                            ArrayBuffer ->
-//                      IO (Promise RelativePath)
+//                            IO (Promise RelativePath)
-exports.createWriter = (root) => {
+exports.createWriterForNew = (root) => {
   if (!isString(root)) {
-    throw new TypeError('`root` must be a path');
+    throw new TypeError('"root" must be a path');
   }
 
   return async (arrayBuffer) => {
     if (!isArrayBuffer(arrayBuffer)) {
-      throw new TypeError('`arrayBuffer` must be an array buffer');
+      throw new TypeError('"arrayBuffer" must be an array buffer');
+    }
+
+    const name = exports.createName();
+    const relativePath = exports.getRelativePath(name);
+    return exports.createWriterForExisting(root)({
+      data: arrayBuffer,
+      path: relativePath,
+    });
+  };
+};
+
+//      createWriter :: AttachmentsPath ->
+//                      { data: ArrayBuffer, path: RelativePath } ->
+//                      IO (Promise RelativePath)
+exports.createWriterForExisting = (root) => {
+  if (!isString(root)) {
+    throw new TypeError('"root" must be a path');
+  }
+
+  return async ({ data: arrayBuffer, path: relativePath } = {}) => {
+    if (!isString(relativePath)) {
+      throw new TypeError('"relativePath" must be a path');
+    }
+
+    if (!isArrayBuffer(arrayBuffer)) {
+      throw new TypeError('"arrayBuffer" must be an array buffer');
     }
 
     const buffer = Buffer.from(arrayBuffer);
-    const name = exports.createName();
-    const relativePath = exports.getRelativePath(name);
     const absolutePath = path.join(root, relativePath);
     await fse.ensureFile(absolutePath);
     await fse.writeFile(absolutePath, buffer);
@@ -71,12 +95,12 @@ exports.createWriter = (root) => {
 //                       IO Unit
 exports.createDeleter = (root) => {
   if (!isString(root)) {
-    throw new TypeError('`root` must be a path');
+    throw new TypeError('"root" must be a path');
   }
 
   return async (relativePath) => {
     if (!isString(relativePath)) {
-      throw new TypeError('`relativePath` must be a string');
+      throw new TypeError('"relativePath" must be a string');
     }
 
     const absolutePath = path.join(root, relativePath);
@@ -93,7 +117,7 @@ exports.createName = () => {
 //      getRelativePath :: String -> IO Path
 exports.getRelativePath = (name) => {
   if (!isString(name)) {
-    throw new TypeError('`name` must be a string');
+    throw new TypeError('"name" must be a string');
   }
 
   const prefix = name.slice(0, 2);

js/modules/backup.js

@@ -1,3 +1,4 @@
+/* global Signal: false */
 /* global Whisper: false */
 /* global dcodeIO: false */
 /* global _: false */
@@ -19,6 +20,7 @@ const archiver = require('archiver');
 const rimraf = require('rimraf');
 const electronRemote = require('electron').remote;
 
+const Attachment = require('./types/attachment');
 const crypto = require('./crypto');
 
 
@@ -30,7 +32,7 @@ const {
 
 module.exports = {
   getDirectoryForExport,
-  backupToDirectory,
+  exportToDirectory,
   getDirectoryForImport,
   importFromDirectory,
   // for testing
@@ -453,7 +455,7 @@ function _getAnonymousAttachmentFileName(message, index) {
 
 async function readAttachment(dir, attachment, name, options) {
   options = options || {};
-  const { key, encrypted } = options;
+  const { key } = options;
 
   const anonymousName = _sanitizeFileName(name);
   const targetPath = path.join(dir, anonymousName);
@@ -465,7 +467,8 @@ async function readAttachment(dir, attachment, name, options) {
 
   const data = await readFileAsArrayBuffer(targetPath);
 
-  if (encrypted && key) {
+  const isEncrypted = !_.isUndefined(key);
+  if (isEncrypted) {
     attachment.data = await crypto.decryptSymmetric(key, data);
   } else {
     attachment.data = data;
@@ -492,17 +495,23 @@ async function writeAttachment(attachment, options) {
     }
   }
 
-  const encrypted = await crypto.encryptSymmetric(key, attachment.data);
+  if (!Attachment.hasData(attachment)) {
+    throw new TypeError('"attachment.data" is required');
+  }
+
+  const ciphertext = await crypto.encryptSymmetric(key, attachment.data);
 
   const writer = await createFileAndWriter(dir, filename);
   const stream = createOutputStream(writer);
-  stream.write(Buffer.from(encrypted));
+  stream.write(Buffer.from(ciphertext));
   await stream.close();
 }
 
-async function writeAttachments(attachments, options) {
+async function writeAttachments(rawAttachments, options) {
   const { name } = options;
 
+  const { loadAttachmentData } = Signal.Migrations;
+  const attachments = await Promise.all(rawAttachments.map(loadAttachmentData));
   const promises = _.map(
     attachments,
     (attachment, index) => writeAttachment(attachment, Object.assign({}, options, {
@@ -620,7 +629,7 @@ async function exportConversation(db, conversation, options) {
         const jsonString = JSON.stringify(stringify(message));
         stream.write(jsonString);
 
-        if (attachments && attachments.length) {
+        if (attachments && attachments.length > 0) {
           const exportAttachments = () => writeAttachments(attachments, {
             dir: attachmentsDir,
             name,
@@ -810,11 +819,17 @@ function saveMessage(db, message) {
   return saveAllMessages(db, [message]);
 }
 
-function saveAllMessages(db, messages) {
+async function saveAllMessages(db, rawMessages) {
-  if (!messages.length) {
+  if (rawMessages.length === 0) {
     return Promise.resolve();
   }
 
+  const { writeMessageAttachments, upgradeMessageSchema } = Signal.Migrations;
+  const importAndUpgrade = async message =>
+    upgradeMessageSchema(await writeMessageAttachments(message));
+
+  const messages = await Promise.all(rawMessages.map(importAndUpgrade));
+
   return new Promise((resolve, reject) => {
     let finished = false;
     const finish = (via) => {
@@ -931,9 +946,7 @@ async function importConversation(db, dir, options) {
     return true;
   });
 
-  if (messages.length > 0) {
+  await saveAllMessages(db, messages);
-    await saveAllMessages(db, messages);
-  }
 
   await promiseChain;
   console.log(
@@ -1079,8 +1092,8 @@ async function encryptFile(sourcePath, targetPath, options) {
   }
 
   const plaintext = await readFileAsArrayBuffer(sourcePath);
-  const encrypted = await crypto.encryptSymmetric(key, plaintext);
+  const ciphertext = await crypto.encryptSymmetric(key, plaintext);
-  return writeFile(targetPath, encrypted);
+  return writeFile(targetPath, ciphertext);
 }
 
 async function decryptFile(sourcePath, targetPath, options) {
@@ -1091,8 +1104,8 @@ async function decryptFile(sourcePath, targetPath, options) {
     throw new Error('Need key to do encryption!');
   }
 
-  const encrypted = await readFileAsArrayBuffer(sourcePath);
+  const ciphertext = await readFileAsArrayBuffer(sourcePath);
-  const plaintext = await crypto.decryptSymmetric(key, encrypted);
+  const plaintext = await crypto.decryptSymmetric(key, ciphertext);
   return writeFile(targetPath, Buffer.from(plaintext));
 }
 
@@ -1105,7 +1118,7 @@ function deleteAll(pattern) {
   return pify(rimraf)(pattern);
 }
 
-async function backupToDirectory(directory, options) {
+async function exportToDirectory(directory, options) {
   options = options || {};
 
   if (!options.key) {
@@ -1196,9 +1209,7 @@ async function importFromDirectory(directory, options) {
           attachmentsDir,
         });
         const result = await importNonMessages(db, stagingDir, options);
-        await importConversations(db, stagingDir, Object.assign({}, options, {
+        await importConversations(db, stagingDir, Object.assign({}, options));
-          encrypted: true,
-        }));
 
         console.log('Done importing from backup!');
         return result;

js/modules/types/attachment/migrate_data_to_file_system.js

@@ -7,15 +7,15 @@ const {
 
 
 // type Context :: {
-//   writeAttachmentData :: ArrayBuffer -> Promise (IO Path)
+//   writeNewAttachmentData :: ArrayBuffer -> Promise (IO Path)
 // }
 //
 //      migrateDataToFileSystem :: Attachment ->
 //                                 Context ->
 //                                 Promise Attachment
-exports.migrateDataToFileSystem = async (attachment, { writeAttachmentData } = {}) => {
+exports.migrateDataToFileSystem = async (attachment, { writeNewAttachmentData } = {}) => {
-  if (!isFunction(writeAttachmentData)) {
+  if (!isFunction(writeNewAttachmentData)) {
-    throw new TypeError('"writeAttachmentData" must be a function');
+    throw new TypeError('"writeNewAttachmentData" must be a function');
   }
 
   const { data } = attachment;
@@ -32,7 +32,7 @@ exports.migrateDataToFileSystem = async (attachment, { writeAttachmentData } = {
       ` got: ${typeof attachment.data}`);
   }
 
-  const path = await writeAttachmentData(data);
+  const path = await writeNewAttachmentData(data);
 
   const attachmentWithoutData = omit(
     Object.assign({}, attachment, { path }),

js/modules/types/message.js

@@ -1,4 +1,4 @@
-const { isFunction } = require('lodash');
+const { isFunction, isString, omit } = require('lodash');
 
 const Attachment = require('./attachment');
 const Errors = require('./errors');
@@ -166,13 +166,72 @@ const toVersion3 = exports._withSchemaVersion(
 );
 
 // UpgradeStep
-exports.upgradeSchema = async (message, { writeAttachmentData } = {}) => {
+exports.upgradeSchema = async (message, { writeNewAttachmentData } = {}) => {
-  if (!isFunction(writeAttachmentData)) {
+  if (!isFunction(writeNewAttachmentData)) {
-    throw new TypeError('`context.writeAttachmentData` is required');
+    throw new TypeError('`context.writeNewAttachmentData` is required');
   }
 
   return toVersion3(
     await toVersion2(await toVersion1(await toVersion0(message))),
-    { writeAttachmentData }
+    { writeNewAttachmentData }
   );
 };
+
+exports.createAttachmentLoader = (loadAttachmentData) => {
+  if (!isFunction(loadAttachmentData)) {
+    throw new TypeError('`loadAttachmentData` is required');
+  }
+
+  return async message => (Object.assign({}, message, {
+    attachments: await Promise.all(message.attachments.map(loadAttachmentData)),
+  }));
+};
+
+//      createAttachmentDataWriter :: (RelativePath -> IO Unit)
+//                                    Message ->
+//                                    IO (Promise Message)
+exports.createAttachmentDataWriter = (writeExistingAttachmentData) => {
+  if (!isFunction(writeExistingAttachmentData)) {
+    throw new TypeError('"writeExistingAttachmentData" must be a function');
+  }
+
+  return async (rawMessage) => {
+    if (!exports.isValid(rawMessage)) {
+      throw new TypeError('"rawMessage" is not valid');
+    }
+
+    const message = exports.initializeSchemaVersion(rawMessage);
+
+    const { attachments } = message;
+    const hasAttachments = attachments && attachments.length > 0;
+    if (!hasAttachments) {
+      return message;
+    }
+
+    const lastVersionWithAttachmentDataInMemory = 2;
+    const willAttachmentsGoToFileSystemOnUpgrade =
+      message.schemaVersion <= lastVersionWithAttachmentDataInMemory;
+    if (willAttachmentsGoToFileSystemOnUpgrade) {
+      return message;
+    }
+
+    attachments.forEach((attachment) => {
+      if (!Attachment.hasData(attachment)) {
+        throw new TypeError('"attachment.data" is required during message import');
+      }
+
+      if (!isString(attachment.path)) {
+        throw new TypeError('"attachment.path" is required during message import');
+      }
+    });
+
+    const messageWithoutAttachmentData = Object.assign({}, message, {
+      attachments: await Promise.all(attachments.map(async (attachment) => {
+        await writeExistingAttachmentData(attachment);
+        return omit(attachment, ['data']);
+      })),
+    });
+
+    return messageWithoutAttachmentData;
+  };
+};

preload.js

@@ -113,11 +113,14 @@ window.ProxyAgent = require('proxy-agent');
 const attachmentsPath = Attachments.getPath(app.getPath('userData'));
 const deleteAttachmentData = Attachments.createDeleter(attachmentsPath);
 const readAttachmentData = Attachments.createReader(attachmentsPath);
-const writeAttachmentData = Attachments.createWriter(attachmentsPath);
+const writeNewAttachmentData = Attachments.createWriterForNew(attachmentsPath);
+const writeExistingAttachmentData = Attachments.createWriterForExisting(attachmentsPath);
+
+const loadAttachmentData = Attachment.loadData(readAttachmentData);
 
 // Injected context functions to keep `Message` agnostic from Electron:
 const upgradeSchemaContext = {
-  writeAttachmentData,
+  writeNewAttachmentData,
 };
 const upgradeMessageSchema = message =>
   Message.upgradeSchema(message, upgradeSchemaContext);
@@ -137,7 +140,10 @@ window.Signal.Migrations = {};
 window.Signal.Migrations.deleteAttachmentData =
   Attachment.deleteData(deleteAttachmentData);
 window.Signal.Migrations.getPlaceholderMigrations = getPlaceholderMigrations;
-window.Signal.Migrations.loadAttachmentData = Attachment.loadData(readAttachmentData);
+window.Signal.Migrations.writeMessageAttachments =
+  Message.createAttachmentDataWriter(writeExistingAttachmentData);
+window.Signal.Migrations.loadAttachmentData = loadAttachmentData;
+window.Signal.Migrations.loadMessage = Message.createAttachmentLoader(loadAttachmentData);
 window.Signal.Migrations.Migrations0DatabaseWithAttachmentData =
   require('./js/modules/migrations/migrations_0_database_with_attachment_data');
 window.Signal.Migrations.Migrations1DatabaseWithoutAttachmentData =

scripts/start-backup

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [[ "$BACKUP" == "" ]]; then
+  echo "BACKUP environment variable is required"
+  exit 1
+fi
+
+if [[ "$PROFILE" == "" ]]; then
+  echo "PROFILE environment variable is required"
+  exit 1
+fi
+
+backupPath="$HOME/Library/Application Support/Signal-$BACKUP"
+profilePath="$HOME/Library/Application Support/Signal-$PROFILE"
+
+rm -rf "$profilePath" && \
+  cp -R "$backupPath" "$profilePath" && \
+  NODE_APP_INSTANCE="$PROFILE" yarn start

test/app/attachments_test.js

@@ -13,7 +13,7 @@ const NAME_LENGTH = 64;
 const PATH_LENGTH = PREFIX_LENGTH + NUM_SEPARATORS + NAME_LENGTH;
 
 describe('Attachments', () => {
-  describe('createWriter', () => {
+  describe('createWriterForNew', () => {
     let tempRootDirectory = null;
     before(() => {
       tempRootDirectory = tmp.dirSync().name;
@@ -25,9 +25,12 @@ describe('Attachments', () => {
 
     it('should write file to disk and return path', async () => {
       const input = stringToArrayBuffer('test string');
-      const tempDirectory = path.join(tempRootDirectory, 'Attachments_createWriter');
+      const tempDirectory = path.join(
+        tempRootDirectory,
+        'Attachments_createWriterForNew'
+      );
 
-      const outputPath = await Attachments.createWriter(tempDirectory)(input);
+      const outputPath = await Attachments.createWriterForNew(tempDirectory)(input);
       const output = await fse.readFile(path.join(tempDirectory, outputPath));
 
       assert.lengthOf(outputPath, PATH_LENGTH);
@@ -37,6 +40,39 @@ describe('Attachments', () => {
     });
   });
 
+  describe('createWriterForExisting', () => {
+    let tempRootDirectory = null;
+    before(() => {
+      tempRootDirectory = tmp.dirSync().name;
+    });
+
+    after(async () => {
+      await fse.remove(tempRootDirectory);
+    });
+
+    it('should write file to disk on given path and return path', async () => {
+      const input = stringToArrayBuffer('test string');
+      const tempDirectory = path.join(
+        tempRootDirectory,
+        'Attachments_createWriterForExisting'
+      );
+
+      const relativePath = Attachments.getRelativePath(Attachments.createName());
+      const attachment = {
+        path: relativePath,
+        data: input,
+      };
+      const outputPath =
+        await Attachments.createWriterForExisting(tempDirectory)(attachment);
+      const output = await fse.readFile(path.join(tempDirectory, outputPath));
+
+      assert.equal(outputPath, relativePath);
+
+      const inputBuffer = Buffer.from(input);
+      assert.deepEqual(inputBuffer, output);
+    });
+  });
+
   describe('createReader', () => {
     let tempRootDirectory = null;
     before(() => {

test/modules/types/attachment_test.js

@@ -120,14 +120,14 @@ describe('Attachment', () => {
       };
 
       const expectedAttachmentData = stringToArrayBuffer('Above us only sky');
-      const writeAttachmentData = async (attachmentData) => {
+      const writeNewAttachmentData = async (attachmentData) => {
         assert.deepEqual(attachmentData, expectedAttachmentData);
         return 'abc/abcdefgh123456789';
       };
 
       const actual = await Attachment.migrateDataToFileSystem(
         input,
-        { writeAttachmentData }
+        { writeNewAttachmentData }
       );
       assert.deepEqual(actual, expected);
     });
@@ -145,12 +145,12 @@ describe('Attachment', () => {
         size: 1111,
       };
 
-      const writeAttachmentData = async () =>
+      const writeNewAttachmentData = async () =>
         'abc/abcdefgh123456789';
 
       const actual = await Attachment.migrateDataToFileSystem(
         input,
-        { writeAttachmentData }
+        { writeNewAttachmentData }
       );
       assert.deepEqual(actual, expected);
     });
@@ -163,11 +163,11 @@ describe('Attachment', () => {
         size: 1111,
       };
 
-      const writeAttachmentData = async () =>
+      const writeNewAttachmentData = async () =>
         'abc/abcdefgh123456789';
 
       try {
-        await Attachment.migrateDataToFileSystem(input, { writeAttachmentData });
+        await Attachment.migrateDataToFileSystem(input, { writeNewAttachmentData });
       } catch (error) {
         assert.strictEqual(
           error.message,

test/modules/types/message_test.js

@@ -5,6 +5,69 @@ const { stringToArrayBuffer } = require('../../../js/modules/string_to_array_buf
 
 
 describe('Message', () => {
+  describe('createAttachmentDataWriter', () => {
+    it('should ignore messages that didn’t go through attachment migration', async () => {
+      const input = {
+        body: 'Imagine there is no heaven…',
+        schemaVersion: 2,
+      };
+      const expected = {
+        body: 'Imagine there is no heaven…',
+        schemaVersion: 2,
+      };
+      const writeExistingAttachmentData = () => {};
+
+      const actual =
+        await Message.createAttachmentDataWriter(writeExistingAttachmentData)(input);
+      assert.deepEqual(actual, expected);
+    });
+
+    it('should ignore messages without attachments', async () => {
+      const input = {
+        body: 'Imagine there is no heaven…',
+        schemaVersion: 4,
+        attachments: [],
+      };
+      const expected = {
+        body: 'Imagine there is no heaven…',
+        schemaVersion: 4,
+        attachments: [],
+      };
+      const writeExistingAttachmentData = () => {};
+
+      const actual =
+        await Message.createAttachmentDataWriter(writeExistingAttachmentData)(input);
+      assert.deepEqual(actual, expected);
+    });
+
+    it('should write attachments to file system on original path', async () => {
+      const input = {
+        body: 'Imagine there is no heaven…',
+        schemaVersion: 4,
+        attachments: [{
+          path: 'ab/abcdefghi',
+          data: stringToArrayBuffer('It’s easy if you try'),
+        }],
+      };
+      const expected = {
+        body: 'Imagine there is no heaven…',
+        schemaVersion: 4,
+        attachments: [{
+          path: 'ab/abcdefghi',
+        }],
+      };
+
+      const writeExistingAttachmentData = (attachment) => {
+        assert.equal(attachment.path, 'ab/abcdefghi');
+        assert.deepEqual(attachment.data, stringToArrayBuffer('It’s easy if you try'));
+      };
+
+      const actual =
+        await Message.createAttachmentDataWriter(writeExistingAttachmentData)(input);
+      assert.deepEqual(actual, expected);
+    });
+  });
+
   describe('initializeSchemaVersion', () => {
     it('should ignore messages with previously inherited schema', () => {
       const input = {
@@ -85,7 +148,7 @@ describe('Message', () => {
 
       const expectedAttachmentData = stringToArrayBuffer('It’s easy if you try');
       const context = {
-        writeAttachmentData: async (attachmentData) => {
+        writeNewAttachmentData: async (attachmentData) => {
           assert.deepEqual(attachmentData, expectedAttachmentData);
           return 'abc/abcdefg';
         },