Backup and end-to-end test!

js/modules/backup.js

@@ -468,6 +468,7 @@ async function readAttachment(dir, attachment, name, options) {
   const data = await readFileAsArrayBuffer(targetPath);
 
   const isEncrypted = !_.isUndefined(key);
+
   if (isEncrypted) {
     attachment.data = await crypto.decryptSymmetric(key, data);
   } else {
@@ -475,6 +476,65 @@ async function readAttachment(dir, attachment, name, options) {
   }
 }
 
+async function writeThumbnail(attachment, options) {
+  const {
+    dir,
+    message,
+    index,
+    key,
+    newKey,
+  } = options;
+  const filename = `${_getAnonymousAttachmentFileName(message, index)}-thumbnail`;
+  const target = path.join(dir, filename);
+  const { thumbnail } = attachment;
+
+  if (!thumbnail || !thumbnail.data) {
+    return;
+  }
+
+  await writeEncryptedAttachment(target, thumbnail.data, {
+    key,
+    newKey,
+    filename,
+    dir,
+  });
+}
+
+async function writeThumbnails(rawQuotedAttachments, options) {
+  const { name } = options;
+
+  const { loadAttachmentData } = Signal.Migrations;
+  const promises = rawQuotedAttachments.map(async (attachment) => {
+    if (!attachment || !attachment.thumbnail || !attachment.thumbnail.path) {
+      return attachment;
+    }
+
+    return Object.assign(
+      {},
+      attachment,
+      { thumbnail: await loadAttachmentData(attachment.thumbnail) }
+    );
+  });
+
+  const attachments = await Promise.all(promises);
+  try {
+    await Promise.all(_.map(
+      attachments,
+      (attachment, index) => writeThumbnail(attachment, Object.assign({}, options, {
+        index,
+      }))
+    ));
+  } catch (error) {
+    console.log(
+      'writeThumbnails: error exporting conversation',
+      name,
+      ':',
+      error && error.stack ? error.stack : error
+    );
+    throw error;
+  }
+}
+
 async function writeAttachment(attachment, options) {
   const {
     dir,
@@ -485,26 +545,16 @@ async function writeAttachment(attachment, options) {
   } = options;
   const filename = _getAnonymousAttachmentFileName(message, index);
   const target = path.join(dir, filename);
-  if (fs.existsSync(target)) {
-    if (newKey) {
-      console.log(`Deleting attachment ${filename}; key has changed`);
-      fs.unlinkSync(target);
-    } else {
-      console.log(`Skipping attachment ${filename}; already exists`);
-      return;
-    }
-  }
-
   if (!Attachment.hasData(attachment)) {
     throw new TypeError("'attachment.data' is required");
   }
 
-  const ciphertext = await crypto.encryptSymmetric(key, attachment.data);
+  await writeEncryptedAttachment(target, attachment.data, {
-
+    key,
-  const writer = await createFileAndWriter(dir, filename);
+    newKey,
-  const stream = createOutputStream(writer);
+    filename,
-  stream.write(Buffer.from(ciphertext));
+    dir,
-  await stream.close();
+  });
 }
 
 async function writeAttachments(rawAttachments, options) {
@@ -531,6 +581,32 @@ async function writeAttachments(rawAttachments, options) {
   }
 }
 
+async function writeEncryptedAttachment(target, data, options = {}) {
+  const {
+    key,
+    newKey,
+    filename,
+    dir,
+  } = options;
+
+  if (fs.existsSync(target)) {
+    if (newKey) {
+      console.log(`Deleting attachment ${filename}; key has changed`);
+      fs.unlinkSync(target);
+    } else {
+      console.log(`Skipping attachment ${filename}; already exists`);
+      return;
+    }
+  }
+
+  const ciphertext = await crypto.encryptSymmetric(key, data);
+
+  const writer = await createFileAndWriter(dir, filename);
+  const stream = createOutputStream(writer);
+  stream.write(Buffer.from(ciphertext));
+  await stream.close();
+}
+
 function _sanitizeFileName(filename) {
   return filename.toString().replace(/[^a-z0-9.,+()'#\- ]/gi, '_');
 }
@@ -542,6 +618,7 @@ async function exportConversation(db, conversation, options) {
     dir,
     attachmentsDir,
     key,
+    newKey,
   } = options;
   if (!name) {
     throw new Error('Need a name!');
@@ -610,6 +687,7 @@ async function exportConversation(db, conversation, options) {
         }
 
         // eliminate attachment data from the JSON, since it will go to disk
+        // Note: this is for legacy messages only, which stored attachment data in the db
         message.attachments = _.map(
           attachments,
           attachment => _.omit(attachment, ['data'])
@@ -629,18 +707,34 @@ async function exportConversation(db, conversation, options) {
         const jsonString = JSON.stringify(stringify(message));
         stream.write(jsonString);
 
+        console.log({ backupMessage: message });
         if (attachments && attachments.length > 0) {
           const exportAttachments = () => writeAttachments(attachments, {
             dir: attachmentsDir,
             name,
             message,
             key,
+            newKey,
           });
 
           // eslint-disable-next-line more/no-then
           promiseChain = promiseChain.then(exportAttachments);
         }
 
+        const quoteThumbnails = message.quote && message.quote.attachments;
+        if (quoteThumbnails && quoteThumbnails.length > 0) {
+          const exportQuoteThumbnails = () => writeThumbnails(quoteThumbnails, {
+            dir: attachmentsDir,
+            name,
+            message,
+            key,
+            newKey,
+          });
+
+          // eslint-disable-next-line more/no-then
+          promiseChain = promiseChain.then(exportQuoteThumbnails);
+        }
+
         count += 1;
         cursor.continue();
       } else {
@@ -701,6 +795,7 @@ function exportConversations(db, options) {
     messagesDir,
     attachmentsDir,
     key,
+    newKey,
   } = options;
 
   if (!messagesDir) {
@@ -747,6 +842,7 @@ function exportConversations(db, options) {
             dir,
             attachmentsDir,
             key,
+            newKey,
           });
         };
 
@@ -808,11 +904,23 @@ function loadAttachments(dir, getName, options) {
   options = options || {};
   const { message } = options;
 
-  const promises = _.map(message.attachments, (attachment, index) => {
+  const attachmentPromises = _.map(message.attachments, (attachment, index) => {
     const name = getName(message, index, attachment);
     return readAttachment(dir, attachment, name, options);
   });
-  return Promise.all(promises);
+
+  const quoteAttachments = message.quote && message.quote.attachments;
+  const thumbnailPromises = _.map(quoteAttachments, (attachment, index) => {
+    const thumbnail = attachment && attachment.thumbnail;
+    if (!thumbnail) {
+      return null;
+    }
+
+    const name = `${getName(message, index, thumbnail)}-thumbnail`;
+    return readAttachment(dir, thumbnail, name, options);
+  });
+
+  return Promise.all(attachmentPromises.concat(thumbnailPromises));
 }
 
 function saveMessage(db, message) {
@@ -922,7 +1030,8 @@ async function importConversation(db, dir, options) {
       return false;
     }
 
-    if (message.attachments && message.attachments.length) {
+    if ((message.attachments && message.attachments.length) ||
+      (message.quote && message.quote.attachments && message.quote.attachments.length)) {
       const importMessage = async () => {
         const getName = attachmentsDir
           ? _getAnonymousAttachmentFileName

js/modules/types/message.js

@@ -243,9 +243,12 @@ exports.createAttachmentDataWriter = (writeExistingAttachmentData) => {
 
     const message = exports.initializeSchemaVersion(rawMessage);
 
-    const { attachments } = message;
+    const { attachments, quote } = message;
-    const hasAttachments = attachments && attachments.length > 0;
+    const hasFilesToWrite =
-    if (!hasAttachments) {
+      (quote && quote.attachments && quote.attachments.length > 0) ||
+      (attachments && attachments.length > 0);
+
+    if (!hasFilesToWrite) {
       return message;
     }
 
@@ -256,7 +259,7 @@ exports.createAttachmentDataWriter = (writeExistingAttachmentData) => {
       return message;
     }
 
-    attachments.forEach((attachment) => {
+    (attachments || []).forEach((attachment) => {
       if (!Attachment.hasData(attachment)) {
         throw new TypeError("'attachment.data' is required during message import");
       }
@@ -266,13 +269,29 @@ exports.createAttachmentDataWriter = (writeExistingAttachmentData) => {
       }
     });
 
-    const messageWithoutAttachmentData = Object.assign({}, message, {
+    const writeThumbnails = exports._mapQuotedAttachments(async (thumbnail) => {
-      attachments: await Promise.all(attachments.map(async (attachment) => {
+      const { data, path } = thumbnail;
-        await writeExistingAttachmentData(attachment);
+
-        return omit(attachment, ['data']);
+      // we want to be bulletproof to thumbnails without data
-      })),
+      if (!data || !path) {
+        return thumbnail;
+      }
+
+      await writeExistingAttachmentData(thumbnail);
+      return omit(thumbnail, ['data']);
     });
 
+    const messageWithoutAttachmentData = Object.assign(
+      {},
+      await writeThumbnails(message),
+      {
+        attachments: await Promise.all((attachments || []).map(async (attachment) => {
+          await writeExistingAttachmentData(attachment);
+          return omit(attachment, ['data']);
+        })),
+      }
+    );
+
     return messageWithoutAttachmentData;
   };
 };

package.json

@@ -114,6 +114,7 @@
     "eslint-plugin-mocha": "^4.12.1",
     "eslint-plugin-more": "^0.3.1",
     "extract-zip": "^1.6.6",
+    "glob": "^7.1.2",
     "grunt": "^1.0.1",
     "grunt-cli": "^1.2.0",
     "grunt-contrib-concat": "^1.0.1",

preload.js

@@ -205,3 +205,17 @@ window.Signal.Workflow.MessageDataMigrator =
 // We pull this in last, because the native module involved appears to be sensitive to
 //   /tmp mounted as noexec on Linux.
 require('./js/spell_check');
+
+if (window.config.environment === 'test') {
+  /* eslint-disable global-require, import/no-extraneous-dependencies */
+  window.test = {
+    fs: require('fs'),
+    glob: require('glob'),
+    fse: require('fs-extra'),
+    tmp: require('tmp'),
+    path: require('path'),
+    basePath: __dirname,
+    attachmentsPath,
+  };
+  /* eslint-enable global-require, import/no-extraneous-dependencies */
+}

test/backup_test.js

@@ -1,5 +1,8 @@
 /* global Signal: false */
+/* global Whisper: false */
 /* global assert: false */
+/* global textsecure: false */
+/* global _: false */
 
 'use strict';
 
@@ -223,4 +226,270 @@ describe('Backup', () => {
       );
     });
   });
+
+  describe('end-to-end', () => {
+    it('exports then imports to produce the same data we started with', async () => {
+      const {
+        attachmentsPath,
+        fs,
+        fse,
+        glob,
+        path,
+        tmp,
+      } = window.test;
+      const {
+        upgradeMessageSchema,
+        loadAttachmentData,
+      } = window.Signal.Migrations;
+
+      const key = new Uint8Array([
+        1, 3, 4, 5, 6, 7, 8, 11,
+        23, 34, 1, 34, 3, 5, 45, 45,
+        1, 3, 4, 5, 6, 7, 8, 11,
+        23, 34, 1, 34, 3, 5, 45, 45,
+      ]);
+      const attachmentsPattern = path.join(attachmentsPath, '**');
+
+      const OUR_NUMBER = '+12025550000';
+      const CONTACT_ONE_NUMBER = '+12025550001';
+
+      async function wrappedLoadAttachment(attachment) {
+        return _.omit(await loadAttachmentData(attachment), ['path']);
+      }
+
+      async function clearAllData() {
+        await textsecure.storage.protocol.removeAllData();
+        await fse.emptyDir(attachmentsPath);
+      }
+
+      function removeId(model) {
+        return _.omit(model, ['id']);
+      }
+
+      const twoSlashes = /.*\/.*\/.*/;
+      function removeDirs(dirs) {
+        return _.filter(dirs, (fullDir) => {
+          const dir = fullDir.replace(attachmentsPath, '');
+          return twoSlashes.test(dir);
+        });
+      }
+
+      function _mapQuotedAttachments(mapper) {
+        return async (message, context) => {
+          if (!message.quote) {
+            return message;
+          }
+
+          const wrappedMapper = async (attachment) => {
+            if (!attachment || !attachment.thumbnail) {
+              return attachment;
+            }
+
+            return Object.assign({}, attachment, {
+              thumbnail: await mapper(attachment.thumbnail, context),
+            });
+          };
+
+          const quotedAttachments = (message.quote && message.quote.attachments) || [];
+
+          return Object.assign({}, message, {
+            quote: Object.assign({}, message.quote, {
+              attachments: await Promise.all(quotedAttachments.map(wrappedMapper)),
+            }),
+          });
+        };
+      }
+
+      async function loadAllFilesFromDisk(message) {
+        const loadThumbnails = _mapQuotedAttachments((thumbnail) => {
+          // we want to be bulletproof to thumbnails without data
+          if (!thumbnail.path) {
+            return thumbnail;
+          }
+
+          return wrappedLoadAttachment(thumbnail);
+        });
+
+        const promises = (message.attachments || []).map(attachment =>
+          wrappedLoadAttachment(attachment));
+
+        return Object.assign(
+          {},
+          await loadThumbnails(message),
+          {
+            attachments: await Promise.all(promises),
+          }
+        );
+      }
+
+      let backupDir;
+      try {
+        const ATTACHMENT_COUNT = 2;
+        const MESSAGE_COUNT = 1;
+        const CONVERSATION_COUNT = 1;
+
+        const messageWithAttachments = {
+          conversationId: CONTACT_ONE_NUMBER,
+          body: 'Totally!',
+          source: OUR_NUMBER,
+          received_at: 1524185933350,
+          timestamp: 1524185933350,
+          errors: [],
+          attachments: [{
+            contentType: 'image/gif',
+            fileName: 'sad_cat.gif',
+            data: new Uint8Array([
+              1, 2, 3, 4, 5, 6, 7, 8,
+              1, 2, 3, 4, 5, 6, 7, 8,
+              1, 2, 3, 4, 5, 6, 7, 8,
+              1, 2, 3, 4, 5, 6, 7, 8,
+            ]).buffer,
+          }],
+          quote: {
+            text: "Isn't it cute?",
+            author: CONTACT_ONE_NUMBER,
+            id: 12345678,
+            attachments: [{
+              contentType: 'audio/mp3',
+              fileName: 'song.mp3',
+            }, {
+              contentType: 'image/gif',
+              fileName: 'happy_cat.gif',
+              thumbnail: {
+                contentType: 'image/png',
+                data: new Uint8Array([
+                  2, 2, 3, 4, 5, 6, 7, 8,
+                  1, 2, 3, 4, 5, 6, 7, 8,
+                  1, 2, 3, 4, 5, 6, 7, 8,
+                  1, 2, 3, 4, 5, 6, 7, 8,
+                ]).buffer,
+              },
+            }],
+          },
+        };
+
+        console.log('Backup test: Clear all data');
+        await clearAllData();
+
+        console.log('Backup test: Create models, save to db/disk');
+        const message = await upgradeMessageSchema(messageWithAttachments);
+        console.log({ message });
+        const messageModel = new Whisper.Message(message);
+        await window.wrapDeferred(messageModel.save());
+
+        const conversation = {
+          active_at: 1524185933350,
+          color: 'orange',
+          expireTimer: 0,
+          id: CONTACT_ONE_NUMBER,
+          lastMessage: 'Heyo!',
+          name: 'Someone Somewhere',
+          profileAvatar: {
+            contentType: 'image/jpeg',
+            data: new Uint8Array([
+              3, 2, 3, 4, 5, 6, 7, 8,
+              1, 2, 3, 4, 5, 6, 7, 8,
+              1, 2, 3, 4, 5, 6, 7, 8,
+              1, 2, 3, 4, 5, 6, 7, 8,
+            ]).buffer,
+            size: 64,
+          },
+          profileKey: new Uint8Array([
+            4, 2, 3, 4, 5, 6, 7, 8,
+            1, 2, 3, 4, 5, 6, 7, 8,
+            1, 2, 3, 4, 5, 6, 7, 8,
+            1, 2, 3, 4, 5, 6, 7, 8,
+          ]).buffer,
+          profileName: 'Someone! 🤔',
+          profileSharing: true,
+          timestamp: 1524185933350,
+          tokens: [
+            'someone somewhere',
+            'someone',
+            'somewhere',
+            '2025550001',
+            '12025550001',
+          ],
+          type: 'private',
+          unreadCount: 0,
+          verified: 0,
+        };
+        console.log({ conversation });
+        const conversationModel = new Whisper.Conversation(conversation);
+        await window.wrapDeferred(conversationModel.save());
+
+        console.log('Backup test: Ensure that all attachments were saved to disk');
+        const attachmentFiles = removeDirs(glob.sync(attachmentsPattern));
+        console.log({ attachmentFiles });
+        assert.strictEqual(ATTACHMENT_COUNT, attachmentFiles.length);
+
+        console.log('Backup test: Export!');
+        backupDir = tmp.dirSync().name;
+        console.log({ backupDir });
+        await Signal.Backup.exportToDirectory(backupDir, { key });
+
+        console.log('Backup test: Ensure that messages.zip exists');
+        const zipPath = path.join(backupDir, 'messages.zip');
+        const messageZipExists = fs.existsSync(zipPath);
+        assert.strictEqual(true, messageZipExists);
+
+        console.log('Backup test: Ensure that all attachments made it to backup dir');
+        const backupAttachmentPattern = path.join(backupDir, 'attachments/*');
+        const backupAttachments = glob.sync(backupAttachmentPattern);
+        console.log({ backupAttachments });
+        assert.strictEqual(ATTACHMENT_COUNT, backupAttachments.length);
+
+        console.log('Backup test: Clear all data');
+        await clearAllData();
+
+        console.log('Backup test: Import!');
+        await Signal.Backup.importFromDirectory(backupDir, { key });
+
+        console.log('Backup test: ensure that all attachments were imported');
+        const recreatedAttachmentFiles = removeDirs(glob.sync(attachmentsPattern));
+        console.log({ recreatedAttachmentFiles });
+        assert.strictEqual(ATTACHMENT_COUNT, recreatedAttachmentFiles.length);
+        assert.deepEqual(attachmentFiles, recreatedAttachmentFiles);
+
+        console.log('Backup test: Check messages');
+        const messageCollection = new Whisper.MessageCollection();
+        await window.wrapDeferred(messageCollection.fetch());
+        assert.strictEqual(messageCollection.length, MESSAGE_COUNT);
+        const messageFromDB = removeId(messageCollection.at(0).attributes);
+        console.log({ messageFromDB, message });
+        assert.deepEqual(messageFromDB, message);
+
+        console.log('Backup test: check that all attachments were successfully imported');
+        const messageWithAttachmentsFromDB = await loadAllFilesFromDisk(messageFromDB);
+        console.log({ messageWithAttachmentsFromDB, messageWithAttachments });
+        assert.deepEqual(
+          _.omit(messageWithAttachmentsFromDB, ['schemaVersion']),
+          messageWithAttachments
+        );
+
+        console.log('Backup test: check conversations');
+        const conversationCollection = new Whisper.ConversationCollection();
+        await window.wrapDeferred(conversationCollection.fetch());
+        assert.strictEqual(conversationCollection.length, CONVERSATION_COUNT);
+
+        const conversationFromDB = conversationCollection.at(0).attributes;
+        console.log({ conversationFromDB, conversation });
+        assert.deepEqual(
+          conversationFromDB,
+          _.omit(conversation, ['profileAvatar'])
+        );
+
+        console.log('Backup test: Clear all data');
+        await clearAllData();
+
+        console.log('Backup test: Complete!');
+      } finally {
+        if (backupDir) {
+          console.log({ backupDir });
+          console.log('Deleting', backupDir);
+          await fse.remove(backupDir);
+        }
+      }
+    });
+  });
 });

test/modules/types/message_test.js

@@ -67,6 +67,43 @@ describe('Message', () => {
         await Message.createAttachmentDataWriter(writeExistingAttachmentData)(input);
       assert.deepEqual(actual, expected);
     });
+
+    it('should process quote attachment thumbnails', async () => {
+      const input = {
+        body: 'Imagine there is no heaven…',
+        schemaVersion: 4,
+        attachments: [],
+        quote: {
+          attachments: [{
+            thumbnail: {
+              path: 'ab/abcdefghi',
+              data: stringToArrayBuffer('It’s easy if you try'),
+            },
+          }],
+        },
+      };
+      const expected = {
+        body: 'Imagine there is no heaven…',
+        schemaVersion: 4,
+        attachments: [],
+        quote: {
+          attachments: [{
+            thumbnail: {
+              path: 'ab/abcdefghi',
+            },
+          }],
+        },
+      };
+
+      const writeExistingAttachmentData = (attachment) => {
+        assert.equal(attachment.path, 'ab/abcdefghi');
+        assert.deepEqual(attachment.data, stringToArrayBuffer('It’s easy if you try'));
+      };
+
+      const actual =
+        await Message.createAttachmentDataWriter(writeExistingAttachmentData)(input);
+      assert.deepEqual(actual, expected);
+    });
   });
 
   describe('initializeSchemaVersion', () => {