From ac0b50d20f9e2385752ac843db60107f121b1c8a Mon Sep 17 00:00:00 2001
From: Scott Nonnenberg <scott@signal.org>
Date: Wed, 25 Apr 2018 18:32:46 -0700
Subject: [PATCH] Generate thumbnails for new video attachments, video quotes

---
 background.html                |   7 +-
 js/models/conversations.js     |  83 ++++++-------
 js/models/messages.js          |   4 -
 js/views/file_input_view.js    | 208 +++++++++++++++++++++++----------
 stylesheets/_conversation.scss |  29 +++++
 5 files changed, 222 insertions(+), 109 deletions(-)

diff --git a/background.html b/background.html
index f6aefa5974..0a5665df31 100644
--- a/background.html
+++ b/background.html
@@ -237,7 +237,12 @@
     </div>
   </script>
   <script type='text/x-tmpl-mustache' id='attachment-preview'>
-        <img src='{{ source }}' class='preview' />
+        <div class='image-container'>
+          <img src='{{ source }}' class='preview' />
+          <div class='outer'>
+            <div class='play icon'></div>
+          </div>
+        </div>
         <a class='x close' alt='remove attachment' href='#'></a>
   </script>
   <script type='text/x-tmpl-mustache' id='file-view'>
diff --git a/js/models/conversations.js b/js/models/conversations.js
index 349e4ff6e7..6876a50536 100644
--- a/js/models/conversations.js
+++ b/js/models/conversations.js
@@ -629,7 +629,11 @@
       const attachmentWithData = await loadAttachmentData(attachment);
       const { data, contentType } = attachmentWithData;
       const objectUrl = this.makeObjectUrl(data, contentType);
-      const thumbnail = await Whisper.FileInputView.makeThumbnail(128, objectUrl);
+
+      const thumbnail = Signal.Util.GoogleChrome.isImageTypeSupported(contentType)
+        ? await Whisper.FileInputView.makeThumbnail(128, objectUrl)
+        : await Whisper.FileInputView.makeVideoThumbnail(128, objectUrl);
+
       URL.revokeObjectURL(objectUrl);
 
       const arrayBuffer = await this.blobToArrayBuffer(thumbnail);
@@ -654,7 +658,8 @@
         attachments: await Promise.all((attachments || []).map(async (attachment) => {
           const { contentType } = attachment;
           const willMakeThumbnail =
-            Signal.Util.GoogleChrome.isImageTypeSupported(contentType);
+            Signal.Util.GoogleChrome.isImageTypeSupported(contentType) ||
+            Signal.Util.GoogleChrome.isVideoTypeSupported(contentType);
 
           return {
             contentType,
@@ -1154,9 +1159,14 @@
       const { attachments, id, author } = quote;
       const first = attachments[0];
 
+      if (!first || first.thumbnail) {
+        return true;
+      }
+
       // Maybe in the future we could try to pull the thumbnail from a video ourselves,
       //   but for now we will rely on incoming thumbnails only.
-      if (!Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType)) {
+      if (!Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType) &&
+        !Signal.Util.GoogleChrome.isVideoTypeSupported(first.contentType)) {
         return false;
       }
 
@@ -1194,9 +1204,12 @@
       const { attachments } = quote;
       const first = attachments[0];
 
-      // Maybe in the future we could try to pull thumbnails video ourselves,
-      //   but for now we will rely on incoming thumbnails only.
-      if (!first || !Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType)) {
+      if (!first || first.thumbnail) {
+        return;
+      }
+
+      if (!Signal.Util.GoogleChrome.isImageTypeSupported(first.contentType) &&
+        !Signal.Util.GoogleChrome.isVideoTypeSupported(first.contentType)) {
         return;
       }
 
@@ -1230,18 +1243,26 @@
       if (!thumbnail) {
         return false;
       }
-      const thumbnailWithData = await loadAttachmentData(thumbnail);
-      thumbnailWithData.objectUrl = this.makeObjectUrl(
-        thumbnailWithData.data,
-        thumbnailWithData.contentType
-      );
+      try {
+        const thumbnailWithData = await loadAttachmentData(thumbnail);
+        thumbnailWithData.objectUrl = this.makeObjectUrl(
+          thumbnailWithData.data,
+          thumbnailWithData.contentType
+        );
 
-      // If we update this data in place, there's the risk that this data could be
-      //   saved back to the database
-      // eslint-disable-next-line no-param-reassign
-      message.quoteThumbnail = thumbnailWithData;
+        // If we update this data in place, there's the risk that this data could be
+        //   saved back to the database
+        // eslint-disable-next-line no-param-reassign
+        message.quoteThumbnail = thumbnailWithData;
 
-      return true;
+        return true;
+      } catch (error) {
+        console.log(
+          'loadQuoteThumbnail: had trouble loading thumbnail data from disk',
+          error && error.stack ? error.stack : error
+        );
+        return false;
+      }
     },
     async processQuotes(messages) {
       const lookup = this.makeMessagesLookup(messages);
@@ -1259,21 +1280,16 @@
           return;
         }
 
-        // 1. Check to see if we've already loaded the target message into memory
+        // 1. Load provided thumbnail
+        const gotThumbnail = await this.loadQuoteThumbnail(message, quote);
+
+        // 2. Check to see if we've already loaded the target message into memory
         const { author, id } = quote;
         const key = this.makeKey(author, id);
         const quotedMessage = lookup[key];
 
         if (quotedMessage) {
-          // eslint-disable-next-line no-param-reassign
           await this.loadQuotedMessage(message, quotedMessage);
-
-          // Note: in the future when we generate our own thumbnail we won't need to rely
-          //   on incoming thumbnail if we have our local message in hand.
-          if (!message.quotedMessage.imageUrl) {
-            await this.loadQuoteThumbnail(message, quote);
-          }
-
           this.forceRender(message);
           return;
         }
@@ -1286,28 +1302,15 @@
         }
 
         // We've don't want to go to the database or load thumbnails a second time.
-        if (message.quoteIsProcessed) {
+        if (message.quoteIsProcessed || gotThumbnail) {
           return;
         }
         // eslint-disable-next-line no-param-reassign
         message.quoteIsProcessed = true;
 
-        // 2. Go to the database for the real referenced attachment
+        // 3. As a last resort, go to the database to generate a thumbnail on-demand
         const loaded = await this.loadQuotedMessageFromDatabase(message, id);
         if (loaded) {
-          // Note: in the future when we generate our own thumbnail we won't need to rely
-          //   on incoming thumbnail if we have our local message in hand.
-          if (!message.quotedMessageFromDatabase.imageUrl) {
-            await this.loadQuoteThumbnail(message, quote);
-          }
-
-          this.forceRender(message);
-          return;
-        }
-
-        // 3. Finally, use the provided thumbnail
-        const gotThumbnail = await this.loadQuoteThumbnail(message, quote);
-        if (gotThumbnail) {
           this.forceRender(message);
         }
       });
diff --git a/js/models/messages.js b/js/models/messages.js
index 7a9f4134c6..46464f1427 100644
--- a/js/models/messages.js
+++ b/js/models/messages.js
@@ -181,10 +181,6 @@
         URL.revokeObjectURL(this.quoteThumbnail.objectUrl);
         this.quoteThumbnail = null;
       }
-      if (this.quotedMessageFromDatabase) {
-        this.quotedMessageFromDatabase.unload();
-        this.quotedMessageFromDatabase = null;
-      }
       if (this.quotedMessage) {
         this.quotedMessage = null;
       }
diff --git a/js/views/file_input_view.js b/js/views/file_input_view.js
index 084d10bf7b..21896fa477 100644
--- a/js/views/file_input_view.js
+++ b/js/views/file_input_view.js
@@ -3,6 +3,8 @@
 /* global i18n: false */
 /* global loadImage: false */
 /* global Backbone: false */
+/* global _: false */
+/* global Signal: false */
 
 // eslint-disable-next-line func-names
 (function () {
@@ -61,6 +63,63 @@
     }));
   }
 
+  function makeVideoScreenshot(objectUrl) {
+    return new Promise(((resolve, reject) => {
+      const video = document.createElement('video');
+
+      function capture() {
+        const canvas = document.createElement('canvas');
+        canvas.width = video.videoWidth;
+        canvas.height = video.videoHeight;
+        canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height);
+
+        const image = window.dataURLToBlobSync(canvas.toDataURL('image/png'));
+
+        video.removeEventListener('canplay', capture);
+
+        resolve(image);
+      }
+
+      video.addEventListener('canplay', capture);
+      video.addEventListener('error', (error) => {
+        console.log(
+          'makeVideoThumbnail error',
+          error && error.stack ? error.stack : error
+        );
+        reject(error);
+      });
+
+      video.src = objectUrl;
+    }));
+  }
+
+  function makeObjectUrl(data, contentType) {
+    const blob = new Blob([data], {
+      type: contentType,
+    });
+    return URL.createObjectURL(blob);
+  }
+
+  function blobToArrayBuffer(blob) {
+    return new Promise((resolve, reject) => {
+      const fileReader = new FileReader();
+
+      fileReader.onload = e => resolve(e.target.result);
+      fileReader.onerror = reject;
+      fileReader.onabort = reject;
+
+      fileReader.readAsArrayBuffer(blob);
+    });
+  }
+
+  async function makeVideoThumbnail(size, videoObjectUrl) {
+    const blob = await makeVideoScreenshot(videoObjectUrl);
+    const arrayBuffer = await blobToArrayBuffer(blob);
+    const screenshotObjectUrl = makeObjectUrl(arrayBuffer, 'image/png');
+
+    return makeThumbnail(size, screenshotObjectUrl);
+  }
+
   Whisper.FileInputView = Backbone.View.extend({
     tagName: 'span',
     className: 'file-input',
@@ -103,10 +162,18 @@
       }
     },
 
-    addThumb(src) {
+    addThumb(src, options = {}) {
+      _.defaults(options, { addPlayIcon: false });
       this.$('.avatar').hide();
       this.thumb.src = src;
       this.$('.attachment-previews').append(this.thumb.render().el);
+
+      if (options.addPlayIcon) {
+        this.$el.addClass('video-attachment');
+      } else {
+        this.$el.removeClass('video-attachment');
+      }
+
       this.thumb.$('img')[0].onload = () => {
         this.$el.trigger('force-resize');
       };
@@ -160,69 +227,81 @@
       }));
     },
 
-    previewImages() {
+    async previewImages() {
       this.clearForm();
       const file = this.file || this.$input.prop('files')[0];
-      if (!file) { return; }
-
-      let type = file.type.split('/')[0];
-      if (file.type === 'image/tiff') {
-        type = 'file';
+      if (!file) {
+        return;
       }
-      switch (type) {
-        case 'audio': this.addThumb('images/audio.svg'); break;
-        case 'video': this.addThumb('images/video.svg'); break;
+
+      const contentType = file.type;
+
+      const renderVideoPreview = async () => {
+        // we use the variable on this here to ensure cleanup if we're interrupted
+        this.previewObjectUrl = URL.createObjectURL(file);
+        const thumbnail = await makeVideoScreenshot(this.previewObjectUrl);
+        URL.revokeObjectURL(this.previewObjectUrl);
+
+        const arrayBuffer = await blobToArrayBuffer(thumbnail);
+
+        this.previewObjectUrl = makeObjectUrl(arrayBuffer, 'image/png');
+        this.addThumb(this.previewObjectUrl, { addPlayIcon: true });
+      };
+
+      const renderImagePreview = async () => {
+        if (!MIME.isJPEG(file.type)) {
+          this.previewObjectUrl = URL.createObjectURL(file);
+          this.addThumb(this.previewObjectUrl);
+          return;
+        }
+
+        const dataUrl = await window.autoOrientImage(file);
+        this.addThumb(dataUrl);
+      };
+
+      if (Signal.Util.GoogleChrome.isImageTypeSupported(contentType)) {
+        renderImagePreview();
+      } else if (Signal.Util.GoogleChrome.isVideoTypeSupported(contentType)) {
+        renderVideoPreview();
+      } else if (MIME.isAudio(contentType)) {
+        this.addThumb('images/audio.svg');
+      } else {
+        this.addThumb('images/file.svg');
+      }
+
+      const blob = await this.autoScale(file);
+      let limitKb = 1000000;
+      const blobType = file.type === 'image/gif'
+        ? 'gif'
+        : contentType.split('/')[0];
+
+      switch (blobType) {
         case 'image':
-          if (!MIME.isJPEG(file.type)) {
-            this.previewObjectUrl = URL.createObjectURL(file);
-            this.addThumb(this.previewObjectUrl);
-            break;
-          }
-
-          // NOTE: Temporarily allow `then` until we convert the entire file
-          // to `async` / `await`:
-          // eslint-disable-next-line more/no-then
-          window.autoOrientImage(file)
-            .then(dataURL => this.addThumb(dataURL));
-          break;
+          limitKb = 6000; break;
+        case 'gif':
+          limitKb = 25000; break;
+        case 'audio':
+          limitKb = 100000; break;
+        case 'video':
+          limitKb = 100000; break;
         default:
-          this.addThumb('images/file.svg'); break;
+          limitKb = 100000; break;
+      }
+      if ((blob.size / 1024).toFixed(4) >= limitKb) {
+        const units = ['kB', 'MB', 'GB'];
+        let u = -1;
+        let limit = limitKb * 1000;
+        do {
+          limit /= 1000;
+          u += 1;
+        } while (limit >= 1000 && u < units.length - 1);
+        const toast = new Whisper.FileSizeToast({
+          model: { limit, units: units[u] },
+        });
+        toast.$el.insertAfter(this.$el);
+        toast.render();
+        this.deleteFiles();
       }
-
-      // NOTE: Temporarily allow `then` until we convert the entire file
-      // to `async` / `await`:
-      // eslint-disable-next-line more/no-then
-      this.autoScale(file).then((blob) => {
-        let limitKb = 1000000;
-        const blobType = file.type === 'image/gif' ? 'gif' : type;
-        switch (blobType) {
-          case 'image':
-            limitKb = 6000; break;
-          case 'gif':
-            limitKb = 25000; break;
-          case 'audio':
-            limitKb = 100000; break;
-          case 'video':
-            limitKb = 100000; break;
-          default:
-            limitKb = 100000; break;
-        }
-        if ((blob.size / 1024).toFixed(4) >= limitKb) {
-          const units = ['kB', 'MB', 'GB'];
-          let u = -1;
-          let limit = limitKb * 1000;
-          do {
-            limit /= 1000;
-            u += 1;
-          } while (limit >= 1000 && u < units.length - 1);
-          const toast = new Whisper.FileSizeToast({
-            model: { limit, units: units[u] },
-          });
-          toast.$el.insertAfter(this.$el);
-          toast.render();
-          this.deleteFiles();
-        }
-      });
     },
 
     hasFiles() {
@@ -262,7 +341,7 @@
         .then(setFlags(attachmentFlags));
     },
 
-    getThumbnail() {
+    async getThumbnail() {
       // Scale and crop an image to 256px square
       const size = 256;
       const file = this.file || this.$input.prop('files')[0];
@@ -275,11 +354,10 @@
 
       const objectUrl = URL.createObjectURL(file);
 
-      // eslint-disable-next-line more/no-then
-      return makeThumbnail(size, objectUrl).then((arrayBuffer) => {
-        URL.revokeObjectURL(objectUrl);
-        return this.readFile(arrayBuffer);
-      });
+      const arrayBuffer = await makeThumbnail(size, objectUrl);
+      URL.revokeObjectURL(objectUrl);
+
+      return this.readFile(arrayBuffer);
     },
 
     // File -> Promise Attachment
@@ -370,4 +448,6 @@
   });
 
   Whisper.FileInputView.makeThumbnail = makeThumbnail;
+  Whisper.FileInputView.makeVideoThumbnail = makeVideoThumbnail;
+  Whisper.FileInputView.makeVideoScreenshot = makeVideoScreenshot;
 }());
diff --git a/stylesheets/_conversation.scss b/stylesheets/_conversation.scss
index a16a21a02c..b1738dd013 100644
--- a/stylesheets/_conversation.scss
+++ b/stylesheets/_conversation.scss
@@ -927,6 +927,33 @@ span.status {
 
   form.send {
     background: #ffffff;
+
+    &.video-attachment {
+      .image-container {
+        position: relative;
+      }
+
+      .outer {
+        position: absolute;
+        top: 0;
+        right: 0;
+        bottom: 0;
+        left: 0;
+
+        display: flex;
+        text-align: center;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+
+        .play.icon {
+          height: 30px;
+          width: 30px;
+
+          @include color-svg('../images/play.svg', white);
+        }
+      }
+    }
   }
 
   input, textarea {
@@ -935,8 +962,10 @@ span.status {
 
   .attachment-previews {
     padding: 0 36px;
+
     .attachment-preview {
       padding: 13px 10px 0;
+
     }
     img {
       border: 2px solid #ddd;