From feff619a3847ad059549adb5882b9357ab4b413a Mon Sep 17 00:00:00 2001
From: trevor-signal <131492920+trevor-signal@users.noreply.github.com>
Date: Wed, 6 Dec 2023 12:27:05 -0500
Subject: [PATCH] Generate compressed thumbnail for images and videos

---
 ts/types/VisualAttachment.ts | 147 ++++++++++++++++++++++++-----------
 1 file changed, 103 insertions(+), 44 deletions(-)

diff --git a/ts/types/VisualAttachment.ts b/ts/types/VisualAttachment.ts
index 5d360eccf2c..59aebbfa3ae 100644
--- a/ts/types/VisualAttachment.ts
+++ b/ts/types/VisualAttachment.ts
@@ -5,11 +5,11 @@ import loadImage from 'blueimp-load-image';
 import { blobToArrayBuffer } from 'blob-util';
 import { toLogFormat } from './errors';
 import type { MIMEType } from './MIME';
-import { IMAGE_PNG } from './MIME';
+import { IMAGE_JPEG, IMAGE_PNG } from './MIME';
 import type { LoggerType } from './Logging';
-import { arrayBufferToObjectURL } from '../util/arrayBufferToObjectURL';
 import { strictAssert } from '../util/assert';
 import { canvasToBlob } from '../util/canvasToBlob';
+import { KIBIBYTE } from './AttachmentSize';
 
 export { blobToArrayBuffer };
 
@@ -101,6 +101,107 @@ export function makeImageThumbnail({
   });
 }
 
+export type MakeImageThumbnailForBackupOptionsType = Readonly<{
+  maxDimension?: number;
+  maxSize?: number;
+  objectUrl: string;
+  logger: LoggerType;
+}>;
+
+// 0.7 quality seems to result in a good result in 1 interation for most images
+const STARTING_JPEG_QUALITY = 0.7;
+const MINIMUM_JPEG_QUALITY = 0.1;
+const ADDITIONAL_QUALITY_DECREASE_PER_ITERATION = 0.1;
+
+export type CreatedThumbnailType = {
+  data: Uint8Array;
+  height: number;
+  width: number;
+  mimeType: MIMEType;
+};
+
+export function makeImageThumbnailForBackup({
+  maxDimension = 256,
+  maxSize = 8 * KIBIBYTE,
+  objectUrl,
+  logger,
+}: MakeImageThumbnailForBackupOptionsType): Promise<CreatedThumbnailType> {
+  return new Promise((resolve, reject) => {
+    const image = document.createElement('img');
+
+    image.addEventListener('load', async () => {
+      const start = performance.now();
+
+      // Scale image to the right size and draw it on a canvas
+      const canvas = loadImage.scale(image, {
+        canvas: true,
+        maxWidth: maxDimension,
+        maxHeight: maxDimension,
+      });
+
+      strictAssert(
+        canvas instanceof HTMLCanvasElement,
+        'loadImage must produce canvas'
+      );
+
+      let jpegQuality = STARTING_JPEG_QUALITY;
+
+      try {
+        let blob = await canvasToBlob(canvas, IMAGE_JPEG, jpegQuality);
+        let iterations = 1;
+
+        while (
+          blob.size > maxSize &&
+          jpegQuality > MINIMUM_JPEG_QUALITY &&
+          // iterations should be capped by the minimum JPEG quality condition, but for
+          // peace of mind, let's cap them explicitly as well.
+          iterations < 5
+        ) {
+          jpegQuality = Math.max(
+            MINIMUM_JPEG_QUALITY,
+            // In testing, the relationship between quality and size in this range is
+            // relatively linear, so we guess at the appropriate quality by scaling by the
+            // size and adding an additional quality decrease as a buffer
+            (maxSize / blob.size) * jpegQuality -
+              ADDITIONAL_QUALITY_DECREASE_PER_ITERATION
+          );
+          // eslint-disable-next-line no-await-in-loop
+          blob = await canvasToBlob(canvas, IMAGE_JPEG, jpegQuality);
+          iterations += 1;
+        }
+
+        const duration = (performance.now() - start).toFixed(1);
+
+        const logMethod = blob.size > maxSize ? logger.warn : logger.info;
+        const sizeInKiB = blob.size / KIBIBYTE;
+        logMethod(
+          'makeImageThumbnail: generated thumbnail of dimensions: ' +
+            `${canvas.width} x ${canvas.height}, and size: ${sizeInKiB}(KiB) ` +
+            `at quality: ${jpegQuality}, iterations: ${iterations}, time: ${duration}ms`
+        );
+
+        const buffer = await blobToArrayBuffer(blob);
+
+        resolve({
+          data: new Uint8Array(buffer),
+          height: canvas.height,
+          width: canvas.width,
+          mimeType: IMAGE_JPEG,
+        });
+      } catch (err) {
+        reject(err);
+      }
+    });
+
+    image.addEventListener('error', error => {
+      logger.error('makeImageThumbnail error', toLogFormat(error));
+      reject(error);
+    });
+
+    image.src = objectUrl;
+  });
+}
+
 export type MakeVideoScreenshotOptionsType = Readonly<{
   objectUrl: string;
   contentType?: MIMEType;
@@ -150,48 +251,6 @@ export function makeVideoScreenshot({
   });
 }
 
-export type MakeVideoThumbnailOptionsType = Readonly<{
-  size: number;
-  videoObjectUrl: string;
-  logger: Pick<LoggerType, 'error'>;
-  contentType: MIMEType;
-}>;
-
-export async function makeVideoThumbnail({
-  size,
-  videoObjectUrl,
-  logger,
-  contentType,
-}: MakeVideoThumbnailOptionsType): Promise<Blob> {
-  let screenshotObjectUrl: string | undefined;
-  try {
-    const blob = await makeVideoScreenshot({
-      objectUrl: videoObjectUrl,
-      contentType,
-      logger,
-    });
-    const data = await blobToArrayBuffer(blob);
-    screenshotObjectUrl = arrayBufferToObjectURL({
-      data,
-      type: contentType,
-    });
-
-    // We need to wait for this, otherwise the finally below will run first
-    const resultBlob = await makeImageThumbnail({
-      size,
-      objectUrl: screenshotObjectUrl,
-      contentType,
-      logger,
-    });
-
-    return resultBlob;
-  } finally {
-    if (screenshotObjectUrl !== undefined) {
-      revokeObjectUrl(screenshotObjectUrl);
-    }
-  }
-}
-
 export function makeObjectUrl(
   data: Uint8Array | ArrayBuffer,
   contentType: MIMEType