From e7a23659057d747d3f9e34d2a008fd86a6c94813 Mon Sep 17 00:00:00 2001 From: Josh Perez <60019601+josh-signal@users.noreply.github.com> Date: Tue, 27 Jul 2021 20:09:10 -0400 Subject: [PATCH] Fix image contentType when transcoding --- ts/util/autoOrientImage.ts | 31 ------ ts/util/handleImageAttachment.ts | 61 +++++++++++ ts/util/scaleImageToLevel.ts | 3 - ts/views/conversation_view.ts | 177 +++---------------------------- 4 files changed, 78 insertions(+), 194 deletions(-) delete mode 100644 ts/util/autoOrientImage.ts create mode 100644 ts/util/handleImageAttachment.ts diff --git a/ts/util/autoOrientImage.ts b/ts/util/autoOrientImage.ts deleted file mode 100644 index dc2a23d05..000000000 --- a/ts/util/autoOrientImage.ts +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2018-2021 Signal Messenger, LLC -// SPDX-License-Identifier: AGPL-3.0-only - -import loadImage, { LoadImageOptions } from 'blueimp-load-image'; -import { IMAGE_JPEG } from '../types/MIME'; -import { canvasToBlob } from './canvasToBlob'; - -const DEFAULT_JPEG_QUALITY = 0.85; - -export async function autoOrientImage(blob: Blob): Promise { - const options: LoadImageOptions = { - canvas: true, - orientation: true, - }; - - try { - const data = await loadImage(blob, options); - const { image } = data; - if (image instanceof HTMLCanvasElement) { - // We `return await`, instead of just `return`, so we capture the rejection in this - // try/catch block. See [this blog post][0] for more background. - // [0]: https://jakearchibald.com/2017/await-vs-return-vs-return-await/ - return await canvasToBlob(image, IMAGE_JPEG, DEFAULT_JPEG_QUALITY); - } - throw new Error('image not a canvas'); - } catch (err) { - const error = new Error('autoOrientImage: Failed to process image'); - error.originalError = err; - throw error; - } -} diff --git a/ts/util/handleImageAttachment.ts b/ts/util/handleImageAttachment.ts new file mode 100644 index 000000000..48340c92a --- /dev/null +++ b/ts/util/handleImageAttachment.ts @@ -0,0 +1,61 @@ +// Copyright 2021 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only + +import path from 'path'; +import { MIMEType, IMAGE_JPEG } from '../types/MIME'; +import { + InMemoryAttachmentDraftType, + canBeTranscoded, +} from '../types/Attachment'; +import { imageToBlurHash } from './imageToBlurHash'; +import { scaleImageToLevel } from './scaleImageToLevel'; + +export async function handleImageAttachment( + file: File +): Promise { + const blurHash = await imageToBlurHash(file); + + const { contentType, file: resizedBlob, fileName } = await autoScale({ + contentType: file.type as MIMEType, + fileName: file.name, + file, + }); + const data = await window.Signal.Types.VisualAttachment.blobToArrayBuffer( + resizedBlob + ); + return { + fileName: fileName || file.name, + contentType, + data, + size: data.byteLength, + blurHash, + }; +} + +export async function autoScale({ + contentType, + file, + fileName, +}: { + contentType: MIMEType; + file: File | Blob; + fileName: string; +}): Promise<{ + contentType: MIMEType; + file: Blob; + fileName: string; +}> { + if (!canBeTranscoded({ contentType })) { + return { contentType, file, fileName }; + } + + const blob = await scaleImageToLevel(file, true); + + const { name } = path.parse(fileName); + + return { + contentType: IMAGE_JPEG, + file: blob, + fileName: `${name}.jpeg`, + }; +} diff --git a/ts/util/scaleImageToLevel.ts b/ts/util/scaleImageToLevel.ts index a994d3b60..02ab09cfe 100644 --- a/ts/util/scaleImageToLevel.ts +++ b/ts/util/scaleImageToLevel.ts @@ -112,9 +112,6 @@ export async function scaleImageToLevel( throw new Error('image not a canvas'); } ({ image } = data); - if (!(image instanceof HTMLCanvasElement)) { - throw new Error('image not a canvas'); - } } catch (err) { const error = new Error('scaleImageToLevel: Failed to process image'); error.originalError = err; diff --git a/ts/views/conversation_view.ts b/ts/views/conversation_view.ts index 13b3a8cc2..08406a8b7 100644 --- a/ts/views/conversation_view.ts +++ b/ts/views/conversation_view.ts @@ -12,7 +12,7 @@ import { } from '../types/Attachment'; import type { StickerPackType as StickerPackDBType } from '../sql/Interface'; import * as Stickers from '../types/Stickers'; -import { IMAGE_JPEG, IMAGE_WEBP } from '../types/MIME'; +import { MIMEType, IMAGE_JPEG, IMAGE_WEBP } from '../types/MIME'; import { ConversationModel } from '../models/conversations'; import { GroupV2PendingMemberType, @@ -43,8 +43,6 @@ import { import { getMessagesByConversation } from '../state/selectors/conversations'; import { ConversationDetailsMembershipList } from '../components/conversation/conversation-details/ConversationDetailsMembershipList'; import { showSafetyNumberChangeDialog } from '../shims/showSafetyNumberChangeDialog'; -import { autoOrientImage } from '../util/autoOrientImage'; -import { canvasToBlob } from '../util/canvasToBlob'; import { LinkPreviewImage, LinkPreviewResult, @@ -52,6 +50,10 @@ import { } from '../types/LinkPreview'; import * as LinkPreview from '../types/LinkPreview'; import { SignalService as Proto } from '../protobuf'; +import { + autoScale, + handleImageAttachment, +} from '../util/handleImageAttachment'; type AttachmentOptions = { messageId: string; @@ -1858,7 +1860,7 @@ Whisper.ConversationView = Whisper.View.extend({ return toWrite; }, - async maybeAddAttachment(file: any) { + async maybeAddAttachment(file: File): Promise { if (!file) { return; } @@ -1892,8 +1894,10 @@ Whisper.ConversationView = Whisper.View.extend({ return; } + const fileType = file.type as MIMEType; + // You can't add a non-image attachment if you already have attachments staged - if (!MIME.isImage(file.type) && draftAttachments.length > 0) { + if (!MIME.isImage(fileType) && draftAttachments.length > 0) { this.showToast(Whisper.CannotMixImageAndNonImageAttachmentsToast); return; } @@ -1901,10 +1905,10 @@ Whisper.ConversationView = Whisper.View.extend({ let attachment: InMemoryAttachmentDraftType; try { - if (window.Signal.Util.GoogleChrome.isImageTypeSupported(file.type)) { - attachment = await this.handleImageAttachment(file); + if (window.Signal.Util.GoogleChrome.isImageTypeSupported(fileType)) { + attachment = await handleImageAttachment(file); } else if ( - window.Signal.Util.GoogleChrome.isVideoTypeSupported(file.type) + window.Signal.Util.GoogleChrome.isVideoTypeSupported(fileType) ) { attachment = await this.handleVideoAttachment(file); } else { @@ -1912,20 +1916,20 @@ Whisper.ConversationView = Whisper.View.extend({ attachment = { data, size: data.byteLength, - contentType: file.type, + contentType: fileType, fileName: file.name, }; } } catch (e) { window.log.error( - `Was unable to generate thumbnail for file type ${file.type}`, + `Was unable to generate thumbnail for fileType ${fileType}`, e && e.stack ? e.stack : e ); const data = await this.arrayBufferFromFile(file); attachment = { data, size: data.byteLength, - contentType: file.type, + contentType: fileType, fileName: file.name, }; } @@ -2009,154 +2013,6 @@ Whisper.ConversationView = Whisper.View.extend({ } }, - async handleImageAttachment(file: any): Promise { - const blurHash = await window.imageToBlurHash(file); - if (MIME.isJPEG(file.type)) { - const rotatedBlob = await autoOrientImage(file); - const { contentType, file: resizedBlob, fileName } = await this.autoScale( - { - contentType: file.type, - fileName: file.name, - file: rotatedBlob, - } - ); - const data = await VisualAttachment.blobToArrayBuffer(resizedBlob); - - return { - fileName: fileName || file.name, - contentType, - data, - size: data.byteLength, - blurHash, - }; - } - - const { contentType, file: resizedBlob, fileName } = await this.autoScale({ - contentType: file.type, - fileName: file.name, - file, - }); - const data = await VisualAttachment.blobToArrayBuffer(resizedBlob); - return { - fileName: fileName || file.name, - contentType, - data, - size: data.byteLength, - blurHash, - }; - }, - - autoScale(attachment: any) { - const { contentType, file, fileName } = attachment; - if (contentType.split('/')[0] !== 'image' || contentType === 'image/tiff') { - // nothing to do - return Promise.resolve(attachment); - } - - return new Promise((resolve, reject) => { - const url = URL.createObjectURL(file); - const img = document.createElement('img'); - img.onload = async () => { - URL.revokeObjectURL(url); - - const maxSize = 6000 * 1024; - const maxHeight = 4096; - const maxWidth = 4096; - if ( - img.naturalWidth <= maxWidth && - img.naturalHeight <= maxHeight && - file.size <= maxSize - ) { - resolve(attachment); - return; - } - - const gifMaxSize = 25000 * 1024; - if (file.type === 'image/gif' && file.size <= gifMaxSize) { - resolve(attachment); - return; - } - - if (file.type === 'image/gif') { - reject(new Error('GIF is too large')); - return; - } - - const targetContentType = IMAGE_JPEG; - const canvas = window.loadImage.scale(img, { - canvas: true, - maxWidth, - maxHeight, - }); - - let quality = 0.95; - let i = 4; - let blob; - do { - i -= 1; - // We want to do these operations in serial. - // eslint-disable-next-line no-await-in-loop - blob = await canvasToBlob(canvas, targetContentType, quality); - quality = (quality * maxSize) / blob.size; - // NOTE: During testing with a large image, we observed the - // `quality` value being > 1. Should we clamp it to [0.5, 1.0]? - // See: https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement/toBlob#Syntax - if (quality < 0.5) { - quality = 0.5; - } - } while (i > 0 && blob.size > maxSize); - - resolve({ - ...attachment, - fileName: this.fixExtension(fileName, targetContentType), - contentType: targetContentType, - file: blob, - }); - }; - img.onerror = ( - _event: unknown, - _source: unknown, - _lineno: unknown, - _colno: unknown, - error: Error = new Error('Failed to load image for auto-scaling') - ) => { - URL.revokeObjectURL(url); - reject(error); - }; - img.src = url; - }); - }, - - getFileName(fileName?: string) { - if (!fileName) { - return ''; - } - - if (!fileName.includes('.')) { - return fileName; - } - - return fileName.split('.').slice(0, -1).join('.'); - }, - - getType(contentType?: string) { - if (!contentType) { - return ''; - } - - if (!contentType.includes('/')) { - return contentType; - } - - return contentType.split('/')[1]; - }, - - fixExtension(fileName: string, contentType: string) { - const extension = this.getType(contentType); - const name = this.getFileName(fileName); - return `${name}.${extension}`; - }, - markAllAsVerifiedDefault(unverified: any) { return Promise.all( unverified.map((contact: any) => { @@ -4324,11 +4180,12 @@ Whisper.ConversationView = Whisper.View.extend({ // Ensure that this file is either small enough or is resized to meet our // requirements for attachments - const withBlob = await this.autoScale({ + const withBlob = await autoScale({ contentType: fullSizeImage.contentType, file: new Blob([fullSizeImage.data], { type: fullSizeImage.contentType, }), + fileName: title, }); const data = await this.arrayBufferFromFile(withBlob.file);