Transcode heic/heif images

ACKNOWLEDGMENTS.md

@@ -1494,6 +1494,10 @@ Signal Desktop makes use of the following open source projects.
 
     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+## heic-convert
+
+    License: ISC
+
 ## history
 
     MIT License

main.js

@@ -125,8 +125,10 @@ const { ChallengeMainHandler } = require('./ts/main/challengeMain');
 const { NativeThemeNotifier } = require('./ts/main/NativeThemeNotifier');
 const { PowerChannel } = require('./ts/main/powerChannel');
 const { maybeParseUrl, setUrlSearchParams } = require('./ts/util/url');
+const { getHeicConverter } = require('./ts/workers/heicConverterMain');
 
 const sql = new MainSQL();
+const heicConverter = getHeicConverter();
 
 let systemTrayService;
 const systemTraySettingCache = new SystemTraySettingCache(
@@ -640,6 +642,11 @@ ipc.on('title-bar-double-click', () => {
   }
 });
 
+ipc.on('convert-image', async (event, uuid, data) => {
+  const { error, response } = await heicConverter(uuid, data);
+  event.reply(`convert-image:${uuid}`, { error, response });
+});
+
 let isReadyForUpdates = false;
 async function readyForUpdates() {
   if (isReadyForUpdates) {

package.json

@@ -55,10 +55,11 @@
     "build:dev": "run-s --print-label build:grunt build:typed-scss build:webpack",
     "build:grunt": "yarn grunt",
     "build:typed-scss": "tsm sticker-creator",
-    "build:webpack": "run-p build:webpack:sticker-creator build:webpack:preload build:webpack:sql-worker",
+    "build:webpack": "run-p build:webpack:sticker-creator build:webpack:preload build:webpack:sql-worker build:webpack:heic-worker",
     "build:webpack:sticker-creator": "cross-env NODE_ENV=production webpack",
     "build:webpack:preload": "cross-env NODE_ENV=production webpack -c webpack-preload.config.ts",
     "build:webpack:sql-worker": "cross-env NODE_ENV=production webpack -c webpack-sql-worker.config.ts",
+    "build:webpack:heic-worker": "cross-env NODE_ENV=production webpack -c webpack-heic-worker.config.ts",
     "build:electron": "electron-builder --config.extraMetadata.environment=$SIGNAL_ENV",
     "build:release": "cross-env SIGNAL_ENV=production yarn build:electron -- --config.directories.output=release",
     "build:fuses": "node scripts/fuse-electron.js",
@@ -101,6 +102,7 @@
     "glob": "7.1.6",
     "google-libphonenumber": "3.2.17",
     "got": "8.3.2",
+    "heic-convert": "^1.2.4",
     "history": "4.9.0",
     "humanize-duration": "3.26.0",
     "intl-tel-input": "12.1.15",
@@ -385,6 +387,7 @@
       ]
     },
     "asarUnpack": [
+      "ts/workers/heicConverter.bundle.js",
       "ts/sql/mainWorker.bundle.js",
       "node_modules/better-sqlite3/build/Release/better_sqlite3.node"
     ],

ts/components/ForwardMessageModal.stories.tsx

@@ -10,17 +10,16 @@ import { text } from '@storybook/addon-knobs';
 import enMessages from '../../_locales/en/messages.json';
 import { AttachmentType } from '../types/Attachment';
 import { ForwardMessageModal, PropsType } from './ForwardMessageModal';
-import { IMAGE_JPEG, MIMEType, VIDEO_MP4 } from '../types/MIME';
+import { IMAGE_JPEG, VIDEO_MP4, stringToMIMEType } from '../types/MIME';
 import { getDefaultConversation } from '../test-both/helpers/getDefaultConversation';
 import { setup as setupI18n } from '../../js/modules/i18n';
 
 const createAttachment = (
   props: Partial<AttachmentType> = {}
 ): AttachmentType => ({
-  contentType: text(
+  contentType: stringToMIMEType(
-    'attachment contentType',
+    text('attachment contentType', props.contentType || '')
-    props.contentType || ''
+  ),
-  ) as MIMEType,
   fileName: text('attachment fileName', props.fileName || ''),
   screenshot: props.screenshot,
   url: text('attachment url', props.url || ''),

ts/components/Lightbox.stories.tsx

@@ -11,9 +11,9 @@ import { Lightbox, Props } from './Lightbox';
 import {
   AUDIO_MP3,
   IMAGE_JPEG,
-  MIMEType,
   VIDEO_MP4,
   VIDEO_QUICKTIME,
+  stringToMIMEType,
 } from '../types/MIME';
 import { setup as setupI18n } from '../../js/modules/i18n';
 import enMessages from '../../_locales/en/messages.json';
@@ -94,7 +94,7 @@ story.add('Video (View Once)', () => {
 
 story.add('Unsupported Image Type', () => {
   const props = createProps({
-    contentType: 'image/tiff' as MIMEType,
+    contentType: stringToMIMEType('image/tiff'),
     objectURL: 'unsupported-image.tiff',
   });
 

ts/components/conversation/AttachmentList.stories.tsx

@@ -11,8 +11,8 @@ import {
   AUDIO_MP3,
   IMAGE_GIF,
   IMAGE_JPEG,
-  MIMEType,
   VIDEO_MP4,
+  stringToMIMEType,
 } from '../../types/MIME';
 import { setup as setupI18n } from '../../../js/modules/i18n';
 import enMessages from '../../../_locales/en/messages.json';
@@ -83,7 +83,7 @@ story.add('Multiple with Non-Visual Types', () => {
         url: '/fixtures/tina-rolf-269345-unsplash.jpg',
       },
       {
-        contentType: 'text/plain' as MIMEType,
+        contentType: stringToMIMEType('text/plain'),
         fileName: 'lorem-ipsum.txt',
         url: '/fixtures/lorem-ipsum.txt',
       },

ts/components/conversation/ImageGrid.stories.tsx

@@ -13,8 +13,8 @@ import {
   IMAGE_JPEG,
   IMAGE_PNG,
   IMAGE_WEBP,
-  MIMEType,
   VIDEO_MP4,
+  stringToMIMEType,
 } from '../../types/MIME';
 import { setup as setupI18n } from '../../../js/modules/i18n';
 import enMessages from '../../../_locales/en/messages.json';
@@ -273,7 +273,7 @@ story.add('Mixed Content Types', () => {
         width: 800,
       },
       {
-        contentType: 'text/plain' as MIMEType,
+        contentType: stringToMIMEType('text/plain'),
         fileName: 'lorem-ipsum.txt',
         url: '/fixtures/lorem-ipsum.txt',
       },

ts/components/conversation/Message.stories.tsx

@@ -17,8 +17,8 @@ import {
   IMAGE_JPEG,
   IMAGE_PNG,
   IMAGE_WEBP,
-  MIMEType,
   VIDEO_MP4,
+  stringToMIMEType,
 } from '../../types/MIME';
 import { MessageAudio } from './MessageAudio';
 import { computePeaks } from '../GlobalAudioContext';
@@ -959,7 +959,7 @@ story.add('Other File Type', () => {
   const props = createProps({
     attachments: [
       {
-        contentType: 'text/plain' as MIMEType,
+        contentType: stringToMIMEType('text/plain'),
         fileName: 'my-resume.txt',
         url: 'my-resume.txt',
       },
@@ -974,7 +974,7 @@ story.add('Other File Type with Caption', () => {
   const props = createProps({
     attachments: [
       {
-        contentType: 'text/plain' as MIMEType,
+        contentType: stringToMIMEType('text/plain'),
         fileName: 'my-resume.txt',
         url: 'my-resume.txt',
       },
@@ -990,7 +990,7 @@ story.add('Other File Type with Long Filename', () => {
   const props = createProps({
     attachments: [
       {
-        contentType: 'text/plain' as MIMEType,
+        contentType: stringToMIMEType('text/plain'),
         fileName:
           'INSERT-APP-NAME_INSERT-APP-APPLE-ID_AppStore_AppsGamesWatch.psd.zip',
         url: 'a2/a2334324darewer4234',
@@ -1081,7 +1081,9 @@ story.add('Dangerous File Type', () => {
   const props = createProps({
     attachments: [
       {
-        contentType: 'application/vnd.microsoft.portable-executable' as MIMEType,
+        contentType: stringToMIMEType(
+          'application/vnd.microsoft.portable-executable'
+        ),
         fileName: 'terrible.exe',
         url: 'terrible.exe',
       },

ts/components/conversation/Quote.stories.tsx

@@ -15,8 +15,8 @@ import {
   AUDIO_MP3,
   IMAGE_PNG,
   LONG_MESSAGE,
-  MIMEType,
   VIDEO_MP4,
+  stringToMIMEType,
 } from '../../types/MIME';
 import { Props, Quote } from './Quote';
 import { setup as setupI18n } from '../../../js/modules/i18n';
@@ -392,7 +392,7 @@ story.add('Voice Message Attachment', () => {
 story.add('Other File Only', () => {
   const props = createProps({
     rawAttachment: {
-      contentType: 'application/json' as MIMEType,
+      contentType: stringToMIMEType('application/json'),
       fileName: 'great-data.json',
       isVoiceMessage: false,
     },
@@ -420,7 +420,7 @@ story.add('Media Tap-to-View', () => {
 story.add('Other File Attachment', () => {
   const props = createProps({
     rawAttachment: {
-      contentType: 'application/json' as MIMEType,
+      contentType: stringToMIMEType('application/json'),
       fileName: 'great-data.json',
       isVoiceMessage: false,
     },

ts/components/conversation/StagedGenericAttachment.stories.tsx

@@ -7,7 +7,7 @@ import { text } from '@storybook/addon-knobs';
 import { action } from '@storybook/addon-actions';
 
 import { AttachmentType } from '../../types/Attachment';
-import { MIMEType } from '../../types/MIME';
+import { stringToMIMEType } from '../../types/MIME';
 import { setup as setupI18n } from '../../../js/modules/i18n';
 import enMessages from '../../../_locales/en/messages.json';
 import { Props, StagedGenericAttachment } from './StagedGenericAttachment';
@@ -28,17 +28,16 @@ const createProps = (overrideProps: Partial<Props> = {}): Props => ({
 const createAttachment = (
   props: Partial<AttachmentType> = {}
 ): AttachmentType => ({
-  contentType: text(
+  contentType: stringToMIMEType(
-    'attachment contentType',
+    text('attachment contentType', props.contentType || '')
-    props.contentType || ''
+  ),
-  ) as MIMEType,
   fileName: text('attachment fileName', props.fileName || ''),
   url: '',
 });
 
 story.add('Text File', () => {
   const attachment = createAttachment({
-    contentType: 'text/plain' as MIMEType,
+    contentType: stringToMIMEType('text/plain'),
     fileName: 'manifesto.txt',
   });
   const props = createProps({ attachment });
@@ -48,7 +47,7 @@ story.add('Text File', () => {
 
 story.add('Long Name', () => {
   const attachment = createAttachment({
-    contentType: 'text/plain' as MIMEType,
+    contentType: stringToMIMEType('text/plain'),
     fileName: 'this-is-my-very-important-manifesto-you-must-read-it.txt',
   });
   const props = createProps({ attachment });
@@ -58,7 +57,7 @@ story.add('Long Name', () => {
 
 story.add('Long Extension', () => {
   const attachment = createAttachment({
-    contentType: 'text/plain' as MIMEType,
+    contentType: stringToMIMEType('text/plain'),
     fileName: 'manifesto.reallylongtxt',
   });
   const props = createProps({ attachment });

ts/components/conversation/StagedLinkPreview.stories.tsx

@@ -7,7 +7,7 @@ import { date, text, withKnobs } from '@storybook/addon-knobs';
 import { action } from '@storybook/addon-actions';
 
 import { AttachmentType } from '../../types/Attachment';
-import { MIMEType } from '../../types/MIME';
+import { stringToMIMEType } from '../../types/MIME';
 import { setup as setupI18n } from '../../../js/modules/i18n';
 import enMessages from '../../../_locales/en/messages.json';
 import { Props, StagedLinkPreview } from './StagedLinkPreview';
@@ -27,10 +27,9 @@ story.addDecorator((withKnobs as any)({ escapeHTML: false }));
 const createAttachment = (
   props: Partial<AttachmentType> = {}
 ): AttachmentType => ({
-  contentType: text(
+  contentType: stringToMIMEType(
-    'attachment contentType',
+    text('attachment contentType', props.contentType || '')
-    props.contentType || ''
+  ),
-  ) as MIMEType,
   fileName: text('attachment fileName', props.fileName || ''),
   url: text('attachment url', props.url || ''),
 });
@@ -69,7 +68,7 @@ story.add('Image', () => {
   const props = createProps({
     image: createAttachment({
       url: '/fixtures/kitten-4-112-112.jpg',
-      contentType: 'image/jpeg' as MIMEType,
+      contentType: stringToMIMEType('image/jpeg'),
     }),
   });
 
@@ -83,7 +82,7 @@ story.add('Image, No Title Or Description', () => {
     domain: 'instagram.com',
     image: createAttachment({
       url: '/fixtures/kitten-4-112-112.jpg',
-      contentType: 'image/jpeg' as MIMEType,
+      contentType: stringToMIMEType('image/jpeg'),
     }),
   });
 
@@ -112,7 +111,7 @@ story.add('Image, Long Title Without Description', () => {
     title: LONG_TITLE,
     image: createAttachment({
       url: '/fixtures/kitten-4-112-112.jpg',
-      contentType: 'image/jpeg' as MIMEType,
+      contentType: stringToMIMEType('image/jpeg'),
     }),
   });
 
@@ -125,7 +124,7 @@ story.add('Image, Long Title And Description', () => {
     description: LONG_DESCRIPTION,
     image: createAttachment({
       url: '/fixtures/kitten-4-112-112.jpg',
-      contentType: 'image/jpeg' as MIMEType,
+      contentType: stringToMIMEType('image/jpeg'),
     }),
   });
 
@@ -139,7 +138,7 @@ story.add('Everything: image, title, description, and date', () => {
     date: Date.now(),
     image: createAttachment({
       url: '/fixtures/kitten-4-112-112.jpg',
-      contentType: 'image/jpeg' as MIMEType,
+      contentType: stringToMIMEType('image/jpeg'),
     }),
   });
 

ts/components/conversation/media-gallery/MediaGridItem.stories.tsx

@@ -10,7 +10,7 @@ import { setup as setupI18n } from '../../../../js/modules/i18n';
 import enMessages from '../../../../_locales/en/messages.json';
 import { MediaItemType } from '../../LightboxGallery';
 import { AttachmentType } from '../../../types/Attachment';
-import { MIMEType } from '../../../types/MIME';
+import { stringToMIMEType } from '../../../types/MIME';
 
 import { MediaGridItem, Props } from './MediaGridItem';
 import { Message } from './types/Message';
@@ -40,7 +40,9 @@ const createMediaItem = (
     'thumbnailObjectUrl',
     overrideProps.thumbnailObjectUrl || ''
   ),
-  contentType: text('contentType', overrideProps.contentType || '') as MIMEType,
+  contentType: stringToMIMEType(
+    text('contentType', overrideProps.contentType || '')
+  ),
   index: 0,
   attachment: {} as AttachmentType, // attachment not useful in the component
   message: {} as Message, // message not used in the component
@@ -49,7 +51,7 @@ const createMediaItem = (
 story.add('Image', () => {
   const mediaItem = createMediaItem({
     thumbnailObjectUrl: '/fixtures/kitten-1-64-64.jpg',
-    contentType: 'image/jpeg' as MIMEType,
+    contentType: stringToMIMEType('image/jpeg'),
   });
 
   const props = createProps({
@@ -62,7 +64,7 @@ story.add('Image', () => {
 story.add('Video', () => {
   const mediaItem = createMediaItem({
     thumbnailObjectUrl: '/fixtures/kitten-2-64-64.jpg',
-    contentType: 'video/mp4' as MIMEType,
+    contentType: stringToMIMEType('video/mp4'),
   });
 
   const props = createProps({
@@ -74,7 +76,7 @@ story.add('Video', () => {
 
 story.add('Missing Image', () => {
   const mediaItem = createMediaItem({
-    contentType: 'image/jpeg' as MIMEType,
+    contentType: stringToMIMEType('image/jpeg'),
   });
 
   const props = createProps({
@@ -86,7 +88,7 @@ story.add('Missing Image', () => {
 
 story.add('Missing Video', () => {
   const mediaItem = createMediaItem({
-    contentType: 'video/mp4' as MIMEType,
+    contentType: stringToMIMEType('video/mp4'),
   });
 
   const props = createProps({
@@ -99,7 +101,7 @@ story.add('Missing Video', () => {
 story.add('Broken Image', () => {
   const mediaItem = createMediaItem({
     thumbnailObjectUrl: '/missing-fixtures/nope.jpg',
-    contentType: 'image/jpeg' as MIMEType,
+    contentType: stringToMIMEType('image/jpeg'),
   });
 
   const props = createProps({
@@ -112,7 +114,7 @@ story.add('Broken Image', () => {
 story.add('Broken Video', () => {
   const mediaItem = createMediaItem({
     thumbnailObjectUrl: '/missing-fixtures/nope.mp4',
-    contentType: 'video/mp4' as MIMEType,
+    contentType: stringToMIMEType('video/mp4'),
   });
 
   const props = createProps({
@@ -124,7 +126,7 @@ story.add('Broken Video', () => {
 
 story.add('Other ContentType', () => {
   const mediaItem = createMediaItem({
-    contentType: 'application/text' as MIMEType,
+    contentType: stringToMIMEType('application/text'),
   });
 
   const props = createProps({

ts/heic-convert.d.ts

@@ -0,0 +1,10 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+declare module 'heic-convert' {
+  export default function heicConvert(options: {
+    buffer: Uint8Array;
+    format: string;
+    quality: number;
+  }): Promise<File>;
+}

ts/linkPreviews/linkPreviewFetch.ts

@@ -11,6 +11,7 @@ import {
   IMAGE_PNG,
   IMAGE_WEBP,
   MIMEType,
+  stringToMIMEType,
 } from '../types/MIME';
 
 const USER_AGENT = 'WhatsApp/2';
@@ -163,7 +164,7 @@ const parseContentType = (headerValue: string | null): ParsedContentType => {
   }
 
   return {
-    type: rawType as MIMEType,
+    type: stringToMIMEType(rawType),
     charset,
   };
 };

ts/model-types.d.ts

@@ -207,7 +207,9 @@ export type ConversationAttributesType = {
   customColorId?: string;
   discoveredUnregisteredAt?: number;
   draftAttachments?: Array<{
+    fileName?: string;
     path?: string;
+    pending?: boolean;
     screenshotPath?: string;
   }>;
   draftBodyRanges?: Array<BodyRangeType>;

ts/models/messages.ts

@@ -42,7 +42,7 @@ import {
 } from '../types/Stickers';
 import * as Stickers from '../types/Stickers';
 import { AttachmentType, isImage, isVideo } from '../types/Attachment';
-import { MIMEType, IMAGE_WEBP } from '../types/MIME';
+import { IMAGE_WEBP, stringToMIMEType } from '../types/MIME';
 import { ourProfileKeyService } from '../services/ourProfileKey';
 import {
   SendActionType,
@@ -2425,10 +2425,10 @@ export class MessageModel extends window.Backbone.Model<MessageAttributesType> {
       !firstAttachment ||
       !firstAttachment.contentType ||
       (!GoogleChrome.isImageTypeSupported(
-        firstAttachment.contentType as MIMEType
+        stringToMIMEType(firstAttachment.contentType)
       ) &&
         !GoogleChrome.isVideoTypeSupported(
-          firstAttachment.contentType as MIMEType
+          stringToMIMEType(firstAttachment.contentType)
         ))
     ) {
       return;

ts/state/ducks/composer.ts

@@ -1,6 +1,9 @@
 // Copyright 2021 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
+import { ThunkAction } from 'redux-thunk';
+
+import { StateType as RootStateType } from '../reducer';
 import { AttachmentType } from '../../types/Attachment';
 import { MessageAttributesType } from '../../model-types.d';
 import { LinkPreviewWithDomain } from '../../types/LinkPreview';
@@ -68,11 +71,20 @@ export const actions = {
 };
 
 function replaceAttachments(
+  conversationId: string,
   payload: ReadonlyArray<AttachmentType>
-): ReplaceAttachmentsActionType {
+): ThunkAction<void, RootStateType, unknown, ReplaceAttachmentsActionType> {
-  return {
+  return (dispatch, getState) => {
-    type: REPLACE_ATTACHMENTS,
+    // If the call came from a conversation we are no longer in we do not
-    payload,
+    // update the state.
+    if (getState().conversations.selectedConversationId !== conversationId) {
+      return;
+    }
+
+    dispatch({
+      type: REPLACE_ATTACHMENTS,
+      payload,
+    });
   };
 }
 

ts/test-both/state/ducks/composer_test.ts

@@ -2,8 +2,11 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import { assert } from 'chai';
+import * as sinon from 'sinon';
 
 import { actions, getEmptyState, reducer } from '../../../state/ducks/composer';
+import { noopAction } from '../../../state/ducks/noop';
+import { reducer as rootReducer } from '../../../state/reducer';
 
 import { IMAGE_JPEG } from '../../../types/MIME';
 import { AttachmentType } from '../../../types/Attachment';
@@ -20,27 +23,71 @@ describe('both/state/ducks/composer', () => {
     },
   };
 
+  const getRootStateFunction = (selectedConversationId?: string) => {
+    const state = rootReducer(undefined, noopAction());
+    return () => ({
+      ...state,
+      conversations: {
+        ...state.conversations,
+        selectedConversationId,
+      },
+    });
+  };
+
   describe('replaceAttachments', () => {
     it('replaces the attachments state', () => {
       const { replaceAttachments } = actions;
-      const state = getEmptyState();
+      const dispatch = sinon.spy();
-      const attachments: Array<AttachmentType> = [{ contentType: IMAGE_JPEG }];
-      const nextState = reducer(state, replaceAttachments(attachments));
 
-      assert.deepEqual(nextState.attachments, attachments);
+      const attachments: Array<AttachmentType> = [{ contentType: IMAGE_JPEG }];
+      replaceAttachments('123', attachments)(
+        dispatch,
+        getRootStateFunction('123'),
+        null
+      );
+
+      const action = dispatch.getCall(0).args[0];
+      const state = reducer(getEmptyState(), action);
+      assert.deepEqual(state.attachments, attachments);
     });
 
     it('sets the high quality setting to false when there are no attachments', () => {
       const { replaceAttachments } = actions;
-      const state = getEmptyState();
+      const dispatch = sinon.spy();
       const attachments: Array<AttachmentType> = [];
-      const nextState = reducer(
+
-        { ...state, shouldSendHighQualityAttachments: true },
+      replaceAttachments('123', attachments)(
-        replaceAttachments(attachments)
+        dispatch,
+        getRootStateFunction('123'),
+        null
       );
 
-      assert.deepEqual(nextState.attachments, attachments);
+      const action = dispatch.getCall(0).args[0];
-      assert.isFalse(nextState.shouldSendHighQualityAttachments);
+      const state = reducer(
+        {
+          ...getEmptyState(),
+          shouldSendHighQualityAttachments: true,
+        },
+        action
+      );
+      assert.deepEqual(state.attachments, attachments);
+
+      assert.deepEqual(state.attachments, attachments);
+      assert.isFalse(state.shouldSendHighQualityAttachments);
+    });
+
+    it('does not update redux if the conversation is not selected', () => {
+      const { replaceAttachments } = actions;
+      const dispatch = sinon.spy();
+
+      const attachments: Array<AttachmentType> = [{ contentType: IMAGE_JPEG }];
+      replaceAttachments('123', attachments)(
+        dispatch,
+        getRootStateFunction('456'),
+        null
+      );
+
+      assert.isNull(dispatch.getCall(0));
     });
   });
 

ts/test-electron/linkPreviews/linkPreviewFetch_test.ts

@@ -6,7 +6,7 @@ import * as sinon from 'sinon';
 import * as fs from 'fs';
 import * as path from 'path';
 import AbortController from 'abort-controller';
-import { MIMEType, IMAGE_JPEG } from '../../types/MIME';
+import { IMAGE_JPEG, stringToMIMEType } from '../../types/MIME';
 
 import { typedArrayToArrayBuffer } from '../../Crypto';
 
@@ -1155,7 +1155,7 @@ describe('link preview fetching', () => {
           ),
           {
             data: typedArrayToArrayBuffer(fixture),
-            contentType: contentType as MIMEType,
+            contentType: stringToMIMEType(contentType),
           }
         );
       });

ts/textsecure/downloadAttachment.ts

@@ -54,7 +54,7 @@ export async function downloadAttachment(
     ...omit(attachment, 'digest', 'key'),
 
     contentType: contentType
-      ? MIME.fromString(contentType)
+      ? MIME.stringToMIMEType(contentType)
       : MIME.APPLICATION_OCTET_STREAM,
     data,
   };

ts/types/Attachment.ts

@@ -196,8 +196,12 @@ export async function autoOrientJPEG(
     return attachment;
   }
 
-  // If we haven't downloaded the attachment yet, we won't have the data
+  // If we haven't downloaded the attachment yet, we won't have the data.
-  if (!attachment.data) {
+  // All images go through handleImageAttachment before being sent and thus have
+  // already been scaled to level, oriented, stripped of exif data, and saved
+  // in high quality format. If we want to send the image in HQ we can return
+  // the attachement as-is. Otherwise we'll have to further scale it down.
+  if (!attachment.data || sendHQImages) {
     return attachment;
   }
 
@@ -205,10 +209,7 @@ export async function autoOrientJPEG(
     attachment.data,
     attachment.contentType
   );
-  const xcodedDataBlob = await scaleImageToLevel(
+  const xcodedDataBlob = await scaleImageToLevel(dataBlob, isIncoming);
-    dataBlob,
-    sendHQImages || isIncoming
-  );
   const xcodedDataArrayBuffer = await blobToArrayBuffer(xcodedDataBlob);
 
   // IMPORTANT: We overwrite the existing `data` `ArrayBuffer` losing the original

ts/types/MIME.ts

@@ -3,20 +3,28 @@
 
 export type MIMEType = string & { _mimeTypeBrand: never };
 
-export const APPLICATION_OCTET_STREAM = 'application/octet-stream' as MIMEType;
+export const stringToMIMEType = (value: string): MIMEType => {
-export const APPLICATION_JSON = 'application/json' as MIMEType;
+  return value as MIMEType;
-export const AUDIO_AAC = 'audio/aac' as MIMEType;
+};
-export const AUDIO_MP3 = 'audio/mp3' as MIMEType;
-export const IMAGE_GIF = 'image/gif' as MIMEType;
-export const IMAGE_JPEG = 'image/jpeg' as MIMEType;
-export const IMAGE_PNG = 'image/png' as MIMEType;
-export const IMAGE_WEBP = 'image/webp' as MIMEType;
-export const IMAGE_ICO = 'image/x-icon' as MIMEType;
-export const IMAGE_BMP = 'image/bmp' as MIMEType;
-export const VIDEO_MP4 = 'video/mp4' as MIMEType;
-export const VIDEO_QUICKTIME = 'video/quicktime' as MIMEType;
-export const LONG_MESSAGE = 'text/x-signal-plain' as MIMEType;
 
+export const APPLICATION_OCTET_STREAM = stringToMIMEType(
+  'application/octet-stream'
+);
+export const APPLICATION_JSON = stringToMIMEType('application/json');
+export const AUDIO_AAC = stringToMIMEType('audio/aac');
+export const AUDIO_MP3 = stringToMIMEType('audio/mp3');
+export const IMAGE_GIF = stringToMIMEType('image/gif');
+export const IMAGE_JPEG = stringToMIMEType('image/jpeg');
+export const IMAGE_PNG = stringToMIMEType('image/png');
+export const IMAGE_WEBP = stringToMIMEType('image/webp');
+export const IMAGE_ICO = stringToMIMEType('image/x-icon');
+export const IMAGE_BMP = stringToMIMEType('image/bmp');
+export const VIDEO_MP4 = stringToMIMEType('video/mp4');
+export const VIDEO_QUICKTIME = stringToMIMEType('video/quicktime');
+export const LONG_MESSAGE = stringToMIMEType('text/x-signal-plain');
+
+export const isHeic = (value: string): boolean =>
+  value === 'image/heic' || value === 'image/heif';
 export const isGif = (value: string): value is MIMEType =>
   value === 'image/gif';
 export const isJPEG = (value: string): value is MIMEType =>
@@ -31,7 +39,3 @@ export const isAudio = (value: string): value is MIMEType =>
   Boolean(value) && value.startsWith('audio/') && !value.endsWith('aiff');
 export const isLongMessage = (value: unknown): value is MIMEType =>
   value === LONG_MESSAGE;
-
-export const fromString = (value: string): MIMEType => {
-  return value as MIMEType;
-};

ts/util/handleImageAttachment.ts

@@ -2,7 +2,10 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import path from 'path';
-import { MIMEType, IMAGE_JPEG } from '../types/MIME';
+import { ipcRenderer } from 'electron';
+import { v4 as genUuid } from 'uuid';
+
+import { IMAGE_JPEG, MIMEType, isHeic, stringToMIMEType } from '../types/MIME';
 import {
   InMemoryAttachmentDraftType,
   canBeTranscoded,
@@ -13,16 +16,37 @@ import { scaleImageToLevel } from './scaleImageToLevel';
 export async function handleImageAttachment(
   file: File
 ): Promise<InMemoryAttachmentDraftType> {
-  const blurHash = await imageToBlurHash(file);
+  let processedFile: File | Blob = file;
+
+  if (isHeic(file.type)) {
+    const uuid = genUuid();
+    const arrayBuffer = await file.arrayBuffer();
+
+    const convertedFile = await new Promise<File>((resolve, reject) => {
+      ipcRenderer.once(`convert-image:${uuid}`, (_, { error, response }) => {
+        if (response) {
+          resolve(response);
+        } else {
+          reject(error);
+        }
+      });
+      ipcRenderer.send('convert-image', uuid, arrayBuffer);
+    });
+
+    processedFile = new Blob([convertedFile]);
+  }
 
   const { contentType, file: resizedBlob, fileName } = await autoScale({
-    contentType: file.type as MIMEType,
+    contentType: isHeic(file.type) ? IMAGE_JPEG : stringToMIMEType(file.type),
     fileName: file.name,
-    file,
+    file: processedFile,
   });
+
   const data = await window.Signal.Types.VisualAttachment.blobToArrayBuffer(
     resizedBlob
   );
+  const blurHash = await imageToBlurHash(resizedBlob);
+
   return {
     fileName: fileName || file.name,
     contentType,

ts/util/lint/exceptions.json

@@ -8384,6 +8384,12 @@
     "updated": "2021-05-07T20:07:48.358Z",
     "reasonDetail": "isn't jquery"
   },
+  {
+    "rule": "jQuery-$(",
+    "path": "node_modules/libheif-js/libheif/libheif.js",
+    "reasonCategory": "falseMatch",
+    "updated": "2021-07-16T22:15:43.772Z"
+  },
   {
     "rule": "jQuery-append(",
     "path": "node_modules/liftup/node_modules/braces/lib/expand.js",

ts/util/lint/linter.ts

@@ -56,6 +56,7 @@ const excludedFilesRegexps = [
   '^sticker-creator/dist/bundle.js',
   '^test/test.js',
   '^ts/test[^/]*/.+',
+  '^ts/workers/heicConverter.bundle.js',
   '^ts/sql/mainWorker.bundle.js',
 
   // Copied from dependency

ts/views/conversation_view.ts

@@ -3,6 +3,7 @@
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
+import nodePath from 'path';
 import {
   AttachmentDraftType,
   AttachmentType,
@@ -12,7 +13,12 @@ import {
 } from '../types/Attachment';
 import type { StickerPackType as StickerPackDBType } from '../sql/Interface';
 import * as Stickers from '../types/Stickers';
-import { MIMEType, IMAGE_JPEG, IMAGE_WEBP } from '../types/MIME';
+import {
+  IMAGE_JPEG,
+  IMAGE_WEBP,
+  isHeic,
+  stringToMIMEType,
+} from '../types/MIME';
 import { ConversationModel } from '../models/conversations';
 import {
   GroupV2PendingMemberType,
@@ -1721,7 +1727,13 @@ Whisper.ConversationView = Whisper.View.extend({
     const { model }: { model: ConversationModel } = this;
     const onDisk = await this.writeDraftAttachment(attachment);
 
-    const draftAttachments = model.get('draftAttachments') || [];
+    // Remove any pending attachments that were transcoding
+    const draftAttachments = (model.get('draftAttachments') || []).filter(
+      draftAttachment =>
+        !draftAttachment.pending &&
+        nodePath.parse(String(draftAttachment.fileName)).name !==
+          attachment.fileName
+    );
     this.model.set({
       draftAttachments: [...draftAttachments, onDisk],
     });
@@ -1859,8 +1871,10 @@ Whisper.ConversationView = Whisper.View.extend({
   },
 
   updateAttachmentsView() {
+    const { model }: { model: ConversationModel } = this;
     const draftAttachments = this.model.get('draftAttachments') || [];
     window.reduxActions.composer.replaceAttachments(
+      model.get('id'),
       draftAttachments.map((att: AttachmentType) =>
         this.resolveOnDiskAttachment(att)
       )
@@ -1928,7 +1942,7 @@ Whisper.ConversationView = Whisper.View.extend({
       return;
     }
 
-    const fileType = file.type as MIMEType;
+    const fileType = stringToMIMEType(file.type);
 
     // You can't add a non-image attachment if you already have attachments staged
     if (!MIME.isImage(fileType) && draftAttachments.length > 0) {
@@ -1939,7 +1953,23 @@ Whisper.ConversationView = Whisper.View.extend({
     let attachment: InMemoryAttachmentDraftType;
 
     try {
-      if (window.Signal.Util.GoogleChrome.isImageTypeSupported(fileType)) {
+      if (
+        window.Signal.Util.GoogleChrome.isImageTypeSupported(fileType) ||
+        isHeic(fileType)
+      ) {
+        // Add a pending attachment since transcoding may take a while
+        this.model.set({
+          draftAttachments: [
+            ...draftAttachments,
+            {
+              contentType: IMAGE_JPEG,
+              fileName: nodePath.parse(file.name).name,
+              pending: true,
+            },
+          ],
+        });
+        this.updateAttachmentsView();
+
         attachment = await handleImageAttachment(file);
       } else if (
         window.Signal.Util.GoogleChrome.isVideoTypeSupported(fileType)

ts/workers/heicConverterMain.ts

@@ -0,0 +1,68 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { join } from 'path';
+import { Worker } from 'worker_threads';
+
+export type WrappedWorkerRequest = {
+  readonly uuid: string;
+  readonly data: ArrayBuffer;
+};
+
+export type WrappedWorkerResponse = {
+  readonly uuid: string;
+  readonly error: string | undefined;
+  readonly response?: File;
+};
+
+const ASAR_PATTERN = /app\.asar$/;
+
+export function getHeicConverter(): (
+  uuid: string,
+  data: ArrayBuffer
+) => Promise<WrappedWorkerResponse> {
+  let appDir = join(__dirname, '..', '..');
+  let isBundled = false;
+  if (ASAR_PATTERN.test(appDir)) {
+    appDir = appDir.replace(ASAR_PATTERN, 'app.asar.unpacked');
+    isBundled = true;
+  }
+  const scriptDir = join(appDir, 'ts', 'workers');
+  const worker = new Worker(
+    join(
+      scriptDir,
+      isBundled ? 'heicConverter.bundle.js' : 'heicConverterWorker.js'
+    )
+  );
+
+  const ResponseMap = new Map<
+    string,
+    (response: WrappedWorkerResponse) => void
+  >();
+
+  worker.on('message', (wrappedResponse: WrappedWorkerResponse) => {
+    const { uuid } = wrappedResponse;
+
+    const resolve = ResponseMap.get(uuid);
+    if (!resolve) {
+      throw new Error(`Cannot find resolver for ${uuid}`);
+    }
+
+    resolve(wrappedResponse);
+  });
+
+  return async (uuid, data) => {
+    const wrappedRequest: WrappedWorkerRequest = {
+      uuid,
+      data,
+    };
+
+    const result = new Promise<WrappedWorkerResponse>(resolve => {
+      ResponseMap.set(uuid, resolve);
+    });
+
+    worker.postMessage(wrappedRequest);
+
+    return result;
+  };
+}

ts/workers/heicConverterWorker.ts

@@ -0,0 +1,39 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import heicConvert from 'heic-convert';
+import { parentPort } from 'worker_threads';
+
+import {
+  WrappedWorkerRequest,
+  WrappedWorkerResponse,
+} from './heicConverterMain';
+
+if (!parentPort) {
+  throw new Error('Must run as a worker thread');
+}
+
+const port = parentPort;
+
+function respond(uuid: string, error: Error | undefined, response?: File) {
+  const wrappedResponse: WrappedWorkerResponse = {
+    uuid,
+    error: error ? error.stack : undefined,
+    response,
+  };
+  port.postMessage(wrappedResponse);
+}
+
+port.on('message', async ({ uuid, data }: WrappedWorkerRequest) => {
+  try {
+    const file = await heicConvert({
+      buffer: new Uint8Array(data),
+      format: 'JPEG',
+      quality: 0.75,
+    });
+
+    respond(uuid, undefined, file);
+  } catch (error) {
+    respond(uuid, error, undefined);
+  }
+});

webpack-heic-worker.config.ts

@@ -0,0 +1,23 @@
+// Copyright 2019-2020 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { resolve } from 'path';
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { Configuration } from 'webpack';
+
+const context = __dirname;
+
+const workerConfig: Configuration = {
+  context,
+  mode: 'development',
+  devtool: false,
+  entry: ['./ts/workers/heicConverterMain.js'],
+  target: 'node',
+  output: {
+    path: resolve(context, 'ts', 'workers'),
+    filename: 'heicConverter.bundle.js',
+    publicPath: './',
+  },
+};
+
+export default [workerConfig];

yarn.lock

@@ -9708,6 +9708,22 @@ he@1.2.0, he@^1.2.0:
   resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
   integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
 
+heic-convert@^1.2.4:
+  version "1.2.4"
+  resolved "https://registry.yarnpkg.com/heic-convert/-/heic-convert-1.2.4.tgz#605820f98ace3949a40fc7b263ee0bc573a0176b"
+  integrity sha512-klJHyv+BqbgKiCQvCqI9IKIvweCcohDuDl0Jphearj8+16+v8eff2piVevHqq4dW9TK0r1onTR6PKHP1I4hdbA==
+  dependencies:
+    heic-decode "^1.1.2"
+    jpeg-js "^0.4.1"
+    pngjs "^3.4.0"
+
+heic-decode@^1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/heic-decode/-/heic-decode-1.1.2.tgz#974701666432e31ed64b2263a1ece7cff5218209"
+  integrity sha512-UF8teegxvzQPdSTcx5frIUhitNDliz/9Pui0JFdIqVRE00spVE33DcCYtZqaLNyd4y5RP/QQWZFIc1YWVKKm2A==
+  dependencies:
+    libheif-js "^1.10.0"
+
 highlight.js@~9.12.0:
   version "9.12.0"
   resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.12.0.tgz#e6d9dbe57cbefe60751f02af336195870c90c01e"
@@ -10973,7 +10989,7 @@ jest-worker@^26.6.2:
     merge-stream "^2.0.0"
     supports-color "^7.0.0"
 
-jpeg-js@^0.4.2:
+jpeg-js@^0.4.1, jpeg-js@^0.4.2:
   version "0.4.3"
   resolved "https://registry.yarnpkg.com/jpeg-js/-/jpeg-js-0.4.3.tgz#6158e09f1983ad773813704be80680550eff977b"
   integrity sha512-ru1HWKek8octvUHFHvE5ZzQ1yAsJmIvRdGWvSoKV52XKyuyYA437QWDttXT8eZXDSbuMpHlLzPDZUPd6idIz+Q==
@@ -11375,6 +11391,11 @@ levn@~0.3.0:
     prelude-ls "~1.1.2"
     type-check "~0.3.2"
 
+libheif-js@^1.10.0:
+  version "1.12.0"
+  resolved "https://registry.yarnpkg.com/libheif-js/-/libheif-js-1.12.0.tgz#9ad1ed16a8e6412b4d3d83565d285465a00e7305"
+  integrity sha512-hDs6xQ7028VOwAFwEtM0Q+B2x2NW69Jb2MhQFUbk3rUrHzz4qo5mqS8VrqNgYnSc8TiUGnR691LnO4uIfEE23w==
+
 lie@~3.1.0:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
@@ -13864,6 +13885,11 @@ plist@^3.0.1:
     xmlbuilder "^9.0.7"
     xmldom "^0.5.0"
 
+pngjs@^3.4.0:
+  version "3.4.0"
+  resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f"
+  integrity sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==
+
 pngjs@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-5.0.0.tgz#e79dd2b215767fd9c04561c01236df960bce7fbb"