Padded attachments, attachments v2

* Handle incoming padded attachments
* Attachments v2 - multipart form POST, and direct CDN GET access
* Pad outgoing attachments before encryption (disabled for now)

js/modules/crypto.js

@@ -5,6 +5,7 @@
 
 module.exports = {
   arrayBufferToBase64,
+  typedArrayToArrayBuffer,
   base64ToArrayBuffer,
   bytesFromString,
   concatenateBytes,
@@ -22,6 +23,7 @@ module.exports = {
   encryptSymmetric,
   fromEncodedBinaryToArrayBuffer,
   getAccessKeyVerifier,
+  getFirstBytes,
   getRandomBytes,
   getViewOfArrayBuffer,
   getZeroes,
@@ -34,6 +36,11 @@ module.exports = {
   verifyAccessKey,
 };
 
+function typedArrayToArrayBuffer(typedArray) {
+  const { buffer, byteOffset, byteLength } = typedArray;
+  return buffer.slice(byteOffset, byteLength + byteOffset);
+}
+
 function arrayBufferToBase64(arrayBuffer) {
   return dcodeIO.ByteBuffer.wrap(arrayBuffer).toString('base64');
 }
@@ -63,7 +70,7 @@ async function encryptDeviceName(deviceName, identityPublic) {
   );
 
   const key1 = await hmacSha256(masterSecret, bytesFromString('auth'));
-  const syntheticIv = _getFirstBytes(await hmacSha256(key1, plaintext), 16);
+  const syntheticIv = getFirstBytes(await hmacSha256(key1, plaintext), 16);
 
   const key2 = await hmacSha256(masterSecret, bytesFromString('cipher'));
   const cipherKey = await hmacSha256(key2, syntheticIv);
@@ -94,7 +101,7 @@ async function decryptDeviceName(
   const plaintext = await decryptAesCtr(cipherKey, ciphertext, counter);
 
   const key1 = await hmacSha256(masterSecret, bytesFromString('auth'));
-  const ourSyntheticIv = _getFirstBytes(await hmacSha256(key1, plaintext), 16);
+  const ourSyntheticIv = getFirstBytes(await hmacSha256(key1, plaintext), 16);
 
   if (!constantTimeEqual(ourSyntheticIv, syntheticIv)) {
     throw new Error('decryptDeviceName: synthetic IV did not match');
@@ -133,7 +140,7 @@ async function encryptFile(staticPublicKey, uniqueId, plaintext) {
 }
 
 async function decryptFile(staticPrivateKey, uniqueId, data) {
-  const ephemeralPublicKey = _getFirstBytes(data, PUB_KEY_LENGTH);
+  const ephemeralPublicKey = getFirstBytes(data, PUB_KEY_LENGTH);
   const ciphertext = _getBytes(data, PUB_KEY_LENGTH, data.byteLength);
   const agreement = await libsignal.Curve.async.calculateAgreement(
     ephemeralPublicKey,
@@ -149,7 +156,7 @@ async function deriveAccessKey(profileKey) {
   const iv = getZeroes(12);
   const plaintext = getZeroes(16);
   const accessKey = await _encrypt_aes_gcm(profileKey, iv, plaintext);
-  return _getFirstBytes(accessKey, 16);
+  return getFirstBytes(accessKey, 16);
 }
 
 async function getAccessKeyVerifier(accessKey) {
@@ -185,7 +192,7 @@ async function encryptSymmetric(key, plaintext) {
     iv,
     plaintext
   );
-  const mac = _getFirstBytes(await hmacSha256(macKey, cipherText), MAC_LENGTH);
+  const mac = getFirstBytes(await hmacSha256(macKey, cipherText), MAC_LENGTH);
 
   return concatenateBytes(nonce, cipherText, mac);
 }
@@ -193,7 +200,7 @@ async function encryptSymmetric(key, plaintext) {
 async function decryptSymmetric(key, data) {
   const iv = getZeroes(IV_LENGTH);
 
-  const nonce = _getFirstBytes(data, NONCE_LENGTH);
+  const nonce = getFirstBytes(data, NONCE_LENGTH);
   const cipherText = _getBytes(
     data,
     NONCE_LENGTH,
@@ -204,7 +211,7 @@ async function decryptSymmetric(key, data) {
   const cipherKey = await hmacSha256(key, nonce);
   const macKey = await hmacSha256(key, cipherKey);
 
-  const ourMac = _getFirstBytes(
+  const ourMac = getFirstBytes(
     await hmacSha256(macKey, cipherText),
     MAC_LENGTH
   );
@@ -379,7 +386,7 @@ function intsToByteHighAndLow(highValue, lowValue) {
 }
 
 function trimBytes(buffer, length) {
-  return _getFirstBytes(buffer, length);
+  return getFirstBytes(buffer, length);
 }
 
 function getViewOfArrayBuffer(buffer, start, finish) {
@@ -437,13 +444,13 @@ function splitBytes(buffer, ...lengths) {
   return results;
 }
 
-// Internal-only
+function getFirstBytes(data, n) {
-
-function _getFirstBytes(data, n) {
   const source = new Uint8Array(data);
   return source.subarray(0, n);
 }
 
+// Internal-only
+
 function _getBytes(data, start, n) {
   const source = new Uint8Array(data);
   return source.subarray(start, start + n);

js/modules/web_api.js

@@ -5,7 +5,7 @@ const { Agent } = require('https');
 
 const is = require('@sindresorhus/is');
 
-/* global Buffer, setTimeout, log, _ */
+/* global Buffer, setTimeout, log, _, getGuid */
 
 /* eslint-disable more/no-then, no-bitwise, no-nested-ternary */
 
@@ -388,7 +388,7 @@ const URL_CALLS = {
   accounts: 'v1/accounts',
   updateDeviceName: 'v1/accounts/name',
   removeSignalingKey: 'v1/accounts/signaling_key',
-  attachment: 'v1/attachments',
+  attachmentId: 'v2/attachments/form/upload',
   deliveryCert: 'v1/certificate/delivery',
   supportUnauthenticatedDelivery: 'v1/devices/unauthenticated_delivery',
   devices: 'v1/devices',
@@ -834,41 +834,88 @@ function initialize({
       });
     }
 
-    function getAttachment(id) {
+    async function getAttachment(id) {
-      return _ajax({
+      // This is going to the CDN, not the service, so we use _outerAjax
-        call: 'attachment',
+      return _outerAjax(`${cdnUrl}/attachments/${id}`, {
-        httpType: 'GET',
+        certificateAuthority,
-        urlParameters: `/${id}`,
+        proxyUrl,
-        responseType: 'json',
+        responseType: 'arraybuffer',
-        validateResponse: { location: 'string' },
+        timeout: 0,
-      }).then(response =>
+        type: 'GET',
-        // Using _outerAJAX, since it's not hardcoded to the Signal Server
+      });
-        _outerAjax(response.location, {
-          contentType: 'application/octet-stream',
-          proxyUrl,
-          responseType: 'arraybuffer',
-          timeout: 0,
-          type: 'GET',
-        })
-      );
     }
 
-    function putAttachment(encryptedBin) {
+    async function putAttachment(encryptedBin) {
-      return _ajax({
+      const response = await _ajax({
-        call: 'attachment',
+        call: 'attachmentId',
         httpType: 'GET',
         responseType: 'json',
-      }).then(response =>
+      });
-        // Using _outerAJAX, since it's not hardcoded to the Signal Server
+
-        _outerAjax(response.location, {
+      const {
-          contentType: 'application/octet-stream',
+        key,
-          data: encryptedBin,
+        credential,
-          processData: false,
+        acl,
-          proxyUrl,
+        algorithm,
-          timeout: 0,
+        date,
-          type: 'PUT',
+        policy,
-        }).then(() => response.idString)
+        signature,
+        attachmentIdString,
+      } = response;
+
+      // Note: when using the boundary string in the POST body, it needs to be prefixed by
+      //   an extra --, and the final boundary string at the end gets a -- prefix and a --
+      //   suffix.
+      const boundaryString = `----------------${getGuid().replace(/-/g, '')}`;
+      const CRLF = '\r\n';
+      const getSection = (name, value) =>
+        [
+          `--${boundaryString}`,
+          `Content-Disposition: form-data; name="${name}"${CRLF}`,
+          value,
+        ].join(CRLF);
+
+      const start = [
+        getSection('key', key),
+        getSection('x-amz-credential', credential),
+        getSection('acl', acl),
+        getSection('x-amz-algorithm', algorithm),
+        getSection('x-amz-date', date),
+        getSection('policy', policy),
+        getSection('x-amz-signature', signature),
+        getSection('Content-Type', 'application/octet-stream'),
+        `--${boundaryString}`,
+        'Content-Disposition: form-data; name="file"',
+        `Content-Type: application/octet-stream${CRLF}${CRLF}`,
+      ].join(CRLF);
+      const end = `${CRLF}--${boundaryString}--${CRLF}`;
+
+      const startBuffer = Buffer.from(start, 'utf8');
+      const attachmentBuffer = Buffer.from(encryptedBin);
+      const endBuffer = Buffer.from(end, 'utf8');
+
+      const contentLength =
+        startBuffer.length + attachmentBuffer.length + endBuffer.length;
+      const data = Buffer.concat(
+        [startBuffer, attachmentBuffer, endBuffer],
+        contentLength
       );
+
+      // This is going to the CDN, not the service, so we use _outerAjax
+      await _outerAjax(`${cdnUrl}/attachments/`, {
+        certificateAuthority,
+        contentType: `multipart/form-data; boundary=${boundaryString}`,
+        data,
+        proxyUrl,
+        timeout: 0,
+        type: 'POST',
+        headers: {
+          'Content-Length': contentLength,
+        },
+        processData: false,
+      });
+
+      return attachmentIdString;
     }
 
     // eslint-disable-next-line no-shadow

libtextsecure/message_receiver.js

@@ -1234,17 +1234,19 @@ MessageReceiver.prototype.extend({
       window.Signal.Crypto.base64ToArrayBuffer(digest)
     );
 
-    if (!size || size !== data.byteLength) {
+    if (!size) {
       throw new Error(
-        `downloadAttachment: Size ${size} did not match downloaded attachment size ${
+        `downloadAttachment: Size was not provided, actual size was ${
           data.byteLength
         }`
       );
     }
 
+    const typedArray = window.Signal.Crypto.getFirstBytes(data, size);
+
     return {
       ..._.omit(attachment, 'digest', 'key'),
-      data,
+      data: window.Signal.Crypto.typedArrayToArrayBuffer(typedArray),
     };
   },
   handleAttachment(attachment) {

libtextsecure/sendmessage.js

@@ -155,60 +155,80 @@ function MessageSender(username, password) {
   this.pendingMessages = {};
 }
 
+const DISABLE_PADDING = true;
+
 MessageSender.prototype = {
   constructor: MessageSender,
 
-  //  makeAttachmentPointer :: Attachment -> Promise AttachmentPointerProto
+  _getAttachmentSizeBucket(size) {
-  makeAttachmentPointer(attachment) {
+    return Math.max(
+      541,
+      Math.floor(1.05 ** Math.ceil(Math.log(size) / Math.log(1.05)))
+    );
+  },
+
+  getPaddedAttachment(data) {
+    if (DISABLE_PADDING) {
+      return data;
+    }
+
+    const size = data.byteLength;
+    const paddedSize = this._getAttachmentSizeBucket(size);
+    const padding = window.Signal.Crypto.getZeroes(paddedSize - size);
+
+    return window.Signal.Crypto.concatenateBytes(data, padding);
+  },
+
+  async makeAttachmentPointer(attachment) {
     if (typeof attachment !== 'object' || attachment == null) {
       return Promise.resolve(undefined);
     }
 
-    if (
+    const { data, size } = attachment;
-      !(attachment.data instanceof ArrayBuffer) &&
+    if (!(data instanceof ArrayBuffer) && !ArrayBuffer.isView(data)) {
-      !ArrayBuffer.isView(attachment.data)
+      throw new Error(
-    ) {
+        `makeAttachmentPointer: data was a '${typeof data}' instead of ArrayBuffer/ArrayBufferView`
-      return Promise.reject(
+      );
-        new TypeError(
+    }
-          `\`attachment.data\` must be an \`ArrayBuffer\` or \`ArrayBufferView\`; got: ${typeof attachment.data}`
+    if (data.byteLength !== size) {
-        )
+      throw new Error(
+        `makeAttachmentPointer: Size ${size} did not match data.byteLength ${
+          data.byteLength
+        }`
       );
     }
 
-    const proto = new textsecure.protobuf.AttachmentPointer();
+    const padded = this.getPaddedAttachment(data);
-    proto.key = libsignal.crypto.getRandomBytes(64);
+    const key = libsignal.crypto.getRandomBytes(64);
-
     const iv = libsignal.crypto.getRandomBytes(16);
-    return textsecure.crypto
-      .encryptAttachment(attachment.data, proto.key, iv)
-      .then(result =>
-        this.server.putAttachment(result.ciphertext).then(id => {
-          proto.id = id;
-          proto.contentType = attachment.contentType;
-          proto.digest = result.digest;
 
-          if (attachment.size) {
+    const result = await textsecure.crypto.encryptAttachment(padded, key, iv);
-            proto.size = attachment.size;
+    const id = await this.server.putAttachment(result.ciphertext);
-          }
-          if (attachment.fileName) {
-            proto.fileName = attachment.fileName;
-          }
-          if (attachment.flags) {
-            proto.flags = attachment.flags;
-          }
-          if (attachment.width) {
-            proto.width = attachment.width;
-          }
-          if (attachment.height) {
-            proto.height = attachment.height;
-          }
-          if (attachment.caption) {
-            proto.caption = attachment.caption;
-          }
 
-          return proto;
+    const proto = new textsecure.protobuf.AttachmentPointer();
-        })
+    proto.id = id;
-      );
+    proto.contentType = attachment.contentType;
+    proto.key = key;
+    proto.size = attachment.size;
+    proto.digest = result.digest;
+
+    if (attachment.fileName) {
+      proto.fileName = attachment.fileName;
+    }
+    if (attachment.flags) {
+      proto.flags = attachment.flags;
+    }
+    if (attachment.width) {
+      proto.width = attachment.width;
+    }
+    if (attachment.height) {
+      proto.height = attachment.height;
+    }
+    if (attachment.caption) {
+      proto.caption = attachment.caption;
+    }
+
+    return proto;
   },
 
   queueJobForNumber(number, runJob) {
@@ -1124,6 +1144,8 @@ textsecure.MessageSender = function MessageSenderWrapper(username, password) {
   this.makeProxiedRequest = sender.makeProxiedRequest.bind(sender);
   this.getProxiedSize = sender.getProxiedSize.bind(sender);
   this.getMessageProto = sender.getMessageProto.bind(sender);
+
+  this._getAttachmentSizeBucket = sender._getAttachmentSizeBucket.bind(sender);
 };
 
 textsecure.MessageSender.prototype = {

libtextsecure/test/index.html

@@ -45,6 +45,7 @@
   <script type="text/javascript" src="websocket-resources_test.js"></script>
   <script type="text/javascript" src="task_with_timeout_test.js"></script>
   <script type="text/javascript" src="account_manager_test.js"></script>
+  <script type="text/javascript" src="sendmessage_test.js"></script>
 
   <!-- Comment out to turn off code coverage. Useful for getting real callstacks. -->
   <!--   NOTE: blanket doesn't support modern syntax and will choke until we find a replacement. :0( -->

libtextsecure/test/sendmessage_test.js

@@ -0,0 +1,335 @@
+/* global textsecure, WebAPI */
+/* eslint-disable no-console */
+
+const BUCKET_SIZES = [
+  541,
+  568,
+  596,
+  626,
+  657,
+  690,
+  725,
+  761,
+  799,
+  839,
+  881,
+  925,
+  972,
+  1020,
+  1071,
+  1125,
+  1181,
+  1240,
+  1302,
+  1367,
+  1436,
+  1507,
+  1583,
+  1662,
+  1745,
+  1832,
+  1924,
+  2020,
+  2121,
+  2227,
+  2339,
+  2456,
+  2579,
+  2708,
+  2843,
+  2985,
+  3134,
+  3291,
+  3456,
+  3629,
+  3810,
+  4001,
+  4201,
+  4411,
+  4631,
+  4863,
+  5106,
+  5361,
+  5629,
+  5911,
+  6207,
+  6517,
+  6843,
+  7185,
+  7544,
+  7921,
+  8318,
+  8733,
+  9170,
+  9629,
+  10110,
+  10616,
+  11146,
+  11704,
+  12289,
+  12903,
+  13549,
+  14226,
+  14937,
+  15684,
+  16469,
+  17292,
+  18157,
+  19065,
+  20018,
+  21019,
+  22070,
+  23173,
+  24332,
+  25549,
+  26826,
+  28167,
+  29576,
+  31054,
+  32607,
+  34238,
+  35950,
+  37747,
+  39634,
+  41616,
+  43697,
+  45882,
+  48176,
+  50585,
+  53114,
+  55770,
+  58558,
+  61486,
+  64561,
+  67789,
+  71178,
+  74737,
+  78474,
+  82398,
+  86518,
+  90843,
+  95386,
+  100155,
+  105163,
+  110421,
+  115942,
+  121739,
+  127826,
+  134217,
+  140928,
+  147975,
+  155373,
+  163142,
+  171299,
+  179864,
+  188858,
+  198300,
+  208215,
+  218626,
+  229558,
+  241036,
+  253087,
+  265742,
+  279029,
+  292980,
+  307629,
+  323011,
+  339161,
+  356119,
+  373925,
+  392622,
+  412253,
+  432866,
+  454509,
+  477234,
+  501096,
+  526151,
+  552458,
+  580081,
+  609086,
+  639540,
+  671517,
+  705093,
+  740347,
+  777365,
+  816233,
+  857045,
+  899897,
+  944892,
+  992136,
+  1041743,
+  1093831,
+  1148522,
+  1205948,
+  1266246,
+  1329558,
+  1396036,
+  1465838,
+  1539130,
+  1616086,
+  1696890,
+  1781735,
+  1870822,
+  1964363,
+  2062581,
+  2165710,
+  2273996,
+  2387695,
+  2507080,
+  2632434,
+  2764056,
+  2902259,
+  3047372,
+  3199740,
+  3359727,
+  3527714,
+  3704100,
+  3889305,
+  4083770,
+  4287958,
+  4502356,
+  4727474,
+  4963848,
+  5212040,
+  5472642,
+  5746274,
+  6033588,
+  6335268,
+  6652031,
+  6984633,
+  7333864,
+  7700558,
+  8085585,
+  8489865,
+  8914358,
+  9360076,
+  9828080,
+  10319484,
+  10835458,
+  11377231,
+  11946092,
+  12543397,
+  13170567,
+  13829095,
+  14520550,
+  15246578,
+  16008907,
+  16809352,
+  17649820,
+  18532311,
+  19458926,
+  20431872,
+  21453466,
+  22526139,
+  23652446,
+  24835069,
+  26076822,
+  27380663,
+  28749697,
+  30187181,
+  31696540,
+  33281368,
+  34945436,
+  36692708,
+  38527343,
+  40453710,
+  42476396,
+  44600216,
+  46830227,
+  49171738,
+  51630325,
+  54211841,
+  56922433,
+  59768555,
+  62756983,
+  65894832,
+  69189573,
+  72649052,
+  76281505,
+  80095580,
+  84100359,
+  88305377,
+  92720646,
+  97356678,
+  102224512,
+  107335738,
+];
+
+describe('sendmessage', () => {
+  let originalWebAPIConnect = null;
+  let sendmessage = null;
+  before(() => {
+    originalWebAPIConnect = WebAPI.connect;
+    WebAPI.connect = () => null;
+
+    sendmessage = new textsecure.MessageSender();
+  });
+  after(() => {
+    WebAPI.connect = originalWebAPIConnect;
+  });
+
+  describe('#_getAttachmentSizeBucket', () => {
+    it('properly calculates first bucket', () => {
+      for (let size = 0, max = BUCKET_SIZES[0]; size < max; size += 1) {
+        assert.strictEqual(
+          BUCKET_SIZES[0],
+          sendmessage._getAttachmentSizeBucket(size)
+        );
+      }
+    });
+
+    it('properly calculates entire table', () => {
+      let count = 0;
+
+      for (let i = 0, max = BUCKET_SIZES.length - 1; i < max; i += 1) {
+        // Exact
+        if (
+          BUCKET_SIZES[i] !==
+          sendmessage._getAttachmentSizeBucket(BUCKET_SIZES[i])
+        ) {
+          count += 1;
+          console.log(
+            `${
+              BUCKET_SIZES[i]
+            } does not equal ${sendmessage._getAttachmentSizeBucket(
+              BUCKET_SIZES[i]
+            )}`
+          );
+        }
+
+        // Just under
+        if (
+          BUCKET_SIZES[i] !==
+          sendmessage._getAttachmentSizeBucket(BUCKET_SIZES[i] - 1)
+        ) {
+          count += 1;
+          console.log(
+            `${
+              BUCKET_SIZES[i]
+            } does not equal ${sendmessage._getAttachmentSizeBucket(
+              BUCKET_SIZES[i] - 1
+            )}`
+          );
+        }
+
+        // Just over
+        if (
+          BUCKET_SIZES[i + 1] !==
+          sendmessage._getAttachmentSizeBucket(BUCKET_SIZES[i] + 1)
+        ) {
+          count += 1;
+          console.log(
+            `${
+              BUCKET_SIZES[i + 1]
+            } does not equal ${sendmessage._getAttachmentSizeBucket(
+              BUCKET_SIZES[i] + 1
+            )}`
+          );
+        }
+      }
+
+      console.log(`Failures: ${count}`);
+      assert.strictEqual(count, 0);
+    });
+  });
+});

ts/util/lint/exceptions.json

@@ -207,31 +207,31 @@
     "rule": "jQuery-wrap(",
     "path": "js/modules/crypto.js",
     "line": "  return dcodeIO.ByteBuffer.wrap(arrayBuffer).toString('base64');",
-    "lineNumber": 38,
-    "reasonCategory": "falseMatch",
-    "updated": "2018-10-05T23:12:28.961Z"
-  },
-  {
-    "rule": "jQuery-wrap(",
-    "path": "js/modules/crypto.js",
-    "line": "  return dcodeIO.ByteBuffer.wrap(base64string, 'base64').toArrayBuffer();",
-    "lineNumber": 41,
-    "reasonCategory": "falseMatch",
-    "updated": "2018-10-05T23:12:28.961Z"
-  },
-  {
-    "rule": "jQuery-wrap(",
-    "path": "js/modules/crypto.js",
-    "line": "  return dcodeIO.ByteBuffer.wrap(key, 'binary').toArrayBuffer();",
     "lineNumber": 45,
     "reasonCategory": "falseMatch",
     "updated": "2018-10-05T23:12:28.961Z"
   },
+  {
+    "rule": "jQuery-wrap(",
+    "path": "js/modules/crypto.js",
+    "line": "  return dcodeIO.ByteBuffer.wrap(base64string, 'base64').toArrayBuffer();",
+    "lineNumber": 48,
+    "reasonCategory": "falseMatch",
+    "updated": "2018-10-05T23:12:28.961Z"
+  },
+  {
+    "rule": "jQuery-wrap(",
+    "path": "js/modules/crypto.js",
+    "line": "  return dcodeIO.ByteBuffer.wrap(key, 'binary').toArrayBuffer();",
+    "lineNumber": 52,
+    "reasonCategory": "falseMatch",
+    "updated": "2018-10-05T23:12:28.961Z"
+  },
   {
     "rule": "jQuery-wrap(",
     "path": "js/modules/crypto.js",
     "line": "  return dcodeIO.ByteBuffer.wrap(string, 'utf8').toArrayBuffer();",
-    "lineNumber": 49,
+    "lineNumber": 56,
     "reasonCategory": "falseMatch",
     "updated": "2018-10-05T23:12:28.961Z"
   },
@@ -239,7 +239,7 @@
     "rule": "jQuery-wrap(",
     "path": "js/modules/crypto.js",
     "line": "  return dcodeIO.ByteBuffer.wrap(buffer).toString('utf8');",
-    "lineNumber": 52,
+    "lineNumber": 59,
     "reasonCategory": "falseMatch",
     "updated": "2018-10-05T23:12:28.961Z"
   },