Add support for receiving attachments from CDN 2

.gitignore

@@ -32,3 +32,5 @@ ts/protobuf/*.d.ts
 
 # Sticker Creator
 sticker-creator/dist/*
+
+/.idea

CONTRIBUTING.md

@@ -267,7 +267,10 @@ the auto update infrastructure doesn't kick in while you are developing.
 {
   "serverUrl": "https://textsecure-service.whispersystems.org",
   "serverTrustRoot": "SOME_ALPHANUMERIC_STRING_MATCHING_PRODUCTION_JSON",
-  "cdnUrl": "https://cdn.signal.org"
+  "cdn": {
+    "0": "https://cdn.signal.org",
+    "2": "https://cdn2.signal.org"
+  }
 }
 ```
 

config/default.json

@@ -1,6 +1,9 @@
 {
   "serverUrl": "https://textsecure-service-staging.whispersystems.org",
-  "cdnUrl": "https://cdn-staging.signal.org",
+  "cdn": {
+    "0": "https://cdn-staging.signal.org",
+    "2": "https://cdn2-staging.signal.org"
+  },
   "contentProxyUrl": "http://contentproxy.signal.org:443",
   "updatesUrl": "https://updates2.signal.org/desktop",
   "updatesPublicKey": "fd7dd3de7149dc0a127909fee7de0f7620ddd0de061b37a2c303e37de802a401",

config/production.json

@@ -1,6 +1,9 @@
 {
   "serverUrl": "https://textsecure-service.whispersystems.org",
-  "cdnUrl": "https://cdn.signal.org",
+  "cdn": {
+    "0": "https://cdn.signal.org",
+    "2": "https://cdn2.signal.org"
+  },
   "serverPublicParams": "DDZM414H2QbA3brAa6NCMaZIN1ZRY+B46PWDvw4LmwrY6CEQArF4OF/yHdBL7HW/JPgjjauzJau+cpikvqH3dDZQ7KFKgx/MGsbw49ATUj6fhBXko9iyPwVwC3+kjNY6PGZuSoYpD4SJJIgzTJ8Gnuk23tSbX1aQWAWNlc8WiyWIHm/A+22w/D1zQmGuFCEGImU4blMK+HhNfC7jM5leBQ==",
   "serverTrustRoot": "BXu6QIKVz5MA8gstzfOgRQGqyLqOwNKHL6INkv3IHWMF",
   "updatesEnabled": true

js/modules/attachment_downloads.js

@@ -193,9 +193,10 @@ async function _runJob(job) {
       // Attachments on the server expire after 30 days, then start returning 404
       if (error && error.code === 404) {
         logger.warn(
-          `_runJob: Got 404 from server, marking attachment ${
+          `_runJob: Got 404 from server for CDN ${
-            attachment.id
+            attachment.cdnNumber
-          } from message ${message.idForLogging()} as permanent error`
+          }, marking attachment ${attachment.cdnId ||
+            attachment.cdnKey} from message ${message.idForLogging()} as permanent error`
         );
 
         await _finishJob(message, id);

js/modules/backup.js

@@ -387,7 +387,7 @@ function _getExportAttachmentFileName(message, index, attachment) {
     return _trimFileName(attachment.fileName);
   }
 
-  let name = attachment.id;
+  let name = attachment.cdnId || attachment.cdnKey;
 
   if (attachment.contentType) {
     const components = attachment.contentType.split('/');

main.js

@@ -188,7 +188,8 @@ function prepareURL(pathSegments, moreKeys) {
       version: app.getVersion(),
       buildExpiration: config.get('buildExpiration'),
       serverUrl: config.get('serverUrl'),
-      cdnUrl: config.get('cdnUrl'),
+      cdnUrl0: config.get('cdn').get('0'),
+      cdnUrl2: config.get('cdn').get('2'),
       certificateAuthority: config.get('certificateAuthority'),
       environment: config.environment,
       node_version: process.versions.node,

preload.js

@@ -230,7 +230,10 @@ try {
 
   window.WebAPI = window.textsecure.WebAPI.initialize({
     url: config.serverUrl,
-    cdnUrl: config.cdnUrl,
+    cdnUrlObject: {
+      '0': config.cdnUrl0,
+      '2': config.cdnUrl2,
+    },
     certificateAuthority: config.certificateAuthority,
     contentProxyUrl: config.contentProxyUrl,
     proxyUrl: config.proxyUrl,

protos/SignalService.proto

@@ -182,12 +182,13 @@ message DataMessage {
   enum ProtocolVersion {
     option allow_alias = true;
 
-    INITIAL         = 0;
+    INITIAL                  = 0;
-    MESSAGE_TIMERS  = 1;
+    MESSAGE_TIMERS           = 1;
-    VIEW_ONCE       = 2;
+    VIEW_ONCE                = 2;
-    VIEW_ONCE_VIDEO = 3;
+    VIEW_ONCE_VIDEO          = 3;
-    REACTIONS       = 4;
+    REACTIONS                = 4;
-    CURRENT         = 4;
+    CDN_SELECTOR_ATTACHMENTS = 5;
+    CURRENT                  = 5;
   }
 
   optional string             body                    = 1;
@@ -337,17 +338,24 @@ message AttachmentPointer {
     VOICE_MESSAGE = 1;
   }
 
-  optional fixed64 id          = 1;
+  oneof attachment_identifier {
-  optional string  contentType = 2;
+    fixed64        cdnId           = 1;
-  optional bytes   key         = 3;
+    string         cdnKey          = 15;
-  optional uint32  size        = 4;
+  }
-  optional bytes   thumbnail   = 5;
+  optional string  contentType     = 2;
-  optional bytes   digest      = 6;
+  optional bytes   key             = 3;
-  optional string  fileName    = 7;
+  optional uint32  size            = 4;
-  optional uint32  flags       = 8;
+  optional bytes   thumbnail       = 5;
-  optional uint32  width       = 9;
+  optional bytes   digest          = 6;
-  optional uint32  height      = 10;
+  optional string  fileName        = 7;
-  optional string  caption     = 11;
+  optional uint32  flags           = 8;
+  optional uint32  width           = 9;
+  optional uint32  height          = 10;
+  optional string  caption         = 11;
+  optional string  blurHash        = 12;
+  optional uint64  uploadTimestamp = 13;
+  optional uint32  cdnNumber       = 14;
+  // Next ID: 16
 }
 
 message GroupContext {

sticker-creator/preload.js

@@ -33,7 +33,10 @@ const { initialize: initializeWebAPI } = require('../ts/textsecure/WebAPI');
 
 const WebAPI = initializeWebAPI({
   url: config.serverUrl,
-  cdnUrl: config.cdnUrl,
+  cdn: {
+    '0': config.cdnUrl0,
+    '2': config.cdnUrl2,
+  },
   certificateAuthority: config.certificateAuthority,
   contentProxyUrl: config.contentProxyUrl,
   proxyUrl: config.proxyUrl,

test/backup_test.js

@@ -65,7 +65,7 @@ describe('Backup', () => {
       };
       const index = 0;
       const attachment = {
-        id: '123',
+        cdnId: '123',
       };
       const expected = '123';
 
@@ -77,13 +77,13 @@ describe('Backup', () => {
       assert.strictEqual(actual, expected);
     });
 
-    it('uses filename and contentType if available', () => {
+    it('uses attachment id and contentType if available', () => {
       const message = {
         body: 'something',
       };
       const index = 0;
       const attachment = {
-        id: '123',
+        cdnId: '123',
         contentType: 'image/jpeg',
       };
       const expected = '123.jpeg';
@@ -102,7 +102,7 @@ describe('Backup', () => {
       };
       const index = 0;
       const attachment = {
-        id: '123',
+        cdnId: '123',
         contentType: 'something',
       };
       const expected = '123.something';
@@ -114,6 +114,43 @@ describe('Backup', () => {
       );
       assert.strictEqual(actual, expected);
     });
+
+    it('uses CDN key if attachment ID not available', () => {
+      const message = {
+        body: 'something',
+      };
+      const index = 0;
+      const attachment = {
+        cdnKey: 'abc',
+      };
+      const expected = 'abc';
+
+      const actual = Signal.Backup._getExportAttachmentFileName(
+        message,
+        index,
+        attachment
+      );
+      assert.strictEqual(actual, expected);
+    });
+
+    it('uses CDN key and contentType if available', () => {
+      const message = {
+        body: 'something',
+      };
+      const index = 0;
+      const attachment = {
+        cdnKey: 'def',
+        contentType: 'image/jpeg',
+      };
+      const expected = 'def.jpeg';
+
+      const actual = Signal.Backup._getExportAttachmentFileName(
+        message,
+        index,
+        attachment
+      );
+      assert.strictEqual(actual, expected);
+    });
   });
 
   describe('_getAnonymousAttachmentFileName', () => {

ts/textsecure.d.ts

@@ -153,7 +153,8 @@ export declare class AttachmentPointerClass {
     encoding?: string
   ) => AttachmentPointerClass;
 
-  id?: ProtoBigNumberType;
+  cdnId?: ProtoBigNumberType;
+  cdnKey?: string;
   contentType?: string;
   key?: ProtoBinaryType;
   size?: number;
@@ -164,6 +165,9 @@ export declare class AttachmentPointerClass {
   width?: number;
   height?: number;
   caption?: string;
+  blurHash?: string;
+  uploadTimestamp?: ProtoBigNumberType;
+  cdnNumber?: number;
 }
 
 export declare class ContactDetailsClass {

ts/textsecure/MessageReceiver.ts

@@ -21,6 +21,7 @@ import {
   AttachmentPointerClass,
   DataMessageClass,
   EnvelopeClass,
+  ProtoBigNumberType,
   ReceiptMessageClass,
   SyncMessageClass,
   TypingMessageClass,
@@ -62,7 +63,8 @@ declare global {
 }
 
 type AttachmentType = {
-  id?: string;
+  cdnId?: string;
+  cdnKey?: string;
   data: ArrayBuffer;
   contentType?: string;
   size?: number;
@@ -71,6 +73,9 @@ type AttachmentType = {
   width?: number;
   height?: number;
   caption?: string;
+  blurHash?: string;
+  uploadTimestamp?: ProtoBigNumberType;
+  cdnNumber?: number;
 };
 
 type CacheAddItemType = {
@@ -1571,13 +1576,16 @@ class MessageReceiverInner extends EventTarget {
   cleanAttachment(attachment: AttachmentPointerClass) {
     return {
       ...omit(attachment, 'thumbnail'),
-      id: attachment.id.toString(),
+      cdnId: attachment.cdnId?.toString(),
       key: attachment.key ? attachment.key.toString('base64') : null,
       digest: attachment.digest ? attachment.digest.toString('base64') : null,
     };
   }
   async downloadAttachment(attachment: AttachmentPointerClass) {
-    const encrypted = await this.server.getAttachment(attachment.id);
+    const encrypted = await this.server.getAttachment(
+      attachment.cdnId || attachment.cdnKey,
+      attachment.cdnNumber || 0
+    );
     const { key, digest, size } = attachment;
 
     if (!digest) {

ts/textsecure/SendMessage.ts

@@ -334,7 +334,7 @@ export default class MessageSender {
     const id = await this.server.putAttachment(result.ciphertext);
 
     const proto = new window.textsecure.protobuf.AttachmentPointer();
-    proto.id = id;
+    proto.cdnId = id;
     proto.contentType = attachment.contentType;
     proto.key = key;
     proto.size = attachment.size;

ts/textsecure/WebAPI.ts

@@ -491,7 +491,10 @@ const URL_CALLS = {
 
 type InitializeOptionsType = {
   url: string;
-  cdnUrl: string;
+  cdnUrlObject: {
+    readonly '0': string;
+    readonly [propName: string]: string;
+  };
   certificateAuthority: string;
   contentProxyUrl: string;
   proxyUrl: string;
@@ -532,7 +535,7 @@ export type WebAPIType = {
     deviceName?: string | null,
     options?: { accessKey?: ArrayBuffer }
   ) => Promise<any>;
-  getAttachment: (id: string) => Promise<any>;
+  getAttachment: (cdnKey: string, cdnNumber: number) => Promise<any>;
   getAvatar: (path: string) => Promise<any>;
   getDevices: () => Promise<any>;
   getKeysForIdentifier: (
@@ -643,7 +646,7 @@ export type ProxiedRequestOptionsType = {
 // tslint:disable-next-line max-func-body-length
 export function initialize({
   url,
-  cdnUrl,
+  cdnUrlObject,
   certificateAuthority,
   contentProxyUrl,
   proxyUrl,
@@ -652,8 +655,14 @@ export function initialize({
   if (!is.string(url)) {
     throw new Error('WebAPI.initialize: Invalid server url');
   }
-  if (!is.string(cdnUrl)) {
+  if (!is.object(cdnUrlObject)) {
-    throw new Error('WebAPI.initialize: Invalid cdnUrl');
+    throw new Error('WebAPI.initialize: Invalid cdnUrlObject');
+  }
+  if (!is.string(cdnUrlObject['0'])) {
+    throw new Error('WebAPI.initialize: Missing CDN 0 configuration');
+  }
+  if (!is.string(cdnUrlObject['2'])) {
+    throw new Error('WebAPI.initialize: Missing CDN 2 configuration');
   }
   if (!is.string(certificateAuthority)) {
     throw new Error('WebAPI.initialize: Invalid certificateAuthority');
@@ -871,7 +880,7 @@ export function initialize({
     async function getAvatar(path: string) {
       // Using _outerAJAX, since it's not hardcoded to the Signal Server. Unlike our
       //   attachment CDN, it uses our self-signed certificate, so we pass it in.
-      return _outerAjax(`${cdnUrl}/${path}`, {
+      return _outerAjax(`${cdnUrlObject['0']}/${path}`, {
         certificateAuthority,
         contentType: 'application/octet-stream',
         proxyUrl,
@@ -1198,25 +1207,31 @@ export function initialize({
     }
 
     async function getSticker(packId: string, stickerId: string) {
-      return _outerAjax(`${cdnUrl}/stickers/${packId}/full/${stickerId}`, {
+      return _outerAjax(
-        certificateAuthority,
+        `${cdnUrlObject['0']}/stickers/${packId}/full/${stickerId}`,
-        proxyUrl,
+        {
-        responseType: 'arraybuffer',
+          certificateAuthority,
-        type: 'GET',
+          proxyUrl,
-        redactUrl: redactStickerUrl,
+          responseType: 'arraybuffer',
-        version,
+          type: 'GET',
-      });
+          redactUrl: redactStickerUrl,
+          version,
+        }
+      );
     }
 
     async function getStickerPackManifest(packId: string) {
-      return _outerAjax(`${cdnUrl}/stickers/${packId}/manifest.proto`, {
+      return _outerAjax(
-        certificateAuthority,
+        `${cdnUrlObject['0']}/stickers/${packId}/manifest.proto`,
-        proxyUrl,
+        {
-        responseType: 'arraybuffer',
+          certificateAuthority,
-        type: 'GET',
+          proxyUrl,
-        redactUrl: redactStickerUrl,
+          responseType: 'arraybuffer',
-        version,
+          type: 'GET',
-      });
+          redactUrl: redactStickerUrl,
+          version,
+        }
+      );
     }
 
     type ServerAttachmentType = {
@@ -1304,7 +1319,7 @@ export function initialize({
       // Upload manifest
       const manifestParams = makePutParams(manifest, encryptedManifest);
       // This is going to the CDN, not the service, so we use _outerAjax
-      await _outerAjax(`${cdnUrl}/`, {
+      await _outerAjax(`${cdnUrlObject['0']}/`, {
         ...manifestParams,
         certificateAuthority,
         proxyUrl,
@@ -1322,7 +1337,7 @@ export function initialize({
             encryptedStickers[index]
           );
           await queue.add(async () =>
-            _outerAjax(`${cdnUrl}/`, {
+            _outerAjax(`${cdnUrlObject['0']}/`, {
               ...stickerParams,
               certificateAuthority,
               proxyUrl,
@@ -1341,9 +1356,10 @@ export function initialize({
       return packId;
     }
 
-    async function getAttachment(id: string) {
+    async function getAttachment(cdnKey: string, cdnNumber: number) {
+      const cdnUrl = cdnUrlObject[cdnNumber] || cdnUrlObject['0'];
       // This is going to the CDN, not the service, so we use _outerAjax
-      return _outerAjax(`${cdnUrl}/attachments/${id}`, {
+      return _outerAjax(`${cdnUrl}/attachments/${cdnKey}`, {
         certificateAuthority,
         proxyUrl,
         responseType: 'arraybuffer',
@@ -1365,7 +1381,7 @@ export function initialize({
       const params = makePutParams(response, encryptedBin);
 
       // This is going to the CDN, not the service, so we use _outerAjax
-      await _outerAjax(`${cdnUrl}/attachments/`, {
+      await _outerAjax(`${cdnUrlObject['0']}/attachments/`, {
         ...params,
         certificateAuthority,
         proxyUrl,

ts/util/lint/exceptions.json

@@ -11817,4 +11817,4 @@
     "reasonCategory": "falseMatch",
     "updated": "2020-04-05T23:45:16.746Z"
   }
-]
+]