diff --git a/.gitignore b/.gitignore index b051b4c296..1f5db62a9f 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ ts/protobuf/*.d.ts # Sticker Creator sticker-creator/dist/* + +/.idea diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2a79331bf..73d42e2677 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -267,7 +267,10 @@ the auto update infrastructure doesn't kick in while you are developing. { "serverUrl": "https://textsecure-service.whispersystems.org", "serverTrustRoot": "SOME_ALPHANUMERIC_STRING_MATCHING_PRODUCTION_JSON", - "cdnUrl": "https://cdn.signal.org" + "cdn": { + "0": "https://cdn.signal.org", + "2": "https://cdn2.signal.org" + } } ``` diff --git a/config/default.json b/config/default.json index c8983e88c7..6a1c01c010 100644 --- a/config/default.json +++ b/config/default.json @@ -1,6 +1,9 @@ { "serverUrl": "https://textsecure-service-staging.whispersystems.org", - "cdnUrl": "https://cdn-staging.signal.org", + "cdn": { + "0": "https://cdn-staging.signal.org", + "2": "https://cdn2-staging.signal.org" + }, "contentProxyUrl": "http://contentproxy.signal.org:443", "updatesUrl": "https://updates2.signal.org/desktop", "updatesPublicKey": "fd7dd3de7149dc0a127909fee7de0f7620ddd0de061b37a2c303e37de802a401", diff --git a/config/production.json b/config/production.json index 06d2321502..e3813073c3 100644 --- a/config/production.json +++ b/config/production.json @@ -1,6 +1,9 @@ { "serverUrl": "https://textsecure-service.whispersystems.org", - "cdnUrl": "https://cdn.signal.org", + "cdn": { + "0": "https://cdn.signal.org", + "2": "https://cdn2.signal.org" + }, "serverPublicParams": "DDZM414H2QbA3brAa6NCMaZIN1ZRY+B46PWDvw4LmwrY6CEQArF4OF/yHdBL7HW/JPgjjauzJau+cpikvqH3dDZQ7KFKgx/MGsbw49ATUj6fhBXko9iyPwVwC3+kjNY6PGZuSoYpD4SJJIgzTJ8Gnuk23tSbX1aQWAWNlc8WiyWIHm/A+22w/D1zQmGuFCEGImU4blMK+HhNfC7jM5leBQ==", "serverTrustRoot": "BXu6QIKVz5MA8gstzfOgRQGqyLqOwNKHL6INkv3IHWMF", "updatesEnabled": true diff --git a/js/modules/attachment_downloads.js b/js/modules/attachment_downloads.js index 49e10f927d..2994fbc5ea 100644 --- a/js/modules/attachment_downloads.js +++ b/js/modules/attachment_downloads.js @@ -193,9 +193,10 @@ async function _runJob(job) { // Attachments on the server expire after 30 days, then start returning 404 if (error && error.code === 404) { logger.warn( - `_runJob: Got 404 from server, marking attachment ${ - attachment.id - } from message ${message.idForLogging()} as permanent error` + `_runJob: Got 404 from server for CDN ${ + attachment.cdnNumber + }, marking attachment ${attachment.cdnId || + attachment.cdnKey} from message ${message.idForLogging()} as permanent error` ); await _finishJob(message, id); diff --git a/js/modules/backup.js b/js/modules/backup.js index 829ece344e..a1a56f003b 100644 --- a/js/modules/backup.js +++ b/js/modules/backup.js @@ -387,7 +387,7 @@ function _getExportAttachmentFileName(message, index, attachment) { return _trimFileName(attachment.fileName); } - let name = attachment.id; + let name = attachment.cdnId || attachment.cdnKey; if (attachment.contentType) { const components = attachment.contentType.split('/'); diff --git a/main.js b/main.js index 2c70871535..570de97f79 100644 --- a/main.js +++ b/main.js @@ -188,7 +188,8 @@ function prepareURL(pathSegments, moreKeys) { version: app.getVersion(), buildExpiration: config.get('buildExpiration'), serverUrl: config.get('serverUrl'), - cdnUrl: config.get('cdnUrl'), + cdnUrl0: config.get('cdn').get('0'), + cdnUrl2: config.get('cdn').get('2'), certificateAuthority: config.get('certificateAuthority'), environment: config.environment, node_version: process.versions.node, diff --git a/preload.js b/preload.js index 5e6538f5de..009e723e7e 100644 --- a/preload.js +++ b/preload.js @@ -230,7 +230,10 @@ try { window.WebAPI = window.textsecure.WebAPI.initialize({ url: config.serverUrl, - cdnUrl: config.cdnUrl, + cdnUrlObject: { + '0': config.cdnUrl0, + '2': config.cdnUrl2, + }, certificateAuthority: config.certificateAuthority, contentProxyUrl: config.contentProxyUrl, proxyUrl: config.proxyUrl, diff --git a/protos/SignalService.proto b/protos/SignalService.proto index 5dfdf15de0..8b904e2bb2 100644 --- a/protos/SignalService.proto +++ b/protos/SignalService.proto @@ -182,12 +182,13 @@ message DataMessage { enum ProtocolVersion { option allow_alias = true; - INITIAL = 0; - MESSAGE_TIMERS = 1; - VIEW_ONCE = 2; - VIEW_ONCE_VIDEO = 3; - REACTIONS = 4; - CURRENT = 4; + INITIAL = 0; + MESSAGE_TIMERS = 1; + VIEW_ONCE = 2; + VIEW_ONCE_VIDEO = 3; + REACTIONS = 4; + CDN_SELECTOR_ATTACHMENTS = 5; + CURRENT = 5; } optional string body = 1; @@ -337,17 +338,24 @@ message AttachmentPointer { VOICE_MESSAGE = 1; } - optional fixed64 id = 1; - optional string contentType = 2; - optional bytes key = 3; - optional uint32 size = 4; - optional bytes thumbnail = 5; - optional bytes digest = 6; - optional string fileName = 7; - optional uint32 flags = 8; - optional uint32 width = 9; - optional uint32 height = 10; - optional string caption = 11; + oneof attachment_identifier { + fixed64 cdnId = 1; + string cdnKey = 15; + } + optional string contentType = 2; + optional bytes key = 3; + optional uint32 size = 4; + optional bytes thumbnail = 5; + optional bytes digest = 6; + optional string fileName = 7; + optional uint32 flags = 8; + optional uint32 width = 9; + optional uint32 height = 10; + optional string caption = 11; + optional string blurHash = 12; + optional uint64 uploadTimestamp = 13; + optional uint32 cdnNumber = 14; + // Next ID: 16 } message GroupContext { diff --git a/sticker-creator/preload.js b/sticker-creator/preload.js index 545cf8970c..5399178eaa 100644 --- a/sticker-creator/preload.js +++ b/sticker-creator/preload.js @@ -33,7 +33,10 @@ const { initialize: initializeWebAPI } = require('../ts/textsecure/WebAPI'); const WebAPI = initializeWebAPI({ url: config.serverUrl, - cdnUrl: config.cdnUrl, + cdn: { + '0': config.cdnUrl0, + '2': config.cdnUrl2, + }, certificateAuthority: config.certificateAuthority, contentProxyUrl: config.contentProxyUrl, proxyUrl: config.proxyUrl, diff --git a/test/backup_test.js b/test/backup_test.js index 218cbe6d3d..76e03ad55c 100644 --- a/test/backup_test.js +++ b/test/backup_test.js @@ -65,7 +65,7 @@ describe('Backup', () => { }; const index = 0; const attachment = { - id: '123', + cdnId: '123', }; const expected = '123'; @@ -77,13 +77,13 @@ describe('Backup', () => { assert.strictEqual(actual, expected); }); - it('uses filename and contentType if available', () => { + it('uses attachment id and contentType if available', () => { const message = { body: 'something', }; const index = 0; const attachment = { - id: '123', + cdnId: '123', contentType: 'image/jpeg', }; const expected = '123.jpeg'; @@ -102,7 +102,7 @@ describe('Backup', () => { }; const index = 0; const attachment = { - id: '123', + cdnId: '123', contentType: 'something', }; const expected = '123.something'; @@ -114,6 +114,43 @@ describe('Backup', () => { ); assert.strictEqual(actual, expected); }); + + it('uses CDN key if attachment ID not available', () => { + const message = { + body: 'something', + }; + const index = 0; + const attachment = { + cdnKey: 'abc', + }; + const expected = 'abc'; + + const actual = Signal.Backup._getExportAttachmentFileName( + message, + index, + attachment + ); + assert.strictEqual(actual, expected); + }); + + it('uses CDN key and contentType if available', () => { + const message = { + body: 'something', + }; + const index = 0; + const attachment = { + cdnKey: 'def', + contentType: 'image/jpeg', + }; + const expected = 'def.jpeg'; + + const actual = Signal.Backup._getExportAttachmentFileName( + message, + index, + attachment + ); + assert.strictEqual(actual, expected); + }); }); describe('_getAnonymousAttachmentFileName', () => { diff --git a/ts/textsecure.d.ts b/ts/textsecure.d.ts index 02536ecc92..147d252e27 100644 --- a/ts/textsecure.d.ts +++ b/ts/textsecure.d.ts @@ -153,7 +153,8 @@ export declare class AttachmentPointerClass { encoding?: string ) => AttachmentPointerClass; - id?: ProtoBigNumberType; + cdnId?: ProtoBigNumberType; + cdnKey?: string; contentType?: string; key?: ProtoBinaryType; size?: number; @@ -164,6 +165,9 @@ export declare class AttachmentPointerClass { width?: number; height?: number; caption?: string; + blurHash?: string; + uploadTimestamp?: ProtoBigNumberType; + cdnNumber?: number; } export declare class ContactDetailsClass { diff --git a/ts/textsecure/MessageReceiver.ts b/ts/textsecure/MessageReceiver.ts index 9b10b2cecc..731eb68700 100644 --- a/ts/textsecure/MessageReceiver.ts +++ b/ts/textsecure/MessageReceiver.ts @@ -21,6 +21,7 @@ import { AttachmentPointerClass, DataMessageClass, EnvelopeClass, + ProtoBigNumberType, ReceiptMessageClass, SyncMessageClass, TypingMessageClass, @@ -62,7 +63,8 @@ declare global { } type AttachmentType = { - id?: string; + cdnId?: string; + cdnKey?: string; data: ArrayBuffer; contentType?: string; size?: number; @@ -71,6 +73,9 @@ type AttachmentType = { width?: number; height?: number; caption?: string; + blurHash?: string; + uploadTimestamp?: ProtoBigNumberType; + cdnNumber?: number; }; type CacheAddItemType = { @@ -1571,13 +1576,16 @@ class MessageReceiverInner extends EventTarget { cleanAttachment(attachment: AttachmentPointerClass) { return { ...omit(attachment, 'thumbnail'), - id: attachment.id.toString(), + cdnId: attachment.cdnId?.toString(), key: attachment.key ? attachment.key.toString('base64') : null, digest: attachment.digest ? attachment.digest.toString('base64') : null, }; } async downloadAttachment(attachment: AttachmentPointerClass) { - const encrypted = await this.server.getAttachment(attachment.id); + const encrypted = await this.server.getAttachment( + attachment.cdnId || attachment.cdnKey, + attachment.cdnNumber || 0 + ); const { key, digest, size } = attachment; if (!digest) { diff --git a/ts/textsecure/SendMessage.ts b/ts/textsecure/SendMessage.ts index db4dd43e6a..3a8dd34e8a 100644 --- a/ts/textsecure/SendMessage.ts +++ b/ts/textsecure/SendMessage.ts @@ -334,7 +334,7 @@ export default class MessageSender { const id = await this.server.putAttachment(result.ciphertext); const proto = new window.textsecure.protobuf.AttachmentPointer(); - proto.id = id; + proto.cdnId = id; proto.contentType = attachment.contentType; proto.key = key; proto.size = attachment.size; diff --git a/ts/textsecure/WebAPI.ts b/ts/textsecure/WebAPI.ts index 41ba225912..7e229a041c 100644 --- a/ts/textsecure/WebAPI.ts +++ b/ts/textsecure/WebAPI.ts @@ -491,7 +491,10 @@ const URL_CALLS = { type InitializeOptionsType = { url: string; - cdnUrl: string; + cdnUrlObject: { + readonly '0': string; + readonly [propName: string]: string; + }; certificateAuthority: string; contentProxyUrl: string; proxyUrl: string; @@ -532,7 +535,7 @@ export type WebAPIType = { deviceName?: string | null, options?: { accessKey?: ArrayBuffer } ) => Promise; - getAttachment: (id: string) => Promise; + getAttachment: (cdnKey: string, cdnNumber: number) => Promise; getAvatar: (path: string) => Promise; getDevices: () => Promise; getKeysForIdentifier: ( @@ -643,7 +646,7 @@ export type ProxiedRequestOptionsType = { // tslint:disable-next-line max-func-body-length export function initialize({ url, - cdnUrl, + cdnUrlObject, certificateAuthority, contentProxyUrl, proxyUrl, @@ -652,8 +655,14 @@ export function initialize({ if (!is.string(url)) { throw new Error('WebAPI.initialize: Invalid server url'); } - if (!is.string(cdnUrl)) { - throw new Error('WebAPI.initialize: Invalid cdnUrl'); + if (!is.object(cdnUrlObject)) { + throw new Error('WebAPI.initialize: Invalid cdnUrlObject'); + } + if (!is.string(cdnUrlObject['0'])) { + throw new Error('WebAPI.initialize: Missing CDN 0 configuration'); + } + if (!is.string(cdnUrlObject['2'])) { + throw new Error('WebAPI.initialize: Missing CDN 2 configuration'); } if (!is.string(certificateAuthority)) { throw new Error('WebAPI.initialize: Invalid certificateAuthority'); @@ -871,7 +880,7 @@ export function initialize({ async function getAvatar(path: string) { // Using _outerAJAX, since it's not hardcoded to the Signal Server. Unlike our // attachment CDN, it uses our self-signed certificate, so we pass it in. - return _outerAjax(`${cdnUrl}/${path}`, { + return _outerAjax(`${cdnUrlObject['0']}/${path}`, { certificateAuthority, contentType: 'application/octet-stream', proxyUrl, @@ -1198,25 +1207,31 @@ export function initialize({ } async function getSticker(packId: string, stickerId: string) { - return _outerAjax(`${cdnUrl}/stickers/${packId}/full/${stickerId}`, { - certificateAuthority, - proxyUrl, - responseType: 'arraybuffer', - type: 'GET', - redactUrl: redactStickerUrl, - version, - }); + return _outerAjax( + `${cdnUrlObject['0']}/stickers/${packId}/full/${stickerId}`, + { + certificateAuthority, + proxyUrl, + responseType: 'arraybuffer', + type: 'GET', + redactUrl: redactStickerUrl, + version, + } + ); } async function getStickerPackManifest(packId: string) { - return _outerAjax(`${cdnUrl}/stickers/${packId}/manifest.proto`, { - certificateAuthority, - proxyUrl, - responseType: 'arraybuffer', - type: 'GET', - redactUrl: redactStickerUrl, - version, - }); + return _outerAjax( + `${cdnUrlObject['0']}/stickers/${packId}/manifest.proto`, + { + certificateAuthority, + proxyUrl, + responseType: 'arraybuffer', + type: 'GET', + redactUrl: redactStickerUrl, + version, + } + ); } type ServerAttachmentType = { @@ -1304,7 +1319,7 @@ export function initialize({ // Upload manifest const manifestParams = makePutParams(manifest, encryptedManifest); // This is going to the CDN, not the service, so we use _outerAjax - await _outerAjax(`${cdnUrl}/`, { + await _outerAjax(`${cdnUrlObject['0']}/`, { ...manifestParams, certificateAuthority, proxyUrl, @@ -1322,7 +1337,7 @@ export function initialize({ encryptedStickers[index] ); await queue.add(async () => - _outerAjax(`${cdnUrl}/`, { + _outerAjax(`${cdnUrlObject['0']}/`, { ...stickerParams, certificateAuthority, proxyUrl, @@ -1341,9 +1356,10 @@ export function initialize({ return packId; } - async function getAttachment(id: string) { + async function getAttachment(cdnKey: string, cdnNumber: number) { + const cdnUrl = cdnUrlObject[cdnNumber] || cdnUrlObject['0']; // This is going to the CDN, not the service, so we use _outerAjax - return _outerAjax(`${cdnUrl}/attachments/${id}`, { + return _outerAjax(`${cdnUrl}/attachments/${cdnKey}`, { certificateAuthority, proxyUrl, responseType: 'arraybuffer', @@ -1365,7 +1381,7 @@ export function initialize({ const params = makePutParams(response, encryptedBin); // This is going to the CDN, not the service, so we use _outerAjax - await _outerAjax(`${cdnUrl}/attachments/`, { + await _outerAjax(`${cdnUrlObject['0']}/attachments/`, { ...params, certificateAuthority, proxyUrl, diff --git a/ts/util/lint/exceptions.json b/ts/util/lint/exceptions.json index cdb58450e8..14c5304a64 100644 --- a/ts/util/lint/exceptions.json +++ b/ts/util/lint/exceptions.json @@ -11817,4 +11817,4 @@ "reasonCategory": "falseMatch", "updated": "2020-04-05T23:45:16.746Z" } -] \ No newline at end of file +]