From 1ce0959fa178efce3dad04e2339c3a4fb2297a01 Mon Sep 17 00:00:00 2001 From: Josh Perez <60019601+josh-signal@users.noreply.github.com> Date: Tue, 8 Sep 2020 20:56:23 -0400 Subject: [PATCH] Storage Service: Write --- js/background.js | 11 +- js/models/conversations.js | 119 +++++- js/modules/signal.js | 12 +- ts/model-types.d.ts | 30 +- ts/services/storage.ts | 707 ++++++++++++++++++++++++++++++++ ts/services/storageRecordOps.ts | 403 ++++++++++++++++++ ts/sql/Client.ts | 6 +- ts/sql/Interface.ts | 2 +- ts/sql/Server.ts | 6 +- ts/textsecure.d.ts | 72 +++- ts/textsecure/SendMessage.ts | 37 +- ts/textsecure/WebAPI.ts | 21 + ts/util/index.ts | 6 - ts/util/lint/exceptions.json | 4 +- ts/util/storageService.ts | 478 --------------------- 15 files changed, 1374 insertions(+), 540 deletions(-) create mode 100644 ts/services/storage.ts create mode 100644 ts/services/storageRecordOps.ts delete mode 100644 ts/util/storageService.ts diff --git a/js/background.js b/js/background.js index 6fe608ec1104..844f172b6f0f 100644 --- a/js/background.js +++ b/js/background.js @@ -549,7 +549,7 @@ window.isBeforeVersion(lastVersion, 'v1.35.0-beta.11') && window.isAfterVersion(lastVersion, 'v1.35.0-beta.1') ) { - await window.Signal.Util.eraseAllStorageServiceState(); + await window.Signal.Services.eraseAllStorageServiceState(); } // This one should always be last - it could restart the app @@ -2818,7 +2818,7 @@ break; case FETCH_LATEST_ENUM.STORAGE_MANIFEST: window.log.info('onFetchLatestSync: fetching latest manifest'); - await window.Signal.Util.runStorageServiceSyncJob(); + await window.Signal.Services.runStorageServiceSyncJob(); break; default: window.log.info( @@ -2832,6 +2832,11 @@ const { storageServiceKey } = ev; + if (storageServiceKey === null) { + window.log.info('onKeysSync: deleting storageKey'); + storage.remove('storageKey'); + } + if (storageServiceKey) { window.log.info('onKeysSync: received keys'); const storageServiceKeyBase64 = window.Signal.Crypto.arrayBufferToBase64( @@ -2839,7 +2844,7 @@ ); storage.put('storageKey', storageServiceKeyBase64); - await window.Signal.Util.runStorageServiceSyncJob(); + await window.Signal.Services.runStorageServiceSyncJob(); } } diff --git a/js/models/conversations.js b/js/models/conversations.js index c8fb669055b9..c01f232e6d33 100644 --- a/js/models/conversations.js +++ b/js/models/conversations.js @@ -25,7 +25,7 @@ UNRESTRICTED: 3, }; - const { Util } = window.Signal; + const { Services, Util } = window.Signal; const { Contact, Message } = window.Signal.Types; const { deleteAttachmentData, @@ -234,48 +234,84 @@ return false; }, - block() { + block({ viaStorageServiceSync = false } = {}) { + let blocked = false; + const isBlocked = this.isBlocked(); + const uuid = this.get('uuid'); if (uuid) { window.storage.addBlockedUuid(uuid); + blocked = true; } const e164 = this.get('e164'); if (e164) { window.storage.addBlockedNumber(e164); + blocked = true; } const groupId = this.get('groupId'); if (groupId) { window.storage.addBlockedGroup(groupId); + blocked = true; + } + + if (!viaStorageServiceSync && !isBlocked && blocked) { + this.captureChange(); } }, - unblock() { + unblock({ viaStorageServiceSync = false } = {}) { + let unblocked = false; + const isBlocked = this.isBlocked(); + const uuid = this.get('uuid'); if (uuid) { window.storage.removeBlockedUuid(uuid); + unblocked = true; } const e164 = this.get('e164'); if (e164) { window.storage.removeBlockedNumber(e164); + unblocked = true; } const groupId = this.get('groupId'); if (groupId) { window.storage.removeBlockedGroup(groupId); + unblocked = true; } - return false; + if (!viaStorageServiceSync && isBlocked && unblocked) { + this.captureChange(); + } + + return unblocked; }, - enableProfileSharing() { + enableProfileSharing({ viaStorageServiceSync = false } = {}) { + const before = this.get('profileSharing'); + this.set({ profileSharing: true }); + + const after = this.get('profileSharing'); + + if (!viaStorageServiceSync && Boolean(before) !== Boolean(after)) { + this.captureChange(); + } }, - disableProfileSharing() { + disableProfileSharing({ viaStorageServiceSync = false } = {}) { + const before = this.get('profileSharing'); + this.set({ profileSharing: false }); + + const after = this.get('profileSharing'); + + if (!viaStorageServiceSync && Boolean(before) !== Boolean(after)) { + this.captureChange(); + } }, hasDraft() { @@ -662,7 +698,10 @@ } while (messages.length > 0); }, - async applyMessageRequestResponse(response, { fromSync = false } = {}) { + async applyMessageRequestResponse( + response, + { fromSync = false, viaStorageServiceSync = false } = {} + ) { // Apply message request response locally this.set({ messageRequestResponseType: response, @@ -670,8 +709,8 @@ window.Signal.Data.updateConversation(this.attributes); if (response === this.messageRequestEnum.ACCEPT) { - this.unblock(); - this.enableProfileSharing(); + this.unblock({ viaStorageServiceSync }); + this.enableProfileSharing({ viaStorageServiceSync }); if (!fromSync) { this.sendProfileKeyUpdate(); @@ -680,13 +719,13 @@ } } else if (response === this.messageRequestEnum.BLOCK) { // Block locally, other devices should block upon receiving the sync message - this.block(); - this.disableProfileSharing(); + this.block({ viaStorageServiceSync }); + this.disableProfileSharing({ viaStorageServiceSync }); } else if (response === this.messageRequestEnum.DELETE) { // Delete messages locally, other devices should delete upon receiving // the sync message this.destroyMessages(); - this.disableProfileSharing(); + this.disableProfileSharing({ viaStorageServiceSync }); this.updateLastMessage(); if (!fromSync) { this.trigger('unload', 'deleted from message request'); @@ -695,10 +734,10 @@ // Delete messages locally, other devices should delete upon receiving // the sync message this.destroyMessages(); - this.disableProfileSharing(); + this.disableProfileSharing({ viaStorageServiceSync }); this.updateLastMessage(); // Block locally, other devices should block upon receiving the sync message - this.block(); + this.block({ viaStorageServiceSync }); // Leave group if this was a local action if (!fromSync) { this.leaveGroup(); @@ -780,6 +819,7 @@ async _setVerified(verified, providedOptions) { const options = providedOptions || {}; _.defaults(options, { + viaStorageServiceSync: false, viaSyncMessage: false, viaContactSync: false, key: null, @@ -814,6 +854,14 @@ this.set({ verified }); window.Signal.Data.updateConversation(this.attributes); + if ( + !options.viaStorageServiceSync && + !keyChange && + beginningVerified !== verified + ) { + this.captureChange(); + } + // Three situations result in a verification notice in the conversation: // 1) The message came from an explicit verification in another client (not // a contact sync) @@ -1982,9 +2030,17 @@ window.Signal.Data.updateConversation(this.attributes); }, - async setArchived(isArchived) { + setArchived(isArchived) { + const before = this.get('isArchived'); + this.set({ isArchived }); window.Signal.Data.updateConversation(this.attributes); + + const after = this.get('isArchived'); + + if (Boolean(before) !== Boolean(after)) { + this.captureChange(); + } }, async updateExpirationTimer( @@ -2573,7 +2629,7 @@ } try { - await c.setProfileName(profile.name); + await c.setEncryptedProfileName(profile.name); } catch (error) { window.log.warn( 'getProfile decryption failure:', @@ -2598,7 +2654,7 @@ window.Signal.Data.updateConversation(c.attributes); }, - async setProfileName(encryptedName) { + async setEncryptedProfileName(encryptedName) { if (!encryptedName) { return; } @@ -2648,6 +2704,10 @@ return; } + if (this.isMe()) { + window.storage.put('avatarUrl', avatarPath); + } + const avatar = await textsecure.messaging.getAvatar(avatarPath); const key = this.get('profileKey'); if (!key) { @@ -2675,7 +2735,7 @@ this.set(newAttributes); } }, - async setProfileKey(profileKey) { + async setProfileKey(profileKey, { viaStorageServiceSync = false } = {}) { // profileKey is a string so we can compare it directly if (this.get('profileKey') !== profileKey) { window.log.info( @@ -2689,6 +2749,10 @@ sealedSender: SEALED_SENDER.UNKNOWN, }); + if (!viaStorageServiceSync) { + this.captureChange(); + } + await Promise.all([ this.deriveAccessKeyIfNeeded(), this.deriveProfileKeyVersionIfNeeded(), @@ -2883,6 +2947,25 @@ return null; }, + // Set of items to captureChanges on: + // [-] uuid + // [-] e164 + // [X] profileKey + // [-] identityKey + // [X] verified! + // [-] profileName + // [-] profileFamilyName + // [X] blocked + // [X] whitelisted + // [X] archived + captureChange() { + this.set({ needsStorageServiceSync: true }); + + this.queueJob(() => { + Services.storageServiceUploadJob(); + }); + }, + async notify(message, reaction) { if (this.get('muteExpiresAt') && Date.now() < this.get('muteExpiresAt')) { return; diff --git a/js/modules/signal.js b/js/modules/signal.js index ce0b309b9637..87985922cf1e 100644 --- a/js/modules/signal.js +++ b/js/modules/signal.js @@ -116,6 +116,12 @@ const { } = require('../../ts/services/updateListener'); const { notify } = require('../../ts/services/notify'); const { calling } = require('../../ts/services/calling'); +const { + eraseAllStorageServiceState, + handleUnknownRecords, + runStorageServiceSyncJob, + storageServiceUploadJob, +} = require('../../ts/services/storage'); function initializeMigrations({ userDataPath, @@ -324,10 +330,14 @@ exports.setup = (options = {}) => { }; const Services = { + calling, + eraseAllStorageServiceState, + handleUnknownRecords, initializeNetworkObserver, initializeUpdateListener, notify, - calling, + runStorageServiceSyncJob, + storageServiceUploadJob, }; const State = { diff --git a/ts/model-types.d.ts b/ts/model-types.d.ts index 6c156b5fd9e1..807c6b4d6f79 100644 --- a/ts/model-types.d.ts +++ b/ts/model-types.d.ts @@ -58,18 +58,27 @@ type ConversationAttributesType = { isArchived?: boolean; lastMessage?: string; members?: Array; + needsStorageServiceSync?: boolean; needsVerification?: boolean; profileFamilyName?: string | null; profileKey?: string | null; profileName?: string | null; profileSharing: boolean; storageID?: string; + storageUnknownFields: string; type: ConversationTypeType; unreadCount?: number; verified?: number; version: number; }; +type VerificationOptions = { + key?: null | ArrayBuffer; + viaContactSync?: boolean; + viaStorageServiceSync?: boolean; + viaSyncMessage?: boolean; +}; + export declare class ConversationModelType extends Backbone.Model< ConversationAttributesType > { @@ -81,11 +90,12 @@ export declare class ConversationModelType extends Backbone.Model< addCallHistory(details: CallHistoryDetailsType): void; applyMessageRequestResponse( response: number, - options?: { fromSync: boolean } + options?: { fromSync: boolean; viaStorageServiceSync?: boolean } ): void; cleanup(): Promise; - disableProfileSharing(): void; + disableProfileSharing(options?: { viaStorageServiceSync?: boolean }): void; dropProfileKey(): Promise; + enableProfileSharing(options?: { viaStorageServiceSync?: boolean }): void; generateProps(): void; getAccepted(): boolean; getAvatarPath(): string | undefined; @@ -99,11 +109,23 @@ export declare class ConversationModelType extends Backbone.Model< getTitle(): string; idForLogging(): string; isFromOrAddedByTrustedContact(): boolean; + isBlocked(): boolean; + isMe(): boolean; + isPrivate(): boolean; isVerified(): boolean; safeGetVerified(): Promise; - setProfileKey(profileKey?: string | null): Promise; + setArchived(isArchived: boolean): void; + setProfileKey( + profileKey?: string | null, + options?: { viaStorageServiceSync?: boolean } + ): Promise; + setProfileAvatar(avatarPath: string): Promise; + setUnverified(options: VerificationOptions): Promise; + setVerified(options: VerificationOptions): Promise; + setVerifiedDefault(options: VerificationOptions): Promise; toggleVerified(): Promise; - unblock(): boolean | undefined; + block(options?: { viaStorageServiceSync?: boolean }): void; + unblock(options?: { viaStorageServiceSync?: boolean }): boolean; updateE164: (e164?: string) => void; updateLastMessage: () => Promise; updateUuid: (uuid?: string) => void; diff --git a/ts/services/storage.ts b/ts/services/storage.ts new file mode 100644 index 000000000000..8f2592bb1c0f --- /dev/null +++ b/ts/services/storage.ts @@ -0,0 +1,707 @@ +/* tslint:disable no-backbone-get-set-outside-model */ +import _ from 'lodash'; +import pMap from 'p-map'; + +import Crypto from '../textsecure/Crypto'; +import dataInterface from '../sql/Client'; +import { + arrayBufferToBase64, + base64ToArrayBuffer, + deriveStorageItemKey, + deriveStorageManifestKey, +} from '../Crypto'; +import { + ManifestRecordClass, + ManifestRecordIdentifierClass, + StorageItemClass, + StorageManifestClass, + StorageRecordClass, +} from '../textsecure.d'; +import { ConversationModelType } from '../model-types.d'; +import { + mergeAccountRecord, + mergeContactRecord, + mergeGroupV1Record, + toAccountRecord, + toContactRecord, + toGroupV1Record, +} from './storageRecordOps'; + +const { + eraseStorageServiceStateFromConversations, + updateConversation, +} = dataInterface; + +let consecutiveConflicts = 0; + +type UnknownRecord = { + itemType: number; + storageID: string; +}; + +function createWriteOperation( + storageManifest: StorageManifestClass, + newItems: Array, + deleteKeys: Array, + clearAll = false +) { + const writeOperation = new window.textsecure.protobuf.WriteOperation(); + writeOperation.manifest = storageManifest; + writeOperation.insertItem = newItems; + writeOperation.deleteKey = deleteKeys; + writeOperation.clearAll = clearAll; + + return writeOperation; +} + +async function encryptRecord( + storageID: string | undefined, + storageRecord: StorageRecordClass +): Promise { + const storageItem = new window.textsecure.protobuf.StorageItem(); + + const storageKeyBuffer = storageID + ? base64ToArrayBuffer(String(storageID)) + : generateStorageID(); + + const storageKeyBase64 = window.storage.get('storageKey'); + const storageKey = base64ToArrayBuffer(storageKeyBase64); + const storageItemKey = await deriveStorageItemKey( + storageKey, + arrayBufferToBase64(storageKeyBuffer) + ); + + const encryptedRecord = await Crypto.encryptProfile( + storageRecord.toArrayBuffer(), + storageItemKey + ); + + storageItem.key = storageKeyBuffer; + storageItem.value = encryptedRecord; + + return storageItem; +} + +function generateStorageID(): ArrayBuffer { + return Crypto.getRandomBytes(16); +} + +type GeneratedManifestType = { + conversationsToUpdate: Array<{ + conversation: ConversationModelType; + storageID: string | undefined; + }>; + deleteKeys: Array; + newItems: Set; + storageManifest: StorageManifestClass; +}; + +/* tslint:disable-next-line max-func-body-length */ +async function generateManifest( + version: number, + isNewManifest = false +): Promise { + window.log.info( + `storageService.generateManifest: generating manifest for version ${version}. Is new? ${isNewManifest}` + ); + + const ITEM_TYPE = window.textsecure.protobuf.ManifestRecord.Identifier.Type; + + const conversationsToUpdate = []; + const deleteKeys = []; + const manifestRecordKeys: Set = new Set(); + const newItems: Set = new Set(); + + const conversations = window.getConversations(); + for (let i = 0; i < conversations.length; i += 1) { + const conversation = conversations.models[i]; + const identifier = new window.textsecure.protobuf.ManifestRecord.Identifier(); + + let storageRecord; + if (conversation.isMe()) { + storageRecord = new window.textsecure.protobuf.StorageRecord(); + // eslint-disable-next-line no-await-in-loop + storageRecord.account = await toAccountRecord(conversation); + identifier.type = ITEM_TYPE.ACCOUNT; + } else if (conversation.isPrivate()) { + storageRecord = new window.textsecure.protobuf.StorageRecord(); + // eslint-disable-next-line no-await-in-loop + storageRecord.contact = await toContactRecord(conversation); + identifier.type = ITEM_TYPE.CONTACT; + } else { + storageRecord = new window.textsecure.protobuf.StorageRecord(); + // eslint-disable-next-line no-await-in-loop + storageRecord.groupV1 = await toGroupV1Record(conversation); + identifier.type = ITEM_TYPE.GROUPV1; + } + + if (storageRecord) { + const isNewItem = + isNewManifest || Boolean(conversation.get('needsStorageServiceSync')); + + const storageID = isNewItem + ? arrayBufferToBase64(generateStorageID()) + : conversation.get('storageID'); + + // eslint-disable-next-line no-await-in-loop + const storageItem = await encryptRecord(storageID, storageRecord); + identifier.raw = storageItem.key; + + // When a client needs to update a given record it should create it + // under a new key and delete the existing key. + if (isNewItem) { + newItems.add(storageItem); + + const oldStorageID = conversation.get('storageID'); + if (oldStorageID) { + deleteKeys.push(base64ToArrayBuffer(oldStorageID)); + } + + conversationsToUpdate.push({ + conversation, + storageID, + }); + } + + manifestRecordKeys.add(identifier); + } + } + + const unknownRecordsArray = + window.storage.get('storage-service-unknown-records') || []; + + window.log.info( + `storageService.generateManifest: adding ${unknownRecordsArray.length} unknown records` + ); + + // When updating the manifest, ensure all "unknown" keys are added to the + // new manifest, so we don't inadvertently delete something we don't understand + unknownRecordsArray.forEach((record: UnknownRecord) => { + const identifier = new window.textsecure.protobuf.ManifestRecord.Identifier(); + identifier.type = record.itemType; + identifier.raw = base64ToArrayBuffer(record.storageID); + + manifestRecordKeys.add(identifier); + }); + + const manifestRecord = new window.textsecure.protobuf.ManifestRecord(); + manifestRecord.version = version; + manifestRecord.keys = Array.from(manifestRecordKeys); + + const storageKeyBase64 = window.storage.get('storageKey'); + const storageKey = base64ToArrayBuffer(storageKeyBase64); + const storageManifestKey = await deriveStorageManifestKey( + storageKey, + version + ); + const encryptedManifest = await Crypto.encryptProfile( + manifestRecord.toArrayBuffer(), + storageManifestKey + ); + + const storageManifest = new window.textsecure.protobuf.StorageManifest(); + storageManifest.version = version; + storageManifest.value = encryptedManifest; + + return { + conversationsToUpdate, + deleteKeys, + newItems, + storageManifest, + }; +} + +async function uploadManifest( + version: number, + { + conversationsToUpdate, + deleteKeys, + newItems, + storageManifest, + }: GeneratedManifestType +): Promise { + if (!window.textsecure.messaging) { + throw new Error('storageService.uploadManifest: We are offline!'); + } + + const credentials = window.storage.get('storageCredentials'); + try { + window.log.info( + `storageService.uploadManifest: inserting ${newItems.size} items, deleting ${deleteKeys.length} keys` + ); + + const writeOperation = createWriteOperation( + storageManifest, + Array.from(newItems), + deleteKeys + ); + + window.log.info('storageService.uploadManifest: uploading...'); + await window.textsecure.messaging.modifyStorageRecords( + writeOperation.toArrayBuffer(), + { + credentials, + } + ); + + window.log.info( + `storageService.uploadManifest: upload done, updating ${conversationsToUpdate.length} conversation(s) with new storageIDs` + ); + + // update conversations with the new storageID + conversationsToUpdate.forEach(({ conversation, storageID }) => { + conversation.set({ + needsStorageServiceSync: false, + storageID, + }); + updateConversation(conversation.attributes); + }); + } catch (err) { + window.log.error( + `storageService.uploadManifest: failed! ${ + err && err.stack ? err.stack : String(err) + }` + ); + + if (err.code === 409) { + window.log.info( + `storageService.uploadManifest: Conflict found, running sync job times(${consecutiveConflicts})` + ); + + if (consecutiveConflicts > 3) { + window.log.error( + 'storageService.uploadManifest: Exceeded maximum consecutive conflicts' + ); + return; + } + consecutiveConflicts += 1; + + throw err; + } + + throw err; + } + + window.log.info( + 'storageService.uploadManifest: setting new manifestVersion', + version + ); + window.storage.put('manifestVersion', version); + consecutiveConflicts = 0; + await window.textsecure.messaging.sendFetchManifestSyncMessage(); +} + +async function stopStorageServiceSync() { + window.log.info('storageService.stopStorageServiceSync'); + + await window.storage.remove('storageKey'); + + if (!window.textsecure.messaging) { + throw new Error('storageService.stopStorageServiceSync: We are offline!'); + } + + await window.textsecure.messaging.sendRequestKeySyncMessage(); +} + +async function createNewManifest() { + window.log.info('storageService.createNewManifest: creating new manifest'); + + const version = window.storage.get('manifestVersion') || 0; + + const { + conversationsToUpdate, + newItems, + storageManifest, + } = await generateManifest(version, true); + + await uploadManifest(version, { + conversationsToUpdate, + // we have created a new manifest, there should be no keys to delete + deleteKeys: [], + newItems, + storageManifest, + }); +} + +async function decryptManifest( + encryptedManifest: StorageManifestClass +): Promise { + const { version, value } = encryptedManifest; + + const storageKeyBase64 = window.storage.get('storageKey'); + const storageKey = base64ToArrayBuffer(storageKeyBase64); + const storageManifestKey = await deriveStorageManifestKey( + storageKey, + typeof version === 'number' ? version : version.toNumber() + ); + + const decryptedManifest = await Crypto.decryptProfile( + typeof value.toArrayBuffer === 'function' ? value.toArrayBuffer() : value, + storageManifestKey + ); + + return window.textsecure.protobuf.ManifestRecord.decode(decryptedManifest); +} + +async function fetchManifest( + manifestVersion: string +): Promise { + window.log.info('storageService.fetchManifest'); + + if (!window.textsecure.messaging) { + throw new Error('storageService.fetchManifest: We are offline!'); + } + + try { + const credentials = await window.textsecure.messaging.getStorageCredentials(); + window.storage.put('storageCredentials', credentials); + + const manifestBinary = await window.textsecure.messaging.getStorageManifest( + { + credentials, + greaterThanVersion: manifestVersion, + } + ); + const encryptedManifest = window.textsecure.protobuf.StorageManifest.decode( + manifestBinary + ); + + // if we don't get a value we're assuming that there's no newer manifest + if (!encryptedManifest.value || !encryptedManifest.version) { + window.log.info('storageService.fetchManifest: nothing changed'); + return; + } + + try { + // eslint-disable-next-line consistent-return + return decryptManifest(encryptedManifest); + } catch (err) { + await stopStorageServiceSync(); + return; + } + } catch (err) { + window.log.error( + `storageService.fetchManifest: failed! ${ + err && err.stack ? err.stack : String(err) + }` + ); + + if (err.code === 404) { + await createNewManifest(); + return; + } else if (err.code === 204) { + // noNewerManifest we're ok + return; + } + + throw err; + } +} + +type MergeableItemType = { + itemType: number; + storageID: string; + storageRecord: StorageRecordClass; +}; + +type MergedRecordType = UnknownRecord & { + hasConflict: boolean; + isUnsupported: boolean; +}; + +async function mergeRecord( + itemToMerge: MergeableItemType +): Promise { + const { itemType, storageID, storageRecord } = itemToMerge; + + const ITEM_TYPE = window.textsecure.protobuf.ManifestRecord.Identifier.Type; + + let hasConflict = false; + let isUnsupported = false; + + try { + if (itemType === ITEM_TYPE.UNKNOWN) { + window.log.info( + 'storageService.mergeRecord: Unknown item type', + storageID + ); + } else if (itemType === ITEM_TYPE.CONTACT && storageRecord.contact) { + hasConflict = await mergeContactRecord(storageID, storageRecord.contact); + } else if (itemType === ITEM_TYPE.GROUPV1 && storageRecord.groupV1) { + hasConflict = await mergeGroupV1Record(storageID, storageRecord.groupV1); + } else if (itemType === ITEM_TYPE.ACCOUNT && storageRecord.account) { + hasConflict = await mergeAccountRecord(storageID, storageRecord.account); + } else { + isUnsupported = true; + window.log.info( + `storageService.mergeRecord: Unknown record: ${itemType}::${storageID}` + ); + } + } catch (err) { + window.log.error( + `storageService.mergeRecord: merging record failed ${storageID}` + ); + } + + return { + hasConflict, + isUnsupported, + itemType, + storageID, + }; +} + +/* tslint:disable-next-line max-func-body-length */ +async function processManifest( + manifest: ManifestRecordClass +): Promise { + const storageKeyBase64 = window.storage.get('storageKey'); + const storageKey = base64ToArrayBuffer(storageKeyBase64); + + if (!window.textsecure.messaging) { + throw new Error('storageService.processManifest: We are offline!'); + } + + const remoteKeysTypeMap = new Map(); + manifest.keys.forEach((identifier: ManifestRecordIdentifierClass) => { + remoteKeysTypeMap.set( + arrayBufferToBase64(identifier.raw.toArrayBuffer()), + identifier.type + ); + }); + + const localKeys = window + .getConversations() + .map((conversation: ConversationModelType) => conversation.get('storageID')) + .filter(Boolean); + + const unknownRecordsArray = + window.storage.get('storage-service-unknown-records') || []; + + unknownRecordsArray.forEach((record: UnknownRecord) => { + localKeys.push(record.storageID); + }); + + window.log.info( + `storageService.processManifest: localKeys.length ${localKeys.length}` + ); + + const remoteKeys = Array.from(remoteKeysTypeMap.keys()); + + const remoteOnly = remoteKeys.filter( + (key: string) => !localKeys.includes(key) + ); + + window.log.info( + `storageService.processManifest: remoteOnly.length ${remoteOnly.length}` + ); + + const readOperation = new window.textsecure.protobuf.ReadOperation(); + readOperation.readKey = remoteOnly.map(base64ToArrayBuffer); + + const credentials = window.storage.get('storageCredentials'); + const storageItemsBuffer = await window.textsecure.messaging.getStorageRecords( + readOperation.toArrayBuffer(), + { + credentials, + } + ); + + const storageItems = window.textsecure.protobuf.StorageItems.decode( + storageItemsBuffer + ); + + if (!storageItems.items) { + window.log.info( + 'storageService.processManifest: No storage items retrieved' + ); + return false; + } + + const decryptedStorageItems = await pMap( + storageItems.items, + async (storageRecordWrapper: StorageItemClass) => { + const { key, value: storageItemCiphertext } = storageRecordWrapper; + + if (!key || !storageItemCiphertext) { + await stopStorageServiceSync(); + throw new Error( + 'storageService.processManifest: Missing key and/or Ciphertext' + ); + } + + const base64ItemID = arrayBufferToBase64(key.toArrayBuffer()); + + const storageItemKey = await deriveStorageItemKey( + storageKey, + base64ItemID + ); + + let storageItemPlaintext; + try { + storageItemPlaintext = await Crypto.decryptProfile( + storageItemCiphertext.toArrayBuffer(), + storageItemKey + ); + } catch (err) { + await stopStorageServiceSync(); + throw err; + } + + const storageRecord = window.textsecure.protobuf.StorageRecord.decode( + storageItemPlaintext + ); + + return { + itemType: remoteKeysTypeMap.get(base64ItemID), + storageID: base64ItemID, + storageRecord, + }; + }, + { concurrency: 50 } + ); + + try { + const mergedRecords = await pMap(decryptedStorageItems, mergeRecord, { + concurrency: 5, + }); + + const unknownRecords: Map = new Map(); + unknownRecordsArray.forEach((record: UnknownRecord) => { + unknownRecords.set(record.storageID, record); + }); + + const hasConflict = mergedRecords.some((mergedRecord: MergedRecordType) => { + if (mergedRecord.isUnsupported) { + unknownRecords.set(mergedRecord.storageID, { + itemType: mergedRecord.itemType, + storageID: mergedRecord.storageID, + }); + } + return mergedRecord.hasConflict; + }); + + window.storage.put( + 'storage-service-unknown-records', + Array.from(unknownRecords.values()) + ); + + if (hasConflict) { + window.log.info( + 'storageService.processManifest: Conflict found, uploading changes' + ); + + return true; + } else { + consecutiveConflicts = 0; + } + } catch (err) { + window.log.error( + `storageService.processManifest: failed! ${ + err && err.stack ? err.stack : String(err) + }` + ); + } + + return false; +} + +// Exported functions + +export async function runStorageServiceSyncJob(): Promise { + if (!window.storage.get('storageKey')) { + throw new Error( + 'storageService.runStorageServiceSyncJob: Cannot start; no storage key!' + ); + } + + window.log.info('storageService.runStorageServiceSyncJob: starting...'); + + try { + const localManifestVersion = window.storage.get('manifestVersion') || 0; + const manifest = await fetchManifest(localManifestVersion); + + // Guarding against no manifests being returned, everything should be ok + if (!manifest) { + window.log.info( + 'storageService.runStorageServiceSyncJob: no manifest, returning early' + ); + return; + } + + const version = manifest.version.toNumber(); + + window.log.info( + `storageService.runStorageServiceSyncJob: manifest versions - previous: ${localManifestVersion}, current: ${version}` + ); + + const hasConflicts = await processManifest(manifest); + if (hasConflicts) { + await storageServiceUploadJob(); + } + + window.storage.put('manifestVersion', version); + } catch (err) { + window.log.error( + `storageService.runStorageServiceSyncJob: error processing manifest ${ + err && err.stack ? err.stack : String(err) + }` + ); + } + + window.log.info('storageService.runStorageServiceSyncJob: complete'); +} + +// Note: this function must be called at startup once we handle unknown records +// of a certain type. This way once the runStorageServiceSyncJob function runs +// it'll pick up the new storage IDs and process them accordingly. +export function handleUnknownRecords(itemType: number): void { + const unknownRecordsArray = + window.storage.get('storage-service-unknown-records') || []; + const newUnknownRecords = unknownRecordsArray.filter( + (record: UnknownRecord) => record.itemType !== itemType + ); + window.storage.put('storage-service-unknown-records', newUnknownRecords); +} + +// Note: this function is meant to be called before ConversationController is hydrated. +// It goes directly to the database, so in-memory conversations will be out of date. +export async function eraseAllStorageServiceState(): Promise { + window.log.info('storageService.eraseAllStorageServiceState: starting...'); + await Promise.all([ + window.storage.remove('manifestVersion'), + window.storage.remove('storage-service-unknown-records'), + window.storage.remove('storageCredentials'), + ]); + await eraseStorageServiceStateFromConversations(); + window.log.info('storageService.eraseAllStorageServiceState: complete'); +} + +async function nondebouncedStorageServiceUploadJob(): Promise { + if (!window.storage.get('storageKey')) { + throw new Error( + 'storageService.storageServiceUploadJob: Cannot start; no storage key!' + ); + } + + const localManifestVersion = window.storage.get('manifestVersion') || 0; + const version = Number(localManifestVersion) + 1; + + window.log.info( + 'storageService.storageServiceUploadJob: will update to manifest version', + version + ); + + try { + await uploadManifest(version, await generateManifest(version)); + } catch (err) { + if (err.code === 409) { + await runStorageServiceSyncJob(); + } + } +} + +export const storageServiceUploadJob = _.debounce( + nondebouncedStorageServiceUploadJob, + 500 +); diff --git a/ts/services/storageRecordOps.ts b/ts/services/storageRecordOps.ts new file mode 100644 index 000000000000..f5b7ee13c081 --- /dev/null +++ b/ts/services/storageRecordOps.ts @@ -0,0 +1,403 @@ +/* tslint:disable no-backbone-get-set-outside-model */ +import _ from 'lodash'; + +import { + arrayBufferToBase64, + base64ToArrayBuffer, + fromEncodedBinaryToArrayBuffer, +} from '../Crypto'; +import dataInterface from '../sql/Client'; +import { + AccountRecordClass, + ContactRecordClass, + GroupV1RecordClass, +} from '../textsecure.d'; +import { ConversationModelType } from '../model-types.d'; + +const { updateConversation } = dataInterface; + +type RecordClass = AccountRecordClass | ContactRecordClass | GroupV1RecordClass; + +function toRecordVerified(verified: number): number { + const VERIFIED_ENUM = window.textsecure.storage.protocol.VerifiedStatus; + const STATE_ENUM = window.textsecure.protobuf.ContactRecord.IdentityState; + + switch (verified) { + case VERIFIED_ENUM.VERIFIED: + return STATE_ENUM.VERIFIED; + case VERIFIED_ENUM.UNVERIFIED: + return STATE_ENUM.UNVERIFIED; + default: + return STATE_ENUM.DEFAULT; + } +} + +function addUnknownFields( + record: RecordClass, + conversation: ConversationModelType +): void { + if (record.__unknownFields) { + conversation.set({ + storageUnknownFields: arrayBufferToBase64(record.__unknownFields), + }); + } +} + +function applyUnknownFields( + record: RecordClass, + conversation: ConversationModelType +): void { + if (conversation.get('storageUnknownFields')) { + // eslint-disable-next-line no-param-reassign + record.__unknownFields = base64ToArrayBuffer( + conversation.get('storageUnknownFields') + ); + } +} + +export async function toContactRecord( + conversation: ConversationModelType +): Promise { + const contactRecord = new window.textsecure.protobuf.ContactRecord(); + if (conversation.get('uuid')) { + contactRecord.serviceUuid = conversation.get('uuid'); + } + if (conversation.get('e164')) { + contactRecord.serviceE164 = conversation.get('e164'); + } + if (conversation.get('profileKey')) { + contactRecord.profileKey = base64ToArrayBuffer( + String(conversation.get('profileKey')) + ); + } + const identityKey = await window.textsecure.storage.protocol.loadIdentityKey( + conversation.id + ); + if (identityKey) { + contactRecord.identityKey = identityKey; + } + if (conversation.get('verified')) { + contactRecord.identityState = toRecordVerified( + Number(conversation.get('verified')) + ); + } + if (conversation.get('profileName')) { + contactRecord.givenName = conversation.get('profileName'); + } + if (conversation.get('profileFamilyName')) { + contactRecord.familyName = conversation.get('profileFamilyName'); + } + contactRecord.blocked = conversation.isBlocked(); + contactRecord.whitelisted = Boolean(conversation.get('profileSharing')); + contactRecord.archived = Boolean(conversation.get('isArchived')); + + applyUnknownFields(contactRecord, conversation); + + return contactRecord; +} + +export async function toAccountRecord( + conversation: ConversationModelType +): Promise { + const accountRecord = new window.textsecure.protobuf.AccountRecord(); + + if (conversation.get('profileKey')) { + accountRecord.profileKey = base64ToArrayBuffer( + String(conversation.get('profileKey')) + ); + } + if (conversation.get('profileName')) { + accountRecord.givenName = conversation.get('profileName') || ''; + } + if (conversation.get('profileFamilyName')) { + accountRecord.familyName = conversation.get('profileFamilyName') || ''; + } + accountRecord.avatarUrl = window.storage.get('avatarUrl') || ''; + accountRecord.noteToSelfArchived = Boolean(conversation.get('isArchived')); + accountRecord.readReceipts = Boolean( + window.storage.get('read-receipt-setting') + ); + accountRecord.sealedSenderIndicators = Boolean( + window.storage.get('sealedSenderIndicators') + ); + accountRecord.typingIndicators = Boolean( + window.storage.get('typingIndicators') + ); + accountRecord.linkPreviews = Boolean(window.storage.get('linkPreviews')); + + applyUnknownFields(accountRecord, conversation); + + return accountRecord; +} + +export async function toGroupV1Record( + conversation: ConversationModelType +): Promise { + const groupV1Record = new window.textsecure.protobuf.GroupV1Record(); + + groupV1Record.id = fromEncodedBinaryToArrayBuffer( + String(conversation.get('groupId')) + ); + groupV1Record.blocked = conversation.isBlocked(); + groupV1Record.whitelisted = Boolean(conversation.get('profileSharing')); + groupV1Record.archived = Boolean(conversation.get('isArchived')); + + applyUnknownFields(groupV1Record, conversation); + + return groupV1Record; +} + +type MessageRequestCapableRecord = ContactRecordClass | GroupV1RecordClass; + +function applyMessageRequestState( + record: MessageRequestCapableRecord, + conversation: ConversationModelType +): void { + if (record.blocked) { + conversation.applyMessageRequestResponse( + conversation.messageRequestEnum.BLOCK, + { fromSync: true, viaStorageServiceSync: true } + ); + } else if (record.whitelisted) { + // unblocking is also handled by this function which is why the next + // condition is part of the else-if and not separate + conversation.applyMessageRequestResponse( + conversation.messageRequestEnum.ACCEPT, + { fromSync: true, viaStorageServiceSync: true } + ); + } else if (!record.blocked) { + // if the condition above failed the state could still be blocked=false + // in which case we should unblock the conversation + conversation.unblock({ viaStorageServiceSync: true }); + } + + if (!record.whitelisted) { + conversation.disableProfileSharing({ viaStorageServiceSync: true }); + } +} + +function doesRecordHavePendingChanges( + mergedRecord: RecordClass, + serviceRecord: RecordClass, + conversation: ConversationModelType +): boolean { + const shouldSync = Boolean(conversation.get('needsStorageServiceSync')); + + const hasConflict = !_.isEqual(mergedRecord, serviceRecord); + + if (shouldSync && !hasConflict) { + conversation.set({ needsStorageServiceSync: false }); + } + + return shouldSync && hasConflict; +} + +export async function mergeGroupV1Record( + storageID: string, + groupV1Record: GroupV1RecordClass +): Promise { + window.log.info(`storageService.mergeGroupV1Record: merging ${storageID}`); + + if (!groupV1Record.id) { + window.log.info( + `storageService.mergeGroupV1Record: no ID for ${storageID}` + ); + return false; + } + + const groupId = groupV1Record.id.toBinary(); + + // We do a get here because we don't get enough information from just this source to + // be able to do the right thing with this group. So we'll update the local group + // record if we have one; otherwise we'll just drop this update. + const conversation = window.ConversationController.get(groupId); + if (!conversation) { + window.log.warn( + `storageService.mergeGroupV1Record: No conversation for group(${groupId})` + ); + return false; + } + + conversation.set({ + isArchived: Boolean(groupV1Record.archived), + storageID, + }); + + applyMessageRequestState(groupV1Record, conversation); + + addUnknownFields(groupV1Record, conversation); + + const hasPendingChanges = doesRecordHavePendingChanges( + await toGroupV1Record(conversation), + groupV1Record, + conversation + ); + + updateConversation(conversation.attributes); + + window.log.info(`storageService.mergeGroupV1Record: merged ${storageID}`); + + return hasPendingChanges; +} + +export async function mergeContactRecord( + storageID: string, + contactRecord: ContactRecordClass +): Promise { + window.log.info(`storageService.mergeContactRecord: merging ${storageID}`); + + window.normalizeUuids( + contactRecord, + ['serviceUuid'], + 'storageService.mergeContactRecord' + ); + + const e164 = contactRecord.serviceE164 || undefined; + const uuid = contactRecord.serviceUuid || undefined; + + const id = window.ConversationController.ensureContactIds({ + e164, + uuid, + highTrust: true, + }); + + if (!id) { + window.log.info( + `storageService.mergeContactRecord: no ID for ${storageID}` + ); + return false; + } + + const conversation = await window.ConversationController.getOrCreateAndWait( + id, + 'private' + ); + + if (contactRecord.profileKey) { + await conversation.setProfileKey( + arrayBufferToBase64(contactRecord.profileKey.toArrayBuffer()), + { viaStorageServiceSync: true } + ); + } + + const verified = await conversation.safeGetVerified(); + const storageServiceVerified = contactRecord.identityState || 0; + if (verified !== storageServiceVerified) { + const verifiedOptions = { viaStorageServiceSync: true }; + const STATE_ENUM = window.textsecure.protobuf.ContactRecord.IdentityState; + + switch (storageServiceVerified) { + case STATE_ENUM.VERIFIED: + await conversation.setVerified(verifiedOptions); + break; + case STATE_ENUM.UNVERIFIED: + await conversation.setUnverified(verifiedOptions); + break; + default: + await conversation.setVerifiedDefault(verifiedOptions); + } + } + + applyMessageRequestState(contactRecord, conversation); + + addUnknownFields(contactRecord, conversation); + + conversation.set({ + isArchived: Boolean(contactRecord.archived), + storageID, + }); + + const hasPendingChanges = doesRecordHavePendingChanges( + await toContactRecord(conversation), + contactRecord, + conversation + ); + + updateConversation(conversation.attributes); + + window.log.info(`storageService.mergeContactRecord: merged ${storageID}`); + + return hasPendingChanges; +} + +export async function mergeAccountRecord( + storageID: string, + accountRecord: AccountRecordClass +): Promise { + window.log.info(`storageService.mergeAccountRecord: merging ${storageID}`); + + const { + avatarUrl, + linkPreviews, + noteToSelfArchived, + profileKey, + readReceipts, + sealedSenderIndicators, + typingIndicators, + } = accountRecord; + + window.storage.put('read-receipt-setting', readReceipts); + + if (typeof sealedSenderIndicators === 'boolean') { + window.storage.put('sealedSenderIndicators', sealedSenderIndicators); + } + + if (typeof typingIndicators === 'boolean') { + window.storage.put('typingIndicators', typingIndicators); + } + + if (typeof linkPreviews === 'boolean') { + window.storage.put('linkPreviews', linkPreviews); + } + + if (profileKey) { + window.storage.put('profileKey', profileKey.toArrayBuffer()); + } + + window.log.info( + `storageService.mergeAccountRecord: merged settings ${storageID}` + ); + + const ourID = window.ConversationController.getOurConversationId(); + + if (!ourID) { + return false; + } + + const conversation = await window.ConversationController.getOrCreateAndWait( + ourID, + 'private' + ); + + addUnknownFields(accountRecord, conversation); + + conversation.set({ + isArchived: Boolean(noteToSelfArchived), + storageID, + }); + + if (accountRecord.profileKey) { + await conversation.setProfileKey( + arrayBufferToBase64(accountRecord.profileKey.toArrayBuffer()) + ); + } + + if (avatarUrl) { + await conversation.setProfileAvatar(avatarUrl); + window.storage.put('avatarUrl', avatarUrl); + } + + const hasPendingChanges = doesRecordHavePendingChanges( + await toAccountRecord(conversation), + accountRecord, + conversation + ); + + updateConversation(conversation.attributes); + + window.log.info( + `storageService.mergeAccountRecord: merged profile ${storageID}` + ); + + return hasPendingChanges; +} diff --git a/ts/sql/Client.ts b/ts/sql/Client.ts index 8789228f96e9..c85036d6edc2 100644 --- a/ts/sql/Client.ts +++ b/ts/sql/Client.ts @@ -129,7 +129,7 @@ const dataInterface: ClientInterface = { updateConversations, removeConversation, - eraseStorageIdFromConversations, + eraseStorageServiceStateFromConversations, getAllConversations, getAllConversationIds, getAllPrivateConversations, @@ -773,8 +773,8 @@ async function _removeConversations(ids: Array) { await channels.removeConversation(ids); } -async function eraseStorageIdFromConversations() { - await channels.eraseStorageIdFromConversations(); +async function eraseStorageServiceStateFromConversations() { + await channels.eraseStorageServiceStateFromConversations(); } async function getAllConversations({ diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 4635bdc4a7b5..dce3a81cecf5 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -67,7 +67,7 @@ export interface DataInterface { removeAllSessions: () => Promise; getAllSessions: () => Promise>; - eraseStorageIdFromConversations: () => Promise; + eraseStorageServiceStateFromConversations: () => Promise; getConversationCount: () => Promise; saveConversation: (data: ConversationType) => Promise; saveConversations: (array: Array) => Promise; diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index 7d30d6751c76..89149e164808 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -104,7 +104,7 @@ const dataInterface: ServerInterface = { updateConversation, updateConversations, removeConversation, - eraseStorageIdFromConversations, + eraseStorageServiceStateFromConversations, getAllConversations, getAllConversationIds, getAllPrivateConversations, @@ -2243,12 +2243,12 @@ async function getConversationById(id: string) { return jsonToObject(row.json); } -async function eraseStorageIdFromConversations() { +async function eraseStorageServiceStateFromConversations() { const db = getInstance(); await db.run( `UPDATE conversations SET - json = json_remove(json, '$.storageID'); + json = json_remove(json, '$.storageID', '$.needsStorageServiceSync', '$.unknownFields'); ` ); } diff --git a/ts/textsecure.d.ts b/ts/textsecure.d.ts index 8c6729f53996..0abe620e465d 100644 --- a/ts/textsecure.d.ts +++ b/ts/textsecure.d.ts @@ -151,11 +151,12 @@ type StorageServiceProtobufTypes = { GroupV1Record: typeof GroupV1RecordClass; GroupV2Record: typeof GroupV2RecordClass; ManifestRecord: typeof ManifestRecordClass; - ReadOperation: typeof ReadOperation; + ReadOperation: typeof ReadOperationClass; StorageItem: typeof StorageItemClass; StorageItems: typeof StorageItemsClass; - StorageManifest: typeof StorageManifest; + StorageManifest: typeof StorageManifestClass; StorageRecord: typeof StorageRecordClass; + WriteOperation: typeof WriteOperationClass; }; type ProtobufCollectionType = StorageServiceProtobufTypes & { @@ -490,23 +491,23 @@ declare enum ManifestType { ACCOUNT, } -type ManifestRecordIdentifier = { +export declare class ManifestRecordIdentifierClass { + static Type: typeof ManifestType; raw: ProtoBinaryType; type: ManifestType; -}; + toArrayBuffer: () => ArrayBuffer; +} export declare class ManifestRecordClass { static decode: ( data: ArrayBuffer | ByteBufferClass, encoding?: string ) => ManifestRecordClass; - - static Identifier: { - Type: typeof ManifestType; - }; + toArrayBuffer: () => ArrayBuffer; + static Identifier: typeof ManifestRecordIdentifierClass; version: ProtoBigNumberType; - keys: ManifestRecordIdentifier[]; + keys: ManifestRecordIdentifierClass[]; } export declare class NullMessageClass { @@ -579,14 +580,14 @@ export declare namespace ReceiptMessageClass { // Storage Service related types -declare class StorageManifest { +export declare class StorageManifestClass { static decode: ( data: ArrayBuffer | ByteBufferClass, encoding?: string - ) => StorageManifest; + ) => StorageManifestClass; version?: ProtoBigNumberType | null; - value?: ByteBufferClass | null; + value?: ProtoBinaryType; } export declare class StorageRecordClass { @@ -594,6 +595,7 @@ export declare class StorageRecordClass { data: ArrayBuffer | ByteBufferClass, encoding?: string ) => StorageRecordClass; + toArrayBuffer: () => ArrayBuffer; contact?: ContactRecordClass | null; groupV1?: GroupV1RecordClass | null; @@ -607,8 +609,8 @@ export declare class StorageItemClass { encoding?: string ) => StorageItemClass; - key?: ByteBufferClass | null; - value?: ByteBufferClass | null; + key?: ProtoBinaryType; + value?: ProtoBinaryType; } export declare class StorageItemsClass { @@ -633,11 +635,12 @@ export declare class ContactRecordClass { data: ArrayBuffer | ByteBufferClass, encoding?: string ) => ContactRecordClass; + toArrayBuffer: () => ArrayBuffer; serviceUuid?: string | null; serviceE164?: string | null; - profileKey?: ByteBufferClass | null; - identityKey?: ByteBufferClass | null; + profileKey?: ProtoBinaryType; + identityKey?: ProtoBinaryType; identityState?: ContactRecordIdentityState | null; givenName?: string | null; familyName?: string | null; @@ -645,6 +648,8 @@ export declare class ContactRecordClass { blocked?: boolean | null; whitelisted?: boolean | null; archived?: boolean | null; + + __unknownFields?: ArrayBuffer; } export declare class GroupV1RecordClass { @@ -652,11 +657,14 @@ export declare class GroupV1RecordClass { data: ArrayBuffer | ByteBufferClass, encoding?: string ) => GroupV1RecordClass; + toArrayBuffer: () => ArrayBuffer; - id?: ByteBufferClass | null; + id?: ProtoBinaryType; blocked?: boolean | null; whitelisted?: boolean | null; archived?: boolean | null; + + __unknownFields?: ArrayBuffer; } export declare class GroupV2RecordClass { @@ -664,11 +672,14 @@ export declare class GroupV2RecordClass { data: ArrayBuffer | ByteBufferClass, encoding?: string ) => GroupV2RecordClass; + toArrayBuffer: () => ArrayBuffer; masterKey?: ByteBufferClass | null; blocked?: boolean | null; whitelisted?: boolean | null; archived?: boolean | null; + + __unknownFields?: ArrayBuffer; } export declare class AccountRecordClass { @@ -676,8 +687,9 @@ export declare class AccountRecordClass { data: ArrayBuffer | ByteBufferClass, encoding?: string ) => AccountRecordClass; + toArrayBuffer: () => ArrayBuffer; - profileKey?: ByteBufferClass | null; + profileKey?: ProtoBinaryType; givenName?: string | null; familyName?: string | null; avatarUrl?: string | null; @@ -686,18 +698,33 @@ export declare class AccountRecordClass { sealedSenderIndicators?: boolean | null; typingIndicators?: boolean | null; linkPreviews?: boolean | null; + + __unknownFields?: ArrayBuffer; } -declare class ReadOperation { +declare class ReadOperationClass { static decode: ( data: ArrayBuffer | ByteBufferClass, encoding?: string - ) => ReadOperation; + ) => ReadOperationClass; readKey: ArrayBuffer[] | ByteBufferClass[]; toArrayBuffer: () => ArrayBuffer; } +declare class WriteOperationClass { + static decode: ( + data: ArrayBuffer | ByteBufferClass, + encoding?: string + ) => WriteOperationClass; + toArrayBuffer: () => ArrayBuffer; + + manifest: StorageManifestClass; + insertItem: StorageItemClass[]; + deleteKey: ArrayBuffer[] | ByteBufferClass[]; + clearAll: boolean; +} + export declare class SyncMessageClass { static decode: ( data: ArrayBuffer | ByteBufferClass, @@ -770,6 +797,11 @@ export declare namespace SyncMessageClass { timestamp?: ProtoBinaryType; } class FetchLatest { + static Type: { + UNKNOWN: number; + LOCAL_PROFILE: number; + STORAGE_MANIFEST: number; + }; type?: number; } class Keys { diff --git a/ts/textsecure/SendMessage.ts b/ts/textsecure/SendMessage.ts index 6d61b7579f5a..62df4109ff4f 100644 --- a/ts/textsecure/SendMessage.ts +++ b/ts/textsecure/SendMessage.ts @@ -823,7 +823,35 @@ export default class MessageSender { return Promise.resolve(); } - async sendRequestKeySyncMessage(options: SendOptionsType) { + async sendFetchManifestSyncMessage(options?: SendOptionsType) { + const myUuid = window.textsecure.storage.user.getUuid(); + const myNumber = window.textsecure.storage.user.getNumber(); + const myDevice = window.textsecure.storage.user.getDeviceId(); + + if (myDevice === 1 || myDevice === '1') { + return; + } + + const fetchLatest = new window.textsecure.protobuf.SyncMessage.FetchLatest(); + fetchLatest.type = + window.textsecure.protobuf.SyncMessage.FetchLatest.Type.STORAGE_MANIFEST; + + const syncMessage = this.createSyncMessage(); + syncMessage.fetchLatest = fetchLatest; + const contentMessage = new window.textsecure.protobuf.Content(); + contentMessage.syncMessage = syncMessage; + + const silent = true; + await this.sendIndividualProto( + myUuid || myNumber, + contentMessage, + Date.now(), + silent, + options + ); + } + + async sendRequestKeySyncMessage(options?: SendOptionsType) { const myUuid = window.textsecure.storage.user.getUuid(); const myNumber = window.textsecure.storage.user.getNumber(); const myDevice = window.textsecure.storage.user.getDeviceId(); @@ -1694,4 +1722,11 @@ export default class MessageSender { ): Promise { return this.server.getStorageRecords(data, options); } + + async modifyStorageRecords( + data: ArrayBuffer, + options: StorageServiceCallOptionsType + ): Promise { + return this.server.modifyStorageRecords(data, options); + } } diff --git a/ts/textsecure/WebAPI.ts b/ts/textsecure/WebAPI.ts index 398288232c29..bbe61e7b8627 100644 --- a/ts/textsecure/WebAPI.ts +++ b/ts/textsecure/WebAPI.ts @@ -701,6 +701,7 @@ export type WebAPIType = { targetUrl: string, options?: ProxiedRequestOptionsType ) => Promise; + modifyStorageRecords: MessageSender['modifyStorageRecords']; putAttachment: (encryptedBin: ArrayBuffer) => Promise; registerCapabilities: (capabilities: any) => Promise; putStickers: ( @@ -860,6 +861,7 @@ export function initialize({ getStorageRecords, getUuidsForE164s, makeProxiedRequest, + modifyStorageRecords, putAttachment, registerCapabilities, putStickers, @@ -1012,6 +1014,25 @@ export function initialize({ }); } + async function modifyStorageRecords( + data: ArrayBuffer, + options: StorageServiceCallOptionsType = {} + ): Promise { + const { credentials } = options; + + return _ajax({ + call: 'storageModify', + contentType: 'application/x-protobuf', + data, + host: storageUrl, + httpType: 'PUT', + // If we run into a conflict, the current manifest is returned - + // it will will be an ArrayBuffer at the response key on the Error + responseType: 'arraybuffer', + ...credentials, + }); + } + async function registerSupportForUnauthenticatedDelivery() { return _ajax({ call: 'supportUnauthenticatedDelivery', diff --git a/ts/util/index.ts b/ts/util/index.ts index 565060f14b54..cab6c6596507 100644 --- a/ts/util/index.ts +++ b/ts/util/index.ts @@ -16,10 +16,6 @@ import { isFileDangerous } from './isFileDangerous'; import { makeLookup } from './makeLookup'; import { migrateColor } from './migrateColor'; import { missingCaseError } from './missingCaseError'; -import { - eraseAllStorageServiceState, - runStorageServiceSyncJob, -} from './storageService'; import * as zkgroup from './zkgroup'; export { @@ -29,7 +25,6 @@ export { createWaitBatcher, deleteForEveryone, downloadAttachment, - eraseAllStorageServiceState, generateSecurityNumber, getSafetyNumberPlaceholder, getStringForProfileChange, @@ -40,6 +35,5 @@ export { migrateColor, missingCaseError, Registration, - runStorageServiceSyncJob, zkgroup, }; diff --git a/ts/util/lint/exceptions.json b/ts/util/lint/exceptions.json index 6eb3e0fbd4f5..05bb4edd827f 100644 --- a/ts/util/lint/exceptions.json +++ b/ts/util/lint/exceptions.json @@ -12952,7 +12952,7 @@ "rule": "jQuery-wrap(", "path": "ts/textsecure/WebAPI.js", "line": " const byteBuffer = window.dcodeIO.ByteBuffer.wrap(quote, 'binary', window.dcodeIO.ByteBuffer.LITTLE_ENDIAN);", - "lineNumber": 1049, + "lineNumber": 1057, "reasonCategory": "falseMatch", "updated": "2020-09-08T23:07:22.682Z" }, @@ -12960,7 +12960,7 @@ "rule": "jQuery-wrap(", "path": "ts/textsecure/WebAPI.ts", "line": " const byteBuffer = window.dcodeIO.ByteBuffer.wrap(", - "lineNumber": 1748, + "lineNumber": 1769, "reasonCategory": "falseMatch", "updated": "2020-09-08T23:07:22.682Z" } diff --git a/ts/util/storageService.ts b/ts/util/storageService.ts deleted file mode 100644 index 5098dd6c6db7..000000000000 --- a/ts/util/storageService.ts +++ /dev/null @@ -1,478 +0,0 @@ -/* tslint:disable no-backbone-get-set-outside-model */ -import _ from 'lodash'; -import PQueue from 'p-queue'; - -import Crypto from '../textsecure/Crypto'; -import { - arrayBufferToBase64, - base64ToArrayBuffer, - constantTimeEqual, - deriveStorageItemKey, - deriveStorageManifestKey, -} from '../Crypto'; -import dataInterface from '../sql/Client'; -const { eraseStorageIdFromConversations, updateConversation } = dataInterface; -import { - AccountRecordClass, - ContactRecordClass, - GroupV1RecordClass, - ManifestRecordClass, - StorageItemClass, -} from '../textsecure.d'; -import { ConversationModelType } from '../model-types.d'; - -function fromRecordVerified(verified: number): number { - const VERIFIED_ENUM = window.textsecure.storage.protocol.VerifiedStatus; - const STATE_ENUM = window.textsecure.protobuf.ContactRecord.IdentityState; - - switch (verified) { - case STATE_ENUM.VERIFIED: - return VERIFIED_ENUM.VERIFIED; - case STATE_ENUM.UNVERIFIED: - return VERIFIED_ENUM.UNVERIFIED; - default: - return VERIFIED_ENUM.DEFAULT; - } -} - -async function fetchManifest(manifestVersion: string) { - window.log.info('storageService.fetchManifest'); - - if (!window.textsecure.messaging) { - throw new Error('fetchManifest: We are offline!'); - } - - try { - const credentials = await window.textsecure.messaging.getStorageCredentials(); - window.storage.put('storageCredentials', credentials); - - const manifestBinary = await window.textsecure.messaging.getStorageManifest( - { - credentials, - greaterThanVersion: manifestVersion, - } - ); - const encryptedManifest = window.textsecure.protobuf.StorageManifest.decode( - manifestBinary - ); - - // If we don't get a value we're assuming we're receiving a 204 - // it would be nice to get an actual e.code 204 and check against that. - if (!encryptedManifest.value || !encryptedManifest.version) { - window.log.info('storageService.fetchManifest: nothing changed'); - return; - } - - const storageKeyBase64 = window.storage.get('storageKey'); - const storageKey = base64ToArrayBuffer(storageKeyBase64); - const storageManifestKey = await deriveStorageManifestKey( - storageKey, - encryptedManifest.version.toNumber() - ); - - const decryptedManifest = await Crypto.decryptProfile( - encryptedManifest.value.toArrayBuffer(), - storageManifestKey - ); - - return window.textsecure.protobuf.ManifestRecord.decode(decryptedManifest); - } catch (err) { - window.log.error(`storageService.fetchManifest: ${err}`); - - if (err.code === 404) { - // No manifest exists, we create one - return { version: 0, keys: [] }; - } else if (err.code === 204) { - // noNewerManifest we're ok - return; - } - - throw err; - } -} - -type MessageRequestCapableRecord = ContactRecordClass | GroupV1RecordClass; - -function applyMessageRequestState( - record: MessageRequestCapableRecord, - conversation: ConversationModelType -): void { - if (record.blocked) { - conversation.applyMessageRequestResponse( - conversation.messageRequestEnum.BLOCK, - { fromSync: true } - ); - } else if (record.whitelisted) { - // unblocking is also handled by this function which is why the next - // condition is part of the else-if and not separate - conversation.applyMessageRequestResponse( - conversation.messageRequestEnum.ACCEPT, - { fromSync: true } - ); - } else if (!record.blocked) { - // if the condition above failed the state could still be blocked=false - // in which case we should unblock the conversation - conversation.unblock(); - } - - if (!record.whitelisted) { - conversation.disableProfileSharing(); - } -} - -async function mergeGroupV1Record( - storageID: string, - groupV1Record: GroupV1RecordClass -): Promise { - window.log.info(`storageService.mergeGroupV1Record: merging ${storageID}`); - - if (!groupV1Record.id) { - window.log.info( - `storageService.mergeGroupV1Record: no ID for ${storageID}` - ); - return; - } - - const groupId = groupV1Record.id.toBinary(); - - // We do a get here because we don't get enough information from just this source to - // be able to do the right thing with this group. So we'll update the local group - // record if we have one; otherwise we'll just drop this update. - const conversation = window.ConversationController.get(groupId); - if (!conversation) { - window.log.warn( - `storageService.mergeGroupV1Record: No conversation for group(${groupId})` - ); - return; - } - - conversation.set({ - isArchived: Boolean(groupV1Record.archived), - storageID, - }); - - applyMessageRequestState(groupV1Record, conversation); - - updateConversation(conversation.attributes); - - window.log.info(`storageService.mergeGroupV1Record: merged ${storageID}`); -} - -async function mergeContactRecord( - storageID: string, - contactRecord: ContactRecordClass -): Promise { - window.log.info(`storageService.mergeContactRecord: merging ${storageID}`); - - window.normalizeUuids( - contactRecord, - ['serviceUuid'], - 'storageService.mergeContactRecord' - ); - - const e164 = contactRecord.serviceE164 || undefined; - const uuid = contactRecord.serviceUuid || undefined; - - const id = window.ConversationController.ensureContactIds({ - e164, - uuid, - highTrust: true, - }); - - if (!id) { - window.log.info( - `storageService.mergeContactRecord: no ID for ${storageID}` - ); - return; - } - - const conversation = await window.ConversationController.getOrCreateAndWait( - id, - 'private' - ); - - const verified = contactRecord.identityState - ? fromRecordVerified(contactRecord.identityState) - : window.textsecure.storage.protocol.VerifiedStatus.DEFAULT; - - conversation.set({ - isArchived: Boolean(contactRecord.archived), - storageID, - verified, - }); - - if (contactRecord.profileKey) { - await conversation.setProfileKey( - arrayBufferToBase64(contactRecord.profileKey.toArrayBuffer()) - ); - } else { - await conversation.dropProfileKey(); - } - - applyMessageRequestState(contactRecord, conversation); - - const identityKey = await window.textsecure.storage.protocol.loadIdentityKey( - conversation.id - ); - - const identityKeyChanged = - identityKey && contactRecord.identityKey - ? !constantTimeEqual( - identityKey, - contactRecord.identityKey.toArrayBuffer() - ) - : false; - - if (identityKeyChanged && contactRecord.identityKey) { - await window.textsecure.storage.protocol.processVerifiedMessage( - conversation.id, - verified, - contactRecord.identityKey.toArrayBuffer() - ); - } else if (conversation.get('verified')) { - await window.textsecure.storage.protocol.setVerified( - conversation.id, - verified - ); - } - - updateConversation(conversation.attributes); - - window.log.info(`storageService.mergeContactRecord: merged ${storageID}`); -} - -async function mergeAccountRecord( - storageID: string, - accountRecord: AccountRecordClass -): Promise { - window.log.info(`storageService.mergeAccountRecord: merging ${storageID}`); - - const { - profileKey, - linkPreviews, - readReceipts, - sealedSenderIndicators, - typingIndicators, - } = accountRecord; - - window.storage.put('read-receipt-setting', readReceipts); - - if (typeof sealedSenderIndicators === 'boolean') { - window.storage.put('sealedSenderIndicators', sealedSenderIndicators); - } - - if (typeof typingIndicators === 'boolean') { - window.storage.put('typingIndicators', typingIndicators); - } - - if (typeof linkPreviews === 'boolean') { - window.storage.put('linkPreviews', linkPreviews); - } - - if (profileKey) { - window.storage.put('profileKey', profileKey.toArrayBuffer()); - } - - window.log.info( - `storageService.mergeAccountRecord: merged settings ${storageID}` - ); - - const ourID = window.ConversationController.getOurConversationId(); - - if (!ourID) { - return; - } - - const conversation = await window.ConversationController.getOrCreateAndWait( - ourID, - 'private' - ); - - conversation.set({ - storageID, - }); - - if (accountRecord.profileKey) { - await conversation.setProfileKey( - arrayBufferToBase64(accountRecord.profileKey.toArrayBuffer()) - ); - } else { - await conversation.dropProfileKey(); - } - - updateConversation(conversation.attributes); - - window.log.info( - `storageService.mergeAccountRecord: merged profile ${storageID}` - ); -} - -// tslint:disable-next-line max-func-body-length -async function processManifest( - manifest: ManifestRecordClass -): Promise { - const credentials = window.storage.get('storageCredentials'); - const storageKeyBase64 = window.storage.get('storageKey'); - const storageKey = base64ToArrayBuffer(storageKeyBase64); - - if (!window.textsecure.messaging) { - throw new Error('processManifest: We are offline!'); - } - - const remoteKeysTypeMap = new Map(); - manifest.keys.forEach(key => { - remoteKeysTypeMap.set( - arrayBufferToBase64(key.raw.toArrayBuffer()), - key.type - ); - }); - - const localKeys = window - .getConversations() - .map((conversation: ConversationModelType) => conversation.get('storageID')) - .filter(Boolean); - - window.log.info( - `storageService.processManifest localKeys.length ${localKeys.length}` - ); - - const remoteKeys = Array.from(remoteKeysTypeMap.keys()); - - const remoteOnly = remoteKeys.filter( - (key: string) => !localKeys.includes(key) - ); - - window.log.info( - `storageService.processManifest remoteOnly.length ${remoteOnly.length}` - ); - - const readOperation = new window.textsecure.protobuf.ReadOperation(); - readOperation.readKey = remoteOnly.map(base64ToArrayBuffer); - - const storageItemsBuffer = await window.textsecure.messaging.getStorageRecords( - readOperation.toArrayBuffer(), - { - credentials, - } - ); - - const storageItems = window.textsecure.protobuf.StorageItems.decode( - storageItemsBuffer - ); - - if (!storageItems.items) { - return false; - } - - const queue = new PQueue({ concurrency: 4 }); - - const mergedItems = storageItems.items.map( - (storageRecordWrapper: StorageItemClass) => async () => { - const { key, value: storageItemCiphertext } = storageRecordWrapper; - - if (!key || !storageItemCiphertext) { - return; - } - - const base64ItemID = arrayBufferToBase64(key.toArrayBuffer()); - - const storageItemKey = await deriveStorageItemKey( - storageKey, - base64ItemID - ); - - const storageItemPlaintext = await Crypto.decryptProfile( - storageItemCiphertext.toArrayBuffer(), - storageItemKey - ); - const storageRecord = window.textsecure.protobuf.StorageRecord.decode( - storageItemPlaintext - ); - - const itemType = remoteKeysTypeMap.get(base64ItemID); - - const ITEM_TYPE = - window.textsecure.protobuf.ManifestRecord.Identifier.Type; - - try { - if (itemType === ITEM_TYPE.UNKNOWN) { - window.log.info('storageService.processManifest: Unknown item type'); - } else if (itemType === ITEM_TYPE.CONTACT && storageRecord.contact) { - await mergeContactRecord(base64ItemID, storageRecord.contact); - } else if (itemType === ITEM_TYPE.GROUPV1 && storageRecord.groupV1) { - await mergeGroupV1Record(base64ItemID, storageRecord.groupV1); - } else if (itemType === ITEM_TYPE.GROUPV2 && storageRecord.groupV2) { - window.log.info( - 'storageService.processManifest: Skipping GroupV2 item' - ); - } else if (itemType === ITEM_TYPE.ACCOUNT && storageRecord.account) { - await mergeAccountRecord(base64ItemID, storageRecord.account); - } - } catch (err) { - window.log.error( - `storageService.processManifest: merging record failed ${base64ItemID}` - ); - } - } - ); - - try { - await queue.addAll(mergedItems); - await queue.onEmpty(); - return true; - } catch (err) { - window.log.error('storageService.processManifest: merging failed'); - return false; - } -} - -export async function runStorageServiceSyncJob() { - if (!window.storage.get('storageKey')) { - throw new Error('runStorageServiceSyncJob: Cannot start; no storage key!'); - } - - window.log.info('runStorageServiceSyncJob: starting...'); - - const localManifestVersion = window.storage.get('manifestVersion') || 0; - - let manifest; - try { - manifest = await fetchManifest(localManifestVersion); - - // Guarding against no manifests being returned, everything should be ok - if (!manifest) { - window.log.info('runStorageServiceSyncJob: no manifest, returning early'); - return; - } - } catch (err) { - // We are supposed to retry here if it's a retryable error - window.log.error( - `storageService.runStorageServiceSyncJob: failed! ${ - err && err.stack ? err.stack : String(err) - }` - ); - return; - } - - const version = manifest.version.toNumber(); - - window.log.info( - `runStorageServiceSyncJob: manifest versions - previous: ${localManifestVersion}, current: ${version}` - ); - - const shouldUpdateVersion = await processManifest(manifest); - - if (shouldUpdateVersion) { - window.storage.put('manifestVersion', version); - } - window.log.info('runStorageServiceSyncJob: complete'); -} - -// Note: this function is meant to be called before ConversationController is hydrated. -// It goes directly to the database, so in-memory conversations will be out of date. -export async function eraseAllStorageServiceState() { - window.log.info('eraseAllStorageServiceState: starting...'); - await window.storage.remove('manifestVersion'); - await eraseStorageIdFromConversations(); - window.log.info('eraseAllStorageServiceState: complete'); -}