Fully move to protobufjs
This commit is contained in:
Fedor Indutny
GitHub
parent
20ea409d9e
commit
570fb182d4
46 changed files with 1133 additions and 12401 deletions
|
|
@ -6,19 +6,14 @@ import pMap from 'p-map';
|
|||
|
||||
import Crypto from '../textsecure/Crypto';
|
||||
import dataInterface from '../sql/Client';
|
||||
import * as Bytes from '../Bytes';
|
||||
import {
|
||||
arrayBufferToBase64,
|
||||
base64ToArrayBuffer,
|
||||
deriveStorageItemKey,
|
||||
deriveStorageManifestKey,
|
||||
typedArrayToArrayBuffer,
|
||||
} from '../Crypto';
|
||||
import {
|
||||
ManifestRecordClass,
|
||||
ManifestRecordIdentifierClass,
|
||||
StorageItemClass,
|
||||
StorageManifestClass,
|
||||
StorageRecordClass,
|
||||
} from '../textsecure.d';
|
||||
import {
|
||||
mergeAccountRecord,
|
||||
mergeContactRecord,
|
||||
|
|
@ -30,16 +25,24 @@ import {
|
|||
toGroupV2Record,
|
||||
} from './storageRecordOps';
|
||||
import { ConversationModel } from '../models/conversations';
|
||||
import { strictAssert } from '../util/assert';
|
||||
import { BackOff } from '../util/BackOff';
|
||||
import { storageJobQueue } from '../util/JobQueue';
|
||||
import { sleep } from '../util/sleep';
|
||||
import { isMoreRecentThan } from '../util/timestamp';
|
||||
import { normalizeNumber } from '../util/normalizeNumber';
|
||||
import { isStorageWriteFeatureEnabled } from '../storage/isFeatureEnabled';
|
||||
import { ourProfileKeyService } from './ourProfileKey';
|
||||
import {
|
||||
ConversationTypes,
|
||||
typeofConversation,
|
||||
} from '../util/whatTypeOfConversation';
|
||||
import { SignalService as Proto } from '../protobuf';
|
||||
|
||||
type IManifestRecordIdentifier = Proto.ManifestRecord.IIdentifier;
|
||||
|
||||
// TODO: remove once we move away from ArrayBuffers
|
||||
const FIXMEU8 = Uint8Array;
|
||||
|
||||
const {
|
||||
eraseStorageServiceStateFromConversations,
|
||||
|
|
@ -82,9 +85,9 @@ type UnknownRecord = RemoteRecord;
|
|||
|
||||
async function encryptRecord(
|
||||
storageID: string | undefined,
|
||||
storageRecord: StorageRecordClass
|
||||
): Promise<StorageItemClass> {
|
||||
const storageItem = new window.textsecure.protobuf.StorageItem();
|
||||
storageRecord: Proto.IStorageRecord
|
||||
): Promise<Proto.StorageItem> {
|
||||
const storageItem = new Proto.StorageItem();
|
||||
|
||||
const storageKeyBuffer = storageID
|
||||
? base64ToArrayBuffer(String(storageID))
|
||||
|
|
@ -101,12 +104,12 @@ async function encryptRecord(
|
|||
);
|
||||
|
||||
const encryptedRecord = await Crypto.encryptProfile(
|
||||
storageRecord.toArrayBuffer(),
|
||||
typedArrayToArrayBuffer(Proto.StorageRecord.encode(storageRecord).finish()),
|
||||
storageItemKey
|
||||
);
|
||||
|
||||
storageItem.key = storageKeyBuffer;
|
||||
storageItem.value = encryptedRecord;
|
||||
storageItem.key = new FIXMEU8(storageKeyBuffer);
|
||||
storageItem.value = new FIXMEU8(encryptedRecord);
|
||||
|
||||
return storageItem;
|
||||
}
|
||||
|
|
@ -121,13 +124,13 @@ type GeneratedManifestType = {
|
|||
storageID: string | undefined;
|
||||
}>;
|
||||
deleteKeys: Array<ArrayBuffer>;
|
||||
newItems: Set<StorageItemClass>;
|
||||
storageManifest: StorageManifestClass;
|
||||
newItems: Set<Proto.IStorageItem>;
|
||||
storageManifest: Proto.IStorageManifest;
|
||||
};
|
||||
|
||||
async function generateManifest(
|
||||
version: number,
|
||||
previousManifest?: ManifestRecordClass,
|
||||
previousManifest?: Proto.IManifestRecord,
|
||||
isNewManifest = false
|
||||
): Promise<GeneratedManifestType> {
|
||||
window.log.info(
|
||||
|
|
@ -138,39 +141,39 @@ async function generateManifest(
|
|||
|
||||
await window.ConversationController.checkForConflicts();
|
||||
|
||||
const ITEM_TYPE = window.textsecure.protobuf.ManifestRecord.Identifier.Type;
|
||||
const ITEM_TYPE = Proto.ManifestRecord.Identifier.Type;
|
||||
|
||||
const conversationsToUpdate = [];
|
||||
const insertKeys: Array<string> = [];
|
||||
const deleteKeys: Array<ArrayBuffer> = [];
|
||||
const manifestRecordKeys: Set<ManifestRecordIdentifierClass> = new Set();
|
||||
const newItems: Set<StorageItemClass> = new Set();
|
||||
const manifestRecordKeys: Set<IManifestRecordIdentifier> = new Set();
|
||||
const newItems: Set<Proto.IStorageItem> = new Set();
|
||||
|
||||
const conversations = window.getConversations();
|
||||
for (let i = 0; i < conversations.length; i += 1) {
|
||||
const conversation = conversations.models[i];
|
||||
const identifier = new window.textsecure.protobuf.ManifestRecord.Identifier();
|
||||
const identifier = new Proto.ManifestRecord.Identifier();
|
||||
|
||||
let storageRecord;
|
||||
|
||||
const conversationType = typeofConversation(conversation.attributes);
|
||||
if (conversationType === ConversationTypes.Me) {
|
||||
storageRecord = new window.textsecure.protobuf.StorageRecord();
|
||||
storageRecord = new Proto.StorageRecord();
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
storageRecord.account = await toAccountRecord(conversation);
|
||||
identifier.type = ITEM_TYPE.ACCOUNT;
|
||||
} else if (conversationType === ConversationTypes.Direct) {
|
||||
storageRecord = new window.textsecure.protobuf.StorageRecord();
|
||||
storageRecord = new Proto.StorageRecord();
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
storageRecord.contact = await toContactRecord(conversation);
|
||||
identifier.type = ITEM_TYPE.CONTACT;
|
||||
} else if (conversationType === ConversationTypes.GroupV2) {
|
||||
storageRecord = new window.textsecure.protobuf.StorageRecord();
|
||||
storageRecord = new Proto.StorageRecord();
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
storageRecord.groupV2 = await toGroupV2Record(conversation);
|
||||
identifier.type = ITEM_TYPE.GROUPV2;
|
||||
} else if (conversationType === ConversationTypes.GroupV1) {
|
||||
storageRecord = new window.textsecure.protobuf.StorageRecord();
|
||||
storageRecord = new Proto.StorageRecord();
|
||||
// eslint-disable-next-line no-await-in-loop
|
||||
storageRecord.groupV1 = await toGroupV1Record(conversation);
|
||||
identifier.type = ITEM_TYPE.GROUPV1;
|
||||
|
|
@ -256,9 +259,9 @@ async function generateManifest(
|
|||
// When updating the manifest, ensure all "unknown" keys are added to the
|
||||
// new manifest, so we don't inadvertently delete something we don't understand
|
||||
unknownRecordsArray.forEach((record: UnknownRecord) => {
|
||||
const identifier = new window.textsecure.protobuf.ManifestRecord.Identifier();
|
||||
const identifier = new Proto.ManifestRecord.Identifier();
|
||||
identifier.type = record.itemType;
|
||||
identifier.raw = base64ToArrayBuffer(record.storageID);
|
||||
identifier.raw = Bytes.fromBase64(record.storageID);
|
||||
|
||||
manifestRecordKeys.add(identifier);
|
||||
});
|
||||
|
|
@ -276,9 +279,9 @@ async function generateManifest(
|
|||
// These records failed to merge in the previous fetchManifest, but we still
|
||||
// need to include them so that the manifest is complete
|
||||
recordsWithErrors.forEach((record: UnknownRecord) => {
|
||||
const identifier = new window.textsecure.protobuf.ManifestRecord.Identifier();
|
||||
const identifier = new Proto.ManifestRecord.Identifier();
|
||||
identifier.type = record.itemType;
|
||||
identifier.raw = base64ToArrayBuffer(record.storageID);
|
||||
identifier.raw = Bytes.fromBase64(record.storageID);
|
||||
|
||||
manifestRecordKeys.add(identifier);
|
||||
});
|
||||
|
|
@ -293,7 +296,8 @@ async function generateManifest(
|
|||
// This can be broken down into two parts:
|
||||
// There are no duplicate type+raw pairs
|
||||
// There are no duplicate raw bytes
|
||||
const storageID = arrayBufferToBase64(identifier.raw);
|
||||
strictAssert(identifier.raw, 'manifest record key without raw identifier');
|
||||
const storageID = Bytes.toBase64(identifier.raw);
|
||||
const typeAndRaw = `${identifier.type}+${storageID}`;
|
||||
if (
|
||||
rawDuplicates.has(identifier.raw) ||
|
||||
|
|
@ -335,11 +339,13 @@ async function generateManifest(
|
|||
rawDuplicates.clear();
|
||||
typeRawDuplicates.clear();
|
||||
|
||||
const storageKeyDuplicates = new Set();
|
||||
const storageKeyDuplicates = new Set<string>();
|
||||
|
||||
newItems.forEach(storageItem => {
|
||||
// Ensure there are no duplicate StorageIdentifiers in your list of inserts
|
||||
const storageID = storageItem.key;
|
||||
strictAssert(storageItem.key, 'New storage item without key');
|
||||
|
||||
const storageID = Bytes.toBase64(storageItem.key);
|
||||
if (storageKeyDuplicates.has(storageID)) {
|
||||
window.log.info(
|
||||
'storageService.generateManifest: removing duplicate identifier from inserts',
|
||||
|
|
@ -360,16 +366,18 @@ async function generateManifest(
|
|||
const pendingDeletes: Set<string> = new Set();
|
||||
|
||||
const remoteKeys: Set<string> = new Set();
|
||||
previousManifest.keys.forEach(
|
||||
(identifier: ManifestRecordIdentifierClass) => {
|
||||
const storageID = arrayBufferToBase64(identifier.raw.toArrayBuffer());
|
||||
(previousManifest.keys ?? []).forEach(
|
||||
(identifier: IManifestRecordIdentifier) => {
|
||||
strictAssert(identifier.raw, 'Identifier without raw field');
|
||||
const storageID = Bytes.toBase64(identifier.raw);
|
||||
remoteKeys.add(storageID);
|
||||
}
|
||||
);
|
||||
|
||||
const localKeys: Set<string> = new Set();
|
||||
manifestRecordKeys.forEach((identifier: ManifestRecordIdentifierClass) => {
|
||||
const storageID = arrayBufferToBase64(identifier.raw);
|
||||
manifestRecordKeys.forEach((identifier: IManifestRecordIdentifier) => {
|
||||
strictAssert(identifier.raw, 'Identifier without raw field');
|
||||
const storageID = Bytes.toBase64(identifier.raw);
|
||||
localKeys.add(storageID);
|
||||
|
||||
if (!remoteKeys.has(storageID)) {
|
||||
|
|
@ -406,7 +414,7 @@ async function generateManifest(
|
|||
});
|
||||
}
|
||||
|
||||
const manifestRecord = new window.textsecure.protobuf.ManifestRecord();
|
||||
const manifestRecord = new Proto.ManifestRecord();
|
||||
manifestRecord.version = version;
|
||||
manifestRecord.keys = Array.from(manifestRecordKeys);
|
||||
|
||||
|
|
@ -420,13 +428,15 @@ async function generateManifest(
|
|||
version
|
||||
);
|
||||
const encryptedManifest = await Crypto.encryptProfile(
|
||||
manifestRecord.toArrayBuffer(),
|
||||
typedArrayToArrayBuffer(
|
||||
Proto.ManifestRecord.encode(manifestRecord).finish()
|
||||
),
|
||||
storageManifestKey
|
||||
);
|
||||
|
||||
const storageManifest = new window.textsecure.protobuf.StorageManifest();
|
||||
const storageManifest = new Proto.StorageManifest();
|
||||
storageManifest.version = version;
|
||||
storageManifest.value = encryptedManifest;
|
||||
storageManifest.value = new FIXMEU8(encryptedManifest);
|
||||
|
||||
return {
|
||||
conversationsToUpdate,
|
||||
|
|
@ -462,14 +472,16 @@ async function uploadManifest(
|
|||
deleteKeys.length
|
||||
);
|
||||
|
||||
const writeOperation = new window.textsecure.protobuf.WriteOperation();
|
||||
const writeOperation = new Proto.WriteOperation();
|
||||
writeOperation.manifest = storageManifest;
|
||||
writeOperation.insertItem = Array.from(newItems);
|
||||
writeOperation.deleteKey = deleteKeys;
|
||||
writeOperation.deleteKey = deleteKeys.map(key => new FIXMEU8(key));
|
||||
|
||||
window.log.info('storageService.uploadManifest: uploading...', version);
|
||||
await window.textsecure.messaging.modifyStorageRecords(
|
||||
writeOperation.toArrayBuffer(),
|
||||
typedArrayToArrayBuffer(
|
||||
Proto.WriteOperation.encode(writeOperation).finish()
|
||||
),
|
||||
{
|
||||
credentials,
|
||||
}
|
||||
|
|
@ -565,8 +577,8 @@ async function createNewManifest() {
|
|||
}
|
||||
|
||||
async function decryptManifest(
|
||||
encryptedManifest: StorageManifestClass
|
||||
): Promise<ManifestRecordClass> {
|
||||
encryptedManifest: Proto.IStorageManifest
|
||||
): Promise<Proto.ManifestRecord> {
|
||||
const { version, value } = encryptedManifest;
|
||||
|
||||
const storageKeyBase64 = window.storage.get('storageKey');
|
||||
|
|
@ -576,20 +588,21 @@ async function decryptManifest(
|
|||
const storageKey = base64ToArrayBuffer(storageKeyBase64);
|
||||
const storageManifestKey = await deriveStorageManifestKey(
|
||||
storageKey,
|
||||
typeof version === 'number' ? version : version.toNumber()
|
||||
normalizeNumber(version ?? 0)
|
||||
);
|
||||
|
||||
strictAssert(value, 'StorageManifest has no value field');
|
||||
const decryptedManifest = await Crypto.decryptProfile(
|
||||
typeof value.toArrayBuffer === 'function' ? value.toArrayBuffer() : value,
|
||||
typedArrayToArrayBuffer(value),
|
||||
storageManifestKey
|
||||
);
|
||||
|
||||
return window.textsecure.protobuf.ManifestRecord.decode(decryptedManifest);
|
||||
return Proto.ManifestRecord.decode(new FIXMEU8(decryptedManifest));
|
||||
}
|
||||
|
||||
async function fetchManifest(
|
||||
manifestVersion: number
|
||||
): Promise<ManifestRecordClass | undefined> {
|
||||
): Promise<Proto.ManifestRecord | undefined> {
|
||||
window.log.info('storageService.fetchManifest');
|
||||
|
||||
if (!window.textsecure.messaging) {
|
||||
|
|
@ -606,8 +619,8 @@ async function fetchManifest(
|
|||
greaterThanVersion: manifestVersion,
|
||||
}
|
||||
);
|
||||
const encryptedManifest = window.textsecure.protobuf.StorageManifest.decode(
|
||||
manifestBinary
|
||||
const encryptedManifest = Proto.StorageManifest.decode(
|
||||
new FIXMEU8(manifestBinary)
|
||||
);
|
||||
|
||||
// if we don't get a value we're assuming that there's no newer manifest
|
||||
|
|
@ -645,7 +658,7 @@ async function fetchManifest(
|
|||
type MergeableItemType = {
|
||||
itemType: number;
|
||||
storageID: string;
|
||||
storageRecord: StorageRecordClass;
|
||||
storageRecord: Proto.IStorageRecord;
|
||||
};
|
||||
|
||||
type MergedRecordType = UnknownRecord & {
|
||||
|
|
@ -659,7 +672,7 @@ async function mergeRecord(
|
|||
): Promise<MergedRecordType> {
|
||||
const { itemType, storageID, storageRecord } = itemToMerge;
|
||||
|
||||
const ITEM_TYPE = window.textsecure.protobuf.ManifestRecord.Identifier.Type;
|
||||
const ITEM_TYPE = Proto.ManifestRecord.Identifier.Type;
|
||||
|
||||
let hasConflict = false;
|
||||
let isUnsupported = false;
|
||||
|
|
@ -709,18 +722,16 @@ async function mergeRecord(
|
|||
}
|
||||
|
||||
async function processManifest(
|
||||
manifest: ManifestRecordClass
|
||||
manifest: Proto.IManifestRecord
|
||||
): Promise<boolean> {
|
||||
if (!window.textsecure.messaging) {
|
||||
throw new Error('storageService.processManifest: We are offline!');
|
||||
}
|
||||
|
||||
const remoteKeysTypeMap = new Map();
|
||||
manifest.keys.forEach((identifier: ManifestRecordIdentifierClass) => {
|
||||
remoteKeysTypeMap.set(
|
||||
arrayBufferToBase64(identifier.raw.toArrayBuffer()),
|
||||
identifier.type
|
||||
);
|
||||
(manifest.keys || []).forEach(({ raw, type }: IManifestRecordIdentifier) => {
|
||||
strictAssert(raw, 'Identifier without raw field');
|
||||
remoteKeysTypeMap.set(Bytes.toBase64(raw), type);
|
||||
});
|
||||
|
||||
const remoteKeys = new Set(remoteKeysTypeMap.keys());
|
||||
|
|
@ -820,21 +831,21 @@ async function processRemoteRecords(
|
|||
remoteOnlyRecords.size
|
||||
);
|
||||
|
||||
const readOperation = new window.textsecure.protobuf.ReadOperation();
|
||||
const readOperation = new Proto.ReadOperation();
|
||||
readOperation.readKey = Array.from(remoteOnlyRecords.keys()).map(
|
||||
base64ToArrayBuffer
|
||||
Bytes.fromBase64
|
||||
);
|
||||
|
||||
const credentials = window.storage.get('storageCredentials');
|
||||
const storageItemsBuffer = await window.textsecure.messaging.getStorageRecords(
|
||||
readOperation.toArrayBuffer(),
|
||||
typedArrayToArrayBuffer(Proto.ReadOperation.encode(readOperation).finish()),
|
||||
{
|
||||
credentials,
|
||||
}
|
||||
);
|
||||
|
||||
const storageItems = window.textsecure.protobuf.StorageItems.decode(
|
||||
storageItemsBuffer
|
||||
const storageItems = Proto.StorageItems.decode(
|
||||
new FIXMEU8(storageItemsBuffer)
|
||||
);
|
||||
|
||||
if (!storageItems.items) {
|
||||
|
|
@ -847,7 +858,7 @@ async function processRemoteRecords(
|
|||
const decryptedStorageItems = await pMap(
|
||||
storageItems.items,
|
||||
async (
|
||||
storageRecordWrapper: StorageItemClass
|
||||
storageRecordWrapper: Proto.IStorageItem
|
||||
): Promise<MergeableItemType> => {
|
||||
const { key, value: storageItemCiphertext } = storageRecordWrapper;
|
||||
|
||||
|
|
@ -861,7 +872,7 @@ async function processRemoteRecords(
|
|||
);
|
||||
}
|
||||
|
||||
const base64ItemID = arrayBufferToBase64(key.toArrayBuffer());
|
||||
const base64ItemID = Bytes.toBase64(key);
|
||||
|
||||
const storageItemKey = await deriveStorageItemKey(
|
||||
storageKey,
|
||||
|
|
@ -871,7 +882,7 @@ async function processRemoteRecords(
|
|||
let storageItemPlaintext;
|
||||
try {
|
||||
storageItemPlaintext = await Crypto.decryptProfile(
|
||||
storageItemCiphertext.toArrayBuffer(),
|
||||
typedArrayToArrayBuffer(storageItemCiphertext),
|
||||
storageItemKey
|
||||
);
|
||||
} catch (err) {
|
||||
|
|
@ -882,8 +893,8 @@ async function processRemoteRecords(
|
|||
throw err;
|
||||
}
|
||||
|
||||
const storageRecord = window.textsecure.protobuf.StorageRecord.decode(
|
||||
storageItemPlaintext
|
||||
const storageRecord = Proto.StorageRecord.decode(
|
||||
new FIXMEU8(storageItemPlaintext)
|
||||
);
|
||||
|
||||
const remoteRecord = remoteOnlyRecords.get(base64ItemID);
|
||||
|
|
@ -906,7 +917,7 @@ async function processRemoteRecords(
|
|||
// Merge Account records last since it contains the pinned conversations
|
||||
// and we need all other records merged first before we can find the pinned
|
||||
// records in our db
|
||||
const ITEM_TYPE = window.textsecure.protobuf.ManifestRecord.Identifier.Type;
|
||||
const ITEM_TYPE = Proto.ManifestRecord.Identifier.Type;
|
||||
const sortedStorageItems = decryptedStorageItems.sort((_, b) =>
|
||||
b.itemType === ITEM_TYPE.ACCOUNT ? -1 : 1
|
||||
);
|
||||
|
|
@ -995,7 +1006,7 @@ async function processRemoteRecords(
|
|||
return 0;
|
||||
}
|
||||
|
||||
async function sync(): Promise<ManifestRecordClass | undefined> {
|
||||
async function sync(): Promise<Proto.ManifestRecord | undefined> {
|
||||
if (!isStorageWriteFeatureEnabled()) {
|
||||
window.log.info(
|
||||
'storageService.sync: Not starting desktop.storage is falsey'
|
||||
|
|
@ -1010,7 +1021,7 @@ async function sync(): Promise<ManifestRecordClass | undefined> {
|
|||
|
||||
window.log.info('storageService.sync: starting...');
|
||||
|
||||
let manifest: ManifestRecordClass | undefined;
|
||||
let manifest: Proto.ManifestRecord | undefined;
|
||||
try {
|
||||
// If we've previously interacted with strage service, update 'fetchComplete' record
|
||||
const previousFetchComplete = window.storage.get('storageFetchComplete');
|
||||
|
|
@ -1028,7 +1039,11 @@ async function sync(): Promise<ManifestRecordClass | undefined> {
|
|||
return undefined;
|
||||
}
|
||||
|
||||
const version = manifest.version.toNumber();
|
||||
strictAssert(
|
||||
manifest.version !== undefined && manifest.version !== null,
|
||||
'Manifest without version'
|
||||
);
|
||||
const version = normalizeNumber(manifest.version);
|
||||
|
||||
window.log.info(
|
||||
`storageService.sync: manifest versions - previous: ${localManifestVersion}, current: ${version}`
|
||||
|
|
@ -1095,7 +1110,7 @@ async function upload(fromSync = false): Promise<void> {
|
|||
return;
|
||||
}
|
||||
|
||||
let previousManifest: ManifestRecordClass | undefined;
|
||||
let previousManifest: Proto.ManifestRecord | undefined;
|
||||
if (!fromSync) {
|
||||
// Syncing before we upload so that we repair any unknown records and
|
||||
// records with errors as well as ensure that we have the latest up to date
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue