Fully move to protobufjs

This commit is contained in:
Fedor Indutny 2021-07-13 11:54:53 -07:00 committed by GitHub
parent 20ea409d9e
commit 570fb182d4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
46 changed files with 1133 additions and 12401 deletions

View file

@ -6,19 +6,14 @@ import pMap from 'p-map';
import Crypto from '../textsecure/Crypto';
import dataInterface from '../sql/Client';
import * as Bytes from '../Bytes';
import {
arrayBufferToBase64,
base64ToArrayBuffer,
deriveStorageItemKey,
deriveStorageManifestKey,
typedArrayToArrayBuffer,
} from '../Crypto';
import {
ManifestRecordClass,
ManifestRecordIdentifierClass,
StorageItemClass,
StorageManifestClass,
StorageRecordClass,
} from '../textsecure.d';
import {
mergeAccountRecord,
mergeContactRecord,
@ -30,16 +25,24 @@ import {
toGroupV2Record,
} from './storageRecordOps';
import { ConversationModel } from '../models/conversations';
import { strictAssert } from '../util/assert';
import { BackOff } from '../util/BackOff';
import { storageJobQueue } from '../util/JobQueue';
import { sleep } from '../util/sleep';
import { isMoreRecentThan } from '../util/timestamp';
import { normalizeNumber } from '../util/normalizeNumber';
import { isStorageWriteFeatureEnabled } from '../storage/isFeatureEnabled';
import { ourProfileKeyService } from './ourProfileKey';
import {
ConversationTypes,
typeofConversation,
} from '../util/whatTypeOfConversation';
import { SignalService as Proto } from '../protobuf';
type IManifestRecordIdentifier = Proto.ManifestRecord.IIdentifier;
// TODO: remove once we move away from ArrayBuffers
const FIXMEU8 = Uint8Array;
const {
eraseStorageServiceStateFromConversations,
@ -82,9 +85,9 @@ type UnknownRecord = RemoteRecord;
async function encryptRecord(
storageID: string | undefined,
storageRecord: StorageRecordClass
): Promise<StorageItemClass> {
const storageItem = new window.textsecure.protobuf.StorageItem();
storageRecord: Proto.IStorageRecord
): Promise<Proto.StorageItem> {
const storageItem = new Proto.StorageItem();
const storageKeyBuffer = storageID
? base64ToArrayBuffer(String(storageID))
@ -101,12 +104,12 @@ async function encryptRecord(
);
const encryptedRecord = await Crypto.encryptProfile(
storageRecord.toArrayBuffer(),
typedArrayToArrayBuffer(Proto.StorageRecord.encode(storageRecord).finish()),
storageItemKey
);
storageItem.key = storageKeyBuffer;
storageItem.value = encryptedRecord;
storageItem.key = new FIXMEU8(storageKeyBuffer);
storageItem.value = new FIXMEU8(encryptedRecord);
return storageItem;
}
@ -121,13 +124,13 @@ type GeneratedManifestType = {
storageID: string | undefined;
}>;
deleteKeys: Array<ArrayBuffer>;
newItems: Set<StorageItemClass>;
storageManifest: StorageManifestClass;
newItems: Set<Proto.IStorageItem>;
storageManifest: Proto.IStorageManifest;
};
async function generateManifest(
version: number,
previousManifest?: ManifestRecordClass,
previousManifest?: Proto.IManifestRecord,
isNewManifest = false
): Promise<GeneratedManifestType> {
window.log.info(
@ -138,39 +141,39 @@ async function generateManifest(
await window.ConversationController.checkForConflicts();
const ITEM_TYPE = window.textsecure.protobuf.ManifestRecord.Identifier.Type;
const ITEM_TYPE = Proto.ManifestRecord.Identifier.Type;
const conversationsToUpdate = [];
const insertKeys: Array<string> = [];
const deleteKeys: Array<ArrayBuffer> = [];
const manifestRecordKeys: Set<ManifestRecordIdentifierClass> = new Set();
const newItems: Set<StorageItemClass> = new Set();
const manifestRecordKeys: Set<IManifestRecordIdentifier> = new Set();
const newItems: Set<Proto.IStorageItem> = new Set();
const conversations = window.getConversations();
for (let i = 0; i < conversations.length; i += 1) {
const conversation = conversations.models[i];
const identifier = new window.textsecure.protobuf.ManifestRecord.Identifier();
const identifier = new Proto.ManifestRecord.Identifier();
let storageRecord;
const conversationType = typeofConversation(conversation.attributes);
if (conversationType === ConversationTypes.Me) {
storageRecord = new window.textsecure.protobuf.StorageRecord();
storageRecord = new Proto.StorageRecord();
// eslint-disable-next-line no-await-in-loop
storageRecord.account = await toAccountRecord(conversation);
identifier.type = ITEM_TYPE.ACCOUNT;
} else if (conversationType === ConversationTypes.Direct) {
storageRecord = new window.textsecure.protobuf.StorageRecord();
storageRecord = new Proto.StorageRecord();
// eslint-disable-next-line no-await-in-loop
storageRecord.contact = await toContactRecord(conversation);
identifier.type = ITEM_TYPE.CONTACT;
} else if (conversationType === ConversationTypes.GroupV2) {
storageRecord = new window.textsecure.protobuf.StorageRecord();
storageRecord = new Proto.StorageRecord();
// eslint-disable-next-line no-await-in-loop
storageRecord.groupV2 = await toGroupV2Record(conversation);
identifier.type = ITEM_TYPE.GROUPV2;
} else if (conversationType === ConversationTypes.GroupV1) {
storageRecord = new window.textsecure.protobuf.StorageRecord();
storageRecord = new Proto.StorageRecord();
// eslint-disable-next-line no-await-in-loop
storageRecord.groupV1 = await toGroupV1Record(conversation);
identifier.type = ITEM_TYPE.GROUPV1;
@ -256,9 +259,9 @@ async function generateManifest(
// When updating the manifest, ensure all "unknown" keys are added to the
// new manifest, so we don't inadvertently delete something we don't understand
unknownRecordsArray.forEach((record: UnknownRecord) => {
const identifier = new window.textsecure.protobuf.ManifestRecord.Identifier();
const identifier = new Proto.ManifestRecord.Identifier();
identifier.type = record.itemType;
identifier.raw = base64ToArrayBuffer(record.storageID);
identifier.raw = Bytes.fromBase64(record.storageID);
manifestRecordKeys.add(identifier);
});
@ -276,9 +279,9 @@ async function generateManifest(
// These records failed to merge in the previous fetchManifest, but we still
// need to include them so that the manifest is complete
recordsWithErrors.forEach((record: UnknownRecord) => {
const identifier = new window.textsecure.protobuf.ManifestRecord.Identifier();
const identifier = new Proto.ManifestRecord.Identifier();
identifier.type = record.itemType;
identifier.raw = base64ToArrayBuffer(record.storageID);
identifier.raw = Bytes.fromBase64(record.storageID);
manifestRecordKeys.add(identifier);
});
@ -293,7 +296,8 @@ async function generateManifest(
// This can be broken down into two parts:
// There are no duplicate type+raw pairs
// There are no duplicate raw bytes
const storageID = arrayBufferToBase64(identifier.raw);
strictAssert(identifier.raw, 'manifest record key without raw identifier');
const storageID = Bytes.toBase64(identifier.raw);
const typeAndRaw = `${identifier.type}+${storageID}`;
if (
rawDuplicates.has(identifier.raw) ||
@ -335,11 +339,13 @@ async function generateManifest(
rawDuplicates.clear();
typeRawDuplicates.clear();
const storageKeyDuplicates = new Set();
const storageKeyDuplicates = new Set<string>();
newItems.forEach(storageItem => {
// Ensure there are no duplicate StorageIdentifiers in your list of inserts
const storageID = storageItem.key;
strictAssert(storageItem.key, 'New storage item without key');
const storageID = Bytes.toBase64(storageItem.key);
if (storageKeyDuplicates.has(storageID)) {
window.log.info(
'storageService.generateManifest: removing duplicate identifier from inserts',
@ -360,16 +366,18 @@ async function generateManifest(
const pendingDeletes: Set<string> = new Set();
const remoteKeys: Set<string> = new Set();
previousManifest.keys.forEach(
(identifier: ManifestRecordIdentifierClass) => {
const storageID = arrayBufferToBase64(identifier.raw.toArrayBuffer());
(previousManifest.keys ?? []).forEach(
(identifier: IManifestRecordIdentifier) => {
strictAssert(identifier.raw, 'Identifier without raw field');
const storageID = Bytes.toBase64(identifier.raw);
remoteKeys.add(storageID);
}
);
const localKeys: Set<string> = new Set();
manifestRecordKeys.forEach((identifier: ManifestRecordIdentifierClass) => {
const storageID = arrayBufferToBase64(identifier.raw);
manifestRecordKeys.forEach((identifier: IManifestRecordIdentifier) => {
strictAssert(identifier.raw, 'Identifier without raw field');
const storageID = Bytes.toBase64(identifier.raw);
localKeys.add(storageID);
if (!remoteKeys.has(storageID)) {
@ -406,7 +414,7 @@ async function generateManifest(
});
}
const manifestRecord = new window.textsecure.protobuf.ManifestRecord();
const manifestRecord = new Proto.ManifestRecord();
manifestRecord.version = version;
manifestRecord.keys = Array.from(manifestRecordKeys);
@ -420,13 +428,15 @@ async function generateManifest(
version
);
const encryptedManifest = await Crypto.encryptProfile(
manifestRecord.toArrayBuffer(),
typedArrayToArrayBuffer(
Proto.ManifestRecord.encode(manifestRecord).finish()
),
storageManifestKey
);
const storageManifest = new window.textsecure.protobuf.StorageManifest();
const storageManifest = new Proto.StorageManifest();
storageManifest.version = version;
storageManifest.value = encryptedManifest;
storageManifest.value = new FIXMEU8(encryptedManifest);
return {
conversationsToUpdate,
@ -462,14 +472,16 @@ async function uploadManifest(
deleteKeys.length
);
const writeOperation = new window.textsecure.protobuf.WriteOperation();
const writeOperation = new Proto.WriteOperation();
writeOperation.manifest = storageManifest;
writeOperation.insertItem = Array.from(newItems);
writeOperation.deleteKey = deleteKeys;
writeOperation.deleteKey = deleteKeys.map(key => new FIXMEU8(key));
window.log.info('storageService.uploadManifest: uploading...', version);
await window.textsecure.messaging.modifyStorageRecords(
writeOperation.toArrayBuffer(),
typedArrayToArrayBuffer(
Proto.WriteOperation.encode(writeOperation).finish()
),
{
credentials,
}
@ -565,8 +577,8 @@ async function createNewManifest() {
}
async function decryptManifest(
encryptedManifest: StorageManifestClass
): Promise<ManifestRecordClass> {
encryptedManifest: Proto.IStorageManifest
): Promise<Proto.ManifestRecord> {
const { version, value } = encryptedManifest;
const storageKeyBase64 = window.storage.get('storageKey');
@ -576,20 +588,21 @@ async function decryptManifest(
const storageKey = base64ToArrayBuffer(storageKeyBase64);
const storageManifestKey = await deriveStorageManifestKey(
storageKey,
typeof version === 'number' ? version : version.toNumber()
normalizeNumber(version ?? 0)
);
strictAssert(value, 'StorageManifest has no value field');
const decryptedManifest = await Crypto.decryptProfile(
typeof value.toArrayBuffer === 'function' ? value.toArrayBuffer() : value,
typedArrayToArrayBuffer(value),
storageManifestKey
);
return window.textsecure.protobuf.ManifestRecord.decode(decryptedManifest);
return Proto.ManifestRecord.decode(new FIXMEU8(decryptedManifest));
}
async function fetchManifest(
manifestVersion: number
): Promise<ManifestRecordClass | undefined> {
): Promise<Proto.ManifestRecord | undefined> {
window.log.info('storageService.fetchManifest');
if (!window.textsecure.messaging) {
@ -606,8 +619,8 @@ async function fetchManifest(
greaterThanVersion: manifestVersion,
}
);
const encryptedManifest = window.textsecure.protobuf.StorageManifest.decode(
manifestBinary
const encryptedManifest = Proto.StorageManifest.decode(
new FIXMEU8(manifestBinary)
);
// if we don't get a value we're assuming that there's no newer manifest
@ -645,7 +658,7 @@ async function fetchManifest(
type MergeableItemType = {
itemType: number;
storageID: string;
storageRecord: StorageRecordClass;
storageRecord: Proto.IStorageRecord;
};
type MergedRecordType = UnknownRecord & {
@ -659,7 +672,7 @@ async function mergeRecord(
): Promise<MergedRecordType> {
const { itemType, storageID, storageRecord } = itemToMerge;
const ITEM_TYPE = window.textsecure.protobuf.ManifestRecord.Identifier.Type;
const ITEM_TYPE = Proto.ManifestRecord.Identifier.Type;
let hasConflict = false;
let isUnsupported = false;
@ -709,18 +722,16 @@ async function mergeRecord(
}
async function processManifest(
manifest: ManifestRecordClass
manifest: Proto.IManifestRecord
): Promise<boolean> {
if (!window.textsecure.messaging) {
throw new Error('storageService.processManifest: We are offline!');
}
const remoteKeysTypeMap = new Map();
manifest.keys.forEach((identifier: ManifestRecordIdentifierClass) => {
remoteKeysTypeMap.set(
arrayBufferToBase64(identifier.raw.toArrayBuffer()),
identifier.type
);
(manifest.keys || []).forEach(({ raw, type }: IManifestRecordIdentifier) => {
strictAssert(raw, 'Identifier without raw field');
remoteKeysTypeMap.set(Bytes.toBase64(raw), type);
});
const remoteKeys = new Set(remoteKeysTypeMap.keys());
@ -820,21 +831,21 @@ async function processRemoteRecords(
remoteOnlyRecords.size
);
const readOperation = new window.textsecure.protobuf.ReadOperation();
const readOperation = new Proto.ReadOperation();
readOperation.readKey = Array.from(remoteOnlyRecords.keys()).map(
base64ToArrayBuffer
Bytes.fromBase64
);
const credentials = window.storage.get('storageCredentials');
const storageItemsBuffer = await window.textsecure.messaging.getStorageRecords(
readOperation.toArrayBuffer(),
typedArrayToArrayBuffer(Proto.ReadOperation.encode(readOperation).finish()),
{
credentials,
}
);
const storageItems = window.textsecure.protobuf.StorageItems.decode(
storageItemsBuffer
const storageItems = Proto.StorageItems.decode(
new FIXMEU8(storageItemsBuffer)
);
if (!storageItems.items) {
@ -847,7 +858,7 @@ async function processRemoteRecords(
const decryptedStorageItems = await pMap(
storageItems.items,
async (
storageRecordWrapper: StorageItemClass
storageRecordWrapper: Proto.IStorageItem
): Promise<MergeableItemType> => {
const { key, value: storageItemCiphertext } = storageRecordWrapper;
@ -861,7 +872,7 @@ async function processRemoteRecords(
);
}
const base64ItemID = arrayBufferToBase64(key.toArrayBuffer());
const base64ItemID = Bytes.toBase64(key);
const storageItemKey = await deriveStorageItemKey(
storageKey,
@ -871,7 +882,7 @@ async function processRemoteRecords(
let storageItemPlaintext;
try {
storageItemPlaintext = await Crypto.decryptProfile(
storageItemCiphertext.toArrayBuffer(),
typedArrayToArrayBuffer(storageItemCiphertext),
storageItemKey
);
} catch (err) {
@ -882,8 +893,8 @@ async function processRemoteRecords(
throw err;
}
const storageRecord = window.textsecure.protobuf.StorageRecord.decode(
storageItemPlaintext
const storageRecord = Proto.StorageRecord.decode(
new FIXMEU8(storageItemPlaintext)
);
const remoteRecord = remoteOnlyRecords.get(base64ItemID);
@ -906,7 +917,7 @@ async function processRemoteRecords(
// Merge Account records last since it contains the pinned conversations
// and we need all other records merged first before we can find the pinned
// records in our db
const ITEM_TYPE = window.textsecure.protobuf.ManifestRecord.Identifier.Type;
const ITEM_TYPE = Proto.ManifestRecord.Identifier.Type;
const sortedStorageItems = decryptedStorageItems.sort((_, b) =>
b.itemType === ITEM_TYPE.ACCOUNT ? -1 : 1
);
@ -995,7 +1006,7 @@ async function processRemoteRecords(
return 0;
}
async function sync(): Promise<ManifestRecordClass | undefined> {
async function sync(): Promise<Proto.ManifestRecord | undefined> {
if (!isStorageWriteFeatureEnabled()) {
window.log.info(
'storageService.sync: Not starting desktop.storage is falsey'
@ -1010,7 +1021,7 @@ async function sync(): Promise<ManifestRecordClass | undefined> {
window.log.info('storageService.sync: starting...');
let manifest: ManifestRecordClass | undefined;
let manifest: Proto.ManifestRecord | undefined;
try {
// If we've previously interacted with strage service, update 'fetchComplete' record
const previousFetchComplete = window.storage.get('storageFetchComplete');
@ -1028,7 +1039,11 @@ async function sync(): Promise<ManifestRecordClass | undefined> {
return undefined;
}
const version = manifest.version.toNumber();
strictAssert(
manifest.version !== undefined && manifest.version !== null,
'Manifest without version'
);
const version = normalizeNumber(manifest.version);
window.log.info(
`storageService.sync: manifest versions - previous: ${localManifestVersion}, current: ${version}`
@ -1095,7 +1110,7 @@ async function upload(fromSync = false): Promise<void> {
return;
}
let previousManifest: ManifestRecordClass | undefined;
let previousManifest: Proto.ManifestRecord | undefined;
if (!fromSync) {
// Syncing before we upload so that we repair any unknown records and
// records with errors as well as ensure that we have the latest up to date