From fa8c66410d9221fc4fbf0b38c5376a04c5de34ab Mon Sep 17 00:00:00 2001 From: automated-signal <37887102+automated-signal@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:28:25 -0500 Subject: [PATCH] Use storage service for call links Co-authored-by: ayumi-signal <143036029+ayumi-signal@users.noreply.github.com> --- protos/SignalService.proto | 2 +- protos/SignalStorage.proto | 9 + .../CallingAdhocCallInfo.stories.tsx | 1 + ts/services/backups/import.ts | 1 + ts/services/calling.ts | 8 +- ts/services/storage.ts | 98 +++++++++ ts/services/storageRecordOps.ts | 191 ++++++++++++++++++ ts/sql/Interface.ts | 12 +- ts/sql/Server.ts | 14 ++ ts/sql/migrations/1190-call-links-storage.ts | 38 ++++ ts/sql/migrations/index.ts | 6 +- ts/sql/server/callLinks.ts | 110 +++++++++- ts/state/ducks/calling.ts | 49 +++-- ts/test-both/helpers/fakeCallLink.ts | 8 + ts/test-electron/backup/calling_test.ts | 8 + ts/test-electron/state/ducks/calling_test.ts | 8 + ts/test-node/Proto_unknown_field_test.ts | 23 +++ ts/textsecure/MessageReceiver.ts | 4 - ts/types/CallLink.ts | 16 +- ts/util/callDisposition.ts | 2 + ts/util/callLinks.ts | 6 +- ts/util/callLinksRingrtc.ts | 8 + ts/util/sendCallLinkUpdateSync.ts | 11 - 23 files changed, 583 insertions(+), 50 deletions(-) create mode 100644 ts/sql/migrations/1190-call-links-storage.ts diff --git a/protos/SignalService.proto b/protos/SignalService.proto index 428824f20..ce106b505 100644 --- a/protos/SignalService.proto +++ b/protos/SignalService.proto @@ -622,7 +622,7 @@ message SyncMessage { message CallLinkUpdate { enum Type { UPDATE = 0; - DELETE = 1; + reserved 1; // was DELETE, superseded by storage service } optional bytes rootKey = 1; diff --git a/protos/SignalStorage.proto b/protos/SignalStorage.proto index 49875a253..8266ebe9e 100644 --- a/protos/SignalStorage.proto +++ b/protos/SignalStorage.proto @@ -47,6 +47,7 @@ message ManifestRecord { ACCOUNT = 4; STORY_DISTRIBUTION_LIST = 5; STICKER_PACK = 6; + CALL_LINK = 7; } optional bytes raw = 1; @@ -67,6 +68,7 @@ message StorageRecord { AccountRecord account = 4; StoryDistributionListRecord storyDistributionList = 5; StickerPackRecord stickerPack = 6; + CallLinkRecord callLink = 7; } } @@ -241,3 +243,10 @@ message StickerPackRecord { // non-zero - `packKey` and `position` should // be unset } + +message CallLinkRecord { + optional bytes rootKey = 1; // 16 bytes + optional bytes adminPasskey = 2; // Non-empty when the current user is an admin + optional uint64 deletedAtTimestampMs = 3; // When present and non-zero, `adminPasskey` + // should be cleared +} diff --git a/ts/components/CallingAdhocCallInfo.stories.tsx b/ts/components/CallingAdhocCallInfo.stories.tsx index 65684d5fb..38a58beb8 100644 --- a/ts/components/CallingAdhocCallInfo.stories.tsx +++ b/ts/components/CallingAdhocCallInfo.stories.tsx @@ -54,6 +54,7 @@ function getCallLink(overrideProps: Partial = {}): CallLinkType { restrictions: CallLinkRestrictions.None, revoked: false, expiration: Date.now() + 30 * 24 * 60 * 60 * 1000, + storageNeedsSync: false, ...overrideProps, }; } diff --git a/ts/services/backups/import.ts b/ts/services/backups/import.ts index ceaa58857..11480c9de 100644 --- a/ts/services/backups/import.ts +++ b/ts/services/backups/import.ts @@ -1074,6 +1074,7 @@ export class BackupImportStream extends Writable { restrictions: fromCallLinkRestrictionsProto(restrictions), revoked: false, expiration: expirationMs?.toNumber() || null, + storageNeedsSync: false, }; this.recipientIdToCallLink.set(recipientId, callLink); diff --git a/ts/services/calling.ts b/ts/services/calling.ts index 91c098e15..122cdb917 100644 --- a/ts/services/calling.ts +++ b/ts/services/calling.ts @@ -153,10 +153,7 @@ import { import type { CallLinkType, CallLinkStateType } from '../types/CallLink'; import { CallLinkRestrictions } from '../types/CallLink'; import { getConversationIdForLogging } from '../util/idForLogging'; -import { - sendCallLinkDeleteSync, - sendCallLinkUpdateSync, -} from '../util/sendCallLinkUpdateSync'; +import { sendCallLinkUpdateSync } from '../util/sendCallLinkUpdateSync'; import { createIdenticon } from '../util/createIdenticon'; import { getColorForCallLink } from '../util/getColorForCallLink'; @@ -677,6 +674,7 @@ export class CallingClass { roomId: roomIdHex, rootKey: rootKey.toString(), adminKey: adminKey.toString('base64'), + storageNeedsSync: true, ...state, }; @@ -716,8 +714,6 @@ export class CallingClass { log.error(`${logId}: ${message}`); throw new Error(message); } - - drop(sendCallLinkDeleteSync(callLink)); } async updateCallLinkName( diff --git a/ts/services/storage.ts b/ts/services/storage.ts index f83be1e70..e1d9663dd 100644 --- a/ts/services/storage.ts +++ b/ts/services/storage.ts @@ -29,6 +29,8 @@ import { toGroupV2Record, toStoryDistributionListRecord, toStickerPackRecord, + toCallLinkRecord, + mergeCallLinkRecord, } from './storageRecordOps'; import type { MergeResultType } from './storageRecordOps'; import { MAX_READ_KEYS } from './storageConstants'; @@ -68,6 +70,8 @@ import { MY_STORY_ID } from '../types/Stories'; import { isNotNil } from '../util/isNotNil'; import { isSignalConversation } from '../util/isSignalConversation'; import { redactExtendedStorageID, redactStorageID } from '../util/privacy'; +import type { CallLinkRecord } from '../types/CallLink'; +import { callLinkFromRecord } from '../util/callLinksRingrtc'; type IManifestRecordIdentifier = Proto.ManifestRecord.IIdentifier; @@ -90,6 +94,7 @@ const validRecordTypes = new Set([ 4, // ACCOUNT 5, // STORY_DISTRIBUTION_LIST 6, // STICKER_PACK + 7, // CALL_LINK ]); const backOff = new BackOff([ @@ -326,6 +331,7 @@ async function generateManifest( } const { + callLinkDbRecords, storyDistributionLists, installedStickerPacks, uninstalledStickerPacks, @@ -460,6 +466,57 @@ async function generateManifest( } }); + log.info( + `storageService.upload(${version}): ` + + `adding callLinks=${callLinkDbRecords.length}` + ); + + for (const callLinkDbRecord of callLinkDbRecords) { + const { roomId } = callLinkDbRecord; + if (callLinkDbRecord.adminKey == null || callLinkDbRecord.rootKey == null) { + log.warn( + `storageService.upload(${version}): ` + + `call link ${roomId} has empty rootKey` + ); + continue; + } + + const storageRecord = new Proto.StorageRecord(); + storageRecord.callLink = toCallLinkRecord(callLinkDbRecord); + + const callLink = callLinkFromRecord(callLinkDbRecord); + const { isNewItem, storageID } = processStorageRecord({ + currentStorageID: callLink.storageID, + currentStorageVersion: callLink.storageVersion, + identifierType: ITEM_TYPE.CALL_LINK, + storageNeedsSync: callLink.storageNeedsSync, + storageRecord, + }); + + const storageFields = { + storageID, + storageVersion: version, + storageNeedsSync: false, + }; + + if (isNewItem) { + postUploadUpdateFunctions.push(async () => { + const freshCallLink = await DataReader.getCallLinkByRoomId(roomId); + if (freshCallLink == null) { + log.warn( + `storageService.upload(${version}): ` + + `call link ${roomId} removed locally from DB while we were uploading to storage` + ); + return; + } + + const callLinkToSave = { ...freshCallLink, ...storageFields }; + await DataWriter.updateCallLink(callLinkToSave); + window.reduxActions.calling.handleCallLinkUpdateLocal(callLinkToSave); + }); + } + } + const unknownRecordsArray: ReadonlyArray = ( window.storage.get('storage-service-unknown-records') || [] ).filter((record: UnknownRecord) => !validRecordTypes.has(record.itemType)); @@ -1023,6 +1080,12 @@ async function mergeRecord( storageVersion, storageRecord.stickerPack ); + } else if (itemType === ITEM_TYPE.CALL_LINK && storageRecord.callLink) { + mergeResult = await mergeCallLinkRecord( + storageID, + storageVersion, + storageRecord.callLink + ); } else { isUnsupported = true; log.warn( @@ -1077,6 +1140,7 @@ async function mergeRecord( } type NonConversationRecordsResultType = Readonly<{ + callLinkDbRecords: ReadonlyArray; installedStickerPacks: ReadonlyArray; uninstalledStickerPacks: ReadonlyArray; storyDistributionLists: ReadonlyArray; @@ -1085,16 +1149,19 @@ type NonConversationRecordsResultType = Readonly<{ // TODO: DESKTOP-3929 async function getNonConversationRecords(): Promise { const [ + callLinkDbRecords, storyDistributionLists, uninstalledStickerPacks, installedStickerPacks, ] = await Promise.all([ + DataReader.getAllCallLinkRecordsWithAdminKey(), DataReader.getAllStoryDistributionsWithMembers(), DataReader.getUninstalledStickerPacks(), DataReader.getInstalledStickerPacks(), ]); return { + callLinkDbRecords, storyDistributionLists, uninstalledStickerPacks, installedStickerPacks, @@ -1130,6 +1197,7 @@ async function processManifest( { const { + callLinkDbRecords, storyDistributionLists, installedStickerPacks, uninstalledStickerPacks, @@ -1144,6 +1212,11 @@ async function processManifest( } }; + callLinkDbRecords.forEach(dbRecord => + collectLocalKeysFromFields(callLinkFromRecord(dbRecord)) + ); + localRecordCount += callLinkDbRecords.length; + storyDistributionLists.forEach(collectLocalKeysFromFields); localRecordCount += storyDistributionLists.length; @@ -1264,6 +1337,7 @@ async function processManifest( // Refetch various records post-merge { const { + callLinkDbRecords, storyDistributionLists, installedStickerPacks, uninstalledStickerPacks, @@ -1352,6 +1426,30 @@ async function processManifest( conflictCount += 1; } + + callLinkDbRecords.forEach(callLinkDbRecord => { + const { storageID, storageVersion } = callLinkDbRecord; + if (!storageID || remoteKeys.has(storageID)) { + return; + } + + const missingKey = redactStorageID( + storageID, + storageVersion || undefined + ); + log.info( + `storageService.process(${version}): localKey=${missingKey} was not ` + + 'in remote manifest' + ); + const callLink = callLinkFromRecord(callLinkDbRecord); + drop( + DataWriter.updateCallLink({ + ...callLink, + storageID: undefined, + storageVersion: undefined, + }) + ); + }); } log.info( diff --git a/ts/services/storageRecordOps.ts b/ts/services/storageRecordOps.ts index 244a5839e..fc2b25497 100644 --- a/ts/services/storageRecordOps.ts +++ b/ts/services/storageRecordOps.ts @@ -4,6 +4,7 @@ import { isEqual, isNumber } from 'lodash'; import Long from 'long'; +import { CallLinkRootKey } from '@signalapp/ringrtc'; import { uuidToBytes, bytesToUuid } from '../util/uuidToBytes'; import { deriveMasterKeyFromGroupV1 } from '../Crypto'; import * as Bytes from '../Bytes'; @@ -65,6 +66,18 @@ import { findAndDeleteOnboardingStoryIfExists } from '../util/findAndDeleteOnboa import { downloadOnboardingStory } from '../util/downloadOnboardingStory'; import { drop } from '../util/drop'; import { redactExtendedStorageID } from '../util/privacy'; +import type { CallLinkRecord } from '../types/CallLink'; +import { + callLinkFromRecord, + fromRootKeyBytes, + getRoomIdFromRootKey, +} from '../util/callLinksRingrtc'; +import { + CALL_LINK_DELETED_STORAGE_RECORD_TTL, + fromAdminKeyBytes, + toCallHistoryFromUnusedCallLink, +} from '../util/callLinks'; +import { isOlderThan } from '../util/timestamp'; const MY_STORY_BYTES = uuidToBytes(MY_STORY_ID); @@ -603,6 +616,35 @@ export function toStickerPackRecord( return stickerPackRecord; } +// callLinkDbRecord exposes additional fields not available on CallLinkType +export function toCallLinkRecord( + callLinkDbRecord: CallLinkRecord +): Proto.CallLinkRecord { + strictAssert(callLinkDbRecord.rootKey, 'toCallLinkRecord: no rootKey'); + + const callLinkRecord = new Proto.CallLinkRecord(); + + callLinkRecord.rootKey = callLinkDbRecord.rootKey; + if (callLinkDbRecord.deleted === 1) { + // adminKey is intentionally omitted for deleted call links. + callLinkRecord.deletedAtTimestampMs = Long.fromNumber( + callLinkDbRecord.deletedAt || new Date().getTime() + ); + } else { + strictAssert( + callLinkDbRecord.adminKey, + 'toCallLinkRecord: no adminPasskey' + ); + callLinkRecord.adminPasskey = callLinkDbRecord.adminKey; + } + + if (callLinkDbRecord.storageUnknownFields) { + callLinkRecord.$unknownFields = [callLinkDbRecord.storageUnknownFields]; + } + + return callLinkRecord; +} + type MessageRequestCapableRecord = Proto.IContactRecord | Proto.IGroupV1Record; function applyMessageRequestState( @@ -1906,3 +1948,152 @@ export async function mergeStickerPackRecord( oldStorageVersion, }; } + +export async function mergeCallLinkRecord( + storageID: string, + storageVersion: number, + callLinkRecord: Proto.ICallLinkRecord +): Promise { + const redactedStorageID = redactExtendedStorageID({ + storageID, + storageVersion, + }); + // callLinkRecords must have rootKey + if (!callLinkRecord.rootKey) { + return { hasConflict: false, shouldDrop: true, details: ['no rootKey'] }; + } + + const details: Array = []; + + const rootKeyString = fromRootKeyBytes(callLinkRecord.rootKey); + const adminKeyString = callLinkRecord.adminPasskey + ? fromAdminKeyBytes(callLinkRecord.adminPasskey) + : null; + + const callLinkRootKey = CallLinkRootKey.parse(rootKeyString); + const roomId = getRoomIdFromRootKey(callLinkRootKey); + const logId = `mergeCallLinkRecord(${redactedStorageID}, ${roomId})`; + + const localCallLinkDbRecord = + await DataReader.getCallLinkRecordByRoomId(roomId); + + const deletedAt: number | null = + callLinkRecord.deletedAtTimestampMs != null + ? getTimestampFromLong(callLinkRecord.deletedAtTimestampMs) + : null; + const shouldDrop = + deletedAt != null && + isOlderThan(deletedAt, CALL_LINK_DELETED_STORAGE_RECORD_TTL); + if (shouldDrop) { + details.push('expired deleted call link; scheduling for removal'); + } + + const callLinkDbRecord: CallLinkRecord = { + roomId, + rootKey: callLinkRecord.rootKey, + adminKey: callLinkRecord.adminPasskey ?? null, + name: localCallLinkDbRecord?.name ?? '', + restrictions: localCallLinkDbRecord?.restrictions ?? 0, + expiration: localCallLinkDbRecord?.expiration ?? null, + revoked: localCallLinkDbRecord?.revoked === 1 ? 1 : 0, + deleted: deletedAt ? 1 : 0, + deletedAt, + + storageID, + storageVersion, + storageUnknownFields: callLinkRecord.$unknownFields + ? Bytes.concatenate(callLinkRecord.$unknownFields) + : null, + storageNeedsSync: localCallLinkDbRecord?.storageNeedsSync === 1 ? 1 : 0, + }; + + if (!localCallLinkDbRecord) { + if (deletedAt) { + log.info( + `${logId}: Found deleted call link with no matching local record, skipping` + ); + } else { + log.info(`${logId}: Discovered new call link, creating locally`); + details.push('creating call link'); + + // Create CallLink and call history item + const callLink = callLinkFromRecord(callLinkDbRecord); + const callHistory = toCallHistoryFromUnusedCallLink(callLink); + await Promise.all([ + DataWriter.insertCallLink(callLink), + DataWriter.saveCallHistory(callHistory), + ]); + + // Refresh call link state via RingRTC and update in redux + window.reduxActions.calling.handleCallLinkUpdate({ + rootKey: rootKeyString, + adminKey: adminKeyString, + }); + window.reduxActions.callHistory.addCallHistory(callHistory); + } + + return { + details, + hasConflict: false, + shouldDrop, + }; + } + + const oldStorageID = localCallLinkDbRecord.storageID || undefined; + const oldStorageVersion = localCallLinkDbRecord.storageVersion || undefined; + + const needsToClearUnknownFields = + !callLinkRecord.$unknownFields && + localCallLinkDbRecord.storageUnknownFields; + if (needsToClearUnknownFields) { + details.push('clearing unknown fields'); + } + + const isBadRemoteData = Boolean(deletedAt && adminKeyString); + if (isBadRemoteData) { + log.warn( + `${logId}: Found bad remote data: deletedAtTimestampMs and adminPasskey were both present. Assuming deleted.` + ); + } + + const { hasConflict, details: conflictDetails } = doRecordsConflict( + toCallLinkRecord(callLinkDbRecord), + callLinkRecord + ); + + // First update local record + details.push('updated'); + const callLink = callLinkFromRecord(callLinkDbRecord); + await DataWriter.updateCallLink(callLink); + + // Deleted in storage but we have it locally: Delete locally too and update redux + if (deletedAt && localCallLinkDbRecord.deleted !== 1) { + // Another device deleted the link and uploaded to storage, and we learned about it + log.info(`${logId}: Discovered deleted call link, deleting locally`); + details.push('deleting locally'); + await DataWriter.beginDeleteCallLink(roomId, { + storageNeedsSync: false, + deletedAt, + }); + // No need to delete via RingRTC as we assume the originating device did that already + await DataWriter.finalizeDeleteCallLink(roomId); + window.reduxActions.calling.handleCallLinkDelete({ roomId }); + } else if (!deletedAt && localCallLinkDbRecord.deleted === 1) { + // Not deleted in storage, but we've marked it as deleted locally. + // Skip doing anything, we will update things locally after sync. + log.warn(`${logId}: Found call link, but it was marked deleted locally.`); + } else { + window.reduxActions.calling.handleCallLinkUpdate({ + rootKey: rootKeyString, + adminKey: adminKeyString, + }); + } + + return { + details: [...details, ...conflictDetails], + hasConflict, + shouldDrop, + oldStorageID, + oldStorageVersion, + }; +} diff --git a/ts/sql/Interface.ts b/ts/sql/Interface.ts index 8e572bf3f..91702d9d7 100644 --- a/ts/sql/Interface.ts +++ b/ts/sql/Interface.ts @@ -31,12 +31,17 @@ import type { CallHistoryPagination, CallLogEventTarget, } from '../types/CallDisposition'; -import type { CallLinkStateType, CallLinkType } from '../types/CallLink'; +import type { + CallLinkRecord, + CallLinkStateType, + CallLinkType, +} from '../types/CallLink'; import type { AttachmentDownloadJobType } from '../types/AttachmentDownload'; import type { GroupSendEndorsementsData } from '../types/GroupSendEndorsements'; import type { SyncTaskType } from '../util/syncTasks'; import type { AttachmentBackupJobType } from '../types/AttachmentBackup'; import type { SingleProtoJobQueue } from '../jobs/singleProtoJobQueue'; +import type { DeleteCallLinkOptions } from './server/callLinks'; export type ReadableDB = Database & { __readable_db: never }; export type WritableDB = ReadableDB & { __writable_db: never }; @@ -568,6 +573,8 @@ type ReadableInterface = { callLinkExists(roomId: string): boolean; getAllCallLinks: () => ReadonlyArray; getCallLinkByRoomId: (roomId: string) => CallLinkType | undefined; + getCallLinkRecordByRoomId: (roomId: string) => CallLinkRecord | undefined; + getAllCallLinkRecordsWithAdminKey(): ReadonlyArray; getAllMarkedDeletedCallLinks(): ReadonlyArray; getMessagesBetween: ( conversationId: string, @@ -799,13 +806,14 @@ type WritableInterface = { markCallHistoryMissed(callIds: ReadonlyArray): void; getRecentStaleRingsAndMarkOlderMissed(): ReadonlyArray; insertCallLink(callLink: CallLinkType): void; + updateCallLink(callLink: CallLinkType): void; updateCallLinkAdminKeyByRoomId(roomId: string, adminKey: string): void; updateCallLinkState( roomId: string, callLinkState: CallLinkStateType ): CallLinkType; beginDeleteAllCallLinks(): void; - beginDeleteCallLink(roomId: string): void; + beginDeleteCallLink(roomId: string, options: DeleteCallLinkOptions): void; finalizeDeleteCallLink(roomId: string): void; _removeAllCallLinks(): void; deleteCallLinkFromSync(roomId: string): void; diff --git a/ts/sql/Server.ts b/ts/sql/Server.ts index cc77d083a..0b72e22fa 100644 --- a/ts/sql/Server.ts +++ b/ts/sql/Server.ts @@ -174,10 +174,13 @@ import { callLinkExists, getAllCallLinks, getCallLinkByRoomId, + getCallLinkRecordByRoomId, insertCallLink, + updateCallLink, updateCallLinkAdminKeyByRoomId, updateCallLinkState, beginDeleteAllCallLinks, + getAllCallLinkRecordsWithAdminKey, getAllMarkedDeletedCallLinks, finalizeDeleteCallLink, beginDeleteCallLink, @@ -304,6 +307,8 @@ export const DataReader: ServerReadableInterface = { callLinkExists, getAllCallLinks, getCallLinkByRoomId, + getCallLinkRecordByRoomId, + getAllCallLinkRecordsWithAdminKey, getAllMarkedDeletedCallLinks, getMessagesBetween, getNearbyMessageFromDeletedSet, @@ -439,6 +444,7 @@ export const DataWriter: ServerWritableInterface = { saveCallHistory, markCallHistoryMissed, insertCallLink, + updateCallLink, updateCallLinkAdminKeyByRoomId, updateCallLinkState, beginDeleteAllCallLinks, @@ -6445,6 +6451,14 @@ function eraseStorageServiceState(db: WritableDB): void { storageVersion = null, storageUnknownFields = null, storageNeedsSync = 0; + + -- Call links + UPDATE callLinks + SET + storageID = null, + storageVersion = null, + storageUnknownFields = null, + storageNeedsSync = 0; `); } diff --git a/ts/sql/migrations/1190-call-links-storage.ts b/ts/sql/migrations/1190-call-links-storage.ts new file mode 100644 index 000000000..9642ba626 --- /dev/null +++ b/ts/sql/migrations/1190-call-links-storage.ts @@ -0,0 +1,38 @@ +// Copyright 2024 Signal Messenger, LLC +// SPDX-License-Identifier: AGPL-3.0-only +import type { Database } from '@signalapp/better-sqlite3'; +import type { LoggerType } from '../../types/Logging'; + +export const version = 1190; + +export function updateToSchemaVersion1190( + currentVersion: number, + db: Database, + logger: LoggerType +): void { + if (currentVersion >= 1190) { + return; + } + + db.transaction(() => { + db.exec(` + ALTER TABLE callLinks ADD COLUMN storageID TEXT; + ALTER TABLE callLinks ADD COLUMN storageVersion INTEGER; + ALTER TABLE callLinks ADD COLUMN storageUnknownFields BLOB; + ALTER TABLE callLinks ADD COLUMN storageNeedsSync INTEGER NOT NULL DEFAULT 0; + ALTER TABLE callLinks ADD COLUMN deletedAt INTEGER; + `); + db.prepare( + ` + UPDATE callLinks + SET deletedAt = $deletedAt + WHERE deleted = 1; + ` + ).run({ + deletedAt: new Date().getTime(), + }); + + db.pragma('user_version = 1190'); + })(); + logger.info('updateToSchemaVersion1190: success!'); +} diff --git a/ts/sql/migrations/index.ts b/ts/sql/migrations/index.ts index cd3506bf7..bfc4bbe49 100644 --- a/ts/sql/migrations/index.ts +++ b/ts/sql/migrations/index.ts @@ -94,10 +94,11 @@ import { updateToSchemaVersion1140 } from './1140-call-links-deleted-column'; import { updateToSchemaVersion1150 } from './1150-expire-timer-version'; import { updateToSchemaVersion1160 } from './1160-optimize-calls-unread-count'; import { updateToSchemaVersion1170 } from './1170-update-call-history-unread-index'; +import { updateToSchemaVersion1180 } from './1180-add-attachment-download-source'; import { - updateToSchemaVersion1180, + updateToSchemaVersion1190, version as MAX_VERSION, -} from './1180-add-attachment-download-source'; +} from './1190-call-links-storage'; function updateToSchemaVersion1( currentVersion: number, @@ -2060,6 +2061,7 @@ export const SCHEMA_VERSIONS = [ updateToSchemaVersion1160, updateToSchemaVersion1170, updateToSchemaVersion1180, + updateToSchemaVersion1190, ]; export class DBVersionFromFutureError extends Error { diff --git a/ts/sql/server/callLinks.ts b/ts/sql/server/callLinks.ts index 444f63be7..959670e9e 100644 --- a/ts/sql/server/callLinks.ts +++ b/ts/sql/server/callLinks.ts @@ -2,7 +2,11 @@ // SPDX-License-Identifier: AGPL-3.0-only import { CallLinkRootKey } from '@signalapp/ringrtc'; -import type { CallLinkStateType, CallLinkType } from '../../types/CallLink'; +import type { + CallLinkRecord, + CallLinkStateType, + CallLinkType, +} from '../../types/CallLink'; import { callLinkRestrictionsSchema, callLinkRecordSchema, @@ -31,6 +35,19 @@ export function getCallLinkByRoomId( db: ReadableDB, roomId: string ): CallLinkType | undefined { + const callLinkRecord = getCallLinkRecordByRoomId(db, roomId); + if (!callLinkRecord) { + return undefined; + } + + return callLinkFromRecord(callLinkRecord); +} + +// When you need to access all the fields (such as deleted and storage fields) +export function getCallLinkRecordByRoomId( + db: ReadableDB, + roomId: string +): CallLinkRecord | undefined { const row = prepare(db, 'SELECT * FROM callLinks WHERE roomId = $roomId').get( { roomId, @@ -41,7 +58,7 @@ export function getCallLinkByRoomId( return undefined; } - return callLinkFromRecord(callLinkRecordSchema.parse(row)); + return callLinkRecordSchema.parse(row); } export function getAllCallLinks(db: ReadableDB): ReadonlyArray { @@ -69,7 +86,11 @@ function _insertCallLink(db: WritableDB, callLink: CallLinkType): void { name, restrictions, revoked, - expiration + expiration, + storageID, + storageVersion, + storageUnknownFields, + storageNeedsSync ) VALUES ( $roomId, $rootKey, @@ -77,7 +98,11 @@ function _insertCallLink(db: WritableDB, callLink: CallLinkType): void { $name, $restrictions, $revoked, - $expiration + $expiration, + $storageID, + $storageVersion, + $storageUnknownFields, + $storageNeedsSync ) ` ).run(data); @@ -87,6 +112,30 @@ export function insertCallLink(db: WritableDB, callLink: CallLinkType): void { _insertCallLink(db, callLink); } +export function updateCallLink(db: WritableDB, callLink: CallLinkType): void { + const { roomId, rootKey } = callLink; + assertRoomIdMatchesRootKey(roomId, rootKey); + + const data = callLinkToRecord(callLink); + // Do not write roomId or rootKey since they should never change + db.prepare( + ` + UPDATE callLinks + SET + adminKey = $adminKey, + name = $name, + restrictions = $restrictions, + revoked = $revoked, + expiration = $expiration, + storageID = $storageID, + storageVersion = $storageVersion, + storageUnknownFields = $storageUnknownFields, + storageNeedsSync = $storageNeedsSync + WHERE roomId = $roomId + ` + ).run(data); +} + export function updateCallLinkState( db: WritableDB, roomId: string, @@ -167,7 +216,16 @@ export function deleteCallLinkFromSync(db: WritableDB, roomId: string): void { })(); } -export function beginDeleteCallLink(db: WritableDB, roomId: string): void { +export type DeleteCallLinkOptions = { + storageNeedsSync: boolean; + deletedAt?: number; +}; + +export function beginDeleteCallLink( + db: WritableDB, + roomId: string, + options: DeleteCallLinkOptions +): void { db.transaction(() => { // If adminKey is null, then we should delete the call link const [deleteNonAdminCallLinksQuery, deleteNonAdminCallLinksParams] = sql` @@ -182,11 +240,17 @@ export function beginDeleteCallLink(db: WritableDB, roomId: string): void { // Skip this query if the call is already deleted if (result.changes === 0) { + const { storageNeedsSync } = options; + const deletedAt = options.deletedAt ?? new Date().getTime(); + // If the admin key is not null, we should mark it for deletion const [markAdminCallLinksDeletedQuery, markAdminCallLinksDeletedParams] = sql` UPDATE callLinks - SET deleted = 1 + SET + deleted = 1, + deletedAt = ${deletedAt}, + storageNeedsSync = ${storageNeedsSync ? 1 : 0} WHERE adminKey IS NOT NULL AND roomId = ${roomId}; `; @@ -201,14 +265,21 @@ export function beginDeleteCallLink(db: WritableDB, roomId: string): void { } export function beginDeleteAllCallLinks(db: WritableDB): void { + const deletedAt = new Date().getTime(); db.transaction(() => { - const [markAdminCallLinksDeletedQuery] = sql` + const [markAdminCallLinksDeletedQuery, markAdminCallLinksDeletedParams] = + sql` UPDATE callLinks - SET deleted = 1 + SET + deleted = 1, + deletedAt = ${deletedAt}, + storageNeedsSync = 1 WHERE adminKey IS NOT NULL; `; - db.prepare(markAdminCallLinksDeletedQuery).run(); + db.prepare(markAdminCallLinksDeletedQuery).run( + markAdminCallLinksDeletedParams + ); const [deleteNonAdminCallLinksQuery] = sql` DELETE FROM callLinks @@ -219,6 +290,21 @@ export function beginDeleteAllCallLinks(db: WritableDB): void { })(); } +// When you need to access the deleted field +export function getAllCallLinkRecordsWithAdminKey( + db: ReadableDB +): ReadonlyArray { + const [query] = sql` + SELECT * FROM callLinks + WHERE adminKey IS NOT NULL + AND rootKey IS NOT NULL; + `; + return db + .prepare(query) + .all() + .map(item => callLinkRecordSchema.parse(item)); +} + export function getAllMarkedDeletedCallLinks( db: ReadableDB ): ReadonlyArray { @@ -231,9 +317,13 @@ export function getAllMarkedDeletedCallLinks( .map(item => callLinkFromRecord(callLinkRecordSchema.parse(item))); } +// TODO: Run this after uploading storage records, maybe periodically on startup export function finalizeDeleteCallLink(db: WritableDB, roomId: string): void { const [query, params] = sql` - DELETE FROM callLinks WHERE roomId = ${roomId} AND deleted = 1; + DELETE FROM callLinks + WHERE roomId = ${roomId} + AND deleted = 1 + AND storageNeedsSync = 0; `; db.prepare(query).run(params); } diff --git a/ts/state/ducks/calling.ts b/ts/state/ducks/calling.ts index eaaab1c84..4c557c333 100644 --- a/ts/state/ducks/calling.ts +++ b/ts/state/ducks/calling.ts @@ -6,7 +6,7 @@ import { hasScreenCapturePermission, openSystemPreferences, } from 'mac-screen-capture-permissions'; -import { omit, pick } from 'lodash'; +import { omit } from 'lodash'; import type { ReadonlyDeep } from 'type-fest'; import { CallLinkRootKey, @@ -98,6 +98,7 @@ import type { CallHistoryDetails } from '../../types/CallDisposition'; import type { StartCallData } from '../../components/ConfirmLeaveCallModal'; import { callLinksDeleteJobQueue } from '../../jobs/callLinksDeleteJobQueue'; import { getCallLinksByRoomId } from '../selectors/calling'; +import { storageServiceUploadJob } from '../../services/storage'; // State @@ -1425,25 +1426,31 @@ function handleCallLinkUpdate( const logId = `handleCallLinkUpdate(${roomId})`; const freshCallLinkState = await calling.readCallLink(callLinkRootKey); + const existingCallLink = await DataReader.getCallLinkByRoomId(roomId); // Only give up when server confirms the call link is gone. If we fail to fetch // state due to unexpected errors, continue to save rootKey and adminKey. if (freshCallLinkState == null) { - log.info(`${logId}: Call link not found, ignoring`); + log.info(`${logId}: Call link not found on server`); + if (!existingCallLink) { + return; + } + + // If the call link is gone remotely (for example if it expired on the server), + // then delete local call link. + log.info(`${logId}: Deleting existing call link`); + await DataWriter.beginDeleteCallLink(roomId, { + storageNeedsSync: true, + }); + storageServiceUploadJob(); + handleCallLinkDelete({ roomId }); return; } - const existingCallLink = await DataReader.getCallLinkByRoomId(roomId); - const existingCallLinkState = pick(existingCallLink, [ - 'name', - 'restrictions', - 'expiration', - 'revoked', - ]); - const callLink: CallLinkType = { ...CALL_LINK_DEFAULT_STATE, - ...existingCallLinkState, + storageNeedsSync: false, + ...existingCallLink, ...freshCallLinkState, roomId, rootKey, @@ -1482,6 +1489,17 @@ function handleCallLinkUpdate( }; } +function handleCallLinkUpdateLocal( + callLink: CallLinkType +): ThunkAction { + return dispatch => { + dispatch({ + type: HANDLE_CALL_LINK_UPDATE, + payload: { callLink }, + }); + }; +} + function handleCallLinkDelete( payload: HandleCallLinkDeleteType ): ThunkAction { @@ -1990,6 +2008,9 @@ function createCallLink( DataWriter.insertCallLink(callLink), DataWriter.saveCallHistory(callHistory), ]); + + storageServiceUploadJob(); + dispatch({ type: HANDLE_CALL_LINK_UPDATE, payload: { callLink }, @@ -2004,7 +2025,8 @@ function deleteCallLink( roomId: string ): ThunkAction { return async dispatch => { - await DataWriter.beginDeleteCallLink(roomId); + await DataWriter.beginDeleteCallLink(roomId, { storageNeedsSync: true }); + storageServiceUploadJob(); await callLinksDeleteJobQueue.add({ source: 'deleteCallLink' }); dispatch(handleCallLinkDelete({ roomId })); }; @@ -2171,6 +2193,7 @@ const _startCallLinkLobby = async ({ restrictions, revoked, expiration, + storageNeedsSync: false, }); log.info('startCallLinkLobby: Saved new call link', roomId); } @@ -2446,6 +2469,7 @@ export const actions = { groupCallStateChange, hangUpActiveCall, handleCallLinkUpdate, + handleCallLinkUpdateLocal, handleCallLinkDelete, joinedAdhocCall, leaveCurrentCallAndStartCallingLobby, @@ -2670,6 +2694,7 @@ export function reducer( callLinks[conversationId]?.rootKey ?? action.payload.callLinkRootKey, adminKey: callLinks[conversationId]?.adminKey, + storageNeedsSync: false, }, } : callLinks, diff --git a/ts/test-both/helpers/fakeCallLink.ts b/ts/test-both/helpers/fakeCallLink.ts index 4bc3c0532..87ac3fa90 100644 --- a/ts/test-both/helpers/fakeCallLink.ts +++ b/ts/test-both/helpers/fakeCallLink.ts @@ -13,6 +13,10 @@ export const FAKE_CALL_LINK: CallLinkType = { revoked: false, roomId: 'd517b48dd118bee24068d4938886c8abe192706d84936d52594a9157189d2759', rootKey: 'dxbb-xfqz-xkgp-nmrx-bpqn-ptkb-spdt-pdgt', + storageID: undefined, + storageVersion: undefined, + storageUnknownFields: undefined, + storageNeedsSync: false, }; // Please set expiration @@ -24,6 +28,10 @@ export const FAKE_CALL_LINK_WITH_ADMIN_KEY: CallLinkType = { revoked: false, roomId: 'c097eb04cc278d6bc7ed9fb2ddeac00dc9646ae6ddb38513dad9a8a4fe3c38f4', rootKey: 'bpmc-mrgn-hntf-mffd-mndd-xbxk-zmgq-qszg', + storageID: undefined, + storageVersion: undefined, + storageUnknownFields: undefined, + storageNeedsSync: false, }; export function getCallLinkState(callLink: CallLinkType): CallLinkStateType { diff --git a/ts/test-electron/backup/calling_test.ts b/ts/test-electron/backup/calling_test.ts index 5e2d66045..ae00d8e5a 100644 --- a/ts/test-electron/backup/calling_test.ts +++ b/ts/test-electron/backup/calling_test.ts @@ -73,6 +73,10 @@ describe('backup/calling', () => { restrictions: CallLinkRestrictions.AdminApproval, revoked: false, expiration: null, + storageID: undefined, + storageVersion: undefined, + storageUnknownFields: undefined, + storageNeedsSync: false, }; await DataWriter.insertCallLink(callLink); @@ -201,6 +205,10 @@ describe('backup/calling', () => { restrictions: CallLinkRestrictions.AdminApproval, revoked: false, expiration: null, + storageID: undefined, + storageVersion: undefined, + storageUnknownFields: undefined, + storageNeedsSync: false, }; await DataWriter.insertCallLink(callLinkNoAdmin); diff --git a/ts/test-electron/state/ducks/calling_test.ts b/ts/test-electron/state/ducks/calling_test.ts index 53c0b7eeb..488f0f29d 100644 --- a/ts/test-electron/state/ducks/calling_test.ts +++ b/ts/test-electron/state/ducks/calling_test.ts @@ -1368,6 +1368,10 @@ describe('calling duck', () => { roomId, rootKey, adminKey, + storageID: undefined, + storageVersion: undefined, + storageUnknownFields: undefined, + storageNeedsSync: false, }, }, }); @@ -1388,6 +1392,10 @@ describe('calling duck', () => { roomId, rootKey, adminKey: 'banana', + storageID: undefined, + storageVersion: undefined, + storageUnknownFields: undefined, + storageNeedsSync: false, }, }, }); diff --git a/ts/test-node/Proto_unknown_field_test.ts b/ts/test-node/Proto_unknown_field_test.ts index 0d67c5ca5..ff74bc914 100644 --- a/ts/test-node/Proto_unknown_field_test.ts +++ b/ts/test-node/Proto_unknown_field_test.ts @@ -115,4 +115,27 @@ describe('Proto#$unknownFields', () => { assert.strictEqual(decoded.c, 42); assert.strictEqual(Buffer.from(decoded.d).toString(), 'ohai'); }); + + it('should not set unknown fields if all fields were known', () => { + const encoded = Partial.encode({ + a: 'hello', + c: 42, + }).finish(); + const decoded = Full.decode(encoded); + + assert.strictEqual(decoded.a, 'hello'); + assert.strictEqual(decoded.c, 42); + assert.isUndefined(decoded.$unknownFields); + + const encodedWithEmptyArray = Partial.encode({ + a: 'hi', + c: 69, + $unkownFields: [], + }).finish(); + const decodedWithEmptyArray = Full.decode(encodedWithEmptyArray); + + assert.strictEqual(decodedWithEmptyArray.a, 'hi'); + assert.strictEqual(decodedWithEmptyArray.c, 69); + assert.isUndefined(decodedWithEmptyArray.$unknownFields); + }); }); diff --git a/ts/textsecure/MessageReceiver.ts b/ts/textsecure/MessageReceiver.ts index 3bd5baa58..e8e1e62d7 100644 --- a/ts/textsecure/MessageReceiver.ts +++ b/ts/textsecure/MessageReceiver.ts @@ -3557,10 +3557,6 @@ export default class MessageReceiver callLinkUpdate.type === Proto.SyncMessage.CallLinkUpdate.Type.UPDATE ) { callLinkUpdateSyncType = CallLinkUpdateSyncType.Update; - } else if ( - callLinkUpdate.type === Proto.SyncMessage.CallLinkUpdate.Type.DELETE - ) { - callLinkUpdateSyncType = CallLinkUpdateSyncType.Delete; } else { throw new Error( `MessageReceiver.handleCallLinkUpdate: unknown type ${callLinkUpdate.type}` diff --git a/ts/types/CallLink.ts b/ts/types/CallLink.ts index 12a2f98b7..d7862241b 100644 --- a/ts/types/CallLink.ts +++ b/ts/types/CallLink.ts @@ -5,6 +5,7 @@ import { z } from 'zod'; import type { ConversationType } from '../state/ducks/conversations'; import { safeParseInteger } from '../util/numbers'; import { byteLength } from '../Bytes'; +import type { StorageServiceFieldsType } from '../sql/Interface'; export enum CallLinkUpdateSyncType { Update = 'Update', @@ -61,7 +62,8 @@ export type CallLinkType = Readonly<{ // Guaranteed from RingRTC readCallLink, but locally may be null immediately after // CallLinkUpdate sync and before readCallLink expiration: number | null; -}>; +}> & + StorageServiceFieldsType; export type CallLinkStateType = Pick< CallLinkType, @@ -86,6 +88,12 @@ export type CallLinkRecord = Readonly<{ restrictions: number; expiration: number | null; revoked: 1 | 0; // sqlite's version of boolean + deleted?: 1 | 0; + deletedAt?: number | null; + storageID: string | null; + storageVersion: number | null; + storageUnknownFields: Uint8Array | null; + storageNeedsSync: 1 | 0; }>; export const callLinkRecordSchema = z.object({ @@ -98,6 +106,12 @@ export const callLinkRecordSchema = z.object({ restrictions: callLinkRestrictionsSchema, expiration: z.number().int().nullable(), revoked: z.union([z.literal(1), z.literal(0)]), + deleted: z.union([z.literal(1), z.literal(0)]).optional(), + deletedAt: z.number().int().nullable().optional(), + storageID: z.string().nullable(), + storageVersion: z.number().int().nullable(), + storageUnknownFields: z.instanceof(Uint8Array).nullable(), + storageNeedsSync: z.union([z.literal(1), z.literal(0)]), }) satisfies z.ZodType; export function isCallLinkAdmin(callLink: CallLinkType): boolean { diff --git a/ts/util/callDisposition.ts b/ts/util/callDisposition.ts index a11b31de7..9ab675223 100644 --- a/ts/util/callDisposition.ts +++ b/ts/util/callDisposition.ts @@ -67,6 +67,7 @@ import type { ConversationModel } from '../models/conversations'; import { drop } from './drop'; import { sendCallLinkUpdateSync } from './sendCallLinkUpdateSync'; import { callLinksDeleteJobQueue } from '../jobs/callLinksDeleteJobQueue'; +import { storageServiceUploadJob } from '../services/storage'; // utils // ----- @@ -1303,6 +1304,7 @@ export async function clearCallHistoryDataAndSync( ); const messageIds = await DataWriter.clearCallHistory(latestCall); await DataWriter.beginDeleteAllCallLinks(); + storageServiceUploadJob(); updateDeletedMessages(messageIds); log.info('clearCallHistory: Queueing sync message'); await singleProtoJobQueue.add( diff --git a/ts/util/callLinks.ts b/ts/util/callLinks.ts index dc688b3d1..16bb25258 100644 --- a/ts/util/callLinks.ts +++ b/ts/util/callLinks.ts @@ -15,14 +15,18 @@ import { type CallHistoryDetails, CallMode, } from '../types/CallDisposition'; +import { DAY } from './durations'; -export const CALL_LINK_DEFAULT_STATE = { +export const CALL_LINK_DEFAULT_STATE: Partial = { name: '', restrictions: CallLinkRestrictions.Unknown, revoked: false, expiration: null, + storageNeedsSync: false, }; +export const CALL_LINK_DELETED_STORAGE_RECORD_TTL = 30 * DAY; + export function getKeyFromCallLink(callLink: string): string { const url = new URL(callLink); if (url == null) { diff --git a/ts/util/callLinksRingrtc.ts b/ts/util/callLinksRingrtc.ts index a25ac3d5f..20dfdcee6 100644 --- a/ts/util/callLinksRingrtc.ts +++ b/ts/util/callLinksRingrtc.ts @@ -136,6 +136,10 @@ export function callLinkFromRecord(record: CallLinkRecord): CallLinkType { restrictions: toCallLinkRestrictions(record.restrictions), revoked: record.revoked === 1, expiration: record.expiration, + storageID: record.storageID || undefined, + storageVersion: record.storageVersion || undefined, + storageUnknownFields: record.storageUnknownFields || undefined, + storageNeedsSync: record.storageNeedsSync === 1, }; } @@ -156,5 +160,9 @@ export function callLinkToRecord(callLink: CallLinkType): CallLinkRecord { restrictions: callLink.restrictions, revoked: callLink.revoked ? 1 : 0, expiration: callLink.expiration, + storageID: callLink.storageID || null, + storageVersion: callLink.storageVersion || null, + storageUnknownFields: callLink.storageUnknownFields || null, + storageNeedsSync: callLink.storageNeedsSync ? 1 : 0, }); } diff --git a/ts/util/sendCallLinkUpdateSync.ts b/ts/util/sendCallLinkUpdateSync.ts index 2d544fe88..4fab44be6 100644 --- a/ts/util/sendCallLinkUpdateSync.ts +++ b/ts/util/sendCallLinkUpdateSync.ts @@ -21,15 +21,6 @@ export async function sendCallLinkUpdateSync( return _sendCallLinkUpdateSync(callLink, CallLinkUpdateSyncType.Update); } -/** - * Underlying sync message is CallLinkUpdate with type set to DELETE. - */ -export async function sendCallLinkDeleteSync( - callLink: sendCallLinkUpdateSyncCallLinkType -): Promise { - return _sendCallLinkUpdateSync(callLink, CallLinkUpdateSyncType.Delete); -} - async function _sendCallLinkUpdateSync( callLink: sendCallLinkUpdateSyncCallLinkType, type: CallLinkUpdateSyncType @@ -37,8 +28,6 @@ async function _sendCallLinkUpdateSync( let protoType: Proto.SyncMessage.CallLinkUpdate.Type; if (type === CallLinkUpdateSyncType.Update) { protoType = Proto.SyncMessage.CallLinkUpdate.Type.UPDATE; - } else if (type === CallLinkUpdateSyncType.Delete) { - protoType = Proto.SyncMessage.CallLinkUpdate.Type.DELETE; } else { throw new Error(`sendCallLinkUpdateSync: unknown type ${type}`); }