From a3b9e97b8299bc8dc656252015bb2c6b7333c3fc Mon Sep 17 00:00:00 2001 From: ayumi-signal <143036029+ayumi-signal@users.noreply.github.com> Date: Tue, 7 May 2024 11:21:57 -0700 Subject: [PATCH] Use global screen share cache for group calls --- ts/components/CallManager.tsx | 15 ++++ ts/components/CallScreen.stories.tsx | 2 + ts/components/CallScreen.tsx | 4 + ts/components/CallingPip.tsx | 4 + ts/components/CallingPipRemoteVideo.tsx | 4 + .../GroupCallOverflowArea.stories.tsx | 3 + ts/components/GroupCallOverflowArea.tsx | 4 + .../GroupCallRemoteParticipant.stories.tsx | 2 + ts/components/GroupCallRemoteParticipant.tsx | 87 ++++++++++++++----- ts/components/GroupCallRemoteParticipants.tsx | 5 ++ ts/util/lint/exceptions.json | 7 ++ 11 files changed, 114 insertions(+), 23 deletions(-) diff --git a/ts/components/CallManager.tsx b/ts/components/CallManager.tsx index d4d839aee1a0..0b7bccbc6aa9 100644 --- a/ts/components/CallManager.tsx +++ b/ts/components/CallManager.tsx @@ -56,6 +56,7 @@ import { callLinkRootKeyToUrl } from '../util/callLinkRootKeyToUrl'; import { ToastType } from '../types/Toast'; import type { ShowToastAction } from '../state/ducks/toast'; import { isSharingPhoneNumberWithEverybody } from '../util/phoneNumberSharingMode'; +import { usePrevious } from '../hooks/usePrevious'; const GROUP_CALL_RING_DURATION = 60 * 1000; @@ -77,6 +78,8 @@ export type GroupIncomingCall = Readonly<{ remoteParticipants: Array; }>; +export type CallingImageDataCache = Map; + export type PropsType = { activeCall?: ActiveCallType; availableCameras: Array; @@ -228,6 +231,16 @@ function ActiveCallManager({ pauseVoiceNotePlayer, ]); + // For caching screenshare frames which update slowly, between Pip and CallScreen. + const imageDataCache = React.useRef(new Map()); + + const previousConversationId = usePrevious(conversation.id, conversation.id); + useEffect(() => { + if (conversation.id !== previousConversationId) { + imageDataCache.current.clear(); + } + }, [conversation.id, previousConversationId]); + const getGroupCallVideoFrameSourceForActiveCall = useCallback( (demuxId: number) => { return getGroupCallVideoFrameSource(conversation.id, demuxId); @@ -313,6 +326,7 @@ function ActiveCallManager({ (), isCallLinkAdmin: true, isGroupCallRaiseHandEnabled: true, me: getDefaultConversation({ diff --git a/ts/components/CallScreen.tsx b/ts/components/CallScreen.tsx index c1d6252f88a7..ebf99b720e6c 100644 --- a/ts/components/CallScreen.tsx +++ b/ts/components/CallScreen.tsx @@ -90,6 +90,7 @@ import { isGroupOrAdhocActiveCall } from '../util/isGroupOrAdhocCall'; import { assertDev } from '../util/assert'; import { emojiToData } from './emoji/lib'; import { CallingPendingParticipants } from './CallingPendingParticipants'; +import type { CallingImageDataCache } from './CallManager'; export type PropsType = { activeCall: ActiveCallType; @@ -100,6 +101,7 @@ export type PropsType = { groupMembers?: Array>; hangUpActiveCall: (reason: string) => void; i18n: LocalizerType; + imageDataCache: React.RefObject; isCallLinkAdmin: boolean; isGroupCallRaiseHandEnabled: boolean; me: ConversationType; @@ -191,6 +193,7 @@ export function CallScreen({ groupMembers, hangUpActiveCall, i18n, + imageDataCache, isCallLinkAdmin, isGroupCallRaiseHandEnabled, me, @@ -688,6 +691,7 @@ export function CallScreen({ void; hasLocalVideo: boolean; i18n: LocalizerType; + imageDataCache: React.RefObject; setGroupCallVideoRequest: ( _: Array, speakerHeight: number @@ -75,6 +77,7 @@ export function CallingPip({ getGroupCallVideoFrameSource, hangUpActiveCall, hasLocalVideo, + imageDataCache, i18n, setGroupCallVideoRequest, setLocalPreview, @@ -304,6 +307,7 @@ export function CallingPip({ VideoFrameSource; i18n: LocalizerType; + imageDataCache: React.RefObject; setGroupCallVideoRequest: ( _: Array, speakerHeight: number @@ -88,6 +90,7 @@ export type PropsType = { export function CallingPipRemoteVideo({ activeCall, getGroupCallVideoFrameSource, + imageDataCache, i18n, setGroupCallVideoRequest, setRendererCanvas, @@ -181,6 +184,7 @@ export function CallingPipRemoteVideo({ Buffer.alloc(FRAME_BUFFER_SIZE)), + getCallingImageDataCache: memoize(() => new Map()), getGroupCallVideoFrameSource: fakeGetGroupCallVideoFrameSource, + imageDataCache: React.createRef(), i18n, isCallReconnecting: false, onParticipantVisibilityChanged: action('onParticipantVisibilityChanged'), diff --git a/ts/components/GroupCallOverflowArea.tsx b/ts/components/GroupCallOverflowArea.tsx index 7eccc1ab65ce..eb28fca7583f 100644 --- a/ts/components/GroupCallOverflowArea.tsx +++ b/ts/components/GroupCallOverflowArea.tsx @@ -8,6 +8,7 @@ import type { VideoFrameSource } from '@signalapp/ringrtc'; import type { LocalizerType } from '../types/Util'; import type { GroupCallRemoteParticipantType } from '../types/Calling'; import { GroupCallRemoteParticipant } from './GroupCallRemoteParticipant'; +import type { CallingImageDataCache } from './CallManager'; const OVERFLOW_SCROLLED_TO_EDGE_THRESHOLD = 20; const OVERFLOW_SCROLL_BUTTON_RATIO = 0.75; @@ -19,6 +20,7 @@ export type PropsType = { getFrameBuffer: () => Buffer; getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; i18n: LocalizerType; + imageDataCache: React.RefObject; isCallReconnecting: boolean; onClickRaisedHand?: () => void; onParticipantVisibilityChanged: ( @@ -33,6 +35,7 @@ export type PropsType = { export function GroupCallOverflowArea({ getFrameBuffer, getGroupCallVideoFrameSource, + imageDataCache, i18n, isCallReconnecting, onClickRaisedHand, @@ -121,6 +124,7 @@ export function GroupCallOverflowArea({ key={remoteParticipant.demuxId} getFrameBuffer={getFrameBuffer} getGroupCallVideoFrameSource={getGroupCallVideoFrameSource} + imageDataCache={imageDataCache} i18n={i18n} audioLevel={remoteAudioLevels.get(remoteParticipant.demuxId) ?? 0} onClickRaisedHand={onClickRaisedHand} diff --git a/ts/components/GroupCallRemoteParticipant.stories.tsx b/ts/components/GroupCallRemoteParticipant.stories.tsx index f02846ea68be..384a1155904e 100644 --- a/ts/components/GroupCallRemoteParticipant.stories.tsx +++ b/ts/components/GroupCallRemoteParticipant.stories.tsx @@ -11,6 +11,7 @@ import { FRAME_BUFFER_SIZE } from '../calling/constants'; import { setupI18n } from '../util/setupI18n'; import { generateAci } from '../types/ServiceId'; import enMessages from '../../_locales/en/messages.json'; +import type { CallingImageDataCache } from './CallManager'; const i18n = setupI18n('en', enMessages); @@ -54,6 +55,7 @@ const createProps = ( getGroupCallVideoFrameSource: () => { return { receiveVideoFrame: () => undefined }; }, + imageDataCache: React.createRef(), i18n, audioLevel: 0, remoteParticipant: { diff --git a/ts/components/GroupCallRemoteParticipant.tsx b/ts/components/GroupCallRemoteParticipant.tsx index c0e6752aff19..a669968fd4f0 100644 --- a/ts/components/GroupCallRemoteParticipant.tsx +++ b/ts/components/GroupCallRemoteParticipant.tsx @@ -29,6 +29,8 @@ import { MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH } from '../calling/constants'; import { useValueAtFixedRate } from '../hooks/useValueAtFixedRate'; import { Theme } from '../util/theme'; import { isOlderThan } from '../util/timestamp'; +import type { CallingImageDataCache } from './CallManager'; +import { usePrevious } from '../hooks/usePrevious'; const MAX_TIME_TO_SHOW_STALE_VIDEO_FRAMES = 10000; const MAX_TIME_TO_SHOW_STALE_SCREENSHARE_FRAMES = 60000; @@ -38,6 +40,7 @@ type BasePropsType = { getFrameBuffer: () => Buffer; getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; i18n: LocalizerType; + imageDataCache: React.RefObject; isActiveSpeakerInSpeakerView: boolean; isCallReconnecting: boolean; onClickRaisedHand?: () => void; @@ -70,6 +73,7 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( const { getFrameBuffer, getGroupCallVideoFrameSource, + imageDataCache, i18n, onClickRaisedHand, onVisibilityChanged, @@ -101,9 +105,12 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( !props.isInPip ? props.audioLevel > 0 : false, SPEAKING_LINGER_MS ); + const previousSharingScreen = usePrevious(sharingScreen, sharingScreen); + const isImageDataCached = + sharingScreen && imageDataCache.current?.has(demuxId); const [hasReceivedVideoRecently, setHasReceivedVideoRecently] = - useState(false); + useState(isImageDataCached); const [isWide, setIsWide] = useState( videoAspectRatio ? videoAspectRatio >= 1 : true ); @@ -132,6 +139,12 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( onVisibilityChanged?.(demuxId, isVisible); }, [demuxId, isVisible, onVisibilityChanged]); + useEffect(() => { + if (sharingScreen !== previousSharingScreen) { + imageDataCache.current?.delete(demuxId); + } + }, [demuxId, imageDataCache, previousSharingScreen, sharingScreen]); + const wantsToShowVideo = hasRemoteVideo && !isBlocked && isVisible; const hasVideoToShow = wantsToShowVideo && hasReceivedVideoRecently; const showMissingMediaKeys = Boolean( @@ -173,46 +186,74 @@ export const GroupCallRemoteParticipant: React.FC = React.memo( // This frame buffer is shared by all participants, so it may contain pixel data // for other participants, or pixel data from a previous frame. That's why we // return early and use the `frameWidth` and `frameHeight`. + let frameWidth: number | undefined; + let frameHeight: number | undefined; + let imageData = imageDataRef.current; + const frameBuffer = getFrameBuffer(); const frameDimensions = videoFrameSource.receiveVideoFrame( frameBuffer, MAX_FRAME_WIDTH, MAX_FRAME_HEIGHT ); - if (!frameDimensions) { - return; + if (frameDimensions) { + [frameWidth, frameHeight] = frameDimensions; + + if ( + frameWidth < 2 || + frameHeight < 2 || + frameWidth > MAX_FRAME_WIDTH || + frameHeight > MAX_FRAME_HEIGHT + ) { + return; + } + + if ( + imageData?.width !== frameWidth || + imageData?.height !== frameHeight + ) { + imageData = new ImageData(frameWidth, frameHeight); + imageDataRef.current = imageData; + } + imageData.data.set( + frameBuffer.subarray(0, frameWidth * frameHeight * 4) + ); + + // Screen share is at a slow FPS so updates slowly if we PiP then restore. + // Cache the image data so we can quickly show the most recent frame. + if (sharingScreen) { + imageDataCache.current?.set(demuxId, imageData); + } + } else if (sharingScreen && !imageData) { + // Try to use the screenshare cache the first time we show + const cachedImageData = imageDataCache.current?.get(demuxId); + if (cachedImageData) { + frameWidth = cachedImageData.width; + frameHeight = cachedImageData.height; + imageDataRef.current = cachedImageData; + imageData = cachedImageData; + } } - const [frameWidth, frameHeight] = frameDimensions; - - if ( - frameWidth < 2 || - frameHeight < 2 || - frameWidth > MAX_FRAME_WIDTH || - frameHeight > MAX_FRAME_HEIGHT - ) { + if (!frameWidth || !frameHeight || !imageData) { return; } canvasEl.width = frameWidth; canvasEl.height = frameHeight; - - let imageData = imageDataRef.current; - if ( - imageData?.width !== frameWidth || - imageData?.height !== frameHeight - ) { - imageData = new ImageData(frameWidth, frameHeight); - imageDataRef.current = imageData; - } - imageData.data.set(frameBuffer.subarray(0, frameWidth * frameHeight * 4)); canvasContext.putImageData(imageData, 0, 0); - lastReceivedVideoAt.current = Date.now(); setHasReceivedVideoRecently(true); setIsWide(frameWidth > frameHeight); - }, [getFrameBuffer, videoFrameSource, sharingScreen, isCallReconnecting]); + }, [ + demuxId, + imageDataCache, + isCallReconnecting, + sharingScreen, + videoFrameSource, + getFrameBuffer, + ]); useEffect(() => { if (!hasRemoteVideo) { diff --git a/ts/components/GroupCallRemoteParticipants.tsx b/ts/components/GroupCallRemoteParticipants.tsx index 22d07ba93057..e2c2e338bcc6 100644 --- a/ts/components/GroupCallRemoteParticipants.tsx +++ b/ts/components/GroupCallRemoteParticipants.tsx @@ -27,6 +27,7 @@ import * as log from '../logging/log'; import { MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH } from '../calling/constants'; import { SizeObserver } from '../hooks/useSizeObserver'; import { strictAssert } from '../util/assert'; +import type { CallingImageDataCache } from './CallManager'; const SMALL_TILES_MIN_HEIGHT = 80; const LARGE_TILES_MIN_HEIGHT = 200; @@ -60,6 +61,7 @@ type PropsType = { callViewMode: CallViewMode; getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; i18n: LocalizerType; + imageDataCache: React.RefObject; isCallReconnecting: boolean; remoteParticipants: ReadonlyArray; setGroupCallVideoRequest: ( @@ -110,6 +112,7 @@ enum VideoRequestMode { export function GroupCallRemoteParticipants({ callViewMode, getGroupCallVideoFrameSource, + imageDataCache, i18n, isCallReconnecting, remoteParticipants, @@ -343,6 +346,7 @@ export function GroupCallRemoteParticipants({ (new Map());", + "reasonCategory": "usageTrusted", + "updated": "2024-05-06T20:18:59.647Z" } ]