Use global screen share cache for group calls

ts/components/CallManager.tsx

@@ -56,6 +56,7 @@ import { callLinkRootKeyToUrl } from '../util/callLinkRootKeyToUrl';
 import { ToastType } from '../types/Toast';
 import type { ShowToastAction } from '../state/ducks/toast';
 import { isSharingPhoneNumberWithEverybody } from '../util/phoneNumberSharingMode';
+import { usePrevious } from '../hooks/usePrevious';
 
 const GROUP_CALL_RING_DURATION = 60 * 1000;
 
@@ -77,6 +78,8 @@ export type GroupIncomingCall = Readonly<{
   remoteParticipants: Array<GroupCallParticipantInfoType>;
 }>;
 
+export type CallingImageDataCache = Map<number, ImageData>;
+
 export type PropsType = {
   activeCall?: ActiveCallType;
   availableCameras: Array<MediaDeviceInfo>;
@@ -228,6 +231,16 @@ function ActiveCallManager({
     pauseVoiceNotePlayer,
   ]);
 
+  // For caching screenshare frames which update slowly, between Pip and CallScreen.
+  const imageDataCache = React.useRef<CallingImageDataCache>(new Map());
+
+  const previousConversationId = usePrevious(conversation.id, conversation.id);
+  useEffect(() => {
+    if (conversation.id !== previousConversationId) {
+      imageDataCache.current.clear();
+    }
+  }, [conversation.id, previousConversationId]);
+
   const getGroupCallVideoFrameSourceForActiveCall = useCallback(
     (demuxId: number) => {
       return getGroupCallVideoFrameSource(conversation.id, demuxId);
@@ -313,6 +326,7 @@ function ActiveCallManager({
       <CallingPip
         activeCall={activeCall}
         getGroupCallVideoFrameSource={getGroupCallVideoFrameSourceForActiveCall}
+        imageDataCache={imageDataCache}
         hangUpActiveCall={hangUpActiveCall}
         hasLocalVideo={hasLocalVideo}
         i18n={i18n}
@@ -417,6 +431,7 @@ function ActiveCallManager({
         groupMembers={groupMembers}
         hangUpActiveCall={hangUpActiveCall}
         i18n={i18n}
+        imageDataCache={imageDataCache}
         isCallLinkAdmin={isCallLinkAdmin}
         isGroupCallRaiseHandEnabled={isGroupCallRaiseHandEnabled}
         me={me}

ts/components/CallScreen.stories.tsx

@@ -33,6 +33,7 @@ import {
 import { fakeGetGroupCallVideoFrameSource } from '../test-both/helpers/fakeGetGroupCallVideoFrameSource';
 import enMessages from '../../_locales/en/messages.json';
 import { CallingToastProvider, useCallingToasts } from './CallingToast';
+import type { CallingImageDataCache } from './CallManager';
 
 const MAX_PARTICIPANTS = 75;
 const LOCAL_DEMUX_ID = 1;
@@ -190,6 +191,7 @@ const createProps = (
   getPresentingSources: action('get-presenting-sources'),
   hangUpActiveCall: action('hang-up'),
   i18n,
+  imageDataCache: React.createRef<CallingImageDataCache>(),
   isCallLinkAdmin: true,
   isGroupCallRaiseHandEnabled: true,
   me: getDefaultConversation({

ts/components/CallScreen.tsx

@@ -90,6 +90,7 @@ import { isGroupOrAdhocActiveCall } from '../util/isGroupOrAdhocCall';
 import { assertDev } from '../util/assert';
 import { emojiToData } from './emoji/lib';
 import { CallingPendingParticipants } from './CallingPendingParticipants';
+import type { CallingImageDataCache } from './CallManager';
 
 export type PropsType = {
   activeCall: ActiveCallType;
@@ -100,6 +101,7 @@ export type PropsType = {
   groupMembers?: Array<Pick<ConversationType, 'id' | 'firstName' | 'title'>>;
   hangUpActiveCall: (reason: string) => void;
   i18n: LocalizerType;
+  imageDataCache: React.RefObject<CallingImageDataCache>;
   isCallLinkAdmin: boolean;
   isGroupCallRaiseHandEnabled: boolean;
   me: ConversationType;
@@ -191,6 +193,7 @@ export function CallScreen({
   groupMembers,
   hangUpActiveCall,
   i18n,
+  imageDataCache,
   isCallLinkAdmin,
   isGroupCallRaiseHandEnabled,
   me,
@@ -688,6 +691,7 @@ export function CallScreen({
         <GroupCallRemoteParticipants
           callViewMode={activeCall.viewMode}
           getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
+          imageDataCache={imageDataCache}
           i18n={i18n}
           remoteParticipants={activeCall.remoteParticipants}
           setGroupCallVideoRequest={setGroupCallVideoRequest}

ts/components/CallingPip.tsx

@@ -13,6 +13,7 @@ import type {
 } from '../state/ducks/calling';
 import { missingCaseError } from '../util/missingCaseError';
 import { useActivateSpeakerViewOnPresenting } from '../hooks/useActivateSpeakerViewOnPresenting';
+import type { CallingImageDataCache } from './CallManager';
 
 enum PositionMode {
   BeingDragged,
@@ -54,6 +55,7 @@ export type PropsType = {
   hangUpActiveCall: (reason: string) => void;
   hasLocalVideo: boolean;
   i18n: LocalizerType;
+  imageDataCache: React.RefObject<CallingImageDataCache>;
   setGroupCallVideoRequest: (
     _: Array<GroupCallVideoRequest>,
     speakerHeight: number
@@ -75,6 +77,7 @@ export function CallingPip({
   getGroupCallVideoFrameSource,
   hangUpActiveCall,
   hasLocalVideo,
+  imageDataCache,
   i18n,
   setGroupCallVideoRequest,
   setLocalPreview,
@@ -304,6 +307,7 @@ export function CallingPip({
       <CallingPipRemoteVideo
         activeCall={activeCall}
         getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
+        imageDataCache={imageDataCache}
         i18n={i18n}
         setRendererCanvas={setRendererCanvas}
         setGroupCallVideoRequest={setGroupCallVideoRequest}

ts/components/CallingPipRemoteVideo.tsx

@@ -25,6 +25,7 @@ import { nonRenderedRemoteParticipant } from '../util/ringrtc/nonRenderedRemoteP
 import { isReconnecting } from '../util/callingIsReconnecting';
 import { isGroupOrAdhocActiveCall } from '../util/isGroupOrAdhocCall';
 import { assertDev } from '../util/assert';
+import type { CallingImageDataCache } from './CallManager';
 
 // This value should be kept in sync with the hard-coded CSS height. It should also be
 //   less than `MAX_FRAME_HEIGHT`.
@@ -78,6 +79,7 @@ export type PropsType = {
   activeCall: ActiveCallType;
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
+  imageDataCache: React.RefObject<CallingImageDataCache>;
   setGroupCallVideoRequest: (
     _: Array<GroupCallVideoRequest>,
     speakerHeight: number
@@ -88,6 +90,7 @@ export type PropsType = {
 export function CallingPipRemoteVideo({
   activeCall,
   getGroupCallVideoFrameSource,
+  imageDataCache,
   i18n,
   setGroupCallVideoRequest,
   setRendererCanvas,
@@ -181,6 +184,7 @@ export function CallingPipRemoteVideo({
           <GroupCallRemoteParticipant
             getFrameBuffer={getGroupCallFrameBuffer}
             getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
+            imageDataCache={imageDataCache}
             i18n={i18n}
             isInPip
             remoteParticipant={activeGroupCallSpeaker}

ts/components/GroupCallOverflowArea.stories.tsx

@@ -13,6 +13,7 @@ import { fakeGetGroupCallVideoFrameSource } from '../test-both/helpers/fakeGetGr
 import { FRAME_BUFFER_SIZE } from '../calling/constants';
 import enMessages from '../../_locales/en/messages.json';
 import { generateAci } from '../types/ServiceId';
+import type { CallingImageDataCache } from './CallManager';
 
 const MAX_PARTICIPANTS = 32;
 
@@ -42,7 +43,9 @@ export default {
 
 const defaultProps = {
   getFrameBuffer: memoize(() => Buffer.alloc(FRAME_BUFFER_SIZE)),
+  getCallingImageDataCache: memoize(() => new Map()),
   getGroupCallVideoFrameSource: fakeGetGroupCallVideoFrameSource,
+  imageDataCache: React.createRef<CallingImageDataCache>(),
   i18n,
   isCallReconnecting: false,
   onParticipantVisibilityChanged: action('onParticipantVisibilityChanged'),

ts/components/GroupCallOverflowArea.tsx

@@ -8,6 +8,7 @@ import type { VideoFrameSource } from '@signalapp/ringrtc';
 import type { LocalizerType } from '../types/Util';
 import type { GroupCallRemoteParticipantType } from '../types/Calling';
 import { GroupCallRemoteParticipant } from './GroupCallRemoteParticipant';
+import type { CallingImageDataCache } from './CallManager';
 
 const OVERFLOW_SCROLLED_TO_EDGE_THRESHOLD = 20;
 const OVERFLOW_SCROLL_BUTTON_RATIO = 0.75;
@@ -19,6 +20,7 @@ export type PropsType = {
   getFrameBuffer: () => Buffer;
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
+  imageDataCache: React.RefObject<CallingImageDataCache>;
   isCallReconnecting: boolean;
   onClickRaisedHand?: () => void;
   onParticipantVisibilityChanged: (
@@ -33,6 +35,7 @@ export type PropsType = {
 export function GroupCallOverflowArea({
   getFrameBuffer,
   getGroupCallVideoFrameSource,
+  imageDataCache,
   i18n,
   isCallReconnecting,
   onClickRaisedHand,
@@ -121,6 +124,7 @@ export function GroupCallOverflowArea({
             key={remoteParticipant.demuxId}
             getFrameBuffer={getFrameBuffer}
             getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
+            imageDataCache={imageDataCache}
             i18n={i18n}
             audioLevel={remoteAudioLevels.get(remoteParticipant.demuxId) ?? 0}
             onClickRaisedHand={onClickRaisedHand}

ts/components/GroupCallRemoteParticipant.stories.tsx

@@ -11,6 +11,7 @@ import { FRAME_BUFFER_SIZE } from '../calling/constants';
 import { setupI18n } from '../util/setupI18n';
 import { generateAci } from '../types/ServiceId';
 import enMessages from '../../_locales/en/messages.json';
+import type { CallingImageDataCache } from './CallManager';
 
 const i18n = setupI18n('en', enMessages);
 
@@ -54,6 +55,7 @@ const createProps = (
   getGroupCallVideoFrameSource: () => {
     return { receiveVideoFrame: () => undefined };
   },
+  imageDataCache: React.createRef<CallingImageDataCache>(),
   i18n,
   audioLevel: 0,
   remoteParticipant: {

ts/components/GroupCallRemoteParticipant.tsx

@@ -29,6 +29,8 @@ import { MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH } from '../calling/constants';
 import { useValueAtFixedRate } from '../hooks/useValueAtFixedRate';
 import { Theme } from '../util/theme';
 import { isOlderThan } from '../util/timestamp';
+import type { CallingImageDataCache } from './CallManager';
+import { usePrevious } from '../hooks/usePrevious';
 
 const MAX_TIME_TO_SHOW_STALE_VIDEO_FRAMES = 10000;
 const MAX_TIME_TO_SHOW_STALE_SCREENSHARE_FRAMES = 60000;
@@ -38,6 +40,7 @@ type BasePropsType = {
   getFrameBuffer: () => Buffer;
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
+  imageDataCache: React.RefObject<CallingImageDataCache>;
   isActiveSpeakerInSpeakerView: boolean;
   isCallReconnecting: boolean;
   onClickRaisedHand?: () => void;
@@ -70,6 +73,7 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
     const {
       getFrameBuffer,
       getGroupCallVideoFrameSource,
+      imageDataCache,
       i18n,
       onClickRaisedHand,
       onVisibilityChanged,
@@ -101,9 +105,12 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
       !props.isInPip ? props.audioLevel > 0 : false,
       SPEAKING_LINGER_MS
     );
+    const previousSharingScreen = usePrevious(sharingScreen, sharingScreen);
 
+    const isImageDataCached =
+      sharingScreen && imageDataCache.current?.has(demuxId);
     const [hasReceivedVideoRecently, setHasReceivedVideoRecently] =
-      useState(false);
+      useState(isImageDataCached);
     const [isWide, setIsWide] = useState<boolean>(
       videoAspectRatio ? videoAspectRatio >= 1 : true
     );
@@ -132,6 +139,12 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
       onVisibilityChanged?.(demuxId, isVisible);
     }, [demuxId, isVisible, onVisibilityChanged]);
 
+    useEffect(() => {
+      if (sharingScreen !== previousSharingScreen) {
+        imageDataCache.current?.delete(demuxId);
+      }
+    }, [demuxId, imageDataCache, previousSharingScreen, sharingScreen]);
+
     const wantsToShowVideo = hasRemoteVideo && !isBlocked && isVisible;
     const hasVideoToShow = wantsToShowVideo && hasReceivedVideoRecently;
     const showMissingMediaKeys = Boolean(
@@ -173,46 +186,74 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
       // This frame buffer is shared by all participants, so it may contain pixel data
       //   for other participants, or pixel data from a previous frame. That's why we
       //   return early and use the `frameWidth` and `frameHeight`.
+      let frameWidth: number | undefined;
+      let frameHeight: number | undefined;
+      let imageData = imageDataRef.current;
+
       const frameBuffer = getFrameBuffer();
       const frameDimensions = videoFrameSource.receiveVideoFrame(
         frameBuffer,
         MAX_FRAME_WIDTH,
         MAX_FRAME_HEIGHT
       );
-      if (!frameDimensions) {
+      if (frameDimensions) {
-        return;
+        [frameWidth, frameHeight] = frameDimensions;
+
+        if (
+          frameWidth < 2 ||
+          frameHeight < 2 ||
+          frameWidth > MAX_FRAME_WIDTH ||
+          frameHeight > MAX_FRAME_HEIGHT
+        ) {
+          return;
+        }
+
+        if (
+          imageData?.width !== frameWidth ||
+          imageData?.height !== frameHeight
+        ) {
+          imageData = new ImageData(frameWidth, frameHeight);
+          imageDataRef.current = imageData;
+        }
+        imageData.data.set(
+          frameBuffer.subarray(0, frameWidth * frameHeight * 4)
+        );
+
+        // Screen share is at a slow FPS so updates slowly if we PiP then restore.
+        // Cache the image data so we can quickly show the most recent frame.
+        if (sharingScreen) {
+          imageDataCache.current?.set(demuxId, imageData);
+        }
+      } else if (sharingScreen && !imageData) {
+        // Try to use the screenshare cache the first time we show
+        const cachedImageData = imageDataCache.current?.get(demuxId);
+        if (cachedImageData) {
+          frameWidth = cachedImageData.width;
+          frameHeight = cachedImageData.height;
+          imageDataRef.current = cachedImageData;
+          imageData = cachedImageData;
+        }
       }
 
-      const [frameWidth, frameHeight] = frameDimensions;
+      if (!frameWidth || !frameHeight || !imageData) {
-
-      if (
-        frameWidth < 2 ||
-        frameHeight < 2 ||
-        frameWidth > MAX_FRAME_WIDTH ||
-        frameHeight > MAX_FRAME_HEIGHT
-      ) {
         return;
       }
 
       canvasEl.width = frameWidth;
       canvasEl.height = frameHeight;
-
-      let imageData = imageDataRef.current;
-      if (
-        imageData?.width !== frameWidth ||
-        imageData?.height !== frameHeight
-      ) {
-        imageData = new ImageData(frameWidth, frameHeight);
-        imageDataRef.current = imageData;
-      }
-      imageData.data.set(frameBuffer.subarray(0, frameWidth * frameHeight * 4));
       canvasContext.putImageData(imageData, 0, 0);
-
       lastReceivedVideoAt.current = Date.now();
 
       setHasReceivedVideoRecently(true);
       setIsWide(frameWidth > frameHeight);
-    }, [getFrameBuffer, videoFrameSource, sharingScreen, isCallReconnecting]);
+    }, [
+      demuxId,
+      imageDataCache,
+      isCallReconnecting,
+      sharingScreen,
+      videoFrameSource,
+      getFrameBuffer,
+    ]);
 
     useEffect(() => {
       if (!hasRemoteVideo) {

ts/components/GroupCallRemoteParticipants.tsx

@@ -27,6 +27,7 @@ import * as log from '../logging/log';
 import { MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH } from '../calling/constants';
 import { SizeObserver } from '../hooks/useSizeObserver';
 import { strictAssert } from '../util/assert';
+import type { CallingImageDataCache } from './CallManager';
 
 const SMALL_TILES_MIN_HEIGHT = 80;
 const LARGE_TILES_MIN_HEIGHT = 200;
@@ -60,6 +61,7 @@ type PropsType = {
   callViewMode: CallViewMode;
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
+  imageDataCache: React.RefObject<CallingImageDataCache>;
   isCallReconnecting: boolean;
   remoteParticipants: ReadonlyArray<GroupCallRemoteParticipantType>;
   setGroupCallVideoRequest: (
@@ -110,6 +112,7 @@ enum VideoRequestMode {
 export function GroupCallRemoteParticipants({
   callViewMode,
   getGroupCallVideoFrameSource,
+  imageDataCache,
   i18n,
   isCallReconnecting,
   remoteParticipants,
@@ -343,6 +346,7 @@ export function GroupCallRemoteParticipants({
           <GroupCallRemoteParticipant
             key={tile.demuxId}
             getFrameBuffer={getFrameBuffer}
+            imageDataCache={imageDataCache}
             getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
             onClickRaisedHand={onClickRaisedHand}
             height={gridParticipantHeight}
@@ -510,6 +514,7 @@ export function GroupCallRemoteParticipants({
         <GroupCallOverflowArea
           getFrameBuffer={getFrameBuffer}
           getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
+          imageDataCache={imageDataCache}
           i18n={i18n}
           isCallReconnecting={isCallReconnecting}
           onClickRaisedHand={onClickRaisedHand}

ts/util/lint/exceptions.json

@@ -3899,5 +3899,12 @@
     "line": "  message.innerHTML = window.i18n('icu:optimizingApplication');",
     "reasonCategory": "usageTrusted",
     "updated": "2021-09-17T21:02:59.414Z"
+  },
+  {
+    "rule": "React-useRef",
+    "path": "ts/components/CallManager.tsx",
+    "line": "  const imageDataCache = React.useRef<CallingImageDataCache>(new Map());",
+    "reasonCategory": "usageTrusted",
+    "updated": "2024-05-06T20:18:59.647Z"
   }
 ]