Share group calling frame buffers to reduce memory usage

ts/calling/constants.ts

@@ -1,6 +1,9 @@
-// Copyright 2020 Signal Messenger, LLC
+// Copyright 2020-2021 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
 export const REQUESTED_VIDEO_WIDTH = 640;
 export const REQUESTED_VIDEO_HEIGHT = 480;
 export const REQUESTED_VIDEO_FRAMERATE = 30;
+
+export const MAX_FRAME_SIZE = 1920 * 1080;
+export const FRAME_BUFFER_SIZE = MAX_FRAME_SIZE * 4;

ts/calling/useGetCallingFrameBuffer.ts

@@ -0,0 +1,24 @@
+// Copyright 2021 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { useRef, useCallback } from 'react';
+import { FRAME_BUFFER_SIZE } from './constants';
+
+/**
+ * A hook that returns a function. This function returns a "singleton" `ArrayBuffer` to be
+ * used in call video rendering.
+ *
+ * This is most useful for group calls, where we can reuse the same frame buffer instead
+ * of allocating one per participant. Be careful when using this buffer elsewhere, as it
+ * is not cleaned up and may hold stale data.
+ */
+export function useGetCallingFrameBuffer(): () => ArrayBuffer {
+  const ref = useRef<ArrayBuffer | null>(null);
+
+  return useCallback(() => {
+    if (!ref.current) {
+      ref.current = new ArrayBuffer(FRAME_BUFFER_SIZE);
+    }
+    return ref.current;
+  }, []);
+}

ts/components/CallingPipRemoteVideo.tsx

@@ -1,4 +1,4 @@
-// Copyright 2020 Signal Messenger, LLC
+// Copyright 2020-2021 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import React, { useMemo, useEffect } from 'react';
@@ -16,6 +16,7 @@ import {
   VideoFrameSource,
 } from '../types/Calling';
 import { SetRendererCanvasType } from '../state/ducks/calling';
+import { useGetCallingFrameBuffer } from '../calling/useGetCallingFrameBuffer';
 import { usePageVisibility } from '../util/hooks';
 import { nonRenderedRemoteParticipant } from '../util/ringrtc/nonRenderedRemoteParticipant';
 
@@ -77,6 +78,8 @@ export const CallingPipRemoteVideo = ({
 }: PropsType): JSX.Element => {
   const { conversation } = activeCall;
 
+  const getGroupCallFrameBuffer = useGetCallingFrameBuffer();
+
   const isPageVisible = usePageVisibility();
 
   const activeGroupCallSpeaker:
@@ -155,6 +158,7 @@ export const CallingPipRemoteVideo = ({
     return (
       <div className="module-calling-pip__video--remote">
         <GroupCallRemoteParticipant
+          getFrameBuffer={getGroupCallFrameBuffer}
           getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
           i18n={i18n}
           isInPip

ts/components/GroupCallRemoteParticipant.stories.tsx

@@ -1,8 +1,8 @@
-// Copyright 2020 Signal Messenger, LLC
+// Copyright 2020-2021 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import * as React from 'react';
-import { noop } from 'lodash';
+import { memoize, noop } from 'lodash';
 import { storiesOf } from '@storybook/react';
 
 import {
@@ -10,6 +10,7 @@ import {
   PropsType,
 } from './GroupCallRemoteParticipant';
 import { getDefaultConversation } from '../test-both/helpers/getDefaultConversation';
+import { FRAME_BUFFER_SIZE } from '../calling/constants';
 import { setup as setupI18n } from '../../js/modules/i18n';
 import enMessages from '../../_locales/en/messages.json';
 
@@ -27,10 +28,13 @@ type OverridePropsType =
       width: number;
     };
 
+const getFrameBuffer = memoize(() => new ArrayBuffer(FRAME_BUFFER_SIZE));
+
 const createProps = (
   overrideProps: OverridePropsType,
   isBlocked?: boolean
 ): PropsType => ({
+  getFrameBuffer,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   getGroupCallVideoFrameSource: noop as any,
   i18n,

ts/components/GroupCallRemoteParticipant.tsx

@@ -1,4 +1,4 @@
-// Copyright 2020 Signal Messenger, LLC
+// Copyright 2020-2021 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import React, {
@@ -21,11 +21,10 @@ import { Avatar, AvatarSize } from './Avatar';
 import { ConfirmationModal } from './ConfirmationModal';
 import { Intl } from './Intl';
 import { ContactName } from './conversation/ContactName';
-
+import { MAX_FRAME_SIZE } from '../calling/constants';
-// The max size video frame we'll support (in RGBA)
-const FRAME_BUFFER_SIZE = 1920 * 1080 * 4;
 
 interface BasePropsType {
+  getFrameBuffer: () => ArrayBuffer;
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
   remoteParticipant: GroupCallRemoteParticipantType;
@@ -47,7 +46,7 @@ export type PropsType = BasePropsType & (InPipPropsType | NotInPipPropsType);
 
 export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
   props => {
-    const { getGroupCallVideoFrameSource, i18n } = props;
+    const { getFrameBuffer, getGroupCallVideoFrameSource, i18n } = props;
 
     const {
       avatarPath,
@@ -66,9 +65,6 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
 
     const remoteVideoRef = useRef<HTMLCanvasElement | null>(null);
     const canvasContextRef = useRef<CanvasRenderingContext2D | null>(null);
-    const frameBufferRef = useRef<ArrayBuffer>(
-      new ArrayBuffer(FRAME_BUFFER_SIZE)
-    );
 
     const videoFrameSource = useMemo(
       () => getGroupCallVideoFrameSource(demuxId),
@@ -86,15 +82,22 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
         return;
       }
 
-      const frameDimensions = videoFrameSource.receiveVideoFrame(
+      // This frame buffer is shared by all participants, so it may contain pixel data
-        frameBufferRef.current
+      //   for other participants, or pixel data from a previous frame. That's why we
-      );
+      //   return early and use the `frameWidth` and `frameHeight`.
+      const frameBuffer = getFrameBuffer();
+      const frameDimensions = videoFrameSource.receiveVideoFrame(frameBuffer);
       if (!frameDimensions) {
         return;
       }
 
       const [frameWidth, frameHeight] = frameDimensions;
-      if (frameWidth < 2 || frameHeight < 2) {
+
+      if (
+        frameWidth < 2 ||
+        frameHeight < 2 ||
+        frameWidth * frameHeight > MAX_FRAME_SIZE
+      ) {
         return;
       }
 
@@ -103,11 +106,7 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
 
       canvasContext.putImageData(
         new ImageData(
-          new Uint8ClampedArray(
+          new Uint8ClampedArray(frameBuffer, 0, frameWidth * frameHeight * 4),
-            frameBufferRef.current,
-            0,
-            frameWidth * frameHeight * 4
-          ),
           frameWidth,
           frameHeight
         ),
@@ -116,7 +115,7 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
       );
 
       setIsWide(frameWidth > frameHeight);
-    }, [videoFrameSource]);
+    }, [getFrameBuffer, videoFrameSource]);
 
     useEffect(() => {
       if (!hasRemoteVideo) {

ts/components/GroupCallRemoteParticipants.tsx

@@ -1,4 +1,4 @@
-// Copyright 2020 Signal Messenger, LLC
+// Copyright 2020-2021 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import React, { useState, useMemo, useEffect } from 'react';
@@ -10,6 +10,7 @@ import {
   GroupCallVideoRequest,
   VideoFrameSource,
 } from '../types/Calling';
+import { useGetCallingFrameBuffer } from '../calling/useGetCallingFrameBuffer';
 import { LocalizerType } from '../types/Util';
 import { usePageVisibility } from '../util/hooks';
 import { nonRenderedRemoteParticipant } from '../util/ringrtc/nonRenderedRemoteParticipant';
@@ -72,6 +73,7 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
     height: 0,
   });
   const isPageVisible = usePageVisibility();
+  const getFrameBuffer = useGetCallingFrameBuffer();
 
   // 1. Figure out the maximum number of possible rows that could fit on the screen.
   //
@@ -216,6 +218,7 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
         return (
           <GroupCallRemoteParticipant
             key={remoteParticipant.demuxId}
+            getFrameBuffer={getFrameBuffer}
             getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
             height={gridParticipantHeight}
             i18n={i18n}

ts/util/lint/exceptions.json

@@ -14325,6 +14325,15 @@
     "updated": "2018-09-17T20:50:40.689Z",
     "reasonDetail": "Hard-coded value"
   },
+  {
+    "rule": "React-useRef",
+    "path": "ts/calling/useGetCallingFrameBuffer.js",
+    "line": "    const ref = react_1.useRef(null);",
+    "lineNumber": 17,
+    "reasonCategory": "usageTrusted",
+    "updated": "2021-01-06T00:47:54.313Z",
+    "reasonDetail": "Needed to render remote video elements. Doesn't interact with the DOM."
+  },
   {
     "rule": "React-useRef",
     "path": "ts/components/AvatarPopup.js",
@@ -14518,7 +14527,7 @@
     "rule": "React-useRef",
     "path": "ts/components/GroupCallRemoteParticipant.js",
     "line": "    const remoteVideoRef = react_1.useRef(null);",
-    "lineNumber": 44,
+    "lineNumber": 43,
     "reasonCategory": "usageTrusted",
     "updated": "2020-11-11T21:56:04.179Z",
     "reasonDetail": "Needed to render the remote video element."
@@ -14527,20 +14536,11 @@
     "rule": "React-useRef",
     "path": "ts/components/GroupCallRemoteParticipant.js",
     "line": "    const canvasContextRef = react_1.useRef(null);",
-    "lineNumber": 45,
+    "lineNumber": 44,
     "reasonCategory": "usageTrusted",
     "updated": "2020-11-17T23:29:38.698Z",
     "reasonDetail": "Doesn't touch the DOM."
   },
-  {
-    "rule": "React-useRef",
-    "path": "ts/components/GroupCallRemoteParticipant.js",
-    "line": "    const frameBufferRef = react_1.useRef(new ArrayBuffer(FRAME_BUFFER_SIZE));",
-    "lineNumber": 46,
-    "reasonCategory": "usageTrusted",
-    "updated": "2020-11-17T16:24:25.480Z",
-    "reasonDetail": "Doesn't touch the DOM."
-  },
   {
     "rule": "jQuery-$(",
     "path": "ts/components/Intl.js",