Group calls: Make renderVideoFrame generate less garbage

ts/calling/useGetCallingFrameBuffer.ts

@@ -12,12 +12,12 @@ import { FRAME_BUFFER_SIZE } from './constants';
  * of allocating one per participant. Be careful when using this buffer elsewhere, as it
  * is not cleaned up and may hold stale data.
  */
-export function useGetCallingFrameBuffer(): () => ArrayBuffer {
-  const ref = useRef<ArrayBuffer | null>(null);
+export function useGetCallingFrameBuffer(): () => Buffer {
+  const ref = useRef<Buffer | null>(null);
 
   return useCallback(() => {
     if (!ref.current) {
-      ref.current = new ArrayBuffer(FRAME_BUFFER_SIZE);
+      ref.current = Buffer.alloc(FRAME_BUFFER_SIZE);
     }
     return ref.current;
   }, []);

ts/components/GroupCallOverflowArea.stories.tsx

@@ -35,7 +35,7 @@ const allRemoteParticipants = times(MAX_PARTICIPANTS).map(index => ({
 const story = storiesOf('Components/GroupCallOverflowArea', module);
 
 const defaultProps = {
-  getFrameBuffer: memoize(() => new ArrayBuffer(FRAME_BUFFER_SIZE)),
+  getFrameBuffer: memoize(() => Buffer.alloc(FRAME_BUFFER_SIZE)),
   getGroupCallVideoFrameSource: fakeGetGroupCallVideoFrameSource,
   i18n,
   onParticipantVisibilityChanged: action('onParticipantVisibilityChanged'),

ts/components/GroupCallOverflowArea.tsx

@@ -16,7 +16,7 @@ const OVERFLOW_SCROLL_BUTTON_RATIO = 0.75;
 export const OVERFLOW_PARTICIPANT_WIDTH = 140;
 
 type PropsType = {
-  getFrameBuffer: () => ArrayBuffer;
+  getFrameBuffer: () => Buffer;
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
   onParticipantVisibilityChanged: (

ts/components/GroupCallRemoteParticipant.stories.tsx

@@ -26,7 +26,7 @@ type OverridePropsType =
       width: number;
     };
 
-const getFrameBuffer = memoize(() => new ArrayBuffer(FRAME_BUFFER_SIZE));
+const getFrameBuffer = memoize(() => Buffer.alloc(FRAME_BUFFER_SIZE));
 
 const createProps = (
   overrideProps: OverridePropsType,

ts/components/GroupCallRemoteParticipant.tsx

@@ -26,7 +26,7 @@ import { MAX_FRAME_SIZE } from '../calling/constants';
 const MAX_TIME_TO_SHOW_STALE_VIDEO_FRAMES = 5000;
 
 type BasePropsType = {
-  getFrameBuffer: () => ArrayBuffer;
+  getFrameBuffer: () => Buffer;
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
   onVisibilityChanged?: (demuxId: number, isVisible: boolean) => unknown;
@@ -93,6 +93,7 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
     const lastReceivedVideoAt = useRef(-Infinity);
     const remoteVideoRef = useRef<HTMLCanvasElement | null>(null);
     const canvasContextRef = useRef<CanvasRenderingContext2D | null>(null);
+    const imageDataRef = useRef<ImageData | null>(null);
 
     const [intersectionRef, intersectionObserverEntry] =
       useIntersectionObserver();
@@ -134,9 +135,7 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
       //   for other participants, or pixel data from a previous frame. That's why we
       //   return early and use the `frameWidth` and `frameHeight`.
       const frameBuffer = getFrameBuffer();
-      const frameDimensions = videoFrameSource.receiveVideoFrame(
-        Buffer.from(frameBuffer)
-      );
+      const frameDimensions = videoFrameSource.receiveVideoFrame(frameBuffer);
       if (!frameDimensions) {
         return;
       }
@@ -154,15 +153,16 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
       canvasEl.width = frameWidth;
       canvasEl.height = frameHeight;
 
-      canvasContext.putImageData(
-        new ImageData(
-          new Uint8ClampedArray(frameBuffer, 0, frameWidth * frameHeight * 4),
-          frameWidth,
-          frameHeight
-        ),
-        0,
-        0
-      );
+      let imageData = imageDataRef.current;
+      if (
+        imageData?.width !== frameWidth ||
+        imageData?.height !== frameHeight
+      ) {
+        imageData = new ImageData(frameWidth, frameHeight);
+        imageDataRef.current = imageData;
+      }
+      imageData.data.set(frameBuffer.subarray(0, frameWidth * frameHeight * 4));
+      canvasContext.putImageData(imageData, 0, 0);
 
       lastReceivedVideoAt.current = Date.now();
 

ts/util/lint/exceptions.json

@@ -6965,9 +6965,10 @@
   {
     "rule": "React-useRef",
     "path": "ts/calling/useGetCallingFrameBuffer.ts",
-    "line": "  const ref = useRef<ArrayBuffer | null>(null);",
+    "line": "  const ref = useRef<Buffer | null>(null);",
     "reasonCategory": "usageTrusted",
-    "updated": "2021-07-30T16:57:33.618Z"
+    "updated": "2021-12-10T23:24:03.829Z",
+    "reasonDetail": "Doesn't touch the DOM."
   },
   {
     "rule": "jQuery-load(",
@@ -7331,6 +7332,14 @@
     "reasonCategory": "usageTrusted",
     "updated": "2021-07-30T16:57:33.618Z"
   },
+  {
+    "rule": "React-useRef",
+    "path": "ts/components/GroupCallRemoteParticipant.tsx",
+    "line": "    const imageDataRef = useRef<ImageData | null>(null);",
+    "reasonCategory": "usageTrusted",
+    "updated": "2021-12-10T23:24:31.237Z",
+    "reasonDetail": "Doesn't touch the DOM."
+  },
   {
     "rule": "React-useRef",
     "path": "ts/components/Inbox.tsx",