Group call video rendering performance improvements

ts/components/GroupCallRemoteParticipant.tsx

@@ -38,134 +38,151 @@ interface NotInPipPropsType {
 
 type PropsType = BasePropsType & (InPipPropsType | NotInPipPropsType);
 
-export const GroupCallRemoteParticipant: React.FC<PropsType> = props => {
+export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
-  const {
+  props => {
-    demuxId,
+    const {
-    getGroupCallVideoFrameSource,
+      demuxId,
-    hasRemoteAudio,
+      getGroupCallVideoFrameSource,
-    hasRemoteVideo,
+      hasRemoteAudio,
-  } = props;
+      hasRemoteVideo,
+    } = props;
 
-  const [canvasStyles, setCanvasStyles] = useState<CSSProperties>({});
+    const [isWide, setIsWide] = useState(true);
 
-  const remoteVideoRef = useRef<HTMLCanvasElement | null>(null);
+    const remoteVideoRef = useRef<HTMLCanvasElement | null>(null);
-  const rafIdRef = useRef<number | null>(null);
+    const canvasContextRef = useRef<CanvasRenderingContext2D | null>(null);
-  const frameBufferRef = useRef<ArrayBuffer>(
+    const frameBufferRef = useRef<ArrayBuffer>(
-    new ArrayBuffer(FRAME_BUFFER_SIZE)
+      new ArrayBuffer(FRAME_BUFFER_SIZE)
-  );
-
-  const videoFrameSource = useMemo(
-    () => getGroupCallVideoFrameSource(demuxId),
-    [getGroupCallVideoFrameSource, demuxId]
-  );
-
-  const renderVideoFrame = useCallback(() => {
-    const canvasEl = remoteVideoRef.current;
-    if (!canvasEl) {
-      return;
-    }
-
-    const context = canvasEl.getContext('2d');
-    if (!context) {
-      return;
-    }
-
-    const frameDimensions = videoFrameSource.receiveVideoFrame(
-      frameBufferRef.current
     );
-    if (!frameDimensions) {
-      return;
-    }
 
-    const [frameWidth, frameHeight] = frameDimensions;
+    const videoFrameSource = useMemo(
-    canvasEl.width = frameWidth;
+      () => getGroupCallVideoFrameSource(demuxId),
-    canvasEl.height = frameHeight;
+      [getGroupCallVideoFrameSource, demuxId]
+    );
 
-    context.putImageData(
+    const renderVideoFrame = useCallback(() => {
-      new ImageData(
+      const canvasEl = remoteVideoRef.current;
-        new Uint8ClampedArray(
+      if (!canvasEl) {
-          frameBufferRef.current,
+        return;
-          0,
+      }
-          frameWidth * frameHeight * 4
+
+      const canvasContext = canvasContextRef.current;
+      if (!canvasContext) {
+        return;
+      }
+
+      const frameDimensions = videoFrameSource.receiveVideoFrame(
+        frameBufferRef.current
+      );
+      if (!frameDimensions) {
+        return;
+      }
+
+      const [frameWidth, frameHeight] = frameDimensions;
+      if (frameWidth < 2 || frameHeight < 2) {
+        return;
+      }
+
+      canvasEl.width = frameWidth;
+      canvasEl.height = frameHeight;
+
+      canvasContext.putImageData(
+        new ImageData(
+          new Uint8ClampedArray(
+            frameBufferRef.current,
+            0,
+            frameWidth * frameHeight * 4
+          ),
+          frameWidth,
+          frameHeight
         ),
-        frameWidth,
+        0,
-        frameHeight
+        0
-      ),
+      );
-      0,
+
-      0
+      setIsWide(frameWidth > frameHeight);
-    );
+    }, [videoFrameSource]);
+
+    useEffect(() => {
+      if (!hasRemoteVideo) {
+        return noop;
+      }
+
+      let rafId = requestAnimationFrame(tick);
+
+      function tick() {
+        renderVideoFrame();
+        rafId = requestAnimationFrame(tick);
+      }
+
+      return () => {
+        cancelAnimationFrame(rafId);
+      };
+    }, [hasRemoteVideo, renderVideoFrame, videoFrameSource]);
+
+    let canvasStyles: CSSProperties;
+    let containerStyles: CSSProperties;
 
     // If our `width` and `height` props don't match the canvas's aspect ratio, we want to
     //   fill the container. This can happen when RingRTC gives us an inaccurate
     //   `videoAspectRatio`, or if the container is an unexpected size.
-    if (frameWidth > frameHeight) {
+    if (isWide) {
-      setCanvasStyles({ width: '100%' });
+      canvasStyles = { width: '100%' };
     } else {
-      setCanvasStyles({ height: '100%' });
+      canvasStyles = { height: '100%' };
-    }
-  }, [videoFrameSource]);
-
-  useEffect(() => {
-    if (!hasRemoteVideo) {
-      return noop;
     }
 
-    const tick = () => {
+    // TypeScript isn't smart enough to know that `isInPip` by itself disambiguates the
-      renderVideoFrame();
+    //   types, so we have to use `props.isInPip` instead.
-      rafIdRef.current = requestAnimationFrame(tick);
+    // eslint-disable-next-line react/destructuring-assignment
-    };
+    if (props.isInPip) {
+      containerStyles = canvasStyles;
+    } else {
+      const { top, left, width, height } = props;
 
-    rafIdRef.current = requestAnimationFrame(tick);
+      containerStyles = {
+        height,
+        left,
+        position: 'absolute',
+        top,
+        width,
+      };
+    }
 
-    return () => {
+    return (
-      if (rafIdRef.current) {
+      <div
-        cancelAnimationFrame(rafIdRef.current);
+        className={classNames(
-        rafIdRef.current = null;
+          'module-ongoing-call__group-call-remote-participant',
-      }
+          {
-    };
+            'module-ongoing-call__group-call-remote-participant--audio-muted': !hasRemoteAudio,
-  }, [hasRemoteVideo, renderVideoFrame, videoFrameSource]);
+          }
-
+        )}
-  let containerStyles: CSSProperties;
+        style={containerStyles}
-
+      >
-  // TypeScript isn't smart enough to know that `isInPip` by itself disambiguates the
+        {hasRemoteVideo ? (
-  //   types, so we have to use `props.isInPip` instead.
+          <canvas
-  // eslint-disable-next-line react/destructuring-assignment
+            className="module-ongoing-call__group-call-remote-participant__remote-video"
-  if (props.isInPip) {
+            style={canvasStyles}
-    containerStyles = canvasStyles;
+            ref={canvasEl => {
-  } else {
+              remoteVideoRef.current = canvasEl;
-    const { top, left, width, height } = props;
+              if (canvasEl) {
-
+                canvasContextRef.current = canvasEl.getContext('2d', {
-    containerStyles = {
+                  alpha: false,
-      height,
+                  desynchronized: true,
-      left,
+                  storage: 'discardable',
-      position: 'absolute',
+                } as CanvasRenderingContext2DSettings);
-      top,
+              } else {
-      width,
+                canvasContextRef.current = null;
-    };
+              }
+            }}
+          />
+        ) : (
+          <CallBackgroundBlur>
+            {/* TODO: Improve the styling here. See DESKTOP-894. */}
+            <span />
+          </CallBackgroundBlur>
+        )}
+      </div>
+    );
   }
-
+);
-  return (
-    <div
-      className={classNames(
-        'module-ongoing-call__group-call-remote-participant',
-        {
-          'module-ongoing-call__group-call-remote-participant--audio-muted': !hasRemoteAudio,
-        }
-      )}
-      style={containerStyles}
-    >
-      {hasRemoteVideo ? (
-        <canvas
-          className="module-ongoing-call__group-call-remote-participant__remote-video"
-          style={canvasStyles}
-          ref={remoteVideoRef}
-        />
-      ) : (
-        <CallBackgroundBlur>
-          {/* TODO: Improve the styling here. See DESKTOP-894. */}
-          <span />
-        </CallBackgroundBlur>
-      )}
-    </div>
-  );
-};

ts/components/GroupCallRemoteParticipants.tsx

@@ -162,7 +162,9 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
   ]);
 
   // 4. Lay out this arrangement on the screen.
-  const gridParticipantHeight = gridArrangement.scalar * MIN_RENDERED_HEIGHT;
+  const gridParticipantHeight = Math.floor(
+    gridArrangement.scalar * MIN_RENDERED_HEIGHT
+  );
   const gridParticipantHeightWithMargin =
     gridParticipantHeight + PARTICIPANT_MARGIN;
   const gridTotalRowHeightWithMargin =
@@ -181,12 +183,15 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
       const totalRowWidth =
         totalRowWidthWithoutMargins +
         PARTICIPANT_MARGIN * (remoteParticipantsInRow.length - 1);
-      const leftOffset = (containerDimensions.width - totalRowWidth) / 2;
+      const leftOffset = Math.floor(
+        (containerDimensions.width - totalRowWidth) / 2
+      );
 
       let rowWidthSoFar = 0;
       return remoteParticipantsInRow.map(remoteParticipant => {
-        const renderedWidth =
+        const renderedWidth = Math.floor(
-          remoteParticipant.videoAspectRatio * gridParticipantHeight;
+          remoteParticipant.videoAspectRatio * gridParticipantHeight
+        );
         const left = rowWidthSoFar + leftOffset;
 
         rowWidthSoFar += renderedWidth + PARTICIPANT_MARGIN;

ts/util/lint/exceptions.json

@@ -14579,10 +14579,10 @@
   {
     "rule": "React-useRef",
     "path": "ts/components/GroupCallRemoteParticipant.js",
-    "line": "    const rafIdRef = react_1.useRef(null);",
+    "line": "    const canvasContextRef = react_1.useRef(null);",
     "lineNumber": 25,
     "reasonCategory": "usageTrusted",
-    "updated": "2020-11-17T16:24:25.480Z",
+    "updated": "2020-11-17T23:29:38.698Z",
     "reasonDetail": "Doesn't touch the DOM."
   },
   {