Group calling: tell RingRTC about our rendered resolutions for perf

stylesheets/_modules.scss

@@ -6552,7 +6552,7 @@ button.module-image__border-overlay:focus {
       background-color: $color-gray-95;
       border-radius: 4px 4px 0 0;
       display: flex;
-      height: 120px;
+      height: 120px; // This height should be kept in sync with <CallingPipRemoteVideo>'s hard-coded height.
       justify-content: center;
       overflow: hidden;
       position: relative;

ts/components/CallManager.stories.tsx

@@ -73,6 +73,7 @@ const createProps = (storyProps: Partial<PropsType> = {}): PropsType => ({
     title: text('Caller Title', 'Morty Smith'),
   },
   renderDeviceSelection: () => <div />,
+  setGroupCallVideoRequest: action('set-group-call-video-request'),
   setLocalAudio: action('set-local-audio'),
   setLocalPreview: action('set-local-preview'),
   setLocalVideo: action('set-local-video'),

ts/components/CallManager.tsx

@@ -13,6 +13,7 @@ import {
   CallMode,
   CallState,
   GroupCallJoinState,
+  GroupCallVideoRequest,
   VideoFrameSource,
 } from '../types/Calling';
 import { ConversationType } from '../state/ducks/conversations';
@@ -23,6 +24,7 @@ import {
   DeclineCallType,
   DirectCallStateType,
   HangUpType,
+  SetGroupCallVideoRequestType,
   SetLocalAudioType,
   SetLocalPreviewType,
   SetLocalVideoType,
@@ -60,6 +62,7 @@ export interface PropsType {
     profileName?: string;
     title: string;
   };
+  setGroupCallVideoRequest: (_: SetGroupCallVideoRequestType) => void;
   setLocalAudio: (_: SetLocalAudioType) => void;
   setLocalVideo: (_: SetLocalVideoType) => void;
   setLocalPreview: (_: SetLocalPreviewType) => void;
@@ -83,6 +86,7 @@ const ActiveCallManager: React.FC<ActiveCallManagerPropsType> = ({
   getGroupCallVideoFrameSource,
   me,
   renderDeviceSelection,
+  setGroupCallVideoRequest,
   setLocalAudio,
   setLocalPreview,
   setLocalVideo,
@@ -129,6 +133,16 @@ const ActiveCallManager: React.FC<ActiveCallManagerPropsType> = ({
     [getGroupCallVideoFrameSource, conversation.id]
   );
 
+  const setGroupCallVideoRequestForConversation = useCallback(
+    (resolutions: Array<GroupCallVideoRequest>) => {
+      setGroupCallVideoRequest({
+        conversationId: conversation.id,
+        resolutions,
+      });
+    },
+    [setGroupCallVideoRequest, conversation.id]
+  );
+
   let showCallLobby: boolean;
 
   switch (call.callMode) {
@@ -205,6 +219,7 @@ const ActiveCallManager: React.FC<ActiveCallManagerPropsType> = ({
         hangUp={hangUp}
         hasLocalVideo={hasLocalVideo}
         i18n={i18n}
+        setGroupCallVideoRequest={setGroupCallVideoRequestForConversation}
         setLocalPreview={setLocalPreview}
         setRendererCanvas={setRendererCanvas}
         togglePip={togglePip}
@@ -223,6 +238,7 @@ const ActiveCallManager: React.FC<ActiveCallManagerPropsType> = ({
         i18n={i18n}
         joinedAt={joinedAt}
         me={me}
+        setGroupCallVideoRequest={setGroupCallVideoRequestForConversation}
         setLocalPreview={setLocalPreview}
         setRendererCanvas={setRendererCanvas}
         setLocalAudio={setLocalAudio}

ts/components/CallScreen.stories.tsx

@@ -114,6 +114,7 @@ const createProps = (
     profileName: 'Morty Smith',
     title: 'Morty Smith',
   },
+  setGroupCallVideoRequest: action('set-group-call-video-request'),
   setLocalAudio: action('set-local-audio'),
   setLocalPreview: action('set-local-preview'),
   setLocalVideo: action('set-local-video'),

ts/components/CallScreen.tsx

@@ -20,6 +20,7 @@ import {
   CallMode,
   CallState,
   GroupCallConnectionState,
+  GroupCallVideoRequest,
   VideoFrameSource,
 } from '../types/Calling';
 import { ColorType } from '../types/Colors';
@@ -45,6 +46,7 @@ export type PropsType = {
     profileName?: string;
     title: string;
   };
+  setGroupCallVideoRequest: (_: Array<GroupCallVideoRequest>) => void;
   setLocalAudio: (_: SetLocalAudioType) => void;
   setLocalVideo: (_: SetLocalVideoType) => void;
   setLocalPreview: (_: SetLocalPreviewType) => void;
@@ -64,6 +66,7 @@ export const CallScreen: React.FC<PropsType> = ({
   i18n,
   joinedAt,
   me,
+  setGroupCallVideoRequest,
   setLocalAudio,
   setLocalVideo,
   setLocalPreview,
@@ -187,6 +190,7 @@ export const CallScreen: React.FC<PropsType> = ({
           getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
           i18n={i18n}
           remoteParticipants={groupCallParticipants}
+          setGroupCallVideoRequest={setGroupCallVideoRequest}
         />
       );
       break;

ts/components/CallingPip.stories.tsx

@@ -68,6 +68,7 @@ const createProps = (
   hangUp: action('hang-up'),
   hasLocalVideo: boolean('hasLocalVideo', overrideProps.hasLocalVideo || false),
   i18n,
+  setGroupCallVideoRequest: action('set-group-call-video-request'),
   setLocalPreview: action('set-local-preview'),
   setRendererCanvas: action('set-renderer-canvas'),
   togglePip: action('toggle-pip'),

ts/components/CallingPip.tsx

@@ -5,7 +5,7 @@ import React from 'react';
 import { minBy, debounce, noop } from 'lodash';
 import { CallingPipRemoteVideo } from './CallingPipRemoteVideo';
 import { LocalizerType } from '../types/Util';
-import { VideoFrameSource } from '../types/Calling';
+import { GroupCallVideoRequest, VideoFrameSource } from '../types/Calling';
 import {
   ActiveCallType,
   HangUpType,
@@ -54,6 +54,7 @@ export type PropsType = {
   hangUp: (_: HangUpType) => void;
   hasLocalVideo: boolean;
   i18n: LocalizerType;
+  setGroupCallVideoRequest: (_: Array<GroupCallVideoRequest>) => void;
   setLocalPreview: (_: SetLocalPreviewType) => void;
   setRendererCanvas: (_: SetRendererCanvasType) => void;
   togglePip: () => void;
@@ -70,6 +71,7 @@ export const CallingPip = ({
   hangUp,
   hasLocalVideo,
   i18n,
+  setGroupCallVideoRequest,
   setLocalPreview,
   setRendererCanvas,
   togglePip,
@@ -269,6 +271,7 @@ export const CallingPip = ({
         getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
         i18n={i18n}
         setRendererCanvas={setRendererCanvas}
+        setGroupCallVideoRequest={setGroupCallVideoRequest}
       />
       {hasLocalVideo ? (
         <video

ts/components/CallingPipRemoteVideo.tsx

@@ -1,7 +1,7 @@
 // Copyright 2020 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
-import React, { useMemo } from 'react';
+import React, { useMemo, useEffect } from 'react';
 import { maxBy } from 'lodash';
 import { Avatar } from './Avatar';
 import { CallBackgroundBlur } from './CallBackgroundBlur';
@@ -11,9 +11,15 @@ import { LocalizerType } from '../types/Util';
 import {
   CallMode,
   GroupCallRemoteParticipantType,
+  GroupCallVideoRequest,
   VideoFrameSource,
 } from '../types/Calling';
 import { ActiveCallType, SetRendererCanvasType } from '../state/ducks/calling';
+import { usePageVisibility } from '../util/hooks';
+import { nonRenderedRemoteParticipant } from '../util/ringrtc/nonRenderedRemoteParticipant';
+
+// This value should be kept in sync with the hard-coded CSS height.
+const PIP_VIDEO_HEIGHT_PX = 120;
 
 const NoVideo = ({
   activeCall,
@@ -57,6 +63,7 @@ export interface PropsType {
   activeCall: ActiveCallType;
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
+  setGroupCallVideoRequest: (_: Array<GroupCallVideoRequest>) => void;
   setRendererCanvas: (_: SetRendererCanvasType) => void;
 }
 
@@ -64,10 +71,13 @@ export const CallingPipRemoteVideo = ({
   activeCall,
   getGroupCallVideoFrameSource,
   i18n,
+  setGroupCallVideoRequest,
   setRendererCanvas,
 }: PropsType): JSX.Element => {
   const { call, conversation, groupCallParticipants } = activeCall;
 
+  const isPageVisible = usePageVisibility();
+
   const activeGroupCallSpeaker:
     | undefined
     | GroupCallRemoteParticipantType = useMemo(() => {
@@ -81,6 +91,42 @@ export const CallingPipRemoteVideo = ({
     );
   }, [call.callMode, groupCallParticipants]);
 
+  useEffect(() => {
+    if (call.callMode !== CallMode.Group) {
+      return;
+    }
+
+    if (isPageVisible) {
+      setGroupCallVideoRequest(
+        groupCallParticipants.map(participant => {
+          const isVisible =
+            participant === activeGroupCallSpeaker &&
+            participant.hasRemoteVideo;
+          if (isVisible) {
+            return {
+              demuxId: participant.demuxId,
+              width: Math.floor(
+                PIP_VIDEO_HEIGHT_PX * participant.videoAspectRatio
+              ),
+              height: PIP_VIDEO_HEIGHT_PX,
+            };
+          }
+          return nonRenderedRemoteParticipant(participant);
+        })
+      );
+    } else {
+      setGroupCallVideoRequest(
+        groupCallParticipants.map(nonRenderedRemoteParticipant)
+      );
+    }
+  }, [
+    call.callMode,
+    groupCallParticipants,
+    activeGroupCallSpeaker,
+    isPageVisible,
+    setGroupCallVideoRequest,
+  ]);
+
   if (call.callMode === CallMode.Direct) {
     if (!call.hasRemoteVideo) {
       return <NoVideo activeCall={activeCall} i18n={i18n} />;

ts/components/GroupCallRemoteParticipants.tsx

@@ -1,19 +1,25 @@
 // Copyright 2020 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
-import React, { useState, useMemo } from 'react';
+import React, { useState, useMemo, useEffect } from 'react';
 import Measure from 'react-measure';
 import { takeWhile, chunk, maxBy, flatten } from 'lodash';
 import { GroupCallRemoteParticipant } from './GroupCallRemoteParticipant';
 import {
   GroupCallRemoteParticipantType,
+  GroupCallVideoRequest,
   VideoFrameSource,
 } from '../types/Calling';
 import { LocalizerType } from '../types/Util';
+import { usePageVisibility } from '../util/hooks';
+import { nonRenderedRemoteParticipant } from '../util/ringrtc/nonRenderedRemoteParticipant';
 
 const MIN_RENDERED_HEIGHT = 10;
 const PARTICIPANT_MARGIN = 10;
 
+// We scale our video requests down for performance. This number is somewhat arbitrary.
+const VIDEO_REQUEST_SCALAR = 0.75;
+
 interface Dimensions {
   width: number;
   height: number;
@@ -28,6 +34,7 @@ interface PropsType {
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
   remoteParticipants: ReadonlyArray<GroupCallRemoteParticipantType>;
+  setGroupCallVideoRequest: (_: Array<GroupCallVideoRequest>) => void;
 }
 
 // This component lays out group call remote participants. It uses a custom layout
@@ -58,11 +65,13 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
   getGroupCallVideoFrameSource,
   i18n,
   remoteParticipants,
+  setGroupCallVideoRequest,
 }) => {
   const [containerDimensions, setContainerDimensions] = useState<Dimensions>({
     width: 0,
     height: 0,
   });
+  const isPageVisible = usePageVisibility();
 
   // 1. Figure out the maximum number of possible rows that could fit on the screen.
   //
@@ -101,6 +110,9 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
       return totalWidth < maxTotalWidth;
     });
   }, [maxRowCount, containerDimensions.width, remoteParticipants]);
+  const overflowedParticipants: Array<GroupCallRemoteParticipantType> = remoteParticipants.slice(
+    visibleParticipants.length
+  );
 
   // 3. For each possible number of rows (starting at 0 and ending at `maxRowCount`),
   //   distribute participants across the rows at the minimum height. Then find the
@@ -216,7 +228,39 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
       });
     }
   );
-  const remoteParticipantElements = flatten(rowElements);
+
+  useEffect(() => {
+    if (isPageVisible) {
+      setGroupCallVideoRequest([
+        ...visibleParticipants.map(participant => {
+          if (participant.hasRemoteVideo) {
+            return {
+              demuxId: participant.demuxId,
+              width: Math.floor(
+                gridParticipantHeight *
+                  participant.videoAspectRatio *
+                  VIDEO_REQUEST_SCALAR
+              ),
+              height: Math.floor(gridParticipantHeight * VIDEO_REQUEST_SCALAR),
+            };
+          }
+          return nonRenderedRemoteParticipant(participant);
+        }),
+        ...overflowedParticipants.map(nonRenderedRemoteParticipant),
+      ]);
+    } else {
+      setGroupCallVideoRequest(
+        remoteParticipants.map(nonRenderedRemoteParticipant)
+      );
+    }
+  }, [
+    gridParticipantHeight,
+    isPageVisible,
+    overflowedParticipants,
+    remoteParticipants,
+    setGroupCallVideoRequest,
+    visibleParticipants,
+  ]);
 
   return (
     <Measure
@@ -231,7 +275,7 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
     >
       {({ measureRef }) => (
         <div className="module-ongoing-call__grid" ref={measureRef}>
-          {remoteParticipantElements}
+          {flatten(rowElements)}
         </div>
       )}
     </Measure>

ts/services/calling.ts

@@ -27,6 +27,7 @@ import {
   RingRTC,
   UserId,
   VideoFrameSource,
+  VideoRequest,
 } from 'ringrtc';
 import { uniqBy, noop } from 'lodash';
 
@@ -526,6 +527,13 @@ export class CallingClass {
     return this.getDirectCall(conversationId)?.callId;
   }
 
+  public setGroupCallVideoRequest(
+    conversationId: string,
+    resolutions: Array<VideoRequest>
+  ): void {
+    this.getGroupCall(conversationId)?.requestVideo(resolutions);
+  }
+
   // See the comment in types/Calling.ts to explain why we have to do this conversion.
   private convertRingRtcConnectionState(
     connectionState: ConnectionState

ts/state/ducks/calling.ts

@@ -19,6 +19,7 @@ import {
   GroupCallJoinState,
   GroupCallPeekedParticipantType,
   GroupCallRemoteParticipantType,
+  GroupCallVideoRequest,
   MediaDeviceSettings,
 } from '../../types/Calling';
 import { callingTones } from '../../util/callingTones';
@@ -168,6 +169,11 @@ export type SetLocalVideoType = {
   enabled: boolean;
 };
 
+export type SetGroupCallVideoRequestType = {
+  conversationId: string;
+  resolutions: Array<GroupCallVideoRequest>;
+};
+
 export type ShowCallLobbyType =
   | {
       callMode: CallMode.Direct;
@@ -684,6 +690,22 @@ function setLocalVideo(
   };
 }
 
+function setGroupCallVideoRequest(
+  payload: SetGroupCallVideoRequestType
+): ThunkAction<void, RootStateType, unknown, never> {
+  return () => {
+    calling.setGroupCallVideoRequest(
+      payload.conversationId,
+      payload.resolutions.map(resolution => ({
+        ...resolution,
+        // The `framerate` property in RingRTC has to be set, even if it's set to
+        //   `undefined`.
+        framerate: undefined,
+      }))
+    );
+  };
+}
+
 function showCallLobby(payload: ShowCallLobbyType): CallLobbyActionType {
   return {
     type: SHOW_CALL_LOBBY,
@@ -758,6 +780,7 @@ export const actions = {
   setRendererCanvas,
   setLocalAudio,
   setLocalVideo,
+  setGroupCallVideoRequest,
   showCallLobby,
   startCall,
   toggleParticipants,

ts/test/util/ringrtc/nonRenderedRemoteParticipant_test.ts

@@ -0,0 +1,16 @@
+// Copyright 2020 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { assert } from 'chai';
+
+import { nonRenderedRemoteParticipant } from '../../../util/ringrtc/nonRenderedRemoteParticipant';
+
+describe('nonRenderedRemoteParticipant', () => {
+  it('returns a video request object a width and height of 0', () => {
+    assert.deepEqual(nonRenderedRemoteParticipant({ demuxId: 123 }), {
+      demuxId: 123,
+      width: 0,
+      height: 0,
+    });
+  });
+});

ts/types/Calling.ts

@@ -84,6 +84,13 @@ export interface GroupCallRemoteParticipantType {
   videoAspectRatio: number;
 }
 
+// Similar to RingRTC's `VideoRequest` but without the `framerate` property.
+export interface GroupCallVideoRequest {
+  demuxId: number;
+  width: number;
+  height: number;
+}
+
 // Should match RingRTC's VideoFrameSource
 export interface VideoFrameSource {
   receiveVideoFrame(buffer: ArrayBuffer): [number, number] | undefined;

ts/util/hooks.ts

@@ -44,3 +44,25 @@ export const useBoundActions = <T extends ActionCreatorsMapObject>(
     return bindActionCreators(actions, dispatch);
   }, [actions, dispatch]);
 };
+
+export const usePageVisibility = (): boolean => {
+  const [result, setResult] = React.useState(!document.hidden);
+
+  React.useEffect(() => {
+    const onVisibilityChange = () => {
+      setResult(!document.hidden);
+    };
+
+    document.addEventListener('visibilitychange', onVisibilityChange, false);
+
+    return () => {
+      document.removeEventListener(
+        'visibilitychange',
+        onVisibilityChange,
+        false
+      );
+    };
+  }, []);
+
+  return result;
+};

ts/util/lint/exceptions.json

@@ -14427,7 +14427,7 @@
     "rule": "React-useRef",
     "path": "ts/components/CallingPip.tsx",
     "line": "  const localVideoRef = React.useRef(null);",
-    "lineNumber": 78,
+    "lineNumber": 80,
     "reasonCategory": "usageTrusted",
     "updated": "2020-10-26T19:12:24.410Z",
     "reasonDetail": "Used to get the local video element for rendering."

ts/util/ringrtc/nonRenderedRemoteParticipant.ts

@@ -0,0 +1,12 @@
+// Copyright 2020 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import type { GroupCallVideoRequest } from '../../types/Calling';
+
+export const nonRenderedRemoteParticipant = ({
+  demuxId,
+}: Readonly<{ demuxId: number }>): GroupCallVideoRequest => ({
+  demuxId,
+  width: 0,
+  height: 0,
+});