// Copyright 2020-2022 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only import React, { useCallback, useState, useMemo, useEffect } from 'react'; import Measure from 'react-measure'; import { takeWhile, clamp, chunk, maxBy, flatten, noop } from 'lodash'; import type { VideoFrameSource } from 'ringrtc'; import { GroupCallRemoteParticipant } from './GroupCallRemoteParticipant'; import { GroupCallOverflowArea, OVERFLOW_PARTICIPANT_WIDTH, } from './GroupCallOverflowArea'; import type { GroupCallRemoteParticipantType, GroupCallVideoRequest, } from '../types/Calling'; import { useGetCallingFrameBuffer } from '../calling/useGetCallingFrameBuffer'; import type { LocalizerType } from '../types/Util'; import { usePageVisibility } from '../hooks/usePageVisibility'; import { useDevicePixelRatio } from '../hooks/useDevicePixelRatio'; import { nonRenderedRemoteParticipant } from '../util/ringrtc/nonRenderedRemoteParticipant'; import { missingCaseError } from '../util/missingCaseError'; import { SECOND } from '../util/durations'; import { filter, join } from '../util/iterables'; import * as setUtil from '../util/setUtil'; import * as log from '../logging/log'; import { MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH } from '../calling/constants'; const MIN_RENDERED_HEIGHT = 180; const PARTICIPANT_MARGIN = 10; const TIME_TO_STOP_REQUESTING_VIDEO_WHEN_PAGE_INVISIBLE = 20 * SECOND; // We scale our video requests down for performance. This number is somewhat arbitrary. const VIDEO_REQUEST_SCALAR = 0.75; type Dimensions = { width: number; height: number; }; type GridArrangement = { rows: Array>; scalar: number; }; type PropsType = { getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource; i18n: LocalizerType; isInSpeakerView: boolean; remoteParticipants: ReadonlyArray; setGroupCallVideoRequest: ( _: Array, speakerHeight: number ) => void; remoteAudioLevels: Map; }; enum VideoRequestMode { Normal = 'Normal', LowResolution = 'LowResolution', NoVideo = 'NoVideo', } // This component lays out group call remote participants. It uses a custom layout // algorithm (in other words, nothing that the browser provides, like flexbox) in // order to animate the boxes as they move around, and to figure out the right fits. // // It's worth looking at the UI (or a design of it) to get an idea of how it works. Some // things to notice: // // * Participants are arranged in 0 or more rows. // * Each row is the same height, but each participant may have a different width. // * It's possible, on small screens with lots of participants, to have participants // removed from the grid. This is because participants have a minimum rendered height. // // There should be more specific comments throughout, but the high-level steps are: // // 1. Figure out the maximum number of possible rows that could fit on the screen; this is // `maxRowCount`. // 2. Split the participants into two groups: ones in the main grid and ones in the // overflow area. The grid should prioritize participants who have recently spoken. // 3. For each possible number of rows (starting at 0 and ending at `maxRowCount`), // distribute participants across the rows at the minimum height. Then find the // "scalar": how much can we scale these boxes up while still fitting them on the // screen? The biggest scalar wins as the "best arrangement". // 4. Lay out this arrangement on the screen. export function GroupCallRemoteParticipants({ getGroupCallVideoFrameSource, i18n, isInSpeakerView, remoteParticipants, setGroupCallVideoRequest, remoteAudioLevels, }: PropsType): JSX.Element { const [containerDimensions, setContainerDimensions] = useState({ width: 0, height: 0, }); const [gridDimensions, setGridDimensions] = useState({ width: 0, height: 0, }); const devicePixelRatio = useDevicePixelRatio(); const getFrameBuffer = useGetCallingFrameBuffer(); const { invisibleDemuxIds, onParticipantVisibilityChanged } = useInvisibleParticipants(remoteParticipants); // 1. Figure out the maximum number of possible rows that could fit on the screen. // // We choose the smaller of these two options: // // - The number of participants, which means there'd be one participant per row. // - The number of possible rows in the container, assuming all participants were // rendered at minimum height. Doesn't rely on the number of participants—it's some // simple division. // // Could be 0 if (a) there are no participants (b) the container's height is small. const maxRowCount = Math.min( remoteParticipants.length, Math.floor( containerDimensions.height / (MIN_RENDERED_HEIGHT + PARTICIPANT_MARGIN) ) ); // 2. Split participants into two groups: ones in the main grid and ones in the overflow // sidebar. // // We start by sorting by `presenting` first since presenters should be on the main grid // then we sort by `speakerTime` so that the most recent speakers are next in // line for the main grid. Then we split the list in two: one for the grid and one for // the overflow area. // // Once we've sorted participants into their respective groups, we sort them on // something stable (the `demuxId`, but we could choose something else) so that people // don't jump around within the group. // // These are primarily memoized for clarity, not performance. const sortedParticipants: Array = useMemo( () => remoteParticipants .concat() .sort( (a, b) => Number(b.presenting || 0) - Number(a.presenting || 0) || (b.speakerTime || -Infinity) - (a.speakerTime || -Infinity) ), [remoteParticipants] ); const gridParticipants: Array = useMemo(() => { if (!sortedParticipants.length) { return []; } const candidateParticipants = isInSpeakerView ? [sortedParticipants[0]] : sortedParticipants; // Imagine that we laid out all of the rows end-to-end. That's the maximum total // width. So if there were 5 rows and the container was 100px wide, then we can't // possibly fit more than 500px of participants. const maxTotalWidth = maxRowCount * containerDimensions.width; // We do the same thing for participants, "laying them out end-to-end" until they // exceed the maximum total width. let totalWidth = 0; return takeWhile(candidateParticipants, remoteParticipant => { totalWidth += remoteParticipant.videoAspectRatio * MIN_RENDERED_HEIGHT; return totalWidth < maxTotalWidth; }).sort(stableParticipantComparator); }, [ containerDimensions.width, isInSpeakerView, maxRowCount, sortedParticipants, ]); const overflowedParticipants: Array = useMemo( () => sortedParticipants .slice(gridParticipants.length) .sort(stableParticipantComparator), [sortedParticipants, gridParticipants.length] ); // 3. For each possible number of rows (starting at 0 and ending at `maxRowCount`), // distribute participants across the rows at the minimum height. Then find the // "scalar": how much can we scale these boxes up while still fitting them on the // screen? The biggest scalar wins as the "best arrangement". const gridArrangement: GridArrangement = useMemo(() => { let bestArrangement: GridArrangement = { scalar: -1, rows: [], }; if (!gridParticipants.length) { return bestArrangement; } for (let rowCount = 1; rowCount <= maxRowCount; rowCount += 1) { // We do something pretty naïve here and chunk the grid's participants into rows. // For example, if there were 12 grid participants and `rowCount === 3`, there // would be 4 participants per row. // // This naïve chunking is suboptimal in terms of absolute best fit, but it is much // faster and simpler than trying to do this perfectly. In practice, this works // fine in the UI from our testing. const numberOfParticipantsInRow = Math.ceil( gridParticipants.length / rowCount ); const rows = chunk(gridParticipants, numberOfParticipantsInRow); // We need to find the scalar for this arrangement. Imagine that we have these // participants at the minimum heights, and we want to scale everything up until // it's about to overflow. // // We don't want it to overflow horizontally or vertically, so we calculate a // "width scalar" and "height scalar" and choose the smaller of the two. (Choosing // the LARGER of the two could cause overflow.) const widestRow = maxBy(rows, totalRemoteParticipantWidthAtMinHeight); if (!widestRow) { log.error('Unable to find the widest row, which should be impossible'); continue; } const widthScalar = (gridDimensions.width - (widestRow.length + 1) * PARTICIPANT_MARGIN) / totalRemoteParticipantWidthAtMinHeight(widestRow); const heightScalar = (gridDimensions.height - (rowCount + 1) * PARTICIPANT_MARGIN) / (rowCount * MIN_RENDERED_HEIGHT); const scalar = Math.min(widthScalar, heightScalar); // If this scalar is the best one so far, we use that. if (scalar > bestArrangement.scalar) { bestArrangement = { scalar, rows }; } } return bestArrangement; }, [ gridParticipants, maxRowCount, gridDimensions.width, gridDimensions.height, ]); // 4. Lay out this arrangement on the screen. const gridParticipantHeight = Math.floor( gridArrangement.scalar * MIN_RENDERED_HEIGHT ); const gridParticipantHeightWithMargin = gridParticipantHeight + PARTICIPANT_MARGIN; const gridTotalRowHeightWithMargin = gridParticipantHeightWithMargin * gridArrangement.rows.length; const gridTopOffset = Math.floor( (gridDimensions.height - gridTotalRowHeightWithMargin) / 2 ); const rowElements: Array> = gridArrangement.rows.map( (remoteParticipantsInRow, index) => { const top = gridTopOffset + index * gridParticipantHeightWithMargin; const totalRowWidthWithoutMargins = totalRemoteParticipantWidthAtMinHeight(remoteParticipantsInRow) * gridArrangement.scalar; const totalRowWidth = totalRowWidthWithoutMargins + PARTICIPANT_MARGIN * (remoteParticipantsInRow.length - 1); const leftOffset = Math.floor((gridDimensions.width - totalRowWidth) / 2); let rowWidthSoFar = 0; return remoteParticipantsInRow.map(remoteParticipant => { const { demuxId, videoAspectRatio } = remoteParticipant; const audioLevel = remoteAudioLevels.get(demuxId) ?? 0; const renderedWidth = Math.floor( videoAspectRatio * gridParticipantHeight ); const left = rowWidthSoFar + leftOffset; rowWidthSoFar += renderedWidth + PARTICIPANT_MARGIN; return ( ); }); } ); const videoRequestMode = useVideoRequestMode(); useEffect(() => { log.info(`Group call now using ${videoRequestMode} video request mode`); }, [videoRequestMode]); useEffect(() => { let videoRequest: Array; switch (videoRequestMode) { case VideoRequestMode.Normal: videoRequest = [ ...gridParticipants.map(participant => { let scalar: number; if (participant.sharingScreen) { // We want best-resolution video if someone is sharing their screen. scalar = Math.max(devicePixelRatio, 1); } else { scalar = VIDEO_REQUEST_SCALAR; } return { demuxId: participant.demuxId, width: clamp( Math.floor( gridParticipantHeight * participant.videoAspectRatio * scalar ), 1, MAX_FRAME_WIDTH ), height: clamp( Math.floor(gridParticipantHeight * scalar), 1, MAX_FRAME_HEIGHT ), }; }), ...overflowedParticipants.map(participant => { if (invisibleDemuxIds.has(participant.demuxId)) { return nonRenderedRemoteParticipant(participant); } return { demuxId: participant.demuxId, width: clamp( Math.floor(OVERFLOW_PARTICIPANT_WIDTH * VIDEO_REQUEST_SCALAR), 1, MAX_FRAME_WIDTH ), height: clamp( Math.floor( (OVERFLOW_PARTICIPANT_WIDTH / participant.videoAspectRatio) * VIDEO_REQUEST_SCALAR ), 1, MAX_FRAME_HEIGHT ), }; }), ]; break; case VideoRequestMode.LowResolution: videoRequest = remoteParticipants.map(participant => participant.hasRemoteVideo ? { demuxId: participant.demuxId, width: 1, height: 1, } : nonRenderedRemoteParticipant(participant) ); break; case VideoRequestMode.NoVideo: videoRequest = remoteParticipants.map(nonRenderedRemoteParticipant); break; default: log.error(missingCaseError(videoRequestMode)); videoRequest = remoteParticipants.map(nonRenderedRemoteParticipant); break; } setGroupCallVideoRequest( videoRequest, clamp(gridParticipantHeight, 0, MAX_FRAME_HEIGHT) ); }, [ devicePixelRatio, gridParticipantHeight, gridParticipants, invisibleDemuxIds, overflowedParticipants, remoteParticipants, setGroupCallVideoRequest, videoRequestMode, ]); return ( { if (!bounds) { log.error('We should be measuring the bounds'); return; } setContainerDimensions(bounds); }} > {containerMeasure => (
{ if (!bounds) { log.error('We should be measuring the bounds'); return; } setGridDimensions(bounds); }} > {gridMeasure => (
{flatten(rowElements)}
)}
)}
); } // This function is only meant for use with `useInvisibleParticipants`. It helps avoid // returning new set instances when the underlying values are equal. function pickDifferentSet(a: Readonly>, b: Readonly>): Set { return a.size === b.size ? a : b; } function useInvisibleParticipants( remoteParticipants: ReadonlyArray ): Readonly<{ invisibleDemuxIds: Set; onParticipantVisibilityChanged: (demuxId: number, isVisible: boolean) => void; }> { const [invisibleDemuxIds, setInvisibleDemuxIds] = useState(new Set()); const onParticipantVisibilityChanged = useCallback( (demuxId: number, isVisible: boolean) => { setInvisibleDemuxIds(oldInvisibleDemuxIds => { const toggled = setUtil.toggle( oldInvisibleDemuxIds, demuxId, !isVisible ); return pickDifferentSet(oldInvisibleDemuxIds, toggled); }); }, [] ); useEffect(() => { log.info( `Invisible demux IDs changed to [${join(invisibleDemuxIds, ',')}]` ); }, [invisibleDemuxIds]); useEffect(() => { const remoteParticipantDemuxIds = new Set( remoteParticipants.map(r => r.demuxId) ); setInvisibleDemuxIds(oldInvisibleDemuxIds => { const staleIds = filter( oldInvisibleDemuxIds, id => !remoteParticipantDemuxIds.has(id) ); const withoutStaleIds = setUtil.remove(oldInvisibleDemuxIds, ...staleIds); return pickDifferentSet(oldInvisibleDemuxIds, withoutStaleIds); }); }, [remoteParticipants]); return { invisibleDemuxIds, onParticipantVisibilityChanged, }; } function useVideoRequestMode(): VideoRequestMode { const isPageVisible = usePageVisibility(); const [result, setResult] = useState( isPageVisible ? VideoRequestMode.Normal : VideoRequestMode.LowResolution ); useEffect(() => { if (isPageVisible) { setResult(VideoRequestMode.Normal); return noop; } setResult(VideoRequestMode.LowResolution); const timeout = setTimeout(() => { setResult(VideoRequestMode.NoVideo); }, TIME_TO_STOP_REQUESTING_VIDEO_WHEN_PAGE_INVISIBLE); return () => { clearTimeout(timeout); }; }, [isPageVisible]); return result; } function totalRemoteParticipantWidthAtMinHeight( remoteParticipants: ReadonlyArray ): number { return remoteParticipants.reduce( (result, { videoAspectRatio }) => result + videoAspectRatio * MIN_RENDERED_HEIGHT, 0 ); } function stableParticipantComparator( a: Readonly<{ demuxId: number }>, b: Readonly<{ demuxId: number }> ): number { return a.demuxId - b.demuxId; }