mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use global screen share cache for group calls

Browse source
This commit is contained in:
ayumi-signal 2024-05-07 11:21:57 -07:00 committed by GitHub
parent 7cd07eb7b4
commit a3b9e97b82
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 114 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
ts/components/CallManager.tsx
View file

@ -56,6 +56,7 @@ import { callLinkRootKeyToUrl } from '../util/callLinkRootKeyToUrl';
import { ToastType } from '../types/Toast';
import type { ShowToastAction } from '../state/ducks/toast';
import { isSharingPhoneNumberWithEverybody } from '../util/phoneNumberSharingMode';
import { usePrevious } from '../hooks/usePrevious';
const GROUP_CALL_RING_DURATION = 60 * 1000;
@ -77,6 +78,8 @@ export type GroupIncomingCall = Readonly<{
remoteParticipants: Array<GroupCallParticipantInfoType>;
}>;
export type CallingImageDataCache = Map<number, ImageData>;
export type PropsType = {
activeCall?: ActiveCallType;
availableCameras: Array<MediaDeviceInfo>;
@ -228,6 +231,16 @@ function ActiveCallManager({
pauseVoiceNotePlayer,
]);
// For caching screenshare frames which update slowly, between Pip and CallScreen.
const imageDataCache = React.useRef<CallingImageDataCache>(new Map());
const previousConversationId = usePrevious(conversation.id, conversation.id);
useEffect(() => {
if (conversation.id !== previousConversationId) {
imageDataCache.current.clear();
}
}, [conversation.id, previousConversationId]);
const getGroupCallVideoFrameSourceForActiveCall = useCallback(
(demuxId: number) => {
return getGroupCallVideoFrameSource(conversation.id, demuxId);
@ -313,6 +326,7 @@ function ActiveCallManager({
<CallingPip
activeCall={activeCall}
getGroupCallVideoFrameSource={getGroupCallVideoFrameSourceForActiveCall}
imageDataCache={imageDataCache}
hangUpActiveCall={hangUpActiveCall}
hasLocalVideo={hasLocalVideo}
i18n={i18n}
@ -417,6 +431,7 @@ function ActiveCallManager({
groupMembers={groupMembers}
hangUpActiveCall={hangUpActiveCall}
i18n={i18n}
imageDataCache={imageDataCache}
isCallLinkAdmin={isCallLinkAdmin}
isGroupCallRaiseHandEnabled={isGroupCallRaiseHandEnabled}
me={me}

2
ts/components/CallScreen.stories.tsx
View file

@ -33,6 +33,7 @@ import {
import { fakeGetGroupCallVideoFrameSource } from '../test-both/helpers/fakeGetGroupCallVideoFrameSource';
import enMessages from '../../_locales/en/messages.json';
import { CallingToastProvider, useCallingToasts } from './CallingToast';
import type { CallingImageDataCache } from './CallManager';
const MAX_PARTICIPANTS = 75;
const LOCAL_DEMUX_ID = 1;
@ -190,6 +191,7 @@ const createProps = (
getPresentingSources: action('get-presenting-sources'),
hangUpActiveCall: action('hang-up'),
i18n,
imageDataCache: React.createRef<CallingImageDataCache>(),
isCallLinkAdmin: true,
isGroupCallRaiseHandEnabled: true,
me: getDefaultConversation({

4
ts/components/CallScreen.tsx
View file

@ -90,6 +90,7 @@ import { isGroupOrAdhocActiveCall } from '../util/isGroupOrAdhocCall';
import { assertDev } from '../util/assert';
import { emojiToData } from './emoji/lib';
import { CallingPendingParticipants } from './CallingPendingParticipants';
import type { CallingImageDataCache } from './CallManager';
export type PropsType = {
activeCall: ActiveCallType;
@ -100,6 +101,7 @@ export type PropsType = {
groupMembers?: Array<Pick<ConversationType, 'id' | 'firstName' | 'title'>>;
hangUpActiveCall: (reason: string) => void;
i18n: LocalizerType;
imageDataCache: React.RefObject<CallingImageDataCache>;
isCallLinkAdmin: boolean;
isGroupCallRaiseHandEnabled: boolean;
me: ConversationType;
@ -191,6 +193,7 @@ export function CallScreen({
groupMembers,
hangUpActiveCall,
i18n,
imageDataCache,
isCallLinkAdmin,
isGroupCallRaiseHandEnabled,
me,
@ -688,6 +691,7 @@ export function CallScreen({
<GroupCallRemoteParticipants
callViewMode={activeCall.viewMode}
getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
imageDataCache={imageDataCache}
i18n={i18n}
remoteParticipants={activeCall.remoteParticipants}
setGroupCallVideoRequest={setGroupCallVideoRequest}

4
ts/components/CallingPip.tsx
View file

@ -13,6 +13,7 @@ import type {
} from '../state/ducks/calling';
import { missingCaseError } from '../util/missingCaseError';
import { useActivateSpeakerViewOnPresenting } from '../hooks/useActivateSpeakerViewOnPresenting';
import type { CallingImageDataCache } from './CallManager';
enum PositionMode {
BeingDragged,
@ -54,6 +55,7 @@ export type PropsType = {
hangUpActiveCall: (reason: string) => void;
hasLocalVideo: boolean;
i18n: LocalizerType;
imageDataCache: React.RefObject<CallingImageDataCache>;
setGroupCallVideoRequest: (
_: Array<GroupCallVideoRequest>,
speakerHeight: number
@ -75,6 +77,7 @@ export function CallingPip({
getGroupCallVideoFrameSource,
hangUpActiveCall,
hasLocalVideo,
imageDataCache,
i18n,
setGroupCallVideoRequest,
setLocalPreview,
@ -304,6 +307,7 @@ export function CallingPip({
<CallingPipRemoteVideo
activeCall={activeCall}
getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
imageDataCache={imageDataCache}
i18n={i18n}
setRendererCanvas={setRendererCanvas}
setGroupCallVideoRequest={setGroupCallVideoRequest}

4
ts/components/CallingPipRemoteVideo.tsx
View file

@ -25,6 +25,7 @@ import { nonRenderedRemoteParticipant } from '../util/ringrtc/nonRenderedRemoteP
import { isReconnecting } from '../util/callingIsReconnecting';
import { isGroupOrAdhocActiveCall } from '../util/isGroupOrAdhocCall';
import { assertDev } from '../util/assert';
import type { CallingImageDataCache } from './CallManager';
// This value should be kept in sync with the hard-coded CSS height. It should also be
// less than `MAX_FRAME_HEIGHT`.
@ -78,6 +79,7 @@ export type PropsType = {
activeCall: ActiveCallType;
getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
i18n: LocalizerType;
imageDataCache: React.RefObject<CallingImageDataCache>;
setGroupCallVideoRequest: (
_: Array<GroupCallVideoRequest>,
speakerHeight: number
@ -88,6 +90,7 @@ export type PropsType = {
export function CallingPipRemoteVideo({
activeCall,
getGroupCallVideoFrameSource,
imageDataCache,
i18n,
setGroupCallVideoRequest,
setRendererCanvas,
@ -181,6 +184,7 @@ export function CallingPipRemoteVideo({
<GroupCallRemoteParticipant
getFrameBuffer={getGroupCallFrameBuffer}
getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
imageDataCache={imageDataCache}
i18n={i18n}
isInPip
remoteParticipant={activeGroupCallSpeaker}

3
ts/components/GroupCallOverflowArea.stories.tsx
View file

@ -13,6 +13,7 @@ import { fakeGetGroupCallVideoFrameSource } from '../test-both/helpers/fakeGetGr
import { FRAME_BUFFER_SIZE } from '../calling/constants';
import enMessages from '../../_locales/en/messages.json';
import { generateAci } from '../types/ServiceId';
import type { CallingImageDataCache } from './CallManager';
const MAX_PARTICIPANTS = 32;
@ -42,7 +43,9 @@ export default {
const defaultProps = {
getFrameBuffer: memoize(() => Buffer.alloc(FRAME_BUFFER_SIZE)),
getCallingImageDataCache: memoize(() => new Map()),
getGroupCallVideoFrameSource: fakeGetGroupCallVideoFrameSource,
imageDataCache: React.createRef<CallingImageDataCache>(),
i18n,
isCallReconnecting: false,
onParticipantVisibilityChanged: action('onParticipantVisibilityChanged'),

4
ts/components/GroupCallOverflowArea.tsx
View file

@ -8,6 +8,7 @@ import type { VideoFrameSource } from '@signalapp/ringrtc';
import type { LocalizerType } from '../types/Util';
import type { GroupCallRemoteParticipantType } from '../types/Calling';
import { GroupCallRemoteParticipant } from './GroupCallRemoteParticipant';
import type { CallingImageDataCache } from './CallManager';
const OVERFLOW_SCROLLED_TO_EDGE_THRESHOLD = 20;
const OVERFLOW_SCROLL_BUTTON_RATIO = 0.75;
@ -19,6 +20,7 @@ export type PropsType = {
getFrameBuffer: () => Buffer;
getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
i18n: LocalizerType;
imageDataCache: React.RefObject<CallingImageDataCache>;
isCallReconnecting: boolean;
onClickRaisedHand?: () => void;
onParticipantVisibilityChanged: (
@ -33,6 +35,7 @@ export type PropsType = {
export function GroupCallOverflowArea({
getFrameBuffer,
getGroupCallVideoFrameSource,
imageDataCache,
i18n,
isCallReconnecting,
onClickRaisedHand,
@ -121,6 +124,7 @@ export function GroupCallOverflowArea({
key={remoteParticipant.demuxId}
getFrameBuffer={getFrameBuffer}
getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
imageDataCache={imageDataCache}
i18n={i18n}
audioLevel={remoteAudioLevels.get(remoteParticipant.demuxId) ?? 0}
onClickRaisedHand={onClickRaisedHand}

2
ts/components/GroupCallRemoteParticipant.stories.tsx
View file

@ -11,6 +11,7 @@ import { FRAME_BUFFER_SIZE } from '../calling/constants';
import { setupI18n } from '../util/setupI18n';
import { generateAci } from '../types/ServiceId';
import enMessages from '../../_locales/en/messages.json';
import type { CallingImageDataCache } from './CallManager';
const i18n = setupI18n('en', enMessages);
@ -54,6 +55,7 @@ const createProps = (
getGroupCallVideoFrameSource: () => {
return { receiveVideoFrame: () => undefined };
},
imageDataCache: React.createRef<CallingImageDataCache>(),
i18n,
audioLevel: 0,
remoteParticipant: {

67
ts/components/GroupCallRemoteParticipant.tsx
View file

@ -29,6 +29,8 @@ import { MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH } from '../calling/constants';
import { useValueAtFixedRate } from '../hooks/useValueAtFixedRate';
import { Theme } from '../util/theme';
import { isOlderThan } from '../util/timestamp';
import type { CallingImageDataCache } from './CallManager';
import { usePrevious } from '../hooks/usePrevious';
const MAX_TIME_TO_SHOW_STALE_VIDEO_FRAMES = 10000;
const MAX_TIME_TO_SHOW_STALE_SCREENSHARE_FRAMES = 60000;
@ -38,6 +40,7 @@ type BasePropsType = {
getFrameBuffer: () => Buffer;
getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
i18n: LocalizerType;
imageDataCache: React.RefObject<CallingImageDataCache>;
isActiveSpeakerInSpeakerView: boolean;
isCallReconnecting: boolean;
onClickRaisedHand?: () => void;
@ -70,6 +73,7 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
const {
getFrameBuffer,
getGroupCallVideoFrameSource,
imageDataCache,
i18n,
onClickRaisedHand,
onVisibilityChanged,
@ -101,9 +105,12 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
!props.isInPip ? props.audioLevel > 0 : false,
SPEAKING_LINGER_MS
);
const previousSharingScreen = usePrevious(sharingScreen, sharingScreen);
const isImageDataCached =
sharingScreen && imageDataCache.current?.has(demuxId);
const [hasReceivedVideoRecently, setHasReceivedVideoRecently] =
useState(false);
useState(isImageDataCached);
const [isWide, setIsWide] = useState<boolean>(
videoAspectRatio ? videoAspectRatio >= 1 : true
);
@ -132,6 +139,12 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
onVisibilityChanged?.(demuxId, isVisible);
}, [demuxId, isVisible, onVisibilityChanged]);
useEffect(() => {
if (sharingScreen !== previousSharingScreen) {
imageDataCache.current?.delete(demuxId);
}
}, [demuxId, imageDataCache, previousSharingScreen, sharingScreen]);
const wantsToShowVideo = hasRemoteVideo && !isBlocked && isVisible;
const hasVideoToShow = wantsToShowVideo && hasReceivedVideoRecently;
const showMissingMediaKeys = Boolean(
@ -173,17 +186,18 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
// This frame buffer is shared by all participants, so it may contain pixel data
// for other participants, or pixel data from a previous frame. That's why we
// return early and use the `frameWidth` and `frameHeight`.
let frameWidth: number | undefined;
let frameHeight: number | undefined;
let imageData = imageDataRef.current;
const frameBuffer = getFrameBuffer();
const frameDimensions = videoFrameSource.receiveVideoFrame(
frameBuffer,
MAX_FRAME_WIDTH,
MAX_FRAME_HEIGHT
);
if (!frameDimensions) {
return;
}
const [frameWidth, frameHeight] = frameDimensions;
if (frameDimensions) {
[frameWidth, frameHeight] = frameDimensions;
if (
frameWidth < 2 ||
@ -194,10 +208,6 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
return;
}
canvasEl.width = frameWidth;
canvasEl.height = frameHeight;
let imageData = imageDataRef.current;
if (
imageData?.width !== frameWidth ||
imageData?.height !== frameHeight
@ -205,14 +215,45 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
imageData = new ImageData(frameWidth, frameHeight);
imageDataRef.current = imageData;
}
imageData.data.set(frameBuffer.subarray(0, frameWidth * frameHeight * 4));
canvasContext.putImageData(imageData, 0, 0);
imageData.data.set(
frameBuffer.subarray(0, frameWidth * frameHeight * 4)
);
// Screen share is at a slow FPS so updates slowly if we PiP then restore.
// Cache the image data so we can quickly show the most recent frame.
if (sharingScreen) {
imageDataCache.current?.set(demuxId, imageData);
}
} else if (sharingScreen && !imageData) {
// Try to use the screenshare cache the first time we show
const cachedImageData = imageDataCache.current?.get(demuxId);
if (cachedImageData) {
frameWidth = cachedImageData.width;
frameHeight = cachedImageData.height;
imageDataRef.current = cachedImageData;
imageData = cachedImageData;
}
}
if (!frameWidth || !frameHeight || !imageData) {
return;
}
canvasEl.width = frameWidth;
canvasEl.height = frameHeight;
canvasContext.putImageData(imageData, 0, 0);
lastReceivedVideoAt.current = Date.now();
setHasReceivedVideoRecently(true);
setIsWide(frameWidth > frameHeight);
}, [getFrameBuffer, videoFrameSource, sharingScreen, isCallReconnecting]);
}, [
demuxId,
imageDataCache,
isCallReconnecting,
sharingScreen,
videoFrameSource,
getFrameBuffer,
]);
useEffect(() => {
if (!hasRemoteVideo) {

5
ts/components/GroupCallRemoteParticipants.tsx
View file

@ -27,6 +27,7 @@ import * as log from '../logging/log';
import { MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH } from '../calling/constants';
import { SizeObserver } from '../hooks/useSizeObserver';
import { strictAssert } from '../util/assert';
import type { CallingImageDataCache } from './CallManager';
const SMALL_TILES_MIN_HEIGHT = 80;
const LARGE_TILES_MIN_HEIGHT = 200;
@ -60,6 +61,7 @@ type PropsType = {
callViewMode: CallViewMode;
getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
i18n: LocalizerType;
imageDataCache: React.RefObject<CallingImageDataCache>;
isCallReconnecting: boolean;
remoteParticipants: ReadonlyArray<GroupCallRemoteParticipantType>;
setGroupCallVideoRequest: (
@ -110,6 +112,7 @@ enum VideoRequestMode {
export function GroupCallRemoteParticipants({
callViewMode,
getGroupCallVideoFrameSource,
imageDataCache,
i18n,
isCallReconnecting,
remoteParticipants,
@ -343,6 +346,7 @@ export function GroupCallRemoteParticipants({
<GroupCallRemoteParticipant
key={tile.demuxId}
getFrameBuffer={getFrameBuffer}
imageDataCache={imageDataCache}
getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
onClickRaisedHand={onClickRaisedHand}
height={gridParticipantHeight}
@ -510,6 +514,7 @@ export function GroupCallRemoteParticipants({
<GroupCallOverflowArea
getFrameBuffer={getFrameBuffer}
getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
imageDataCache={imageDataCache}
i18n={i18n}
isCallReconnecting={isCallReconnecting}
onClickRaisedHand={onClickRaisedHand}

7
ts/util/lint/exceptions.json
View file

@ -3899,5 +3899,12 @@
"line": " message.innerHTML = window.i18n('icu:optimizingApplication');",
"reasonCategory": "usageTrusted",
"updated": "2021-09-17T21:02:59.414Z"
},
{
"rule": "React-useRef",
"path": "ts/components/CallManager.tsx",
"line": " const imageDataCache = React.useRef<CallingImageDataCache>(new Map());",
"reasonCategory": "usageTrusted",
"updated": "2024-05-06T20:18:59.647Z"
}
]