mirrors/signal-desktop
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Use global screen share cache for group calls

Browse source
This commit is contained in:
ayumi-signal 2024-05-07 11:21:57 -07:00 • committed by GitHub
parent 7cd07eb7b4
commit a3b9e97b82
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 114 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

87
ts/components/GroupCallRemoteParticipant.tsx
View file

@ -29,6 +29,8 @@ import { MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH } from '../calling/constants';
import { useValueAtFixedRate } from '../hooks/useValueAtFixedRate';
import { Theme } from '../util/theme';
import { isOlderThan } from '../util/timestamp';
import type { CallingImageDataCache } from './CallManager';
import { usePrevious } from '../hooks/usePrevious';
const MAX_TIME_TO_SHOW_STALE_VIDEO_FRAMES = 10000;
const MAX_TIME_TO_SHOW_STALE_SCREENSHARE_FRAMES = 60000;
@ -38,6 +40,7 @@ type BasePropsType = {
getFrameBuffer: () => Buffer;
getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
i18n: LocalizerType;
imageDataCache: React.RefObject<CallingImageDataCache>;
isActiveSpeakerInSpeakerView: boolean;
isCallReconnecting: boolean;
onClickRaisedHand?: () => void;
@ -70,6 +73,7 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
const {
getFrameBuffer,
getGroupCallVideoFrameSource,
imageDataCache,
i18n,
onClickRaisedHand,
onVisibilityChanged,
@ -101,9 +105,12 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
!props.isInPip ? props.audioLevel > 0 : false,
SPEAKING_LINGER_MS
);
const previousSharingScreen = usePrevious(sharingScreen, sharingScreen);
const isImageDataCached =
sharingScreen && imageDataCache.current?.has(demuxId);
const [hasReceivedVideoRecently, setHasReceivedVideoRecently] =
useState(false);
useState(isImageDataCached);
const [isWide, setIsWide] = useState<boolean>(
videoAspectRatio ? videoAspectRatio >= 1 : true
);
@ -132,6 +139,12 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
onVisibilityChanged?.(demuxId, isVisible);
}, [demuxId, isVisible, onVisibilityChanged]);
useEffect(() => {
if (sharingScreen !== previousSharingScreen) {
imageDataCache.current?.delete(demuxId);
}
}, [demuxId, imageDataCache, previousSharingScreen, sharingScreen]);
const wantsToShowVideo = hasRemoteVideo && !isBlocked && isVisible;
const hasVideoToShow = wantsToShowVideo && hasReceivedVideoRecently;
const showMissingMediaKeys = Boolean(
@ -173,46 +186,74 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
// This frame buffer is shared by all participants, so it may contain pixel data
// for other participants, or pixel data from a previous frame. That's why we
// return early and use the `frameWidth` and `frameHeight`.
let frameWidth: number | undefined;
let frameHeight: number | undefined;
let imageData = imageDataRef.current;
const frameBuffer = getFrameBuffer();
const frameDimensions = videoFrameSource.receiveVideoFrame(
frameBuffer,
MAX_FRAME_WIDTH,
MAX_FRAME_HEIGHT
);
if (!frameDimensions) {
return;
if (frameDimensions) {
[frameWidth, frameHeight] = frameDimensions;
if (
frameWidth < 2 ||
frameHeight < 2 ||
frameWidth > MAX_FRAME_WIDTH ||
frameHeight > MAX_FRAME_HEIGHT
) {
return;
}
if (
imageData?.width !== frameWidth ||
imageData?.height !== frameHeight
) {
imageData = new ImageData(frameWidth, frameHeight);
imageDataRef.current = imageData;
}
imageData.data.set(
frameBuffer.subarray(0, frameWidth * frameHeight * 4)
);
// Screen share is at a slow FPS so updates slowly if we PiP then restore.
// Cache the image data so we can quickly show the most recent frame.
if (sharingScreen) {
imageDataCache.current?.set(demuxId, imageData);
}
} else if (sharingScreen && !imageData) {
// Try to use the screenshare cache the first time we show
const cachedImageData = imageDataCache.current?.get(demuxId);
if (cachedImageData) {
frameWidth = cachedImageData.width;
frameHeight = cachedImageData.height;
imageDataRef.current = cachedImageData;
imageData = cachedImageData;
}
}
const [frameWidth, frameHeight] = frameDimensions;
if (
frameWidth < 2 ||
frameHeight < 2 ||
frameWidth > MAX_FRAME_WIDTH ||
frameHeight > MAX_FRAME_HEIGHT
) {
if (!frameWidth || !frameHeight || !imageData) {
return;
}
canvasEl.width = frameWidth;
canvasEl.height = frameHeight;
let imageData = imageDataRef.current;
if (
imageData?.width !== frameWidth ||
imageData?.height !== frameHeight
) {
imageData = new ImageData(frameWidth, frameHeight);
imageDataRef.current = imageData;
}
imageData.data.set(frameBuffer.subarray(0, frameWidth * frameHeight * 4));
canvasContext.putImageData(imageData, 0, 0);
lastReceivedVideoAt.current = Date.now();
setHasReceivedVideoRecently(true);
setIsWide(frameWidth > frameHeight);
}, [getFrameBuffer, videoFrameSource, sharingScreen, isCallReconnecting]);
}, [
demuxId,
imageDataCache,
isCallReconnecting,
sharingScreen,
videoFrameSource,
getFrameBuffer,
]);
useEffect(() => {
if (!hasRemoteVideo) {