Blur participant videos when calls are reconnecting

stylesheets/_modules.scss

@@ -3655,6 +3655,9 @@ button.module-image__border-overlay:focus {
     background-color: $color-gray-95;
     height: 100%;
     width: 100%;
+    &--reconnecting {
+      filter: blur(15px);
+    }
   }
 
   &__remote-video-disabled {
@@ -3844,6 +3847,9 @@ button.module-image__border-overlay:focus {
     &__remote-video {
       // The background-color is seen while the video loads.
       background-color: $color-gray-75;
+      &--reconnecting {
+        filter: blur(15px);
+      }
     }
 
     &__blocked {

ts/components/CallScreen.tsx

@@ -49,6 +49,7 @@ import {
   useKeyboardShortcuts,
 } from '../hooks/useKeyboardShortcuts';
 import { useValueAtFixedRate } from '../hooks/useValueAtFixedRate';
+import { isReconnecting } from '../util/callingIsReconnecting';
 
 export type PropsType = {
   activeCall: ActiveCallType;
@@ -113,9 +114,6 @@ function DirectCallHeaderMessage({
     return clearInterval.bind(null, interval);
   }, [joinedAt]);
 
-  if (callState === CallState.Reconnecting) {
-    return <>{i18n('icu:callReconnecting')}</>;
-  }
   if (callState === CallState.Accepted && acceptedDuration) {
     return (
       <>
@@ -298,6 +296,7 @@ export function CallScreen({
           conversation={conversation}
           hasRemoteVideo={hasRemoteVideo}
           i18n={i18n}
+          isReconnecting={isReconnecting(activeCall)}
           setRendererCanvas={setRendererCanvas}
         />
       ) : (
@@ -333,6 +332,7 @@ export function CallScreen({
           remoteParticipants={activeCall.remoteParticipants}
           setGroupCallVideoRequest={setGroupCallVideoRequest}
           remoteAudioLevels={activeCall.remoteAudioLevels}
+          isCallReconnecting={isReconnecting(activeCall)}
         />
       );
       break;
@@ -343,15 +343,9 @@ export function CallScreen({
   let lonelyInCallNode: ReactNode;
   let localPreviewNode: ReactNode;
 
-  const isLonelyInGroup =
+  const isLonelyInCall = !activeCall.remoteParticipants.length;
-    activeCall.callMode === CallMode.Group &&
-    !activeCall.remoteParticipants.length;
 
-  const isLonelyInDirectCall =
+  if (isLonelyInCall) {
-    activeCall.callMode === CallMode.Direct &&
-    activeCall.callState !== CallState.Accepted;
-
-  if (isLonelyInGroup || isLonelyInDirectCall) {
     lonelyInCallNode = (
       <div
         className={classNames(

ts/components/CallingPipRemoteVideo.tsx

@@ -22,6 +22,7 @@ import { MAX_FRAME_WIDTH } from '../calling/constants';
 import { usePageVisibility } from '../hooks/usePageVisibility';
 import { missingCaseError } from '../util/missingCaseError';
 import { nonRenderedRemoteParticipant } from '../util/ringrtc/nonRenderedRemoteParticipant';
+import { isReconnecting } from '../util/callingIsReconnecting';
 
 // This value should be kept in sync with the hard-coded CSS height. It should also be
 //   less than `MAX_FRAME_HEIGHT`.
@@ -154,6 +155,7 @@ export function CallingPipRemoteVideo({
             conversation={conversation}
             hasRemoteVideo={hasRemoteVideo}
             i18n={i18n}
+            isReconnecting={isReconnecting(activeCall)}
             setRendererCanvas={setRendererCanvas}
           />
         </div>
@@ -173,6 +175,7 @@ export function CallingPipRemoteVideo({
             remoteParticipant={activeGroupCallSpeaker}
             remoteParticipantsCount={activeCall.remoteParticipants.length}
             isActiveSpeakerInSpeakerView={false}
+            isCallReconnecting={isReconnecting(activeCall)}
           />
         </div>
       );

ts/components/CallingToastManager.tsx

@@ -3,11 +3,12 @@
 
 import React, { useCallback, useEffect, useRef, useState } from 'react';
 import type { ActiveCallType } from '../types/Calling';
-import { CallMode, GroupCallConnectionState } from '../types/Calling';
+import { CallMode } from '../types/Calling';
 import type { ConversationType } from '../state/ducks/conversations';
 import type { LocalizerType } from '../types/Util';
 import { clearTimeoutIfNecessary } from '../util/clearTimeoutIfNecessary';
 import { CallingToast, DEFAULT_LIFETIME } from './CallingToast';
+import { isReconnecting } from '../util/callingIsReconnecting';
 
 type PropsType = {
   activeCall: ActiveCallType;
@@ -22,10 +23,7 @@ type ToastType =
   | undefined;
 
 function getReconnectingToast({ activeCall, i18n }: PropsType): ToastType {
-  if (
+  if (isReconnecting(activeCall)) {
-    activeCall.callMode === CallMode.Group &&
-    activeCall.connectionState === GroupCallConnectionState.Reconnecting
-  ) {
     return {
       message: i18n('icu:callReconnecting'),
       type: 'static',

ts/components/DirectCallRemoteParticipant.tsx

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 
 import React, { useRef, useEffect } from 'react';
+import classNames from 'classnames';
 import type { SetRendererCanvasType } from '../state/ducks/calling';
 import type { ConversationType } from '../state/ducks/conversations';
 import type { LocalizerType } from '../types/Util';
@@ -12,6 +13,7 @@ type PropsType = {
   conversation: ConversationType;
   hasRemoteVideo: boolean;
   i18n: LocalizerType;
+  isReconnecting: boolean;
   setRendererCanvas: (_: SetRendererCanvasType) => void;
 };
 
@@ -19,6 +21,7 @@ export function DirectCallRemoteParticipant({
   conversation,
   hasRemoteVideo,
   i18n,
+  isReconnecting,
   setRendererCanvas,
 }: PropsType): JSX.Element {
   const remoteVideoRef = useRef<HTMLCanvasElement | null>(null);
@@ -32,7 +35,11 @@ export function DirectCallRemoteParticipant({
 
   return hasRemoteVideo ? (
     <canvas
-      className="module-ongoing-call__remote-video-enabled"
+      className={classNames(
+        'module-ongoing-call__remote-video-enabled',
+        isReconnecting &&
+          'module-ongoing-call__remote-video-enabled--reconnecting'
+      )}
       ref={remoteVideoRef}
     />
   ) : (

ts/components/GroupCallOverflowArea.stories.tsx

@@ -42,6 +42,7 @@ const defaultProps = {
   getFrameBuffer: memoize(() => Buffer.alloc(FRAME_BUFFER_SIZE)),
   getGroupCallVideoFrameSource: fakeGetGroupCallVideoFrameSource,
   i18n,
+  isCallReconnecting: false,
   onParticipantVisibilityChanged: action('onParticipantVisibilityChanged'),
   remoteAudioLevels: new Map<number, number>(),
   remoteParticipantsCount: 1,

ts/components/GroupCallOverflowArea.tsx

@@ -19,6 +19,7 @@ export type PropsType = {
   getFrameBuffer: () => Buffer;
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
+  isCallReconnecting: boolean;
   onParticipantVisibilityChanged: (
     demuxId: number,
     isVisible: boolean
@@ -32,6 +33,7 @@ export function GroupCallOverflowArea({
   getFrameBuffer,
   getGroupCallVideoFrameSource,
   i18n,
+  isCallReconnecting,
   onParticipantVisibilityChanged,
   overflowedParticipants,
   remoteAudioLevels,
@@ -127,6 +129,7 @@ export function GroupCallOverflowArea({
             remoteParticipant={remoteParticipant}
             remoteParticipantsCount={remoteParticipantsCount}
             isActiveSpeakerInSpeakerView={false}
+            isCallReconnecting={isCallReconnecting}
           />
         ))}
       </div>

ts/components/GroupCallRemoteParticipant.stories.tsx

@@ -67,6 +67,7 @@ const createProps = (
   },
   remoteParticipantsCount: 1,
   isActiveSpeakerInSpeakerView: false,
+  isCallReconnecting: false,
   ...overrideProps,
 });
 

ts/components/GroupCallRemoteParticipant.tsx

@@ -28,7 +28,7 @@ import { useIntersectionObserver } from '../hooks/useIntersectionObserver';
 import { MAX_FRAME_HEIGHT, MAX_FRAME_WIDTH } from '../calling/constants';
 import { useValueAtFixedRate } from '../hooks/useValueAtFixedRate';
 
-const MAX_TIME_TO_SHOW_STALE_VIDEO_FRAMES = 5000;
+const MAX_TIME_TO_SHOW_STALE_VIDEO_FRAMES = 10000;
 const MAX_TIME_TO_SHOW_STALE_SCREENSHARE_FRAMES = 60000;
 
 type BasePropsType = {
@@ -36,6 +36,7 @@ type BasePropsType = {
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
   isActiveSpeakerInSpeakerView: boolean;
+  isCallReconnecting: boolean;
   onVisibilityChanged?: (demuxId: number, isVisible: boolean) => unknown;
   remoteParticipant: GroupCallRemoteParticipantType;
   remoteParticipantsCount: number;
@@ -69,6 +70,7 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
       onVisibilityChanged,
       remoteParticipantsCount,
       isActiveSpeakerInSpeakerView,
+      isCallReconnecting,
     } = props;
 
     const {
@@ -136,7 +138,13 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
         ? MAX_TIME_TO_SHOW_STALE_SCREENSHARE_FRAMES
         : MAX_TIME_TO_SHOW_STALE_VIDEO_FRAMES;
       if (frameAge > maxFrameAge) {
-        setHasReceivedVideoRecently(false);
+        // We consider that we have received video recently from a remote participant if
+        // we have received it recently relative to the last time we had a connection. If
+        // we lost their video due to our reconnecting, we still want to show the last
+        // frame of video (blurred out) until we have reconnected.
+        if (!isCallReconnecting) {
+          setHasReceivedVideoRecently(false);
+        }
       }
 
       const canvasEl = remoteVideoRef.current;
@@ -191,7 +199,7 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
 
       setHasReceivedVideoRecently(true);
       setIsWide(frameWidth > frameHeight);
-    }, [getFrameBuffer, videoFrameSource, sharingScreen]);
+    }, [getFrameBuffer, videoFrameSource, sharingScreen, isCallReconnecting]);
 
     useEffect(() => {
       if (!hasRemoteVideo) {
@@ -310,7 +318,11 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
           )}
           {wantsToShowVideo && (
             <canvas
-              className="module-ongoing-call__group-call-remote-participant__remote-video"
+              className={classNames(
+                'module-ongoing-call__group-call-remote-participant__remote-video',
+                isCallReconnecting &&
+                  'module-ongoing-call__group-call-remote-participant__remote-video--reconnecting'
+              )}
               style={{
                 ...canvasStyles,
                 // If we want to show video but don't have any yet, we still render the

ts/components/GroupCallRemoteParticipants.tsx

@@ -46,6 +46,7 @@ type GridArrangement = {
 type PropsType = {
   getGroupCallVideoFrameSource: (demuxId: number) => VideoFrameSource;
   i18n: LocalizerType;
+  isCallReconnecting: boolean;
   isInSpeakerView: boolean;
   remoteParticipants: ReadonlyArray<GroupCallRemoteParticipantType>;
   setGroupCallVideoRequest: (
@@ -87,6 +88,7 @@ enum VideoRequestMode {
 export function GroupCallRemoteParticipants({
   getGroupCallVideoFrameSource,
   i18n,
+  isCallReconnecting,
   isInSpeakerView,
   remoteParticipants,
   setGroupCallVideoRequest,
@@ -297,6 +299,7 @@ export function GroupCallRemoteParticipants({
             width={renderedWidth}
             remoteParticipantsCount={remoteParticipants.length}
             isActiveSpeakerInSpeakerView={isInSpeakerView}
+            isCallReconnecting={isCallReconnecting}
           />
         );
       });
@@ -424,6 +427,7 @@ export function GroupCallRemoteParticipants({
             getFrameBuffer={getFrameBuffer}
             getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
             i18n={i18n}
+            isCallReconnecting={isCallReconnecting}
             onParticipantVisibilityChanged={onParticipantVisibilityChanged}
             overflowedParticipants={overflowedParticipants}
             remoteAudioLevels={remoteAudioLevels}

ts/util/callingIsReconnecting.ts

@@ -0,0 +1,18 @@
+// Copyright 2023 Signal Messenger, LLC
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import {
+  CallMode,
+  CallState,
+  GroupCallConnectionState,
+} from '../types/Calling';
+import type { ActiveCallType } from '../types/Calling';
+
+export function isReconnecting(activeCall: ActiveCallType): boolean {
+  return (
+    (activeCall.callMode === CallMode.Group &&
+      activeCall.connectionState === GroupCallConnectionState.Reconnecting) ||
+    (activeCall.callMode === CallMode.Direct &&
+      activeCall.callState === CallState.Reconnecting)
+  );
+}