Dynamic audio level indicator

This commit is contained in:
Fedor Indutny 2022-05-18 20:28:51 -07:00 committed by GitHub
parent ac59dec5aa
commit e6223b6a11
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 323 additions and 123 deletions

File diff suppressed because one or more lines are too long

View file

@ -22,11 +22,6 @@
width: $size; width: $size;
height: $size; height: $size;
/* Center Lottie animation */
display: flex;
align-items: center;
justify-content: center;
&--muted { &--muted {
@include color-svg( @include color-svg(
'../images/icons/v2/mic-off-solid-28.svg', '../images/icons/v2/mic-off-solid-28.svg',

View file

@ -1,8 +1,8 @@
// Copyright 2020-2022 Signal Messenger, LLC // Copyright 2020-2022 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only // SPDX-License-Identifier: AGPL-3.0-only
export const AUDIO_LEVEL_INTERVAL_MS = 250; // See `TICK_INTERVAL` in group_call.rs in RingRTC
export const DEFAULT_AUDIO_LEVEL_FOR_SPEAKING = 0.15; export const AUDIO_LEVEL_INTERVAL_MS = 200;
export const REQUESTED_VIDEO_WIDTH = 640; export const REQUESTED_VIDEO_WIDTH = 640;
export const REQUESTED_VIDEO_HEIGHT = 480; export const REQUESTED_VIDEO_HEIGHT = 480;

View file

@ -1,20 +0,0 @@
// Copyright 2022 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import type * as RemoteConfig from '../RemoteConfig';
import { DEFAULT_AUDIO_LEVEL_FOR_SPEAKING } from './constants';
export function getAudioLevelForSpeaking(
getValueFromRemoteConfig: typeof RemoteConfig.getValue
): number {
const configValue = getValueFromRemoteConfig(
'desktop.calling.audioLevelForSpeaking'
);
if (typeof configValue !== 'string') {
return DEFAULT_AUDIO_LEVEL_FOR_SPEAKING;
}
const result = parseFloat(configValue);
const isResultValid = result > 0 && result <= 1;
return isResultValid ? result : DEFAULT_AUDIO_LEVEL_FOR_SPEAKING;
}

View file

@ -0,0 +1,24 @@
// Copyright 2022 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
// See https://github.com/signalapp/Signal-Android/blob/b527b2ffb94fb82a7508c3b33ddbffef28085349/app/src/main/java/org/thoughtcrime/securesms/events/CallParticipant.kt#L87-L100
const LOWEST = 500 / 32767;
const LOW = 1000 / 32767;
const MEDIUM = 5000 / 32767;
const HIGH = 16000 / 32767;
export function truncateAudioLevel(audioLevel: number): number {
if (audioLevel < LOWEST) {
return 0;
}
if (audioLevel < LOW) {
return 0.25;
}
if (audioLevel < MEDIUM) {
return 0.5;
}
if (audioLevel < HIGH) {
return 0.75;
}
return 1;
}

View file

@ -50,7 +50,7 @@ const getCommonActiveCallData = () => ({
joinedAt: Date.now(), joinedAt: Date.now(),
hasLocalAudio: boolean('hasLocalAudio', true), hasLocalAudio: boolean('hasLocalAudio', true),
hasLocalVideo: boolean('hasLocalVideo', false), hasLocalVideo: boolean('hasLocalVideo', false),
amISpeaking: boolean('amISpeaking', false), localAudioLevel: select('localAudioLevel', [0, 0.5, 1], 0),
isInSpeakerView: boolean('isInSpeakerView', false), isInSpeakerView: boolean('isInSpeakerView', false),
outgoingRing: boolean('outgoingRing', true), outgoingRing: boolean('outgoingRing', true),
pip: boolean('pip', false), pip: boolean('pip', false),
@ -145,7 +145,7 @@ story.add('Ongoing Group Call', () => (
groupMembers: [], groupMembers: [],
peekedParticipants: [], peekedParticipants: [],
remoteParticipants: [], remoteParticipants: [],
speakingDemuxIds: new Set<number>(), remoteAudioLevels: new Map<number, number>(),
}, },
})} })}
/> />
@ -220,7 +220,7 @@ story.add('Group call - Safety Number Changed', () => (
groupMembers: [], groupMembers: [],
peekedParticipants: [], peekedParticipants: [],
remoteParticipants: [], remoteParticipants: [],
speakingDemuxIds: new Set<number>(), remoteAudioLevels: new Map<number, number>(),
}, },
})} })}
/> />

View file

@ -44,7 +44,7 @@ const conversation = getDefaultConversation({
type OverridePropsBase = { type OverridePropsBase = {
hasLocalAudio?: boolean; hasLocalAudio?: boolean;
hasLocalVideo?: boolean; hasLocalVideo?: boolean;
amISpeaking?: boolean; localAudioLevel?: number;
isInSpeakerView?: boolean; isInSpeakerView?: boolean;
}; };
@ -104,7 +104,7 @@ const createActiveGroupCallProp = (overrideProps: GroupCallOverrideProps) => ({
peekedParticipants: peekedParticipants:
overrideProps.peekedParticipants || overrideProps.remoteParticipants || [], overrideProps.peekedParticipants || overrideProps.remoteParticipants || [],
remoteParticipants: overrideProps.remoteParticipants || [], remoteParticipants: overrideProps.remoteParticipants || [],
speakingDemuxIds: new Set<number>(), remoteAudioLevels: new Map<number, number>(),
}); });
const createActiveCallProp = ( const createActiveCallProp = (
@ -121,7 +121,11 @@ const createActiveCallProp = (
'hasLocalVideo', 'hasLocalVideo',
overrideProps.hasLocalVideo || false overrideProps.hasLocalVideo || false
), ),
amISpeaking: boolean('amISpeaking', overrideProps.amISpeaking || false), localAudioLevel: select(
'localAudioLevel',
[0, 0.5, 1],
overrideProps.localAudioLevel || 0
),
isInSpeakerView: boolean( isInSpeakerView: boolean(
'isInSpeakerView', 'isInSpeakerView',
overrideProps.isInSpeakerView || false overrideProps.isInSpeakerView || false

View file

@ -130,7 +130,7 @@ export const CallScreen: React.FC<PropsType> = ({
conversation, conversation,
hasLocalAudio, hasLocalAudio,
hasLocalVideo, hasLocalVideo,
amISpeaking, localAudioLevel,
isInSpeakerView, isInSpeakerView,
presentingSource, presentingSource,
remoteParticipants, remoteParticipants,
@ -296,7 +296,7 @@ export const CallScreen: React.FC<PropsType> = ({
isInSpeakerView={isInSpeakerView} isInSpeakerView={isInSpeakerView}
remoteParticipants={activeCall.remoteParticipants} remoteParticipants={activeCall.remoteParticipants}
setGroupCallVideoRequest={setGroupCallVideoRequest} setGroupCallVideoRequest={setGroupCallVideoRequest}
speakingDemuxIds={activeCall.speakingDemuxIds} remoteAudioLevels={activeCall.remoteAudioLevels}
/> />
); );
break; break;
@ -514,7 +514,7 @@ export const CallScreen: React.FC<PropsType> = ({
{localPreviewNode} {localPreviewNode}
<CallingAudioIndicator <CallingAudioIndicator
hasAudio={hasLocalAudio} hasAudio={hasLocalAudio}
isSpeaking={amISpeaking} audioLevel={localAudioLevel}
/> />
</div> </div>
</div> </div>

View file

@ -0,0 +1,17 @@
// Copyright 2022 Signal Messenger, LLC
// SPDX-License-Identifier: AGPL-3.0-only
import React from 'react';
import { storiesOf } from '@storybook/react';
import { boolean, select } from '@storybook/addon-knobs';
import { CallingAudioIndicator } from './CallingAudioIndicator';
const story = storiesOf('Components/CallingAudioIndicator', module);
story.add('Default', () => (
<CallingAudioIndicator
hasAudio={boolean('hasAudio', true)}
audioLevel={select('audioLevel', [0, 0.25, 0.5, 0.75, 1], 0.5)}
/>
));

View file

@ -4,23 +4,112 @@
import classNames from 'classnames'; import classNames from 'classnames';
import { noop } from 'lodash'; import { noop } from 'lodash';
import type { ReactElement } from 'react'; import type { ReactElement } from 'react';
import React, { useEffect, useState } from 'react'; import React, { useEffect, useCallback, useState } from 'react';
import animationData from '../../images/lottie-animations/CallingSpeakingIndicator.json'; import { useSpring, animated } from '@react-spring/web';
import { Lottie } from './Lottie';
const SPEAKING_LINGER_MS = 100; import { AUDIO_LEVEL_INTERVAL_MS } from '../calling/constants';
import { missingCaseError } from '../util/missingCaseError';
const SPEAKING_LINGER_MS = 500;
const BASE_CLASS_NAME = 'CallingAudioIndicator'; const BASE_CLASS_NAME = 'CallingAudioIndicator';
const CONTENT_CLASS_NAME = `${BASE_CLASS_NAME}__content`; const CONTENT_CLASS_NAME = `${BASE_CLASS_NAME}__content`;
export function CallingAudioIndicator({ const MIN_BAR_HEIGHT = 2;
hasAudio, const SIDE_SCALE_FACTOR = 0.75;
isSpeaking, const MAX_CENTRAL_BAR_DELTA = 9;
}: Readonly<{ hasAudio: boolean; isSpeaking: boolean }>): ReactElement {
const [shouldShowSpeaking, setShouldShowSpeaking] = useState(isSpeaking); /* Should match css */
const CONTENT_WIDTH = 14;
const CONTENT_HEIGHT = 14;
const BAR_WIDTH = 2;
const CONTENT_PADDING = 1;
enum BarPosition {
Left,
Center,
Right,
}
function generateBarPath(position: BarPosition, audioLevel: number): string {
let x: number;
if (position === BarPosition.Left) {
x = CONTENT_PADDING;
} else if (position === BarPosition.Center) {
x = CONTENT_WIDTH / 2 - CONTENT_PADDING;
} else if (position === BarPosition.Right) {
x = CONTENT_WIDTH - CONTENT_PADDING - BAR_WIDTH;
} else {
throw missingCaseError(position);
}
let height: number;
if (position === BarPosition.Left || position === BarPosition.Right) {
height =
MIN_BAR_HEIGHT + audioLevel * MAX_CENTRAL_BAR_DELTA * SIDE_SCALE_FACTOR;
} else if (position === BarPosition.Center) {
height = MIN_BAR_HEIGHT + audioLevel * MAX_CENTRAL_BAR_DELTA;
} else {
throw missingCaseError(position);
}
// Take the round corners off the height
height -= 2;
const y = (CONTENT_HEIGHT - height) / 2;
const top = y;
const bottom = top + height;
return (
`M ${x} ${top} ` +
`L ${x} ${bottom} ` +
`A 0.5 0.5 0 0 0 ${x + BAR_WIDTH} ${bottom} ` +
`L ${x + BAR_WIDTH} ${top} ` +
`A 0.5 0.5 0 0 0 ${x} ${top}`
);
}
function Bar({
position,
audioLevel,
}: {
position: BarPosition;
audioLevel: number;
}): ReactElement {
const animatedProps = useSpring({
from: { audioLevel: 0 },
config: { duration: AUDIO_LEVEL_INTERVAL_MS },
});
const levelToPath = useCallback(
(animatedLevel: number): string => {
return generateBarPath(position, animatedLevel);
},
[position]
);
useEffect(() => { useEffect(() => {
if (isSpeaking) { animatedProps.audioLevel.stop();
animatedProps.audioLevel.start(audioLevel);
}, [audioLevel, animatedProps]);
return (
<animated.path
d={animatedProps.audioLevel.to(levelToPath)}
fill="#ffffff"
/>
);
}
export function CallingAudioIndicator({
hasAudio,
audioLevel,
}: Readonly<{ hasAudio: boolean; audioLevel: number }>): ReactElement {
const [shouldShowSpeaking, setShouldShowSpeaking] = useState(audioLevel > 0);
useEffect(() => {
if (audioLevel > 0) {
setShouldShowSpeaking(true); setShouldShowSpeaking(true);
} else if (shouldShowSpeaking) { } else if (shouldShowSpeaking) {
const timeout = setTimeout(() => { const timeout = setTimeout(() => {
@ -31,7 +120,7 @@ export function CallingAudioIndicator({
}; };
} }
return noop; return noop;
}, [isSpeaking, shouldShowSpeaking]); }, [audioLevel, shouldShowSpeaking]);
if (!hasAudio) { if (!hasAudio) {
return ( return (
@ -59,7 +148,11 @@ export function CallingAudioIndicator({
`${BASE_CLASS_NAME}--with-content` `${BASE_CLASS_NAME}--with-content`
)} )}
> >
<Lottie animationData={animationData} className={CONTENT_CLASS_NAME} /> <svg className={CONTENT_CLASS_NAME}>
<Bar position={BarPosition.Left} audioLevel={audioLevel} />
<Bar position={BarPosition.Center} audioLevel={audioLevel} />
<Bar position={BarPosition.Right} audioLevel={audioLevel} />
</svg>
</div> </div>
); );
} }

View file

@ -4,7 +4,7 @@
import * as React from 'react'; import * as React from 'react';
import { times } from 'lodash'; import { times } from 'lodash';
import { storiesOf } from '@storybook/react'; import { storiesOf } from '@storybook/react';
import { boolean } from '@storybook/addon-knobs'; import { boolean, select } from '@storybook/addon-knobs';
import { action } from '@storybook/addon-actions'; import { action } from '@storybook/addon-actions';
import { AvatarColors } from '../types/Colors'; import { AvatarColors } from '../types/Colors';
@ -39,7 +39,7 @@ const getCommonActiveCallData = () => ({
conversation, conversation,
hasLocalAudio: boolean('hasLocalAudio', true), hasLocalAudio: boolean('hasLocalAudio', true),
hasLocalVideo: boolean('hasLocalVideo', false), hasLocalVideo: boolean('hasLocalVideo', false),
amISpeaking: boolean('amISpeaking', false), localAudioLevel: select('localAudioLevel', [0, 0.5, 1], 0),
isInSpeakerView: boolean('isInSpeakerView', false), isInSpeakerView: boolean('isInSpeakerView', false),
joinedAt: Date.now(), joinedAt: Date.now(),
outgoingRing: true, outgoingRing: true,
@ -120,7 +120,7 @@ story.add('Group Call', () => {
deviceCount: 0, deviceCount: 0,
peekedParticipants: [], peekedParticipants: [],
remoteParticipants: [], remoteParticipants: [],
speakingDemuxIds: new Set<number>(), remoteAudioLevels: new Map<number, number>(),
}, },
}); });
return <CallingPip {...props} />; return <CallingPip {...props} />;

View file

@ -39,7 +39,7 @@ const defaultProps = {
getGroupCallVideoFrameSource: fakeGetGroupCallVideoFrameSource, getGroupCallVideoFrameSource: fakeGetGroupCallVideoFrameSource,
i18n, i18n,
onParticipantVisibilityChanged: action('onParticipantVisibilityChanged'), onParticipantVisibilityChanged: action('onParticipantVisibilityChanged'),
speakingDemuxIds: new Set<number>(), remoteAudioLevels: new Map<number, number>(),
}; };
// This component is usually rendered on a call screen. // This component is usually rendered on a call screen.

View file

@ -24,7 +24,7 @@ type PropsType = {
isVisible: boolean isVisible: boolean
) => unknown; ) => unknown;
overflowedParticipants: ReadonlyArray<GroupCallRemoteParticipantType>; overflowedParticipants: ReadonlyArray<GroupCallRemoteParticipantType>;
speakingDemuxIds: Set<number>; remoteAudioLevels: Map<number, number>;
}; };
export const GroupCallOverflowArea: FC<PropsType> = ({ export const GroupCallOverflowArea: FC<PropsType> = ({
@ -33,7 +33,7 @@ export const GroupCallOverflowArea: FC<PropsType> = ({
i18n, i18n,
onParticipantVisibilityChanged, onParticipantVisibilityChanged,
overflowedParticipants, overflowedParticipants,
speakingDemuxIds, remoteAudioLevels,
}) => { }) => {
const overflowRef = useRef<HTMLDivElement | null>(null); const overflowRef = useRef<HTMLDivElement | null>(null);
const [overflowScrollTop, setOverflowScrollTop] = useState(0); const [overflowScrollTop, setOverflowScrollTop] = useState(0);
@ -116,7 +116,7 @@ export const GroupCallOverflowArea: FC<PropsType> = ({
getFrameBuffer={getFrameBuffer} getFrameBuffer={getFrameBuffer}
getGroupCallVideoFrameSource={getGroupCallVideoFrameSource} getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
i18n={i18n} i18n={i18n}
isSpeaking={speakingDemuxIds.has(remoteParticipant.demuxId)} audioLevel={remoteAudioLevels.get(remoteParticipant.demuxId) ?? 0}
onVisibilityChanged={onParticipantVisibilityChanged} onVisibilityChanged={onParticipantVisibilityChanged}
width={OVERFLOW_PARTICIPANT_WIDTH} width={OVERFLOW_PARTICIPANT_WIDTH}
height={Math.floor( height={Math.floor(

View file

@ -4,6 +4,7 @@
import * as React from 'react'; import * as React from 'react';
import { memoize, noop } from 'lodash'; import { memoize, noop } from 'lodash';
import { storiesOf } from '@storybook/react'; import { storiesOf } from '@storybook/react';
import { select } from '@storybook/addon-knobs';
import type { PropsType } from './GroupCallRemoteParticipant'; import type { PropsType } from './GroupCallRemoteParticipant';
import { GroupCallRemoteParticipant } from './GroupCallRemoteParticipant'; import { GroupCallRemoteParticipant } from './GroupCallRemoteParticipant';
@ -14,7 +15,9 @@ import enMessages from '../../_locales/en/messages.json';
const i18n = setupI18n('en', enMessages); const i18n = setupI18n('en', enMessages);
type OverridePropsType = type OverridePropsType = {
audioLevel?: number;
} & (
| { | {
isInPip: true; isInPip: true;
} }
@ -24,22 +27,29 @@ type OverridePropsType =
left: number; left: number;
top: number; top: number;
width: number; width: number;
}; }
);
const getFrameBuffer = memoize(() => Buffer.alloc(FRAME_BUFFER_SIZE)); const getFrameBuffer = memoize(() => Buffer.alloc(FRAME_BUFFER_SIZE));
const createProps = ( const createProps = (
overrideProps: OverridePropsType, overrideProps: OverridePropsType,
isBlocked?: boolean {
isBlocked = false,
hasRemoteAudio = false,
}: {
isBlocked?: boolean;
hasRemoteAudio?: boolean;
} = {}
): PropsType => ({ ): PropsType => ({
getFrameBuffer, getFrameBuffer,
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
getGroupCallVideoFrameSource: noop as any, getGroupCallVideoFrameSource: noop as any,
i18n, i18n,
isSpeaking: false, audioLevel: 0,
remoteParticipant: { remoteParticipant: {
demuxId: 123, demuxId: 123,
hasRemoteAudio: false, hasRemoteAudio,
hasRemoteVideo: true, hasRemoteVideo: true,
presenting: false, presenting: false,
sharingScreen: false, sharingScreen: false,
@ -52,7 +62,6 @@ const createProps = (
}), }),
}, },
...overrideProps, ...overrideProps,
...(overrideProps.isInPip ? {} : { isSpeaking: false }),
}); });
const story = storiesOf('Components/GroupCallRemoteParticipant', module); const story = storiesOf('Components/GroupCallRemoteParticipant', module);
@ -69,6 +78,22 @@ story.add('Default', () => (
/> />
)); ));
story.add('Speaking', () => (
<GroupCallRemoteParticipant
{...createProps(
{
isInPip: false,
height: 120,
left: 0,
top: 0,
width: 120,
audioLevel: select('audioLevel', [0, 0.5, 1], 0.5),
},
{ hasRemoteAudio: true }
)}
/>
));
story.add('isInPip', () => ( story.add('isInPip', () => (
<GroupCallRemoteParticipant <GroupCallRemoteParticipant
{...createProps({ {...createProps({
@ -87,7 +112,7 @@ story.add('Blocked', () => (
top: 0, top: 0,
width: 120, width: 120,
}, },
true { isBlocked: true }
)} )}
/> />
)); ));

View file

@ -42,7 +42,7 @@ type InPipPropsType = {
type InOverflowAreaPropsType = { type InOverflowAreaPropsType = {
height: number; height: number;
isInPip?: false; isInPip?: false;
isSpeaking: boolean; audioLevel: number;
width: number; width: number;
}; };
@ -282,7 +282,7 @@ export const GroupCallRemoteParticipant: React.FC<PropsType> = React.memo(
/> />
<CallingAudioIndicator <CallingAudioIndicator
hasAudio={hasRemoteAudio} hasAudio={hasRemoteAudio}
isSpeaking={props.isSpeaking} audioLevel={props.audioLevel}
/> />
</div> </div>
)} )}

View file

@ -48,7 +48,7 @@ type PropsType = {
isInSpeakerView: boolean; isInSpeakerView: boolean;
remoteParticipants: ReadonlyArray<GroupCallRemoteParticipantType>; remoteParticipants: ReadonlyArray<GroupCallRemoteParticipantType>;
setGroupCallVideoRequest: (_: Array<GroupCallVideoRequest>) => void; setGroupCallVideoRequest: (_: Array<GroupCallVideoRequest>) => void;
speakingDemuxIds: Set<number>; remoteAudioLevels: Map<number, number>;
}; };
enum VideoRequestMode { enum VideoRequestMode {
@ -86,7 +86,7 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
isInSpeakerView, isInSpeakerView,
remoteParticipants, remoteParticipants,
setGroupCallVideoRequest, setGroupCallVideoRequest,
speakingDemuxIds, remoteAudioLevels,
}) => { }) => {
const [containerDimensions, setContainerDimensions] = useState<Dimensions>({ const [containerDimensions, setContainerDimensions] = useState<Dimensions>({
width: 0, width: 0,
@ -270,7 +270,7 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
return remoteParticipantsInRow.map(remoteParticipant => { return remoteParticipantsInRow.map(remoteParticipant => {
const { demuxId, videoAspectRatio } = remoteParticipant; const { demuxId, videoAspectRatio } = remoteParticipant;
const isSpeaking = speakingDemuxIds.has(demuxId); const audioLevel = remoteAudioLevels.get(demuxId) ?? 0;
const renderedWidth = Math.floor( const renderedWidth = Math.floor(
videoAspectRatio * gridParticipantHeight videoAspectRatio * gridParticipantHeight
@ -286,7 +286,7 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
getGroupCallVideoFrameSource={getGroupCallVideoFrameSource} getGroupCallVideoFrameSource={getGroupCallVideoFrameSource}
height={gridParticipantHeight} height={gridParticipantHeight}
i18n={i18n} i18n={i18n}
isSpeaking={isSpeaking} audioLevel={audioLevel}
left={left} left={left}
remoteParticipant={remoteParticipant} remoteParticipant={remoteParticipant}
top={top} top={top}
@ -418,7 +418,7 @@ export const GroupCallRemoteParticipants: React.FC<PropsType> = ({
i18n={i18n} i18n={i18n}
onParticipantVisibilityChanged={onParticipantVisibilityChanged} onParticipantVisibilityChanged={onParticipantVisibilityChanged}
overflowedParticipants={overflowedParticipants} overflowedParticipants={overflowedParticipants}
speakingDemuxIds={speakingDemuxIds} remoteAudioLevels={remoteAudioLevels}
/> />
</div> </div>
)} )}

View file

@ -37,7 +37,6 @@ import {
} from 'ringrtc'; } from 'ringrtc';
import { uniqBy, noop } from 'lodash'; import { uniqBy, noop } from 'lodash';
import * as RemoteConfig from '../RemoteConfig';
import type { import type {
ActionsType as UxActionsType, ActionsType as UxActionsType,
GroupCallParticipantInfoType, GroupCallParticipantInfoType,
@ -63,7 +62,6 @@ import {
getAudioDeviceModule, getAudioDeviceModule,
parseAudioDeviceModule, parseAudioDeviceModule,
} from '../calling/audioDeviceModule'; } from '../calling/audioDeviceModule';
import { getAudioLevelForSpeaking } from '../calling/getAudioLevelForSpeaking';
import { import {
findBestMatchingAudioDeviceIndex, findBestMatchingAudioDeviceIndex,
findBestMatchingCameraId, findBestMatchingCameraId,
@ -684,9 +682,6 @@ export class CallingClass {
const localAudioLevel = groupCall.getLocalDeviceState().audioLevel; const localAudioLevel = groupCall.getLocalDeviceState().audioLevel;
this.uxActions?.groupCallAudioLevelsChange({ this.uxActions?.groupCallAudioLevelsChange({
audioLevelForSpeaking: getAudioLevelForSpeaking(
RemoteConfig.getValue
),
conversationId, conversationId,
localAudioLevel, localAudioLevel,
remoteDeviceStates, remoteDeviceStates,

View file

@ -15,6 +15,7 @@ import { getPlatform } from '../selectors/user';
import { isConversationTooBigToRing } from '../../conversations/isConversationTooBigToRing'; import { isConversationTooBigToRing } from '../../conversations/isConversationTooBigToRing';
import { missingCaseError } from '../../util/missingCaseError'; import { missingCaseError } from '../../util/missingCaseError';
import { calling } from '../../services/calling'; import { calling } from '../../services/calling';
import { truncateAudioLevel } from '../../calling/truncateAudioLevel';
import type { StateType as RootStateType } from '../reducer'; import type { StateType as RootStateType } from '../reducer';
import type { import type {
ChangeIODevicePayloadType, ChangeIODevicePayloadType,
@ -44,7 +45,7 @@ import { getConversationCallMode } from './conversations';
import * as log from '../../logging/log'; import * as log from '../../logging/log';
import { strictAssert } from '../../util/assert'; import { strictAssert } from '../../util/assert';
import { waitForOnline } from '../../util/waitForOnline'; import { waitForOnline } from '../../util/waitForOnline';
import * as setUtil from '../../util/setUtil'; import * as mapUtil from '../../util/mapUtil';
// State // State
@ -95,14 +96,14 @@ export type GroupCallStateType = {
joinState: GroupCallJoinState; joinState: GroupCallJoinState;
peekInfo?: GroupCallPeekInfoType; peekInfo?: GroupCallPeekInfoType;
remoteParticipants: Array<GroupCallParticipantInfoType>; remoteParticipants: Array<GroupCallParticipantInfoType>;
speakingDemuxIds?: Set<number>; remoteAudioLevels?: Map<number, number>;
} & GroupCallRingStateType; } & GroupCallRingStateType;
export type ActiveCallStateType = { export type ActiveCallStateType = {
conversationId: string; conversationId: string;
hasLocalAudio: boolean; hasLocalAudio: boolean;
hasLocalVideo: boolean; hasLocalVideo: boolean;
amISpeaking: boolean; localAudioLevel: number;
isInSpeakerView: boolean; isInSpeakerView: boolean;
joinedAt?: number; joinedAt?: number;
outgoingRing: boolean; outgoingRing: boolean;
@ -467,7 +468,6 @@ type DeclineCallActionType = {
}; };
type GroupCallAudioLevelsChangeActionPayloadType = Readonly<{ type GroupCallAudioLevelsChangeActionPayloadType = Readonly<{
audioLevelForSpeaking: number;
conversationId: string; conversationId: string;
localAudioLevel: number; localAudioLevel: number;
remoteDeviceStates: ReadonlyArray<{ audioLevel: number; demuxId: number }>; remoteDeviceStates: ReadonlyArray<{ audioLevel: number; demuxId: number }>;
@ -1456,7 +1456,7 @@ export function reducer(
conversationId: action.payload.conversationId, conversationId: action.payload.conversationId,
hasLocalAudio: action.payload.hasLocalAudio, hasLocalAudio: action.payload.hasLocalAudio,
hasLocalVideo: action.payload.hasLocalVideo, hasLocalVideo: action.payload.hasLocalVideo,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
pip: false, pip: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],
@ -1484,7 +1484,7 @@ export function reducer(
conversationId: action.payload.conversationId, conversationId: action.payload.conversationId,
hasLocalAudio: action.payload.hasLocalAudio, hasLocalAudio: action.payload.hasLocalAudio,
hasLocalVideo: action.payload.hasLocalVideo, hasLocalVideo: action.payload.hasLocalVideo,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
pip: false, pip: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],
@ -1507,7 +1507,7 @@ export function reducer(
conversationId: action.payload.conversationId, conversationId: action.payload.conversationId,
hasLocalAudio: true, hasLocalAudio: true,
hasLocalVideo: action.payload.asVideoCall, hasLocalVideo: action.payload.asVideoCall,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
pip: false, pip: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],
@ -1661,7 +1661,7 @@ export function reducer(
conversationId: action.payload.conversationId, conversationId: action.payload.conversationId,
hasLocalAudio: action.payload.hasLocalAudio, hasLocalAudio: action.payload.hasLocalAudio,
hasLocalVideo: action.payload.hasLocalVideo, hasLocalVideo: action.payload.hasLocalVideo,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
pip: false, pip: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],
@ -1719,12 +1719,7 @@ export function reducer(
} }
if (action.type === GROUP_CALL_AUDIO_LEVELS_CHANGE) { if (action.type === GROUP_CALL_AUDIO_LEVELS_CHANGE) {
const { const { conversationId, remoteDeviceStates } = action.payload;
audioLevelForSpeaking,
conversationId,
localAudioLevel,
remoteDeviceStates,
} = action.payload;
const { activeCallState } = state; const { activeCallState } = state;
const existingCall = getGroupCall(conversationId, state); const existingCall = getGroupCall(conversationId, state);
@ -1735,36 +1730,38 @@ export function reducer(
return state; return state;
} }
const amISpeaking = localAudioLevel > audioLevelForSpeaking; const localAudioLevel = truncateAudioLevel(action.payload.localAudioLevel);
const speakingDemuxIds = new Set<number>(); const remoteAudioLevels = new Map<number, number>();
remoteDeviceStates.forEach(({ audioLevel, demuxId }) => { remoteDeviceStates.forEach(({ audioLevel, demuxId }) => {
// We expect `audioLevel` to be a number but have this check just in case. // We expect `audioLevel` to be a number but have this check just in case.
if ( if (typeof audioLevel !== 'number') {
typeof audioLevel === 'number' && return;
audioLevel > audioLevelForSpeaking }
) {
speakingDemuxIds.add(demuxId); const graded = truncateAudioLevel(audioLevel);
if (graded > 0) {
remoteAudioLevels.set(demuxId, graded);
} }
}); });
// This action is dispatched frequently. This equality check helps avoid re-renders. // This action is dispatched frequently. This equality check helps avoid re-renders.
const oldAmISpeaking = activeCallState.amISpeaking; const oldLocalAudioLevel = activeCallState.localAudioLevel;
const oldSpeakingDemuxIds = existingCall.speakingDemuxIds; const oldRemoteAudioLevels = existingCall.remoteAudioLevels;
if ( if (
oldAmISpeaking === amISpeaking && oldLocalAudioLevel === localAudioLevel &&
oldSpeakingDemuxIds && oldRemoteAudioLevels &&
setUtil.isEqual(oldSpeakingDemuxIds, speakingDemuxIds) mapUtil.isEqual(oldRemoteAudioLevels, remoteAudioLevels)
) { ) {
return state; return state;
} }
return { return {
...state, ...state,
activeCallState: { ...activeCallState, amISpeaking }, activeCallState: { ...activeCallState, localAudioLevel },
callsByConversation: { callsByConversation: {
...callsByConversation, ...callsByConversation,
[conversationId]: { ...existingCall, speakingDemuxIds }, [conversationId]: { ...existingCall, remoteAudioLevels },
}, },
}; };
} }

View file

@ -130,7 +130,7 @@ const mapStateToActiveCallProp = (
conversation, conversation,
hasLocalAudio: activeCallState.hasLocalAudio, hasLocalAudio: activeCallState.hasLocalAudio,
hasLocalVideo: activeCallState.hasLocalVideo, hasLocalVideo: activeCallState.hasLocalVideo,
amISpeaking: activeCallState.amISpeaking, localAudioLevel: activeCallState.localAudioLevel,
isInSpeakerView: activeCallState.isInSpeakerView, isInSpeakerView: activeCallState.isInSpeakerView,
joinedAt: activeCallState.joinedAt, joinedAt: activeCallState.joinedAt,
outgoingRing: activeCallState.outgoingRing, outgoingRing: activeCallState.outgoingRing,
@ -263,7 +263,7 @@ const mapStateToActiveCallProp = (
maxDevices: peekInfo.maxDevices, maxDevices: peekInfo.maxDevices,
peekedParticipants, peekedParticipants,
remoteParticipants, remoteParticipants,
speakingDemuxIds: call.speakingDemuxIds || new Set<number>(), remoteAudioLevels: call.remoteAudioLevels || new Map<number, number>(),
}; };
} }
default: default:

View file

@ -4,7 +4,7 @@
import { assert } from 'chai'; import { assert } from 'chai';
import * as sinon from 'sinon'; import * as sinon from 'sinon';
import { groupBy } from '../../util/mapUtil'; import { groupBy, isEqual } from '../../util/mapUtil';
describe('map utilities', () => { describe('map utilities', () => {
describe('groupBy', () => { describe('groupBy', () => {
@ -27,4 +27,45 @@ describe('map utilities', () => {
); );
}); });
}); });
describe('isEqual', () => {
it('returns false on different maps', () => {
assert.isFalse(
isEqual<string, number>(new Map([]), new Map([['key', 1]]))
);
assert.isFalse(
isEqual<string, number>(new Map([['key', 0]]), new Map([['key', 1]]))
);
assert.isFalse(
isEqual<string, number>(
new Map([
['key', 1],
['another-key', 2],
]),
new Map([['key', 1]])
)
);
});
it('returns true on equal maps', () => {
assert.isTrue(isEqual<string, number>(new Map([]), new Map([])));
assert.isTrue(
isEqual<string, number>(new Map([['key', 1]]), new Map([['key', 1]]))
);
assert.isTrue(
isEqual<string, number>(
new Map([
['a', 1],
['b', 2],
]),
new Map([
['b', 2],
['a', 1],
])
)
);
});
});
}); });

View file

@ -18,6 +18,7 @@ import {
isAnybodyElseInGroupCall, isAnybodyElseInGroupCall,
reducer, reducer,
} from '../../../state/ducks/calling'; } from '../../../state/ducks/calling';
import { truncateAudioLevel } from '../../../calling/truncateAudioLevel';
import { calling as callingService } from '../../../services/calling'; import { calling as callingService } from '../../../services/calling';
import { import {
CallMode, CallMode,
@ -50,7 +51,7 @@ describe('calling duck', () => {
conversationId: 'fake-direct-call-conversation-id', conversationId: 'fake-direct-call-conversation-id',
hasLocalAudio: true, hasLocalAudio: true,
hasLocalVideo: false, hasLocalVideo: false,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
showParticipantsList: false, showParticipantsList: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],
@ -129,7 +130,7 @@ describe('calling duck', () => {
conversationId: 'fake-group-call-conversation-id', conversationId: 'fake-group-call-conversation-id',
hasLocalAudio: true, hasLocalAudio: true,
hasLocalVideo: false, hasLocalVideo: false,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
showParticipantsList: false, showParticipantsList: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],
@ -435,7 +436,7 @@ describe('calling duck', () => {
conversationId: 'fake-direct-call-conversation-id', conversationId: 'fake-direct-call-conversation-id',
hasLocalAudio: true, hasLocalAudio: true,
hasLocalVideo: true, hasLocalVideo: true,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
showParticipantsList: false, showParticipantsList: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],
@ -528,7 +529,7 @@ describe('calling duck', () => {
conversationId: 'fake-group-call-conversation-id', conversationId: 'fake-group-call-conversation-id',
hasLocalAudio: true, hasLocalAudio: true,
hasLocalVideo: true, hasLocalVideo: true,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
showParticipantsList: false, showParticipantsList: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],
@ -763,9 +764,16 @@ describe('calling duck', () => {
{ audioLevel: 0, demuxId: 9 }, { audioLevel: 0, demuxId: 9 },
]; ];
const remoteAudioLevels = new Map<number, number>([
[1, truncateAudioLevel(0.3)],
[2, truncateAudioLevel(0.4)],
[3, truncateAudioLevel(0.5)],
[7, truncateAudioLevel(0.2)],
[8, truncateAudioLevel(0.1)],
]);
it("does nothing if there's no relevant call", () => { it("does nothing if there's no relevant call", () => {
const action = groupCallAudioLevelsChange({ const action = groupCallAudioLevelsChange({
audioLevelForSpeaking: 0.25,
conversationId: 'garbage', conversationId: 'garbage',
localAudioLevel: 1, localAudioLevel: 1,
remoteDeviceStates, remoteDeviceStates,
@ -784,14 +792,13 @@ describe('calling duck', () => {
...stateWithActiveGroupCall.callsByConversation[ ...stateWithActiveGroupCall.callsByConversation[
'fake-group-call-conversation-id' 'fake-group-call-conversation-id'
], ],
speakingDemuxIds: new Set([3, 2, 1]), remoteAudioLevels,
}, },
}, },
}; };
const action = groupCallAudioLevelsChange({ const action = groupCallAudioLevelsChange({
audioLevelForSpeaking: 0.25,
conversationId: 'fake-group-call-conversation-id', conversationId: 'fake-group-call-conversation-id',
localAudioLevel: 0.1, localAudioLevel: 0.001,
remoteDeviceStates, remoteDeviceStates,
}); });
@ -802,21 +809,23 @@ describe('calling duck', () => {
it('updates the set of speaking participants, including yourself', () => { it('updates the set of speaking participants, including yourself', () => {
const action = groupCallAudioLevelsChange({ const action = groupCallAudioLevelsChange({
audioLevelForSpeaking: 0.25,
conversationId: 'fake-group-call-conversation-id', conversationId: 'fake-group-call-conversation-id',
localAudioLevel: 0.8, localAudioLevel: 0.8,
remoteDeviceStates, remoteDeviceStates,
}); });
const result = reducer(stateWithActiveGroupCall, action); const result = reducer(stateWithActiveGroupCall, action);
assert.isTrue(result.activeCallState?.amISpeaking); assert.strictEqual(
result.activeCallState?.localAudioLevel,
truncateAudioLevel(0.8)
);
const call = const call =
result.callsByConversation['fake-group-call-conversation-id']; result.callsByConversation['fake-group-call-conversation-id'];
if (call?.callMode !== CallMode.Group) { if (call?.callMode !== CallMode.Group) {
throw new Error('Expected a group call to be found'); throw new Error('Expected a group call to be found');
} }
assert.deepStrictEqual(call.speakingDemuxIds, new Set([1, 2, 3])); assert.deepStrictEqual(call.remoteAudioLevels, remoteAudioLevels);
}); });
}); });
@ -1112,7 +1121,7 @@ describe('calling duck', () => {
conversationId: 'fake-group-call-conversation-id', conversationId: 'fake-group-call-conversation-id',
hasLocalAudio: true, hasLocalAudio: true,
hasLocalVideo: false, hasLocalVideo: false,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
showParticipantsList: false, showParticipantsList: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],
@ -1641,7 +1650,7 @@ describe('calling duck', () => {
conversationId: 'fake-conversation-id', conversationId: 'fake-conversation-id',
hasLocalAudio: true, hasLocalAudio: true,
hasLocalVideo: true, hasLocalVideo: true,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
showParticipantsList: false, showParticipantsList: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],
@ -1927,7 +1936,7 @@ describe('calling duck', () => {
conversationId: 'fake-conversation-id', conversationId: 'fake-conversation-id',
hasLocalAudio: true, hasLocalAudio: true,
hasLocalVideo: false, hasLocalVideo: false,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
showParticipantsList: false, showParticipantsList: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],

View file

@ -51,7 +51,7 @@ describe('state/selectors/calling', () => {
conversationId: 'fake-direct-call-conversation-id', conversationId: 'fake-direct-call-conversation-id',
hasLocalAudio: true, hasLocalAudio: true,
hasLocalVideo: false, hasLocalVideo: false,
amISpeaking: false, localAudioLevel: 0,
isInSpeakerView: false, isInSpeakerView: false,
showParticipantsList: false, showParticipantsList: false,
safetyNumberChangedUuids: [], safetyNumberChangedUuids: [],

View file

@ -28,7 +28,7 @@ type ActiveCallBaseType = {
conversation: ConversationType; conversation: ConversationType;
hasLocalAudio: boolean; hasLocalAudio: boolean;
hasLocalVideo: boolean; hasLocalVideo: boolean;
amISpeaking: boolean; localAudioLevel: number;
isInSpeakerView: boolean; isInSpeakerView: boolean;
isSharingScreen?: boolean; isSharingScreen?: boolean;
joinedAt?: number; joinedAt?: number;
@ -66,7 +66,7 @@ type ActiveGroupCallType = ActiveCallBaseType & {
groupMembers: Array<Pick<ConversationType, 'id' | 'firstName' | 'title'>>; groupMembers: Array<Pick<ConversationType, 'id' | 'firstName' | 'title'>>;
peekedParticipants: Array<ConversationType>; peekedParticipants: Array<ConversationType>;
remoteParticipants: Array<GroupCallRemoteParticipantType>; remoteParticipants: Array<GroupCallRemoteParticipantType>;
speakingDemuxIds: Set<number>; remoteAudioLevels: Map<number, number>;
}; };
export type ActiveCallType = ActiveDirectCallType | ActiveGroupCallType; export type ActiveCallType = ActiveDirectCallType | ActiveGroupCallType;

View file

@ -24,3 +24,24 @@ export const groupBy = <T, ResultT>(
}, },
new Map<ResultT, Array<T>>() new Map<ResultT, Array<T>>()
); );
export const isEqual = <K, V>(
left: ReadonlyMap<K, V>,
right: ReadonlyMap<K, V>
): boolean => {
if (left.size !== right.size) {
return false;
}
for (const [key, value] of left) {
if (!right.has(key)) {
return false;
}
if (right.get(key) !== value) {
return false;
}
}
return true;
};