From 69e1326b40508d480a38dc87b6389903bb3aba66 Mon Sep 17 00:00:00 2001 From: automated-signal <37887102+automated-signal@users.noreply.github.com> Date: Thu, 26 Sep 2024 18:47:44 -0500 Subject: [PATCH] Native macOS Sequoia screen sharing Co-authored-by: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com> --- ACKNOWLEDGMENTS.md | 22 +++++++++ package-lock.json | 12 +++++ package.json | 3 ++ scripts/esbuild.js | 1 + ts/state/ducks/calling.ts | 75 ++++++++++++++++++++++-------- ts/util/desktopCapturer.ts | 93 +++++++++++++++++++++++++++++++++++++- 6 files changed, 184 insertions(+), 22 deletions(-) diff --git a/ACKNOWLEDGMENTS.md b/ACKNOWLEDGMENTS.md index 038f95ca12..281efdb729 100644 --- a/ACKNOWLEDGMENTS.md +++ b/ACKNOWLEDGMENTS.md @@ -63,6 +63,28 @@ Signal Desktop makes use of the following open source projects. FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +## @indutny/mac-screen-share + + Copyright Fedor Indutny, 2024. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + ## @indutny/range-finder Copyright Fedor Indutny, 2022. diff --git a/package-lock.json b/package-lock.json index 8969a56ba3..4001e1eee5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@formatjs/icu-messageformat-parser": "2.3.0", "@formatjs/intl-localematcher": "0.2.32", "@indutny/dicer": "0.3.2", + "@indutny/mac-screen-share": "1.0.3", "@indutny/range-finder": "1.3.4", "@indutny/simple-windows-notifications": "2.0.6", "@indutny/sneequals": "4.0.0", @@ -4062,6 +4063,17 @@ "node": ">=10.0.0" } }, + "node_modules/@indutny/mac-screen-share": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@indutny/mac-screen-share/-/mac-screen-share-1.0.3.tgz", + "integrity": "sha512-oBeZQGw4mvY654pc4muQUzttZvNBlnwU0ywMDFkNGYO1UdH1H20Fx+XGZbtDl1t7827U+swRNajCROuzQztUrw==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "node-addon-api": "*" + } + }, "node_modules/@indutny/parallel-prettier": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@indutny/parallel-prettier/-/parallel-prettier-3.0.0.tgz", diff --git a/package.json b/package.json index 7555fa9dae..43288346cc 100644 --- a/package.json +++ b/package.json @@ -98,6 +98,7 @@ "@formatjs/icu-messageformat-parser": "2.3.0", "@formatjs/intl-localematcher": "0.2.32", "@indutny/dicer": "0.3.2", + "@indutny/mac-screen-share": "1.0.3", "@indutny/range-finder": "1.3.4", "@indutny/simple-windows-notifications": "2.0.6", "@indutny/sneequals": "4.0.0", @@ -580,8 +581,10 @@ "node_modules/@signalapp/ringrtc/build/${platform}/*${arch}*.node", "node_modules/mac-screen-capture-permissions/build/Release/*.node", "node_modules/@indutny/simple-windows-notifications/build/Release/*.node", + "node_modules/@indutny/mac-screen-share/build/Release/*.node", "node_modules/fs-xattr/build/Release/*.node", "!node_modules/@indutny/simple-windows-notifications/*.cpp", + "!node_modules/@indutny/mac-screen-share/*.{mm,h}", "!node_modules/libheif-js/libheif", "!node_modules/libheif-js/libheif-wasm/libheif-bundle.mjs", "!node_modules/libheif-js/libheif-wasm/*.wasm", diff --git a/scripts/esbuild.js b/scripts/esbuild.js index d767e163f8..791d0de78a 100644 --- a/scripts/esbuild.js +++ b/scripts/esbuild.js @@ -35,6 +35,7 @@ const bundleDefaults = { '@signalapp/libsignal-client/zkgroup', '@signalapp/ringrtc', '@signalapp/better-sqlite3', + '@indutny/mac-screen-share', 'electron', 'fs-xattr', 'fsevents', diff --git a/ts/state/ducks/calling.ts b/ts/state/ducks/calling.ts index 1bdc78614c..5e668ed6e2 100644 --- a/ts/state/ducks/calling.ts +++ b/ts/state/ducks/calling.ts @@ -185,7 +185,6 @@ export type ActiveCallStateType = { pip: boolean; presentingSource?: PresentedSource; presentingSourcesAvailable?: ReadonlyArray; - capturerBaton?: DesktopCapturerBaton; settingsDialogOpen: boolean; showNeedsScreenRecordingPermissionsWarning?: boolean; showParticipantsList: boolean; @@ -216,6 +215,7 @@ export type CallingStateType = MediaDeviceSettings & { adhocCalls: AdhocCallsType; callLinks: CallLinksByRoomIdType; activeCallState?: ActiveCallStateType | WaitingCallStateType; + capturerBaton?: DesktopCapturerBaton; }; export type AcceptCallType = ReadonlyDeep<{ @@ -649,6 +649,7 @@ const SET_LOCAL_VIDEO_FULFILLED = 'calling/SET_LOCAL_VIDEO_FULFILLED'; const SET_OUTGOING_RING = 'calling/SET_OUTGOING_RING'; const SET_PRESENTING = 'calling/SET_PRESENTING'; const SET_PRESENTING_SOURCES = 'calling/SET_PRESENTING_SOURCES'; +const SET_CAPTURER_BATON = 'calling/SET_CAPTURER_BATON'; const TOGGLE_NEEDS_SCREEN_RECORDING_PERMISSIONS = 'calling/TOGGLE_NEEDS_SCREEN_RECORDING_PERMISSIONS'; const START_DIRECT_CALL = 'calling/START_DIRECT_CALL'; @@ -897,10 +898,14 @@ type SetPresentingSourcesActionType = ReadonlyDeep<{ type: 'calling/SET_PRESENTING_SOURCES'; payload: { presentableSources: ReadonlyArray; - capturerBaton: DesktopCapturerBaton; }; }>; +type SetCapturerBatonActionType = ReadonlyDeep<{ + type: 'calling/SET_CAPTURER_BATON'; + payload: DesktopCapturerBaton; +}>; + type SetOutgoingRingActionType = ReadonlyDeep<{ type: 'calling/SET_OUTGOING_RING'; payload: boolean; @@ -977,6 +982,7 @@ export type CallingActionType = | ReturnToActiveCallActionType | SendGroupCallReactionActionType | SelectPresentingSourceActionType + | SetCapturerBatonActionType | SetLocalAudioActionType | SetLocalVideoFulfilledActionType | SetPresentingSourcesActionType @@ -1296,6 +1302,7 @@ function getPresentingSources(): ThunkAction< void, RootStateType, unknown, + | SetCapturerBatonActionType | SetPresentingSourcesActionType | ToggleNeedsScreenRecordingPermissionsActionType > { @@ -1318,7 +1325,7 @@ function getPresentingSources(): ThunkAction< onPresentableSources(presentableSources) { if (needsPermission) { // Abort - capturer.selectSource(undefined); + capturer.abort(); return; } @@ -1326,7 +1333,6 @@ function getPresentingSources(): ThunkAction< type: SET_PRESENTING_SOURCES, payload: { presentableSources, - capturerBaton: capturer.baton, }, }); }, @@ -1353,6 +1359,11 @@ function getPresentingSources(): ThunkAction< }); globalCapturers.set(capturer.baton, capturer); + dispatch({ + type: SET_CAPTURER_BATON, + payload: capturer.baton, + }); + if (needsPermission) { dispatch({ type: TOGGLE_NEEDS_SCREEN_RECORDING_PERMISSIONS, @@ -2753,6 +2764,25 @@ function mergeCallWithGroupCallLookups({ }; } +function abortCapturer( + state: Readonly +): Readonly { + const { capturerBaton } = state; + if (capturerBaton == null) { + return state; + } + + // Cancel source selection if running + const capturer = globalCapturers.get(capturerBaton); + strictAssert(capturer != null, 'Capturer reference exists, but not capturer'); + capturer.abort(); + + return { + ...state, + capturerBaton: undefined, + }; +} + export function reducer( state: Readonly = getEmptyState(), action: Readonly @@ -2972,17 +3002,22 @@ export function reducer( action.type === HANG_UP || action.type === CLOSE_NEED_PERMISSION_SCREEN ) { - const activeCall = getActiveCall(state); + const updatedState = abortCapturer(state); + const activeCall = getActiveCall(updatedState); if (!activeCall) { log.warn(`${action.type}: No active call to remove`); - return state; + return updatedState; } + switch (activeCall.callMode) { case CallMode.Direct: - return removeConversationFromState(state, activeCall.conversationId); + return removeConversationFromState( + updatedState, + activeCall.conversationId + ); case CallMode.Group: case CallMode.Adhoc: - return omit(state, 'activeCallState'); + return omit(updatedState, 'activeCallState'); default: throw missingCaseError(activeCall); } @@ -3471,6 +3506,13 @@ export function reducer( }; } + if (action.type === SET_CAPTURER_BATON) { + return { + ...abortCapturer(state), + capturerBaton: action.payload, + }; + } + if ( action.type === SEND_GROUP_CALL_REACTION || action.type === GROUP_CALL_REACTIONS_RECEIVED @@ -3753,25 +3795,19 @@ export function reducer( if (action.type === SET_PRESENTING) { const { activeCallState } = state; + if (activeCallState?.state !== 'Active') { log.warn('Cannot toggle presenting when there is no active call'); return state; } - // Cancel source selection if running - const { capturerBaton } = activeCallState; - if (capturerBaton != null) { - const capturer = globalCapturers.get(capturerBaton); - capturer?.selectSource(undefined); - } - return { ...state, + capturerBaton: undefined, activeCallState: { ...activeCallState, presentingSource: action.payload, presentingSourcesAvailable: undefined, - capturerBaton: undefined, }, }; } @@ -3788,19 +3824,18 @@ export function reducer( activeCallState: { ...activeCallState, presentingSourcesAvailable: action.payload.presentableSources, - capturerBaton: action.payload.capturerBaton, }, }; } if (action.type === SELECT_PRESENTING_SOURCE) { - const { activeCallState } = state; + const { activeCallState, capturerBaton } = state; if (activeCallState?.state !== 'Active') { log.warn('Cannot set presenting sources when there is no active call'); return state; } - const { capturerBaton, presentingSourcesAvailable } = activeCallState; + const { presentingSourcesAvailable } = activeCallState; if (!capturerBaton || !presentingSourcesAvailable) { log.warn( 'Cannot set presenting sources when there is no presenting modal' @@ -3817,13 +3852,13 @@ export function reducer( return { ...state, + capturerBaton: undefined, activeCallState: { ...activeCallState, presentingSource: presentingSourcesAvailable.find( source => source.id === action.payload ), presentingSourcesAvailable: undefined, - capturerBaton: undefined, }, }; } diff --git a/ts/util/desktopCapturer.ts b/ts/util/desktopCapturer.ts index 16a7db7380..0e9b528342 100644 --- a/ts/util/desktopCapturer.ts +++ b/ts/util/desktopCapturer.ts @@ -1,7 +1,9 @@ // Copyright 2024 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only +/* eslint-disable max-classes-per-file */ import { ipcRenderer, type DesktopCapturerSource } from 'electron'; +import * as macScreenShare from '@indutny/mac-screen-share'; import * as log from '../logging/log'; import * as Errors from '../types/errors'; @@ -15,6 +17,14 @@ import { import { strictAssert } from './assert'; import { explodePromise } from './explodePromise'; import { isNotNil } from './isNotNil'; +import { drop } from './drop'; + +// Chrome-only API for now, thus a declaration: +declare class MediaStreamTrackGenerator extends MediaStreamTrack { + constructor(options: { kind: 'video' }); + + public writable: WritableStream; +} enum Step { RequestingMedia = 'RequestingMedia', @@ -24,6 +34,9 @@ enum Step { // Skipped on macOS Sequoia SelectingSource = 'SelectingSource', SelectedSource = 'SelectedSource', + + // macOS Sequoia + NativeMacOS = 'NativeMacOS', } type State = Readonly< @@ -41,6 +54,10 @@ type State = Readonly< step: Step.SelectedSource; promise: Promise; } + | { + step: Step.NativeMacOS; + stream: macScreenShare.Stream; + } | { step: Step.Done; } @@ -80,10 +97,29 @@ export class DesktopCapturer { DesktopCapturer.initialize(); } - this.state = { step: Step.RequestingMedia, promise: this.getStream() }; + if (macScreenShare.isSupported) { + this.state = { + step: Step.NativeMacOS, + stream: this.getNativeMacOSStream(), + }; + } else { + this.state = { step: Step.RequestingMedia, promise: this.getStream() }; + } } - public selectSource(id: string | undefined): void { + public abort(): void { + if (this.state.step === Step.NativeMacOS) { + this.state.stream.stop(); + } + + if (this.state.step === Step.SelectingSource) { + this.state.onSource(undefined); + } + + this.state = { step: Step.Error }; + } + + public selectSource(id: string): void { strictAssert( this.state.step === Step.SelectingSource, `Invalid state in "selectSource" ${this.state.step}` @@ -177,6 +213,59 @@ export class DesktopCapturer { } } + private getNativeMacOSStream(): macScreenShare.Stream { + const track = new MediaStreamTrackGenerator({ kind: 'video' }); + const writer = track.writable.getWriter(); + + const mediaStream = new MediaStream(); + mediaStream.addTrack(track); + + let isRunning = false; + + const stream = new macScreenShare.Stream({ + width: REQUESTED_VIDEO_WIDTH, + height: REQUESTED_VIDEO_HEIGHT, + frameRate: REQUESTED_VIDEO_FRAMERATE, + + onStart: () => { + isRunning = true; + + this.options.onMediaStream(mediaStream); + }, + onStop() { + if (!isRunning) { + return; + } + isRunning = false; + + if (track.readyState === 'ended') { + stream.stop(); + return; + } + drop(writer.close()); + }, + onFrame(frame, width, height) { + if (!isRunning) { + return; + } + if (track.readyState === 'ended') { + stream.stop(); + return; + } + + const videoFrame = new VideoFrame(frame, { + format: 'NV12', + codedWidth: width, + codedHeight: height, + timestamp: 0, + }); + drop(writer.write(videoFrame)); + }, + }); + + return stream; + } + private translateSourceName(source: DesktopCapturerSource): string { const { i18n } = this.options;