Native macOS Sequoia screen sharing

ACKNOWLEDGMENTS.md

@@ -63,6 +63,28 @@ Signal Desktop makes use of the following open source projects.
     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
     IN THE SOFTWARE.
 
+## @indutny/mac-screen-share
+
+    Copyright Fedor Indutny, 2024.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+
 ## @indutny/range-finder
 
     Copyright Fedor Indutny, 2022.

package-lock.json

@@ -14,6 +14,7 @@
         "@formatjs/icu-messageformat-parser": "2.3.0",
         "@formatjs/intl-localematcher": "0.2.32",
         "@indutny/dicer": "0.3.2",
+        "@indutny/mac-screen-share": "1.0.3",
         "@indutny/range-finder": "1.3.4",
         "@indutny/simple-windows-notifications": "2.0.6",
         "@indutny/sneequals": "4.0.0",
@@ -4062,6 +4063,17 @@
         "node": ">=10.0.0"
       }
     },
+    "node_modules/@indutny/mac-screen-share": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@indutny/mac-screen-share/-/mac-screen-share-1.0.3.tgz",
+      "integrity": "sha512-oBeZQGw4mvY654pc4muQUzttZvNBlnwU0ywMDFkNGYO1UdH1H20Fx+XGZbtDl1t7827U+swRNajCROuzQztUrw==",
+      "hasInstallScript": true,
+      "license": "MIT",
+      "dependencies": {
+        "bindings": "^1.5.0",
+        "node-addon-api": "*"
+      }
+    },
     "node_modules/@indutny/parallel-prettier": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/@indutny/parallel-prettier/-/parallel-prettier-3.0.0.tgz",

package.json

@@ -98,6 +98,7 @@
     "@formatjs/icu-messageformat-parser": "2.3.0",
     "@formatjs/intl-localematcher": "0.2.32",
     "@indutny/dicer": "0.3.2",
+    "@indutny/mac-screen-share": "1.0.3",
     "@indutny/range-finder": "1.3.4",
     "@indutny/simple-windows-notifications": "2.0.6",
     "@indutny/sneequals": "4.0.0",
@@ -580,8 +581,10 @@
       "node_modules/@signalapp/ringrtc/build/${platform}/*${arch}*.node",
       "node_modules/mac-screen-capture-permissions/build/Release/*.node",
       "node_modules/@indutny/simple-windows-notifications/build/Release/*.node",
+      "node_modules/@indutny/mac-screen-share/build/Release/*.node",
       "node_modules/fs-xattr/build/Release/*.node",
       "!node_modules/@indutny/simple-windows-notifications/*.cpp",
+      "!node_modules/@indutny/mac-screen-share/*.{mm,h}",
       "!node_modules/libheif-js/libheif",
       "!node_modules/libheif-js/libheif-wasm/libheif-bundle.mjs",
       "!node_modules/libheif-js/libheif-wasm/*.wasm",

scripts/esbuild.js

@@ -35,6 +35,7 @@ const bundleDefaults = {
     '@signalapp/libsignal-client/zkgroup',
     '@signalapp/ringrtc',
     '@signalapp/better-sqlite3',
+    '@indutny/mac-screen-share',
     'electron',
     'fs-xattr',
     'fsevents',

ts/state/ducks/calling.ts

@@ -185,7 +185,6 @@ export type ActiveCallStateType = {
   pip: boolean;
   presentingSource?: PresentedSource;
   presentingSourcesAvailable?: ReadonlyArray<PresentableSource>;
-  capturerBaton?: DesktopCapturerBaton;
   settingsDialogOpen: boolean;
   showNeedsScreenRecordingPermissionsWarning?: boolean;
   showParticipantsList: boolean;
@@ -216,6 +215,7 @@ export type CallingStateType = MediaDeviceSettings & {
   adhocCalls: AdhocCallsType;
   callLinks: CallLinksByRoomIdType;
   activeCallState?: ActiveCallStateType | WaitingCallStateType;
+  capturerBaton?: DesktopCapturerBaton;
 };
 
 export type AcceptCallType = ReadonlyDeep<{
@@ -649,6 +649,7 @@ const SET_LOCAL_VIDEO_FULFILLED = 'calling/SET_LOCAL_VIDEO_FULFILLED';
 const SET_OUTGOING_RING = 'calling/SET_OUTGOING_RING';
 const SET_PRESENTING = 'calling/SET_PRESENTING';
 const SET_PRESENTING_SOURCES = 'calling/SET_PRESENTING_SOURCES';
+const SET_CAPTURER_BATON = 'calling/SET_CAPTURER_BATON';
 const TOGGLE_NEEDS_SCREEN_RECORDING_PERMISSIONS =
   'calling/TOGGLE_NEEDS_SCREEN_RECORDING_PERMISSIONS';
 const START_DIRECT_CALL = 'calling/START_DIRECT_CALL';
@@ -897,10 +898,14 @@ type SetPresentingSourcesActionType = ReadonlyDeep<{
   type: 'calling/SET_PRESENTING_SOURCES';
   payload: {
     presentableSources: ReadonlyArray<PresentableSource>;
-    capturerBaton: DesktopCapturerBaton;
   };
 }>;
 
+type SetCapturerBatonActionType = ReadonlyDeep<{
+  type: 'calling/SET_CAPTURER_BATON';
+  payload: DesktopCapturerBaton;
+}>;
+
 type SetOutgoingRingActionType = ReadonlyDeep<{
   type: 'calling/SET_OUTGOING_RING';
   payload: boolean;
@@ -977,6 +982,7 @@ export type CallingActionType =
   | ReturnToActiveCallActionType
   | SendGroupCallReactionActionType
   | SelectPresentingSourceActionType
+  | SetCapturerBatonActionType
   | SetLocalAudioActionType
   | SetLocalVideoFulfilledActionType
   | SetPresentingSourcesActionType
@@ -1296,6 +1302,7 @@ function getPresentingSources(): ThunkAction<
   void,
   RootStateType,
   unknown,
+  | SetCapturerBatonActionType
   | SetPresentingSourcesActionType
   | ToggleNeedsScreenRecordingPermissionsActionType
 > {
@@ -1318,7 +1325,7 @@ function getPresentingSources(): ThunkAction<
       onPresentableSources(presentableSources) {
         if (needsPermission) {
           // Abort
-          capturer.selectSource(undefined);
+          capturer.abort();
           return;
         }
 
@@ -1326,7 +1333,6 @@ function getPresentingSources(): ThunkAction<
           type: SET_PRESENTING_SOURCES,
           payload: {
             presentableSources,
-            capturerBaton: capturer.baton,
           },
         });
       },
@@ -1353,6 +1359,11 @@ function getPresentingSources(): ThunkAction<
     });
     globalCapturers.set(capturer.baton, capturer);
 
+    dispatch({
+      type: SET_CAPTURER_BATON,
+      payload: capturer.baton,
+    });
+
     if (needsPermission) {
       dispatch({
         type: TOGGLE_NEEDS_SCREEN_RECORDING_PERMISSIONS,
@@ -2753,6 +2764,25 @@ function mergeCallWithGroupCallLookups({
   };
 }
 
+function abortCapturer(
+  state: Readonly<CallingStateType>
+): Readonly<CallingStateType> {
+  const { capturerBaton } = state;
+  if (capturerBaton == null) {
+    return state;
+  }
+
+  // Cancel source selection if running
+  const capturer = globalCapturers.get(capturerBaton);
+  strictAssert(capturer != null, 'Capturer reference exists, but not capturer');
+  capturer.abort();
+
+  return {
+    ...state,
+    capturerBaton: undefined,
+  };
+}
+
 export function reducer(
   state: Readonly<CallingStateType> = getEmptyState(),
   action: Readonly<CallingActionType>
@@ -2972,17 +3002,22 @@ export function reducer(
     action.type === HANG_UP ||
     action.type === CLOSE_NEED_PERMISSION_SCREEN
   ) {
-    const activeCall = getActiveCall(state);
+    const updatedState = abortCapturer(state);
+    const activeCall = getActiveCall(updatedState);
     if (!activeCall) {
       log.warn(`${action.type}: No active call to remove`);
-      return state;
+      return updatedState;
     }
+
     switch (activeCall.callMode) {
       case CallMode.Direct:
-        return removeConversationFromState(state, activeCall.conversationId);
+        return removeConversationFromState(
+          updatedState,
+          activeCall.conversationId
+        );
       case CallMode.Group:
       case CallMode.Adhoc:
-        return omit(state, 'activeCallState');
+        return omit(updatedState, 'activeCallState');
       default:
         throw missingCaseError(activeCall);
     }
@@ -3471,6 +3506,13 @@ export function reducer(
     };
   }
 
+  if (action.type === SET_CAPTURER_BATON) {
+    return {
+      ...abortCapturer(state),
+      capturerBaton: action.payload,
+    };
+  }
+
   if (
     action.type === SEND_GROUP_CALL_REACTION ||
     action.type === GROUP_CALL_REACTIONS_RECEIVED
@@ -3753,25 +3795,19 @@ export function reducer(
 
   if (action.type === SET_PRESENTING) {
     const { activeCallState } = state;
+
     if (activeCallState?.state !== 'Active') {
       log.warn('Cannot toggle presenting when there is no active call');
       return state;
     }
 
-    // Cancel source selection if running
-    const { capturerBaton } = activeCallState;
-    if (capturerBaton != null) {
-      const capturer = globalCapturers.get(capturerBaton);
-      capturer?.selectSource(undefined);
-    }
-
     return {
       ...state,
+      capturerBaton: undefined,
       activeCallState: {
         ...activeCallState,
         presentingSource: action.payload,
         presentingSourcesAvailable: undefined,
-        capturerBaton: undefined,
       },
     };
   }
@@ -3788,19 +3824,18 @@ export function reducer(
       activeCallState: {
         ...activeCallState,
         presentingSourcesAvailable: action.payload.presentableSources,
-        capturerBaton: action.payload.capturerBaton,
       },
     };
   }
 
   if (action.type === SELECT_PRESENTING_SOURCE) {
-    const { activeCallState } = state;
+    const { activeCallState, capturerBaton } = state;
     if (activeCallState?.state !== 'Active') {
       log.warn('Cannot set presenting sources when there is no active call');
       return state;
     }
 
-    const { capturerBaton, presentingSourcesAvailable } = activeCallState;
+    const { presentingSourcesAvailable } = activeCallState;
     if (!capturerBaton || !presentingSourcesAvailable) {
       log.warn(
         'Cannot set presenting sources when there is no presenting modal'
@@ -3817,13 +3852,13 @@ export function reducer(
 
     return {
       ...state,
+      capturerBaton: undefined,
       activeCallState: {
         ...activeCallState,
         presentingSource: presentingSourcesAvailable.find(
           source => source.id === action.payload
         ),
         presentingSourcesAvailable: undefined,
-        capturerBaton: undefined,
       },
     };
   }

ts/util/desktopCapturer.ts

@@ -1,7 +1,9 @@
 // Copyright 2024 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
+/* eslint-disable max-classes-per-file */
 
 import { ipcRenderer, type DesktopCapturerSource } from 'electron';
+import * as macScreenShare from '@indutny/mac-screen-share';
 
 import * as log from '../logging/log';
 import * as Errors from '../types/errors';
@@ -15,6 +17,14 @@ import {
 import { strictAssert } from './assert';
 import { explodePromise } from './explodePromise';
 import { isNotNil } from './isNotNil';
+import { drop } from './drop';
+
+// Chrome-only API for now, thus a declaration:
+declare class MediaStreamTrackGenerator extends MediaStreamTrack {
+  constructor(options: { kind: 'video' });
+
+  public writable: WritableStream;
+}
 
 enum Step {
   RequestingMedia = 'RequestingMedia',
@@ -24,6 +34,9 @@ enum Step {
   // Skipped on macOS Sequoia
   SelectingSource = 'SelectingSource',
   SelectedSource = 'SelectedSource',
+
+  // macOS Sequoia
+  NativeMacOS = 'NativeMacOS',
 }
 
 type State = Readonly<
@@ -41,6 +54,10 @@ type State = Readonly<
       step: Step.SelectedSource;
       promise: Promise<void>;
     }
+  | {
+      step: Step.NativeMacOS;
+      stream: macScreenShare.Stream;
+    }
   | {
       step: Step.Done;
     }
@@ -80,10 +97,29 @@ export class DesktopCapturer {
       DesktopCapturer.initialize();
     }
 
-    this.state = { step: Step.RequestingMedia, promise: this.getStream() };
+    if (macScreenShare.isSupported) {
+      this.state = {
+        step: Step.NativeMacOS,
+        stream: this.getNativeMacOSStream(),
+      };
+    } else {
+      this.state = { step: Step.RequestingMedia, promise: this.getStream() };
+    }
   }
 
-  public selectSource(id: string | undefined): void {
+  public abort(): void {
+    if (this.state.step === Step.NativeMacOS) {
+      this.state.stream.stop();
+    }
+
+    if (this.state.step === Step.SelectingSource) {
+      this.state.onSource(undefined);
+    }
+
+    this.state = { step: Step.Error };
+  }
+
+  public selectSource(id: string): void {
     strictAssert(
       this.state.step === Step.SelectingSource,
       `Invalid state in "selectSource" ${this.state.step}`
@@ -177,6 +213,59 @@ export class DesktopCapturer {
     }
   }
 
+  private getNativeMacOSStream(): macScreenShare.Stream {
+    const track = new MediaStreamTrackGenerator({ kind: 'video' });
+    const writer = track.writable.getWriter();
+
+    const mediaStream = new MediaStream();
+    mediaStream.addTrack(track);
+
+    let isRunning = false;
+
+    const stream = new macScreenShare.Stream({
+      width: REQUESTED_VIDEO_WIDTH,
+      height: REQUESTED_VIDEO_HEIGHT,
+      frameRate: REQUESTED_VIDEO_FRAMERATE,
+
+      onStart: () => {
+        isRunning = true;
+
+        this.options.onMediaStream(mediaStream);
+      },
+      onStop() {
+        if (!isRunning) {
+          return;
+        }
+        isRunning = false;
+
+        if (track.readyState === 'ended') {
+          stream.stop();
+          return;
+        }
+        drop(writer.close());
+      },
+      onFrame(frame, width, height) {
+        if (!isRunning) {
+          return;
+        }
+        if (track.readyState === 'ended') {
+          stream.stop();
+          return;
+        }
+
+        const videoFrame = new VideoFrame(frame, {
+          format: 'NV12',
+          codedWidth: width,
+          codedHeight: height,
+          timestamp: 0,
+        });
+        drop(writer.write(videoFrame));
+      },
+    });
+
+    return stream;
+  }
+
   private translateSourceName(source: DesktopCapturerSource): string {
     const { i18n } = this.options;