Merge pull request #976 from MyPureCloud/STREAM-1153-support-multiple-screens

STREAM-1153: support multiple screens

Author: AidanWZ

changelog.md

@@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * [STREAM-1178](https://inindca.atlassian.net/browse/STREAM-1178) - REVERT STREAM-825, removed ability to send multiple tracks when screensharing.
 * [STREAM-1130](https://inindca.atlassian.net/browse/STREAM-1130) - Updated TypeScript to v5.9.3 and Webpack to v5.105.0, removed legacy Spigot testing, added `.nvmrc`, demo app dependency updates, updated Node versions in testing matrices.
 
+### Added
+* [STREAM-1153](https://inindca.atlassian.net/browse/STREAM-1153) - Add support for monitoring multiple screens during live monitoring sessions
+
 # [v11.5.1](https://github.com/MyPureCloud/genesys-cloud-webrtc-sdk/compare/v11.5.0...v11.5.1)
 ### Added
 * [STREAM-1034](https://inindca.atlassian.net/browse/STREAM-1034) - Added JWT support for live monitoring sessions

src/sessions/live-monitoring-session-handler.ts

@@ -63,54 +63,73 @@ export class LiveMonitoringSessionHandler extends BaseSessionHandler {
     let addMediaPromise: Promise<any> = Promise.resolve();
     params.mediaStream.getTracks().forEach((track) => {
       addMediaPromise = addMediaPromise.then(() => {
-        this.sdk.logger.info('Adding screen track to live screen monitoring session', { trackId: track.id, label: track.label, conversationId: session.conversationId, sessionType: this.sessionType });
-        return session.pc.addTrack(track);
+        const trackStream = createNewStreamWithTrack(track);
+        this.sdk.logger.info('Adding screen track to live screen monitoring session', { streamId: trackStream.id, trackId: track.id, label: track.label, conversationId: session.conversationId, sessionType: this.sessionType });
+        return session.pc.addTrack(track, trackStream);
       });
     });
     await addMediaPromise;
+
+    // Set unused video transceivers direction: inactive to not get sent to the observer clients
+    const unusedTransceivers = session.pc.getTransceivers()
+      .filter(transceiver => transceiver.receiver.track?.kind === 'video' && !transceiver.sender.track)
+    this.sdk.logger.info(`Setting ${unusedTransceivers.length} unused video transceivers to inactive`, { conversationId: session.conversationId, sessionId: session.id });
+    unusedTransceivers.forEach(transceiver => {
+      transceiver.direction = "inactive";
+    });
   }
 
   async acceptSessionForObserver(session: LiveScreenMonitoringSession, params: IAcceptSessionRequest) {
-    const videoElement = params.videoElement || this.sdk._config.defaults.videoElement;
+    const videoElements = params.videoElements || (params.videoElement ? [params.videoElement] : [this.sdk._config.defaults.videoElement].filter(Boolean));
     const sessionInfo = { conversationId: session.conversationId, sessionId: session.id };
 
-    if (!videoElement) {
+    if (!videoElements.length) {
       throw createAndEmitSdkError.call(this.sdk, SdkErrorTypes.invalid_options,
-        'acceptSession for live monitoring observer requires a videoElement to be provided or in the default config',
+        'acceptSession for live monitoring observer requires videoElements array or videoElement to be provided or in the default config',
         sessionInfo);
     }
 
-    const attachParams = { videoElement };
-
-    const handleIncomingTracks = (session: IExtendedMediaSession, tracks: MediaStreamTrack | MediaStreamTrack[]) => {
-      if (!Array.isArray(tracks)) tracks = [tracks];
-
-      for (const track of tracks) {
-        this.log('info', 'Incoming track from live monitoring session', {
-          track,
-          conversationId: session.conversationId,
-          sessionId: session.id,
-          sessionType: session.sessionType
+    const tracks = session.pc.getReceivers()
+      .filter(receiver => receiver.track)
+      .map(receiver => receiver.track)
+      .filter(track => track.kind === 'video');
+
+    this.log('info', `Accepting live screen monitoring session as observer with ${videoElements.length} available video elements for ${session.pc.getReceivers().length} receivers with ${tracks.length} video tracks`);
+
+    try {
+      let addEmptyMediaPromise: Promise<any> = Promise.resolve();
+      tracks.forEach((targetTrack) => {
+        addEmptyMediaPromise = addEmptyMediaPromise.then(() => {
+          const canvas = document.createElement('canvas');
+          canvas.width = 1;
+          canvas.height = 1;
+          const emptyStream = canvas.captureStream(0);
+          const emptyVideoTrack = emptyStream.getVideoTracks()[0];
+          this.sdk.logger.info('Adding empty screen track to live screen monitoring session', { streamId: emptyStream.id, trackId: emptyVideoTrack.id, label: emptyVideoTrack.label, conversationId: session.conversationId, sessionType: this.sessionType });
+          return session.pc.addTrack(emptyVideoTrack, emptyStream);
         });
+      });
+      await addEmptyMediaPromise;
+    } catch (error: any) {
+      this.sdk.logger.error('Error when adding empty video streams', error);
+    }
 
-        this.attachIncomingTrackToElement(track, attachParams);
+    let videoElementIndex = 0;
+    for (const track of tracks) {
+      if (videoElementIndex < videoElements.length) {
+        const stream = createNewStreamWithTrack(track);
+        const videoElement = videoElements[videoElementIndex];
+        videoElement.muted = true;
+        videoElement.autoplay = true;
+        videoElement.srcObject = stream;
+        this.log('info', `Attached stream to video element at index ${videoElementIndex}`, {
+          streamId: videoElement.srcObject?.id,
+          track: track,
+          ...sessionInfo,
+        });
+        videoElementIndex++;
       }
-
       session.emit('incomingMedia');
-    };
-
-    // Get existing tracks
-    const tracks = session.pc.getReceivers()
-      .filter(receiver => receiver.track)
-      .map(receiver => receiver.track);
-
-    if (tracks.length) {
-      handleIncomingTracks(session, tracks);
-    } else {
-      // Listen for tracks that arrive later
-      session.on('peerTrackAdded', (track: MediaStreamTrack) => {
-        handleIncomingTracks(session, track);
-      });
     }
   }
 
@@ -129,29 +148,6 @@ export class LiveMonitoringSessionHandler extends BaseSessionHandler {
     this.log('warn', 'Cannot update outgoing media for live monitoring sessions', { sessionId: session.id, sessionType: session.sessionType });
     throw createAndEmitSdkError.call(this.sdk, SdkErrorTypes.not_supported, 'Cannot update outgoing media for live monitoring sessions');
   }
-
-  /**
-   * Attach incoming track to HTML element
-   */
-  attachIncomingTrackToElement(
-    track: MediaStreamTrack,
-    { videoElement }: { videoElement: HTMLVideoElement }
-  ): HTMLAudioElement | HTMLVideoElement {
-    const element = videoElement;
-
-    if (track.kind === 'video') {
-      if (element) {
-        element.muted = true;
-      }
-    }
-
-    if (element) {
-      element.autoplay = true;
-      element.srcObject = createNewStreamWithTrack(track);
-    }
-
-    return element;
-  }
 }
 
 export default LiveMonitoringSessionHandler;

src/types/interfaces.ts

@@ -678,6 +678,9 @@ export interface IAcceptSessionRequest extends ISdkMediaDeviceIds {
   /** video element to attach incoming video to. default is sdk `defaults.videoElement` */
   videoElement?: HTMLVideoElement;
 
+  /** array of video elements for live monitoring observers to attach multiple video streams */
+  videoElements?: HTMLVideoElement[];
+
   /** Flag set to true when the participant is a monitoring observer. default is `false` */
   liveMonitoringObserver?: boolean
 }