STREAM-1153: use multiple streams instead of multiple tracks

Author: Aidan Zimmermann

src/sessions/live-monitoring-session-handler.ts

@@ -71,46 +71,34 @@ export class LiveMonitoringSessionHandler extends BaseSessionHandler {
   }
 
   async acceptSessionForObserver(session: LiveScreenMonitoringSession, params: IAcceptSessionRequest) {
-    const videoElement = params.videoElement || this.sdk._config.defaults.videoElement;
+    const videoElements = params.videoElements || (params.videoElement ? [params.videoElement] : [this.sdk._config.defaults.videoElement].filter(Boolean));
     const sessionInfo = { conversationId: session.conversationId, sessionId: session.id };
 
-    if (!videoElement) {
+    if (!videoElements.length) {
       throw createAndEmitSdkError.call(this.sdk, SdkErrorTypes.invalid_options,
-        'acceptSession for live monitoring observer requires a videoElement to be provided or in the default config',
+        'acceptSession for live monitoring observer requires videoElements array or videoElement to be provided or in the default config',
         sessionInfo);
     }
 
-    const attachParams = { videoElement };
-
-    const handleIncomingTracks = (session: IExtendedMediaSession, tracks: MediaStreamTrack | MediaStreamTrack[]) => {
-      if (!Array.isArray(tracks)) tracks = [tracks];
-
-      for (const track of tracks) {
-        this.log('info', 'Incoming track from live monitoring session', {
-          track,
-          conversationId: session.conversationId,
-          sessionId: session.id,
-          sessionType: session.sessionType
-        });
-
-        this.attachIncomingTrackToElement(track, attachParams);
-      }
+    // Use mediaStreams if provided
+    if (params.mediaStreams && params.mediaStreams.length > 0) {
+      this.log('info', `Attaching ${params.mediaStreams.length} media streams to ${videoElements.length} video elements`, sessionInfo);
+
+      params.mediaStreams.forEach((mediaStreamItem, index) => {
+        if (index < videoElements.length) {
+          const videoElement = videoElements[index];
+          videoElement.muted = true;
+          videoElement.autoplay = true;
+          videoElement.srcObject = mediaStreamItem.stream;
+          this.log('info', `Attached media stream to video element at index ${index}`, {
+            streamId: mediaStreamItem.stream.id,
+            metadata: mediaStreamItem.metadata,
+            ...sessionInfo
+          });
+        }
+      });
 
       session.emit('incomingMedia');
-    };
-
-    // Get existing tracks
-    const tracks = session.pc.getReceivers()
-      .filter(receiver => receiver.track)
-      .map(receiver => receiver.track);
-
-    if (tracks.length) {
-      handleIncomingTracks(session, tracks);
-    } else {
-      // Listen for tracks that arrive later
-      session.on('peerTrackAdded', (track: MediaStreamTrack) => {
-        handleIncomingTracks(session, track);
-      });
     }
   }
 
@@ -129,29 +117,6 @@ export class LiveMonitoringSessionHandler extends BaseSessionHandler {
     this.log('warn', 'Cannot update outgoing media for live monitoring sessions', { sessionId: session.id, sessionType: session.sessionType });
     throw createAndEmitSdkError.call(this.sdk, SdkErrorTypes.not_supported, 'Cannot update outgoing media for live monitoring sessions');
   }
-
-  /**
-   * Attach incoming track to HTML element
-   */
-  attachIncomingTrackToElement(
-    track: MediaStreamTrack,
-    { videoElement }: { videoElement: HTMLVideoElement }
-  ): HTMLAudioElement | HTMLVideoElement {
-    const element = videoElement;
-
-    if (track.kind === 'video') {
-      if (element) {
-        element.muted = true;
-      }
-    }
-
-    if (element) {
-      element.autoplay = true;
-      element.srcObject = createNewStreamWithTrack(track);
-    }
-
-    return element;
-  }
 }
 
 export default LiveMonitoringSessionHandler;

src/types/interfaces.ts

@@ -667,6 +667,11 @@ export interface IAcceptSessionRequest extends ISdkMediaDeviceIds {
    */
   mediaStream?: MediaStream;
 
+  /**
+   * array of media streams with associated metadata
+   */
+  mediaStreams?: { stream: MediaStream; metadata: ScreenRecordingMetadata }[];
+
   /**
    * metadata about screens and tracks. This is required for screen recording sessions
    */
@@ -678,6 +683,9 @@ export interface IAcceptSessionRequest extends ISdkMediaDeviceIds {
   /** video element to attach incoming video to. default is sdk `defaults.videoElement` */
   videoElement?: HTMLVideoElement;
 
+  /** array of video elements for live monitoring observers to attach multiple video streams */
+  videoElements?: HTMLVideoElement[];
+
   /** Flag set to true when the participant is a monitoring observer. default is `false` */
   liveMonitoringObserver?: boolean
 }

test/unit/sessions/live-monitoring-session-handler.test.ts

@@ -62,7 +62,7 @@ describe('handlePropose', () => {
 
     mockSdk._config.autoAcceptPendingLiveScreenMonitoringRequests = false;
     const pendingSession = { fromUserId: 'user123' } as any;
-    
+
     await handler.handlePropose(pendingSession);
 
     expect(handler._liveMonitoringObserver).toBe(false);
@@ -76,7 +76,7 @@ describe('handlePropose', () => {
 
     mockSdk._config.autoAcceptPendingLiveScreenMonitoringRequests = false;
     const pendingSession = { fromUserId: 'different-user' } as any;
-    
+
     await handler.handlePropose(pendingSession);
 
     expect(handler._liveMonitoringObserver).toBe(false);
@@ -157,55 +157,113 @@ describe('acceptSessionForTarget', () => {
 });
 
 describe('acceptSessionForObserver', () => {
-  let parentHandlerSpy: jest.SpyInstance<Promise<any>>;
-  let addMediaToSessionSpy: jest.SpyInstance<Promise<void>>;
-  let attachIncomingTrackToElementSpy: jest.SpyInstance<HTMLAudioElement>;
-  let startMediaSpy: jest.SpyInstance<Promise<MediaStream>>;
-  let initialMutesSpy: jest.SpyInstance<Promise<any>>; /* keep this spy */
   let session: LiveScreenMonitoringSession;
-  let media: MediaStream;
 
   beforeEach(() => {
-    media = new MockStream() as any;
-    parentHandlerSpy = jest.spyOn(BaseSessionHandler.prototype, 'acceptSession').mockResolvedValue(null);
-    attachIncomingTrackToElementSpy = jest.spyOn(handler, 'attachIncomingTrackToElement').mockReturnValue({} as HTMLMediaElement);
-    startMediaSpy = jest.spyOn(mockSdk.media, 'startMedia').mockResolvedValue(media);
     session = new MockSession() as any;
   });
 
   it('should throw if no video element is provided', async () => {
-    await expect(handler.acceptSession(session, { conversationId: session.conversationId, liveMonitoringObserver: true, audioElement: document.createElement('audio') })).rejects.toThrowError(/requires a videoElement/);
+    await expect(handler.acceptSession(session, { conversationId: session.conversationId, liveMonitoringObserver: true, audioElement: document.createElement('audio') })).rejects.toThrowError(/requires videoElements array or videoElement/);
+  });
+
+  it('should attach mediaStreams to video elements when provided', async () => {
+    const video1 = document.createElement('video');
+    const video2 = document.createElement('video');
+    const videoElements = [video1, video2];
+
+    const stream1 = new MockStream({ video: true }) as any;
+    const stream2 = new MockStream({ video: true }) as any;
+    const metadata1 = { screenId: 'screen1', trackId: 'track1', originX: 0, originY: 0, resolutionX: 1920, resolutionY: 1080, primary: true };
+    const metadata2 = { screenId: 'screen2', trackId: 'track2', originX: 1920, originY: 0, resolutionX: 1920, resolutionY: 1080, primary: false };
+
+    const mediaStreams = [
+      { stream: stream1, metadata: metadata1 },
+      { stream: stream2, metadata: metadata2 }
+    ];
+
+    const emitSpy = jest.spyOn(session, 'emit');
+
+    await handler.acceptSession(session, {
+      conversationId: session.conversationId,
+      liveMonitoringObserver: true,
+      videoElements,
+      mediaStreams
+    });
+
+    expect(video1.srcObject).toBe(stream1);
+    expect(video1.muted).toBe(true);
+    expect(video1.autoplay).toBe(true);
+
+    expect(video2.srcObject).toBe(stream2);
+    expect(video2.muted).toBe(true);
+    expect(video2.autoplay).toBe(true);
+
+    expect(emitSpy).toHaveBeenCalledWith('incomingMedia');
   });
 
-  it('should use default elements', async () => {
-    const video = mockSdk._config.defaults!.videoElement = document.createElement('video');
+  it('should only attach streams up to the number of available video elements', async () => {
+    const video1 = document.createElement('video');
+    const videoElements = [video1]; // Only one video element
 
-    const incomingTrack = {} as any;
-    jest.spyOn(session.pc, 'getReceivers').mockReturnValue([{ track: incomingTrack }] as any);
+    const stream1 = new MockStream({ video: true }) as any;
+    const stream2 = new MockStream({ video: true }) as any;
+    const metadata = { screenId: 'screen1', trackId: 'track1', originX: 0, originY: 0, resolutionX: 1920, resolutionY: 1080, primary: true };
 
-    attachIncomingTrackToElementSpy.mockReturnValue(video);
+    const mediaStreams = [
+      { stream: stream1, metadata },
+      { stream: stream2, metadata } // This won't be attached
+    ];
 
-    await handler.acceptSession(session, { conversationId: session.conversationId, liveMonitoringObserver: true });
+    await handler.acceptSession(session, {
+      conversationId: session.conversationId,
+      liveMonitoringObserver: true,
+      videoElements,
+      mediaStreams
+    });
 
-    expect(parentHandlerSpy).toHaveBeenCalled();
-    expect(attachIncomingTrackToElementSpy).toHaveBeenCalledWith(incomingTrack, { videoElement: video });
+    expect(video1.srcObject).toBe(stream1);
+    // stream2 should not be attached anywhere
   });
 
+  it('should use videoElement field when no videoElements provided ', async () => {
+    const videoElement = document.createElement('video');
+
+    const stream1 = new MockStream({ video: true }) as any;
+    const stream2 = new MockStream({ video: true }) as any;
+    const metadata = { screenId: 'screen1', trackId: 'track1', originX: 0, originY: 0, resolutionX: 1920, resolutionY: 1080, primary: true };
 
-  it('should attach tracks later if not available', async () => {
-    const video = document.createElement('video');
+    const mediaStreams = [
+      { stream: stream1, metadata },
+      { stream: stream2, metadata } // This won't be attached
+    ];
 
-    attachIncomingTrackToElementSpy.mockReturnValue(video);
+    await handler.acceptSession(session, {
+      conversationId: session.conversationId,
+      liveMonitoringObserver: true,
+      videoElement,
+      mediaStreams
+    });
 
-    await handler.acceptSession(session, { conversationId: session.conversationId, liveMonitoringObserver: true, videoElement: video });
-    expect(attachIncomingTrackToElementSpy).not.toHaveBeenCalled();
-    expect(parentHandlerSpy).toHaveBeenCalled();
+    expect(videoElement.srcObject).toBe(stream1);
+    // stream2 should not be attached anywhere
+  });
 
-    const incomingTrack = {} as any;
+  it('should use default video element when no videoElements or videoElement provided', async () => {
+    const defaultVideo = document.createElement('video');
+    mockSdk._config.defaults!.videoElement = defaultVideo;
 
-    session.emit('peerTrackAdded', incomingTrack);
+    const stream = new MockStream({ video: true }) as any;
+    const metadata = { screenId: 'screen1', trackId: 'track1', originX: 0, originY: 0, resolutionX: 1920, resolutionY: 1080, primary: true };
+    const mediaStreams = [{ stream, metadata }];
 
-    expect(attachIncomingTrackToElementSpy).toHaveBeenCalledWith(incomingTrack, { videoElement: video });
+    await handler.acceptSession(session, {
+      conversationId: session.conversationId,
+      liveMonitoringObserver: true,
+      mediaStreams
+    });
+
+    expect(defaultVideo.srcObject).toBe(stream);
   });
 });
 
@@ -243,20 +301,3 @@ describe('updateOutgoingMedia', () => {
     );
   });
 });
-
-describe('attachIncomingTrackToElement', () => {
-  it('should attach to video element', () => {
-    const video = document.createElement('video');
-
-    const track = new MockTrack();
-    track.kind = 'video';
-    const fakeStream = {};
-    jest.spyOn(mediaUtils, 'createNewStreamWithTrack').mockReturnValue(fakeStream as any);
-
-    handler.attachIncomingTrackToElement(track as any, { videoElement: video });
-
-    expect(video.srcObject).toBe(fakeStream);
-    expect(video.autoplay).toBeTruthy();
-    expect(video.muted).toBeTruthy();
-  });
-});