Populate localVideoStreamState state with video effects status

change-beta/@azure-communication-react-e89997fe-4d19-4ee4-9b83-746b5bda4520.json

@@ -0,0 +1,7 @@
+{
+  "type": "prerelease",
+  "comment": "In the stateful call client subscribe to video effects changes and populate state when those subscriptions fire'",
+  "packageName": "@azure/communication-react",
+  "email": "2684369+JamesBurnside@users.noreply.github.com",
+  "dependentChangeType": "patch"
+}

change/@azure-communication-react-e89997fe-4d19-4ee4-9b83-746b5bda4520.json

@@ -0,0 +1,7 @@
+{
+  "type": "prerelease",
+  "comment": "In the stateful call client subscribe to video effects changes and populate state when those subscriptions fire'",
+  "packageName": "@azure/communication-react",
+  "email": "2684369+JamesBurnside@users.noreply.github.com",
+  "dependentChangeType": "patch"
+}

packages/calling-stateful-client/src/CallContext.ts

@@ -37,6 +37,12 @@ import {
 } from './CallClientState';
 import { callingStatefulLogger } from './Logger';
 import { CallIdHistory } from './CallIdHistory';
+/* @conditional-compile-remove(video-background-effects) */
+import { LocalVideoStreamVideoEffectsState } from './CallClientState';
+/* @conditional-compile-remove(video-background-effects) */
+import { LocalVideoStreamVideoEffectsSubscriber } from './LocalVideoStreamVideoEffectsSubscriber';
+/* @conditional-compile-remove(video-background-effects) */
+import { VideoEffectsFeature } from '@azure/communication-calling';
 
 enableMapSet();
 // Needed to generate state diff for verbose logging.
@@ -57,6 +63,8 @@ export class CallContext {
   private _emitter: EventEmitter;
   private _atomicId: number;
   private _callIdHistory: CallIdHistory = new CallIdHistory();
+  /* @conditional-compile-remove(video-background-effects) */
+  private _unparentedViewVideoEffectsSubscriber: LocalVideoStreamVideoEffectsSubscriber | undefined;
 
   constructor(
     userId: CommunicationIdentifierKind,
@@ -262,6 +270,26 @@ export class CallContext {
     });
   }
 
+  /* @conditional-compile-remove(video-background-effects) */
+  public setCallLocalVideoStreamVideoEffects(
+    callId: string,
+    localVideoStream: LocalVideoStreamState,
+    videoEffects: Partial<LocalVideoStreamVideoEffectsState>
+  ): void {
+    this.modifyState((draft: CallClientState) => {
+      const call = draft.calls[this._callIdHistory.latestCallId(callId)];
+      if (call) {
+        const stream = call.localVideoStreams?.find((i) => i.mediaStreamType === 'Video');
+        if (stream) {
+          stream.videoEffects = {
+            isActive: videoEffects.isActive ?? stream.videoEffects?.isActive ?? false,
+            effectName: videoEffects.effectName ?? stream.videoEffects?.effectName
+          };
+        }
+      }
+    });
+  }
+
   public setCallIsMicrophoneMuted(callId: string, isMicrophoneMuted: boolean): void {
     this.modifyState((draft: CallClientState) => {
       const call = draft.calls[this._callIdHistory.latestCallId(callId)];
@@ -625,20 +653,36 @@ export class CallContext {
     });
   }
 
-  public setDeviceManagerUnparentedView(
-    localVideoStream: LocalVideoStreamState,
-    view: VideoStreamRendererViewState | undefined
-  ): void {
+  public setDeviceManagerUnparentedView(args: {
+    localVideoStream: LocalVideoStreamState;
+    view: VideoStreamRendererViewState | undefined;
+    /* @conditional-compile-remove(video-background-effects) */
+    localVideoStreamEffectsAPI: VideoEffectsFeature;
+  }): void {
+    /* @conditional-compile-remove(video-background-effects) */
+    {
+      this._unparentedViewVideoEffectsSubscriber?.unsubscribe();
+      this._unparentedViewVideoEffectsSubscriber = new LocalVideoStreamVideoEffectsSubscriber({
+        parent: 'unparented',
+        context: this,
+        localVideoStream: args.localVideoStream,
+        localVideoStreamEffectsAPI: args.localVideoStreamEffectsAPI
+      });
+    }
+
     this.modifyState((draft: CallClientState) => {
       draft.deviceManager.unparentedViews.push({
-        source: localVideoStream.source,
-        mediaStreamType: localVideoStream.mediaStreamType,
-        view: view
+        source: args.localVideoStream.source,
+        mediaStreamType: args.localVideoStream.mediaStreamType,
+        view: args.view
       });
     });
   }
 
   public deleteDeviceManagerUnparentedView(localVideoStream: LocalVideoStreamState): void {
+    /* @conditional-compile-remove(video-background-effects) */
+    this._unparentedViewVideoEffectsSubscriber?.unsubscribe();
+
     this.modifyState((draft: CallClientState) => {
       const foundIndex = draft.deviceManager.unparentedViews.findIndex(
         (stream) =>
@@ -650,6 +694,26 @@ export class CallContext {
     });
   }
 
+  /* @conditional-compile-remove(video-background-effects) */
+  public setDeviceManagerUnparentedViewVideoEffects(
+    localVideoStream: LocalVideoStreamState,
+    videoEffects: LocalVideoStreamVideoEffectsState
+  ): void {
+    this.modifyState((draft: CallClientState) => {
+      const foundIndex = draft.deviceManager.unparentedViews.findIndex(
+        (stream) =>
+          stream.source.id === localVideoStream.source.id && stream.mediaStreamType === localVideoStream.mediaStreamType
+      );
+      if (foundIndex !== -1) {
+        const draftStream = draft.deviceManager.unparentedViews[foundIndex];
+        draftStream.videoEffects = {
+          isActive: videoEffects.isActive ?? draftStream.videoEffects?.isActive ?? false,
+          effectName: videoEffects.effectName ?? draftStream.videoEffects?.effectName
+        };
+      }
+    });
+  }
+
   public getAndIncrementAtomicId(): number {
     const id = this._atomicId;
     this._atomicId++;

packages/calling-stateful-client/src/CallSubscriber.ts

@@ -11,6 +11,8 @@ import {
   convertSdkParticipantToDeclarativeParticipant
 } from './Converter';
 import { InternalCallContext } from './InternalCallContext';
+/* @conditional-compile-remove(video-background-effects) */
+import { LocalVideoStreamVideoEffectsSubscriber } from './LocalVideoStreamVideoEffectsSubscriber';
 import { ParticipantSubscriber } from './ParticipantSubscriber';
 import { RecordingSubscriber } from './RecordingSubscriber';
 import { disposeView } from './StreamUtils';
@@ -32,6 +34,8 @@ export class CallSubscriber {
   private _participantSubscribers: Map<string, ParticipantSubscriber>;
   private _recordingSubscriber: RecordingSubscriber;
   private _transcriptionSubscriber: TranscriptionSubscriber;
+  /* @conditional-compile-remove(video-background-effects) */
+  private _localVideoStreamVideoEffectsSubscribers: Map<string, LocalVideoStreamVideoEffectsSubscriber>;
 
   constructor(call: CallCommon, context: CallContext, internalContext: InternalCallContext) {
     this._call = call;
@@ -55,6 +59,8 @@ export class CallSubscriber {
       this._context,
       this._call.feature(Features.Transcription)
     );
+    /* @conditional-compile-remove(video-background-effects) */
+    this._localVideoStreamVideoEffectsSubscribers = new Map();
 
     this.subscribe();
   }
@@ -218,8 +224,28 @@ export class CallSubscriber {
         undefined
       );
       this._context.setCallLocalVideoStream(this._callIdRef.callId, [...localVideoStreams]);
+
+      /* @conditional-compile-remove(video-background-effects) */
+      {
+        const localVideoStreamKey = event.added[0].source.id;
+        this._localVideoStreamVideoEffectsSubscribers.get(localVideoStreamKey)?.unsubscribe();
+        this._localVideoStreamVideoEffectsSubscribers.set(
+          localVideoStreamKey,
+          new LocalVideoStreamVideoEffectsSubscriber({
+            parent: this._callIdRef,
+            context: this._context,
+            localVideoStream: this._call.localVideoStreams[0],
+            localVideoStreamEffectsAPI: this._call.localVideoStreams[0].feature(Features.VideoEffects)
+          })
+        );
+      }
     }
     if (event.removed.length > 0) {
+      /* @conditional-compile-remove(video-background-effects) */
+      {
+        const localVideoStreamKey = event.removed[0].source.id;
+        this._localVideoStreamVideoEffectsSubscribers.get(localVideoStreamKey)?.unsubscribe();
+      }
       disposeView(
         this._context,
         this._internalContext,

packages/calling-stateful-client/src/Converter.ts

@@ -32,6 +32,7 @@ export function convertSdkLocalStreamToDeclarativeLocalStream(
     source: stream.source,
     mediaStreamType: stream.mediaStreamType,
     view: undefined
+    // TODO [video-background-effects]: Add video effects state when it is added to the SDK
   };
 }
 

packages/calling-stateful-client/src/LocalVideoStreamVideoEffectsSubscriber.ts

@@ -0,0 +1,85 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+/* @conditional-compile-remove(video-background-effects) */
+import { VideoEffectErrorPayload, VideoEffectName, VideoEffectsFeature } from '@azure/communication-calling';
+/* @conditional-compile-remove(video-background-effects) */
+import { LocalVideoStreamState, LocalVideoStreamVideoEffectsState } from './CallClientState';
+/* @conditional-compile-remove(video-background-effects) */
+import { CallContext } from './CallContext';
+/* @conditional-compile-remove(video-background-effects) */
+import { CallIdRef } from './CallIdRef';
+
+/* @conditional-compile-remove(video-background-effects) */
+/**
+ * Subscribes to a LocalVideoStream's video effects events and updates the call context appropriately.
+ * @private
+ */
+export class LocalVideoStreamVideoEffectsSubscriber {
+  private _parent: CallIdRef | 'unparented';
+  private _context: CallContext;
+  private _localVideoStream: LocalVideoStreamState;
+  private _localVideoStreamEffectsAPI: VideoEffectsFeature;
+
+  constructor(args: {
+    /** Owner of the local video stream. This is either the Call (referenced by CallIdRef) or is the device manager's unparented view (referenced by 'unparented') */
+    parent: CallIdRef | 'unparented';
+    context: CallContext;
+    localVideoStream: LocalVideoStreamState;
+    localVideoStreamEffectsAPI: VideoEffectsFeature;
+  }) {
+    this._parent = args.parent;
+    this._context = args.context;
+    this._localVideoStream = args.localVideoStream;
+    this._localVideoStreamEffectsAPI = args.localVideoStreamEffectsAPI;
+
+    this.subscribe();
+  }
+
+  private subscribe = (): void => {
+    this._localVideoStreamEffectsAPI.on('effectsStarted', this.effectsStarted);
+    this._localVideoStreamEffectsAPI.on('effectsStopped', this.effectsStopped);
+    this._localVideoStreamEffectsAPI.on('effectsError', this.effectsError);
+  };
+
+  public unsubscribe = (): void => {
+    this._localVideoStreamEffectsAPI.off('effectsStarted', this.effectsStarted);
+    this._localVideoStreamEffectsAPI.off('effectsStopped', this.effectsStopped);
+    this._localVideoStreamEffectsAPI.off('effectsError', this.effectsError);
+  };
+
+  private effectsStarted = (effectName: VideoEffectName): void => {
+    this.updateEffectsState({
+      isActive: true,
+      effectName: effectName
+    });
+  };
+
+  private effectsStopped = (effectName: VideoEffectName): void => {
+    this.updateEffectsState({
+      isActive: false,
+      effectName: effectName
+    });
+  };
+
+  private effectsError = (error: VideoEffectErrorPayload): void => {
+    // When there is an error the effects have stopped. Update the state to reflect this.
+    this.updateEffectsState({
+      isActive: false
+    });
+    // TODO [video-background-effects]: future PR will tee error to state
+    console.error(
+      `LocalVideoStreamVideoEffectsSubscriber effectError: Method not fully implemented. Error needs teed to state. ${error}`
+    );
+  };
+
+  private updateEffectsState = (newEffectsState: LocalVideoStreamVideoEffectsState): void => {
+    if (this._parent === 'unparented') {
+      this._context.setDeviceManagerUnparentedViewVideoEffects(this._localVideoStream, newEffectsState);
+    } else {
+      this._context.setCallLocalVideoStreamVideoEffects(this._parent.callId, this._localVideoStream, newEffectsState);
+    }
+  };
+}
+
+export {};

packages/calling-stateful-client/src/StatefulCallClient.test.ts

@@ -6,7 +6,6 @@ import {
   DeviceManager,
   UserFacingDiagnosticsFeature,
   Features,
-  LocalVideoStream,
   RecordingCallFeature,
   TranscriptionCallFeature,
   VideoStreamRendererView,
@@ -25,6 +24,7 @@ import {
   createMockCall,
   createMockCallAgent,
   createMockCallClient,
+  createMockLocalVideoStream,
   createMockRemoteParticipant,
   createMockRemoteScreenshareStream,
   createMockRemoteVideoStream,
@@ -237,7 +237,7 @@ describe('Stateful call client', () => {
 
     {
       const listener = new StateChangeListener(client);
-      call.testHelperPushLocalVideoStream({} as LocalVideoStream);
+      call.testHelperPushLocalVideoStream(createMockLocalVideoStream());
       expect(await waitWithBreakCondition(() => client.getState().calls[callId]?.localVideoStreams.length !== 0)).toBe(
         true
       );
@@ -346,7 +346,7 @@ describe('Stateful call client', () => {
     const participantKey = toFlatCommunicationIdentifier(participant.identifier);
 
     // Add a local video stream as well.
-    call.testHelperPushLocalVideoStream({} as LocalVideoStream);
+    call.testHelperPushLocalVideoStream(createMockLocalVideoStream());
     expect(await waitWithBreakCondition(() => client.getState().calls[callId]?.localVideoStreams.length !== 0)).toBe(
       true
     );
@@ -638,15 +638,15 @@ describe('errors should be reported correctly from Call when', () => {
 
     {
       const listener = new StateChangeListener(client);
-      await expect(call.startVideo({} as LocalVideoStream)).rejects.toThrow(
+      await expect(call.startVideo(createMockLocalVideoStream())).rejects.toThrow(
         new CallError('Call.startVideo', new Error('startVideo: injected error'))
       );
       expect(listener.onChangeCalledCount).toBe(1);
       expect(client.getState().latestErrors['Call.startVideo']).toBeDefined();
     }
     {
       const listener = new StateChangeListener(client);
-      await expect(call.stopVideo({} as LocalVideoStream)).rejects.toThrow(
+      await expect(call.stopVideo(createMockLocalVideoStream())).rejects.toThrow(
         new CallError('Call.stopVideo', new Error('stopVideo: injected error'))
       );
       expect(listener.onChangeCalledCount).toBe(1);
@@ -715,7 +715,7 @@ const prepareCall = async (): Promise<PreparedCall> => {
 const prepareCallWithLocalVideoStream = async (): Promise<PreparedCall> => {
   const prepared = await prepareCall();
   const { client, callId, call } = prepared;
-  call.testHelperPushLocalVideoStream({} as LocalVideoStream);
+  call.testHelperPushLocalVideoStream(createMockLocalVideoStream());
   expect(await waitWithBreakCondition(() => client.getState().calls[callId]?.localVideoStreams.length !== 0)).toBe(
     true
   );

packages/calling-stateful-client/src/StreamUtils.test.ts

@@ -17,7 +17,7 @@ import { CallState, LocalVideoStreamState, RemoteParticipantState, RemoteVideoSt
 import { CallContext } from './CallContext';
 import { InternalCallContext } from './InternalCallContext';
 import { createView, disposeView, disposeAllViewsFromCall, disposeAllViews } from './StreamUtils';
-import { createMockRemoteVideoStream } from './TestUtils';
+import { createMockLocalVideoStream, createMockRemoteVideoStream } from './TestUtils';
 import { toFlatCommunicationIdentifier } from '@internal/acs-ui-common';
 
 jest.mock('@azure/communication-calling', () => {
@@ -27,18 +27,7 @@ jest.mock('@azure/communication-calling', () => {
     }),
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     LocalVideoStream: jest.fn().mockImplementation((info: VideoDeviceInfo) => {
-      return {
-        source: () => {
-          return {} as VideoDeviceInfo;
-        },
-        mediaStreamType: () => {
-          return 'Video';
-        },
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        switchSource: (videoDeviceInfo: VideoDeviceInfo) => {
-          return Promise.resolve();
-        }
-      };
+      return createMockLocalVideoStream();
     }),
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     VideoStreamRenderer: jest.fn().mockImplementation((videoStream: SdkLocalVideoStream | SdkRemoteVideoStream) => {
@@ -50,7 +39,10 @@ jest.mock('@azure/communication-calling', () => {
         // eslint-disable-next-line @typescript-eslint/no-empty-function
         dispose: () => {}
       };
-    })
+    }),
+    Features: {
+      VideoEffects: undefined
+    }
   };
 });