Merge branch 'master' of https://github.com/MyPureCloud/genesys-cloud-webrtc-sdk into WC-801

Author: djhouseknecht

changelog.md

@@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 # [Unreleased](https://github.com/MyPureCloud/genesys-cloud-webrtc-sdk/compare/v4.1.0...HEAD)
 
+### Fixed
+* [PCM-1428](https://inindca.atlassian.net/browse/PCM-1428) – in chrome, if you were using the system default audio device and then changed the default on the system level, the sdk would not start a audio with the new system default. 
+
+### Added 
+* [PCM-1426](https://inindca.atlassian.net/browse/PCM-1426) – better logging around devices. It will log devices in use we a session starts and when the devices change. 
+
+
 # [v4.1.0](https://github.com/MyPureCloud/genesys-cloud-webrtc-sdk/compare/v4.0.1...v4.1.0)
 ### Added
 * added client logger to spigot tests for test debugability

package.json

@@ -32,6 +32,7 @@
     "test:unit": "NODE_ENV=test jest --runInBand",
     "test:watch": "jest --watch --collectCoverage=false --runInBand",
     "lint": "npm run tslint",
+    "lint:fix": "npm run tslint:fix",
     "tslint": "tslint --project . --config tslint.json",
     "tslint:fix": "tslint --project . --config tslint.json --fix",
     "greenkeep": "npx npm-check --update",
@@ -102,4 +103,4 @@
     "ws": "^7.1.0"
   },
   "false": {}
-}
+}

src/media-utils.ts

@@ -1,7 +1,7 @@
 import browserama from 'browserama';
 
 import { GenesysCloudWebrtcSdk } from './client';
-import { IMediaRequestOptions, IEnumeratedDevices } from './types/interfaces';
+import { IMediaRequestOptions, IEnumeratedDevices, IJingleSession } from './types/interfaces';
 import { SdkErrorTypes } from './types/enums';
 import { throwSdkError } from './utils';
 
@@ -227,28 +227,131 @@ export const stopListeningForDeviceChanges = function () {
   navigator.mediaDevices.removeEventListener('devicechange', handleDeviceChange);
 };
 
+/**
+ * Look through the cached enumerated devices and match based on
+ * the passed in track's `kind` and `label`
+ * @param track media stream track with the label to search for
+ */
+export const findCachedDeviceByTrackLabel = (track?: MediaStreamTrack): MediaDeviceInfo | undefined => {
+  if (!track?.kind) return;
+  const availableDevices = getCachedEnumeratedDevices();
+  const devicesToSearch = track.kind === 'video' ? availableDevices.videoDevices : availableDevices.audioDevices;
+  return devicesToSearch.find(d => d.label === track.label);
+};
+
+/**
+ * Look through the cached enumerated devices and match based on
+ *  the passed in output deviceId
+ * @param id output deviceId
+ */
+export const findCachedOutputDeviceById = (id?: string): MediaDeviceInfo | undefined => {
+  return getCachedEnumeratedDevices().outputDevices.find(d => d.deviceId === id);
+};
+
+/**
+ * Utility to log device changes. It will use the passed in `from` track _or_
+ *  look up the currently used devices via the sender (based on labels) to see
+ *  what device is currently in use. Then it will log out the device changing `to`
+ *  as the new device.
+ *
+ * NOTE: if the system default is being used and then changes (which will force
+ *  devices to be enumerated) the device will not be able to be looked up in the
+ *  cached devices. So the caller will need to pass in the "old" system default(s)
+ *  as `fromVideoTrack` and/or `fromAudioTrack`.
+ *
+ * @param sdk sdk instance
+ * @param session session devices are changing for
+ * @param action action taken
+ * @param devicesChange devices changing to/from
+ */
+export function logDeviceChange (
+  sdk: GenesysCloudWebrtcSdk,
+  session: IJingleSession,
+  action: 'sessionStarted' | 'changingDevices',
+  devicesChange: {
+    toVideoTrack?: MediaStreamTrack,
+    fromVideoTrack?: MediaStreamTrack,
+    toAudioTrack?: MediaStreamTrack,
+    fromAudioTrack?: MediaStreamTrack,
+    toOutputDeviceId?: string
+  } = {}
+): void {
+  let currentVideoTrack: MediaStreamTrack;
+  let currentAudioTrack: MediaStreamTrack;
+  const currentOutputDeviceId: string = session._outputAudioElement?.sinkId;
+  const screenShareTrackId = session._screenShareStream?.getVideoTracks()[0]?.id;
+
+  /* grab the currect device being used */
+  session.pc.getSenders()
+    .filter(s => s.track && s.track.id !== screenShareTrackId)
+    .forEach(sender => {
+      if (sender.track.kind === 'audio') {
+        currentAudioTrack = sender.track;
+      } else /* if (sender.track.kind === 'video') */ {
+        currentVideoTrack = sender.track;
+      }
+    });
+
+  const availableDevices = getCachedEnumeratedDevices();
+  const details = {
+    /* meta data */
+    action,
+    availableDevices,
+    sessionId: session.id,
+    conversationId: session.conversationId,
+    /* the device being switched from and/or currently being used.
+      if a track was passed in, we will assume the caller knows what it is doing
+      and we will use that track for logging. Otherwise, we will look up the device */
+    currentVideoDevice: devicesChange.fromVideoTrack ? { deviceId: undefined, groupId: undefined, label: devicesChange.fromVideoTrack.label } : findCachedDeviceByTrackLabel(currentVideoTrack),
+    currentAudioDevice: devicesChange.fromAudioTrack ? { deviceId: undefined, groupId: undefined, label: devicesChange.fromAudioTrack.label } : findCachedDeviceByTrackLabel(currentAudioTrack),
+    currentOutputDevice: findCachedOutputDeviceById(currentOutputDeviceId),
+    /* the device being switched to */
+    newVideoDevice: findCachedDeviceByTrackLabel(devicesChange.toVideoTrack),
+    newAudioDevice: findCachedDeviceByTrackLabel(devicesChange.toAudioTrack),
+    newOutputDevice: findCachedOutputDeviceById(devicesChange.toOutputDeviceId),
+    /* other potentially useful information to log */
+    sessionVideoMute: session.videoMuted,
+    sessionAudioMute: session.audioMuted,
+    hasDevicePermissions: hasDevicePermissions(availableDevices)
+  };
+
+  sdk.logger.info('media devices changing for session', details);
+}
+
+/**
+ * Get the currently cached enumerated devices
+ */
+export function getCachedEnumeratedDevices (): IEnumeratedDevices {
+  /* return a copy of cached devices */
+  return {
+    audioDevices: enumeratedDevices.audioDevices.slice(),
+    videoDevices: enumeratedDevices.videoDevices.slice(),
+    outputDevices: enumeratedDevices.outputDevices.slice()
+  };
+}
+
 export async function getEnumeratedDevices (sdk: GenesysCloudWebrtcSdk, forceRefresh: boolean = false): Promise<IEnumeratedDevices> {
   if (!window.navigator.mediaDevices || !window.navigator.mediaDevices.enumerateDevices) {
     sdk.logger.warn('Unable to enumerate devices');
     enumeratedDevices.videoDevices = [];
     enumeratedDevices.audioDevices = [];
     enumeratedDevices.outputDevices = [];
-    return enumeratedDevices;
+    return getCachedEnumeratedDevices();
   }
 
   if (!isListeningForDeviceChanges) {
     navigator.mediaDevices.addEventListener('devicechange', handleDeviceChange.bind(sdk));
     isListeningForDeviceChanges = true;
   }
 
-  /* if devices haven't changed, no forceRefresh, & we have permission - return the cache devices */
+  /* if devices haven't changed, no forceRefresh, & we have permissions - return the cache devices */
   if (!refreshDevices && !forceRefresh && hasDevicePermissions(enumeratedDevices)) {
     sdk.logger.debug('Returning cached enumerated devices', { devices: enumeratedDevices });
-    return enumeratedDevices;
+    return getCachedEnumeratedDevices();
   }
 
   try {
-    const oldDevices = { ...enumeratedDevices };
+    const oldDevices = getCachedEnumeratedDevices();
     enumeratedDevices.videoDevices = [];
     enumeratedDevices.audioDevices = [];
     enumeratedDevices.outputDevices = [];
@@ -271,7 +374,8 @@ export async function getEnumeratedDevices (sdk: GenesysCloudWebrtcSdk, forceRef
   }
 
   sdk.logger.debug('Enumerated devices', { devices: enumeratedDevices });
-  return enumeratedDevices;
+  /* the "cache" was just updated with the new devices, this will make a copy */
+  return getCachedEnumeratedDevices();
 }
 
 function mapOldToNewDevices (oldEnumeratedDevices: IEnumeratedDevices | undefined, newDevices: MediaDeviceInfo[]): MediaDeviceInfo[] {

src/sessions/base-session-handler.ts

@@ -4,10 +4,9 @@ import { GenesysCloudWebrtcSdk } from '../client';
 import { LogLevels, SessionTypes, SdkErrorTypes } from '../types/enums';
 import { SessionManager } from './session-manager';
 import { IPendingSession, IStartSessionParams, IAcceptSessionRequest, ISessionMuteRequest, IJingleSession, IUpdateOutgoingMedia, IJingleReason } from '../types/interfaces';
-import { checkHasTransceiverFunctionality, startMedia } from '../media-utils';
+import { checkHasTransceiverFunctionality, startMedia, logDeviceChange } from '../media-utils';
 import { throwSdkError } from '../utils';
 import { ConversationUpdate } from '../types/conversation-update';
-import browserama from 'browserama';
 
 type ExtendedHTMLAudioElement = HTMLAudioElement & {
   setSinkId (deviceId: string): Promise<undefined>;
@@ -171,6 +170,7 @@ export default abstract class BaseSessionHandler {
     if (session._screenShareStream) {
       trackIdsToIgnore.push(...session._screenShareStream.getTracks().map((track) => track.id));
     }
+
     /*
       if we aren't updating a media type, leave the existing track(s) alone
       also true if our video is on mute, we don't need to touch it
@@ -187,13 +187,23 @@ export default abstract class BaseSessionHandler {
         )
       );
 
-    /* stop media first because FF does not allow more than one audio/video track */
-    if (browserama.isFirefox) {
-      senders.forEach(sender => {
-        sender.track.stop();
-        destroyMediaPromises.push(sender.replaceTrack(null));
-      });
-    }
+    /*
+      stop media first because FF does not allow more than one audio/video track
+      AND stop the media in case we are using system default because chrome will not let you switch
+      system default devices _while_ you have an active media track with the old system default
+    */
+    let fromVideoTrack: MediaStreamTrack;
+    let fromAudioTrack: MediaStreamTrack;
+
+    senders.forEach(sender => {
+      if (sender.track.kind === 'video') {
+        fromVideoTrack = sender.track;
+      } else {
+        fromAudioTrack = sender.track;
+      }
+      sender.track.stop();
+      destroyMediaPromises.push(sender.replaceTrack(null));
+    });
 
     await Promise.all(destroyMediaPromises);
 
@@ -239,6 +249,15 @@ export default abstract class BaseSessionHandler {
       }
     });
 
+    /* just in case the system default is changing, we need to tell this log function what was the
+      previous device being used (since it won't show up in the enumerated device list anymore) */
+    logDeviceChange(this.sdk, session, 'changingDevices', {
+      fromVideoTrack: fromVideoTrack,
+      fromAudioTrack: fromAudioTrack,
+      toVideoTrack: stream.getVideoTracks()[0],
+      toAudioTrack: stream.getAudioTracks()[0]
+    });
+
     const newMediaPromises: Promise<any>[] = stream.getTracks().map(async track => {
       await this.addReplaceTrackToSession(session, track);
       outboundStream.addTrack(track);
@@ -275,7 +294,7 @@ export default abstract class BaseSessionHandler {
       throwSdkError.call(this.sdk, SdkErrorTypes.not_supported, err, { conversationId: session.conversationId, sessionId: session.id });
     }
 
-    this.log(LogLevels.info, 'Setting output deviceId', { deviceId, conversationId: session.conversationId, sessionId: session.id });
+    logDeviceChange(this.sdk, session, 'changingDevices', { toOutputDeviceId: deviceId });
     return el.setSinkId(deviceId);
   }
 
@@ -337,11 +356,11 @@ export default abstract class BaseSessionHandler {
   }
 
   _warnNegotiationNeeded (session: IJingleSession): void {
-    this.log(LogLevels.error, 'negotiation needed and not supported', { conversationId: session.conversationId });
+    this.log(LogLevels.error, 'negotiation needed and not supported', { conversationId: session.conversationId, sessionId: session.id });
   }
 
   removeMediaFromSession (session: IJingleSession, track: MediaStreamTrack): Promise<void> {
-    this.log(LogLevels.debug, 'Removing track from session', { track, conversationId: session.conversationId });
+    this.log(LogLevels.debug, 'Removing track from session', { track, conversationId: session.conversationId, sessionId: session.id });
     return session.removeTrack(track);
   }
 

src/sessions/softphone-session-handler.ts

@@ -1,7 +1,7 @@
 import BaseSessionHandler from './base-session-handler';
 import { IPendingSession, IAcceptSessionRequest, ISessionMuteRequest, IConversationParticipant, IJingleSession } from '../types/interfaces';
 import { SessionTypes, LogLevels, SdkErrorTypes } from '../types/enums';
-import { attachAudioMedia, startMedia } from '../media-utils';
+import { attachAudioMedia, startMedia, logDeviceChange } from '../media-utils';
 import { requestApi, throwSdkError, isSoftphoneJid } from '../utils';
 import { pick } from 'lodash';
 
@@ -47,7 +47,8 @@ export default class SoftphoneSessionHandler extends BaseSessionHandler {
       });
     }
 
-    return super.acceptSession(session, params);
+    await super.acceptSession(session, params);
+    logDeviceChange(this.sdk, session, 'sessionStarted');
   }
 
   async endSession (session: IJingleSession): Promise<void> {

src/sessions/video-session-handler.ts

@@ -15,7 +15,7 @@ import {
 } from '../types/interfaces';
 import BaseSessionHandler from './base-session-handler';
 import { SessionTypes, LogLevels, SdkErrorTypes, CommunicationStates } from '../types/enums';
-import { createNewStreamWithTrack, startMedia, startDisplayMedia, getEnumeratedDevices } from '../media-utils';
+import { createNewStreamWithTrack, startMedia, startDisplayMedia, getEnumeratedDevices, logDeviceChange } from '../media-utils';
 import { throwSdkError, requestApi, isVideoJid, isPeerVideoJid } from '../utils';
 import { ConversationUpdate } from '../types/conversation-update';
 
@@ -308,6 +308,8 @@ export default class VideoSessionHandler extends BaseSessionHandler {
 
     await super.acceptSession(session, params);
     this.setInitialMuteStates(session);
+
+    logDeviceChange(this.sdk, session, 'sessionStarted');
   }
 
   setInitialMuteStates (session: IJingleSession): void {
@@ -318,13 +320,17 @@ export default class VideoSessionHandler extends BaseSessionHandler {
       session.videoMuted = true;
       this.log(LogLevels.info, 'Sending initial video mute', { conversationId: session.conversationId });
       session.mute(userId, 'video');
+    } else {
+      session.videoMuted = false;
     }
 
     const audioSender = session.pc.getSenders().find((sender) => sender.track && sender.track.kind === 'audio');
     if (!audioSender || !audioSender.track.enabled) {
       session.audioMuted = true;
       this.log(LogLevels.info, 'Sending initial audio mute', { conversationId: session.conversationId });
       session.mute(userId, 'audio');
+    } else {
+      session.audioMuted = false;
     }
   }
 

test/test-utils.ts

@@ -162,6 +162,9 @@ class MockStream {
   getVideoTracks () {
     return this.getTracks().filter((t) => t.kind === 'video');
   }
+  getAudioTracks () {
+    return this.getTracks().filter((t) => t.kind === 'audio');
+  }
   getTracks () {
     return this._tracks;
   }