Emit integration invoke error metric for destination load failures (#1341)

Author: MichaelGHSeg

.changeset/silent-destinations-metrics.md

@@ -0,0 +1,5 @@
+---
+'@segment/analytics-next': patch
+---
+
+Emit analytics_js.integration.invoke.error metric for destination load and build failures that were previously silent

packages/browser/src/browser/index.ts

@@ -173,7 +173,10 @@ async function registerPlugins(
     options,
     tsubMiddleware,
     pluginSources
-  ).catch(() => [])
+  ).catch((err) => {
+    console.error('Failed to load remote plugins', err)
+    return [] as Plugin[]
+  })
 
   const basePlugins = [envEnrichment, ...legacyDestinations, ...remotePlugins]
 

packages/browser/src/plugins/ajs-destination/__tests__/index.test.ts

@@ -11,6 +11,8 @@ import { AMPLITUDE_WRITEKEY } from '../../../test-helpers/test-writekeys'
 import { PersistedPriorityQueue } from '../../../lib/priority-queue/persisted'
 import * as Factory from '../../../test-helpers/factories'
 import { cdnSettingsMinimal } from '../../../test-helpers/fixtures'
+import * as MetricHelpers from '../../../core/stats/metric-helpers'
+import * as ScriptLoader from '../../../lib/load-script'
 
 const cdnResponse: CDNSettings = {
   ...cdnSettingsMinimal,
@@ -838,3 +840,65 @@ describe('option overrides', () => {
     )
   })
 })
+
+describe('load error metrics', () => {
+  let metricSpy: jest.SpyInstance
+
+  beforeEach(() => {
+    jest.restoreAllMocks()
+    jest.resetAllMocks()
+    metricSpy = jest.spyOn(MetricHelpers, 'recordIntegrationMetric')
+  })
+
+  it('records integration invoke error metric when loadIntegration fails', async () => {
+    jest
+      .spyOn(ScriptLoader, 'loadScript')
+      .mockRejectedValue(new Error('Script load failed'))
+
+    const dest = new LegacyDestination(
+      'Broken Integration',
+      'latest',
+      'writeKey',
+      {},
+      {}
+    )
+
+    const ctx = Context.system()
+
+    await expect(dest.load(ctx, {} as Analytics)).rejects.toThrow(
+      'Script load failed'
+    )
+
+    expect(metricSpy).toHaveBeenCalledWith(ctx, {
+      integrationName: 'Broken Integration',
+      methodName: 'load',
+      type: 'classic',
+      didError: true,
+    })
+  })
+
+  it('records integration invoke error metric when buildIntegration fails', async () => {
+    // Provide an integrationSource that will cause buildIntegration to throw
+    const badSource = {} as any
+
+    const dest = new LegacyDestination(
+      'Bad Build Integration',
+      'latest',
+      'writeKey',
+      {},
+      {},
+      badSource
+    )
+
+    const ctx = Context.system()
+
+    await expect(dest.load(ctx, {} as Analytics)).rejects.toThrow()
+
+    expect(metricSpy).toHaveBeenCalledWith(ctx, {
+      integrationName: 'Bad Build Integration',
+      methodName: 'load',
+      type: 'classic',
+      didError: true,
+    })
+  })
+})

packages/browser/src/plugins/ajs-destination/index.ts

@@ -134,20 +134,30 @@ export class LegacyDestination implements InternalPluginWithAddMiddleware {
       return
     }
 
-    const integrationSource =
-      this.integrationSource ??
-      (await loadIntegration(
-        ctx,
-        this.name,
-        this.version,
-        this.options.obfuscate
-      ))
-
-    this.integration = buildIntegration(
-      integrationSource,
-      this.settings,
-      analyticsInstance
-    )
+    try {
+      const integrationSource =
+        this.integrationSource ??
+        (await loadIntegration(
+          ctx,
+          this.name,
+          this.version,
+          this.options.obfuscate
+        ))
+
+      this.integration = buildIntegration(
+        integrationSource,
+        this.settings,
+        analyticsInstance
+      )
+    } catch (error) {
+      recordIntegrationMetric(ctx, {
+        integrationName: this.name,
+        methodName: 'load',
+        type: 'classic',
+        didError: true,
+      })
+      throw error
+    }
 
     this.onReady = new Promise((resolve) => {
       const onReadyFn = (): void => {

packages/browser/src/plugins/remote-loader/__tests__/index.test.ts

@@ -7,6 +7,7 @@ import { InitOptions } from '../../../core/analytics'
 import { Context } from '../../../core/context'
 import { tsubMiddleware } from '../../routing-middleware'
 import { cdnSettingsMinimal } from '../../../test-helpers/fixtures'
+import * as MetricHelpers from '../../../core/stats/metric-helpers'
 
 const pluginFactory = jest.fn()
 
@@ -966,4 +967,40 @@ describe('Remote Loader', () => {
 
     expect(newCtx.event.name).toEqual('foobar')
   })
+
+  it('records integration invoke error metric when plugin fails to load', async () => {
+    const metricSpy = jest.spyOn(MetricHelpers, 'recordIntegrationMetric')
+
+    // @ts-expect-error not gonna return a script tag sorry
+    jest.spyOn(loader, 'loadScript').mockImplementation(() => {
+      window['flaky'] = (): never => {
+        throw Error('aaay')
+      }
+      return Promise.resolve(true)
+    })
+
+    await remoteLoader(
+      {
+        ...cdnSettingsMinimal,
+        remotePlugins: [
+          {
+            name: 'flaky plugin',
+            creationName: 'Flaky Plugin',
+            url: 'cdn/path/to/flaky.js',
+            libraryName: 'flaky',
+            settings: {},
+          },
+        ],
+      },
+      {},
+      {}
+    )
+
+    expect(metricSpy).toHaveBeenCalledWith(expect.any(Context), {
+      integrationName: 'flaky plugin',
+      methodName: 'load',
+      type: 'action',
+      didError: true,
+    })
+  })
 })

packages/browser/src/plugins/remote-loader/index.ts

@@ -309,6 +309,12 @@ export async function remoteLoader(
         }
       } catch (error) {
         console.warn('Failed to load Remote Plugin', error)
+        recordIntegrationMetric(Context.system(), {
+          integrationName: remotePlugin.name,
+          methodName: 'load',
+          type: 'action',
+          didError: true,
+        })
       }
     }
   )