Initial code for code hotspots in wall profiler

* Hide functionality behind DD_PROFILING_EXPERIMENTAL_HOTSPOTS flag
* Add a debug message naming the FF to enable

Author: nsavoire

packages/datadog-core/src/storage/async_resource.js

@@ -5,6 +5,7 @@ const { channel } = require('../../../diagnostics_channel')
 
 const beforeCh = channel('dd-trace:storage:before')
 const afterCh = channel('dd-trace:storage:after')
+const enterCh = channel('dd-trace:storage:enter')
 
 let PrivateSymbol = Symbol
 function makePrivateSymbol () {
@@ -52,6 +53,7 @@ class AsyncResourceStorage {
     const resource = this._executionAsyncResource()
 
     resource[this._ddResourceStore] = store
+    enterCh.publish()
   }
 
   run (store, callback, ...args) {
@@ -61,11 +63,13 @@ class AsyncResourceStorage {
     const oldStore = resource[this._ddResourceStore]
 
     resource[this._ddResourceStore] = store
+    enterCh.publish()
 
     try {
       return callback(...args)
     } finally {
       resource[this._ddResourceStore] = oldStore
+      enterCh.publish()
     }
   }
 

packages/dd-trace/src/profiling/config.js

@@ -36,7 +36,8 @@ class Config {
       DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED,
       DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE,
       DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT,
-      DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES
+      DD_PROFILING_EXPERIMENTAL_OOM_EXPORT_STRATEGIES,
+      DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED
     } = process.env
 
     const enabled = isTrue(coalesce(options.enabled, DD_PROFILING_ENABLED, true))
@@ -110,6 +111,8 @@ class Config {
     const profilers = options.profilers
       ? options.profilers
       : getProfilers({ DD_PROFILING_HEAP_ENABLED, DD_PROFILING_WALLTIME_ENABLED, DD_PROFILING_PROFILERS })
+    this.codeHotspotsEnabled = isTrue(coalesce(options.codeHotspotsEnabled,
+      DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED, false))
 
     this.profilers = ensureProfilers(profilers, this)
   }

packages/dd-trace/src/profiling/profilers/wall.js

@@ -1,5 +1,24 @@
 'use strict'
 
+const { storage } = require('../../../../datadog-core')
+
+const dc = require('../../../../diagnostics_channel')
+
+const beforeCh = dc.channel('dd-trace:storage:before')
+const afterCh = dc.channel('dd-trace:storage:after')
+const enterCh = dc.channel('dd-trace:storage:enter')
+
+function getActiveSpan () {
+  const store = storage.getStore()
+  if (!store) return
+  return store.span
+}
+
+function getStartedSpans (context) {
+  if (!context) return
+  return context._trace.started
+}
+
 class NativeWallProfiler {
   constructor (options = {}) {
     this.type = 'wall'
@@ -9,13 +28,26 @@ class NativeWallProfiler {
     this._mapper = undefined
     this._pprof = undefined
 
+    // Bind to this so the same value can be used to unsubscribe later
+    this._enter = this._enter.bind(this)
+    this._exit = this._exit.bind(this)
     this._logger = options.logger
     this._started = false
   }
 
+  codeHotspotsEnabled () {
+    return this._codeHotspotsEnabled
+  }
+
   start ({ mapper } = {}) {
     if (this._started) return
 
+    if (this._hotspots && !this._emittedFFMessage && this._logger) {
+      this._logger.debug(
+        `Wall profiler: Enable config_trace_show_breakdown_profiling_for_node feature flag to see code hotspots.`)
+      this._emittedFFMessage = true
+    }
+
     this._mapper = mapper
     this._pprof = require('@datadog/pprof')
 
@@ -35,9 +67,45 @@ class NativeWallProfiler {
       lineNumbers: false
     })
 
+    if (this._hotspots) {
+      beforeCh.subscribe(this._enter)
+      enterCh.subscribe(this._enter)
+      afterCh.subscribe(this._exit)
+    }
+
     this._started = true
   }
 
+  setLabels (labels) {
+    this._setLabels(labels)
+  }
+
+  _enter () {
+    if (!this._started) return
+
+    const currentSpan = getActiveSpan() || null
+    const currentContext = currentSpan ? currentSpan.context() : null
+
+    if (!currentContext) return
+
+    const startedSpans = getStartedSpans(currentContext)
+    if (!startedSpans || startedSpans.length === 0) return
+    const rootContext = startedSpans[0].context()
+    if (!rootContext) return
+
+    const labels = currentContext ? {
+      'local root span id': rootContext.toSpanId(),
+      'span id': currentContext.toSpanId()
+    } : null
+
+    this.setLabels(labels)
+  }
+
+  _exit () {
+    if (!this._started) return
+    this.setLabels(undefined)
+  }
+
   profile () {
     if (!this._started) return
     return this._pprof.time.stop(true)
@@ -51,6 +119,12 @@ class NativeWallProfiler {
     if (!this._started) return
 
     const profile = this._pprof.time.stop()
+    if (this._codeHotspotsEnabled) {
+      beforeCh.unsubscribe(this._enter)
+      afterCh.unsubscribe(this._exit)
+      enterCh.unsubscribe(this._enter)
+    }
+
     this._started = false
     return profile
   }

packages/dd-trace/test/profiling/config.spec.js

@@ -42,6 +42,7 @@ describe('config', () => {
     expect(config.logger).to.be.an.instanceof(ConsoleLogger)
     expect(config.exporters[0]).to.be.an.instanceof(AgentExporter)
     expect(config.profilers[0]).to.be.an.instanceof(WallProfiler)
+    expect(config.profilers[0].codeHotspotsEnabled()).false
     expect(config.profilers[1]).to.be.an.instanceof(SpaceProfiler)
   })
 
@@ -57,8 +58,9 @@ describe('config', () => {
         error () { }
       },
       exporters: 'agent,file',
-      profilers: 'wall',
-      url: 'http://localhost:1234/'
+      profilers: 'space,wall',
+      url: 'http://localhost:1234/',
+      codeHotspotsEnabled: true
     }
 
     const config = new Config(options)
@@ -78,8 +80,10 @@ describe('config', () => {
     expect(config.exporters[0]._url.toString()).to.equal(options.url)
     expect(config.exporters[1]).to.be.an.instanceof(FileExporter)
     expect(config.profilers).to.be.an('array')
-    expect(config.profilers.length).to.equal(1)
-    expect(config.profilers[0]).to.be.an.instanceOf(WallProfiler)
+    expect(config.profilers.length).to.equal(2)
+    expect(config.profilers[0]).to.be.an.instanceOf(SpaceProfiler)
+    expect(config.profilers[1]).to.be.an.instanceOf(WallProfiler)
+    expect(config.profilers[1].codeHotspotsEnabled()).true
   })
 
   it('should filter out invalid profilers', () => {
@@ -127,7 +131,8 @@ describe('config', () => {
 
   it('should support profiler config with DD_PROFILING_PROFILERS', () => {
     process.env = {
-      DD_PROFILING_PROFILERS: 'wall'
+      DD_PROFILING_PROFILERS: 'wall',
+      DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED: '1'
     }
     const options = {
       logger: {
@@ -143,6 +148,7 @@ describe('config', () => {
     expect(config.profilers).to.be.an('array')
     expect(config.profilers.length).to.equal(1)
     expect(config.profilers[0]).to.be.an.instanceOf(WallProfiler)
+    expect(config.profilers[0].codeHotspotsEnabled()).true
   })
 
   it('should support profiler config with DD_PROFILING_XXX_ENABLED', () => {
@@ -190,7 +196,8 @@ describe('config', () => {
 
   it('should prioritize options over env variables', () => {
     process.env = {
-      DD_PROFILING_PROFILERS: 'wall'
+      DD_PROFILING_PROFILERS: 'space',
+      DD_PROFILING_EXPERIMENTAL_CODEHOTSPOTS_ENABLED: '1'
     }
     const options = {
       logger: {
@@ -199,14 +206,16 @@ describe('config', () => {
         warn () {},
         error () {}
       },
-      profilers: ['space']
+      profilers: ['wall'],
+      codeHotspotsEnabled: false
     }
 
     const config = new Config(options)
 
     expect(config.profilers).to.be.an('array')
     expect(config.profilers.length).to.equal(1)
-    expect(config.profilers[0]).to.be.an.instanceOf(SpaceProfiler)
+    expect(config.profilers[0]).to.be.an.instanceOf(WallProfiler)
+    expect(config.profilers[0].codeHotspotsEnabled()).false
   })
 
   it('should support tags', () => {