Implement distro detection for statsbeat feature (#33761)

Author: lzchen

sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md

@@ -22,6 +22,8 @@
     ([#33667](https://github.com/Azure/azure-sdk-for-python/pull/33667))
 - Readme examples are updated with correct imports
     ([#33691](https://github.com/Azure/azure-sdk-for-python/pull/33691))
+- Implement distro detection for statsbeat feature
+    ([#33761](https://github.com/Azure/azure-sdk-for-python/pull/33761))
 
 ## 1.0.0b19 (2023-11-20)
 

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/_constants.py

@@ -46,6 +46,10 @@
 _REQUEST_ENVELOPE_NAME = "Microsoft.ApplicationInsights.Request"
 _REMOTE_DEPENDENCY_ENVELOPE_NAME = "Microsoft.ApplicationInsights.RemoteDependency"
 
+# Feature constants
+_APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE = "APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE"
+_AZURE_MONITOR_DISTRO_VERSION_ARG = "distro_version"
+
 # Statsbeat
 
 # (OpenTelemetry metric name, Statsbeat metric name)
@@ -119,6 +123,8 @@
     "urllib",
     "urllib3",
     _AZURE_SDK_OPENTELEMETRY_NAME,
+    "cassandra",
+    "tortoiseorm",
 ]
 
 _INSTRUMENTATIONS_BIT_MAP = {_INSTRUMENTATIONS_LIST[i]: _BASE**i for i in range(len(_INSTRUMENTATIONS_LIST))}

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/_base.py

@@ -29,6 +29,7 @@
     TelemetryItem,
 )
 from azure.monitor.opentelemetry.exporter._constants import (
+    _AZURE_MONITOR_DISTRO_VERSION_ARG,
     _INVALID_STATUS_CODES,
     _REACHED_INGESTION_STATUS_CODES,
     _REDIRECT_STATUS_CODES,
@@ -100,6 +101,7 @@ def __init__(self, **kwargs: Any) -> None:
         self._storage_directory = kwargs.get('storage_directory', default_storage_directory)  # Storage path in which to store retry files.
         self._storage_retention_period = kwargs.get('storage_retention_period', 48 * 60 * 60)  # Retention period in seconds (default 48 hrs)
         self._timeout = kwargs.get('timeout', 10.0)  # networking timeout in seconds
+        self._distro_version = kwargs.get(_AZURE_MONITOR_DISTRO_VERSION_ARG, '')  # If set, indicates the exporter is instantiated via Azure monitor OpenTelemetry distro. Versions corresponds to distro version.
 
         config = AzureMonitorClientConfiguration(self._endpoint, **kwargs)
         policies = [

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/logs/_exporter.py

@@ -26,6 +26,9 @@
     BaseExporter,
     ExportResult,
 )
+from azure.monitor.opentelemetry.exporter._constants import (
+    _APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE,
+)
 from azure.monitor.opentelemetry.exporter.statsbeat._state import (
     get_statsbeat_shutdown,
     get_statsbeat_custom_events_feature_set,
@@ -40,7 +43,6 @@
 
 __all__ = ["AzureMonitorLogExporter"]
 
-_APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE = "APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE"
 
 class AzureMonitorLogExporter(BaseExporter, LogExporter):
     """Azure Monitor Log exporter for OpenTelemetry."""

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/statsbeat/_statsbeat.py

@@ -62,6 +62,7 @@ def collect_statsbeat_metrics(exporter) -> None:
                 exporter._disable_offline_storage,
                 long_interval_threshold,
                 exporter._credential is not None,
+                exporter._distro_version,
             )
         # Export some initial stats on program start
         mp.force_flush()

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/statsbeat/_statsbeat_metrics.py

@@ -53,6 +53,7 @@ class _StatsbeatFeature:
     DISK_RETRY = 1
     AAD = 2
     CUSTOM_EVENTS_EXTENSION = 4
+    DISTRO = 8
 
 
 class _AttachTypes:
@@ -97,13 +98,16 @@ def __init__(
         disable_offline_storage: bool,
         long_interval_threshold: int,
         has_credential: bool,
+        distro_version: str = "",
     ) -> None:
         self._ikey = instrumentation_key
         self._feature = _StatsbeatFeature.NONE
         if not disable_offline_storage:
             self._feature |= _StatsbeatFeature.DISK_RETRY
         if has_credential:
             self._feature |= _StatsbeatFeature.AAD
+        if distro_version:
+            self._feature |= _StatsbeatFeature.DISTRO
         if get_statsbeat_custom_events_feature_set():
             self._feature |= _StatsbeatFeature.CUSTOM_EVENTS_EXTENSION
         self._ikey = instrumentation_key

sdk/monitor/azure-monitor-opentelemetry-exporter/tests/logs/test_logs.py

@@ -17,11 +17,13 @@
 
 from azure.monitor.opentelemetry.exporter.export._base import ExportResult
 from azure.monitor.opentelemetry.exporter.export.logs._exporter import (
-    _APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE,
     AzureMonitorLogExporter,
     _get_log_export_result,
     _get_severity_level,
 )
+from azure.monitor.opentelemetry.exporter._constants import (
+    _APPLICATION_INSIGHTS_EVENT_MARKER_ATTRIBUTE,
+)
 from azure.monitor.opentelemetry.exporter._generated.models import ContextTagKeys
 from azure.monitor.opentelemetry.exporter._utils import (
     azure_monitor_context,

sdk/monitor/azure-monitor-opentelemetry-exporter/tests/statsbeat/test_statsbeat.py

@@ -176,6 +176,7 @@ def test_collect_statsbeat_metrics_aad(
         exporter._instrumentation_key = TEST_IKEY
         exporter._disable_offline_storage = False
         exporter._credential = TEST_CREDENTIAL
+        exporter._distro_version = ""
         _statsbeat.collect_statsbeat_metrics(exporter)
         mock_statsbeat_metrics.assert_called_once_with(
             mock.ANY,
@@ -184,6 +185,7 @@ def test_collect_statsbeat_metrics_aad(
             False,
             _DEFAULT_STATS_LONG_EXPORT_INTERVAL / _DEFAULT_STATS_SHORT_EXPORT_INTERVAL,
             True,
+            "",
         )
 
 
@@ -207,6 +209,7 @@ def test_collect_statsbeat_metrics_no_aad(
         exporter._instrumentation_key = TEST_IKEY
         exporter._disable_offline_storage = False
         exporter._credential = TEST_CREDENTIAL
+        exporter._distro_version = ""
         _statsbeat.collect_statsbeat_metrics(exporter)
         mock_statsbeat_metrics.assert_called_once_with(
             mock.ANY,
@@ -215,6 +218,39 @@ def test_collect_statsbeat_metrics_no_aad(
             False,
             _DEFAULT_STATS_LONG_EXPORT_INTERVAL / _DEFAULT_STATS_SHORT_EXPORT_INTERVAL,
             False,
+            "",
+        )
+
+    @mock.patch('azure.monitor.opentelemetry.exporter.statsbeat._statsbeat._StatsbeatMetrics')
+    @mock.patch.dict(
+        "os.environ",
+        {
+            "APPLICATION_INSIGHTS_STATS_SHORT_EXPORT_INTERVAL": "",
+            "APPLICATION_INSIGHTS_STATS_LONG_EXPORT_INTERVAL": "",
+        },
+    )
+    def test_collect_statsbeat_metrics_distro_version(
+        self,
+        mock_statsbeat_metrics,
+    ):
+        exporter = mock.Mock()
+        TEST_ENDPOINT = "test endpoint"
+        TEST_IKEY = "test ikey"
+        TEST_CREDENTIAL = None
+        exporter._endpoint = TEST_ENDPOINT
+        exporter._instrumentation_key = TEST_IKEY
+        exporter._disable_offline_storage = False
+        exporter._credential = TEST_CREDENTIAL
+        exporter._distro_version = "1.0.0"
+        _statsbeat.collect_statsbeat_metrics(exporter)
+        mock_statsbeat_metrics.assert_called_once_with(
+            mock.ANY,
+            TEST_IKEY,
+            TEST_ENDPOINT,
+            False,
+            _DEFAULT_STATS_LONG_EXPORT_INTERVAL / _DEFAULT_STATS_SHORT_EXPORT_INTERVAL,
+            False,
+            "1.0.0",
         )
 
     def test_shutdown_statsbeat_metrics(self):
@@ -634,7 +670,7 @@ def test_get_feature_metric_does_not_meet_threshold(self):
         self.assertEqual(metric._long_interval_count_map[_FEATURE_METRIC_NAME[0]], 1)
 
     # pylint: disable=protected-access
-    def test_get_feature_metric(self):
+    def test_get_feature_metric_storage(self):
         mp = MeterProvider()
         ikey = "1aa11111-bbbb-1ccc-8ddd-eeeeffff3334"
         endpoint = "https://westus-1.in.applicationinsights.azure.com/"
@@ -648,7 +684,7 @@ def test_get_feature_metric(self):
         )
         attributes = dict(_StatsbeatMetrics._COMMON_ATTRIBUTES)
         attributes.update(_StatsbeatMetrics._FEATURE_ATTRIBUTES)
-        self.assertEqual(attributes["feature"], 1)
+        self.assertEqual(attributes["feature"], _StatsbeatFeature.DISK_RETRY)
         self.assertEqual(attributes["type"], _FEATURE_TYPES.FEATURE)
         observations = metric._get_feature_metric(options=None)
         for obs in observations:
@@ -740,6 +776,25 @@ def test_get_feature_metric_custom_events_runtime(self):
             self.assertEqual(obs.value, 1)
             self.assertEqual(obs.attributes, attributes)
 
+    # pylint: disable=protected-access
+    def test_get_feature_metric_distro(self):
+        mp = MeterProvider()
+        ikey = "1aa11111-bbbb-1ccc-8ddd-eeeeffff3334"
+        endpoint = "https://westus-1.in.applicationinsights.azure.com/"
+        metric = _StatsbeatMetrics(
+            mp,
+            ikey,
+            endpoint,
+            True,
+            0,
+            False,
+            "1.0.0"
+        )
+        self.assertEqual(_StatsbeatMetrics._FEATURE_ATTRIBUTES["feature"], _StatsbeatFeature.DISTRO)
+        observations = metric._get_feature_metric(options=None)
+        self.assertEqual(len(observations), 1)
+        self.assertEqual(_StatsbeatMetrics._FEATURE_ATTRIBUTES["feature"], _StatsbeatFeature.DISTRO)
+
     # pylint: disable=protected-access
     def test_get_feature_metric_instrumentation(self):
         mp = MeterProvider()

sdk/monitor/azure-monitor-opentelemetry-exporter/tests/test_base_exporter.py

@@ -101,6 +101,7 @@ def test_constructor(self):
             api_version="2021-02-10_Preview",
             connection_string="InstrumentationKey=4321abcd-5678-4efa-8abc-1234567890ab;IngestionEndpoint=https://westus-0.in.applicationinsights.azure.com/",
             disable_offline_storage=False,
+            distro_version="1.0.0",
             storage_maintenance_period=30,
             storage_max_size=1000,
             storage_min_retry_interval=100,
@@ -115,6 +116,7 @@ def test_constructor(self):
             base._endpoint,
             "https://westus-0.in.applicationinsights.azure.com/",
         )
+        self.assertEqual(base._distro_version, "1.0.0")
         self.assertIsNotNone(base.storage)
         self.assertEqual(base.storage._max_size, 1000)
         self.assertEqual(base.storage._retention_period, 2000)

sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md

@@ -10,8 +10,8 @@
 
 ### Other Changes
 
-- Shutdown statsbeat on customer exporter 400 status code
-    ([#31881](https://github.com/Azure/azure-sdk-for-python/pull/31881))
+- Implement distro detection for statsbeat feature
+    ([#33761](https://github.com/Azure/azure-sdk-for-python/pull/33761))
 
 ## 1.1.1 (2023-12-04)