Introduce enable live metrics to distro

sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md

@@ -10,6 +10,9 @@
 
 ### Other Changes
 
+- Update live metrics to use typespec generated swagger
+    ([#34840](https://github.com/Azure/azure-sdk-for-python/pull/34840))
+
 ## 1.0.0b25 (2024-04-19)
 
 ### Features Added

sdk/monitor/azure-monitor-opentelemetry/CHANGELOG.md

@@ -4,6 +4,9 @@
 
 ### Features Added
 
+- Enable live metrics feature
+    ([#35566](https://github.com/Azure/azure-sdk-for-python/pull/35566))
+
 ### Breaking Changes
 
 ### Bugs Fixed

sdk/monitor/azure-monitor-opentelemetry/README.md

@@ -60,6 +60,7 @@ You can use `configure_azure_monitor` to set up instrumentation for your app to
 | Parameter | Description | Environment Variable |
 |-------------------|----------------------------------------------------|----------------------|
 | `connection_string` | The [connection string][connection_string_doc] for your Application Insights resource. The connection string will be automatically populated from the `APPLICATIONINSIGHTS_CONNECTION_STRING` environment variable if not explicitly passed in. | `APPLICATIONINSIGHTS_CONNECTION_STRING` |
+| `enable_live_metrics` | Enable [live metrics][application_insights_live_metrics] feature. | `N/A` |
 | `logger_name` | The name of the [Python logger][python_logger] under which telemetry is collected. | `N/A` |
 | `instrumentation_options` | A nested dictionary that determines which instrumentations to enable or disable. Instrumentations are referred to by their [Library Names](#officially-supported-instrumentations). For example, `{"azure_sdk": {"enabled": False}, "flask": {"enabled": False}, "django": {"enabled": True}}` will disable Azure Core Tracing and the Flask instrumentation but leave Django and the other default instrumentations enabled. The `OTEL_PYTHON_DISABLED_INSTRUMENTATIONS` environment variable explained below can also be used to disable instrumentations. | `N/A` |
 | `resource` | Specifies the OpenTelemetry [Resource][ot_spec_resource] associated with your application. Passed in [Resource Attributes][ot_spec_resource_attributes] take priority over default attributes and those from [Resource Detectors][ot_python_resource_detectors]. | [OTEL_SERVICE_NAME][ot_spec_service_name], [OTEL_RESOURCE_ATTRIBUTES][ot_spec_resource_attributes], [OTEL_EXPERIMENTAL_RESOURCE_DETECTORS][ot_python_resource_detectors] |
@@ -210,6 +211,7 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio
 [azure_monitor_opentelemetry_exporters]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/monitor/azure-monitor-opentelemetry-exporter#microsoft-opentelemetry-exporter-for-azure-monitor
 [azure_portal]: https://portal.azure.com
 [azure_sub]: https://azure.microsoft.com/free/
+[application_insights_live_metrics]: https://learn.microsoft.com/azure/azure-monitor/app/live-stream
 [application_insights_namespace]: https://learn.microsoft.com/azure/azure-monitor/app/app-insights-overview
 [application_insights_sampling]: https://learn.microsoft.com/azure/azure-monitor/app/sampling
 [connection_string_doc]: https://learn.microsoft.com/azure/azure-monitor/app/sdk-connection-string

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_configure.py

@@ -32,12 +32,18 @@
     DISABLE_LOGGING_ARG,
     DISABLE_METRICS_ARG,
     DISABLE_TRACING_ARG,
+    ENABLE_LIVE_METRICS_ARG,
     LOGGER_NAME_ARG,
     RESOURCE_ARG,
     SAMPLING_RATIO_ARG,
     SPAN_PROCESSORS_ARG,
 )
 from azure.monitor.opentelemetry._types import ConfigurationValue
+from azure.monitor.opentelemetry.exporter._quickpulse import enable_live_metrics  # pylint: disable=import-error,no-name-in-module
+from azure.monitor.opentelemetry.exporter._quickpulse._processor import (  # pylint: disable=import-error,no-name-in-module
+    _QuickpulseLogRecordProcessor,
+    _QuickpulseSpanProcessor,
+)
 from azure.monitor.opentelemetry.exporter import (  # pylint: disable=import-error,no-name-in-module
     ApplicationInsightsSampler,
     AzureMonitorLogExporter,
@@ -78,6 +84,8 @@ def configure_azure_monitor(**kwargs) -> None:  # pylint: disable=C4758
      Attributes take priority over default attributes and those from Resource Detectors.
     :keyword list[~opentelemetry.sdk.trace.SpanProcessor] span_processors: List of `SpanProcessor` objects
      to process every span prior to exporting. Will be run sequentially.
+    :keyword bool enable_live_metrics: Boolean value to determine whether to enable live metrics feature.
+     Defaults to `False`.
     :keyword str storage_directory: Storage directory in which to store retry files. Defaults to
      `<tempfile.gettempdir()>/Microsoft/AzureMonitor/opentelemetry-python-<your-instrumentation-key>`.
     :rtype: None
@@ -90,6 +98,7 @@ def configure_azure_monitor(**kwargs) -> None:  # pylint: disable=C4758
     disable_tracing = configurations[DISABLE_TRACING_ARG]
     disable_logging = configurations[DISABLE_LOGGING_ARG]
     disable_metrics = configurations[DISABLE_METRICS_ARG]
+    enable_live_metrics_config = configurations[ENABLE_LIVE_METRICS_ARG]
 
     # Setup tracing pipeline
     if not disable_tracing:
@@ -103,6 +112,10 @@ def configure_azure_monitor(**kwargs) -> None:  # pylint: disable=C4758
     if not disable_metrics:
         _setup_metrics(configurations)
 
+    # Setup live metrics
+    if enable_live_metrics_config:
+        _setup_live_metrics(configurations)
+
     # Setup instrumentations
     # Instrumentations need to be setup last so to use the global providers
     # instanstiated in the other setup steps
@@ -119,6 +132,9 @@ def _setup_tracing(configurations: Dict[str, ConfigurationValue]):
     set_tracer_provider(tracer_provider)
     for span_processor in configurations[SPAN_PROCESSORS_ARG]: # type: ignore
         get_tracer_provider().add_span_processor(span_processor) # type: ignore
+    if configurations.get(ENABLE_LIVE_METRICS_ARG):
+        qsp = _QuickpulseSpanProcessor()
+        get_tracer_provider().add_span_processor(qsp) # type: ignore
     trace_exporter = AzureMonitorTraceExporter(**configurations)
     bsp = BatchSpanProcessor(
         trace_exporter,
@@ -132,6 +148,9 @@ def _setup_logging(configurations: Dict[str, ConfigurationValue]):
     resource: Resource = configurations[RESOURCE_ARG] # type: ignore
     logger_provider = LoggerProvider(resource=resource)
     set_logger_provider(logger_provider)
+    if configurations.get(ENABLE_LIVE_METRICS_ARG):
+        qlp = _QuickpulseLogRecordProcessor()
+        get_logger_provider().add_log_record_processor(qlp) # type: ignore
     log_exporter = AzureMonitorLogExporter(**configurations)
     log_record_processor = BatchLogRecordProcessor(
         log_exporter,
@@ -153,6 +172,10 @@ def _setup_metrics(configurations: Dict[str, ConfigurationValue]):
     set_meter_provider(meter_provider)
 
 
+def _setup_live_metrics(configurations):
+    enable_live_metrics(**configurations)
+
+
 def _setup_instrumentations(configurations: Dict[str, ConfigurationValue]):
     # use pkg_resources for now until https://github.com/open-telemetry/opentelemetry-python/pull/3168 is merged
     for entry_point in iter_entry_points(

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_constants.py

@@ -11,6 +11,7 @@
 # --------------------Configuration------------------------------------------
 
 CONNECTION_STRING_ARG = "connection_string"
+ENABLE_LIVE_METRICS_ARG = "enable_live_metrics"
 DISABLE_AZURE_CORE_TRACING_ARG = "disable_azure_core_tracing"
 DISABLE_LOGGING_ARG = "disable_logging"
 DISABLE_METRICS_ARG = "disable_metrics"

sdk/monitor/azure-monitor-opentelemetry/azure/monitor/opentelemetry/_utils/configurations.py

@@ -31,6 +31,7 @@
     DISABLE_METRICS_ARG,
     DISABLE_TRACING_ARG,
     DISTRO_VERSION_ARG,
+    ENABLE_LIVE_METRICS_ARG,
     INSTRUMENTATION_OPTIONS_ARG,
     LOGGER_NAME_ARG,
     RESOURCE_ARG,
@@ -68,6 +69,7 @@ def _get_configurations(**kwargs) -> Dict[str, ConfigurationValue]:
     _default_sampling_ratio(configurations)
     _default_instrumentation_options(configurations)
     _default_span_processors(configurations)
+    _default_enable_live_metrics(configurations)
 
     return configurations
 
@@ -151,6 +153,11 @@ def _default_span_processors(configurations):
         configurations[SPAN_PROCESSORS_ARG] = []
 
 
+def _default_enable_live_metrics(configurations):
+    if ENABLE_LIVE_METRICS_ARG not in configurations:
+        configurations[ENABLE_LIVE_METRICS_ARG] = False
+
+
 def _get_otel_disabled_instrumentations():
     disabled_instrumentation = environ.get(
         OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, ""

sdk/monitor/azure-monitor-opentelemetry/samples/metrics/live.py

@@ -0,0 +1,37 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+"""
+This example shows how configure live metrics to be enabled. It sets up a minimal example of sending dependency,
+trace and exception telemetry to demonstrate the capabilities and collection set of live metrics.
+"""
+import logging
+import requests
+import time
+
+from azure.monitor.opentelemetry import configure_azure_monitor
+from opentelemetry import trace
+
+from opentelemetry.sdk.resources import Resource
+
+configure_azure_monitor(
+    resource=Resource.create({
+        "service.name": "live_metrics_service",
+        "service.instance.id": "qp_instance_id",
+    }),
+    logger_name=__name__,
+    enable_live_metrics=True,  # Enable live metrics configuration
+)
+
+tracer = trace.get_tracer(__name__)
+logger = logging.getLogger(__name__)
+
+while True:
+    with tracer.start_as_current_span("parent"):
+        logger.warning("sending request")
+        response = requests.get("https://azure.microsoft.com/", timeout=5)
+        try:
+            val = 1 / 0
+            print(val)
+        except ZeroDivisionError:
+            logger.error("Error: Division by zero", stack_info=True, exc_info=True)
+    time.sleep(2)

sdk/monitor/azure-monitor-opentelemetry/tests/test_configure.py

@@ -20,6 +20,7 @@
 from azure.monitor.opentelemetry._configure import (
     _send_attach_warning,
     _setup_instrumentations,
+    _setup_live_metrics,
     _setup_logging,
     _setup_metrics,
     _setup_tracing,
@@ -38,6 +39,9 @@ class TestConfigure(unittest.TestCase):
     @patch(
         "azure.monitor.opentelemetry._configure._setup_instrumentations",
     )
+    @patch(
+        "azure.monitor.opentelemetry._configure._setup_live_metrics",
+    )
     @patch(
         "azure.monitor.opentelemetry._configure._setup_metrics",
     )
@@ -52,6 +56,7 @@ def test_configure_azure_monitor(
         tracing_mock,
         logging_mock,
         metrics_mock,
+        live_metrics_mock,
         instrumentation_mock,
         detect_attach_mock,
     ):
@@ -62,12 +67,16 @@ def test_configure_azure_monitor(
         tracing_mock.assert_called_once()
         logging_mock.assert_called_once()
         metrics_mock.assert_called_once()
+        live_metrics_mock.assert_not_called()
         instrumentation_mock.assert_called_once()
         detect_attach_mock.assert_called_once()
 
     @patch(
         "azure.monitor.opentelemetry._configure._setup_instrumentations",
     )
+    @patch(
+        "azure.monitor.opentelemetry._configure._setup_live_metrics",
+    )
     @patch(
         "azure.monitor.opentelemetry._configure._setup_metrics",
     )
@@ -86,6 +95,7 @@ def test_configure_azure_monitor_disable_tracing(
         tracing_mock,
         logging_mock,
         metrics_mock,
+        live_metrics_mock,
         instrumentation_mock,
     ):
         configurations = {
@@ -104,18 +114,23 @@ def test_configure_azure_monitor_disable_tracing(
                     "enabled": False
                 },
             },
+            "enable_live_metrics": False,
             "resource": TEST_RESOURCE,
         }
         config_mock.return_value = configurations
         configure_azure_monitor()
         tracing_mock.assert_not_called()
         logging_mock.assert_called_once_with(configurations)
         metrics_mock.assert_called_once_with(configurations)
+        live_metrics_mock.assert_not_called()
         instrumentation_mock.assert_called_once_with(configurations)
 
     @patch(
         "azure.monitor.opentelemetry._configure._setup_instrumentations",
     )
+    @patch(
+        "azure.monitor.opentelemetry._configure._setup_live_metrics",
+    )
     @patch(
         "azure.monitor.opentelemetry._configure._setup_metrics",
     )
@@ -134,25 +149,31 @@ def test_configure_azure_monitor_disable_logging(
         tracing_mock,
         logging_mock,
         metrics_mock,
+        live_metrics_mock,
         instrumentation_mock,
     ):
         configurations = {
             "connection_string": "test_cs",
             "disable_tracing": False,
             "disable_logging": True,
             "disable_metrics": False,
+            "enable_live_metrics": False,
             "resource": TEST_RESOURCE,
         }
         config_mock.return_value = configurations
         configure_azure_monitor()
         tracing_mock.assert_called_once_with(configurations)
         logging_mock.assert_not_called()
         metrics_mock.assert_called_once_with(configurations)
+        live_metrics_mock.assert_not_called()
         instrumentation_mock.assert_called_once_with(configurations)
 
     @patch(
         "azure.monitor.opentelemetry._configure._setup_instrumentations",
     )
+    @patch(
+        "azure.monitor.opentelemetry._configure._setup_live_metrics",
+    )
     @patch(
         "azure.monitor.opentelemetry._configure._setup_metrics",
     )
@@ -171,20 +192,66 @@ def test_configure_azure_monitor_disable_metrics(
         tracing_mock,
         logging_mock,
         metrics_mock,
+        live_metrics_mock,
         instrumentation_mock,
     ):
         configurations = {
             "connection_string": "test_cs",
             "disable_tracing": False,
             "disable_logging": False,
             "disable_metrics": True,
+            "enable_live_metrics": False,
             "resource": TEST_RESOURCE,
         }
         config_mock.return_value = configurations
         configure_azure_monitor()
         tracing_mock.assert_called_once_with(configurations)
         logging_mock.assert_called_once_with(configurations)
         metrics_mock.assert_not_called()
+        live_metrics_mock.assert_not_called()
+        instrumentation_mock.assert_called_once_with(configurations)
+
+    @patch(
+        "azure.monitor.opentelemetry._configure._setup_instrumentations",
+    )
+    @patch(
+        "azure.monitor.opentelemetry._configure._setup_live_metrics",
+    )
+    @patch(
+        "azure.monitor.opentelemetry._configure._setup_metrics",
+    )
+    @patch(
+        "azure.monitor.opentelemetry._configure._setup_logging",
+    )
+    @patch(
+        "azure.monitor.opentelemetry._configure._setup_tracing",
+    )
+    @patch(
+        "azure.monitor.opentelemetry._configure._get_configurations",
+    )
+    def test_configure_azure_monitor_enable_live_metrics(
+        self,
+        config_mock,
+        tracing_mock,
+        logging_mock,
+        metrics_mock,
+        live_metrics_mock,
+        instrumentation_mock,
+    ):
+        configurations = {
+            "connection_string": "test_cs",
+            "disable_tracing": False,
+            "disable_logging": False,
+            "disable_metrics": False,
+            "enable_live_metrics": True,
+            "resource": TEST_RESOURCE,
+        }
+        config_mock.return_value = configurations
+        configure_azure_monitor()
+        tracing_mock.assert_called_once_with(configurations)
+        logging_mock.assert_called_once_with(configurations)
+        metrics_mock.assert_called_once_with(configurations)
+        live_metrics_mock.assert_called_once_with(configurations)
         instrumentation_mock.assert_called_once_with(configurations)
 
     @patch(
@@ -364,6 +431,21 @@ def test_setup_metrics(
         metric_exporter_mock.assert_called_once_with(**configurations)
         reader_mock.assert_called_once_with(metric_exp_init_mock)
 
+    @patch(
+        "azure.monitor.opentelemetry._configure.enable_live_metrics",
+    )
+    def test_setup_live_metrics(
+        self,
+        enable_live_metrics_mock,
+    ):
+        configurations = {
+            "connection_string": "test_cs",
+            "resource": TEST_RESOURCE,
+        }
+        _setup_live_metrics(configurations)
+
+        enable_live_metrics_mock.assert_called_once_with(**configurations)
+
     @patch("azure.monitor.opentelemetry._configure._ALL_SUPPORTED_INSTRUMENTED_LIBRARIES", ("test_instr2"))
     @patch("azure.monitor.opentelemetry._configure._is_instrumentation_enabled")
     @patch("azure.monitor.opentelemetry._configure.get_dist_dependency_conflicts")

sdk/monitor/azure-monitor-opentelemetry/tests/utils/test_configurations.py

@@ -106,6 +106,7 @@ def test_get_configurations_defaults(self, resource_create_mock):
         self.assertEqual(configurations["sampling_ratio"], 1.0)
         self.assertTrue("credential" not in configurations)
         self.assertTrue("storage_directory" not in configurations)
+        self.assertEqual(configurations["enable_live_metrics"], False)
 
     @patch.dict(
         "os.environ",