Implement metrics exporter (#23960)

Author: lzchen

sdk/monitor/azure-monitor-opentelemetry-exporter/CHANGELOG.md

@@ -9,6 +9,8 @@
     ([#23633](https://github.com/Azure/azure-sdk-for-python/pull/23633))
 - Implement exporting span events as message/exception telemetry
     ([#23708](https://github.com/Azure/azure-sdk-for-python/pull/23708))
+- Implement metrics exporter using experimental OT metrics sdk
+    ([#23960](https://github.com/Azure/azure-sdk-for-python/pull/23960))
 
 ### Breaking Changes
 

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/__init__.py

@@ -5,8 +5,13 @@
 # --------------------------------------------------------------------------
 
 from azure.monitor.opentelemetry.exporter.export.logs._exporter import AzureMonitorLogExporter
+from azure.monitor.opentelemetry.exporter.export.metrics._exporter import AzureMonitorMetricExporter
 from azure.monitor.opentelemetry.exporter.export.trace._exporter import AzureMonitorTraceExporter
 from ._version import VERSION
 
-__all__ = ["AzureMonitorLogExporter", "AzureMonitorTraceExporter"]
+__all__ = [
+    "AzureMonitorMetricExporter",
+    "AzureMonitorLogExporter",
+    "AzureMonitorTraceExporter",
+]
 __version__ = VERSION

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/metrics/__init__.py

@@ -0,0 +1,132 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import logging
+from typing import Sequence, Any
+
+from opentelemetry.sdk._metrics.export import MetricExporter, MetricExportResult
+from opentelemetry.sdk._metrics.point import (
+    Gauge,
+    Histogram,
+    Metric,
+    Sum,
+)
+
+from azure.monitor.opentelemetry.exporter import _utils
+from azure.monitor.opentelemetry.exporter._generated.models import (
+    MetricDataPoint,
+    MetricsData,
+    MonitorBase,
+    TelemetryItem,
+)
+from azure.monitor.opentelemetry.exporter.export._base import (
+    BaseExporter,
+    ExportResult,
+)
+
+_logger = logging.getLogger(__name__)
+
+__all__ = ["AzureMonitorMetricExporter"]
+
+
+class AzureMonitorMetricExporter(BaseExporter, MetricExporter):
+    """Azure Monitor Metric exporter for OpenTelemetry."""
+
+    def export(
+        self, metrics: Sequence[Metric], **kwargs: Any  # pylint: disable=unused-argument
+    ) -> MetricExportResult:
+        """Exports a batch of metric data
+        :param metrics: Open Telemetry Metric(s) to export.
+        :type metrics: Sequence[~opentelemetry._metrics.point.Metric]
+        :rtype: ~opentelemetry.sdk._metrics.export.MetricExportResult
+        """
+        envelopes = [self._metric_to_envelope(metric) for metric in metrics]
+        try:
+            result = self._transmit(envelopes)
+            if result == ExportResult.FAILED_RETRYABLE:
+                envelopes_to_store = [x.as_dict() for x in envelopes]
+                self.storage.put(envelopes_to_store, 1)
+            if result == ExportResult.SUCCESS:
+                # Try to send any cached events
+                self._transmit_from_storage()
+            return _get_metric_export_result(result)
+        except Exception:  # pylint: disable=broad-except
+            _logger.exception("Exception occurred while exporting the data.")
+            return _get_metric_export_result(ExportResult.FAILED_NOT_RETRYABLE)
+
+    def shutdown(self) -> None:
+        """Shuts down the exporter.
+
+        Called when the SDK is shut down.
+        """
+        self.storage.close()
+
+    def _metric_to_envelope(self, metric: Metric) -> TelemetryItem:
+        if not metric:
+            return None
+        envelope = _convert_metric_to_envelope(metric)
+        envelope.instrumentation_key = self._instrumentation_key
+        return envelope
+
+    @classmethod
+    def from_connection_string(
+        cls, conn_str: str, **kwargs: Any
+    ) -> "AzureMonitorMetricExporter":
+        """
+        Create an AzureMonitorMetricExporter from a connection string.
+
+        This is the recommended way of instantation if a connection string is passed in explicitly.
+        If a user wants to use a connection string provided by environment variable, the constructor
+        of the exporter can be called directly.
+
+        :param str conn_str: The connection string to be used for authentication.
+        :keyword str api_version: The service API version used. Defaults to latest.
+        :returns an instance of ~AzureMonitorMetricExporter
+        """
+        return cls(connection_string=conn_str, **kwargs)
+
+
+# pylint: disable=protected-access
+def _convert_metric_to_envelope(metric: Metric) -> TelemetryItem:
+    point = metric.point
+    envelope = _utils._create_telemetry_item(point.time_unix_nano)
+    envelope.name = "Microsoft.ApplicationInsights.Metric"
+    envelope.tags.update(_utils._populate_part_a_fields(metric.resource))
+    properties = metric.attributes
+    value = 0
+    # TODO
+    count = 1
+    # min = None
+    # max = None
+    # std_dev = None
+
+    if isinstance(point, (Gauge, Sum)):
+        value = point.value
+    elif isinstance(point, Histogram):
+        value = sum(point.bucket_counts)
+        count = sum(point.bucket_counts)
+
+    data_point = MetricDataPoint(
+        name=metric.name,
+        value=value,
+        data_point_type="Aggregation",
+        count=count,
+    )
+    data = MetricsData(
+        properties=properties,
+        metrics=[data_point],
+    )
+
+    envelope.data = MonitorBase(base_data=data, base_type="MetricData")
+
+    return envelope
+
+
+def _get_metric_export_result(result: ExportResult) -> MetricExportResult:
+    if result == ExportResult.SUCCESS:
+        return MetricExportResult.SUCCESS
+    if result in (
+        ExportResult.FAILED_RETRYABLE,
+        ExportResult.FAILED_NOT_RETRYABLE,
+    ):
+        return MetricExportResult.FAILURE
+    return None

sdk/monitor/azure-monitor-opentelemetry-exporter/azure/monitor/opentelemetry/exporter/export/metrics/_exporter.py

@@ -10,6 +10,7 @@
 from opentelemetry.sdk._logs import (
     LogEmitterProvider,
     OTLPHandler,
+    get_log_emitter_provider,
     set_log_emitter_provider,
 )
 from opentelemetry.sdk._logs.export import BatchLogProcessor
@@ -19,17 +20,15 @@
 
 trace.set_tracer_provider(TracerProvider())
 tracer = trace.get_tracer(__name__)
-log_emitter_provider = LogEmitterProvider()
-set_log_emitter_provider(log_emitter_provider)
+set_log_emitter_provider(LogEmitterProvider())
 
 exporter = AzureMonitorLogExporter.from_connection_string(
     os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
-
-log_emitter_provider.add_log_processor(BatchLogProcessor(exporter))
-handler = OTLPHandler()
+get_log_emitter_provider().add_log_processor(BatchLogProcessor(exporter))
 
 # Attach OTel handler to namespaced logger
+handler = OTLPHandler()
 logger = logging.getLogger(__name__)
 logger.addHandler(handler)
 logger.setLevel(logging.NOTSET)

sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/sample_correlate.py

@@ -9,23 +9,21 @@
 from opentelemetry.sdk._logs import (
     LogEmitterProvider,
     OTLPHandler,
+    get_log_emitter_provider,
     set_log_emitter_provider,
 )
 from opentelemetry.sdk._logs.export import BatchLogProcessor
 
 from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter
 
-log_emitter_provider = LogEmitterProvider()
-set_log_emitter_provider(log_emitter_provider)
-
+set_log_emitter_provider(LogEmitterProvider())
 exporter = AzureMonitorLogExporter.from_connection_string(
     os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
-
-log_emitter_provider.add_log_processor(BatchLogProcessor(exporter))
-handler = OTLPHandler()
+get_log_emitter_provider().add_log_processor(BatchLogProcessor(exporter))
 
 # Attach OTel handler to namespaced logger
+handler = OTLPHandler()
 logger = logging.getLogger(__name__)
 logger.addHandler(handler)
 logger.setLevel(logging.NOTSET)

sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/sample_exception.py

@@ -10,23 +10,21 @@
 from opentelemetry.sdk._logs import (
     LogEmitterProvider,
     OTLPHandler,
+    get_log_emitter_provider,
     set_log_emitter_provider,
 )
 from opentelemetry.sdk._logs.export import BatchLogProcessor
 
 from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter
 
-log_emitter_provider = LogEmitterProvider()
-set_log_emitter_provider(log_emitter_provider)
-
+set_log_emitter_provider(LogEmitterProvider())
 exporter = AzureMonitorLogExporter.from_connection_string(
     os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
-
-log_emitter_provider.add_log_processor(BatchLogProcessor(exporter))
-handler = OTLPHandler()
+get_log_emitter_provider().add_log_processor(BatchLogProcessor(exporter))
 
 # Attach OTel handler to namespaced logger
+handler = OTLPHandler()
 logger = logging.getLogger(__name__)
 logger.addHandler(handler)
 logger.setLevel(logging.NOTSET)

sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/sample_log.py

@@ -9,23 +9,21 @@
 from opentelemetry.sdk._logs import (
     LogEmitterProvider,
     OTLPHandler,
+    get_log_emitter_provider,
     set_log_emitter_provider,
 )
 from opentelemetry.sdk._logs.export import BatchLogProcessor
 
 from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter
 
-log_emitter_provider = LogEmitterProvider()
-set_log_emitter_provider(log_emitter_provider)
-
+set_log_emitter_provider(LogEmitterProvider())
 exporter = AzureMonitorLogExporter.from_connection_string(
     os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
 )
-
-log_emitter_provider.add_log_processor(BatchLogProcessor(exporter))
-handler = OTLPHandler()
+get_log_emitter_provider().add_log_processor(BatchLogProcessor(exporter))
 
 # Attach OTel handler to namespaced logger
+handler = OTLPHandler()
 logger = logging.getLogger(__name__)
 logger.addHandler(handler)
 logger.setLevel(logging.NOTSET)

sdk/monitor/azure-monitor-opentelemetry-exporter/samples/logs/sample_properties.py

@@ -0,0 +1,50 @@
+---
+page_type: sample
+languages:
+  - python
+products:
+  - azure-monitor
+---
+
+# Microsoft Azure Monitor Opentelemetry Exporter Metric Python Samples
+
+These code samples show common champion scenario operations with the AzureMonitorMetricExporter.
+
+* Metrics: [sample_metrics.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/monitor/azure-monitor-opentelemetry-exporter/samples/metrics/sample_metrics.py)
+* Instruments: [sample_instruments.py](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/monitor/azure-monitor-opentelemetry-exporter/samples/metrics/sample_instruments.py)
+
+
+## Installation
+
+```sh
+$ pip install azure-monitor-opentelemetry-exporter --pre
+```
+
+## Run the Applications
+
+### Metrics
+
+* Update `APPLICATIONINSIGHTS_CONNECTION_STRING` environment variable
+
+* Run the sample
+
+```sh
+$ # from this directory
+$ python sample_metrics.py
+```
+
+### Instrument usage
+
+* Update `APPLICATIONINSIGHTS_CONNECTION_STRING` environment variable
+
+* Run the sample
+
+```sh
+$ # from this directory
+$ python sample_instruments.py
+```
+
+## Explore the data
+
+After running the applications, data would be available in [Azure](
+https://docs.microsoft.com/azure/azure-monitor/app/app-insights-overview#where-do-i-see-my-telemetry)

sdk/monitor/azure-monitor-opentelemetry-exporter/samples/metrics/README.md

@@ -0,0 +1,64 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+"""
+An example to show an application using all instruments in the OpenTelemetry SDK. Metrics created
+and recorded using the sdk are tracked and telemetry is exported to application insights with the
+AzureMonitorMetricsExporter.
+"""
+import os
+
+from opentelemetry import _metrics
+from opentelemetry._metrics.measurement import Measurement
+from opentelemetry.sdk._metrics import MeterProvider
+from opentelemetry.sdk._metrics.export import PeriodicExportingMetricReader
+
+from azure.monitor.opentelemetry.exporter import AzureMonitorMetricExporter
+
+exporter = AzureMonitorMetricExporter.from_connection_string(
+    os.environ["APPLICATIONINSIGHTS_CONNECTION_STRING"]
+)
+reader = PeriodicExportingMetricReader(exporter, export_interval_millis=5000)
+_metrics.set_meter_provider(MeterProvider(metric_readers=[reader]))
+
+# Create a namespaced meter
+meter = _metrics.get_meter_provider().get_meter("sample")
+
+# Callback functions for observable instruments
+def observable_counter_func():
+    yield Measurement(1, {})
+
+
+def observable_up_down_counter_func():
+    yield Measurement(-10, {})
+
+
+def observable_gauge_func():
+    yield Measurement(9, {})
+
+# Counter
+counter = meter.create_counter("counter")
+counter.add(1)
+
+# Async Counter
+observable_counter = meter.create_observable_counter(
+    "observable_counter", observable_counter_func
+)
+
+# UpDownCounter
+updown_counter = meter.create_up_down_counter("updown_counter")
+updown_counter.add(1)
+updown_counter.add(-5)
+
+# Async UpDownCounter
+observable_updown_counter = meter.create_observable_up_down_counter(
+    "observable_updown_counter", observable_up_down_counter_func
+)
+
+# Histogram
+histogram = meter.create_histogram("histogram")
+histogram.record(99.9)
+
+# Async Gauge
+gauge = meter.create_observable_gauge("gauge", observable_gauge_func)
+
+input(...)