Add duration metric to tool calls

Author: keith-decker

util/opentelemetry-util-genai/src/opentelemetry/util/genai/handler.py

@@ -137,33 +137,25 @@ def __init__(
             schema_url=schema_url,
         )
 
-    def _record_llm_metrics(
+    def _record_metrics(
         self,
-        invocation: LLMInvocation,
+        invocation: GenAIInvocation,
         span: Span | None = None,
         *,
         error_type: str | None = None,
     ) -> None:
+        """Record metrics for an invocation."""
         if self._metrics_recorder is None or span is None:
             return
+        # Only LLMInvocation and ToolCall metrics are currently supported
+        if not isinstance(invocation, (LLMInvocation, ToolCall)):
+            return
         self._metrics_recorder.record(
             span,
             invocation,
             error_type=error_type,
         )
 
-    @staticmethod
-    def _record_embedding_metrics(
-        invocation: EmbeddingInvocation,
-        span: Span | None = None,
-        *,
-        error_type: str | None = None,
-    ) -> None:
-        # Metrics recorder currently supports LLMInvocation fields only.
-        # Keep embedding metrics as a no-op until dedicated embedding
-        # metric support is added.
-        return
-
     def _start(self, invocation: _T) -> _T:
         """Start a GenAI invocation and create a pending span entry."""
         span_kind = SpanKind.CLIENT
@@ -203,13 +195,14 @@ def _stop(self, invocation: _T) -> _T:
         try:
             if isinstance(invocation, LLMInvocation):
                 _apply_llm_finish_attributes(span, invocation)
-                self._record_llm_metrics(invocation, span)
+                self._record_metrics(invocation, span)
                 _maybe_emit_llm_event(self._logger, span, invocation)
             elif isinstance(invocation, EmbeddingInvocation):
                 _apply_embedding_finish_attributes(span, invocation)
-                self._record_embedding_metrics(invocation, span)
+                self._record_metrics(invocation, span)
             elif isinstance(invocation, ToolCall):
                 _finish_tool_call_span(span, invocation, capture_content=True)
+                self._record_metrics(invocation, span)
         finally:
             # Detach context and end span even if finishing fails
             otel_context.detach(invocation.context_token)
@@ -228,21 +221,18 @@ def _fail(self, invocation: _T, error: Error) -> _T:
             if isinstance(invocation, LLMInvocation):
                 _apply_llm_finish_attributes(span, invocation)
                 _apply_error_attributes(span, error, error_type)
-                self._record_llm_metrics(
-                    invocation, span, error_type=error_type
-                )
+                self._record_metrics(invocation, span, error_type=error_type)
                 _maybe_emit_llm_event(
                     self._logger, span, invocation, error_type
                 )
             elif isinstance(invocation, EmbeddingInvocation):
                 _apply_embedding_finish_attributes(span, invocation)
                 _apply_error_attributes(span, error, error_type)
-                self._record_embedding_metrics(
-                    invocation, span, error_type=error_type
-                )
+                self._record_metrics(invocation, span, error_type=error_type)
             elif isinstance(invocation, ToolCall):
                 invocation.error_type = error_type
                 _finish_tool_call_span(span, invocation, capture_content=True)
+                self._record_metrics(invocation, span, error_type=error_type)
                 span.set_status(Status(StatusCode.ERROR, error.message))
         finally:
             # Detach context and end span even if finishing fails

util/opentelemetry-util-genai/src/opentelemetry/util/genai/metrics.py

@@ -1,9 +1,9 @@
-"""Helpers for emitting GenAI metrics from LLM invocations."""
+"""Helpers for emitting GenAI metrics from invocations."""
 
 from __future__ import annotations
 
 import timeit
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
 
 from opentelemetry.metrics import Histogram, Meter
 from opentelemetry.semconv._incubating.attributes import (
@@ -18,7 +18,7 @@
     create_duration_histogram,
     create_token_histogram,
 )
-from opentelemetry.util.genai.types import LLMInvocation
+from opentelemetry.util.genai.types import LLMInvocation, ToolCall
 from opentelemetry.util.types import AttributeValue
 
 
@@ -32,44 +32,41 @@ def __init__(self, meter: Meter):
     def record(
         self,
         span: Optional[Span],
-        invocation: LLMInvocation,
+        invocation: Union[LLMInvocation, ToolCall],
         *,
         error_type: Optional[str] = None,
     ) -> None:
-        """Record duration and token metrics for an invocation if possible."""
+        """Record duration and token metrics for an invocation.
 
+        Supports LLMInvocation (with token metrics) and ToolCall (duration only).
+        """
         # pylint: disable=too-many-branches
 
         if span is None:
             return
 
-        token_counts: list[tuple[int, str]] = []
-        if invocation.input_tokens is not None:
-            token_counts.append(
-                (
-                    invocation.input_tokens,
-                    GenAI.GenAiTokenTypeValues.INPUT.value,
-                )
+        # Build attributes based on invocation type
+        attributes: Dict[str, AttributeValue] = {}
+
+        if isinstance(invocation, LLMInvocation):
+            attributes[GenAI.GEN_AI_OPERATION_NAME] = (
+                GenAI.GenAiOperationNameValues.CHAT.value
             )
-        if invocation.output_tokens is not None:
-            token_counts.append(
-                (
-                    invocation.output_tokens,
-                    GenAI.GenAiTokenTypeValues.OUTPUT.value,
+            if invocation.request_model:
+                attributes[GenAI.GEN_AI_REQUEST_MODEL] = (
+                    invocation.request_model
                 )
-            )
+            if invocation.response_model_name:
+                attributes[GenAI.GEN_AI_RESPONSE_MODEL] = (
+                    invocation.response_model_name
+                )
+        else:
+            # ToolCall
+            attributes[GenAI.GEN_AI_OPERATION_NAME] = "execute_tool"
 
-        attributes: Dict[str, AttributeValue] = {
-            GenAI.GEN_AI_OPERATION_NAME: GenAI.GenAiOperationNameValues.CHAT.value
-        }
-        if invocation.request_model:
-            attributes[GenAI.GEN_AI_REQUEST_MODEL] = invocation.request_model
+        # Common attributes across invocation types
         if invocation.provider:
             attributes[GenAI.GEN_AI_PROVIDER_NAME] = invocation.provider
-        if invocation.response_model_name:
-            attributes[GenAI.GEN_AI_RESPONSE_MODEL] = (
-                invocation.response_model_name
-            )
         if invocation.server_address:
             attributes[server_attributes.SERVER_ADDRESS] = (
                 invocation.server_address
@@ -79,7 +76,7 @@ def record(
         if invocation.metric_attributes:
             attributes.update(invocation.metric_attributes)
 
-        # Calculate duration from span timing or invocation monotonic start
+        # Calculate duration from monotonic start time
         duration_seconds: Optional[float] = None
         if invocation.monotonic_start_s is not None:
             duration_seconds = max(
@@ -98,12 +95,31 @@ def record(
                 context=span_context,
             )
 
-        for token_count, token_type in token_counts:
-            self._token_histogram.record(
-                token_count,
-                attributes=attributes | {GenAI.GEN_AI_TOKEN_TYPE: token_type},
-                context=span_context,
-            )
+        # Token metrics only for LLMInvocation
+        if isinstance(invocation, LLMInvocation):
+            token_counts: list[tuple[int, str]] = []
+            if invocation.input_tokens is not None:
+                token_counts.append(
+                    (
+                        invocation.input_tokens,
+                        GenAI.GenAiTokenTypeValues.INPUT.value,
+                    )
+                )
+            if invocation.output_tokens is not None:
+                token_counts.append(
+                    (
+                        invocation.output_tokens,
+                        GenAI.GenAiTokenTypeValues.OUTPUT.value,
+                    )
+                )
+
+            for token_count, token_type in token_counts:
+                self._token_histogram.record(
+                    token_count,
+                    attributes=attributes
+                    | {GenAI.GEN_AI_TOKEN_TYPE: token_type},
+                    context=span_context,
+                )
 
 
 __all__ = ["InvocationMetricsRecorder"]

util/opentelemetry-util-genai/src/opentelemetry/util/genai/types.py

@@ -411,6 +411,17 @@ class ToolCall(GenAIInvocation):
     # gen_ai.tool.call.result - Result returned by the tool (Opt-In, may contain sensitive data)
     tool_result: Any = None
 
+    # Metric-related fields (for gen_ai.client.operation.duration)
+    provider: str | None = None  # gen_ai.provider.name (Required for metrics)
+    server_address: str | None = None  # server.address (Recommended)
+    server_port: int | None = None  # server.port (Conditionally Required)
+    metric_attributes: dict[str, Any] = field(
+        default_factory=_new_str_any_dict
+    )
+    """
+    Additional attributes to set on metrics. Must be of low cardinality.
+    """
+
     # Timing field (not inherited from GenAIInvocation, matches LLMInvocation pattern)
     monotonic_start_s: float | None = None
 

util/opentelemetry-util-genai/tests/test_handler_metrics.py

@@ -382,3 +382,150 @@ class ToolExecutionError(RuntimeError):
             "ToolExecutionError",
             span.attributes[error_attributes.ERROR_TYPE],
         )
+
+
+class TelemetryHandlerToolMetricsTest(TestBase):
+    """Tests for tool call metrics recording"""
+
+    def _harvest_metrics(self) -> Dict[str, List[Any]]:
+        """Returns metrics_by_name mapping metric name to list of data points."""
+        metrics = self.get_sorted_metrics(SCOPE)
+        metrics_by_name: Dict[str, List[Any]] = {}
+        for metric in metrics or []:
+            points = metric.data.data_points or []
+            metrics_by_name.setdefault(metric.name, []).extend(points)
+        return metrics_by_name
+
+    def test_stop_tool_call_records_duration(self) -> None:
+        """Test stop records duration metric for tool call"""
+        handler = TelemetryHandler(
+            tracer_provider=self.tracer_provider,
+            meter_provider=self.meter_provider,
+        )
+        tool = ToolCall(
+            name="get_weather",
+            arguments={"location": "Paris"},
+            id="call_123",
+            provider="test-provider",
+        )
+
+        with patch("timeit.default_timer", return_value=1000.0):
+            handler.start(tool)
+
+        with patch("timeit.default_timer", return_value=1002.5):
+            handler.stop(tool)
+
+        metrics = self._harvest_metrics()
+        self.assertIn("gen_ai.client.operation.duration", metrics)
+        duration_points = metrics["gen_ai.client.operation.duration"]
+        self.assertEqual(len(duration_points), 1)
+        duration_point = duration_points[0]
+
+        # Check required attributes
+        self.assertEqual(
+            duration_point.attributes[GenAI.GEN_AI_OPERATION_NAME],
+            "execute_tool",
+        )
+        self.assertEqual(
+            duration_point.attributes[GenAI.GEN_AI_PROVIDER_NAME],
+            "test-provider",
+        )
+
+        # Check duration value
+        self.assertAlmostEqual(duration_point.sum, 2.5, places=3)
+
+        # Token metrics should NOT be recorded for tool calls
+        self.assertNotIn("gen_ai.client.token.usage", metrics)
+
+    def test_stop_tool_call_records_duration_with_server_address(self) -> None:
+        """Test stop records duration with server attributes for tool call"""
+        handler = TelemetryHandler(
+            tracer_provider=self.tracer_provider,
+            meter_provider=self.meter_provider,
+        )
+        tool = ToolCall(
+            name="api_call",
+            arguments={},
+            id="call_456",
+            provider="custom-provider",
+            server_address="api.example.com",
+            server_port=443,
+        )
+
+        with patch("timeit.default_timer", return_value=100.0):
+            handler.start(tool)
+
+        with patch("timeit.default_timer", return_value=100.5):
+            handler.stop(tool)
+
+        metrics = self._harvest_metrics()
+        duration_points = metrics["gen_ai.client.operation.duration"]
+        self.assertEqual(len(duration_points), 1)
+        duration_point = duration_points[0]
+
+        self.assertEqual(
+            duration_point.attributes["server.address"], "api.example.com"
+        )
+        self.assertEqual(duration_point.attributes["server.port"], 443)
+
+    def test_stop_tool_call_records_metric_attributes(self) -> None:
+        """Test stop includes custom metric_attributes for tool call"""
+        handler = TelemetryHandler(
+            tracer_provider=self.tracer_provider,
+            meter_provider=self.meter_provider,
+        )
+        tool = ToolCall(
+            name="custom_tool",
+            arguments={},
+            provider="my-provider",
+        )
+        tool.metric_attributes = {"custom.key": "custom_value"}
+
+        with patch("timeit.default_timer", return_value=0.0):
+            handler.start(tool)
+
+        with patch("timeit.default_timer", return_value=1.0):
+            handler.stop(tool)
+
+        metrics = self._harvest_metrics()
+        duration_point = metrics["gen_ai.client.operation.duration"][0]
+
+        self.assertEqual(
+            duration_point.attributes["custom.key"], "custom_value"
+        )
+
+    def test_fail_tool_call_records_duration_with_error(self) -> None:
+        """Test fail records duration with error.type for tool call"""
+        handler = TelemetryHandler(
+            tracer_provider=self.tracer_provider,
+            meter_provider=self.meter_provider,
+        )
+        tool = ToolCall(
+            name="failing_tool",
+            arguments={},
+            id="call_err",
+            provider="err-provider",
+        )
+
+        with patch("timeit.default_timer", return_value=500.0):
+            handler.start(tool)
+
+        error = Error(message="Tool execution failed", type=RuntimeError)
+        with patch("timeit.default_timer", return_value=501.5):
+            handler.fail(tool, error)
+
+        metrics = self._harvest_metrics()
+        self.assertIn("gen_ai.client.operation.duration", metrics)
+        duration_points = metrics["gen_ai.client.operation.duration"]
+        self.assertEqual(len(duration_points), 1)
+        duration_point = duration_points[0]
+
+        # Check error.type is recorded
+        self.assertEqual(
+            duration_point.attributes["error.type"], "RuntimeError"
+        )
+        self.assertEqual(
+            duration_point.attributes[GenAI.GEN_AI_OPERATION_NAME],
+            "execute_tool",
+        )
+        self.assertAlmostEqual(duration_point.sum, 1.5, places=3)