feat(anthropic): add support for Claude thinking API

nirga · claude · nirga · commit d97bd7ea6748 · 2025-08-21T15:04:07.000+03:00
Adds instrumentation support for Claude's extended thinking feature via the beta messages API. Changes: - Add instrumentation for anthropic.beta.messages.create() calls - Capture thinking parameters (type and budget_tokens) in span attributes - Handle response content with both thinking and text blocks - Add comprehensive test for thinking API functionality This enables proper observability for Claude thinking API calls, capturing both the reasoning process and final response in telemetry data. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
diff --git a/packages/instrumentation-anthropic/recordings/Test-Anthropic-instrumentation_3769946143/should-set-attributes-in-span-for-beta-messages-with-thinking_2020128547/recording.har b/packages/instrumentation-anthropic/recordings/Test-Anthropic-instrumentation_3769946143/should-set-attributes-in-span-for-beta-messages-with-thinking_2020128547/recording.har
@@ -0,0 +1,223 @@
+{
+  "log": {
+    "_recordingName": "Test Anthropic instrumentation/should set attributes in span for beta messages with thinking",
+    "creator": {
+      "comment": "persister:fs",
+      "name": "Polly.JS",
+      "version": "6.0.6"
+    },
+    "entries": [
+      {
+        "_id": "36fcbb1741e0f2a632c89d8c928a5d53",
+        "_order": 0,
+        "cache": {},
+        "request": {
+          "bodySize": 192,
+          "cookies": [],
+          "headers": [
+            {
+              "name": "accept",
+              "value": "application/json"
+            },
+            {
+              "name": "anthropic-beta",
+              "value": "interleaved-thinking-2025-05-14"
+            },
+            {
+              "name": "anthropic-version",
+              "value": "2023-06-01"
+            },
+            {
+              "name": "content-type",
+              "value": "application/json"
+            },
+            {
+              "name": "user-agent",
+              "value": "Anthropic/JS 0.56.0"
+            },
+            {
+              "name": "x-stainless-arch",
+              "value": "arm64"
+            },
+            {
+              "name": "x-stainless-lang",
+              "value": "js"
+            },
+            {
+              "name": "x-stainless-os",
+              "value": "MacOS"
+            },
+            {
+              "name": "x-stainless-package-version",
+              "value": "0.56.0"
+            },
+            {
+              "name": "x-stainless-retry-count",
+              "value": "0"
+            },
+            {
+              "name": "x-stainless-runtime",
+              "value": "node"
+            },
+            {
+              "name": "x-stainless-runtime-version",
+              "value": "v20.11.1"
+            },
+            {
+              "name": "x-stainless-timeout",
+              "value": "600"
+            }
+          ],
+          "headersSize": 584,
+          "httpVersion": "HTTP/1.1",
+          "method": "POST",
+          "postData": {
+            "mimeType": "application/json",
+            "params": [],
+            "text": "{\"max_tokens\":2048,\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2? Think through this step by step.\"}],\"model\":\"claude-opus-4-1-20250805\",\"thinking\":{\"type\":\"enabled\",\"budget_tokens\":1024}}"
+          },
+          "queryString": [
+            {
+              "name": "beta",
+              "value": "true"
+            }
+          ],
+          "url": "https://api.anthropic.com/v1/messages?beta=true"
+        },
+        "response": {
+          "bodySize": 1570,
+          "content": {
+            "mimeType": "application/json",
+            "size": 1570,
+            "text": "{\"id\":\"msg_018V3xGyrq6nc25GVuWiaKHx\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-opus-4-1-20250805\",\"content\":[{\"type\":\"thinking\",\"thinking\":\"This is a very simple arithmetic question. The user is asking for 2+2, which equals 4. They've asked me to think through it step by step, so I should show the basic addition process even though it's elementary.\",\"signature\":\"EvsCCkYIBhgCKkDcMyQ9Uh8CsGT5WmyeuwbI5yYSB1cbUyx5DC/zqmUMb0n5Zyi+Oz/fXpxGLJUmfnKp3zXuuOhybxupMRhBmK+3EgxVj1F8BGfmpqpOxjcaDIbIp9dBQHkej5KsTyIwJDTMjuH/q/vu4Pk/Zf4w9htqsZOPLfYdg/EbXIdeBNV4sJ6Jtiu+KzBg4O5fTgjPKuIBuD8ob8cR9xna6cV8JHxfUT9IeX3huQ2oF/vJC/99vqn4F//OEjiN8kCKPlJo28+S72odghUyF8TUITL/UIBWZ3kcQtwCdmytlB1+2Bld5osVVmOi4KApBkl9cRTOemDzkJHBFmhJ1AuUyZ2Fl2hVGmE2ACE8CPYU+iCZpZX2l4tWCT2M1wCaNTwNSqHcQtC/C0H9geP6Vyc2K2P6TcUIuUv8CFVIdqwcYDnbREhlY2Jv7nmaVDSraCvCXWj3Y/sQulDsBOqp6drQAITcWPJI8wbDmw8fIEhcCyujlcpanKzBhhgB\"},{\"type\":\"text\",\"text\":\"I'll work through this simple addition step by step.\\n\\n**Step 1:** Identify what we're adding\\n- We have two numbers: 2 and 2\\n- We need to add them together\\n\\n**Step 2:** Perform the addition\\n- Start with the first number: 2\\n- Add the second number: + 2\\n- When we combine 2 items with 2 more items, we get 4 items total\\n\\n**Step 3:** State the result\\n- 2 + 2 = 4\\n\\nThe answer is **4**.\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":49,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":186,\"service_tier\":\"standard\"}}"
+          },
+          "cookies": [],
+          "headers": [
+            {
+              "name": "anthropic-organization-id",
+              "value": "617d109c-a187-4902-889d-689223d134aa"
+            },
+            {
+              "name": "anthropic-ratelimit-input-tokens-limit",
+              "value": "2000000"
+            },
+            {
+              "name": "anthropic-ratelimit-input-tokens-remaining",
+              "value": "2000000"
+            },
+            {
+              "name": "anthropic-ratelimit-input-tokens-reset",
+              "value": "2025-08-21T11:41:58Z"
+            },
+            {
+              "name": "anthropic-ratelimit-output-tokens-limit",
+              "value": "400000"
+            },
+            {
+              "name": "anthropic-ratelimit-output-tokens-remaining",
+              "value": "400000"
+            },
+            {
+              "name": "anthropic-ratelimit-output-tokens-reset",
+              "value": "2025-08-21T11:42:02Z"
+            },
+            {
+              "name": "anthropic-ratelimit-requests-limit",
+              "value": "4000"
+            },
+            {
+              "name": "anthropic-ratelimit-requests-remaining",
+              "value": "3999"
+            },
+            {
+              "name": "anthropic-ratelimit-requests-reset",
+              "value": "2025-08-21T11:41:57Z"
+            },
+            {
+              "name": "anthropic-ratelimit-tokens-limit",
+              "value": "2400000"
+            },
+            {
+              "name": "anthropic-ratelimit-tokens-remaining",
+              "value": "2400000"
+            },
+            {
+              "name": "anthropic-ratelimit-tokens-reset",
+              "value": "2025-08-21T11:41:58Z"
+            },
+            {
+              "name": "cf-cache-status",
+              "value": "DYNAMIC"
+            },
+            {
+              "name": "cf-ray",
+              "value": "9729dd411cdd6756-ATL"
+            },
+            {
+              "name": "connection",
+              "value": "keep-alive"
+            },
+            {
+              "name": "content-encoding",
+              "value": "gzip"
+            },
+            {
+              "name": "content-type",
+              "value": "application/json"
+            },
+            {
+              "name": "date",
+              "value": "Thu, 21 Aug 2025 11:42:02 GMT"
+            },
+            {
+              "name": "request-id",
+              "value": "req_011CSLo11ceKMKF1kTBWoKxZ"
+            },
+            {
+              "name": "server",
+              "value": "cloudflare"
+            },
+            {
+              "name": "strict-transport-security",
+              "value": "max-age=31536000; includeSubDomains; preload"
+            },
+            {
+              "name": "transfer-encoding",
+              "value": "chunked"
+            },
+            {
+              "name": "via",
+              "value": "1.1 google"
+            },
+            {
+              "name": "x-envoy-upstream-service-time",
+              "value": "5555"
+            },
+            {
+              "name": "x-robots-tag",
+              "value": "none"
+            }
+          ],
+          "headersSize": 1098,
+          "httpVersion": "HTTP/1.1",
+          "redirectURL": "",
+          "status": 200,
+          "statusText": "OK"
+        },
+        "startedDateTime": "2025-08-21T11:41:56.089Z",
+        "time": 6594,
+        "timings": {
+          "blocked": -1,
+          "connect": -1,
+          "dns": -1,
+          "receive": 0,
+          "send": 0,
+          "ssl": -1,
+          "wait": 6594
+        }
+      }
+    ],
+    "pages": [],
+    "version": "1.2"
+  }
+}
diff --git a/packages/instrumentation-anthropic/src/instrumentation.ts b/packages/instrumentation-anthropic/src/instrumentation.ts
@@ -72,6 +72,11 @@ export class AnthropicInstrumentation extends InstrumentationBase {
       "create",
       this.patchAnthropic("chat", module),
     );
+    this._wrap(
+      module.Anthropic.Beta.Messages.prototype,
+      "create",
+      this.patchAnthropic("chat", module),
+    );
   }
 
   protected init(): InstrumentationModuleDefinition {
@@ -97,6 +102,11 @@ export class AnthropicInstrumentation extends InstrumentationBase {
       "create",
       this.patchAnthropic("chat", moduleExports),
     );
+    this._wrap(
+      moduleExports.Anthropic.Beta.Messages.prototype,
+      "create",
+      this.patchAnthropic("chat", moduleExports),
+    );
     return moduleExports;
   }
 
@@ -108,6 +118,7 @@ export class AnthropicInstrumentation extends InstrumentationBase {
 
     this._unwrap(moduleExports.Anthropic.Completions.prototype, "create");
     this._unwrap(moduleExports.Anthropic.Messages.prototype, "create");
+    this._unwrap(moduleExports.Anthropic.Beta.Messages.prototype, "create");
   }
 
   private patchAnthropic(
@@ -202,6 +213,17 @@ export class AnthropicInstrumentation extends InstrumentationBase {
       attributes[SpanAttributes.LLM_REQUEST_TOP_P] = params.top_p;
       attributes[SpanAttributes.LLM_TOP_K] = params.top_k;
 
+      // Handle thinking parameters
+      if ((params as any).thinking) {
+        const thinking = (params as any).thinking;
+        if (thinking.type) {
+          attributes["llm.request.thinking.type"] = thinking.type;
+        }
+        if (thinking.budget_tokens) {
+          attributes["llm.request.thinking.budget_tokens"] = thinking.budget_tokens;
+        }
+      }
+
       if (type === "completion") {
         attributes[SpanAttributes.LLM_REQUEST_MAX_TOKENS] =
           params.max_tokens_to_sample;
diff --git a/packages/instrumentation-anthropic/test/instrumentation.test.ts b/packages/instrumentation-anthropic/test/instrumentation.test.ts
@@ -328,4 +328,82 @@ describe("Test Anthropic instrumentation", async function () {
       "user",
     );
   }).timeout(30000);
+
+  it("should set attributes in span for beta messages with thinking", async () => {
+    const message = await anthropic.beta.messages.create({
+      max_tokens: 2048,
+      betas: ['interleaved-thinking-2025-05-14'],
+      messages: [
+        { role: "user", content: "What is 2+2? Think through this step by step." },
+      ],
+      model: "claude-opus-4-1-20250805",
+      thinking: {
+        type: "enabled",
+        budget_tokens: 1024,
+      },
+    });
+
+    const spans = memoryExporter.getFinishedSpans();
+    const chatSpan = spans.find((span) => span.name === "anthropic.chat");
+
+    assert.ok(message);
+    assert.ok(chatSpan);
+    assert.strictEqual(
+      chatSpan.attributes[`${SpanAttributes.LLM_REQUEST_MODEL}`],
+      "claude-opus-4-1-20250805",
+    );
+    assert.strictEqual(
+      chatSpan.attributes[`${SpanAttributes.LLM_RESPONSE_MODEL}`],
+      "claude-opus-4-1-20250805",
+    );
+    assert.strictEqual(
+      chatSpan.attributes[`${SpanAttributes.LLM_REQUEST_MAX_TOKENS}`],
+      2048,
+    );
+    
+    // Check if thinking parameters are captured (these will fail initially)
+    assert.strictEqual(
+      chatSpan.attributes["llm.request.thinking.type"],
+      "enabled",
+    );
+    assert.strictEqual(
+      chatSpan.attributes["llm.request.thinking.budget_tokens"],
+      1024,
+    );
+
+    // Check prompts
+    assert.strictEqual(
+      chatSpan.attributes[`${SpanAttributes.LLM_PROMPTS}.0.role`],
+      "user",
+    );
+    assert.strictEqual(
+      chatSpan.attributes[`${SpanAttributes.LLM_PROMPTS}.0.content`],
+      "What is 2+2? Think through this step by step.",
+    );
+
+    // Check that we capture both thinking and regular content blocks
+    const content = JSON.parse(chatSpan.attributes[`${SpanAttributes.LLM_COMPLETIONS}.0.content`] as string);
+    assert.ok(Array.isArray(content));
+    
+    const thinkingBlock = content.find((block: any) => block.type === "thinking");
+    const textBlock = content.find((block: any) => block.type === "text");
+    
+    assert.ok(thinkingBlock, "Should contain a thinking block");
+    assert.ok(thinkingBlock.thinking, "Thinking block should have thinking content");
+    assert.ok(textBlock, "Should contain a text block");
+    assert.ok(textBlock.text, "Text block should have text content");
+
+    // Verify token usage includes thinking tokens
+    assert.ok(
+      +chatSpan.attributes[`${SpanAttributes.LLM_USAGE_COMPLETION_TOKENS}`]! > 0,
+    );
+    assert.ok(
+      +chatSpan.attributes[`${SpanAttributes.LLM_USAGE_PROMPT_TOKENS}`]! > 0,
+    );
+    assert.equal(
+      +chatSpan.attributes[`${SpanAttributes.LLM_USAGE_PROMPT_TOKENS}`]! +
+        +chatSpan.attributes[`${SpanAttributes.LLM_USAGE_COMPLETION_TOKENS}`]!,
+      chatSpan.attributes[`${SpanAttributes.LLM_USAGE_TOTAL_TOKENS}`],
+    );
+  }).timeout(30000);
 });
