@@ -19,11 +19,7 @@ import { BaseMessage } from "@langchain/core/messages";
1919import { LLMResult } from "@langchain/core/outputs" ;
2020import { Serialized } from "@langchain/core/load/serializable" ;
2121import { ChainValues } from "@langchain/core/utils/types" ;
22- import {
23- Tracer ,
24- SpanKind ,
25- SpanStatusCode ,
26- } from "@opentelemetry/api" ;
22+ import { Tracer , SpanKind , SpanStatusCode } from "@opentelemetry/api" ;
2723import { SpanAttributes } from "@traceloop/ai-semantic-conventions" ;
2824
2925interface SpanData {
@@ -55,8 +51,11 @@ export class TraceloopCallbackHandler extends BaseCallbackHandler {
5551 _metadata ?: Record < string , unknown > ,
5652 _runName ?: string ,
5753 ) : Promise < void > {
54+ console . log ( "📞 handleChatModelStart called!" , {
55+ runId,
56+ className : llm . id ?. [ llm . id . length - 1 ] ,
57+ } ) ;
5858 const className = llm . id ?. [ llm . id . length - 1 ] || "unknown" ;
59- const modelName = this . extractModelName ( llm ) ;
6059 const vendor = this . detectVendor ( llm ) ;
6160 const spanBaseName = this . convertClassNameToSpanName ( className ) ;
6261
@@ -68,8 +67,7 @@ export class TraceloopCallbackHandler extends BaseCallbackHandler {
6867 const flatMessages = messages . flat ( ) ;
6968 span . setAttributes ( {
7069 [ SpanAttributes . LLM_SYSTEM ] : vendor ,
71- [ SpanAttributes . LLM_REQUEST_TYPE ] : "completion" ,
72- [ SpanAttributes . LLM_REQUEST_MODEL ] : modelName ,
70+ [ SpanAttributes . LLM_REQUEST_TYPE ] : "chat" ,
7371 } ) ;
7472
7573 // Add prompts if tracing content
@@ -100,7 +98,6 @@ export class TraceloopCallbackHandler extends BaseCallbackHandler {
10098 _runName ?: string ,
10199 ) : Promise < void > {
102100 const className = llm . id ?. [ llm . id . length - 1 ] || "unknown" ;
103- const modelName = this . extractModelName ( llm ) ;
104101 const vendor = this . detectVendor ( llm ) ;
105102 const spanBaseName = this . convertClassNameToSpanName ( className ) ;
106103
@@ -112,7 +109,6 @@ export class TraceloopCallbackHandler extends BaseCallbackHandler {
112109 span . setAttributes ( {
113110 [ SpanAttributes . LLM_SYSTEM ] : vendor ,
114111 [ SpanAttributes . LLM_REQUEST_TYPE ] : "completion" ,
115- [ SpanAttributes . LLM_REQUEST_MODEL ] : modelName ,
116112 } ) ;
117113
118114 if ( this . traceContent && prompts . length > 0 ) {
@@ -139,17 +135,31 @@ export class TraceloopCallbackHandler extends BaseCallbackHandler {
139135
140136 const { span } = spanData ;
141137
142- if ( this . traceContent && output . generations && output . generations . length > 0 ) {
138+ if (
139+ this . traceContent &&
140+ output . generations &&
141+ output . generations . length > 0
142+ ) {
143143 output . generations . forEach ( ( generation , idx ) => {
144144 if ( generation && generation . length > 0 ) {
145145 span . setAttributes ( {
146146 [ `${ SpanAttributes . LLM_COMPLETIONS } .${ idx } .role` ] : "assistant" ,
147- [ `${ SpanAttributes . LLM_COMPLETIONS } .${ idx } .content` ] : generation [ 0 ] . text ,
147+ [ `${ SpanAttributes . LLM_COMPLETIONS } .${ idx } .content` ] :
148+ generation [ 0 ] . text ,
148149 } ) ;
149150 }
150151 } ) ;
151152 }
152153
154+ // Extract model name from response only, like Python implementation
155+ const modelName = this . extractModelNameFromResponse ( output ) ;
156+
157+ // Set both request and response model attributes like Python implementation
158+ span . setAttributes ( {
159+ [ SpanAttributes . LLM_REQUEST_MODEL ] : modelName || "unknown" ,
160+ [ SpanAttributes . LLM_RESPONSE_MODEL ] : modelName || "unknown" ,
161+ } ) ;
162+
153163 // Add usage metrics if available
154164 if ( output . llmOutput ?. usage ) {
155165 const usage = output . llmOutput . usage ;
@@ -163,7 +173,8 @@ export class TraceloopCallbackHandler extends BaseCallbackHandler {
163173 [ SpanAttributes . LLM_USAGE_COMPLETION_TOKENS ] : usage . output_tokens ,
164174 } ) ;
165175 }
166- const totalTokens = ( usage . input_tokens || 0 ) + ( usage . output_tokens || 0 ) ;
176+ const totalTokens =
177+ ( usage . input_tokens || 0 ) + ( usage . output_tokens || 0 ) ;
167178 if ( totalTokens > 0 ) {
168179 span . setAttributes ( {
169180 [ SpanAttributes . LLM_USAGE_TOTAL_TOKENS ] : totalTokens ,
@@ -191,11 +202,6 @@ export class TraceloopCallbackHandler extends BaseCallbackHandler {
191202 }
192203 }
193204
194- // Set response model (same as request model for most cases)
195- span . setAttributes ( {
196- [ SpanAttributes . LLM_RESPONSE_MODEL ] : span . attributes ?. [ SpanAttributes . LLM_REQUEST_MODEL ] || "unknown" ,
197- } ) ;
198-
199205 span . setStatus ( { code : SpanStatusCode . OK } ) ;
200206 span . end ( ) ;
201207 this . spans . delete ( runId ) ;
@@ -368,27 +374,19 @@ export class TraceloopCallbackHandler extends BaseCallbackHandler {
368374 this . spans . delete ( runId ) ;
369375 }
370376
371- private extractModelName ( llm : Serialized ) : string {
372- // Extract from class hierarchy - last element is usually the class name
373- const className = llm . id ?. [ llm . id . length - 1 ] || "unknown" ;
374-
375- // For BedrockChat, try to get the actual model name
376- if ( className === "BedrockChat" ) {
377- // The model name might be available in kwargs - cast to any to access kwargs
378- const llmAny = llm as any ;
379- const modelId = llmAny . kwargs ?. model || llmAny . kwargs ?. model_id ;
380- if ( modelId && typeof modelId === "string" ) {
381- // Extract clean model name from full ID (e.g., "us.anthropic.claude-3-7-sonnet-20250219-v1:0" -> "claude-3-7-sonnet")
382- const parts = modelId . split ( "." ) ;
383- if ( parts . length >= 3 ) {
384- const modelPart = parts . slice ( 2 ) . join ( "." ) . split ( ":" ) [ 0 ] ; // Remove region and version
385- return modelPart . replace ( "-20250219-v1" , "" ) ; // Clean up version suffix
386- }
387- return modelId ;
377+ private extractModelNameFromResponse ( output : LLMResult ) : string | null {
378+ // Follow Python implementation - extract from llm_output first
379+ if ( output . llmOutput ) {
380+ const modelName =
381+ output . llmOutput . model_name ||
382+ output . llmOutput . model_id ||
383+ output . llmOutput . model ;
384+ if ( modelName && typeof modelName === "string" ) {
385+ return modelName ;
388386 }
389387 }
390388
391- return className ;
389+ return null ;
392390 }
393391
394392 private convertClassNameToSpanName ( className : string ) : string {
@@ -403,34 +401,115 @@ export class TraceloopCallbackHandler extends BaseCallbackHandler {
403401 private detectVendor ( llm : Serialized ) : string {
404402 const className = llm . id ?. [ llm . id . length - 1 ] || "" ;
405403
406- // Follow Python implementation - map class names to vendors
407- if ( className . includes ( "OpenAI" ) || className . includes ( "GPT" ) ) {
408- return "OpenAI" ;
404+ if ( ! className ) {
405+ return "Langchain" ;
409406 }
410- if ( className . includes ( "Anthropic" ) || className . includes ( "Claude" ) ) {
411- return "Anthropic" ;
407+
408+ // Follow Python implementation with exact matches and patterns
409+ // Ordered by specificity (most specific first)
410+
411+ // Azure (most specific - check first)
412+ if (
413+ [ "AzureChatOpenAI" , "AzureOpenAI" , "AzureOpenAIEmbeddings" ] . includes (
414+ className ,
415+ ) ||
416+ className . toLowerCase ( ) . includes ( "azure" )
417+ ) {
418+ return "Azure" ;
412419 }
413- if ( className . includes ( "Bedrock" ) || className === "BedrockChat" ) {
414- // Python implementation returns "AWS" for all Bedrock classes
420+
421+ // OpenAI
422+ if (
423+ [ "ChatOpenAI" , "OpenAI" , "OpenAIEmbeddings" ] . includes ( className ) ||
424+ className . toLowerCase ( ) . includes ( "openai" )
425+ ) {
426+ return "OpenAI" ;
427+ }
428+
429+ // AWS Bedrock
430+ if (
431+ [ "ChatBedrock" , "BedrockEmbeddings" , "Bedrock" , "BedrockChat" ] . includes (
432+ className ,
433+ ) ||
434+ className . toLowerCase ( ) . includes ( "bedrock" ) ||
435+ className . toLowerCase ( ) . includes ( "aws" )
436+ ) {
415437 return "AWS" ;
416438 }
417- if ( className . includes ( "Vertex" ) ) {
439+
440+ // Anthropic
441+ if (
442+ [ "ChatAnthropic" , "AnthropicLLM" ] . includes ( className ) ||
443+ className . toLowerCase ( ) . includes ( "anthropic" )
444+ ) {
445+ return "Anthropic" ;
446+ }
447+
448+ // Google (Vertex/PaLM/Gemini)
449+ if (
450+ [
451+ "ChatVertexAI" ,
452+ "VertexAI" ,
453+ "VertexAIEmbeddings" ,
454+ "ChatGoogleGenerativeAI" ,
455+ "GoogleGenerativeAI" ,
456+ "GooglePaLM" ,
457+ "ChatGooglePaLM" ,
458+ ] . includes ( className ) ||
459+ className . toLowerCase ( ) . includes ( "vertex" ) ||
460+ className . toLowerCase ( ) . includes ( "google" ) ||
461+ className . toLowerCase ( ) . includes ( "palm" ) ||
462+ className . toLowerCase ( ) . includes ( "gemini" )
463+ ) {
418464 return "Google" ;
419465 }
420- if ( className . includes ( "Azure" ) ) {
421- return "Azure" ;
466+
467+ // Cohere
468+ if (
469+ [ "ChatCohere" , "CohereEmbeddings" , "Cohere" ] . includes ( className ) ||
470+ className . toLowerCase ( ) . includes ( "cohere" )
471+ ) {
472+ return "Cohere" ;
422473 }
423- if ( className . includes ( "Hugging" ) ) {
474+
475+ // HuggingFace
476+ if (
477+ [
478+ "HuggingFacePipeline" ,
479+ "HuggingFaceTextGenInference" ,
480+ "HuggingFaceEmbeddings" ,
481+ "ChatHuggingFace" ,
482+ ] . includes ( className ) ||
483+ className . toLowerCase ( ) . includes ( "huggingface" )
484+ ) {
424485 return "HuggingFace" ;
425486 }
426- if ( className . includes ( "Ollama" ) ) {
487+
488+ // Ollama
489+ if (
490+ [ "ChatOllama" , "OllamaEmbeddings" , "Ollama" ] . includes ( className ) ||
491+ className . toLowerCase ( ) . includes ( "ollama" )
492+ ) {
427493 return "Ollama" ;
428494 }
429- if ( className . includes ( "Cohere" ) ) {
430- return "Cohere" ;
495+
496+ // Together
497+ if (
498+ [ "Together" , "ChatTogether" ] . includes ( className ) ||
499+ className . toLowerCase ( ) . includes ( "together" )
500+ ) {
501+ return "Together" ;
502+ }
503+
504+ // Replicate
505+ if (
506+ [ "Replicate" , "ChatReplicate" ] . includes ( className ) ||
507+ className . toLowerCase ( ) . includes ( "replicate" )
508+ ) {
509+ return "Replicate" ;
431510 }
432511
433- return "LangChain " ;
512+ return "Langchain " ;
434513 }
435514
436515 private mapMessageTypeToRole ( messageType : string ) : string {
0 commit comments