99
1010import asyncio
1111import json
12+ import logging
1213from enum import Enum
1314from typing import Any , ClassVar , cast
1415
3132 resolve_interpolatable_enum ,
3233 resolve_interpolatable_numeric ,
3334)
35+ from .llm_config import LLMConfigLoader
36+
37+ logger = logging .getLogger (__name__ )
3438
3539# ===========================================================================
3640# Type Definitions
@@ -192,23 +196,30 @@ def validate_response_schema(cls, v: dict[str, Any] | str | None) -> dict[str, A
192196
193197 @model_validator (mode = "after" )
194198 def validate_profile_or_provider_model (self ) -> LLMCallInput :
195- """Ensure either profile OR (provider + model) is specified.
199+ """Validate LLM configuration - profile and provider are mutually exclusive.
200+
201+ Valid configurations:
202+ 1. profile specified (resolved from config, may fallback to default_profile)
203+ 2. provider specified (model is optional, can be empty string)
204+ 3. neither specified (will error at execution with better context)
205+
206+ Invalid:
207+ - Both profile and provider specified (ambiguous)
196208
197209 Raises:
198- ValueError: Missing required configuration
210+ ValueError: Invalid configuration
199211 """
200- if self .profile is None :
201- # No profile - provider and model are required
202- if self .provider is None :
203- raise ValueError (
204- "Either 'profile' OR 'provider' must be specified. "
205- "Use profile for config-based setup, or direct provider+model."
206- )
207- if self .model is None :
208- raise ValueError (
209- "When 'provider' is specified without 'profile', 'model' is required."
210- )
212+ # Error: Both profile and provider specified
213+ if self .profile is not None and self .provider is not None :
214+ raise ValueError (
215+ "Cannot specify both 'profile' and 'provider'. Choose one:\n "
216+ " - Use 'profile' for config-based setup, OR\n "
217+ " - Use 'provider' (+ optional 'model') for direct specification"
218+ )
211219
220+ # OK: Profile specified (validated at execution)
221+ # OK: Provider specified (model is optional)
222+ # OK: Neither specified (error at execution with better context)
212223 return self
213224
214225
@@ -305,21 +316,41 @@ class LLMCallExecutor(BlockExecutor):
305316 async def execute ( # type: ignore[override]
306317 self , inputs : LLMCallInput , context : Execution
307318 ) -> LLMCallOutput :
308- """Execute LLM call with retry logic and client-side schema validation .
319+ """Execute LLM call with retry logic and profile fallback .
309320
310- Resolves profile configuration, calls provider API with exponential backoff
311- retry, and validates responses client-side for maximum compatibility.
321+ Resolves profile configuration with fallback to default_profile, calls provider
322+ API with exponential backoff retry, and validates responses client-side for
323+ maximum compatibility.
312324
313325 Raises:
314326 ValueError: Invalid configuration
315327 httpx.*: Network errors after retry exhaustion
316328 """
317- # Step 1: Profile Resolution (if profile specified)
329+ # Get execution context for config access
330+ execution_context = context .execution_context
331+ if execution_context is None :
332+ raise ValueError ("ExecutionContext not available. Cannot resolve LLM configuration." )
333+
334+ llm_config_loader = execution_context .llm_config_loader
335+
336+ # Step 1: Resolve profile (with fallback logic)
337+ effective_profile = self ._resolve_profile_with_fallback (inputs , llm_config_loader )
338+
339+ # Step 2: Profile resolution (if profile determined)
318340 effective_inputs = inputs
319- if inputs .profile is not None :
320- effective_inputs = await self ._resolve_profile_to_inputs (inputs , context )
341+ profile_fallback_occurred = False
342+
343+ if effective_profile is not None :
344+ # Track if fallback occurred (profile requested but different one used)
345+ profile_fallback_occurred = (
346+ inputs .profile is not None and inputs .profile != effective_profile
347+ )
348+
349+ # Create new input with resolved profile
350+ inputs_with_profile = inputs .model_copy (update = {"profile" : effective_profile })
351+ effective_inputs = await self ._resolve_profile_to_inputs (inputs_with_profile , context )
321352
322- # Step 2 : Resolve interpolatable numeric fields to their actual types
353+ # Step 3 : Resolve interpolatable numeric fields to their actual types
323354 max_retries = resolve_interpolatable_numeric (
324355 effective_inputs .max_retries , int , "max_retries" , ge = 1 , le = 10
325356 )
@@ -382,13 +413,21 @@ async def execute( # type: ignore[override]
382413 )
383414
384415 # Success - return validated JSON structure directly
416+ metadata = {
417+ "attempts" : attempts ,
418+ ** provider_metadata ,
419+ }
420+ # Add fallback info if applicable
421+ if profile_fallback_occurred :
422+ metadata ["profile_fallback" ] = {
423+ "requested" : inputs .profile ,
424+ "resolved" : effective_profile ,
425+ }
426+
385427 return LLMCallOutput (
386428 response = validated_response ,
387429 success = True ,
388- metadata = {
389- "attempts" : attempts ,
390- ** provider_metadata ,
391- },
430+ metadata = metadata ,
392431 )
393432 except ValueError as e :
394433 # Validation failed
@@ -397,15 +436,23 @@ async def execute( # type: ignore[override]
397436
398437 # If this is the last attempt, return with validation failure
399438 if attempt == max_retries - 1 :
439+ metadata = {
440+ "attempts" : attempts ,
441+ "validation_failed" : True ,
442+ "validation_error" : validation_error ,
443+ ** provider_metadata ,
444+ }
445+ # Add fallback info if applicable
446+ if profile_fallback_occurred :
447+ metadata ["profile_fallback" ] = {
448+ "requested" : inputs .profile ,
449+ "resolved" : effective_profile ,
450+ }
451+
400452 return LLMCallOutput (
401453 response = {"content" : response_text },
402454 success = True , # API call succeeded, validation failed
403- metadata = {
404- "attempts" : attempts ,
405- "validation_failed" : True ,
406- "validation_error" : validation_error ,
407- ** provider_metadata ,
408- },
455+ metadata = metadata ,
409456 )
410457
411458 # Otherwise, wait and retry with feedback
@@ -414,13 +461,21 @@ async def execute( # type: ignore[override]
414461 continue
415462 else :
416463 # No schema provided - return raw text in content key
464+ metadata = {
465+ "attempts" : attempts ,
466+ ** provider_metadata ,
467+ }
468+ # Add fallback info if applicable
469+ if profile_fallback_occurred :
470+ metadata ["profile_fallback" ] = {
471+ "requested" : inputs .profile ,
472+ "resolved" : effective_profile ,
473+ }
474+
417475 return LLMCallOutput (
418476 response = {"content" : response_text },
419477 success = True ,
420- metadata = {
421- "attempts" : attempts ,
422- ** provider_metadata ,
423- },
478+ metadata = metadata ,
424479 )
425480
426481 except (
@@ -528,6 +583,68 @@ async def _resolve_profile_to_inputs(
528583 validation_prompt_template = inputs .validation_prompt_template ,
529584 )
530585
586+ def _resolve_profile_with_fallback (
587+ self ,
588+ inputs : LLMCallInput ,
589+ llm_config_loader : LLMConfigLoader ,
590+ ) -> str | None :
591+ """Resolve profile with fallback to default_profile.
592+
593+ Resolution logic:
594+ 1. Direct provider/model specified → None (bypass profiles)
595+ 2. Profile exists in config → use it
596+ 3. Profile missing + default_profile exists → WARN and use default_profile
597+ 4. Profile missing + no default_profile → ERROR
598+ 5. No profile and no provider → ERROR (explicit required)
599+
600+ Args:
601+ inputs: LLMCall inputs with profile/provider configuration
602+ llm_config_loader: Config loader with profile definitions
603+
604+ Returns:
605+ Profile name to use, or None for direct provider config
606+
607+ Raises:
608+ ValueError: Missing or invalid configuration
609+ """
610+ config = llm_config_loader .load_config ()
611+
612+ # Case 1: Direct provider/model bypasses profiles
613+ if inputs .provider is not None :
614+ return None
615+
616+ # Case 2: Profile specified
617+ if inputs .profile is not None :
618+ # Profile exists - use it
619+ if inputs .profile in config .profiles :
620+ return inputs .profile
621+
622+ # Profile missing - try fallback to default_profile
623+ if config .default_profile is not None :
624+ logger .warning (
625+ f"Profile '{ inputs .profile } ' not found in config. "
626+ f"Falling back to default_profile '{ config .default_profile } '. "
627+ f"Available profiles: { ', ' .join (config .profiles .keys ())} "
628+ )
629+ return config .default_profile
630+
631+ # No fallback available - error
632+ available = ", " .join (config .profiles .keys ()) if config .profiles else "none"
633+ raise ValueError (
634+ f"Profile '{ inputs .profile } ' not found and no default_profile set.\n "
635+ f"Available profiles: { available } \n "
636+ f"Either:\n "
637+ f" 1. Add '{ inputs .profile } ' profile to ~/.workflows/llm-config.yml, OR\n "
638+ f" 2. Set 'default_profile' in ~/.workflows/llm-config.yml"
639+ )
640+
641+ # Case 3: Neither profile nor provider - error (explicit required)
642+ raise ValueError (
643+ "LLM configuration required. Either:\n "
644+ " 1. Specify 'profile' in LLMCall block, OR\n "
645+ " 2. Specify 'provider' and 'model' directly"
646+ )
647+
531648 async def _call_provider (
532649 self ,
533650 inputs : LLMCallInput ,
0 commit comments