Enhance LiteLLM integration with debug toggling and provider overrides

- Added configuration options for enabling LiteLLM debugging in README.md and settings.py.
- Implemented provider-specific model routing and credential handling in review.py and task_executor.py.
- Updated automated review functionality to respect the LiteLLM debug setting and allow for provider overrides in review configurations.
- Enhanced test coverage for model resolution and JSON serialization in test_review_engine.py.

Author: Wojciech Napierała

README.md

@@ -49,6 +49,11 @@ The project targets **Python 3.12**. Create a virtual environment with that inte
     and optionally `AZURE_OPENAI_API_VERSION`.
   - Ollama: override `OLLAMA_BASE_URL` when running a remote instance.
   - Missing credentials cause a descriptive `WorkflowError` to surface.
+- Enable LiteLLM debugging by setting `llm.enable_debug` to `true` in the
+  configuration when deeper request/response tracing is needed.
+- Automated review can use a distinct provider by setting
+  `review.auto_reviewer_provider`; if omitted it inherits the default workflow's
+  provider.
 - Embeddings (default: Azure OpenAI `text-embedding-3-large`):
   - Set `AZURE_OPENAI_EMBEDDING_DEPLOYMENT` if your embedding deployment name
     differs from the model identifier.

config/settings.py

@@ -1,6 +1,9 @@
 """Helpers for loading and validating DRM configuration files.
 
-Updates: v0.1 - 2025-11-06 - Added Pydantic-based loader for core configuration.
+Updates:
+    v0.1 - 2025-11-06 - Added Pydantic-based loader for core configuration.
+    v0.2 - 2025-11-07 - Added LiteLLM debug toggle to LLM configuration schema.
+    v0.3 - 2025-11-07 - Added review model provider overrides.
 """
 
 from __future__ import annotations
@@ -44,6 +47,10 @@ class LLMConfig(BaseModel):
     default_workflow: str = Field(..., min_length=1)
     workflows: Dict[str, WorkflowModelConfig]
     timeouts: WorkflowTimeoutConfig
+    enable_debug: bool = Field(
+        False,
+        description="Turn on LiteLLM's verbose debug logging across workflows.",
+    )
 
     @model_validator(mode="after")
     def _ensure_default_present(self) -> "LLMConfig":
@@ -86,6 +93,13 @@ class ReviewConfig(BaseModel):
         default=None,
         description="Model identifier used for automated audits.",
     )
+    auto_reviewer_provider: Optional[str] = Field(
+        default=None,
+        description=(
+            "Optional provider override for the automated review model; defaults "
+            "to the application's default workflow provider."
+        ),
+    )
 
 
 class TelemetryConfig(BaseModel):

core/review.py

@@ -4,6 +4,9 @@
     v0.1 - 2025-11-06 - Added ReviewEngine with optional LiteLLM-based automated audits and structured review records.
     v0.2 - 2025-11-06 - Expanded automated review rubric and context framing.
     v0.3 - 2025-11-07 - Parsed automated review verdict, score, and suggestions into structured fields.
+    v0.4 - 2025-11-07 - Honoured LiteLLM debug toggle for automated reviews.
+    v0.5 - 2025-11-07 - Normalised metadata serialisation for automated review payloads.
+    v0.6 - 2025-11-07 - Applied provider-aware routing for automated review models.
 """
 
 from __future__ import annotations
@@ -13,7 +16,8 @@
 import re
 import uuid
 from dataclasses import dataclass
-from typing import Any, List, Optional, Sequence, Tuple, cast
+from os import getenv
+from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, cast
 
 from config.settings import AppConfig
 from core.exceptions import ReviewError
@@ -45,6 +49,7 @@ class ReviewEngine:
     def __init__(self, config: AppConfig) -> None:
         self._config = config
         self._logger = LOGGER
+        self._activate_litellm_debug()
 
     def perform_review(
         self,
@@ -106,19 +111,21 @@ def _run_automated_review(
 
         try:
             self._logger.debug("Running automated review with model %s", model)
+            model_name, provider_kwargs = self._resolve_model_configuration()
+            payload = {
+                "task_prompt": request.prompt,
+                "workflow": request.workflow,
+                "context": self._to_json_safe(request.context),
+                "result": result.content,
+                "metadata": self._to_json_safe(result.metadata),
+            }
             review_payload = json.dumps(
-                {
-                    "task_prompt": request.prompt,
-                    "workflow": request.workflow,
-                    "context": request.context,
-                    "result": result.content,
-                    "metadata": result.metadata,
-                },
+                payload,
                 ensure_ascii=False,
                 indent=2,
             )
             response = litellm.completion(
-                model=model,
+                model=model_name,
                 messages=[
                     {"role": "system", "content": REVIEW_SYSTEM_PROMPT},
                     {
@@ -132,6 +139,7 @@ def _run_automated_review(
                 ],
                 temperature=0.0,
                 request_timeout=self._config.llm.timeouts.request_seconds,
+                **provider_kwargs,
             )
             auto_notes = response["choices"][0]["message"]["content"]
             parsed = self._parse_automated_review(auto_notes)
@@ -178,6 +186,88 @@ def _normalise_verdict(raw_verdict: Optional[str]) -> str:
             return "fail-auto"
         return verdict
 
+    def _activate_litellm_debug(self) -> None:
+        """Enable LiteLLM debug logging for automated review when configured."""
+        if not self._config.llm.enable_debug:
+            return
+
+        if litellm is None:
+            self._logger.warning(
+                "LiteLLM debug requested for reviews but the library is not installed."
+            )
+            return
+
+        debug_hook = getattr(litellm, "_turn_on_debug", None)
+        if callable(debug_hook):
+            debug_hook()
+            self._logger.info("LiteLLM debug logging enabled for review engine.")
+        else:
+            self._logger.warning(
+                "LiteLLM debug requested but '_turn_on_debug' is unavailable on the library."
+            )
+
+    @staticmethod
+    def _to_json_safe(value: Any) -> Any:
+        """Convert the value into JSON-serialisable primitives."""
+        if value is None or isinstance(value, (str, int, float, bool)):
+            return value
+
+        if isinstance(value, Mapping):
+            return {str(key): ReviewEngine._to_json_safe(item) for key, item in value.items()}
+
+        if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+            return [ReviewEngine._to_json_safe(item) for item in value]
+
+        model_dump = getattr(value, "model_dump", None)
+        if callable(model_dump):
+            return ReviewEngine._to_json_safe(model_dump())
+
+        if hasattr(value, "__dict__"):
+            return ReviewEngine._to_json_safe(vars(value))
+
+        return str(value)
+
+    def _resolve_model_configuration(self) -> Tuple[str, Dict[str, object]]:
+        """Return the provider-aware model identifier and kwargs for LiteLLM."""
+        model_name = self._config.review.auto_reviewer_model
+        if not model_name:
+            raise ReviewError("Automated review model is not configured.")
+
+        provider = self._config.review.auto_reviewer_provider
+        if not provider:
+            default = self._config.llm.default_workflow
+            default_cfg = self._config.llm.workflows.get(default)
+            provider = default_cfg.provider if default_cfg else None
+
+        if not provider:
+            return model_name, {}
+
+        provider_lower = provider.lower()
+        if provider_lower == "azure":
+            api_key = getenv("AZURE_OPENAI_API_KEY")
+            endpoint = getenv("AZURE_OPENAI_ENDPOINT")
+            api_version = getenv("AZURE_OPENAI_API_VERSION", "2024-08-01-preview")
+            if not api_key or not endpoint:
+                raise ReviewError(
+                    "Azure OpenAI credentials missing for automated review."
+                )
+            base = endpoint.rstrip("/")
+            if not model_name.startswith("azure/"):
+                model_name = f"azure/{model_name}"
+            return model_name, {
+                "api_key": api_key,
+                "api_base": base,
+                "base_url": base,
+                "api_version": api_version,
+                "custom_llm_provider": "azure",
+            }
+
+        if provider_lower == "ollama":
+            base_url = getenv("OLLAMA_BASE_URL", "http://localhost:11434")
+            return model_name, {"base_url": base_url.rstrip("/")}
+
+        return model_name, {}
+
     def _parse_automated_review(self, content: str) -> "AutomatedReview":
         lines = [line.rstrip() for line in content.splitlines()]
         verdict: Optional[str] = None

core/task_executor.py

@@ -5,6 +5,8 @@
         scaffold with retries and telemetry hooks.
     v0.2 - 2025-11-06 - Wired provider credential handling for Azure and Ollama.
     v0.3 - 2025-11-06 - Integrated controller bias into workflow selection metadata.
+    v0.4 - 2025-11-07 - Enabled optional LiteLLM debug toggling from configuration.
+    v0.5 - 2025-11-07 - Normalised Azure provider routing for LiteLLM compatibility.
 """
 
 from __future__ import annotations
@@ -14,7 +16,7 @@
 from os import getenv
 from typing import Any, Dict, Mapping, Optional, Tuple, cast
 
-from config.settings import AppConfig
+from config.settings import AppConfig, WorkflowModelConfig
 from core.controller import SelfAdjustingController
 from core.exceptions import WorkflowError
 from models.workflows import TaskRequest, TaskResult, WorkflowSelection
@@ -40,6 +42,7 @@ def __init__(
         self._config = config
         self._logger = LOGGER
         self._controller = controller
+        self._activate_litellm_debug()
 
     def select_workflow(self, requested: Optional[str] = None) -> WorkflowSelection:
         """Select the best workflow given request metadata."""
@@ -124,6 +127,7 @@ def execute(self, request: TaskRequest) -> TaskResult:
         workflow_cfg = workflows[request.workflow]
         timeout_cfg = self._config.llm.timeouts
         provider_kwargs = self._build_provider_kwargs(workflow_cfg.provider)
+        model_identifier = self._resolve_model_name(workflow_cfg)
 
         attempt = 0
         delay = timeout_cfg.retry_backoff_seconds
@@ -136,7 +140,7 @@ def execute(self, request: TaskRequest) -> TaskResult:
                     "Executing workflow '%s' attempt %s", request.workflow, attempt
                 )
                 response = litellm.completion(
-                    model=workflow_cfg.model,
+                    model=model_identifier,
                     messages=[
                         {"role": "system", "content": request.context.get("system", "")},
                         {"role": "user", "content": request.prompt},
@@ -195,10 +199,13 @@ def _build_provider_kwargs(self, provider: str) -> Dict[str, object]:
                     "Azure OpenAI credentials missing. Set AZURE_OPENAI_API_KEY and "
                     "AZURE_OPENAI_ENDPOINT environment variables."
                 )
+            base = endpoint.rstrip("/")
             return {
                 "api_key": api_key,
-                "base_url": endpoint.rstrip("/"),
+                "api_base": base,
+                "base_url": base,
                 "api_version": api_version,
+                "custom_llm_provider": "azure",
             }
 
         if provider.lower() == "ollama":
@@ -217,3 +224,34 @@ def _redact_sensitive(payload: Dict[str, object]) -> Dict[str, object]:
             else:
                 redacted[key] = value
         return redacted
+
+    def _activate_litellm_debug(self) -> None:
+        """Enable LiteLLM debug logging when requested via configuration."""
+        if not self._config.llm.enable_debug:
+            return
+
+        if litellm is None:
+            self._logger.warning(
+                "LiteLLM debug requested but the library is not installed."
+            )
+            return
+
+        debug_hook = getattr(litellm, "_turn_on_debug", None)
+        if callable(debug_hook):
+            debug_hook()
+            self._logger.info("LiteLLM debug logging enabled.")
+        else:
+            self._logger.warning(
+                "LiteLLM debug requested but '_turn_on_debug' is unavailable on the library."
+            )
+
+    def _resolve_model_name(self, workflow_cfg: WorkflowModelConfig) -> str:
+        """Normalise provider-specific model identifiers for LiteLLM."""
+        model_name = workflow_cfg.model
+        if workflow_cfg.provider.lower() != "azure":
+            return model_name
+
+        if model_name.startswith("azure/"):
+            return model_name
+
+        return f"azure/{model_name}"

tests/test_review_engine.py

@@ -55,6 +55,7 @@ def test_parse_automated_review_structured_fields(tmp_path: Path) -> None:
 def test_live_task_loop_persists_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     config = _load_sample_config(tmp_path)
     config.review.auto_reviewer_model = "review-stub"
+    config.review.auto_reviewer_provider = "ollama"
     config.llm.workflows["fast"].provider = "ollama"
     config.llm.workflows["fast"].model = "stub-fast-model"
 
@@ -126,3 +127,62 @@ class DummyTimeout(Exception):
 
     working_items = loop._memory_manager.list_working_items()
     assert any(item.key.endswith(":result") for item in working_items)
+
+
+def test_to_json_safe_serialises_usage_objects() -> None:
+    payload = {
+        "usage": SimpleNamespace(total_tokens=42, prompt_tokens=10),
+        "sequence": [SimpleNamespace(value="a")],
+        "primitive": "ok",
+    }
+    safe = ReviewEngine._to_json_safe(payload)
+    assert safe == {
+        "usage": {"total_tokens": 42, "prompt_tokens": 10},
+        "sequence": [{"value": "a"}],
+        "primitive": "ok",
+    }
+
+
+def test_resolve_model_configuration_uses_azure_provider(monkeypatch: pytest.MonkeyPatch) -> None:
+    config = settings.AppConfig.model_validate(
+        {
+            "version": "0.1",
+            "llm": {
+                "default_workflow": "fast",
+                "workflows": {
+                    "fast": {
+                        "provider": "azure",
+                        "model": "gpt-4.1",
+                        "temperature": 0.2,
+                    }
+                },
+                "timeouts": {
+                    "request_seconds": 10,
+                    "retry_attempts": 1,
+                    "retry_backoff_seconds": 1,
+                },
+                "enable_debug": False,
+            },
+            "memory": {
+                "redis": {"host": "localhost", "port": 6379, "db": 0, "ttl_seconds": 120},
+                "chromadb": {"persist_directory": "data/chromadb", "collection": "test"},
+            },
+            "review": {
+                "enabled": True,
+                "auto_reviewer_model": "gpt-4.1",
+                "auto_reviewer_provider": None,
+            },
+            "embedding": None,
+            "telemetry": {"log_level": "INFO"},
+        }
+    )
+
+    monkeypatch.setenv("AZURE_OPENAI_API_KEY", "test-key")
+    monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "https://example.openai.azure.com")
+    monkeypatch.setenv("AZURE_OPENAI_API_VERSION", "2024-08-01-preview")
+
+    engine = ReviewEngine(config)
+    model_name, kwargs = engine._resolve_model_configuration()
+    assert model_name == "azure/gpt-4.1"
+    assert kwargs["custom_llm_provider"] == "azure"
+    assert kwargs["api_base"] == "https://example.openai.azure.com"