diff --git a/rag_eval/metrics/factory.py b/rag_eval/metrics/factory.py index b2e4458..97edbf8 100644 --- a/rag_eval/metrics/factory.py +++ b/rag_eval/metrics/factory.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from typing import Any from openai import AsyncOpenAI @@ -28,14 +29,17 @@ from .judge_prompts import localize_pipeline_prompts from .pipeline import MetricPipeline +logger = logging.getLogger("rag_eval.metrics.factory") + + def _resolve_openai_client_kwargs( - judge_model: str, + model: str, settings: EvaluationSettings, ) -> dict[str, Any]: """Return AsyncOpenAI kwargs, preferring a matching LLM Profile over .env settings. Lookup order: - 1. LLM Profile whose model name equals judge_model (exact match) + 1. LLM Profile whose model name equals `model` (exact match) 2. Fall back to EvaluationSettings (.env) """ try: @@ -43,26 +47,35 @@ def _resolve_openai_client_kwargs( from webapp.services.profile_manager import profile_manager profiles = profile_manager.list_all() for profile in profiles: - if profile.model == judge_model: + if profile.model == model: kwargs: dict[str, Any] = { "api_key": profile.api_key or "sk-placeholder", "timeout": float(profile.timeout_seconds or 30), } if profile.base_url and profile.base_url.strip(): kwargs["base_url"] = profile.base_url.strip() + logger.debug( + "[factory] model=%s source=profile base_url=%s", + model, kwargs.get("base_url", "(not set, using default)") + ) return kwargs - except Exception: # noqa: BLE001 + except Exception as exc: # noqa: BLE001 # If profile lookup fails for any reason, fall through to .env settings. - pass + logger.warning("[factory] profile lookup failed for model=%s: %s", model, exc) - return settings.openai_client_kwargs + fallback = settings.openai_client_kwargs + logger.debug( + "[factory] model=%s source=env base_url=%s", + model, fallback.get("base_url", "(not set)") + ) + return fallback def resolve_openai_client_kwargs( judge_model: str, settings: EvaluationSettings, ) -> dict[str, Any]: - """Public accessor for profile-aware AsyncOpenAI kwargs (matched by judge_model). + """Public accessor for profile-aware AsyncOpenAI kwargs (matched by model name). Exposed so other components (e.g. the optimization advisor's direct LLM call) can build a client that honors the same saved-profile/.env resolution used by @@ -106,6 +119,12 @@ def build_models( llm_kwargs = _resolve_openai_client_kwargs(judge_model, settings) emb_kwargs = _resolve_openai_client_kwargs(embedding_model, settings) + logger.info( + "[factory] build_models judge=%s→%s embedding=%s→%s", + judge_model, llm_kwargs.get("base_url", "(env default)"), + embedding_model, emb_kwargs.get("base_url", "(env default)"), + ) + llm_client = AsyncOpenAI(**llm_kwargs) # Only allocate a second client when the embedding model needs different settings. emb_client = AsyncOpenAI(**emb_kwargs) if emb_kwargs != llm_kwargs else llm_client