diff --git a/rag_eval/metrics/factory.py b/rag_eval/metrics/factory.py index 3aafb05..b2e4458 100644 --- a/rag_eval/metrics/factory.py +++ b/rag_eval/metrics/factory.py @@ -98,19 +98,26 @@ def build_models( ) -> tuple[Any, Any]: """Create the LLM and embedding clients required by the selected RAGAS metrics. - Dynamically resolves connection settings from the stored LLM Profiles first - (matched by model name), falling back to .env settings when no profile matches. + Resolves connection settings independently for the judge LLM and the embedding + model by looking up each in the stored LLM Profiles (matched by model name). + This allows judge_model and embedding_model to use different gateways / API keys. + Falls back to .env settings when no matching profile is found. """ - client_kwargs = _resolve_openai_client_kwargs(judge_model, settings) - client = AsyncOpenAI(**client_kwargs) + llm_kwargs = _resolve_openai_client_kwargs(judge_model, settings) + emb_kwargs = _resolve_openai_client_kwargs(embedding_model, settings) + + llm_client = AsyncOpenAI(**llm_kwargs) + # Only allocate a second client when the embedding model needs different settings. + emb_client = AsyncOpenAI(**emb_kwargs) if emb_kwargs != llm_kwargs else llm_client + # RAGAS structured-output judge calls can be truncated by the upstream default # 1024 completion budget, especially for faithfulness and GPT-5 family models. llm = llm_factory( judge_model, - client=client, + client=llm_client, max_tokens=max(1, int(settings.ragas_llm_max_tokens)), ) - embeddings = embedding_factory(provider="openai", model=embedding_model, client=client) + embeddings = embedding_factory(provider="openai", model=embedding_model, client=emb_client) return llm, embeddings diff --git a/tests/test_build_models_separate_clients.py b/tests/test_build_models_separate_clients.py new file mode 100644 index 0000000..31f9ada --- /dev/null +++ b/tests/test_build_models_separate_clients.py @@ -0,0 +1,134 @@ +"""Tests that build_models resolves separate clients for judge and embedding models.""" + +from unittest.mock import MagicMock, patch, call +from rag_eval.metrics.factory import build_models +from rag_eval.settings import EvaluationSettings + + +def _settings(): + return EvaluationSettings(_env_file=None) + + +def _patch_profiles(profiles_by_model: dict): + """Patch profile_manager.list_all() to return fake profiles.""" + fake_profiles = [] + for model, base_url in profiles_by_model.items(): + p = MagicMock() + p.model = model + p.base_url = base_url + p.api_key = "sk-test" + p.timeout_seconds = 30 + fake_profiles.append(p) + return fake_profiles + + +def test_build_models_uses_separate_clients_for_different_profiles(monkeypatch): + """When judge and embedding models have different profiles, two AsyncOpenAI clients are created.""" + created_clients = [] + + class _FakeClient: + def __init__(self, **kwargs): + created_clients.append(kwargs.copy()) + + fake_profiles = _patch_profiles({ + "gpt-5": "http://llm-gateway/v1", + "Qwen3-Embedding-4B": "https://api.siliconflow.cn/v1", + }) + + import webapp.services.profile_manager as pm_mod + monkeypatch.setattr(pm_mod.profile_manager, "list_all", lambda: fake_profiles) + + from ragas.llms.base import InstructorBaseRagasLLM + from ragas.embeddings.base import BaseRagasEmbedding + + with patch("rag_eval.metrics.factory.AsyncOpenAI", side_effect=_FakeClient), \ + patch("rag_eval.metrics.factory.llm_factory", return_value=MagicMock(spec=InstructorBaseRagasLLM)), \ + patch("rag_eval.metrics.factory.embedding_factory", return_value=MagicMock(spec=BaseRagasEmbedding)): + build_models("gpt-5", "Qwen3-Embedding-4B", _settings()) + + # Two distinct clients should have been created with different base_urls. + assert len(created_clients) == 2 + base_urls = {c["base_url"] for c in created_clients} + assert "http://llm-gateway/v1" in base_urls + assert "https://api.siliconflow.cn/v1" in base_urls + + +def test_build_models_shares_client_when_same_profile(monkeypatch): + """When judge and embedding models resolve to the same settings, only one client is created.""" + created_clients = [] + + class _FakeClient: + def __init__(self, **kwargs): + created_clients.append(kwargs.copy()) + + # Both models map to the same gateway. + fake_profiles = _patch_profiles({ + "gpt-5": "http://same-gateway/v1", + "text-embedding-3-small": "http://same-gateway/v1", + }) + + import webapp.services.profile_manager as pm_mod + monkeypatch.setattr(pm_mod.profile_manager, "list_all", lambda: fake_profiles) + + from ragas.llms.base import InstructorBaseRagasLLM + from ragas.embeddings.base import BaseRagasEmbedding + + with patch("rag_eval.metrics.factory.AsyncOpenAI", side_effect=_FakeClient), \ + patch("rag_eval.metrics.factory.llm_factory", return_value=MagicMock(spec=InstructorBaseRagasLLM)), \ + patch("rag_eval.metrics.factory.embedding_factory", return_value=MagicMock(spec=BaseRagasEmbedding)): + build_models("gpt-5", "text-embedding-3-small", _settings()) + + # Same settings → only one client needed. + assert len(created_clients) == 1 + + +def test_embedding_factory_receives_embedding_client(monkeypatch): + """embedding_factory is called with the client resolved from the embedding model's profile.""" + emb_client_ref = [] + llm_client_ref = [] + + class _LLMClient: + label = "llm" + def __init__(self, **kwargs): + llm_client_ref.append(self) + + class _EmbClient: + label = "emb" + def __init__(self, **kwargs): + emb_client_ref.append(self) + + fake_profiles = _patch_profiles({ + "gpt-5": "http://llm-gw/v1", + "Qwen3-Embedding-4B": "https://api.siliconflow.cn/v1", + }) + + import webapp.services.profile_manager as pm_mod + monkeypatch.setattr(pm_mod.profile_manager, "list_all", lambda: fake_profiles) + + from ragas.llms.base import InstructorBaseRagasLLM + from ragas.embeddings.base import BaseRagasEmbedding + + emb_factory_calls = [] + + def _fake_emb_factory(provider, model, client): + emb_factory_calls.append({"model": model, "client": client}) + return MagicMock(spec=BaseRagasEmbedding) + + call_count = [0] + + def _client_factory(**kwargs): + call_count[0] += 1 + if "llm-gw" in kwargs.get("base_url", ""): + return _LLMClient(**kwargs) + return _EmbClient(**kwargs) + + with patch("rag_eval.metrics.factory.AsyncOpenAI", side_effect=_client_factory), \ + patch("rag_eval.metrics.factory.llm_factory", return_value=MagicMock(spec=InstructorBaseRagasLLM)), \ + patch("rag_eval.metrics.factory.embedding_factory", side_effect=_fake_emb_factory): + build_models("gpt-5", "Qwen3-Embedding-4B", _settings()) + + assert len(emb_factory_calls) == 1 + # The client passed to embedding_factory must be the siliconflow client, not the LLM client. + assert isinstance(emb_factory_calls[0]["client"], _EmbClient), ( + "embedding_factory should receive the embedding-model client, not the judge-model client" + )