Fix: build_models resolves separate AsyncOpenAI clients for judge and embedding models

Previously, only the judge_model's profile was looked up and its AsyncOpenAI

client was shared with embedding_factory. When embedding_model has a different

base_url/api_key (e.g. Qwen3-Embedding-4B on SiliconFlow vs gpt-5 on another

gateway), the embedding calls silently used the wrong URL, causing 402/404 errors.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

rag_eval/metrics/factory.py

@@ -98,19 +98,26 @@ def build_models(
 ) -> tuple[Any, Any]:
     """Create the LLM and embedding clients required by the selected RAGAS metrics.
 
-    Dynamically resolves connection settings from the stored LLM Profiles first
+    Resolves connection settings independently for the judge LLM and the embedding
-    (matched by model name), falling back to .env settings when no profile matches.
+    model by looking up each in the stored LLM Profiles (matched by model name).
+    This allows judge_model and embedding_model to use different gateways / API keys.
+    Falls back to .env settings when no matching profile is found.
     """
-    client_kwargs = _resolve_openai_client_kwargs(judge_model, settings)
+    llm_kwargs = _resolve_openai_client_kwargs(judge_model, settings)
-    client = AsyncOpenAI(**client_kwargs)
+    emb_kwargs = _resolve_openai_client_kwargs(embedding_model, settings)
+
+    llm_client = AsyncOpenAI(**llm_kwargs)
+    # Only allocate a second client when the embedding model needs different settings.
+    emb_client = AsyncOpenAI(**emb_kwargs) if emb_kwargs != llm_kwargs else llm_client
+
     # RAGAS structured-output judge calls can be truncated by the upstream default
     # 1024 completion budget, especially for faithfulness and GPT-5 family models.
     llm = llm_factory(
         judge_model,
-        client=client,
+        client=llm_client,
         max_tokens=max(1, int(settings.ragas_llm_max_tokens)),
     )
-    embeddings = embedding_factory(provider="openai", model=embedding_model, client=client)
+    embeddings = embedding_factory(provider="openai", model=embedding_model, client=emb_client)
     return llm, embeddings
 
 

tests/test_build_models_separate_clients.py

@@ -0,0 +1,134 @@
+"""Tests that build_models resolves separate clients for judge and embedding models."""
+
+from unittest.mock import MagicMock, patch, call
+from rag_eval.metrics.factory import build_models
+from rag_eval.settings import EvaluationSettings
+
+
+def _settings():
+    return EvaluationSettings(_env_file=None)
+
+
+def _patch_profiles(profiles_by_model: dict):
+    """Patch profile_manager.list_all() to return fake profiles."""
+    fake_profiles = []
+    for model, base_url in profiles_by_model.items():
+        p = MagicMock()
+        p.model = model
+        p.base_url = base_url
+        p.api_key = "sk-test"
+        p.timeout_seconds = 30
+        fake_profiles.append(p)
+    return fake_profiles
+
+
+def test_build_models_uses_separate_clients_for_different_profiles(monkeypatch):
+    """When judge and embedding models have different profiles, two AsyncOpenAI clients are created."""
+    created_clients = []
+
+    class _FakeClient:
+        def __init__(self, **kwargs):
+            created_clients.append(kwargs.copy())
+
+    fake_profiles = _patch_profiles({
+        "gpt-5": "http://llm-gateway/v1",
+        "Qwen3-Embedding-4B": "https://api.siliconflow.cn/v1",
+    })
+
+    import webapp.services.profile_manager as pm_mod
+    monkeypatch.setattr(pm_mod.profile_manager, "list_all", lambda: fake_profiles)
+
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbedding
+
+    with patch("rag_eval.metrics.factory.AsyncOpenAI", side_effect=_FakeClient), \
+         patch("rag_eval.metrics.factory.llm_factory", return_value=MagicMock(spec=InstructorBaseRagasLLM)), \
+         patch("rag_eval.metrics.factory.embedding_factory", return_value=MagicMock(spec=BaseRagasEmbedding)):
+        build_models("gpt-5", "Qwen3-Embedding-4B", _settings())
+
+    # Two distinct clients should have been created with different base_urls.
+    assert len(created_clients) == 2
+    base_urls = {c["base_url"] for c in created_clients}
+    assert "http://llm-gateway/v1" in base_urls
+    assert "https://api.siliconflow.cn/v1" in base_urls
+
+
+def test_build_models_shares_client_when_same_profile(monkeypatch):
+    """When judge and embedding models resolve to the same settings, only one client is created."""
+    created_clients = []
+
+    class _FakeClient:
+        def __init__(self, **kwargs):
+            created_clients.append(kwargs.copy())
+
+    # Both models map to the same gateway.
+    fake_profiles = _patch_profiles({
+        "gpt-5": "http://same-gateway/v1",
+        "text-embedding-3-small": "http://same-gateway/v1",
+    })
+
+    import webapp.services.profile_manager as pm_mod
+    monkeypatch.setattr(pm_mod.profile_manager, "list_all", lambda: fake_profiles)
+
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbedding
+
+    with patch("rag_eval.metrics.factory.AsyncOpenAI", side_effect=_FakeClient), \
+         patch("rag_eval.metrics.factory.llm_factory", return_value=MagicMock(spec=InstructorBaseRagasLLM)), \
+         patch("rag_eval.metrics.factory.embedding_factory", return_value=MagicMock(spec=BaseRagasEmbedding)):
+        build_models("gpt-5", "text-embedding-3-small", _settings())
+
+    # Same settings → only one client needed.
+    assert len(created_clients) == 1
+
+
+def test_embedding_factory_receives_embedding_client(monkeypatch):
+    """embedding_factory is called with the client resolved from the embedding model's profile."""
+    emb_client_ref = []
+    llm_client_ref = []
+
+    class _LLMClient:
+        label = "llm"
+        def __init__(self, **kwargs):
+            llm_client_ref.append(self)
+
+    class _EmbClient:
+        label = "emb"
+        def __init__(self, **kwargs):
+            emb_client_ref.append(self)
+
+    fake_profiles = _patch_profiles({
+        "gpt-5": "http://llm-gw/v1",
+        "Qwen3-Embedding-4B": "https://api.siliconflow.cn/v1",
+    })
+
+    import webapp.services.profile_manager as pm_mod
+    monkeypatch.setattr(pm_mod.profile_manager, "list_all", lambda: fake_profiles)
+
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbedding
+
+    emb_factory_calls = []
+
+    def _fake_emb_factory(provider, model, client):
+        emb_factory_calls.append({"model": model, "client": client})
+        return MagicMock(spec=BaseRagasEmbedding)
+
+    call_count = [0]
+
+    def _client_factory(**kwargs):
+        call_count[0] += 1
+        if "llm-gw" in kwargs.get("base_url", ""):
+            return _LLMClient(**kwargs)
+        return _EmbClient(**kwargs)
+
+    with patch("rag_eval.metrics.factory.AsyncOpenAI", side_effect=_client_factory), \
+         patch("rag_eval.metrics.factory.llm_factory", return_value=MagicMock(spec=InstructorBaseRagasLLM)), \
+         patch("rag_eval.metrics.factory.embedding_factory", side_effect=_fake_emb_factory):
+        build_models("gpt-5", "Qwen3-Embedding-4B", _settings())
+
+    assert len(emb_factory_calls) == 1
+    # The client passed to embedding_factory must be the siliconflow client, not the LLM client.
+    assert isinstance(emb_factory_calls[0]["client"], _EmbClient), (
+        "embedding_factory should receive the embedding-model client, not the judge-model client"
+    )