update

tests/webapp/test_profile_manager.py

@@ -1,4 +1,6 @@
 import pytest
+from unittest.mock import sentinel
+
 from webapp.models import LLMProfile, ProfileApplyRequest, ProfileApplyResponse
 
 def test_llm_profile_defaults():
@@ -147,3 +149,57 @@ def test_resolve_openai_client_kwargs_falls_back_to_env(tmp_path, monkeypatch):
     assert kwargs["api_key"] == "sk-env"
     assert kwargs["base_url"] == "http://env-base/v1"
     assert kwargs["timeout"] == 45.0
+
+
+def test_build_models_uses_high_default_max_tokens_for_structured_judge(monkeypatch):
+    """Structured RAGAS judge calls should use a larger completion budget by default."""
+    import rag_eval.metrics.factory as factory
+    from rag_eval.settings import EvaluationSettings
+
+    captured: dict[str, object] = {}
+
+    def fake_llm_factory(model, client=None, **kwargs):
+        captured["model"] = model
+        captured["client"] = client
+        captured["kwargs"] = kwargs
+        return sentinel.llm
+
+    monkeypatch.setattr(factory, "AsyncOpenAI", lambda **kwargs: sentinel.client)
+    monkeypatch.setattr(factory, "llm_factory", fake_llm_factory)
+    monkeypatch.setattr(factory, "embedding_factory", lambda **kwargs: sentinel.embeddings)
+
+    llm, embeddings = factory.build_models(
+        "gpt-5",
+        "text-embedding-3-small",
+        EvaluationSettings(),
+    )
+
+    assert llm is sentinel.llm
+    assert embeddings is sentinel.embeddings
+    assert captured["model"] == "gpt-5"
+    assert captured["client"] is sentinel.client
+    assert captured["kwargs"] == {"max_tokens": 4096}
+
+
+def test_build_models_allows_env_override_for_judge_max_tokens(monkeypatch):
+    """Operators should be able to raise the judge completion budget via settings."""
+    import rag_eval.metrics.factory as factory
+    from rag_eval.settings import EvaluationSettings
+
+    captured: dict[str, object] = {}
+
+    def fake_llm_factory(model, client=None, **kwargs):
+        captured["kwargs"] = kwargs
+        return sentinel.llm
+
+    monkeypatch.setattr(factory, "AsyncOpenAI", lambda **kwargs: sentinel.client)
+    monkeypatch.setattr(factory, "llm_factory", fake_llm_factory)
+    monkeypatch.setattr(factory, "embedding_factory", lambda **kwargs: sentinel.embeddings)
+
+    factory.build_models(
+        "gpt-5",
+        "text-embedding-3-small",
+        EvaluationSettings(RAGAS_LLM_MAX_TOKENS=8192),
+    )
+
+    assert captured["kwargs"] == {"max_tokens": 8192}