import pytest from webapp.models import LLMProfile, ProfileApplyRequest, ProfileApplyResponse def test_llm_profile_defaults(): p = LLMProfile( profile_id="abc", name="Test", model="gpt-4", base_url="http://localhost/v1", api_key="sk-test", ) assert p.timeout_seconds == 30 assert p.created_at != "" assert p.updated_at != "" def test_profile_apply_request_fields(): req = ProfileApplyRequest( scenario_path="scenarios/offline/sample.yaml", judge_profile_id="id1", answer_profile_id="id2", dataset_profile_id=None, ) assert req.judge_profile_id == "id1" assert req.dataset_profile_id is None def test_profile_apply_response(): resp = ProfileApplyResponse(scenario_path="scenarios/offline/sample.yaml", patched_fields=["judge_model"]) assert "judge_model" in resp.patched_fields # --------------------------------------------------------------------------- # ProfileManager service tests # --------------------------------------------------------------------------- import json from webapp.services.profile_manager import ProfileManager def _make_manager(tmp_path): store = tmp_path / "profiles.json" return ProfileManager(store_path=store) def test_create_profile(tmp_path): mgr = _make_manager(tmp_path) p = mgr.create(name="Local", model="deepseek-v4-flash", base_url="http://localhost/v1", api_key="sk-x") assert p.profile_id != "" assert p.name == "Local" def test_list_profiles(tmp_path): mgr = _make_manager(tmp_path) mgr.create(name="A", model="m1", base_url="http://a/v1", api_key="k1") mgr.create(name="B", model="m2", base_url="http://b/v1", api_key="k2") profiles = mgr.list_all() assert len(profiles) == 2 def test_get_profile(tmp_path): mgr = _make_manager(tmp_path) created = mgr.create(name="X", model="m", base_url="http://x/v1", api_key="k") fetched = mgr.get(created.profile_id) assert fetched is not None assert fetched.name == "X" def test_update_profile(tmp_path): mgr = _make_manager(tmp_path) p = mgr.create(name="Old", model="m", base_url="http://x/v1", api_key="k") updated = mgr.update(p.profile_id, name="New", model="m2", base_url="http://x/v1", api_key="k", timeout_seconds=60) assert updated is not None assert updated.name == "New" assert updated.model == "m2" assert updated.timeout_seconds == 60 def test_delete_profile(tmp_path): mgr = _make_manager(tmp_path) p = mgr.create(name="Del", model="m", base_url="http://x/v1", api_key="k") assert mgr.delete(p.profile_id) is True assert mgr.get(p.profile_id) is None def test_persistence(tmp_path): store = tmp_path / "profiles.json" mgr1 = ProfileManager(store_path=store) p = mgr1.create(name="Persist", model="m", base_url="http://x/v1", api_key="k") mgr2 = ProfileManager(store_path=store) assert mgr2.get(p.profile_id) is not None def test_get_nonexistent(tmp_path): mgr = _make_manager(tmp_path) assert mgr.get("does-not-exist") is None def test_delete_nonexistent(tmp_path): mgr = _make_manager(tmp_path) assert mgr.delete("does-not-exist") is False def test_resolve_openai_client_kwargs_prefers_matching_profile(tmp_path, monkeypatch): """Metric runtime should prefer the saved LLM Profile over .env defaults.""" from rag_eval.metrics.factory import _resolve_openai_client_kwargs from rag_eval.settings import EvaluationSettings import webapp.services.profile_manager as pm_mod mgr = _make_manager(tmp_path) mgr.create( name="Judge", model="gpt-5.5", base_url="http://39.107.88.131:13000", api_key="sk-profile", timeout_seconds=300, ) monkeypatch.setattr(pm_mod, "profile_manager", mgr) settings = EvaluationSettings( OPENAI_API_KEY="sk-env", OPENAI_BASE_URL="http://env-base/v1", OPENAI_TIMEOUT_SECONDS=30, ) kwargs = _resolve_openai_client_kwargs("gpt-5.5", settings) assert kwargs["api_key"] == "sk-profile" assert kwargs["base_url"] == "http://39.107.88.131:13000" assert kwargs["timeout"] == 300.0 def test_resolve_openai_client_kwargs_falls_back_to_env(tmp_path, monkeypatch): """When no saved profile matches, .env settings remain the fallback.""" from rag_eval.metrics.factory import _resolve_openai_client_kwargs from rag_eval.settings import EvaluationSettings import webapp.services.profile_manager as pm_mod mgr = _make_manager(tmp_path) monkeypatch.setattr(pm_mod, "profile_manager", mgr) settings = EvaluationSettings( OPENAI_API_KEY="sk-env", OPENAI_BASE_URL="http://env-base/v1", OPENAI_TIMEOUT_SECONDS=45, ) kwargs = _resolve_openai_client_kwargs("gpt-5", settings) assert kwargs["api_key"] == "sk-env" assert kwargs["base_url"] == "http://env-base/v1" assert kwargs["timeout"] == 45.0