140 lines
5.2 KiB
Python
140 lines
5.2 KiB
Python
"""Integration tests for /api/llm-profiles endpoints."""
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
|
|
@pytest.fixture()
|
|
def client(tmp_path, monkeypatch):
|
|
"""TestClient with a fresh ProfileManager backed by a temp file."""
|
|
store = tmp_path / "profiles.json"
|
|
import webapp.services.profile_manager as pm_mod
|
|
from webapp.services.profile_manager import ProfileManager
|
|
fresh_mgr = ProfileManager(store_path=store)
|
|
monkeypatch.setattr(pm_mod, "profile_manager", fresh_mgr)
|
|
import webapp.api.llm_profiles as api_mod
|
|
monkeypatch.setattr(api_mod, "profile_manager", fresh_mgr)
|
|
|
|
from webapp.server import create_app
|
|
return TestClient(create_app())
|
|
|
|
|
|
def test_list_empty(client):
|
|
resp = client.get("/api/llm-profiles")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["profiles"] == []
|
|
|
|
|
|
def test_create_and_list(client):
|
|
body = {"name": "Test", "model": "m1", "base_url": "http://x/v1", "api_key": "k"}
|
|
resp = client.post("/api/llm-profiles", json=body)
|
|
assert resp.status_code == 201
|
|
data = resp.json()
|
|
assert data["name"] == "Test"
|
|
assert data["profile_id"] != ""
|
|
|
|
resp2 = client.get("/api/llm-profiles")
|
|
assert len(resp2.json()["profiles"]) == 1
|
|
|
|
|
|
def test_update_profile(client):
|
|
body = {"name": "Old", "model": "m1", "base_url": "http://x/v1", "api_key": "k"}
|
|
pid = client.post("/api/llm-profiles", json=body).json()["profile_id"]
|
|
|
|
upd = {"name": "New", "model": "m2", "base_url": "http://x/v1", "api_key": "k", "timeout_seconds": 60}
|
|
resp = client.put(f"/api/llm-profiles/{pid}", json=upd)
|
|
assert resp.status_code == 200
|
|
assert resp.json()["name"] == "New"
|
|
assert resp.json()["timeout_seconds"] == 60
|
|
|
|
|
|
def test_delete_profile(client):
|
|
body = {"name": "Del", "model": "m", "base_url": "http://x/v1", "api_key": "k"}
|
|
pid = client.post("/api/llm-profiles", json=body).json()["profile_id"]
|
|
resp = client.delete(f"/api/llm-profiles/{pid}")
|
|
assert resp.status_code == 200
|
|
assert resp.json()["deleted"] is True
|
|
assert len(client.get("/api/llm-profiles").json()["profiles"]) == 0
|
|
|
|
|
|
def test_update_nonexistent(client):
|
|
resp = client.put("/api/llm-profiles/nope",
|
|
json={"name": "X", "model": "m", "base_url": "http://x/v1", "api_key": "k"})
|
|
assert resp.status_code == 404
|
|
|
|
|
|
def test_delete_nonexistent(client):
|
|
resp = client.delete("/api/llm-profiles/nope")
|
|
assert resp.status_code == 404
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# YAML patcher tests
|
|
# ---------------------------------------------------------------------------
|
|
import yaml as yaml_lib
|
|
from webapp.services.yaml_patcher import apply_profiles_to_scenario
|
|
from webapp.models import LLMProfile
|
|
|
|
|
|
def test_apply_judge_profile(tmp_path):
|
|
"""Applying a judge profile patches judge_model in the YAML."""
|
|
scenario_file = tmp_path / "test-scenario.yaml"
|
|
scenario_file.write_text(
|
|
"scenario_name: test\nmode: offline\njudge_model: old-model\nembedding_model: emb\n"
|
|
"dataset: data.csv\nmetrics:\n- faithfulness\noutput_dir: outputs/test\n",
|
|
encoding="utf-8",
|
|
)
|
|
judge_p = LLMProfile(
|
|
profile_id="x", name="J", model="new-model",
|
|
base_url="http://x/v1", api_key="k", created_at="t", updated_at="t",
|
|
)
|
|
patched = apply_profiles_to_scenario(
|
|
scenario_path=str(scenario_file),
|
|
judge_profile=judge_p,
|
|
answer_profile=None,
|
|
dataset_profile=None,
|
|
_resolve_absolute=True,
|
|
)
|
|
assert "judge_model" in patched
|
|
data = yaml_lib.safe_load(scenario_file.read_text())
|
|
assert data["judge_model"] == "new-model"
|
|
|
|
|
|
def test_apply_answer_profile(tmp_path):
|
|
"""Applying an answer profile patches app_adapter.static_kwargs.model."""
|
|
scenario_file = tmp_path / "online.yaml"
|
|
scenario_file.write_text(
|
|
"scenario_name: online\nmode: online\njudge_model: j\nembedding_model: emb\n"
|
|
"dataset: d.csv\nmetrics:\n- faithfulness\noutput_dir: out\n"
|
|
"app_adapter:\n type: python\n callable: apps.foo:run\n"
|
|
" static_kwargs:\n model: old\n source_chunks_path: chunks.jsonl\n",
|
|
encoding="utf-8",
|
|
)
|
|
answer_p = LLMProfile(
|
|
profile_id="y", name="A", model="new-answer-model",
|
|
base_url="http://x/v1", api_key="k", created_at="t", updated_at="t",
|
|
)
|
|
patched = apply_profiles_to_scenario(
|
|
scenario_path=str(scenario_file),
|
|
judge_profile=None,
|
|
answer_profile=answer_p,
|
|
dataset_profile=None,
|
|
_resolve_absolute=True,
|
|
)
|
|
assert "app_adapter.static_kwargs.model" in patched
|
|
data = yaml_lib.safe_load(scenario_file.read_text())
|
|
assert data["app_adapter"]["static_kwargs"]["model"] == "new-answer-model"
|
|
|
|
|
|
def test_apply_no_profiles_returns_empty(tmp_path):
|
|
"""When no profiles are given, no fields are patched."""
|
|
scenario_file = tmp_path / "noop.yaml"
|
|
scenario_file.write_text("scenario_name: noop\njudge_model: m\n", encoding="utf-8")
|
|
patched = apply_profiles_to_scenario(
|
|
scenario_path=str(scenario_file),
|
|
judge_profile=None,
|
|
answer_profile=None,
|
|
dataset_profile=None,
|
|
_resolve_absolute=True,
|
|
)
|
|
assert patched == []
|