diff --git a/tests/webapp/test_llm_profiles_api.py b/tests/webapp/test_llm_profiles_api.py new file mode 100644 index 0000000..6cc2c4a --- /dev/null +++ b/tests/webapp/test_llm_profiles_api.py @@ -0,0 +1,67 @@ +"""Integration tests for /api/llm-profiles endpoints.""" +import pytest +from fastapi.testclient import TestClient + + +@pytest.fixture() +def client(tmp_path, monkeypatch): + """TestClient with a fresh ProfileManager backed by a temp file.""" + store = tmp_path / "profiles.json" + import webapp.services.profile_manager as pm_mod + from webapp.services.profile_manager import ProfileManager + fresh_mgr = ProfileManager(store_path=store) + monkeypatch.setattr(pm_mod, "profile_manager", fresh_mgr) + import webapp.api.llm_profiles as api_mod + monkeypatch.setattr(api_mod, "profile_manager", fresh_mgr) + + from webapp.server import create_app + return TestClient(create_app()) + + +def test_list_empty(client): + resp = client.get("/api/llm-profiles") + assert resp.status_code == 200 + assert resp.json()["profiles"] == [] + + +def test_create_and_list(client): + body = {"name": "Test", "model": "m1", "base_url": "http://x/v1", "api_key": "k"} + resp = client.post("/api/llm-profiles", json=body) + assert resp.status_code == 201 + data = resp.json() + assert data["name"] == "Test" + assert data["profile_id"] != "" + + resp2 = client.get("/api/llm-profiles") + assert len(resp2.json()["profiles"]) == 1 + + +def test_update_profile(client): + body = {"name": "Old", "model": "m1", "base_url": "http://x/v1", "api_key": "k"} + pid = client.post("/api/llm-profiles", json=body).json()["profile_id"] + + upd = {"name": "New", "model": "m2", "base_url": "http://x/v1", "api_key": "k", "timeout_seconds": 60} + resp = client.put(f"/api/llm-profiles/{pid}", json=upd) + assert resp.status_code == 200 + assert resp.json()["name"] == "New" + assert resp.json()["timeout_seconds"] == 60 + + +def test_delete_profile(client): + body = {"name": "Del", "model": "m", "base_url": "http://x/v1", "api_key": "k"} + pid = client.post("/api/llm-profiles", json=body).json()["profile_id"] + resp = client.delete(f"/api/llm-profiles/{pid}") + assert resp.status_code == 200 + assert resp.json()["deleted"] is True + assert len(client.get("/api/llm-profiles").json()["profiles"]) == 0 + + +def test_update_nonexistent(client): + resp = client.put("/api/llm-profiles/nope", + json={"name": "X", "model": "m", "base_url": "http://x/v1", "api_key": "k"}) + assert resp.status_code == 404 + + +def test_delete_nonexistent(client): + resp = client.delete("/api/llm-profiles/nope") + assert resp.status_code == 404 diff --git a/webapp/api/llm_profiles.py b/webapp/api/llm_profiles.py new file mode 100644 index 0000000..5504190 --- /dev/null +++ b/webapp/api/llm_profiles.py @@ -0,0 +1,96 @@ +"""CRUD routes for LLM profiles plus the scenario-patching apply endpoint.""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from webapp.models import ( + CreateProfileRequest, + LLMProfile, + ProfileApplyRequest, + ProfileApplyResponse, +) +from webapp.services.profile_manager import profile_manager +from webapp.services.yaml_patcher import apply_profiles_to_scenario + +router = APIRouter(prefix="/api/llm-profiles", tags=["llm-profiles"]) + + +@router.get("", response_model=dict) +def list_profiles() -> dict: + """Return all saved LLM profiles.""" + return {"profiles": [p.model_dump() for p in profile_manager.list_all()]} + + +@router.post("", status_code=201, response_model=LLMProfile) +def create_profile(request: CreateProfileRequest) -> LLMProfile: + """Create a new LLM profile.""" + return profile_manager.create( + name=request.name, + model=request.model, + base_url=request.base_url, + api_key=request.api_key, + timeout_seconds=request.timeout_seconds, + ) + + +@router.put("/{profile_id}", response_model=LLMProfile) +def update_profile(profile_id: str, request: CreateProfileRequest) -> LLMProfile: + """Update an existing LLM profile by id.""" + updated = profile_manager.update( + profile_id=profile_id, + name=request.name, + model=request.model, + base_url=request.base_url, + api_key=request.api_key, + timeout_seconds=request.timeout_seconds, + ) + if updated is None: + raise HTTPException(status_code=404, detail=f"Profile not found: {profile_id}") + return updated + + +@router.delete("/{profile_id}", response_model=dict) +def delete_profile(profile_id: str) -> dict: + """Delete an LLM profile by id.""" + deleted = profile_manager.delete(profile_id) + if not deleted: + raise HTTPException(status_code=404, detail=f"Profile not found: {profile_id}") + return {"deleted": True} + + +@router.post("/apply", response_model=ProfileApplyResponse) +def apply_profiles(request: ProfileApplyRequest) -> ProfileApplyResponse: + """Patch selected LLM profiles into the target scenario YAML file.""" + role_profiles: dict[str, LLMProfile | None] = { + "judge": profile_manager.get(request.judge_profile_id) if request.judge_profile_id else None, + "answer": profile_manager.get(request.answer_profile_id) if request.answer_profile_id else None, + "dataset": profile_manager.get(request.dataset_profile_id) if request.dataset_profile_id else None, + } + + missing = [ + role + for role, pid in [ + ("judge", request.judge_profile_id), + ("answer", request.answer_profile_id), + ("dataset", request.dataset_profile_id), + ] + if pid and role_profiles[role] is None + ] + + if missing: + raise HTTPException( + status_code=400, + detail=f"Profile(s) not found for roles: {', '.join(missing)}", + ) + + patched = apply_profiles_to_scenario( + scenario_path=request.scenario_path, + judge_profile=role_profiles["judge"], + answer_profile=role_profiles["answer"], + dataset_profile=role_profiles["dataset"], + ) + return ProfileApplyResponse( + scenario_path=request.scenario_path, + patched_fields=patched, + ) diff --git a/webapp/server.py b/webapp/server.py index 49ea03d..06bdfe1 100644 --- a/webapp/server.py +++ b/webapp/server.py @@ -13,7 +13,7 @@ from fastapi import FastAPI from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles -from webapp.api import evaluations, runs, scenarios +from webapp.api import evaluations, llm_profiles, runs, scenarios STATIC_DIR = Path(__file__).resolve().parent / "static" @@ -29,6 +29,7 @@ def create_app() -> FastAPI: app.include_router(runs.router) app.include_router(scenarios.router) app.include_router(evaluations.router) + app.include_router(llm_profiles.router) @app.get("/api/health", tags=["meta"]) def health() -> dict[str, str]: