feat: add /api/llm-profiles CRUD router

tests/webapp/test_llm_profiles_api.py

@@ -0,0 +1,67 @@
+"""Integration tests for /api/llm-profiles endpoints."""
+import pytest
+from fastapi.testclient import TestClient
+
+
+@pytest.fixture()
+def client(tmp_path, monkeypatch):
+    """TestClient with a fresh ProfileManager backed by a temp file."""
+    store = tmp_path / "profiles.json"
+    import webapp.services.profile_manager as pm_mod
+    from webapp.services.profile_manager import ProfileManager
+    fresh_mgr = ProfileManager(store_path=store)
+    monkeypatch.setattr(pm_mod, "profile_manager", fresh_mgr)
+    import webapp.api.llm_profiles as api_mod
+    monkeypatch.setattr(api_mod, "profile_manager", fresh_mgr)
+
+    from webapp.server import create_app
+    return TestClient(create_app())
+
+
+def test_list_empty(client):
+    resp = client.get("/api/llm-profiles")
+    assert resp.status_code == 200
+    assert resp.json()["profiles"] == []
+
+
+def test_create_and_list(client):
+    body = {"name": "Test", "model": "m1", "base_url": "http://x/v1", "api_key": "k"}
+    resp = client.post("/api/llm-profiles", json=body)
+    assert resp.status_code == 201
+    data = resp.json()
+    assert data["name"] == "Test"
+    assert data["profile_id"] != ""
+
+    resp2 = client.get("/api/llm-profiles")
+    assert len(resp2.json()["profiles"]) == 1
+
+
+def test_update_profile(client):
+    body = {"name": "Old", "model": "m1", "base_url": "http://x/v1", "api_key": "k"}
+    pid = client.post("/api/llm-profiles", json=body).json()["profile_id"]
+
+    upd = {"name": "New", "model": "m2", "base_url": "http://x/v1", "api_key": "k", "timeout_seconds": 60}
+    resp = client.put(f"/api/llm-profiles/{pid}", json=upd)
+    assert resp.status_code == 200
+    assert resp.json()["name"] == "New"
+    assert resp.json()["timeout_seconds"] == 60
+
+
+def test_delete_profile(client):
+    body = {"name": "Del", "model": "m", "base_url": "http://x/v1", "api_key": "k"}
+    pid = client.post("/api/llm-profiles", json=body).json()["profile_id"]
+    resp = client.delete(f"/api/llm-profiles/{pid}")
+    assert resp.status_code == 200
+    assert resp.json()["deleted"] is True
+    assert len(client.get("/api/llm-profiles").json()["profiles"]) == 0
+
+
+def test_update_nonexistent(client):
+    resp = client.put("/api/llm-profiles/nope",
+                      json={"name": "X", "model": "m", "base_url": "http://x/v1", "api_key": "k"})
+    assert resp.status_code == 404
+
+
+def test_delete_nonexistent(client):
+    resp = client.delete("/api/llm-profiles/nope")
+    assert resp.status_code == 404

webapp/api/llm_profiles.py

@@ -0,0 +1,96 @@
+"""CRUD routes for LLM profiles plus the scenario-patching apply endpoint."""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException
+
+from webapp.models import (
+    CreateProfileRequest,
+    LLMProfile,
+    ProfileApplyRequest,
+    ProfileApplyResponse,
+)
+from webapp.services.profile_manager import profile_manager
+from webapp.services.yaml_patcher import apply_profiles_to_scenario
+
+router = APIRouter(prefix="/api/llm-profiles", tags=["llm-profiles"])
+
+
+@router.get("", response_model=dict)
+def list_profiles() -> dict:
+    """Return all saved LLM profiles."""
+    return {"profiles": [p.model_dump() for p in profile_manager.list_all()]}
+
+
+@router.post("", status_code=201, response_model=LLMProfile)
+def create_profile(request: CreateProfileRequest) -> LLMProfile:
+    """Create a new LLM profile."""
+    return profile_manager.create(
+        name=request.name,
+        model=request.model,
+        base_url=request.base_url,
+        api_key=request.api_key,
+        timeout_seconds=request.timeout_seconds,
+    )
+
+
+@router.put("/{profile_id}", response_model=LLMProfile)
+def update_profile(profile_id: str, request: CreateProfileRequest) -> LLMProfile:
+    """Update an existing LLM profile by id."""
+    updated = profile_manager.update(
+        profile_id=profile_id,
+        name=request.name,
+        model=request.model,
+        base_url=request.base_url,
+        api_key=request.api_key,
+        timeout_seconds=request.timeout_seconds,
+    )
+    if updated is None:
+        raise HTTPException(status_code=404, detail=f"Profile not found: {profile_id}")
+    return updated
+
+
+@router.delete("/{profile_id}", response_model=dict)
+def delete_profile(profile_id: str) -> dict:
+    """Delete an LLM profile by id."""
+    deleted = profile_manager.delete(profile_id)
+    if not deleted:
+        raise HTTPException(status_code=404, detail=f"Profile not found: {profile_id}")
+    return {"deleted": True}
+
+
+@router.post("/apply", response_model=ProfileApplyResponse)
+def apply_profiles(request: ProfileApplyRequest) -> ProfileApplyResponse:
+    """Patch selected LLM profiles into the target scenario YAML file."""
+    role_profiles: dict[str, LLMProfile | None] = {
+        "judge": profile_manager.get(request.judge_profile_id) if request.judge_profile_id else None,
+        "answer": profile_manager.get(request.answer_profile_id) if request.answer_profile_id else None,
+        "dataset": profile_manager.get(request.dataset_profile_id) if request.dataset_profile_id else None,
+    }
+
+    missing = [
+        role
+        for role, pid in [
+            ("judge", request.judge_profile_id),
+            ("answer", request.answer_profile_id),
+            ("dataset", request.dataset_profile_id),
+        ]
+        if pid and role_profiles[role] is None
+    ]
+
+    if missing:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Profile(s) not found for roles: {', '.join(missing)}",
+        )
+
+    patched = apply_profiles_to_scenario(
+        scenario_path=request.scenario_path,
+        judge_profile=role_profiles["judge"],
+        answer_profile=role_profiles["answer"],
+        dataset_profile=role_profiles["dataset"],
+    )
+    return ProfileApplyResponse(
+        scenario_path=request.scenario_path,
+        patched_fields=patched,
+    )

webapp/server.py

@@ -13,7 +13,7 @@ from fastapi import FastAPI
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
 
-from webapp.api import evaluations, runs, scenarios
+from webapp.api import evaluations, llm_profiles, runs, scenarios
 
 STATIC_DIR = Path(__file__).resolve().parent / "static"
 
@@ -29,6 +29,7 @@ def create_app() -> FastAPI:
     app.include_router(runs.router)
     app.include_router(scenarios.router)
     app.include_router(evaluations.router)
+    app.include_router(llm_profiles.router)
 
     @app.get("/api/health", tags=["meta"])
     def health() -> dict[str, str]: