siemens_ragas/webapp/api/llm_profiles.py

"""CRUD routes for LLM profiles plus the scenario-patching apply endpoint."""

from __future__ import annotations

import time

from fastapi import APIRouter, HTTPException
from openai import OpenAI

from webapp.models import (
    CreateProfileRequest,
    LLMProfile,
    ProfileApplyRequest,
    ProfileApplyResponse,
    ProfileProbeRequest,
    ProfileTestResponse,
)
from webapp.services.profile_manager import profile_manager
from webapp.services.yaml_patcher import apply_profiles_to_scenario

router = APIRouter(prefix="/api/llm-profiles", tags=["llm-profiles"])


def _do_connectivity_test(
    model: str,
    base_url: str,
    api_key: str,
    timeout_seconds: int,
) -> ProfileTestResponse:
    """Send a minimal chat completion request and return the test result."""
    client = OpenAI(
        api_key=api_key,
        base_url=base_url.rstrip("/"),
        timeout=float(timeout_seconds),
    )
    t0 = time.monotonic()
    try:
        client.chat.completions.create(
            model=model,
            messages=[{"role": "user", "content": "hi"}],
            max_tokens=1,
        )
        latency_ms = int((time.monotonic() - t0) * 1000)
        return ProfileTestResponse(ok=True, message="连接成功", latency_ms=latency_ms)
    except Exception as exc:  # noqa: BLE001
        latency_ms = int((time.monotonic() - t0) * 1000)
        return ProfileTestResponse(ok=False, message=str(exc), latency_ms=latency_ms)


@router.post("/probe", response_model=ProfileTestResponse, tags=["llm-profiles"])
def probe_connectivity(request: ProfileProbeRequest) -> ProfileTestResponse:
    """Test LLM connectivity with inline credentials (no saved profile required)."""
    return _do_connectivity_test(
        model=request.model,
        base_url=request.base_url,
        api_key=request.api_key,
        timeout_seconds=request.timeout_seconds,
    )


@router.get("", response_model=dict)
def list_profiles() -> dict:
    """Return all saved LLM profiles."""
    return {"profiles": [p.model_dump() for p in profile_manager.list_all()]}


@router.post("", status_code=201, response_model=LLMProfile)
def create_profile(request: CreateProfileRequest) -> LLMProfile:
    """Create a new LLM profile."""
    return profile_manager.create(
        name=request.name,
        model=request.model,
        base_url=request.base_url,
        api_key=request.api_key,
        timeout_seconds=request.timeout_seconds,
    )


@router.put("/{profile_id}", response_model=LLMProfile)
def update_profile(profile_id: str, request: CreateProfileRequest) -> LLMProfile:
    """Update an existing LLM profile by id."""
    updated = profile_manager.update(
        profile_id=profile_id,
        name=request.name,
        model=request.model,
        base_url=request.base_url,
        api_key=request.api_key,
        timeout_seconds=request.timeout_seconds,
    )
    if updated is None:
        raise HTTPException(status_code=404, detail=f"Profile not found: {profile_id}")
    return updated


@router.delete("/{profile_id}", response_model=dict)
def delete_profile(profile_id: str) -> dict:
    """Delete an LLM profile by id."""
    deleted = profile_manager.delete(profile_id)
    if not deleted:
        raise HTTPException(status_code=404, detail=f"Profile not found: {profile_id}")
    return {"deleted": True}


@router.post("/{profile_id}/test", response_model=ProfileTestResponse)
def test_profile(profile_id: str) -> ProfileTestResponse:
    """Test LLM connectivity for a saved profile."""
    profile = profile_manager.get(profile_id)
    if profile is None:
        raise HTTPException(status_code=404, detail=f"Profile not found: {profile_id}")
    return _do_connectivity_test(
        model=profile.model,
        base_url=profile.base_url,
        api_key=profile.api_key,
        timeout_seconds=profile.timeout_seconds,
    )


@router.post("/apply", response_model=ProfileApplyResponse)
def apply_profiles(request: ProfileApplyRequest) -> ProfileApplyResponse:
    """Patch selected LLM profiles into the target scenario YAML file."""
    role_profiles: dict[str, LLMProfile | None] = {
        "judge": profile_manager.get(request.judge_profile_id) if request.judge_profile_id else None,
        "answer": profile_manager.get(request.answer_profile_id) if request.answer_profile_id else None,
        "dataset": profile_manager.get(request.dataset_profile_id) if request.dataset_profile_id else None,
    }

    missing = [
        role
        for role, pid in [
            ("judge", request.judge_profile_id),
            ("answer", request.answer_profile_id),
            ("dataset", request.dataset_profile_id),
        ]
        if pid and role_profiles[role] is None
    ]

    if missing:
        raise HTTPException(
            status_code=400,
            detail=f"Profile(s) not found for roles: {', '.join(missing)}",
        )

    patched = apply_profiles_to_scenario(
        scenario_path=request.scenario_path,
        judge_profile=role_profiles["judge"],
        answer_profile=role_profiles["answer"],
        dataset_profile=role_profiles["dataset"],
        metric_weights=request.metric_weights,
        doc_weights=request.doc_weights,
    )
    return ProfileApplyResponse(
        scenario_path=request.scenario_path,
        patched_fields=patched,
    )