"""Routes for session-grouped async RAGAS scoring (Dify multi-call integration). Use case: Dify evaluates multiple Q&A pairs in a session. Each pair gets its own `POST /api/score/session_async` call with a shared `session_id`. All results are accumulated into one report, visible in 「运行列表」→「报告详情」. Key behaviour: - Deterministic run_id: derived from session_id — same session always maps to the same report directory (outputs/score-session/session-/). - Append semantics: each call adds a new sample row. Previous rows are preserved. - Advisor regeneration: optimization_advice.md is regenerated after every call using the full set of accumulated rows. - Each call returns its own `job_id` for individual status polling, plus the shared `run_id` and `session_id`. Endpoints: POST /api/score/session_async Submit one call (returns job_id + run_id) GET /api/score/sessions List all sessions GET /api/score/sessions/{session_id} Session aggregate (call_count, metric_means, jobs) GET /api/score/session/jobs/{job_id} Status of one individual call """ from __future__ import annotations import logging from fastapi import APIRouter, HTTPException from webapp.models import ( AsyncScoreJobStatus, ScoreRequest, SessionScoreJobResponse, SessionScoreRequest, SessionStatus, ) from webapp.services.session_score_manager import session_score_manager router = APIRouter(prefix="/api/score", tags=["score"]) logger = logging.getLogger("webapp.api.session_score_jobs") @router.post( "/session_async", status_code=202, response_model=SessionScoreJobResponse, summary="提交 Session 异步评分(多样本批量聚合)", description=( "**用途**\n" "- 适合 Dify 循环节点、批量问答评测、同一对话多轮累计评分。\n" "- 相同 `session_id` 的多次调用不会生成多个独立报告,而是持续追加到同一个 session 报告。\n\n" "**请求字段说明**\n" "- `session_id`:会话唯一标识,同一会话必须保持一致。\n" "- `question` / `answer`:本次待评分的问答对。\n" "- `contexts`:检索片段拼接字符串,按 `context_separator` 拆分。\n" "- `ground_truth`:标准答案,可选;缺失时会自动跳过依赖它的指标。\n" "- `metrics`:本次需要计算的指标列表。\n" "- `judge_model` / `embedding_model`:可选;为空时回退到系统默认配置。\n\n" "**处理行为**\n" "1. 服务端立即返回 `202 Accepted`,并生成本次调用的 `job_id`。\n" "2. 系统根据 `session_id` 计算固定 `run_id`,格式为 `session-`。\n" "3. 本次评分完成后,会向该 session 的 `scores.csv` 追加一行样本数据。\n" "4. 系统会基于当前 session 的全量样本重写 `summary.md`,并重新生成 `optimization_advice.md`。\n" "5. 报告可在「运行列表」中按 `run_id` 查看;同一 session 的后续调用会持续增量更新该报告。\n\n" "**后续查询接口**\n" "- `GET /api/score/session/jobs/{job_id}`:查询本次调用状态与得分。\n" "- `GET /api/score/sessions/{session_id}`:查询整个 session 的累计调用次数、指标均值、所有作业记录。\n" "- `GET /api/runs/{run_id}`:查看完整评估报告内容。\n\n" "**典型请求示例**\n" "```json\n" "{\n" " \"session_id\": \"dify-session-001\",\n" " \"question\": \"单源CT与双源CT在球管配置上有何本质区别?\",\n" " \"answer\": \"单源CT只有一套球管-探测器系统,双源CT有两套独立的球管-探测器系统。\",\n" " \"contexts\": \"双源CT采用两套管-探测器系统 |||| 单源CT只有一个球管\",\n" " \"context_separator\": \" |||| \",\n" " \"metrics\": [\"answer_relevancy\", \"faithfulness\"],\n" " \"judge_model\": \"gpt-5.5\",\n" " \"embedding_model\": \"text-embedding-3-small\"\n" "}\n" "```" ), responses={ 202: { "description": ( "调用已排队,立即返回 job_id + run_id(202 Accepted)。\n\n" "相同 `session_id` 的多次调用合并为同一报告,每次调用新增一个样本行。\n" "评分完成后,`summary.md` 和 `optimization_advice.md` 增量更新。\n" "通过 `GET /api/score/sessions/{session_id}` 查看 session 聚合状态," "通过 `GET /api/score/session/jobs/{job_id}` 查询单次调用状态," "在「运行列表」中查看完整报告(run_id 即 `session-` 形式)。" ), "content": { "application/json": { "example": { "job_id": "abc123def456", "session_id": "dify-session-001", "run_id": "session-dify-session-001", "status": "queued", "call_count": 1, } } }, }, }, ) def submit_session_async_score(request: SessionScoreRequest) -> SessionScoreJobResponse: """提交 Session 异步 RAGAS 评分,立即返回 job_id。 相同 `session_id` 的多次调用合并到同一评估报告中,每次调用: 1. 新增一个样本行到 `scores.csv` 2. 重写 `summary.md`(包含所有累积样本的指标均值) 3. 重新生成 `optimization_advice.md`(基于全量样本的 LLM 优化建议) **适合 Dify 工作流**:在循环节点中批量调用,所有轮次共用同一 `session_id`, 最终在 RAGAS 平台「运行列表」中查看完整的批量评估报告。 """ logger.info( "[session_async] submit session_id=%s metrics=%s has_ctx=%s has_gt=%s", request.session_id, request.metrics, bool(request.contexts), bool(request.ground_truth), ) # Strip session_id to build a plain ScoreRequest for the manager score_request = ScoreRequest( question=request.question, answer=request.answer, contexts=request.contexts, ground_truth=request.ground_truth, context_separator=request.context_separator, metrics=request.metrics, judge_model=request.judge_model, embedding_model=request.embedding_model, ) status, run_id = session_score_manager.submit(request.session_id, score_request) # Compute call_count from current session state session_status = session_score_manager.get_session(request.session_id) call_count = session_status.call_count if session_status else 1 logger.info( "[session_async] queued job_id=%s session_id=%s run_id=%s call=%d", status.job_id, request.session_id, run_id, call_count, ) return SessionScoreJobResponse( job_id=status.job_id, session_id=request.session_id, run_id=run_id, status=status.status, call_count=call_count, ) @router.get( "/sessions", response_model=dict, summary="列出所有 Session 聚合状态", ) def list_sessions() -> dict: """返回所有 session 的聚合状态,按最近完成时间倒序排列。""" sessions = session_score_manager.list_sessions() logger.info("[session_score] list_sessions count=%d", len(sessions)) return {"sessions": [s.model_dump() for s in sessions]} @router.get( "/sessions/{session_id}", response_model=SessionStatus, summary="查询 Session 聚合状态(指标均值 + 所有调用记录)", responses={404: {"description": "指定 session_id 不存在。"}}, ) def get_session(session_id: str) -> SessionStatus: """查询 session 的聚合评分状态。 返回内容: - `run_id`:在「运行列表」中查看完整报告 - `call_count`:本 session 累计调用次数 - `metric_means`:所有已累积样本的各指标均值(实时读取 scores.csv) - `jobs`:本 session 所有调用记录列表 """ status = session_score_manager.get_session(session_id) if status is None: raise HTTPException(status_code=404, detail=f"Session not found: {session_id}") return status @router.get( "/session/jobs/{job_id}", response_model=AsyncScoreJobStatus, summary="查询 Session 单次调用状态", responses={404: {"description": "指定 job_id 不存在。"}}, ) def get_session_job(job_id: str) -> AsyncScoreJobStatus: """查询 session 评分中某次调用的状态和评分结果。 `status` 为 `completed` 时,`run_id` 即所属 session 的报告目录, `scores` 包含本次调用的各指标得分。 """ status = session_score_manager.get_job(job_id) if status is None: raise HTTPException( status_code=404, detail=f"Session score job not found: {job_id}" ) return status