update for ragas
This commit is contained in:
wangwei
@@ -1,4 +1,11 @@
|
||||
"""LLM-powered analysis of rule diagnostics and low-score samples."""
|
||||
"""LLM-powered analysis of rule diagnostics and low-score samples.
|
||||
|
||||
The analyzer issues a single, direct OpenAI-compatible chat call (no langchain,
|
||||
no instructor structured output) to produce a detailed Chinese optimization
|
||||
report. It deliberately uses plain text generation so it works with any
|
||||
OpenAI-compatible gateway and never trips the structured-output response-shape
|
||||
issues seen with the scoring path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
@@ -8,27 +15,41 @@ from .rules import Diagnosis
|
||||
|
||||
logger = logging.getLogger("rag_eval.advisor")
|
||||
|
||||
# Worked-example oriented prompt: forces per-question decomposition (为什么低 →
|
||||
# 拆解 → 怎么改) plus cross-metric causal reasoning, so the report is concrete
|
||||
# instead of a generic restatement of the static rule templates.
|
||||
_PROMPT_TEMPLATE = """\
|
||||
你是一个 RAG 系统优化专家，正在分析西门子医疗 CT 文档问答系统的评测结果。
|
||||
请用中文撰写一份优化建议报告，格式为 Markdown。
|
||||
你是一位资深的 RAG（检索增强生成）系统优化专家，正在分析西门子医疗 CT 文档问答系统的 RAGAS 评测结果。
|
||||
请基于以下诊断数据与低分样本，用中文撰写一份**详细、具体、可落地**的优化建议报告（Markdown 格式）。
|
||||
|
||||
## 评测诊断摘要
|
||||
|
||||
{diagnosis_summary}
|
||||
|
||||
## 低分样本示例
|
||||
## 低分样本明细（含检索片段 contexts，用于定位问题出在检索还是生成环节）
|
||||
|
||||
{low_sample_text}
|
||||
|
||||
## 报告要求
|
||||
## 撰写要求
|
||||
|
||||
1. 按指标分节（## 指标名 [严重程度]），先解释"为什么低"（结合低分样本具体分析），再给出"具体怎么改"
|
||||
2. 严重程度说明：critical=严重（<阈值50%），warning=警告（<阈值70%），low=待优化（低于0.85，有提升空间）
|
||||
3. "具体怎么改"要结合低分样本的实际内容，而不只是泛泛建议
|
||||
4. 最后写一节 **## 优先优化次序**，按性价比排序（不增加 LLM 调用次数的优化优先），critical 和 warning 项优先于 low 项
|
||||
5. 语言简洁，面向工程师，不要废话，不要重复列表内容
|
||||
1. **按指标分节**：每个指标一个 `## 指标名 [严重程度]` 小节。
|
||||
2. **每节必须包含「举例拆解」**：从该指标的低分样本中挑 1-2 个最典型的，逐条按如下结构拆解：
|
||||
- **问题**：简述该样本的 question
|
||||
- **当前得分**：该样本在此指标上的分数
|
||||
- **为什么低**：结合该样本的 answer / 检索片段 contexts / 标准答案 ground_truth，**具体指出**问题所在
|
||||
（例如：答案里哪句话没有被检索片段支持；检索片段里缺了哪个关键信息；答案偏离了问题的哪个点）
|
||||
- **拆解定位**：判断问题出在「检索环节」「生成环节」还是「两者兼有」
|
||||
- **优化动作**：针对这个具体样本，给出 1-3 条**可操作**的改法，不要泛泛而谈
|
||||
3. **跨指标关联分析**：若多个指标同时偏低，分析其因果关系。例如：
|
||||
- faithfulness 低且 context_recall 低 → 多半是检索缺失关键信息，导致模型臆造（幻觉）
|
||||
- context_recall 正常但 context_precision 低 → 检索引入噪声，稀释了有效信息
|
||||
- faithfulness 低但 context_recall 高 → 生成环节 grounding 不足，需收紧生成 prompt
|
||||
4. 最后写一节 `## 优先优化次序`：按性价比排序（不增加 LLM 调用次数的优化优先；critical/warning 优先于 low）。
|
||||
5. 语言简洁，面向工程师，重点是「具体、可操作」。不要复述本提示词，不要无意义的客套。
|
||||
|
||||
只输出 Markdown 报告正文，不要任何前置说明。
|
||||
严重程度说明：critical=严重（远低于阈值），warning=警告（低于阈值），low=待优化（达标但低于 0.85，仍有提升空间）。
|
||||
|
||||
只输出 Markdown 报告正文，不要任何前置说明或代码块包裹。
|
||||
"""
|
||||
|
||||
|
||||
@@ -38,8 +59,15 @@ _SEVERITY_LABEL_ZH: dict[str, str] = {
|
||||
"low": "待优化",
|
||||
}
|
||||
|
||||
# Per-sample text limits keep the prompt bounded regardless of sample size.
|
||||
_ANSWER_LIMIT = 400
|
||||
_GT_LIMIT = 300
|
||||
_CONTEXT_LIMIT = 600
|
||||
_CONTEXT_SEPARATOR = " |||| "
|
||||
|
||||
|
||||
def _build_diagnosis_summary(diagnoses: list[Diagnosis]) -> str:
|
||||
"""Render the per-metric diagnosis block fed to the LLM."""
|
||||
lines = []
|
||||
for d in diagnoses:
|
||||
direction = "（越低越好）" if d.metric == "noise_sensitivity" else ""
|
||||
@@ -53,7 +81,22 @@ def _build_diagnosis_summary(diagnoses: list[Diagnosis]) -> str:
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _format_contexts(raw: Any) -> str:
|
||||
"""Render a sample's contexts (stored as a joined string) as a short list."""
|
||||
text = str(raw or "").strip()
|
||||
if not text:
|
||||
return "（无检索片段）"
|
||||
parts = [p.strip() for p in text.split(_CONTEXT_SEPARATOR) if p.strip()]
|
||||
if not parts:
|
||||
return "（无检索片段）"
|
||||
rendered = "; ".join(f"[{i + 1}] {p}" for i, p in enumerate(parts))
|
||||
if len(rendered) > _CONTEXT_LIMIT:
|
||||
rendered = rendered[:_CONTEXT_LIMIT] + "…"
|
||||
return rendered
|
||||
|
||||
|
||||
def _build_low_sample_text(diagnoses: list[Diagnosis]) -> str:
|
||||
"""Render low-score samples (now including contexts) for grounding analysis."""
|
||||
lines = []
|
||||
for d in diagnoses:
|
||||
if not d.low_samples:
|
||||
@@ -61,24 +104,63 @@ def _build_low_sample_text(diagnoses: list[Diagnosis]) -> str:
|
||||
lines.append(f"### {d.metric} 低分样本（最多 3 条）")
|
||||
for i, s in enumerate(d.low_samples, 1):
|
||||
score = s.get(d.metric, "N/A")
|
||||
lines.append(f"\n**样本 {i}**（分数={score}）")
|
||||
lines.append(f"- 问题：{s.get('question', '')}")
|
||||
lines.append(f"- 回答：{s.get('answer', '')[:300]}")
|
||||
lines.append(f"- 标准答案：{s.get('ground_truth', '')[:200]}")
|
||||
lines.append(f"\n**样本 {i}**（{d.metric}={score}）")
|
||||
lines.append(f"- 问题 question：{s.get('question', '')}")
|
||||
lines.append(f"- 生成答案 answer：{str(s.get('answer', ''))[:_ANSWER_LIMIT]}")
|
||||
lines.append(f"- 检索片段 contexts：{_format_contexts(s.get('contexts'))}")
|
||||
lines.append(f"- 标准答案 ground_truth：{str(s.get('ground_truth', ''))[:_GT_LIMIT]}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _is_reasoning_model(model: str) -> bool:
|
||||
"""Return True for OpenAI reasoning models (gpt-5+/o-series/codex-mini).
|
||||
|
||||
These require `max_completion_tokens` instead of `max_tokens` and do not
|
||||
accept a custom temperature. Mirrors RAGAS's own detection logic so advice
|
||||
generation stays consistent with the scoring path.
|
||||
"""
|
||||
m = (model or "").lower()
|
||||
# O-series: o1..o9 optionally followed by - or _
|
||||
if len(m) >= 2 and m[0] == "o" and m[1] in "123456789":
|
||||
if len(m) == 2 or m[2] in ("-", "_"):
|
||||
return True
|
||||
# GPT-5 through GPT-19 generation
|
||||
if m.startswith("gpt-"):
|
||||
version_str = m[4:].split("-")[0].split("_")[0].split(".")[0]
|
||||
try:
|
||||
if 5 <= int(version_str) <= 19:
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
if m == "codex-mini":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _chat_token_kwargs(model: str, max_tokens: int) -> dict[str, Any]:
|
||||
"""Return the completion-budget kwargs appropriate for the model family."""
|
||||
if _is_reasoning_model(model):
|
||||
# Reasoning models: only max_completion_tokens; temperature must stay default (1).
|
||||
return {"max_completion_tokens": max_tokens}
|
||||
return {"max_tokens": max_tokens, "temperature": 0.2}
|
||||
|
||||
|
||||
async def analyze(
|
||||
diagnoses: list[Diagnosis],
|
||||
llm: Any,
|
||||
scenario_name: str,
|
||||
judge_model: str,
|
||||
settings: Any,
|
||||
*,
|
||||
chat_client: Any | None = None,
|
||||
) -> str:
|
||||
"""Call the judge LLM to generate a Chinese optimization report.
|
||||
"""Call the judge LLM directly to generate a Chinese optimization report.
|
||||
|
||||
Args:
|
||||
diagnoses: Non-empty list of Diagnosis from rules.diagnose().
|
||||
llm: RAGAS LLM wrapper (has .agenerate() method).
|
||||
scenario_name: Used only for logging.
|
||||
judge_model: Model name to call (also selects the matching LLM profile).
|
||||
settings: EvaluationSettings (supplies client kwargs + token budget).
|
||||
chat_client: Optional pre-built AsyncOpenAI-compatible client (for tests).
|
||||
|
||||
Returns:
|
||||
LLM-generated Markdown string, or "" on failure (triggers writer fallback).
|
||||
@@ -86,22 +168,47 @@ async def analyze(
|
||||
if not diagnoses:
|
||||
return ""
|
||||
|
||||
diagnosis_summary = _build_diagnosis_summary(diagnoses)
|
||||
low_sample_text = _build_low_sample_text(diagnoses)
|
||||
prompt = _PROMPT_TEMPLATE.format(
|
||||
diagnosis_summary=diagnosis_summary,
|
||||
low_sample_text=low_sample_text,
|
||||
diagnosis_summary=_build_diagnosis_summary(diagnoses),
|
||||
low_sample_text=_build_low_sample_text(diagnoses),
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info("[advisor] calling LLM for optimization analysis scenario=%s", scenario_name)
|
||||
from langchain_core.messages import HumanMessage
|
||||
# Use the underlying langchain chat model directly (RAGAS LangchainLLMWrapper wraps BaseChatModel)
|
||||
response = await llm.langchain_llm.ainvoke([HumanMessage(content=prompt)])
|
||||
text = response.content.strip()
|
||||
logger.info("[advisor] LLM analysis complete chars=%d", len(text))
|
||||
return text
|
||||
except Exception as exc:
|
||||
client = chat_client
|
||||
owns_client = False
|
||||
if client is None:
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from rag_eval.metrics.factory import resolve_openai_client_kwargs
|
||||
|
||||
client = AsyncOpenAI(**resolve_openai_client_kwargs(judge_model, settings))
|
||||
owns_client = True
|
||||
|
||||
try:
|
||||
# Advice is a longer document than per-metric scoring; give it headroom.
|
||||
max_tokens = max(2048, int(getattr(settings, "ragas_llm_max_tokens", 4096) or 4096))
|
||||
token_kwargs = _chat_token_kwargs(judge_model, max_tokens)
|
||||
|
||||
response = await client.chat.completions.create(
|
||||
model=judge_model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
**token_kwargs,
|
||||
)
|
||||
text = (response.choices[0].message.content or "").strip()
|
||||
logger.info("[advisor] LLM analysis complete chars=%d", len(text))
|
||||
return text
|
||||
finally:
|
||||
# Close the httpx connection pool inside THIS event loop. run_advisor
|
||||
# drives analyze() via asyncio.run(), which closes the loop on return;
|
||||
# a later GC-time aclose() on the dead loop would otherwise log
|
||||
# "RuntimeError: Event loop is closed". Only close clients we created.
|
||||
if owns_client:
|
||||
try:
|
||||
await client.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"[advisor] LLM analysis failed (%s: %s) — falling back to rule report",
|
||||
type(exc).__name__, exc,
|
||||
|
||||
Reference in New Issue
Block a user