update for ragas
This commit is contained in:
wangwei
@@ -20,7 +20,10 @@ __all__ = ["run_advisor", "Diagnosis", "diagnose"]
|
||||
def run_advisor(
|
||||
result: EvaluationResult,
|
||||
scenario: Scenario,
|
||||
llm: Any,
|
||||
llm: Any = None,
|
||||
*,
|
||||
settings: Any | None = None,
|
||||
chat_client: Any | None = None,
|
||||
) -> None:
|
||||
"""Run the full optimization advisor pipeline after an evaluation completes.
|
||||
|
||||
@@ -30,7 +33,10 @@ def run_advisor(
|
||||
Args:
|
||||
result: Completed EvaluationResult from Evaluator.evaluate().
|
||||
scenario: The resolved Scenario (provides metrics, judge_model, output_dir).
|
||||
llm: Pre-built RAGAS LLM instance (from build_models()) for LLM analysis.
|
||||
llm: Deprecated/unused — kept for backward-compatible call sites. The
|
||||
advisor now issues its own direct LLM call resolved from judge_model.
|
||||
settings: Optional EvaluationSettings; defaults to EvaluationSettings().
|
||||
chat_client: Optional pre-built chat client (used by tests to avoid network).
|
||||
"""
|
||||
if not scenario.optimization_advisor:
|
||||
return
|
||||
@@ -38,6 +44,10 @@ def run_advisor(
|
||||
logger.info("[advisor] starting optimization analysis scenario=%s", scenario.scenario_name)
|
||||
|
||||
try:
|
||||
if settings is None:
|
||||
from rag_eval.settings import EvaluationSettings
|
||||
settings = EvaluationSettings()
|
||||
|
||||
artifact_paths = build_artifact_paths(scenario.output_dir, result.run_id)
|
||||
if artifact_paths.advice_md is None:
|
||||
logger.warning("[advisor] advice_md path not set in RunArtifactPaths — skipping")
|
||||
@@ -47,7 +57,15 @@ def run_advisor(
|
||||
logger.info("[advisor] rule diagnosis complete: %d metric(s) triggered", len(diagnoses))
|
||||
|
||||
if diagnoses:
|
||||
llm_markdown = asyncio.run(analyze(diagnoses, llm, scenario.scenario_name))
|
||||
llm_markdown = asyncio.run(
|
||||
analyze(
|
||||
diagnoses,
|
||||
scenario.scenario_name,
|
||||
scenario.judge_model,
|
||||
settings,
|
||||
chat_client=chat_client,
|
||||
)
|
||||
)
|
||||
else:
|
||||
llm_markdown = ""
|
||||
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
"""LLM-powered analysis of rule diagnostics and low-score samples."""
|
||||
"""LLM-powered analysis of rule diagnostics and low-score samples.
|
||||
|
||||
The analyzer issues a single, direct OpenAI-compatible chat call (no langchain,
|
||||
no instructor structured output) to produce a detailed Chinese optimization
|
||||
report. It deliberately uses plain text generation so it works with any
|
||||
OpenAI-compatible gateway and never trips the structured-output response-shape
|
||||
issues seen with the scoring path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
@@ -8,27 +15,41 @@ from .rules import Diagnosis
|
||||
|
||||
logger = logging.getLogger("rag_eval.advisor")
|
||||
|
||||
# Worked-example oriented prompt: forces per-question decomposition (为什么低 →
|
||||
# 拆解 → 怎么改) plus cross-metric causal reasoning, so the report is concrete
|
||||
# instead of a generic restatement of the static rule templates.
|
||||
_PROMPT_TEMPLATE = """\
|
||||
你是一个 RAG 系统优化专家，正在分析西门子医疗 CT 文档问答系统的评测结果。
|
||||
请用中文撰写一份优化建议报告，格式为 Markdown。
|
||||
你是一位资深的 RAG（检索增强生成）系统优化专家，正在分析西门子医疗 CT 文档问答系统的 RAGAS 评测结果。
|
||||
请基于以下诊断数据与低分样本，用中文撰写一份**详细、具体、可落地**的优化建议报告（Markdown 格式）。
|
||||
|
||||
## 评测诊断摘要
|
||||
|
||||
{diagnosis_summary}
|
||||
|
||||
## 低分样本示例
|
||||
## 低分样本明细（含检索片段 contexts，用于定位问题出在检索还是生成环节）
|
||||
|
||||
{low_sample_text}
|
||||
|
||||
## 报告要求
|
||||
## 撰写要求
|
||||
|
||||
1. 按指标分节（## 指标名 [严重程度]），先解释"为什么低"（结合低分样本具体分析），再给出"具体怎么改"
|
||||
2. 严重程度说明：critical=严重（<阈值50%），warning=警告（<阈值70%），low=待优化（低于0.85，有提升空间）
|
||||
3. "具体怎么改"要结合低分样本的实际内容，而不只是泛泛建议
|
||||
4. 最后写一节 **## 优先优化次序**，按性价比排序（不增加 LLM 调用次数的优化优先），critical 和 warning 项优先于 low 项
|
||||
5. 语言简洁，面向工程师，不要废话，不要重复列表内容
|
||||
1. **按指标分节**：每个指标一个 `## 指标名 [严重程度]` 小节。
|
||||
2. **每节必须包含「举例拆解」**：从该指标的低分样本中挑 1-2 个最典型的，逐条按如下结构拆解：
|
||||
- **问题**：简述该样本的 question
|
||||
- **当前得分**：该样本在此指标上的分数
|
||||
- **为什么低**：结合该样本的 answer / 检索片段 contexts / 标准答案 ground_truth，**具体指出**问题所在
|
||||
（例如：答案里哪句话没有被检索片段支持；检索片段里缺了哪个关键信息；答案偏离了问题的哪个点）
|
||||
- **拆解定位**：判断问题出在「检索环节」「生成环节」还是「两者兼有」
|
||||
- **优化动作**：针对这个具体样本，给出 1-3 条**可操作**的改法，不要泛泛而谈
|
||||
3. **跨指标关联分析**：若多个指标同时偏低，分析其因果关系。例如：
|
||||
- faithfulness 低且 context_recall 低 → 多半是检索缺失关键信息，导致模型臆造（幻觉）
|
||||
- context_recall 正常但 context_precision 低 → 检索引入噪声，稀释了有效信息
|
||||
- faithfulness 低但 context_recall 高 → 生成环节 grounding 不足，需收紧生成 prompt
|
||||
4. 最后写一节 `## 优先优化次序`：按性价比排序（不增加 LLM 调用次数的优化优先；critical/warning 优先于 low）。
|
||||
5. 语言简洁，面向工程师，重点是「具体、可操作」。不要复述本提示词，不要无意义的客套。
|
||||
|
||||
只输出 Markdown 报告正文，不要任何前置说明。
|
||||
严重程度说明：critical=严重（远低于阈值），warning=警告（低于阈值），low=待优化（达标但低于 0.85，仍有提升空间）。
|
||||
|
||||
只输出 Markdown 报告正文，不要任何前置说明或代码块包裹。
|
||||
"""
|
||||
|
||||
|
||||
@@ -38,8 +59,15 @@ _SEVERITY_LABEL_ZH: dict[str, str] = {
|
||||
"low": "待优化",
|
||||
}
|
||||
|
||||
# Per-sample text limits keep the prompt bounded regardless of sample size.
|
||||
_ANSWER_LIMIT = 400
|
||||
_GT_LIMIT = 300
|
||||
_CONTEXT_LIMIT = 600
|
||||
_CONTEXT_SEPARATOR = " |||| "
|
||||
|
||||
|
||||
def _build_diagnosis_summary(diagnoses: list[Diagnosis]) -> str:
|
||||
"""Render the per-metric diagnosis block fed to the LLM."""
|
||||
lines = []
|
||||
for d in diagnoses:
|
||||
direction = "（越低越好）" if d.metric == "noise_sensitivity" else ""
|
||||
@@ -53,7 +81,22 @@ def _build_diagnosis_summary(diagnoses: list[Diagnosis]) -> str:
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _format_contexts(raw: Any) -> str:
|
||||
"""Render a sample's contexts (stored as a joined string) as a short list."""
|
||||
text = str(raw or "").strip()
|
||||
if not text:
|
||||
return "（无检索片段）"
|
||||
parts = [p.strip() for p in text.split(_CONTEXT_SEPARATOR) if p.strip()]
|
||||
if not parts:
|
||||
return "（无检索片段）"
|
||||
rendered = "; ".join(f"[{i + 1}] {p}" for i, p in enumerate(parts))
|
||||
if len(rendered) > _CONTEXT_LIMIT:
|
||||
rendered = rendered[:_CONTEXT_LIMIT] + "…"
|
||||
return rendered
|
||||
|
||||
|
||||
def _build_low_sample_text(diagnoses: list[Diagnosis]) -> str:
|
||||
"""Render low-score samples (now including contexts) for grounding analysis."""
|
||||
lines = []
|
||||
for d in diagnoses:
|
||||
if not d.low_samples:
|
||||
@@ -61,24 +104,63 @@ def _build_low_sample_text(diagnoses: list[Diagnosis]) -> str:
|
||||
lines.append(f"### {d.metric} 低分样本（最多 3 条）")
|
||||
for i, s in enumerate(d.low_samples, 1):
|
||||
score = s.get(d.metric, "N/A")
|
||||
lines.append(f"\n**样本 {i}**（分数={score}）")
|
||||
lines.append(f"- 问题：{s.get('question', '')}")
|
||||
lines.append(f"- 回答：{s.get('answer', '')[:300]}")
|
||||
lines.append(f"- 标准答案：{s.get('ground_truth', '')[:200]}")
|
||||
lines.append(f"\n**样本 {i}**（{d.metric}={score}）")
|
||||
lines.append(f"- 问题 question：{s.get('question', '')}")
|
||||
lines.append(f"- 生成答案 answer：{str(s.get('answer', ''))[:_ANSWER_LIMIT]}")
|
||||
lines.append(f"- 检索片段 contexts：{_format_contexts(s.get('contexts'))}")
|
||||
lines.append(f"- 标准答案 ground_truth：{str(s.get('ground_truth', ''))[:_GT_LIMIT]}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _is_reasoning_model(model: str) -> bool:
|
||||
"""Return True for OpenAI reasoning models (gpt-5+/o-series/codex-mini).
|
||||
|
||||
These require `max_completion_tokens` instead of `max_tokens` and do not
|
||||
accept a custom temperature. Mirrors RAGAS's own detection logic so advice
|
||||
generation stays consistent with the scoring path.
|
||||
"""
|
||||
m = (model or "").lower()
|
||||
# O-series: o1..o9 optionally followed by - or _
|
||||
if len(m) >= 2 and m[0] == "o" and m[1] in "123456789":
|
||||
if len(m) == 2 or m[2] in ("-", "_"):
|
||||
return True
|
||||
# GPT-5 through GPT-19 generation
|
||||
if m.startswith("gpt-"):
|
||||
version_str = m[4:].split("-")[0].split("_")[0].split(".")[0]
|
||||
try:
|
||||
if 5 <= int(version_str) <= 19:
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
if m == "codex-mini":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _chat_token_kwargs(model: str, max_tokens: int) -> dict[str, Any]:
|
||||
"""Return the completion-budget kwargs appropriate for the model family."""
|
||||
if _is_reasoning_model(model):
|
||||
# Reasoning models: only max_completion_tokens; temperature must stay default (1).
|
||||
return {"max_completion_tokens": max_tokens}
|
||||
return {"max_tokens": max_tokens, "temperature": 0.2}
|
||||
|
||||
|
||||
async def analyze(
|
||||
diagnoses: list[Diagnosis],
|
||||
llm: Any,
|
||||
scenario_name: str,
|
||||
judge_model: str,
|
||||
settings: Any,
|
||||
*,
|
||||
chat_client: Any | None = None,
|
||||
) -> str:
|
||||
"""Call the judge LLM to generate a Chinese optimization report.
|
||||
"""Call the judge LLM directly to generate a Chinese optimization report.
|
||||
|
||||
Args:
|
||||
diagnoses: Non-empty list of Diagnosis from rules.diagnose().
|
||||
llm: RAGAS LLM wrapper (has .agenerate() method).
|
||||
scenario_name: Used only for logging.
|
||||
judge_model: Model name to call (also selects the matching LLM profile).
|
||||
settings: EvaluationSettings (supplies client kwargs + token budget).
|
||||
chat_client: Optional pre-built AsyncOpenAI-compatible client (for tests).
|
||||
|
||||
Returns:
|
||||
LLM-generated Markdown string, or "" on failure (triggers writer fallback).
|
||||
@@ -86,22 +168,47 @@ async def analyze(
|
||||
if not diagnoses:
|
||||
return ""
|
||||
|
||||
diagnosis_summary = _build_diagnosis_summary(diagnoses)
|
||||
low_sample_text = _build_low_sample_text(diagnoses)
|
||||
prompt = _PROMPT_TEMPLATE.format(
|
||||
diagnosis_summary=diagnosis_summary,
|
||||
low_sample_text=low_sample_text,
|
||||
diagnosis_summary=_build_diagnosis_summary(diagnoses),
|
||||
low_sample_text=_build_low_sample_text(diagnoses),
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info("[advisor] calling LLM for optimization analysis scenario=%s", scenario_name)
|
||||
from langchain_core.messages import HumanMessage
|
||||
# Use the underlying langchain chat model directly (RAGAS LangchainLLMWrapper wraps BaseChatModel)
|
||||
response = await llm.langchain_llm.ainvoke([HumanMessage(content=prompt)])
|
||||
text = response.content.strip()
|
||||
logger.info("[advisor] LLM analysis complete chars=%d", len(text))
|
||||
return text
|
||||
except Exception as exc:
|
||||
client = chat_client
|
||||
owns_client = False
|
||||
if client is None:
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from rag_eval.metrics.factory import resolve_openai_client_kwargs
|
||||
|
||||
client = AsyncOpenAI(**resolve_openai_client_kwargs(judge_model, settings))
|
||||
owns_client = True
|
||||
|
||||
try:
|
||||
# Advice is a longer document than per-metric scoring; give it headroom.
|
||||
max_tokens = max(2048, int(getattr(settings, "ragas_llm_max_tokens", 4096) or 4096))
|
||||
token_kwargs = _chat_token_kwargs(judge_model, max_tokens)
|
||||
|
||||
response = await client.chat.completions.create(
|
||||
model=judge_model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
**token_kwargs,
|
||||
)
|
||||
text = (response.choices[0].message.content or "").strip()
|
||||
logger.info("[advisor] LLM analysis complete chars=%d", len(text))
|
||||
return text
|
||||
finally:
|
||||
# Close the httpx connection pool inside THIS event loop. run_advisor
|
||||
# drives analyze() via asyncio.run(), which closes the loop on return;
|
||||
# a later GC-time aclose() on the dead loop would otherwise log
|
||||
# "RuntimeError: Event loop is closed". Only close clients we created.
|
||||
if owns_client:
|
||||
try:
|
||||
await client.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"[advisor] LLM analysis failed (%s: %s) — falling back to rule report",
|
||||
type(exc).__name__, exc,
|
||||
|
||||
@@ -159,7 +159,8 @@ def _select_low_samples(
|
||||
valid = [r for r in rows if metric in r and not math.isnan(float(r[metric]))]
|
||||
sorted_rows = sorted(valid, key=lambda r: float(r[metric]), reverse=not higher_is_better)
|
||||
worst = sorted_rows[:top_n]
|
||||
keep_keys = {"sample_id", "question", "answer", "ground_truth", metric}
|
||||
# contexts is included so the LLM/fallback can judge grounding (retrieval vs generation).
|
||||
keep_keys = {"sample_id", "question", "answer", "ground_truth", "contexts", metric}
|
||||
return [{k: v for k, v in row.items() if k in keep_keys} for row in worst]
|
||||
|
||||
|
||||
|
||||
@@ -29,10 +29,15 @@ def _format_log_summary(diagnoses: list[Diagnosis], advice_path: Path) -> str:
|
||||
|
||||
|
||||
def _build_fallback_report(diagnoses: list[Diagnosis]) -> str:
|
||||
"""Build a rules-only report when LLM analysis is unavailable."""
|
||||
"""Build a rules-only report when LLM analysis is unavailable.
|
||||
|
||||
Even without the LLM, embed each metric's worst sample(s) — question,
|
||||
answer, ground truth — so the advice still references concrete problems
|
||||
instead of reading as a purely generic template.
|
||||
"""
|
||||
if not diagnoses:
|
||||
return ""
|
||||
lines = ["## 规则诊断（LLM 分析不可用）\n"]
|
||||
lines = ["## 规则诊断（LLM 分析不可用，以下为规则引擎输出）\n"]
|
||||
for d in diagnoses:
|
||||
label = _SEVERITY_LABEL.get(d.severity, d.severity)
|
||||
lines.append(f"### {d.metric} [{label}] 均值={d.mean_score:.4f}")
|
||||
@@ -42,6 +47,22 @@ def _build_fallback_report(diagnoses: list[Diagnosis]) -> str:
|
||||
lines.append("\n**建议动作：**")
|
||||
for action in d.suggested_actions:
|
||||
lines.append(f"- {action}")
|
||||
if d.low_samples:
|
||||
lines.append("\n**低分样本举例拆解：**")
|
||||
for i, sample in enumerate(d.low_samples, 1):
|
||||
score = sample.get(d.metric, "N/A")
|
||||
question = str(sample.get("question", "")).strip() or "（无问题文本）"
|
||||
lines.append(f"\n- **样本 {i}**（{d.metric}={score}）问题：{question}")
|
||||
answer = str(sample.get("answer", "")).strip()
|
||||
if answer:
|
||||
lines.append(f" - 生成答案：{answer[:200]}")
|
||||
ground_truth = str(sample.get("ground_truth", "")).strip()
|
||||
if ground_truth:
|
||||
lines.append(f" - 标准答案：{ground_truth[:160]}")
|
||||
lines.append(
|
||||
f" - 拆解：该样本 {d.metric} 偏低，请对照上述「可能原因 / 建议动作」"
|
||||
f"重点排查本问题的检索片段与生成答案。"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@@ -79,6 +79,6 @@ def run_scenario(
|
||||
logger.info("[runner] artifacts written for run_id=%s", result.run_id)
|
||||
|
||||
# Optimization advisor — runs only if scenario.optimization_advisor is True.
|
||||
run_advisor(result, scenario, llm)
|
||||
run_advisor(result, scenario, settings=settings)
|
||||
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user