wangwei
f5c2dce64a
- rag_eval/advisor/: new package with rules engine, LLM analyzer, writer - rules.py: 7-metric diagnostic rules (warning/critical thresholds, top-3 low samples) - llm_analyzer.py: Chinese optimization report via judge_model, graceful fallback - writer.py: writes optimization_advice.md + log summary - __init__.py: run_advisor() entry point (no-op when optimization_advisor=False) - Scenario.optimization_advisor: new bool field (default False) - ScenarioModel: same field added, loader.py透传 - RunArtifactPaths.advice_md: new path field - factory.py: build_models() now public; build_metric_pipeline() accepts pre-built llm/embeddings - runner.py: lifts llm, passes to pipeline and advisor; calls run_advisor() at end - siemens online YAML: optimization_advisor: true enabled - tests: 9 rules tests + 6 writer tests, all pass - docs: advisor section added to engine-flow.md and architecture.md Co-Authored-By: Claude <noreply@anthropic.com>
83 lines
2.8 KiB
Python
83 lines
2.8 KiB
Python
"""Write optimization advice to markdown file and emit log summary."""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from pathlib import Path
|
||
|
||
from .rules import Diagnosis
|
||
|
||
logger = logging.getLogger("rag_eval.advisor")
|
||
|
||
|
||
def _format_log_summary(diagnoses: list[Diagnosis], advice_path: Path) -> str:
|
||
"""Return a single-line log summary of triggered diagnoses."""
|
||
if not diagnoses:
|
||
return "[advisor] 所有指标正常，无需优化建议。"
|
||
parts = [f"{d.metric}({d.mean_score:.2f}, {d.severity})" for d in diagnoses]
|
||
triggered = " ".join(parts)
|
||
return f"[advisor] 触发诊断 {len(diagnoses)} 项: {triggered} → {advice_path}"
|
||
|
||
|
||
def _build_fallback_report(diagnoses: list[Diagnosis]) -> str:
|
||
"""Build a rules-only report when LLM analysis is unavailable."""
|
||
if not diagnoses:
|
||
return ""
|
||
lines = ["## 规则诊断（LLM 分析不可用）\n"]
|
||
for d in diagnoses:
|
||
lines.append(f"### {d.metric} [{d.severity}] 均值={d.mean_score:.4f}")
|
||
lines.append("\n**可能原因：**")
|
||
for cause in d.root_causes:
|
||
lines.append(f"- {cause}")
|
||
lines.append("\n**建议动作：**")
|
||
for action in d.suggested_actions:
|
||
lines.append(f"- {action}")
|
||
lines.append("")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def write_advice(
|
||
diagnoses: list[Diagnosis],
|
||
llm_markdown: str,
|
||
advice_path: Path,
|
||
scenario_name: str,
|
||
run_id: str,
|
||
judge_model: str,
|
||
) -> None:
|
||
"""Write optimization_advice.md and emit a log summary line.
|
||
|
||
Args:
|
||
diagnoses: List of Diagnosis from rules.diagnose().
|
||
llm_markdown: LLM-generated Markdown body. Empty string triggers fallback.
|
||
advice_path: Full path to write the .md file.
|
||
scenario_name: Human-readable scenario identifier for the report header.
|
||
run_id: Run identifier string.
|
||
judge_model: Model used for LLM analysis (shown in header).
|
||
"""
|
||
advice_path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
||
from rag_eval.shared.utils import utc_now_iso
|
||
header_lines = [
|
||
f"# 优化建议报告 — {scenario_name}",
|
||
"",
|
||
f"- run_id: `{run_id}`",
|
||
f"- 生成时间: `{utc_now_iso()}`",
|
||
f"- judge_model: `{judge_model}`",
|
||
"",
|
||
"---",
|
||
"",
|
||
]
|
||
|
||
if not diagnoses:
|
||
body = "## ✅ 未发现明显指标异常\n\n所有指标均在正常范围内，当前 RAG 链路表现良好。\n"
|
||
elif llm_markdown:
|
||
body = llm_markdown
|
||
else:
|
||
body = _build_fallback_report(diagnoses)
|
||
|
||
content = "\n".join(header_lines) + body
|
||
advice_path.write_text(content, encoding="utf-8")
|
||
|
||
summary = _format_log_summary(diagnoses, advice_path)
|
||
logger.info(summary)
|
||
logger.info("[advisor] 优化建议已写出: %s", advice_path)
|