feat(advisor): add optimization advisor module
- rag_eval/advisor/: new package with rules engine, LLM analyzer, writer - rules.py: 7-metric diagnostic rules (warning/critical thresholds, top-3 low samples) - llm_analyzer.py: Chinese optimization report via judge_model, graceful fallback - writer.py: writes optimization_advice.md + log summary - __init__.py: run_advisor() entry point (no-op when optimization_advisor=False) - Scenario.optimization_advisor: new bool field (default False) - ScenarioModel: same field added, loader.py透传 - RunArtifactPaths.advice_md: new path field - factory.py: build_models() now public; build_metric_pipeline() accepts pre-built llm/embeddings - runner.py: lifts llm, passes to pipeline and advisor; calls run_advisor() at end - siemens online YAML: optimization_advisor: true enabled - tests: 9 rules tests + 6 writer tests, all pass - docs: advisor section added to engine-flow.md and architecture.md Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
82
rag_eval/advisor/writer.py
Normal file
82
rag_eval/advisor/writer.py
Normal file
@@ -0,0 +1,82 @@
|
||||
"""Write optimization advice to markdown file and emit log summary."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from .rules import Diagnosis
|
||||
|
||||
logger = logging.getLogger("rag_eval.advisor")
|
||||
|
||||
|
||||
def _format_log_summary(diagnoses: list[Diagnosis], advice_path: Path) -> str:
|
||||
"""Return a single-line log summary of triggered diagnoses."""
|
||||
if not diagnoses:
|
||||
return "[advisor] 所有指标正常,无需优化建议。"
|
||||
parts = [f"{d.metric}({d.mean_score:.2f}, {d.severity})" for d in diagnoses]
|
||||
triggered = " ".join(parts)
|
||||
return f"[advisor] 触发诊断 {len(diagnoses)} 项: {triggered} → {advice_path}"
|
||||
|
||||
|
||||
def _build_fallback_report(diagnoses: list[Diagnosis]) -> str:
|
||||
"""Build a rules-only report when LLM analysis is unavailable."""
|
||||
if not diagnoses:
|
||||
return ""
|
||||
lines = ["## 规则诊断(LLM 分析不可用)\n"]
|
||||
for d in diagnoses:
|
||||
lines.append(f"### {d.metric} [{d.severity}] 均值={d.mean_score:.4f}")
|
||||
lines.append("\n**可能原因:**")
|
||||
for cause in d.root_causes:
|
||||
lines.append(f"- {cause}")
|
||||
lines.append("\n**建议动作:**")
|
||||
for action in d.suggested_actions:
|
||||
lines.append(f"- {action}")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write_advice(
|
||||
diagnoses: list[Diagnosis],
|
||||
llm_markdown: str,
|
||||
advice_path: Path,
|
||||
scenario_name: str,
|
||||
run_id: str,
|
||||
judge_model: str,
|
||||
) -> None:
|
||||
"""Write optimization_advice.md and emit a log summary line.
|
||||
|
||||
Args:
|
||||
diagnoses: List of Diagnosis from rules.diagnose().
|
||||
llm_markdown: LLM-generated Markdown body. Empty string triggers fallback.
|
||||
advice_path: Full path to write the .md file.
|
||||
scenario_name: Human-readable scenario identifier for the report header.
|
||||
run_id: Run identifier string.
|
||||
judge_model: Model used for LLM analysis (shown in header).
|
||||
"""
|
||||
advice_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
from rag_eval.shared.utils import utc_now_iso
|
||||
header_lines = [
|
||||
f"# 优化建议报告 — {scenario_name}",
|
||||
"",
|
||||
f"- run_id: `{run_id}`",
|
||||
f"- 生成时间: `{utc_now_iso()}`",
|
||||
f"- judge_model: `{judge_model}`",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
|
||||
if not diagnoses:
|
||||
body = "## ✅ 未发现明显指标异常\n\n所有指标均在正常范围内,当前 RAG 链路表现良好。\n"
|
||||
elif llm_markdown:
|
||||
body = llm_markdown
|
||||
else:
|
||||
body = _build_fallback_report(diagnoses)
|
||||
|
||||
content = "\n".join(header_lines) + body
|
||||
advice_path.write_text(content, encoding="utf-8")
|
||||
|
||||
summary = _format_log_summary(diagnoses, advice_path)
|
||||
logger.info(summary)
|
||||
logger.info("[advisor] 优化建议已写出: %s", advice_path)
|
||||
Reference in New Issue
Block a user