"""Offline smoke-check for the advisor module wiring (no network required).""" import math import sys import tempfile from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent)) from rag_eval.advisor.rules import diagnose from rag_eval.advisor.writer import write_advice, _format_log_summary # Simulate score_rows with low faithfulness and high noise_sensitivity rows = [ { "sample_id": f"s{i}", "question": f"问题{i}:西门子CT扫描的Flash技术原理是什么?", "answer": f"答案{i}:Flash技术采用双源CT扫描", "ground_truth": f"标准答案{i}:Flash扫描利用双源CT和大螺距实现超低辐射剂量扫描", "faithfulness": 0.3 + i * 0.05, "noise_sensitivity": 0.4 + i * 0.02, "context_recall": 0.75, "semantic_similarity": 0.65, } for i in range(5) ] diags = diagnose(rows, metrics=["faithfulness", "noise_sensitivity", "context_recall", "semantic_similarity"]) print(f"Diagnosed {len(diags)} metric(s):") for d in diags: print(f" {d.metric}: mean={d.mean_score}, severity={d.severity}, low_samples={len(d.low_samples)}") assert len(diags) >= 2, f"Expected at least 2 diagnoses, got {len(diags)}" metrics_hit = {d.metric for d in diags} assert "faithfulness" in metrics_hit, "faithfulness should be triggered" assert "noise_sensitivity" in metrics_hit, "noise_sensitivity should be triggered" with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "optimization_advice.md" write_advice( diagnoses=diags, llm_markdown="", # fallback mode (no LLM) advice_path=path, scenario_name="smoke-test-siemens", run_id="2026-06-16T00-00-00", judge_model="deepseek-v4-flash", ) content = path.read_text(encoding="utf-8") assert "smoke-test-siemens" in content, "scenario name missing from report" assert "faithfulness" in content, "faithfulness missing from report" assert "noise_sensitivity" in content, "noise_sensitivity missing from report" print(f"\nAdvice file ({len(content)} chars) — assertions OK") # Verify log summary format summary = _format_log_summary(diags, Path("optimization_advice.md")) print(f"\nLog summary length: {len(summary)} chars, faithfulness present: {'faithfulness' in summary}") assert "触发诊断" in summary assert "faithfulness" in summary print("\nSmoke check PASSED")