update for ragas

rag_eval/advisor/__init__.py

@@ -20,7 +20,10 @@ __all__ = ["run_advisor", "Diagnosis", "diagnose"]
 def run_advisor(
     result: EvaluationResult,
     scenario: Scenario,
-    llm: Any,
+    llm: Any = None,
+    *,
+    settings: Any | None = None,
+    chat_client: Any | None = None,
 ) -> None:
     """Run the full optimization advisor pipeline after an evaluation completes.
 
@@ -30,7 +33,10 @@ def run_advisor(
     Args:
         result: Completed EvaluationResult from Evaluator.evaluate().
         scenario: The resolved Scenario (provides metrics, judge_model, output_dir).
-        llm: Pre-built RAGAS LLM instance (from build_models()) for LLM analysis.
+        llm: Deprecated/unused — kept for backward-compatible call sites. The
+            advisor now issues its own direct LLM call resolved from judge_model.
+        settings: Optional EvaluationSettings; defaults to EvaluationSettings().
+        chat_client: Optional pre-built chat client (used by tests to avoid network).
     """
     if not scenario.optimization_advisor:
         return
@@ -38,6 +44,10 @@ def run_advisor(
     logger.info("[advisor] starting optimization analysis  scenario=%s", scenario.scenario_name)
 
     try:
+        if settings is None:
+            from rag_eval.settings import EvaluationSettings
+            settings = EvaluationSettings()
+
         artifact_paths = build_artifact_paths(scenario.output_dir, result.run_id)
         if artifact_paths.advice_md is None:
             logger.warning("[advisor] advice_md path not set in RunArtifactPaths — skipping")
@@ -47,7 +57,15 @@ def run_advisor(
         logger.info("[advisor] rule diagnosis complete: %d metric(s) triggered", len(diagnoses))
 
         if diagnoses:
-            llm_markdown = asyncio.run(analyze(diagnoses, llm, scenario.scenario_name))
+            llm_markdown = asyncio.run(
+                analyze(
+                    diagnoses,
+                    scenario.scenario_name,
+                    scenario.judge_model,
+                    settings,
+                    chat_client=chat_client,
+                )
+            )
         else:
             llm_markdown = ""
 

rag_eval/advisor/llm_analyzer.py

@@ -1,4 +1,11 @@
-"""LLM-powered analysis of rule diagnostics and low-score samples."""
+"""LLM-powered analysis of rule diagnostics and low-score samples.
+
+The analyzer issues a single, direct OpenAI-compatible chat call (no langchain,
+no instructor structured output) to produce a detailed Chinese optimization
+report. It deliberately uses plain text generation so it works with any
+OpenAI-compatible gateway and never trips the structured-output response-shape
+issues seen with the scoring path.
+"""
 from __future__ import annotations
 
 import logging
@@ -8,27 +15,41 @@ from .rules import Diagnosis
 
 logger = logging.getLogger("rag_eval.advisor")
 
+# Worked-example oriented prompt: forces per-question decomposition (为什么低 →
+# 拆解 → 怎么改) plus cross-metric causal reasoning, so the report is concrete
+# instead of a generic restatement of the static rule templates.
 _PROMPT_TEMPLATE = """\
-你是一个 RAG 系统优化专家，正在分析西门子医疗 CT 文档问答系统的评测结果。
-请用中文撰写一份优化建议报告，格式为 Markdown。
+你是一位资深的 RAG（检索增强生成）系统优化专家，正在分析西门子医疗 CT 文档问答系统的 RAGAS 评测结果。
+请基于以下诊断数据与低分样本，用中文撰写一份**详细、具体、可落地**的优化建议报告（Markdown 格式）。
 
 ## 评测诊断摘要
 
 {diagnosis_summary}
 
-## 低分样本示例
+## 低分样本明细（含检索片段 contexts，用于定位问题出在检索还是生成环节）
 
 {low_sample_text}
 
-## 报告要求
+## 撰写要求
 
-1. 按指标分节（## 指标名  [严重程度]），先解释"为什么低"（结合低分样本具体分析），再给出"具体怎么改"
-2. 严重程度说明：critical=严重（<阈值50%），warning=警告（<阈值70%），low=待优化（低于0.85，有提升空间）
-3. "具体怎么改"要结合低分样本的实际内容，而不只是泛泛建议
-4. 最后写一节 **## 优先优化次序**，按性价比排序（不增加 LLM 调用次数的优化优先），critical 和 warning 项优先于 low 项
-5. 语言简洁，面向工程师，不要废话，不要重复列表内容
+1. **按指标分节**：每个指标一个 `## 指标名 [严重程度]` 小节。
+2. **每节必须包含「举例拆解」**：从该指标的低分样本中挑 1-2 个最典型的，逐条按如下结构拆解：
+   - **问题**：简述该样本的 question
+   - **当前得分**：该样本在此指标上的分数
+   - **为什么低**：结合该样本的 answer / 检索片段 contexts / 标准答案 ground_truth，**具体指出**问题所在
+     （例如：答案里哪句话没有被检索片段支持；检索片段里缺了哪个关键信息；答案偏离了问题的哪个点）
+   - **拆解定位**：判断问题出在「检索环节」「生成环节」还是「两者兼有」
+   - **优化动作**：针对这个具体样本，给出 1-3 条**可操作**的改法，不要泛泛而谈
+3. **跨指标关联分析**：若多个指标同时偏低，分析其因果关系。例如：
+   - faithfulness 低且 context_recall 低 → 多半是检索缺失关键信息，导致模型臆造（幻觉）
+   - context_recall 正常但 context_precision 低 → 检索引入噪声，稀释了有效信息
+   - faithfulness 低但 context_recall 高 → 生成环节 grounding 不足，需收紧生成 prompt
+4. 最后写一节 `## 优先优化次序`：按性价比排序（不增加 LLM 调用次数的优化优先；critical/warning 优先于 low）。
+5. 语言简洁，面向工程师，重点是「具体、可操作」。不要复述本提示词，不要无意义的客套。
 
-只输出 Markdown 报告正文，不要任何前置说明。
+严重程度说明：critical=严重（远低于阈值），warning=警告（低于阈值），low=待优化（达标但低于 0.85，仍有提升空间）。
+
+只输出 Markdown 报告正文，不要任何前置说明或代码块包裹。
 """
 
 
@@ -38,8 +59,15 @@ _SEVERITY_LABEL_ZH: dict[str, str] = {
     "low": "待优化",
 }
 
+# Per-sample text limits keep the prompt bounded regardless of sample size.
+_ANSWER_LIMIT = 400
+_GT_LIMIT = 300
+_CONTEXT_LIMIT = 600
+_CONTEXT_SEPARATOR = " |||| "
+
 
 def _build_diagnosis_summary(diagnoses: list[Diagnosis]) -> str:
+    """Render the per-metric diagnosis block fed to the LLM."""
     lines = []
     for d in diagnoses:
         direction = "（越低越好）" if d.metric == "noise_sensitivity" else ""
@@ -53,7 +81,22 @@ def _build_diagnosis_summary(diagnoses: list[Diagnosis]) -> str:
     return "\n".join(lines)
 
 
+def _format_contexts(raw: Any) -> str:
+    """Render a sample's contexts (stored as a joined string) as a short list."""
+    text = str(raw or "").strip()
+    if not text:
+        return "（无检索片段）"
+    parts = [p.strip() for p in text.split(_CONTEXT_SEPARATOR) if p.strip()]
+    if not parts:
+        return "（无检索片段）"
+    rendered = "; ".join(f"[{i + 1}] {p}" for i, p in enumerate(parts))
+    if len(rendered) > _CONTEXT_LIMIT:
+        rendered = rendered[:_CONTEXT_LIMIT] + "…"
+    return rendered
+
+
 def _build_low_sample_text(diagnoses: list[Diagnosis]) -> str:
+    """Render low-score samples (now including contexts) for grounding analysis."""
     lines = []
     for d in diagnoses:
         if not d.low_samples:
@@ -61,24 +104,63 @@ def _build_low_sample_text(diagnoses: list[Diagnosis]) -> str:
         lines.append(f"### {d.metric} 低分样本（最多 3 条）")
         for i, s in enumerate(d.low_samples, 1):
             score = s.get(d.metric, "N/A")
-            lines.append(f"\n**样本 {i}**（分数={score}）")
-            lines.append(f"- 问题：{s.get('question', '')}")
-            lines.append(f"- 回答：{s.get('answer', '')[:300]}")
-            lines.append(f"- 标准答案：{s.get('ground_truth', '')[:200]}")
+            lines.append(f"\n**样本 {i}**（{d.metric}={score}）")
+            lines.append(f"- 问题 question：{s.get('question', '')}")
+            lines.append(f"- 生成答案 answer：{str(s.get('answer', ''))[:_ANSWER_LIMIT]}")
+            lines.append(f"- 检索片段 contexts：{_format_contexts(s.get('contexts'))}")
+            lines.append(f"- 标准答案 ground_truth：{str(s.get('ground_truth', ''))[:_GT_LIMIT]}")
     return "\n".join(lines)
 
 
+def _is_reasoning_model(model: str) -> bool:
+    """Return True for OpenAI reasoning models (gpt-5+/o-series/codex-mini).
+
+    These require `max_completion_tokens` instead of `max_tokens` and do not
+    accept a custom temperature. Mirrors RAGAS's own detection logic so advice
+    generation stays consistent with the scoring path.
+    """
+    m = (model or "").lower()
+    # O-series: o1..o9 optionally followed by - or _
+    if len(m) >= 2 and m[0] == "o" and m[1] in "123456789":
+        if len(m) == 2 or m[2] in ("-", "_"):
+            return True
+    # GPT-5 through GPT-19 generation
+    if m.startswith("gpt-"):
+        version_str = m[4:].split("-")[0].split("_")[0].split(".")[0]
+        try:
+            if 5 <= int(version_str) <= 19:
+                return True
+        except ValueError:
+            pass
+    if m == "codex-mini":
+        return True
+    return False
+
+
+def _chat_token_kwargs(model: str, max_tokens: int) -> dict[str, Any]:
+    """Return the completion-budget kwargs appropriate for the model family."""
+    if _is_reasoning_model(model):
+        # Reasoning models: only max_completion_tokens; temperature must stay default (1).
+        return {"max_completion_tokens": max_tokens}
+    return {"max_tokens": max_tokens, "temperature": 0.2}
+
+
 async def analyze(
     diagnoses: list[Diagnosis],
-    llm: Any,
     scenario_name: str,
+    judge_model: str,
+    settings: Any,
+    *,
+    chat_client: Any | None = None,
 ) -> str:
-    """Call the judge LLM to generate a Chinese optimization report.
+    """Call the judge LLM directly to generate a Chinese optimization report.
 
     Args:
         diagnoses: Non-empty list of Diagnosis from rules.diagnose().
-        llm: RAGAS LLM wrapper (has .agenerate() method).
         scenario_name: Used only for logging.
+        judge_model: Model name to call (also selects the matching LLM profile).
+        settings: EvaluationSettings (supplies client kwargs + token budget).
+        chat_client: Optional pre-built AsyncOpenAI-compatible client (for tests).
 
     Returns:
         LLM-generated Markdown string, or "" on failure (triggers writer fallback).
@@ -86,22 +168,47 @@ async def analyze(
     if not diagnoses:
         return ""
 
-    diagnosis_summary = _build_diagnosis_summary(diagnoses)
-    low_sample_text = _build_low_sample_text(diagnoses)
     prompt = _PROMPT_TEMPLATE.format(
-        diagnosis_summary=diagnosis_summary,
-        low_sample_text=low_sample_text,
+        diagnosis_summary=_build_diagnosis_summary(diagnoses),
+        low_sample_text=_build_low_sample_text(diagnoses),
     )
 
     try:
         logger.info("[advisor] calling LLM for optimization analysis  scenario=%s", scenario_name)
-        from langchain_core.messages import HumanMessage
-        # Use the underlying langchain chat model directly (RAGAS LangchainLLMWrapper wraps BaseChatModel)
-        response = await llm.langchain_llm.ainvoke([HumanMessage(content=prompt)])
-        text = response.content.strip()
-        logger.info("[advisor] LLM analysis complete  chars=%d", len(text))
-        return text
-    except Exception as exc:
+        client = chat_client
+        owns_client = False
+        if client is None:
+            from openai import AsyncOpenAI
+
+            from rag_eval.metrics.factory import resolve_openai_client_kwargs
+
+            client = AsyncOpenAI(**resolve_openai_client_kwargs(judge_model, settings))
+            owns_client = True
+
+        try:
+            # Advice is a longer document than per-metric scoring; give it headroom.
+            max_tokens = max(2048, int(getattr(settings, "ragas_llm_max_tokens", 4096) or 4096))
+            token_kwargs = _chat_token_kwargs(judge_model, max_tokens)
+
+            response = await client.chat.completions.create(
+                model=judge_model,
+                messages=[{"role": "user", "content": prompt}],
+                **token_kwargs,
+            )
+            text = (response.choices[0].message.content or "").strip()
+            logger.info("[advisor] LLM analysis complete  chars=%d", len(text))
+            return text
+        finally:
+            # Close the httpx connection pool inside THIS event loop. run_advisor
+            # drives analyze() via asyncio.run(), which closes the loop on return;
+            # a later GC-time aclose() on the dead loop would otherwise log
+            # "RuntimeError: Event loop is closed". Only close clients we created.
+            if owns_client:
+                try:
+                    await client.close()
+                except Exception:  # noqa: BLE001
+                    pass
+    except Exception as exc:  # noqa: BLE001
         logger.warning(
             "[advisor] LLM analysis failed (%s: %s) — falling back to rule report",
             type(exc).__name__, exc,

rag_eval/advisor/rules.py

@@ -159,7 +159,8 @@ def _select_low_samples(
     valid = [r for r in rows if metric in r and not math.isnan(float(r[metric]))]
     sorted_rows = sorted(valid, key=lambda r: float(r[metric]), reverse=not higher_is_better)
     worst = sorted_rows[:top_n]
-    keep_keys = {"sample_id", "question", "answer", "ground_truth", metric}
+    # contexts is included so the LLM/fallback can judge grounding (retrieval vs generation).
+    keep_keys = {"sample_id", "question", "answer", "ground_truth", "contexts", metric}
     return [{k: v for k, v in row.items() if k in keep_keys} for row in worst]
 
 

rag_eval/advisor/writer.py

@@ -29,10 +29,15 @@ def _format_log_summary(diagnoses: list[Diagnosis], advice_path: Path) -> str:
 
 
 def _build_fallback_report(diagnoses: list[Diagnosis]) -> str:
-    """Build a rules-only report when LLM analysis is unavailable."""
+    """Build a rules-only report when LLM analysis is unavailable.
+
+    Even without the LLM, embed each metric's worst sample(s) — question,
+    answer, ground truth — so the advice still references concrete problems
+    instead of reading as a purely generic template.
+    """
     if not diagnoses:
         return ""
-    lines = ["## 规则诊断（LLM 分析不可用）\n"]
+    lines = ["## 规则诊断（LLM 分析不可用，以下为规则引擎输出）\n"]
     for d in diagnoses:
         label = _SEVERITY_LABEL.get(d.severity, d.severity)
         lines.append(f"### {d.metric}  [{label}]  均值={d.mean_score:.4f}")
@@ -42,6 +47,22 @@ def _build_fallback_report(diagnoses: list[Diagnosis]) -> str:
         lines.append("\n**建议动作：**")
         for action in d.suggested_actions:
             lines.append(f"- {action}")
+        if d.low_samples:
+            lines.append("\n**低分样本举例拆解：**")
+            for i, sample in enumerate(d.low_samples, 1):
+                score = sample.get(d.metric, "N/A")
+                question = str(sample.get("question", "")).strip() or "（无问题文本）"
+                lines.append(f"\n- **样本 {i}**（{d.metric}={score}）问题：{question}")
+                answer = str(sample.get("answer", "")).strip()
+                if answer:
+                    lines.append(f"  - 生成答案：{answer[:200]}")
+                ground_truth = str(sample.get("ground_truth", "")).strip()
+                if ground_truth:
+                    lines.append(f"  - 标准答案：{ground_truth[:160]}")
+                lines.append(
+                    f"  - 拆解：该样本 {d.metric} 偏低，请对照上述「可能原因 / 建议动作」"
+                    f"重点排查本问题的检索片段与生成答案。"
+                )
         lines.append("")
     return "\n".join(lines)
 

rag_eval/execution/runner.py

@@ -79,6 +79,6 @@ def run_scenario(
     logger.info("[runner] artifacts written for run_id=%s", result.run_id)
 
     # Optimization advisor — runs only if scenario.optimization_advisor is True.
-    run_advisor(result, scenario, llm)
+    run_advisor(result, scenario, settings=settings)
 
     return result

tests/test_metric_presenter.py

@@ -40,7 +40,14 @@ const result = {{
   desc: p.describeMetric("faithfulness"),
   noiseDesc: p.describeMetric("noise_sensitivity"),
   noiseBin: p.binColor("noise_sensitivity", 0.0),
-  faithBin: p.binColor("faithfulness", 0.8)
+  faithBin: p.binColor("faithfulness", 0.8),
+  lowerBetterNoise: p.isLowerBetter("noise_sensitivity"),
+  lowerBetterFaith: p.isLowerBetter("faithfulness"),
+  upHigher: p.deltaInfo("faithfulness", 0.80, 0.60),
+  downHigher: p.deltaInfo("faithfulness", 0.60, 0.80),
+  noiseImproved: p.deltaInfo("noise_sensitivity", 0.10, 0.30),
+  noiseWorse: p.deltaInfo("noise_sensitivity", 0.30, 0.10),
+  noBaseline: p.deltaInfo("faithfulness", 0.80, null)
 }};
 console.log(JSON.stringify(result));
 """
@@ -55,6 +62,15 @@ console.log(JSON.stringify(result));
     assert '"noiseDesc":"' in output
     assert '"noiseBin":"#16a34a"' in output
     assert '"faithBin":"#16a34a"' in output
+    assert '"lowerBetterNoise":true' in output
+    assert '"lowerBetterFaith":false' in output
+    # higher-better: rising value is an improvement (green ▲); falling is a regression (red ▼)
+    assert '"upHigher":{"hasData":true,"delta":0.2,"improved":true,"arrow":"▲","magnitude":"0.20","cls":"delta-good"}' in output
+    assert '"downHigher":{"hasData":true,"delta":-0.2,"improved":false,"arrow":"▼","magnitude":"0.20","cls":"delta-bad"}' in output
+    # noise_sensitivity (lower-better): falling value is an improvement (green ▼)
+    assert '"noiseImproved":{"hasData":true,"delta":-0.2,"improved":true,"arrow":"▼","magnitude":"0.20","cls":"delta-good"}' in output
+    assert '"noiseWorse":{"hasData":true,"delta":0.2,"improved":false,"arrow":"▲","magnitude":"0.20","cls":"delta-bad"}' in output
+    assert '"noBaseline":{"hasData":false' in output
 
 
 def test_report_and_index_load_metric_presenter_helper() -> None:
@@ -66,3 +82,6 @@ def test_report_and_index_load_metric_presenter_helper() -> None:
     assert "js/metric_presenter.js" in index_html
     assert "MetricPresenter.describeMetric" in report_js
     assert "MetricPresenter.scoreClass" in app_js
+    # history comparison table uses the direction-aware delta helper
+    assert "MetricPresenter.deltaInfo" in report_js
+    assert "history-table" in report_js

tests/test_webapp_report_builder.py

@@ -4,8 +4,11 @@ from __future__ import annotations
 
 from pathlib import Path
 
+import pandas as pd
 import pytest
 
+from webapp.services import question_history
+from webapp.services import report_builder
 from webapp.services.report_builder import build_report
 from webapp.services.run_reader import _infer_metrics_from_scores, _read_weights_from_snapshot
 
@@ -115,3 +118,71 @@ def test_build_report_ranks_noise_sensitivity_with_lower_values_as_better(tmp_pa
         "s-warn",
         "s-good",
     ]
+
+
+def test_lowest_samples_attaches_history_from_index() -> None:
+    """Surfaced samples are annotated with the same question's prior-run scores."""
+    frame = pd.DataFrame(
+        [
+            {"sample_id": "s1", "question": "  How LONG is the tube? ", "faithfulness": 0.40},
+            {"sample_id": "s2", "question": "unrelated question", "faithfulness": 0.30},
+        ]
+    )
+    history_index = {
+        question_history.normalize_question("How long is the tube?"): [
+            {
+                "run_id": "prev",
+                "scenario_name": "scn",
+                "finished_at": "2026-01-01T00:00:00",
+                "metrics": {"faithfulness": 0.90},
+            }
+        ]
+    }
+
+    samples = report_builder._lowest_samples(frame, ["faithfulness"], history_index)
+    by_id = {s.sample_id: s for s in samples}
+
+    assert len(by_id["s1"].history) == 1
+    assert by_id["s1"].history[0].run_id == "prev"
+    assert by_id["s1"].history[0].metrics["faithfulness"] == 0.90
+    assert by_id["s2"].history == []  # no match → no history
+
+
+def test_build_report_attaches_question_history(tmp_path: Path, monkeypatch) -> None:
+    """build_report wires the question-history index into surfaced samples."""
+    run_dir = tmp_path / "run"
+    run_dir.mkdir(parents=True, exist_ok=True)
+    (run_dir / "scores.csv").write_text(
+        "\n".join(
+            [
+                "sample_id,question,faithfulness",
+                "s1,How long is the tube?,0.40",
+            ]
+        ),
+        encoding="utf-8",
+    )
+    (run_dir / "summary.md").write_text("summary", encoding="utf-8")
+    (run_dir / "optimization_advice.md").write_text("", encoding="utf-8")
+
+    captured: dict = {}
+
+    def _fake_index(exclude_run_id=None, extra_roots=None):
+        captured["exclude_run_id"] = exclude_run_id
+        return {
+            question_history.normalize_question("How long is the tube?"): [
+                {
+                    "run_id": "older",
+                    "scenario_name": "scn",
+                    "finished_at": "2026-01-01T00:00:00",
+                    "metrics": {"faithfulness": 0.95},
+                }
+            ]
+        }
+
+    monkeypatch.setattr(question_history, "build_question_history_index", _fake_index)
+
+    report = build_report(run_dir, ["faithfulness"])
+
+    assert captured["exclude_run_id"] == "run"  # current run excluded from history
+    assert report.lowest_samples[0].history[0].run_id == "older"
+    assert report.lowest_samples[0].history[0].metrics["faithfulness"] == 0.95

webapp/services/report_builder.py

@@ -23,9 +23,10 @@ from webapp.models import (
     DistributionBin,
     GroupStat,
     ReportData,
+    SampleHistoryEntry,
     SampleScore,
 )
-from webapp.services import run_reader
+from webapp.services import question_history, run_reader
 
 
 # Number of equal-width buckets used for metric score histograms.
@@ -37,6 +38,9 @@ GROUPING_FIELDS = ("difficulty", "question_type", "language")
 # How many lowest-scoring samples to surface for manual review.
 LOWEST_SAMPLE_COUNT = 10
 
+# How many past evaluations of the same question to show in the history table.
+HISTORY_LIMIT = 5
+
 # Metrics whose lower raw value means stronger performance.
 LOWER_IS_BETTER_METRICS = {"noise_sensitivity"}
 
@@ -124,8 +128,16 @@ def _cell_text(row: pd.Series, column: str) -> str:
     return str(row[column]).strip()
 
 
-def _lowest_samples(frame: pd.DataFrame, metrics: list[str]) -> list[SampleScore]:
-    """Select and shape the lowest-scoring samples for the review table."""
+def _lowest_samples(
+    frame: pd.DataFrame,
+    metrics: list[str],
+    history_index: dict[str, list[dict]] | None = None,
+) -> list[SampleScore]:
+    """Select and shape the lowest-scoring samples for the review table.
+
+    When a history_index is supplied, each surfaced sample is annotated with the
+    same question's scores from previous runs (newest first) for comparison.
+    """
     if frame.empty:
         return []
 
@@ -154,7 +166,19 @@ def _lowest_samples(frame: pd.DataFrame, metrics: list[str]) -> list[SampleScore
         enriched.append((sort_key, sample))
 
     enriched.sort(key=lambda item: item[0])
-    return [sample for _, sample in enriched[:LOWEST_SAMPLE_COUNT]]
+    selected = [sample for _, sample in enriched[:LOWEST_SAMPLE_COUNT]]
+
+    # Attach per-question history only for the surfaced samples (keeps lookups cheap).
+    if history_index is not None:
+        for sample in selected:
+            if not sample.question:
+                continue
+            entries = question_history.lookup(
+                history_index, sample.question, limit=HISTORY_LIMIT
+            )
+            sample.history = [SampleHistoryEntry(**entry) for entry in entries]
+
+    return selected
 
 
 def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
@@ -192,12 +216,20 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
         if metric in frame.columns
     }
 
+    # Cross-run history: scores of the same question in *other* runs (Approach A —
+    # on-demand global scan, excluding the run currently being viewed).
+    metadata = run_reader._read_json(run_dir / "metadata.json")
+    current_run_id = str(metadata.get("run_id") or run_dir.name)
+    history_index = question_history.build_question_history_index(
+        exclude_run_id=current_run_id
+    )
+
     return ReportData(
         metrics=metrics,
         metric_means=rounded_means,
         distributions=distributions,
         groupings=_groupings(frame, metrics),
-        lowest_samples=_lowest_samples(frame, metrics),
+        lowest_samples=_lowest_samples(frame, metrics, history_index),
         summary_markdown=summary_markdown,
         advice_markdown=advice_markdown,
         weighted_score_mean=_round_or_none(overall_ws),

webapp/services/score_job_manager.py

@@ -107,7 +107,6 @@ class ScoreJobManager:
 
         # Lazy imports to keep web server bootable if ragas is not installed.
         from rag_eval.advisor import run_advisor
-        from rag_eval.metrics.factory import build_models
         from rag_eval.metrics.weights import compute_weighted_score
         from rag_eval.reporting.writers import write_run_artifacts
         from rag_eval.settings import EvaluationSettings
@@ -206,8 +205,7 @@ class ScoreJobManager:
 
             # Run optimization advisor (builds optimization_advice.md)
             try:
-                llm, _ = build_models(judge_model, embedding_model, settings)
-                run_advisor(result, scenario, llm)
+                run_advisor(result, scenario, settings=settings)
                 logger.info("[score_job] advisor done  job_id=%s", job_id)
             except Exception as adv_exc:  # noqa: BLE001
                 logger.warning("[score_job] advisor failed  job_id=%s  err=%s", job_id, adv_exc)

webapp/services/session_score_manager.py

@@ -192,7 +192,6 @@ class SessionScoreJobManager:
 
         # Lazy imports — keep web server bootable if ragas is not installed.
         from rag_eval.advisor import run_advisor
-        from rag_eval.metrics.factory import build_models
         from rag_eval.metrics.weights import compute_weighted_score
         from rag_eval.reporting.writers import write_run_artifacts
         from rag_eval.settings import EvaluationSettings
@@ -320,8 +319,7 @@ class SessionScoreJobManager:
 
                 # Regenerate optimization advice over all accumulated rows
                 try:
-                    llm, _ = build_models(judge_model, embedding_model, settings)
-                    run_advisor(result, scenario, llm)
+                    run_advisor(result, scenario, settings=settings)
                     logger.info("[session_job] advisor done  job_id=%s  session=%s", job_id, session_id)
                 except Exception as adv_exc:  # noqa: BLE001
                     logger.warning(

webapp/static/css/app.css

@@ -253,6 +253,23 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t
 }
 .detail-gt { color: var(--good); }
 
+/* 历史评分小表格：本次行高亮 + 涨跌着色（绿=改善 红=退步） */
+table.history-table { width: 100%; border-collapse: collapse; font-size: 12px; margin-top: 4px; }
+table.history-table th, table.history-table td {
+  padding: 5px 8px; text-align: left; border-bottom: 1px solid #f1f5f9;
+}
+table.history-table th { color: var(--slate); font-weight: 600; border-bottom: 1px solid var(--line); }
+table.history-table td { font-variant-numeric: tabular-nums; }
+.history-table tr.hist-current { background: #f0f9ff; }
+.history-table tr.hist-current .hist-label { font-weight: 700; color: #0369a1; }
+.hist-when { white-space: nowrap; }
+.hist-label { display: inline-block; }
+.hist-sub { display: block; font-size: 11px; color: var(--slate-light); }
+.hist-delta { font-size: 11px; font-weight: 700; font-variant-numeric: tabular-nums; }
+.hist-delta.delta-good { color: #16a34a; }
+.hist-delta.delta-bad { color: #dc2626; }
+.hist-delta.delta-flat { color: var(--slate-light); }
+
 .empty { text-align: center; padding: 60px 20px; color: var(--slate); }
 .empty p { margin-bottom: 8px; }
 
@@ -514,6 +531,14 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t
   table.group-table td { padding: 4pt 6pt; border-bottom: 1px solid #e2e8f0; }
   table.group-table th { font-weight: 700; color: #64748b; }
 
+  /* ── 历史评分表 ── */
+  table.history-table { width: 100%; font-size: 9pt; border-collapse: collapse; }
+  table.history-table th,
+  table.history-table td { padding: 3pt 6pt; border-bottom: 1px solid #e2e8f0; }
+  .history-table tr.hist-current { background: #f0f9ff !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
+  .hist-delta.delta-good { color: #16a34a !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
+  .hist-delta.delta-bad  { color: #dc2626 !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
+
   /* ── 颜色保留（部分浏览器打印默认去色） ── */
   .good  { color: #16a34a !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
   .warn  { color: #eab308 !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
@@ -546,3 +571,67 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t
 .advice-md ul { padding-left: 20px; margin: 6px 0; }
 .advice-md li { margin: 3px 0; font-size: 13px; }
 .advice-md strong { color: var(--ink); font-weight: 600; }
+
+/* ---------- 指标看板 Dashboard ---------- */
+.dashboard-charts {
+  display: flex;
+  flex-direction: column;
+  gap: 20px;
+}
+.dashboard-chart-panel {
+  min-width: 0;
+  /* 上下布局时给图表面板稍微更宽裕的高度 */
+}
+.dashboard-chart-panel canvas {
+  max-height: 340px;
+  height: 320px !important;
+}
+
+/* 运行选择器列表 */
+.db-run-list {
+  display: flex;
+  flex-direction: column;
+  gap: 5px;
+  max-height: 240px;
+  overflow-y: auto;
+  margin-top: 10px;
+  padding-right: 4px;
+}
+.db-run-row {
+  display: flex;
+  align-items: center;
+  gap: 10px;
+  padding: 8px 12px;
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  cursor: pointer;
+  transition: background 0.12s, border-color 0.12s;
+  background: var(--surface);
+}
+.db-run-row:hover { background: #f0fbfb; border-color: var(--petrol); }
+.db-run-row:has(input:checked) {
+  background: #e8f7f7;
+  border-color: #7ecece;
+}
+.db-run-row input[type="checkbox"] { flex-shrink: 0; accent-color: var(--petrol); width: 15px; height: 15px; }
+.db-run-label { display: flex; flex-direction: column; gap: 2px; flex: 1; min-width: 0; }
+.db-run-name { font-size: 13px; font-weight: 600; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
+.db-run-chips { display: flex; flex-wrap: wrap; gap: 6px; flex-shrink: 0; }
+.db-chip-name { color: var(--slate); }
+.btn-sm { padding: 5px 12px; font-size: 12px; }
+
+/* 看板图表面板头：标题左 + 下拉右 对齐优化 */
+.db-panel-head-bar {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  flex-wrap: wrap;
+  gap: 8px;
+  margin-bottom: 14px;
+}
+.db-chart-hint {
+  font-size: 11px;
+  color: var(--slate-light);
+  margin-top: 6px;
+  text-align: center;
+}

webapp/static/index.html

@@ -31,6 +31,9 @@
         <button class="nav-item" data-view="scorejobs">
           <span class="nav-ico">📋</span><span>评分记录</span>
         </button>
+        <button class="nav-item" data-view="dashboard">
+          <span class="nav-ico">📊</span><span>指标看板</span>
+        </button>
         <button class="nav-item" data-view="apidocs">
           <span class="nav-ico">⎔</span><span>API 文档</span>
         </button>
@@ -263,6 +266,11 @@
           allowfullscreen>
         </iframe>
       </section>
+
+      <!-- 指标看板视图 -->
+      <section class="view" id="view-dashboard" hidden>
+        <div id="dashboard-wrap"></div>
+      </section>
     </main>
   </div>
 
@@ -272,6 +280,7 @@
   <script src="/static/js/profiles.js"></script>
   <script src="/static/js/runner.js"></script>
   <script src="/static/js/score_jobs.js"></script>
+  <script src="/static/js/dashboard.js"></script>
   <script src="/static/js/app.js"></script>
 </body>
 </html>

webapp/static/js/app.js

@@ -5,8 +5,8 @@
 const App = {
   currentRunId: null,
   activeView: null,
-  views: ["runs", "new", "report", "profiles", "scorejobs", "apidocs"],
-  titles: { runs: "运行列表", new: "新建评估", report: "报告详情", profiles: "LLM 配置", scorejobs: "评分记录", apidocs: "API 文档" },
+  views: ["runs", "new", "report", "profiles", "scorejobs", "dashboard", "apidocs"],
+  titles: { runs: "运行列表", new: "新建评估", report: "报告详情", profiles: "LLM 配置", scorejobs: "评分记录", dashboard: "指标看板", apidocs: "API 文档" },
 
   // 初始化：绑定导航、从 URL/sessionStorage 恢复上次位置、启动健康检查。
   init() {
@@ -73,6 +73,7 @@ const App = {
     if (view === "report")     Report.render(App.currentRunId);
     if (view === "profiles")   Profiles.load();
     if (view === "scorejobs")  ScoreJobs.load();
+    if (view === "dashboard")  Dashboard.load();
   },
 
   // ----------------------------------------------------------------

webapp/static/js/metric_presenter.js

@@ -69,9 +69,42 @@
     return "#dc2626";
   }
 
+  // 计算某指标本次相对上一次的涨跌信息，方向语义随指标而定
+  // （noise_sensitivity 越低越好：下降=改善）。
+  function deltaInfo(metricName, current, previous) {
+    const isNum = (v) => v !== null && v !== undefined && !Number.isNaN(Number(v));
+    if (!isNum(current) || !isNum(previous)) {
+      return { hasData: false, delta: null, improved: null, arrow: "", magnitude: "", cls: "delta-flat" };
+    }
+    const delta = Number(current) - Number(previous);
+    const rounded = Math.round(delta * 10000) / 10000;
+    const arrow = rounded > 0 ? "▲" : rounded < 0 ? "▼" : "→";
+    const magnitude = Math.abs(rounded).toFixed(2);
+    const improved = isLowerBetter(metricName) ? rounded < 0 : rounded > 0;
+    const cls = rounded === 0 ? "delta-flat" : improved ? "delta-good" : "delta-bad";
+    return { hasData: true, delta: rounded, improved, arrow, magnitude, cls };
+  }
+
+  // 返回指标的"达标阈值"（柱状图对比用，方向感知）。
+  // higher-better 指标：0.85；lower-better (noise_sensitivity)：0.15。
+  function passThreshold(metricName) {
+    return isLowerBetter(metricName) ? 0.15 : 0.85;
+  }
+
+  // 判断某指标的值是否达标。
+  function meetsTarget(metricName, value) {
+    if (value === null || value === undefined || Number.isNaN(Number(value))) return false;
+    const v = Number(value);
+    return isLowerBetter(metricName) ? v <= passThreshold(metricName) : v >= passThreshold(metricName);
+  }
+
   globalObj.MetricPresenter = {
     scoreClass,
     describeMetric,
     binColor,
+    isLowerBetter,
+    deltaInfo,
+    passThreshold,
+    meetsTarget,
   };
 })(window);

webapp/static/js/report.js

@@ -283,7 +283,7 @@ const Report = {
       const detail = document.createElement("div");
       detail.className = "lowest-detail";
       detail.hidden = true;
-      detail.innerHTML = Report._detailHtml(sample);
+      detail.innerHTML = Report._detailHtml(sample, metrics);
 
       row.addEventListener("click", () => {
         detail.hidden = !detail.hidden;
@@ -293,8 +293,8 @@ const Report = {
     });
   },
 
-  // 单条样本的展开详情：question / contexts / answer / ground_truth。
-  _detailHtml(sample) {
+  // 单条样本的展开详情：question / contexts / answer / ground_truth / 历史评分。
+  _detailHtml(sample, metrics) {
     const contexts = (sample.contexts || [])
       .map((c, i) => `<div class="ctx-item">[${i + 1}] ${App.escape(c)}</div>`)
       .join("");
@@ -320,6 +320,63 @@ const Report = {
           <div class="detail-gt">${App.escape(sample.ground_truth || "—")}</div>
         </div>
         ${errorBlock}
+        ${Report._historyHtml(sample, metrics || [])}
+      </div>
+    `;
+  },
+
+  // 同一问题的历史评分小表格：本次 + 历次（按时间倒序），逐行标注较更早一次的涨跌。
+  _historyHtml(sample, metrics) {
+    const history = sample.history || [];
+    if (!history.length) return "";
+
+    // 只展示当前样本与历史中实际出现过的指标列，避免空列。
+    const cols = metrics.filter(
+      (m) =>
+        (sample.metrics && sample.metrics[m] !== undefined && sample.metrics[m] !== null) ||
+        history.some((h) => h.metrics && h.metrics[m] !== undefined && h.metrics[m] !== null),
+    );
+    if (!cols.length) return "";
+
+    // 组合行：[本次, 历次...]，相邻两行做涨跌对比（行 r 对比更早的行 r+1）。
+    const rows = [
+      { label: "本次", sub: "", metrics: sample.metrics || {}, current: true },
+      ...history.map((h) => ({
+        label: App.escape(h.scenario_name || h.run_id || "历史"),
+        sub: App.escape(App.shortTime(h.finished_at)),
+        metrics: h.metrics || {},
+        current: false,
+      })),
+    ];
+
+    let head = "<tr><th>评测</th>";
+    cols.forEach((m) => (head += `<th>${App.escape(App.shortMetric(m))}</th>`));
+    head += "</tr>";
+
+    let body = "";
+    rows.forEach((row, r) => {
+      const older = rows[r + 1];
+      body += `<tr class="${row.current ? "hist-current" : ""}">`;
+      body += `<td class="hist-when"><span class="hist-label">${row.label}</span>${row.sub ? `<span class="hist-sub">${row.sub}</span>` : ""}</td>`;
+      cols.forEach((m) => {
+        const v = row.metrics ? row.metrics[m] : null;
+        const cls = App.scoreClass(m, v);
+        const text = v === null || v === undefined ? "—" : Number(v).toFixed(2);
+        let deltaHtml = "";
+        const baseline = older && older.metrics ? older.metrics[m] : undefined;
+        const d = MetricPresenter.deltaInfo(m, v, baseline);
+        if (d.hasData && d.delta !== 0) {
+          deltaHtml = ` <span class="hist-delta ${d.cls}">${d.arrow}${d.magnitude}</span>`;
+        }
+        body += `<td><span class="score-badge ${cls}">${text}</span>${deltaHtml}</td>`;
+      });
+      body += "</tr>";
+    });
+
+    return `
+      <div class="detail-field">
+        <div class="detail-label">历史评分 history（同一问题，最近 ${history.length} 次，含本次对比）</div>
+        <table class="history-table">${head}${body}</table>
       </div>
     `;
   },