diff --git a/rag_eval/advisor/__init__.py b/rag_eval/advisor/__init__.py
index 7ea2cb6..42168a2 100644
--- a/rag_eval/advisor/__init__.py
+++ b/rag_eval/advisor/__init__.py
@@ -20,7 +20,10 @@ __all__ = ["run_advisor", "Diagnosis", "diagnose"]
def run_advisor(
result: EvaluationResult,
scenario: Scenario,
- llm: Any,
+ llm: Any = None,
+ *,
+ settings: Any | None = None,
+ chat_client: Any | None = None,
) -> None:
"""Run the full optimization advisor pipeline after an evaluation completes.
@@ -30,7 +33,10 @@ def run_advisor(
Args:
result: Completed EvaluationResult from Evaluator.evaluate().
scenario: The resolved Scenario (provides metrics, judge_model, output_dir).
- llm: Pre-built RAGAS LLM instance (from build_models()) for LLM analysis.
+ llm: Deprecated/unused — kept for backward-compatible call sites. The
+ advisor now issues its own direct LLM call resolved from judge_model.
+ settings: Optional EvaluationSettings; defaults to EvaluationSettings().
+ chat_client: Optional pre-built chat client (used by tests to avoid network).
"""
if not scenario.optimization_advisor:
return
@@ -38,6 +44,10 @@ def run_advisor(
logger.info("[advisor] starting optimization analysis scenario=%s", scenario.scenario_name)
try:
+ if settings is None:
+ from rag_eval.settings import EvaluationSettings
+ settings = EvaluationSettings()
+
artifact_paths = build_artifact_paths(scenario.output_dir, result.run_id)
if artifact_paths.advice_md is None:
logger.warning("[advisor] advice_md path not set in RunArtifactPaths — skipping")
@@ -47,7 +57,15 @@ def run_advisor(
logger.info("[advisor] rule diagnosis complete: %d metric(s) triggered", len(diagnoses))
if diagnoses:
- llm_markdown = asyncio.run(analyze(diagnoses, llm, scenario.scenario_name))
+ llm_markdown = asyncio.run(
+ analyze(
+ diagnoses,
+ scenario.scenario_name,
+ scenario.judge_model,
+ settings,
+ chat_client=chat_client,
+ )
+ )
else:
llm_markdown = ""
diff --git a/rag_eval/advisor/llm_analyzer.py b/rag_eval/advisor/llm_analyzer.py
index 99d9997..749e008 100644
--- a/rag_eval/advisor/llm_analyzer.py
+++ b/rag_eval/advisor/llm_analyzer.py
@@ -1,4 +1,11 @@
-"""LLM-powered analysis of rule diagnostics and low-score samples."""
+"""LLM-powered analysis of rule diagnostics and low-score samples.
+
+The analyzer issues a single, direct OpenAI-compatible chat call (no langchain,
+no instructor structured output) to produce a detailed Chinese optimization
+report. It deliberately uses plain text generation so it works with any
+OpenAI-compatible gateway and never trips the structured-output response-shape
+issues seen with the scoring path.
+"""
from __future__ import annotations
import logging
@@ -8,27 +15,41 @@ from .rules import Diagnosis
logger = logging.getLogger("rag_eval.advisor")
+# Worked-example oriented prompt: forces per-question decomposition (为什么低 →
+# 拆解 → 怎么改) plus cross-metric causal reasoning, so the report is concrete
+# instead of a generic restatement of the static rule templates.
_PROMPT_TEMPLATE = """\
-你是一个 RAG 系统优化专家，正在分析西门子医疗 CT 文档问答系统的评测结果。
-请用中文撰写一份优化建议报告，格式为 Markdown。
+你是一位资深的 RAG（检索增强生成）系统优化专家，正在分析西门子医疗 CT 文档问答系统的 RAGAS 评测结果。
+请基于以下诊断数据与低分样本，用中文撰写一份**详细、具体、可落地**的优化建议报告（Markdown 格式）。
## 评测诊断摘要
{diagnosis_summary}
-## 低分样本示例
+## 低分样本明细（含检索片段 contexts，用于定位问题出在检索还是生成环节）
{low_sample_text}
-## 报告要求
+## 撰写要求
-1. 按指标分节（## 指标名 [严重程度]），先解释"为什么低"（结合低分样本具体分析），再给出"具体怎么改"
-2. 严重程度说明：critical=严重（<阈值50%），warning=警告（<阈值70%），low=待优化（低于0.85，有提升空间）
-3. "具体怎么改"要结合低分样本的实际内容，而不只是泛泛建议
-4. 最后写一节 **## 优先优化次序**，按性价比排序（不增加 LLM 调用次数的优化优先），critical 和 warning 项优先于 low 项
-5. 语言简洁，面向工程师，不要废话，不要重复列表内容
+1. **按指标分节**：每个指标一个 `## 指标名 [严重程度]` 小节。
+2. **每节必须包含「举例拆解」**：从该指标的低分样本中挑 1-2 个最典型的，逐条按如下结构拆解：
+ - **问题**：简述该样本的 question
+ - **当前得分**：该样本在此指标上的分数
+ - **为什么低**：结合该样本的 answer / 检索片段 contexts / 标准答案 ground_truth，**具体指出**问题所在
+ （例如：答案里哪句话没有被检索片段支持；检索片段里缺了哪个关键信息；答案偏离了问题的哪个点）
+ - **拆解定位**：判断问题出在「检索环节」「生成环节」还是「两者兼有」
+ - **优化动作**：针对这个具体样本，给出 1-3 条**可操作**的改法，不要泛泛而谈
+3. **跨指标关联分析**：若多个指标同时偏低，分析其因果关系。例如：
+ - faithfulness 低且 context_recall 低 → 多半是检索缺失关键信息，导致模型臆造（幻觉）
+ - context_recall 正常但 context_precision 低 → 检索引入噪声，稀释了有效信息
+ - faithfulness 低但 context_recall 高 → 生成环节 grounding 不足，需收紧生成 prompt
+4. 最后写一节 `## 优先优化次序`：按性价比排序（不增加 LLM 调用次数的优化优先；critical/warning 优先于 low）。
+5. 语言简洁，面向工程师，重点是「具体、可操作」。不要复述本提示词，不要无意义的客套。
-只输出 Markdown 报告正文，不要任何前置说明。
+严重程度说明：critical=严重（远低于阈值），warning=警告（低于阈值），low=待优化（达标但低于 0.85，仍有提升空间）。
+
+只输出 Markdown 报告正文，不要任何前置说明或代码块包裹。
"""
@@ -38,8 +59,15 @@ _SEVERITY_LABEL_ZH: dict[str, str] = {
"low": "待优化",
}
+# Per-sample text limits keep the prompt bounded regardless of sample size.
+_ANSWER_LIMIT = 400
+_GT_LIMIT = 300
+_CONTEXT_LIMIT = 600
+_CONTEXT_SEPARATOR = " |||| "
+
def _build_diagnosis_summary(diagnoses: list[Diagnosis]) -> str:
+ """Render the per-metric diagnosis block fed to the LLM."""
lines = []
for d in diagnoses:
direction = "（越低越好）" if d.metric == "noise_sensitivity" else ""
@@ -53,7 +81,22 @@ def _build_diagnosis_summary(diagnoses: list[Diagnosis]) -> str:
return "\n".join(lines)
+def _format_contexts(raw: Any) -> str:
+ """Render a sample's contexts (stored as a joined string) as a short list."""
+ text = str(raw or "").strip()
+ if not text:
+ return "（无检索片段）"
+ parts = [p.strip() for p in text.split(_CONTEXT_SEPARATOR) if p.strip()]
+ if not parts:
+ return "（无检索片段）"
+ rendered = "; ".join(f"[{i + 1}] {p}" for i, p in enumerate(parts))
+ if len(rendered) > _CONTEXT_LIMIT:
+ rendered = rendered[:_CONTEXT_LIMIT] + "…"
+ return rendered
+
+
def _build_low_sample_text(diagnoses: list[Diagnosis]) -> str:
+ """Render low-score samples (now including contexts) for grounding analysis."""
lines = []
for d in diagnoses:
if not d.low_samples:
@@ -61,24 +104,63 @@ def _build_low_sample_text(diagnoses: list[Diagnosis]) -> str:
lines.append(f"### {d.metric} 低分样本（最多 3 条）")
for i, s in enumerate(d.low_samples, 1):
score = s.get(d.metric, "N/A")
- lines.append(f"\n**样本 {i}**（分数={score}）")
- lines.append(f"- 问题：{s.get('question', '')}")
- lines.append(f"- 回答：{s.get('answer', '')[:300]}")
- lines.append(f"- 标准答案：{s.get('ground_truth', '')[:200]}")
+ lines.append(f"\n**样本 {i}**（{d.metric}={score}）")
+ lines.append(f"- 问题 question：{s.get('question', '')}")
+ lines.append(f"- 生成答案 answer：{str(s.get('answer', ''))[:_ANSWER_LIMIT]}")
+ lines.append(f"- 检索片段 contexts：{_format_contexts(s.get('contexts'))}")
+ lines.append(f"- 标准答案 ground_truth：{str(s.get('ground_truth', ''))[:_GT_LIMIT]}")
return "\n".join(lines)
+def _is_reasoning_model(model: str) -> bool:
+ """Return True for OpenAI reasoning models (gpt-5+/o-series/codex-mini).
+
+ These require `max_completion_tokens` instead of `max_tokens` and do not
+ accept a custom temperature. Mirrors RAGAS's own detection logic so advice
+ generation stays consistent with the scoring path.
+ """
+ m = (model or "").lower()
+ # O-series: o1..o9 optionally followed by - or _
+ if len(m) >= 2 and m[0] == "o" and m[1] in "123456789":
+ if len(m) == 2 or m[2] in ("-", "_"):
+ return True
+ # GPT-5 through GPT-19 generation
+ if m.startswith("gpt-"):
+ version_str = m[4:].split("-")[0].split("_")[0].split(".")[0]
+ try:
+ if 5 <= int(version_str) <= 19:
+ return True
+ except ValueError:
+ pass
+ if m == "codex-mini":
+ return True
+ return False
+
+
+def _chat_token_kwargs(model: str, max_tokens: int) -> dict[str, Any]:
+ """Return the completion-budget kwargs appropriate for the model family."""
+ if _is_reasoning_model(model):
+ # Reasoning models: only max_completion_tokens; temperature must stay default (1).
+ return {"max_completion_tokens": max_tokens}
+ return {"max_tokens": max_tokens, "temperature": 0.2}
+
+
async def analyze(
diagnoses: list[Diagnosis],
- llm: Any,
scenario_name: str,
+ judge_model: str,
+ settings: Any,
+ *,
+ chat_client: Any | None = None,
) -> str:
- """Call the judge LLM to generate a Chinese optimization report.
+ """Call the judge LLM directly to generate a Chinese optimization report.
Args:
diagnoses: Non-empty list of Diagnosis from rules.diagnose().
- llm: RAGAS LLM wrapper (has .agenerate() method).
scenario_name: Used only for logging.
+ judge_model: Model name to call (also selects the matching LLM profile).
+ settings: EvaluationSettings (supplies client kwargs + token budget).
+ chat_client: Optional pre-built AsyncOpenAI-compatible client (for tests).
Returns:
LLM-generated Markdown string, or "" on failure (triggers writer fallback).
@@ -86,22 +168,47 @@ async def analyze(
if not diagnoses:
return ""
- diagnosis_summary = _build_diagnosis_summary(diagnoses)
- low_sample_text = _build_low_sample_text(diagnoses)
prompt = _PROMPT_TEMPLATE.format(
- diagnosis_summary=diagnosis_summary,
- low_sample_text=low_sample_text,
+ diagnosis_summary=_build_diagnosis_summary(diagnoses),
+ low_sample_text=_build_low_sample_text(diagnoses),
)
try:
logger.info("[advisor] calling LLM for optimization analysis scenario=%s", scenario_name)
- from langchain_core.messages import HumanMessage
- # Use the underlying langchain chat model directly (RAGAS LangchainLLMWrapper wraps BaseChatModel)
- response = await llm.langchain_llm.ainvoke([HumanMessage(content=prompt)])
- text = response.content.strip()
- logger.info("[advisor] LLM analysis complete chars=%d", len(text))
- return text
- except Exception as exc:
+ client = chat_client
+ owns_client = False
+ if client is None:
+ from openai import AsyncOpenAI
+
+ from rag_eval.metrics.factory import resolve_openai_client_kwargs
+
+ client = AsyncOpenAI(**resolve_openai_client_kwargs(judge_model, settings))
+ owns_client = True
+
+ try:
+ # Advice is a longer document than per-metric scoring; give it headroom.
+ max_tokens = max(2048, int(getattr(settings, "ragas_llm_max_tokens", 4096) or 4096))
+ token_kwargs = _chat_token_kwargs(judge_model, max_tokens)
+
+ response = await client.chat.completions.create(
+ model=judge_model,
+ messages=[{"role": "user", "content": prompt}],
+ **token_kwargs,
+ )
+ text = (response.choices[0].message.content or "").strip()
+ logger.info("[advisor] LLM analysis complete chars=%d", len(text))
+ return text
+ finally:
+ # Close the httpx connection pool inside THIS event loop. run_advisor
+ # drives analyze() via asyncio.run(), which closes the loop on return;
+ # a later GC-time aclose() on the dead loop would otherwise log
+ # "RuntimeError: Event loop is closed". Only close clients we created.
+ if owns_client:
+ try:
+ await client.close()
+ except Exception: # noqa: BLE001
+ pass
+ except Exception as exc: # noqa: BLE001
logger.warning(
"[advisor] LLM analysis failed (%s: %s) — falling back to rule report",
type(exc).__name__, exc,
diff --git a/rag_eval/advisor/rules.py b/rag_eval/advisor/rules.py
index eee1829..fd26514 100644
--- a/rag_eval/advisor/rules.py
+++ b/rag_eval/advisor/rules.py
@@ -159,7 +159,8 @@ def _select_low_samples(
valid = [r for r in rows if metric in r and not math.isnan(float(r[metric]))]
sorted_rows = sorted(valid, key=lambda r: float(r[metric]), reverse=not higher_is_better)
worst = sorted_rows[:top_n]
- keep_keys = {"sample_id", "question", "answer", "ground_truth", metric}
+ # contexts is included so the LLM/fallback can judge grounding (retrieval vs generation).
+ keep_keys = {"sample_id", "question", "answer", "ground_truth", "contexts", metric}
return [{k: v for k, v in row.items() if k in keep_keys} for row in worst]
diff --git a/rag_eval/advisor/writer.py b/rag_eval/advisor/writer.py
index b60e6dd..a60bd3c 100644
--- a/rag_eval/advisor/writer.py
+++ b/rag_eval/advisor/writer.py
@@ -29,10 +29,15 @@ def _format_log_summary(diagnoses: list[Diagnosis], advice_path: Path) -> str:
def _build_fallback_report(diagnoses: list[Diagnosis]) -> str:
- """Build a rules-only report when LLM analysis is unavailable."""
+ """Build a rules-only report when LLM analysis is unavailable.
+
+ Even without the LLM, embed each metric's worst sample(s) — question,
+ answer, ground truth — so the advice still references concrete problems
+ instead of reading as a purely generic template.
+ """
if not diagnoses:
return ""
- lines = ["## 规则诊断（LLM 分析不可用）\n"]
+ lines = ["## 规则诊断（LLM 分析不可用，以下为规则引擎输出）\n"]
for d in diagnoses:
label = _SEVERITY_LABEL.get(d.severity, d.severity)
lines.append(f"### {d.metric} [{label}] 均值={d.mean_score:.4f}")
@@ -42,6 +47,22 @@ def _build_fallback_report(diagnoses: list[Diagnosis]) -> str:
lines.append("\n**建议动作：**")
for action in d.suggested_actions:
lines.append(f"- {action}")
+ if d.low_samples:
+ lines.append("\n**低分样本举例拆解：**")
+ for i, sample in enumerate(d.low_samples, 1):
+ score = sample.get(d.metric, "N/A")
+ question = str(sample.get("question", "")).strip() or "（无问题文本）"
+ lines.append(f"\n- **样本 {i}**（{d.metric}={score}）问题：{question}")
+ answer = str(sample.get("answer", "")).strip()
+ if answer:
+ lines.append(f" - 生成答案：{answer[:200]}")
+ ground_truth = str(sample.get("ground_truth", "")).strip()
+ if ground_truth:
+ lines.append(f" - 标准答案：{ground_truth[:160]}")
+ lines.append(
+ f" - 拆解：该样本 {d.metric} 偏低，请对照上述「可能原因 / 建议动作」"
+ f"重点排查本问题的检索片段与生成答案。"
+ )
lines.append("")
return "\n".join(lines)
diff --git a/rag_eval/execution/runner.py b/rag_eval/execution/runner.py
index a1e4b03..e119e0e 100644
--- a/rag_eval/execution/runner.py
+++ b/rag_eval/execution/runner.py
@@ -79,6 +79,6 @@ def run_scenario(
logger.info("[runner] artifacts written for run_id=%s", result.run_id)
# Optimization advisor — runs only if scenario.optimization_advisor is True.
- run_advisor(result, scenario, llm)
+ run_advisor(result, scenario, settings=settings)
return result
diff --git a/tests/test_metric_presenter.py b/tests/test_metric_presenter.py
index d4f94f8..baf87c3 100644
--- a/tests/test_metric_presenter.py
+++ b/tests/test_metric_presenter.py
@@ -40,7 +40,14 @@ const result = {{
desc: p.describeMetric("faithfulness"),
noiseDesc: p.describeMetric("noise_sensitivity"),
noiseBin: p.binColor("noise_sensitivity", 0.0),
- faithBin: p.binColor("faithfulness", 0.8)
+ faithBin: p.binColor("faithfulness", 0.8),
+ lowerBetterNoise: p.isLowerBetter("noise_sensitivity"),
+ lowerBetterFaith: p.isLowerBetter("faithfulness"),
+ upHigher: p.deltaInfo("faithfulness", 0.80, 0.60),
+ downHigher: p.deltaInfo("faithfulness", 0.60, 0.80),
+ noiseImproved: p.deltaInfo("noise_sensitivity", 0.10, 0.30),
+ noiseWorse: p.deltaInfo("noise_sensitivity", 0.30, 0.10),
+ noBaseline: p.deltaInfo("faithfulness", 0.80, null)
}};
console.log(JSON.stringify(result));
"""
@@ -55,6 +62,15 @@ console.log(JSON.stringify(result));
assert '"noiseDesc":"' in output
assert '"noiseBin":"#16a34a"' in output
assert '"faithBin":"#16a34a"' in output
+ assert '"lowerBetterNoise":true' in output
+ assert '"lowerBetterFaith":false' in output
+ # higher-better: rising value is an improvement (green ▲); falling is a regression (red ▼)
+ assert '"upHigher":{"hasData":true,"delta":0.2,"improved":true,"arrow":"▲","magnitude":"0.20","cls":"delta-good"}' in output
+ assert '"downHigher":{"hasData":true,"delta":-0.2,"improved":false,"arrow":"▼","magnitude":"0.20","cls":"delta-bad"}' in output
+ # noise_sensitivity (lower-better): falling value is an improvement (green ▼)
+ assert '"noiseImproved":{"hasData":true,"delta":-0.2,"improved":true,"arrow":"▼","magnitude":"0.20","cls":"delta-good"}' in output
+ assert '"noiseWorse":{"hasData":true,"delta":0.2,"improved":false,"arrow":"▲","magnitude":"0.20","cls":"delta-bad"}' in output
+ assert '"noBaseline":{"hasData":false' in output
def test_report_and_index_load_metric_presenter_helper() -> None:
@@ -66,3 +82,6 @@ def test_report_and_index_load_metric_presenter_helper() -> None:
assert "js/metric_presenter.js" in index_html
assert "MetricPresenter.describeMetric" in report_js
assert "MetricPresenter.scoreClass" in app_js
+ # history comparison table uses the direction-aware delta helper
+ assert "MetricPresenter.deltaInfo" in report_js
+ assert "history-table" in report_js
diff --git a/tests/test_webapp_report_builder.py b/tests/test_webapp_report_builder.py
index b1be92d..2b60f4f 100644
--- a/tests/test_webapp_report_builder.py
+++ b/tests/test_webapp_report_builder.py
@@ -4,8 +4,11 @@ from __future__ import annotations
from pathlib import Path
+import pandas as pd
import pytest
+from webapp.services import question_history
+from webapp.services import report_builder
from webapp.services.report_builder import build_report
from webapp.services.run_reader import _infer_metrics_from_scores, _read_weights_from_snapshot
@@ -115,3 +118,71 @@ def test_build_report_ranks_noise_sensitivity_with_lower_values_as_better(tmp_pa
"s-warn",
"s-good",
]
+
+
+def test_lowest_samples_attaches_history_from_index() -> None:
+ """Surfaced samples are annotated with the same question's prior-run scores."""
+ frame = pd.DataFrame(
+ [
+ {"sample_id": "s1", "question": " How LONG is the tube? ", "faithfulness": 0.40},
+ {"sample_id": "s2", "question": "unrelated question", "faithfulness": 0.30},
+ ]
+ )
+ history_index = {
+ question_history.normalize_question("How long is the tube?"): [
+ {
+ "run_id": "prev",
+ "scenario_name": "scn",
+ "finished_at": "2026-01-01T00:00:00",
+ "metrics": {"faithfulness": 0.90},
+ }
+ ]
+ }
+
+ samples = report_builder._lowest_samples(frame, ["faithfulness"], history_index)
+ by_id = {s.sample_id: s for s in samples}
+
+ assert len(by_id["s1"].history) == 1
+ assert by_id["s1"].history[0].run_id == "prev"
+ assert by_id["s1"].history[0].metrics["faithfulness"] == 0.90
+ assert by_id["s2"].history == [] # no match → no history
+
+
+def test_build_report_attaches_question_history(tmp_path: Path, monkeypatch) -> None:
+ """build_report wires the question-history index into surfaced samples."""
+ run_dir = tmp_path / "run"
+ run_dir.mkdir(parents=True, exist_ok=True)
+ (run_dir / "scores.csv").write_text(
+ "\n".join(
+ [
+ "sample_id,question,faithfulness",
+ "s1,How long is the tube?,0.40",
+ ]
+ ),
+ encoding="utf-8",
+ )
+ (run_dir / "summary.md").write_text("summary", encoding="utf-8")
+ (run_dir / "optimization_advice.md").write_text("", encoding="utf-8")
+
+ captured: dict = {}
+
+ def _fake_index(exclude_run_id=None, extra_roots=None):
+ captured["exclude_run_id"] = exclude_run_id
+ return {
+ question_history.normalize_question("How long is the tube?"): [
+ {
+ "run_id": "older",
+ "scenario_name": "scn",
+ "finished_at": "2026-01-01T00:00:00",
+ "metrics": {"faithfulness": 0.95},
+ }
+ ]
+ }
+
+ monkeypatch.setattr(question_history, "build_question_history_index", _fake_index)
+
+ report = build_report(run_dir, ["faithfulness"])
+
+ assert captured["exclude_run_id"] == "run" # current run excluded from history
+ assert report.lowest_samples[0].history[0].run_id == "older"
+ assert report.lowest_samples[0].history[0].metrics["faithfulness"] == 0.95
diff --git a/webapp/services/report_builder.py b/webapp/services/report_builder.py
index 0a1e204..989fb55 100644
--- a/webapp/services/report_builder.py
+++ b/webapp/services/report_builder.py
@@ -23,9 +23,10 @@ from webapp.models import (
DistributionBin,
GroupStat,
ReportData,
+ SampleHistoryEntry,
SampleScore,
)
-from webapp.services import run_reader
+from webapp.services import question_history, run_reader
# Number of equal-width buckets used for metric score histograms.
@@ -37,6 +38,9 @@ GROUPING_FIELDS = ("difficulty", "question_type", "language")
# How many lowest-scoring samples to surface for manual review.
LOWEST_SAMPLE_COUNT = 10
+# How many past evaluations of the same question to show in the history table.
+HISTORY_LIMIT = 5
+
# Metrics whose lower raw value means stronger performance.
LOWER_IS_BETTER_METRICS = {"noise_sensitivity"}
@@ -124,8 +128,16 @@ def _cell_text(row: pd.Series, column: str) -> str:
return str(row[column]).strip()
-def _lowest_samples(frame: pd.DataFrame, metrics: list[str]) -> list[SampleScore]:
- """Select and shape the lowest-scoring samples for the review table."""
+def _lowest_samples(
+ frame: pd.DataFrame,
+ metrics: list[str],
+ history_index: dict[str, list[dict]] | None = None,
+) -> list[SampleScore]:
+ """Select and shape the lowest-scoring samples for the review table.
+
+ When a history_index is supplied, each surfaced sample is annotated with the
+ same question's scores from previous runs (newest first) for comparison.
+ """
if frame.empty:
return []
@@ -154,7 +166,19 @@ def _lowest_samples(frame: pd.DataFrame, metrics: list[str]) -> list[SampleScore
enriched.append((sort_key, sample))
enriched.sort(key=lambda item: item[0])
- return [sample for _, sample in enriched[:LOWEST_SAMPLE_COUNT]]
+ selected = [sample for _, sample in enriched[:LOWEST_SAMPLE_COUNT]]
+
+ # Attach per-question history only for the surfaced samples (keeps lookups cheap).
+ if history_index is not None:
+ for sample in selected:
+ if not sample.question:
+ continue
+ entries = question_history.lookup(
+ history_index, sample.question, limit=HISTORY_LIMIT
+ )
+ sample.history = [SampleHistoryEntry(**entry) for entry in entries]
+
+ return selected
def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
@@ -192,12 +216,20 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
if metric in frame.columns
}
+ # Cross-run history: scores of the same question in *other* runs (Approach A —
+ # on-demand global scan, excluding the run currently being viewed).
+ metadata = run_reader._read_json(run_dir / "metadata.json")
+ current_run_id = str(metadata.get("run_id") or run_dir.name)
+ history_index = question_history.build_question_history_index(
+ exclude_run_id=current_run_id
+ )
+
return ReportData(
metrics=metrics,
metric_means=rounded_means,
distributions=distributions,
groupings=_groupings(frame, metrics),
- lowest_samples=_lowest_samples(frame, metrics),
+ lowest_samples=_lowest_samples(frame, metrics, history_index),
summary_markdown=summary_markdown,
advice_markdown=advice_markdown,
weighted_score_mean=_round_or_none(overall_ws),
diff --git a/webapp/services/score_job_manager.py b/webapp/services/score_job_manager.py
index a3964a1..f538b5f 100644
--- a/webapp/services/score_job_manager.py
+++ b/webapp/services/score_job_manager.py
@@ -107,7 +107,6 @@ class ScoreJobManager:
# Lazy imports to keep web server bootable if ragas is not installed.
from rag_eval.advisor import run_advisor
- from rag_eval.metrics.factory import build_models
from rag_eval.metrics.weights import compute_weighted_score
from rag_eval.reporting.writers import write_run_artifacts
from rag_eval.settings import EvaluationSettings
@@ -206,8 +205,7 @@ class ScoreJobManager:
# Run optimization advisor (builds optimization_advice.md)
try:
- llm, _ = build_models(judge_model, embedding_model, settings)
- run_advisor(result, scenario, llm)
+ run_advisor(result, scenario, settings=settings)
logger.info("[score_job] advisor done job_id=%s", job_id)
except Exception as adv_exc: # noqa: BLE001
logger.warning("[score_job] advisor failed job_id=%s err=%s", job_id, adv_exc)
diff --git a/webapp/services/session_score_manager.py b/webapp/services/session_score_manager.py
index d145975..82c5785 100644
--- a/webapp/services/session_score_manager.py
+++ b/webapp/services/session_score_manager.py
@@ -192,7 +192,6 @@ class SessionScoreJobManager:
# Lazy imports — keep web server bootable if ragas is not installed.
from rag_eval.advisor import run_advisor
- from rag_eval.metrics.factory import build_models
from rag_eval.metrics.weights import compute_weighted_score
from rag_eval.reporting.writers import write_run_artifacts
from rag_eval.settings import EvaluationSettings
@@ -320,8 +319,7 @@ class SessionScoreJobManager:
# Regenerate optimization advice over all accumulated rows
try:
- llm, _ = build_models(judge_model, embedding_model, settings)
- run_advisor(result, scenario, llm)
+ run_advisor(result, scenario, settings=settings)
logger.info("[session_job] advisor done job_id=%s session=%s", job_id, session_id)
except Exception as adv_exc: # noqa: BLE001
logger.warning(
diff --git a/webapp/static/css/app.css b/webapp/static/css/app.css
index 7282750..5661e8b 100644
--- a/webapp/static/css/app.css
+++ b/webapp/static/css/app.css
@@ -253,6 +253,23 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t
}
.detail-gt { color: var(--good); }
+/* 历史评分小表格：本次行高亮 + 涨跌着色（绿=改善 红=退步） */
+table.history-table { width: 100%; border-collapse: collapse; font-size: 12px; margin-top: 4px; }
+table.history-table th, table.history-table td {
+ padding: 5px 8px; text-align: left; border-bottom: 1px solid #f1f5f9;
+}
+table.history-table th { color: var(--slate); font-weight: 600; border-bottom: 1px solid var(--line); }
+table.history-table td { font-variant-numeric: tabular-nums; }
+.history-table tr.hist-current { background: #f0f9ff; }
+.history-table tr.hist-current .hist-label { font-weight: 700; color: #0369a1; }
+.hist-when { white-space: nowrap; }
+.hist-label { display: inline-block; }
+.hist-sub { display: block; font-size: 11px; color: var(--slate-light); }
+.hist-delta { font-size: 11px; font-weight: 700; font-variant-numeric: tabular-nums; }
+.hist-delta.delta-good { color: #16a34a; }
+.hist-delta.delta-bad { color: #dc2626; }
+.hist-delta.delta-flat { color: var(--slate-light); }
+
.empty { text-align: center; padding: 60px 20px; color: var(--slate); }
.empty p { margin-bottom: 8px; }
@@ -514,6 +531,14 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t
table.group-table td { padding: 4pt 6pt; border-bottom: 1px solid #e2e8f0; }
table.group-table th { font-weight: 700; color: #64748b; }
+ /* ── 历史评分表 ── */
+ table.history-table { width: 100%; font-size: 9pt; border-collapse: collapse; }
+ table.history-table th,
+ table.history-table td { padding: 3pt 6pt; border-bottom: 1px solid #e2e8f0; }
+ .history-table tr.hist-current { background: #f0f9ff !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
+ .hist-delta.delta-good { color: #16a34a !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
+ .hist-delta.delta-bad { color: #dc2626 !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
+
/* ── 颜色保留（部分浏览器打印默认去色） ── */
.good { color: #16a34a !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
.warn { color: #eab308 !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
@@ -546,3 +571,67 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t
.advice-md ul { padding-left: 20px; margin: 6px 0; }
.advice-md li { margin: 3px 0; font-size: 13px; }
.advice-md strong { color: var(--ink); font-weight: 600; }
+
+/* ---------- 指标看板 Dashboard ---------- */
+.dashboard-charts {
+ display: flex;
+ flex-direction: column;
+ gap: 20px;
+}
+.dashboard-chart-panel {
+ min-width: 0;
+ /* 上下布局时给图表面板稍微更宽裕的高度 */
+}
+.dashboard-chart-panel canvas {
+ max-height: 340px;
+ height: 320px !important;
+}
+
+/* 运行选择器列表 */
+.db-run-list {
+ display: flex;
+ flex-direction: column;
+ gap: 5px;
+ max-height: 240px;
+ overflow-y: auto;
+ margin-top: 10px;
+ padding-right: 4px;
+}
+.db-run-row {
+ display: flex;
+ align-items: center;
+ gap: 10px;
+ padding: 8px 12px;
+ border: 1px solid var(--line);
+ border-radius: 8px;
+ cursor: pointer;
+ transition: background 0.12s, border-color 0.12s;
+ background: var(--surface);
+}
+.db-run-row:hover { background: #f0fbfb; border-color: var(--petrol); }
+.db-run-row:has(input:checked) {
+ background: #e8f7f7;
+ border-color: #7ecece;
+}
+.db-run-row input[type="checkbox"] { flex-shrink: 0; accent-color: var(--petrol); width: 15px; height: 15px; }
+.db-run-label { display: flex; flex-direction: column; gap: 2px; flex: 1; min-width: 0; }
+.db-run-name { font-size: 13px; font-weight: 600; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
+.db-run-chips { display: flex; flex-wrap: wrap; gap: 6px; flex-shrink: 0; }
+.db-chip-name { color: var(--slate); }
+.btn-sm { padding: 5px 12px; font-size: 12px; }
+
+/* 看板图表面板头：标题左 + 下拉右 对齐优化 */
+.db-panel-head-bar {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ flex-wrap: wrap;
+ gap: 8px;
+ margin-bottom: 14px;
+}
+.db-chart-hint {
+ font-size: 11px;
+ color: var(--slate-light);
+ margin-top: 6px;
+ text-align: center;
+}
diff --git a/webapp/static/index.html b/webapp/static/index.html
index 5cdd311..02c1fa4 100644
--- a/webapp/static/index.html
+++ b/webapp/static/index.html
@@ -31,6 +31,9 @@
+
@@ -263,6 +266,11 @@
allowfullscreen>
+
+
+
@@ -272,6 +280,7 @@
+