From 4e74e1b247cb56243327b22e6d148e33d63f013c Mon Sep 17 00:00:00 2001 From: wangwei Date: Wed, 1 Jul 2026 17:53:00 +0800 Subject: [PATCH] update for ragas --- rag_eval/advisor/__init__.py | 24 +++- rag_eval/advisor/llm_analyzer.py | 165 +++++++++++++++++++---- rag_eval/advisor/rules.py | 3 +- rag_eval/advisor/writer.py | 25 +++- rag_eval/execution/runner.py | 2 +- tests/test_metric_presenter.py | 21 ++- tests/test_webapp_report_builder.py | 71 ++++++++++ webapp/services/report_builder.py | 42 +++++- webapp/services/score_job_manager.py | 4 +- webapp/services/session_score_manager.py | 4 +- webapp/static/css/app.css | 89 ++++++++++++ webapp/static/index.html | 9 ++ webapp/static/js/app.js | 5 +- webapp/static/js/metric_presenter.js | 33 +++++ webapp/static/js/report.js | 63 ++++++++- 15 files changed, 507 insertions(+), 53 deletions(-) diff --git a/rag_eval/advisor/__init__.py b/rag_eval/advisor/__init__.py index 7ea2cb6..42168a2 100644 --- a/rag_eval/advisor/__init__.py +++ b/rag_eval/advisor/__init__.py @@ -20,7 +20,10 @@ __all__ = ["run_advisor", "Diagnosis", "diagnose"] def run_advisor( result: EvaluationResult, scenario: Scenario, - llm: Any, + llm: Any = None, + *, + settings: Any | None = None, + chat_client: Any | None = None, ) -> None: """Run the full optimization advisor pipeline after an evaluation completes. @@ -30,7 +33,10 @@ def run_advisor( Args: result: Completed EvaluationResult from Evaluator.evaluate(). scenario: The resolved Scenario (provides metrics, judge_model, output_dir). - llm: Pre-built RAGAS LLM instance (from build_models()) for LLM analysis. + llm: Deprecated/unused — kept for backward-compatible call sites. The + advisor now issues its own direct LLM call resolved from judge_model. + settings: Optional EvaluationSettings; defaults to EvaluationSettings(). + chat_client: Optional pre-built chat client (used by tests to avoid network). """ if not scenario.optimization_advisor: return @@ -38,6 +44,10 @@ def run_advisor( logger.info("[advisor] starting optimization analysis scenario=%s", scenario.scenario_name) try: + if settings is None: + from rag_eval.settings import EvaluationSettings + settings = EvaluationSettings() + artifact_paths = build_artifact_paths(scenario.output_dir, result.run_id) if artifact_paths.advice_md is None: logger.warning("[advisor] advice_md path not set in RunArtifactPaths — skipping") @@ -47,7 +57,15 @@ def run_advisor( logger.info("[advisor] rule diagnosis complete: %d metric(s) triggered", len(diagnoses)) if diagnoses: - llm_markdown = asyncio.run(analyze(diagnoses, llm, scenario.scenario_name)) + llm_markdown = asyncio.run( + analyze( + diagnoses, + scenario.scenario_name, + scenario.judge_model, + settings, + chat_client=chat_client, + ) + ) else: llm_markdown = "" diff --git a/rag_eval/advisor/llm_analyzer.py b/rag_eval/advisor/llm_analyzer.py index 99d9997..749e008 100644 --- a/rag_eval/advisor/llm_analyzer.py +++ b/rag_eval/advisor/llm_analyzer.py @@ -1,4 +1,11 @@ -"""LLM-powered analysis of rule diagnostics and low-score samples.""" +"""LLM-powered analysis of rule diagnostics and low-score samples. + +The analyzer issues a single, direct OpenAI-compatible chat call (no langchain, +no instructor structured output) to produce a detailed Chinese optimization +report. It deliberately uses plain text generation so it works with any +OpenAI-compatible gateway and never trips the structured-output response-shape +issues seen with the scoring path. +""" from __future__ import annotations import logging @@ -8,27 +15,41 @@ from .rules import Diagnosis logger = logging.getLogger("rag_eval.advisor") +# Worked-example oriented prompt: forces per-question decomposition (为什么低 → +# 拆解 → 怎么改) plus cross-metric causal reasoning, so the report is concrete +# instead of a generic restatement of the static rule templates. _PROMPT_TEMPLATE = """\ -你是一个 RAG 系统优化专家，正在分析西门子医疗 CT 文档问答系统的评测结果。 -请用中文撰写一份优化建议报告，格式为 Markdown。 +你是一位资深的 RAG（检索增强生成）系统优化专家，正在分析西门子医疗 CT 文档问答系统的 RAGAS 评测结果。 +请基于以下诊断数据与低分样本，用中文撰写一份**详细、具体、可落地**的优化建议报告（Markdown 格式）。 ## 评测诊断摘要 {diagnosis_summary} -## 低分样本示例 +## 低分样本明细（含检索片段 contexts，用于定位问题出在检索还是生成环节） {low_sample_text} -## 报告要求 +## 撰写要求 -1. 按指标分节（## 指标名 [严重程度]），先解释"为什么低"（结合低分样本具体分析），再给出"具体怎么改" -2. 严重程度说明：critical=严重（<阈值50%），warning=警告（<阈值70%），low=待优化（低于0.85，有提升空间） -3. "具体怎么改"要结合低分样本的实际内容，而不只是泛泛建议 -4. 最后写一节 **## 优先优化次序**，按性价比排序（不增加 LLM 调用次数的优化优先），critical 和 warning 项优先于 low 项 -5. 语言简洁，面向工程师，不要废话，不要重复列表内容 +1. **按指标分节**：每个指标一个 `## 指标名 [严重程度]` 小节。 +2. **每节必须包含「举例拆解」**：从该指标的低分样本中挑 1-2 个最典型的，逐条按如下结构拆解： + - **问题**：简述该样本的 question + - **当前得分**：该样本在此指标上的分数 + - **为什么低**：结合该样本的 answer / 检索片段 contexts / 标准答案 ground_truth，**具体指出**问题所在 + （例如：答案里哪句话没有被检索片段支持；检索片段里缺了哪个关键信息；答案偏离了问题的哪个点） + - **拆解定位**：判断问题出在「检索环节」「生成环节」还是「两者兼有」 + - **优化动作**：针对这个具体样本，给出 1-3 条**可操作**的改法，不要泛泛而谈 +3. **跨指标关联分析**：若多个指标同时偏低，分析其因果关系。例如： + - faithfulness 低且 context_recall 低 → 多半是检索缺失关键信息，导致模型臆造（幻觉） + - context_recall 正常但 context_precision 低 → 检索引入噪声，稀释了有效信息 + - faithfulness 低但 context_recall 高 → 生成环节 grounding 不足，需收紧生成 prompt +4. 最后写一节 `## 优先优化次序`：按性价比排序（不增加 LLM 调用次数的优化优先；critical/warning 优先于 low）。 +5. 语言简洁，面向工程师，重点是「具体、可操作」。不要复述本提示词，不要无意义的客套。 -只输出 Markdown 报告正文，不要任何前置说明。 +严重程度说明：critical=严重（远低于阈值），warning=警告（低于阈值），low=待优化（达标但低于 0.85，仍有提升空间）。 + +只输出 Markdown 报告正文，不要任何前置说明或代码块包裹。 """ @@ -38,8 +59,15 @@ _SEVERITY_LABEL_ZH: dict[str, str] = { "low": "待优化", } +# Per-sample text limits keep the prompt bounded regardless of sample size. +_ANSWER_LIMIT = 400 +_GT_LIMIT = 300 +_CONTEXT_LIMIT = 600 +_CONTEXT_SEPARATOR = " |||| " + def _build_diagnosis_summary(diagnoses: list[Diagnosis]) -> str: + """Render the per-metric diagnosis block fed to the LLM.""" lines = [] for d in diagnoses: direction = "（越低越好）" if d.metric == "noise_sensitivity" else "" @@ -53,7 +81,22 @@ def _build_diagnosis_summary(diagnoses: list[Diagnosis]) -> str: return "\n".join(lines) +def _format_contexts(raw: Any) -> str: + """Render a sample's contexts (stored as a joined string) as a short list.""" + text = str(raw or "").strip() + if not text: + return "（无检索片段）" + parts = [p.strip() for p in text.split(_CONTEXT_SEPARATOR) if p.strip()] + if not parts: + return "（无检索片段）" + rendered = "; ".join(f"[{i + 1}] {p}" for i, p in enumerate(parts)) + if len(rendered) > _CONTEXT_LIMIT: + rendered = rendered[:_CONTEXT_LIMIT] + "…" + return rendered + + def _build_low_sample_text(diagnoses: list[Diagnosis]) -> str: + """Render low-score samples (now including contexts) for grounding analysis.""" lines = [] for d in diagnoses: if not d.low_samples: @@ -61,24 +104,63 @@ def _build_low_sample_text(diagnoses: list[Diagnosis]) -> str: lines.append(f"### {d.metric} 低分样本（最多 3 条）") for i, s in enumerate(d.low_samples, 1): score = s.get(d.metric, "N/A") - lines.append(f"\n**样本 {i}**（分数={score}）") - lines.append(f"- 问题：{s.get('question', '')}") - lines.append(f"- 回答：{s.get('answer', '')[:300]}") - lines.append(f"- 标准答案：{s.get('ground_truth', '')[:200]}") + lines.append(f"\n**样本 {i}**（{d.metric}={score}）") + lines.append(f"- 问题 question：{s.get('question', '')}") + lines.append(f"- 生成答案 answer：{str(s.get('answer', ''))[:_ANSWER_LIMIT]}") + lines.append(f"- 检索片段 contexts：{_format_contexts(s.get('contexts'))}") + lines.append(f"- 标准答案 ground_truth：{str(s.get('ground_truth', ''))[:_GT_LIMIT]}") return "\n".join(lines) +def _is_reasoning_model(model: str) -> bool: + """Return True for OpenAI reasoning models (gpt-5+/o-series/codex-mini). + + These require `max_completion_tokens` instead of `max_tokens` and do not + accept a custom temperature. Mirrors RAGAS's own detection logic so advice + generation stays consistent with the scoring path. + """ + m = (model or "").lower() + # O-series: o1..o9 optionally followed by - or _ + if len(m) >= 2 and m[0] == "o" and m[1] in "123456789": + if len(m) == 2 or m[2] in ("-", "_"): + return True + # GPT-5 through GPT-19 generation + if m.startswith("gpt-"): + version_str = m[4:].split("-")[0].split("_")[0].split(".")[0] + try: + if 5 <= int(version_str) <= 19: + return True + except ValueError: + pass + if m == "codex-mini": + return True + return False + + +def _chat_token_kwargs(model: str, max_tokens: int) -> dict[str, Any]: + """Return the completion-budget kwargs appropriate for the model family.""" + if _is_reasoning_model(model): + # Reasoning models: only max_completion_tokens; temperature must stay default (1). + return {"max_completion_tokens": max_tokens} + return {"max_tokens": max_tokens, "temperature": 0.2} + + async def analyze( diagnoses: list[Diagnosis], - llm: Any, scenario_name: str, + judge_model: str, + settings: Any, + *, + chat_client: Any | None = None, ) -> str: - """Call the judge LLM to generate a Chinese optimization report. + """Call the judge LLM directly to generate a Chinese optimization report. Args: diagnoses: Non-empty list of Diagnosis from rules.diagnose(). - llm: RAGAS LLM wrapper (has .agenerate() method). scenario_name: Used only for logging. + judge_model: Model name to call (also selects the matching LLM profile). + settings: EvaluationSettings (supplies client kwargs + token budget). + chat_client: Optional pre-built AsyncOpenAI-compatible client (for tests). Returns: LLM-generated Markdown string, or "" on failure (triggers writer fallback). @@ -86,22 +168,47 @@ async def analyze( if not diagnoses: return "" - diagnosis_summary = _build_diagnosis_summary(diagnoses) - low_sample_text = _build_low_sample_text(diagnoses) prompt = _PROMPT_TEMPLATE.format( - diagnosis_summary=diagnosis_summary, - low_sample_text=low_sample_text, + diagnosis_summary=_build_diagnosis_summary(diagnoses), + low_sample_text=_build_low_sample_text(diagnoses), ) try: logger.info("[advisor] calling LLM for optimization analysis scenario=%s", scenario_name) - from langchain_core.messages import HumanMessage - # Use the underlying langchain chat model directly (RAGAS LangchainLLMWrapper wraps BaseChatModel) - response = await llm.langchain_llm.ainvoke([HumanMessage(content=prompt)]) - text = response.content.strip() - logger.info("[advisor] LLM analysis complete chars=%d", len(text)) - return text - except Exception as exc: + client = chat_client + owns_client = False + if client is None: + from openai import AsyncOpenAI + + from rag_eval.metrics.factory import resolve_openai_client_kwargs + + client = AsyncOpenAI(**resolve_openai_client_kwargs(judge_model, settings)) + owns_client = True + + try: + # Advice is a longer document than per-metric scoring; give it headroom. + max_tokens = max(2048, int(getattr(settings, "ragas_llm_max_tokens", 4096) or 4096)) + token_kwargs = _chat_token_kwargs(judge_model, max_tokens) + + response = await client.chat.completions.create( + model=judge_model, + messages=[{"role": "user", "content": prompt}], + **token_kwargs, + ) + text = (response.choices[0].message.content or "").strip() + logger.info("[advisor] LLM analysis complete chars=%d", len(text)) + return text + finally: + # Close the httpx connection pool inside THIS event loop. run_advisor + # drives analyze() via asyncio.run(), which closes the loop on return; + # a later GC-time aclose() on the dead loop would otherwise log + # "RuntimeError: Event loop is closed". Only close clients we created. + if owns_client: + try: + await client.close() + except Exception: # noqa: BLE001 + pass + except Exception as exc: # noqa: BLE001 logger.warning( "[advisor] LLM analysis failed (%s: %s) — falling back to rule report", type(exc).__name__, exc, diff --git a/rag_eval/advisor/rules.py b/rag_eval/advisor/rules.py index eee1829..fd26514 100644 --- a/rag_eval/advisor/rules.py +++ b/rag_eval/advisor/rules.py @@ -159,7 +159,8 @@ def _select_low_samples( valid = [r for r in rows if metric in r and not math.isnan(float(r[metric]))] sorted_rows = sorted(valid, key=lambda r: float(r[metric]), reverse=not higher_is_better) worst = sorted_rows[:top_n] - keep_keys = {"sample_id", "question", "answer", "ground_truth", metric} + # contexts is included so the LLM/fallback can judge grounding (retrieval vs generation). + keep_keys = {"sample_id", "question", "answer", "ground_truth", "contexts", metric} return [{k: v for k, v in row.items() if k in keep_keys} for row in worst] diff --git a/rag_eval/advisor/writer.py b/rag_eval/advisor/writer.py index b60e6dd..a60bd3c 100644 --- a/rag_eval/advisor/writer.py +++ b/rag_eval/advisor/writer.py @@ -29,10 +29,15 @@ def _format_log_summary(diagnoses: list[Diagnosis], advice_path: Path) -> str: def _build_fallback_report(diagnoses: list[Diagnosis]) -> str: - """Build a rules-only report when LLM analysis is unavailable.""" + """Build a rules-only report when LLM analysis is unavailable. + + Even without the LLM, embed each metric's worst sample(s) — question, + answer, ground truth — so the advice still references concrete problems + instead of reading as a purely generic template. + """ if not diagnoses: return "" - lines = ["## 规则诊断（LLM 分析不可用）\n"] + lines = ["## 规则诊断（LLM 分析不可用，以下为规则引擎输出）\n"] for d in diagnoses: label = _SEVERITY_LABEL.get(d.severity, d.severity) lines.append(f"### {d.metric} [{label}] 均值={d.mean_score:.4f}") @@ -42,6 +47,22 @@ def _build_fallback_report(diagnoses: list[Diagnosis]) -> str: lines.append("\n**建议动作：**") for action in d.suggested_actions: lines.append(f"- {action}") + if d.low_samples: + lines.append("\n**低分样本举例拆解：**") + for i, sample in enumerate(d.low_samples, 1): + score = sample.get(d.metric, "N/A") + question = str(sample.get("question", "")).strip() or "（无问题文本）" + lines.append(f"\n- **样本 {i}**（{d.metric}={score}）问题：{question}") + answer = str(sample.get("answer", "")).strip() + if answer: + lines.append(f" - 生成答案：{answer[:200]}") + ground_truth = str(sample.get("ground_truth", "")).strip() + if ground_truth: + lines.append(f" - 标准答案：{ground_truth[:160]}") + lines.append( + f" - 拆解：该样本 {d.metric} 偏低，请对照上述「可能原因 / 建议动作」" + f"重点排查本问题的检索片段与生成答案。" + ) lines.append("") return "\n".join(lines) diff --git a/rag_eval/execution/runner.py b/rag_eval/execution/runner.py index a1e4b03..e119e0e 100644 --- a/rag_eval/execution/runner.py +++ b/rag_eval/execution/runner.py @@ -79,6 +79,6 @@ def run_scenario( logger.info("[runner] artifacts written for run_id=%s", result.run_id) # Optimization advisor — runs only if scenario.optimization_advisor is True. - run_advisor(result, scenario, llm) + run_advisor(result, scenario, settings=settings) return result diff --git a/tests/test_metric_presenter.py b/tests/test_metric_presenter.py index d4f94f8..baf87c3 100644 --- a/tests/test_metric_presenter.py +++ b/tests/test_metric_presenter.py @@ -40,7 +40,14 @@ const result = {{ desc: p.describeMetric("faithfulness"), noiseDesc: p.describeMetric("noise_sensitivity"), noiseBin: p.binColor("noise_sensitivity", 0.0), - faithBin: p.binColor("faithfulness", 0.8) + faithBin: p.binColor("faithfulness", 0.8), + lowerBetterNoise: p.isLowerBetter("noise_sensitivity"), + lowerBetterFaith: p.isLowerBetter("faithfulness"), + upHigher: p.deltaInfo("faithfulness", 0.80, 0.60), + downHigher: p.deltaInfo("faithfulness", 0.60, 0.80), + noiseImproved: p.deltaInfo("noise_sensitivity", 0.10, 0.30), + noiseWorse: p.deltaInfo("noise_sensitivity", 0.30, 0.10), + noBaseline: p.deltaInfo("faithfulness", 0.80, null) }}; console.log(JSON.stringify(result)); """ @@ -55,6 +62,15 @@ console.log(JSON.stringify(result)); assert '"noiseDesc":"' in output assert '"noiseBin":"#16a34a"' in output assert '"faithBin":"#16a34a"' in output + assert '"lowerBetterNoise":true' in output + assert '"lowerBetterFaith":false' in output + # higher-better: rising value is an improvement (green ▲); falling is a regression (red ▼) + assert '"upHigher":{"hasData":true,"delta":0.2,"improved":true,"arrow":"▲","magnitude":"0.20","cls":"delta-good"}' in output + assert '"downHigher":{"hasData":true,"delta":-0.2,"improved":false,"arrow":"▼","magnitude":"0.20","cls":"delta-bad"}' in output + # noise_sensitivity (lower-better): falling value is an improvement (green ▼) + assert '"noiseImproved":{"hasData":true,"delta":-0.2,"improved":true,"arrow":"▼","magnitude":"0.20","cls":"delta-good"}' in output + assert '"noiseWorse":{"hasData":true,"delta":0.2,"improved":false,"arrow":"▲","magnitude":"0.20","cls":"delta-bad"}' in output + assert '"noBaseline":{"hasData":false' in output def test_report_and_index_load_metric_presenter_helper() -> None: @@ -66,3 +82,6 @@ def test_report_and_index_load_metric_presenter_helper() -> None: assert "js/metric_presenter.js" in index_html assert "MetricPresenter.describeMetric" in report_js assert "MetricPresenter.scoreClass" in app_js + # history comparison table uses the direction-aware delta helper + assert "MetricPresenter.deltaInfo" in report_js + assert "history-table" in report_js diff --git a/tests/test_webapp_report_builder.py b/tests/test_webapp_report_builder.py index b1be92d..2b60f4f 100644 --- a/tests/test_webapp_report_builder.py +++ b/tests/test_webapp_report_builder.py @@ -4,8 +4,11 @@ from __future__ import annotations from pathlib import Path +import pandas as pd import pytest +from webapp.services import question_history +from webapp.services import report_builder from webapp.services.report_builder import build_report from webapp.services.run_reader import _infer_metrics_from_scores, _read_weights_from_snapshot @@ -115,3 +118,71 @@ def test_build_report_ranks_noise_sensitivity_with_lower_values_as_better(tmp_pa "s-warn", "s-good", ] + + +def test_lowest_samples_attaches_history_from_index() -> None: + """Surfaced samples are annotated with the same question's prior-run scores.""" + frame = pd.DataFrame( + [ + {"sample_id": "s1", "question": " How LONG is the tube? ", "faithfulness": 0.40}, + {"sample_id": "s2", "question": "unrelated question", "faithfulness": 0.30}, + ] + ) + history_index = { + question_history.normalize_question("How long is the tube?"): [ + { + "run_id": "prev", + "scenario_name": "scn", + "finished_at": "2026-01-01T00:00:00", + "metrics": {"faithfulness": 0.90}, + } + ] + } + + samples = report_builder._lowest_samples(frame, ["faithfulness"], history_index) + by_id = {s.sample_id: s for s in samples} + + assert len(by_id["s1"].history) == 1 + assert by_id["s1"].history[0].run_id == "prev" + assert by_id["s1"].history[0].metrics["faithfulness"] == 0.90 + assert by_id["s2"].history == [] # no match → no history + + +def test_build_report_attaches_question_history(tmp_path: Path, monkeypatch) -> None: + """build_report wires the question-history index into surfaced samples.""" + run_dir = tmp_path / "run" + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "scores.csv").write_text( + "\n".join( + [ + "sample_id,question,faithfulness", + "s1,How long is the tube?,0.40", + ] + ), + encoding="utf-8", + ) + (run_dir / "summary.md").write_text("summary", encoding="utf-8") + (run_dir / "optimization_advice.md").write_text("", encoding="utf-8") + + captured: dict = {} + + def _fake_index(exclude_run_id=None, extra_roots=None): + captured["exclude_run_id"] = exclude_run_id + return { + question_history.normalize_question("How long is the tube?"): [ + { + "run_id": "older", + "scenario_name": "scn", + "finished_at": "2026-01-01T00:00:00", + "metrics": {"faithfulness": 0.95}, + } + ] + } + + monkeypatch.setattr(question_history, "build_question_history_index", _fake_index) + + report = build_report(run_dir, ["faithfulness"]) + + assert captured["exclude_run_id"] == "run" # current run excluded from history + assert report.lowest_samples[0].history[0].run_id == "older" + assert report.lowest_samples[0].history[0].metrics["faithfulness"] == 0.95 diff --git a/webapp/services/report_builder.py b/webapp/services/report_builder.py index 0a1e204..989fb55 100644 --- a/webapp/services/report_builder.py +++ b/webapp/services/report_builder.py @@ -23,9 +23,10 @@ from webapp.models import ( DistributionBin, GroupStat, ReportData, + SampleHistoryEntry, SampleScore, ) -from webapp.services import run_reader +from webapp.services import question_history, run_reader # Number of equal-width buckets used for metric score histograms. @@ -37,6 +38,9 @@ GROUPING_FIELDS = ("difficulty", "question_type", "language") # How many lowest-scoring samples to surface for manual review. LOWEST_SAMPLE_COUNT = 10 +# How many past evaluations of the same question to show in the history table. +HISTORY_LIMIT = 5 + # Metrics whose lower raw value means stronger performance. LOWER_IS_BETTER_METRICS = {"noise_sensitivity"} @@ -124,8 +128,16 @@ def _cell_text(row: pd.Series, column: str) -> str: return str(row[column]).strip() -def _lowest_samples(frame: pd.DataFrame, metrics: list[str]) -> list[SampleScore]: - """Select and shape the lowest-scoring samples for the review table.""" +def _lowest_samples( + frame: pd.DataFrame, + metrics: list[str], + history_index: dict[str, list[dict]] | None = None, +) -> list[SampleScore]: + """Select and shape the lowest-scoring samples for the review table. + + When a history_index is supplied, each surfaced sample is annotated with the + same question's scores from previous runs (newest first) for comparison. + """ if frame.empty: return [] @@ -154,7 +166,19 @@ def _lowest_samples(frame: pd.DataFrame, metrics: list[str]) -> list[SampleScore enriched.append((sort_key, sample)) enriched.sort(key=lambda item: item[0]) - return [sample for _, sample in enriched[:LOWEST_SAMPLE_COUNT]] + selected = [sample for _, sample in enriched[:LOWEST_SAMPLE_COUNT]] + + # Attach per-question history only for the surfaced samples (keeps lookups cheap). + if history_index is not None: + for sample in selected: + if not sample.question: + continue + entries = question_history.lookup( + history_index, sample.question, limit=HISTORY_LIMIT + ) + sample.history = [SampleHistoryEntry(**entry) for entry in entries] + + return selected def build_report(run_dir: Path, metrics: list[str]) -> ReportData: @@ -192,12 +216,20 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData: if metric in frame.columns } + # Cross-run history: scores of the same question in *other* runs (Approach A — + # on-demand global scan, excluding the run currently being viewed). + metadata = run_reader._read_json(run_dir / "metadata.json") + current_run_id = str(metadata.get("run_id") or run_dir.name) + history_index = question_history.build_question_history_index( + exclude_run_id=current_run_id + ) + return ReportData( metrics=metrics, metric_means=rounded_means, distributions=distributions, groupings=_groupings(frame, metrics), - lowest_samples=_lowest_samples(frame, metrics), + lowest_samples=_lowest_samples(frame, metrics, history_index), summary_markdown=summary_markdown, advice_markdown=advice_markdown, weighted_score_mean=_round_or_none(overall_ws), diff --git a/webapp/services/score_job_manager.py b/webapp/services/score_job_manager.py index a3964a1..f538b5f 100644 --- a/webapp/services/score_job_manager.py +++ b/webapp/services/score_job_manager.py @@ -107,7 +107,6 @@ class ScoreJobManager: # Lazy imports to keep web server bootable if ragas is not installed. from rag_eval.advisor import run_advisor - from rag_eval.metrics.factory import build_models from rag_eval.metrics.weights import compute_weighted_score from rag_eval.reporting.writers import write_run_artifacts from rag_eval.settings import EvaluationSettings @@ -206,8 +205,7 @@ class ScoreJobManager: # Run optimization advisor (builds optimization_advice.md) try: - llm, _ = build_models(judge_model, embedding_model, settings) - run_advisor(result, scenario, llm) + run_advisor(result, scenario, settings=settings) logger.info("[score_job] advisor done job_id=%s", job_id) except Exception as adv_exc: # noqa: BLE001 logger.warning("[score_job] advisor failed job_id=%s err=%s", job_id, adv_exc) diff --git a/webapp/services/session_score_manager.py b/webapp/services/session_score_manager.py index d145975..82c5785 100644 --- a/webapp/services/session_score_manager.py +++ b/webapp/services/session_score_manager.py @@ -192,7 +192,6 @@ class SessionScoreJobManager: # Lazy imports — keep web server bootable if ragas is not installed. from rag_eval.advisor import run_advisor - from rag_eval.metrics.factory import build_models from rag_eval.metrics.weights import compute_weighted_score from rag_eval.reporting.writers import write_run_artifacts from rag_eval.settings import EvaluationSettings @@ -320,8 +319,7 @@ class SessionScoreJobManager: # Regenerate optimization advice over all accumulated rows try: - llm, _ = build_models(judge_model, embedding_model, settings) - run_advisor(result, scenario, llm) + run_advisor(result, scenario, settings=settings) logger.info("[session_job] advisor done job_id=%s session=%s", job_id, session_id) except Exception as adv_exc: # noqa: BLE001 logger.warning( diff --git a/webapp/static/css/app.css b/webapp/static/css/app.css index 7282750..5661e8b 100644 --- a/webapp/static/css/app.css +++ b/webapp/static/css/app.css @@ -253,6 +253,23 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t } .detail-gt { color: var(--good); } +/* 历史评分小表格：本次行高亮 + 涨跌着色（绿=改善 红=退步） */ +table.history-table { width: 100%; border-collapse: collapse; font-size: 12px; margin-top: 4px; } +table.history-table th, table.history-table td { + padding: 5px 8px; text-align: left; border-bottom: 1px solid #f1f5f9; +} +table.history-table th { color: var(--slate); font-weight: 600; border-bottom: 1px solid var(--line); } +table.history-table td { font-variant-numeric: tabular-nums; } +.history-table tr.hist-current { background: #f0f9ff; } +.history-table tr.hist-current .hist-label { font-weight: 700; color: #0369a1; } +.hist-when { white-space: nowrap; } +.hist-label { display: inline-block; } +.hist-sub { display: block; font-size: 11px; color: var(--slate-light); } +.hist-delta { font-size: 11px; font-weight: 700; font-variant-numeric: tabular-nums; } +.hist-delta.delta-good { color: #16a34a; } +.hist-delta.delta-bad { color: #dc2626; } +.hist-delta.delta-flat { color: var(--slate-light); } + .empty { text-align: center; padding: 60px 20px; color: var(--slate); } .empty p { margin-bottom: 8px; } @@ -514,6 +531,14 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t table.group-table td { padding: 4pt 6pt; border-bottom: 1px solid #e2e8f0; } table.group-table th { font-weight: 700; color: #64748b; } + /* ── 历史评分表 ── */ + table.history-table { width: 100%; font-size: 9pt; border-collapse: collapse; } + table.history-table th, + table.history-table td { padding: 3pt 6pt; border-bottom: 1px solid #e2e8f0; } + .history-table tr.hist-current { background: #f0f9ff !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; } + .hist-delta.delta-good { color: #16a34a !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; } + .hist-delta.delta-bad { color: #dc2626 !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; } + /* ── 颜色保留（部分浏览器打印默认去色） ── */ .good { color: #16a34a !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; } .warn { color: #eab308 !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; } @@ -546,3 +571,67 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t .advice-md ul { padding-left: 20px; margin: 6px 0; } .advice-md li { margin: 3px 0; font-size: 13px; } .advice-md strong { color: var(--ink); font-weight: 600; } + +/* ---------- 指标看板 Dashboard ---------- */ +.dashboard-charts { + display: flex; + flex-direction: column; + gap: 20px; +} +.dashboard-chart-panel { + min-width: 0; + /* 上下布局时给图表面板稍微更宽裕的高度 */ +} +.dashboard-chart-panel canvas { + max-height: 340px; + height: 320px !important; +} + +/* 运行选择器列表 */ +.db-run-list { + display: flex; + flex-direction: column; + gap: 5px; + max-height: 240px; + overflow-y: auto; + margin-top: 10px; + padding-right: 4px; +} +.db-run-row { + display: flex; + align-items: center; + gap: 10px; + padding: 8px 12px; + border: 1px solid var(--line); + border-radius: 8px; + cursor: pointer; + transition: background 0.12s, border-color 0.12s; + background: var(--surface); +} +.db-run-row:hover { background: #f0fbfb; border-color: var(--petrol); } +.db-run-row:has(input:checked) { + background: #e8f7f7; + border-color: #7ecece; +} +.db-run-row input[type="checkbox"] { flex-shrink: 0; accent-color: var(--petrol); width: 15px; height: 15px; } +.db-run-label { display: flex; flex-direction: column; gap: 2px; flex: 1; min-width: 0; } +.db-run-name { font-size: 13px; font-weight: 600; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +.db-run-chips { display: flex; flex-wrap: wrap; gap: 6px; flex-shrink: 0; } +.db-chip-name { color: var(--slate); } +.btn-sm { padding: 5px 12px; font-size: 12px; } + +/* 看板图表面板头：标题左 + 下拉右 对齐优化 */ +.db-panel-head-bar { + display: flex; + align-items: center; + justify-content: space-between; + flex-wrap: wrap; + gap: 8px; + margin-bottom: 14px; +} +.db-chart-hint { + font-size: 11px; + color: var(--slate-light); + margin-top: 6px; + text-align: center; +} diff --git a/webapp/static/index.html b/webapp/static/index.html index 5cdd311..02c1fa4 100644 --- a/webapp/static/index.html +++ b/webapp/static/index.html @@ -31,6 +31,9 @@ + @@ -263,6 +266,11 @@ allowfullscreen> + + + @@ -272,6 +280,7 @@ + diff --git a/webapp/static/js/app.js b/webapp/static/js/app.js index bdfcb0c..7cbf6f0 100644 --- a/webapp/static/js/app.js +++ b/webapp/static/js/app.js @@ -5,8 +5,8 @@ const App = { currentRunId: null, activeView: null, - views: ["runs", "new", "report", "profiles", "scorejobs", "apidocs"], - titles: { runs: "运行列表", new: "新建评估", report: "报告详情", profiles: "LLM 配置", scorejobs: "评分记录", apidocs: "API 文档" }, + views: ["runs", "new", "report", "profiles", "scorejobs", "dashboard", "apidocs"], + titles: { runs: "运行列表", new: "新建评估", report: "报告详情", profiles: "LLM 配置", scorejobs: "评分记录", dashboard: "指标看板", apidocs: "API 文档" }, // 初始化：绑定导航、从 URL/sessionStorage 恢复上次位置、启动健康检查。 init() { @@ -73,6 +73,7 @@ const App = { if (view === "report") Report.render(App.currentRunId); if (view === "profiles") Profiles.load(); if (view === "scorejobs") ScoreJobs.load(); + if (view === "dashboard") Dashboard.load(); }, // ---------------------------------------------------------------- diff --git a/webapp/static/js/metric_presenter.js b/webapp/static/js/metric_presenter.js index 99a5b0b..d6242ef 100644 --- a/webapp/static/js/metric_presenter.js +++ b/webapp/static/js/metric_presenter.js @@ -69,9 +69,42 @@ return "#dc2626"; } + // 计算某指标本次相对上一次的涨跌信息，方向语义随指标而定 + // （noise_sensitivity 越低越好：下降=改善）。 + function deltaInfo(metricName, current, previous) { + const isNum = (v) => v !== null && v !== undefined && !Number.isNaN(Number(v)); + if (!isNum(current) || !isNum(previous)) { + return { hasData: false, delta: null, improved: null, arrow: "", magnitude: "", cls: "delta-flat" }; + } + const delta = Number(current) - Number(previous); + const rounded = Math.round(delta * 10000) / 10000; + const arrow = rounded > 0 ? "▲" : rounded < 0 ? "▼" : "→"; + const magnitude = Math.abs(rounded).toFixed(2); + const improved = isLowerBetter(metricName) ? rounded < 0 : rounded > 0; + const cls = rounded === 0 ? "delta-flat" : improved ? "delta-good" : "delta-bad"; + return { hasData: true, delta: rounded, improved, arrow, magnitude, cls }; + } + + // 返回指标的"达标阈值"（柱状图对比用，方向感知）。 + // higher-better 指标：0.85；lower-better (noise_sensitivity)：0.15。 + function passThreshold(metricName) { + return isLowerBetter(metricName) ? 0.15 : 0.85; + } + + // 判断某指标的值是否达标。 + function meetsTarget(metricName, value) { + if (value === null || value === undefined || Number.isNaN(Number(value))) return false; + const v = Number(value); + return isLowerBetter(metricName) ? v <= passThreshold(metricName) : v >= passThreshold(metricName); + } + globalObj.MetricPresenter = { scoreClass, describeMetric, binColor, + isLowerBetter, + deltaInfo, + passThreshold, + meetsTarget, }; })(window); diff --git a/webapp/static/js/report.js b/webapp/static/js/report.js index 42c5287..6a2a2ed 100644 --- a/webapp/static/js/report.js +++ b/webapp/static/js/report.js @@ -283,7 +283,7 @@ const Report = { const detail = document.createElement("div"); detail.className = "lowest-detail"; detail.hidden = true; - detail.innerHTML = Report._detailHtml(sample); + detail.innerHTML = Report._detailHtml(sample, metrics); row.addEventListener("click", () => { detail.hidden = !detail.hidden; @@ -293,8 +293,8 @@ const Report = { }); }, - // 单条样本的展开详情：question / contexts / answer / ground_truth。 - _detailHtml(sample) { + // 单条样本的展开详情：question / contexts / answer / ground_truth / 历史评分。 + _detailHtml(sample, metrics) { const contexts = (sample.contexts || []) .map((c, i) => `
[${i + 1}] ${App.escape(c)}
`) .join(""); @@ -320,6 +320,63 @@ const Report = {
${App.escape(sample.ground_truth || "—")}
${errorBlock} + ${Report._historyHtml(sample, metrics || [])} + + `; + }, + + // 同一问题的历史评分小表格：本次 + 历次（按时间倒序），逐行标注较更早一次的涨跌。 + _historyHtml(sample, metrics) { + const history = sample.history || []; + if (!history.length) return ""; + + // 只展示当前样本与历史中实际出现过的指标列，避免空列。 + const cols = metrics.filter( + (m) => + (sample.metrics && sample.metrics[m] !== undefined && sample.metrics[m] !== null) || + history.some((h) => h.metrics && h.metrics[m] !== undefined && h.metrics[m] !== null), + ); + if (!cols.length) return ""; + + // 组合行：[本次, 历次...]，相邻两行做涨跌对比（行 r 对比更早的行 r+1）。 + const rows = [ + { label: "本次", sub: "", metrics: sample.metrics || {}, current: true }, + ...history.map((h) => ({ + label: App.escape(h.scenario_name || h.run_id || "历史"), + sub: App.escape(App.shortTime(h.finished_at)), + metrics: h.metrics || {}, + current: false, + })), + ]; + + let head = "评测"; + cols.forEach((m) => (head += `${App.escape(App.shortMetric(m))}`)); + head += ""; + + let body = ""; + rows.forEach((row, r) => { + const older = rows[r + 1]; + body += ``; + body += `${row.label}${row.sub ? `${row.sub}` : ""}`; + cols.forEach((m) => { + const v = row.metrics ? row.metrics[m] : null; + const cls = App.scoreClass(m, v); + const text = v === null || v === undefined ? "—" : Number(v).toFixed(2); + let deltaHtml = ""; + const baseline = older && older.metrics ? older.metrics[m] : undefined; + const d = MetricPresenter.deltaInfo(m, v, baseline); + if (d.hasData && d.delta !== 0) { + deltaHtml = ` ${d.arrow}${d.magnitude}`; + } + body += `${text}${deltaHtml}`; + }); + body += ""; + }); + + return ` +
+
历史评分 history（同一问题，最近 ${history.length} 次，含本次对比）
+ ${head}${body}
`; },