Extract shared build_metric_registry factory (DRY)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

webapp/services/inline_scorer.py

@@ -13,35 +13,17 @@ import threading
 from typing import Any
 
 from rag_eval.compat import ensure_ragas_import_compat
-from rag_eval.metrics.factory import build_models
+from rag_eval.metrics.factory import build_metric_registry, build_models
 from rag_eval.metrics.pipeline import MetricPipeline
 from rag_eval.settings import EvaluationSettings
 from rag_eval.shared.models import NormalizedSample
 
 ensure_ragas_import_compat()
 
-from ragas.metrics.collections import (  # noqa: E402
-    AnswerRelevancy,
-    ContextPrecision,
-    ContextRecall,
-    FactualCorrectness,
-    Faithfulness,
-    NoiseSensitivity,
-    SemanticSimilarity,
-)
-
 
 def _build_metric_instances(metrics: list[str], llm: Any, embeddings: Any) -> dict[str, Any]:
     """Instantiate only the RAGAS metric objects requested."""
-    registry: dict[str, Any] = {
-        "faithfulness": Faithfulness(llm=llm),
-        "answer_relevancy": AnswerRelevancy(llm=llm, embeddings=embeddings),
-        "context_recall": ContextRecall(llm=llm),
-        "context_precision": ContextPrecision(llm=llm),
-        "noise_sensitivity": NoiseSensitivity(llm=llm),
-        "factual_correctness": FactualCorrectness(llm=llm),
-        "semantic_similarity": SemanticSimilarity(embeddings=embeddings),
-    }
+    registry = build_metric_registry(llm, embeddings)
     return {name: registry[name] for name in metrics if name in registry}
 
 

webapp/services/question_history.py

@@ -0,0 +1,102 @@
+"""Build a cross-run index of per-question RAGAS scores for historical comparison.
+
+The report detail page surfaces, for each low-scoring sample, how the same
+question scored in previous evaluations. Matching is by normalized question text
+(case-insensitive, whitespace-collapsed) across all discovered run directories,
+so a question evaluated in any earlier run shows up as history.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from webapp.services import run_reader
+from webapp.services.run_reader import NON_METRIC_COLUMNS, _read_json
+
+
+def normalize_question(question: Any) -> str:
+    """Return a stable match key for a question (case/whitespace-insensitive)."""
+    return " ".join(str(question or "").split()).lower()
+
+
+def _row_metrics(row: dict[str, Any]) -> dict[str, float | None]:
+    """Extract numeric metric scores from a single scores.csv row."""
+    metrics: dict[str, float | None] = {}
+    for key, value in row.items():
+        if key in NON_METRIC_COLUMNS:
+            continue
+        try:
+            num = float(value)
+        except (TypeError, ValueError):
+            continue
+        if pd.isna(num):
+            continue
+        metrics[str(key)] = round(num, 4)
+    return metrics
+
+
+def build_question_history_index(
+    exclude_run_id: str | None = None,
+    extra_roots: list[Path] | None = None,
+) -> dict[str, list[dict[str, Any]]]:
+    """Scan all run dirs and group per-question score entries for history lookup.
+
+    Args:
+        exclude_run_id: Run whose rows are skipped, so "history" means *other*
+            evaluations (typically the run currently being viewed).
+        extra_roots: Additional output roots to scan (beyond the defaults).
+
+    Returns:
+        Map of normalized_question -> list of entries, each shaped as
+        ``{run_id, scenario_name, finished_at, metrics: {metric: value}}`` and
+        sorted by finished_at descending (most recent first). Within a single
+        run, only the last occurrence of a question is kept.
+    """
+    # question_key -> run_id -> entry (last write wins within the same run)
+    grouped: dict[str, dict[str, dict[str, Any]]] = {}
+
+    for run_dir in run_reader.discover_run_dirs(extra_roots):
+        metadata = _read_json(run_dir / "metadata.json")
+        run_id = str(metadata.get("run_id") or run_dir.name)
+        if exclude_run_id and run_id == exclude_run_id:
+            continue
+        scenario_name = str(metadata.get("scenario_name", ""))
+        finished_at = str(metadata.get("finished_at") or metadata.get("started_at") or "")
+
+        frame = run_reader.read_scores_frame(run_dir)
+        if frame.empty or "question" not in frame.columns:
+            continue
+
+        for record in frame.where(pd.notnull(frame), None).to_dict("records"):
+            key = normalize_question(record.get("question"))
+            if not key:
+                continue
+            metrics = _row_metrics(record)
+            if not metrics:
+                continue
+            grouped.setdefault(key, {})[run_id] = {
+                "run_id": run_id,
+                "scenario_name": scenario_name,
+                "finished_at": finished_at,
+                "metrics": metrics,
+            }
+
+    index: dict[str, list[dict[str, Any]]] = {}
+    for key, per_run in grouped.items():
+        entries = list(per_run.values())
+        entries.sort(key=lambda entry: entry["finished_at"], reverse=True)
+        index[key] = entries
+    return index
+
+
+def lookup(
+    index: dict[str, list[dict[str, Any]]],
+    question: Any,
+    limit: int = 5,
+) -> list[dict[str, Any]]:
+    """Return up to ``limit`` historical entries for a question (newest first)."""
+    entries = index.get(normalize_question(question), [])
+    return entries[: max(0, limit)]

webapp/static/js/dashboard.js

@@ -0,0 +1,418 @@
+// dashboard.js — 指标看板：运行选择器 + 折线图（指标趋势） + 柱状图（vs 达标阈值）。
+// 纯前端，数据来自 GET /api/runs，复用 MetricPresenter 的方向语义与阈值。
+
+(function attachDashboard(globalObj) {
+const Dashboard = {
+  _runs: [],          // 全量 runs（已按 finished_at 倒序来自 API）
+  _selected: new Set(), // 当前勾选的 run_id 集合
+  _focusId: null,     // 柱状图聚焦的 run_id
+  _trendChart: null,
+  _barChart: null,
+
+  // ── 入口 ─────────────────────────────────────────────────────────────────
+
+  async load() {
+    const wrap = document.getElementById("dashboard-wrap");
+    if (!wrap) return;
+    wrap.innerHTML = '<p class="muted">加载中…</p>';
+
+    try {
+      const data = await API.runs();
+      Dashboard._runs = (data.runs || []).slice().sort(
+        (a, b) => (a.finished_at || "").localeCompare(b.finished_at || "")
+      );
+      Dashboard._selected = new Set(Dashboard._runs.map((r) => r.run_id));
+      Dashboard._focusId = Dashboard._runs.length
+        ? Dashboard._runs[Dashboard._runs.length - 1].run_id
+        : null;
+      Dashboard._render(wrap);
+    } catch (err) {
+      wrap.innerHTML = `<p class="muted">加载失败：${App.escape(err.message)}</p>`;
+    }
+  },
+
+  // ── 渲染 ─────────────────────────────────────────────────────────────────
+
+  _render(wrap) {
+    if (!Dashboard._runs.length) {
+      wrap.innerHTML = `
+        <div class="empty">
+          <p>暂无评测运行数据。</p>
+          <p class="muted">触发一次评测或通过 Dify 工具调用后，数据将在此显示。</p>
+        </div>`;
+      return;
+    }
+
+    wrap.innerHTML = "";
+
+    // 运行选择器面板
+    wrap.appendChild(Dashboard._buildSelector());
+
+    // 图表区
+    const chartRow = document.createElement("div");
+    chartRow.className = "dashboard-charts";
+    chartRow.innerHTML = `
+      <div class="panel dashboard-chart-panel">
+        <div class="db-panel-head-bar">
+          <div>
+            <div class="section-label tight">📈 指标趋势折线图</div>
+            <div class="muted" style="font-size:12px;margin-top:2px">按时间顺序展示所选运行的各指标均值变化</div>
+          </div>
+        </div>
+        <canvas id="db-trend-chart"></canvas>
+        <p class="db-chart-hint" id="db-trend-hint"></p>
+      </div>
+      <div class="panel dashboard-chart-panel">
+        <div class="db-panel-head-bar">
+          <div>
+            <div class="section-label tight">📊 指标达标对比柱状图</div>
+            <div class="muted" style="font-size:12px;margin-top:2px">实际均值 vs 达标阈值（深绿柱）</div>
+          </div>
+          <select class="select" id="db-focus-select" style="min-width:220px"></select>
+        </div>
+        <canvas id="db-bar-chart"></canvas>
+        <p class="db-chart-hint">达标阈值：higher-better 指标 0.85 · noise_sensitivity 0.15</p>
+      </div>
+    `;
+    wrap.appendChild(chartRow);
+
+    Dashboard._populateFocusSelect();
+    Dashboard._drawTrend();
+    Dashboard._drawBar();
+  },
+
+  // 运行选择器
+  _buildSelector() {
+    const panel = document.createElement("div");
+    panel.className = "panel";
+    panel.innerHTML = `
+      <div class="panel-head">
+        <div>
+          <span class="section-label tight">运行选择（折线图数据源）</span>
+          <span class="muted" style="font-size:12px; margin-left:10px">勾选≥2个运行可看趋势</span>
+        </div>
+        <div style="display:flex;gap:8px">
+          <button class="btn btn-sm" id="db-sel-all">全选</button>
+          <button class="btn btn-sm" id="db-sel-none">清空</button>
+          <input class="form-input" id="db-filter-input" placeholder="按场景名过滤…" style="width:180px;padding:6px 10px" />
+        </div>
+      </div>
+      <div class="db-run-list" id="db-run-list"></div>
+    `;
+    setTimeout(() => {
+      Dashboard._renderRunList();
+      document.getElementById("db-sel-all").onclick = () => {
+        Dashboard._runs.forEach((r) => Dashboard._selected.add(r.run_id));
+        Dashboard._renderRunList();
+        Dashboard._drawTrend();
+      };
+      document.getElementById("db-sel-none").onclick = () => {
+        Dashboard._selected.clear();
+        Dashboard._renderRunList();
+        Dashboard._drawTrend();
+      };
+      document.getElementById("db-filter-input").oninput = (e) => {
+        Dashboard._renderRunList(e.target.value.toLowerCase());
+      };
+    });
+    return panel;
+  },
+
+  _renderRunList(filter) {
+    const list = document.getElementById("db-run-list");
+    if (!list) return;
+    list.innerHTML = "";
+    const visible = filter
+      ? Dashboard._runs.filter((r) =>
+          (r.scenario_name || r.run_id).toLowerCase().includes(filter)
+        )
+      : Dashboard._runs;
+    [...visible].reverse().forEach((run) => {
+      const row = document.createElement("label");
+      row.className = "db-run-row";
+      const chips = (run.metrics || [])
+        .slice(0, 4)
+        .map((m) => {
+          const v = run.metric_means ? run.metric_means[m] : null;
+          const cls = App.scoreClass(m, v);
+          const text = v === null || v === undefined ? "n/a" : Number(v).toFixed(2);
+          return `<span class="metric-chip"><span class="db-chip-name">${App.escape(App.shortMetric(m))}</span> <b class="${cls}">${text}</b></span>`;
+        })
+        .join("");
+      row.innerHTML = `
+        <input type="checkbox" class="db-run-cb" data-id="${App.escape(run.run_id)}"
+          ${Dashboard._selected.has(run.run_id) ? "checked" : ""} />
+        <span class="db-run-label">
+          <span class="db-run-name">${App.escape(run.scenario_name || run.run_id)}</span>
+          <span class="muted" style="font-size:11px">${App.escape(App.shortTime(run.finished_at))} · ${App.escape(run.judge_model || "")}</span>
+        </span>
+        <span class="db-run-chips">${chips}</span>
+      `;
+      row.querySelector(".db-run-cb").addEventListener("change", (e) => {
+        if (e.target.checked) Dashboard._selected.add(run.run_id);
+        else Dashboard._selected.delete(run.run_id);
+        Dashboard._drawTrend();
+      });
+      list.appendChild(row);
+    });
+  },
+
+  // 填充柱状图聚焦下拉
+  _populateFocusSelect() {
+    const sel = document.getElementById("db-focus-select");
+    if (!sel) return;
+    sel.innerHTML = "";
+    [...Dashboard._runs].reverse().forEach((run) => {
+      const opt = document.createElement("option");
+      opt.value = run.run_id;
+      opt.textContent = `${run.scenario_name || run.run_id}  ${App.shortTime(run.finished_at)}`;
+      if (run.run_id === Dashboard._focusId) opt.selected = true;
+      sel.appendChild(opt);
+    });
+    sel.onchange = () => {
+      Dashboard._focusId = sel.value;
+      Dashboard._drawBar();
+    };
+  },
+
+  // ── 折线图 ────────────────────────────────────────────────────────────────
+
+  _drawTrend() {
+    const canvas = document.getElementById("db-trend-chart");
+    const hint = document.getElementById("db-trend-hint");
+    if (!canvas) return;
+
+    const selected = Dashboard._runs.filter((r) => Dashboard._selected.has(r.run_id));
+    if (selected.length === 0) {
+      if (Dashboard._trendChart) { Dashboard._trendChart.destroy(); Dashboard._trendChart = null; }
+      if (hint) hint.textContent = "请在上方勾选至少 1 个运行。";
+      return;
+    }
+    if (hint) {
+      hint.textContent = selected.length === 1
+        ? "只有 1 个运行，折线退化为单点——勾选更多运行可看趋势。"
+        : `共 ${selected.length} 个运行 · 横轴按完成时间升序`;
+    }
+
+    const { labels, datasets } = Dashboard._buildTrendDatasets(selected);
+    // 精选对比度高、色盲友好的颜色组合
+    const colors = [
+      "#009999", // petrol brand
+      "#3b82f6", // blue
+      "#f97316", // orange
+      "#8b5cf6", // violet
+      "#ec4899", // pink
+      "#06b6d4", // cyan
+      "#f59e0b", // amber
+    ];
+
+    if (Dashboard._trendChart) Dashboard._trendChart.destroy();
+    Dashboard._trendChart = new Chart(canvas, {
+      type: "line",
+      data: {
+        labels,
+        datasets: [
+          // 达标参考线（0.85，虚线，不显示在图例前列）
+          {
+            label: "达标参考线 0.85",
+            data: Array(labels.length).fill(0.85),
+            borderColor: "#16a34a",
+            borderDash: [6, 4],
+            borderWidth: 1.5,
+            pointRadius: 0,
+            fill: false,
+            order: 99,
+          },
+          ...datasets.map((ds, i) => ({
+            label: ds.label,
+            data: ds.data,
+            borderColor: colors[i % colors.length],
+            backgroundColor: colors[i % colors.length] + "18",
+            borderWidth: 2.5,
+            pointRadius: 5,
+            pointHoverRadius: 7,
+            pointBackgroundColor: colors[i % colors.length],
+            pointBorderColor: "#fff",
+            pointBorderWidth: 2,
+            tension: 0.25,
+            spanGaps: false,
+          })),
+        ],
+      },
+      options: {
+        responsive: true,
+        maintainAspectRatio: false,
+        interaction: { mode: "index", intersect: false },
+        plugins: {
+          legend: {
+            position: "bottom",
+            labels: { font: { size: 12 }, boxWidth: 14, padding: 16, usePointStyle: true, pointStyleWidth: 12 },
+          },
+          tooltip: {
+            backgroundColor: "#1a2942",
+            titleColor: "#e2e8f0",
+            bodyColor: "#cbd5e1",
+            borderColor: "#334155",
+            borderWidth: 1,
+            padding: 10,
+            callbacks: {
+              label: (ctx) => {
+                if (ctx.raw === null) return `  ${ctx.dataset.label}: —`;
+                return `  ${ctx.dataset.label}: ${Number(ctx.raw).toFixed(3)}`;
+              },
+            },
+          },
+        },
+        scales: {
+          y: {
+            min: 0, max: 1,
+            ticks: { stepSize: 0.1, font: { size: 11 }, color: "#94a3b8" },
+            grid: { color: "#f1f5f9" },
+            border: { display: false },
+          },
+          x: {
+            ticks: { font: { size: 11 }, color: "#64748b", maxRotation: 30 },
+            grid: { display: false },
+            border: { display: false },
+          },
+        },
+      },
+    });
+  },
+
+  // 柱状图 ──────────────────────────────────────────────────────────────────
+
+  _drawBar() {
+    const canvas = document.getElementById("db-bar-chart");
+    if (!canvas) return;
+
+    const run = Dashboard._runs.find((r) => r.run_id === Dashboard._focusId);
+    if (!run) return;
+
+    const { labels, actual, thresholds, colors, targetMet } =
+      Dashboard._buildComparisonData(run);
+
+    if (Dashboard._barChart) Dashboard._barChart.destroy();
+    Dashboard._barChart = new Chart(canvas, {
+      type: "bar",
+      data: {
+        labels,
+        datasets: [
+          {
+            label: "实际分数",
+            data: actual,
+            backgroundColor: colors,
+            borderRadius: 5,
+            barPercentage: 0.6,
+            categoryPercentage: 0.75,
+          },
+          {
+            label: "达标阈值",
+            data: thresholds,
+            // 深绿实心柱（不透明，无边框线）
+            backgroundColor: "#15803d",
+            borderRadius: 5,
+            barPercentage: 0.6,
+            categoryPercentage: 0.75,
+          },
+        ],
+      },
+      options: {
+        responsive: true,
+        maintainAspectRatio: false,
+        interaction: { mode: "index", intersect: false },
+        plugins: {
+          legend: {
+            position: "bottom",
+            labels: { font: { size: 12 }, boxWidth: 14, padding: 16, usePointStyle: false },
+          },
+          tooltip: {
+            backgroundColor: "#1a2942",
+            titleColor: "#e2e8f0",
+            bodyColor: "#cbd5e1",
+            borderColor: "#334155",
+            borderWidth: 1,
+            padding: 10,
+            callbacks: {
+              afterBody: (ctx) => {
+                if (!ctx.length) return [];
+                const idx = ctx[0].dataIndex;
+                return [targetMet[idx] ? "✓ 达标" : "✗ 未达标"];
+              },
+            },
+          },
+        },
+        scales: {
+          y: {
+            min: 0, max: 1,
+            ticks: { stepSize: 0.1, font: { size: 11 }, color: "#94a3b8" },
+            grid: { color: "#f1f5f9" },
+            border: { display: false },
+          },
+          x: {
+            ticks: { font: { size: 11 }, color: "#64748b" },
+            grid: { display: false },
+            border: { display: false },
+          },
+        },
+      },
+    });
+  },
+
+  // ── 纯数据函数（便于测试）────────────────────────────────────────────────
+
+  /**
+   * 从选中的 runs（按时间升序）构建折线图数据集。
+   * 每条线 = 一个指标；X 轴 = 各运行的简短标签；无值处补 null（断线）。
+   * @param {Array} runs - 已按 finished_at 升序排序的 run 对象数组
+   * @returns {{ labels: string[], datasets: Array<{label:string, data:Array<number|null>}> }}
+   */
+  _buildTrendDatasets(runs) {
+    // 合并所有 runs 出现过的指标（保持首次出现顺序）
+    const metricSet = [];
+    runs.forEach((r) => {
+      (r.metrics || []).forEach((m) => {
+        if (!metricSet.includes(m)) metricSet.push(m);
+      });
+    });
+
+    const labels = runs.map(
+      (r) => `${r.scenario_name || r.run_id}\n${App.shortTime(r.finished_at)}`
+    );
+
+    const datasets = metricSet.map((m) => ({
+      label: m + (MetricPresenter.isLowerBetter(m) ? " (越低越好)" : ""),
+      data: runs.map((r) => {
+        const v = r.metric_means ? r.metric_means[m] : null;
+        return v !== null && v !== undefined ? Number(v) : null;
+      }),
+    }));
+
+    return { labels, datasets };
+  },
+
+  /**
+   * 从单个 run 构建柱状图对比数据。
+   * @param {Object} run
+   * @returns {{ labels, actual, thresholds, colors, targetMet }}
+   */
+  _buildComparisonData(run) {
+    const metrics = run.metrics || [];
+    const labels = metrics.map((m) => App.shortMetric(m));
+    const actual = metrics.map((m) => {
+      const v = run.metric_means ? run.metric_means[m] : null;
+      return v !== null && v !== undefined ? Number(v) : null;
+    });
+    const thresholds = metrics.map((m) => MetricPresenter.passThreshold(m));
+    const targetMet = metrics.map((m, i) =>
+      MetricPresenter.meetsTarget(m, actual[i])
+    );
+    const colorMap = { good: "#4ade80", warn: "#fbbf24", bad: "#f87171", na: "#cbd5e1" };
+    const colors = metrics.map((m, i) => colorMap[App.scoreClass(m, actual[i])] || "#cbd5e1");
+
+    return { labels, actual, thresholds, colors, targetMet };
+  },
+};
+
+  globalObj.Dashboard = Dashboard;
+})(typeof window !== "undefined" ? window : this);