AIRegulation/siemens_ragas
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update for ragas

Browse Source
This commit is contained in:
wangwei
2026-07-01 17:53:00 +08:00
parent 2bb804b059
commit 4e74e1b247
15 changed files with 507 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webapp/services/report_builder.py
+37 -5
View File
@@ -23,9 +23,10 @@ from webapp.models import (
DistributionBin,
GroupStat,
ReportData,
SampleHistoryEntry,
SampleScore,
)
from webapp.services import run_reader
from webapp.services import question_history, run_reader
# Number of equal-width buckets used for metric score histograms.
@@ -37,6 +38,9 @@ GROUPING_FIELDS = ("difficulty", "question_type", "language")
# How many lowest-scoring samples to surface for manual review.
LOWEST_SAMPLE_COUNT = 10
# How many past evaluations of the same question to show in the history table.
HISTORY_LIMIT = 5
# Metrics whose lower raw value means stronger performance.
LOWER_IS_BETTER_METRICS = {"noise_sensitivity"}
@@ -124,8 +128,16 @@ def _cell_text(row: pd.Series, column: str) -> str:
return str(row[column]).strip()
def _lowest_samples(frame: pd.DataFrame, metrics: list[str]) -> list[SampleScore]:
"""Select and shape the lowest-scoring samples for the review table."""
def _lowest_samples(
frame: pd.DataFrame,
metrics: list[str],
history_index: dict[str, list[dict]] | None = None,
) -> list[SampleScore]:
"""Select and shape the lowest-scoring samples for the review table.
When a history_index is supplied, each surfaced sample is annotated with the
same question's scores from previous runs (newest first) for comparison.
"""
if frame.empty:
return []
@@ -154,7 +166,19 @@ def _lowest_samples(frame: pd.DataFrame, metrics: list[str]) -> list[SampleScore
enriched.append((sort_key, sample))
enriched.sort(key=lambda item: item[0])
return [sample for _, sample in enriched[:LOWEST_SAMPLE_COUNT]]
selected = [sample for _, sample in enriched[:LOWEST_SAMPLE_COUNT]]
# Attach per-question history only for the surfaced samples (keeps lookups cheap).
if history_index is not None:
for sample in selected:
if not sample.question:
continue
entries = question_history.lookup(
history_index, sample.question, limit=HISTORY_LIMIT
)
sample.history = [SampleHistoryEntry(**entry) for entry in entries]
return selected
def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
@@ -192,12 +216,20 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
if metric in frame.columns
}
# Cross-run history: scores of the same question in *other* runs (Approach A —
# on-demand global scan, excluding the run currently being viewed).
metadata = run_reader._read_json(run_dir / "metadata.json")
current_run_id = str(metadata.get("run_id") or run_dir.name)
history_index = question_history.build_question_history_index(
exclude_run_id=current_run_id
)
return ReportData(
metrics=metrics,
metric_means=rounded_means,
distributions=distributions,
groupings=_groupings(frame, metrics),
lowest_samples=_lowest_samples(frame, metrics),
lowest_samples=_lowest_samples(frame, metrics, history_index),
summary_markdown=summary_markdown,
advice_markdown=advice_markdown,
weighted_score_mean=_round_or_none(overall_ws),
webapp/services/score_job_manager.py
+1 -3
View File
@@ -107,7 +107,6 @@ class ScoreJobManager:
# Lazy imports to keep web server bootable if ragas is not installed.
from rag_eval.advisor import run_advisor
from rag_eval.metrics.factory import build_models
from rag_eval.metrics.weights import compute_weighted_score
from rag_eval.reporting.writers import write_run_artifacts
from rag_eval.settings import EvaluationSettings
@@ -206,8 +205,7 @@ class ScoreJobManager:
# Run optimization advisor (builds optimization_advice.md)
try:
llm, _ = build_models(judge_model, embedding_model, settings)
run_advisor(result, scenario, llm)
run_advisor(result, scenario, settings=settings)
logger.info("[score_job] advisor done job_id=%s", job_id)
except Exception as adv_exc: # noqa: BLE001
logger.warning("[score_job] advisor failed job_id=%s err=%s", job_id, adv_exc)
webapp/services/session_score_manager.py
+1 -3
View File
@@ -192,7 +192,6 @@ class SessionScoreJobManager:
# Lazy imports — keep web server bootable if ragas is not installed.
from rag_eval.advisor import run_advisor
from rag_eval.metrics.factory import build_models
from rag_eval.metrics.weights import compute_weighted_score
from rag_eval.reporting.writers import write_run_artifacts
from rag_eval.settings import EvaluationSettings
@@ -320,8 +319,7 @@ class SessionScoreJobManager:
# Regenerate optimization advice over all accumulated rows
try:
llm, _ = build_models(judge_model, embedding_model, settings)
run_advisor(result, scenario, llm)
run_advisor(result, scenario, settings=settings)
logger.info("[session_job] advisor done job_id=%s session=%s", job_id, session_id)
except Exception as adv_exc: # noqa: BLE001
logger.warning(
webapp/static/css/app.css
+89
View File
@@ -253,6 +253,23 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t
}
.detail-gt { color: var(--good); }
/* 历史评分小表格：本次行高亮 + 涨跌着色（绿=改善 红=退步） */
table.history-table { width: 100%; border-collapse: collapse; font-size: 12px; margin-top: 4px; }
table.history-table th, table.history-table td {
padding: 5px 8px; text-align: left; border-bottom: 1px solid #f1f5f9;
}
table.history-table th { color: var(--slate); font-weight: 600; border-bottom: 1px solid var(--line); }
table.history-table td { font-variant-numeric: tabular-nums; }
.history-table tr.hist-current { background: #f0f9ff; }
.history-table tr.hist-current .hist-label { font-weight: 700; color: #0369a1; }
.hist-when { white-space: nowrap; }
.hist-label { display: inline-block; }
.hist-sub { display: block; font-size: 11px; color: var(--slate-light); }
.hist-delta { font-size: 11px; font-weight: 700; font-variant-numeric: tabular-nums; }
.hist-delta.delta-good { color: #16a34a; }
.hist-delta.delta-bad { color: #dc2626; }
.hist-delta.delta-flat { color: var(--slate-light); }
.empty { text-align: center; padding: 60px 20px; color: var(--slate); }
.empty p { margin-bottom: 8px; }
@@ -514,6 +531,14 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t
table.group-table td { padding: 4pt 6pt; border-bottom: 1px solid #e2e8f0; }
table.group-table th { font-weight: 700; color: #64748b; }
/* ── 历史评分表 ── */
table.history-table { width: 100%; font-size: 9pt; border-collapse: collapse; }
table.history-table th,
table.history-table td { padding: 3pt 6pt; border-bottom: 1px solid #e2e8f0; }
.history-table tr.hist-current { background: #f0f9ff !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
.hist-delta.delta-good { color: #16a34a !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
.hist-delta.delta-bad { color: #dc2626 !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
/* ── 颜色保留（部分浏览器打印默认去色） ── */
.good { color: #16a34a !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
.warn { color: #eab308 !important; -webkit-print-color-adjust: exact; print-color-adjust: exact; }
@@ -546,3 +571,67 @@ table.group-table td { border-bottom: 1px solid #f1f5f9; font-variant-numeric: t
.advice-md ul { padding-left: 20px; margin: 6px 0; }
.advice-md li { margin: 3px 0; font-size: 13px; }
.advice-md strong { color: var(--ink); font-weight: 600; }
/* ---------- 指标看板 Dashboard ---------- */
.dashboard-charts {
display: flex;
flex-direction: column;
gap: 20px;
}
.dashboard-chart-panel {
min-width: 0;
/* 上下布局时给图表面板稍微更宽裕的高度 */
}
.dashboard-chart-panel canvas {
max-height: 340px;
height: 320px !important;
}
/* 运行选择器列表 */
.db-run-list {
display: flex;
flex-direction: column;
gap: 5px;
max-height: 240px;
overflow-y: auto;
margin-top: 10px;
padding-right: 4px;
}
.db-run-row {
display: flex;
align-items: center;
gap: 10px;
padding: 8px 12px;
border: 1px solid var(--line);
border-radius: 8px;
cursor: pointer;
transition: background 0.12s, border-color 0.12s;
background: var(--surface);
}
.db-run-row:hover { background: #f0fbfb; border-color: var(--petrol); }
.db-run-row:has(input:checked) {
background: #e8f7f7;
border-color: #7ecece;
}
.db-run-row input[type="checkbox"] { flex-shrink: 0; accent-color: var(--petrol); width: 15px; height: 15px; }
.db-run-label { display: flex; flex-direction: column; gap: 2px; flex: 1; min-width: 0; }
.db-run-name { font-size: 13px; font-weight: 600; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
.db-run-chips { display: flex; flex-wrap: wrap; gap: 6px; flex-shrink: 0; }
.db-chip-name { color: var(--slate); }
.btn-sm { padding: 5px 12px; font-size: 12px; }
/* 看板图表面板头：标题左 + 下拉右 对齐优化 */
.db-panel-head-bar {
display: flex;
align-items: center;
justify-content: space-between;
flex-wrap: wrap;
gap: 8px;
margin-bottom: 14px;
}
.db-chart-hint {
font-size: 11px;
color: var(--slate-light);
margin-top: 6px;
text-align: center;
}
webapp/static/index.html
+9
View File
@@ -31,6 +31,9 @@
<button class="nav-item" data-view="scorejobs">
<span class="nav-ico">📋</span><span>评分记录</span>
</button>
<button class="nav-item" data-view="dashboard">
<span class="nav-ico">📊</span><span>指标看板</span>
</button>
<button class="nav-item" data-view="apidocs">
<span class="nav-ico">⎔</span><span>API 文档</span>
</button>
@@ -263,6 +266,11 @@
allowfullscreen>
</iframe>
</section>
<!-- 指标看板视图 -->
<section class="view" id="view-dashboard" hidden>
<div id="dashboard-wrap"></div>
</section>
</main>
</div>
@@ -272,6 +280,7 @@
<script src="/static/js/profiles.js"></script>
<script src="/static/js/runner.js"></script>
<script src="/static/js/score_jobs.js"></script>
<script src="/static/js/dashboard.js"></script>
<script src="/static/js/app.js"></script>
</body>
</html>
webapp/static/js/app.js
+3 -2
View File
@@ -5,8 +5,8 @@
const App = {
currentRunId: null,
activeView: null,
views: ["runs", "new", "report", "profiles", "scorejobs", "apidocs"],
titles: { runs: "运行列表", new: "新建评估", report: "报告详情", profiles: "LLM 配置", scorejobs: "评分记录", apidocs: "API 文档" },
views: ["runs", "new", "report", "profiles", "scorejobs", "dashboard", "apidocs"],
titles: { runs: "运行列表", new: "新建评估", report: "报告详情", profiles: "LLM 配置", scorejobs: "评分记录", dashboard: "指标看板", apidocs: "API 文档" },
// 初始化：绑定导航、从 URL/sessionStorage 恢复上次位置、启动健康检查。
init() {
@@ -73,6 +73,7 @@ const App = {
if (view === "report") Report.render(App.currentRunId);
if (view === "profiles") Profiles.load();
if (view === "scorejobs") ScoreJobs.load();
if (view === "dashboard") Dashboard.load();
},
// ----------------------------------------------------------------
webapp/static/js/metric_presenter.js
+33
View File
@@ -69,9 +69,42 @@
return "#dc2626";
}
// 计算某指标本次相对上一次的涨跌信息，方向语义随指标而定
// （noise_sensitivity 越低越好：下降=改善）。
function deltaInfo(metricName, current, previous) {
const isNum = (v) => v !== null && v !== undefined && !Number.isNaN(Number(v));
if (!isNum(current) || !isNum(previous)) {
return { hasData: false, delta: null, improved: null, arrow: "", magnitude: "", cls: "delta-flat" };
}
const delta = Number(current) - Number(previous);
const rounded = Math.round(delta * 10000) / 10000;
const arrow = rounded > 0 ? "▲" : rounded < 0 ? "▼" : "→";
const magnitude = Math.abs(rounded).toFixed(2);
const improved = isLowerBetter(metricName) ? rounded < 0 : rounded > 0;
const cls = rounded === 0 ? "delta-flat" : improved ? "delta-good" : "delta-bad";
return { hasData: true, delta: rounded, improved, arrow, magnitude, cls };
}
// 返回指标的"达标阈值"（柱状图对比用，方向感知）。
// higher-better 指标：0.85；lower-better (noise_sensitivity)：0.15。
function passThreshold(metricName) {
return isLowerBetter(metricName) ? 0.15 : 0.85;
}
// 判断某指标的值是否达标。
function meetsTarget(metricName, value) {
if (value === null || value === undefined || Number.isNaN(Number(value))) return false;
const v = Number(value);
return isLowerBetter(metricName) ? v <= passThreshold(metricName) : v >= passThreshold(metricName);
}
globalObj.MetricPresenter = {
scoreClass,
describeMetric,
binColor,
isLowerBetter,
deltaInfo,
passThreshold,
meetsTarget,
};
})(window);
webapp/static/js/report.js
+60 -3
View File
@@ -283,7 +283,7 @@ const Report = {
const detail = document.createElement("div");
detail.className = "lowest-detail";
detail.hidden = true;
detail.innerHTML = Report._detailHtml(sample);
detail.innerHTML = Report._detailHtml(sample, metrics);
row.addEventListener("click", () => {
detail.hidden = !detail.hidden;
@@ -293,8 +293,8 @@ const Report = {
});
},
// 单条样本的展开详情：question / contexts / answer / ground_truth。
_detailHtml(sample) {
// 单条样本的展开详情：question / contexts / answer / ground_truth / 历史评分。
_detailHtml(sample, metrics) {
const contexts = (sample.contexts || [])
.map((c, i) => `<div class="ctx-item">[${i + 1}] ${App.escape(c)}</div>`)
.join("");
@@ -320,6 +320,63 @@ const Report = {
<div class="detail-gt">${App.escape(sample.ground_truth || "—")}</div>
</div>
${errorBlock}
${Report._historyHtml(sample, metrics || [])}
</div>
`;
},
// 同一问题的历史评分小表格：本次 + 历次（按时间倒序），逐行标注较更早一次的涨跌。
_historyHtml(sample, metrics) {
const history = sample.history || [];
if (!history.length) return "";
// 只展示当前样本与历史中实际出现过的指标列，避免空列。
const cols = metrics.filter(
(m) =>
(sample.metrics && sample.metrics[m] !== undefined && sample.metrics[m] !== null) ||
history.some((h) => h.metrics && h.metrics[m] !== undefined && h.metrics[m] !== null),
);
if (!cols.length) return "";
// 组合行：[本次, 历次...]，相邻两行做涨跌对比（行 r 对比更早的行 r+1）。
const rows = [
{ label: "本次", sub: "", metrics: sample.metrics || {}, current: true },
...history.map((h) => ({
label: App.escape(h.scenario_name || h.run_id || "历史"),
sub: App.escape(App.shortTime(h.finished_at)),
metrics: h.metrics || {},
current: false,
})),
];
let head = "<tr><th>评测</th>";
cols.forEach((m) => (head += `<th>${App.escape(App.shortMetric(m))}</th>`));
head += "</tr>";
let body = "";
rows.forEach((row, r) => {
const older = rows[r + 1];
body += `<tr class="${row.current ? "hist-current" : ""}">`;
body += `<td class="hist-when"><span class="hist-label">${row.label}</span>${row.sub ? `<span class="hist-sub">${row.sub}</span>` : ""}</td>`;
cols.forEach((m) => {
const v = row.metrics ? row.metrics[m] : null;
const cls = App.scoreClass(m, v);
const text = v === null || v === undefined ? "—" : Number(v).toFixed(2);
let deltaHtml = "";
const baseline = older && older.metrics ? older.metrics[m] : undefined;
const d = MetricPresenter.deltaInfo(m, v, baseline);
if (d.hasData && d.delta !== 0) {
deltaHtml = ` <span class="hist-delta ${d.cls}">${d.arrow}${d.magnitude}</span>`;
}
body += `<td><span class="score-badge ${cls}">${text}</span>${deltaHtml}</td>`;
});
body += "</tr>";
});
return `
<div class="detail-field">
<div class="detail-label">历史评分 history（同一问题，最近 ${history.length} 次，含本次对比）</div>
<table class="history-table">${head}${body}</table>
</div>
`;
},