Extract shared build_metric_registry factory (DRY)
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
wangwei
+35
-14
@@ -57,6 +57,39 @@ def _resolve_openai_client_kwargs(
|
||||
return settings.openai_client_kwargs
|
||||
|
||||
|
||||
def resolve_openai_client_kwargs(
|
||||
judge_model: str,
|
||||
settings: EvaluationSettings,
|
||||
) -> dict[str, Any]:
|
||||
"""Public accessor for profile-aware AsyncOpenAI kwargs (matched by judge_model).
|
||||
|
||||
Exposed so other components (e.g. the optimization advisor's direct LLM call)
|
||||
can build a client that honors the same saved-profile/.env resolution used by
|
||||
the scoring pipeline, instead of duplicating the lookup logic.
|
||||
"""
|
||||
return _resolve_openai_client_kwargs(judge_model, settings)
|
||||
|
||||
|
||||
def build_metric_registry(llm: Any, embeddings: Any) -> dict[str, Any]:
|
||||
"""Instantiate the full set of supported RAGAS metrics keyed by canonical name.
|
||||
|
||||
Shared by the scenario pipeline, the inline scorer, and the prompt-cache
|
||||
bootstrap so the metric set is defined in exactly one place.
|
||||
"""
|
||||
return {
|
||||
"faithfulness": Faithfulness(llm=llm),
|
||||
"answer_relevancy": AnswerRelevancy(llm=llm, embeddings=embeddings),
|
||||
"context_recall": ContextRecall(llm=llm),
|
||||
"context_precision": ContextPrecision(llm=llm),
|
||||
# NoiseSensitivity mode='relevant': sensitivity to noise from relevant contexts.
|
||||
"noise_sensitivity": NoiseSensitivity(llm=llm),
|
||||
# FactualCorrectness mode='f1': balances claim precision and recall vs. ground truth.
|
||||
"factual_correctness": FactualCorrectness(llm=llm),
|
||||
# SemanticSimilarity: embedding cosine between answer and ground truth (no LLM call).
|
||||
"semantic_similarity": SemanticSimilarity(embeddings=embeddings),
|
||||
}
|
||||
|
||||
|
||||
def build_models(
|
||||
judge_model: str,
|
||||
embedding_model: str,
|
||||
@@ -98,20 +131,8 @@ def build_metric_pipeline(
|
||||
settings,
|
||||
)
|
||||
|
||||
# Build the full registry once, then slice it by configured metric names.
|
||||
registry: dict[str, Any] = {
|
||||
"faithfulness": Faithfulness(llm=llm),
|
||||
"answer_relevancy": AnswerRelevancy(llm=llm, embeddings=embeddings),
|
||||
"context_recall": ContextRecall(llm=llm),
|
||||
"context_precision": ContextPrecision(llm=llm),
|
||||
# Robustness / end-to-end metrics (架构设计 §10.2).
|
||||
# NoiseSensitivity mode='relevant': sensitivity to noise from relevant contexts.
|
||||
"noise_sensitivity": NoiseSensitivity(llm=llm),
|
||||
# FactualCorrectness mode='f1': balances claim precision and recall vs. ground truth.
|
||||
"factual_correctness": FactualCorrectness(llm=llm),
|
||||
# SemanticSimilarity: embedding cosine between answer and ground truth (no LLM call).
|
||||
"semantic_similarity": SemanticSimilarity(embeddings=embeddings),
|
||||
}
|
||||
# Build the full registry once using the shared factory, then slice by requested metrics.
|
||||
registry = build_metric_registry(llm, embeddings)
|
||||
return MetricPipeline(
|
||||
metrics={name: registry[name] for name in scenario.metrics},
|
||||
metric_timeout_seconds=settings.ragas_metric_timeout_seconds,
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Tests for the optimization advisor's direct-LLM analyzer.
|
||||
|
||||
These tests inject a fake async chat client so no network call is made. They
|
||||
verify that analyze() uses a plain chat.completions call (not the removed
|
||||
langchain path), returns the text from choices[0].message.content, embeds the
|
||||
worked-example instructions and the low-sample contexts in the prompt, and
|
||||
selects the correct token parameter for reasoning vs. legacy models.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from rag_eval.advisor.llm_analyzer import analyze, _is_reasoning_model
|
||||
from rag_eval.advisor.rules import Diagnosis
|
||||
|
||||
|
||||
class _FakeMessage:
|
||||
def __init__(self, content: str) -> None:
|
||||
self.content = content
|
||||
|
||||
|
||||
class _FakeChoice:
|
||||
def __init__(self, content: str) -> None:
|
||||
self.message = _FakeMessage(content)
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, content: str) -> None:
|
||||
self.choices = [_FakeChoice(content)]
|
||||
|
||||
|
||||
class _FakeCompletions:
|
||||
def __init__(self, captured: dict) -> None:
|
||||
self._captured = captured
|
||||
|
||||
async def create(self, **kwargs):
|
||||
self._captured.update(kwargs)
|
||||
return _FakeResponse("## faithfulness [警告]\n\n针对该问题的具体优化建议")
|
||||
|
||||
|
||||
class _FakeChat:
|
||||
def __init__(self, captured: dict) -> None:
|
||||
self.completions = _FakeCompletions(captured)
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, captured: dict) -> None:
|
||||
self.chat = _FakeChat(captured)
|
||||
self.closed = False
|
||||
|
||||
async def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _Settings:
|
||||
ragas_llm_max_tokens = 4096
|
||||
|
||||
|
||||
def _diagnosis() -> Diagnosis:
|
||||
return Diagnosis(
|
||||
metric="faithfulness",
|
||||
mean_score=0.55,
|
||||
threshold=0.7,
|
||||
severity="warning",
|
||||
root_causes=["生成未严格 grounding"],
|
||||
suggested_actions=["强化 grounding 约束"],
|
||||
low_samples=[
|
||||
{
|
||||
"sample_id": "s1",
|
||||
"question": "球管寿命如何评估？",
|
||||
"answer": "球管寿命约 3 年。",
|
||||
"ground_truth": "球管寿命取决于使用强度。",
|
||||
"contexts": "球管寿命与扫描负载相关 |||| 高负载会缩短寿命",
|
||||
"faithfulness": 0.4,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_analyze_uses_direct_chat_and_returns_content() -> None:
|
||||
captured: dict = {}
|
||||
text = asyncio.run(
|
||||
analyze([_diagnosis()], "scn", "gpt-4o", _Settings(), chat_client=_FakeClient(captured))
|
||||
)
|
||||
assert "优化建议" in text
|
||||
assert captured["model"] == "gpt-4o"
|
||||
prompt = captured["messages"][0]["content"]
|
||||
assert "举例拆解" in prompt # worked-example instruction present
|
||||
assert "球管寿命与扫描负载相关" in prompt # low-sample contexts embedded
|
||||
assert "max_tokens" in captured # legacy model uses max_tokens
|
||||
assert "max_completion_tokens" not in captured
|
||||
|
||||
|
||||
def test_analyze_reasoning_model_uses_max_completion_tokens() -> None:
|
||||
captured: dict = {}
|
||||
asyncio.run(
|
||||
analyze([_diagnosis()], "scn", "gpt-5", _Settings(), chat_client=_FakeClient(captured))
|
||||
)
|
||||
assert "max_completion_tokens" in captured
|
||||
assert "max_tokens" not in captured
|
||||
|
||||
|
||||
def test_analyze_empty_diagnoses_returns_empty() -> None:
|
||||
assert asyncio.run(analyze([], "scn", "gpt-4o", _Settings())) == ""
|
||||
|
||||
|
||||
def test_analyze_closes_client_it_creates(monkeypatch) -> None:
|
||||
"""A self-created client is closed in-loop to avoid 'Event loop is closed'."""
|
||||
captured: dict = {}
|
||||
fake = _FakeClient(captured)
|
||||
|
||||
import openai
|
||||
import rag_eval.metrics.factory as factory_mod
|
||||
|
||||
monkeypatch.setattr(openai, "AsyncOpenAI", lambda **kwargs: fake)
|
||||
monkeypatch.setattr(
|
||||
factory_mod, "resolve_openai_client_kwargs", lambda *a, **k: {"api_key": "x"}
|
||||
)
|
||||
|
||||
# No chat_client passed → analyze() builds (and must close) its own client.
|
||||
text = asyncio.run(analyze([_diagnosis()], "scn", "gpt-4o", _Settings()))
|
||||
|
||||
assert "优化建议" in text
|
||||
assert fake.closed is True
|
||||
|
||||
|
||||
def test_analyze_does_not_close_injected_client() -> None:
|
||||
"""An injected client is owned by the caller and must not be closed."""
|
||||
fake = _FakeClient({})
|
||||
asyncio.run(analyze([_diagnosis()], "scn", "gpt-4o", _Settings(), chat_client=fake))
|
||||
assert fake.closed is False
|
||||
|
||||
|
||||
def test_is_reasoning_model_detection() -> None:
|
||||
assert _is_reasoning_model("gpt-5")
|
||||
assert _is_reasoning_model("gpt-5.5")
|
||||
assert _is_reasoning_model("o1-mini")
|
||||
assert _is_reasoning_model("o3")
|
||||
assert not _is_reasoning_model("gpt-4o")
|
||||
assert not _is_reasoning_model("deepseek-v4-flash")
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Tests for the Dashboard module's pure data functions and MetricPresenter additions.
|
||||
|
||||
The pure data functions (_buildTrendDatasets, _buildComparisonData) and the new
|
||||
MetricPresenter helpers (passThreshold, meetsTarget) are tested via Node.js so no
|
||||
browser or network is required.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def _run_node(script: str) -> str:
|
||||
"""Execute a Node.js script and return stdout."""
|
||||
completed = subprocess.run(
|
||||
["node", "-e", script],
|
||||
cwd=REPO_ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
check=True,
|
||||
)
|
||||
return completed.stdout.strip()
|
||||
|
||||
|
||||
def _load_js() -> str:
|
||||
"""Return a Node-runnable bootstrap that loads MetricPresenter + Dashboard."""
|
||||
presenter_path = (REPO_ROOT / "webapp" / "static" / "js" / "metric_presenter.js").as_posix()
|
||||
dashboard_path = (REPO_ROOT / "webapp" / "static" / "js" / "dashboard.js").as_posix()
|
||||
return f"""
|
||||
const fs = require("fs");
|
||||
const vm = require("vm");
|
||||
|
||||
// Shared sandbox (window object shared by both scripts)
|
||||
const sandbox = {{ window: {{}}, console }};
|
||||
|
||||
// MetricPresenter
|
||||
vm.runInNewContext(fs.readFileSync("{presenter_path}", "utf8"), sandbox);
|
||||
const MetricPresenter = sandbox.window.MetricPresenter;
|
||||
|
||||
// Minimal App stub required by dashboard.js
|
||||
sandbox.App = {{
|
||||
escape: (s) => String(s == null ? "" : s),
|
||||
shortMetric: (m) => m,
|
||||
shortTime: (t) => (t || "").slice(0, 16),
|
||||
scoreClass: (m, v) => MetricPresenter.scoreClass(m, v),
|
||||
}};
|
||||
sandbox.MetricPresenter = MetricPresenter;
|
||||
// Stub Chart.js (not exercised by pure functions)
|
||||
sandbox.Chart = function() {{ this.destroy = () => {{}}; }};
|
||||
|
||||
// Dashboard (attaches itself to sandbox.window.Dashboard)
|
||||
vm.runInNewContext(fs.readFileSync("{dashboard_path}", "utf8"), sandbox);
|
||||
const Dashboard = sandbox.window.Dashboard;
|
||||
"""
|
||||
|
||||
|
||||
def test_pass_threshold_higher_better() -> None:
|
||||
"""All higher-better metrics should have passThreshold 0.85."""
|
||||
script = _load_js() + """
|
||||
const result = {
|
||||
faith: MetricPresenter.passThreshold("faithfulness"),
|
||||
ans: MetricPresenter.passThreshold("answer_relevancy"),
|
||||
recall: MetricPresenter.passThreshold("context_recall"),
|
||||
prec: MetricPresenter.passThreshold("context_precision"),
|
||||
fact: MetricPresenter.passThreshold("factual_correctness"),
|
||||
sem: MetricPresenter.passThreshold("semantic_similarity"),
|
||||
};
|
||||
console.log(JSON.stringify(result));
|
||||
"""
|
||||
out = _run_node(script)
|
||||
assert '"faith":0.85' in out
|
||||
assert '"ans":0.85' in out
|
||||
assert '"recall":0.85' in out
|
||||
assert '"prec":0.85' in out
|
||||
assert '"fact":0.85' in out
|
||||
assert '"sem":0.85' in out
|
||||
|
||||
|
||||
def test_pass_threshold_noise_sensitivity_lower_better() -> None:
|
||||
"""noise_sensitivity (lower-better) should have passThreshold 0.15."""
|
||||
script = _load_js() + """
|
||||
console.log(JSON.stringify(MetricPresenter.passThreshold("noise_sensitivity")));
|
||||
"""
|
||||
out = _run_node(script)
|
||||
assert out.strip() == "0.15"
|
||||
|
||||
|
||||
def test_meets_target_higher_better() -> None:
|
||||
"""meetsTarget should return true only at/above 0.85 for higher-better metrics."""
|
||||
script = _load_js() + """
|
||||
const result = {
|
||||
at085: MetricPresenter.meetsTarget("faithfulness", 0.85),
|
||||
above: MetricPresenter.meetsTarget("faithfulness", 0.90),
|
||||
below: MetricPresenter.meetsTarget("faithfulness", 0.84),
|
||||
zero: MetricPresenter.meetsTarget("faithfulness", 0),
|
||||
nul: MetricPresenter.meetsTarget("faithfulness", null),
|
||||
};
|
||||
console.log(JSON.stringify(result));
|
||||
"""
|
||||
out = _run_node(script)
|
||||
assert '"at085":true' in out
|
||||
assert '"above":true' in out
|
||||
assert '"below":false' in out
|
||||
assert '"zero":false' in out
|
||||
assert '"nul":false' in out
|
||||
|
||||
|
||||
def test_meets_target_noise_sensitivity() -> None:
|
||||
"""meetsTarget for noise_sensitivity: true only at/below 0.15."""
|
||||
script = _load_js() + """
|
||||
const result = {
|
||||
at015: MetricPresenter.meetsTarget("noise_sensitivity", 0.15),
|
||||
below: MetricPresenter.meetsTarget("noise_sensitivity", 0.10),
|
||||
above: MetricPresenter.meetsTarget("noise_sensitivity", 0.16),
|
||||
};
|
||||
console.log(JSON.stringify(result));
|
||||
"""
|
||||
out = _run_node(script)
|
||||
assert '"at015":true' in out
|
||||
assert '"below":true' in out
|
||||
assert '"above":false' in out
|
||||
|
||||
|
||||
def test_build_trend_datasets_time_order_and_null_gap() -> None:
|
||||
"""_buildTrendDatasets returns metrics in appearance order, null for missing values."""
|
||||
script = (
|
||||
_load_js()
|
||||
+ """
|
||||
const runs = [
|
||||
{ run_id: "r1", scenario_name: "scn", finished_at: "2026-01-01T00:00:00",
|
||||
metrics: ["faithfulness"], metric_means: { faithfulness: 0.60 } },
|
||||
{ run_id: "r2", scenario_name: "scn", finished_at: "2026-02-01T00:00:00",
|
||||
metrics: ["faithfulness", "noise_sensitivity"],
|
||||
metric_means: { faithfulness: 0.80, noise_sensitivity: 0.20 } },
|
||||
{ run_id: "r3", scenario_name: "scn", finished_at: "2026-03-01T00:00:00",
|
||||
metrics: ["faithfulness"], metric_means: { faithfulness: 0.90 } },
|
||||
];
|
||||
const { labels, datasets } = Dashboard._buildTrendDatasets(runs);
|
||||
console.log(JSON.stringify({ labels, datasets }));
|
||||
"""
|
||||
)
|
||||
import json
|
||||
|
||||
out = json.loads(_run_node(script))
|
||||
assert len(out["labels"]) == 3
|
||||
# faithfulness dataset: all 3 points
|
||||
faith_ds = next(d for d in out["datasets"] if "faithfulness" in d["label"])
|
||||
assert faith_ds["data"] == [0.60, 0.80, 0.90]
|
||||
# noise_sensitivity only present in r2 → null in r1 and r3
|
||||
noise_ds = next(d for d in out["datasets"] if "noise_sensitivity" in d["label"])
|
||||
assert noise_ds["data"] == [None, 0.20, None]
|
||||
# noise label should note lower-is-better
|
||||
assert "越低越好" in noise_ds["label"]
|
||||
|
||||
|
||||
def test_build_comparison_data_structure() -> None:
|
||||
"""_buildComparisonData returns correct labels, actuals, thresholds and targetMet."""
|
||||
script = (
|
||||
_load_js()
|
||||
+ """
|
||||
const run = {
|
||||
run_id: "r1", scenario_name: "scn",
|
||||
metrics: ["faithfulness", "noise_sensitivity"],
|
||||
metric_means: { faithfulness: 0.90, noise_sensitivity: 0.10 },
|
||||
};
|
||||
const result = Dashboard._buildComparisonData(run);
|
||||
console.log(JSON.stringify(result));
|
||||
"""
|
||||
)
|
||||
import json
|
||||
|
||||
out = json.loads(_run_node(script))
|
||||
assert out["labels"] == ["faithfulness", "noise_sensitivity"]
|
||||
assert out["actual"] == [0.90, 0.10]
|
||||
assert out["thresholds"] == [0.85, 0.15]
|
||||
assert out["targetMet"] == [True, True] # 0.90 >= 0.85 ✓; 0.10 <= 0.15 ✓
|
||||
|
||||
|
||||
def test_build_comparison_data_unmet_targets() -> None:
|
||||
"""targetMet is False when metrics are below threshold."""
|
||||
script = (
|
||||
_load_js()
|
||||
+ """
|
||||
const run = {
|
||||
run_id: "r1", scenario_name: "scn",
|
||||
metrics: ["faithfulness", "noise_sensitivity"],
|
||||
metric_means: { faithfulness: 0.60, noise_sensitivity: 0.40 },
|
||||
};
|
||||
const result = Dashboard._buildComparisonData(run);
|
||||
console.log(JSON.stringify(result.targetMet));
|
||||
"""
|
||||
)
|
||||
out = _run_node(script)
|
||||
assert out.strip() == "[false,false]"
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Tests for the shared metric registry factory."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from ragas.llms.base import InstructorBaseRagasLLM
|
||||
from ragas.embeddings.base import BaseRagasEmbedding
|
||||
|
||||
from rag_eval.metrics.factory import build_metric_registry
|
||||
|
||||
|
||||
def _mock_llm():
|
||||
"""Return a mock that passes RAGAS InstructorLLM type checks."""
|
||||
return MagicMock(spec=InstructorBaseRagasLLM)
|
||||
|
||||
|
||||
def _mock_emb():
|
||||
"""Return a mock that passes RAGAS embedding type checks."""
|
||||
return MagicMock(spec=BaseRagasEmbedding)
|
||||
|
||||
|
||||
def test_build_metric_registry_has_all_seven_metrics():
|
||||
"""The registry exposes every supported metric keyed by its canonical name."""
|
||||
registry = build_metric_registry(llm=_mock_llm(), embeddings=_mock_emb())
|
||||
assert set(registry) == {
|
||||
"faithfulness", "answer_relevancy", "context_recall", "context_precision",
|
||||
"noise_sensitivity", "factual_correctness", "semantic_similarity",
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Tests for the cross-run question-history index used by the report page."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from webapp.services.question_history import (
|
||||
build_question_history_index,
|
||||
lookup,
|
||||
normalize_question,
|
||||
)
|
||||
|
||||
|
||||
def _write_run(
|
||||
run_dir: Path,
|
||||
*,
|
||||
run_id: str,
|
||||
scenario_name: str,
|
||||
finished_at: str,
|
||||
rows: list[tuple[str, float]],
|
||||
metric: str = "faithfulness",
|
||||
) -> None:
|
||||
"""Create a minimal run directory (metadata.json + scores.csv)."""
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
import json
|
||||
|
||||
(run_dir / "metadata.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"scenario_name": scenario_name,
|
||||
"finished_at": finished_at,
|
||||
"valid_samples": len(rows),
|
||||
"invalid_samples": 0,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
lines = [f"sample_id,question,{metric}"]
|
||||
for i, (question, score) in enumerate(rows):
|
||||
lines.append(f"s{i},{question},{score}")
|
||||
(run_dir / "scores.csv").write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
# A question unlikely to collide with anything under the real outputs/ tree.
|
||||
_Q = "UNIQTESTQ ball tube lifetime evaluation method 9f3a"
|
||||
|
||||
|
||||
def test_normalize_question_is_case_and_whitespace_insensitive() -> None:
|
||||
assert normalize_question(" Hello World ") == normalize_question("hello world")
|
||||
|
||||
|
||||
def test_index_matches_question_across_runs_newest_first(tmp_path: Path) -> None:
|
||||
_write_run(
|
||||
tmp_path / "runA",
|
||||
run_id="runA",
|
||||
scenario_name="scnA",
|
||||
finished_at="2026-01-01T00:00:00",
|
||||
rows=[(_Q, 0.40)],
|
||||
)
|
||||
_write_run(
|
||||
tmp_path / "runB",
|
||||
run_id="runB",
|
||||
scenario_name="scnB",
|
||||
finished_at="2026-02-01T00:00:00",
|
||||
rows=[(_Q, 0.80)],
|
||||
)
|
||||
|
||||
index = build_question_history_index(extra_roots=[tmp_path])
|
||||
entries = lookup(index, _Q)
|
||||
|
||||
assert [e["run_id"] for e in entries] == ["runB", "runA"] # newest first
|
||||
assert entries[0]["metrics"]["faithfulness"] == 0.80
|
||||
assert entries[1]["scenario_name"] == "scnA"
|
||||
|
||||
|
||||
def test_index_excludes_current_run(tmp_path: Path) -> None:
|
||||
_write_run(
|
||||
tmp_path / "cur",
|
||||
run_id="cur",
|
||||
scenario_name="scn",
|
||||
finished_at="2026-03-01T00:00:00",
|
||||
rows=[(_Q, 0.50)],
|
||||
)
|
||||
_write_run(
|
||||
tmp_path / "prev",
|
||||
run_id="prev",
|
||||
scenario_name="scn",
|
||||
finished_at="2026-01-01T00:00:00",
|
||||
rows=[(_Q, 0.60)],
|
||||
)
|
||||
|
||||
index = build_question_history_index(exclude_run_id="cur", extra_roots=[tmp_path])
|
||||
entries = lookup(index, _Q)
|
||||
|
||||
assert [e["run_id"] for e in entries] == ["prev"]
|
||||
|
||||
|
||||
def test_index_keeps_latest_occurrence_within_a_run(tmp_path: Path) -> None:
|
||||
_write_run(
|
||||
tmp_path / "run1",
|
||||
run_id="run1",
|
||||
scenario_name="scn",
|
||||
finished_at="2026-01-01T00:00:00",
|
||||
rows=[(_Q, 0.30), (_Q, 0.70)], # same question twice
|
||||
)
|
||||
|
||||
index = build_question_history_index(extra_roots=[tmp_path])
|
||||
entries = lookup(index, _Q)
|
||||
|
||||
assert len(entries) == 1
|
||||
assert entries[0]["metrics"]["faithfulness"] == 0.70 # last occurrence wins
|
||||
|
||||
|
||||
def test_lookup_caps_at_limit(tmp_path: Path) -> None:
|
||||
for i in range(7):
|
||||
_write_run(
|
||||
tmp_path / f"r{i}",
|
||||
run_id=f"r{i}",
|
||||
scenario_name="scn",
|
||||
finished_at=f"2026-01-0{i + 1}T00:00:00",
|
||||
rows=[(_Q, 0.1 * i)],
|
||||
)
|
||||
|
||||
index = build_question_history_index(extra_roots=[tmp_path])
|
||||
assert len(lookup(index, _Q, limit=3)) == 3
|
||||
|
||||
|
||||
def test_lookup_unknown_question_returns_empty(tmp_path: Path) -> None:
|
||||
index = build_question_history_index(extra_roots=[tmp_path])
|
||||
assert lookup(index, "NO SUCH QUESTION zzz 0000") == []
|
||||
@@ -13,35 +13,17 @@ import threading
|
||||
from typing import Any
|
||||
|
||||
from rag_eval.compat import ensure_ragas_import_compat
|
||||
from rag_eval.metrics.factory import build_models
|
||||
from rag_eval.metrics.factory import build_metric_registry, build_models
|
||||
from rag_eval.metrics.pipeline import MetricPipeline
|
||||
from rag_eval.settings import EvaluationSettings
|
||||
from rag_eval.shared.models import NormalizedSample
|
||||
|
||||
ensure_ragas_import_compat()
|
||||
|
||||
from ragas.metrics.collections import ( # noqa: E402
|
||||
AnswerRelevancy,
|
||||
ContextPrecision,
|
||||
ContextRecall,
|
||||
FactualCorrectness,
|
||||
Faithfulness,
|
||||
NoiseSensitivity,
|
||||
SemanticSimilarity,
|
||||
)
|
||||
|
||||
|
||||
def _build_metric_instances(metrics: list[str], llm: Any, embeddings: Any) -> dict[str, Any]:
|
||||
"""Instantiate only the RAGAS metric objects requested."""
|
||||
registry: dict[str, Any] = {
|
||||
"faithfulness": Faithfulness(llm=llm),
|
||||
"answer_relevancy": AnswerRelevancy(llm=llm, embeddings=embeddings),
|
||||
"context_recall": ContextRecall(llm=llm),
|
||||
"context_precision": ContextPrecision(llm=llm),
|
||||
"noise_sensitivity": NoiseSensitivity(llm=llm),
|
||||
"factual_correctness": FactualCorrectness(llm=llm),
|
||||
"semantic_similarity": SemanticSimilarity(embeddings=embeddings),
|
||||
}
|
||||
registry = build_metric_registry(llm, embeddings)
|
||||
return {name: registry[name] for name in metrics if name in registry}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Build a cross-run index of per-question RAGAS scores for historical comparison.
|
||||
|
||||
The report detail page surfaces, for each low-scoring sample, how the same
|
||||
question scored in previous evaluations. Matching is by normalized question text
|
||||
(case-insensitive, whitespace-collapsed) across all discovered run directories,
|
||||
so a question evaluated in any earlier run shows up as history.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from webapp.services import run_reader
|
||||
from webapp.services.run_reader import NON_METRIC_COLUMNS, _read_json
|
||||
|
||||
|
||||
def normalize_question(question: Any) -> str:
|
||||
"""Return a stable match key for a question (case/whitespace-insensitive)."""
|
||||
return " ".join(str(question or "").split()).lower()
|
||||
|
||||
|
||||
def _row_metrics(row: dict[str, Any]) -> dict[str, float | None]:
|
||||
"""Extract numeric metric scores from a single scores.csv row."""
|
||||
metrics: dict[str, float | None] = {}
|
||||
for key, value in row.items():
|
||||
if key in NON_METRIC_COLUMNS:
|
||||
continue
|
||||
try:
|
||||
num = float(value)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if pd.isna(num):
|
||||
continue
|
||||
metrics[str(key)] = round(num, 4)
|
||||
return metrics
|
||||
|
||||
|
||||
def build_question_history_index(
|
||||
exclude_run_id: str | None = None,
|
||||
extra_roots: list[Path] | None = None,
|
||||
) -> dict[str, list[dict[str, Any]]]:
|
||||
"""Scan all run dirs and group per-question score entries for history lookup.
|
||||
|
||||
Args:
|
||||
exclude_run_id: Run whose rows are skipped, so "history" means *other*
|
||||
evaluations (typically the run currently being viewed).
|
||||
extra_roots: Additional output roots to scan (beyond the defaults).
|
||||
|
||||
Returns:
|
||||
Map of normalized_question -> list of entries, each shaped as
|
||||
``{run_id, scenario_name, finished_at, metrics: {metric: value}}`` and
|
||||
sorted by finished_at descending (most recent first). Within a single
|
||||
run, only the last occurrence of a question is kept.
|
||||
"""
|
||||
# question_key -> run_id -> entry (last write wins within the same run)
|
||||
grouped: dict[str, dict[str, dict[str, Any]]] = {}
|
||||
|
||||
for run_dir in run_reader.discover_run_dirs(extra_roots):
|
||||
metadata = _read_json(run_dir / "metadata.json")
|
||||
run_id = str(metadata.get("run_id") or run_dir.name)
|
||||
if exclude_run_id and run_id == exclude_run_id:
|
||||
continue
|
||||
scenario_name = str(metadata.get("scenario_name", ""))
|
||||
finished_at = str(metadata.get("finished_at") or metadata.get("started_at") or "")
|
||||
|
||||
frame = run_reader.read_scores_frame(run_dir)
|
||||
if frame.empty or "question" not in frame.columns:
|
||||
continue
|
||||
|
||||
for record in frame.where(pd.notnull(frame), None).to_dict("records"):
|
||||
key = normalize_question(record.get("question"))
|
||||
if not key:
|
||||
continue
|
||||
metrics = _row_metrics(record)
|
||||
if not metrics:
|
||||
continue
|
||||
grouped.setdefault(key, {})[run_id] = {
|
||||
"run_id": run_id,
|
||||
"scenario_name": scenario_name,
|
||||
"finished_at": finished_at,
|
||||
"metrics": metrics,
|
||||
}
|
||||
|
||||
index: dict[str, list[dict[str, Any]]] = {}
|
||||
for key, per_run in grouped.items():
|
||||
entries = list(per_run.values())
|
||||
entries.sort(key=lambda entry: entry["finished_at"], reverse=True)
|
||||
index[key] = entries
|
||||
return index
|
||||
|
||||
|
||||
def lookup(
|
||||
index: dict[str, list[dict[str, Any]]],
|
||||
question: Any,
|
||||
limit: int = 5,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return up to ``limit`` historical entries for a question (newest first)."""
|
||||
entries = index.get(normalize_question(question), [])
|
||||
return entries[: max(0, limit)]
|
||||
@@ -0,0 +1,418 @@
|
||||
// dashboard.js — 指标看板：运行选择器 + 折线图（指标趋势） + 柱状图（vs 达标阈值）。
|
||||
// 纯前端，数据来自 GET /api/runs，复用 MetricPresenter 的方向语义与阈值。
|
||||
|
||||
(function attachDashboard(globalObj) {
|
||||
const Dashboard = {
|
||||
_runs: [], // 全量 runs（已按 finished_at 倒序来自 API）
|
||||
_selected: new Set(), // 当前勾选的 run_id 集合
|
||||
_focusId: null, // 柱状图聚焦的 run_id
|
||||
_trendChart: null,
|
||||
_barChart: null,
|
||||
|
||||
// ── 入口 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
async load() {
|
||||
const wrap = document.getElementById("dashboard-wrap");
|
||||
if (!wrap) return;
|
||||
wrap.innerHTML = '<p class="muted">加载中…</p>';
|
||||
|
||||
try {
|
||||
const data = await API.runs();
|
||||
Dashboard._runs = (data.runs || []).slice().sort(
|
||||
(a, b) => (a.finished_at || "").localeCompare(b.finished_at || "")
|
||||
);
|
||||
Dashboard._selected = new Set(Dashboard._runs.map((r) => r.run_id));
|
||||
Dashboard._focusId = Dashboard._runs.length
|
||||
? Dashboard._runs[Dashboard._runs.length - 1].run_id
|
||||
: null;
|
||||
Dashboard._render(wrap);
|
||||
} catch (err) {
|
||||
wrap.innerHTML = `<p class="muted">加载失败：${App.escape(err.message)}</p>`;
|
||||
}
|
||||
},
|
||||
|
||||
// ── 渲染 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
_render(wrap) {
|
||||
if (!Dashboard._runs.length) {
|
||||
wrap.innerHTML = `
|
||||
<div class="empty">
|
||||
<p>暂无评测运行数据。</p>
|
||||
<p class="muted">触发一次评测或通过 Dify 工具调用后，数据将在此显示。</p>
|
||||
</div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
wrap.innerHTML = "";
|
||||
|
||||
// 运行选择器面板
|
||||
wrap.appendChild(Dashboard._buildSelector());
|
||||
|
||||
// 图表区
|
||||
const chartRow = document.createElement("div");
|
||||
chartRow.className = "dashboard-charts";
|
||||
chartRow.innerHTML = `
|
||||
<div class="panel dashboard-chart-panel">
|
||||
<div class="db-panel-head-bar">
|
||||
<div>
|
||||
<div class="section-label tight">📈 指标趋势折线图</div>
|
||||
<div class="muted" style="font-size:12px;margin-top:2px">按时间顺序展示所选运行的各指标均值变化</div>
|
||||
</div>
|
||||
</div>
|
||||
<canvas id="db-trend-chart"></canvas>
|
||||
<p class="db-chart-hint" id="db-trend-hint"></p>
|
||||
</div>
|
||||
<div class="panel dashboard-chart-panel">
|
||||
<div class="db-panel-head-bar">
|
||||
<div>
|
||||
<div class="section-label tight">📊 指标达标对比柱状图</div>
|
||||
<div class="muted" style="font-size:12px;margin-top:2px">实际均值 vs 达标阈值（深绿柱）</div>
|
||||
</div>
|
||||
<select class="select" id="db-focus-select" style="min-width:220px"></select>
|
||||
</div>
|
||||
<canvas id="db-bar-chart"></canvas>
|
||||
<p class="db-chart-hint">达标阈值：higher-better 指标 0.85 · noise_sensitivity 0.15</p>
|
||||
</div>
|
||||
`;
|
||||
wrap.appendChild(chartRow);
|
||||
|
||||
Dashboard._populateFocusSelect();
|
||||
Dashboard._drawTrend();
|
||||
Dashboard._drawBar();
|
||||
},
|
||||
|
||||
// 运行选择器
|
||||
_buildSelector() {
|
||||
const panel = document.createElement("div");
|
||||
panel.className = "panel";
|
||||
panel.innerHTML = `
|
||||
<div class="panel-head">
|
||||
<div>
|
||||
<span class="section-label tight">运行选择（折线图数据源）</span>
|
||||
<span class="muted" style="font-size:12px; margin-left:10px">勾选≥2个运行可看趋势</span>
|
||||
</div>
|
||||
<div style="display:flex;gap:8px">
|
||||
<button class="btn btn-sm" id="db-sel-all">全选</button>
|
||||
<button class="btn btn-sm" id="db-sel-none">清空</button>
|
||||
<input class="form-input" id="db-filter-input" placeholder="按场景名过滤…" style="width:180px;padding:6px 10px" />
|
||||
</div>
|
||||
</div>
|
||||
<div class="db-run-list" id="db-run-list"></div>
|
||||
`;
|
||||
setTimeout(() => {
|
||||
Dashboard._renderRunList();
|
||||
document.getElementById("db-sel-all").onclick = () => {
|
||||
Dashboard._runs.forEach((r) => Dashboard._selected.add(r.run_id));
|
||||
Dashboard._renderRunList();
|
||||
Dashboard._drawTrend();
|
||||
};
|
||||
document.getElementById("db-sel-none").onclick = () => {
|
||||
Dashboard._selected.clear();
|
||||
Dashboard._renderRunList();
|
||||
Dashboard._drawTrend();
|
||||
};
|
||||
document.getElementById("db-filter-input").oninput = (e) => {
|
||||
Dashboard._renderRunList(e.target.value.toLowerCase());
|
||||
};
|
||||
});
|
||||
return panel;
|
||||
},
|
||||
|
||||
_renderRunList(filter) {
|
||||
const list = document.getElementById("db-run-list");
|
||||
if (!list) return;
|
||||
list.innerHTML = "";
|
||||
const visible = filter
|
||||
? Dashboard._runs.filter((r) =>
|
||||
(r.scenario_name || r.run_id).toLowerCase().includes(filter)
|
||||
)
|
||||
: Dashboard._runs;
|
||||
[...visible].reverse().forEach((run) => {
|
||||
const row = document.createElement("label");
|
||||
row.className = "db-run-row";
|
||||
const chips = (run.metrics || [])
|
||||
.slice(0, 4)
|
||||
.map((m) => {
|
||||
const v = run.metric_means ? run.metric_means[m] : null;
|
||||
const cls = App.scoreClass(m, v);
|
||||
const text = v === null || v === undefined ? "n/a" : Number(v).toFixed(2);
|
||||
return `<span class="metric-chip"><span class="db-chip-name">${App.escape(App.shortMetric(m))}</span> <b class="${cls}">${text}</b></span>`;
|
||||
})
|
||||
.join("");
|
||||
row.innerHTML = `
|
||||
<input type="checkbox" class="db-run-cb" data-id="${App.escape(run.run_id)}"
|
||||
${Dashboard._selected.has(run.run_id) ? "checked" : ""} />
|
||||
<span class="db-run-label">
|
||||
<span class="db-run-name">${App.escape(run.scenario_name || run.run_id)}</span>
|
||||
<span class="muted" style="font-size:11px">${App.escape(App.shortTime(run.finished_at))} · ${App.escape(run.judge_model || "")}</span>
|
||||
</span>
|
||||
<span class="db-run-chips">${chips}</span>
|
||||
`;
|
||||
row.querySelector(".db-run-cb").addEventListener("change", (e) => {
|
||||
if (e.target.checked) Dashboard._selected.add(run.run_id);
|
||||
else Dashboard._selected.delete(run.run_id);
|
||||
Dashboard._drawTrend();
|
||||
});
|
||||
list.appendChild(row);
|
||||
});
|
||||
},
|
||||
|
||||
// 填充柱状图聚焦下拉
|
||||
_populateFocusSelect() {
|
||||
const sel = document.getElementById("db-focus-select");
|
||||
if (!sel) return;
|
||||
sel.innerHTML = "";
|
||||
[...Dashboard._runs].reverse().forEach((run) => {
|
||||
const opt = document.createElement("option");
|
||||
opt.value = run.run_id;
|
||||
opt.textContent = `${run.scenario_name || run.run_id} ${App.shortTime(run.finished_at)}`;
|
||||
if (run.run_id === Dashboard._focusId) opt.selected = true;
|
||||
sel.appendChild(opt);
|
||||
});
|
||||
sel.onchange = () => {
|
||||
Dashboard._focusId = sel.value;
|
||||
Dashboard._drawBar();
|
||||
};
|
||||
},
|
||||
|
||||
// ── 折线图 ────────────────────────────────────────────────────────────────
|
||||
|
||||
_drawTrend() {
|
||||
const canvas = document.getElementById("db-trend-chart");
|
||||
const hint = document.getElementById("db-trend-hint");
|
||||
if (!canvas) return;
|
||||
|
||||
const selected = Dashboard._runs.filter((r) => Dashboard._selected.has(r.run_id));
|
||||
if (selected.length === 0) {
|
||||
if (Dashboard._trendChart) { Dashboard._trendChart.destroy(); Dashboard._trendChart = null; }
|
||||
if (hint) hint.textContent = "请在上方勾选至少 1 个运行。";
|
||||
return;
|
||||
}
|
||||
if (hint) {
|
||||
hint.textContent = selected.length === 1
|
||||
? "只有 1 个运行，折线退化为单点——勾选更多运行可看趋势。"
|
||||
: `共 ${selected.length} 个运行 · 横轴按完成时间升序`;
|
||||
}
|
||||
|
||||
const { labels, datasets } = Dashboard._buildTrendDatasets(selected);
|
||||
// 精选对比度高、色盲友好的颜色组合
|
||||
const colors = [
|
||||
"#009999", // petrol brand
|
||||
"#3b82f6", // blue
|
||||
"#f97316", // orange
|
||||
"#8b5cf6", // violet
|
||||
"#ec4899", // pink
|
||||
"#06b6d4", // cyan
|
||||
"#f59e0b", // amber
|
||||
];
|
||||
|
||||
if (Dashboard._trendChart) Dashboard._trendChart.destroy();
|
||||
Dashboard._trendChart = new Chart(canvas, {
|
||||
type: "line",
|
||||
data: {
|
||||
labels,
|
||||
datasets: [
|
||||
// 达标参考线（0.85，虚线，不显示在图例前列）
|
||||
{
|
||||
label: "达标参考线 0.85",
|
||||
data: Array(labels.length).fill(0.85),
|
||||
borderColor: "#16a34a",
|
||||
borderDash: [6, 4],
|
||||
borderWidth: 1.5,
|
||||
pointRadius: 0,
|
||||
fill: false,
|
||||
order: 99,
|
||||
},
|
||||
...datasets.map((ds, i) => ({
|
||||
label: ds.label,
|
||||
data: ds.data,
|
||||
borderColor: colors[i % colors.length],
|
||||
backgroundColor: colors[i % colors.length] + "18",
|
||||
borderWidth: 2.5,
|
||||
pointRadius: 5,
|
||||
pointHoverRadius: 7,
|
||||
pointBackgroundColor: colors[i % colors.length],
|
||||
pointBorderColor: "#fff",
|
||||
pointBorderWidth: 2,
|
||||
tension: 0.25,
|
||||
spanGaps: false,
|
||||
})),
|
||||
],
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
interaction: { mode: "index", intersect: false },
|
||||
plugins: {
|
||||
legend: {
|
||||
position: "bottom",
|
||||
labels: { font: { size: 12 }, boxWidth: 14, padding: 16, usePointStyle: true, pointStyleWidth: 12 },
|
||||
},
|
||||
tooltip: {
|
||||
backgroundColor: "#1a2942",
|
||||
titleColor: "#e2e8f0",
|
||||
bodyColor: "#cbd5e1",
|
||||
borderColor: "#334155",
|
||||
borderWidth: 1,
|
||||
padding: 10,
|
||||
callbacks: {
|
||||
label: (ctx) => {
|
||||
if (ctx.raw === null) return ` ${ctx.dataset.label}: —`;
|
||||
return ` ${ctx.dataset.label}: ${Number(ctx.raw).toFixed(3)}`;
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
scales: {
|
||||
y: {
|
||||
min: 0, max: 1,
|
||||
ticks: { stepSize: 0.1, font: { size: 11 }, color: "#94a3b8" },
|
||||
grid: { color: "#f1f5f9" },
|
||||
border: { display: false },
|
||||
},
|
||||
x: {
|
||||
ticks: { font: { size: 11 }, color: "#64748b", maxRotation: 30 },
|
||||
grid: { display: false },
|
||||
border: { display: false },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
|
||||
// 柱状图 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
_drawBar() {
|
||||
const canvas = document.getElementById("db-bar-chart");
|
||||
if (!canvas) return;
|
||||
|
||||
const run = Dashboard._runs.find((r) => r.run_id === Dashboard._focusId);
|
||||
if (!run) return;
|
||||
|
||||
const { labels, actual, thresholds, colors, targetMet } =
|
||||
Dashboard._buildComparisonData(run);
|
||||
|
||||
if (Dashboard._barChart) Dashboard._barChart.destroy();
|
||||
Dashboard._barChart = new Chart(canvas, {
|
||||
type: "bar",
|
||||
data: {
|
||||
labels,
|
||||
datasets: [
|
||||
{
|
||||
label: "实际分数",
|
||||
data: actual,
|
||||
backgroundColor: colors,
|
||||
borderRadius: 5,
|
||||
barPercentage: 0.6,
|
||||
categoryPercentage: 0.75,
|
||||
},
|
||||
{
|
||||
label: "达标阈值",
|
||||
data: thresholds,
|
||||
// 深绿实心柱（不透明，无边框线）
|
||||
backgroundColor: "#15803d",
|
||||
borderRadius: 5,
|
||||
barPercentage: 0.6,
|
||||
categoryPercentage: 0.75,
|
||||
},
|
||||
],
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
interaction: { mode: "index", intersect: false },
|
||||
plugins: {
|
||||
legend: {
|
||||
position: "bottom",
|
||||
labels: { font: { size: 12 }, boxWidth: 14, padding: 16, usePointStyle: false },
|
||||
},
|
||||
tooltip: {
|
||||
backgroundColor: "#1a2942",
|
||||
titleColor: "#e2e8f0",
|
||||
bodyColor: "#cbd5e1",
|
||||
borderColor: "#334155",
|
||||
borderWidth: 1,
|
||||
padding: 10,
|
||||
callbacks: {
|
||||
afterBody: (ctx) => {
|
||||
if (!ctx.length) return [];
|
||||
const idx = ctx[0].dataIndex;
|
||||
return [targetMet[idx] ? "✓ 达标" : "✗ 未达标"];
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
scales: {
|
||||
y: {
|
||||
min: 0, max: 1,
|
||||
ticks: { stepSize: 0.1, font: { size: 11 }, color: "#94a3b8" },
|
||||
grid: { color: "#f1f5f9" },
|
||||
border: { display: false },
|
||||
},
|
||||
x: {
|
||||
ticks: { font: { size: 11 }, color: "#64748b" },
|
||||
grid: { display: false },
|
||||
border: { display: false },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
|
||||
// ── 纯数据函数（便于测试）────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* 从选中的 runs（按时间升序）构建折线图数据集。
|
||||
* 每条线 = 一个指标；X 轴 = 各运行的简短标签；无值处补 null（断线）。
|
||||
* @param {Array} runs - 已按 finished_at 升序排序的 run 对象数组
|
||||
* @returns {{ labels: string[], datasets: Array<{label:string, data:Array<number|null>}> }}
|
||||
*/
|
||||
_buildTrendDatasets(runs) {
|
||||
// 合并所有 runs 出现过的指标（保持首次出现顺序）
|
||||
const metricSet = [];
|
||||
runs.forEach((r) => {
|
||||
(r.metrics || []).forEach((m) => {
|
||||
if (!metricSet.includes(m)) metricSet.push(m);
|
||||
});
|
||||
});
|
||||
|
||||
const labels = runs.map(
|
||||
(r) => `${r.scenario_name || r.run_id}\n${App.shortTime(r.finished_at)}`
|
||||
);
|
||||
|
||||
const datasets = metricSet.map((m) => ({
|
||||
label: m + (MetricPresenter.isLowerBetter(m) ? " (越低越好)" : ""),
|
||||
data: runs.map((r) => {
|
||||
const v = r.metric_means ? r.metric_means[m] : null;
|
||||
return v !== null && v !== undefined ? Number(v) : null;
|
||||
}),
|
||||
}));
|
||||
|
||||
return { labels, datasets };
|
||||
},
|
||||
|
||||
/**
|
||||
* 从单个 run 构建柱状图对比数据。
|
||||
* @param {Object} run
|
||||
* @returns {{ labels, actual, thresholds, colors, targetMet }}
|
||||
*/
|
||||
_buildComparisonData(run) {
|
||||
const metrics = run.metrics || [];
|
||||
const labels = metrics.map((m) => App.shortMetric(m));
|
||||
const actual = metrics.map((m) => {
|
||||
const v = run.metric_means ? run.metric_means[m] : null;
|
||||
return v !== null && v !== undefined ? Number(v) : null;
|
||||
});
|
||||
const thresholds = metrics.map((m) => MetricPresenter.passThreshold(m));
|
||||
const targetMet = metrics.map((m, i) =>
|
||||
MetricPresenter.meetsTarget(m, actual[i])
|
||||
);
|
||||
const colorMap = { good: "#4ade80", warn: "#fbbf24", bad: "#f87171", na: "#cbd5e1" };
|
||||
const colors = metrics.map((m, i) => colorMap[App.scoreClass(m, actual[i])] || "#cbd5e1");
|
||||
|
||||
return { labels, actual, thresholds, colors, targetMet };
|
||||
},
|
||||
};
|
||||
|
||||
globalObj.Dashboard = Dashboard;
|
||||
})(typeof window !== "undefined" ? window : this);
|
||||
Reference in New Issue
Block a user