siemens_ragas/tests/test_dashboard.py

"""Tests for the Dashboard module's pure data functions and MetricPresenter additions.

The pure data functions (_buildTrendDatasets, _buildComparisonData) and the new
MetricPresenter helpers (passThreshold, meetsTarget) are tested via Node.js so no
browser or network is required.
"""
from __future__ import annotations

import subprocess
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parents[1]


def _run_node(script: str) -> str:
    """Execute a Node.js script and return stdout."""
    completed = subprocess.run(
        ["node", "-e", script],
        cwd=REPO_ROOT,
        capture_output=True,
        text=True,
        encoding="utf-8",
        check=True,
    )
    return completed.stdout.strip()


def _load_js() -> str:
    """Return a Node-runnable bootstrap that loads MetricPresenter + Dashboard."""
    presenter_path = (REPO_ROOT / "webapp" / "static" / "js" / "metric_presenter.js").as_posix()
    dashboard_path = (REPO_ROOT / "webapp" / "static" / "js" / "dashboard.js").as_posix()
    return f"""
const fs = require("fs");
const vm = require("vm");

// Shared sandbox (window object shared by both scripts)
const sandbox = {{ window: {{}}, console }};

// MetricPresenter
vm.runInNewContext(fs.readFileSync("{presenter_path}", "utf8"), sandbox);
const MetricPresenter = sandbox.window.MetricPresenter;

// Minimal App stub required by dashboard.js
sandbox.App = {{
  escape: (s) => String(s == null ? "" : s),
  shortMetric: (m) => m,
  shortTime: (t) => (t || "").slice(0, 16),
  scoreClass: (m, v) => MetricPresenter.scoreClass(m, v),
}};
sandbox.MetricPresenter = MetricPresenter;
// Stub Chart.js (not exercised by pure functions)
sandbox.Chart = function() {{ this.destroy = () => {{}}; }};

// Dashboard (attaches itself to sandbox.window.Dashboard)
vm.runInNewContext(fs.readFileSync("{dashboard_path}", "utf8"), sandbox);
const Dashboard = sandbox.window.Dashboard;
"""


def test_pass_threshold_higher_better() -> None:
    """All higher-better metrics should have passThreshold 0.85."""
    script = _load_js() + """
const result = {
  faith: MetricPresenter.passThreshold("faithfulness"),
  ans: MetricPresenter.passThreshold("answer_relevancy"),
  recall: MetricPresenter.passThreshold("context_recall"),
  prec: MetricPresenter.passThreshold("context_precision"),
  fact: MetricPresenter.passThreshold("factual_correctness"),
  sem: MetricPresenter.passThreshold("semantic_similarity"),
};
console.log(JSON.stringify(result));
"""
    out = _run_node(script)
    assert '"faith":0.85' in out
    assert '"ans":0.85' in out
    assert '"recall":0.85' in out
    assert '"prec":0.85' in out
    assert '"fact":0.85' in out
    assert '"sem":0.85' in out


def test_pass_threshold_noise_sensitivity_lower_better() -> None:
    """noise_sensitivity (lower-better) should have passThreshold 0.15."""
    script = _load_js() + """
console.log(JSON.stringify(MetricPresenter.passThreshold("noise_sensitivity")));
"""
    out = _run_node(script)
    assert out.strip() == "0.15"


def test_meets_target_higher_better() -> None:
    """meetsTarget should return true only at/above 0.85 for higher-better metrics."""
    script = _load_js() + """
const result = {
  at085: MetricPresenter.meetsTarget("faithfulness", 0.85),
  above: MetricPresenter.meetsTarget("faithfulness", 0.90),
  below: MetricPresenter.meetsTarget("faithfulness", 0.84),
  zero: MetricPresenter.meetsTarget("faithfulness", 0),
  nul: MetricPresenter.meetsTarget("faithfulness", null),
};
console.log(JSON.stringify(result));
"""
    out = _run_node(script)
    assert '"at085":true' in out
    assert '"above":true' in out
    assert '"below":false' in out
    assert '"zero":false' in out
    assert '"nul":false' in out


def test_meets_target_noise_sensitivity() -> None:
    """meetsTarget for noise_sensitivity: true only at/below 0.15."""
    script = _load_js() + """
const result = {
  at015: MetricPresenter.meetsTarget("noise_sensitivity", 0.15),
  below: MetricPresenter.meetsTarget("noise_sensitivity", 0.10),
  above: MetricPresenter.meetsTarget("noise_sensitivity", 0.16),
};
console.log(JSON.stringify(result));
"""
    out = _run_node(script)
    assert '"at015":true' in out
    assert '"below":true' in out
    assert '"above":false' in out


def test_build_trend_datasets_time_order_and_null_gap() -> None:
    """_buildTrendDatasets returns metrics in appearance order, null for missing values."""
    script = (
        _load_js()
        + """
const runs = [
  { run_id: "r1", scenario_name: "scn", finished_at: "2026-01-01T00:00:00",
    metrics: ["faithfulness"], metric_means: { faithfulness: 0.60 } },
  { run_id: "r2", scenario_name: "scn", finished_at: "2026-02-01T00:00:00",
    metrics: ["faithfulness", "noise_sensitivity"],
    metric_means: { faithfulness: 0.80, noise_sensitivity: 0.20 } },
  { run_id: "r3", scenario_name: "scn", finished_at: "2026-03-01T00:00:00",
    metrics: ["faithfulness"], metric_means: { faithfulness: 0.90 } },
];
const { labels, datasets } = Dashboard._buildTrendDatasets(runs);
console.log(JSON.stringify({ labels, datasets }));
"""
    )
    import json

    out = json.loads(_run_node(script))
    assert len(out["labels"]) == 3
    # faithfulness dataset: all 3 points
    faith_ds = next(d for d in out["datasets"] if "faithfulness" in d["label"])
    assert faith_ds["data"] == [0.60, 0.80, 0.90]
    # noise_sensitivity only present in r2 → null in r1 and r3
    noise_ds = next(d for d in out["datasets"] if "noise_sensitivity" in d["label"])
    assert noise_ds["data"] == [None, 0.20, None]
    # noise label should note lower-is-better
    assert "越低越好" in noise_ds["label"]


def test_build_comparison_data_structure() -> None:
    """_buildComparisonData returns correct labels, actuals, thresholds and targetMet."""
    script = (
        _load_js()
        + """
const run = {
  run_id: "r1", scenario_name: "scn",
  metrics: ["faithfulness", "noise_sensitivity"],
  metric_means: { faithfulness: 0.90, noise_sensitivity: 0.10 },
};
const result = Dashboard._buildComparisonData(run);
console.log(JSON.stringify(result));
"""
    )
    import json

    out = json.loads(_run_node(script))
    assert out["labels"] == ["faithfulness", "noise_sensitivity"]
    assert out["actual"] == [0.90, 0.10]
    assert out["thresholds"] == [0.85, 0.15]
    assert out["targetMet"] == [True, True]  # 0.90 >= 0.85 ✓; 0.10 <= 0.15 ✓


def test_build_comparison_data_unmet_targets() -> None:
    """targetMet is False when metrics are below threshold."""
    script = (
        _load_js()
        + """
const run = {
  run_id: "r1", scenario_name: "scn",
  metrics: ["faithfulness", "noise_sensitivity"],
  metric_means: { faithfulness: 0.60, noise_sensitivity: 0.40 },
};
const result = Dashboard._buildComparisonData(run);
console.log(JSON.stringify(result.targetMet));
"""
    )
    out = _run_node(script)
    assert out.strip() == "[false,false]"