89 lines
3.4 KiB
Python
89 lines
3.4 KiB
Python
"""Tests for webapp.services.advisor_comparison: same-scenario run comparison."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
from webapp.services.advisor_comparison import find_previous_run
|
|
|
|
|
|
def _write_fake_run(
|
|
root: Path,
|
|
run_id: str,
|
|
scenario_name: str,
|
|
finished_at: str,
|
|
judge_model: str,
|
|
rows: list[dict],
|
|
) -> Path:
|
|
"""Write a minimal run directory discoverable by run_reader.list_run_summaries()."""
|
|
run_dir = root / run_id
|
|
run_dir.mkdir(parents=True, exist_ok=True)
|
|
pd.DataFrame(rows).to_csv(run_dir / "scores.csv", index=False)
|
|
metadata = {
|
|
"run_id": run_id,
|
|
"scenario_name": scenario_name,
|
|
"judge_model": judge_model,
|
|
"embedding_model": "embed-model",
|
|
"finished_at": finished_at,
|
|
"started_at": finished_at,
|
|
"valid_samples": len(rows),
|
|
"invalid_samples": 0,
|
|
}
|
|
(run_dir / "metadata.json").write_text(json.dumps(metadata), encoding="utf-8")
|
|
return run_dir
|
|
|
|
|
|
class TestFindPreviousRun:
|
|
def test_finds_most_recent_prior_run_with_same_scenario(self, tmp_path: Path) -> None:
|
|
_write_fake_run(tmp_path, "r1", "scn-a", "2026-01-01T00:00:00+00:00", "gpt-5",
|
|
[{"sample_id": "s1", "faithfulness": 0.5}])
|
|
_write_fake_run(tmp_path, "r2", "scn-a", "2026-01-02T00:00:00+00:00", "gpt-5",
|
|
[{"sample_id": "s1", "faithfulness": 0.6}])
|
|
_write_fake_run(tmp_path, "r3", "scn-a", "2026-01-03T00:00:00+00:00", "gpt-5",
|
|
[{"sample_id": "s1", "faithfulness": 0.9}])
|
|
|
|
previous = find_previous_run(
|
|
"scn-a", "r3", "2026-01-03T00:00:00+00:00", extra_roots=[tmp_path]
|
|
)
|
|
|
|
assert previous is not None
|
|
assert previous.run_id == "r2"
|
|
|
|
def test_excludes_runs_with_different_scenario_name(self, tmp_path: Path) -> None:
|
|
_write_fake_run(tmp_path, "r1", "scn-other", "2026-01-01T00:00:00+00:00", "gpt-5",
|
|
[{"sample_id": "s1", "faithfulness": 0.5}])
|
|
_write_fake_run(tmp_path, "r2", "scn-a", "2026-01-02T00:00:00+00:00", "gpt-5",
|
|
[{"sample_id": "s1", "faithfulness": 0.6}])
|
|
|
|
previous = find_previous_run(
|
|
"scn-a", "r2", "2026-01-02T00:00:00+00:00", extra_roots=[tmp_path]
|
|
)
|
|
|
|
assert previous is None
|
|
|
|
def test_returns_none_when_no_history(self, tmp_path: Path) -> None:
|
|
_write_fake_run(tmp_path, "r1", "scn-a", "2026-01-01T00:00:00+00:00", "gpt-5",
|
|
[{"sample_id": "s1", "faithfulness": 0.5}])
|
|
|
|
previous = find_previous_run(
|
|
"scn-a", "r1", "2026-01-01T00:00:00+00:00", extra_roots=[tmp_path]
|
|
)
|
|
|
|
assert previous is None
|
|
|
|
def test_ignores_runs_at_or_after_current_time(self, tmp_path: Path) -> None:
|
|
_write_fake_run(tmp_path, "r1", "scn-a", "2026-01-01T00:00:00+00:00", "gpt-5",
|
|
[{"sample_id": "s1", "faithfulness": 0.5}])
|
|
_write_fake_run(tmp_path, "r2", "scn-a", "2026-01-05T00:00:00+00:00", "gpt-5",
|
|
[{"sample_id": "s1", "faithfulness": 0.6}])
|
|
|
|
# Current run finished at 2026-01-02, i.e. AFTER r1 but BEFORE r2.
|
|
previous = find_previous_run(
|
|
"scn-a", "r-current", "2026-01-02T00:00:00+00:00", extra_roots=[tmp_path]
|
|
)
|
|
|
|
assert previous is not None
|
|
assert previous.run_id == "r1"
|