103 lines
3.7 KiB
Python
103 lines
3.7 KiB
Python
"""Build a cross-run index of per-question RAGAS scores for historical comparison.
|
|
|
|
The report detail page surfaces, for each low-scoring sample, how the same
|
|
question scored in previous evaluations. Matching is by normalized question text
|
|
(case-insensitive, whitespace-collapsed) across all discovered run directories,
|
|
so a question evaluated in any earlier run shows up as history.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pandas as pd
|
|
|
|
from webapp.services import run_reader
|
|
from webapp.services.run_reader import NON_METRIC_COLUMNS, _read_json
|
|
|
|
|
|
def normalize_question(question: Any) -> str:
|
|
"""Return a stable match key for a question (case/whitespace-insensitive)."""
|
|
return " ".join(str(question or "").split()).lower()
|
|
|
|
|
|
def _row_metrics(row: dict[str, Any]) -> dict[str, float | None]:
|
|
"""Extract numeric metric scores from a single scores.csv row."""
|
|
metrics: dict[str, float | None] = {}
|
|
for key, value in row.items():
|
|
if key in NON_METRIC_COLUMNS:
|
|
continue
|
|
try:
|
|
num = float(value)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if pd.isna(num):
|
|
continue
|
|
metrics[str(key)] = round(num, 4)
|
|
return metrics
|
|
|
|
|
|
def build_question_history_index(
|
|
exclude_run_id: str | None = None,
|
|
extra_roots: list[Path] | None = None,
|
|
) -> dict[str, list[dict[str, Any]]]:
|
|
"""Scan all run dirs and group per-question score entries for history lookup.
|
|
|
|
Args:
|
|
exclude_run_id: Run whose rows are skipped, so "history" means *other*
|
|
evaluations (typically the run currently being viewed).
|
|
extra_roots: Additional output roots to scan (beyond the defaults).
|
|
|
|
Returns:
|
|
Map of normalized_question -> list of entries, each shaped as
|
|
``{run_id, scenario_name, finished_at, metrics: {metric: value}}`` and
|
|
sorted by finished_at descending (most recent first). Within a single
|
|
run, only the last occurrence of a question is kept.
|
|
"""
|
|
# question_key -> run_id -> entry (last write wins within the same run)
|
|
grouped: dict[str, dict[str, dict[str, Any]]] = {}
|
|
|
|
for run_dir in run_reader.discover_run_dirs(extra_roots):
|
|
metadata = _read_json(run_dir / "metadata.json")
|
|
run_id = str(metadata.get("run_id") or run_dir.name)
|
|
if exclude_run_id and run_id == exclude_run_id:
|
|
continue
|
|
scenario_name = str(metadata.get("scenario_name", ""))
|
|
finished_at = str(metadata.get("finished_at") or metadata.get("started_at") or "")
|
|
|
|
frame = run_reader.read_scores_frame(run_dir)
|
|
if frame.empty or "question" not in frame.columns:
|
|
continue
|
|
|
|
for record in frame.where(pd.notnull(frame), None).to_dict("records"):
|
|
key = normalize_question(record.get("question"))
|
|
if not key:
|
|
continue
|
|
metrics = _row_metrics(record)
|
|
if not metrics:
|
|
continue
|
|
grouped.setdefault(key, {})[run_id] = {
|
|
"run_id": run_id,
|
|
"scenario_name": scenario_name,
|
|
"finished_at": finished_at,
|
|
"metrics": metrics,
|
|
}
|
|
|
|
index: dict[str, list[dict[str, Any]]] = {}
|
|
for key, per_run in grouped.items():
|
|
entries = list(per_run.values())
|
|
entries.sort(key=lambda entry: entry["finished_at"], reverse=True)
|
|
index[key] = entries
|
|
return index
|
|
|
|
|
|
def lookup(
|
|
index: dict[str, list[dict[str, Any]]],
|
|
question: Any,
|
|
limit: int = 5,
|
|
) -> list[dict[str, Any]]:
|
|
"""Return up to ``limit`` historical entries for a question (newest first)."""
|
|
entries = index.get(normalize_question(question), [])
|
|
return entries[: max(0, limit)]
|