150 lines
5.7 KiB
Python
150 lines
5.7 KiB
Python
"""Advisor-diagnosis comparison against the same scenario's previous run.
|
|
|
|
The report detail page shows, for the run currently being viewed, whether
|
|
metrics flagged by the optimization advisor in the immediately preceding run
|
|
of the same scenario have since improved, regressed, or remain unresolved.
|
|
|
|
No new artifact files are written: diagnoses are recomputed on demand from
|
|
each run's scores.csv via the existing rag_eval.advisor.diagnose(), so the
|
|
comparison always reflects the current threshold rules in rules.py.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pandas as pd
|
|
|
|
from rag_eval.advisor import diagnose
|
|
from webapp.models import AdvisorComparison, AdvisorComparisonEntry, RunSummary
|
|
from webapp.services import run_reader
|
|
|
|
logger = logging.getLogger("webapp.services.advisor_comparison")
|
|
|
|
# Worse-first ordering so regressions and unresolved issues surface above
|
|
# resolved ones in the rendered comparison list.
|
|
_STATUS_ORDER = {"regressed": 0, "still_triggered": 1, "new_metric": 2, "resolved": 3}
|
|
|
|
|
|
def find_previous_run(
|
|
scenario_name: str,
|
|
current_run_id: str,
|
|
current_finished_at: str,
|
|
extra_roots: list[Path] | None = None,
|
|
) -> RunSummary | None:
|
|
"""Return the most recent prior run with the same scenario_name, or None.
|
|
|
|
"Prior" means finished_at strictly earlier than current_finished_at, so a
|
|
run that finished later (e.g. a concurrent run) is never mistaken for a
|
|
historical baseline.
|
|
"""
|
|
candidates = [
|
|
summary
|
|
for summary in run_reader.list_run_summaries(extra_roots)
|
|
if summary.scenario_name == scenario_name
|
|
and summary.run_id != current_run_id
|
|
and (summary.finished_at or "") < (current_finished_at or "")
|
|
]
|
|
if not candidates:
|
|
return None
|
|
candidates.sort(key=lambda summary: summary.finished_at or "", reverse=True)
|
|
return candidates[0]
|
|
|
|
|
|
def _rows_as_records(frame: pd.DataFrame) -> list[dict[str, Any]]:
|
|
"""Convert a scores dataframe into plain dict records, NaN -> None."""
|
|
if frame.empty:
|
|
return []
|
|
return frame.where(pd.notnull(frame), None).to_dict("records")
|
|
|
|
|
|
def build_advisor_comparison(
|
|
run_dir: Path,
|
|
scenario_name: str,
|
|
metrics: list[str],
|
|
extra_roots: list[Path] | None = None,
|
|
) -> AdvisorComparison | None:
|
|
"""Build the current run's advisor-diagnosis delta vs. its same-scenario predecessor.
|
|
|
|
Returns None when there is no predecessor, the predecessor's data cannot be
|
|
read, or there is nothing worth surfacing (no metric changed status). Never
|
|
raises — any failure degrades to None so the report detail page is
|
|
unaffected (mirrors run_advisor()'s defensive error handling).
|
|
"""
|
|
try:
|
|
metadata = run_reader._read_json(run_dir / "metadata.json")
|
|
current_run_id = str(metadata.get("run_id") or run_dir.name)
|
|
current_finished_at = str(metadata.get("finished_at") or "")
|
|
current_judge_model = str(metadata.get("judge_model", ""))
|
|
|
|
previous = find_previous_run(
|
|
scenario_name, current_run_id, current_finished_at, extra_roots
|
|
)
|
|
if previous is None:
|
|
return None
|
|
|
|
previous_dir = Path(previous.output_path)
|
|
previous_frame = run_reader.read_scores_frame(previous_dir)
|
|
if previous_frame.empty:
|
|
return None
|
|
previous_rows = _rows_as_records(previous_frame)
|
|
previous_metrics = previous.metrics
|
|
|
|
current_frame = run_reader.read_scores_frame(run_dir)
|
|
current_rows = _rows_as_records(current_frame)
|
|
|
|
previous_diagnoses = {d.metric: d for d in diagnose(previous_rows, previous_metrics)}
|
|
current_diagnoses = {d.metric: d for d in diagnose(current_rows, metrics)}
|
|
|
|
entries: list[AdvisorComparisonEntry] = []
|
|
for metric in sorted(set(previous_diagnoses) | set(current_diagnoses)):
|
|
prev_d = previous_diagnoses.get(metric)
|
|
curr_d = current_diagnoses.get(metric)
|
|
|
|
if prev_d is not None and curr_d is None:
|
|
if metric not in metrics:
|
|
# Dropped from the current scope entirely — cannot fairly
|
|
# claim "resolved" without a current-side measurement.
|
|
continue
|
|
status = "resolved"
|
|
elif prev_d is None and curr_d is not None:
|
|
status = "regressed" if metric in previous_metrics else "new_metric"
|
|
else:
|
|
status = "still_triggered"
|
|
|
|
entries.append(AdvisorComparisonEntry(
|
|
metric=metric,
|
|
status=status,
|
|
previous_score=prev_d.mean_score if prev_d else None,
|
|
previous_severity=prev_d.severity if prev_d else None,
|
|
current_score=curr_d.mean_score if curr_d else None,
|
|
current_severity=curr_d.severity if curr_d else None,
|
|
))
|
|
|
|
if not entries:
|
|
return None
|
|
|
|
entries.sort(key=lambda e: (_STATUS_ORDER.get(e.status, 99), e.metric))
|
|
|
|
previous_judge_model = previous.judge_model
|
|
return AdvisorComparison(
|
|
previous_run_id=previous.run_id,
|
|
previous_finished_at=previous.finished_at,
|
|
previous_judge_model=previous_judge_model,
|
|
current_judge_model=current_judge_model,
|
|
judge_model_changed=(
|
|
bool(previous_judge_model)
|
|
and bool(current_judge_model)
|
|
and previous_judge_model != current_judge_model
|
|
),
|
|
entries=entries,
|
|
)
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning(
|
|
"[advisor_comparison] failed to build comparison for run_dir=%s: %s",
|
|
run_dir, exc,
|
|
)
|
|
return None
|