"""Advisor-diagnosis comparison against the same scenario's previous run. The report detail page shows, for the run currently being viewed, whether metrics flagged by the optimization advisor in the immediately preceding run of the same scenario have since improved, regressed, or remain unresolved. No new artifact files are written: diagnoses are recomputed on demand from each run's scores.csv via the existing rag_eval.advisor.diagnose(), so the comparison always reflects the current threshold rules in rules.py. """ from __future__ import annotations import logging from pathlib import Path from typing import Any import pandas as pd from rag_eval.advisor import diagnose from webapp.models import AdvisorComparison, AdvisorComparisonEntry, RunSummary from webapp.services import run_reader logger = logging.getLogger("webapp.services.advisor_comparison") # Worse-first ordering so regressions and unresolved issues surface above # resolved ones in the rendered comparison list. _STATUS_ORDER = {"regressed": 0, "still_triggered": 1, "new_metric": 2, "resolved": 3} def find_previous_run( scenario_name: str, current_run_id: str, current_finished_at: str, extra_roots: list[Path] | None = None, ) -> RunSummary | None: """Return the most recent prior run with the same scenario_name, or None. "Prior" means finished_at strictly earlier than current_finished_at, so a run that finished later (e.g. a concurrent run) is never mistaken for a historical baseline. """ candidates = [ summary for summary in run_reader.list_run_summaries(extra_roots) if summary.scenario_name == scenario_name and summary.run_id != current_run_id and (summary.finished_at or "") < (current_finished_at or "") ] if not candidates: return None candidates.sort(key=lambda summary: summary.finished_at or "", reverse=True) return candidates[0] def _rows_as_records(frame: pd.DataFrame) -> list[dict[str, Any]]: """Convert a scores dataframe into plain dict records, NaN -> None.""" if frame.empty: return [] return frame.where(pd.notnull(frame), None).to_dict("records") def build_advisor_comparison( run_dir: Path, scenario_name: str, metrics: list[str], extra_roots: list[Path] | None = None, ) -> AdvisorComparison | None: """Build the current run's advisor-diagnosis delta vs. its same-scenario predecessor. Returns None when there is no predecessor, the predecessor's data cannot be read, or there is nothing worth surfacing (no metric changed status). Never raises — any failure degrades to None so the report detail page is unaffected (mirrors run_advisor()'s defensive error handling). """ try: metadata = run_reader._read_json(run_dir / "metadata.json") current_run_id = str(metadata.get("run_id") or run_dir.name) current_finished_at = str(metadata.get("finished_at") or "") current_judge_model = str(metadata.get("judge_model", "")) previous = find_previous_run( scenario_name, current_run_id, current_finished_at, extra_roots ) if previous is None: return None previous_dir = Path(previous.output_path) previous_frame = run_reader.read_scores_frame(previous_dir) if previous_frame.empty: return None previous_rows = _rows_as_records(previous_frame) previous_metrics = previous.metrics current_frame = run_reader.read_scores_frame(run_dir) current_rows = _rows_as_records(current_frame) previous_diagnoses = {d.metric: d for d in diagnose(previous_rows, previous_metrics)} current_diagnoses = {d.metric: d for d in diagnose(current_rows, metrics)} entries: list[AdvisorComparisonEntry] = [] for metric in sorted(set(previous_diagnoses) | set(current_diagnoses)): prev_d = previous_diagnoses.get(metric) curr_d = current_diagnoses.get(metric) if prev_d is not None and curr_d is None: if metric not in metrics: # Dropped from the current scope entirely — cannot fairly # claim "resolved" without a current-side measurement. continue status = "resolved" elif prev_d is None and curr_d is not None: status = "regressed" if metric in previous_metrics else "new_metric" else: status = "still_triggered" entries.append(AdvisorComparisonEntry( metric=metric, status=status, previous_score=prev_d.mean_score if prev_d else None, previous_severity=prev_d.severity if prev_d else None, current_score=curr_d.mean_score if curr_d else None, current_severity=curr_d.severity if curr_d else None, )) if not entries: return None entries.sort(key=lambda e: (_STATUS_ORDER.get(e.status, 99), e.metric)) previous_judge_model = previous.judge_model return AdvisorComparison( previous_run_id=previous.run_id, previous_finished_at=previous.finished_at, previous_judge_model=previous_judge_model, current_judge_model=current_judge_model, judge_model_changed=( bool(previous_judge_model) and bool(current_judge_model) and previous_judge_model != current_judge_model ), entries=entries, ) except Exception as exc: # noqa: BLE001 logger.warning( "[advisor_comparison] failed to build comparison for run_dir=%s: %s", run_dir, exc, ) return None