update for ragas

webapp/services/report_builder.py

@@ -23,9 +23,10 @@ from webapp.models import (
     DistributionBin,
     GroupStat,
     ReportData,
+    SampleHistoryEntry,
     SampleScore,
 )
-from webapp.services import run_reader
+from webapp.services import question_history, run_reader
 
 
 # Number of equal-width buckets used for metric score histograms.
@@ -37,6 +38,9 @@ GROUPING_FIELDS = ("difficulty", "question_type", "language")
 # How many lowest-scoring samples to surface for manual review.
 LOWEST_SAMPLE_COUNT = 10
 
+# How many past evaluations of the same question to show in the history table.
+HISTORY_LIMIT = 5
+
 # Metrics whose lower raw value means stronger performance.
 LOWER_IS_BETTER_METRICS = {"noise_sensitivity"}
 
@@ -124,8 +128,16 @@ def _cell_text(row: pd.Series, column: str) -> str:
     return str(row[column]).strip()
 
 
-def _lowest_samples(frame: pd.DataFrame, metrics: list[str]) -> list[SampleScore]:
-    """Select and shape the lowest-scoring samples for the review table."""
+def _lowest_samples(
+    frame: pd.DataFrame,
+    metrics: list[str],
+    history_index: dict[str, list[dict]] | None = None,
+) -> list[SampleScore]:
+    """Select and shape the lowest-scoring samples for the review table.
+
+    When a history_index is supplied, each surfaced sample is annotated with the
+    same question's scores from previous runs (newest first) for comparison.
+    """
     if frame.empty:
         return []
 
@@ -154,7 +166,19 @@ def _lowest_samples(frame: pd.DataFrame, metrics: list[str]) -> list[SampleScore
         enriched.append((sort_key, sample))
 
     enriched.sort(key=lambda item: item[0])
-    return [sample for _, sample in enriched[:LOWEST_SAMPLE_COUNT]]
+    selected = [sample for _, sample in enriched[:LOWEST_SAMPLE_COUNT]]
+
+    # Attach per-question history only for the surfaced samples (keeps lookups cheap).
+    if history_index is not None:
+        for sample in selected:
+            if not sample.question:
+                continue
+            entries = question_history.lookup(
+                history_index, sample.question, limit=HISTORY_LIMIT
+            )
+            sample.history = [SampleHistoryEntry(**entry) for entry in entries]
+
+    return selected
 
 
 def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
@@ -192,12 +216,20 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
         if metric in frame.columns
     }
 
+    # Cross-run history: scores of the same question in *other* runs (Approach A —
+    # on-demand global scan, excluding the run currently being viewed).
+    metadata = run_reader._read_json(run_dir / "metadata.json")
+    current_run_id = str(metadata.get("run_id") or run_dir.name)
+    history_index = question_history.build_question_history_index(
+        exclude_run_id=current_run_id
+    )
+
     return ReportData(
         metrics=metrics,
         metric_means=rounded_means,
         distributions=distributions,
         groupings=_groupings(frame, metrics),
-        lowest_samples=_lowest_samples(frame, metrics),
+        lowest_samples=_lowest_samples(frame, metrics, history_index),
         summary_markdown=summary_markdown,
         advice_markdown=advice_markdown,
         weighted_score_mean=_round_or_none(overall_ws),

webapp/services/score_job_manager.py

@@ -107,7 +107,6 @@ class ScoreJobManager:
 
         # Lazy imports to keep web server bootable if ragas is not installed.
         from rag_eval.advisor import run_advisor
-        from rag_eval.metrics.factory import build_models
         from rag_eval.metrics.weights import compute_weighted_score
         from rag_eval.reporting.writers import write_run_artifacts
         from rag_eval.settings import EvaluationSettings
@@ -206,8 +205,7 @@ class ScoreJobManager:
 
             # Run optimization advisor (builds optimization_advice.md)
             try:
-                llm, _ = build_models(judge_model, embedding_model, settings)
-                run_advisor(result, scenario, llm)
+                run_advisor(result, scenario, settings=settings)
                 logger.info("[score_job] advisor done  job_id=%s", job_id)
             except Exception as adv_exc:  # noqa: BLE001
                 logger.warning("[score_job] advisor failed  job_id=%s  err=%s", job_id, adv_exc)

webapp/services/session_score_manager.py

@@ -192,7 +192,6 @@ class SessionScoreJobManager:
 
         # Lazy imports — keep web server bootable if ragas is not installed.
         from rag_eval.advisor import run_advisor
-        from rag_eval.metrics.factory import build_models
         from rag_eval.metrics.weights import compute_weighted_score
         from rag_eval.reporting.writers import write_run_artifacts
         from rag_eval.settings import EvaluationSettings
@@ -320,8 +319,7 @@ class SessionScoreJobManager:
 
                 # Regenerate optimization advice over all accumulated rows
                 try:
-                    llm, _ = build_models(judge_model, embedding_model, settings)
-                    run_advisor(result, scenario, llm)
+                    run_advisor(result, scenario, settings=settings)
                     logger.info("[session_job] advisor done  job_id=%s  session=%s", job_id, session_id)
                 except Exception as adv_exc:  # noqa: BLE001
                     logger.warning(