diff --git a/tests/test_webapp_report_builder.py b/tests/test_webapp_report_builder.py index 2b60f4f..eaf83df 100644 --- a/tests/test_webapp_report_builder.py +++ b/tests/test_webapp_report_builder.py @@ -2,11 +2,13 @@ from __future__ import annotations +import json from pathlib import Path import pandas as pd import pytest +from webapp.models import AdvisorComparison, AdvisorComparisonEntry from webapp.services import question_history from webapp.services import report_builder from webapp.services.report_builder import build_report @@ -186,3 +188,63 @@ def test_build_report_attaches_question_history(tmp_path: Path, monkeypatch) -> assert captured["exclude_run_id"] == "run" # current run excluded from history assert report.lowest_samples[0].history[0].run_id == "older" assert report.lowest_samples[0].history[0].metrics["faithfulness"] == 0.95 + + +def test_build_report_attaches_advisor_comparison(tmp_path: Path, monkeypatch) -> None: + """build_report wires advisor_comparison.build_advisor_comparison() into ReportData.""" + run_dir = tmp_path / "run" + _write_run_artifacts(run_dir) + (run_dir / "metadata.json").write_text( + json.dumps({"run_id": "run-1", "scenario_name": "my-scenario"}), encoding="utf-8" + ) + + fake_comparison = AdvisorComparison( + previous_run_id="prev-1", + previous_finished_at="2026-01-01T00:00:00+00:00", + entries=[ + AdvisorComparisonEntry( + metric="faithfulness", + status="resolved", + previous_score=0.5, + previous_severity="warning", + current_score=None, + current_severity=None, + ) + ], + ) + + captured_args = {} + + def _fake_build(run_dir_arg, scenario_name_arg, metrics_arg): + captured_args["scenario_name"] = scenario_name_arg + captured_args["metrics"] = metrics_arg + return fake_comparison + + monkeypatch.setattr( + report_builder.advisor_comparison, "build_advisor_comparison", _fake_build + ) + + report = build_report(run_dir, ["faithfulness", "context_recall"]) + + assert report.advisor_comparison == fake_comparison + assert captured_args["scenario_name"] == "my-scenario" + assert captured_args["metrics"] == ["faithfulness", "context_recall"] + + +def test_build_report_advisor_comparison_none_when_no_previous_run( + tmp_path: Path, monkeypatch +) -> None: + """build_report leaves advisor_comparison as None when no predecessor exists.""" + run_dir = tmp_path / "run" + _write_run_artifacts(run_dir) + (run_dir / "metadata.json").write_text( + json.dumps({"run_id": "run-1", "scenario_name": "my-scenario"}), encoding="utf-8" + ) + + monkeypatch.setattr( + report_builder.advisor_comparison, "build_advisor_comparison", lambda *a, **k: None + ) + + report = build_report(run_dir, ["faithfulness", "context_recall"]) + + assert report.advisor_comparison is None diff --git a/webapp/services/report_builder.py b/webapp/services/report_builder.py index 887dac8..71d1699 100644 --- a/webapp/services/report_builder.py +++ b/webapp/services/report_builder.py @@ -26,7 +26,7 @@ from webapp.models import ( SampleHistoryEntry, SampleScore, ) -from webapp.services import question_history, run_reader +from webapp.services import advisor_comparison, question_history, run_reader # Number of equal-width buckets used for metric score histograms. @@ -200,6 +200,7 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData: metric_weights=metric_weights, doc_weights=doc_weights, token_usage=token_usage, + advisor_comparison=None, ) score_rows_list = frame.to_dict(orient="records") @@ -227,6 +228,13 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData: exclude_run_id=current_run_id ) + scenario_name = str(metadata.get("scenario_name") or "") + comparison = ( + advisor_comparison.build_advisor_comparison(run_dir, scenario_name, metrics) + if scenario_name + else None + ) + return ReportData( metrics=metrics, metric_means=rounded_means, @@ -239,4 +247,5 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData: metric_weights=metric_weights, doc_weights=doc_weights, token_usage=token_usage, + advisor_comparison=comparison, )