feat(advisor-comparison): wire build_advisor_comparison into report_builder

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
wangwei
2026-07-02 17:00:07 +08:00
co-authored by Copilot
parent 86e3aef7d5
commit f4016a1f09
2 changed files with 72 additions and 1 deletions
+62
View File
@@ -2,11 +2,13 @@
from __future__ import annotations from __future__ import annotations
import json
from pathlib import Path from pathlib import Path
import pandas as pd import pandas as pd
import pytest import pytest
from webapp.models import AdvisorComparison, AdvisorComparisonEntry
from webapp.services import question_history from webapp.services import question_history
from webapp.services import report_builder from webapp.services import report_builder
from webapp.services.report_builder import build_report from webapp.services.report_builder import build_report
@@ -186,3 +188,63 @@ def test_build_report_attaches_question_history(tmp_path: Path, monkeypatch) ->
assert captured["exclude_run_id"] == "run" # current run excluded from history assert captured["exclude_run_id"] == "run" # current run excluded from history
assert report.lowest_samples[0].history[0].run_id == "older" assert report.lowest_samples[0].history[0].run_id == "older"
assert report.lowest_samples[0].history[0].metrics["faithfulness"] == 0.95 assert report.lowest_samples[0].history[0].metrics["faithfulness"] == 0.95
def test_build_report_attaches_advisor_comparison(tmp_path: Path, monkeypatch) -> None:
"""build_report wires advisor_comparison.build_advisor_comparison() into ReportData."""
run_dir = tmp_path / "run"
_write_run_artifacts(run_dir)
(run_dir / "metadata.json").write_text(
json.dumps({"run_id": "run-1", "scenario_name": "my-scenario"}), encoding="utf-8"
)
fake_comparison = AdvisorComparison(
previous_run_id="prev-1",
previous_finished_at="2026-01-01T00:00:00+00:00",
entries=[
AdvisorComparisonEntry(
metric="faithfulness",
status="resolved",
previous_score=0.5,
previous_severity="warning",
current_score=None,
current_severity=None,
)
],
)
captured_args = {}
def _fake_build(run_dir_arg, scenario_name_arg, metrics_arg):
captured_args["scenario_name"] = scenario_name_arg
captured_args["metrics"] = metrics_arg
return fake_comparison
monkeypatch.setattr(
report_builder.advisor_comparison, "build_advisor_comparison", _fake_build
)
report = build_report(run_dir, ["faithfulness", "context_recall"])
assert report.advisor_comparison == fake_comparison
assert captured_args["scenario_name"] == "my-scenario"
assert captured_args["metrics"] == ["faithfulness", "context_recall"]
def test_build_report_advisor_comparison_none_when_no_previous_run(
tmp_path: Path, monkeypatch
) -> None:
"""build_report leaves advisor_comparison as None when no predecessor exists."""
run_dir = tmp_path / "run"
_write_run_artifacts(run_dir)
(run_dir / "metadata.json").write_text(
json.dumps({"run_id": "run-1", "scenario_name": "my-scenario"}), encoding="utf-8"
)
monkeypatch.setattr(
report_builder.advisor_comparison, "build_advisor_comparison", lambda *a, **k: None
)
report = build_report(run_dir, ["faithfulness", "context_recall"])
assert report.advisor_comparison is None
+10 -1
View File
@@ -26,7 +26,7 @@ from webapp.models import (
SampleHistoryEntry, SampleHistoryEntry,
SampleScore, SampleScore,
) )
from webapp.services import question_history, run_reader from webapp.services import advisor_comparison, question_history, run_reader
# Number of equal-width buckets used for metric score histograms. # Number of equal-width buckets used for metric score histograms.
@@ -200,6 +200,7 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
metric_weights=metric_weights, metric_weights=metric_weights,
doc_weights=doc_weights, doc_weights=doc_weights,
token_usage=token_usage, token_usage=token_usage,
advisor_comparison=None,
) )
score_rows_list = frame.to_dict(orient="records") score_rows_list = frame.to_dict(orient="records")
@@ -227,6 +228,13 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
exclude_run_id=current_run_id exclude_run_id=current_run_id
) )
scenario_name = str(metadata.get("scenario_name") or "")
comparison = (
advisor_comparison.build_advisor_comparison(run_dir, scenario_name, metrics)
if scenario_name
else None
)
return ReportData( return ReportData(
metrics=metrics, metrics=metrics,
metric_means=rounded_means, metric_means=rounded_means,
@@ -239,4 +247,5 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
metric_weights=metric_weights, metric_weights=metric_weights,
doc_weights=doc_weights, doc_weights=doc_weights,
token_usage=token_usage, token_usage=token_usage,
advisor_comparison=comparison,
) )