feat(advisor-comparison): wire build_advisor_comparison into report_builder
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -2,11 +2,13 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from webapp.models import AdvisorComparison, AdvisorComparisonEntry
|
||||||
from webapp.services import question_history
|
from webapp.services import question_history
|
||||||
from webapp.services import report_builder
|
from webapp.services import report_builder
|
||||||
from webapp.services.report_builder import build_report
|
from webapp.services.report_builder import build_report
|
||||||
@@ -186,3 +188,63 @@ def test_build_report_attaches_question_history(tmp_path: Path, monkeypatch) ->
|
|||||||
assert captured["exclude_run_id"] == "run" # current run excluded from history
|
assert captured["exclude_run_id"] == "run" # current run excluded from history
|
||||||
assert report.lowest_samples[0].history[0].run_id == "older"
|
assert report.lowest_samples[0].history[0].run_id == "older"
|
||||||
assert report.lowest_samples[0].history[0].metrics["faithfulness"] == 0.95
|
assert report.lowest_samples[0].history[0].metrics["faithfulness"] == 0.95
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_report_attaches_advisor_comparison(tmp_path: Path, monkeypatch) -> None:
|
||||||
|
"""build_report wires advisor_comparison.build_advisor_comparison() into ReportData."""
|
||||||
|
run_dir = tmp_path / "run"
|
||||||
|
_write_run_artifacts(run_dir)
|
||||||
|
(run_dir / "metadata.json").write_text(
|
||||||
|
json.dumps({"run_id": "run-1", "scenario_name": "my-scenario"}), encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
fake_comparison = AdvisorComparison(
|
||||||
|
previous_run_id="prev-1",
|
||||||
|
previous_finished_at="2026-01-01T00:00:00+00:00",
|
||||||
|
entries=[
|
||||||
|
AdvisorComparisonEntry(
|
||||||
|
metric="faithfulness",
|
||||||
|
status="resolved",
|
||||||
|
previous_score=0.5,
|
||||||
|
previous_severity="warning",
|
||||||
|
current_score=None,
|
||||||
|
current_severity=None,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
captured_args = {}
|
||||||
|
|
||||||
|
def _fake_build(run_dir_arg, scenario_name_arg, metrics_arg):
|
||||||
|
captured_args["scenario_name"] = scenario_name_arg
|
||||||
|
captured_args["metrics"] = metrics_arg
|
||||||
|
return fake_comparison
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
report_builder.advisor_comparison, "build_advisor_comparison", _fake_build
|
||||||
|
)
|
||||||
|
|
||||||
|
report = build_report(run_dir, ["faithfulness", "context_recall"])
|
||||||
|
|
||||||
|
assert report.advisor_comparison == fake_comparison
|
||||||
|
assert captured_args["scenario_name"] == "my-scenario"
|
||||||
|
assert captured_args["metrics"] == ["faithfulness", "context_recall"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_report_advisor_comparison_none_when_no_previous_run(
|
||||||
|
tmp_path: Path, monkeypatch
|
||||||
|
) -> None:
|
||||||
|
"""build_report leaves advisor_comparison as None when no predecessor exists."""
|
||||||
|
run_dir = tmp_path / "run"
|
||||||
|
_write_run_artifacts(run_dir)
|
||||||
|
(run_dir / "metadata.json").write_text(
|
||||||
|
json.dumps({"run_id": "run-1", "scenario_name": "my-scenario"}), encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
report_builder.advisor_comparison, "build_advisor_comparison", lambda *a, **k: None
|
||||||
|
)
|
||||||
|
|
||||||
|
report = build_report(run_dir, ["faithfulness", "context_recall"])
|
||||||
|
|
||||||
|
assert report.advisor_comparison is None
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ from webapp.models import (
|
|||||||
SampleHistoryEntry,
|
SampleHistoryEntry,
|
||||||
SampleScore,
|
SampleScore,
|
||||||
)
|
)
|
||||||
from webapp.services import question_history, run_reader
|
from webapp.services import advisor_comparison, question_history, run_reader
|
||||||
|
|
||||||
|
|
||||||
# Number of equal-width buckets used for metric score histograms.
|
# Number of equal-width buckets used for metric score histograms.
|
||||||
@@ -200,6 +200,7 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
|
|||||||
metric_weights=metric_weights,
|
metric_weights=metric_weights,
|
||||||
doc_weights=doc_weights,
|
doc_weights=doc_weights,
|
||||||
token_usage=token_usage,
|
token_usage=token_usage,
|
||||||
|
advisor_comparison=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
score_rows_list = frame.to_dict(orient="records")
|
score_rows_list = frame.to_dict(orient="records")
|
||||||
@@ -227,6 +228,13 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
|
|||||||
exclude_run_id=current_run_id
|
exclude_run_id=current_run_id
|
||||||
)
|
)
|
||||||
|
|
||||||
|
scenario_name = str(metadata.get("scenario_name") or "")
|
||||||
|
comparison = (
|
||||||
|
advisor_comparison.build_advisor_comparison(run_dir, scenario_name, metrics)
|
||||||
|
if scenario_name
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
|
||||||
return ReportData(
|
return ReportData(
|
||||||
metrics=metrics,
|
metrics=metrics,
|
||||||
metric_means=rounded_means,
|
metric_means=rounded_means,
|
||||||
@@ -239,4 +247,5 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
|
|||||||
metric_weights=metric_weights,
|
metric_weights=metric_weights,
|
||||||
doc_weights=doc_weights,
|
doc_weights=doc_weights,
|
||||||
token_usage=token_usage,
|
token_usage=token_usage,
|
||||||
|
advisor_comparison=comparison,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user