feat(token-tracking): surface token_usage in ReportData
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,55 @@
|
|||||||
|
"""Tests for token_usage passthrough in the webapp report builder."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from webapp.services.report_builder import build_report
|
||||||
|
|
||||||
|
|
||||||
|
def _write_minimal_run(run_dir: Path, token_usage: dict | None) -> None:
|
||||||
|
run_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
(run_dir / "scores.csv").write_text(
|
||||||
|
"sample_id,faithfulness\ns1,0.9\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
(run_dir / "summary.md").write_text("summary", encoding="utf-8")
|
||||||
|
metadata = {"run_id": run_dir.name}
|
||||||
|
if token_usage is not None:
|
||||||
|
metadata["token_usage"] = token_usage
|
||||||
|
(run_dir / "metadata.json").write_text(json.dumps(metadata), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_report_passes_through_token_usage(tmp_path: Path) -> None:
|
||||||
|
run_dir = tmp_path / "run"
|
||||||
|
_write_minimal_run(
|
||||||
|
run_dir,
|
||||||
|
token_usage={"gpt-5": {"input_tokens": 100, "output_tokens": 50, "calls": 2}},
|
||||||
|
)
|
||||||
|
|
||||||
|
report = build_report(run_dir, ["faithfulness"])
|
||||||
|
|
||||||
|
assert report.token_usage == {
|
||||||
|
"gpt-5": {"input_tokens": 100, "output_tokens": 50, "calls": 2}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_report_defaults_token_usage_to_empty_dict(tmp_path: Path) -> None:
|
||||||
|
run_dir = tmp_path / "run"
|
||||||
|
_write_minimal_run(run_dir, token_usage=None)
|
||||||
|
|
||||||
|
report = build_report(run_dir, ["faithfulness"])
|
||||||
|
|
||||||
|
assert report.token_usage == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_report_early_return_branch_still_surfaces_token_usage(tmp_path: Path) -> None:
|
||||||
|
"""metrics=[] forces the early-return branch; token_usage must still surface."""
|
||||||
|
run_dir = tmp_path / "run"
|
||||||
|
_write_minimal_run(
|
||||||
|
run_dir,
|
||||||
|
token_usage={"gpt-5": {"input_tokens": 5, "output_tokens": 5, "calls": 1}},
|
||||||
|
)
|
||||||
|
|
||||||
|
report = build_report(run_dir, [])
|
||||||
|
|
||||||
|
assert report.token_usage == {"gpt-5": {"input_tokens": 5, "output_tokens": 5, "calls": 1}}
|
||||||
@@ -99,6 +99,10 @@ class ReportData(BaseModel):
|
|||||||
default_factory=dict,
|
default_factory=dict,
|
||||||
description="该次运行使用的文档权重配置(来自 scenario.snapshot.yaml)。",
|
description="该次运行使用的文档权重配置(来自 scenario.snapshot.yaml)。",
|
||||||
)
|
)
|
||||||
|
token_usage: dict[str, dict[str, int]] = Field(
|
||||||
|
default_factory=dict,
|
||||||
|
description="按模型累计的 token 用量:{model: {input_tokens, output_tokens, calls}}。",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class RunDetail(BaseModel):
|
class RunDetail(BaseModel):
|
||||||
|
|||||||
@@ -187,6 +187,9 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
|
|||||||
summary_markdown = run_reader.read_summary_markdown(run_dir)
|
summary_markdown = run_reader.read_summary_markdown(run_dir)
|
||||||
advice_markdown = run_reader.read_advice_markdown(run_dir)
|
advice_markdown = run_reader.read_advice_markdown(run_dir)
|
||||||
metric_weights, doc_weights = _read_weights_from_snapshot(run_dir)
|
metric_weights, doc_weights = _read_weights_from_snapshot(run_dir)
|
||||||
|
# Read once up front so both the empty-frame and full branches can surface it.
|
||||||
|
metadata = run_reader._read_json(run_dir / "metadata.json")
|
||||||
|
token_usage = metadata.get("token_usage") or {}
|
||||||
|
|
||||||
if frame.empty or not metrics:
|
if frame.empty or not metrics:
|
||||||
return ReportData(
|
return ReportData(
|
||||||
@@ -196,6 +199,7 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
|
|||||||
advice_markdown=advice_markdown,
|
advice_markdown=advice_markdown,
|
||||||
metric_weights=metric_weights,
|
metric_weights=metric_weights,
|
||||||
doc_weights=doc_weights,
|
doc_weights=doc_weights,
|
||||||
|
token_usage=token_usage,
|
||||||
)
|
)
|
||||||
|
|
||||||
score_rows_list = frame.to_dict(orient="records")
|
score_rows_list = frame.to_dict(orient="records")
|
||||||
@@ -218,7 +222,6 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
|
|||||||
|
|
||||||
# Cross-run history: scores of the same question in *other* runs (Approach A —
|
# Cross-run history: scores of the same question in *other* runs (Approach A —
|
||||||
# on-demand global scan, excluding the run currently being viewed).
|
# on-demand global scan, excluding the run currently being viewed).
|
||||||
metadata = run_reader._read_json(run_dir / "metadata.json")
|
|
||||||
current_run_id = str(metadata.get("run_id") or run_dir.name)
|
current_run_id = str(metadata.get("run_id") or run_dir.name)
|
||||||
history_index = question_history.build_question_history_index(
|
history_index = question_history.build_question_history_index(
|
||||||
exclude_run_id=current_run_id
|
exclude_run_id=current_run_id
|
||||||
@@ -235,4 +238,5 @@ def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
|
|||||||
weighted_score_mean=_round_or_none(overall_ws),
|
weighted_score_mean=_round_or_none(overall_ws),
|
||||||
metric_weights=metric_weights,
|
metric_weights=metric_weights,
|
||||||
doc_weights=doc_weights,
|
doc_weights=doc_weights,
|
||||||
|
token_usage=token_usage,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user