feat(token-tracking): add Token usage section to summary.md

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
wangwei
2026-07-02 14:50:07 +08:00
co-authored by Copilot
parent b50b911de0
commit cd044c8416
2 changed files with 82 additions and 0 deletions
+19
View File
@@ -35,6 +35,23 @@ def _table_from_frame(frame: pd.DataFrame) -> str:
return "\n".join([header, separator, *body])
def _token_usage_section(token_usage: dict[str, dict[str, int]]) -> list[str]:
"""Render the '## Token 用量' section as a list of markdown lines."""
lines = ["", "## Token 用量", ""]
if not token_usage:
lines.append("未记录 token 用量。")
return lines
lines.append("| 模型 | input_tokens | output_tokens | 调用次数 |")
lines.append("|---|---|---|---|")
for model in sorted(token_usage):
usage = token_usage[model]
lines.append(
f"| {model} | {usage.get('input_tokens', 0)} "
f"| {usage.get('output_tokens', 0)} | {usage.get('calls', 0)} |"
)
return lines
def build_summary_markdown(result: EvaluationResult) -> str:
"""Build the human-readable markdown summary written for each evaluation run."""
total = len(result.valid_samples) + len(result.invalid_samples)
@@ -57,6 +74,7 @@ def build_summary_markdown(result: EvaluationResult) -> str:
if scores.empty:
lines.append("No valid samples were scored.")
lines.extend(_token_usage_section(result.token_usage))
return "\n".join(lines) + "\n"
score_rows_list = scores.to_dict(orient="records")
@@ -97,4 +115,5 @@ def build_summary_markdown(result: EvaluationResult) -> str:
_table_from_frame(detail),
"```",
])
lines.extend(_token_usage_section(result.token_usage))
return "\n".join(lines) + "\n"
@@ -0,0 +1,63 @@
"""Tests for the '## Token 用量' section rendered by build_summary_markdown."""
from __future__ import annotations
from pathlib import Path
from rag_eval.reporting.summary import build_summary_markdown
from rag_eval.shared.models import DatasetConfig, EvaluationResult, RuntimeConfig, Scenario
def _scenario(tmp_path: Path) -> Scenario:
return Scenario(
scenario_name="summary-token-test",
mode="offline",
dataset=DatasetConfig(path=tmp_path / "dataset.csv"),
judge_model="gpt-5",
embedding_model="embedding-model",
metrics=["faithfulness"],
output_dir=tmp_path / "outputs",
runtime=RuntimeConfig(batch_size=1),
)
def test_summary_includes_token_usage_table(tmp_path: Path) -> None:
scenario = _scenario(tmp_path)
result = EvaluationResult(
scenario=scenario,
run_id="run-1",
started_at="t0",
finished_at="t1",
valid_samples=[],
invalid_samples=[],
score_rows=[{"sample_id": "s1", "faithfulness": 0.9, "error": ""}],
token_usage={
"gpt-5": {"input_tokens": 12450, "output_tokens": 3200, "calls": 60},
"Qwen/Qwen3-Embedding-4B": {"input_tokens": 45000, "output_tokens": 0, "calls": 30},
},
)
markdown = build_summary_markdown(result)
assert "## Token 用量" in markdown
assert "gpt-5" in markdown
assert "12450" in markdown
assert "Qwen/Qwen3-Embedding-4B" in markdown
assert "45000" in markdown
def test_summary_shows_fallback_text_when_token_usage_empty(tmp_path: Path) -> None:
scenario = _scenario(tmp_path)
result = EvaluationResult(
scenario=scenario,
run_id="run-2",
started_at="t0",
finished_at="t1",
valid_samples=[],
invalid_samples=[],
score_rows=[{"sample_id": "s1", "faithfulness": 0.9, "error": ""}],
)
markdown = build_summary_markdown(result)
assert "## Token 用量" in markdown
assert "未记录 token 用量" in markdown