feat(token-tracking): add Token usage section to summary.md
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -35,6 +35,23 @@ def _table_from_frame(frame: pd.DataFrame) -> str:
|
||||
return "\n".join([header, separator, *body])
|
||||
|
||||
|
||||
def _token_usage_section(token_usage: dict[str, dict[str, int]]) -> list[str]:
|
||||
"""Render the '## Token 用量' section as a list of markdown lines."""
|
||||
lines = ["", "## Token 用量", ""]
|
||||
if not token_usage:
|
||||
lines.append("未记录 token 用量。")
|
||||
return lines
|
||||
lines.append("| 模型 | input_tokens | output_tokens | 调用次数 |")
|
||||
lines.append("|---|---|---|---|")
|
||||
for model in sorted(token_usage):
|
||||
usage = token_usage[model]
|
||||
lines.append(
|
||||
f"| {model} | {usage.get('input_tokens', 0)} "
|
||||
f"| {usage.get('output_tokens', 0)} | {usage.get('calls', 0)} |"
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def build_summary_markdown(result: EvaluationResult) -> str:
|
||||
"""Build the human-readable markdown summary written for each evaluation run."""
|
||||
total = len(result.valid_samples) + len(result.invalid_samples)
|
||||
@@ -57,6 +74,7 @@ def build_summary_markdown(result: EvaluationResult) -> str:
|
||||
|
||||
if scores.empty:
|
||||
lines.append("No valid samples were scored.")
|
||||
lines.extend(_token_usage_section(result.token_usage))
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
score_rows_list = scores.to_dict(orient="records")
|
||||
@@ -97,4 +115,5 @@ def build_summary_markdown(result: EvaluationResult) -> str:
|
||||
_table_from_frame(detail),
|
||||
"```",
|
||||
])
|
||||
lines.extend(_token_usage_section(result.token_usage))
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Tests for the '## Token 用量' section rendered by build_summary_markdown."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from rag_eval.reporting.summary import build_summary_markdown
|
||||
from rag_eval.shared.models import DatasetConfig, EvaluationResult, RuntimeConfig, Scenario
|
||||
|
||||
|
||||
def _scenario(tmp_path: Path) -> Scenario:
|
||||
return Scenario(
|
||||
scenario_name="summary-token-test",
|
||||
mode="offline",
|
||||
dataset=DatasetConfig(path=tmp_path / "dataset.csv"),
|
||||
judge_model="gpt-5",
|
||||
embedding_model="embedding-model",
|
||||
metrics=["faithfulness"],
|
||||
output_dir=tmp_path / "outputs",
|
||||
runtime=RuntimeConfig(batch_size=1),
|
||||
)
|
||||
|
||||
|
||||
def test_summary_includes_token_usage_table(tmp_path: Path) -> None:
|
||||
scenario = _scenario(tmp_path)
|
||||
result = EvaluationResult(
|
||||
scenario=scenario,
|
||||
run_id="run-1",
|
||||
started_at="t0",
|
||||
finished_at="t1",
|
||||
valid_samples=[],
|
||||
invalid_samples=[],
|
||||
score_rows=[{"sample_id": "s1", "faithfulness": 0.9, "error": ""}],
|
||||
token_usage={
|
||||
"gpt-5": {"input_tokens": 12450, "output_tokens": 3200, "calls": 60},
|
||||
"Qwen/Qwen3-Embedding-4B": {"input_tokens": 45000, "output_tokens": 0, "calls": 30},
|
||||
},
|
||||
)
|
||||
|
||||
markdown = build_summary_markdown(result)
|
||||
|
||||
assert "## Token 用量" in markdown
|
||||
assert "gpt-5" in markdown
|
||||
assert "12450" in markdown
|
||||
assert "Qwen/Qwen3-Embedding-4B" in markdown
|
||||
assert "45000" in markdown
|
||||
|
||||
|
||||
def test_summary_shows_fallback_text_when_token_usage_empty(tmp_path: Path) -> None:
|
||||
scenario = _scenario(tmp_path)
|
||||
result = EvaluationResult(
|
||||
scenario=scenario,
|
||||
run_id="run-2",
|
||||
started_at="t0",
|
||||
finished_at="t1",
|
||||
valid_samples=[],
|
||||
invalid_samples=[],
|
||||
score_rows=[{"sample_id": "s1", "faithfulness": 0.9, "error": ""}],
|
||||
)
|
||||
|
||||
markdown = build_summary_markdown(result)
|
||||
|
||||
assert "## Token 用量" in markdown
|
||||
assert "未记录 token 用量" in markdown
|
||||
Reference in New Issue
Block a user