diff --git a/rag_eval/reporting/summary.py b/rag_eval/reporting/summary.py index 7953e7f..e90b8dd 100644 --- a/rag_eval/reporting/summary.py +++ b/rag_eval/reporting/summary.py @@ -35,6 +35,23 @@ def _table_from_frame(frame: pd.DataFrame) -> str: return "\n".join([header, separator, *body]) +def _token_usage_section(token_usage: dict[str, dict[str, int]]) -> list[str]: + """Render the '## Token 用量' section as a list of markdown lines.""" + lines = ["", "## Token 用量", ""] + if not token_usage: + lines.append("未记录 token 用量。") + return lines + lines.append("| 模型 | input_tokens | output_tokens | 调用次数 |") + lines.append("|---|---|---|---|") + for model in sorted(token_usage): + usage = token_usage[model] + lines.append( + f"| {model} | {usage.get('input_tokens', 0)} " + f"| {usage.get('output_tokens', 0)} | {usage.get('calls', 0)} |" + ) + return lines + + def build_summary_markdown(result: EvaluationResult) -> str: """Build the human-readable markdown summary written for each evaluation run.""" total = len(result.valid_samples) + len(result.invalid_samples) @@ -57,6 +74,7 @@ def build_summary_markdown(result: EvaluationResult) -> str: if scores.empty: lines.append("No valid samples were scored.") + lines.extend(_token_usage_section(result.token_usage)) return "\n".join(lines) + "\n" score_rows_list = scores.to_dict(orient="records") @@ -97,4 +115,5 @@ def build_summary_markdown(result: EvaluationResult) -> str: _table_from_frame(detail), "```", ]) + lines.extend(_token_usage_section(result.token_usage)) return "\n".join(lines) + "\n" diff --git a/tests/test_reporting_summary_token_usage.py b/tests/test_reporting_summary_token_usage.py new file mode 100644 index 0000000..a623821 --- /dev/null +++ b/tests/test_reporting_summary_token_usage.py @@ -0,0 +1,63 @@ +"""Tests for the '## Token 用量' section rendered by build_summary_markdown.""" +from __future__ import annotations + +from pathlib import Path + +from rag_eval.reporting.summary import build_summary_markdown +from rag_eval.shared.models import DatasetConfig, EvaluationResult, RuntimeConfig, Scenario + + +def _scenario(tmp_path: Path) -> Scenario: + return Scenario( + scenario_name="summary-token-test", + mode="offline", + dataset=DatasetConfig(path=tmp_path / "dataset.csv"), + judge_model="gpt-5", + embedding_model="embedding-model", + metrics=["faithfulness"], + output_dir=tmp_path / "outputs", + runtime=RuntimeConfig(batch_size=1), + ) + + +def test_summary_includes_token_usage_table(tmp_path: Path) -> None: + scenario = _scenario(tmp_path) + result = EvaluationResult( + scenario=scenario, + run_id="run-1", + started_at="t0", + finished_at="t1", + valid_samples=[], + invalid_samples=[], + score_rows=[{"sample_id": "s1", "faithfulness": 0.9, "error": ""}], + token_usage={ + "gpt-5": {"input_tokens": 12450, "output_tokens": 3200, "calls": 60}, + "Qwen/Qwen3-Embedding-4B": {"input_tokens": 45000, "output_tokens": 0, "calls": 30}, + }, + ) + + markdown = build_summary_markdown(result) + + assert "## Token 用量" in markdown + assert "gpt-5" in markdown + assert "12450" in markdown + assert "Qwen/Qwen3-Embedding-4B" in markdown + assert "45000" in markdown + + +def test_summary_shows_fallback_text_when_token_usage_empty(tmp_path: Path) -> None: + scenario = _scenario(tmp_path) + result = EvaluationResult( + scenario=scenario, + run_id="run-2", + started_at="t0", + finished_at="t1", + valid_samples=[], + invalid_samples=[], + score_rows=[{"sample_id": "s1", "faithfulness": 0.9, "error": ""}], + ) + + markdown = build_summary_markdown(result) + + assert "## Token 用量" in markdown + assert "未记录 token 用量" in markdown