64 lines
2.0 KiB
Python
64 lines
2.0 KiB
Python
|
"""Tests for the '## Token 用量' section rendered by build_summary_markdown."""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
from rag_eval.reporting.summary import build_summary_markdown
|
||
|
|
from rag_eval.shared.models import DatasetConfig, EvaluationResult, RuntimeConfig, Scenario
|
||
|
|
|
||
|
|
|
||
|
|
def _scenario(tmp_path: Path) -> Scenario:
|
||
|
|
return Scenario(
|
||
|
|
scenario_name="summary-token-test",
|
||
|
|
mode="offline",
|
||
|
|
dataset=DatasetConfig(path=tmp_path / "dataset.csv"),
|
||
|
|
judge_model="gpt-5",
|
||
|
|
embedding_model="embedding-model",
|
||
|
|
metrics=["faithfulness"],
|
||
|
|
output_dir=tmp_path / "outputs",
|
||
|
|
runtime=RuntimeConfig(batch_size=1),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_summary_includes_token_usage_table(tmp_path: Path) -> None:
|
||
|
|
scenario = _scenario(tmp_path)
|
||
|
|
result = EvaluationResult(
|
||
|
|
scenario=scenario,
|
||
|
|
run_id="run-1",
|
||
|
|
started_at="t0",
|
||
|
|
finished_at="t1",
|
||
|
|
valid_samples=[],
|
||
|
|
invalid_samples=[],
|
||
|
|
score_rows=[{"sample_id": "s1", "faithfulness": 0.9, "error": ""}],
|
||
|
|
token_usage={
|
||
|
|
"gpt-5": {"input_tokens": 12450, "output_tokens": 3200, "calls": 60},
|
||
|
|
"Qwen/Qwen3-Embedding-4B": {"input_tokens": 45000, "output_tokens": 0, "calls": 30},
|
||
|
|
},
|
||
|
|
)
|
||
|
|
|
||
|
|
markdown = build_summary_markdown(result)
|
||
|
|
|
||
|
|
assert "## Token 用量" in markdown
|
||
|
|
assert "gpt-5" in markdown
|
||
|
|
assert "12450" in markdown
|
||
|
|
assert "Qwen/Qwen3-Embedding-4B" in markdown
|
||
|
|
assert "45000" in markdown
|
||
|
|
|
||
|
|
|
||
|
|
def test_summary_shows_fallback_text_when_token_usage_empty(tmp_path: Path) -> None:
|
||
|
|
scenario = _scenario(tmp_path)
|
||
|
|
result = EvaluationResult(
|
||
|
|
scenario=scenario,
|
||
|
|
run_id="run-2",
|
||
|
|
started_at="t0",
|
||
|
|
finished_at="t1",
|
||
|
|
valid_samples=[],
|
||
|
|
invalid_samples=[],
|
||
|
|
score_rows=[{"sample_id": "s1", "faithfulness": 0.9, "error": ""}],
|
||
|
|
)
|
||
|
|
|
||
|
|
markdown = build_summary_markdown(result)
|
||
|
|
|
||
|
|
assert "## Token 用量" in markdown
|
||
|
|
assert "未记录 token 用量" in markdown
|