feat(token-tracking): wrap CLI evaluator metric scoring in track_token_usage

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
wangwei
2026-07-02 14:57:19 +08:00
co-authored by Copilot
parent cd044c8416
commit 5494840431
2 changed files with 138 additions and 5 deletions
+9 -5
View File
@@ -12,6 +12,7 @@ from rag_eval.datasets.loader import load_dataset_records
from rag_eval.datasets.normalizers import normalize_records
from rag_eval.execution.concurrency import gather_with_limit
from rag_eval.metrics.pipeline import MetricPipeline
from rag_eval.metrics.token_tracker import track_token_usage
from rag_eval.metrics.weights import compute_weighted_score, resolve_weight
from rag_eval.shared.models import EvaluationResult, InvalidSample, NormalizedSample, Scenario
from rag_eval.shared.utils import utc_now_iso
@@ -67,14 +68,16 @@ class Evaluator:
logger.info("[eval] scoring %d samples with metric pipeline ...", len(samples))
t0 = time.monotonic()
metric_scores = asyncio.run(
self.metric_pipeline.score_samples(
samples,
max_concurrency=self.scenario.runtime.metric_limit(),
with track_token_usage() as usage_tracker:
metric_scores = asyncio.run(
self.metric_pipeline.score_samples(
samples,
max_concurrency=self.scenario.runtime.metric_limit(),
)
)
)
elapsed = time.monotonic() - t0
logger.info("[eval] metric scoring done elapsed=%.1fs", elapsed)
logger.info("[eval] token_usage=%s", usage_tracker.as_dict())
finished_at = utc_now_iso()
score_rows = [self._merge_score(sample, score) for sample, score in zip(samples, metric_scores)]
@@ -99,6 +102,7 @@ class Evaluator:
valid_samples=samples,
invalid_samples=invalid_samples,
score_rows=score_rows,
token_usage=usage_tracker.as_dict(),
)
async def _enrich_online_samples(