feat(token-tracking): wrap CLI evaluator metric scoring in track_token_usage
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -12,6 +12,7 @@ from rag_eval.datasets.loader import load_dataset_records
|
||||
from rag_eval.datasets.normalizers import normalize_records
|
||||
from rag_eval.execution.concurrency import gather_with_limit
|
||||
from rag_eval.metrics.pipeline import MetricPipeline
|
||||
from rag_eval.metrics.token_tracker import track_token_usage
|
||||
from rag_eval.metrics.weights import compute_weighted_score, resolve_weight
|
||||
from rag_eval.shared.models import EvaluationResult, InvalidSample, NormalizedSample, Scenario
|
||||
from rag_eval.shared.utils import utc_now_iso
|
||||
@@ -67,14 +68,16 @@ class Evaluator:
|
||||
|
||||
logger.info("[eval] scoring %d samples with metric pipeline ...", len(samples))
|
||||
t0 = time.monotonic()
|
||||
metric_scores = asyncio.run(
|
||||
self.metric_pipeline.score_samples(
|
||||
samples,
|
||||
max_concurrency=self.scenario.runtime.metric_limit(),
|
||||
with track_token_usage() as usage_tracker:
|
||||
metric_scores = asyncio.run(
|
||||
self.metric_pipeline.score_samples(
|
||||
samples,
|
||||
max_concurrency=self.scenario.runtime.metric_limit(),
|
||||
)
|
||||
)
|
||||
)
|
||||
elapsed = time.monotonic() - t0
|
||||
logger.info("[eval] metric scoring done elapsed=%.1fs", elapsed)
|
||||
logger.info("[eval] token_usage=%s", usage_tracker.as_dict())
|
||||
|
||||
finished_at = utc_now_iso()
|
||||
score_rows = [self._merge_score(sample, score) for sample, score in zip(samples, metric_scores)]
|
||||
@@ -99,6 +102,7 @@ class Evaluator:
|
||||
valid_samples=samples,
|
||||
invalid_samples=invalid_samples,
|
||||
score_rows=score_rows,
|
||||
token_usage=usage_tracker.as_dict(),
|
||||
)
|
||||
|
||||
async def _enrich_online_samples(
|
||||
|
||||
Reference in New Issue
Block a user