feat(token-tracking): accumulate token usage across session_async calls

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-07-02 15:09:30 +08:00
parent 6a2bbf8239
commit 8245c7f9c3
2 changed files with 119 additions and 14 deletions
@@ -0,0 +1,84 @@
+"""Tests that session-grouped async scoring accumulates token usage across calls."""
+from __future__ import annotations
+
+import json
+import time
+
+from webapp.models import ScoreRequest
+from webapp.services.session_score_manager import SessionScoreJobManager
+
+
+def _wait_for_call_count(mgr: SessionScoreJobManager, session_id: str, expected: int, timeout: float = 2.0):
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        session = mgr.get_session(session_id)
+        if session is not None and session.call_count >= expected:
+            all_done = all(j.status in ("completed", "failed") for j in session.jobs)
+            if all_done:
+                return session
+        time.sleep(0.02)
+    raise TimeoutError(f"session {session_id} did not reach {expected} completed calls in time")
+
+
+def test_session_accumulates_token_usage_across_calls(tmp_path, monkeypatch):
+    from rag_eval.metrics.token_tracker import get_current_tracker
+
+    mgr = SessionScoreJobManager(
+        output_dir=tmp_path / "score-session",
+        index_dir=tmp_path / "score-session-jobs",
+        max_workers=1,
+    )
+
+    call_usages = iter([(100, 40), (30, 10)])
+
+    def _fake_score(**kwargs):
+        tracker = get_current_tracker()
+        input_tok, output_tok = next(call_usages)
+        if tracker is not None:
+            tracker.record("gpt-5", input_tok, output_tok)
+        return {m: 0.9 for m in kwargs["metrics"]}
+
+    monkeypatch.setattr(
+        "webapp.services.inline_scorer.inline_scorer.score", _fake_score
+    )
+
+    request = ScoreRequest(question="q?", answer="a.", metrics=["answer_relevancy"])
+
+    _, run_id = mgr.submit("session-token-test", request)
+    _wait_for_call_count(mgr, "session-token-test", 1)
+    mgr.submit("session-token-test", request)
+    _wait_for_call_count(mgr, "session-token-test", 2)
+
+    run_dir = tmp_path / "score-session" / run_id
+    metadata = json.loads((run_dir / "metadata.json").read_text(encoding="utf-8"))
+    assert metadata["token_usage"] == {
+        "gpt-5": {"input_tokens": 130, "output_tokens": 50, "calls": 2}
+    }
+
+
+def test_session_first_call_writes_token_usage_from_scratch(tmp_path, monkeypatch):
+    from rag_eval.metrics.token_tracker import get_current_tracker
+
+    mgr = SessionScoreJobManager(
+        output_dir=tmp_path / "score-session",
+        index_dir=tmp_path / "score-session-jobs",
+        max_workers=1,
+    )
+
+    def _fake_score(**kwargs):
+        tracker = get_current_tracker()
+        if tracker is not None:
+            tracker.record("gpt-5", 50, 20)
+        return {m: 0.9 for m in kwargs["metrics"]}
+
+    monkeypatch.setattr(
+        "webapp.services.inline_scorer.inline_scorer.score", _fake_score
+    )
+
+    request = ScoreRequest(question="q?", answer="a.", metrics=["answer_relevancy"])
+    _, run_id = mgr.submit("session-first-call-test", request)
+    _wait_for_call_count(mgr, "session-first-call-test", 1)
+
+    run_dir = tmp_path / "score-session" / run_id
+    metadata = json.loads((run_dir / "metadata.json").read_text(encoding="utf-8"))
+    assert metadata["token_usage"] == {"gpt-5": {"input_tokens": 50, "output_tokens": 20, "calls": 1}}