80 lines
3.5 KiB
Python
80 lines
3.5 KiB
Python
|
"""Unit tests for ModelUsageTracker — no mocking needed, pure in-memory state."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from app.shared.model_usage_tracker import ModelUsageEntry, ModelUsageTracker, get_model_usage_tracker
|
||
|
|
|
||
|
|
|
||
|
|
def test_never_called_model_has_no_entry():
|
||
|
|
"""A tracker that has never recorded a call returns None from get()."""
|
||
|
|
tracker = ModelUsageTracker()
|
||
|
|
assert tracker.get("deepseek", "deepseek-v4-flash") is None
|
||
|
|
|
||
|
|
|
||
|
|
def test_record_success_accumulates_tokens_and_calls():
|
||
|
|
"""Two successful calls accumulate tokens and call_count_ok."""
|
||
|
|
tracker = ModelUsageTracker()
|
||
|
|
tracker.record(
|
||
|
|
provider="deepseek", model="deepseek-v4-flash", success=True,
|
||
|
|
usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, latency_ms=100,
|
||
|
|
)
|
||
|
|
tracker.record(
|
||
|
|
provider="deepseek", model="deepseek-v4-flash", success=True,
|
||
|
|
usage={"prompt_tokens": 20, "completion_tokens": 8, "total_tokens": 28}, latency_ms=200,
|
||
|
|
)
|
||
|
|
entry = tracker.get("deepseek", "deepseek-v4-flash")
|
||
|
|
assert entry is not None
|
||
|
|
assert entry.total_tokens == 43
|
||
|
|
assert entry.prompt_tokens == 30
|
||
|
|
assert entry.completion_tokens == 13
|
||
|
|
assert entry.call_count_ok == 2
|
||
|
|
assert entry.call_count_error == 0
|
||
|
|
assert entry.status == "ok"
|
||
|
|
assert entry.last_latency_ms == 200
|
||
|
|
|
||
|
|
|
||
|
|
def test_record_error_sets_error_status_without_losing_prior_tokens():
|
||
|
|
"""A failed call after successful ones flips status to 'error' but keeps accumulated tokens."""
|
||
|
|
tracker = ModelUsageTracker()
|
||
|
|
tracker.record(provider="qwen", model="qwen3.5-flash", success=True, usage={"total_tokens": 50}, latency_ms=50)
|
||
|
|
tracker.record(provider="qwen", model="qwen3.5-flash", success=False, error="HTTP 500", latency_ms=30)
|
||
|
|
entry = tracker.get("qwen", "qwen3.5-flash")
|
||
|
|
assert entry.total_tokens == 50
|
||
|
|
assert entry.call_count_ok == 1
|
||
|
|
assert entry.call_count_error == 1
|
||
|
|
assert entry.status == "error"
|
||
|
|
assert entry.last_error == "HTTP 500"
|
||
|
|
|
||
|
|
|
||
|
|
def test_record_success_after_error_clears_last_error():
|
||
|
|
"""A later successful call clears last_error and status returns to 'ok'."""
|
||
|
|
tracker = ModelUsageTracker()
|
||
|
|
tracker.record(provider="qwen", model="qwen3.5-flash", success=False, error="timeout", latency_ms=30)
|
||
|
|
tracker.record(provider="qwen", model="qwen3.5-flash", success=True, usage={"total_tokens": 5}, latency_ms=40)
|
||
|
|
entry = tracker.get("qwen", "qwen3.5-flash")
|
||
|
|
assert entry.status == "ok"
|
||
|
|
assert entry.last_error is None
|
||
|
|
|
||
|
|
|
||
|
|
def test_record_never_raises_on_bad_usage_dict():
|
||
|
|
"""A malformed usage value (wrong type) is swallowed, not raised, and does not corrupt other entries."""
|
||
|
|
tracker = ModelUsageTracker()
|
||
|
|
tracker.record(provider="embedding", model="text-embedding-v3", success=True, usage="not-a-dict", latency_ms=10) # type: ignore[arg-type]
|
||
|
|
# Must not raise, and must not have created a corrupted entry that breaks snapshot().
|
||
|
|
snapshot = tracker.snapshot()
|
||
|
|
assert isinstance(snapshot, dict)
|
||
|
|
|
||
|
|
|
||
|
|
def test_snapshot_returns_independent_copy():
|
||
|
|
"""snapshot() returns a dict that can be safely mutated without affecting the tracker."""
|
||
|
|
tracker = ModelUsageTracker()
|
||
|
|
tracker.record(provider="deepseek", model="deepseek-v4-flash", success=True, usage={"total_tokens": 1}, latency_ms=1)
|
||
|
|
snap = tracker.snapshot()
|
||
|
|
snap.clear()
|
||
|
|
assert tracker.get("deepseek", "deepseek-v4-flash") is not None
|
||
|
|
|
||
|
|
|
||
|
|
def test_get_model_usage_tracker_returns_singleton():
|
||
|
|
"""get_model_usage_tracker() always returns the same process-wide instance."""
|
||
|
|
assert get_model_usage_tracker() is get_model_usage_tracker()
|