diff --git a/backend/app/infrastructure/vectorstore/cross_encoder_reranker.py b/backend/app/infrastructure/vectorstore/cross_encoder_reranker.py index d902bc9..f3c0326 100644 --- a/backend/app/infrastructure/vectorstore/cross_encoder_reranker.py +++ b/backend/app/infrastructure/vectorstore/cross_encoder_reranker.py @@ -9,6 +9,7 @@ from loguru import logger from app.config.settings import settings from app.domain.retrieval import Reranker, RetrievedChunk +from app.shared.model_usage_tracker import get_model_usage_tracker class OpenAICompatibleReranker(Reranker): @@ -37,10 +38,26 @@ class OpenAICompatibleReranker(Reranker): scores = self._call_reranker(query, texts) except Exception as exc: logger.warning("Reranker call failed ({}), falling back to original order: {}", type(exc).__name__, exc) + # Record the failure so the Status page reflects real reranker health. + get_model_usage_tracker().record( + provider="reranker", + model=self._model, + success=False, + latency_ms=int((time.time() - start) * 1000), + error=str(exc), + ) return chunks[:top_k] elapsed_ms = int((time.time() - start) * 1000) logger.debug("Reranker scored {} chunks in {}ms", len(chunks), elapsed_ms) + # TEI/Cohere-style rerank responses carry no token usage field — + # only call success/latency is meaningful for this role. + get_model_usage_tracker().record( + provider="reranker", + model=self._model, + success=True, + latency_ms=elapsed_ms, + ) ranked = sorted( [(score, chunk) for score, chunk in zip(scores, chunks)], diff --git a/backend/tests/observability/test_reranker_usage_tracking.py b/backend/tests/observability/test_reranker_usage_tracking.py new file mode 100644 index 0000000..e36f7f2 --- /dev/null +++ b/backend/tests/observability/test_reranker_usage_tracking.py @@ -0,0 +1,50 @@ +"""Verifies OpenAICompatibleReranker records call outcome (no tokens) into ModelUsageTracker.""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from app.domain.retrieval import RetrievedChunk +from app.infrastructure.vectorstore.cross_encoder_reranker import OpenAICompatibleReranker +from app.shared.model_usage_tracker import get_model_usage_tracker + + +@pytest.fixture(autouse=True) +def _reset_tracker(): + """Clear the process-wide tracker before and after each test in this file.""" + get_model_usage_tracker()._entries.clear() + yield + get_model_usage_tracker()._entries.clear() + + +def _chunk(chunk_id: str, text: str) -> RetrievedChunk: + """Build a minimal RetrievedChunk for reranker tests.""" + return RetrievedChunk(chunk_id=chunk_id, doc_id="doc-1", doc_title="Doc", text=text, score=0.0) + + +def test_successful_rerank_records_call_without_tokens(): + """A successful rerank() call is recorded with call_count_ok but zero tokens.""" + reranker = OpenAICompatibleReranker(base_url="http://example.test", model="bge-reranker-v2-m3") + with patch.object(reranker, "_call_reranker", return_value=[0.9, 0.1]): + result = reranker.rerank("query", [_chunk("c1", "a"), _chunk("c2", "b")], top_k=2) + + assert len(result) == 2 + entry = get_model_usage_tracker().get("reranker", "bge-reranker-v2-m3") + assert entry is not None + assert entry.call_count_ok == 1 + assert entry.total_tokens == 0 + + +def test_failed_rerank_records_error_and_falls_back(): + """A rerank() call that raises internally is recorded as an error but still returns a fallback list.""" + reranker = OpenAICompatibleReranker(base_url="http://example.test", model="bge-reranker-v2-m3") + with patch.object(reranker, "_call_reranker", side_effect=RuntimeError("gateway down")): + result = reranker.rerank("query", [_chunk("c1", "a")], top_k=1) + + assert len(result) == 1 # existing fallback behavior: original order, unscored + entry = get_model_usage_tracker().get("reranker", "bge-reranker-v2-m3") + assert entry is not None + assert entry.call_count_error == 1 + assert entry.status == "error"