"""Integration tests for the /status/models routes. Uses FastAPI TestClient with mocked LLM/embedding/reranker clients so no external gateway or database is required. """ from __future__ import annotations from unittest.mock import MagicMock, patch import pytest from fastapi.testclient import TestClient from app.services.llm.base_client import LLMResponse from app.services.llm.llm_factory import LLMFactory, get_llm_client from app.shared.model_usage_tracker import get_model_usage_tracker @pytest.fixture(autouse=True) def _reset_tracker(): """Clear the process-wide tracker before and after each test in this file.""" get_model_usage_tracker()._entries.clear() yield get_model_usage_tracker()._entries.clear() @pytest.fixture def _reset_llm_factory_instances(): """Clear LLMFactory's process-wide client cache before/after this test only. Mirrors backend/tests/observability/test_llm_factory_tracking.py's _reset_singletons fixture: LLMFactory._global_instances persists for the life of the process, so without this the cache entry created by driving a real get_llm_client() call in a test would leak into other tests. """ LLMFactory._global_instances.clear() yield LLMFactory._global_instances.clear() @pytest.fixture def client(): """Return a TestClient for the real app (status routes require no auth).""" from app.api.main import app with TestClient(app, raise_server_exceptions=False) as c: yield c def test_get_models_returns_four_roles_never_called_by_default(client): """With no calls made yet, all 4 roles are returned with status 'never_called' or 'disabled'.""" resp = client.get("/api/v1/status/models") assert resp.status_code == 200 body = resp.json() roles = {m["role"] for m in body["models"]} assert roles == {"main_llm", "hyde_llm", "embedding", "reranker"} reranker_row = next(m for m in body["models"] if m["role"] == "reranker") # Default .env.example ships RERANKER_ENABLED=false. from app.config.settings import settings assert reranker_row["enabled"] == settings.reranker_enabled if not settings.reranker_enabled: assert reranker_row["status"] == "disabled" def test_get_models_reflects_recorded_usage(client): """A previously recorded call must show up in total_tokens/status.""" from app.config.settings import settings get_model_usage_tracker().record( provider=settings.llm_provider, model=settings.llm_model, success=True, usage={"total_tokens": 99}, ) resp = client.get("/api/v1/status/models") main_row = next(m for m in resp.json()["models"] if m["role"] == "main_llm") assert main_row["total_tokens"] == 99 assert main_row["status"] == "ok" def test_get_models_hyde_llm_disabled_forces_disabled_status(client): """settings.hyde_enabled=False must force hyde_llm to enabled=False/status='disabled', mirroring the reranker override, even if HyDE previously ran successfully.""" from app.config.settings import settings get_model_usage_tracker().record( provider=settings.hyde_llm_provider or settings.llm_provider, model=settings.hyde_llm_model or settings.llm_model, success=True, ) with patch.object(settings, "hyde_enabled", False): resp = client.get("/api/v1/status/models") assert resp.status_code == 200 hyde_row = next(m for m in resp.json()["models"] if m["role"] == "hyde_llm") assert hyde_row["enabled"] is False assert hyde_row["status"] == "disabled" def test_ping_models_calls_each_enabled_model_once(client): """POST /status/models/ping must invoke chat()/embed_query() and return fresh statuses.""" mock_llm_response = LLMResponse(content="pong", model="test-model", usage={"total_tokens": 1}) mock_llm_client = MagicMock() mock_llm_client.chat.return_value = mock_llm_response mock_embedding = MagicMock() mock_embedding.embed_query.return_value = [0.1] with patch("app.api.routes.status.get_llm_client", return_value=mock_llm_client), \ patch("app.api.routes.status.get_embedding_provider", return_value=mock_embedding), \ patch("app.api.routes.status.get_reranker", return_value=None): resp = client.post("/api/v1/status/models/ping") assert resp.status_code == 200 body = resp.json() assert len(body["models"]) == 4 assert mock_llm_client.chat.call_count >= 1 mock_embedding.embed_query.assert_called_once() def test_ping_models_survives_one_model_failing(client): """If the LLM ping raises, embedding/reranker pings must still be attempted and a 200 returned.""" mock_embedding = MagicMock() mock_embedding.embed_query.return_value = [0.1] with patch("app.api.routes.status.get_llm_client", side_effect=RuntimeError("gateway down")), \ patch("app.api.routes.status.get_embedding_provider", return_value=mock_embedding), \ patch("app.api.routes.status.get_reranker", return_value=None): resp = client.post("/api/v1/status/models/ping") assert resp.status_code == 200 mock_embedding.embed_query.assert_called_once() def test_ping_records_get_llm_client_failure_instead_of_dropping_it(client): """A get_llm_client() failure (raised before any TrackedLLMClient exists) must still be recorded into the tracker, so it is visible afterwards via _build_model_status() instead of being silently discarded by asyncio.gather(return_exceptions=True).""" mock_embedding = MagicMock() mock_embedding.embed_query.return_value = [0.1] with patch("app.api.routes.status.get_llm_client", side_effect=RuntimeError("missing api key")), \ patch("app.api.routes.status.get_embedding_provider", return_value=mock_embedding), \ patch("app.api.routes.status.get_reranker", return_value=None): resp = client.post("/api/v1/status/models/ping") assert resp.status_code == 200 body = resp.json() main_row = next(m for m in body["models"] if m["role"] == "main_llm") hyde_row = next(m for m in body["models"] if m["role"] == "hyde_llm") assert main_row["status"] == "error" assert main_row["call_count_error"] == 1 assert main_row["last_error"] == "missing api key" assert hyde_row["status"] == "error" assert hyde_row["call_count_error"] == 1 assert hyde_row["last_error"] == "missing api key" def test_ping_skips_hyde_llm_when_hyde_disabled(client): """POST /status/models/ping must NOT ping hyde_llm when settings.hyde_enabled is False: HyDE often reuses the main LLM, so a live ping would just be a redundant duplicate chat call against the same model for no benefit. main_llm must still always be pinged.""" from app.config.settings import settings mock_llm_response = LLMResponse(content="pong", model="test-model", usage={"total_tokens": 1}) mock_llm_client = MagicMock() mock_llm_client.chat.return_value = mock_llm_response mock_embedding = MagicMock() mock_embedding.embed_query.return_value = [0.1] with patch.object(settings, "hyde_enabled", False), \ patch("app.api.routes.status.get_llm_client", return_value=mock_llm_client) as mock_get_llm_client, \ patch("app.api.routes.status.get_embedding_provider", return_value=mock_embedding), \ patch("app.api.routes.status.get_reranker", return_value=None): resp = client.post("/api/v1/status/models/ping") assert resp.status_code == 200 # Only main_llm's ping should reach get_llm_client()/.chat(); hyde_llm's must be skipped. assert mock_get_llm_client.call_count == 1 assert mock_llm_client.chat.call_count == 1 hyde_row = next(m for m in resp.json()["models"] if m["role"] == "hyde_llm") assert hyde_row["status"] == "disabled" def test_ping_write_path_and_status_read_path_agree_for_non_canonical_provider_alias( client, _reset_llm_factory_instances ): """Regression for the tracker key mismatch bug: TrackedLLMClient.chat() always records usage under the NORMALIZED LLMProvider enum value from LLMFactory._parse_provider() (see tracked_client.py), never the raw provider string a caller passed to get_llm_client(). Before the fix, _resolve_role_provider_model() returned settings.llm_provider verbatim with no normalization. So whenever LLM_PROVIDER held a non-canonical alias (e.g. "deepseek-v3" instead of "deepseek"), the read-side lookup key ("deepseek-v3:") stopped matching the write-side key TrackedLLMClient actually recorded under ("deepseek:"), and /status/models showed main_llm as perpetually "never_called" even though it was being actively tracked. This drives the FULL real path (LLMFactory.create() -> TrackedLLMClient -> ModelUsageTracker -> the /status/models route), the same way real call sites like compliance.py invoke get_llm_client(provider=settings.llm_provider, model=settings.llm_model) -- unlike test_get_models_reflects_recorded_usage above, which shortcuts by recording directly into the tracker. """ from app.config.settings import settings mock_response = LLMResponse(content="pong", model=settings.llm_model, usage={"total_tokens": 7}) def _fake_deepseek_client(config): # Carry the REAL LLMConfig built by LLMFactory.create() so TrackedLLMClient.chat() # records under config.provider.value exactly like it does in production. fake = MagicMock() fake.config = config fake.chat.return_value = mock_response return fake with patch.object(settings, "llm_provider", "deepseek-v3"), \ patch("app.services.llm.llm_factory.DeepSeekClient", side_effect=_fake_deepseek_client): tracked_client = get_llm_client(provider=settings.llm_provider, model=settings.llm_model, api_key="test-key") tracked_client.chat([{"role": "user", "content": "hi"}]) resp = client.get("/api/v1/status/models") assert resp.status_code == 200 main_row = next(m for m in resp.json()["models"] if m["role"] == "main_llm") assert main_row["status"] == "ok" assert main_row["total_tokens"] == 7 assert main_row["provider"] == "deepseek"