fix(llm): resolve score runtime config from saved profiles

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

webapp/api/llm_profiles.py

@@ -148,6 +148,13 @@ def update_profile(profile_id: str, request: CreateProfileRequest) -> LLMProfile
     if updated is None:
         logger.warning("[update_profile] not found  id=%s", profile_id)
         raise HTTPException(status_code=404, detail=f"Profile not found: {profile_id}")
+    # Invalidate scorer cache so next request picks up the new profile settings.
+    try:
+        from webapp.services.inline_scorer import inline_scorer
+        inline_scorer.invalidate_cache()
+        logger.info("[update_profile] scorer cache invalidated  id=%s", profile_id)
+    except Exception:  # noqa: BLE001
+        pass
     logger.info("[update_profile] updated  id=%s", profile_id)
     return updated
 
@@ -160,6 +167,12 @@ def delete_profile(profile_id: str) -> dict:
     if not deleted:
         logger.warning("[delete_profile] not found  id=%s", profile_id)
         raise HTTPException(status_code=404, detail=f"Profile not found: {profile_id}")
+    # Invalidate scorer cache in case the deleted profile was in use.
+    try:
+        from webapp.services.inline_scorer import inline_scorer
+        inline_scorer.invalidate_cache()
+    except Exception:  # noqa: BLE001
+        pass
     logger.info("[delete_profile] deleted  id=%s", profile_id)
     return {"deleted": True}
 

webapp/services/inline_scorer.py

@@ -54,13 +54,22 @@ class InlineScorer:
         self._model_cache: dict[tuple[str, str], tuple[Any, Any]] = {}
         self._lock = threading.Lock()
 
+    def invalidate_cache(self) -> None:
+        """Clear the model cache so the next call rebuilds clients from current profiles."""
+        with self._lock:
+            self._model_cache.clear()
+
     def _get_models(
         self,
         judge_model: str,
         embedding_model: str,
         settings: EvaluationSettings,
     ) -> tuple[Any, Any]:
-        """Return cached LLM/embedding clients, building them on first use."""
+        """Return cached LLM/embedding clients, building them on first use.
+
+        Cache is keyed by (judge_model, embedding_model). Call invalidate_cache()
+        after updating an LLM Profile to force a fresh client on the next request.
+        """
         cache_key = (judge_model, embedding_model)
         with self._lock:
             if cache_key not in self._model_cache: