first commit

rag_eval/adapters/__init__.py

@@ -0,0 +1,7 @@
+"""Adapter implementations that connect evaluation flows to target applications."""
+
+from .base import AppAdapter
+from .http import HttpAppAdapter
+from .python import PythonFunctionAdapter
+
+__all__ = ["AppAdapter", "HttpAppAdapter", "PythonFunctionAdapter"]

rag_eval/adapters/base.py

@@ -0,0 +1,37 @@
+"""Shared adapter interfaces for online application execution."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from rag_eval.shared.models import NormalizedSample
+
+
+class AppAdapter(ABC):
+    """Abstract base class for adapters that fetch answers and contexts from apps."""
+
+    @abstractmethod
+    async def run(self, question: str, **kwargs: Any) -> dict[str, Any]:
+        """Execute the target application for a single question."""
+        raise NotImplementedError
+
+    async def enrich_sample(self, sample: NormalizedSample) -> NormalizedSample:
+        """Merge adapter output into an existing normalized sample."""
+        response = await self.run(question=sample.question, **sample.metadata)
+        answer = str(response.get("answer", "")).strip()
+        contexts = response.get("contexts") or []
+        # Drop empty context fragments so downstream metrics receive clean lists.
+        normalized_contexts = [str(item).strip() for item in contexts if str(item).strip()]
+        return NormalizedSample(
+            sample_id=sample.sample_id,
+            question=sample.question,
+            contexts=normalized_contexts,
+            answer=answer,
+            ground_truth=sample.ground_truth,
+            scenario=sample.scenario,
+            language=sample.language,
+            retrieval_config=sample.retrieval_config,
+            metadata={**sample.metadata, "raw_response": response.get("raw_response")},
+            raw=sample.raw,
+        )

rag_eval/adapters/http.py

@@ -0,0 +1,45 @@
+"""HTTP adapter implementation for online evaluation scenarios."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from rag_eval.shared.models import AppAdapterConfig
+
+from .base import AppAdapter
+
+
+class HttpAppAdapter(AppAdapter):
+    """Call an HTTP endpoint and map its JSON response into the normalized adapter shape."""
+
+    def __init__(self, config: AppAdapterConfig):
+        """Store the HTTP adapter configuration for later requests."""
+        self.config = config
+
+    async def run(self, question: str, **kwargs: Any) -> dict[str, Any]:
+        """Send one HTTP request and return the normalized response payload."""
+        payload = dict(self.config.request_template)
+        payload["question"] = question
+        payload.update(self.config.static_kwargs)
+        payload.update(kwargs)
+
+        async with httpx.AsyncClient(timeout=self.config.timeout_seconds) as client:
+            response = await client.request(
+                self.config.method.upper(),
+                self.config.endpoint or "",
+                json=payload,
+            )
+            response.raise_for_status()
+            body = response.json()
+
+        # Allow scenario config to rename answer/context fields without custom code.
+        mapping = self.config.response_mapping or {}
+        answer_key = mapping.get("answer", "answer")
+        contexts_key = mapping.get("contexts", "contexts")
+        return {
+            "answer": body.get(answer_key, ""),
+            "contexts": body.get(contexts_key, []),
+            "raw_response": body,
+        }

rag_eval/adapters/python.py

@@ -0,0 +1,38 @@
+"""Python callable adapter for in-process application integrations."""
+
+from __future__ import annotations
+
+from importlib import import_module
+from typing import Any, Callable
+
+from rag_eval.shared.models import AppAdapterConfig
+
+from .base import AppAdapter
+
+
+class PythonFunctionAdapter(AppAdapter):
+    """Wrap a configured Python callable so it can participate in online evaluation."""
+
+    def __init__(self, config: AppAdapterConfig):
+        """Load and cache the configured callable during adapter initialization."""
+        self.config = config
+        self._callable = self._load_callable(config.callable or "")
+
+    @staticmethod
+    def _load_callable(target: str) -> Callable[..., dict[str, Any]]:
+        """Resolve a `module:function` target into a callable object."""
+        module_name, _, attr_name = target.partition(":")
+        if not module_name or not attr_name:
+            raise ValueError("Python adapter callable must use module:function syntax.")
+        module = import_module(module_name)
+        fn = getattr(module, attr_name)
+        if not callable(fn):
+            raise TypeError(f"Configured callable is not callable: {target}")
+        return fn
+
+    async def run(self, question: str, **kwargs: Any) -> dict[str, Any]:
+        """Invoke the configured callable and enforce the adapter response contract."""
+        result = self._callable(question=question, **self.config.static_kwargs, **kwargs)
+        if not isinstance(result, dict):
+            raise TypeError("Python adapter callable must return a dict.")
+        return result