Wire judge-prompt localization into factory and inline scorer

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
wangwei
2026-07-01 17:58:43 +08:00
co-authored by Copilot
parent 555328cb3b
commit bd5658c3ac
3 changed files with 135 additions and 5 deletions
+10 -4
View File
@@ -14,6 +14,7 @@ from typing import Any
from rag_eval.compat import ensure_ragas_import_compat
from rag_eval.metrics.factory import build_metric_registry, build_models
from rag_eval.metrics.judge_prompts import localize_pipeline_prompts
from rag_eval.metrics.pipeline import MetricPipeline
from rag_eval.settings import EvaluationSettings
from rag_eval.shared.models import NormalizedSample
@@ -21,10 +22,14 @@ from rag_eval.shared.models import NormalizedSample
ensure_ragas_import_compat()
def _build_metric_instances(metrics: list[str], llm: Any, embeddings: Any) -> dict[str, Any]:
"""Instantiate only the RAGAS metric objects requested."""
def _build_metric_instances(
metrics: list[str], llm: Any, embeddings: Any, judge_language: str = "en"
) -> dict[str, Any]:
"""Instantiate only the RAGAS metric objects requested, localized if needed."""
registry = build_metric_registry(llm, embeddings)
return {name: registry[name] for name in metrics if name in registry}
selected = {name: registry[name] for name in metrics if name in registry}
localize_pipeline_prompts(selected, judge_language)
return selected
class InlineScorer:
@@ -69,10 +74,11 @@ class InlineScorer:
judge_model: str,
embedding_model: str,
settings: EvaluationSettings,
judge_language: str = "en",
) -> dict[str, float | None]:
"""Score one sample synchronously and return {metric_name: score | None}."""
llm, embeddings = self._get_models(judge_model, embedding_model, settings)
metric_instances = _build_metric_instances(metrics, llm, embeddings)
metric_instances = _build_metric_instances(metrics, llm, embeddings, judge_language)
pipeline = MetricPipeline(
metrics=metric_instances,