Wire judge-prompt localization into factory and inline scorer
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -14,6 +14,7 @@ from typing import Any
|
||||
|
||||
from rag_eval.compat import ensure_ragas_import_compat
|
||||
from rag_eval.metrics.factory import build_metric_registry, build_models
|
||||
from rag_eval.metrics.judge_prompts import localize_pipeline_prompts
|
||||
from rag_eval.metrics.pipeline import MetricPipeline
|
||||
from rag_eval.settings import EvaluationSettings
|
||||
from rag_eval.shared.models import NormalizedSample
|
||||
@@ -21,10 +22,14 @@ from rag_eval.shared.models import NormalizedSample
|
||||
ensure_ragas_import_compat()
|
||||
|
||||
|
||||
def _build_metric_instances(metrics: list[str], llm: Any, embeddings: Any) -> dict[str, Any]:
|
||||
"""Instantiate only the RAGAS metric objects requested."""
|
||||
def _build_metric_instances(
|
||||
metrics: list[str], llm: Any, embeddings: Any, judge_language: str = "en"
|
||||
) -> dict[str, Any]:
|
||||
"""Instantiate only the RAGAS metric objects requested, localized if needed."""
|
||||
registry = build_metric_registry(llm, embeddings)
|
||||
return {name: registry[name] for name in metrics if name in registry}
|
||||
selected = {name: registry[name] for name in metrics if name in registry}
|
||||
localize_pipeline_prompts(selected, judge_language)
|
||||
return selected
|
||||
|
||||
|
||||
class InlineScorer:
|
||||
@@ -69,10 +74,11 @@ class InlineScorer:
|
||||
judge_model: str,
|
||||
embedding_model: str,
|
||||
settings: EvaluationSettings,
|
||||
judge_language: str = "en",
|
||||
) -> dict[str, float | None]:
|
||||
"""Score one sample synchronously and return {metric_name: score | None}."""
|
||||
llm, embeddings = self._get_models(judge_model, embedding_model, settings)
|
||||
metric_instances = _build_metric_instances(metrics, llm, embeddings)
|
||||
metric_instances = _build_metric_instances(metrics, llm, embeddings, judge_language)
|
||||
|
||||
pipeline = MetricPipeline(
|
||||
metrics=metric_instances,
|
||||
|
||||
Reference in New Issue
Block a user