Wire judge-prompt localization into factory and inline scorer

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

rag_eval/metrics/factory.py

@@ -24,6 +24,7 @@ from ragas.metrics.collections import (
     SemanticSimilarity,
 )
 
+from .judge_prompts import localize_pipeline_prompts
 from .pipeline import MetricPipeline
 
 
@@ -133,7 +134,11 @@ def build_metric_pipeline(
 
     # Build the full registry once using the shared factory, then slice by requested metrics.
     registry = build_metric_registry(llm, embeddings)
+    selected = {name: registry[name] for name in scenario.metrics}
+    # Apply language-specific judge prompt localization (no-op for 'en').
+    language = scenario.judge_language or settings.ragas_judge_language or "en"
+    localize_pipeline_prompts(selected, language)
     return MetricPipeline(
-        metrics={name: registry[name] for name in scenario.metrics},
+        metrics=selected,
         metric_timeout_seconds=settings.ragas_metric_timeout_seconds,
     )

tests/test_judge_language_wiring.py

@@ -0,0 +1,119 @@
+"""Tests that the factory and inline scorer invoke the localizer per language."""
+
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import rag_eval.metrics.factory as factory_mod
+import webapp.services.inline_scorer as inline_mod
+
+
+def test_build_pipeline_localizes_when_zh(monkeypatch):
+    """build_metric_pipeline calls localize_pipeline_prompts with 'zh'."""
+    calls = []
+    monkeypatch.setattr(factory_mod, "localize_pipeline_prompts",
+                        lambda registry, language: calls.append(language))
+
+    from rag_eval.shared.models import DatasetConfig, Scenario
+    from rag_eval.settings import EvaluationSettings
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbedding
+
+    scenario = Scenario(
+        scenario_name="t", mode="offline",
+        dataset=DatasetConfig(path=Path("x.csv")),
+        judge_model="gpt-5", embedding_model="text-embedding-3-small",
+        metrics=["faithfulness"], output_dir=Path("out"),
+        judge_language="zh",
+    )
+    factory_mod.build_metric_pipeline(
+        scenario, EvaluationSettings(_env_file=None),
+        llm=MagicMock(spec=InstructorBaseRagasLLM),
+        embeddings=MagicMock(spec=BaseRagasEmbedding),
+    )
+    assert calls == ["zh"]
+
+
+def test_build_pipeline_falls_back_to_settings_default(monkeypatch):
+    """When scenario.judge_language is None, settings.ragas_judge_language is used."""
+    calls = []
+    monkeypatch.setattr(factory_mod, "localize_pipeline_prompts",
+                        lambda registry, language: calls.append(language))
+
+    from rag_eval.shared.models import DatasetConfig, Scenario
+    from rag_eval.settings import EvaluationSettings
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbedding
+
+    scenario = Scenario(
+        scenario_name="t", mode="offline",
+        dataset=DatasetConfig(path=Path("x.csv")),
+        judge_model="gpt-5", embedding_model="text-embedding-3-small",
+        metrics=["faithfulness"], output_dir=Path("out"),
+        judge_language=None,
+    )
+    settings = EvaluationSettings(_env_file=None)
+    settings.ragas_judge_language = "zh"
+    factory_mod.build_metric_pipeline(
+        scenario, settings,
+        llm=MagicMock(spec=InstructorBaseRagasLLM),
+        embeddings=MagicMock(spec=BaseRagasEmbedding),
+    )
+    assert calls == ["zh"]
+
+
+def test_build_pipeline_en_is_noop(monkeypatch):
+    """Default language 'en' still calls localize (which is a no-op inside)."""
+    calls = []
+    monkeypatch.setattr(factory_mod, "localize_pipeline_prompts",
+                        lambda registry, language: calls.append(language))
+
+    from rag_eval.shared.models import DatasetConfig, Scenario
+    from rag_eval.settings import EvaluationSettings
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbedding
+
+    scenario = Scenario(
+        scenario_name="t", mode="offline",
+        dataset=DatasetConfig(path=Path("x.csv")),
+        judge_model="gpt-5", embedding_model="text-embedding-3-small",
+        metrics=["faithfulness"], output_dir=Path("out"),
+    )
+    factory_mod.build_metric_pipeline(
+        scenario, EvaluationSettings(_env_file=None),
+        llm=MagicMock(spec=InstructorBaseRagasLLM),
+        embeddings=MagicMock(spec=BaseRagasEmbedding),
+    )
+    assert calls == ["en"]
+
+
+def test_inline_score_threads_judge_language(monkeypatch):
+    """InlineScorer.score forwards judge_language to _build_metric_instances."""
+    seen: dict = {}
+
+    def fake_build_instances(metrics, llm, embeddings, judge_language="en"):
+        seen["lang"] = judge_language
+        return {}
+
+    monkeypatch.setattr(inline_mod, "_build_metric_instances", fake_build_instances)
+    monkeypatch.setattr(inline_mod.InlineScorer, "_get_models",
+                        lambda self, j, e, s: (object(), object()))
+
+    class _FakePipeline:
+        def __init__(self, *a, **k):
+            pass
+
+        async def score_sample(self, sample):
+            return _FakeResult()
+
+    class _FakeResult:
+        metrics: dict = {}
+
+    import asyncio
+    monkeypatch.setattr(inline_mod, "MetricPipeline", _FakePipeline)
+    monkeypatch.setattr(asyncio, "run", lambda coro: _FakeResult())
+
+    scorer = inline_mod.InlineScorer()
+    scorer.score(question="q", answer="a", contexts=[], ground_truth=None,
+                 metrics=["faithfulness"], judge_model="gpt-5",
+                 embedding_model="e", settings=MagicMock(), judge_language="zh")
+    assert seen.get("lang") == "zh"

webapp/services/inline_scorer.py

@@ -14,6 +14,7 @@ from typing import Any
 
 from rag_eval.compat import ensure_ragas_import_compat
 from rag_eval.metrics.factory import build_metric_registry, build_models
+from rag_eval.metrics.judge_prompts import localize_pipeline_prompts
 from rag_eval.metrics.pipeline import MetricPipeline
 from rag_eval.settings import EvaluationSettings
 from rag_eval.shared.models import NormalizedSample
@@ -21,10 +22,14 @@ from rag_eval.shared.models import NormalizedSample
 ensure_ragas_import_compat()
 
 
-def _build_metric_instances(metrics: list[str], llm: Any, embeddings: Any) -> dict[str, Any]:
+def _build_metric_instances(
-    """Instantiate only the RAGAS metric objects requested."""
+    metrics: list[str], llm: Any, embeddings: Any, judge_language: str = "en"
+) -> dict[str, Any]:
+    """Instantiate only the RAGAS metric objects requested, localized if needed."""
     registry = build_metric_registry(llm, embeddings)
-    return {name: registry[name] for name in metrics if name in registry}
+    selected = {name: registry[name] for name in metrics if name in registry}
+    localize_pipeline_prompts(selected, judge_language)
+    return selected
 
 
 class InlineScorer:
@@ -69,10 +74,11 @@ class InlineScorer:
         judge_model: str,
         embedding_model: str,
         settings: EvaluationSettings,
+        judge_language: str = "en",
     ) -> dict[str, float | None]:
         """Score one sample synchronously and return {metric_name: score | None}."""
         llm, embeddings = self._get_models(judge_model, embedding_model, settings)
-        metric_instances = _build_metric_instances(metrics, llm, embeddings)
+        metric_instances = _build_metric_instances(metrics, llm, embeddings, judge_language)
 
         pipeline = MetricPipeline(
             metrics=metric_instances,