Wire judge-prompt localization into factory and inline scorer

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-07-01 17:58:43 +08:00
parent 555328cb3b
commit bd5658c3ac
3 changed files with 135 additions and 5 deletions
@@ -0,0 +1,119 @@
+"""Tests that the factory and inline scorer invoke the localizer per language."""
+
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import rag_eval.metrics.factory as factory_mod
+import webapp.services.inline_scorer as inline_mod
+
+
+def test_build_pipeline_localizes_when_zh(monkeypatch):
+    """build_metric_pipeline calls localize_pipeline_prompts with 'zh'."""
+    calls = []
+    monkeypatch.setattr(factory_mod, "localize_pipeline_prompts",
+                        lambda registry, language: calls.append(language))
+
+    from rag_eval.shared.models import DatasetConfig, Scenario
+    from rag_eval.settings import EvaluationSettings
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbedding
+
+    scenario = Scenario(
+        scenario_name="t", mode="offline",
+        dataset=DatasetConfig(path=Path("x.csv")),
+        judge_model="gpt-5", embedding_model="text-embedding-3-small",
+        metrics=["faithfulness"], output_dir=Path("out"),
+        judge_language="zh",
+    )
+    factory_mod.build_metric_pipeline(
+        scenario, EvaluationSettings(_env_file=None),
+        llm=MagicMock(spec=InstructorBaseRagasLLM),
+        embeddings=MagicMock(spec=BaseRagasEmbedding),
+    )
+    assert calls == ["zh"]
+
+
+def test_build_pipeline_falls_back_to_settings_default(monkeypatch):
+    """When scenario.judge_language is None, settings.ragas_judge_language is used."""
+    calls = []
+    monkeypatch.setattr(factory_mod, "localize_pipeline_prompts",
+                        lambda registry, language: calls.append(language))
+
+    from rag_eval.shared.models import DatasetConfig, Scenario
+    from rag_eval.settings import EvaluationSettings
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbedding
+
+    scenario = Scenario(
+        scenario_name="t", mode="offline",
+        dataset=DatasetConfig(path=Path("x.csv")),
+        judge_model="gpt-5", embedding_model="text-embedding-3-small",
+        metrics=["faithfulness"], output_dir=Path("out"),
+        judge_language=None,
+    )
+    settings = EvaluationSettings(_env_file=None)
+    settings.ragas_judge_language = "zh"
+    factory_mod.build_metric_pipeline(
+        scenario, settings,
+        llm=MagicMock(spec=InstructorBaseRagasLLM),
+        embeddings=MagicMock(spec=BaseRagasEmbedding),
+    )
+    assert calls == ["zh"]
+
+
+def test_build_pipeline_en_is_noop(monkeypatch):
+    """Default language 'en' still calls localize (which is a no-op inside)."""
+    calls = []
+    monkeypatch.setattr(factory_mod, "localize_pipeline_prompts",
+                        lambda registry, language: calls.append(language))
+
+    from rag_eval.shared.models import DatasetConfig, Scenario
+    from rag_eval.settings import EvaluationSettings
+    from ragas.llms.base import InstructorBaseRagasLLM
+    from ragas.embeddings.base import BaseRagasEmbedding
+
+    scenario = Scenario(
+        scenario_name="t", mode="offline",
+        dataset=DatasetConfig(path=Path("x.csv")),
+        judge_model="gpt-5", embedding_model="text-embedding-3-small",
+        metrics=["faithfulness"], output_dir=Path("out"),
+    )
+    factory_mod.build_metric_pipeline(
+        scenario, EvaluationSettings(_env_file=None),
+        llm=MagicMock(spec=InstructorBaseRagasLLM),
+        embeddings=MagicMock(spec=BaseRagasEmbedding),
+    )
+    assert calls == ["en"]
+
+
+def test_inline_score_threads_judge_language(monkeypatch):
+    """InlineScorer.score forwards judge_language to _build_metric_instances."""
+    seen: dict = {}
+
+    def fake_build_instances(metrics, llm, embeddings, judge_language="en"):
+        seen["lang"] = judge_language
+        return {}
+
+    monkeypatch.setattr(inline_mod, "_build_metric_instances", fake_build_instances)
+    monkeypatch.setattr(inline_mod.InlineScorer, "_get_models",
+                        lambda self, j, e, s: (object(), object()))
+
+    class _FakePipeline:
+        def __init__(self, *a, **k):
+            pass
+
+        async def score_sample(self, sample):
+            return _FakeResult()
+
+    class _FakeResult:
+        metrics: dict = {}
+
+    import asyncio
+    monkeypatch.setattr(inline_mod, "MetricPipeline", _FakePipeline)
+    monkeypatch.setattr(asyncio, "run", lambda coro: _FakeResult())
+
+    scorer = inline_mod.InlineScorer()
+    scorer.score(question="q", answer="a", contexts=[], ground_truth=None,
+                 metrics=["faithfulness"], judge_model="gpt-5",
+                 embedding_model="e", settings=MagicMock(), judge_language="zh")
+    assert seen.get("lang") == "zh"