From bd5658c3ac6d2f88703c9be8ac8199e25fcf761e Mon Sep 17 00:00:00 2001 From: wangwei Date: Wed, 1 Jul 2026 17:58:43 +0800 Subject: [PATCH] Wire judge-prompt localization into factory and inline scorer Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- rag_eval/metrics/factory.py | 7 +- tests/test_judge_language_wiring.py | 119 ++++++++++++++++++++++++++++ webapp/services/inline_scorer.py | 14 +++- 3 files changed, 135 insertions(+), 5 deletions(-) create mode 100644 tests/test_judge_language_wiring.py diff --git a/rag_eval/metrics/factory.py b/rag_eval/metrics/factory.py index 1697606..3aafb05 100644 --- a/rag_eval/metrics/factory.py +++ b/rag_eval/metrics/factory.py @@ -24,6 +24,7 @@ from ragas.metrics.collections import ( SemanticSimilarity, ) +from .judge_prompts import localize_pipeline_prompts from .pipeline import MetricPipeline @@ -133,7 +134,11 @@ def build_metric_pipeline( # Build the full registry once using the shared factory, then slice by requested metrics. registry = build_metric_registry(llm, embeddings) + selected = {name: registry[name] for name in scenario.metrics} + # Apply language-specific judge prompt localization (no-op for 'en'). + language = scenario.judge_language or settings.ragas_judge_language or "en" + localize_pipeline_prompts(selected, language) return MetricPipeline( - metrics={name: registry[name] for name in scenario.metrics}, + metrics=selected, metric_timeout_seconds=settings.ragas_metric_timeout_seconds, ) diff --git a/tests/test_judge_language_wiring.py b/tests/test_judge_language_wiring.py new file mode 100644 index 0000000..23987e3 --- /dev/null +++ b/tests/test_judge_language_wiring.py @@ -0,0 +1,119 @@ +"""Tests that the factory and inline scorer invoke the localizer per language.""" + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import rag_eval.metrics.factory as factory_mod +import webapp.services.inline_scorer as inline_mod + + +def test_build_pipeline_localizes_when_zh(monkeypatch): + """build_metric_pipeline calls localize_pipeline_prompts with 'zh'.""" + calls = [] + monkeypatch.setattr(factory_mod, "localize_pipeline_prompts", + lambda registry, language: calls.append(language)) + + from rag_eval.shared.models import DatasetConfig, Scenario + from rag_eval.settings import EvaluationSettings + from ragas.llms.base import InstructorBaseRagasLLM + from ragas.embeddings.base import BaseRagasEmbedding + + scenario = Scenario( + scenario_name="t", mode="offline", + dataset=DatasetConfig(path=Path("x.csv")), + judge_model="gpt-5", embedding_model="text-embedding-3-small", + metrics=["faithfulness"], output_dir=Path("out"), + judge_language="zh", + ) + factory_mod.build_metric_pipeline( + scenario, EvaluationSettings(_env_file=None), + llm=MagicMock(spec=InstructorBaseRagasLLM), + embeddings=MagicMock(spec=BaseRagasEmbedding), + ) + assert calls == ["zh"] + + +def test_build_pipeline_falls_back_to_settings_default(monkeypatch): + """When scenario.judge_language is None, settings.ragas_judge_language is used.""" + calls = [] + monkeypatch.setattr(factory_mod, "localize_pipeline_prompts", + lambda registry, language: calls.append(language)) + + from rag_eval.shared.models import DatasetConfig, Scenario + from rag_eval.settings import EvaluationSettings + from ragas.llms.base import InstructorBaseRagasLLM + from ragas.embeddings.base import BaseRagasEmbedding + + scenario = Scenario( + scenario_name="t", mode="offline", + dataset=DatasetConfig(path=Path("x.csv")), + judge_model="gpt-5", embedding_model="text-embedding-3-small", + metrics=["faithfulness"], output_dir=Path("out"), + judge_language=None, + ) + settings = EvaluationSettings(_env_file=None) + settings.ragas_judge_language = "zh" + factory_mod.build_metric_pipeline( + scenario, settings, + llm=MagicMock(spec=InstructorBaseRagasLLM), + embeddings=MagicMock(spec=BaseRagasEmbedding), + ) + assert calls == ["zh"] + + +def test_build_pipeline_en_is_noop(monkeypatch): + """Default language 'en' still calls localize (which is a no-op inside).""" + calls = [] + monkeypatch.setattr(factory_mod, "localize_pipeline_prompts", + lambda registry, language: calls.append(language)) + + from rag_eval.shared.models import DatasetConfig, Scenario + from rag_eval.settings import EvaluationSettings + from ragas.llms.base import InstructorBaseRagasLLM + from ragas.embeddings.base import BaseRagasEmbedding + + scenario = Scenario( + scenario_name="t", mode="offline", + dataset=DatasetConfig(path=Path("x.csv")), + judge_model="gpt-5", embedding_model="text-embedding-3-small", + metrics=["faithfulness"], output_dir=Path("out"), + ) + factory_mod.build_metric_pipeline( + scenario, EvaluationSettings(_env_file=None), + llm=MagicMock(spec=InstructorBaseRagasLLM), + embeddings=MagicMock(spec=BaseRagasEmbedding), + ) + assert calls == ["en"] + + +def test_inline_score_threads_judge_language(monkeypatch): + """InlineScorer.score forwards judge_language to _build_metric_instances.""" + seen: dict = {} + + def fake_build_instances(metrics, llm, embeddings, judge_language="en"): + seen["lang"] = judge_language + return {} + + monkeypatch.setattr(inline_mod, "_build_metric_instances", fake_build_instances) + monkeypatch.setattr(inline_mod.InlineScorer, "_get_models", + lambda self, j, e, s: (object(), object())) + + class _FakePipeline: + def __init__(self, *a, **k): + pass + + async def score_sample(self, sample): + return _FakeResult() + + class _FakeResult: + metrics: dict = {} + + import asyncio + monkeypatch.setattr(inline_mod, "MetricPipeline", _FakePipeline) + monkeypatch.setattr(asyncio, "run", lambda coro: _FakeResult()) + + scorer = inline_mod.InlineScorer() + scorer.score(question="q", answer="a", contexts=[], ground_truth=None, + metrics=["faithfulness"], judge_model="gpt-5", + embedding_model="e", settings=MagicMock(), judge_language="zh") + assert seen.get("lang") == "zh" diff --git a/webapp/services/inline_scorer.py b/webapp/services/inline_scorer.py index 79df55f..87021e1 100644 --- a/webapp/services/inline_scorer.py +++ b/webapp/services/inline_scorer.py @@ -14,6 +14,7 @@ from typing import Any from rag_eval.compat import ensure_ragas_import_compat from rag_eval.metrics.factory import build_metric_registry, build_models +from rag_eval.metrics.judge_prompts import localize_pipeline_prompts from rag_eval.metrics.pipeline import MetricPipeline from rag_eval.settings import EvaluationSettings from rag_eval.shared.models import NormalizedSample @@ -21,10 +22,14 @@ from rag_eval.shared.models import NormalizedSample ensure_ragas_import_compat() -def _build_metric_instances(metrics: list[str], llm: Any, embeddings: Any) -> dict[str, Any]: - """Instantiate only the RAGAS metric objects requested.""" +def _build_metric_instances( + metrics: list[str], llm: Any, embeddings: Any, judge_language: str = "en" +) -> dict[str, Any]: + """Instantiate only the RAGAS metric objects requested, localized if needed.""" registry = build_metric_registry(llm, embeddings) - return {name: registry[name] for name in metrics if name in registry} + selected = {name: registry[name] for name in metrics if name in registry} + localize_pipeline_prompts(selected, judge_language) + return selected class InlineScorer: @@ -69,10 +74,11 @@ class InlineScorer: judge_model: str, embedding_model: str, settings: EvaluationSettings, + judge_language: str = "en", ) -> dict[str, float | None]: """Score one sample synchronously and return {metric_name: score | None}.""" llm, embeddings = self._get_models(judge_model, embedding_model, settings) - metric_instances = _build_metric_instances(metrics, llm, embeddings) + metric_instances = _build_metric_instances(metrics, llm, embeddings, judge_language) pipeline = MetricPipeline( metrics=metric_instances,