siemens_ragas/tests/test_judge_language_wiring.py

"""Tests that the factory and inline scorer invoke the localizer per language."""

from pathlib import Path
from unittest.mock import MagicMock, patch

import rag_eval.metrics.factory as factory_mod
import webapp.services.inline_scorer as inline_mod


def test_build_pipeline_localizes_when_zh(monkeypatch):
    """build_metric_pipeline calls localize_pipeline_prompts with 'zh'."""
    calls = []
    monkeypatch.setattr(factory_mod, "localize_pipeline_prompts",
                        lambda registry, language: calls.append(language))

    from rag_eval.shared.models import DatasetConfig, Scenario
    from rag_eval.settings import EvaluationSettings
    from ragas.llms.base import InstructorBaseRagasLLM
    from ragas.embeddings.base import BaseRagasEmbedding

    scenario = Scenario(
        scenario_name="t", mode="offline",
        dataset=DatasetConfig(path=Path("x.csv")),
        judge_model="gpt-5", embedding_model="text-embedding-3-small",
        metrics=["faithfulness"], output_dir=Path("out"),
        judge_language="zh",
    )
    factory_mod.build_metric_pipeline(
        scenario, EvaluationSettings(_env_file=None),
        llm=MagicMock(spec=InstructorBaseRagasLLM),
        embeddings=MagicMock(spec=BaseRagasEmbedding),
    )
    assert calls == ["zh"]


def test_build_pipeline_falls_back_to_settings_default(monkeypatch):
    """When scenario.judge_language is None, settings.ragas_judge_language is used."""
    calls = []
    monkeypatch.setattr(factory_mod, "localize_pipeline_prompts",
                        lambda registry, language: calls.append(language))

    from rag_eval.shared.models import DatasetConfig, Scenario
    from rag_eval.settings import EvaluationSettings
    from ragas.llms.base import InstructorBaseRagasLLM
    from ragas.embeddings.base import BaseRagasEmbedding

    scenario = Scenario(
        scenario_name="t", mode="offline",
        dataset=DatasetConfig(path=Path("x.csv")),
        judge_model="gpt-5", embedding_model="text-embedding-3-small",
        metrics=["faithfulness"], output_dir=Path("out"),
        judge_language=None,
    )
    settings = EvaluationSettings(_env_file=None)
    settings.ragas_judge_language = "zh"
    factory_mod.build_metric_pipeline(
        scenario, settings,
        llm=MagicMock(spec=InstructorBaseRagasLLM),
        embeddings=MagicMock(spec=BaseRagasEmbedding),
    )
    assert calls == ["zh"]


def test_build_pipeline_en_is_noop(monkeypatch):
    """Default language 'en' still calls localize (which is a no-op inside)."""
    calls = []
    monkeypatch.setattr(factory_mod, "localize_pipeline_prompts",
                        lambda registry, language: calls.append(language))

    from rag_eval.shared.models import DatasetConfig, Scenario
    from rag_eval.settings import EvaluationSettings
    from ragas.llms.base import InstructorBaseRagasLLM
    from ragas.embeddings.base import BaseRagasEmbedding

    scenario = Scenario(
        scenario_name="t", mode="offline",
        dataset=DatasetConfig(path=Path("x.csv")),
        judge_model="gpt-5", embedding_model="text-embedding-3-small",
        metrics=["faithfulness"], output_dir=Path("out"),
    )
    factory_mod.build_metric_pipeline(
        scenario, EvaluationSettings(_env_file=None),
        llm=MagicMock(spec=InstructorBaseRagasLLM),
        embeddings=MagicMock(spec=BaseRagasEmbedding),
    )
    assert calls == ["en"]


def test_inline_score_threads_judge_language(monkeypatch):
    """InlineScorer.score forwards judge_language to _build_metric_instances."""
    seen: dict = {}

    def fake_build_instances(metrics, llm, embeddings, judge_language="en"):
        seen["lang"] = judge_language
        return {}

    monkeypatch.setattr(inline_mod, "_build_metric_instances", fake_build_instances)
    monkeypatch.setattr(inline_mod.InlineScorer, "_get_models",
                        lambda self, j, e, s: (object(), object()))

    class _FakePipeline:
        def __init__(self, *a, **k):
            pass

        async def score_sample(self, sample):
            return _FakeResult()

    class _FakeResult:
        metrics: dict = {}

    import asyncio
    monkeypatch.setattr(inline_mod, "MetricPipeline", _FakePipeline)
    monkeypatch.setattr(asyncio, "run", lambda coro: _FakeResult())

    scorer = inline_mod.InlineScorer()
    scorer.score(question="q", answer="a", contexts=[], ground_truth=None,
                 metrics=["faithfulness"], judge_model="gpt-5",
                 embedding_model="e", settings=MagicMock(), judge_language="zh")
    assert seen.get("lang") == "zh"