73 lines
2.6 KiB
Python
73 lines
2.6 KiB
Python
"""Tests for judge_language plumbing across settings, scenario schema, and loader."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from rag_eval.settings import EvaluationSettings
|
|
from rag_eval.config.loader import load_scenario
|
|
|
|
|
|
def test_settings_default_judge_language_is_en():
|
|
"""ragas_judge_language defaults to 'en' when the env var is absent."""
|
|
settings = EvaluationSettings(_env_file=None)
|
|
assert settings.ragas_judge_language == "en"
|
|
|
|
|
|
def _write_scenario(tmp_path: Path, extra: str) -> Path:
|
|
"""Write a minimal valid offline scenario YAML plus the given extra line(s)."""
|
|
dataset = tmp_path / "data.csv"
|
|
dataset.write_text("sample_id,question,answer,contexts,ground_truth\n", encoding="utf-8")
|
|
text = (
|
|
"scenario_name: t\n"
|
|
"mode: offline\n"
|
|
f"dataset: {dataset.name}\n"
|
|
"judge_model: gpt-5\n"
|
|
"embedding_model: text-embedding-3-small\n"
|
|
"metrics: [faithfulness]\n"
|
|
"output_dir: out\n"
|
|
f"{extra}"
|
|
)
|
|
path = tmp_path / "s.yaml"
|
|
path.write_text(text, encoding="utf-8")
|
|
return path
|
|
|
|
|
|
def test_scenario_loads_judge_language_zh(tmp_path):
|
|
"""A scenario may declare judge_language: zh and it lands on the dataclass."""
|
|
path = _write_scenario(tmp_path, "judge_language: zh\n")
|
|
scenario = load_scenario(path)
|
|
assert scenario.judge_language == "zh"
|
|
|
|
|
|
def test_scenario_defaults_judge_language_none(tmp_path):
|
|
"""Omitting judge_language leaves it None so the factory can apply the settings default."""
|
|
path = _write_scenario(tmp_path, "")
|
|
scenario = load_scenario(path)
|
|
assert scenario.judge_language is None
|
|
|
|
|
|
def test_scenario_rejects_invalid_judge_language(tmp_path):
|
|
"""An unsupported judge_language value is rejected at schema validation."""
|
|
path = _write_scenario(tmp_path, "judge_language: fr\n")
|
|
with pytest.raises(Exception):
|
|
load_scenario(path)
|
|
|
|
|
|
def test_score_request_judge_language_defaults_none():
|
|
"""ScoreRequest exposes an optional judge_language defaulting to None."""
|
|
from webapp.models import ScoreRequest
|
|
|
|
req = ScoreRequest(question="q", answer="a")
|
|
assert req.judge_language is None
|
|
req_zh = ScoreRequest(question="q", answer="a", judge_language="zh")
|
|
assert req_zh.judge_language == "zh"
|
|
|
|
|
|
def test_session_score_request_inherits_judge_language():
|
|
"""SessionScoreRequest inherits the judge_language field from ScoreRequest."""
|
|
from webapp.models import SessionScoreRequest
|
|
|
|
req = SessionScoreRequest(session_id="s1", question="q", answer="a", judge_language="zh")
|
|
assert req.judge_language == "zh"
|