From 4a646b6b9cbc436ad9a7e5bde5bec2c46d9b47ef Mon Sep 17 00:00:00 2001 From: wangwei Date: Wed, 1 Jul 2026 17:47:21 +0800 Subject: [PATCH] Add judge_language config plumbing (settings + scenario + ScoreRequest) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- rag_eval/config/loader.py | 1 + rag_eval/config/schema.py | 1 + rag_eval/settings.py | 1 + rag_eval/shared/models.py | 3 ++ tests/test_judge_language_config.py | 72 +++++++++++++++++++++++++++++ webapp/models.py | 17 +++++++ 6 files changed, 95 insertions(+) create mode 100644 tests/test_judge_language_config.py diff --git a/rag_eval/config/loader.py b/rag_eval/config/loader.py index b68f8bc..35ef0c4 100644 --- a/rag_eval/config/loader.py +++ b/rag_eval/config/loader.py @@ -64,6 +64,7 @@ def load_scenario(path: str | Path) -> Scenario: optimization_advisor=model.optimization_advisor, metric_weights=dict(model.metric_weights), doc_weights=dict(model.doc_weights), + judge_language=model.judge_language, ) # Run cross-field checks after all relative paths have been resolved. validate_scenario(scenario) diff --git a/rag_eval/config/schema.py b/rag_eval/config/schema.py index 3fac72a..913fd0f 100644 --- a/rag_eval/config/schema.py +++ b/rag_eval/config/schema.py @@ -57,6 +57,7 @@ class ScenarioModel(BaseModel): optimization_advisor: bool = False metric_weights: dict[str, float] = Field(default_factory=dict) doc_weights: dict[str, float] = Field(default_factory=dict) + judge_language: Literal["en", "zh"] | None = None @field_validator("metrics") @classmethod diff --git a/rag_eval/settings.py b/rag_eval/settings.py index d19609c..59b7ed3 100644 --- a/rag_eval/settings.py +++ b/rag_eval/settings.py @@ -22,6 +22,7 @@ class EvaluationSettings(BaseSettings): openai_api_key: str | None = Field(default=None, alias="OPENAI_API_KEY") openai_base_url: str = Field(default="http://6.86.80.4:30080/v1", alias="OPENAI_BASE_URL") ragas_judge_model: str = Field(default="gpt-5", alias="RAGAS_JUDGE_MODEL") + ragas_judge_language: str = Field(default="en", alias="RAGAS_JUDGE_LANGUAGE") ragas_embedding_model: str = Field( default="text-embedding-3-small", alias="RAGAS_EMBEDDING_MODEL", diff --git a/rag_eval/shared/models.py b/rag_eval/shared/models.py index 98c6fa3..677c2c8 100644 --- a/rag_eval/shared/models.py +++ b/rag_eval/shared/models.py @@ -79,6 +79,9 @@ class Scenario: optimization_advisor: bool = False metric_weights: dict[str, float] = field(default_factory=dict) doc_weights: dict[str, float] = field(default_factory=dict) + # Language used for judge prompts: 'en' (default, English) or 'zh' (Chinese). + # None means "use the global settings default". + judge_language: str | None = None def snapshot(self) -> dict[str, Any]: """Serialize the scenario into a reporting-friendly dictionary snapshot.""" diff --git a/tests/test_judge_language_config.py b/tests/test_judge_language_config.py new file mode 100644 index 0000000..4d9a6e1 --- /dev/null +++ b/tests/test_judge_language_config.py @@ -0,0 +1,72 @@ +"""Tests for judge_language plumbing across settings, scenario schema, and loader.""" + +from pathlib import Path + +import pytest + +from rag_eval.settings import EvaluationSettings +from rag_eval.config.loader import load_scenario + + +def test_settings_default_judge_language_is_en(): + """ragas_judge_language defaults to 'en' when the env var is absent.""" + settings = EvaluationSettings(_env_file=None) + assert settings.ragas_judge_language == "en" + + +def _write_scenario(tmp_path: Path, extra: str) -> Path: + """Write a minimal valid offline scenario YAML plus the given extra line(s).""" + dataset = tmp_path / "data.csv" + dataset.write_text("sample_id,question,answer,contexts,ground_truth\n", encoding="utf-8") + text = ( + "scenario_name: t\n" + "mode: offline\n" + f"dataset: {dataset.name}\n" + "judge_model: gpt-5\n" + "embedding_model: text-embedding-3-small\n" + "metrics: [faithfulness]\n" + "output_dir: out\n" + f"{extra}" + ) + path = tmp_path / "s.yaml" + path.write_text(text, encoding="utf-8") + return path + + +def test_scenario_loads_judge_language_zh(tmp_path): + """A scenario may declare judge_language: zh and it lands on the dataclass.""" + path = _write_scenario(tmp_path, "judge_language: zh\n") + scenario = load_scenario(path) + assert scenario.judge_language == "zh" + + +def test_scenario_defaults_judge_language_none(tmp_path): + """Omitting judge_language leaves it None so the factory can apply the settings default.""" + path = _write_scenario(tmp_path, "") + scenario = load_scenario(path) + assert scenario.judge_language is None + + +def test_scenario_rejects_invalid_judge_language(tmp_path): + """An unsupported judge_language value is rejected at schema validation.""" + path = _write_scenario(tmp_path, "judge_language: fr\n") + with pytest.raises(Exception): + load_scenario(path) + + +def test_score_request_judge_language_defaults_none(): + """ScoreRequest exposes an optional judge_language defaulting to None.""" + from webapp.models import ScoreRequest + + req = ScoreRequest(question="q", answer="a") + assert req.judge_language is None + req_zh = ScoreRequest(question="q", answer="a", judge_language="zh") + assert req_zh.judge_language == "zh" + + +def test_session_score_request_inherits_judge_language(): + """SessionScoreRequest inherits the judge_language field from ScoreRequest.""" + from webapp.models import SessionScoreRequest + + req = SessionScoreRequest(session_id="s1", question="q", answer="a", judge_language="zh") + assert req.judge_language == "zh" diff --git a/webapp/models.py b/webapp/models.py index 9e46be2..0455bf3 100644 --- a/webapp/models.py +++ b/webapp/models.py @@ -48,6 +48,15 @@ class DistributionBin(BaseModel): count: int +class SampleHistoryEntry(BaseModel): + """One past evaluation of the same question, for the history comparison table.""" + + run_id: str + scenario_name: str = "" + finished_at: str = "" + metrics: dict[str, float | None] = Field(default_factory=dict) + + class SampleScore(BaseModel): """Per-sample row used for the lowest-score review table.""" @@ -62,6 +71,10 @@ class SampleScore(BaseModel): metrics: dict[str, float | None] = Field(default_factory=dict) mean_score: float | None = None error: str = "" + history: list[SampleHistoryEntry] = Field( + default_factory=list, + description="同一问题在以往运行中的评分（按时间倒序），用于历史对比。", + ) class ReportData(BaseModel): @@ -460,6 +473,10 @@ class ScoreRequest(BaseModel): default=None, description="Embedding 模型名称；为 null 时使用 .env 中的 RAGAS_EMBEDDING_MODEL。", ) + judge_language: str | None = Field( + default=None, + description="评判 prompt 语言；'zh' 启用中文评判，为 null 时使用 RAGAS_JUDGE_LANGUAGE（默认 en）。", + ) @field_validator("metrics") @classmethod