"""Tests for the judge-prompt localizer (no RAGAS dependency; uses fake prompts).""" import json from pydantic import BaseModel from rag_eval.metrics import judge_prompts as jp class _In(BaseModel): question: str class _Out(BaseModel): statements: list[str] class _FakePrompt: """Minimal stand-in for a RAGAS BasePrompt with overridable attributes.""" def __init__(self): self.input_model = _In self.output_model = _Out self.instruction = "English instruction." self.examples = [(_In(question="q"), _Out(statements=["s"]))] self.language = "english" class _FakeMetric: def __init__(self): self.prompt = _FakePrompt() def _cache_dict(prompt): """Build a valid cache dict for the given fake prompt.""" return { "metric": "context_recall", "prompt_attr": "prompt", "language": "chinese", "ragas_version": "0.4.3", "source_hash": jp.prompt_source_hash(prompt), "instruction": "中文指令。", "examples": [{"input": {"question": "问题"}, "output": {"statements": ["陈述"]}}], } def setup_function(_): jp.reset_cache() def test_apply_localized_prompt_overrides_instruction_and_examples(): """apply_localized_prompt swaps instruction/examples and rebuilds example models.""" prompt = _FakePrompt() jp.apply_localized_prompt(prompt, _cache_dict(prompt)) assert prompt.instruction == "中文指令。" assert prompt.examples[0][0].question == "问题" assert prompt.examples[0][1].statements == ["陈述"] assert prompt.language == "chinese" def test_localize_english_is_noop(): """language='en' leaves the registry untouched.""" metric = _FakeMetric() report = jp.localize_pipeline_prompts({"context_recall": metric}, "en") assert report.applied == [] assert metric.prompt.instruction == "English instruction." def test_localize_applies_from_cache_file(tmp_path, monkeypatch): """localize reads /zh/context_recall__prompt.json and applies it.""" metric = _FakeMetric() root = tmp_path / "configs" / "judge_prompts" (root / "zh").mkdir(parents=True) (root / "zh" / "context_recall__prompt.json").write_text( json.dumps(_cache_dict(metric.prompt), ensure_ascii=False), encoding="utf-8" ) monkeypatch.setattr(jp, "CACHE_ROOT", root) report = jp.localize_pipeline_prompts({"context_recall": metric}, "zh") assert "context_recall.prompt" in report.applied assert metric.prompt.instruction == "中文指令。" def test_localize_missing_cache_keeps_english(tmp_path, monkeypatch): """A missing cache file degrades gracefully to the English prompt with a warning.""" metric = _FakeMetric() monkeypatch.setattr(jp, "CACHE_ROOT", tmp_path / "empty") report = jp.localize_pipeline_prompts({"context_recall": metric}, "zh") assert metric.prompt.instruction == "English instruction." assert "context_recall.prompt" in report.skipped assert report.warnings def test_localize_stale_hash_warns_but_applies(tmp_path, monkeypatch): """A source_hash mismatch still applies Chinese but records a stale warning.""" metric = _FakeMetric() data = _cache_dict(metric.prompt) data["source_hash"] = "deadbeef" root = tmp_path / "configs" / "judge_prompts" (root / "zh").mkdir(parents=True) (root / "zh" / "context_recall__prompt.json").write_text( json.dumps(data, ensure_ascii=False), encoding="utf-8" ) monkeypatch.setattr(jp, "CACHE_ROOT", root) report = jp.localize_pipeline_prompts({"context_recall": metric}, "zh") assert metric.prompt.instruction == "中文指令。" assert any("stale" in w for w in report.warnings) def test_cache_memoized(tmp_path, monkeypatch): """A second localize call does not re-read the file (in-memory memo).""" metric = _FakeMetric() root = tmp_path / "configs" / "judge_prompts" (root / "zh").mkdir(parents=True) path = root / "zh" / "context_recall__prompt.json" path.write_text(json.dumps(_cache_dict(metric.prompt), ensure_ascii=False), encoding="utf-8") monkeypatch.setattr(jp, "CACHE_ROOT", root) jp.localize_pipeline_prompts({"context_recall": _FakeMetric()}, "zh") path.unlink() # delete file; memo should still serve the parsed data metric2 = _FakeMetric() report = jp.localize_pipeline_prompts({"context_recall": metric2}, "zh") assert "context_recall.prompt" in report.applied