122 lines
4.4 KiB
Python
122 lines
4.4 KiB
Python
"""Tests for the judge-prompt localizer (no RAGAS dependency; uses fake prompts)."""
|
|
|
|
import json
|
|
|
|
from pydantic import BaseModel
|
|
|
|
from rag_eval.metrics import judge_prompts as jp
|
|
|
|
|
|
class _In(BaseModel):
|
|
question: str
|
|
|
|
|
|
class _Out(BaseModel):
|
|
statements: list[str]
|
|
|
|
|
|
class _FakePrompt:
|
|
"""Minimal stand-in for a RAGAS BasePrompt with overridable attributes."""
|
|
|
|
def __init__(self):
|
|
self.input_model = _In
|
|
self.output_model = _Out
|
|
self.instruction = "English instruction."
|
|
self.examples = [(_In(question="q"), _Out(statements=["s"]))]
|
|
self.language = "english"
|
|
|
|
|
|
class _FakeMetric:
|
|
def __init__(self):
|
|
self.prompt = _FakePrompt()
|
|
|
|
|
|
def _cache_dict(prompt):
|
|
"""Build a valid cache dict for the given fake prompt."""
|
|
return {
|
|
"metric": "context_recall",
|
|
"prompt_attr": "prompt",
|
|
"language": "chinese",
|
|
"ragas_version": "0.4.3",
|
|
"source_hash": jp.prompt_source_hash(prompt),
|
|
"instruction": "中文指令。",
|
|
"examples": [{"input": {"question": "问题"}, "output": {"statements": ["陈述"]}}],
|
|
}
|
|
|
|
|
|
def setup_function(_):
|
|
jp.reset_cache()
|
|
|
|
|
|
def test_apply_localized_prompt_overrides_instruction_and_examples():
|
|
"""apply_localized_prompt swaps instruction/examples and rebuilds example models."""
|
|
prompt = _FakePrompt()
|
|
jp.apply_localized_prompt(prompt, _cache_dict(prompt))
|
|
assert prompt.instruction == "中文指令。"
|
|
assert prompt.examples[0][0].question == "问题"
|
|
assert prompt.examples[0][1].statements == ["陈述"]
|
|
assert prompt.language == "chinese"
|
|
|
|
|
|
def test_localize_english_is_noop():
|
|
"""language='en' leaves the registry untouched."""
|
|
metric = _FakeMetric()
|
|
report = jp.localize_pipeline_prompts({"context_recall": metric}, "en")
|
|
assert report.applied == []
|
|
assert metric.prompt.instruction == "English instruction."
|
|
|
|
|
|
def test_localize_applies_from_cache_file(tmp_path, monkeypatch):
|
|
"""localize reads <root>/zh/context_recall__prompt.json and applies it."""
|
|
metric = _FakeMetric()
|
|
root = tmp_path / "configs" / "judge_prompts"
|
|
(root / "zh").mkdir(parents=True)
|
|
(root / "zh" / "context_recall__prompt.json").write_text(
|
|
json.dumps(_cache_dict(metric.prompt), ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
monkeypatch.setattr(jp, "CACHE_ROOT", root)
|
|
report = jp.localize_pipeline_prompts({"context_recall": metric}, "zh")
|
|
assert "context_recall.prompt" in report.applied
|
|
assert metric.prompt.instruction == "中文指令。"
|
|
|
|
|
|
def test_localize_missing_cache_keeps_english(tmp_path, monkeypatch):
|
|
"""A missing cache file degrades gracefully to the English prompt with a warning."""
|
|
metric = _FakeMetric()
|
|
monkeypatch.setattr(jp, "CACHE_ROOT", tmp_path / "empty")
|
|
report = jp.localize_pipeline_prompts({"context_recall": metric}, "zh")
|
|
assert metric.prompt.instruction == "English instruction."
|
|
assert "context_recall.prompt" in report.skipped
|
|
assert report.warnings
|
|
|
|
|
|
def test_localize_stale_hash_warns_but_applies(tmp_path, monkeypatch):
|
|
"""A source_hash mismatch still applies Chinese but records a stale warning."""
|
|
metric = _FakeMetric()
|
|
data = _cache_dict(metric.prompt)
|
|
data["source_hash"] = "deadbeef"
|
|
root = tmp_path / "configs" / "judge_prompts"
|
|
(root / "zh").mkdir(parents=True)
|
|
(root / "zh" / "context_recall__prompt.json").write_text(
|
|
json.dumps(data, ensure_ascii=False), encoding="utf-8"
|
|
)
|
|
monkeypatch.setattr(jp, "CACHE_ROOT", root)
|
|
report = jp.localize_pipeline_prompts({"context_recall": metric}, "zh")
|
|
assert metric.prompt.instruction == "中文指令。"
|
|
assert any("stale" in w for w in report.warnings)
|
|
|
|
|
|
def test_cache_memoized(tmp_path, monkeypatch):
|
|
"""A second localize call does not re-read the file (in-memory memo)."""
|
|
metric = _FakeMetric()
|
|
root = tmp_path / "configs" / "judge_prompts"
|
|
(root / "zh").mkdir(parents=True)
|
|
path = root / "zh" / "context_recall__prompt.json"
|
|
path.write_text(json.dumps(_cache_dict(metric.prompt), ensure_ascii=False), encoding="utf-8")
|
|
monkeypatch.setattr(jp, "CACHE_ROOT", root)
|
|
jp.localize_pipeline_prompts({"context_recall": _FakeMetric()}, "zh")
|
|
path.unlink() # delete file; memo should still serve the parsed data
|
|
metric2 = _FakeMetric()
|
|
report = jp.localize_pipeline_prompts({"context_recall": metric2}, "zh")
|
|
assert "context_recall.prompt" in report.applied
|