AIRegulation/siemens_ragas
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
Guangfei.Zhao
2026-06-12 14:02:15 +08:00
commit 9cbdc1d95d
69 changed files with 9486 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

779
tests/test_dataset_build.py Normal file
View File

@@ -0,0 +1,779 @@
import csv
import json
import shutil
import unittest
from pathlib import Path
from unittest import mock
from pydantic import ValidationError
from rag_eval.dataset_builder.generator.question_generator import OpenAIQuestionGenerator
from rag_eval.dataset_builder.generator.validators import dedupe_samples, validate_draft_sample
from rag_eval.dataset_builder.models import DraftQuestionSample, ParsedDocument, SourceChunk
from rag_eval.dataset_builder.parser.aliyun_document_parser import AliyunDocumentParser
from rag_eval.dataset_builder.parser.aliyun_docmind_gateway import AliyunDocmindGateway
from rag_eval.dataset_builder.parser.aliyun_layout_normalizer import normalize_layouts
from rag_eval.dataset_builder.runner import load_dataset_build_job, run_dataset_build
from rag_eval.dataset_builder.schema import DatasetBuildConfigModel
from rag_eval.dataset_builder.sources import discover_pdf_files
from rag_eval.settings import EvaluationSettings
class FakeParser:
def __init__(self, documents_by_name, failures=None):
self.documents_by_name = documents_by_name
self.failures = failures or set()
def parse(self, pdf_path: Path):
if pdf_path.name in self.failures:
raise RuntimeError(f"parse failed for {pdf_path.name}")
return self.documents_by_name[pdf_path.name]
class FakeGenerator:
def __init__(self, outputs_by_doc_id):
self.outputs_by_doc_id = outputs_by_doc_id
def generate(self, document, *, max_questions, max_chunks_per_question, job_name):
return list(self.outputs_by_doc_id.get(document.doc_id, []))
class FakeGateway(AliyunDocmindGateway):
def __init__(self, settings, *, statuses=None, layouts=None):
super().__init__(settings)
self.statuses = list(statuses or [])
self.layouts = list(layouts or [])
def submit_parse_task(self, pdf_path: Path) -> str:
return "task-1"
def get_task_status(self, task_id: str):
if self.statuses:
return self.statuses.pop(0)
return {"status": "succeeded", "doc_id": "doc-1", "doc_name": "doc1.pdf"}
def fetch_layouts(self, task_id: str):
return list(self.layouts)
class DatasetBuildTests(unittest.TestCase):
def setUp(self) -> None:
root = Path("tests/.tmp").resolve()
root.mkdir(parents=True, exist_ok=True)
self.temp_dir = root / self._testMethodName
shutil.rmtree(self.temp_dir, ignore_errors=True)
self.temp_dir.mkdir(parents=True, exist_ok=True)
self.input_dir = self.temp_dir / "pdfs"
self.input_dir.mkdir(parents=True, exist_ok=True)
(self.input_dir / "doc1.pdf").write_bytes(b"%PDF-1.4 doc1")
(self.input_dir / "doc2.pdf").write_bytes(b"%PDF-1.4 doc2")
self.config_path = self.temp_dir / "dataset-build.yaml"
self.config_path.write_text(
"\n".join(
[
"job_name: sample-build",
"input:",
f" path: {self.input_dir.as_posix()}",
" glob: '*.pdf'",
"parser:",
" provider: aliyun_docmind",
" failure_mode: skip",
"generation:",
" output_type: online_question_bank",
" review_mode: draft_with_manual_review",
" max_questions_per_document: 3",
" max_source_chunks_per_question: 2",
"output:",
f" dataset_path: {(self.temp_dir / 'generated' / 'draft.csv').as_posix()}",
f" artifact_dir: {(self.temp_dir / 'outputs').as_posix()}",
"runtime:",
" max_documents: 2",
]
),
encoding="utf-8",
)
def tearDown(self) -> None:
shutil.rmtree(self.temp_dir, ignore_errors=True)
def _make_document(self, doc_id: str, doc_name: str) -> ParsedDocument:
chunk = SourceChunk(
chunk_id=f"{doc_id}-chunk-1",
doc_id=doc_id,
doc_name=doc_name,
text="Section content for review.",
page_start=1,
page_end=2,
section_path="Chapter 1 > Scope",
section_title="Scope",
source_layout_ids=["layout-1"],
)
return ParsedDocument(
doc_id=doc_id,
doc_name=doc_name,
raw_text=chunk.text,
structure_nodes=[],
semantic_blocks=[],
source_chunks=[chunk],
metadata={},
)
def test_load_dataset_build_job_resolves_paths_and_defaults(self) -> None:
settings = EvaluationSettings.model_construct(dataset_generator_model="env-model")
job = load_dataset_build_job(self.config_path, settings=settings)
self.assertEqual(job.job_name, "sample-build")
self.assertEqual(job.generation_model, "env-model")
self.assertTrue(job.dataset_path.is_absolute())
self.assertEqual(job.failure_mode, "skip")
def test_load_dataset_build_job_prefers_yaml_generation_model(self) -> None:
config_path = self.temp_dir / "dataset-build-with-model.yaml"
config_path.write_text(
self.config_path.read_text(encoding="utf-8").replace(
"generation:\n",
"generation:\n model: yaml-model\n",
),
encoding="utf-8",
)
settings = EvaluationSettings.model_construct(dataset_generator_model="env-model")
job = load_dataset_build_job(config_path, settings=settings)
self.assertEqual(job.generation_model, "yaml-model")
def test_load_dataset_build_job_uses_env_default_failure_mode(self) -> None:
config_path = self.temp_dir / "dataset-build-without-failure-mode.yaml"
config_path.write_text(
self.config_path.read_text(encoding="utf-8").replace(" failure_mode: skip\n", ""),
encoding="utf-8",
)
settings = EvaluationSettings.model_construct(
dataset_generator_model="env-model",
parser_failure_mode="skip",
)
job = load_dataset_build_job(config_path, settings=settings)
self.assertEqual(job.failure_mode, "skip")
def test_discover_pdf_files_rejects_missing_or_empty_input(self) -> None:
with self.assertRaises(FileNotFoundError):
discover_pdf_files(self.temp_dir / "missing")
empty_dir = self.temp_dir / "empty"
empty_dir.mkdir()
with self.assertRaises(ValueError):
discover_pdf_files(empty_dir)
def test_discover_pdf_files_accepts_single_pdf_file(self) -> None:
pdf_path = self.input_dir / "doc1.pdf"
files = discover_pdf_files(pdf_path)
self.assertEqual(files, [pdf_path])
def test_dataset_build_schema_rejects_missing_required_fields(self) -> None:
with self.assertRaises(ValidationError):
DatasetBuildConfigModel.model_validate(
{
"job_name": "sample-build",
"parser": {"provider": "aliyun_docmind"},
"generation": {
"output_type": "online_question_bank",
"review_mode": "draft_with_manual_review",
},
"output": {
"dataset_path": "draft.csv",
"artifact_dir": "outputs",
},
}
)
def test_dataset_build_schema_rejects_invalid_enums(self) -> None:
with self.assertRaises(ValidationError):
DatasetBuildConfigModel.model_validate(
{
"job_name": "sample-build",
"input": {"path": self.input_dir.as_posix()},
"parser": {"provider": "other-provider", "failure_mode": "ignore"},
"generation": {
"output_type": "other-output",
"review_mode": "auto_publish",
},
"output": {
"dataset_path": "draft.csv",
"artifact_dir": "outputs",
},
}
)
def test_normalize_layouts_applies_core_rules(self) -> None:
layouts = [
{"type": "toc", "text": "目录", "page": 1, "layout_id": "toc-1"},
{"type": "heading", "text": "第一章 总则", "page": 2, "layout_id": "h1", "level": 1},
{"type": "paragraph", "text": "第一段。", "page": 2, "layout_id": "p1"},
{"type": "caption", "text": "系统示意图", "page": 2, "layout_id": "c1"},
{
"type": "table",
"rows": [["字段", "说明"], ["名称", "项目名称"]],
"page": 3,
"layout_id": "t1",
},
]
document = normalize_layouts(doc_id="doc-1", doc_name="sample.pdf", layouts=layouts, max_chunk_chars=80, overlap_chars=10)
self.assertEqual(len(document.structure_nodes), 1)
self.assertEqual(document.structure_nodes[0].section_path, "第一章 总则")
self.assertEqual(len(document.semantic_blocks), 1)
self.assertIn("图注:", document.semantic_blocks[0].text)
self.assertIn("字段 | 说明", document.semantic_blocks[0].text)
self.assertEqual(document.source_chunks[0].page_start, 2)
self.assertEqual(document.source_chunks[0].page_end, 3)
def test_normalize_layouts_splits_long_text_into_multiple_chunks(self) -> None:
long_text = "A" * 220
layouts = [
{"type": "heading", "text": "Chapter 1", "page": 1, "layout_id": "h1", "level": 1},
{"type": "paragraph", "text": long_text, "page": 1, "layout_id": "p1"},
]
document = normalize_layouts(
doc_id="doc-1",
doc_name="sample.pdf",
layouts=layouts,
max_chunk_chars=100,
overlap_chars=20,
)
self.assertGreaterEqual(len(document.source_chunks), 3)
self.assertTrue(all(chunk.section_title == "Chapter 1" for chunk in document.source_chunks))
def test_validate_and_dedupe_generated_samples(self) -> None:
document = self._make_document("doc-1", "doc1.pdf")
valid = DraftQuestionSample(
sample_id="doc-1-q1",
question="这份文档的范围是什么？",
ground_truth="文档说明了适用范围。",
scenario="sample-build",
language="zh",
doc_id="doc-1",
doc_name="doc1.pdf",
section_path="Chapter 1 > Scope",
page_start=1,
page_end=2,
source_chunk_ids=["doc-1-chunk-1"],
question_type="summary",
difficulty="easy",
)
invalid = DraftQuestionSample(
sample_id="doc-1-q2",
question="",
ground_truth="",
scenario="sample-build",
language="zh",
doc_id="doc-2",
doc_name="doc1.pdf",
section_path="",
page_start=0,
page_end=0,
source_chunk_ids=["missing-chunk"],
question_type="invalid",
difficulty="invalid",
)
duplicate = DraftQuestionSample(
sample_id="doc-1-q3",
question=" 这份文档的范围是什么？ ",
ground_truth="文档说明了适用范围",
scenario="sample-build",
language="zh",
doc_id="doc-1",
doc_name="doc1.pdf",
section_path="Chapter 1 > Scope",
page_start=1,
page_end=2,
source_chunk_ids=["doc-1-chunk-1"],
question_type="summary",
difficulty="easy",
)
self.assertEqual(validate_draft_sample(valid, document=document), [])
self.assertTrue(validate_draft_sample(invalid, document=document))
self.assertEqual(len(dedupe_samples([valid, duplicate])), 1)
def test_validate_rejects_too_many_source_chunks(self) -> None:
document = self._make_document("doc-1", "doc1.pdf")
document.source_chunks.append(
SourceChunk(
chunk_id="doc-1-chunk-2",
doc_id="doc-1",
doc_name="doc1.pdf",
text="More content",
page_start=2,
page_end=3,
section_path="Chapter 1 > Scope",
section_title="Scope",
source_layout_ids=["layout-2"],
)
)
sample = DraftQuestionSample(
sample_id="doc-1-q1",
question="What is the scope?",
ground_truth="It defines scope.",
scenario="sample-build",
language="en",
doc_id="doc-1",
doc_name="doc1.pdf",
section_path="Chapter 1 > Scope",
page_start=1,
page_end=3,
source_chunk_ids=["doc-1-chunk-1", "doc-1-chunk-2"],
question_type="fact",
difficulty="easy",
)
errors = validate_draft_sample(
sample,
document=document,
max_source_chunks_per_question=1,
)
self.assertTrue(any("exceeds limit" in error for error in errors))
def test_dedupe_keeps_only_one_question_per_chunk_group(self) -> None:
sample_a = DraftQuestionSample(
sample_id="doc-1-q1",
question="What is the scope?",
ground_truth="It defines the scope.",
scenario="sample-build",
language="en",
doc_id="doc-1",
doc_name="doc1.pdf",
section_path="Chapter 1 > Scope",
page_start=1,
page_end=2,
source_chunk_ids=["doc-1-chunk-1"],
question_type="fact",
difficulty="easy",
)
sample_b = DraftQuestionSample(
sample_id="doc-1-q2",
question="How is the scope described?",
ground_truth="The scope is described in the first section.",
scenario="sample-build",
language="en",
doc_id="doc-1",
doc_name="doc1.pdf",
section_path="Chapter 1 > Scope",
page_start=1,
page_end=2,
source_chunk_ids=["doc-1-chunk-1"],
question_type="summary",
difficulty="medium",
)
self.assertEqual(len(dedupe_samples([sample_a, sample_b])), 1)
def test_aliyun_gateway_parse_success_failure_and_timeout(self) -> None:
settings = EvaluationSettings.model_construct(
aliyun_parse_poll_interval_seconds=1,
aliyun_parse_timeout_seconds=1,
)
pdf_path = self.input_dir / "doc1.pdf"
success_gateway = FakeGateway(
settings,
statuses=[{"status": "running"}, {"status": "succeeded", "doc_id": "doc-1", "doc_name": "doc1.pdf"}],
layouts=[{"type": "paragraph", "text": "hello", "page": 1, "layout_id": "p1"}],
)
with mock.patch("rag_eval.dataset_builder.parser.aliyun_docmind_gateway.time.sleep", return_value=None), mock.patch(
"rag_eval.dataset_builder.parser.aliyun_docmind_gateway.time.monotonic",
side_effect=[0.0, 0.1, 0.2],
):
payload = success_gateway.parse_document(pdf_path)
self.assertEqual(payload["doc_id"], "doc-1")
self.assertEqual(len(payload["layouts"]), 1)
failure_gateway = FakeGateway(settings, statuses=[{"status": "failed", "message": "bad file"}])
with self.assertRaises(RuntimeError):
failure_gateway.parse_document(pdf_path)
timeout_gateway = FakeGateway(settings, statuses=[{"status": "running"}, {"status": "running"}])
with mock.patch("rag_eval.dataset_builder.parser.aliyun_docmind_gateway.time.sleep", return_value=None), mock.patch(
"rag_eval.dataset_builder.parser.aliyun_docmind_gateway.time.monotonic",
side_effect=[0.0, 2.0],
):
with self.assertRaises(TimeoutError):
timeout_gateway.parse_document(pdf_path)
def test_aliyun_gateway_reports_missing_sdk(self) -> None:
settings = EvaluationSettings.model_construct(
aliyun_parse_poll_interval_seconds=1,
aliyun_parse_timeout_seconds=1,
)
gateway = AliyunDocmindGateway(settings)
with mock.patch("rag_eval.dataset_builder.parser.aliyun_docmind_gateway.DocmindClient", None), mock.patch(
"rag_eval.dataset_builder.parser.aliyun_docmind_gateway.docmind_models", None
), mock.patch("rag_eval.dataset_builder.parser.aliyun_docmind_gateway.openapi_models", None), mock.patch(
"rag_eval.dataset_builder.parser.aliyun_docmind_gateway.runtime_models", None
):
with self.assertRaises(ImportError):
gateway._load_sdk()
def test_document_parser_rejects_empty_layouts(self) -> None:
settings = EvaluationSettings.model_construct(
aliyun_parse_poll_interval_seconds=1,
aliyun_parse_timeout_seconds=1,
)
gateway = FakeGateway(
settings,
statuses=[{"status": "succeeded", "doc_id": "doc-1", "doc_name": "doc1.pdf"}],
layouts=[],
)
parser = AliyunDocumentParser(gateway)
with self.assertRaises(ValueError):
parser.parse(self.input_dir / "doc1.pdf")
def test_run_dataset_build_skip_mode_writes_all_artifacts(self) -> None:
doc1 = self._make_document("doc-1", "doc1.pdf")
parser = FakeParser(
{"doc1.pdf": doc1},
failures={"doc2.pdf"},
)
generator = FakeGenerator(
{
"doc-1": [
DraftQuestionSample(
sample_id="doc-1-q1",
question="What is the scope?",
ground_truth="It defines the scope.",
scenario="sample-build",
language="en",
doc_id="doc-1",
doc_name="doc1.pdf",
section_path="Chapter 1 > Scope",
page_start=1,
page_end=2,
source_chunk_ids=["doc-1-chunk-1"],
question_type="fact",
difficulty="easy",
)
]
}
)
result = run_dataset_build(
self.config_path,
settings=EvaluationSettings.model_construct(dataset_generator_model="stub-model"),
parser=parser,
generator=generator,
)
self.assertEqual(len(result.documents), 1)
self.assertEqual(len(result.parse_failures), 1)
self.assertEqual(len(result.draft_samples), 1)
self.assertTrue(result.artifact_paths.documents_jsonl.exists())
self.assertTrue(result.artifact_paths.semantic_blocks_jsonl.exists())
self.assertTrue(result.artifact_paths.source_chunks_jsonl.exists())
self.assertTrue(result.artifact_paths.dataset_draft_csv.exists())
self.assertTrue(result.artifact_paths.parse_failures_csv.exists())
self.assertTrue(result.artifact_paths.metadata_json.exists())
self.assertTrue(result.job.dataset_path.exists())
latest_dir = result.job.artifact_dir / "latest"
self.assertTrue((latest_dir / "source_chunks.jsonl").exists())
self.assertTrue((latest_dir / "dataset_draft.csv").exists())
self.assertTrue((latest_dir / "metadata.json").exists())
with result.artifact_paths.parse_failures_csv.open(encoding="utf-8") as handle:
rows = list(csv.DictReader(handle))
self.assertEqual(len(rows), 1)
self.assertIn("doc2.pdf", rows[0]["file_path"])
metadata = json.loads(result.artifact_paths.metadata_json.read_text(encoding="utf-8"))
self.assertEqual(metadata["stats"]["documents_processed"], 1)
self.assertEqual(metadata["stats"]["parse_failures"], 1)
latest_metadata = json.loads((latest_dir / "metadata.json").read_text(encoding="utf-8"))
self.assertEqual(latest_metadata["run_id"], result.run_id)
with result.artifact_paths.source_chunks_jsonl.open(encoding="utf-8") as handle:
run_chunks = handle.read()
with (latest_dir / "source_chunks.jsonl").open(encoding="utf-8") as handle:
latest_chunks = handle.read()
self.assertEqual(latest_chunks, run_chunks)
def test_run_dataset_build_single_pdf_input(self) -> None:
single_pdf_config = self.temp_dir / "single-pdf-build.yaml"
single_pdf_config.write_text(
self.config_path.read_text(encoding="utf-8").replace(
f" path: {self.input_dir.as_posix()}",
f" path: {(self.input_dir / 'doc1.pdf').as_posix()}",
),
encoding="utf-8",
)
parser = FakeParser({"doc1.pdf": self._make_document("doc-1", "doc1.pdf")})
generator = FakeGenerator(
{
"doc-1": [
DraftQuestionSample(
sample_id="doc-1-q1",
question="What is the scope?",
ground_truth="It defines the scope.",
scenario="sample-build",
language="en",
doc_id="doc-1",
doc_name="doc1.pdf",
section_path="Chapter 1 > Scope",
page_start=1,
page_end=2,
source_chunk_ids=["doc-1-chunk-1"],
question_type="fact",
difficulty="easy",
)
]
}
)
result = run_dataset_build(
single_pdf_config,
settings=EvaluationSettings.model_construct(dataset_generator_model="stub-model"),
parser=parser,
generator=generator,
)
self.assertEqual(len(result.documents), 1)
self.assertEqual(result.documents[0].doc_name, "doc1.pdf")
self.assertEqual(len(result.draft_samples), 1)
def test_run_dataset_build_caps_questions_per_document(self) -> None:
doc1 = self._make_document("doc-1", "doc1.pdf")
parser = FakeParser({"doc1.pdf": doc1, "doc2.pdf": self._make_document("doc-2", "doc2.pdf")})
generator = FakeGenerator(
{
"doc-1": [
DraftQuestionSample(
sample_id=f"doc-1-q{index}",
question=f"Question {index}?",
ground_truth=f"Answer {index}.",
scenario="sample-build",
language="en",
doc_id="doc-1",
doc_name="doc1.pdf",
section_path="Chapter 1 > Scope",
page_start=1,
page_end=2,
source_chunk_ids=[f"doc-1-chunk-{index}"],
question_type="fact",
difficulty="easy",
)
for index in range(1, 5)
]
}
)
# Rebuild the doc with enough chunk ids for validation to pass.
doc1.source_chunks = [
SourceChunk(
chunk_id=f"doc-1-chunk-{index}",
doc_id="doc-1",
doc_name="doc1.pdf",
text=f"Chunk {index}",
page_start=index,
page_end=index,
section_path="Chapter 1 > Scope",
section_title="Scope",
source_layout_ids=[f"layout-{index}"],
)
for index in range(1, 5)
]
result = run_dataset_build(
self.config_path,
settings=EvaluationSettings.model_construct(dataset_generator_model="stub-model"),
parser=parser,
generator=generator,
)
self.assertLessEqual(len([item for item in result.draft_samples if item.doc_id == "doc-1"]), 3)
def test_run_dataset_build_filters_questions_exceeding_chunk_limit(self) -> None:
doc1 = self._make_document("doc-1", "doc1.pdf")
doc1.source_chunks.append(
SourceChunk(
chunk_id="doc-1-chunk-2",
doc_id="doc-1",
doc_name="doc1.pdf",
text="Chunk 2",
page_start=2,
page_end=2,
section_path="Chapter 1 > Scope",
section_title="Scope",
source_layout_ids=["layout-2"],
)
)
parser = FakeParser({"doc1.pdf": doc1}, failures={"doc2.pdf"})
generator = FakeGenerator(
{
"doc-1": [
DraftQuestionSample(
sample_id="doc-1-q1",
question="Too many chunks?",
ground_truth="This cites two chunks.",
scenario="sample-build",
language="en",
doc_id="doc-1",
doc_name="doc1.pdf",
section_path="Chapter 1 > Scope",
page_start=1,
page_end=2,
source_chunk_ids=["doc-1-chunk-1", "doc-1-chunk-2"],
question_type="fact",
difficulty="easy",
)
]
}
)
strict_config = self.temp_dir / "dataset-build-strict.yaml"
strict_config.write_text(
self.config_path.read_text(encoding="utf-8").replace(
" max_source_chunks_per_question: 2",
" max_source_chunks_per_question: 1",
),
encoding="utf-8",
)
result = run_dataset_build(
strict_config,
settings=EvaluationSettings.model_construct(dataset_generator_model="stub-model"),
parser=parser,
generator=generator,
)
self.assertEqual(len(result.draft_samples), 0)
def test_run_dataset_build_fail_mode_raises(self) -> None:
fail_config = self.temp_dir / "dataset-build-fail.yaml"
fail_config.write_text(self.config_path.read_text(encoding="utf-8").replace("failure_mode: skip", "failure_mode: fail"), encoding="utf-8")
parser = FakeParser({}, failures={"doc1.pdf"})
generator = FakeGenerator({})
with self.assertRaises(RuntimeError):
run_dataset_build(
fail_config,
settings=EvaluationSettings.model_construct(dataset_generator_model="stub-model"),
parser=parser,
generator=generator,
)
class QuestionGeneratorTests(unittest.TestCase):
def _make_document(self) -> ParsedDocument:
return ParsedDocument(
doc_id="doc-1",
doc_name="doc1.pdf",
raw_text="source text",
structure_nodes=[],
semantic_blocks=[],
source_chunks=[
SourceChunk(
chunk_id="doc-1-chunk-1",
doc_id="doc-1",
doc_name="doc1.pdf",
text="Scope content",
page_start=1,
page_end=1,
section_path="Chapter 1 > Scope",
section_title="Scope",
source_layout_ids=["layout-1"],
),
SourceChunk(
chunk_id="doc-1-chunk-2",
doc_id="doc-1",
doc_name="doc1.pdf",
text="Procedure content",
page_start=2,
page_end=2,
section_path="Chapter 2 > Process",
section_title="Process",
source_layout_ids=["layout-2"],
),
],
metadata={},
)
def _make_fake_client(self, content: str):
class FakeResponse:
def __init__(self, payload: str):
self.choices = [type("Choice", (), {"message": type("Message", (), {"content": payload})()})()]
class FakeCompletions:
def __init__(self, payload: str):
self.payload = payload
def create(self, **kwargs):
return FakeResponse(self.payload)
return type(
"FakeClient",
(),
{"chat": type("Chat", (), {"completions": FakeCompletions(content)})()},
)()
def test_question_generator_builds_samples_from_json_response(self) -> None:
settings = EvaluationSettings.model_construct(openai_api_key="test-key")
content = json.dumps(
{
"samples": [
{
"question": "What is the scope?",
"ground_truth": "It defines the scope.",
"source_chunk_ids": ["doc-1-chunk-1"],
"question_type": "fact",
"difficulty": "easy",
},
{
"question": "Summarize the process.",
"ground_truth": "It explains the process.",
"source_chunk_ids": ["doc-1-chunk-2"],
"question_type": "summary",
"difficulty": "medium",
},
]
}
)
generator = OpenAIQuestionGenerator(
settings=settings,
model="stub-model",
client=self._make_fake_client(content),
)
samples = generator.generate(
self._make_document(),
max_questions=1,
max_chunks_per_question=2,
job_name="sample-build",
)
self.assertEqual(len(samples), 1)
self.assertEqual(samples[0].sample_id, "doc-1-q1")
self.assertEqual(samples[0].section_path, "Chapter 1 > Scope")
def test_question_generator_rejects_invalid_json(self) -> None:
settings = EvaluationSettings.model_construct(openai_api_key="test-key")
generator = OpenAIQuestionGenerator(
settings=settings,
model="stub-model",
client=self._make_fake_client("not-json"),
)
with self.assertRaises(ValueError):
generator.generate(
self._make_document(),
max_questions=1,
max_chunks_per_question=2,
job_name="sample-build",
)
def test_question_generator_rejects_non_list_samples(self) -> None:
settings = EvaluationSettings.model_construct(openai_api_key="test-key")
content = json.dumps({"samples": {"question": "bad-shape"}})
generator = OpenAIQuestionGenerator(
settings=settings,
model="stub-model",
client=self._make_fake_client(content),
)
with self.assertRaises(ValueError):
generator.generate(
self._make_document(),
max_questions=1,
max_chunks_per_question=2,
job_name="sample-build",
)
class MainCliParseTests(unittest.TestCase):
def test_cli_options_are_mutually_exclusive(self) -> None:
import main
with mock.patch("sys.argv", ["main.py", "--scenario", "a.yaml", "--dataset-build-config", "b.yaml"]):
with self.assertRaises(SystemExit):
main.parse_args()

310
tests/test_offline_eval.py Normal file
View File

@@ -0,0 +1,310 @@
import os
import unittest
from pathlib import Path
from unittest import mock
import pandas as pd
from pydantic_settings import SettingsConfigDict
from rag_eval.config.loader import load_scenario
from rag_eval.datasets.normalizers import normalize_records
from rag_eval.execution.evaluator import Evaluator
from rag_eval.metrics.pipeline import MetricPipeline
from rag_eval.reporting.summary import build_summary_markdown
from rag_eval.reporting.writers import write_run_artifacts
from rag_eval.settings import EvaluationSettings
from rag_eval.shared.models import EvaluationResult
class EnvOnlySettings(EvaluationSettings):
model_config = SettingsConfigDict(env_file=None, extra="ignore")
class FakeMetric:
def __init__(self, value: float):
self.value = value
async def ascore(self, **kwargs):
class Result:
def __init__(self, value: float):
self.value = value
return Result(self.value)
class SlowMetric:
async def ascore(self, **kwargs):
await __import__("asyncio").sleep(0.05)
return type("Result", (), {"value": 1.0})()
class OpenAIConfigTests(unittest.TestCase):
def test_openai_client_kwargs_without_base_url(self) -> None:
with mock.patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=True):
settings = EnvOnlySettings()
self.assertEqual(
settings.openai_client_kwargs,
{"api_key": "test-key", "base_url": "http://6.86.80.4:30080/v1", "timeout": 30.0},
)
def test_openai_client_kwargs_with_base_url(self) -> None:
with mock.patch.dict(
os.environ,
{
"OPENAI_API_KEY": "test-key",
"OPENAI_BASE_URL": "https://proxy.example/v1",
},
clear=True,
):
settings = EnvOnlySettings()
self.assertEqual(
settings.openai_client_kwargs,
{"api_key": "test-key", "base_url": "https://proxy.example/v1", "timeout": 30.0},
)
def test_settings_defaults(self) -> None:
with mock.patch.dict(os.environ, {}, clear=True):
settings = EnvOnlySettings()
self.assertEqual(settings.openai_base_url, "http://6.86.80.4:30080/v1")
self.assertEqual(settings.ragas_judge_model, "deepseek-v4-flash")
self.assertEqual(settings.ragas_embedding_model, "text-embedding-v3")
self.assertEqual(settings.openai_timeout_seconds, 30.0)
self.assertEqual(settings.ragas_metric_timeout_seconds, 45.0)
self.assertEqual(settings.batch_size, 8)
class ScenarioAndDatasetTests(unittest.TestCase):
def test_load_scenario_resolves_relative_paths(self) -> None:
scenario = load_scenario("scenarios/offline/sample-offline.yaml")
self.assertEqual(scenario.mode, "offline")
self.assertTrue(scenario.dataset.path.name.endswith(".csv"))
self.assertTrue(scenario.output_dir.name == "sample-offline-baseline")
def test_scenario_snapshot_serializes_path_static_kwargs(self) -> None:
scenario = load_scenario("scenarios/online/sample-pdf-question-bank-online.yaml")
snapshot = scenario.snapshot()
self.assertIsInstance(snapshot["app_adapter"]["static_kwargs"]["source_chunks_path"], str)
self.assertTrue(
snapshot["app_adapter"]["static_kwargs"]["source_chunks_path"].endswith("source_chunks.jsonl")
)
def test_load_sample_pdf_offline_smoke_scenario(self) -> None:
scenario = load_scenario("scenarios/offline/sample-pdf-offline-smoke.yaml")
self.assertEqual(scenario.mode, "offline")
self.assertEqual(scenario.dataset.path.name, "sample_pdf_offline_smoke.csv")
self.assertEqual(scenario.output_dir.name, "sample-pdf-offline-smoke")
def test_normalize_records_splits_valid_and_invalid(self) -> None:
records = [
{
"question": "Q1",
"contexts": '["C1"]',
"answer": "A1",
"ground_truth": "G1",
},
{
"question": "",
"contexts": "",
"answer": "",
"ground_truth": "",
},
]
valid, invalid = normalize_records(records)
self.assertEqual(len(valid), 1)
self.assertEqual(len(invalid), 1)
self.assertEqual(valid[0].contexts, ["C1"])
def test_normalize_sample_pdf_offline_smoke_row(self) -> None:
frame = pd.read_csv("datasets/normalized/sample_pdf_offline_smoke.csv")
valid, invalid = normalize_records(frame.to_dict(orient="records"))
self.assertEqual(len(invalid), 0)
self.assertEqual(len(valid), 3)
self.assertTrue(valid[0].answer)
self.assertTrue(valid[0].ground_truth)
self.assertTrue(valid[0].contexts)
class EvaluatorAndReportingTests(unittest.TestCase):
def test_metric_pipeline_scores_sample(self) -> None:
pipeline = MetricPipeline(
metrics={
"faithfulness": FakeMetric(0.1),
"answer_relevancy": FakeMetric(0.2),
"context_recall": FakeMetric(0.3),
"context_precision": FakeMetric(0.4),
}
)
valid, _ = normalize_records(
[
{
"question": "What is RAG?",
"contexts": ["RAG combines retrieval and generation."],
"answer": "RAG combines retrieval and generation.",
"ground_truth": "RAG combines retrieval and generation.",
}
]
)
score = __import__("asyncio").run(pipeline.score_sample(valid[0]))
self.assertEqual(score.metrics["faithfulness"], 0.1)
self.assertEqual(score.metrics["context_precision"], 0.4)
def test_metric_pipeline_captures_metric_timeout_without_aborting(self) -> None:
pipeline = MetricPipeline(
metrics={
"faithfulness": SlowMetric(),
"answer_relevancy": FakeMetric(0.2),
},
metric_timeout_seconds=0.01,
)
valid, _ = normalize_records(
[
{
"question": "What is RAG?",
"contexts": ["RAG combines retrieval and generation."],
"answer": "RAG combines retrieval and generation.",
"ground_truth": "RAG combines retrieval and generation.",
}
]
)
score = __import__("asyncio").run(pipeline.score_sample(valid[0]))
self.assertEqual(score.metrics["faithfulness"], 1.0)
self.assertEqual(score.metrics["answer_relevancy"], 0.2)
self.assertEqual(score.error, "")
def test_evaluator_and_reporting_write_run_assets(self) -> None:
temp_root = Path("tests/.tmp/run-assets")
temp_root.mkdir(parents=True, exist_ok=True)
for child in temp_root.iterdir():
if child.is_dir():
import shutil
shutil.rmtree(child)
else:
child.unlink()
output_root = temp_root
try:
scenario = load_scenario("scenarios/offline/sample-offline.yaml")
scenario.output_dir = output_root
pipeline = MetricPipeline(
metrics={
"faithfulness": FakeMetric(0.1),
"answer_relevancy": FakeMetric(0.2),
"context_recall": FakeMetric(0.3),
"context_precision": FakeMetric(0.4),
}
)
evaluator = Evaluator(scenario=scenario, metric_pipeline=pipeline)
result = evaluator.evaluate()
write_run_artifacts(result)
run_dir = output_root / result.run_id
self.assertTrue((run_dir / "scenario.snapshot.yaml").exists())
self.assertTrue((run_dir / "scores.csv").exists())
self.assertTrue((run_dir / "invalid.csv").exists())
self.assertTrue((run_dir / "summary.md").exists())
self.assertTrue((run_dir / "metadata.json").exists())
scores = pd.read_csv(run_dir / "scores.csv")
self.assertEqual(len(scores), 3)
self.assertIn("faithfulness", scores.columns)
finally:
import shutil
shutil.rmtree(temp_root, ignore_errors=True)
def test_summary_markdown_lists_all_scored_samples_and_errors(self) -> None:
scenario = load_scenario("scenarios/offline/sample-offline.yaml")
valid, invalid = normalize_records(
[
{
"sample_id": "sample-1",
"question": "Q1",
"contexts": ["C1"],
"answer": "A1",
"ground_truth": "G1",
},
{
"sample_id": "sample-2",
"question": "Q2",
"contexts": ["C2"],
"answer": "A2",
"ground_truth": "G2",
},
{
"sample_id": "sample-3",
"question": "Q3",
"contexts": ["C3"],
"answer": "A3",
"ground_truth": "G3",
},
{
"sample_id": "sample-4",
"question": "Q4",
"contexts": ["C4"],
"answer": "A4",
"ground_truth": "G4",
},
]
)
summary = build_summary_markdown(
EvaluationResult(
scenario=scenario,
run_id="test-run",
started_at="2026-06-10T00:00:00+00:00",
finished_at="2026-06-10T00:01:00+00:00",
valid_samples=valid,
invalid_samples=invalid,
score_rows=[
{
"sample_id": "sample-1",
"faithfulness": 1.0,
"answer_relevancy": 0.9,
"context_recall": 1.0,
"context_precision": 0.8,
"error": "",
},
{
"sample_id": "sample-2",
"faithfulness": 0.8,
"answer_relevancy": 0.7,
"context_recall": 0.9,
"context_precision": 0.6,
"error": "faithfulness: timeout",
},
{
"sample_id": "sample-3",
"faithfulness": 0.7,
"answer_relevancy": 0.6,
"context_recall": 0.8,
"context_precision": 0.5,
"error": "",
},
{
"sample_id": "sample-4",
"faithfulness": 0.6,
"answer_relevancy": 0.5,
"context_recall": 0.7,
"context_precision": 0.4,
"error": "context_precision: failed",
},
],
)
)
self.assertIn("## Per-sample Scores", summary)
self.assertIn("sample-1", summary)
self.assertIn("sample-2", summary)
self.assertIn("sample-3", summary)
self.assertIn("sample-4", summary)
self.assertIn("faithfulness", summary)
self.assertIn("answer_relevancy", summary)
self.assertIn("context_recall", summary)
self.assertIn("context_precision", summary)
self.assertIn("error", summary)
self.assertIn("faithfulness: timeout", summary)
self.assertIn("context_precision: failed", summary)
if __name__ == "__main__":
unittest.main()

351
tests/test_online_eval.py Normal file
View File

@@ -0,0 +1,351 @@
import shutil
import unittest
from pathlib import Path
from unittest import mock
import pandas as pd
from rag_eval.adapters.base import AppAdapter
from rag_eval.config.loader import load_scenario
from rag_eval.datasets.normalizers import normalize_records
from rag_eval.execution.evaluator import Evaluator
from rag_eval.metrics.pipeline import MetricPipeline
from rag_eval.shared.models import AppAdapterConfig, DatasetConfig, RuntimeConfig, Scenario
from apps.pdf_question_bank import adapter as pdf_question_bank_adapter
class FakeMetric:
def __init__(self, value: float):
self.value = value
async def ascore(self, **kwargs):
class Result:
def __init__(self, value: float):
self.value = value
return Result(self.value)
class FakeOnlineAdapter(AppAdapter):
async def run(self, question: str, **kwargs):
return {
"answer": f"answer for {question}",
"contexts": [f"context for {question}"],
"raw_response": {"question": question, "metadata": kwargs},
}
class ExplodingOnlineAdapter(AppAdapter):
async def run(self, question: str, **kwargs):
raise RuntimeError("boom")
class OnlineDatasetTests(unittest.TestCase):
def setUp(self) -> None:
root = Path("tests/.tmp").resolve()
root.mkdir(parents=True, exist_ok=True)
self.temp_dir = root / self._testMethodName
shutil.rmtree(self.temp_dir, ignore_errors=True)
self.temp_dir.mkdir(parents=True, exist_ok=True)
def tearDown(self) -> None:
shutil.rmtree(self.temp_dir, ignore_errors=True)
def test_online_records_allow_missing_answer_and_contexts_before_adapter(self) -> None:
records = [
{
"sample_id": "sample-1",
"question": "What is the policy scope?",
"ground_truth": "It covers all employees.",
"doc_id": "doc-1",
"source_chunk_ids": '["doc-1-chunk-1"]',
}
]
valid, invalid = normalize_records(records, mode="online")
self.assertEqual(len(valid), 1)
self.assertEqual(len(invalid), 0)
self.assertEqual(valid[0].answer, "")
self.assertEqual(valid[0].contexts, [])
self.assertEqual(valid[0].metadata["source_chunk_ids"], '["doc-1-chunk-1"]')
def test_online_evaluator_enriches_dataset_and_scores(self) -> None:
dataset_path = self.temp_dir / "online.csv"
pd.DataFrame(
[
{
"sample_id": "sample-1",
"question": "What is the policy scope?",
"ground_truth": "It covers all employees.",
"doc_id": "doc-1",
"section_path": "Policy > Scope",
"source_chunk_ids": '["doc-1-chunk-1"]',
}
]
).to_csv(dataset_path, index=False)
scenario = Scenario(
scenario_name="online-test",
mode="online",
dataset=DatasetConfig(path=dataset_path),
judge_model="judge-model",
embedding_model="embedding-model",
metrics=["faithfulness"],
output_dir=self.temp_dir / "outputs",
runtime=RuntimeConfig(batch_size=1),
app_adapter=AppAdapterConfig(type="python", callable="tests.fake:run"),
)
pipeline = MetricPipeline(metrics={"faithfulness": FakeMetric(0.8)})
evaluator = Evaluator(scenario=scenario, metric_pipeline=pipeline, app_adapter=FakeOnlineAdapter())
result = evaluator.evaluate()
self.assertEqual(len(result.valid_samples), 1)
self.assertEqual(len(result.invalid_samples), 0)
self.assertEqual(result.valid_samples[0].answer, "answer for What is the policy scope?")
self.assertEqual(result.valid_samples[0].contexts, ["context for What is the policy scope?"])
self.assertEqual(result.score_rows[0]["faithfulness"], 0.8)
def test_online_evaluator_captures_adapter_exception_type_in_invalid_rows(self) -> None:
dataset_path = self.temp_dir / "online.csv"
pd.DataFrame(
[
{
"sample_id": "sample-1",
"question": "What is the policy scope?",
"ground_truth": "It covers all employees.",
"doc_id": "doc-1",
"source_chunk_ids": '["doc-1-chunk-1"]',
}
]
).to_csv(dataset_path, index=False)
scenario = Scenario(
scenario_name="online-test",
mode="online",
dataset=DatasetConfig(path=dataset_path),
judge_model="judge-model",
embedding_model="embedding-model",
metrics=["faithfulness"],
output_dir=self.temp_dir / "outputs",
runtime=RuntimeConfig(batch_size=1),
app_adapter=AppAdapterConfig(type="python", callable="tests.fake:run"),
)
pipeline = MetricPipeline(metrics={"faithfulness": FakeMetric(0.8)})
evaluator = Evaluator(scenario=scenario, metric_pipeline=pipeline, app_adapter=ExplodingOnlineAdapter())
result = evaluator.evaluate()
self.assertEqual(len(result.valid_samples), 0)
self.assertEqual(len(result.invalid_samples), 1)
self.assertEqual(result.invalid_samples[0].error, "adapter failed [RuntimeError]: boom")
class FakeCompletionResponse:
def __init__(self, content: str):
self.choices = [type("Choice", (), {"message": type("Message", (), {"content": content})()})()]
class FakeCompletions:
def __init__(self, content: str):
self.content = content
self.calls: list[dict] = []
def create(self, **kwargs):
self.calls.append(kwargs)
return FakeCompletionResponse(self.content)
class PdfQuestionBankAdapterTests(unittest.TestCase):
def setUp(self) -> None:
root = Path("tests/.tmp").resolve()
root.mkdir(parents=True, exist_ok=True)
self.temp_dir = root / self._testMethodName
shutil.rmtree(self.temp_dir, ignore_errors=True)
self.temp_dir.mkdir(parents=True, exist_ok=True)
self.source_chunks_path = self.temp_dir / "source_chunks.jsonl"
self.source_chunks_path.write_text(
"\n".join(
[
'{"chunk_id":"doc-1-chunk-1","doc_id":"doc-1","doc_name":"doc1.pdf","text":"Scope covers all employees.","page_start":1,"page_end":1,"section_path":"Policy > Scope","section_title":"Scope","source_layout_ids":["layout-1"]}',
'{"chunk_id":"doc-1-chunk-2","doc_id":"doc-1","doc_name":"doc1.pdf","text":"Managers approve exceptions.","page_start":2,"page_end":2,"section_path":"Policy > Exceptions","section_title":"Exceptions","source_layout_ids":["layout-2"]}',
]
),
encoding="utf-8",
)
pdf_question_bank_adapter._CHUNK_CACHE.clear()
def tearDown(self) -> None:
shutil.rmtree(self.temp_dir, ignore_errors=True)
pdf_question_bank_adapter._CHUNK_CACHE.clear()
def test_adapter_loads_chunks_and_returns_resolved_contexts(self) -> None:
completions = FakeCompletions("It covers all employees.")
client = type(
"FakeClient",
(),
{"chat": type("Chat", (), {"completions": completions})()},
)()
result = pdf_question_bank_adapter.run(
question="What is the policy scope?",
source_chunks_path=str(self.source_chunks_path),
model="stub-model",
client=client,
source_chunk_ids='["doc-1-chunk-1"]',
doc_id="doc-1",
doc_name="doc1.pdf",
section_path="Policy > Scope",
)
self.assertEqual(result["answer"], "It covers all employees.")
self.assertEqual(result["contexts"], ["Scope covers all employees."])
self.assertEqual(result["raw_response"]["resolved_chunk_ids"], ["doc-1-chunk-1"])
self.assertEqual(result["raw_response"]["model"], "stub-model")
self.assertEqual(len(completions.calls), 1)
self.assertEqual(completions.calls[0]["model"], "stub-model")
self.assertEqual(completions.calls[0]["temperature"], 0)
self.assertIn("Evidence chunks:", completions.calls[0]["messages"][1]["content"])
def test_adapter_supports_multiple_chunk_ids(self) -> None:
completions = FakeCompletions("Combined answer.")
client = type(
"FakeClient",
(),
{"chat": type("Chat", (), {"completions": completions})()},
)()
result = pdf_question_bank_adapter.run(
question="What does the policy say?",
source_chunks_path=str(self.source_chunks_path),
model="stub-model",
client=client,
source_chunk_ids='["doc-1-chunk-1", "doc-1-chunk-2"]',
doc_id="doc-1",
doc_name="doc1.pdf",
)
self.assertEqual(
result["contexts"],
["Scope covers all employees.", "Managers approve exceptions."],
)
self.assertEqual(
result["raw_response"]["resolved_chunk_ids"],
["doc-1-chunk-1", "doc-1-chunk-2"],
)
def test_adapter_rejects_missing_source_chunk_ids(self) -> None:
with self.assertRaisesRegex(ValueError, "source_chunk_ids is required"):
pdf_question_bank_adapter.run(
question="What is the policy scope?",
source_chunks_path=str(self.source_chunks_path),
model="stub-model",
client=mock.Mock(),
)
def test_adapter_rejects_unknown_chunk_id(self) -> None:
with self.assertRaisesRegex(ValueError, "source_chunk_ids not found"):
pdf_question_bank_adapter.run(
question="What is the policy scope?",
source_chunks_path=str(self.source_chunks_path),
model="stub-model",
client=mock.Mock(),
source_chunk_ids='["missing-chunk"]',
)
def test_adapter_falls_back_to_latest_run_directory_when_latest_alias_is_missing(self) -> None:
artifact_root = self.temp_dir / "sample-pdf-question-bank"
run_dir = artifact_root / "2026-06-10T02-01-32.508056+00-00"
run_dir.mkdir(parents=True, exist_ok=True)
latest_path = artifact_root / "latest" / "source_chunks.jsonl"
run_chunks_path = run_dir / "source_chunks.jsonl"
run_chunks_path.write_text(self.source_chunks_path.read_text(encoding="utf-8"), encoding="utf-8")
completions = FakeCompletions("It covers all employees.")
client = type(
"FakeClient",
(),
{"chat": type("Chat", (), {"completions": completions})()},
)()
result = pdf_question_bank_adapter.run(
question="What is the policy scope?",
source_chunks_path=str(latest_path),
model="stub-model",
client=client,
source_chunk_ids='["doc-1-chunk-1"]',
doc_id="doc-1",
)
self.assertEqual(result["contexts"], ["Scope covers all employees."])
self.assertEqual(result["raw_response"]["resolved_chunk_ids"], ["doc-1-chunk-1"])
def test_online_evaluator_handles_dataset_build_rows_with_python_adapter(self) -> None:
dataset_path = self.temp_dir / "question_bank.csv"
pd.DataFrame(
[
{
"sample_id": "sample-1",
"question": "What is the policy scope?",
"ground_truth": "It covers all employees.",
"doc_id": "doc-1",
"doc_name": "doc1.pdf",
"section_path": "Policy > Scope",
"source_chunk_ids": '["doc-1-chunk-1"]',
}
]
).to_csv(dataset_path, index=False)
completions = FakeCompletions("It covers all employees.")
client = type(
"FakeClient",
(),
{"chat": type("Chat", (), {"completions": completions})()},
)()
scenario = Scenario(
scenario_name="online-question-bank-test",
mode="online",
dataset=DatasetConfig(path=dataset_path),
judge_model="judge-model",
embedding_model="embedding-model",
metrics=["faithfulness"],
output_dir=self.temp_dir / "outputs",
runtime=RuntimeConfig(batch_size=1),
app_adapter=AppAdapterConfig(
type="python",
callable="apps.pdf_question_bank.adapter:run",
static_kwargs={
"source_chunks_path": str(self.source_chunks_path),
"model": "stub-model",
"client": client,
},
),
)
pipeline = MetricPipeline(metrics={"faithfulness": FakeMetric(0.8)})
from rag_eval.adapters.python import PythonFunctionAdapter
evaluator = Evaluator(
scenario=scenario,
metric_pipeline=pipeline,
app_adapter=PythonFunctionAdapter(scenario.app_adapter),
)
result = evaluator.evaluate()
self.assertEqual(len(result.valid_samples), 1)
self.assertEqual(len(result.invalid_samples), 0)
self.assertEqual(result.valid_samples[0].answer, "It covers all employees.")
self.assertEqual(result.valid_samples[0].contexts, ["Scope covers all employees."])
self.assertEqual(result.score_rows[0]["faithfulness"], 0.8)
self.assertEqual(
result.valid_samples[0].metadata["raw_response"]["resolved_chunk_ids"],
["doc-1-chunk-1"],
)
def test_load_sample_pdf_online_scenario(self) -> None:
scenario = load_scenario("scenarios/online/sample-pdf-question-bank-online.yaml")
self.assertEqual(scenario.mode, "online")
self.assertEqual(scenario.dataset.path.name, "sample-pdf-question-bank.csv")
self.assertEqual(scenario.output_dir.name, "sample-pdf-question-bank")
self.assertEqual(scenario.runtime.max_samples, 45)
self.assertEqual(scenario.app_adapter.callable, "apps.pdf_question_bank.adapter:run")
self.assertTrue(
str(scenario.app_adapter.static_kwargs["source_chunks_path"]).endswith("source_chunks.jsonl")
)