Add Siemens CT document evaluation scenario (three-step pipeline)
- scenarios/siemens_build/siemens-pdf-build.yaml: dataset build for all 17 Siemens medical-imaging PDFs (aliyun_docmind parser, 10 questions/doc, failure_mode=skip, ~170 question total) - scenarios/offline/siemens-pdf-offline-smoke.yaml: offline evaluation using source chunks as contexts and ground_truth as answer (up to 30 samples) - scenarios/online/siemens-pdf-question-bank-online.yaml: online evaluation calling siemens_pdf_qa adapter, batch_size=4, up to 50 samples - apps/siemens_pdf_qa/adapter.py: Siemens-specific adapter with bilingual (zh/en) system prompt and strict evidence-grounding for CT domain - scripts/build_siemens_offline_smoke.py: helper to derive offline smoke CSV from completed dataset build artifacts (run after dataset build step) - docs/superpowers/specs/2026-06-15-siemens-scenario-design.md: design spec All three scenarios are automatically discovered by the web console. Co-Authored-By: Claude Sonnet 4 <noreply@anthropic.com>
This commit is contained in:
15
scenarios/offline/siemens-pdf-offline-smoke.yaml
Normal file
15
scenarios/offline/siemens-pdf-offline-smoke.yaml
Normal file
@@ -0,0 +1,15 @@
|
||||
scenario_name: siemens-pdf-offline-smoke
|
||||
mode: offline
|
||||
app_adapter: null
|
||||
dataset: ../../datasets/normalized/siemens_pdf_offline_smoke.csv
|
||||
judge_model: deepseek-v4-flash
|
||||
embedding_model: text-embedding-v3
|
||||
metrics:
|
||||
- faithfulness
|
||||
- answer_relevancy
|
||||
- context_recall
|
||||
- context_precision
|
||||
output_dir: ../../outputs/siemens-pdf-offline-smoke
|
||||
runtime:
|
||||
batch_size: 4
|
||||
max_samples: 30
|
||||
22
scenarios/online/siemens-pdf-question-bank-online.yaml
Normal file
22
scenarios/online/siemens-pdf-question-bank-online.yaml
Normal file
@@ -0,0 +1,22 @@
|
||||
scenario_name: siemens-pdf-question-bank-online
|
||||
mode: online
|
||||
dataset: ../../datasets/raw/generated/siemens-pdf-question-bank.csv
|
||||
judge_model: deepseek-v4-flash
|
||||
embedding_model: text-embedding-v3
|
||||
metrics:
|
||||
- faithfulness
|
||||
- answer_relevancy
|
||||
- context_recall
|
||||
- context_precision
|
||||
output_dir: ../../outputs/online/siemens-pdf-question-bank
|
||||
runtime:
|
||||
batch_size: 4
|
||||
app_concurrency: 4
|
||||
metric_concurrency: 4
|
||||
max_samples: 50
|
||||
app_adapter:
|
||||
type: python
|
||||
callable: apps.siemens_pdf_qa.adapter:run
|
||||
static_kwargs:
|
||||
source_chunks_path: ../../outputs/dataset-builds/siemens-pdf-question-bank/latest/source_chunks.jsonl
|
||||
model: deepseek-v4-flash
|
||||
17
scenarios/siemens_build/siemens-pdf-build.yaml
Normal file
17
scenarios/siemens_build/siemens-pdf-build.yaml
Normal file
@@ -0,0 +1,17 @@
|
||||
job_name: siemens-pdf-question-bank
|
||||
input:
|
||||
path: ../../datasets/siemens-pdfs
|
||||
glob: "*.pdf"
|
||||
parser:
|
||||
provider: aliyun_docmind
|
||||
failure_mode: skip
|
||||
generation:
|
||||
output_type: online_question_bank
|
||||
review_mode: draft_with_manual_review
|
||||
max_questions_per_document: 10
|
||||
max_source_chunks_per_question: 3
|
||||
output:
|
||||
dataset_path: ../../datasets/raw/generated/siemens-pdf-question-bank.csv
|
||||
artifact_dir: ../../outputs/dataset-builds/siemens-pdf-question-bank
|
||||
runtime:
|
||||
max_documents: 17
|
||||
Reference in New Issue
Block a user