2026-06-15 17:00:52 +08:00
|
|
|
scenario_name: siemens-pdf-question-bank-online
|
|
|
|
|
mode: online
|
|
|
|
|
dataset: ../../datasets/raw/generated/siemens-pdf-question-bank.csv
|
|
|
|
|
judge_model: deepseek-v4-flash
|
|
|
|
|
embedding_model: text-embedding-v3
|
2026-07-01 18:15:53 +08:00
|
|
|
# 使用中文评判 prompt 以提升对中文语料的判定一致性（架构设计 §10.5）
|
|
|
|
|
judge_language: zh
|
2026-06-16 18:12:33 +08:00
|
|
|
optimization_advisor: true
|
2026-06-15 17:00:52 +08:00
|
|
|
metrics:
|
2026-06-16 18:12:33 +08:00
|
|
|
- faithfulness
|
|
|
|
|
- answer_relevancy
|
|
|
|
|
- context_recall
|
|
|
|
|
- context_precision
|
|
|
|
|
- noise_sensitivity
|
|
|
|
|
- factual_correctness
|
|
|
|
|
- semantic_similarity
|
2026-06-15 17:00:52 +08:00
|
|
|
output_dir: ../../outputs/online/siemens-pdf-question-bank
|
|
|
|
|
runtime:
|
2026-06-16 18:12:33 +08:00
|
|
|
batch_size: 3
|
|
|
|
|
app_concurrency: 3
|
|
|
|
|
metric_concurrency: 3
|
|
|
|
|
max_samples: 10
|
2026-06-15 17:00:52 +08:00
|
|
|
app_adapter:
|
|
|
|
|
type: python
|
|
|
|
|
callable: apps.siemens_pdf_qa.adapter:run
|
|
|
|
|
static_kwargs:
|
|
|
|
|
source_chunks_path: ../../outputs/dataset-builds/siemens-pdf-question-bank/latest/source_chunks.jsonl
|
2026-06-16 18:12:33 +08:00
|
|
|
model: glm-5
|
