scenario_name: siemens-pdf-question-bank-online
mode: online
dataset: ../../datasets/raw/generated/siemens-pdf-question-bank.csv
judge_model: deepseek-v4-flash
embedding_model: text-embedding-v3
optimization_advisor: true
metrics:
- faithfulness
- answer_relevancy
- context_recall
- context_precision
- noise_sensitivity
- factual_correctness
- semantic_similarity
output_dir: ../../outputs/online/siemens-pdf-question-bank
runtime:
  batch_size: 3
  app_concurrency: 3
  metric_concurrency: 3
  max_samples: 10
app_adapter:
  type: python
  callable: apps.siemens_pdf_qa.adapter:run
  static_kwargs:
    source_chunks_path: ../../outputs/dataset-builds/siemens-pdf-question-bank/latest/source_chunks.jsonl
    model: glm-5