wangwei
f5c2dce64a
- rag_eval/advisor/: new package with rules engine, LLM analyzer, writer - rules.py: 7-metric diagnostic rules (warning/critical thresholds, top-3 low samples) - llm_analyzer.py: Chinese optimization report via judge_model, graceful fallback - writer.py: writes optimization_advice.md + log summary - __init__.py: run_advisor() entry point (no-op when optimization_advisor=False) - Scenario.optimization_advisor: new bool field (default False) - ScenarioModel: same field added, loader.py透传 - RunArtifactPaths.advice_md: new path field - factory.py: build_models() now public; build_metric_pipeline() accepts pre-built llm/embeddings - runner.py: lifts llm, passes to pipeline and advisor; calls run_advisor() at end - siemens online YAML: optimization_advisor: true enabled - tests: 9 rules tests + 6 writer tests, all pass - docs: advisor section added to engine-flow.md and architecture.md Co-Authored-By: Claude <noreply@anthropic.com>
29 lines
1.1 KiB
YAML
29 lines
1.1 KiB
YAML
scenario_name: siemens-pdf-question-bank-online
|
||
mode: online
|
||
dataset: ../../datasets/raw/generated/siemens-pdf-question-bank.csv
|
||
# judge_model: qwen3.5-flash
|
||
judge_model: deepseek-v4-flash
|
||
embedding_model: text-embedding-v3
|
||
optimization_advisor: true # 评测结束后自动生成优化建议报告
|
||
metrics:
|
||
- faithfulness
|
||
- answer_relevancy
|
||
- context_recall
|
||
- context_precision
|
||
# 已启用：鲁棒性 / 端到端指标（数据集已含 ground_truth）
|
||
- noise_sensitivity # 鲁棒性：对检索噪声的敏感度
|
||
- factual_correctness # 端到端：事实正确性（相对标准答案）
|
||
- semantic_similarity # 端到端：语义相似度（embedding，无 LLM 调用）
|
||
output_dir: ../../outputs/online/siemens-pdf-question-bank
|
||
runtime:
|
||
batch_size: 4
|
||
app_concurrency: 4
|
||
metric_concurrency: 4
|
||
max_samples: 50
|
||
app_adapter:
|
||
type: python
|
||
callable: apps.siemens_pdf_qa.adapter:run
|
||
static_kwargs:
|
||
source_chunks_path: ../../outputs/dataset-builds/siemens-pdf-question-bank/latest/source_chunks.jsonl
|
||
model: deepseek-v4-flash
|