feat(advisor): add optimization advisor module
- rag_eval/advisor/: new package with rules engine, LLM analyzer, writer - rules.py: 7-metric diagnostic rules (warning/critical thresholds, top-3 low samples) - llm_analyzer.py: Chinese optimization report via judge_model, graceful fallback - writer.py: writes optimization_advice.md + log summary - __init__.py: run_advisor() entry point (no-op when optimization_advisor=False) - Scenario.optimization_advisor: new bool field (default False) - ScenarioModel: same field added, loader.py透传 - RunArtifactPaths.advice_md: new path field - factory.py: build_models() now public; build_metric_pipeline() accepts pre-built llm/embeddings - runner.py: lifts llm, passes to pipeline and advisor; calls run_advisor() at end - siemens online YAML: optimization_advisor: true enabled - tests: 9 rules tests + 6 writer tests, all pass - docs: advisor section added to engine-flow.md and architecture.md Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
67
rag_eval/advisor/__init__.py
Normal file
67
rag_eval/advisor/__init__.py
Normal file
@@ -0,0 +1,67 @@
|
||||
"""Optimization advisor: rule-based diagnosis + LLM-powered recommendations."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from rag_eval.reporting.artifacts import build_artifact_paths
|
||||
from rag_eval.shared.models import EvaluationResult, Scenario
|
||||
|
||||
from .llm_analyzer import analyze
|
||||
from .rules import Diagnosis, diagnose
|
||||
from .writer import write_advice
|
||||
|
||||
logger = logging.getLogger("rag_eval.advisor")
|
||||
|
||||
__all__ = ["run_advisor", "Diagnosis", "diagnose"]
|
||||
|
||||
|
||||
def run_advisor(
|
||||
result: EvaluationResult,
|
||||
scenario: Scenario,
|
||||
llm: Any,
|
||||
) -> None:
|
||||
"""Run the full optimization advisor pipeline after an evaluation completes.
|
||||
|
||||
Skips silently if scenario.optimization_advisor is False.
|
||||
Never raises — failures are logged as warnings, not exceptions.
|
||||
|
||||
Args:
|
||||
result: Completed EvaluationResult from Evaluator.evaluate().
|
||||
scenario: The resolved Scenario (provides metrics, judge_model, output_dir).
|
||||
llm: Pre-built RAGAS LLM instance (from build_models()) for LLM analysis.
|
||||
"""
|
||||
if not scenario.optimization_advisor:
|
||||
return
|
||||
|
||||
logger.info("[advisor] starting optimization analysis scenario=%s", scenario.scenario_name)
|
||||
|
||||
try:
|
||||
artifact_paths = build_artifact_paths(scenario.output_dir, result.run_id)
|
||||
if artifact_paths.advice_md is None:
|
||||
logger.warning("[advisor] advice_md path not set in RunArtifactPaths — skipping")
|
||||
return
|
||||
|
||||
diagnoses = diagnose(result.score_rows, scenario.metrics)
|
||||
logger.info("[advisor] rule diagnosis complete: %d metric(s) triggered", len(diagnoses))
|
||||
|
||||
if diagnoses:
|
||||
llm_markdown = asyncio.run(analyze(diagnoses, llm, scenario.scenario_name))
|
||||
else:
|
||||
llm_markdown = ""
|
||||
|
||||
write_advice(
|
||||
diagnoses=diagnoses,
|
||||
llm_markdown=llm_markdown,
|
||||
advice_path=artifact_paths.advice_md,
|
||||
scenario_name=scenario.scenario_name,
|
||||
run_id=result.run_id,
|
||||
judge_model=scenario.judge_model,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"[advisor] advisor failed (%s: %s) — evaluation result is unaffected",
|
||||
type(exc).__name__, exc,
|
||||
)
|
||||
Reference in New Issue
Block a user