wangwei
f5c2dce64a
- rag_eval/advisor/: new package with rules engine, LLM analyzer, writer - rules.py: 7-metric diagnostic rules (warning/critical thresholds, top-3 low samples) - llm_analyzer.py: Chinese optimization report via judge_model, graceful fallback - writer.py: writes optimization_advice.md + log summary - __init__.py: run_advisor() entry point (no-op when optimization_advisor=False) - Scenario.optimization_advisor: new bool field (default False) - ScenarioModel: same field added, loader.py透传 - RunArtifactPaths.advice_md: new path field - factory.py: build_models() now public; build_metric_pipeline() accepts pre-built llm/embeddings - runner.py: lifts llm, passes to pipeline and advisor; calls run_advisor() at end - siemens online YAML: optimization_advisor: true enabled - tests: 9 rules tests + 6 writer tests, all pass - docs: advisor section added to engine-flow.md and architecture.md Co-Authored-By: Claude <noreply@anthropic.com>
68 lines
2.2 KiB
Python
68 lines
2.2 KiB
Python
"""Optimization advisor: rule-based diagnosis + LLM-powered recommendations."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from typing import Any
|
|
|
|
from rag_eval.reporting.artifacts import build_artifact_paths
|
|
from rag_eval.shared.models import EvaluationResult, Scenario
|
|
|
|
from .llm_analyzer import analyze
|
|
from .rules import Diagnosis, diagnose
|
|
from .writer import write_advice
|
|
|
|
logger = logging.getLogger("rag_eval.advisor")
|
|
|
|
__all__ = ["run_advisor", "Diagnosis", "diagnose"]
|
|
|
|
|
|
def run_advisor(
|
|
result: EvaluationResult,
|
|
scenario: Scenario,
|
|
llm: Any,
|
|
) -> None:
|
|
"""Run the full optimization advisor pipeline after an evaluation completes.
|
|
|
|
Skips silently if scenario.optimization_advisor is False.
|
|
Never raises — failures are logged as warnings, not exceptions.
|
|
|
|
Args:
|
|
result: Completed EvaluationResult from Evaluator.evaluate().
|
|
scenario: The resolved Scenario (provides metrics, judge_model, output_dir).
|
|
llm: Pre-built RAGAS LLM instance (from build_models()) for LLM analysis.
|
|
"""
|
|
if not scenario.optimization_advisor:
|
|
return
|
|
|
|
logger.info("[advisor] starting optimization analysis scenario=%s", scenario.scenario_name)
|
|
|
|
try:
|
|
artifact_paths = build_artifact_paths(scenario.output_dir, result.run_id)
|
|
if artifact_paths.advice_md is None:
|
|
logger.warning("[advisor] advice_md path not set in RunArtifactPaths — skipping")
|
|
return
|
|
|
|
diagnoses = diagnose(result.score_rows, scenario.metrics)
|
|
logger.info("[advisor] rule diagnosis complete: %d metric(s) triggered", len(diagnoses))
|
|
|
|
if diagnoses:
|
|
llm_markdown = asyncio.run(analyze(diagnoses, llm, scenario.scenario_name))
|
|
else:
|
|
llm_markdown = ""
|
|
|
|
write_advice(
|
|
diagnoses=diagnoses,
|
|
llm_markdown=llm_markdown,
|
|
advice_path=artifact_paths.advice_md,
|
|
scenario_name=scenario.scenario_name,
|
|
run_id=result.run_id,
|
|
judge_model=scenario.judge_model,
|
|
)
|
|
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"[advisor] advisor failed (%s: %s) — evaluation result is unaffected",
|
|
type(exc).__name__, exc,
|
|
)
|