86 lines
2.9 KiB
Python
86 lines
2.9 KiB
Python
"""Optimization advisor: rule-based diagnosis + LLM-powered recommendations."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from typing import Any
|
|
|
|
from rag_eval.reporting.artifacts import build_artifact_paths
|
|
from rag_eval.shared.models import EvaluationResult, Scenario
|
|
|
|
from .llm_analyzer import analyze
|
|
from .rules import Diagnosis, diagnose
|
|
from .writer import write_advice
|
|
|
|
logger = logging.getLogger("rag_eval.advisor")
|
|
|
|
__all__ = ["run_advisor", "Diagnosis", "diagnose"]
|
|
|
|
|
|
def run_advisor(
|
|
result: EvaluationResult,
|
|
scenario: Scenario,
|
|
llm: Any = None,
|
|
*,
|
|
settings: Any | None = None,
|
|
chat_client: Any | None = None,
|
|
) -> None:
|
|
"""Run the full optimization advisor pipeline after an evaluation completes.
|
|
|
|
Skips silently if scenario.optimization_advisor is False.
|
|
Never raises — failures are logged as warnings, not exceptions.
|
|
|
|
Args:
|
|
result: Completed EvaluationResult from Evaluator.evaluate().
|
|
scenario: The resolved Scenario (provides metrics, judge_model, output_dir).
|
|
llm: Deprecated/unused — kept for backward-compatible call sites. The
|
|
advisor now issues its own direct LLM call resolved from judge_model.
|
|
settings: Optional EvaluationSettings; defaults to EvaluationSettings().
|
|
chat_client: Optional pre-built chat client (used by tests to avoid network).
|
|
"""
|
|
if not scenario.optimization_advisor:
|
|
return
|
|
|
|
logger.info("[advisor] starting optimization analysis scenario=%s", scenario.scenario_name)
|
|
|
|
try:
|
|
if settings is None:
|
|
from rag_eval.settings import EvaluationSettings
|
|
settings = EvaluationSettings()
|
|
|
|
artifact_paths = build_artifact_paths(scenario.output_dir, result.run_id)
|
|
if artifact_paths.advice_md is None:
|
|
logger.warning("[advisor] advice_md path not set in RunArtifactPaths — skipping")
|
|
return
|
|
|
|
diagnoses = diagnose(result.score_rows, scenario.metrics)
|
|
logger.info("[advisor] rule diagnosis complete: %d metric(s) triggered", len(diagnoses))
|
|
|
|
if diagnoses:
|
|
llm_markdown = asyncio.run(
|
|
analyze(
|
|
diagnoses,
|
|
scenario.scenario_name,
|
|
scenario.judge_model,
|
|
settings,
|
|
chat_client=chat_client,
|
|
)
|
|
)
|
|
else:
|
|
llm_markdown = ""
|
|
|
|
write_advice(
|
|
diagnoses=diagnoses,
|
|
llm_markdown=llm_markdown,
|
|
advice_path=artifact_paths.advice_md,
|
|
scenario_name=scenario.scenario_name,
|
|
run_id=result.run_id,
|
|
judge_model=scenario.judge_model,
|
|
)
|
|
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"[advisor] advisor failed (%s: %s) — evaluation result is unaffected",
|
|
type(exc).__name__, exc,
|
|
)
|