"""Optimization advisor: rule-based diagnosis + LLM-powered recommendations.""" from __future__ import annotations import asyncio import logging from typing import Any from rag_eval.reporting.artifacts import build_artifact_paths from rag_eval.shared.models import EvaluationResult, Scenario from .llm_analyzer import analyze from .rules import Diagnosis, diagnose from .writer import write_advice logger = logging.getLogger("rag_eval.advisor") __all__ = ["run_advisor", "Diagnosis", "diagnose"] def run_advisor( result: EvaluationResult, scenario: Scenario, llm: Any = None, *, settings: Any | None = None, chat_client: Any | None = None, ) -> None: """Run the full optimization advisor pipeline after an evaluation completes. Skips silently if scenario.optimization_advisor is False. Never raises — failures are logged as warnings, not exceptions. Args: result: Completed EvaluationResult from Evaluator.evaluate(). scenario: The resolved Scenario (provides metrics, judge_model, output_dir). llm: Deprecated/unused — kept for backward-compatible call sites. The advisor now issues its own direct LLM call resolved from judge_model. settings: Optional EvaluationSettings; defaults to EvaluationSettings(). chat_client: Optional pre-built chat client (used by tests to avoid network). """ if not scenario.optimization_advisor: return logger.info("[advisor] starting optimization analysis scenario=%s", scenario.scenario_name) try: if settings is None: from rag_eval.settings import EvaluationSettings settings = EvaluationSettings() artifact_paths = build_artifact_paths(scenario.output_dir, result.run_id) if artifact_paths.advice_md is None: logger.warning("[advisor] advice_md path not set in RunArtifactPaths — skipping") return diagnoses = diagnose(result.score_rows, scenario.metrics) logger.info("[advisor] rule diagnosis complete: %d metric(s) triggered", len(diagnoses)) if diagnoses: llm_markdown = asyncio.run( analyze( diagnoses, scenario.scenario_name, scenario.judge_model, settings, chat_client=chat_client, ) ) else: llm_markdown = "" write_advice( diagnoses=diagnoses, llm_markdown=llm_markdown, advice_path=artifact_paths.advice_md, scenario_name=scenario.scenario_name, run_id=result.run_id, judge_model=scenario.judge_model, ) except Exception as exc: logger.warning( "[advisor] advisor failed (%s: %s) — evaluation result is unaffected", type(exc).__name__, exc, )