Extract shared build_metric_registry factory (DRY)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
wangwei
2026-07-01 17:52:55 +08:00
co-authored by Copilot
parent 4a646b6b9c
commit 2bb804b059
8 changed files with 1053 additions and 34 deletions
+35 -14
View File
@@ -57,6 +57,39 @@ def _resolve_openai_client_kwargs(
return settings.openai_client_kwargs
def resolve_openai_client_kwargs(
judge_model: str,
settings: EvaluationSettings,
) -> dict[str, Any]:
"""Public accessor for profile-aware AsyncOpenAI kwargs (matched by judge_model).
Exposed so other components (e.g. the optimization advisor's direct LLM call)
can build a client that honors the same saved-profile/.env resolution used by
the scoring pipeline, instead of duplicating the lookup logic.
"""
return _resolve_openai_client_kwargs(judge_model, settings)
def build_metric_registry(llm: Any, embeddings: Any) -> dict[str, Any]:
"""Instantiate the full set of supported RAGAS metrics keyed by canonical name.
Shared by the scenario pipeline, the inline scorer, and the prompt-cache
bootstrap so the metric set is defined in exactly one place.
"""
return {
"faithfulness": Faithfulness(llm=llm),
"answer_relevancy": AnswerRelevancy(llm=llm, embeddings=embeddings),
"context_recall": ContextRecall(llm=llm),
"context_precision": ContextPrecision(llm=llm),
# NoiseSensitivity mode='relevant': sensitivity to noise from relevant contexts.
"noise_sensitivity": NoiseSensitivity(llm=llm),
# FactualCorrectness mode='f1': balances claim precision and recall vs. ground truth.
"factual_correctness": FactualCorrectness(llm=llm),
# SemanticSimilarity: embedding cosine between answer and ground truth (no LLM call).
"semantic_similarity": SemanticSimilarity(embeddings=embeddings),
}
def build_models(
judge_model: str,
embedding_model: str,
@@ -98,20 +131,8 @@ def build_metric_pipeline(
settings,
)
# Build the full registry once, then slice it by configured metric names.
registry: dict[str, Any] = {
"faithfulness": Faithfulness(llm=llm),
"answer_relevancy": AnswerRelevancy(llm=llm, embeddings=embeddings),
"context_recall": ContextRecall(llm=llm),
"context_precision": ContextPrecision(llm=llm),
# Robustness / end-to-end metrics (架构设计 §10.2).
# NoiseSensitivity mode='relevant': sensitivity to noise from relevant contexts.
"noise_sensitivity": NoiseSensitivity(llm=llm),
# FactualCorrectness mode='f1': balances claim precision and recall vs. ground truth.
"factual_correctness": FactualCorrectness(llm=llm),
# SemanticSimilarity: embedding cosine between answer and ground truth (no LLM call).
"semantic_similarity": SemanticSimilarity(embeddings=embeddings),
}
# Build the full registry once using the shared factory, then slice by requested metrics.
registry = build_metric_registry(llm, embeddings)
return MetricPipeline(
metrics={name: registry[name] for name in scenario.metrics},
metric_timeout_seconds=settings.ragas_metric_timeout_seconds,