Extract shared build_metric_registry factory (DRY)
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
+35
-14
@@ -57,6 +57,39 @@ def _resolve_openai_client_kwargs(
|
||||
return settings.openai_client_kwargs
|
||||
|
||||
|
||||
def resolve_openai_client_kwargs(
|
||||
judge_model: str,
|
||||
settings: EvaluationSettings,
|
||||
) -> dict[str, Any]:
|
||||
"""Public accessor for profile-aware AsyncOpenAI kwargs (matched by judge_model).
|
||||
|
||||
Exposed so other components (e.g. the optimization advisor's direct LLM call)
|
||||
can build a client that honors the same saved-profile/.env resolution used by
|
||||
the scoring pipeline, instead of duplicating the lookup logic.
|
||||
"""
|
||||
return _resolve_openai_client_kwargs(judge_model, settings)
|
||||
|
||||
|
||||
def build_metric_registry(llm: Any, embeddings: Any) -> dict[str, Any]:
|
||||
"""Instantiate the full set of supported RAGAS metrics keyed by canonical name.
|
||||
|
||||
Shared by the scenario pipeline, the inline scorer, and the prompt-cache
|
||||
bootstrap so the metric set is defined in exactly one place.
|
||||
"""
|
||||
return {
|
||||
"faithfulness": Faithfulness(llm=llm),
|
||||
"answer_relevancy": AnswerRelevancy(llm=llm, embeddings=embeddings),
|
||||
"context_recall": ContextRecall(llm=llm),
|
||||
"context_precision": ContextPrecision(llm=llm),
|
||||
# NoiseSensitivity mode='relevant': sensitivity to noise from relevant contexts.
|
||||
"noise_sensitivity": NoiseSensitivity(llm=llm),
|
||||
# FactualCorrectness mode='f1': balances claim precision and recall vs. ground truth.
|
||||
"factual_correctness": FactualCorrectness(llm=llm),
|
||||
# SemanticSimilarity: embedding cosine between answer and ground truth (no LLM call).
|
||||
"semantic_similarity": SemanticSimilarity(embeddings=embeddings),
|
||||
}
|
||||
|
||||
|
||||
def build_models(
|
||||
judge_model: str,
|
||||
embedding_model: str,
|
||||
@@ -98,20 +131,8 @@ def build_metric_pipeline(
|
||||
settings,
|
||||
)
|
||||
|
||||
# Build the full registry once, then slice it by configured metric names.
|
||||
registry: dict[str, Any] = {
|
||||
"faithfulness": Faithfulness(llm=llm),
|
||||
"answer_relevancy": AnswerRelevancy(llm=llm, embeddings=embeddings),
|
||||
"context_recall": ContextRecall(llm=llm),
|
||||
"context_precision": ContextPrecision(llm=llm),
|
||||
# Robustness / end-to-end metrics (架构设计 §10.2).
|
||||
# NoiseSensitivity mode='relevant': sensitivity to noise from relevant contexts.
|
||||
"noise_sensitivity": NoiseSensitivity(llm=llm),
|
||||
# FactualCorrectness mode='f1': balances claim precision and recall vs. ground truth.
|
||||
"factual_correctness": FactualCorrectness(llm=llm),
|
||||
# SemanticSimilarity: embedding cosine between answer and ground truth (no LLM call).
|
||||
"semantic_similarity": SemanticSimilarity(embeddings=embeddings),
|
||||
}
|
||||
# Build the full registry once using the shared factory, then slice by requested metrics.
|
||||
registry = build_metric_registry(llm, embeddings)
|
||||
return MetricPipeline(
|
||||
metrics={name: registry[name] for name in scenario.metrics},
|
||||
metric_timeout_seconds=settings.ragas_metric_timeout_seconds,
|
||||
|
||||
Reference in New Issue
Block a user