feat: make contexts optional in /api/score
When contexts is absent, metrics that require retrieved_contexts (faithfulness, context_recall, context_precision, noise_sensitivity) are automatically skipped and appear in skipped_metrics. Only answer_relevancy, factual_correctness, semantic_similarity remain computable without contexts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -384,6 +384,14 @@ _GT_DEPENDENT_METRICS: frozenset[str] = frozenset({
|
||||
"noise_sensitivity",
|
||||
})
|
||||
|
||||
# 需要 contexts 才能计算的指标集合
|
||||
_CONTEXT_DEPENDENT_METRICS: frozenset[str] = frozenset({
|
||||
"faithfulness",
|
||||
"context_recall",
|
||||
"context_precision",
|
||||
"noise_sensitivity",
|
||||
})
|
||||
|
||||
# 所有合法指标名称
|
||||
_VALID_METRICS: frozenset[str] = frozenset({
|
||||
"faithfulness",
|
||||
@@ -428,8 +436,9 @@ class ScoreRequest(BaseModel):
|
||||
|
||||
question: str = Field(description="问题文本。")
|
||||
answer: str = Field(description="待评分的回答。")
|
||||
contexts: str = Field(
|
||||
description="检索上下文字符串,多段之间用 context_separator 拼接。"
|
||||
contexts: str | None = Field(
|
||||
default=None,
|
||||
description="检索上下文字符串,多段之间用 context_separator 拼接。缺失时自动跳过依赖检索内容的指标(faithfulness、context_recall、context_precision、noise_sensitivity)。",
|
||||
)
|
||||
ground_truth: str | None = Field(
|
||||
default=None,
|
||||
@@ -467,15 +476,23 @@ class ScoreRequest(BaseModel):
|
||||
return value
|
||||
|
||||
def contexts_as_list(self) -> list[str]:
|
||||
"""Split the contexts string into a list of non-empty fragments."""
|
||||
"""Split the contexts string into a list of non-empty fragments.
|
||||
|
||||
Returns an empty list when contexts is None or blank.
|
||||
"""
|
||||
if not self.contexts:
|
||||
return []
|
||||
separator = self.context_separator or " |||| "
|
||||
return [part.strip() for part in self.contexts.split(separator) if part.strip()]
|
||||
|
||||
def effective_metrics(self) -> list[str]:
|
||||
"""Return metrics filtered to exclude GT-dependent ones when ground_truth is absent."""
|
||||
if self.ground_truth is not None:
|
||||
return list(self.metrics)
|
||||
return [metric_name for metric_name in self.metrics if metric_name not in _GT_DEPENDENT_METRICS]
|
||||
"""Return metrics filtered to exclude GT-dependent or context-dependent ones when inputs are absent."""
|
||||
result = list(self.metrics)
|
||||
if self.ground_truth is None:
|
||||
result = [m for m in result if m not in _GT_DEPENDENT_METRICS]
|
||||
if not self.contexts:
|
||||
result = [m for m in result if m not in _CONTEXT_DEPENDENT_METRICS]
|
||||
return result
|
||||
|
||||
|
||||
class ScoreResponse(BaseModel):
|
||||
|
||||
Reference in New Issue
Block a user