siemens_ragas/webapp/services/report_builder.py

"""Aggregate a run's per-sample scores into the report payload for the UI.

All aggregation reads only the standard scores.csv produced by the reporting
layer, plus the metric list resolved by run_reader. The output mirrors the
report detail page: metric means, per-metric distribution histograms, grouped
means by difficulty / question_type, and the lowest-scoring samples for review.
"""

from __future__ import annotations

import math
from pathlib import Path

import pandas as pd

from rag_eval.metrics.weights import (
    compute_overall_weighted_score_mean,
    weighted_metric_means as _weighted_metric_means,
)
from webapp.services.run_reader import _read_weights_from_snapshot
from webapp.services.text_utils import parse_contexts
from webapp.models import (
    DistributionBin,
    GroupStat,
    ReportData,
    SampleHistoryEntry,
    SampleScore,
)
from webapp.services import advisor_comparison, question_history, run_reader


# Number of equal-width buckets used for metric score histograms.
DISTRIBUTION_BIN_COUNT = 5

# Metadata columns that we group samples by when present in the data.
GROUPING_FIELDS = ("difficulty", "question_type", "language")

# How many lowest-scoring samples to surface for manual review.
LOWEST_SAMPLE_COUNT = 10

# How many past evaluations of the same question to show in the history table.
HISTORY_LIMIT = 5

# Metrics whose lower raw value means stronger performance.
LOWER_IS_BETTER_METRICS = {"noise_sensitivity"}


def _round_or_none(value: float | None) -> float | None:
    """Round a float to four places, mapping NaN/None to None for clean JSON."""
    if value is None:
        return None
    if isinstance(value, float) and (math.isnan(value) or math.isinf(value)):
        return None
    return round(float(value), 4)


def _distribution(frame: pd.DataFrame, metric: str) -> list[DistributionBin]:
    """Bucket one metric's scores into fixed-width [0,1] histogram bins."""
    bins: list[DistributionBin] = []
    if metric not in frame.columns:
        return bins

    series = pd.to_numeric(frame[metric], errors="coerce").dropna()
    width = 1.0 / DISTRIBUTION_BIN_COUNT
    for index in range(DISTRIBUTION_BIN_COUNT):
        lower = index * width
        upper = (index + 1) * width
        # Include the right edge in the final bin so 1.0 is counted.
        if index == DISTRIBUTION_BIN_COUNT - 1:
            mask = (series >= lower) & (series <= upper)
        else:
            mask = (series >= lower) & (series < upper)
        bins.append(
            DistributionBin(
                label=f"{lower:.1f}–{upper:.1f}",
                lower=round(lower, 2),
                upper=round(upper, 2),
                count=int(mask.sum()),
            )
        )
    return bins


def _groupings(frame: pd.DataFrame, metrics: list[str]) -> dict[str, list[GroupStat]]:
    """Compute per-group metric means for each available grouping field."""
    groupings: dict[str, list[GroupStat]] = {}
    for field in GROUPING_FIELDS:
        if field not in frame.columns:
            continue
        # Skip fields that are entirely empty so the UI does not render noise.
        non_empty = frame[field].astype(str).str.strip().replace("nan", "")
        if non_empty.eq("").all():
            continue

        stats: list[GroupStat] = []
        for key, group in frame.groupby(frame[field].astype(str)):
            key_text = str(key).strip()
            if not key_text or key_text == "nan":
                continue
            means = {
                metric: _round_or_none(group[metric].mean(numeric_only=True))
                for metric in metrics
                if metric in group.columns
            }
            stats.append(GroupStat(key=key_text, count=int(len(group)), means=means))
        if stats:
            stats.sort(key=lambda item: item.key)
            groupings[field] = stats
    return groupings


def _sample_mean(row: pd.Series, metrics: list[str]) -> float | None:
    """Average a single sample's available metric scores for ranking."""
    values = [
        (1.0 - float(row[metric])) if metric in LOWER_IS_BETTER_METRICS else float(row[metric])
        for metric in metrics
        if metric in row and pd.notna(row[metric])
    ]
    if not values:
        return None
    return sum(values) / len(values)


def _cell_text(row: pd.Series, column: str) -> str:
    """Safely read a string cell, returning '' for missing or NaN values."""
    if column not in row or pd.isna(row[column]):
        return ""
    return str(row[column]).strip()


def _lowest_samples(
    frame: pd.DataFrame,
    metrics: list[str],
    history_index: dict[str, list[dict]] | None = None,
) -> list[SampleScore]:
    """Select and shape the lowest-scoring samples for the review table.

    When a history_index is supplied, each surfaced sample is annotated with the
    same question's scores from previous runs (newest first) for comparison.
    """
    if frame.empty:
        return []

    enriched: list[tuple[float, SampleScore]] = []
    for _, row in frame.iterrows():
        mean_score = _sample_mean(row, metrics)
        sample = SampleScore(
            sample_id=_cell_text(row, "sample_id") or "—",
            question=_cell_text(row, "question"),
            contexts=parse_contexts(row["contexts"]) if "contexts" in row else [],
            answer=_cell_text(row, "answer"),
            ground_truth=_cell_text(row, "ground_truth"),
            language=_cell_text(row, "language"),
            difficulty=_cell_text(row, "difficulty"),
            question_type=_cell_text(row, "question_type"),
            metrics={
                metric: _round_or_none(float(row[metric]))
                for metric in metrics
                if metric in row and pd.notna(row[metric])
            },
            mean_score=_round_or_none(mean_score),
            error=_cell_text(row, "error"),
        )
        # Samples without any score sort last (treated as worst for review).
        sort_key = mean_score if mean_score is not None else -1.0
        enriched.append((sort_key, sample))

    enriched.sort(key=lambda item: item[0])
    selected = [sample for _, sample in enriched[:LOWEST_SAMPLE_COUNT]]

    # Attach per-question history only for the surfaced samples (keeps lookups cheap).
    if history_index is not None:
        for sample in selected:
            if not sample.question:
                continue
            entries = question_history.lookup(
                history_index, sample.question, limit=HISTORY_LIMIT
            )
            sample.history = [SampleHistoryEntry(**entry) for entry in entries]

    return selected


def build_report(run_dir: Path, metrics: list[str]) -> ReportData:
    """Build the full aggregated report payload for one run directory."""
    frame = run_reader.read_scores_frame(run_dir)
    summary_markdown = run_reader.read_summary_markdown(run_dir)
    advice_markdown = run_reader.read_advice_markdown(run_dir)
    metric_weights, doc_weights = _read_weights_from_snapshot(run_dir)
    # Read once up front so both the empty-frame and full branches can surface it.
    metadata = run_reader._read_json(run_dir / "metadata.json")
    token_usage = metadata.get("token_usage") or {}

    if frame.empty or not metrics:
        return ReportData(
            metrics=metrics,
            metric_means={metric: None for metric in metrics},
            summary_markdown=summary_markdown,
            advice_markdown=advice_markdown,
            metric_weights=metric_weights,
            doc_weights=doc_weights,
            token_usage=token_usage,
            advisor_comparison=None,
        )

    score_rows_list = frame.to_dict(orient="records")

    # Use weighted metric means (degrades to arithmetic mean when weights are empty).
    w_means = _weighted_metric_means(score_rows_list, metrics, doc_weights)
    rounded_means = {metric: _round_or_none(value) for metric, value in w_means.items()}

    # 综合加权得分计算（已暂时禁用）
    # overall_ws = compute_overall_weighted_score_mean(
    #     score_rows_list, metric_weights, doc_weights
    # )
    overall_ws = None

    distributions = {
        metric: _distribution(frame, metric)
        for metric in metrics
        if metric in frame.columns
    }

    # Cross-run history: scores of the same question in *other* runs (Approach A —
    # on-demand global scan, excluding the run currently being viewed).
    current_run_id = str(metadata.get("run_id") or run_dir.name)
    history_index = question_history.build_question_history_index(
        exclude_run_id=current_run_id
    )

    scenario_name = str(metadata.get("scenario_name") or "")
    comparison = (
        advisor_comparison.build_advisor_comparison(run_dir, scenario_name, metrics)
        if scenario_name
        else None
    )

    return ReportData(
        metrics=metrics,
        metric_means=rounded_means,
        distributions=distributions,
        groupings=_groupings(frame, metrics),
        lowest_samples=_lowest_samples(frame, metrics, history_index),
        summary_markdown=summary_markdown,
        advice_markdown=advice_markdown,
        weighted_score_mean=_round_or_none(overall_ws),
        metric_weights=metric_weights,
        doc_weights=doc_weights,
        token_usage=token_usage,
        advisor_comparison=comparison,
    )