feat(session-async): add /api/score/session_async with incremental session report aggregation
- New POST /api/score/session_async endpoint: same session_id calls append to one shared report
- New GET /api/score/sessions/{session_id}: returns call_count, metric_means, all job records
- New GET /api/score/session/jobs/{job_id}: individual call status
- SessionScoreJobManager: deterministic run_id from session_id, per-session mutex for CSV append, advisor regenerated on every call
- SessionScoreRequest (extends ScoreRequest + session_id), SessionScoreJobResponse, SessionStatus models added
- 24 new tests, all passing
chore(weighted-score): comment out 综合加权得分 display and computation
- report.js: hide 综合加权得分 card in report detail page
- score_jobs.js: hide 综合 chip in async job list
- report_builder.py: overall_ws=None (computation disabled)
- summary.py: weighted_score summary line disabled
- evaluator.py: weighted_score/sample_weight columns no longer written to scores.csv
- score.py /api/score: weighted_score always returns null
- score_job_manager.py + session_score_manager.py: weighted=None
- Updated 3 tests to match new behaviour (6 pre-existing failures unchanged)
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -241,7 +241,8 @@ class TestScoreEndpoint:
|
||||
})
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["weighted_score"] is not None
|
||||
# 综合加权得分已暂时禁用，始终返回 null
|
||||
assert data["weighted_score"] is None
|
||||
|
||||
def test_missing_required_fields_returns_422(self, client):
|
||||
resp = client.post("/api/score", json={"question": "q"})
|
||||
|
||||
299
tests/webapp/test_session_score_jobs_api.py
Normal file
299
tests/webapp/test_session_score_jobs_api.py
Normal file
@@ -0,0 +1,299 @@
|
||||
"""Tests for session-grouped async scoring API and SessionScoreJobManager."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.fixture()
|
||||
def tmp_manager(tmp_path):
|
||||
"""Isolated SessionScoreJobManager backed by tmp dirs (no real LLM calls)."""
|
||||
from webapp.services.session_score_manager import SessionScoreJobManager
|
||||
return SessionScoreJobManager(
|
||||
output_dir=tmp_path / "score-session",
|
||||
index_dir=tmp_path / "score-session-jobs",
|
||||
max_workers=2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def client(tmp_path, monkeypatch):
|
||||
"""TestClient with fresh SessionScoreJobManager backed by tmp dirs."""
|
||||
import webapp.services.session_score_manager as mgr_mod
|
||||
from webapp.services.session_score_manager import SessionScoreJobManager
|
||||
|
||||
fresh_mgr = SessionScoreJobManager(
|
||||
output_dir=tmp_path / "score-session",
|
||||
index_dir=tmp_path / "score-session-jobs",
|
||||
max_workers=2,
|
||||
)
|
||||
monkeypatch.setattr(mgr_mod, "session_score_manager", fresh_mgr)
|
||||
|
||||
import webapp.api.session_score_jobs as api_mod
|
||||
monkeypatch.setattr(api_mod, "session_score_manager", fresh_mgr)
|
||||
|
||||
from webapp.server import create_app
|
||||
return pytest.importorskip("fastapi.testclient").TestClient(create_app())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests for SessionScoreJobManager
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSessionRunId:
|
||||
def test_same_session_always_same_run_id(self, tmp_manager):
|
||||
assert tmp_manager.session_run_id("abc") == tmp_manager.session_run_id("abc")
|
||||
|
||||
def test_different_sessions_different_run_ids(self, tmp_manager):
|
||||
assert tmp_manager.session_run_id("session-A") != tmp_manager.session_run_id("session-B")
|
||||
|
||||
def test_run_id_prefixed_with_session(self, tmp_manager):
|
||||
assert tmp_manager.session_run_id("test123").startswith("session-")
|
||||
|
||||
def test_special_chars_sanitized(self, tmp_manager):
|
||||
run_id = tmp_manager.session_run_id("user@dify:flow/001")
|
||||
assert "/" not in run_id
|
||||
assert "@" not in run_id
|
||||
assert ":" not in run_id
|
||||
|
||||
|
||||
class TestSubmit:
|
||||
def test_submit_returns_job_status_and_run_id(self, tmp_manager):
|
||||
with patch.object(tmp_manager._executor, "submit"):
|
||||
status, run_id = tmp_manager.submit("session-1", _mock_request())
|
||||
assert status.job_id
|
||||
assert status.status == "queued"
|
||||
assert run_id == tmp_manager.session_run_id("session-1")
|
||||
|
||||
def test_submit_adds_job_to_session(self, tmp_manager):
|
||||
with patch.object(tmp_manager._executor, "submit"):
|
||||
status, _ = tmp_manager.submit("session-1", _mock_request())
|
||||
session = tmp_manager.get_session("session-1")
|
||||
assert session is not None
|
||||
assert any(j.job_id == status.job_id for j in session.jobs)
|
||||
|
||||
def test_multiple_submits_same_session_accumulate(self, tmp_manager):
|
||||
with patch.object(tmp_manager._executor, "submit"):
|
||||
tmp_manager.submit("session-X", _mock_request())
|
||||
tmp_manager.submit("session-X", _mock_request())
|
||||
tmp_manager.submit("session-X", _mock_request())
|
||||
session = tmp_manager.get_session("session-X")
|
||||
assert session.call_count == 3
|
||||
|
||||
def test_get_unknown_job_returns_none(self, tmp_manager):
|
||||
assert tmp_manager.get_job("does-not-exist") is None
|
||||
|
||||
def test_get_unknown_session_returns_none(self, tmp_manager):
|
||||
assert tmp_manager.get_session("no-such-session") is None
|
||||
|
||||
|
||||
class TestSessionIndexPersistence:
|
||||
def test_session_index_survives_restart(self, tmp_path):
|
||||
"""Jobs and session mappings loaded from disk on new manager instance."""
|
||||
from webapp.services.session_score_manager import SessionScoreJobManager
|
||||
|
||||
mgr1 = SessionScoreJobManager(
|
||||
output_dir=tmp_path / "score-session",
|
||||
index_dir=tmp_path / "score-session-jobs",
|
||||
)
|
||||
with patch.object(mgr1._executor, "submit"):
|
||||
mgr1.submit("persist-session", _mock_request())
|
||||
mgr1.submit("persist-session", _mock_request())
|
||||
|
||||
# New manager instance loads from disk
|
||||
mgr2 = SessionScoreJobManager(
|
||||
output_dir=tmp_path / "score-session",
|
||||
index_dir=tmp_path / "score-session-jobs",
|
||||
)
|
||||
session = mgr2.get_session("persist-session")
|
||||
assert session is not None
|
||||
assert session.call_count == 2
|
||||
|
||||
def test_job_index_file_created_on_submit(self, tmp_path):
|
||||
from webapp.services.session_score_manager import SessionScoreJobManager
|
||||
mgr = SessionScoreJobManager(
|
||||
output_dir=tmp_path / "score-session",
|
||||
index_dir=tmp_path / "score-session-jobs",
|
||||
)
|
||||
with patch.object(mgr._executor, "submit"):
|
||||
status, _ = mgr.submit("file-test", _mock_request())
|
||||
index_file = tmp_path / "score-session-jobs" / f"{status.job_id}.json"
|
||||
assert index_file.is_file()
|
||||
data = json.loads(index_file.read_text())
|
||||
assert data["job_id"] == status.job_id
|
||||
|
||||
|
||||
class TestAppendBehaviour:
|
||||
"""Test the CSV append / read-all logic in _append_and_regenerate via _read_score_rows."""
|
||||
|
||||
def test_read_score_rows_returns_empty_for_missing_csv(self, tmp_manager, tmp_path):
|
||||
rows = tmp_manager._read_score_rows(tmp_path / "nonexistent")
|
||||
assert rows == []
|
||||
|
||||
def test_read_score_rows_reads_existing_csv(self, tmp_manager, tmp_path):
|
||||
run_dir = tmp_path / "run1"
|
||||
run_dir.mkdir()
|
||||
df = pd.DataFrame([{"sample_id": "s1", "answer_relevancy": 0.9}])
|
||||
df.to_csv(run_dir / "scores.csv", index=False)
|
||||
rows = tmp_manager._read_score_rows(run_dir)
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["sample_id"] == "s1"
|
||||
|
||||
def test_metric_means_computed_from_csv(self, tmp_manager, tmp_path):
|
||||
run_dir = tmp_path / "run2"
|
||||
run_dir.mkdir()
|
||||
df = pd.DataFrame([
|
||||
{"sample_id": "s1", "answer_relevancy": 0.8},
|
||||
{"sample_id": "s2", "answer_relevancy": 0.6},
|
||||
])
|
||||
df.to_csv(run_dir / "scores.csv", index=False)
|
||||
means = tmp_manager._read_metric_means(run_dir)
|
||||
assert means["answer_relevancy"] == pytest.approx(0.7, abs=1e-4)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API endpoint tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSessionAsyncEndpoints:
|
||||
def test_submit_returns_202_with_session_fields(self, client):
|
||||
with patch("webapp.services.session_score_manager.SessionScoreJobManager._run"):
|
||||
resp = client.post("/api/score/session_async", json={
|
||||
"session_id": "test-session-001",
|
||||
"question": "What is CT?",
|
||||
"answer": "CT is computed tomography.",
|
||||
"metrics": ["answer_relevancy"],
|
||||
})
|
||||
assert resp.status_code == 202
|
||||
data = resp.json()
|
||||
assert data["session_id"] == "test-session-001"
|
||||
assert "job_id" in data
|
||||
assert "run_id" in data
|
||||
assert data["status"] == "queued"
|
||||
assert data["call_count"] >= 1
|
||||
|
||||
def test_run_id_deterministic_for_session(self, client):
|
||||
with patch("webapp.services.session_score_manager.SessionScoreJobManager._run"):
|
||||
r1 = client.post("/api/score/session_async", json={
|
||||
"session_id": "det-session",
|
||||
"question": "Q1",
|
||||
"answer": "A1",
|
||||
"metrics": ["answer_relevancy"],
|
||||
})
|
||||
r2 = client.post("/api/score/session_async", json={
|
||||
"session_id": "det-session",
|
||||
"question": "Q2",
|
||||
"answer": "A2",
|
||||
"metrics": ["answer_relevancy"],
|
||||
})
|
||||
assert r1.json()["run_id"] == r2.json()["run_id"]
|
||||
|
||||
def test_different_sessions_different_run_ids(self, client):
|
||||
with patch("webapp.services.session_score_manager.SessionScoreJobManager._run"):
|
||||
r1 = client.post("/api/score/session_async", json={
|
||||
"session_id": "session-A",
|
||||
"question": "Q",
|
||||
"answer": "A",
|
||||
"metrics": ["answer_relevancy"],
|
||||
})
|
||||
r2 = client.post("/api/score/session_async", json={
|
||||
"session_id": "session-B",
|
||||
"question": "Q",
|
||||
"answer": "A",
|
||||
"metrics": ["answer_relevancy"],
|
||||
})
|
||||
assert r1.json()["run_id"] != r2.json()["run_id"]
|
||||
|
||||
def test_call_count_increments_per_session(self, client):
|
||||
with patch("webapp.services.session_score_manager.SessionScoreJobManager._run"):
|
||||
for _ in range(3):
|
||||
client.post("/api/score/session_async", json={
|
||||
"session_id": "count-session",
|
||||
"question": "Q",
|
||||
"answer": "A",
|
||||
"metrics": ["answer_relevancy"],
|
||||
})
|
||||
time.sleep(0.05)
|
||||
resp = client.get("/api/score/sessions/count-session")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["call_count"] == 3
|
||||
|
||||
def test_get_session_returns_jobs_list(self, client):
|
||||
with patch("webapp.services.session_score_manager.SessionScoreJobManager._run"):
|
||||
client.post("/api/score/session_async", json={
|
||||
"session_id": "list-session",
|
||||
"question": "Q",
|
||||
"answer": "A",
|
||||
"metrics": ["answer_relevancy"],
|
||||
})
|
||||
time.sleep(0.05)
|
||||
resp = client.get("/api/score/sessions/list-session")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert len(data["jobs"]) == 1
|
||||
|
||||
def test_get_unknown_session_returns_404(self, client):
|
||||
resp = client.get("/api/score/sessions/no-such-session-xyz")
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_get_session_job_by_id(self, client):
|
||||
with patch("webapp.services.session_score_manager.SessionScoreJobManager._run"):
|
||||
resp = client.post("/api/score/session_async", json={
|
||||
"session_id": "job-lookup-session",
|
||||
"question": "Q",
|
||||
"answer": "A",
|
||||
"metrics": ["answer_relevancy"],
|
||||
})
|
||||
job_id = resp.json()["job_id"]
|
||||
time.sleep(0.05)
|
||||
get_resp = client.get(f"/api/score/session/jobs/{job_id}")
|
||||
assert get_resp.status_code == 200
|
||||
assert get_resp.json()["job_id"] == job_id
|
||||
|
||||
def test_get_unknown_job_returns_404(self, client):
|
||||
resp = client.get("/api/score/session/jobs/nonexistent-job-id")
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_missing_session_id_returns_422(self, client):
|
||||
resp = client.post("/api/score/session_async", json={
|
||||
"question": "Q",
|
||||
"answer": "A",
|
||||
"metrics": ["answer_relevancy"],
|
||||
})
|
||||
assert resp.status_code == 422
|
||||
|
||||
def test_list_sessions_endpoint(self, client):
|
||||
with patch("webapp.services.session_score_manager.SessionScoreJobManager._run"):
|
||||
client.post("/api/score/session_async", json={
|
||||
"session_id": "list-all-session",
|
||||
"question": "Q",
|
||||
"answer": "A",
|
||||
"metrics": ["answer_relevancy"],
|
||||
})
|
||||
resp = client.get("/api/score/sessions")
|
||||
assert resp.status_code == 200
|
||||
assert "sessions" in resp.json()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _mock_request():
|
||||
"""Build a minimal ScoreRequest for testing."""
|
||||
from webapp.models import ScoreRequest
|
||||
return ScoreRequest(
|
||||
question="What is dual-source CT?",
|
||||
answer="It uses two X-ray sources.",
|
||||
metrics=["answer_relevancy"],
|
||||
)
|
||||
Reference in New Issue
Block a user