454 lines
17 KiB
Python
454 lines
17 KiB
Python
"""Integration tests for CrawlService."""
|
|
from __future__ import annotations
|
|
from unittest.mock import MagicMock
|
|
import hashlib
|
|
import pytest
|
|
|
|
from app.infrastructure.perception.crawlers.base import RawEvent
|
|
from app.infrastructure.perception.mock_event_store import MockEventStore
|
|
from app.infrastructure.perception.mock_notification_store import MockNotificationStore
|
|
|
|
|
|
def _make_raw_event(code="TST-001"):
|
|
return RawEvent(
|
|
source="TEST", source_label="Test", standard_code=code,
|
|
title=f"Test {code}", summary="Summary", full_text_url="https://example.com",
|
|
status="enacted", published_at="2026-01-01", effective_at=None,
|
|
category="test", tags=["test"], raw_text="full text",
|
|
)
|
|
|
|
|
|
def _make_crawler(raw_events, full_text="full body text"):
|
|
"""Build a mock crawler. `full_text=""` simulates a failed detail fetch."""
|
|
mock_crawler = MagicMock()
|
|
mock_crawler.fetch.return_value = raw_events
|
|
mock_crawler.fetch_full_text.return_value = full_text
|
|
return mock_crawler
|
|
|
|
|
|
def _make_service(raw_events):
|
|
from app.application.perception.crawl_service import CrawlService
|
|
|
|
mock_crawler = _make_crawler(raw_events)
|
|
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {
|
|
"obligations": [], "deadlines": [], "scope": "test",
|
|
"penalties": None, "impact_level": "low",
|
|
}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
mock_pipeline.compute_diff.return_value = {
|
|
"changed_sections": [], "change_summary": "No changes.",
|
|
}
|
|
|
|
mock_retrieval = MagicMock()
|
|
store = MockEventStore()
|
|
|
|
return CrawlService(
|
|
crawlers={"TEST": mock_crawler},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=mock_retrieval,
|
|
notification_store=MockNotificationStore(),
|
|
embedding_provider=MagicMock(),
|
|
vector_index=MagicMock(),
|
|
)
|
|
|
|
|
|
def test_crawl_yields_progress_and_done():
|
|
svc = _make_service([_make_raw_event("TST-001")])
|
|
events = list(svc.run_crawl())
|
|
event_types = [e.get("event") for e in events]
|
|
assert "done" in event_types
|
|
|
|
|
|
def test_crawl_upserts_to_store():
|
|
store = MockEventStore()
|
|
from app.application.perception.crawl_service import CrawlService
|
|
mock_crawler = _make_crawler([_make_raw_event("NEW-001")])
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {
|
|
"obligations": [], "deadlines": [], "scope": "",
|
|
"penalties": None, "impact_level": "medium",
|
|
}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
mock_pipeline.compute_diff.return_value = {
|
|
"changed_sections": [], "change_summary": "",
|
|
}
|
|
svc = CrawlService(
|
|
crawlers={"TEST": mock_crawler},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=MockNotificationStore(),
|
|
embedding_provider=MagicMock(),
|
|
vector_index=MagicMock(),
|
|
)
|
|
list(svc.run_crawl())
|
|
result = store.get_by_standard_code("NEW-001")
|
|
assert result is not None
|
|
assert result["title"] == "Test NEW-001"
|
|
|
|
|
|
def test_crawl_skips_unchanged_events():
|
|
store = MockEventStore()
|
|
raw = _make_raw_event("SKIP-001")
|
|
body = "full body text"
|
|
content_hash = hashlib.sha256(body.encode()).hexdigest()
|
|
store.upsert({
|
|
"id": hashlib.sha256(f"TEST-SKIP-001".encode()).hexdigest()[:12],
|
|
"standard_code": "SKIP-001",
|
|
"source": "TEST",
|
|
"source_label": "Test",
|
|
"title": "Test SKIP-001",
|
|
"summary": "",
|
|
"full_text_url": "",
|
|
"status": "enacted",
|
|
"impact_level": "low",
|
|
"published_at": "2026-01-01",
|
|
"effective_at": None,
|
|
"category": "test",
|
|
"tags": [],
|
|
"content_hash": content_hash,
|
|
})
|
|
mock_pipeline = MagicMock()
|
|
from app.application.perception.crawl_service import CrawlService
|
|
mock_crawler = _make_crawler([raw], full_text=body)
|
|
svc = CrawlService(
|
|
crawlers={"TEST": mock_crawler},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=MockNotificationStore(),
|
|
embedding_provider=MagicMock(),
|
|
vector_index=MagicMock(),
|
|
)
|
|
list(svc.run_crawl())
|
|
mock_pipeline.extract_structure.assert_not_called()
|
|
|
|
|
|
def test_crawl_stores_the_fetched_body_for_the_next_diff():
|
|
"""The body must be persisted, or the next crawl has no baseline to compare."""
|
|
store = MockEventStore()
|
|
from app.application.perception.crawl_service import CrawlService
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
svc = CrawlService(
|
|
crawlers={"TEST": _make_crawler([_make_raw_event("BODY-001")], full_text="第一条 正文内容。")},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=MockNotificationStore(),
|
|
embedding_provider=MagicMock(),
|
|
vector_index=MagicMock(),
|
|
)
|
|
list(svc.run_crawl())
|
|
|
|
stored = store.get_by_standard_code("BODY-001")
|
|
assert stored["raw_text"] == "第一条 正文内容。"
|
|
|
|
|
|
def test_crawl_falls_back_when_full_text_fetch_fails():
|
|
"""An unreachable detail page degrades to the list-page text, never crashes."""
|
|
store = MockEventStore()
|
|
from app.application.perception.crawl_service import CrawlService
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
svc = CrawlService(
|
|
crawlers={"TEST": _make_crawler([_make_raw_event("FALL-001")], full_text="")},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=MockNotificationStore(),
|
|
embedding_provider=MagicMock(),
|
|
vector_index=MagicMock(),
|
|
)
|
|
list(svc.run_crawl())
|
|
|
|
stored = store.get_by_standard_code("FALL-001")
|
|
assert stored is not None
|
|
assert stored["raw_text"] == "full text"
|
|
|
|
|
|
def test_crawl_skips_diff_when_no_previous_body_exists():
|
|
"""Rows stored before raw_text was persisted must not be diffed against nothing."""
|
|
store = MockEventStore()
|
|
from app.application.perception.crawl_service import CrawlService
|
|
event_id = hashlib.sha256(b"TEST-OLD-001").hexdigest()[:12]
|
|
store.upsert({
|
|
"id": event_id,
|
|
"standard_code": "OLD-001",
|
|
"source": "TEST",
|
|
"title": "Test OLD-001",
|
|
"summary": "legacy row",
|
|
"impact_level": "low",
|
|
"published_at": "2026-01-01",
|
|
"tags": [],
|
|
"content_hash": "stale-hash-from-before-this-change",
|
|
})
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
svc = CrawlService(
|
|
crawlers={"TEST": _make_crawler([_make_raw_event("OLD-001")], full_text="第一条 新正文。")},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=MockNotificationStore(),
|
|
embedding_provider=MagicMock(),
|
|
vector_index=MagicMock(),
|
|
)
|
|
list(svc.run_crawl())
|
|
|
|
mock_pipeline.compute_diff.assert_not_called()
|
|
assert store.get(event_id)["raw_text"] == "第一条 新正文。"
|
|
|
|
|
|
def test_new_event_creates_a_new_notification():
|
|
"""A brand-new event must produce exactly one kind='new' notification."""
|
|
from app.application.perception.crawl_service import CrawlService
|
|
|
|
store = MockEventStore()
|
|
notifications = MockNotificationStore()
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
svc = CrawlService(
|
|
crawlers={"TEST": _make_crawler([_make_raw_event("NOTIF-NEW")])},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=notifications,
|
|
embedding_provider=MagicMock(),
|
|
vector_index=MagicMock(),
|
|
)
|
|
list(svc.run_crawl())
|
|
|
|
items = notifications.list_for_user("any-user")
|
|
assert len(items) == 1
|
|
assert items[0]["kind"] == "new"
|
|
|
|
|
|
def test_significant_change_creates_a_changed_notification():
|
|
"""A numeric or deontic change must produce a kind='changed' notification."""
|
|
from app.application.perception.crawl_service import CrawlService
|
|
|
|
store = MockEventStore()
|
|
event_id = hashlib.sha256(b"TEST-SIG-001").hexdigest()[:12]
|
|
store.upsert({
|
|
"id": event_id, "standard_code": "SIG-001", "source": "TEST",
|
|
"title": "Test SIG-001", "summary": "", "impact_level": "medium",
|
|
"published_at": "2026-01-01", "tags": [],
|
|
"content_hash": "old-hash", "raw_text": "old body",
|
|
})
|
|
notifications = MockNotificationStore()
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
mock_pipeline.compute_diff.return_value = {
|
|
"changed_sections": [{"change_type": "modified", "numeric_changed": True, "deontic_changed": False}],
|
|
"change_summary": "1 paragraph changed (numeric).",
|
|
}
|
|
svc = CrawlService(
|
|
crawlers={"TEST": _make_crawler([_make_raw_event("SIG-001")], full_text="new body")},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=notifications,
|
|
embedding_provider=MagicMock(),
|
|
vector_index=MagicMock(),
|
|
)
|
|
list(svc.run_crawl())
|
|
|
|
items = notifications.list_for_user("any-user")
|
|
assert len(items) == 1
|
|
assert items[0]["kind"] == "changed"
|
|
|
|
|
|
def test_cosmetic_only_change_creates_no_notification():
|
|
"""A change with no numeric/deontic/added/removed section must not notify."""
|
|
from app.application.perception.crawl_service import CrawlService
|
|
|
|
store = MockEventStore()
|
|
event_id = hashlib.sha256(b"TEST-COS-001").hexdigest()[:12]
|
|
store.upsert({
|
|
"id": event_id, "standard_code": "COS-001", "source": "TEST",
|
|
"title": "Test COS-001", "summary": "", "impact_level": "low",
|
|
"published_at": "2026-01-01", "tags": [],
|
|
"content_hash": "old-hash", "raw_text": "old body.",
|
|
})
|
|
notifications = MockNotificationStore()
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
mock_pipeline.compute_diff.return_value = {
|
|
"changed_sections": [{"change_type": "modified", "numeric_changed": False, "deontic_changed": False}],
|
|
"change_summary": "cosmetic only",
|
|
}
|
|
svc = CrawlService(
|
|
crawlers={"TEST": _make_crawler([_make_raw_event("COS-001")], full_text="old body")},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=notifications,
|
|
embedding_provider=MagicMock(),
|
|
vector_index=MagicMock(),
|
|
)
|
|
list(svc.run_crawl())
|
|
|
|
assert notifications.list_for_user("any-user") == []
|
|
|
|
|
|
def test_notification_store_failure_does_not_abort_the_crawl():
|
|
"""A broken notification store must not stop the crawl or raise."""
|
|
from app.application.perception.crawl_service import CrawlService
|
|
|
|
store = MockEventStore()
|
|
broken_notifications = MagicMock()
|
|
broken_notifications.create.side_effect = RuntimeError("notification db down")
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
svc = CrawlService(
|
|
crawlers={"TEST": _make_crawler([_make_raw_event("BROKEN-001")])},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=broken_notifications,
|
|
embedding_provider=MagicMock(),
|
|
vector_index=MagicMock(),
|
|
)
|
|
events = list(svc.run_crawl())
|
|
|
|
assert any(e.get("event") == "done" for e in events)
|
|
assert store.get_by_standard_code("BROKEN-001") is not None
|
|
|
|
|
|
def test_new_event_is_indexed_in_the_knowledge_base():
|
|
"""A brand-new event must be chunked, embedded, and upserted into Milvus."""
|
|
from app.application.perception.crawl_service import CrawlService
|
|
|
|
body = "第一条 本标准规定了车辆制动系统的技术要求。\n第二条 车辆制动系统应在时速50公里条件下于30米内完全停止。"
|
|
embedding_provider = MagicMock()
|
|
embedding_provider.embed_texts.return_value = [[0.1] * 8]
|
|
vector_index = MagicMock()
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
svc = CrawlService(
|
|
crawlers={"TEST": _make_crawler([_make_raw_event("IDX-NEW")], full_text=body)},
|
|
event_store=MockEventStore(),
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=MockNotificationStore(),
|
|
embedding_provider=embedding_provider,
|
|
vector_index=vector_index,
|
|
)
|
|
list(svc.run_crawl())
|
|
|
|
event_id = hashlib.sha256(b"TEST-IDX-NEW").hexdigest()[:12]
|
|
vector_index.delete_by_document.assert_called_once_with(event_id)
|
|
vector_index.upsert.assert_called_once()
|
|
chunks_arg = vector_index.upsert.call_args.args[0]
|
|
assert len(chunks_arg) > 0
|
|
|
|
|
|
def test_significant_change_reindexes_the_knowledge_base():
|
|
"""A numeric/deontic change must delete the stale chunks and upsert new ones."""
|
|
from app.application.perception.crawl_service import CrawlService
|
|
|
|
store = MockEventStore()
|
|
event_id = hashlib.sha256(b"TEST-IDX-SIG").hexdigest()[:12]
|
|
store.upsert({
|
|
"id": event_id, "standard_code": "IDX-SIG", "source": "TEST",
|
|
"title": "Test IDX-SIG", "summary": "", "impact_level": "medium",
|
|
"published_at": "2026-01-01", "tags": [],
|
|
"content_hash": "old-hash", "raw_text": "old body",
|
|
})
|
|
embedding_provider = MagicMock()
|
|
embedding_provider.embed_texts.return_value = [[0.1] * 8]
|
|
vector_index = MagicMock()
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
mock_pipeline.compute_diff.return_value = {
|
|
"changed_sections": [{"change_type": "modified", "numeric_changed": True, "deontic_changed": False}],
|
|
"change_summary": "numeric change",
|
|
}
|
|
svc = CrawlService(
|
|
crawlers={"TEST": _make_crawler([_make_raw_event("IDX-SIG")], full_text="new body with a number 20米")},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=MockNotificationStore(),
|
|
embedding_provider=embedding_provider,
|
|
vector_index=vector_index,
|
|
)
|
|
list(svc.run_crawl())
|
|
|
|
vector_index.delete_by_document.assert_called_once_with(event_id)
|
|
vector_index.upsert.assert_called_once()
|
|
|
|
|
|
def test_cosmetic_only_change_does_not_touch_the_knowledge_base():
|
|
"""A punctuation-only edit must not trigger embedding or a Milvus write."""
|
|
from app.application.perception.crawl_service import CrawlService
|
|
|
|
store = MockEventStore()
|
|
event_id = hashlib.sha256(b"TEST-IDX-COS").hexdigest()[:12]
|
|
store.upsert({
|
|
"id": event_id, "standard_code": "IDX-COS", "source": "TEST",
|
|
"title": "Test IDX-COS", "summary": "", "impact_level": "low",
|
|
"published_at": "2026-01-01", "tags": [],
|
|
"content_hash": "old-hash", "raw_text": "old body.",
|
|
})
|
|
embedding_provider = MagicMock()
|
|
vector_index = MagicMock()
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
mock_pipeline.compute_diff.return_value = {
|
|
"changed_sections": [{"change_type": "modified", "numeric_changed": False, "deontic_changed": False}],
|
|
"change_summary": "cosmetic only",
|
|
}
|
|
svc = CrawlService(
|
|
crawlers={"TEST": _make_crawler([_make_raw_event("IDX-COS")], full_text="old body")},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=MockNotificationStore(),
|
|
embedding_provider=embedding_provider,
|
|
vector_index=vector_index,
|
|
)
|
|
list(svc.run_crawl())
|
|
|
|
embedding_provider.embed_texts.assert_not_called()
|
|
vector_index.upsert.assert_not_called()
|
|
|
|
|
|
def test_vector_index_failure_does_not_abort_the_crawl():
|
|
"""A broken vector index must not stop the crawl or raise."""
|
|
from app.application.perception.crawl_service import CrawlService
|
|
|
|
broken_vector_index = MagicMock()
|
|
broken_vector_index.upsert.side_effect = RuntimeError("milvus unreachable")
|
|
store = MockEventStore()
|
|
mock_pipeline = MagicMock()
|
|
mock_pipeline.extract_structure.return_value = {}
|
|
mock_pipeline.assess_impact.return_value = []
|
|
svc = CrawlService(
|
|
crawlers={"TEST": _make_crawler([_make_raw_event("IDX-FAIL")], full_text="第一条 正文内容。")},
|
|
event_store=store,
|
|
llm_pipeline=mock_pipeline,
|
|
retrieval_service=MagicMock(),
|
|
notification_store=MockNotificationStore(),
|
|
embedding_provider=MagicMock(embed_texts=MagicMock(return_value=[[0.1] * 8])),
|
|
vector_index=broken_vector_index,
|
|
)
|
|
events = list(svc.run_crawl())
|
|
|
|
assert any(e.get("event") == "done" for e in events)
|
|
assert store.get_by_standard_code("IDX-FAIL") is not None
|