update for mcp

2026-08-06 11:08:46 +08:00
parent 31bbf80aeb
commit b2feaeddb4
40 changed files with 1986 additions and 202 deletions
@@ -6,6 +6,7 @@ import pytest

 from app.infrastructure.perception.crawlers.base import RawEvent
 from app.infrastructure.perception.mock_event_store import MockEventStore
+from app.infrastructure.perception.mock_notification_store import MockNotificationStore


 def _make_raw_event(code="TST-001"):
@@ -17,11 +18,18 @@ def _make_raw_event(code="TST-001"):
    )


+def _make_crawler(raw_events, full_text="full body text"):
+    """Build a mock crawler. `full_text=""` simulates a failed detail fetch."""
+    mock_crawler = MagicMock()
+    mock_crawler.fetch.return_value = raw_events
+    mock_crawler.fetch_full_text.return_value = full_text
+    return mock_crawler
+
+
 def _make_service(raw_events):
    from app.application.perception.crawl_service import CrawlService

-    mock_crawler = MagicMock()
-    mock_crawler.fetch.return_value = raw_events
+    mock_crawler = _make_crawler(raw_events)

    mock_pipeline = MagicMock()
    mock_pipeline.extract_structure.return_value = {
@@ -41,6 +49,9 @@ def _make_service(raw_events):
        event_store=store,
        llm_pipeline=mock_pipeline,
        retrieval_service=mock_retrieval,
+        notification_store=MockNotificationStore(),
+        embedding_provider=MagicMock(),
+        vector_index=MagicMock(),
    )


@@ -54,8 +65,7 @@ def test_crawl_yields_progress_and_done():
 def test_crawl_upserts_to_store():
    store = MockEventStore()
    from app.application.perception.crawl_service import CrawlService
-    mock_crawler = MagicMock()
-    mock_crawler.fetch.return_value = [_make_raw_event("NEW-001")]
+    mock_crawler = _make_crawler([_make_raw_event("NEW-001")])
    mock_pipeline = MagicMock()
    mock_pipeline.extract_structure.return_value = {
        "obligations": [], "deadlines": [], "scope": "",
@@ -70,6 +80,9 @@ def test_crawl_upserts_to_store():
        event_store=store,
        llm_pipeline=mock_pipeline,
        retrieval_service=MagicMock(),
+        notification_store=MockNotificationStore(),
+        embedding_provider=MagicMock(),
+        vector_index=MagicMock(),
    )
    list(svc.run_crawl())
    result = store.get_by_standard_code("NEW-001")
@@ -80,7 +93,8 @@ def test_crawl_upserts_to_store():
 def test_crawl_skips_unchanged_events():
    store = MockEventStore()
    raw = _make_raw_event("SKIP-001")
-    content_hash = hashlib.sha256(raw.raw_text.encode()).hexdigest()
+    body = "full body text"
+    content_hash = hashlib.sha256(body.encode()).hexdigest()
    store.upsert({
        "id": hashlib.sha256(f"TEST-SKIP-001".encode()).hexdigest()[:12],
        "standard_code": "SKIP-001",
@@ -99,13 +113,341 @@ def test_crawl_skips_unchanged_events():
    })
    mock_pipeline = MagicMock()
    from app.application.perception.crawl_service import CrawlService
-    mock_crawler = MagicMock()
-    mock_crawler.fetch.return_value = [raw]
+    mock_crawler = _make_crawler([raw], full_text=body)
    svc = CrawlService(
        crawlers={"TEST": mock_crawler},
        event_store=store,
        llm_pipeline=mock_pipeline,
        retrieval_service=MagicMock(),
+        notification_store=MockNotificationStore(),
+        embedding_provider=MagicMock(),
+        vector_index=MagicMock(),
    )
    list(svc.run_crawl())
    mock_pipeline.extract_structure.assert_not_called()
+
+
+def test_crawl_stores_the_fetched_body_for_the_next_diff():
+    """The body must be persisted, or the next crawl has no baseline to compare."""
+    store = MockEventStore()
+    from app.application.perception.crawl_service import CrawlService
+    mock_pipeline = MagicMock()
+    mock_pipeline.extract_structure.return_value = {}
+    mock_pipeline.assess_impact.return_value = []
+    svc = CrawlService(
+        crawlers={"TEST": _make_crawler([_make_raw_event("BODY-001")], full_text="第一条 正文内容。")},
+        event_store=store,
+        llm_pipeline=mock_pipeline,
+        retrieval_service=MagicMock(),
+        notification_store=MockNotificationStore(),
+        embedding_provider=MagicMock(),
+        vector_index=MagicMock(),
+    )
+    list(svc.run_crawl())
+
+    stored = store.get_by_standard_code("BODY-001")
+    assert stored["raw_text"] == "第一条 正文内容。"
+
+
+def test_crawl_falls_back_when_full_text_fetch_fails():
+    """An unreachable detail page degrades to the list-page text, never crashes."""
+    store = MockEventStore()
+    from app.application.perception.crawl_service import CrawlService
+    mock_pipeline = MagicMock()
+    mock_pipeline.extract_structure.return_value = {}
+    mock_pipeline.assess_impact.return_value = []
+    svc = CrawlService(
+        crawlers={"TEST": _make_crawler([_make_raw_event("FALL-001")], full_text="")},
+        event_store=store,
+        llm_pipeline=mock_pipeline,
+        retrieval_service=MagicMock(),
+        notification_store=MockNotificationStore(),
+        embedding_provider=MagicMock(),
+        vector_index=MagicMock(),
+    )
+    list(svc.run_crawl())
+
+    stored = store.get_by_standard_code("FALL-001")
+    assert stored is not None
+    assert stored["raw_text"] == "full text"
+
+
+def test_crawl_skips_diff_when_no_previous_body_exists():
+    """Rows stored before raw_text was persisted must not be diffed against nothing."""
+    store = MockEventStore()
+    from app.application.perception.crawl_service import CrawlService
+    event_id = hashlib.sha256(b"TEST-OLD-001").hexdigest()[:12]
+    store.upsert({
+        "id": event_id,
+        "standard_code": "OLD-001",
+        "source": "TEST",
+        "title": "Test OLD-001",
+        "summary": "legacy row",
+        "impact_level": "low",
+        "published_at": "2026-01-01",
+        "tags": [],
+        "content_hash": "stale-hash-from-before-this-change",
+    })
+    mock_pipeline = MagicMock()
+    mock_pipeline.extract_structure.return_value = {}
+    mock_pipeline.assess_impact.return_value = []
+    svc = CrawlService(
+        crawlers={"TEST": _make_crawler([_make_raw_event("OLD-001")], full_text="第一条 新正文。")},
+        event_store=store,
+        llm_pipeline=mock_pipeline,
+        retrieval_service=MagicMock(),
+        notification_store=MockNotificationStore(),
+        embedding_provider=MagicMock(),
+        vector_index=MagicMock(),
+    )
+    list(svc.run_crawl())
+
+    mock_pipeline.compute_diff.assert_not_called()
+    assert store.get(event_id)["raw_text"] == "第一条 新正文。"
+
+
+def test_new_event_creates_a_new_notification():
+    """A brand-new event must produce exactly one kind='new' notification."""
+    from app.application.perception.crawl_service import CrawlService
+
+    store = MockEventStore()
+    notifications = MockNotificationStore()
+    mock_pipeline = MagicMock()
+    mock_pipeline.extract_structure.return_value = {}
+    mock_pipeline.assess_impact.return_value = []
+    svc = CrawlService(
+        crawlers={"TEST": _make_crawler([_make_raw_event("NOTIF-NEW")])},
+        event_store=store,
+        llm_pipeline=mock_pipeline,
+        retrieval_service=MagicMock(),
+        notification_store=notifications,
+        embedding_provider=MagicMock(),
+        vector_index=MagicMock(),
+    )
+    list(svc.run_crawl())
+
+    items = notifications.list_for_user("any-user")
+    assert len(items) == 1
+    assert items[0]["kind"] == "new"
+
+
+def test_significant_change_creates_a_changed_notification():
+    """A numeric or deontic change must produce a kind='changed' notification."""
+    from app.application.perception.crawl_service import CrawlService
+
+    store = MockEventStore()
+    event_id = hashlib.sha256(b"TEST-SIG-001").hexdigest()[:12]
+    store.upsert({
+        "id": event_id, "standard_code": "SIG-001", "source": "TEST",
+        "title": "Test SIG-001", "summary": "", "impact_level": "medium",
+        "published_at": "2026-01-01", "tags": [],
+        "content_hash": "old-hash", "raw_text": "old body",
+    })
+    notifications = MockNotificationStore()
+    mock_pipeline = MagicMock()
+    mock_pipeline.extract_structure.return_value = {}
+    mock_pipeline.assess_impact.return_value = []
+    mock_pipeline.compute_diff.return_value = {
+        "changed_sections": [{"change_type": "modified", "numeric_changed": True, "deontic_changed": False}],
+        "change_summary": "1 paragraph changed (numeric).",
+    }
+    svc = CrawlService(
+        crawlers={"TEST": _make_crawler([_make_raw_event("SIG-001")], full_text="new body")},
+        event_store=store,
+        llm_pipeline=mock_pipeline,
+        retrieval_service=MagicMock(),
+        notification_store=notifications,
+        embedding_provider=MagicMock(),
+        vector_index=MagicMock(),
+    )
+    list(svc.run_crawl())
+
+    items = notifications.list_for_user("any-user")
+    assert len(items) == 1
+    assert items[0]["kind"] == "changed"
+
+
+def test_cosmetic_only_change_creates_no_notification():
+    """A change with no numeric/deontic/added/removed section must not notify."""
+    from app.application.perception.crawl_service import CrawlService
+
+    store = MockEventStore()
+    event_id = hashlib.sha256(b"TEST-COS-001").hexdigest()[:12]
+    store.upsert({
+        "id": event_id, "standard_code": "COS-001", "source": "TEST",
+        "title": "Test COS-001", "summary": "", "impact_level": "low",
+        "published_at": "2026-01-01", "tags": [],
+        "content_hash": "old-hash", "raw_text": "old body.",
+    })
+    notifications = MockNotificationStore()
+    mock_pipeline = MagicMock()
+    mock_pipeline.extract_structure.return_value = {}
+    mock_pipeline.assess_impact.return_value = []
+    mock_pipeline.compute_diff.return_value = {
+        "changed_sections": [{"change_type": "modified", "numeric_changed": False, "deontic_changed": False}],
+        "change_summary": "cosmetic only",
+    }
+    svc = CrawlService(
+        crawlers={"TEST": _make_crawler([_make_raw_event("COS-001")], full_text="old body")},
+        event_store=store,
+        llm_pipeline=mock_pipeline,
+        retrieval_service=MagicMock(),
+        notification_store=notifications,
+        embedding_provider=MagicMock(),
+        vector_index=MagicMock(),
+    )
+    list(svc.run_crawl())
+
+    assert notifications.list_for_user("any-user") == []
+
+
+def test_notification_store_failure_does_not_abort_the_crawl():
+    """A broken notification store must not stop the crawl or raise."""
+    from app.application.perception.crawl_service import CrawlService
+
+    store = MockEventStore()
+    broken_notifications = MagicMock()
+    broken_notifications.create.side_effect = RuntimeError("notification db down")
+    mock_pipeline = MagicMock()
+    mock_pipeline.extract_structure.return_value = {}
+    mock_pipeline.assess_impact.return_value = []
+    svc = CrawlService(
+        crawlers={"TEST": _make_crawler([_make_raw_event("BROKEN-001")])},
+        event_store=store,
+        llm_pipeline=mock_pipeline,
+        retrieval_service=MagicMock(),
+        notification_store=broken_notifications,
+        embedding_provider=MagicMock(),
+        vector_index=MagicMock(),
+    )
+    events = list(svc.run_crawl())
+
+    assert any(e.get("event") == "done" for e in events)
+    assert store.get_by_standard_code("BROKEN-001") is not None
+
+
+def test_new_event_is_indexed_in_the_knowledge_base():
+    """A brand-new event must be chunked, embedded, and upserted into Milvus."""
+    from app.application.perception.crawl_service import CrawlService
+
+    body = "第一条 本标准规定了车辆制动系统的技术要求。\n第二条 车辆制动系统应在时速50公里条件下于30米内完全停止。"
+    embedding_provider = MagicMock()
+    embedding_provider.embed_texts.return_value = [[0.1] * 8]
+    vector_index = MagicMock()
+    mock_pipeline = MagicMock()
+    mock_pipeline.extract_structure.return_value = {}
+    mock_pipeline.assess_impact.return_value = []
+    svc = CrawlService(
+        crawlers={"TEST": _make_crawler([_make_raw_event("IDX-NEW")], full_text=body)},
+        event_store=MockEventStore(),
+        llm_pipeline=mock_pipeline,
+        retrieval_service=MagicMock(),
+        notification_store=MockNotificationStore(),
+        embedding_provider=embedding_provider,
+        vector_index=vector_index,
+    )
+    list(svc.run_crawl())
+
+    event_id = hashlib.sha256(b"TEST-IDX-NEW").hexdigest()[:12]
+    vector_index.delete_by_document.assert_called_once_with(event_id)
+    vector_index.upsert.assert_called_once()
+    chunks_arg = vector_index.upsert.call_args.args[0]
+    assert len(chunks_arg) > 0
+
+
+def test_significant_change_reindexes_the_knowledge_base():
+    """A numeric/deontic change must delete the stale chunks and upsert new ones."""
+    from app.application.perception.crawl_service import CrawlService
+
+    store = MockEventStore()
+    event_id = hashlib.sha256(b"TEST-IDX-SIG").hexdigest()[:12]
+    store.upsert({
+        "id": event_id, "standard_code": "IDX-SIG", "source": "TEST",
+        "title": "Test IDX-SIG", "summary": "", "impact_level": "medium",
+        "published_at": "2026-01-01", "tags": [],
+        "content_hash": "old-hash", "raw_text": "old body",
+    })
+    embedding_provider = MagicMock()
+    embedding_provider.embed_texts.return_value = [[0.1] * 8]
+    vector_index = MagicMock()
+    mock_pipeline = MagicMock()
+    mock_pipeline.extract_structure.return_value = {}
+    mock_pipeline.assess_impact.return_value = []
+    mock_pipeline.compute_diff.return_value = {
+        "changed_sections": [{"change_type": "modified", "numeric_changed": True, "deontic_changed": False}],
+        "change_summary": "numeric change",
+    }
+    svc = CrawlService(
+        crawlers={"TEST": _make_crawler([_make_raw_event("IDX-SIG")], full_text="new body with a number 20米")},
+        event_store=store,
+        llm_pipeline=mock_pipeline,
+        retrieval_service=MagicMock(),
+        notification_store=MockNotificationStore(),
+        embedding_provider=embedding_provider,
+        vector_index=vector_index,
+    )
+    list(svc.run_crawl())
+
+    vector_index.delete_by_document.assert_called_once_with(event_id)
+    vector_index.upsert.assert_called_once()
+
+
+def test_cosmetic_only_change_does_not_touch_the_knowledge_base():
+    """A punctuation-only edit must not trigger embedding or a Milvus write."""
+    from app.application.perception.crawl_service import CrawlService
+
+    store = MockEventStore()
+    event_id = hashlib.sha256(b"TEST-IDX-COS").hexdigest()[:12]
+    store.upsert({
+        "id": event_id, "standard_code": "IDX-COS", "source": "TEST",
+        "title": "Test IDX-COS", "summary": "", "impact_level": "low",
+        "published_at": "2026-01-01", "tags": [],
+        "content_hash": "old-hash", "raw_text": "old body.",
+    })
+    embedding_provider = MagicMock()
+    vector_index = MagicMock()
+    mock_pipeline = MagicMock()
+    mock_pipeline.extract_structure.return_value = {}
+    mock_pipeline.assess_impact.return_value = []
+    mock_pipeline.compute_diff.return_value = {
+        "changed_sections": [{"change_type": "modified", "numeric_changed": False, "deontic_changed": False}],
+        "change_summary": "cosmetic only",
+    }
+    svc = CrawlService(
+        crawlers={"TEST": _make_crawler([_make_raw_event("IDX-COS")], full_text="old body")},
+        event_store=store,
+        llm_pipeline=mock_pipeline,
+        retrieval_service=MagicMock(),
+        notification_store=MockNotificationStore(),
+        embedding_provider=embedding_provider,
+        vector_index=vector_index,
+    )
+    list(svc.run_crawl())
+
+    embedding_provider.embed_texts.assert_not_called()
+    vector_index.upsert.assert_not_called()
+
+
+def test_vector_index_failure_does_not_abort_the_crawl():
+    """A broken vector index must not stop the crawl or raise."""
+    from app.application.perception.crawl_service import CrawlService
+
+    broken_vector_index = MagicMock()
+    broken_vector_index.upsert.side_effect = RuntimeError("milvus unreachable")
+    store = MockEventStore()
+    mock_pipeline = MagicMock()
+    mock_pipeline.extract_structure.return_value = {}
+    mock_pipeline.assess_impact.return_value = []
+    svc = CrawlService(
+        crawlers={"TEST": _make_crawler([_make_raw_event("IDX-FAIL")], full_text="第一条 正文内容。")},
+        event_store=store,
+        llm_pipeline=mock_pipeline,
+        retrieval_service=MagicMock(),
+        notification_store=MockNotificationStore(),
+        embedding_provider=MagicMock(embed_texts=MagicMock(return_value=[[0.1] * 8])),
+        vector_index=broken_vector_index,
+    )
+    events = list(svc.run_crawl())
+
+    assert any(e.get("event") == "done" for e in events)
+    assert store.get_by_standard_code("IDX-FAIL") is not None