update for mcp

backend/app/infrastructure/tasks/celery_app.py

@@ -28,7 +28,10 @@ celery_app = Celery(
     "compliance_hub",
     broker=_BROKER,
     backend=_BACKEND,
-    include=["app.infrastructure.tasks.document_tasks"],
+    include=[
+        "app.infrastructure.tasks.document_tasks",
+        "app.infrastructure.tasks.perception_tasks",
+    ],
 )
 
 celery_app.conf.update(
@@ -42,4 +45,12 @@ celery_app.conf.update(
     task_reject_on_worker_lost=True,
     # Keep results for 1 hour for status polling.
     result_expires=3600,
+    # Scheduled counterpart to the Perception page's manual "Refresh" button.
+    # Only takes effect while a Beat process is running (./dev.sh start beat).
+    beat_schedule={
+        "crawl-regulations-periodic": {
+            "task": "app.infrastructure.tasks.perception_tasks.crawl_regulations_task",
+            "schedule": settings.perception_crawl_interval_seconds,
+        },
+    },
 )

backend/app/infrastructure/tasks/perception_tasks.py

@@ -0,0 +1,63 @@
+"""Celery task for scheduled regulatory source crawling.
+
+This is the scheduled counterpart to the Perception page's manual "Refresh"
+button (POST /perception/crawl). Every architecture reference document
+describes source monitoring as continuous ("定时爬取"), not operator-triggered,
+so this task is what Celery Beat runs on a fixed interval once an operator
+starts a Beat process.
+"""
+
+from __future__ import annotations
+
+from loguru import logger
+
+from app.infrastructure.tasks.celery_app import celery_app
+
+
+@celery_app.task(
+    name="app.infrastructure.tasks.perception_tasks.crawl_regulations_task",
+    bind=True,
+)
+def crawl_regulations_task(self) -> dict:
+    """Crawl every registered regulatory source and enrich new/changed events.
+
+    Drains CrawlService.run_crawl(), which already isolates each source's
+    fetch and each event's enrichment behind its own try/except — a source
+    outage or a single bad event yields an "error" progress item and the
+    generator continues. Re-catching those here would only hide problems the
+    service has already handled, so this task's job is limited to counting
+    them and logging a summary.
+
+    No automatic retry is configured. An exception escaping run_crawl itself
+    means something broke in a way the service's own error handling did not
+    anticipate; the next scheduled tick already provides a retry within
+    settings.perception_crawl_interval_seconds, so an immediate retry against
+    the same failure is not worth the added complexity.
+
+    ponytail: relies on a single worker process to serialize scheduled runs
+    (Celery's default concurrency processes one task at a time, so a run that
+    outlasts the interval delays the next tick rather than overlapping it).
+    Add a Redis-based lock (e.g. SETNX on a per-task key) if this queue is
+    ever served by more than one worker.
+    """
+    from app.shared.bootstrap import get_crawl_service
+
+    error_count = 0
+    new_count = 0
+    updated_count = 0
+
+    for item in get_crawl_service().run_crawl():
+        event = item.get("event")
+        if event == "error":
+            error_count += 1
+            logger.warning("Scheduled crawl source error: {}", item.get("data"))
+        elif event == "done":
+            data = item.get("data") or {}
+            new_count = data.get("total_new", 0)
+            updated_count = data.get("total_updated", 0)
+
+    logger.info(
+        "Scheduled crawl finished: new={} updated={} source_errors={}",
+        new_count, updated_count, error_count,
+    )
+    return {"new": new_count, "updated": updated_count, "source_errors": error_count}