64 lines
2.6 KiB
Python
64 lines
2.6 KiB
Python
|
"""Celery task for scheduled regulatory source crawling.
|
||
|
|
|
||
|
|
This is the scheduled counterpart to the Perception page's manual "Refresh"
|
||
|
|
button (POST /perception/crawl). Every architecture reference document
|
||
|
|
describes source monitoring as continuous ("定时爬取"), not operator-triggered,
|
||
|
|
so this task is what Celery Beat runs on a fixed interval once an operator
|
||
|
|
starts a Beat process.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from loguru import logger
|
||
|
|
|
||
|
|
from app.infrastructure.tasks.celery_app import celery_app
|
||
|
|
|
||
|
|
|
||
|
|
@celery_app.task(
|
||
|
|
name="app.infrastructure.tasks.perception_tasks.crawl_regulations_task",
|
||
|
|
bind=True,
|
||
|
|
)
|
||
|
|
def crawl_regulations_task(self) -> dict:
|
||
|
|
"""Crawl every registered regulatory source and enrich new/changed events.
|
||
|
|
|
||
|
|
Drains CrawlService.run_crawl(), which already isolates each source's
|
||
|
|
fetch and each event's enrichment behind its own try/except — a source
|
||
|
|
outage or a single bad event yields an "error" progress item and the
|
||
|
|
generator continues. Re-catching those here would only hide problems the
|
||
|
|
service has already handled, so this task's job is limited to counting
|
||
|
|
them and logging a summary.
|
||
|
|
|
||
|
|
No automatic retry is configured. An exception escaping run_crawl itself
|
||
|
|
means something broke in a way the service's own error handling did not
|
||
|
|
anticipate; the next scheduled tick already provides a retry within
|
||
|
|
settings.perception_crawl_interval_seconds, so an immediate retry against
|
||
|
|
the same failure is not worth the added complexity.
|
||
|
|
|
||
|
|
ponytail: relies on a single worker process to serialize scheduled runs
|
||
|
|
(Celery's default concurrency processes one task at a time, so a run that
|
||
|
|
outlasts the interval delays the next tick rather than overlapping it).
|
||
|
|
Add a Redis-based lock (e.g. SETNX on a per-task key) if this queue is
|
||
|
|
ever served by more than one worker.
|
||
|
|
"""
|
||
|
|
from app.shared.bootstrap import get_crawl_service
|
||
|
|
|
||
|
|
error_count = 0
|
||
|
|
new_count = 0
|
||
|
|
updated_count = 0
|
||
|
|
|
||
|
|
for item in get_crawl_service().run_crawl():
|
||
|
|
event = item.get("event")
|
||
|
|
if event == "error":
|
||
|
|
error_count += 1
|
||
|
|
logger.warning("Scheduled crawl source error: {}", item.get("data"))
|
||
|
|
elif event == "done":
|
||
|
|
data = item.get("data") or {}
|
||
|
|
new_count = data.get("total_new", 0)
|
||
|
|
updated_count = data.get("total_updated", 0)
|
||
|
|
|
||
|
|
logger.info(
|
||
|
|
"Scheduled crawl finished: new={} updated={} source_errors={}",
|
||
|
|
new_count, updated_count, error_count,
|
||
|
|
)
|
||
|
|
return {"new": new_count, "updated": updated_count, "source_errors": error_count}
|