update for mcp

This commit is contained in:
wangwei
2026-08-06 11:08:46 +08:00
parent 31bbf80aeb
commit b2feaeddb4
40 changed files with 1986 additions and 202 deletions
@@ -8,6 +8,7 @@ import httpx
from bs4 import BeautifulSoup
from loguru import logger
from app.config.settings import settings
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
from ._utils import extract_tags, parse_date
@@ -33,7 +34,11 @@ class CatarcCrawler(BaseCrawler):
while len(events) < limit and page <= max_pages:
url = f"{_BASE_URL}?page={page}"
try:
resp = httpx.get(url, timeout=30, follow_redirects=True)
resp = httpx.get(
url,
timeout=settings.perception_crawl_timeout_seconds,
follow_redirects=True,
)
resp.raise_for_status()
except Exception as exc:
logger.warning("CATARC fetch failed page={} err={}", page, exc)