76 lines
2.6 KiB
Python
76 lines
2.6 KiB
Python
"""Shared contracts for regulatory source crawlers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass, field
|
|
|
|
import httpx
|
|
import trafilatura
|
|
from loguru import logger
|
|
|
|
from app.config.settings import settings
|
|
|
|
|
|
@dataclass
|
|
class RawEvent:
|
|
"""Raw regulatory event returned by a crawler before enrichment."""
|
|
|
|
source: str
|
|
source_label: str
|
|
standard_code: str
|
|
title: str
|
|
summary: str
|
|
full_text_url: str
|
|
status: str # 'enacted' | 'draft' | 'consultation'
|
|
published_at: str # YYYY-MM-DD string
|
|
effective_at: str | None
|
|
category: str
|
|
tags: list[str] = field(default_factory=list)
|
|
# Whatever text the list page yields. CrawlService upgrades this by calling
|
|
# fetch_full_text(full_text_url); this value is the fallback when that
|
|
# fails. Used for change hashing and for the version diff.
|
|
raw_text: str = ""
|
|
|
|
|
|
class BaseCrawler(ABC):
|
|
"""Abstract regulatory source crawler."""
|
|
|
|
@abstractmethod
|
|
def fetch(self, limit: int = 50) -> list[RawEvent]:
|
|
"""Fetch up to `limit` recent events from the data source."""
|
|
|
|
def fetch_full_text(self, url: str) -> str:
|
|
"""Download a regulation detail page and extract its body text.
|
|
|
|
Change detection is only as good as the text it compares, and list
|
|
pages carry nothing but a standard code and a title. This default
|
|
implementation serves all current sources; a source that needs PDF
|
|
extraction or authentication overrides this one method.
|
|
|
|
Returns an empty string on any failure rather than raising, so one
|
|
unreachable page cannot abort a whole crawl run. The caller decides how
|
|
to degrade.
|
|
"""
|
|
if not url:
|
|
return ""
|
|
try:
|
|
response = httpx.get(
|
|
url,
|
|
timeout=settings.perception_crawl_timeout_seconds,
|
|
follow_redirects=True,
|
|
)
|
|
response.raise_for_status()
|
|
except Exception as exc: # noqa: BLE001 - any transport error degrades the same way
|
|
logger.warning("Full-text fetch failed url={} err={}", url, exc)
|
|
return ""
|
|
|
|
# trafilatura scores 0.92 F1 on government pages against 0.78 for
|
|
# readability-lxml, and handles CJK content; include_tables matters
|
|
# because regulatory limits are frequently tabulated.
|
|
extracted = trafilatura.extract(response.text, include_tables=True)
|
|
if not extracted:
|
|
logger.warning("Full-text extraction returned nothing url={}", url)
|
|
return ""
|
|
return extracted.strip()
|