AIRegulation-DocAnalysis/backend/app/application/perception/crawl_service.py

"""Orchestrates regulatory source crawlers and LLM enrichment pipeline."""

from __future__ import annotations

import hashlib
from typing import Any, Generator

from loguru import logger

from app.config.settings import settings
from app.domain.documents import ParsedDocument
from app.infrastructure.perception.base_event_store import BaseEventStore
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
from app.infrastructure.perception.llm_pipeline import LlmPipeline
from app.infrastructure.parser.local_chunk_builder import LocalRegulationChunkBuilder


def _event_id(source: str, standard_code: str) -> str:
    """Deterministic 12-char ID from source + standard_code."""
    return hashlib.sha256(f"{source}-{standard_code}".encode()).hexdigest()[:12]


def _content_hash(raw_text: str) -> str:
    return hashlib.sha256(raw_text.encode()).hexdigest()


def _is_significant(changed_sections: list[dict]) -> bool:
    """Report whether any changed section is worth notifying every user about.

    changed_sections legitimately includes cosmetic edits — the differ
    (subproject 1) still reports a fixed typo or a dropped trailing period as
    a change, it just doesn't send those to the LLM. Broadcasting a
    notification for every cosmetic edit would train people to ignore it, so
    this reuses the same significance test the differ's own LLM gate applies:
    a numeric or deontic change, or a whole paragraph added or removed.
    """
    return any(
        section.get("numeric_changed")
        or section.get("deontic_changed")
        or section.get("change_type") in ("added", "removed")
        for section in changed_sections
    )


def _index_in_knowledge_base(event: dict, *, embedding_provider: Any, vector_index: Any) -> None:
    """Chunk, embed, and upsert a regulation's text into the shared knowledge base.

    Always uses the local markdown chunker, never get_chunk_builder() — that
    bootstrap function resolves to AliyunVectorChunkBuilder when
    settings.chunk_backend == "aliyun" (the deployed value), which consumes
    Aliyun DocMind's structured parse output. Crawled text has no such parse
    output; it is already plain text (trafilatura, subproject 1), which is
    exactly what LocalRegulationChunkBuilder chunks directly.

    delete_by_document runs unconditionally before upsert — a no-op for a
    brand-new event, and the only way to keep a changed regulation from
    leaving its superseded text retrievable alongside the new version.
    """
    vector_index.delete_by_document(event["id"])

    parsed = ParsedDocument(
        doc_id=event["id"],
        doc_name=event.get("title", ""),
        structure_nodes=[],
        semantic_blocks=[],
        vector_chunks=[],
        parser_name="perception_crawl",
        raw_text=event.get("raw_text") or "",
    )
    builder = LocalRegulationChunkBuilder(
        chunk_size=settings.chunk_size, chunk_overlap=settings.chunk_overlap,
    )
    chunks = builder.build(
        parsed_document=parsed,
        # regulation_type/version fill the same slots a manually uploaded
        # document's form fields would, so the two intake paths are
        # indistinguishable to retrieval and compliance analysis.
        regulation_type=event.get("category", ""),
        version=event.get("standard_code", ""),
    )
    if not chunks:
        return

    vectors = embedding_provider.embed_texts([c.embedding_text for c in chunks])
    vector_index.upsert(chunks, vectors)


def _raw_to_dict(raw: RawEvent, event_id: str, content_hash: str, raw_text: str) -> dict:
    return {
        "id": event_id,
        "source": raw.source,
        "source_label": raw.source_label,
        "standard_code": raw.standard_code,
        "title": raw.title,
        "summary": raw.summary,
        "full_text_url": raw.full_text_url,
        "status": raw.status,
        "impact_level": "medium",
        "published_at": raw.published_at,
        "effective_at": raw.effective_at,
        "category": raw.category,
        "tags": raw.tags,
        # Persisted so the next crawl has a baseline to diff against. Without
        # this the change detector has nothing to compare and every update
        # looks like a first sighting.
        "raw_text": raw_text,
        "content_hash": content_hash,
        "previous_hash": None,
    }


class CrawlService:
    """Orchestrate crawlers, hash-based change detection, and LLM enrichment."""

    def __init__(
        self,
        crawlers: dict[str, BaseCrawler],
        event_store: BaseEventStore,
        llm_pipeline: LlmPipeline,
        retrieval_service: Any,
        notification_store: BaseNotificationStore,
        embedding_provider: Any,
        vector_index: Any,
    ) -> None:
        self._crawlers = crawlers
        self._store = event_store
        self._pipeline = llm_pipeline
        self._retrieval = retrieval_service
        self._notifications = notification_store
        self._embedding_provider = embedding_provider
        self._vector_index = vector_index

    def run_crawl(
        self, sources: list[str] | None = None
    ) -> Generator[dict, None, None]:
        """Run crawl for selected sources. Yields SSE-ready progress dicts."""
        targets = sources or list(self._crawlers.keys())
        total_new = 0
        total_updated = 0

        for source_key in targets:
            crawler = self._crawlers.get(source_key)
            if not crawler:
                yield {"event": "error", "data": f"Unknown source: {source_key}"}
                continue

            yield {"event": "progress", "data": {"source": source_key, "stage": "fetching"}}
            try:
                raw_events = crawler.fetch(limit=settings.perception_max_events_per_source)
            except Exception as exc:
                logger.exception("Crawler failed source={}", source_key)
                yield {"event": "error", "data": {"source": source_key, "message": str(exc)}}
                continue

            yield {
                "event": "progress",
                "data": {"source": source_key, "stage": "processing", "fetched": len(raw_events)},
            }

            new_count = 0
            updated_count = 0

            for raw in raw_events:
                eid = _event_id(raw.source, raw.standard_code)
                # List pages carry only a code and a title, which is not enough
                # to detect a change in the regulation itself. Fetch the body,
                # degrading to whatever the list page gave us if that fails.
                body_text = crawler.fetch_full_text(raw.full_text_url) or raw.raw_text or raw.title
                new_hash = _content_hash(body_text)
                existing = self._store.get(eid)

                if existing and existing.get("content_hash") == new_hash:
                    continue

                is_update = existing is not None
                old_body = existing.get("raw_text") or "" if is_update else ""
                previous_hash = existing.get("content_hash") if is_update else None

                event_dict = _raw_to_dict(raw, eid, new_hash, body_text)
                event_dict["previous_hash"] = previous_hash

                try:
                    structure = self._pipeline.extract_structure(event_dict)
                    event_dict.update(structure)
                except Exception as exc:
                    logger.warning("Structure extraction failed id={} err={}", eid, exc)

                try:
                    affected = self._pipeline.assess_impact(event_dict, self._retrieval)
                    event_dict["affected_docs"] = affected
                except Exception as exc:
                    logger.warning("Impact assessment failed id={} err={}", eid, exc)

                # Events stored before raw_text was persisted have no baseline,
                # so they are treated as a first sighting and establish one now.
                if is_update and old_body and body_text:
                    try:
                        diff = self._pipeline.compute_diff(old_body, body_text)
                        event_dict["change_summary"] = diff.get("change_summary")
                        event_dict["changed_sections"] = diff.get("changed_sections")
                    except Exception as exc:
                        logger.warning("Diff failed id={} err={}", eid, exc)

                self._store.upsert(event_dict)

                should_index = not is_update or _is_significant(event_dict.get("changed_sections") or [])

                try:
                    if not is_update:
                        self._notifications.create(
                            event_id=eid, kind="new", title=raw.title,
                            impact_level=event_dict.get("impact_level"), summary=None,
                        )
                    elif should_index:  # significant change, already computed above
                        self._notifications.create(
                            event_id=eid, kind="changed", title=raw.title,
                            impact_level=event_dict.get("impact_level"),
                            summary=event_dict.get("change_summary"),
                        )
                except Exception as exc:
                    logger.warning("Notification create failed id={} err={}", eid, exc)

                if should_index:
                    try:
                        _index_in_knowledge_base(
                            event_dict,
                            embedding_provider=self._embedding_provider,
                            vector_index=self._vector_index,
                        )
                    except Exception as exc:
                        logger.warning("Knowledge base indexing failed id={} err={}", eid, exc)

                if is_update:
                    updated_count += 1
                else:
                    new_count += 1

            total_new += new_count
            total_updated += updated_count

            yield {
                "event": "progress",
                "data": {
                    "source": source_key,
                    "stage": "done",
                    "new": new_count,
                    "updated": updated_count,
                },
            }

        yield {
            "event": "done",
            "data": {"total_new": total_new, "total_updated": total_updated},
        }