"""Orchestrates regulatory source crawlers and LLM enrichment pipeline.""" from __future__ import annotations import hashlib from typing import Any, Generator from loguru import logger from app.config.settings import settings from app.domain.documents import ParsedDocument from app.infrastructure.perception.base_event_store import BaseEventStore from app.infrastructure.perception.base_notification_store import BaseNotificationStore from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent from app.infrastructure.perception.llm_pipeline import LlmPipeline from app.infrastructure.parser.local_chunk_builder import LocalRegulationChunkBuilder def _event_id(source: str, standard_code: str) -> str: """Deterministic 12-char ID from source + standard_code.""" return hashlib.sha256(f"{source}-{standard_code}".encode()).hexdigest()[:12] def _content_hash(raw_text: str) -> str: return hashlib.sha256(raw_text.encode()).hexdigest() def _is_significant(changed_sections: list[dict]) -> bool: """Report whether any changed section is worth notifying every user about. changed_sections legitimately includes cosmetic edits — the differ (subproject 1) still reports a fixed typo or a dropped trailing period as a change, it just doesn't send those to the LLM. Broadcasting a notification for every cosmetic edit would train people to ignore it, so this reuses the same significance test the differ's own LLM gate applies: a numeric or deontic change, or a whole paragraph added or removed. """ return any( section.get("numeric_changed") or section.get("deontic_changed") or section.get("change_type") in ("added", "removed") for section in changed_sections ) def _index_in_knowledge_base(event: dict, *, embedding_provider: Any, vector_index: Any) -> None: """Chunk, embed, and upsert a regulation's text into the shared knowledge base. Always uses the local markdown chunker, never get_chunk_builder() — that bootstrap function resolves to AliyunVectorChunkBuilder when settings.chunk_backend == "aliyun" (the deployed value), which consumes Aliyun DocMind's structured parse output. Crawled text has no such parse output; it is already plain text (trafilatura, subproject 1), which is exactly what LocalRegulationChunkBuilder chunks directly. delete_by_document runs unconditionally before upsert — a no-op for a brand-new event, and the only way to keep a changed regulation from leaving its superseded text retrievable alongside the new version. """ vector_index.delete_by_document(event["id"]) parsed = ParsedDocument( doc_id=event["id"], doc_name=event.get("title", ""), structure_nodes=[], semantic_blocks=[], vector_chunks=[], parser_name="perception_crawl", raw_text=event.get("raw_text") or "", ) builder = LocalRegulationChunkBuilder( chunk_size=settings.chunk_size, chunk_overlap=settings.chunk_overlap, ) chunks = builder.build( parsed_document=parsed, # regulation_type/version fill the same slots a manually uploaded # document's form fields would, so the two intake paths are # indistinguishable to retrieval and compliance analysis. regulation_type=event.get("category", ""), version=event.get("standard_code", ""), ) if not chunks: return vectors = embedding_provider.embed_texts([c.embedding_text for c in chunks]) vector_index.upsert(chunks, vectors) def _raw_to_dict(raw: RawEvent, event_id: str, content_hash: str, raw_text: str) -> dict: return { "id": event_id, "source": raw.source, "source_label": raw.source_label, "standard_code": raw.standard_code, "title": raw.title, "summary": raw.summary, "full_text_url": raw.full_text_url, "status": raw.status, "impact_level": "medium", "published_at": raw.published_at, "effective_at": raw.effective_at, "category": raw.category, "tags": raw.tags, # Persisted so the next crawl has a baseline to diff against. Without # this the change detector has nothing to compare and every update # looks like a first sighting. "raw_text": raw_text, "content_hash": content_hash, "previous_hash": None, } class CrawlService: """Orchestrate crawlers, hash-based change detection, and LLM enrichment.""" def __init__( self, crawlers: dict[str, BaseCrawler], event_store: BaseEventStore, llm_pipeline: LlmPipeline, retrieval_service: Any, notification_store: BaseNotificationStore, embedding_provider: Any, vector_index: Any, ) -> None: self._crawlers = crawlers self._store = event_store self._pipeline = llm_pipeline self._retrieval = retrieval_service self._notifications = notification_store self._embedding_provider = embedding_provider self._vector_index = vector_index def run_crawl( self, sources: list[str] | None = None ) -> Generator[dict, None, None]: """Run crawl for selected sources. Yields SSE-ready progress dicts.""" targets = sources or list(self._crawlers.keys()) total_new = 0 total_updated = 0 for source_key in targets: crawler = self._crawlers.get(source_key) if not crawler: yield {"event": "error", "data": f"Unknown source: {source_key}"} continue yield {"event": "progress", "data": {"source": source_key, "stage": "fetching"}} try: raw_events = crawler.fetch(limit=settings.perception_max_events_per_source) except Exception as exc: logger.exception("Crawler failed source={}", source_key) yield {"event": "error", "data": {"source": source_key, "message": str(exc)}} continue yield { "event": "progress", "data": {"source": source_key, "stage": "processing", "fetched": len(raw_events)}, } new_count = 0 updated_count = 0 for raw in raw_events: eid = _event_id(raw.source, raw.standard_code) # List pages carry only a code and a title, which is not enough # to detect a change in the regulation itself. Fetch the body, # degrading to whatever the list page gave us if that fails. body_text = crawler.fetch_full_text(raw.full_text_url) or raw.raw_text or raw.title new_hash = _content_hash(body_text) existing = self._store.get(eid) if existing and existing.get("content_hash") == new_hash: continue is_update = existing is not None old_body = existing.get("raw_text") or "" if is_update else "" previous_hash = existing.get("content_hash") if is_update else None event_dict = _raw_to_dict(raw, eid, new_hash, body_text) event_dict["previous_hash"] = previous_hash try: structure = self._pipeline.extract_structure(event_dict) event_dict.update(structure) except Exception as exc: logger.warning("Structure extraction failed id={} err={}", eid, exc) try: affected = self._pipeline.assess_impact(event_dict, self._retrieval) event_dict["affected_docs"] = affected except Exception as exc: logger.warning("Impact assessment failed id={} err={}", eid, exc) # Events stored before raw_text was persisted have no baseline, # so they are treated as a first sighting and establish one now. if is_update and old_body and body_text: try: diff = self._pipeline.compute_diff(old_body, body_text) event_dict["change_summary"] = diff.get("change_summary") event_dict["changed_sections"] = diff.get("changed_sections") except Exception as exc: logger.warning("Diff failed id={} err={}", eid, exc) self._store.upsert(event_dict) should_index = not is_update or _is_significant(event_dict.get("changed_sections") or []) try: if not is_update: self._notifications.create( event_id=eid, kind="new", title=raw.title, impact_level=event_dict.get("impact_level"), summary=None, ) elif should_index: # significant change, already computed above self._notifications.create( event_id=eid, kind="changed", title=raw.title, impact_level=event_dict.get("impact_level"), summary=event_dict.get("change_summary"), ) except Exception as exc: logger.warning("Notification create failed id={} err={}", eid, exc) if should_index: try: _index_in_knowledge_base( event_dict, embedding_provider=self._embedding_provider, vector_index=self._vector_index, ) except Exception as exc: logger.warning("Knowledge base indexing failed id={} err={}", eid, exc) if is_update: updated_count += 1 else: new_count += 1 total_new += new_count total_updated += updated_count yield { "event": "progress", "data": { "source": source_key, "stage": "done", "new": new_count, "updated": updated_count, }, } yield { "event": "done", "data": {"total_new": total_new, "total_updated": total_updated}, }