update for mcp

backend/app/application/agent/hyde_expander.py

@@ -73,7 +73,7 @@ class HyDEExpander:
             return query
 
         # Use the dedicated HyDE model when configured; fall back to main LLM.
-        # A lightweight model (e.g. qwen3.5-flash) is sufficient for generating
+        # A lightweight model (e.g. qwen3.6-flash) is sufficient for generating
         # a short hypothetical passage and significantly reduces cost + latency.
         provider = settings.hyde_llm_provider or settings.llm_provider
         model = settings.hyde_llm_model or settings.llm_model

backend/app/application/perception/crawl_service.py

@@ -7,9 +7,13 @@ from typing import Any, Generator
 
 from loguru import logger
 
+from app.config.settings import settings
+from app.domain.documents import ParsedDocument
 from app.infrastructure.perception.base_event_store import BaseEventStore
+from app.infrastructure.perception.base_notification_store import BaseNotificationStore
 from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
 from app.infrastructure.perception.llm_pipeline import LlmPipeline
+from app.infrastructure.parser.local_chunk_builder import LocalRegulationChunkBuilder
 
 
 def _event_id(source: str, standard_code: str) -> str:
@@ -21,7 +25,68 @@ def _content_hash(raw_text: str) -> str:
     return hashlib.sha256(raw_text.encode()).hexdigest()
 
 
-def _raw_to_dict(raw: RawEvent, event_id: str, content_hash: str) -> dict:
+def _is_significant(changed_sections: list[dict]) -> bool:
+    """Report whether any changed section is worth notifying every user about.
+
+    changed_sections legitimately includes cosmetic edits — the differ
+    (subproject 1) still reports a fixed typo or a dropped trailing period as
+    a change, it just doesn't send those to the LLM. Broadcasting a
+    notification for every cosmetic edit would train people to ignore it, so
+    this reuses the same significance test the differ's own LLM gate applies:
+    a numeric or deontic change, or a whole paragraph added or removed.
+    """
+    return any(
+        section.get("numeric_changed")
+        or section.get("deontic_changed")
+        or section.get("change_type") in ("added", "removed")
+        for section in changed_sections
+    )
+
+
+def _index_in_knowledge_base(event: dict, *, embedding_provider: Any, vector_index: Any) -> None:
+    """Chunk, embed, and upsert a regulation's text into the shared knowledge base.
+
+    Always uses the local markdown chunker, never get_chunk_builder() — that
+    bootstrap function resolves to AliyunVectorChunkBuilder when
+    settings.chunk_backend == "aliyun" (the deployed value), which consumes
+    Aliyun DocMind's structured parse output. Crawled text has no such parse
+    output; it is already plain text (trafilatura, subproject 1), which is
+    exactly what LocalRegulationChunkBuilder chunks directly.
+
+    delete_by_document runs unconditionally before upsert — a no-op for a
+    brand-new event, and the only way to keep a changed regulation from
+    leaving its superseded text retrievable alongside the new version.
+    """
+    vector_index.delete_by_document(event["id"])
+
+    parsed = ParsedDocument(
+        doc_id=event["id"],
+        doc_name=event.get("title", ""),
+        structure_nodes=[],
+        semantic_blocks=[],
+        vector_chunks=[],
+        parser_name="perception_crawl",
+        raw_text=event.get("raw_text") or "",
+    )
+    builder = LocalRegulationChunkBuilder(
+        chunk_size=settings.chunk_size, chunk_overlap=settings.chunk_overlap,
+    )
+    chunks = builder.build(
+        parsed_document=parsed,
+        # regulation_type/version fill the same slots a manually uploaded
+        # document's form fields would, so the two intake paths are
+        # indistinguishable to retrieval and compliance analysis.
+        regulation_type=event.get("category", ""),
+        version=event.get("standard_code", ""),
+    )
+    if not chunks:
+        return
+
+    vectors = embedding_provider.embed_texts([c.embedding_text for c in chunks])
+    vector_index.upsert(chunks, vectors)
+
+
+def _raw_to_dict(raw: RawEvent, event_id: str, content_hash: str, raw_text: str) -> dict:
     return {
         "id": event_id,
         "source": raw.source,
@@ -36,6 +101,10 @@ def _raw_to_dict(raw: RawEvent, event_id: str, content_hash: str) -> dict:
         "effective_at": raw.effective_at,
         "category": raw.category,
         "tags": raw.tags,
+        # Persisted so the next crawl has a baseline to diff against. Without
+        # this the change detector has nothing to compare and every update
+        # looks like a first sighting.
+        "raw_text": raw_text,
         "content_hash": content_hash,
         "previous_hash": None,
     }
@@ -50,11 +119,17 @@ class CrawlService:
         event_store: BaseEventStore,
         llm_pipeline: LlmPipeline,
         retrieval_service: Any,
+        notification_store: BaseNotificationStore,
+        embedding_provider: Any,
+        vector_index: Any,
     ) -> None:
         self._crawlers = crawlers
         self._store = event_store
         self._pipeline = llm_pipeline
         self._retrieval = retrieval_service
+        self._notifications = notification_store
+        self._embedding_provider = embedding_provider
+        self._vector_index = vector_index
 
     def run_crawl(
         self, sources: list[str] | None = None
@@ -72,7 +147,7 @@ class CrawlService:
 
             yield {"event": "progress", "data": {"source": source_key, "stage": "fetching"}}
             try:
-                raw_events = crawler.fetch(limit=100)
+                raw_events = crawler.fetch(limit=settings.perception_max_events_per_source)
             except Exception as exc:
                 logger.exception("Crawler failed source={}", source_key)
                 yield {"event": "error", "data": {"source": source_key, "message": str(exc)}}
@@ -88,17 +163,21 @@ class CrawlService:
 
             for raw in raw_events:
                 eid = _event_id(raw.source, raw.standard_code)
-                new_hash = _content_hash(raw.raw_text or raw.title)
+                # List pages carry only a code and a title, which is not enough
+                # to detect a change in the regulation itself. Fetch the body,
+                # degrading to whatever the list page gave us if that fails.
+                body_text = crawler.fetch_full_text(raw.full_text_url) or raw.raw_text or raw.title
+                new_hash = _content_hash(body_text)
                 existing = self._store.get(eid)
 
                 if existing and existing.get("content_hash") == new_hash:
                     continue
 
                 is_update = existing is not None
-                old_text = existing.get("summary", "") if is_update else ""
+                old_body = existing.get("raw_text") or "" if is_update else ""
                 previous_hash = existing.get("content_hash") if is_update else None
 
-                event_dict = _raw_to_dict(raw, eid, new_hash)
+                event_dict = _raw_to_dict(raw, eid, new_hash, body_text)
                 event_dict["previous_hash"] = previous_hash
 
                 try:
@@ -113,9 +192,11 @@ class CrawlService:
                 except Exception as exc:
                     logger.warning("Impact assessment failed id={} err={}", eid, exc)
 
-                if is_update and old_text and raw.raw_text:
+                # Events stored before raw_text was persisted have no baseline,
+                # so they are treated as a first sighting and establish one now.
+                if is_update and old_body and body_text:
                     try:
-                        diff = self._pipeline.compute_diff(old_text, raw.raw_text)
+                        diff = self._pipeline.compute_diff(old_body, body_text)
                         event_dict["change_summary"] = diff.get("change_summary")
                         event_dict["changed_sections"] = diff.get("changed_sections")
                     except Exception as exc:
@@ -123,6 +204,33 @@ class CrawlService:
 
                 self._store.upsert(event_dict)
 
+                should_index = not is_update or _is_significant(event_dict.get("changed_sections") or [])
+
+                try:
+                    if not is_update:
+                        self._notifications.create(
+                            event_id=eid, kind="new", title=raw.title,
+                            impact_level=event_dict.get("impact_level"), summary=None,
+                        )
+                    elif should_index:  # significant change, already computed above
+                        self._notifications.create(
+                            event_id=eid, kind="changed", title=raw.title,
+                            impact_level=event_dict.get("impact_level"),
+                            summary=event_dict.get("change_summary"),
+                        )
+                except Exception as exc:
+                    logger.warning("Notification create failed id={} err={}", eid, exc)
+
+                if should_index:
+                    try:
+                        _index_in_knowledge_base(
+                            event_dict,
+                            embedding_provider=self._embedding_provider,
+                            vector_index=self._vector_index,
+                        )
+                    except Exception as exc:
+                        logger.warning("Knowledge base indexing failed id={} err={}", eid, exc)
+
                 if is_update:
                     updated_count += 1
                 else: