update for mcp
This commit is contained in:
@@ -7,7 +7,12 @@ import json
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from app.shared.bootstrap import get_crawl_service, get_event_store, get_perception_service
|
||||
from app.shared.bootstrap import (
|
||||
get_crawl_service,
|
||||
get_event_store,
|
||||
get_notification_store,
|
||||
get_perception_service,
|
||||
)
|
||||
from app.api.dependencies.auth import get_current_user
|
||||
from app.domain.auth.models import UserClaims
|
||||
from app.shared.async_utils import iter_in_thread
|
||||
@@ -141,3 +146,26 @@ async def get_event_diff(event_id: str):
|
||||
"previous_hash": event.get("previous_hash"),
|
||||
"content_hash": event.get("content_hash"),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/notifications")
|
||||
async def list_notifications(
|
||||
limit: int = Query(default=20, ge=1, le=100),
|
||||
current_user: UserClaims = Depends(get_current_user),
|
||||
):
|
||||
"""Return the newest in-app notifications plus this user's unread count.
|
||||
|
||||
Every logged-in user sees the same broadcast feed — there is no per-role
|
||||
or per-topic subscription. "read" per item and the aggregate unread_count
|
||||
both reflect only the calling user's own read receipts.
|
||||
"""
|
||||
store = get_notification_store()
|
||||
items = store.list_for_user(current_user.user_id, limit=limit)
|
||||
return {"items": items, "unread_count": store.unread_count(current_user.user_id)}
|
||||
|
||||
|
||||
@router.post("/notifications/read")
|
||||
async def mark_notifications_read(current_user: UserClaims = Depends(get_current_user)):
|
||||
"""Mark every currently-unread notification read for the calling user."""
|
||||
marked = get_notification_store().mark_all_read(current_user.user_id)
|
||||
return {"marked": marked}
|
||||
|
||||
@@ -73,7 +73,7 @@ class HyDEExpander:
|
||||
return query
|
||||
|
||||
# Use the dedicated HyDE model when configured; fall back to main LLM.
|
||||
# A lightweight model (e.g. qwen3.5-flash) is sufficient for generating
|
||||
# A lightweight model (e.g. qwen3.6-flash) is sufficient for generating
|
||||
# a short hypothetical passage and significantly reduces cost + latency.
|
||||
provider = settings.hyde_llm_provider or settings.llm_provider
|
||||
model = settings.hyde_llm_model or settings.llm_model
|
||||
|
||||
@@ -7,9 +7,13 @@ from typing import Any, Generator
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.domain.documents import ParsedDocument
|
||||
from app.infrastructure.perception.base_event_store import BaseEventStore
|
||||
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
|
||||
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
|
||||
from app.infrastructure.perception.llm_pipeline import LlmPipeline
|
||||
from app.infrastructure.parser.local_chunk_builder import LocalRegulationChunkBuilder
|
||||
|
||||
|
||||
def _event_id(source: str, standard_code: str) -> str:
|
||||
@@ -21,7 +25,68 @@ def _content_hash(raw_text: str) -> str:
|
||||
return hashlib.sha256(raw_text.encode()).hexdigest()
|
||||
|
||||
|
||||
def _raw_to_dict(raw: RawEvent, event_id: str, content_hash: str) -> dict:
|
||||
def _is_significant(changed_sections: list[dict]) -> bool:
|
||||
"""Report whether any changed section is worth notifying every user about.
|
||||
|
||||
changed_sections legitimately includes cosmetic edits — the differ
|
||||
(subproject 1) still reports a fixed typo or a dropped trailing period as
|
||||
a change, it just doesn't send those to the LLM. Broadcasting a
|
||||
notification for every cosmetic edit would train people to ignore it, so
|
||||
this reuses the same significance test the differ's own LLM gate applies:
|
||||
a numeric or deontic change, or a whole paragraph added or removed.
|
||||
"""
|
||||
return any(
|
||||
section.get("numeric_changed")
|
||||
or section.get("deontic_changed")
|
||||
or section.get("change_type") in ("added", "removed")
|
||||
for section in changed_sections
|
||||
)
|
||||
|
||||
|
||||
def _index_in_knowledge_base(event: dict, *, embedding_provider: Any, vector_index: Any) -> None:
|
||||
"""Chunk, embed, and upsert a regulation's text into the shared knowledge base.
|
||||
|
||||
Always uses the local markdown chunker, never get_chunk_builder() — that
|
||||
bootstrap function resolves to AliyunVectorChunkBuilder when
|
||||
settings.chunk_backend == "aliyun" (the deployed value), which consumes
|
||||
Aliyun DocMind's structured parse output. Crawled text has no such parse
|
||||
output; it is already plain text (trafilatura, subproject 1), which is
|
||||
exactly what LocalRegulationChunkBuilder chunks directly.
|
||||
|
||||
delete_by_document runs unconditionally before upsert — a no-op for a
|
||||
brand-new event, and the only way to keep a changed regulation from
|
||||
leaving its superseded text retrievable alongside the new version.
|
||||
"""
|
||||
vector_index.delete_by_document(event["id"])
|
||||
|
||||
parsed = ParsedDocument(
|
||||
doc_id=event["id"],
|
||||
doc_name=event.get("title", ""),
|
||||
structure_nodes=[],
|
||||
semantic_blocks=[],
|
||||
vector_chunks=[],
|
||||
parser_name="perception_crawl",
|
||||
raw_text=event.get("raw_text") or "",
|
||||
)
|
||||
builder = LocalRegulationChunkBuilder(
|
||||
chunk_size=settings.chunk_size, chunk_overlap=settings.chunk_overlap,
|
||||
)
|
||||
chunks = builder.build(
|
||||
parsed_document=parsed,
|
||||
# regulation_type/version fill the same slots a manually uploaded
|
||||
# document's form fields would, so the two intake paths are
|
||||
# indistinguishable to retrieval and compliance analysis.
|
||||
regulation_type=event.get("category", ""),
|
||||
version=event.get("standard_code", ""),
|
||||
)
|
||||
if not chunks:
|
||||
return
|
||||
|
||||
vectors = embedding_provider.embed_texts([c.embedding_text for c in chunks])
|
||||
vector_index.upsert(chunks, vectors)
|
||||
|
||||
|
||||
def _raw_to_dict(raw: RawEvent, event_id: str, content_hash: str, raw_text: str) -> dict:
|
||||
return {
|
||||
"id": event_id,
|
||||
"source": raw.source,
|
||||
@@ -36,6 +101,10 @@ def _raw_to_dict(raw: RawEvent, event_id: str, content_hash: str) -> dict:
|
||||
"effective_at": raw.effective_at,
|
||||
"category": raw.category,
|
||||
"tags": raw.tags,
|
||||
# Persisted so the next crawl has a baseline to diff against. Without
|
||||
# this the change detector has nothing to compare and every update
|
||||
# looks like a first sighting.
|
||||
"raw_text": raw_text,
|
||||
"content_hash": content_hash,
|
||||
"previous_hash": None,
|
||||
}
|
||||
@@ -50,11 +119,17 @@ class CrawlService:
|
||||
event_store: BaseEventStore,
|
||||
llm_pipeline: LlmPipeline,
|
||||
retrieval_service: Any,
|
||||
notification_store: BaseNotificationStore,
|
||||
embedding_provider: Any,
|
||||
vector_index: Any,
|
||||
) -> None:
|
||||
self._crawlers = crawlers
|
||||
self._store = event_store
|
||||
self._pipeline = llm_pipeline
|
||||
self._retrieval = retrieval_service
|
||||
self._notifications = notification_store
|
||||
self._embedding_provider = embedding_provider
|
||||
self._vector_index = vector_index
|
||||
|
||||
def run_crawl(
|
||||
self, sources: list[str] | None = None
|
||||
@@ -72,7 +147,7 @@ class CrawlService:
|
||||
|
||||
yield {"event": "progress", "data": {"source": source_key, "stage": "fetching"}}
|
||||
try:
|
||||
raw_events = crawler.fetch(limit=100)
|
||||
raw_events = crawler.fetch(limit=settings.perception_max_events_per_source)
|
||||
except Exception as exc:
|
||||
logger.exception("Crawler failed source={}", source_key)
|
||||
yield {"event": "error", "data": {"source": source_key, "message": str(exc)}}
|
||||
@@ -88,17 +163,21 @@ class CrawlService:
|
||||
|
||||
for raw in raw_events:
|
||||
eid = _event_id(raw.source, raw.standard_code)
|
||||
new_hash = _content_hash(raw.raw_text or raw.title)
|
||||
# List pages carry only a code and a title, which is not enough
|
||||
# to detect a change in the regulation itself. Fetch the body,
|
||||
# degrading to whatever the list page gave us if that fails.
|
||||
body_text = crawler.fetch_full_text(raw.full_text_url) or raw.raw_text or raw.title
|
||||
new_hash = _content_hash(body_text)
|
||||
existing = self._store.get(eid)
|
||||
|
||||
if existing and existing.get("content_hash") == new_hash:
|
||||
continue
|
||||
|
||||
is_update = existing is not None
|
||||
old_text = existing.get("summary", "") if is_update else ""
|
||||
old_body = existing.get("raw_text") or "" if is_update else ""
|
||||
previous_hash = existing.get("content_hash") if is_update else None
|
||||
|
||||
event_dict = _raw_to_dict(raw, eid, new_hash)
|
||||
event_dict = _raw_to_dict(raw, eid, new_hash, body_text)
|
||||
event_dict["previous_hash"] = previous_hash
|
||||
|
||||
try:
|
||||
@@ -113,9 +192,11 @@ class CrawlService:
|
||||
except Exception as exc:
|
||||
logger.warning("Impact assessment failed id={} err={}", eid, exc)
|
||||
|
||||
if is_update and old_text and raw.raw_text:
|
||||
# Events stored before raw_text was persisted have no baseline,
|
||||
# so they are treated as a first sighting and establish one now.
|
||||
if is_update and old_body and body_text:
|
||||
try:
|
||||
diff = self._pipeline.compute_diff(old_text, raw.raw_text)
|
||||
diff = self._pipeline.compute_diff(old_body, body_text)
|
||||
event_dict["change_summary"] = diff.get("change_summary")
|
||||
event_dict["changed_sections"] = diff.get("changed_sections")
|
||||
except Exception as exc:
|
||||
@@ -123,6 +204,33 @@ class CrawlService:
|
||||
|
||||
self._store.upsert(event_dict)
|
||||
|
||||
should_index = not is_update or _is_significant(event_dict.get("changed_sections") or [])
|
||||
|
||||
try:
|
||||
if not is_update:
|
||||
self._notifications.create(
|
||||
event_id=eid, kind="new", title=raw.title,
|
||||
impact_level=event_dict.get("impact_level"), summary=None,
|
||||
)
|
||||
elif should_index: # significant change, already computed above
|
||||
self._notifications.create(
|
||||
event_id=eid, kind="changed", title=raw.title,
|
||||
impact_level=event_dict.get("impact_level"),
|
||||
summary=event_dict.get("change_summary"),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Notification create failed id={} err={}", eid, exc)
|
||||
|
||||
if should_index:
|
||||
try:
|
||||
_index_in_knowledge_base(
|
||||
event_dict,
|
||||
embedding_provider=self._embedding_provider,
|
||||
vector_index=self._vector_index,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Knowledge base indexing failed id={} err={}", eid, exc)
|
||||
|
||||
if is_update:
|
||||
updated_count += 1
|
||||
else:
|
||||
|
||||
@@ -94,9 +94,21 @@ class Settings(BaseSettings):
|
||||
perception_max_events_per_source: int = Field(
|
||||
default=100, description="Maximum events fetched per source per crawl run."
|
||||
)
|
||||
perception_diff_similarity_threshold: float = Field(
|
||||
default=0.85,
|
||||
description="Cosine similarity below which a paragraph is flagged as changed.",
|
||||
perception_diff_min_change_ratio: float = Field(
|
||||
default=0.02,
|
||||
description=(
|
||||
"Fraction of characters that must differ before an otherwise "
|
||||
"unremarkable paragraph edit is worth an LLM classification call. "
|
||||
"Numeric and deontic changes bypass this gate entirely."
|
||||
),
|
||||
)
|
||||
perception_crawl_interval_seconds: int = Field(
|
||||
default=21600,
|
||||
description=(
|
||||
"How often Celery Beat runs the scheduled crawl-all-sources task, "
|
||||
"in seconds. Default 21600 = 6 hours. Only takes effect when a "
|
||||
"Beat process is running (./dev.sh start beat)."
|
||||
),
|
||||
)
|
||||
|
||||
# Keep configuration setup explicit so runtime behavior is easy to reason about.
|
||||
@@ -117,7 +129,7 @@ class Settings(BaseSettings):
|
||||
# Keep configuration setup explicit so runtime behavior is easy to reason about.
|
||||
qwen_api_key: str = Field(default="", description="Qwen API密钥")
|
||||
qwen_base_url: str = Field(default="http://6.86.80.4:30080/v1", description="Qwen API地址")
|
||||
qwen_model: str = Field(default="qwen3.5-flash", description="Qwen文本模型")
|
||||
qwen_model: str = Field(default="qwen3.6-flash", description="Qwen文本模型")
|
||||
qwen_vl_model: str = Field(default="qwen3-vl-plus", description="Qwen视觉模型")
|
||||
|
||||
# Keep configuration setup explicit so runtime behavior is easy to reason about.
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Abstract base class for in-app regulatory-signal notifications.
|
||||
|
||||
A notification is created once per triggering event (a brand-new regulation,
|
||||
or a significant change to an existing one) and broadcast to every logged-in
|
||||
user. There is no per-user subscription targeting — see the design doc for why.
|
||||
Per-user "read" state is tracked separately from the notification itself, so
|
||||
one notification row serves every user rather than being fanned out on create.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class BaseNotificationStore(ABC):
|
||||
"""Port interface for perception notification persistence."""
|
||||
|
||||
@abstractmethod
|
||||
def create(
|
||||
self,
|
||||
*,
|
||||
event_id: str,
|
||||
kind: str,
|
||||
title: str,
|
||||
impact_level: str | None,
|
||||
summary: str | None,
|
||||
) -> None:
|
||||
"""Record a new notification. kind is 'new' or 'changed'."""
|
||||
|
||||
@abstractmethod
|
||||
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
|
||||
"""Return the most recent notifications, newest first.
|
||||
|
||||
Each item includes a "read" boolean reflecting whether `user_id` has
|
||||
marked it read.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def unread_count(self, user_id: str) -> int:
|
||||
"""Return how many notifications `user_id` has not yet read."""
|
||||
|
||||
@abstractmethod
|
||||
def mark_all_read(self, user_id: str) -> int:
|
||||
"""Mark every currently-unread notification read for `user_id`.
|
||||
|
||||
Returns the number of notifications newly marked.
|
||||
"""
|
||||
@@ -5,6 +5,12 @@ from __future__ import annotations
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import httpx
|
||||
import trafilatura
|
||||
from loguru import logger
|
||||
|
||||
from app.config.settings import settings
|
||||
|
||||
|
||||
@dataclass
|
||||
class RawEvent:
|
||||
@@ -21,7 +27,10 @@ class RawEvent:
|
||||
effective_at: str | None
|
||||
category: str
|
||||
tags: list[str] = field(default_factory=list)
|
||||
raw_text: str = "" # full crawled text for hashing + LLM
|
||||
# Whatever text the list page yields. CrawlService upgrades this by calling
|
||||
# fetch_full_text(full_text_url); this value is the fallback when that
|
||||
# fails. Used for change hashing and for the version diff.
|
||||
raw_text: str = ""
|
||||
|
||||
|
||||
class BaseCrawler(ABC):
|
||||
@@ -30,3 +39,37 @@ class BaseCrawler(ABC):
|
||||
@abstractmethod
|
||||
def fetch(self, limit: int = 50) -> list[RawEvent]:
|
||||
"""Fetch up to `limit` recent events from the data source."""
|
||||
|
||||
def fetch_full_text(self, url: str) -> str:
|
||||
"""Download a regulation detail page and extract its body text.
|
||||
|
||||
Change detection is only as good as the text it compares, and list
|
||||
pages carry nothing but a standard code and a title. This default
|
||||
implementation serves all current sources; a source that needs PDF
|
||||
extraction or authentication overrides this one method.
|
||||
|
||||
Returns an empty string on any failure rather than raising, so one
|
||||
unreachable page cannot abort a whole crawl run. The caller decides how
|
||||
to degrade.
|
||||
"""
|
||||
if not url:
|
||||
return ""
|
||||
try:
|
||||
response = httpx.get(
|
||||
url,
|
||||
timeout=settings.perception_crawl_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except Exception as exc: # noqa: BLE001 - any transport error degrades the same way
|
||||
logger.warning("Full-text fetch failed url={} err={}", url, exc)
|
||||
return ""
|
||||
|
||||
# trafilatura scores 0.92 F1 on government pages against 0.78 for
|
||||
# readability-lxml, and handles CJK content; include_tables matters
|
||||
# because regulatory limits are frequently tabulated.
|
||||
extracted = trafilatura.extract(response.text, include_tables=True)
|
||||
if not extracted:
|
||||
logger.warning("Full-text extraction returned nothing url={}", url)
|
||||
return ""
|
||||
return extracted.strip()
|
||||
|
||||
@@ -8,6 +8,7 @@ import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from loguru import logger
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
|
||||
from ._utils import extract_tags, parse_date
|
||||
|
||||
@@ -33,7 +34,11 @@ class CatarcCrawler(BaseCrawler):
|
||||
while len(events) < limit and page <= max_pages:
|
||||
url = f"{_BASE_URL}?page={page}"
|
||||
try:
|
||||
resp = httpx.get(url, timeout=30, follow_redirects=True)
|
||||
resp = httpx.get(
|
||||
url,
|
||||
timeout=settings.perception_crawl_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except Exception as exc:
|
||||
logger.warning("CATARC fetch failed page={} err={}", page, exc)
|
||||
|
||||
@@ -9,6 +9,7 @@ import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from loguru import logger
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
|
||||
from ._utils import parse_date
|
||||
|
||||
@@ -53,7 +54,11 @@ class EurlexCrawler(BaseCrawler):
|
||||
if len(events) >= limit:
|
||||
break
|
||||
try:
|
||||
resp = httpx.get(rss_url, timeout=30, follow_redirects=True)
|
||||
resp = httpx.get(
|
||||
rss_url,
|
||||
timeout=settings.perception_crawl_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except Exception as exc:
|
||||
logger.warning("EUR-Lex RSS fetch failed url={} err={}", rss_url, exc)
|
||||
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import httpx
|
||||
from loguru import logger
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
|
||||
from ._utils import extract_tags, parse_date
|
||||
|
||||
@@ -22,7 +23,12 @@ def _fetch_page(std_type: int, page: int, page_size: int) -> list[dict]:
|
||||
"p.p7": page_size,
|
||||
}
|
||||
try:
|
||||
resp = httpx.get(_BASE_URL, params=params, headers=_HEADERS, timeout=30)
|
||||
resp = httpx.get(
|
||||
_BASE_URL,
|
||||
params=params,
|
||||
headers=_HEADERS,
|
||||
timeout=settings.perception_crawl_timeout_seconds,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
return data.get("rows", []) or []
|
||||
|
||||
@@ -3,14 +3,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.infrastructure.embedding.openai_compatible_embedding_provider import (
|
||||
OpenAICompatibleEmbeddingProvider,
|
||||
from app.infrastructure.perception.regulation_differ import (
|
||||
ParagraphChange,
|
||||
RegulationDiffer,
|
||||
)
|
||||
from app.services.llm.llm_factory import get_llm_client
|
||||
|
||||
@@ -27,21 +27,31 @@ _ASSESS_SYSTEM = (
|
||||
)
|
||||
|
||||
_DIFF_SYSTEM = (
|
||||
"You are a regulatory change analyst. Given an old and new version of a regulation paragraph, "
|
||||
"classify the type of change and summarise it. "
|
||||
"Return JSON only: {\"change_type\": \"tightened|relaxed|added|removed\", \"summary\": \"...\"}"
|
||||
"You are a regulatory change analyst. You are given the OLD and NEW version of "
|
||||
"one regulation paragraph, with the exact edits marked <DEL>removed</DEL> and "
|
||||
"<INS>added</INS>. Classify the legal effect of the change. "
|
||||
"Return JSON only: {\"change_type\": \"tightened|relaxed|numeric|clarified|scope\", "
|
||||
"\"legal_effect\": \"one sentence on what this means for compliance\"}"
|
||||
)
|
||||
|
||||
_SIMILARITY_THRESHOLD = 0.85
|
||||
|
||||
def _marked_diff(change: ParagraphChange) -> str:
|
||||
"""Render a paragraph change with the exact edits marked for the model.
|
||||
|
||||
def _cosine(a: list[float], b: list[float]) -> float:
|
||||
dot = sum(x * y for x, y in zip(a, b))
|
||||
norm_a = math.sqrt(sum(x * x for x in a))
|
||||
norm_b = math.sqrt(sum(x * x for x in b))
|
||||
if norm_a == 0 or norm_b == 0:
|
||||
return 0.0
|
||||
return dot / (norm_a * norm_b)
|
||||
The model is shown where the edit is rather than being asked to find it,
|
||||
and is never asked to reproduce the changed text — the differ already
|
||||
computed those spans exactly, so there is nothing for the model to
|
||||
hallucinate.
|
||||
"""
|
||||
marked = "".join(
|
||||
text if op == 0 else (f"<DEL>{text}</DEL>" if op < 0 else f"<INS>{text}</INS>")
|
||||
for op, text in change.diff_spans
|
||||
)
|
||||
return (
|
||||
f"OLD: {change.old_text[:500]}\n"
|
||||
f"NEW: {change.new_text[:500]}\n"
|
||||
f"MARKED: {marked[:800]}"
|
||||
)
|
||||
|
||||
|
||||
def _llm_json(client: Any, messages: list[dict]) -> Any:
|
||||
@@ -67,7 +77,9 @@ class LlmPipeline:
|
||||
provider=settings.llm_provider,
|
||||
model=settings.llm_model,
|
||||
)
|
||||
self._embedder = OpenAICompatibleEmbeddingProvider()
|
||||
# Change detection is deterministic; the differ needs no model and no
|
||||
# network, so the pipeline no longer constructs an embedding provider.
|
||||
self._differ = RegulationDiffer()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Step 1: Structure extraction
|
||||
@@ -166,76 +178,68 @@ For each document, assess impact and recommend action. Return JSON array:
|
||||
return doc_excerpts
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Step 3: Semantic diff
|
||||
# Step 3: Deterministic diff with gated LLM classification
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def compute_diff(self, old_text: str, new_text: str) -> dict:
|
||||
"""Compare old and new regulation text; return changed sections and summary."""
|
||||
old_paras = [p.strip() for p in old_text.split("\n") if p.strip()]
|
||||
new_paras = [p.strip() for p in new_text.split("\n") if p.strip()]
|
||||
"""Compare old and new regulation text; return changed sections and summary.
|
||||
|
||||
if not old_paras or not new_paras:
|
||||
return {"changed_sections": [], "change_summary": "No comparable text."}
|
||||
Detection is deterministic — see regulation_differ for why embedding
|
||||
similarity was removed. The LLM is called only for paragraphs the
|
||||
differ marked significant, and only to explain the legal effect of a
|
||||
change that has already been located exactly.
|
||||
"""
|
||||
changes = self._differ.diff(old_text, new_text)
|
||||
if not changes:
|
||||
return {
|
||||
"changed_sections": [],
|
||||
"change_summary": "No substantive changes detected between versions.",
|
||||
}
|
||||
|
||||
all_paras = old_paras + new_paras
|
||||
try:
|
||||
all_embeddings = self._embedder.embed_texts(all_paras)
|
||||
except Exception as exc:
|
||||
logger.warning("Embedding for diff failed: {}", exc)
|
||||
return {"changed_sections": [], "change_summary": "Diff unavailable (embedding error)."}
|
||||
changed_sections = [self._describe(change) for change in changes]
|
||||
|
||||
old_embeddings = all_embeddings[: len(old_paras)]
|
||||
new_embeddings = all_embeddings[len(old_paras):]
|
||||
|
||||
changed_sections: list[dict] = []
|
||||
max_len = max(len(old_paras), len(new_paras))
|
||||
|
||||
for i in range(max_len):
|
||||
if i >= len(old_paras):
|
||||
# New paragraph added
|
||||
changed_sections.append({
|
||||
"old_text": "",
|
||||
"new_text": new_paras[i][:300],
|
||||
"similarity": 0.0,
|
||||
"change_type": "added",
|
||||
"summary": "New paragraph added.",
|
||||
})
|
||||
continue
|
||||
if i >= len(new_paras):
|
||||
# Old paragraph removed
|
||||
changed_sections.append({
|
||||
"old_text": old_paras[i][:300],
|
||||
"new_text": "",
|
||||
"similarity": 0.0,
|
||||
"change_type": "removed",
|
||||
"summary": "Paragraph removed.",
|
||||
})
|
||||
continue
|
||||
# Both exist — compare via embeddings
|
||||
sim = _cosine(old_embeddings[i], new_embeddings[i])
|
||||
if sim < _SIMILARITY_THRESHOLD:
|
||||
messages = [
|
||||
{"role": "system", "content": _DIFF_SYSTEM},
|
||||
{"role": "user", "content": f"OLD: {old_paras[i][:500]}\nNEW: {new_paras[i][:500]}"},
|
||||
]
|
||||
classification = _llm_json(self._client, messages) or {}
|
||||
changed_sections.append({
|
||||
"old_text": old_paras[i][:300],
|
||||
"new_text": new_paras[i][:300],
|
||||
"similarity": round(sim, 3),
|
||||
"change_type": classification.get("change_type", "modified"),
|
||||
"summary": classification.get("summary", ""),
|
||||
})
|
||||
|
||||
if not changed_sections:
|
||||
change_summary = "No substantive changes detected between versions."
|
||||
else:
|
||||
types = [s["change_type"] for s in changed_sections]
|
||||
change_summary = (
|
||||
f"{len(changed_sections)} paragraph(s) changed: "
|
||||
+ ", ".join(f"{t}" for t in set(types))
|
||||
+ ". "
|
||||
+ (changed_sections[0].get("summary", "") if changed_sections else "")
|
||||
)
|
||||
types = sorted({section["change_type"] for section in changed_sections})
|
||||
gated = sum(1 for change in changes if change.needs_llm)
|
||||
change_summary = (
|
||||
f"{len(changed_sections)} paragraph(s) changed ({', '.join(types)}); "
|
||||
f"{gated} significant. "
|
||||
+ (changed_sections[0].get("summary") or "")
|
||||
).strip()
|
||||
|
||||
return {"changed_sections": changed_sections, "change_summary": change_summary}
|
||||
|
||||
def _describe(self, change: ParagraphChange) -> dict:
|
||||
"""Turn one detected change into the API payload, classifying if warranted."""
|
||||
section = {
|
||||
"old_text": change.old_text[:300],
|
||||
"new_text": change.new_text[:300],
|
||||
"change_type": change.change_type,
|
||||
"change_ratio": round(change.change_ratio, 3),
|
||||
"numeric_changed": change.numeric_changed,
|
||||
"deontic_changed": change.deontic_changed,
|
||||
"summary": "",
|
||||
}
|
||||
|
||||
if not change.needs_llm:
|
||||
return section
|
||||
|
||||
classification = _llm_json(
|
||||
self._client,
|
||||
[
|
||||
{"role": "system", "content": _DIFF_SYSTEM},
|
||||
{"role": "user", "content": _marked_diff(change)},
|
||||
],
|
||||
)
|
||||
if isinstance(classification, dict):
|
||||
section["change_type"] = classification.get("change_type") or change.change_type
|
||||
section["summary"] = classification.get("legal_effect") or ""
|
||||
# A failed or malformed model response must not discard a change that
|
||||
# deterministic analysis already proved real; the section keeps its
|
||||
# spans, flags, and alignment-derived type with an empty summary.
|
||||
|
||||
if change.numeric_changed:
|
||||
# Models routinely label a changed threshold as "clarified". The
|
||||
# deterministic pass already knows a number moved, so it wins.
|
||||
section["change_type"] = "numeric"
|
||||
|
||||
return section
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""In-memory notification store used when Postgres is not configured.
|
||||
|
||||
Mirrors MockEventStore's role for BaseEventStore: keeps the feature usable in
|
||||
local dev and in tests without a live database, and matches
|
||||
DOCUMENT_REPOSITORY_BACKEND's existing Mock/Postgres split.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
|
||||
|
||||
|
||||
class MockNotificationStore(BaseNotificationStore):
|
||||
"""Dict-backed notification store. Data does not survive a process restart."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Start with an empty feed and no read receipts."""
|
||||
self._notifications: list[dict] = []
|
||||
self._next_id = 1
|
||||
# (notification_id, user_id) pairs — presence means read.
|
||||
self._reads: set[tuple[int, str]] = set()
|
||||
|
||||
def create(
|
||||
self,
|
||||
*,
|
||||
event_id: str,
|
||||
kind: str,
|
||||
title: str,
|
||||
impact_level: str | None,
|
||||
summary: str | None,
|
||||
) -> None:
|
||||
"""Append a notification with an auto-incrementing id."""
|
||||
self._notifications.append({
|
||||
"id": self._next_id,
|
||||
"event_id": event_id,
|
||||
"kind": kind,
|
||||
"title": title,
|
||||
"impact_level": impact_level,
|
||||
"summary": summary,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
})
|
||||
self._next_id += 1
|
||||
|
||||
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
|
||||
"""Return the newest `limit` notifications with this user's read state."""
|
||||
ordered = sorted(self._notifications, key=lambda n: n["id"], reverse=True)
|
||||
return [
|
||||
{**n, "read": (n["id"], user_id) in self._reads}
|
||||
for n in ordered[:limit]
|
||||
]
|
||||
|
||||
def unread_count(self, user_id: str) -> int:
|
||||
"""Count notifications this user has not yet read."""
|
||||
return sum(1 for n in self._notifications if (n["id"], user_id) not in self._reads)
|
||||
|
||||
def mark_all_read(self, user_id: str) -> int:
|
||||
"""Add a read receipt for every currently-unread notification."""
|
||||
marked = 0
|
||||
for n in self._notifications:
|
||||
key = (n["id"], user_id)
|
||||
if key not in self._reads:
|
||||
self._reads.add(key)
|
||||
marked += 1
|
||||
return marked
|
||||
@@ -40,7 +40,8 @@ CREATE TABLE IF NOT EXISTS regulation_events (
|
||||
affected_docs JSONB,
|
||||
crawled_at TIMESTAMPTZ DEFAULT now(),
|
||||
processed_at TIMESTAMPTZ,
|
||||
raw_storage_key TEXT
|
||||
raw_storage_key TEXT,
|
||||
raw_text TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS reg_events_source_date
|
||||
ON regulation_events (source, published_at DESC);
|
||||
@@ -48,12 +49,16 @@ CREATE INDEX IF NOT EXISTS reg_events_impact_date
|
||||
ON regulation_events (impact_level, published_at DESC);
|
||||
"""
|
||||
|
||||
_ADD_COLUMNS = """
|
||||
ALTER TABLE regulation_events ADD COLUMN IF NOT EXISTS raw_text TEXT;
|
||||
"""
|
||||
|
||||
_ALL_COLUMNS = (
|
||||
"id", "source", "source_label", "standard_code", "title", "summary",
|
||||
"full_text_url", "status", "impact_level", "published_at", "effective_at",
|
||||
"category", "tags", "obligations", "deadlines", "scope", "penalties",
|
||||
"content_hash", "previous_hash", "change_summary", "changed_sections",
|
||||
"affected_docs", "crawled_at", "processed_at", "raw_storage_key",
|
||||
"affected_docs", "crawled_at", "processed_at", "raw_storage_key", "raw_text",
|
||||
)
|
||||
|
||||
|
||||
@@ -97,6 +102,10 @@ class PostgresEventStore(BaseEventStore):
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(_CREATE_TABLE)
|
||||
# CREATE TABLE IF NOT EXISTS is a no-op on deployments that
|
||||
# already have this table, so new columns must be added
|
||||
# explicitly or existing installations silently lack them.
|
||||
cur.execute(_ADD_COLUMNS)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
"""PostgreSQL-backed notification store.
|
||||
|
||||
One row per triggering event, shared by every user; a separate read-receipt
|
||||
table tracks per-user read state so broadcasting to everyone needs no fan-out
|
||||
insert per user. See base_notification_store.py for the port contract and the
|
||||
design doc for why this shape was chosen over per-user subscriptions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
from psycopg2.pool import ThreadedConnectionPool
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
|
||||
|
||||
_CREATE_TABLES = """
|
||||
CREATE TABLE IF NOT EXISTS perception_notifications (
|
||||
id SERIAL PRIMARY KEY,
|
||||
event_id TEXT NOT NULL REFERENCES regulation_events(id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
impact_level TEXT,
|
||||
summary TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS perception_notification_reads (
|
||||
notification_id INTEGER NOT NULL REFERENCES perception_notifications(id) ON DELETE CASCADE,
|
||||
user_id TEXT NOT NULL,
|
||||
read_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (notification_id, user_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS perception_notif_created
|
||||
ON perception_notifications (created_at DESC);
|
||||
"""
|
||||
|
||||
|
||||
def _row_to_dict(row: dict[str, Any]) -> dict:
|
||||
"""Convert a psycopg2 RealDictRow to a plain dict with an ISO timestamp."""
|
||||
d = dict(row)
|
||||
if d.get("created_at") is not None:
|
||||
d["created_at"] = d["created_at"].isoformat()
|
||||
return d
|
||||
|
||||
|
||||
class PostgresNotificationStore(BaseNotificationStore):
|
||||
"""Notification store backed by PostgreSQL."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Open a connection pool and ensure both tables exist."""
|
||||
self._pool = ThreadedConnectionPool(
|
||||
minconn=1,
|
||||
maxconn=5,
|
||||
host=settings.postgres_host,
|
||||
port=settings.postgres_port,
|
||||
user=settings.postgres_user,
|
||||
password=settings.postgres_password,
|
||||
dbname=settings.postgres_db,
|
||||
)
|
||||
self._ensure_schema()
|
||||
|
||||
def _ensure_schema(self) -> None:
|
||||
with self._conn() as conn:
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(_CREATE_TABLES)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
|
||||
@contextmanager
|
||||
def _conn(self):
|
||||
conn = None
|
||||
try:
|
||||
conn = self._pool.getconn()
|
||||
yield conn
|
||||
finally:
|
||||
if conn is not None:
|
||||
self._pool.putconn(conn)
|
||||
|
||||
def create(
|
||||
self,
|
||||
*,
|
||||
event_id: str,
|
||||
kind: str,
|
||||
title: str,
|
||||
impact_level: str | None,
|
||||
summary: str | None,
|
||||
) -> None:
|
||||
"""Insert one notification row for the triggering event."""
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO perception_notifications "
|
||||
"(event_id, kind, title, impact_level, summary) "
|
||||
"VALUES (%s, %s, %s, %s, %s)",
|
||||
(event_id, kind, title, impact_level, summary),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
|
||||
"""Return the newest notifications with this user's read state joined in."""
|
||||
with self._conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT n.*, (r.user_id IS NOT NULL) AS read
|
||||
FROM perception_notifications n
|
||||
LEFT JOIN perception_notification_reads r
|
||||
ON r.notification_id = n.id AND r.user_id = %s
|
||||
ORDER BY n.created_at DESC
|
||||
LIMIT %s
|
||||
""",
|
||||
(user_id, limit),
|
||||
)
|
||||
return [_row_to_dict(r) for r in cur.fetchall()]
|
||||
|
||||
def unread_count(self, user_id: str) -> int:
|
||||
"""Count notifications with no read receipt for this user."""
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM perception_notifications n
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM perception_notification_reads r
|
||||
WHERE r.notification_id = n.id AND r.user_id = %s
|
||||
)
|
||||
""",
|
||||
(user_id,),
|
||||
)
|
||||
return cur.fetchone()[0]
|
||||
|
||||
def mark_all_read(self, user_id: str) -> int:
|
||||
"""Insert a read receipt for every notification this user hasn't read."""
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO perception_notification_reads (notification_id, user_id)
|
||||
SELECT n.id, %s FROM perception_notifications n
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM perception_notification_reads r
|
||||
WHERE r.notification_id = n.id AND r.user_id = %s
|
||||
)
|
||||
ON CONFLICT (notification_id, user_id) DO NOTHING
|
||||
""",
|
||||
(user_id, user_id),
|
||||
)
|
||||
marked = cur.rowcount
|
||||
conn.commit()
|
||||
return marked
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Deterministic change detection between two versions of a regulation.
|
||||
|
||||
This module deliberately contains no LLM call, no network access, and no
|
||||
embedding lookup. It exists because the previous implementation decided whether
|
||||
a paragraph had changed by comparing embedding cosine similarity against a 0.85
|
||||
threshold, which is blind to exactly the edits that matter in regulation.
|
||||
Measured against the deployed text-embedding-v3 gateway, tightening a braking
|
||||
limit from 30米 to 20米 scores 0.9153 and relaxing 应当 to 宜 scores 0.9162 —
|
||||
both far above the threshold, both undetected — while an entirely unrelated
|
||||
clause scores 0.6862 and is the only thing that fires. Cosine is scale
|
||||
invariant, so it cannot represent a change in magnitude or certainty
|
||||
(arXiv:2403.05440, ACM Web Conference 2024); no threshold recovers the signal.
|
||||
|
||||
The replacement is the production consensus for legal text: align paragraphs
|
||||
with a longest-common-subsequence matcher, run a literal character diff on the
|
||||
aligned pairs, and let cheap deterministic rules decide whether a change is
|
||||
significant enough to spend an LLM call classifying.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from diff_match_patch import diff_match_patch
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
from app.config.settings import settings
|
||||
|
||||
# Chinese regulatory drafting uses a small, near-unambiguous set of deontic
|
||||
# markers, so a regex pre-pass identifies legally significant edits without an
|
||||
# LLM. Adding or removing any of these changes what the provision compels.
|
||||
_DEONTIC_PATTERN = re.compile(r"应当|须|禁止|不得|可以|允许|宜")
|
||||
|
||||
# Matches digit runs including decimals, so "30" -> "20" and "0.85" -> "0.9"
|
||||
# are both treated as numeric changes.
|
||||
_NUMBER_PATTERN = re.compile(r"\d+(?:\.\d+)?")
|
||||
|
||||
# diff_match_patch operation codes.
|
||||
_DMP_DELETE = -1
|
||||
_DMP_INSERT = 1
|
||||
_DMP_EQUAL = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParagraphChange:
|
||||
"""One detected difference between the old and new version of a regulation.
|
||||
|
||||
`needs_llm` is the gate: it records whether this change is worth the cost of
|
||||
an LLM classification call. The deterministic flags that drive it are kept
|
||||
on the record so downstream code can act on them even when the LLM call
|
||||
fails or is skipped.
|
||||
"""
|
||||
|
||||
change_type: str
|
||||
old_text: str
|
||||
new_text: str
|
||||
numeric_changed: bool
|
||||
deontic_changed: bool
|
||||
change_ratio: float
|
||||
needs_llm: bool
|
||||
# (op, text) pairs from diff_match_patch, for rendering a redline view.
|
||||
diff_spans: list[tuple[int, str]] = field(default_factory=list)
|
||||
|
||||
|
||||
def _split_paragraphs(text: str) -> list[str]:
|
||||
"""Split regulation text into comparable units, dropping blank lines.
|
||||
|
||||
ponytail: newline splitting, not clause parsing. Upgrade to 第X条 / X.X.X
|
||||
segmentation only if paragraph granularity proves too coarse in practice.
|
||||
"""
|
||||
return [line.strip() for line in (text or "").split("\n") if line.strip()]
|
||||
|
||||
|
||||
def _numbers_differ(old: str, new: str) -> bool:
|
||||
"""Report whether the two spans contain a different sequence of numbers."""
|
||||
return _NUMBER_PATTERN.findall(old) != _NUMBER_PATTERN.findall(new)
|
||||
|
||||
|
||||
def _deontic_differs(old: str, new: str) -> bool:
|
||||
"""Report whether obligation markers were added, removed, or swapped."""
|
||||
return sorted(_DEONTIC_PATTERN.findall(old)) != sorted(_DEONTIC_PATTERN.findall(new))
|
||||
|
||||
|
||||
def _is_cosmetic(spans: list[tuple[int, str]]) -> bool:
|
||||
"""Report whether the edit touched nothing but punctuation and whitespace.
|
||||
|
||||
A change ratio alone cannot answer this for Chinese regulation text. Clauses
|
||||
run 20-60 characters, so deleting a single 。 is a 4% change and clears any
|
||||
threshold low enough to still catch real edits in longer paragraphs. Testing
|
||||
what actually changed is both cheaper and exact.
|
||||
"""
|
||||
changed = "".join(text for op, text in spans if op != _DMP_EQUAL)
|
||||
# Unicode categories P (punctuation), Z (separator) and C (control) cover
|
||||
# Chinese and ASCII punctuation plus every flavour of whitespace.
|
||||
return all(unicodedata.category(char)[0] in {"P", "Z", "C"} for char in changed)
|
||||
|
||||
|
||||
class RegulationDiffer:
|
||||
"""Align two regulation versions and classify what changed, without an LLM."""
|
||||
|
||||
def __init__(self, min_change_ratio: float | None = None) -> None:
|
||||
"""Store the gate threshold, defaulting to the configured value.
|
||||
|
||||
The explicit argument exists so tests never depend on the deployed .env.
|
||||
"""
|
||||
self._min_change_ratio = (
|
||||
settings.perception_diff_min_change_ratio
|
||||
if min_change_ratio is None
|
||||
else min_change_ratio
|
||||
)
|
||||
self._dmp = diff_match_patch()
|
||||
|
||||
def diff(self, old_text: str, new_text: str) -> list[ParagraphChange]:
|
||||
"""Return every changed paragraph between two versions.
|
||||
|
||||
Unchanged paragraphs are not returned. An empty old version means there
|
||||
is no baseline to compare against — the caller's first crawl — so no
|
||||
changes are reported rather than the whole document being called new.
|
||||
"""
|
||||
old_paras = _split_paragraphs(old_text)
|
||||
new_paras = _split_paragraphs(new_text)
|
||||
|
||||
if not old_paras or not new_paras:
|
||||
return []
|
||||
|
||||
# autojunk=False is load-bearing: the default treats any element
|
||||
# appearing in over 1% of a sequence of 200+ items as junk, and
|
||||
# regulations repeat boilerplate paragraphs that alignment depends on
|
||||
# as anchors.
|
||||
matcher = SequenceMatcher(None, old_paras, new_paras, autojunk=False)
|
||||
|
||||
changes: list[ParagraphChange] = []
|
||||
for tag, i1, i2, j1, j2 in matcher.get_opcodes():
|
||||
if tag == "equal":
|
||||
continue
|
||||
if tag == "insert":
|
||||
changes.extend(self._added(p) for p in new_paras[j1:j2])
|
||||
elif tag == "delete":
|
||||
changes.extend(self._removed(p) for p in old_paras[i1:i2])
|
||||
elif tag == "replace":
|
||||
changes.extend(self._replaced(old_paras[i1:i2], new_paras[j1:j2]))
|
||||
|
||||
return changes
|
||||
|
||||
def _added(self, paragraph: str) -> ParagraphChange:
|
||||
"""Build a record for a provision present only in the new version."""
|
||||
return ParagraphChange(
|
||||
change_type="added",
|
||||
old_text="",
|
||||
new_text=paragraph,
|
||||
numeric_changed=False,
|
||||
deontic_changed=bool(_DEONTIC_PATTERN.search(paragraph)),
|
||||
change_ratio=1.0,
|
||||
# A new provision always carries new obligations, so it is always
|
||||
# worth classifying.
|
||||
needs_llm=True,
|
||||
diff_spans=[(_DMP_INSERT, paragraph)],
|
||||
)
|
||||
|
||||
def _removed(self, paragraph: str) -> ParagraphChange:
|
||||
"""Build a record for a provision dropped from the new version."""
|
||||
return ParagraphChange(
|
||||
change_type="removed",
|
||||
old_text=paragraph,
|
||||
new_text="",
|
||||
numeric_changed=False,
|
||||
deontic_changed=bool(_DEONTIC_PATTERN.search(paragraph)),
|
||||
change_ratio=1.0,
|
||||
needs_llm=True,
|
||||
diff_spans=[(_DMP_DELETE, paragraph)],
|
||||
)
|
||||
|
||||
def _replaced(self, old_block: list[str], new_block: list[str]) -> list[ParagraphChange]:
|
||||
"""Compare a run of rewritten paragraphs pairwise, reporting the remainder.
|
||||
|
||||
SequenceMatcher emits `replace` for a whole run at once, and the two
|
||||
sides may differ in length. Pairing by position within the run is safe
|
||||
here because alignment has already established that this run as a whole
|
||||
corresponds; any surplus on either side is a genuine insertion or
|
||||
deletion.
|
||||
"""
|
||||
results: list[ParagraphChange] = []
|
||||
for index in range(max(len(old_block), len(new_block))):
|
||||
if index >= len(old_block):
|
||||
results.append(self._added(new_block[index]))
|
||||
elif index >= len(new_block):
|
||||
results.append(self._removed(old_block[index]))
|
||||
else:
|
||||
results.append(self._modified(old_block[index], new_block[index]))
|
||||
return results
|
||||
|
||||
def _modified(self, old: str, new: str) -> ParagraphChange:
|
||||
"""Character-diff an aligned pair and decide whether it warrants an LLM call."""
|
||||
spans = self._dmp.diff_main(old, new)
|
||||
# Merges single-character edits into human-meaningful chunks so the
|
||||
# redline view and the change ratio both reflect real edits.
|
||||
self._dmp.diff_cleanupSemantic(spans)
|
||||
|
||||
changed_chars = sum(len(text) for op, text in spans if op != _DMP_EQUAL)
|
||||
denominator = max(len(old), len(new), 1)
|
||||
change_ratio = changed_chars / denominator
|
||||
|
||||
numeric_changed = _numbers_differ(old, new)
|
||||
deontic_changed = _deontic_differs(old, new)
|
||||
|
||||
# A changed limit or obligation marker is always significant no matter
|
||||
# how few characters moved. Everything else must be substantive and
|
||||
# clear the ratio gate to be worth a model call.
|
||||
significant = numeric_changed or deontic_changed or (
|
||||
not _is_cosmetic(spans) and change_ratio >= self._min_change_ratio
|
||||
)
|
||||
|
||||
return ParagraphChange(
|
||||
change_type="modified",
|
||||
old_text=old,
|
||||
new_text=new,
|
||||
numeric_changed=numeric_changed,
|
||||
deontic_changed=deontic_changed,
|
||||
change_ratio=change_ratio,
|
||||
needs_llm=significant,
|
||||
diff_spans=[(op, text) for op, text in spans],
|
||||
)
|
||||
@@ -28,7 +28,10 @@ celery_app = Celery(
|
||||
"compliance_hub",
|
||||
broker=_BROKER,
|
||||
backend=_BACKEND,
|
||||
include=["app.infrastructure.tasks.document_tasks"],
|
||||
include=[
|
||||
"app.infrastructure.tasks.document_tasks",
|
||||
"app.infrastructure.tasks.perception_tasks",
|
||||
],
|
||||
)
|
||||
|
||||
celery_app.conf.update(
|
||||
@@ -42,4 +45,12 @@ celery_app.conf.update(
|
||||
task_reject_on_worker_lost=True,
|
||||
# Keep results for 1 hour for status polling.
|
||||
result_expires=3600,
|
||||
# Scheduled counterpart to the Perception page's manual "Refresh" button.
|
||||
# Only takes effect while a Beat process is running (./dev.sh start beat).
|
||||
beat_schedule={
|
||||
"crawl-regulations-periodic": {
|
||||
"task": "app.infrastructure.tasks.perception_tasks.crawl_regulations_task",
|
||||
"schedule": settings.perception_crawl_interval_seconds,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Celery task for scheduled regulatory source crawling.
|
||||
|
||||
This is the scheduled counterpart to the Perception page's manual "Refresh"
|
||||
button (POST /perception/crawl). Every architecture reference document
|
||||
describes source monitoring as continuous ("定时爬取"), not operator-triggered,
|
||||
so this task is what Celery Beat runs on a fixed interval once an operator
|
||||
starts a Beat process.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from app.infrastructure.tasks.celery_app import celery_app
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
name="app.infrastructure.tasks.perception_tasks.crawl_regulations_task",
|
||||
bind=True,
|
||||
)
|
||||
def crawl_regulations_task(self) -> dict:
|
||||
"""Crawl every registered regulatory source and enrich new/changed events.
|
||||
|
||||
Drains CrawlService.run_crawl(), which already isolates each source's
|
||||
fetch and each event's enrichment behind its own try/except — a source
|
||||
outage or a single bad event yields an "error" progress item and the
|
||||
generator continues. Re-catching those here would only hide problems the
|
||||
service has already handled, so this task's job is limited to counting
|
||||
them and logging a summary.
|
||||
|
||||
No automatic retry is configured. An exception escaping run_crawl itself
|
||||
means something broke in a way the service's own error handling did not
|
||||
anticipate; the next scheduled tick already provides a retry within
|
||||
settings.perception_crawl_interval_seconds, so an immediate retry against
|
||||
the same failure is not worth the added complexity.
|
||||
|
||||
ponytail: relies on a single worker process to serialize scheduled runs
|
||||
(Celery's default concurrency processes one task at a time, so a run that
|
||||
outlasts the interval delays the next tick rather than overlapping it).
|
||||
Add a Redis-based lock (e.g. SETNX on a per-task key) if this queue is
|
||||
ever served by more than one worker.
|
||||
"""
|
||||
from app.shared.bootstrap import get_crawl_service
|
||||
|
||||
error_count = 0
|
||||
new_count = 0
|
||||
updated_count = 0
|
||||
|
||||
for item in get_crawl_service().run_crawl():
|
||||
event = item.get("event")
|
||||
if event == "error":
|
||||
error_count += 1
|
||||
logger.warning("Scheduled crawl source error: {}", item.get("data"))
|
||||
elif event == "done":
|
||||
data = item.get("data") or {}
|
||||
new_count = data.get("total_new", 0)
|
||||
updated_count = data.get("total_updated", 0)
|
||||
|
||||
logger.info(
|
||||
"Scheduled crawl finished: new={} updated={} source_errors={}",
|
||||
new_count, updated_count, error_count,
|
||||
)
|
||||
return {"new": new_count, "updated": updated_count, "source_errors": error_count}
|
||||
@@ -16,7 +16,7 @@ from app.shared.model_usage_tracker import get_model_usage_tracker
|
||||
# Keep provider-specific behavior explicit so debugging stays straightforward.
|
||||
DEFAULT_MODELS = {
|
||||
LLMProvider.DEEPSEEK: "deepseek-v4-flash",
|
||||
LLMProvider.QWEN: "qwen3.5-flash",
|
||||
LLMProvider.QWEN: "qwen3.6-flash",
|
||||
LLMProvider.QWEN_VL: "qwen3-vl-plus"
|
||||
}
|
||||
|
||||
@@ -101,6 +101,8 @@ class LLMFactory:
|
||||
"qwen-max": LLMProvider.QWEN,
|
||||
"qwen3.5-flash": LLMProvider.QWEN,
|
||||
"qwen3.5-plus": LLMProvider.QWEN,
|
||||
"qwen3.6-flash": LLMProvider.QWEN,
|
||||
"qwen3.6-plus": LLMProvider.QWEN,
|
||||
"qwen_vl": LLMProvider.QWEN_VL,
|
||||
"qwen-vl": LLMProvider.QWEN_VL,
|
||||
"qwen-vl-plus": LLMProvider.QWEN_VL,
|
||||
|
||||
@@ -27,6 +27,8 @@ class QwenClient(BaseLLMClient):
|
||||
"qwen-long",
|
||||
"qwen3.5-flash",
|
||||
"qwen3.5-plus",
|
||||
"qwen3.6-flash",
|
||||
"qwen3.6-plus",
|
||||
"qwen3-plus",
|
||||
"qwen2.5-72b-instruct",
|
||||
"qwen2.5-32b-instruct",
|
||||
@@ -371,7 +373,7 @@ class QwenVLClient(BaseLLMClient):
|
||||
|
||||
def create_qwen_client(
|
||||
api_key: str,
|
||||
model: str = "qwen3.5-flash",
|
||||
model: str = "qwen3.6-flash",
|
||||
base_url: str = "http://6.86.80.4:30080/v1",
|
||||
**kwargs
|
||||
) -> QwenClient:
|
||||
|
||||
@@ -23,8 +23,10 @@ from app.infrastructure.parser.local_chunk_builder import LocalRegulationChunkBu
|
||||
from app.infrastructure.parser.local_document_parser import LocalDocumentParser
|
||||
from app.infrastructure.parser.vector_chunk_builder import AliyunVectorChunkBuilder
|
||||
from app.infrastructure.perception.mock_event_store import MockEventStore
|
||||
from app.infrastructure.perception.mock_notification_store import MockNotificationStore
|
||||
from app.application.perception.crawl_service import CrawlService
|
||||
from app.infrastructure.perception.base_event_store import BaseEventStore
|
||||
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
|
||||
from app.infrastructure.perception.crawlers.catarc_crawler import CatarcCrawler
|
||||
from app.infrastructure.perception.crawlers.guobiao_crawler import (
|
||||
GuobiaoMandatoryCrawler,
|
||||
@@ -327,6 +329,22 @@ def get_event_store() -> BaseEventStore:
|
||||
return MockEventStore()
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_notification_store() -> BaseNotificationStore:
|
||||
"""Return notification store selected by DOCUMENT_REPOSITORY_BACKEND setting.
|
||||
|
||||
Mirrors get_event_store()'s gate: Mock in-memory when Postgres isn't
|
||||
configured, so the feature works in local dev and tests without a
|
||||
database.
|
||||
"""
|
||||
if settings.document_repository_backend == "postgres":
|
||||
from app.infrastructure.perception.postgres_notification_store import (
|
||||
PostgresNotificationStore,
|
||||
)
|
||||
return PostgresNotificationStore()
|
||||
return MockNotificationStore()
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_compliance_repository() -> ComplianceRepository:
|
||||
"""Return the compliance analysis repository.
|
||||
@@ -370,6 +388,9 @@ def get_crawl_service() -> CrawlService:
|
||||
event_store=get_event_store(),
|
||||
llm_pipeline=LlmPipeline(),
|
||||
retrieval_service=get_retrieval_service(),
|
||||
notification_store=get_notification_store(),
|
||||
embedding_provider=get_embedding_provider(),
|
||||
vector_index=get_vector_index(),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user