update for mcp

This commit is contained in:
wangwei
2026-08-06 11:08:46 +08:00
parent 31bbf80aeb
commit b2feaeddb4
40 changed files with 1986 additions and 202 deletions
+29 -1
View File
@@ -7,7 +7,12 @@ import json
from fastapi import APIRouter, Depends, Query
from fastapi.responses import StreamingResponse
from app.shared.bootstrap import get_crawl_service, get_event_store, get_perception_service
from app.shared.bootstrap import (
get_crawl_service,
get_event_store,
get_notification_store,
get_perception_service,
)
from app.api.dependencies.auth import get_current_user
from app.domain.auth.models import UserClaims
from app.shared.async_utils import iter_in_thread
@@ -141,3 +146,26 @@ async def get_event_diff(event_id: str):
"previous_hash": event.get("previous_hash"),
"content_hash": event.get("content_hash"),
}
@router.get("/notifications")
async def list_notifications(
limit: int = Query(default=20, ge=1, le=100),
current_user: UserClaims = Depends(get_current_user),
):
"""Return the newest in-app notifications plus this user's unread count.
Every logged-in user sees the same broadcast feed — there is no per-role
or per-topic subscription. "read" per item and the aggregate unread_count
both reflect only the calling user's own read receipts.
"""
store = get_notification_store()
items = store.list_for_user(current_user.user_id, limit=limit)
return {"items": items, "unread_count": store.unread_count(current_user.user_id)}
@router.post("/notifications/read")
async def mark_notifications_read(current_user: UserClaims = Depends(get_current_user)):
"""Mark every currently-unread notification read for the calling user."""
marked = get_notification_store().mark_all_read(current_user.user_id)
return {"marked": marked}
@@ -73,7 +73,7 @@ class HyDEExpander:
return query
# Use the dedicated HyDE model when configured; fall back to main LLM.
# A lightweight model (e.g. qwen3.5-flash) is sufficient for generating
# A lightweight model (e.g. qwen3.6-flash) is sufficient for generating
# a short hypothetical passage and significantly reduces cost + latency.
provider = settings.hyde_llm_provider or settings.llm_provider
model = settings.hyde_llm_model or settings.llm_model
@@ -7,9 +7,13 @@ from typing import Any, Generator
from loguru import logger
from app.config.settings import settings
from app.domain.documents import ParsedDocument
from app.infrastructure.perception.base_event_store import BaseEventStore
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
from app.infrastructure.perception.llm_pipeline import LlmPipeline
from app.infrastructure.parser.local_chunk_builder import LocalRegulationChunkBuilder
def _event_id(source: str, standard_code: str) -> str:
@@ -21,7 +25,68 @@ def _content_hash(raw_text: str) -> str:
return hashlib.sha256(raw_text.encode()).hexdigest()
def _raw_to_dict(raw: RawEvent, event_id: str, content_hash: str) -> dict:
def _is_significant(changed_sections: list[dict]) -> bool:
"""Report whether any changed section is worth notifying every user about.
changed_sections legitimately includes cosmetic edits — the differ
(subproject 1) still reports a fixed typo or a dropped trailing period as
a change, it just doesn't send those to the LLM. Broadcasting a
notification for every cosmetic edit would train people to ignore it, so
this reuses the same significance test the differ's own LLM gate applies:
a numeric or deontic change, or a whole paragraph added or removed.
"""
return any(
section.get("numeric_changed")
or section.get("deontic_changed")
or section.get("change_type") in ("added", "removed")
for section in changed_sections
)
def _index_in_knowledge_base(event: dict, *, embedding_provider: Any, vector_index: Any) -> None:
"""Chunk, embed, and upsert a regulation's text into the shared knowledge base.
Always uses the local markdown chunker, never get_chunk_builder() — that
bootstrap function resolves to AliyunVectorChunkBuilder when
settings.chunk_backend == "aliyun" (the deployed value), which consumes
Aliyun DocMind's structured parse output. Crawled text has no such parse
output; it is already plain text (trafilatura, subproject 1), which is
exactly what LocalRegulationChunkBuilder chunks directly.
delete_by_document runs unconditionally before upsert — a no-op for a
brand-new event, and the only way to keep a changed regulation from
leaving its superseded text retrievable alongside the new version.
"""
vector_index.delete_by_document(event["id"])
parsed = ParsedDocument(
doc_id=event["id"],
doc_name=event.get("title", ""),
structure_nodes=[],
semantic_blocks=[],
vector_chunks=[],
parser_name="perception_crawl",
raw_text=event.get("raw_text") or "",
)
builder = LocalRegulationChunkBuilder(
chunk_size=settings.chunk_size, chunk_overlap=settings.chunk_overlap,
)
chunks = builder.build(
parsed_document=parsed,
# regulation_type/version fill the same slots a manually uploaded
# document's form fields would, so the two intake paths are
# indistinguishable to retrieval and compliance analysis.
regulation_type=event.get("category", ""),
version=event.get("standard_code", ""),
)
if not chunks:
return
vectors = embedding_provider.embed_texts([c.embedding_text for c in chunks])
vector_index.upsert(chunks, vectors)
def _raw_to_dict(raw: RawEvent, event_id: str, content_hash: str, raw_text: str) -> dict:
return {
"id": event_id,
"source": raw.source,
@@ -36,6 +101,10 @@ def _raw_to_dict(raw: RawEvent, event_id: str, content_hash: str) -> dict:
"effective_at": raw.effective_at,
"category": raw.category,
"tags": raw.tags,
# Persisted so the next crawl has a baseline to diff against. Without
# this the change detector has nothing to compare and every update
# looks like a first sighting.
"raw_text": raw_text,
"content_hash": content_hash,
"previous_hash": None,
}
@@ -50,11 +119,17 @@ class CrawlService:
event_store: BaseEventStore,
llm_pipeline: LlmPipeline,
retrieval_service: Any,
notification_store: BaseNotificationStore,
embedding_provider: Any,
vector_index: Any,
) -> None:
self._crawlers = crawlers
self._store = event_store
self._pipeline = llm_pipeline
self._retrieval = retrieval_service
self._notifications = notification_store
self._embedding_provider = embedding_provider
self._vector_index = vector_index
def run_crawl(
self, sources: list[str] | None = None
@@ -72,7 +147,7 @@ class CrawlService:
yield {"event": "progress", "data": {"source": source_key, "stage": "fetching"}}
try:
raw_events = crawler.fetch(limit=100)
raw_events = crawler.fetch(limit=settings.perception_max_events_per_source)
except Exception as exc:
logger.exception("Crawler failed source={}", source_key)
yield {"event": "error", "data": {"source": source_key, "message": str(exc)}}
@@ -88,17 +163,21 @@ class CrawlService:
for raw in raw_events:
eid = _event_id(raw.source, raw.standard_code)
new_hash = _content_hash(raw.raw_text or raw.title)
# List pages carry only a code and a title, which is not enough
# to detect a change in the regulation itself. Fetch the body,
# degrading to whatever the list page gave us if that fails.
body_text = crawler.fetch_full_text(raw.full_text_url) or raw.raw_text or raw.title
new_hash = _content_hash(body_text)
existing = self._store.get(eid)
if existing and existing.get("content_hash") == new_hash:
continue
is_update = existing is not None
old_text = existing.get("summary", "") if is_update else ""
old_body = existing.get("raw_text") or "" if is_update else ""
previous_hash = existing.get("content_hash") if is_update else None
event_dict = _raw_to_dict(raw, eid, new_hash)
event_dict = _raw_to_dict(raw, eid, new_hash, body_text)
event_dict["previous_hash"] = previous_hash
try:
@@ -113,9 +192,11 @@ class CrawlService:
except Exception as exc:
logger.warning("Impact assessment failed id={} err={}", eid, exc)
if is_update and old_text and raw.raw_text:
# Events stored before raw_text was persisted have no baseline,
# so they are treated as a first sighting and establish one now.
if is_update and old_body and body_text:
try:
diff = self._pipeline.compute_diff(old_text, raw.raw_text)
diff = self._pipeline.compute_diff(old_body, body_text)
event_dict["change_summary"] = diff.get("change_summary")
event_dict["changed_sections"] = diff.get("changed_sections")
except Exception as exc:
@@ -123,6 +204,33 @@ class CrawlService:
self._store.upsert(event_dict)
should_index = not is_update or _is_significant(event_dict.get("changed_sections") or [])
try:
if not is_update:
self._notifications.create(
event_id=eid, kind="new", title=raw.title,
impact_level=event_dict.get("impact_level"), summary=None,
)
elif should_index: # significant change, already computed above
self._notifications.create(
event_id=eid, kind="changed", title=raw.title,
impact_level=event_dict.get("impact_level"),
summary=event_dict.get("change_summary"),
)
except Exception as exc:
logger.warning("Notification create failed id={} err={}", eid, exc)
if should_index:
try:
_index_in_knowledge_base(
event_dict,
embedding_provider=self._embedding_provider,
vector_index=self._vector_index,
)
except Exception as exc:
logger.warning("Knowledge base indexing failed id={} err={}", eid, exc)
if is_update:
updated_count += 1
else:
+16 -4
View File
@@ -94,9 +94,21 @@ class Settings(BaseSettings):
perception_max_events_per_source: int = Field(
default=100, description="Maximum events fetched per source per crawl run."
)
perception_diff_similarity_threshold: float = Field(
default=0.85,
description="Cosine similarity below which a paragraph is flagged as changed.",
perception_diff_min_change_ratio: float = Field(
default=0.02,
description=(
"Fraction of characters that must differ before an otherwise "
"unremarkable paragraph edit is worth an LLM classification call. "
"Numeric and deontic changes bypass this gate entirely."
),
)
perception_crawl_interval_seconds: int = Field(
default=21600,
description=(
"How often Celery Beat runs the scheduled crawl-all-sources task, "
"in seconds. Default 21600 = 6 hours. Only takes effect when a "
"Beat process is running (./dev.sh start beat)."
),
)
# Keep configuration setup explicit so runtime behavior is easy to reason about.
@@ -117,7 +129,7 @@ class Settings(BaseSettings):
# Keep configuration setup explicit so runtime behavior is easy to reason about.
qwen_api_key: str = Field(default="", description="Qwen API密钥")
qwen_base_url: str = Field(default="http://6.86.80.4:30080/v1", description="Qwen API地址")
qwen_model: str = Field(default="qwen3.5-flash", description="Qwen文本模型")
qwen_model: str = Field(default="qwen3.6-flash", description="Qwen文本模型")
qwen_vl_model: str = Field(default="qwen3-vl-plus", description="Qwen视觉模型")
# Keep configuration setup explicit so runtime behavior is easy to reason about.
@@ -0,0 +1,47 @@
"""Abstract base class for in-app regulatory-signal notifications.
A notification is created once per triggering event (a brand-new regulation,
or a significant change to an existing one) and broadcast to every logged-in
user. There is no per-user subscription targeting — see the design doc for why.
Per-user "read" state is tracked separately from the notification itself, so
one notification row serves every user rather than being fanned out on create.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
class BaseNotificationStore(ABC):
"""Port interface for perception notification persistence."""
@abstractmethod
def create(
self,
*,
event_id: str,
kind: str,
title: str,
impact_level: str | None,
summary: str | None,
) -> None:
"""Record a new notification. kind is 'new' or 'changed'."""
@abstractmethod
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
"""Return the most recent notifications, newest first.
Each item includes a "read" boolean reflecting whether `user_id` has
marked it read.
"""
@abstractmethod
def unread_count(self, user_id: str) -> int:
"""Return how many notifications `user_id` has not yet read."""
@abstractmethod
def mark_all_read(self, user_id: str) -> int:
"""Mark every currently-unread notification read for `user_id`.
Returns the number of notifications newly marked.
"""
@@ -5,6 +5,12 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
import httpx
import trafilatura
from loguru import logger
from app.config.settings import settings
@dataclass
class RawEvent:
@@ -21,7 +27,10 @@ class RawEvent:
effective_at: str | None
category: str
tags: list[str] = field(default_factory=list)
raw_text: str = "" # full crawled text for hashing + LLM
# Whatever text the list page yields. CrawlService upgrades this by calling
# fetch_full_text(full_text_url); this value is the fallback when that
# fails. Used for change hashing and for the version diff.
raw_text: str = ""
class BaseCrawler(ABC):
@@ -30,3 +39,37 @@ class BaseCrawler(ABC):
@abstractmethod
def fetch(self, limit: int = 50) -> list[RawEvent]:
"""Fetch up to `limit` recent events from the data source."""
def fetch_full_text(self, url: str) -> str:
"""Download a regulation detail page and extract its body text.
Change detection is only as good as the text it compares, and list
pages carry nothing but a standard code and a title. This default
implementation serves all current sources; a source that needs PDF
extraction or authentication overrides this one method.
Returns an empty string on any failure rather than raising, so one
unreachable page cannot abort a whole crawl run. The caller decides how
to degrade.
"""
if not url:
return ""
try:
response = httpx.get(
url,
timeout=settings.perception_crawl_timeout_seconds,
follow_redirects=True,
)
response.raise_for_status()
except Exception as exc: # noqa: BLE001 - any transport error degrades the same way
logger.warning("Full-text fetch failed url={} err={}", url, exc)
return ""
# trafilatura scores 0.92 F1 on government pages against 0.78 for
# readability-lxml, and handles CJK content; include_tables matters
# because regulatory limits are frequently tabulated.
extracted = trafilatura.extract(response.text, include_tables=True)
if not extracted:
logger.warning("Full-text extraction returned nothing url={}", url)
return ""
return extracted.strip()
@@ -8,6 +8,7 @@ import httpx
from bs4 import BeautifulSoup
from loguru import logger
from app.config.settings import settings
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
from ._utils import extract_tags, parse_date
@@ -33,7 +34,11 @@ class CatarcCrawler(BaseCrawler):
while len(events) < limit and page <= max_pages:
url = f"{_BASE_URL}?page={page}"
try:
resp = httpx.get(url, timeout=30, follow_redirects=True)
resp = httpx.get(
url,
timeout=settings.perception_crawl_timeout_seconds,
follow_redirects=True,
)
resp.raise_for_status()
except Exception as exc:
logger.warning("CATARC fetch failed page={} err={}", page, exc)
@@ -9,6 +9,7 @@ import httpx
from bs4 import BeautifulSoup
from loguru import logger
from app.config.settings import settings
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
from ._utils import parse_date
@@ -53,7 +54,11 @@ class EurlexCrawler(BaseCrawler):
if len(events) >= limit:
break
try:
resp = httpx.get(rss_url, timeout=30, follow_redirects=True)
resp = httpx.get(
rss_url,
timeout=settings.perception_crawl_timeout_seconds,
follow_redirects=True,
)
resp.raise_for_status()
except Exception as exc:
logger.warning("EUR-Lex RSS fetch failed url={} err={}", rss_url, exc)
@@ -5,6 +5,7 @@ from __future__ import annotations
import httpx
from loguru import logger
from app.config.settings import settings
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
from ._utils import extract_tags, parse_date
@@ -22,7 +23,12 @@ def _fetch_page(std_type: int, page: int, page_size: int) -> list[dict]:
"p.p7": page_size,
}
try:
resp = httpx.get(_BASE_URL, params=params, headers=_HEADERS, timeout=30)
resp = httpx.get(
_BASE_URL,
params=params,
headers=_HEADERS,
timeout=settings.perception_crawl_timeout_seconds,
)
resp.raise_for_status()
data = resp.json()
return data.get("rows", []) or []
@@ -3,14 +3,14 @@
from __future__ import annotations
import json
import math
from typing import Any
from loguru import logger
from app.config.settings import settings
from app.infrastructure.embedding.openai_compatible_embedding_provider import (
OpenAICompatibleEmbeddingProvider,
from app.infrastructure.perception.regulation_differ import (
ParagraphChange,
RegulationDiffer,
)
from app.services.llm.llm_factory import get_llm_client
@@ -27,21 +27,31 @@ _ASSESS_SYSTEM = (
)
_DIFF_SYSTEM = (
"You are a regulatory change analyst. Given an old and new version of a regulation paragraph, "
"classify the type of change and summarise it. "
"Return JSON only: {\"change_type\": \"tightened|relaxed|added|removed\", \"summary\": \"...\"}"
"You are a regulatory change analyst. You are given the OLD and NEW version of "
"one regulation paragraph, with the exact edits marked <DEL>removed</DEL> and "
"<INS>added</INS>. Classify the legal effect of the change. "
"Return JSON only: {\"change_type\": \"tightened|relaxed|numeric|clarified|scope\", "
"\"legal_effect\": \"one sentence on what this means for compliance\"}"
)
_SIMILARITY_THRESHOLD = 0.85
def _marked_diff(change: ParagraphChange) -> str:
"""Render a paragraph change with the exact edits marked for the model.
def _cosine(a: list[float], b: list[float]) -> float:
dot = sum(x * y for x, y in zip(a, b))
norm_a = math.sqrt(sum(x * x for x in a))
norm_b = math.sqrt(sum(x * x for x in b))
if norm_a == 0 or norm_b == 0:
return 0.0
return dot / (norm_a * norm_b)
The model is shown where the edit is rather than being asked to find it,
and is never asked to reproduce the changed text — the differ already
computed those spans exactly, so there is nothing for the model to
hallucinate.
"""
marked = "".join(
text if op == 0 else (f"<DEL>{text}</DEL>" if op < 0 else f"<INS>{text}</INS>")
for op, text in change.diff_spans
)
return (
f"OLD: {change.old_text[:500]}\n"
f"NEW: {change.new_text[:500]}\n"
f"MARKED: {marked[:800]}"
)
def _llm_json(client: Any, messages: list[dict]) -> Any:
@@ -67,7 +77,9 @@ class LlmPipeline:
provider=settings.llm_provider,
model=settings.llm_model,
)
self._embedder = OpenAICompatibleEmbeddingProvider()
# Change detection is deterministic; the differ needs no model and no
# network, so the pipeline no longer constructs an embedding provider.
self._differ = RegulationDiffer()
# ------------------------------------------------------------------
# Step 1: Structure extraction
@@ -166,76 +178,68 @@ For each document, assess impact and recommend action. Return JSON array:
return doc_excerpts
# ------------------------------------------------------------------
# Step 3: Semantic diff
# Step 3: Deterministic diff with gated LLM classification
# ------------------------------------------------------------------
def compute_diff(self, old_text: str, new_text: str) -> dict:
"""Compare old and new regulation text; return changed sections and summary."""
old_paras = [p.strip() for p in old_text.split("\n") if p.strip()]
new_paras = [p.strip() for p in new_text.split("\n") if p.strip()]
"""Compare old and new regulation text; return changed sections and summary.
if not old_paras or not new_paras:
return {"changed_sections": [], "change_summary": "No comparable text."}
Detection is deterministic — see regulation_differ for why embedding
similarity was removed. The LLM is called only for paragraphs the
differ marked significant, and only to explain the legal effect of a
change that has already been located exactly.
"""
changes = self._differ.diff(old_text, new_text)
if not changes:
return {
"changed_sections": [],
"change_summary": "No substantive changes detected between versions.",
}
all_paras = old_paras + new_paras
try:
all_embeddings = self._embedder.embed_texts(all_paras)
except Exception as exc:
logger.warning("Embedding for diff failed: {}", exc)
return {"changed_sections": [], "change_summary": "Diff unavailable (embedding error)."}
changed_sections = [self._describe(change) for change in changes]
old_embeddings = all_embeddings[: len(old_paras)]
new_embeddings = all_embeddings[len(old_paras):]
changed_sections: list[dict] = []
max_len = max(len(old_paras), len(new_paras))
for i in range(max_len):
if i >= len(old_paras):
# New paragraph added
changed_sections.append({
"old_text": "",
"new_text": new_paras[i][:300],
"similarity": 0.0,
"change_type": "added",
"summary": "New paragraph added.",
})
continue
if i >= len(new_paras):
# Old paragraph removed
changed_sections.append({
"old_text": old_paras[i][:300],
"new_text": "",
"similarity": 0.0,
"change_type": "removed",
"summary": "Paragraph removed.",
})
continue
# Both exist — compare via embeddings
sim = _cosine(old_embeddings[i], new_embeddings[i])
if sim < _SIMILARITY_THRESHOLD:
messages = [
{"role": "system", "content": _DIFF_SYSTEM},
{"role": "user", "content": f"OLD: {old_paras[i][:500]}\nNEW: {new_paras[i][:500]}"},
]
classification = _llm_json(self._client, messages) or {}
changed_sections.append({
"old_text": old_paras[i][:300],
"new_text": new_paras[i][:300],
"similarity": round(sim, 3),
"change_type": classification.get("change_type", "modified"),
"summary": classification.get("summary", ""),
})
if not changed_sections:
change_summary = "No substantive changes detected between versions."
else:
types = [s["change_type"] for s in changed_sections]
change_summary = (
f"{len(changed_sections)} paragraph(s) changed: "
+ ", ".join(f"{t}" for t in set(types))
+ ". "
+ (changed_sections[0].get("summary", "") if changed_sections else "")
)
types = sorted({section["change_type"] for section in changed_sections})
gated = sum(1 for change in changes if change.needs_llm)
change_summary = (
f"{len(changed_sections)} paragraph(s) changed ({', '.join(types)}); "
f"{gated} significant. "
+ (changed_sections[0].get("summary") or "")
).strip()
return {"changed_sections": changed_sections, "change_summary": change_summary}
def _describe(self, change: ParagraphChange) -> dict:
"""Turn one detected change into the API payload, classifying if warranted."""
section = {
"old_text": change.old_text[:300],
"new_text": change.new_text[:300],
"change_type": change.change_type,
"change_ratio": round(change.change_ratio, 3),
"numeric_changed": change.numeric_changed,
"deontic_changed": change.deontic_changed,
"summary": "",
}
if not change.needs_llm:
return section
classification = _llm_json(
self._client,
[
{"role": "system", "content": _DIFF_SYSTEM},
{"role": "user", "content": _marked_diff(change)},
],
)
if isinstance(classification, dict):
section["change_type"] = classification.get("change_type") or change.change_type
section["summary"] = classification.get("legal_effect") or ""
# A failed or malformed model response must not discard a change that
# deterministic analysis already proved real; the section keeps its
# spans, flags, and alignment-derived type with an empty summary.
if change.numeric_changed:
# Models routinely label a changed threshold as "clarified". The
# deterministic pass already knows a number moved, so it wins.
section["change_type"] = "numeric"
return section
@@ -0,0 +1,66 @@
"""In-memory notification store used when Postgres is not configured.
Mirrors MockEventStore's role for BaseEventStore: keeps the feature usable in
local dev and in tests without a live database, and matches
DOCUMENT_REPOSITORY_BACKEND's existing Mock/Postgres split.
"""
from __future__ import annotations
from datetime import UTC, datetime
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
class MockNotificationStore(BaseNotificationStore):
"""Dict-backed notification store. Data does not survive a process restart."""
def __init__(self) -> None:
"""Start with an empty feed and no read receipts."""
self._notifications: list[dict] = []
self._next_id = 1
# (notification_id, user_id) pairs — presence means read.
self._reads: set[tuple[int, str]] = set()
def create(
self,
*,
event_id: str,
kind: str,
title: str,
impact_level: str | None,
summary: str | None,
) -> None:
"""Append a notification with an auto-incrementing id."""
self._notifications.append({
"id": self._next_id,
"event_id": event_id,
"kind": kind,
"title": title,
"impact_level": impact_level,
"summary": summary,
"created_at": datetime.now(UTC).isoformat(),
})
self._next_id += 1
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
"""Return the newest `limit` notifications with this user's read state."""
ordered = sorted(self._notifications, key=lambda n: n["id"], reverse=True)
return [
{**n, "read": (n["id"], user_id) in self._reads}
for n in ordered[:limit]
]
def unread_count(self, user_id: str) -> int:
"""Count notifications this user has not yet read."""
return sum(1 for n in self._notifications if (n["id"], user_id) not in self._reads)
def mark_all_read(self, user_id: str) -> int:
"""Add a read receipt for every currently-unread notification."""
marked = 0
for n in self._notifications:
key = (n["id"], user_id)
if key not in self._reads:
self._reads.add(key)
marked += 1
return marked
@@ -40,7 +40,8 @@ CREATE TABLE IF NOT EXISTS regulation_events (
affected_docs JSONB,
crawled_at TIMESTAMPTZ DEFAULT now(),
processed_at TIMESTAMPTZ,
raw_storage_key TEXT
raw_storage_key TEXT,
raw_text TEXT
);
CREATE INDEX IF NOT EXISTS reg_events_source_date
ON regulation_events (source, published_at DESC);
@@ -48,12 +49,16 @@ CREATE INDEX IF NOT EXISTS reg_events_impact_date
ON regulation_events (impact_level, published_at DESC);
"""
_ADD_COLUMNS = """
ALTER TABLE regulation_events ADD COLUMN IF NOT EXISTS raw_text TEXT;
"""
_ALL_COLUMNS = (
"id", "source", "source_label", "standard_code", "title", "summary",
"full_text_url", "status", "impact_level", "published_at", "effective_at",
"category", "tags", "obligations", "deadlines", "scope", "penalties",
"content_hash", "previous_hash", "change_summary", "changed_sections",
"affected_docs", "crawled_at", "processed_at", "raw_storage_key",
"affected_docs", "crawled_at", "processed_at", "raw_storage_key", "raw_text",
)
@@ -97,6 +102,10 @@ class PostgresEventStore(BaseEventStore):
try:
with conn.cursor() as cur:
cur.execute(_CREATE_TABLE)
# CREATE TABLE IF NOT EXISTS is a no-op on deployments that
# already have this table, so new columns must be added
# explicitly or existing installations silently lack them.
cur.execute(_ADD_COLUMNS)
conn.commit()
except Exception:
conn.rollback()
@@ -0,0 +1,157 @@
"""PostgreSQL-backed notification store.
One row per triggering event, shared by every user; a separate read-receipt
table tracks per-user read state so broadcasting to everyone needs no fan-out
insert per user. See base_notification_store.py for the port contract and the
design doc for why this shape was chosen over per-user subscriptions.
"""
from __future__ import annotations
from contextlib import contextmanager
from typing import Any
import psycopg2
import psycopg2.extras
from psycopg2.pool import ThreadedConnectionPool
from app.config.settings import settings
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
_CREATE_TABLES = """
CREATE TABLE IF NOT EXISTS perception_notifications (
id SERIAL PRIMARY KEY,
event_id TEXT NOT NULL REFERENCES regulation_events(id) ON DELETE CASCADE,
kind TEXT NOT NULL,
title TEXT NOT NULL,
impact_level TEXT,
summary TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS perception_notification_reads (
notification_id INTEGER NOT NULL REFERENCES perception_notifications(id) ON DELETE CASCADE,
user_id TEXT NOT NULL,
read_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (notification_id, user_id)
);
CREATE INDEX IF NOT EXISTS perception_notif_created
ON perception_notifications (created_at DESC);
"""
def _row_to_dict(row: dict[str, Any]) -> dict:
"""Convert a psycopg2 RealDictRow to a plain dict with an ISO timestamp."""
d = dict(row)
if d.get("created_at") is not None:
d["created_at"] = d["created_at"].isoformat()
return d
class PostgresNotificationStore(BaseNotificationStore):
"""Notification store backed by PostgreSQL."""
def __init__(self) -> None:
"""Open a connection pool and ensure both tables exist."""
self._pool = ThreadedConnectionPool(
minconn=1,
maxconn=5,
host=settings.postgres_host,
port=settings.postgres_port,
user=settings.postgres_user,
password=settings.postgres_password,
dbname=settings.postgres_db,
)
self._ensure_schema()
def _ensure_schema(self) -> None:
with self._conn() as conn:
try:
with conn.cursor() as cur:
cur.execute(_CREATE_TABLES)
conn.commit()
except Exception:
conn.rollback()
raise
@contextmanager
def _conn(self):
conn = None
try:
conn = self._pool.getconn()
yield conn
finally:
if conn is not None:
self._pool.putconn(conn)
def create(
self,
*,
event_id: str,
kind: str,
title: str,
impact_level: str | None,
summary: str | None,
) -> None:
"""Insert one notification row for the triggering event."""
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
"INSERT INTO perception_notifications "
"(event_id, kind, title, impact_level, summary) "
"VALUES (%s, %s, %s, %s, %s)",
(event_id, kind, title, impact_level, summary),
)
conn.commit()
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
"""Return the newest notifications with this user's read state joined in."""
with self._conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
cur.execute(
"""
SELECT n.*, (r.user_id IS NOT NULL) AS read
FROM perception_notifications n
LEFT JOIN perception_notification_reads r
ON r.notification_id = n.id AND r.user_id = %s
ORDER BY n.created_at DESC
LIMIT %s
""",
(user_id, limit),
)
return [_row_to_dict(r) for r in cur.fetchall()]
def unread_count(self, user_id: str) -> int:
"""Count notifications with no read receipt for this user."""
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT COUNT(*) FROM perception_notifications n
WHERE NOT EXISTS (
SELECT 1 FROM perception_notification_reads r
WHERE r.notification_id = n.id AND r.user_id = %s
)
""",
(user_id,),
)
return cur.fetchone()[0]
def mark_all_read(self, user_id: str) -> int:
"""Insert a read receipt for every notification this user hasn't read."""
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO perception_notification_reads (notification_id, user_id)
SELECT n.id, %s FROM perception_notifications n
WHERE NOT EXISTS (
SELECT 1 FROM perception_notification_reads r
WHERE r.notification_id = n.id AND r.user_id = %s
)
ON CONFLICT (notification_id, user_id) DO NOTHING
""",
(user_id, user_id),
)
marked = cur.rowcount
conn.commit()
return marked
@@ -0,0 +1,224 @@
"""Deterministic change detection between two versions of a regulation.
This module deliberately contains no LLM call, no network access, and no
embedding lookup. It exists because the previous implementation decided whether
a paragraph had changed by comparing embedding cosine similarity against a 0.85
threshold, which is blind to exactly the edits that matter in regulation.
Measured against the deployed text-embedding-v3 gateway, tightening a braking
limit from 30米 to 20米 scores 0.9153 and relaxing 应当 to 宜 scores 0.9162 —
both far above the threshold, both undetected — while an entirely unrelated
clause scores 0.6862 and is the only thing that fires. Cosine is scale
invariant, so it cannot represent a change in magnitude or certainty
(arXiv:2403.05440, ACM Web Conference 2024); no threshold recovers the signal.
The replacement is the production consensus for legal text: align paragraphs
with a longest-common-subsequence matcher, run a literal character diff on the
aligned pairs, and let cheap deterministic rules decide whether a change is
significant enough to spend an LLM call classifying.
"""
from __future__ import annotations
import re
import unicodedata
from dataclasses import dataclass, field
from diff_match_patch import diff_match_patch
from difflib import SequenceMatcher
from app.config.settings import settings
# Chinese regulatory drafting uses a small, near-unambiguous set of deontic
# markers, so a regex pre-pass identifies legally significant edits without an
# LLM. Adding or removing any of these changes what the provision compels.
_DEONTIC_PATTERN = re.compile(r"应当|须|禁止|不得|可以|允许|宜")
# Matches digit runs including decimals, so "30" -> "20" and "0.85" -> "0.9"
# are both treated as numeric changes.
_NUMBER_PATTERN = re.compile(r"\d+(?:\.\d+)?")
# diff_match_patch operation codes.
_DMP_DELETE = -1
_DMP_INSERT = 1
_DMP_EQUAL = 0
@dataclass(frozen=True)
class ParagraphChange:
"""One detected difference between the old and new version of a regulation.
`needs_llm` is the gate: it records whether this change is worth the cost of
an LLM classification call. The deterministic flags that drive it are kept
on the record so downstream code can act on them even when the LLM call
fails or is skipped.
"""
change_type: str
old_text: str
new_text: str
numeric_changed: bool
deontic_changed: bool
change_ratio: float
needs_llm: bool
# (op, text) pairs from diff_match_patch, for rendering a redline view.
diff_spans: list[tuple[int, str]] = field(default_factory=list)
def _split_paragraphs(text: str) -> list[str]:
"""Split regulation text into comparable units, dropping blank lines.
ponytail: newline splitting, not clause parsing. Upgrade to 第X条 / X.X.X
segmentation only if paragraph granularity proves too coarse in practice.
"""
return [line.strip() for line in (text or "").split("\n") if line.strip()]
def _numbers_differ(old: str, new: str) -> bool:
"""Report whether the two spans contain a different sequence of numbers."""
return _NUMBER_PATTERN.findall(old) != _NUMBER_PATTERN.findall(new)
def _deontic_differs(old: str, new: str) -> bool:
"""Report whether obligation markers were added, removed, or swapped."""
return sorted(_DEONTIC_PATTERN.findall(old)) != sorted(_DEONTIC_PATTERN.findall(new))
def _is_cosmetic(spans: list[tuple[int, str]]) -> bool:
"""Report whether the edit touched nothing but punctuation and whitespace.
A change ratio alone cannot answer this for Chinese regulation text. Clauses
run 20-60 characters, so deleting a single 。 is a 4% change and clears any
threshold low enough to still catch real edits in longer paragraphs. Testing
what actually changed is both cheaper and exact.
"""
changed = "".join(text for op, text in spans if op != _DMP_EQUAL)
# Unicode categories P (punctuation), Z (separator) and C (control) cover
# Chinese and ASCII punctuation plus every flavour of whitespace.
return all(unicodedata.category(char)[0] in {"P", "Z", "C"} for char in changed)
class RegulationDiffer:
"""Align two regulation versions and classify what changed, without an LLM."""
def __init__(self, min_change_ratio: float | None = None) -> None:
"""Store the gate threshold, defaulting to the configured value.
The explicit argument exists so tests never depend on the deployed .env.
"""
self._min_change_ratio = (
settings.perception_diff_min_change_ratio
if min_change_ratio is None
else min_change_ratio
)
self._dmp = diff_match_patch()
def diff(self, old_text: str, new_text: str) -> list[ParagraphChange]:
"""Return every changed paragraph between two versions.
Unchanged paragraphs are not returned. An empty old version means there
is no baseline to compare against — the caller's first crawl — so no
changes are reported rather than the whole document being called new.
"""
old_paras = _split_paragraphs(old_text)
new_paras = _split_paragraphs(new_text)
if not old_paras or not new_paras:
return []
# autojunk=False is load-bearing: the default treats any element
# appearing in over 1% of a sequence of 200+ items as junk, and
# regulations repeat boilerplate paragraphs that alignment depends on
# as anchors.
matcher = SequenceMatcher(None, old_paras, new_paras, autojunk=False)
changes: list[ParagraphChange] = []
for tag, i1, i2, j1, j2 in matcher.get_opcodes():
if tag == "equal":
continue
if tag == "insert":
changes.extend(self._added(p) for p in new_paras[j1:j2])
elif tag == "delete":
changes.extend(self._removed(p) for p in old_paras[i1:i2])
elif tag == "replace":
changes.extend(self._replaced(old_paras[i1:i2], new_paras[j1:j2]))
return changes
def _added(self, paragraph: str) -> ParagraphChange:
"""Build a record for a provision present only in the new version."""
return ParagraphChange(
change_type="added",
old_text="",
new_text=paragraph,
numeric_changed=False,
deontic_changed=bool(_DEONTIC_PATTERN.search(paragraph)),
change_ratio=1.0,
# A new provision always carries new obligations, so it is always
# worth classifying.
needs_llm=True,
diff_spans=[(_DMP_INSERT, paragraph)],
)
def _removed(self, paragraph: str) -> ParagraphChange:
"""Build a record for a provision dropped from the new version."""
return ParagraphChange(
change_type="removed",
old_text=paragraph,
new_text="",
numeric_changed=False,
deontic_changed=bool(_DEONTIC_PATTERN.search(paragraph)),
change_ratio=1.0,
needs_llm=True,
diff_spans=[(_DMP_DELETE, paragraph)],
)
def _replaced(self, old_block: list[str], new_block: list[str]) -> list[ParagraphChange]:
"""Compare a run of rewritten paragraphs pairwise, reporting the remainder.
SequenceMatcher emits `replace` for a whole run at once, and the two
sides may differ in length. Pairing by position within the run is safe
here because alignment has already established that this run as a whole
corresponds; any surplus on either side is a genuine insertion or
deletion.
"""
results: list[ParagraphChange] = []
for index in range(max(len(old_block), len(new_block))):
if index >= len(old_block):
results.append(self._added(new_block[index]))
elif index >= len(new_block):
results.append(self._removed(old_block[index]))
else:
results.append(self._modified(old_block[index], new_block[index]))
return results
def _modified(self, old: str, new: str) -> ParagraphChange:
"""Character-diff an aligned pair and decide whether it warrants an LLM call."""
spans = self._dmp.diff_main(old, new)
# Merges single-character edits into human-meaningful chunks so the
# redline view and the change ratio both reflect real edits.
self._dmp.diff_cleanupSemantic(spans)
changed_chars = sum(len(text) for op, text in spans if op != _DMP_EQUAL)
denominator = max(len(old), len(new), 1)
change_ratio = changed_chars / denominator
numeric_changed = _numbers_differ(old, new)
deontic_changed = _deontic_differs(old, new)
# A changed limit or obligation marker is always significant no matter
# how few characters moved. Everything else must be substantive and
# clear the ratio gate to be worth a model call.
significant = numeric_changed or deontic_changed or (
not _is_cosmetic(spans) and change_ratio >= self._min_change_ratio
)
return ParagraphChange(
change_type="modified",
old_text=old,
new_text=new,
numeric_changed=numeric_changed,
deontic_changed=deontic_changed,
change_ratio=change_ratio,
needs_llm=significant,
diff_spans=[(op, text) for op, text in spans],
)
+12 -1
View File
@@ -28,7 +28,10 @@ celery_app = Celery(
"compliance_hub",
broker=_BROKER,
backend=_BACKEND,
include=["app.infrastructure.tasks.document_tasks"],
include=[
"app.infrastructure.tasks.document_tasks",
"app.infrastructure.tasks.perception_tasks",
],
)
celery_app.conf.update(
@@ -42,4 +45,12 @@ celery_app.conf.update(
task_reject_on_worker_lost=True,
# Keep results for 1 hour for status polling.
result_expires=3600,
# Scheduled counterpart to the Perception page's manual "Refresh" button.
# Only takes effect while a Beat process is running (./dev.sh start beat).
beat_schedule={
"crawl-regulations-periodic": {
"task": "app.infrastructure.tasks.perception_tasks.crawl_regulations_task",
"schedule": settings.perception_crawl_interval_seconds,
},
},
)
@@ -0,0 +1,63 @@
"""Celery task for scheduled regulatory source crawling.
This is the scheduled counterpart to the Perception page's manual "Refresh"
button (POST /perception/crawl). Every architecture reference document
describes source monitoring as continuous ("定时爬取"), not operator-triggered,
so this task is what Celery Beat runs on a fixed interval once an operator
starts a Beat process.
"""
from __future__ import annotations
from loguru import logger
from app.infrastructure.tasks.celery_app import celery_app
@celery_app.task(
name="app.infrastructure.tasks.perception_tasks.crawl_regulations_task",
bind=True,
)
def crawl_regulations_task(self) -> dict:
"""Crawl every registered regulatory source and enrich new/changed events.
Drains CrawlService.run_crawl(), which already isolates each source's
fetch and each event's enrichment behind its own try/except — a source
outage or a single bad event yields an "error" progress item and the
generator continues. Re-catching those here would only hide problems the
service has already handled, so this task's job is limited to counting
them and logging a summary.
No automatic retry is configured. An exception escaping run_crawl itself
means something broke in a way the service's own error handling did not
anticipate; the next scheduled tick already provides a retry within
settings.perception_crawl_interval_seconds, so an immediate retry against
the same failure is not worth the added complexity.
ponytail: relies on a single worker process to serialize scheduled runs
(Celery's default concurrency processes one task at a time, so a run that
outlasts the interval delays the next tick rather than overlapping it).
Add a Redis-based lock (e.g. SETNX on a per-task key) if this queue is
ever served by more than one worker.
"""
from app.shared.bootstrap import get_crawl_service
error_count = 0
new_count = 0
updated_count = 0
for item in get_crawl_service().run_crawl():
event = item.get("event")
if event == "error":
error_count += 1
logger.warning("Scheduled crawl source error: {}", item.get("data"))
elif event == "done":
data = item.get("data") or {}
new_count = data.get("total_new", 0)
updated_count = data.get("total_updated", 0)
logger.info(
"Scheduled crawl finished: new={} updated={} source_errors={}",
new_count, updated_count, error_count,
)
return {"new": new_count, "updated": updated_count, "source_errors": error_count}
+3 -1
View File
@@ -16,7 +16,7 @@ from app.shared.model_usage_tracker import get_model_usage_tracker
# Keep provider-specific behavior explicit so debugging stays straightforward.
DEFAULT_MODELS = {
LLMProvider.DEEPSEEK: "deepseek-v4-flash",
LLMProvider.QWEN: "qwen3.5-flash",
LLMProvider.QWEN: "qwen3.6-flash",
LLMProvider.QWEN_VL: "qwen3-vl-plus"
}
@@ -101,6 +101,8 @@ class LLMFactory:
"qwen-max": LLMProvider.QWEN,
"qwen3.5-flash": LLMProvider.QWEN,
"qwen3.5-plus": LLMProvider.QWEN,
"qwen3.6-flash": LLMProvider.QWEN,
"qwen3.6-plus": LLMProvider.QWEN,
"qwen_vl": LLMProvider.QWEN_VL,
"qwen-vl": LLMProvider.QWEN_VL,
"qwen-vl-plus": LLMProvider.QWEN_VL,
+3 -1
View File
@@ -27,6 +27,8 @@ class QwenClient(BaseLLMClient):
"qwen-long",
"qwen3.5-flash",
"qwen3.5-plus",
"qwen3.6-flash",
"qwen3.6-plus",
"qwen3-plus",
"qwen2.5-72b-instruct",
"qwen2.5-32b-instruct",
@@ -371,7 +373,7 @@ class QwenVLClient(BaseLLMClient):
def create_qwen_client(
api_key: str,
model: str = "qwen3.5-flash",
model: str = "qwen3.6-flash",
base_url: str = "http://6.86.80.4:30080/v1",
**kwargs
) -> QwenClient:
+21
View File
@@ -23,8 +23,10 @@ from app.infrastructure.parser.local_chunk_builder import LocalRegulationChunkBu
from app.infrastructure.parser.local_document_parser import LocalDocumentParser
from app.infrastructure.parser.vector_chunk_builder import AliyunVectorChunkBuilder
from app.infrastructure.perception.mock_event_store import MockEventStore
from app.infrastructure.perception.mock_notification_store import MockNotificationStore
from app.application.perception.crawl_service import CrawlService
from app.infrastructure.perception.base_event_store import BaseEventStore
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
from app.infrastructure.perception.crawlers.catarc_crawler import CatarcCrawler
from app.infrastructure.perception.crawlers.guobiao_crawler import (
GuobiaoMandatoryCrawler,
@@ -327,6 +329,22 @@ def get_event_store() -> BaseEventStore:
return MockEventStore()
@lru_cache
def get_notification_store() -> BaseNotificationStore:
"""Return notification store selected by DOCUMENT_REPOSITORY_BACKEND setting.
Mirrors get_event_store()'s gate: Mock in-memory when Postgres isn't
configured, so the feature works in local dev and tests without a
database.
"""
if settings.document_repository_backend == "postgres":
from app.infrastructure.perception.postgres_notification_store import (
PostgresNotificationStore,
)
return PostgresNotificationStore()
return MockNotificationStore()
@lru_cache
def get_compliance_repository() -> ComplianceRepository:
"""Return the compliance analysis repository.
@@ -370,6 +388,9 @@ def get_crawl_service() -> CrawlService:
event_store=get_event_store(),
llm_pipeline=LlmPipeline(),
retrieval_service=get_retrieval_service(),
notification_store=get_notification_store(),
embedding_provider=get_embedding_provider(),
vector_index=get_vector_index(),
)