update for mcp
This commit is contained in:
wangwei
@@ -0,0 +1,47 @@
|
||||
"""Abstract base class for in-app regulatory-signal notifications.
|
||||
|
||||
A notification is created once per triggering event (a brand-new regulation,
|
||||
or a significant change to an existing one) and broadcast to every logged-in
|
||||
user. There is no per-user subscription targeting — see the design doc for why.
|
||||
Per-user "read" state is tracked separately from the notification itself, so
|
||||
one notification row serves every user rather than being fanned out on create.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class BaseNotificationStore(ABC):
|
||||
"""Port interface for perception notification persistence."""
|
||||
|
||||
@abstractmethod
|
||||
def create(
|
||||
self,
|
||||
*,
|
||||
event_id: str,
|
||||
kind: str,
|
||||
title: str,
|
||||
impact_level: str | None,
|
||||
summary: str | None,
|
||||
) -> None:
|
||||
"""Record a new notification. kind is 'new' or 'changed'."""
|
||||
|
||||
@abstractmethod
|
||||
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
|
||||
"""Return the most recent notifications, newest first.
|
||||
|
||||
Each item includes a "read" boolean reflecting whether `user_id` has
|
||||
marked it read.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def unread_count(self, user_id: str) -> int:
|
||||
"""Return how many notifications `user_id` has not yet read."""
|
||||
|
||||
@abstractmethod
|
||||
def mark_all_read(self, user_id: str) -> int:
|
||||
"""Mark every currently-unread notification read for `user_id`.
|
||||
|
||||
Returns the number of notifications newly marked.
|
||||
"""
|
||||
@@ -5,6 +5,12 @@ from __future__ import annotations
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import httpx
|
||||
import trafilatura
|
||||
from loguru import logger
|
||||
|
||||
from app.config.settings import settings
|
||||
|
||||
|
||||
@dataclass
|
||||
class RawEvent:
|
||||
@@ -21,7 +27,10 @@ class RawEvent:
|
||||
effective_at: str | None
|
||||
category: str
|
||||
tags: list[str] = field(default_factory=list)
|
||||
raw_text: str = "" # full crawled text for hashing + LLM
|
||||
# Whatever text the list page yields. CrawlService upgrades this by calling
|
||||
# fetch_full_text(full_text_url); this value is the fallback when that
|
||||
# fails. Used for change hashing and for the version diff.
|
||||
raw_text: str = ""
|
||||
|
||||
|
||||
class BaseCrawler(ABC):
|
||||
@@ -30,3 +39,37 @@ class BaseCrawler(ABC):
|
||||
@abstractmethod
|
||||
def fetch(self, limit: int = 50) -> list[RawEvent]:
|
||||
"""Fetch up to `limit` recent events from the data source."""
|
||||
|
||||
def fetch_full_text(self, url: str) -> str:
|
||||
"""Download a regulation detail page and extract its body text.
|
||||
|
||||
Change detection is only as good as the text it compares, and list
|
||||
pages carry nothing but a standard code and a title. This default
|
||||
implementation serves all current sources; a source that needs PDF
|
||||
extraction or authentication overrides this one method.
|
||||
|
||||
Returns an empty string on any failure rather than raising, so one
|
||||
unreachable page cannot abort a whole crawl run. The caller decides how
|
||||
to degrade.
|
||||
"""
|
||||
if not url:
|
||||
return ""
|
||||
try:
|
||||
response = httpx.get(
|
||||
url,
|
||||
timeout=settings.perception_crawl_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
)
|
||||
response.raise_for_status()
|
||||
except Exception as exc: # noqa: BLE001 - any transport error degrades the same way
|
||||
logger.warning("Full-text fetch failed url={} err={}", url, exc)
|
||||
return ""
|
||||
|
||||
# trafilatura scores 0.92 F1 on government pages against 0.78 for
|
||||
# readability-lxml, and handles CJK content; include_tables matters
|
||||
# because regulatory limits are frequently tabulated.
|
||||
extracted = trafilatura.extract(response.text, include_tables=True)
|
||||
if not extracted:
|
||||
logger.warning("Full-text extraction returned nothing url={}", url)
|
||||
return ""
|
||||
return extracted.strip()
|
||||
|
||||
@@ -8,6 +8,7 @@ import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from loguru import logger
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
|
||||
from ._utils import extract_tags, parse_date
|
||||
|
||||
@@ -33,7 +34,11 @@ class CatarcCrawler(BaseCrawler):
|
||||
while len(events) < limit and page <= max_pages:
|
||||
url = f"{_BASE_URL}?page={page}"
|
||||
try:
|
||||
resp = httpx.get(url, timeout=30, follow_redirects=True)
|
||||
resp = httpx.get(
|
||||
url,
|
||||
timeout=settings.perception_crawl_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except Exception as exc:
|
||||
logger.warning("CATARC fetch failed page={} err={}", page, exc)
|
||||
|
||||
@@ -9,6 +9,7 @@ import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from loguru import logger
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
|
||||
from ._utils import parse_date
|
||||
|
||||
@@ -53,7 +54,11 @@ class EurlexCrawler(BaseCrawler):
|
||||
if len(events) >= limit:
|
||||
break
|
||||
try:
|
||||
resp = httpx.get(rss_url, timeout=30, follow_redirects=True)
|
||||
resp = httpx.get(
|
||||
rss_url,
|
||||
timeout=settings.perception_crawl_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except Exception as exc:
|
||||
logger.warning("EUR-Lex RSS fetch failed url={} err={}", rss_url, exc)
|
||||
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import httpx
|
||||
from loguru import logger
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
|
||||
from ._utils import extract_tags, parse_date
|
||||
|
||||
@@ -22,7 +23,12 @@ def _fetch_page(std_type: int, page: int, page_size: int) -> list[dict]:
|
||||
"p.p7": page_size,
|
||||
}
|
||||
try:
|
||||
resp = httpx.get(_BASE_URL, params=params, headers=_HEADERS, timeout=30)
|
||||
resp = httpx.get(
|
||||
_BASE_URL,
|
||||
params=params,
|
||||
headers=_HEADERS,
|
||||
timeout=settings.perception_crawl_timeout_seconds,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
return data.get("rows", []) or []
|
||||
|
||||
@@ -3,14 +3,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.infrastructure.embedding.openai_compatible_embedding_provider import (
|
||||
OpenAICompatibleEmbeddingProvider,
|
||||
from app.infrastructure.perception.regulation_differ import (
|
||||
ParagraphChange,
|
||||
RegulationDiffer,
|
||||
)
|
||||
from app.services.llm.llm_factory import get_llm_client
|
||||
|
||||
@@ -27,21 +27,31 @@ _ASSESS_SYSTEM = (
|
||||
)
|
||||
|
||||
_DIFF_SYSTEM = (
|
||||
"You are a regulatory change analyst. Given an old and new version of a regulation paragraph, "
|
||||
"classify the type of change and summarise it. "
|
||||
"Return JSON only: {\"change_type\": \"tightened|relaxed|added|removed\", \"summary\": \"...\"}"
|
||||
"You are a regulatory change analyst. You are given the OLD and NEW version of "
|
||||
"one regulation paragraph, with the exact edits marked <DEL>removed</DEL> and "
|
||||
"<INS>added</INS>. Classify the legal effect of the change. "
|
||||
"Return JSON only: {\"change_type\": \"tightened|relaxed|numeric|clarified|scope\", "
|
||||
"\"legal_effect\": \"one sentence on what this means for compliance\"}"
|
||||
)
|
||||
|
||||
_SIMILARITY_THRESHOLD = 0.85
|
||||
|
||||
def _marked_diff(change: ParagraphChange) -> str:
|
||||
"""Render a paragraph change with the exact edits marked for the model.
|
||||
|
||||
def _cosine(a: list[float], b: list[float]) -> float:
|
||||
dot = sum(x * y for x, y in zip(a, b))
|
||||
norm_a = math.sqrt(sum(x * x for x in a))
|
||||
norm_b = math.sqrt(sum(x * x for x in b))
|
||||
if norm_a == 0 or norm_b == 0:
|
||||
return 0.0
|
||||
return dot / (norm_a * norm_b)
|
||||
The model is shown where the edit is rather than being asked to find it,
|
||||
and is never asked to reproduce the changed text — the differ already
|
||||
computed those spans exactly, so there is nothing for the model to
|
||||
hallucinate.
|
||||
"""
|
||||
marked = "".join(
|
||||
text if op == 0 else (f"<DEL>{text}</DEL>" if op < 0 else f"<INS>{text}</INS>")
|
||||
for op, text in change.diff_spans
|
||||
)
|
||||
return (
|
||||
f"OLD: {change.old_text[:500]}\n"
|
||||
f"NEW: {change.new_text[:500]}\n"
|
||||
f"MARKED: {marked[:800]}"
|
||||
)
|
||||
|
||||
|
||||
def _llm_json(client: Any, messages: list[dict]) -> Any:
|
||||
@@ -67,7 +77,9 @@ class LlmPipeline:
|
||||
provider=settings.llm_provider,
|
||||
model=settings.llm_model,
|
||||
)
|
||||
self._embedder = OpenAICompatibleEmbeddingProvider()
|
||||
# Change detection is deterministic; the differ needs no model and no
|
||||
# network, so the pipeline no longer constructs an embedding provider.
|
||||
self._differ = RegulationDiffer()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Step 1: Structure extraction
|
||||
@@ -166,76 +178,68 @@ For each document, assess impact and recommend action. Return JSON array:
|
||||
return doc_excerpts
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Step 3: Semantic diff
|
||||
# Step 3: Deterministic diff with gated LLM classification
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def compute_diff(self, old_text: str, new_text: str) -> dict:
|
||||
"""Compare old and new regulation text; return changed sections and summary."""
|
||||
old_paras = [p.strip() for p in old_text.split("\n") if p.strip()]
|
||||
new_paras = [p.strip() for p in new_text.split("\n") if p.strip()]
|
||||
"""Compare old and new regulation text; return changed sections and summary.
|
||||
|
||||
if not old_paras or not new_paras:
|
||||
return {"changed_sections": [], "change_summary": "No comparable text."}
|
||||
Detection is deterministic — see regulation_differ for why embedding
|
||||
similarity was removed. The LLM is called only for paragraphs the
|
||||
differ marked significant, and only to explain the legal effect of a
|
||||
change that has already been located exactly.
|
||||
"""
|
||||
changes = self._differ.diff(old_text, new_text)
|
||||
if not changes:
|
||||
return {
|
||||
"changed_sections": [],
|
||||
"change_summary": "No substantive changes detected between versions.",
|
||||
}
|
||||
|
||||
all_paras = old_paras + new_paras
|
||||
try:
|
||||
all_embeddings = self._embedder.embed_texts(all_paras)
|
||||
except Exception as exc:
|
||||
logger.warning("Embedding for diff failed: {}", exc)
|
||||
return {"changed_sections": [], "change_summary": "Diff unavailable (embedding error)."}
|
||||
changed_sections = [self._describe(change) for change in changes]
|
||||
|
||||
old_embeddings = all_embeddings[: len(old_paras)]
|
||||
new_embeddings = all_embeddings[len(old_paras):]
|
||||
|
||||
changed_sections: list[dict] = []
|
||||
max_len = max(len(old_paras), len(new_paras))
|
||||
|
||||
for i in range(max_len):
|
||||
if i >= len(old_paras):
|
||||
# New paragraph added
|
||||
changed_sections.append({
|
||||
"old_text": "",
|
||||
"new_text": new_paras[i][:300],
|
||||
"similarity": 0.0,
|
||||
"change_type": "added",
|
||||
"summary": "New paragraph added.",
|
||||
})
|
||||
continue
|
||||
if i >= len(new_paras):
|
||||
# Old paragraph removed
|
||||
changed_sections.append({
|
||||
"old_text": old_paras[i][:300],
|
||||
"new_text": "",
|
||||
"similarity": 0.0,
|
||||
"change_type": "removed",
|
||||
"summary": "Paragraph removed.",
|
||||
})
|
||||
continue
|
||||
# Both exist — compare via embeddings
|
||||
sim = _cosine(old_embeddings[i], new_embeddings[i])
|
||||
if sim < _SIMILARITY_THRESHOLD:
|
||||
messages = [
|
||||
{"role": "system", "content": _DIFF_SYSTEM},
|
||||
{"role": "user", "content": f"OLD: {old_paras[i][:500]}\nNEW: {new_paras[i][:500]}"},
|
||||
]
|
||||
classification = _llm_json(self._client, messages) or {}
|
||||
changed_sections.append({
|
||||
"old_text": old_paras[i][:300],
|
||||
"new_text": new_paras[i][:300],
|
||||
"similarity": round(sim, 3),
|
||||
"change_type": classification.get("change_type", "modified"),
|
||||
"summary": classification.get("summary", ""),
|
||||
})
|
||||
|
||||
if not changed_sections:
|
||||
change_summary = "No substantive changes detected between versions."
|
||||
else:
|
||||
types = [s["change_type"] for s in changed_sections]
|
||||
change_summary = (
|
||||
f"{len(changed_sections)} paragraph(s) changed: "
|
||||
+ ", ".join(f"{t}" for t in set(types))
|
||||
+ ". "
|
||||
+ (changed_sections[0].get("summary", "") if changed_sections else "")
|
||||
)
|
||||
types = sorted({section["change_type"] for section in changed_sections})
|
||||
gated = sum(1 for change in changes if change.needs_llm)
|
||||
change_summary = (
|
||||
f"{len(changed_sections)} paragraph(s) changed ({', '.join(types)}); "
|
||||
f"{gated} significant. "
|
||||
+ (changed_sections[0].get("summary") or "")
|
||||
).strip()
|
||||
|
||||
return {"changed_sections": changed_sections, "change_summary": change_summary}
|
||||
|
||||
def _describe(self, change: ParagraphChange) -> dict:
|
||||
"""Turn one detected change into the API payload, classifying if warranted."""
|
||||
section = {
|
||||
"old_text": change.old_text[:300],
|
||||
"new_text": change.new_text[:300],
|
||||
"change_type": change.change_type,
|
||||
"change_ratio": round(change.change_ratio, 3),
|
||||
"numeric_changed": change.numeric_changed,
|
||||
"deontic_changed": change.deontic_changed,
|
||||
"summary": "",
|
||||
}
|
||||
|
||||
if not change.needs_llm:
|
||||
return section
|
||||
|
||||
classification = _llm_json(
|
||||
self._client,
|
||||
[
|
||||
{"role": "system", "content": _DIFF_SYSTEM},
|
||||
{"role": "user", "content": _marked_diff(change)},
|
||||
],
|
||||
)
|
||||
if isinstance(classification, dict):
|
||||
section["change_type"] = classification.get("change_type") or change.change_type
|
||||
section["summary"] = classification.get("legal_effect") or ""
|
||||
# A failed or malformed model response must not discard a change that
|
||||
# deterministic analysis already proved real; the section keeps its
|
||||
# spans, flags, and alignment-derived type with an empty summary.
|
||||
|
||||
if change.numeric_changed:
|
||||
# Models routinely label a changed threshold as "clarified". The
|
||||
# deterministic pass already knows a number moved, so it wins.
|
||||
section["change_type"] = "numeric"
|
||||
|
||||
return section
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""In-memory notification store used when Postgres is not configured.
|
||||
|
||||
Mirrors MockEventStore's role for BaseEventStore: keeps the feature usable in
|
||||
local dev and in tests without a live database, and matches
|
||||
DOCUMENT_REPOSITORY_BACKEND's existing Mock/Postgres split.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
|
||||
|
||||
|
||||
class MockNotificationStore(BaseNotificationStore):
|
||||
"""Dict-backed notification store. Data does not survive a process restart."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Start with an empty feed and no read receipts."""
|
||||
self._notifications: list[dict] = []
|
||||
self._next_id = 1
|
||||
# (notification_id, user_id) pairs — presence means read.
|
||||
self._reads: set[tuple[int, str]] = set()
|
||||
|
||||
def create(
|
||||
self,
|
||||
*,
|
||||
event_id: str,
|
||||
kind: str,
|
||||
title: str,
|
||||
impact_level: str | None,
|
||||
summary: str | None,
|
||||
) -> None:
|
||||
"""Append a notification with an auto-incrementing id."""
|
||||
self._notifications.append({
|
||||
"id": self._next_id,
|
||||
"event_id": event_id,
|
||||
"kind": kind,
|
||||
"title": title,
|
||||
"impact_level": impact_level,
|
||||
"summary": summary,
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
})
|
||||
self._next_id += 1
|
||||
|
||||
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
|
||||
"""Return the newest `limit` notifications with this user's read state."""
|
||||
ordered = sorted(self._notifications, key=lambda n: n["id"], reverse=True)
|
||||
return [
|
||||
{**n, "read": (n["id"], user_id) in self._reads}
|
||||
for n in ordered[:limit]
|
||||
]
|
||||
|
||||
def unread_count(self, user_id: str) -> int:
|
||||
"""Count notifications this user has not yet read."""
|
||||
return sum(1 for n in self._notifications if (n["id"], user_id) not in self._reads)
|
||||
|
||||
def mark_all_read(self, user_id: str) -> int:
|
||||
"""Add a read receipt for every currently-unread notification."""
|
||||
marked = 0
|
||||
for n in self._notifications:
|
||||
key = (n["id"], user_id)
|
||||
if key not in self._reads:
|
||||
self._reads.add(key)
|
||||
marked += 1
|
||||
return marked
|
||||
@@ -40,7 +40,8 @@ CREATE TABLE IF NOT EXISTS regulation_events (
|
||||
affected_docs JSONB,
|
||||
crawled_at TIMESTAMPTZ DEFAULT now(),
|
||||
processed_at TIMESTAMPTZ,
|
||||
raw_storage_key TEXT
|
||||
raw_storage_key TEXT,
|
||||
raw_text TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS reg_events_source_date
|
||||
ON regulation_events (source, published_at DESC);
|
||||
@@ -48,12 +49,16 @@ CREATE INDEX IF NOT EXISTS reg_events_impact_date
|
||||
ON regulation_events (impact_level, published_at DESC);
|
||||
"""
|
||||
|
||||
_ADD_COLUMNS = """
|
||||
ALTER TABLE regulation_events ADD COLUMN IF NOT EXISTS raw_text TEXT;
|
||||
"""
|
||||
|
||||
_ALL_COLUMNS = (
|
||||
"id", "source", "source_label", "standard_code", "title", "summary",
|
||||
"full_text_url", "status", "impact_level", "published_at", "effective_at",
|
||||
"category", "tags", "obligations", "deadlines", "scope", "penalties",
|
||||
"content_hash", "previous_hash", "change_summary", "changed_sections",
|
||||
"affected_docs", "crawled_at", "processed_at", "raw_storage_key",
|
||||
"affected_docs", "crawled_at", "processed_at", "raw_storage_key", "raw_text",
|
||||
)
|
||||
|
||||
|
||||
@@ -97,6 +102,10 @@ class PostgresEventStore(BaseEventStore):
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(_CREATE_TABLE)
|
||||
# CREATE TABLE IF NOT EXISTS is a no-op on deployments that
|
||||
# already have this table, so new columns must be added
|
||||
# explicitly or existing installations silently lack them.
|
||||
cur.execute(_ADD_COLUMNS)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
"""PostgreSQL-backed notification store.
|
||||
|
||||
One row per triggering event, shared by every user; a separate read-receipt
|
||||
table tracks per-user read state so broadcasting to everyone needs no fan-out
|
||||
insert per user. See base_notification_store.py for the port contract and the
|
||||
design doc for why this shape was chosen over per-user subscriptions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
from psycopg2.pool import ThreadedConnectionPool
|
||||
|
||||
from app.config.settings import settings
|
||||
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
|
||||
|
||||
_CREATE_TABLES = """
|
||||
CREATE TABLE IF NOT EXISTS perception_notifications (
|
||||
id SERIAL PRIMARY KEY,
|
||||
event_id TEXT NOT NULL REFERENCES regulation_events(id) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
impact_level TEXT,
|
||||
summary TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS perception_notification_reads (
|
||||
notification_id INTEGER NOT NULL REFERENCES perception_notifications(id) ON DELETE CASCADE,
|
||||
user_id TEXT NOT NULL,
|
||||
read_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (notification_id, user_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS perception_notif_created
|
||||
ON perception_notifications (created_at DESC);
|
||||
"""
|
||||
|
||||
|
||||
def _row_to_dict(row: dict[str, Any]) -> dict:
|
||||
"""Convert a psycopg2 RealDictRow to a plain dict with an ISO timestamp."""
|
||||
d = dict(row)
|
||||
if d.get("created_at") is not None:
|
||||
d["created_at"] = d["created_at"].isoformat()
|
||||
return d
|
||||
|
||||
|
||||
class PostgresNotificationStore(BaseNotificationStore):
|
||||
"""Notification store backed by PostgreSQL."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Open a connection pool and ensure both tables exist."""
|
||||
self._pool = ThreadedConnectionPool(
|
||||
minconn=1,
|
||||
maxconn=5,
|
||||
host=settings.postgres_host,
|
||||
port=settings.postgres_port,
|
||||
user=settings.postgres_user,
|
||||
password=settings.postgres_password,
|
||||
dbname=settings.postgres_db,
|
||||
)
|
||||
self._ensure_schema()
|
||||
|
||||
def _ensure_schema(self) -> None:
|
||||
with self._conn() as conn:
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(_CREATE_TABLES)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
|
||||
@contextmanager
|
||||
def _conn(self):
|
||||
conn = None
|
||||
try:
|
||||
conn = self._pool.getconn()
|
||||
yield conn
|
||||
finally:
|
||||
if conn is not None:
|
||||
self._pool.putconn(conn)
|
||||
|
||||
def create(
|
||||
self,
|
||||
*,
|
||||
event_id: str,
|
||||
kind: str,
|
||||
title: str,
|
||||
impact_level: str | None,
|
||||
summary: str | None,
|
||||
) -> None:
|
||||
"""Insert one notification row for the triggering event."""
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"INSERT INTO perception_notifications "
|
||||
"(event_id, kind, title, impact_level, summary) "
|
||||
"VALUES (%s, %s, %s, %s, %s)",
|
||||
(event_id, kind, title, impact_level, summary),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
|
||||
"""Return the newest notifications with this user's read state joined in."""
|
||||
with self._conn() as conn:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT n.*, (r.user_id IS NOT NULL) AS read
|
||||
FROM perception_notifications n
|
||||
LEFT JOIN perception_notification_reads r
|
||||
ON r.notification_id = n.id AND r.user_id = %s
|
||||
ORDER BY n.created_at DESC
|
||||
LIMIT %s
|
||||
""",
|
||||
(user_id, limit),
|
||||
)
|
||||
return [_row_to_dict(r) for r in cur.fetchall()]
|
||||
|
||||
def unread_count(self, user_id: str) -> int:
|
||||
"""Count notifications with no read receipt for this user."""
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM perception_notifications n
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM perception_notification_reads r
|
||||
WHERE r.notification_id = n.id AND r.user_id = %s
|
||||
)
|
||||
""",
|
||||
(user_id,),
|
||||
)
|
||||
return cur.fetchone()[0]
|
||||
|
||||
def mark_all_read(self, user_id: str) -> int:
|
||||
"""Insert a read receipt for every notification this user hasn't read."""
|
||||
with self._conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO perception_notification_reads (notification_id, user_id)
|
||||
SELECT n.id, %s FROM perception_notifications n
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM perception_notification_reads r
|
||||
WHERE r.notification_id = n.id AND r.user_id = %s
|
||||
)
|
||||
ON CONFLICT (notification_id, user_id) DO NOTHING
|
||||
""",
|
||||
(user_id, user_id),
|
||||
)
|
||||
marked = cur.rowcount
|
||||
conn.commit()
|
||||
return marked
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Deterministic change detection between two versions of a regulation.
|
||||
|
||||
This module deliberately contains no LLM call, no network access, and no
|
||||
embedding lookup. It exists because the previous implementation decided whether
|
||||
a paragraph had changed by comparing embedding cosine similarity against a 0.85
|
||||
threshold, which is blind to exactly the edits that matter in regulation.
|
||||
Measured against the deployed text-embedding-v3 gateway, tightening a braking
|
||||
limit from 30米 to 20米 scores 0.9153 and relaxing 应当 to 宜 scores 0.9162 —
|
||||
both far above the threshold, both undetected — while an entirely unrelated
|
||||
clause scores 0.6862 and is the only thing that fires. Cosine is scale
|
||||
invariant, so it cannot represent a change in magnitude or certainty
|
||||
(arXiv:2403.05440, ACM Web Conference 2024); no threshold recovers the signal.
|
||||
|
||||
The replacement is the production consensus for legal text: align paragraphs
|
||||
with a longest-common-subsequence matcher, run a literal character diff on the
|
||||
aligned pairs, and let cheap deterministic rules decide whether a change is
|
||||
significant enough to spend an LLM call classifying.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from diff_match_patch import diff_match_patch
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
from app.config.settings import settings
|
||||
|
||||
# Chinese regulatory drafting uses a small, near-unambiguous set of deontic
|
||||
# markers, so a regex pre-pass identifies legally significant edits without an
|
||||
# LLM. Adding or removing any of these changes what the provision compels.
|
||||
_DEONTIC_PATTERN = re.compile(r"应当|须|禁止|不得|可以|允许|宜")
|
||||
|
||||
# Matches digit runs including decimals, so "30" -> "20" and "0.85" -> "0.9"
|
||||
# are both treated as numeric changes.
|
||||
_NUMBER_PATTERN = re.compile(r"\d+(?:\.\d+)?")
|
||||
|
||||
# diff_match_patch operation codes.
|
||||
_DMP_DELETE = -1
|
||||
_DMP_INSERT = 1
|
||||
_DMP_EQUAL = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParagraphChange:
|
||||
"""One detected difference between the old and new version of a regulation.
|
||||
|
||||
`needs_llm` is the gate: it records whether this change is worth the cost of
|
||||
an LLM classification call. The deterministic flags that drive it are kept
|
||||
on the record so downstream code can act on them even when the LLM call
|
||||
fails or is skipped.
|
||||
"""
|
||||
|
||||
change_type: str
|
||||
old_text: str
|
||||
new_text: str
|
||||
numeric_changed: bool
|
||||
deontic_changed: bool
|
||||
change_ratio: float
|
||||
needs_llm: bool
|
||||
# (op, text) pairs from diff_match_patch, for rendering a redline view.
|
||||
diff_spans: list[tuple[int, str]] = field(default_factory=list)
|
||||
|
||||
|
||||
def _split_paragraphs(text: str) -> list[str]:
|
||||
"""Split regulation text into comparable units, dropping blank lines.
|
||||
|
||||
ponytail: newline splitting, not clause parsing. Upgrade to 第X条 / X.X.X
|
||||
segmentation only if paragraph granularity proves too coarse in practice.
|
||||
"""
|
||||
return [line.strip() for line in (text or "").split("\n") if line.strip()]
|
||||
|
||||
|
||||
def _numbers_differ(old: str, new: str) -> bool:
|
||||
"""Report whether the two spans contain a different sequence of numbers."""
|
||||
return _NUMBER_PATTERN.findall(old) != _NUMBER_PATTERN.findall(new)
|
||||
|
||||
|
||||
def _deontic_differs(old: str, new: str) -> bool:
|
||||
"""Report whether obligation markers were added, removed, or swapped."""
|
||||
return sorted(_DEONTIC_PATTERN.findall(old)) != sorted(_DEONTIC_PATTERN.findall(new))
|
||||
|
||||
|
||||
def _is_cosmetic(spans: list[tuple[int, str]]) -> bool:
|
||||
"""Report whether the edit touched nothing but punctuation and whitespace.
|
||||
|
||||
A change ratio alone cannot answer this for Chinese regulation text. Clauses
|
||||
run 20-60 characters, so deleting a single 。 is a 4% change and clears any
|
||||
threshold low enough to still catch real edits in longer paragraphs. Testing
|
||||
what actually changed is both cheaper and exact.
|
||||
"""
|
||||
changed = "".join(text for op, text in spans if op != _DMP_EQUAL)
|
||||
# Unicode categories P (punctuation), Z (separator) and C (control) cover
|
||||
# Chinese and ASCII punctuation plus every flavour of whitespace.
|
||||
return all(unicodedata.category(char)[0] in {"P", "Z", "C"} for char in changed)
|
||||
|
||||
|
||||
class RegulationDiffer:
|
||||
"""Align two regulation versions and classify what changed, without an LLM."""
|
||||
|
||||
def __init__(self, min_change_ratio: float | None = None) -> None:
|
||||
"""Store the gate threshold, defaulting to the configured value.
|
||||
|
||||
The explicit argument exists so tests never depend on the deployed .env.
|
||||
"""
|
||||
self._min_change_ratio = (
|
||||
settings.perception_diff_min_change_ratio
|
||||
if min_change_ratio is None
|
||||
else min_change_ratio
|
||||
)
|
||||
self._dmp = diff_match_patch()
|
||||
|
||||
def diff(self, old_text: str, new_text: str) -> list[ParagraphChange]:
|
||||
"""Return every changed paragraph between two versions.
|
||||
|
||||
Unchanged paragraphs are not returned. An empty old version means there
|
||||
is no baseline to compare against — the caller's first crawl — so no
|
||||
changes are reported rather than the whole document being called new.
|
||||
"""
|
||||
old_paras = _split_paragraphs(old_text)
|
||||
new_paras = _split_paragraphs(new_text)
|
||||
|
||||
if not old_paras or not new_paras:
|
||||
return []
|
||||
|
||||
# autojunk=False is load-bearing: the default treats any element
|
||||
# appearing in over 1% of a sequence of 200+ items as junk, and
|
||||
# regulations repeat boilerplate paragraphs that alignment depends on
|
||||
# as anchors.
|
||||
matcher = SequenceMatcher(None, old_paras, new_paras, autojunk=False)
|
||||
|
||||
changes: list[ParagraphChange] = []
|
||||
for tag, i1, i2, j1, j2 in matcher.get_opcodes():
|
||||
if tag == "equal":
|
||||
continue
|
||||
if tag == "insert":
|
||||
changes.extend(self._added(p) for p in new_paras[j1:j2])
|
||||
elif tag == "delete":
|
||||
changes.extend(self._removed(p) for p in old_paras[i1:i2])
|
||||
elif tag == "replace":
|
||||
changes.extend(self._replaced(old_paras[i1:i2], new_paras[j1:j2]))
|
||||
|
||||
return changes
|
||||
|
||||
def _added(self, paragraph: str) -> ParagraphChange:
|
||||
"""Build a record for a provision present only in the new version."""
|
||||
return ParagraphChange(
|
||||
change_type="added",
|
||||
old_text="",
|
||||
new_text=paragraph,
|
||||
numeric_changed=False,
|
||||
deontic_changed=bool(_DEONTIC_PATTERN.search(paragraph)),
|
||||
change_ratio=1.0,
|
||||
# A new provision always carries new obligations, so it is always
|
||||
# worth classifying.
|
||||
needs_llm=True,
|
||||
diff_spans=[(_DMP_INSERT, paragraph)],
|
||||
)
|
||||
|
||||
def _removed(self, paragraph: str) -> ParagraphChange:
|
||||
"""Build a record for a provision dropped from the new version."""
|
||||
return ParagraphChange(
|
||||
change_type="removed",
|
||||
old_text=paragraph,
|
||||
new_text="",
|
||||
numeric_changed=False,
|
||||
deontic_changed=bool(_DEONTIC_PATTERN.search(paragraph)),
|
||||
change_ratio=1.0,
|
||||
needs_llm=True,
|
||||
diff_spans=[(_DMP_DELETE, paragraph)],
|
||||
)
|
||||
|
||||
def _replaced(self, old_block: list[str], new_block: list[str]) -> list[ParagraphChange]:
|
||||
"""Compare a run of rewritten paragraphs pairwise, reporting the remainder.
|
||||
|
||||
SequenceMatcher emits `replace` for a whole run at once, and the two
|
||||
sides may differ in length. Pairing by position within the run is safe
|
||||
here because alignment has already established that this run as a whole
|
||||
corresponds; any surplus on either side is a genuine insertion or
|
||||
deletion.
|
||||
"""
|
||||
results: list[ParagraphChange] = []
|
||||
for index in range(max(len(old_block), len(new_block))):
|
||||
if index >= len(old_block):
|
||||
results.append(self._added(new_block[index]))
|
||||
elif index >= len(new_block):
|
||||
results.append(self._removed(old_block[index]))
|
||||
else:
|
||||
results.append(self._modified(old_block[index], new_block[index]))
|
||||
return results
|
||||
|
||||
def _modified(self, old: str, new: str) -> ParagraphChange:
|
||||
"""Character-diff an aligned pair and decide whether it warrants an LLM call."""
|
||||
spans = self._dmp.diff_main(old, new)
|
||||
# Merges single-character edits into human-meaningful chunks so the
|
||||
# redline view and the change ratio both reflect real edits.
|
||||
self._dmp.diff_cleanupSemantic(spans)
|
||||
|
||||
changed_chars = sum(len(text) for op, text in spans if op != _DMP_EQUAL)
|
||||
denominator = max(len(old), len(new), 1)
|
||||
change_ratio = changed_chars / denominator
|
||||
|
||||
numeric_changed = _numbers_differ(old, new)
|
||||
deontic_changed = _deontic_differs(old, new)
|
||||
|
||||
# A changed limit or obligation marker is always significant no matter
|
||||
# how few characters moved. Everything else must be substantive and
|
||||
# clear the ratio gate to be worth a model call.
|
||||
significant = numeric_changed or deontic_changed or (
|
||||
not _is_cosmetic(spans) and change_ratio >= self._min_change_ratio
|
||||
)
|
||||
|
||||
return ParagraphChange(
|
||||
change_type="modified",
|
||||
old_text=old,
|
||||
new_text=new,
|
||||
numeric_changed=numeric_changed,
|
||||
deontic_changed=deontic_changed,
|
||||
change_ratio=change_ratio,
|
||||
needs_llm=significant,
|
||||
diff_spans=[(op, text) for op, text in spans],
|
||||
)
|
||||
@@ -28,7 +28,10 @@ celery_app = Celery(
|
||||
"compliance_hub",
|
||||
broker=_BROKER,
|
||||
backend=_BACKEND,
|
||||
include=["app.infrastructure.tasks.document_tasks"],
|
||||
include=[
|
||||
"app.infrastructure.tasks.document_tasks",
|
||||
"app.infrastructure.tasks.perception_tasks",
|
||||
],
|
||||
)
|
||||
|
||||
celery_app.conf.update(
|
||||
@@ -42,4 +45,12 @@ celery_app.conf.update(
|
||||
task_reject_on_worker_lost=True,
|
||||
# Keep results for 1 hour for status polling.
|
||||
result_expires=3600,
|
||||
# Scheduled counterpart to the Perception page's manual "Refresh" button.
|
||||
# Only takes effect while a Beat process is running (./dev.sh start beat).
|
||||
beat_schedule={
|
||||
"crawl-regulations-periodic": {
|
||||
"task": "app.infrastructure.tasks.perception_tasks.crawl_regulations_task",
|
||||
"schedule": settings.perception_crawl_interval_seconds,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Celery task for scheduled regulatory source crawling.
|
||||
|
||||
This is the scheduled counterpart to the Perception page's manual "Refresh"
|
||||
button (POST /perception/crawl). Every architecture reference document
|
||||
describes source monitoring as continuous ("定时爬取"), not operator-triggered,
|
||||
so this task is what Celery Beat runs on a fixed interval once an operator
|
||||
starts a Beat process.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from app.infrastructure.tasks.celery_app import celery_app
|
||||
|
||||
|
||||
@celery_app.task(
|
||||
name="app.infrastructure.tasks.perception_tasks.crawl_regulations_task",
|
||||
bind=True,
|
||||
)
|
||||
def crawl_regulations_task(self) -> dict:
|
||||
"""Crawl every registered regulatory source and enrich new/changed events.
|
||||
|
||||
Drains CrawlService.run_crawl(), which already isolates each source's
|
||||
fetch and each event's enrichment behind its own try/except — a source
|
||||
outage or a single bad event yields an "error" progress item and the
|
||||
generator continues. Re-catching those here would only hide problems the
|
||||
service has already handled, so this task's job is limited to counting
|
||||
them and logging a summary.
|
||||
|
||||
No automatic retry is configured. An exception escaping run_crawl itself
|
||||
means something broke in a way the service's own error handling did not
|
||||
anticipate; the next scheduled tick already provides a retry within
|
||||
settings.perception_crawl_interval_seconds, so an immediate retry against
|
||||
the same failure is not worth the added complexity.
|
||||
|
||||
ponytail: relies on a single worker process to serialize scheduled runs
|
||||
(Celery's default concurrency processes one task at a time, so a run that
|
||||
outlasts the interval delays the next tick rather than overlapping it).
|
||||
Add a Redis-based lock (e.g. SETNX on a per-task key) if this queue is
|
||||
ever served by more than one worker.
|
||||
"""
|
||||
from app.shared.bootstrap import get_crawl_service
|
||||
|
||||
error_count = 0
|
||||
new_count = 0
|
||||
updated_count = 0
|
||||
|
||||
for item in get_crawl_service().run_crawl():
|
||||
event = item.get("event")
|
||||
if event == "error":
|
||||
error_count += 1
|
||||
logger.warning("Scheduled crawl source error: {}", item.get("data"))
|
||||
elif event == "done":
|
||||
data = item.get("data") or {}
|
||||
new_count = data.get("total_new", 0)
|
||||
updated_count = data.get("total_updated", 0)
|
||||
|
||||
logger.info(
|
||||
"Scheduled crawl finished: new={} updated={} source_errors={}",
|
||||
new_count, updated_count, error_count,
|
||||
)
|
||||
return {"new": new_count, "updated": updated_count, "source_errors": error_count}
|
||||
Reference in New Issue
Block a user