AIRegulation/AIRegulation-DocAnalysis
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update for mcp

Browse Source
This commit is contained in:
wangwei
2026-08-06 11:08:46 +08:00
parent 31bbf80aeb
commit b2feaeddb4
40 changed files with 1986 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/infrastructure/perception/base_notification_store.py
+47
View File
@@ -0,0 +1,47 @@
"""Abstract base class for in-app regulatory-signal notifications.
A notification is created once per triggering event (a brand-new regulation,
or a significant change to an existing one) and broadcast to every logged-in
user. There is no per-user subscription targeting — see the design doc for why.
Per-user "read" state is tracked separately from the notification itself, so
one notification row serves every user rather than being fanned out on create.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
class BaseNotificationStore(ABC):
"""Port interface for perception notification persistence."""
@abstractmethod
def create(
self,
*,
event_id: str,
kind: str,
title: str,
impact_level: str | None,
summary: str | None,
) -> None:
"""Record a new notification. kind is 'new' or 'changed'."""
@abstractmethod
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
"""Return the most recent notifications, newest first.
Each item includes a "read" boolean reflecting whether `user_id` has
marked it read.
"""
@abstractmethod
def unread_count(self, user_id: str) -> int:
"""Return how many notifications `user_id` has not yet read."""
@abstractmethod
def mark_all_read(self, user_id: str) -> int:
"""Mark every currently-unread notification read for `user_id`.
Returns the number of notifications newly marked.
"""
backend/app/infrastructure/perception/crawlers/base.py
+44 -1
View File
@@ -5,6 +5,12 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
import httpx
import trafilatura
from loguru import logger
from app.config.settings import settings
@dataclass
class RawEvent:
@@ -21,7 +27,10 @@ class RawEvent:
effective_at: str | None
category: str
tags: list[str] = field(default_factory=list)
raw_text: str = "" # full crawled text for hashing + LLM
# Whatever text the list page yields. CrawlService upgrades this by calling
# fetch_full_text(full_text_url); this value is the fallback when that
# fails. Used for change hashing and for the version diff.
raw_text: str = ""
class BaseCrawler(ABC):
@@ -30,3 +39,37 @@ class BaseCrawler(ABC):
@abstractmethod
def fetch(self, limit: int = 50) -> list[RawEvent]:
"""Fetch up to `limit` recent events from the data source."""
def fetch_full_text(self, url: str) -> str:
"""Download a regulation detail page and extract its body text.
Change detection is only as good as the text it compares, and list
pages carry nothing but a standard code and a title. This default
implementation serves all current sources; a source that needs PDF
extraction or authentication overrides this one method.
Returns an empty string on any failure rather than raising, so one
unreachable page cannot abort a whole crawl run. The caller decides how
to degrade.
"""
if not url:
return ""
try:
response = httpx.get(
url,
timeout=settings.perception_crawl_timeout_seconds,
follow_redirects=True,
)
response.raise_for_status()
except Exception as exc: # noqa: BLE001 - any transport error degrades the same way
logger.warning("Full-text fetch failed url={} err={}", url, exc)
return ""
# trafilatura scores 0.92 F1 on government pages against 0.78 for
# readability-lxml, and handles CJK content; include_tables matters
# because regulatory limits are frequently tabulated.
extracted = trafilatura.extract(response.text, include_tables=True)
if not extracted:
logger.warning("Full-text extraction returned nothing url={}", url)
return ""
return extracted.strip()
backend/app/infrastructure/perception/crawlers/catarc_crawler.py
+6 -1
View File
@@ -8,6 +8,7 @@ import httpx
from bs4 import BeautifulSoup
from loguru import logger
from app.config.settings import settings
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
from ._utils import extract_tags, parse_date
@@ -33,7 +34,11 @@ class CatarcCrawler(BaseCrawler):
while len(events) < limit and page <= max_pages:
url = f"{_BASE_URL}?page={page}"
try:
resp = httpx.get(url, timeout=30, follow_redirects=True)
resp = httpx.get(
url,
timeout=settings.perception_crawl_timeout_seconds,
follow_redirects=True,
)
resp.raise_for_status()
except Exception as exc:
logger.warning("CATARC fetch failed page={} err={}", page, exc)
backend/app/infrastructure/perception/crawlers/eurlex_crawler.py
+6 -1
View File
@@ -9,6 +9,7 @@ import httpx
from bs4 import BeautifulSoup
from loguru import logger
from app.config.settings import settings
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
from ._utils import parse_date
@@ -53,7 +54,11 @@ class EurlexCrawler(BaseCrawler):
if len(events) >= limit:
break
try:
resp = httpx.get(rss_url, timeout=30, follow_redirects=True)
resp = httpx.get(
rss_url,
timeout=settings.perception_crawl_timeout_seconds,
follow_redirects=True,
)
resp.raise_for_status()
except Exception as exc:
logger.warning("EUR-Lex RSS fetch failed url={} err={}", rss_url, exc)
backend/app/infrastructure/perception/crawlers/guobiao_crawler.py
+7 -1
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import httpx
from loguru import logger
from app.config.settings import settings
from app.infrastructure.perception.crawlers.base import BaseCrawler, RawEvent
from ._utils import extract_tags, parse_date
@@ -22,7 +23,12 @@ def _fetch_page(std_type: int, page: int, page_size: int) -> list[dict]:
"p.p7": page_size,
}
try:
resp = httpx.get(_BASE_URL, params=params, headers=_HEADERS, timeout=30)
resp = httpx.get(
_BASE_URL,
params=params,
headers=_HEADERS,
timeout=settings.perception_crawl_timeout_seconds,
)
resp.raise_for_status()
data = resp.json()
return data.get("rows", []) or []
backend/app/infrastructure/perception/llm_pipeline.py
+84 -80
View File
@@ -3,14 +3,14 @@
from __future__ import annotations
import json
import math
from typing import Any
from loguru import logger
from app.config.settings import settings
from app.infrastructure.embedding.openai_compatible_embedding_provider import (
OpenAICompatibleEmbeddingProvider,
from app.infrastructure.perception.regulation_differ import (
ParagraphChange,
RegulationDiffer,
)
from app.services.llm.llm_factory import get_llm_client
@@ -27,21 +27,31 @@ _ASSESS_SYSTEM = (
)
_DIFF_SYSTEM = (
"You are a regulatory change analyst. Given an old and new version of a regulation paragraph, "
"classify the type of change and summarise it. "
"Return JSON only: {\"change_type\": \"tightened|relaxed|added|removed\", \"summary\": \"...\"}"
"You are a regulatory change analyst. You are given the OLD and NEW version of "
"one regulation paragraph, with the exact edits marked <DEL>removed</DEL> and "
"<INS>added</INS>. Classify the legal effect of the change. "
"Return JSON only: {\"change_type\": \"tightened|relaxed|numeric|clarified|scope\", "
"\"legal_effect\": \"one sentence on what this means for compliance\"}"
)
_SIMILARITY_THRESHOLD = 0.85
def _marked_diff(change: ParagraphChange) -> str:
"""Render a paragraph change with the exact edits marked for the model.
def _cosine(a: list[float], b: list[float]) -> float:
dot = sum(x * y for x, y in zip(a, b))
norm_a = math.sqrt(sum(x * x for x in a))
norm_b = math.sqrt(sum(x * x for x in b))
if norm_a == 0 or norm_b == 0:
return 0.0
return dot / (norm_a * norm_b)
The model is shown where the edit is rather than being asked to find it,
and is never asked to reproduce the changed text — the differ already
computed those spans exactly, so there is nothing for the model to
hallucinate.
"""
marked = "".join(
text if op == 0 else (f"<DEL>{text}</DEL>" if op < 0 else f"<INS>{text}</INS>")
for op, text in change.diff_spans
)
return (
f"OLD: {change.old_text[:500]}\n"
f"NEW: {change.new_text[:500]}\n"
f"MARKED: {marked[:800]}"
)
def _llm_json(client: Any, messages: list[dict]) -> Any:
@@ -67,7 +77,9 @@ class LlmPipeline:
provider=settings.llm_provider,
model=settings.llm_model,
)
self._embedder = OpenAICompatibleEmbeddingProvider()
# Change detection is deterministic; the differ needs no model and no
# network, so the pipeline no longer constructs an embedding provider.
self._differ = RegulationDiffer()
# ------------------------------------------------------------------
# Step 1: Structure extraction
@@ -166,76 +178,68 @@ For each document, assess impact and recommend action. Return JSON array:
return doc_excerpts
# ------------------------------------------------------------------
# Step 3: Semantic diff
# Step 3: Deterministic diff with gated LLM classification
# ------------------------------------------------------------------
def compute_diff(self, old_text: str, new_text: str) -> dict:
"""Compare old and new regulation text; return changed sections and summary."""
old_paras = [p.strip() for p in old_text.split("\n") if p.strip()]
new_paras = [p.strip() for p in new_text.split("\n") if p.strip()]
"""Compare old and new regulation text; return changed sections and summary.
if not old_paras or not new_paras:
return {"changed_sections": [], "change_summary": "No comparable text."}
Detection is deterministic — see regulation_differ for why embedding
similarity was removed. The LLM is called only for paragraphs the
differ marked significant, and only to explain the legal effect of a
change that has already been located exactly.
"""
changes = self._differ.diff(old_text, new_text)
if not changes:
return {
"changed_sections": [],
"change_summary": "No substantive changes detected between versions.",
}
all_paras = old_paras + new_paras
try:
all_embeddings = self._embedder.embed_texts(all_paras)
except Exception as exc:
logger.warning("Embedding for diff failed: {}", exc)
return {"changed_sections": [], "change_summary": "Diff unavailable (embedding error)."}
changed_sections = [self._describe(change) for change in changes]
old_embeddings = all_embeddings[: len(old_paras)]
new_embeddings = all_embeddings[len(old_paras):]
changed_sections: list[dict] = []
max_len = max(len(old_paras), len(new_paras))
for i in range(max_len):
if i >= len(old_paras):
# New paragraph added
changed_sections.append({
"old_text": "",
"new_text": new_paras[i][:300],
"similarity": 0.0,
"change_type": "added",
"summary": "New paragraph added.",
})
continue
if i >= len(new_paras):
# Old paragraph removed
changed_sections.append({
"old_text": old_paras[i][:300],
"new_text": "",
"similarity": 0.0,
"change_type": "removed",
"summary": "Paragraph removed.",
})
continue
# Both exist — compare via embeddings
sim = _cosine(old_embeddings[i], new_embeddings[i])
if sim < _SIMILARITY_THRESHOLD:
messages = [
{"role": "system", "content": _DIFF_SYSTEM},
{"role": "user", "content": f"OLD: {old_paras[i][:500]}\nNEW: {new_paras[i][:500]}"},
]
classification = _llm_json(self._client, messages) or {}
changed_sections.append({
"old_text": old_paras[i][:300],
"new_text": new_paras[i][:300],
"similarity": round(sim, 3),
"change_type": classification.get("change_type", "modified"),
"summary": classification.get("summary", ""),
})
if not changed_sections:
change_summary = "No substantive changes detected between versions."
else:
types = [s["change_type"] for s in changed_sections]
change_summary = (
f"{len(changed_sections)} paragraph(s) changed: "
+ ", ".join(f"{t}" for t in set(types))
+ ". "
+ (changed_sections[0].get("summary", "") if changed_sections else "")
)
types = sorted({section["change_type"] for section in changed_sections})
gated = sum(1 for change in changes if change.needs_llm)
change_summary = (
f"{len(changed_sections)} paragraph(s) changed ({', '.join(types)}); "
f"{gated} significant. "
+ (changed_sections[0].get("summary") or "")
).strip()
return {"changed_sections": changed_sections, "change_summary": change_summary}
def _describe(self, change: ParagraphChange) -> dict:
"""Turn one detected change into the API payload, classifying if warranted."""
section = {
"old_text": change.old_text[:300],
"new_text": change.new_text[:300],
"change_type": change.change_type,
"change_ratio": round(change.change_ratio, 3),
"numeric_changed": change.numeric_changed,
"deontic_changed": change.deontic_changed,
"summary": "",
}
if not change.needs_llm:
return section
classification = _llm_json(
self._client,
[
{"role": "system", "content": _DIFF_SYSTEM},
{"role": "user", "content": _marked_diff(change)},
],
)
if isinstance(classification, dict):
section["change_type"] = classification.get("change_type") or change.change_type
section["summary"] = classification.get("legal_effect") or ""
# A failed or malformed model response must not discard a change that
# deterministic analysis already proved real; the section keeps its
# spans, flags, and alignment-derived type with an empty summary.
if change.numeric_changed:
# Models routinely label a changed threshold as "clarified". The
# deterministic pass already knows a number moved, so it wins.
section["change_type"] = "numeric"
return section
backend/app/infrastructure/perception/mock_notification_store.py
+66
View File
@@ -0,0 +1,66 @@
"""In-memory notification store used when Postgres is not configured.
Mirrors MockEventStore's role for BaseEventStore: keeps the feature usable in
local dev and in tests without a live database, and matches
DOCUMENT_REPOSITORY_BACKEND's existing Mock/Postgres split.
"""
from __future__ import annotations
from datetime import UTC, datetime
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
class MockNotificationStore(BaseNotificationStore):
"""Dict-backed notification store. Data does not survive a process restart."""
def __init__(self) -> None:
"""Start with an empty feed and no read receipts."""
self._notifications: list[dict] = []
self._next_id = 1
# (notification_id, user_id) pairs — presence means read.
self._reads: set[tuple[int, str]] = set()
def create(
self,
*,
event_id: str,
kind: str,
title: str,
impact_level: str | None,
summary: str | None,
) -> None:
"""Append a notification with an auto-incrementing id."""
self._notifications.append({
"id": self._next_id,
"event_id": event_id,
"kind": kind,
"title": title,
"impact_level": impact_level,
"summary": summary,
"created_at": datetime.now(UTC).isoformat(),
})
self._next_id += 1
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
"""Return the newest `limit` notifications with this user's read state."""
ordered = sorted(self._notifications, key=lambda n: n["id"], reverse=True)
return [
{**n, "read": (n["id"], user_id) in self._reads}
for n in ordered[:limit]
]
def unread_count(self, user_id: str) -> int:
"""Count notifications this user has not yet read."""
return sum(1 for n in self._notifications if (n["id"], user_id) not in self._reads)
def mark_all_read(self, user_id: str) -> int:
"""Add a read receipt for every currently-unread notification."""
marked = 0
for n in self._notifications:
key = (n["id"], user_id)
if key not in self._reads:
self._reads.add(key)
marked += 1
return marked
backend/app/infrastructure/perception/postgres_event_store.py
+11 -2
View File
@@ -40,7 +40,8 @@ CREATE TABLE IF NOT EXISTS regulation_events (
affected_docs JSONB,
crawled_at TIMESTAMPTZ DEFAULT now(),
processed_at TIMESTAMPTZ,
raw_storage_key TEXT
raw_storage_key TEXT,
raw_text TEXT
);
CREATE INDEX IF NOT EXISTS reg_events_source_date
ON regulation_events (source, published_at DESC);
@@ -48,12 +49,16 @@ CREATE INDEX IF NOT EXISTS reg_events_impact_date
ON regulation_events (impact_level, published_at DESC);
"""
_ADD_COLUMNS = """
ALTER TABLE regulation_events ADD COLUMN IF NOT EXISTS raw_text TEXT;
"""
_ALL_COLUMNS = (
"id", "source", "source_label", "standard_code", "title", "summary",
"full_text_url", "status", "impact_level", "published_at", "effective_at",
"category", "tags", "obligations", "deadlines", "scope", "penalties",
"content_hash", "previous_hash", "change_summary", "changed_sections",
"affected_docs", "crawled_at", "processed_at", "raw_storage_key",
"affected_docs", "crawled_at", "processed_at", "raw_storage_key", "raw_text",
)
@@ -97,6 +102,10 @@ class PostgresEventStore(BaseEventStore):
try:
with conn.cursor() as cur:
cur.execute(_CREATE_TABLE)
# CREATE TABLE IF NOT EXISTS is a no-op on deployments that
# already have this table, so new columns must be added
# explicitly or existing installations silently lack them.
cur.execute(_ADD_COLUMNS)
conn.commit()
except Exception:
conn.rollback()
backend/app/infrastructure/perception/postgres_notification_store.py
+157
View File
@@ -0,0 +1,157 @@
"""PostgreSQL-backed notification store.
One row per triggering event, shared by every user; a separate read-receipt
table tracks per-user read state so broadcasting to everyone needs no fan-out
insert per user. See base_notification_store.py for the port contract and the
design doc for why this shape was chosen over per-user subscriptions.
"""
from __future__ import annotations
from contextlib import contextmanager
from typing import Any
import psycopg2
import psycopg2.extras
from psycopg2.pool import ThreadedConnectionPool
from app.config.settings import settings
from app.infrastructure.perception.base_notification_store import BaseNotificationStore
_CREATE_TABLES = """
CREATE TABLE IF NOT EXISTS perception_notifications (
id SERIAL PRIMARY KEY,
event_id TEXT NOT NULL REFERENCES regulation_events(id) ON DELETE CASCADE,
kind TEXT NOT NULL,
title TEXT NOT NULL,
impact_level TEXT,
summary TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS perception_notification_reads (
notification_id INTEGER NOT NULL REFERENCES perception_notifications(id) ON DELETE CASCADE,
user_id TEXT NOT NULL,
read_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (notification_id, user_id)
);
CREATE INDEX IF NOT EXISTS perception_notif_created
ON perception_notifications (created_at DESC);
"""
def _row_to_dict(row: dict[str, Any]) -> dict:
"""Convert a psycopg2 RealDictRow to a plain dict with an ISO timestamp."""
d = dict(row)
if d.get("created_at") is not None:
d["created_at"] = d["created_at"].isoformat()
return d
class PostgresNotificationStore(BaseNotificationStore):
"""Notification store backed by PostgreSQL."""
def __init__(self) -> None:
"""Open a connection pool and ensure both tables exist."""
self._pool = ThreadedConnectionPool(
minconn=1,
maxconn=5,
host=settings.postgres_host,
port=settings.postgres_port,
user=settings.postgres_user,
password=settings.postgres_password,
dbname=settings.postgres_db,
)
self._ensure_schema()
def _ensure_schema(self) -> None:
with self._conn() as conn:
try:
with conn.cursor() as cur:
cur.execute(_CREATE_TABLES)
conn.commit()
except Exception:
conn.rollback()
raise
@contextmanager
def _conn(self):
conn = None
try:
conn = self._pool.getconn()
yield conn
finally:
if conn is not None:
self._pool.putconn(conn)
def create(
self,
*,
event_id: str,
kind: str,
title: str,
impact_level: str | None,
summary: str | None,
) -> None:
"""Insert one notification row for the triggering event."""
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
"INSERT INTO perception_notifications "
"(event_id, kind, title, impact_level, summary) "
"VALUES (%s, %s, %s, %s, %s)",
(event_id, kind, title, impact_level, summary),
)
conn.commit()
def list_for_user(self, user_id: str, limit: int = 20) -> list[dict]:
"""Return the newest notifications with this user's read state joined in."""
with self._conn() as conn:
with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
cur.execute(
"""
SELECT n.*, (r.user_id IS NOT NULL) AS read
FROM perception_notifications n
LEFT JOIN perception_notification_reads r
ON r.notification_id = n.id AND r.user_id = %s
ORDER BY n.created_at DESC
LIMIT %s
""",
(user_id, limit),
)
return [_row_to_dict(r) for r in cur.fetchall()]
def unread_count(self, user_id: str) -> int:
"""Count notifications with no read receipt for this user."""
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT COUNT(*) FROM perception_notifications n
WHERE NOT EXISTS (
SELECT 1 FROM perception_notification_reads r
WHERE r.notification_id = n.id AND r.user_id = %s
)
""",
(user_id,),
)
return cur.fetchone()[0]
def mark_all_read(self, user_id: str) -> int:
"""Insert a read receipt for every notification this user hasn't read."""
with self._conn() as conn:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO perception_notification_reads (notification_id, user_id)
SELECT n.id, %s FROM perception_notifications n
WHERE NOT EXISTS (
SELECT 1 FROM perception_notification_reads r
WHERE r.notification_id = n.id AND r.user_id = %s
)
ON CONFLICT (notification_id, user_id) DO NOTHING
""",
(user_id, user_id),
)
marked = cur.rowcount
conn.commit()
return marked
backend/app/infrastructure/perception/regulation_differ.py
+224
View File
@@ -0,0 +1,224 @@
"""Deterministic change detection between two versions of a regulation.
This module deliberately contains no LLM call, no network access, and no
embedding lookup. It exists because the previous implementation decided whether
a paragraph had changed by comparing embedding cosine similarity against a 0.85
threshold, which is blind to exactly the edits that matter in regulation.
Measured against the deployed text-embedding-v3 gateway, tightening a braking
limit from 30米 to 20米 scores 0.9153 and relaxing 应当 to 宜 scores 0.9162 —
both far above the threshold, both undetected — while an entirely unrelated
clause scores 0.6862 and is the only thing that fires. Cosine is scale
invariant, so it cannot represent a change in magnitude or certainty
(arXiv:2403.05440, ACM Web Conference 2024); no threshold recovers the signal.
The replacement is the production consensus for legal text: align paragraphs
with a longest-common-subsequence matcher, run a literal character diff on the
aligned pairs, and let cheap deterministic rules decide whether a change is
significant enough to spend an LLM call classifying.
"""
from __future__ import annotations
import re
import unicodedata
from dataclasses import dataclass, field
from diff_match_patch import diff_match_patch
from difflib import SequenceMatcher
from app.config.settings import settings
# Chinese regulatory drafting uses a small, near-unambiguous set of deontic
# markers, so a regex pre-pass identifies legally significant edits without an
# LLM. Adding or removing any of these changes what the provision compels.
_DEONTIC_PATTERN = re.compile(r"应当|须|禁止|不得|可以|允许|宜")
# Matches digit runs including decimals, so "30" -> "20" and "0.85" -> "0.9"
# are both treated as numeric changes.
_NUMBER_PATTERN = re.compile(r"\d+(?:\.\d+)?")
# diff_match_patch operation codes.
_DMP_DELETE = -1
_DMP_INSERT = 1
_DMP_EQUAL = 0
@dataclass(frozen=True)
class ParagraphChange:
"""One detected difference between the old and new version of a regulation.
`needs_llm` is the gate: it records whether this change is worth the cost of
an LLM classification call. The deterministic flags that drive it are kept
on the record so downstream code can act on them even when the LLM call
fails or is skipped.
"""
change_type: str
old_text: str
new_text: str
numeric_changed: bool
deontic_changed: bool
change_ratio: float
needs_llm: bool
# (op, text) pairs from diff_match_patch, for rendering a redline view.
diff_spans: list[tuple[int, str]] = field(default_factory=list)
def _split_paragraphs(text: str) -> list[str]:
"""Split regulation text into comparable units, dropping blank lines.
ponytail: newline splitting, not clause parsing. Upgrade to 第X条 / X.X.X
segmentation only if paragraph granularity proves too coarse in practice.
"""
return [line.strip() for line in (text or "").split("\n") if line.strip()]
def _numbers_differ(old: str, new: str) -> bool:
"""Report whether the two spans contain a different sequence of numbers."""
return _NUMBER_PATTERN.findall(old) != _NUMBER_PATTERN.findall(new)
def _deontic_differs(old: str, new: str) -> bool:
"""Report whether obligation markers were added, removed, or swapped."""
return sorted(_DEONTIC_PATTERN.findall(old)) != sorted(_DEONTIC_PATTERN.findall(new))
def _is_cosmetic(spans: list[tuple[int, str]]) -> bool:
"""Report whether the edit touched nothing but punctuation and whitespace.
A change ratio alone cannot answer this for Chinese regulation text. Clauses
run 20-60 characters, so deleting a single 。 is a 4% change and clears any
threshold low enough to still catch real edits in longer paragraphs. Testing
what actually changed is both cheaper and exact.
"""
changed = "".join(text for op, text in spans if op != _DMP_EQUAL)
# Unicode categories P (punctuation), Z (separator) and C (control) cover
# Chinese and ASCII punctuation plus every flavour of whitespace.
return all(unicodedata.category(char)[0] in {"P", "Z", "C"} for char in changed)
class RegulationDiffer:
"""Align two regulation versions and classify what changed, without an LLM."""
def __init__(self, min_change_ratio: float | None = None) -> None:
"""Store the gate threshold, defaulting to the configured value.
The explicit argument exists so tests never depend on the deployed .env.
"""
self._min_change_ratio = (
settings.perception_diff_min_change_ratio
if min_change_ratio is None
else min_change_ratio
)
self._dmp = diff_match_patch()
def diff(self, old_text: str, new_text: str) -> list[ParagraphChange]:
"""Return every changed paragraph between two versions.
Unchanged paragraphs are not returned. An empty old version means there
is no baseline to compare against — the caller's first crawl — so no
changes are reported rather than the whole document being called new.
"""
old_paras = _split_paragraphs(old_text)
new_paras = _split_paragraphs(new_text)
if not old_paras or not new_paras:
return []
# autojunk=False is load-bearing: the default treats any element
# appearing in over 1% of a sequence of 200+ items as junk, and
# regulations repeat boilerplate paragraphs that alignment depends on
# as anchors.
matcher = SequenceMatcher(None, old_paras, new_paras, autojunk=False)
changes: list[ParagraphChange] = []
for tag, i1, i2, j1, j2 in matcher.get_opcodes():
if tag == "equal":
continue
if tag == "insert":
changes.extend(self._added(p) for p in new_paras[j1:j2])
elif tag == "delete":
changes.extend(self._removed(p) for p in old_paras[i1:i2])
elif tag == "replace":
changes.extend(self._replaced(old_paras[i1:i2], new_paras[j1:j2]))
return changes
def _added(self, paragraph: str) -> ParagraphChange:
"""Build a record for a provision present only in the new version."""
return ParagraphChange(
change_type="added",
old_text="",
new_text=paragraph,
numeric_changed=False,
deontic_changed=bool(_DEONTIC_PATTERN.search(paragraph)),
change_ratio=1.0,
# A new provision always carries new obligations, so it is always
# worth classifying.
needs_llm=True,
diff_spans=[(_DMP_INSERT, paragraph)],
)
def _removed(self, paragraph: str) -> ParagraphChange:
"""Build a record for a provision dropped from the new version."""
return ParagraphChange(
change_type="removed",
old_text=paragraph,
new_text="",
numeric_changed=False,
deontic_changed=bool(_DEONTIC_PATTERN.search(paragraph)),
change_ratio=1.0,
needs_llm=True,
diff_spans=[(_DMP_DELETE, paragraph)],
)
def _replaced(self, old_block: list[str], new_block: list[str]) -> list[ParagraphChange]:
"""Compare a run of rewritten paragraphs pairwise, reporting the remainder.
SequenceMatcher emits `replace` for a whole run at once, and the two
sides may differ in length. Pairing by position within the run is safe
here because alignment has already established that this run as a whole
corresponds; any surplus on either side is a genuine insertion or
deletion.
"""
results: list[ParagraphChange] = []
for index in range(max(len(old_block), len(new_block))):
if index >= len(old_block):
results.append(self._added(new_block[index]))
elif index >= len(new_block):
results.append(self._removed(old_block[index]))
else:
results.append(self._modified(old_block[index], new_block[index]))
return results
def _modified(self, old: str, new: str) -> ParagraphChange:
"""Character-diff an aligned pair and decide whether it warrants an LLM call."""
spans = self._dmp.diff_main(old, new)
# Merges single-character edits into human-meaningful chunks so the
# redline view and the change ratio both reflect real edits.
self._dmp.diff_cleanupSemantic(spans)
changed_chars = sum(len(text) for op, text in spans if op != _DMP_EQUAL)
denominator = max(len(old), len(new), 1)
change_ratio = changed_chars / denominator
numeric_changed = _numbers_differ(old, new)
deontic_changed = _deontic_differs(old, new)
# A changed limit or obligation marker is always significant no matter
# how few characters moved. Everything else must be substantive and
# clear the ratio gate to be worth a model call.
significant = numeric_changed or deontic_changed or (
not _is_cosmetic(spans) and change_ratio >= self._min_change_ratio
)
return ParagraphChange(
change_type="modified",
old_text=old,
new_text=new,
numeric_changed=numeric_changed,
deontic_changed=deontic_changed,
change_ratio=change_ratio,
needs_llm=significant,
diff_spans=[(op, text) for op, text in spans],
)