AIRegulation/AIRegulation-DocAnalysis
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix SSE route dependency and align architecture docs

Browse Source
This commit is contained in:
ash66
2026-05-18 16:32:42 +08:00
parent 86b9ac806a
commit 3f69cad404
149 changed files with 4786 additions and 5957 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
backend/app/services/rag/__init__.py
View File

@@ -1,11 +1,29 @@
"""RAG服务模块"""
"""Initialize the app.services.rag package."""
# Keep package boundaries explicit so backend imports stay predictable.
from .retriever import Retriever, retrieve_regulations
from .context_builder import ContextBuilder, build_rag_context
from .prompt_templates import PromptTemplates, get_prompt_template
__all__ = [
"Retriever", "retrieve_regulations",
"ContextBuilder", "build_rag_context",
"PromptTemplates", "get_prompt_template"
"Retriever",
"retrieve_regulations",
"ContextBuilder",
"build_rag_context",
"PromptTemplates",
"get_prompt_template",
]
def __getattr__(name: str):
"""Handle getattr for this module."""
if name in {"Retriever", "retrieve_regulations"}:
from .retriever import Retriever, retrieve_regulations
return {"Retriever": Retriever, "retrieve_regulations": retrieve_regulations}[name]
if name in {"ContextBuilder", "build_rag_context"}:
from .context_builder import ContextBuilder, build_rag_context
return {"ContextBuilder": ContextBuilder, "build_rag_context": build_rag_context}[name]
if name in {"PromptTemplates", "get_prompt_template"}:
from .prompt_templates import PromptTemplates, get_prompt_template
return {"PromptTemplates": PromptTemplates, "get_prompt_template": get_prompt_template}[name]
raise AttributeError(name)

88
backend/app/services/rag/context_builder.py
View File

@@ -1,4 +1,4 @@
"""RAG上下文构建服务 - 构建LLM输入上下文"""
"""Provide service-layer logic for context builder."""
from typing import List, Dict, Optional
from dataclasses import dataclass
@@ -6,11 +6,13 @@ from loguru import logger
from .retriever import RetrievedDocument
from app.config.settings import settings
# Keep service responsibilities explicit so downstream behavior stays predictable.
@dataclass
class RAGContext:
"""RAG构建的上下文"""
"""Represent the R A G Context type."""
system_prompt: str
context_text: str
user_query: str
@@ -20,14 +22,7 @@ class RAGContext:
class ContextBuilder:
"""
RAG上下文构建器
功能：
- 格式化检索结果为上下文文本
- 控制上下文长度（token限制）
- 构建完整的LLM输入格式
"""
"""Provide the Context Builder builder."""
def __init__(
self,
@@ -35,14 +30,7 @@ class ContextBuilder:
include_metadata: bool = True,
citation_format: str = "【条款{clause}】"
):
"""
初始化上下文构建器
Args:
max_context_tokens: 最大上下文token数
include_metadata: 是否包含元数据（文档名、条款号等）
citation_format: 引用格式模板
"""
"""Initialize the Context Builder instance."""
self.max_context_tokens = max_context_tokens or settings.rag_max_context_tokens
self.include_metadata = include_metadata
self.citation_format = citation_format
@@ -56,30 +44,19 @@ class ContextBuilder:
system_prompt: Optional[str] = None,
max_tokens: Optional[int] = None
) -> RAGContext:
"""
构建RAG上下文
Args:
query: 用户查询
documents: 检索到的文档列表
system_prompt: 系统提示词（可选）
max_tokens: 最大token数（可选，覆盖默认值）
Returns:
RAGContext: 构建的上下文对象
"""
"""Handle build for the Context Builder instance."""
max_tokens = max_tokens or self.max_context_tokens
# 格式化文档内容
# Keep service responsibilities explicit so downstream behavior stays predictable.
context_text, sources, truncated = self._format_documents(
documents,
max_tokens
)
# 构建系统提示词
# Keep service responsibilities explicit so downstream behavior stays predictable.
system_prompt = system_prompt or self._default_system_prompt()
# 估算总token数
# Keep service responsibilities explicit so downstream behavior stays predictable.
total_tokens = self._estimate_tokens(system_prompt + context_text + query)
logger.info(f"上下文构建完成: {len(documents)}条文档, {total_tokens}tokens, truncated={truncated}")
@@ -98,29 +75,20 @@ class ContextBuilder:
documents: List[RetrievedDocument],
max_tokens: int
) -> tuple:
"""
格式化文档内容
Args:
documents: 文档列表
max_tokens: 最大token数
Returns:
(context_text, sources, truncated)
"""
"""Handle format documents for this module for the Context Builder instance."""
context_parts = []
sources = []
current_tokens = 0
truncated = False
for i, doc in enumerate(documents):
# 格式化单个文档
# Keep service responsibilities explicit so downstream behavior stays predictable.
formatted = self._format_single_doc(doc, i + 1)
# 估算token数
# Keep service responsibilities explicit so downstream behavior stays predictable.
doc_tokens = self._estimate_tokens(formatted)
# 检查是否超出限制
# Keep service responsibilities explicit so downstream behavior stays predictable.
if current_tokens + doc_tokens > max_tokens:
truncated = True
logger.warning(f"上下文截断: 已达到{max_tokens}tokens限制")
@@ -129,7 +97,7 @@ class ContextBuilder:
context_parts.append(formatted)
current_tokens += doc_tokens
# 记录来源
# Keep service responsibilities explicit so downstream behavior stays predictable.
sources.append({
"index": i + 1,
"doc_id": doc.doc_id,
@@ -148,13 +116,13 @@ class ContextBuilder:
doc: RetrievedDocument,
index: int
) -> str:
"""格式化单个文档"""
"""Handle format single doc for this module for the Context Builder instance."""
parts = []
# 索引编号
# Keep service responsibilities explicit so downstream behavior stays predictable.
parts.append(f"[{index}]")
# 元数据（可选）
# Keep service responsibilities explicit so downstream behavior stays predictable.
if self.include_metadata:
meta_parts = []
@@ -171,13 +139,13 @@ class ContextBuilder:
if meta_parts:
parts.append(" | ".join(meta_parts))
# 内容
# Keep service responsibilities explicit so downstream behavior stays predictable.
parts.append(doc.content)
return "\n".join(parts)
def _default_system_prompt(self) -> str:
"""默认系统提示词"""
"""Handle default system prompt for this module for the Context Builder instance."""
return """你是合规专家助手，基于提供的法规条款回答问题。
回答要求：
@@ -192,8 +160,8 @@ class ContextBuilder:
- 最后给出合规建议"""
def _estimate_tokens(self, text: str) -> int:
"""估算文本token数"""
# 中文字符约1.5 token，英文约0.25 token
"""Handle estimate tokens for this module for the Context Builder instance."""
# Keep service responsibilities explicit so downstream behavior stays predictable.
chinese_chars = sum(1 for c in text if '一' <= c <= '鿿')
other_chars = len(text) - chinese_chars
return int(chinese_chars * 1.5 + other_chars * 0.25)
@@ -202,15 +170,7 @@ class ContextBuilder:
self,
context: RAGContext
) -> List[Dict[str, str]]:
"""
构建LLM消息格式
Args:
context: RAG上下文对象
Returns:
List[Dict]: [{"role": "system/user/assistant", "content": "..."}]
"""
"""Build messages for the Context Builder instance."""
messages = [
{"role": "system", "content": context.system_prompt},
{"role": "user", "content": f"参考以下法规条款回答问题。\n\n{context.context_text}\n\n问题：{context.user_query}"}
@@ -224,6 +184,6 @@ def build_rag_context(
documents: List[RetrievedDocument],
**kwargs
) -> RAGContext:
"""便捷函数：构建RAG上下文"""
"""Build rag context."""
builder = ContextBuilder()
return builder.build(query, documents, **kwargs)

35
backend/app/services/rag/prompt_templates.py
View File

@@ -1,12 +1,14 @@
"""RAG Prompt模板 - 合规问答专用Prompt"""
"""Provide service-layer logic for prompt templates."""
from typing import Dict, Optional
from dataclasses import dataclass
# Keep service responsibilities explicit so downstream behavior stays predictable.
@dataclass
class PromptTemplate:
"""Prompt模板"""
"""Represent the Prompt Template type."""
name: str
system_prompt: str
user_template: str
@@ -14,18 +16,9 @@ class PromptTemplate:
class PromptTemplates:
"""
合规问答Prompt模板库
"""Represent the Prompt Templates type."""
包含多种场景的Prompt模板：
- 合规问答（标准）
- 条款解读（详细解释）
- 合规检查（判断合规状态）
- 差异对比（新旧法规对比）
- 报告生成（合规报告）
"""
# 合规问答标准模板
# Keep service responsibilities explicit so downstream behavior stays predictable.
COMPLIANCE_QA = PromptTemplate(
name="compliance_qa",
system_prompt="""你是合规专家助手，专门解答法规合规问题。
@@ -63,7 +56,7 @@ class PromptTemplates:
description="标准合规问答模板"
)
# 条款解读模板（详细解释）
# Keep service responsibilities explicit so downstream behavior stays predictable.
CLAUSE_INTERPRETATION = PromptTemplate(
name="clause_interpretation",
system_prompt="""你是法规解读专家，负责详细解释法规条款的含义和应用。
@@ -96,7 +89,7 @@ class PromptTemplates:
description="条款详细解读模板"
)
# 合规检查模板（判断合规状态）
# Keep service responsibilities explicit so downstream behavior stays predictable.
COMPLIANCE_CHECK = PromptTemplate(
name="compliance_check",
system_prompt="""你是合规检查专家，负责评估企业行为或产品的合规状态。
@@ -140,7 +133,7 @@ class PromptTemplates:
description="合规检查评估模板"
)
# 差异对比模板（新旧法规对比）
# Keep service responsibilities explicit so downstream behavior stays predictable.
COMPARISON = PromptTemplate(
name="comparison",
system_prompt="""你是法规变更分析专家，负责对比新旧法规版本的差异。
@@ -192,7 +185,7 @@ class PromptTemplates:
description="法规版本对比模板"
)
# 报告生成模板
# Keep service responsibilities explicit so downstream behavior stays predictable.
REPORT_GENERATION = PromptTemplate(
name="report_generation",
system_prompt="""你是合规报告撰写专家，负责生成结构化的合规分析报告。
@@ -222,7 +215,7 @@ class PromptTemplates:
description="合规报告生成模板"
)
# 文档摘要生成模板
# Keep service responsibilities explicit so downstream behavior stays predictable.
DOCUMENT_SUMMARY = PromptTemplate(
name="document_summary",
system_prompt="""你是法规文档摘要专家，负责生成法规文档的核心要点摘要。
@@ -263,7 +256,7 @@ class PromptTemplates:
@classmethod
def get_template(cls, name: str) -> Optional[PromptTemplate]:
"""获取指定模板"""
"""Return template for the Prompt Templates instance."""
templates = {
"compliance_qa": cls.COMPLIANCE_QA,
"clause_interpretation": cls.CLAUSE_INTERPRETATION,
@@ -276,7 +269,7 @@ class PromptTemplates:
@classmethod
def list_templates(cls) -> Dict[str, str]:
"""列出所有模板"""
"""List templates for the Prompt Templates instance."""
return {
"compliance_qa": cls.COMPLIANCE_QA.description,
"clause_interpretation": cls.CLAUSE_INTERPRETATION.description,
@@ -288,7 +281,7 @@ class PromptTemplates:
def get_prompt_template(name: str) -> PromptTemplate:
"""便捷函数：获取Prompt模板"""
"""Return prompt template."""
template = PromptTemplates.get_template(name)
if not template:
raise ValueError(f"不存在的模板: {name}")

220
backend/app/services/rag/retriever.py
View File

@@ -1,192 +1,82 @@
"""RAG检索服务 - 封装Milvus检索"""
"""Provide service-layer logic for retriever."""
from __future__ import annotations
from typing import List, Dict, Optional, Any
from dataclasses import dataclass, field
from loguru import logger
from typing import Any, Optional
from app.shared.bootstrap import get_retrieval_service
# Keep service responsibilities explicit so downstream behavior stays predictable.
from app.services.embedding.bge_m3_embedder import BGEM3Embedder
from app.services.storage.milvus_client import MilvusClient, SearchResult
from app.config.settings import settings
@dataclass
class RetrievedDocument:
"""检索到的文档"""
"""Represent the Retrieved Document type."""
content: str
doc_id: str # 文档ID，用于下载
doc_id: str
doc_name: str
section_title: str
clause_number: str
page_number: int
score: float
metadata: Dict[str, Any] = field(default_factory=dict)
metadata: dict[str, Any] = field(default_factory=dict)
class Retriever:
"""
RAG检索器
功能：
- 向量检索（Dense + Sparse混合）
- 重排序（可选）
- 过滤和筛选
"""
def __init__(
self,
top_k: int = None,
rerank: bool = False,
min_score: float = 0.3
):
"""
初始化检索器
Args:
top_k: 检索召回数量
rerank: 是否启用重排序
min_score: 最低相关性分数阈值
"""
self.top_k = top_k or settings.rag_top_k
"""Provide the Retriever retriever."""
def __init__(self, top_k: int = 5, rerank: bool = False, min_score: float = 0.0):
"""Initialize the Retriever instance."""
self.top_k = top_k
self.rerank = rerank
self.min_score = min_score
# 嵌入模型（延迟加载）
self.embedder: Optional[BGEM3Embedder] = None
# Milvus客户端（延迟连接）
self.milvus: Optional[MilvusClient] = None
logger.info(f"检索器初始化: top_k={self.top_k}, rerank={self.rerank}")
def _init_embedder(self):
"""延迟初始化嵌入模型"""
if self.embedder is None:
logger.info("加载嵌入模型...")
self.embedder = BGEM3Embedder(model_name=settings.embedding_model)
def _init_milvus(self):
"""延迟初始化Milvus"""
if self.milvus is None:
logger.info("连接Milvus...")
self.milvus = MilvusClient()
self.milvus.connect()
self.milvus.create_collection(recreate=False)
self.milvus.load_collection()
def retrieve(
self,
query: str,
filters: Optional[str] = None,
top_k: Optional[int] = None
) -> List[RetrievedDocument]:
"""
检索相关文档
Args:
query: 查询文本
filters: 过滤条件（如 "regulation_type=='车辆安全'"）
top_k: 返回数量（可选，覆盖默认值）
Returns:
List[RetrievedDocument]: 检索结果列表
"""
logger.info(f"执行检索: {query}")
# 初始化组件
self._init_embedder()
self._init_milvus()
# 生成查询向量
query_embedding = self.embedder.embed_single(query)
# 执行混合检索
results = self.milvus.hybrid_search(
query_dense=query_embedding['dense'].tolist(),
query_sparse=query_embedding['sparse'],
top_k=top_k or self.top_k,
filters=filters
)
# 转换为RetrievedDocument格式
documents = []
for r in results:
if r.score >= self.min_score:
doc = RetrievedDocument(
content=r.content,
doc_id=r.metadata.get("doc_id", ""),
doc_name=r.metadata.get("doc_name", ""),
section_title=r.metadata.get("section_title", ""),
clause_number=r.metadata.get("clause_number", ""),
page_number=r.metadata.get("page_number", 0),
score=r.score,
metadata=r.metadata
)
documents.append(doc)
logger.success(f"检索完成，返回{len(documents)}条结果（阈值过滤后）")
return documents
def retrieve_with_scores(
self,
query: str,
filters: Optional[str] = None
) -> List[Dict]:
"""
检索并返回完整结果（包含分数）
Args:
query: 查询文本
filters: 过滤条件
Returns:
List[Dict]: 包含分数的检索结果
"""
documents = self.retrieve(query, filters)
def retrieve(self, query: str, filters: Optional[str] = None, top_k: Optional[int] = None) -> list[RetrievedDocument]:
"""Handle retrieve for the Retriever instance."""
results = get_retrieval_service().retrieve(query=query, top_k=top_k or self.top_k, filters=filters)
return [
{
"content": doc.content,
"doc_id": doc.doc_id,
"doc_name": doc.doc_name,
"section_title": doc.section_title,
"clause_number": doc.clause_number,
"page_number": doc.page_number,
"score": doc.score
}
for doc in documents
RetrievedDocument(
content=item.content,
doc_id=item.doc_id,
doc_name=item.doc_name,
section_title=item.section_title,
clause_number=item.metadata.get("clause_number", ""),
page_number=item.page_number,
score=item.score,
metadata=item.metadata,
)
for item in results
if item.score >= self.min_score
]
def search_by_doc_name(
self,
query: str,
doc_name: str
) -> List[RetrievedDocument]:
"""按文档名称过滤检索"""
filters = f'doc_name=="{doc_name}"'
return self.retrieve(query, filters)
def retrieve_with_scores(self, query: str, filters: Optional[str] = None) -> list[dict]:
"""Handle retrieve with scores for the Retriever instance."""
return [
{
"content": item.content,
"doc_id": item.doc_id,
"doc_name": item.doc_name,
"section_title": item.section_title,
"clause_number": item.clause_number,
"page_number": item.page_number,
"score": item.score,
}
for item in self.retrieve(query, filters)
]
def search_by_regulation_type(
self,
query: str,
regulation_type: str
) -> List[RetrievedDocument]:
"""按法规类型过滤检索"""
filters = f'regulation_type=="{regulation_type}"'
return self.retrieve(query, filters)
def search_by_doc_name(self, query: str, doc_name: str) -> list[RetrievedDocument]:
"""Search by doc name for the Retriever instance."""
return self.retrieve(query, filters=f'doc_name == "{doc_name}"')
def search_by_regulation_type(self, query: str, regulation_type: str) -> list[RetrievedDocument]:
"""Search by regulation type for the Retriever instance."""
return self.retrieve(query, filters=f'regulation_type == "{regulation_type}"')
def close(self):
"""关闭连接"""
if self.milvus:
self.milvus.disconnect()
logger.info("检索器已关闭")
"""Release the resources held by this component."""
return None
def retrieve_regulations(
query: str,
top_k: int = 10,
filters: Optional[str] = None
) -> List[RetrievedDocument]:
"""便捷函数：检索法规"""
retriever = Retriever(top_k=top_k)
results = retriever.retrieve(query, filters)
retriever.close()
return results
def retrieve_regulations(query: str, top_k: int = 10, filters: Optional[str] = None) -> list[RetrievedDocument]:
"""Handle retrieve regulations."""
return Retriever(top_k=top_k).retrieve(query, filters)