first commit
This commit is contained in:
89
.env.example
Normal file
89
.env.example
Normal file
@@ -0,0 +1,89 @@
|
||||
# ══════════════════════════════════════════════════
|
||||
# AI合规智能中枢 — 环境变量配置
|
||||
# 复制本文件为 .env 并填写实际值
|
||||
# cp .env.example .env
|
||||
# ══════════════════════════════════════════════════
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# LLM 云端 API 配置（至少填写一个）
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
# LLM 提供商：deepseek 或 qwen
|
||||
LLM_PROVIDER=deepseek
|
||||
|
||||
# DeepSeek API（推荐，约¥1/百万tokens）
|
||||
# 申请地址：https://platform.deepseek.com
|
||||
DEEPSEEK_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
# 可选模型：deepseek-chat（通用）、deepseek-reasoner（推理增强）
|
||||
DEEPSEEK_MODEL=deepseek-chat
|
||||
|
||||
# 阿里云 DashScope / Qwen API（备用）
|
||||
# 申请地址：https://dashscope.aliyuncs.com
|
||||
DASHSCOPE_API_KEY=sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
||||
# 可选模型：qwen-plus、qwen-max、qwen-turbo
|
||||
QWEN_MODEL=qwen-plus
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# 数据库密码
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
# PostgreSQL 密码（生产环境请使用强密码）
|
||||
POSTGRES_PASSWORD=compliance_secure_2026
|
||||
|
||||
# Redis 密码
|
||||
REDIS_PASSWORD=redis_secure_2026
|
||||
|
||||
# Neo4j 密码（不能包含特殊字符）
|
||||
NEO4J_PASSWORD=neo4j_secure_2026
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# AI 模型配置
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
# HuggingFace 镜像（国内加速，默认使用 hf-mirror.com）
|
||||
HF_ENDPOINT=https://hf-mirror.com
|
||||
|
||||
# 嵌入服务设备：cpu 或 cuda（有 GPU 时改为 cuda）
|
||||
EMBEDDING_DEVICE=cpu
|
||||
|
||||
# MinerU 解析设备：cpu 或 cuda
|
||||
MCP_DEVICE=cpu
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# 应用配置
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
# 运行环境：development / production
|
||||
APP_ENV=development
|
||||
|
||||
# 日志级别：DEBUG / INFO / WARNING / ERROR
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
# API 认证密钥（用于内部服务间调用）
|
||||
API_SECRET_KEY=change_this_to_a_random_secret_key_32chars
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# 监控配置（可选）
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
# Grafana 管理员密码
|
||||
GRAFANA_PASSWORD=admin
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────
|
||||
# 外部推送配置（闭环③法规监控推送用）
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
# 邮件推送（可选）
|
||||
# SMTP_HOST=smtp.example.com
|
||||
# SMTP_PORT=587
|
||||
# SMTP_USER=your@email.com
|
||||
# SMTP_PASSWORD=your_smtp_password
|
||||
|
||||
# Webhook 推送（可选，支持飞书/钉钉/企业微信）
|
||||
# WEBHOOK_URL=https://hooks.slack.com/services/xxx
|
||||
277
00_整体部署规划.md
Normal file
277
00_整体部署规划.md
Normal file
@@ -0,0 +1,277 @@
|
||||
# AI合规智能中枢 — 整体部署规划
|
||||
|
||||
> **版本：** 调研版 v1.0 | **日期：** 2026.04 | **团队：** T-Systems AI Regulations Team
|
||||
|
||||
---
|
||||
|
||||
## 一、项目背景
|
||||
|
||||
AI+合规智能中枢面向车企与工厂，是一个全链路合规智能平台。主要解决以下痛点：
|
||||
|
||||
| 痛点 | 说明 |
|
||||
|------|------|
|
||||
| 法规来源复杂 | GB、MIIT、UN-ECE、IATF 16949、ISO 45001 等多源并行 |
|
||||
| 更新频率高 | 新能源、数据安全、碳排放法规频繁变动 |
|
||||
| 跨语言要求 | 中英德法多语言法规并存 |
|
||||
| 文档管理分散 | 内部文档与外部法规割裂，难以统一检索 |
|
||||
| 被动识别隐患 | EHS 合规靠人工排查，效率低下 |
|
||||
|
||||
**调研目标：** 以最小资源投入（Docker Compose 单机）验证三条核心业务闭环的技术可行性。
|
||||
|
||||
---
|
||||
|
||||
## 二、部署架构概览
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ 单台服务器 │
|
||||
│ ┌──────────────┐ ┌──────────────────────────────────────┐ │
|
||||
│ │ API 网关 │ │ Docker Compose │ │
|
||||
│ │ Nginx :80 │───▶│ │ │
|
||||
│ └──────────────┘ │ ┌──────────────────────────────┐ │ │
|
||||
│ │ │ 业务服务层 │ │ │
|
||||
│ │ │ compliance-backend :8000 │ │ │
|
||||
│ │ │ celery-worker │ │ │
|
||||
│ │ │ celery-beat │ │ │
|
||||
│ │ └──────────┬───────────────────┘ │ │
|
||||
│ │ │ │ │
|
||||
│ │ ┌──────────▼───────────────────┐ │ │
|
||||
│ │ │ AI 模型层 │ │ │
|
||||
│ │ │ embedding-service :8010 │ │ │
|
||||
│ │ │ mcp-server(MinerU) :8011 │ │ │
|
||||
│ │ │ LLM → DeepSeek API (云端) │ │ │
|
||||
│ │ └──────────┬───────────────────┘ │ │
|
||||
│ │ │ │ │
|
||||
│ │ ┌──────────▼───────────────────┐ │ │
|
||||
│ │ │ 数据层 │ │ │
|
||||
│ │ │ PostgreSQL :5432 │ │ │
|
||||
│ │ │ Redis :6379 │ │ │
|
||||
│ │ │ Milvus :19530 │ │ │
|
||||
│ │ │ Neo4j :7474/:7687 │ │ │
|
||||
│ │ │ MinIO (Milvus内置) │ │ │
|
||||
│ │ └──────────────────────────────┘ │ │
|
||||
│ └──────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
┌─────────▼──────────┐
|
||||
│ DeepSeek API │
|
||||
│ (云端 LLM) │
|
||||
└────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 三、原方案 vs 调研方案对比
|
||||
|
||||
| 维度 | 原方案（生产级）| 调研方案 | 降级理由 |
|
||||
|------|--------------|---------|---------|
|
||||
| 编排 | Kubernetes 1.36 + Helm | **Docker Compose** | 无需集群管理，`up -d` 一键启动 |
|
||||
| LLM | vLLM + DeepSeek-V3（4×A100）| **DeepSeek/Qwen 云端 API** | 无 GPU 依赖，秒级就绪 |
|
||||
| 嵌入模型 | BGE-M3 GPU 服务 | **BGE-M3 CPU 容器** | 调研数据量小，CPU 够用 |
|
||||
| Milvus | 分布式集群 + MinIO | **Milvus Standalone**（含内置 MinIO）| 单容器，省去 MinIO 独立部署 |
|
||||
| 消息队列 | Kafka 3 节点 | **Redis + Celery**（复用已有 Redis）| 调研无需高吞吐，大幅简化 |
|
||||
| 监控 | Prometheus + Grafana + ELK | **仅 Prometheus + Grafana**（可选）| 轻量，后期按需加 |
|
||||
| 安全 | JWT + cert-manager + RBAC | **API Key 简单认证** | 调研期无需生产级安全 |
|
||||
| CI/CD | GitLab CI 完整流水线 | **无**（手动部署）| 调研期直接 compose up |
|
||||
|
||||
---
|
||||
|
||||
## 四、硬件最低要求
|
||||
|
||||
| 资源 | 最低配置 | 推荐配置 | 说明 |
|
||||
|------|---------|---------|------|
|
||||
| CPU | 8 核 | 16 核+ | BGE-M3 CPU 模式需要较多核心 |
|
||||
| 内存 | 32 GB | 64 GB | Milvus + BGE-M3 + Neo4j 内存消耗较大 |
|
||||
| 存储 | 200 GB SSD | 500 GB SSD | 含模型文件（约 5GB）+ 数据 |
|
||||
| GPU | **无需** | 1× RTX 3090（24GB）| 有 GPU 可加速嵌入/MinerU |
|
||||
| 网络 | 能访问 DeepSeek API | — | LLM 完全在云端 |
|
||||
| OS | Ubuntu 22.04 LTS | — | 或 Windows 11 + WSL2 |
|
||||
|
||||
**各组件内存估算：**
|
||||
|
||||
| 服务 | 内存占用 |
|
||||
|------|---------|
|
||||
| PostgreSQL | ~1 GB |
|
||||
| Redis | ~512 MB |
|
||||
| Milvus（含 etcd/minio）| ~4 GB |
|
||||
| Neo4j | ~2 GB |
|
||||
| BGE-M3（CPU 模式）| ~6 GB |
|
||||
| MinerU（CPU 模式）| ~4 GB |
|
||||
| compliance-backend | ~1 GB |
|
||||
| celery-worker × 1 | ~1 GB |
|
||||
| **合计** | **~20 GB** |
|
||||
|
||||
---
|
||||
|
||||
## 五、五阶段部署步骤（总览）
|
||||
|
||||
```
|
||||
阶段一：宿主机环境准备
|
||||
└─ 安装 Docker CE / Docker Desktop
|
||||
└─ 配置 nvidia-container-toolkit（有 GPU 时）
|
||||
└─ 创建项目目录，配置 .env
|
||||
|
||||
阶段二：基础中间件启动
|
||||
└─ PostgreSQL + Redis（优先启动）
|
||||
└─ etcd + MinIO（Milvus 依赖）
|
||||
└─ Milvus Standalone（向量检索核心）
|
||||
└─ Neo4j Community（知识图谱）
|
||||
|
||||
阶段三：AI 模型服务构建与启动
|
||||
└─ 构建 embedding-service（BGE-M3 封装）
|
||||
└─ 构建 mcp-server（MinerU 封装）
|
||||
└─ 预下载模型（BGE-M3 ~2.5GB，MinerU ~2GB）
|
||||
|
||||
阶段四：业务微服务启动
|
||||
└─ compliance-backend（FastAPI 主服务）
|
||||
└─ celery-worker（异步任务处理）
|
||||
└─ celery-beat（定时任务调度）
|
||||
└─ nginx（API 网关）
|
||||
|
||||
阶段五：验证与闭环测试
|
||||
└─ 健康检查（bash scripts/check_health.sh）
|
||||
└─ 端到端冒烟测试（bash scripts/07_smoke_test.sh）
|
||||
└─ 三条业务闭环验证
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 六、三条核心业务闭环
|
||||
|
||||
### 闭环①：法规入库 → 检索问答
|
||||
|
||||
```
|
||||
用户上传 PDF
|
||||
│
|
||||
▼
|
||||
API Gateway（Nginx）
|
||||
│
|
||||
▼
|
||||
kbmp-service（文件接收）
|
||||
│ 异步投递
|
||||
▼
|
||||
Celery Worker
|
||||
│
|
||||
├─► parse-worker ──► mcp-server（MinerU 解析）
|
||||
│ │ Markdown + 结构化文本
|
||||
│ ▼
|
||||
└─► vectorize-worker ──► embedding-service（BGE-M3）
|
||||
│ 1024维向量
|
||||
▼
|
||||
Milvus（向量存储）+ PostgreSQL（元数据）
|
||||
|
||||
用户提问
|
||||
│
|
||||
▼
|
||||
BM25 关键词检索 + BGE-M3 向量检索（Milvus hybrid search）
|
||||
│
|
||||
▼
|
||||
Cross-Encoder Reranker（精排 Top-K）
|
||||
│
|
||||
▼
|
||||
DeepSeek API（引文锚定生成）
|
||||
│
|
||||
▼
|
||||
返回答案（含原文引用 + 页码）
|
||||
```
|
||||
|
||||
### 闭环②：文档上传 → 合规审查
|
||||
|
||||
```
|
||||
上传供应商/内部文档
|
||||
│
|
||||
▼
|
||||
MinerU 解析 → 条款级分割
|
||||
│
|
||||
▼
|
||||
法规域匹配（vehicle_safety / data_security / ehs）
|
||||
│
|
||||
▼
|
||||
与法规库语义比对（向量相似度 + 关键字匹配）
|
||||
│
|
||||
▼
|
||||
DeepSeek API 风险评分（条款级分析）
|
||||
│
|
||||
▼
|
||||
生成 Markdown 审查报告（风险等级 + 整改建议）
|
||||
```
|
||||
|
||||
### 闭环③：法规监控 → 变更推送
|
||||
|
||||
```
|
||||
Celery Beat 定时触发（每天）
|
||||
│
|
||||
▼
|
||||
抓取监控源（国标委 / 工信部 / 应急管理部 / 生环部）
|
||||
│
|
||||
▼
|
||||
内容 Hash 比对（检测变更）
|
||||
│
|
||||
▼ [有变更]
|
||||
NLP Diff 分析（DeepSeek 提取新增/修订/废止条款）
|
||||
│
|
||||
▼
|
||||
增量入库（MinerU 解析 → BGE-M3 → Milvus + PostgreSQL + Neo4j）
|
||||
│
|
||||
▼
|
||||
差距分析（与企业现状比对）
|
||||
│
|
||||
▼
|
||||
推送通知（Email / Webhook / 飞书 / 钉钉）
|
||||
│
|
||||
▼
|
||||
记录变更日志 → 触发整改任务
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 七、技术选型决策依据
|
||||
|
||||
| 组件 | 选型 | 决策依据 |
|
||||
|------|------|---------|
|
||||
| 向量数据库 | Milvus 2.4 | 支持 Dense+Sparse 混合检索，BGE-M3 配套，生产可扩展 |
|
||||
| 图数据库 | Neo4j 5.x | 法规实体关系建模成熟，APOC 插件丰富，Cypher 查询友好 |
|
||||
| 嵌入模型 | BGE-M3 | 中英文双语，支持 dense+sparse+multi-vector，8192 token 上下文 |
|
||||
| LLM | DeepSeek API | 推理能力强，成本低（约¥1/百万 tokens），OpenAI 兼容 |
|
||||
| 文档解析 | MinerU | GPU 最快 0.21s/页，支持 109 种语言 OCR，布局感知 |
|
||||
| 任务队列 | Celery + Redis | 调研阶段够用，比 Kafka 轻量，Redis 可复用 |
|
||||
| API 框架 | FastAPI | 异步性能好，OpenAPI 自动生成，Pydantic 数据验证 |
|
||||
| 关系数据库 | PostgreSQL + pgvector | 元数据存储 + 备用向量检索，pgvector 镜像开箱即用 |
|
||||
|
||||
---
|
||||
|
||||
## 八、升级路径（调研 → 生产）
|
||||
|
||||
| 维度 | 升级内容 | 触发条件 |
|
||||
|------|---------|---------|
|
||||
| LLM | API → 本地 vLLM + DeepSeek-V3 | 数据安全要求/API成本超阈值 |
|
||||
| Milvus | Standalone → 分布式集群 | 向量数据 > 1000 万条 |
|
||||
| 消息队列 | Celery+Redis → Kafka | 并发任务 > 100/分钟 |
|
||||
| 编排 | Docker Compose → Kubernetes | 多节点部署/弹性伸缩需求 |
|
||||
| 安全 | API Key → JWT + RBAC | 对外提供服务/多租户 |
|
||||
| 监控 | Grafana → Grafana + ELK | 日志量大/需要复杂分析 |
|
||||
|
||||
---
|
||||
|
||||
## 九、文件结构说明
|
||||
|
||||
```
|
||||
Depolyment/
|
||||
├── 00_整体部署规划.md ← 本文档
|
||||
├── 01_技术架构详解.md ← 六层架构 + 六大微服务详细说明
|
||||
├── 02_组件安装指南.md ← 每个组件的详细安装步骤
|
||||
├── 03_业务闭环说明.md ← 三条闭环的数据流和接口规范
|
||||
├── README.md ← 快速启动指南
|
||||
├── docker-compose.yml ← 全服务编排
|
||||
├── .env.example ← 环境变量模板
|
||||
├── scripts/ ← 安装与运维脚本（13 个）
|
||||
├── services/ ← 服务源码
|
||||
│ ├── embedding/ ← BGE-M3 嵌入服务
|
||||
│ ├── mcp-server/ ← MinerU 文档解析服务
|
||||
│ └── compliance-backend/ ← 核心业务后端
|
||||
├── config/ ← Nginx、Prometheus 配置
|
||||
├── init-sql/ ← PostgreSQL 初始化 SQL
|
||||
├── data/ ← 运行时数据
|
||||
├── logs/ ← 服务日志
|
||||
└── models/ ← AI 模型缓存
|
||||
```
|
||||
263
01_技术架构详解.md
Normal file
263
01_技术架构详解.md
Normal file
@@ -0,0 +1,263 @@
|
||||
# AI合规智能中枢 — 技术架构详解
|
||||
|
||||
> 本文档对应架构文档：`01_分层次技术架构图.html` 和 `02_详细技术架构图.html`
|
||||
|
||||
---
|
||||
|
||||
## 一、六层架构总览
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────────────┐
|
||||
│ L1 应用接入层：Web / Mobile / Bot / API Gateway / RBAC │
|
||||
├──────────────────────────────────────────────────────────────────┤
|
||||
│ L2 业务能力层：知识库问答 / 文档审查 / EHS / 法规监控 / 推荐 │
|
||||
├──────────────────────────────────────────────────────────────────┤
|
||||
│ L3 法规感知层：监控 → 感知 → 解析 → 图谱 → 分析 → 闭环 │
|
||||
├──────────────────────────────────────────────────────────────────┤
|
||||
│ L4 AI引擎层：RAG / LLM / 文档解析 / 知识图谱推理 / NLP │
|
||||
├──────────────────────────────────────────────────────────────────┤
|
||||
│ L5 数据知识层：Milvus / PostgreSQL / Neo4j / Redis / 知识库 │
|
||||
├──────────────────────────────────────────────────────────────────┤
|
||||
│ L6 基础设施层：安全治理 / 容器编排 / 运维观测 / CI/CD │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 二、六大微服务详解
|
||||
|
||||
### 2.1 kbmp-service（知识库公开接口）
|
||||
|
||||
**职责：** 知识库的统一入口，处理文件上传、检索编排、任务投递。
|
||||
|
||||
**核心接口：**
|
||||
|
||||
| 方法 | 路径 | 功能 |
|
||||
|------|------|------|
|
||||
| POST | `/workspace/create` | 创建知识库工作空间 |
|
||||
| POST | `/files/upload` | 上传文件（触发解析任务） |
|
||||
| POST | `/files/parse` | 手动触发解析 |
|
||||
| POST | `/knowledge/retrieval` | 混合检索（BM25 + 向量）|
|
||||
| POST | `/chunks/recall` | 原始 Chunk 召回 |
|
||||
| POST | `/qa` | 检索 + LLM 问答生成 |
|
||||
|
||||
**内部流程：**
|
||||
```
|
||||
文件上传 → 存储 data/uploads → 投递 Celery 任务（parse-queue）
|
||||
→ parse-worker 调用 mcp-server 解析
|
||||
→ vectorize-worker 调用 embedding-service 向量化
|
||||
→ 写入 Milvus（向量）+ PostgreSQL（元数据）
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2.2 mcp-server（文档解析服务）
|
||||
|
||||
**职责：** 将 PDF/Word/Excel 等文档转换为结构化 Markdown，供后续向量化。
|
||||
|
||||
**核心接口：**
|
||||
|
||||
| 方法 | 路径 | 功能 |
|
||||
|------|------|------|
|
||||
| POST | `/parse-document` | 通用解析（自动选择引擎）|
|
||||
| POST | `/mineru-parse` | MinerU 精准解析 |
|
||||
| GET | `/health` | 健康检查 |
|
||||
|
||||
**解析策略（降级链）：**
|
||||
```
|
||||
1. 阿里云文档解析 API（云端高精度）→ [调研版暂不启用]
|
||||
2. MinerU（本地，GPU/CPU 均支持）→ 主用
|
||||
3. python-docx / PyMuPDF（纯文本降级）→ 兜底
|
||||
```
|
||||
|
||||
**MinerU 特性：**
|
||||
- GPU 最快：0.21 秒/页
|
||||
- CPU 模式：约 3-5 秒/页（调研阶段可接受）
|
||||
- 支持 109 种语言 OCR
|
||||
- 布局感知：区分正文/标题/表格/图片/页眉页脚
|
||||
- 输出格式：Markdown + JSON（含结构化元数据）
|
||||
|
||||
---
|
||||
|
||||
### 2.3 合规业务后端（compliance-backend）
|
||||
|
||||
**职责：** 核心业务逻辑，整合三条闭环的业务处理。
|
||||
|
||||
**核心接口：**
|
||||
|
||||
| 方法 | 路径 | 功能 |
|
||||
|------|------|------|
|
||||
| POST | `/compliance/upload` | 上传待审查文档 |
|
||||
| POST | `/compliance/check` | 智能合规审查 |
|
||||
| GET | `/compliance/report/{id}` | 获取审查报告 |
|
||||
| POST | `/compliance/regulations/download` | 下载法规 |
|
||||
| POST | `/compliance/regulations/update` | 更新法规版本 |
|
||||
| POST | `/compliance/access-control` | 权限分级管理 |
|
||||
| POST | `/compliance/subscribe` | 订阅变更推送 |
|
||||
|
||||
---
|
||||
|
||||
### 2.4 法规感知引擎（Regulation Awareness Engine）
|
||||
|
||||
**职责：** 定时监控法规源，自动检测变更，触发增量更新。
|
||||
|
||||
**六步感知闭环：**
|
||||
|
||||
```
|
||||
① 法规源监控
|
||||
- 定时抓取：国家标准委、工信部、UN-ECE、EUR-Lex、碳交易平台
|
||||
- 技术：requests + BeautifulSoup + Playwright（动态页面）
|
||||
|
||||
② 智能变更感知
|
||||
- Hash 对比（快速过滤）
|
||||
- NLP 版本 Diff（精确识别新增/修订/废止条款）
|
||||
|
||||
③ 自动解析入库
|
||||
- MinerU 解析 → 条款级分割
|
||||
- BGE-M3 向量化 → Milvus + PostgreSQL
|
||||
|
||||
④ 知识图谱同步
|
||||
- Neo4j 更新：法规-条款-义务关系
|
||||
- 影响分析：哪些企业文档受影响
|
||||
|
||||
⑤ 差距分析
|
||||
- AI 比对企业现状 vs 新法规要求
|
||||
- 生成差距报告
|
||||
|
||||
⑥ 推送与整改触发
|
||||
- 按角色推送（研发/EHS/采购/法务）
|
||||
- 自动生成整改任务
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2.5 AI 推理引擎（AI Inference Engine）
|
||||
|
||||
**职责：** 混合检索、精排、LLM 生成、知识图谱推理。
|
||||
|
||||
**混合检索流程：**
|
||||
|
||||
```
|
||||
用户查询
|
||||
│
|
||||
├─► BGE-M3 向量化（Dense 1024维）
|
||||
│ │
|
||||
│ └─► Milvus ANN 检索（HNSW，Cosine 相似度）
|
||||
│
|
||||
├─► BM25 关键词检索（稀疏向量/倒排索引）
|
||||
│
|
||||
└─► 结果融合（RRF 排名融合）
|
||||
│
|
||||
▼
|
||||
Cross-Encoder Reranker（精排 Top-K）
|
||||
│
|
||||
▼
|
||||
LLM 生成（DeepSeek API）
|
||||
- System Prompt：引文锚定要求
|
||||
- 输出：答案 + 原文引用 + 来源文档 + 页码
|
||||
```
|
||||
|
||||
**BGE-M3 三种向量输出：**
|
||||
- **Dense Vector**（1024维）：语义相似度，主要用于向量检索
|
||||
- **Sparse Vector**（词汇权重）：关键字匹配，等效 BM25
|
||||
- **Multi-Vector**（ColBERT 风格）：精细粒度 token 级匹配
|
||||
|
||||
---
|
||||
|
||||
### 2.6 Worker 集群
|
||||
|
||||
**职责：** 异步任务处理，解耦主服务压力。
|
||||
|
||||
**Worker 类型：**
|
||||
|
||||
| Worker | 队列 | 职责 |
|
||||
|--------|------|------|
|
||||
| parse-worker | `parse` | 调用 mcp-server 解析文档 |
|
||||
| vectorize-worker | `vectorize` | BGE-M3 向量化 + Milvus 写入 |
|
||||
| compliance-worker | `compliance` | 合规比对 + 风险评分 |
|
||||
| monitor-worker | `monitor` | 法规源定时抓取 |
|
||||
| push-worker | `push` | 推送通知（Email/Webhook）|
|
||||
|
||||
**调度配置（Celery Beat）：**
|
||||
```python
|
||||
CELERY_BEAT_SCHEDULE = {
|
||||
"regulation-monitor": {
|
||||
"task": "app.worker.fetch_regulation_updates",
|
||||
"schedule": crontab(hour=2, minute=0), # 每天凌晨2点
|
||||
},
|
||||
"push-notifications": {
|
||||
"task": "app.worker.send_pending_notifications",
|
||||
"schedule": crontab(minute="*/30"), # 每30分钟
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 三、数据模型
|
||||
|
||||
### 3.1 PostgreSQL 表结构
|
||||
|
||||
```
|
||||
workspaces → 知识库工作空间
|
||||
files → 上传文件记录（含解析状态）
|
||||
tasks → 异步任务状态追踪
|
||||
compliance_reports → 合规审查报告
|
||||
regulation_sources → 法规监控源配置
|
||||
regulation_updates → 法规变更记录
|
||||
subscriptions → 推送订阅配置
|
||||
audit_logs → 全链路审计日志
|
||||
```
|
||||
|
||||
### 3.2 Milvus Collection 结构
|
||||
|
||||
```python
|
||||
# regulation_chunks / doc_chunks / case_library 共用相同 Schema
|
||||
fields = [
|
||||
FieldSchema("id", VARCHAR, primary_key=True),
|
||||
FieldSchema("file_id", VARCHAR), # 关联文件
|
||||
FieldSchema("workspace_id", VARCHAR), # 所属工作空间
|
||||
FieldSchema("chunk_idx", INT64), # 块序号
|
||||
FieldSchema("content", VARCHAR(65535)), # 原文内容
|
||||
FieldSchema("dense_vec", FLOAT_VECTOR(1024)), # BGE-M3 向量
|
||||
FieldSchema("metadata", JSON), # 扩展元数据
|
||||
]
|
||||
# 索引：HNSW，Cosine 相似度，M=16，efConstruction=200
|
||||
```
|
||||
|
||||
### 3.3 Neo4j 图模型
|
||||
|
||||
```cypher
|
||||
// 节点类型
|
||||
(:Regulation {id, title, code, version, domain, effective_date})
|
||||
(:Clause {id, number, content, clause_type})
|
||||
(:Obligation {id, description, obligation_type, subject})
|
||||
(:Enterprise {id, name, industry})
|
||||
(:RiskItem {id, description, severity, domain})
|
||||
(:Domain {name, label})
|
||||
|
||||
// 关系类型
|
||||
(Regulation)-[:CONTAINS]->(Clause)
|
||||
(Clause)-[:REQUIRES]->(Obligation)
|
||||
(Regulation)-[:SUPERSEDES]->(Regulation) // 版本替代
|
||||
(Clause)-[:MAPS_TO]->(RiskItem)
|
||||
(Enterprise)-[:SUBJECT_TO]->(Regulation)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 四、核心技术栈版本锁定
|
||||
|
||||
| 组件 | 版本 | Docker 镜像 |
|
||||
|------|------|------------|
|
||||
| PostgreSQL | 16 + pgvector | `pgvector/pgvector:pg16` |
|
||||
| Redis | 7.x | `redis:7-alpine` |
|
||||
| Milvus | 2.4.13 | `milvusdb/milvus:v2.4.13` |
|
||||
| Neo4j | 5.20 Community | `neo4j:5.20-community` |
|
||||
| BGE-M3 | 最新 | BAAI/bge-m3（HuggingFace）|
|
||||
| MinerU | 1.x | opendatalab/MinerU（pip）|
|
||||
| LangChain | 0.3+ | pip install langchain>=0.3 |
|
||||
| FastAPI | 0.115+ | pip install fastapi>=0.115 |
|
||||
| Celery | 5.4+ | pip install celery[redis]>=5.4 |
|
||||
| Python | 3.12 | python:3.12-slim（Docker）|
|
||||
| Nginx | 1.25 | `nginx:1.25-alpine` |
|
||||
569
02_组件安装指南.md
Normal file
569
02_组件安装指南.md
Normal file
@@ -0,0 +1,569 @@
|
||||
# AI合规智能中枢 — 组件安装指南
|
||||
|
||||
> 本文档提供每个组件的详细安装步骤、配置说明和验证方法。
|
||||
|
||||
---
|
||||
|
||||
## 前置：Docker 环境安装
|
||||
|
||||
### Ubuntu 22.04 LTS
|
||||
|
||||
```bash
|
||||
# 1. 更新包列表
|
||||
sudo apt-get update
|
||||
|
||||
# 2. 安装依赖
|
||||
sudo apt-get install -y ca-certificates curl gnupg lsb-release
|
||||
|
||||
# 3. 添加 Docker GPG 密钥
|
||||
sudo install -m 0755 -d /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \
|
||||
sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
|
||||
sudo chmod a+r /etc/apt/keyrings/docker.gpg
|
||||
|
||||
# 4. 添加 Docker 仓库
|
||||
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \
|
||||
https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | \
|
||||
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
|
||||
|
||||
# 5. 安装 Docker CE
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y docker-ce docker-ce-cli containerd.io \
|
||||
docker-buildx-plugin docker-compose-plugin
|
||||
|
||||
# 6. 加入 docker 组（免 sudo）
|
||||
sudo usermod -aG docker $USER
|
||||
newgrp docker
|
||||
|
||||
# 7. 验证
|
||||
docker --version # Docker version 27.x.x
|
||||
docker compose version # Docker Compose version v2.x.x
|
||||
```
|
||||
|
||||
### Windows 11 + WSL2
|
||||
|
||||
```powershell
|
||||
# PowerShell（管理员）
|
||||
|
||||
# 1. 启用 WSL2
|
||||
wsl --install -d Ubuntu-22.04
|
||||
wsl --set-default-version 2
|
||||
|
||||
# 2. 安装 Docker Desktop（需重启）
|
||||
winget install -e --id Docker.DockerDesktop
|
||||
|
||||
# 3. 重启后，Docker Desktop 设置：
|
||||
# Settings → General → "Use WSL 2 based engine" ✓
|
||||
# Settings → Resources → WSL Integration → Ubuntu-22.04 ✓
|
||||
```
|
||||
|
||||
### GPU 支持（可选，有 NVIDIA GPU 时）
|
||||
|
||||
```bash
|
||||
# Ubuntu 安装 nvidia-container-toolkit
|
||||
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \
|
||||
sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
|
||||
|
||||
curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
|
||||
sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
|
||||
sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
|
||||
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y nvidia-container-toolkit
|
||||
sudo nvidia-ctk runtime configure --runtime=docker
|
||||
sudo systemctl restart docker
|
||||
|
||||
# 验证
|
||||
docker run --rm --gpus all nvidia/cuda:12.4-base nvidia-smi
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 组件一：PostgreSQL 16 + pgvector
|
||||
|
||||
**用途：** 存储元数据（文件记录、任务状态、合规报告、法规变更）
|
||||
|
||||
### 配置参数
|
||||
|
||||
```yaml
|
||||
# docker-compose.yml 中的关键配置
|
||||
image: pgvector/pgvector:pg16 # 内置 pgvector 扩展
|
||||
POSTGRES_USER: compliance
|
||||
POSTGRES_PASSWORD: <your_password>
|
||||
POSTGRES_DB: compliance_db
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data # 数据持久化
|
||||
- ./init-sql:/docker-entrypoint-initdb.d # 自动执行初始化 SQL
|
||||
ports:
|
||||
- "5432:5432"
|
||||
```
|
||||
|
||||
### 启动与验证
|
||||
|
||||
```bash
|
||||
# 启动
|
||||
docker compose up -d postgres
|
||||
|
||||
# 等待健康（约10秒）
|
||||
docker compose ps postgres
|
||||
|
||||
# 连接测试
|
||||
docker compose exec postgres psql -U compliance -d compliance_db -c "\dt"
|
||||
|
||||
# 验证扩展
|
||||
docker compose exec postgres psql -U compliance -d compliance_db \
|
||||
-c "SELECT extname FROM pg_extension WHERE extname IN ('vector', 'uuid-ossp');"
|
||||
```
|
||||
|
||||
### 常用操作
|
||||
|
||||
```bash
|
||||
# 查看所有表
|
||||
docker compose exec postgres psql -U compliance -d compliance_db \
|
||||
-c "\dt"
|
||||
|
||||
# 查询任务状态
|
||||
docker compose exec postgres psql -U compliance -d compliance_db \
|
||||
-c "SELECT id, task_type, status, created_at FROM tasks ORDER BY created_at DESC LIMIT 10;"
|
||||
|
||||
# 备份数据库
|
||||
docker compose exec postgres pg_dump -U compliance compliance_db > backup_$(date +%Y%m%d).sql
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 组件二：Redis 7
|
||||
|
||||
**用途：** Celery 消息中间件、热数据缓存、分布式锁、会话存储
|
||||
|
||||
### 配置参数
|
||||
|
||||
```yaml
|
||||
image: redis:7-alpine
|
||||
command: >
|
||||
redis-server
|
||||
--requirepass <your_password>
|
||||
--maxmemory 2gb
|
||||
--maxmemory-policy allkeys-lru # 内存满时淘汰最近最少使用的 key
|
||||
ports:
|
||||
- "6379:6379"
|
||||
```
|
||||
|
||||
### 启动与验证
|
||||
|
||||
```bash
|
||||
# 启动
|
||||
docker compose up -d redis
|
||||
|
||||
# 连接测试
|
||||
docker compose exec redis redis-cli -a <password> ping
|
||||
# 应返回：PONG
|
||||
|
||||
# 查看 Celery 队列长度
|
||||
docker compose exec redis redis-cli -a <password> llen celery
|
||||
|
||||
# 查看内存使用
|
||||
docker compose exec redis redis-cli -a <password> info memory | grep used_memory_human
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 组件三：Milvus 2.4 Standalone
|
||||
|
||||
**用途：** 向量数据库，存储 BGE-M3 嵌入向量，支持混合检索
|
||||
|
||||
### 架构说明
|
||||
|
||||
Milvus Standalone 包含三个内部组件：
|
||||
- **etcd**：元数据存储（Collection 定义、索引配置）
|
||||
- **MinIO**：向量段文件存储
|
||||
- **milvus**：查询/写入引擎
|
||||
|
||||
### 启动顺序（严格按顺序）
|
||||
|
||||
```bash
|
||||
# 1. 先启动 etcd
|
||||
docker compose up -d etcd
|
||||
sleep 10
|
||||
|
||||
# 2. 再启动 MinIO
|
||||
docker compose up -d minio
|
||||
sleep 10
|
||||
|
||||
# 3. 最后启动 Milvus（依赖前两者）
|
||||
docker compose up -d milvus
|
||||
# Milvus 冷启动约需 60 秒，请耐心等待
|
||||
```
|
||||
|
||||
### 验证
|
||||
|
||||
```bash
|
||||
# HTTP 健康检查
|
||||
curl http://localhost:9091/healthz
|
||||
# 应返回：{"status":"ok"}
|
||||
|
||||
# Python 连接测试
|
||||
python3 -c "
|
||||
from pymilvus import connections, utility
|
||||
connections.connect(host='localhost', port='19530')
|
||||
print('Collections:', utility.list_collections())
|
||||
print('Milvus 连接成功')
|
||||
"
|
||||
```
|
||||
|
||||
### 创建 Collection（向量索引）
|
||||
|
||||
```python
|
||||
from pymilvus import (connections, Collection, CollectionSchema,
|
||||
FieldSchema, DataType, utility)
|
||||
|
||||
connections.connect(host='localhost', port='19530')
|
||||
|
||||
fields = [
|
||||
FieldSchema('id', DataType.VARCHAR, is_primary=True, max_length=128),
|
||||
FieldSchema('content', DataType.VARCHAR, max_length=65535),
|
||||
FieldSchema('dense_vec', DataType.FLOAT_VECTOR, dim=1024), # BGE-M3
|
||||
FieldSchema('metadata', DataType.JSON),
|
||||
]
|
||||
|
||||
schema = CollectionSchema(fields, description='法规条款向量库')
|
||||
col = Collection('regulation_chunks', schema)
|
||||
|
||||
# 创建 HNSW 索引（速度/精度平衡）
|
||||
col.create_index('dense_vec', {
|
||||
'metric_type': 'COSINE',
|
||||
'index_type': 'HNSW',
|
||||
'params': {'M': 16, 'efConstruction': 200}
|
||||
})
|
||||
col.load()
|
||||
print('Collection 创建完成')
|
||||
```
|
||||
|
||||
### 常用查询
|
||||
|
||||
```python
|
||||
# 向量相似度检索
|
||||
results = col.search(
|
||||
data=[query_vector], # 查询向量（1024维）
|
||||
anns_field='dense_vec',
|
||||
param={'metric_type': 'COSINE', 'params': {'ef': 100}},
|
||||
limit=10,
|
||||
output_fields=['content', 'metadata']
|
||||
)
|
||||
|
||||
# 查看 Collection 统计
|
||||
print(col.num_entities) # 向量总数
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 组件四：Neo4j 5 Community
|
||||
|
||||
**用途：** 知识图谱存储，法规-条款-义务实体关系
|
||||
|
||||
### 配置参数
|
||||
|
||||
```yaml
|
||||
image: neo4j:5.20-community
|
||||
environment:
|
||||
NEO4J_AUTH: neo4j/<your_password>
|
||||
NEO4J_PLUGINS: '["apoc"]' # 必须安装 APOC 插件
|
||||
NEO4J_dbms_memory_heap_max__size: 2G
|
||||
ports:
|
||||
- "7474:7474" # Browser UI
|
||||
- "7687:7687" # Bolt 协议（应用连接用）
|
||||
```
|
||||
|
||||
### 启动与验证
|
||||
|
||||
```bash
|
||||
# 启动
|
||||
docker compose up -d neo4j
|
||||
# 首次启动约需 60 秒（下载 APOC 插件）
|
||||
|
||||
# 浏览器访问：http://localhost:7474
|
||||
# 用户名：neo4j，密码：见 .env 中 NEO4J_PASSWORD
|
||||
|
||||
# 命令行连接
|
||||
docker compose exec neo4j cypher-shell -u neo4j -p <password>
|
||||
```
|
||||
|
||||
### 常用 Cypher 查询
|
||||
|
||||
```cypher
|
||||
// 查看所有节点类型
|
||||
CALL apoc.meta.schema() YIELD value RETURN value;
|
||||
|
||||
// 创建法规节点
|
||||
CREATE (r:Regulation {
|
||||
id: 'GB18384-2020',
|
||||
title: 'GB 18384-2020 电动汽车安全要求',
|
||||
domain: 'vehicle_safety',
|
||||
effective_date: date('2021-01-01'),
|
||||
version: '2020'
|
||||
});
|
||||
|
||||
// 法规-条款关系
|
||||
MATCH (r:Regulation {id: 'GB18384-2020'})
|
||||
CREATE (c:Clause {
|
||||
id: 'GB18384-2020-2.1',
|
||||
number: '2.1',
|
||||
content: '绝缘电阻要求：直流电路绝缘电阻不得低于100Ω/V'
|
||||
})
|
||||
CREATE (r)-[:CONTAINS]->(c);
|
||||
|
||||
// 多跳查询：查找某法规所有义务
|
||||
MATCH (r:Regulation {domain: 'vehicle_safety'})-[:CONTAINS]->(c)-[:REQUIRES]->(o)
|
||||
RETURN r.title, c.number, o.description LIMIT 20;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 组件五：BGE-M3 嵌入服务
|
||||
|
||||
**用途：** 将文本转换为 1024 维向量，支持中英双语，支持 Dense+Sparse 混合检索
|
||||
|
||||
### 服务构建
|
||||
|
||||
```bash
|
||||
# 构建镜像
|
||||
docker compose build embedding-service
|
||||
|
||||
# 首次启动（会自动下载 BGE-M3 模型约 2.5GB）
|
||||
docker compose up -d embedding-service
|
||||
|
||||
# 查看下载进度
|
||||
docker compose logs -f embedding-service
|
||||
```
|
||||
|
||||
### 模型预下载（推荐，避免启动超时）
|
||||
|
||||
```bash
|
||||
# 方法1：通过 hf-mirror.com 加速
|
||||
bash scripts/download_models.sh
|
||||
|
||||
# 方法2：通过 ModelScope（国内最快）
|
||||
pip install modelscope
|
||||
python3 -c "
|
||||
from modelscope import snapshot_download
|
||||
snapshot_download('AI-ModelScope/bge-m3', cache_dir='./models/modelscope')
|
||||
"
|
||||
```
|
||||
|
||||
### API 使用
|
||||
|
||||
```bash
|
||||
# 健康检查
|
||||
curl http://localhost:8010/health
|
||||
|
||||
# 生成嵌入向量
|
||||
curl -X POST http://localhost:8010/embed \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"texts": ["GB 18384 电动汽车碰撞安全", "vehicle crash safety requirements"],
|
||||
"batch_size": 2
|
||||
}'
|
||||
# 返回：{"dense": [[...1024个浮点数...], [...]], "sparse": [{...词汇权重...}, {...}]}
|
||||
```
|
||||
|
||||
### 性能参考
|
||||
|
||||
| 模式 | 硬件 | 速度 |
|
||||
|------|------|------|
|
||||
| CPU | 16核，64GB RAM | 约 2-5 秒/批（batch=16）|
|
||||
| GPU | RTX 3090 24GB | 约 0.2-0.5 秒/批（batch=32）|
|
||||
|
||||
---
|
||||
|
||||
## 组件六：MinerU 文档解析服务
|
||||
|
||||
**用途：** 将 PDF/Word/Excel 解析为 Markdown + 结构化 JSON
|
||||
|
||||
### 服务构建
|
||||
|
||||
```bash
|
||||
# 构建镜像（首次约需 10-20 分钟，下载大量依赖）
|
||||
docker compose build mcp-server
|
||||
|
||||
# 启动服务（首次会下载 MinerU 模型约 2GB）
|
||||
docker compose up -d mcp-server
|
||||
|
||||
# 查看启动日志
|
||||
docker compose logs -f mcp-server
|
||||
```
|
||||
|
||||
### API 使用
|
||||
|
||||
```bash
|
||||
# 解析 PDF
|
||||
curl -X POST http://localhost:8011/mineru-parse \
|
||||
-F "file=@/path/to/regulation.pdf"
|
||||
# 返回：{"markdown": "# 法规标题\n\n## 第一章...", "filename": "regulation.pdf"}
|
||||
|
||||
# 解析 Word 文档
|
||||
curl -X POST http://localhost:8011/parse-document \
|
||||
-F "file=@/path/to/document.docx"
|
||||
```
|
||||
|
||||
### 性能参考
|
||||
|
||||
| 模式 | 速度 | 说明 |
|
||||
|------|------|------|
|
||||
| CPU | 3-5 秒/页 | 调研阶段可接受 |
|
||||
| GPU（RTX 3090）| 0.21 秒/页 | 生产推荐 |
|
||||
|
||||
---
|
||||
|
||||
## 组件七：业务后端（compliance-backend）
|
||||
|
||||
**用途：** FastAPI 主服务，整合所有业务逻辑
|
||||
|
||||
### 关键依赖配置
|
||||
|
||||
```bash
|
||||
# .env 中必须设置
|
||||
DEEPSEEK_API_KEY=sk-xxxx # DeepSeek API Key
|
||||
LLM_PROVIDER=deepseek # 或 qwen
|
||||
DATABASE_URL=postgresql+asyncpg://...
|
||||
REDIS_URL=redis://:password@redis:6379/0
|
||||
MILVUS_HOST=milvus
|
||||
NEO4J_URI=bolt://neo4j:7687
|
||||
EMBEDDING_SERVICE_URL=http://embedding-service:8010
|
||||
MCP_SERVER_URL=http://mcp-server:8011
|
||||
```
|
||||
|
||||
### 启动与验证
|
||||
|
||||
```bash
|
||||
# 启动服务
|
||||
docker compose up -d compliance-backend celery-worker celery-beat
|
||||
|
||||
# 验证 API 文档
|
||||
open http://localhost:8000/docs
|
||||
|
||||
# 查看健康状态（包含所有依赖）
|
||||
curl http://localhost:8000/health
|
||||
```
|
||||
|
||||
### Celery Worker 监控
|
||||
|
||||
```bash
|
||||
# 查看 Worker 状态
|
||||
docker compose exec celery-worker celery -A app.worker inspect active
|
||||
|
||||
# 查看队列积压
|
||||
docker compose exec redis redis-cli -a <password> llen celery
|
||||
|
||||
# Worker 日志
|
||||
docker compose logs -f celery-worker
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 组件八：Nginx API 网关
|
||||
|
||||
**用途：** 反向代理，统一路由，TLS 终止（生产）
|
||||
|
||||
### 配置说明（config/nginx.conf）
|
||||
|
||||
```nginx
|
||||
upstream compliance_backend {
|
||||
server compliance-backend:8000;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
client_max_body_size 100M; # 支持大 PDF 上传
|
||||
proxy_read_timeout 300s; # LLM 推理超时设置
|
||||
|
||||
location /api/kb/ { proxy_pass http://compliance_backend; }
|
||||
location /api/compliance/ { proxy_pass http://compliance_backend; }
|
||||
location /api/regulation/ { proxy_pass http://compliance_backend; }
|
||||
location /health { proxy_pass http://compliance_backend; }
|
||||
location /docs { proxy_pass http://compliance_backend; }
|
||||
}
|
||||
```
|
||||
|
||||
### 启动与验证
|
||||
|
||||
```bash
|
||||
# 启动
|
||||
docker compose up -d nginx
|
||||
|
||||
# 测试路由
|
||||
curl http://localhost/health
|
||||
curl http://localhost/docs # 应返回 Swagger UI HTML
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 完整启动顺序
|
||||
|
||||
```bash
|
||||
# 方式1：分步启动（推荐，含健康等待）
|
||||
bash scripts/06_start_all.sh
|
||||
|
||||
# 方式2：手动分步
|
||||
docker compose up -d postgres redis # 等30s
|
||||
docker compose up -d etcd minio # 等30s
|
||||
docker compose up -d milvus # 等60s
|
||||
docker compose up -d neo4j # 等60s
|
||||
docker compose build embedding-service mcp-server compliance-backend
|
||||
docker compose up -d embedding-service mcp-server # 等120s（模型加载）
|
||||
bash scripts/05_init_db.sh # 初始化数据库
|
||||
docker compose up -d compliance-backend celery-worker celery-beat nginx
|
||||
|
||||
# 验证
|
||||
bash scripts/check_health.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 常见问题
|
||||
|
||||
### Q: Milvus 启动失败
|
||||
|
||||
```bash
|
||||
# 检查 etcd 和 minio 是否健康
|
||||
docker compose ps etcd minio
|
||||
|
||||
# 查看 Milvus 日志
|
||||
docker compose logs milvus | tail -50
|
||||
|
||||
# 常见原因：内存不足（Milvus 需要至少 4GB 可用内存）
|
||||
free -h
|
||||
```
|
||||
|
||||
### Q: BGE-M3 模型下载失败
|
||||
|
||||
```bash
|
||||
# 使用镜像加速
|
||||
export HF_ENDPOINT=https://hf-mirror.com
|
||||
docker compose up -d embedding-service
|
||||
|
||||
# 或使用 ModelScope
|
||||
bash scripts/download_models.sh
|
||||
```
|
||||
|
||||
### Q: DeepSeek API 连接超时
|
||||
|
||||
```bash
|
||||
# 测试连通性
|
||||
curl -X POST https://api.deepseek.com/v1/chat/completions \
|
||||
-H "Authorization: Bearer $DEEPSEEK_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "deepseek-chat", "messages": [{"role": "user", "content": "ping"}]}'
|
||||
|
||||
# 常见原因：API Key 未设置或网络问题
|
||||
```
|
||||
|
||||
### Q: 内存不足
|
||||
|
||||
```bash
|
||||
# 查看内存使用
|
||||
docker stats --no-stream
|
||||
|
||||
# 临时解决：减少 BGE-M3 批大小（降低内存峰值）
|
||||
# 编辑 .env，添加：
|
||||
# EMBEDDING_BATCH_SIZE=4 （默认16）
|
||||
```
|
||||
536
03_业务闭环说明.md
Normal file
536
03_业务闭环说明.md
Normal file
@@ -0,0 +1,536 @@
|
||||
# AI合规智能中枢 — 三条业务闭环说明
|
||||
|
||||
> 本文档详细描述三条核心业务闭环的数据流、接口规范和验证方法。
|
||||
|
||||
---
|
||||
|
||||
## 一、闭环①：法规入库 → 检索问答
|
||||
|
||||
### 1.1 业务场景
|
||||
|
||||
**触发场景：**
|
||||
- 法务/研发人员上传新法规 PDF（如 GB 18384-2020、UN-ECE R155）
|
||||
- 系统自动解析、分块、向量化，建立可检索知识库
|
||||
- 用户用自然语言提问，系统返回精准答案并标注来源
|
||||
|
||||
**用户角色：** 车企研发、法务、合规管理员
|
||||
|
||||
### 1.2 数据流
|
||||
|
||||
```
|
||||
[用户] 上传 PDF
|
||||
│
|
||||
▼
|
||||
POST /api/kb/files/upload
|
||||
{workspace_id, file}
|
||||
│
|
||||
▼
|
||||
[kbmp-service]
|
||||
- 存储文件 → data/uploads/{file_id}.pdf
|
||||
- 写入 files 表（status: uploaded）
|
||||
- 投递 Celery 任务 → parse-queue
|
||||
- 返回 {task_id, file_id}
|
||||
│
|
||||
▼ 异步
|
||||
[celery: parse-worker]
|
||||
- 调用 POST http://mcp-server:8011/mineru-parse
|
||||
- 获取 Markdown 文本
|
||||
- 更新 files 表（status: parsed）
|
||||
- 投递 vectorize-queue
|
||||
│
|
||||
▼ 异步
|
||||
[celery: vectorize-worker]
|
||||
- 文本分块（chunk_size=512，overlap=64）
|
||||
- 调用 POST http://embedding-service:8010/embed
|
||||
- 获取 1024维 Dense + Sparse 向量
|
||||
- 写入 Milvus regulation_chunks
|
||||
- 写入 PostgreSQL（chunk 元数据）
|
||||
- 更新 files 表（status: vectorized）
|
||||
- 更新 tasks 表（status: completed）
|
||||
|
||||
[用户] 提问
|
||||
│
|
||||
▼
|
||||
POST /api/kb/qa
|
||||
{query, workspace_id, top_k=5}
|
||||
│
|
||||
▼
|
||||
[rag-service]
|
||||
1. BGE-M3 向量化查询
|
||||
2. Milvus Dense 向量检索（Cosine，top-20）
|
||||
3. Milvus Sparse 向量检索（BM25 等效，top-20）
|
||||
4. RRF 融合（Reciprocal Rank Fusion）
|
||||
5. Cross-Encoder Reranker 精排（top-5）
|
||||
6. 构建 RAG Prompt（含检索片段）
|
||||
7. DeepSeek API 生成答案（引文锚定）
|
||||
│
|
||||
▼
|
||||
返回：{answer, sources: [{content, file, page, score}], tokens_used}
|
||||
```
|
||||
|
||||
### 1.3 关键接口
|
||||
|
||||
```http
|
||||
### 创建工作空间
|
||||
POST /api/kb/workspaces
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"name": "汽车安全法规库",
|
||||
"description": "GB、UN-ECE 系列法规",
|
||||
"domain": "vehicle_safety"
|
||||
}
|
||||
|
||||
### 响应
|
||||
{
|
||||
"id": "uuid-xxx",
|
||||
"name": "汽车安全法规库",
|
||||
"created_at": "2026-04-22T10:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
```http
|
||||
### 上传文件
|
||||
POST /api/kb/files/upload
|
||||
Content-Type: multipart/form-data
|
||||
|
||||
file: <binary>
|
||||
workspace_id: uuid-xxx
|
||||
|
||||
### 响应
|
||||
{
|
||||
"file_id": "uuid-yyy",
|
||||
"task_id": "uuid-zzz",
|
||||
"filename": "GB18384-2020.pdf",
|
||||
"status": "processing"
|
||||
}
|
||||
```
|
||||
|
||||
```http
|
||||
### 查询任务状态
|
||||
GET /api/kb/tasks/{task_id}
|
||||
|
||||
### 响应
|
||||
{
|
||||
"task_id": "uuid-zzz",
|
||||
"status": "completed", // pending / running / completed / failed
|
||||
"progress": 100,
|
||||
"file_id": "uuid-yyy",
|
||||
"completed_at": "2026-04-22T10:05:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
```http
|
||||
### 智能问答
|
||||
POST /api/kb/qa
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"query": "电动汽车碰撞后高压系统的断电时间要求是多少？",
|
||||
"workspace_id": "uuid-xxx",
|
||||
"top_k": 5,
|
||||
"return_sources": true
|
||||
}
|
||||
|
||||
### 响应
|
||||
{
|
||||
"answer": "根据 GB 18384-2020 第 2.2 条，碰撞后 5 秒内，高压系统电压应降至 60V 以下。[来源：GB18384-2020.pdf，第3页]",
|
||||
"sources": [
|
||||
{
|
||||
"content": "碰撞后5秒内，高压系统电压应降至60V以下。",
|
||||
"file": "GB18384-2020.pdf",
|
||||
"page": 3,
|
||||
"chunk_idx": 12,
|
||||
"score": 0.94
|
||||
}
|
||||
],
|
||||
"tokens_used": 1250
|
||||
}
|
||||
```
|
||||
|
||||
### 1.4 分块策略
|
||||
|
||||
```python
|
||||
# 推荐分块配置（调研阶段）
|
||||
CHUNK_SIZE = 512 # 每块最大 token 数
|
||||
CHUNK_OVERLAP = 64 # 块间重叠（保留上下文）
|
||||
SEPARATOR = "\n\n" # 优先按段落分割
|
||||
|
||||
# 法规文档特殊处理
|
||||
# - 识别条款编号（1.1, 2.3.1 等），保证条款完整性
|
||||
# - 表格单独处理（不与正文混合）
|
||||
# - 图片提取 alt text
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 二、闭环②：文档上传 → 合规审查
|
||||
|
||||
### 2.1 业务场景
|
||||
|
||||
**触发场景：**
|
||||
- 采购/供应链人员上传供应商文件（技术规格书、合规声明等）
|
||||
- 研发人员上传设计文档，检查是否符合最新法规
|
||||
- EHS 工程师上传安全操作规程，验证 ISO 45001 合规性
|
||||
|
||||
**用户角色：** 采购、供应链、研发、EHS 工程师
|
||||
|
||||
### 2.2 数据流
|
||||
|
||||
```
|
||||
[用户] 上传供应商文件
|
||||
│
|
||||
▼
|
||||
POST /api/compliance/upload
|
||||
{file, regulation_domains}
|
||||
│
|
||||
▼
|
||||
[compliance-backend]
|
||||
- MinerU 解析文档
|
||||
- 条款级分割（识别条款结构）
|
||||
- 法规域匹配（根据内容自动识别：vehicle_safety / data_security / ehs）
|
||||
- 投递 compliance-queue
|
||||
│
|
||||
▼ 异步
|
||||
[celery: compliance-worker]
|
||||
1. 对每个条款，在 Milvus 中检索相关法规要求
|
||||
2. DeepSeek API 评估合规性
|
||||
Prompt: "对比以下供应商条款与法规要求，评估合规性..."
|
||||
3. 生成风险评分（0-100）
|
||||
4. 汇总生成 Markdown 报告
|
||||
5. 存储 compliance_reports 表
|
||||
│
|
||||
▼
|
||||
[用户] 获取报告
|
||||
GET /api/compliance/report/{id}
|
||||
```
|
||||
|
||||
### 2.3 关键接口
|
||||
|
||||
```http
|
||||
### 上传并审查文档
|
||||
POST /api/compliance/upload
|
||||
Content-Type: multipart/form-data
|
||||
|
||||
file: <binary>
|
||||
regulation_domains: ["vehicle_safety", "data_security"] # 可多选
|
||||
|
||||
### 响应
|
||||
{
|
||||
"report_id": "uuid-aaa",
|
||||
"file_id": "uuid-bbb",
|
||||
"status": "analyzing",
|
||||
"estimated_time_seconds": 60
|
||||
}
|
||||
```
|
||||
|
||||
```http
|
||||
### 直接合规检查（文本输入）
|
||||
POST /api/compliance/check
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"query": "供应商声明：产品绝缘电阻为50Ω/V，满足行业标准",
|
||||
"regulation_domains": ["vehicle_safety"],
|
||||
"top_k": 3
|
||||
}
|
||||
|
||||
### 响应
|
||||
{
|
||||
"risk_level": "high",
|
||||
"risk_score": 78,
|
||||
"findings": [
|
||||
{
|
||||
"clause": "GB 18384-2020 第2.1条",
|
||||
"requirement": "直流电路绝缘电阻不得低于100Ω/V",
|
||||
"actual": "供应商声明50Ω/V",
|
||||
"gap": "不满足，差距50Ω/V",
|
||||
"severity": "critical"
|
||||
}
|
||||
],
|
||||
"recommendations": [
|
||||
"要求供应商提升绝缘电阻至100Ω/V以上",
|
||||
"提供经第三方认证的测试报告"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
```http
|
||||
### 获取完整审查报告
|
||||
GET /api/compliance/report/{report_id}
|
||||
|
||||
### 响应
|
||||
{
|
||||
"report_id": "uuid-aaa",
|
||||
"overall_risk_level": "high",
|
||||
"risk_score": 78,
|
||||
"findings": [...],
|
||||
"recommendations": [...],
|
||||
"report_markdown": "# 合规审查报告\n\n## 总体评估\n...",
|
||||
"regulation_domains": ["vehicle_safety"],
|
||||
"llm_model": "deepseek-chat",
|
||||
"created_at": "2026-04-22T11:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
### 2.4 风险等级定义
|
||||
|
||||
| 风险等级 | 分数 | 说明 | 建议行动 |
|
||||
|---------|------|------|---------|
|
||||
| low | 0-30 | 基本合规，小幅优化 | 记录并监控 |
|
||||
| medium | 31-60 | 部分不符合，需要整改 | 制定整改计划 |
|
||||
| high | 61-80 | 重大不符合，需立即处理 | 暂停合作/紧急整改 |
|
||||
| critical | 81-100 | 严重违规，可能造成法律风险 | 立即停止/上报管理层 |
|
||||
|
||||
---
|
||||
|
||||
## 三、闭环③：法规监控 → 变更推送
|
||||
|
||||
### 3.1 业务场景
|
||||
|
||||
**触发场景：**
|
||||
- 国家发布新的新能源汽车数据安全法规
|
||||
- 现有法规（如 GB 7258）进行修订
|
||||
- 碳排放法规新增企业义务
|
||||
|
||||
系统自动检测变更，分析影响，推送给相关角色。
|
||||
|
||||
**用户角色：** 合规管理员、法务专员、EHS 工程师（订阅对应域）
|
||||
|
||||
### 3.2 数据流
|
||||
|
||||
```
|
||||
[Celery Beat] 每天凌晨 2:00 触发
|
||||
│
|
||||
▼
|
||||
[celery: monitor-worker]
|
||||
- 读取 regulation_sources 表（所有 is_active=True 的监控源）
|
||||
- 对每个监控源：
|
||||
a. HTTP 抓取页面内容
|
||||
b. 计算 MD5 Hash
|
||||
c. 与 last_hash 对比
|
||||
d. 有变化 → 投递变更分析任务
|
||||
│
|
||||
▼ [有变更时]
|
||||
[celery: compliance-worker]
|
||||
- DeepSeek API 分析变更内容
|
||||
- 提取新增/修订/废止条款
|
||||
- 生成变更摘要
|
||||
- 写入 regulation_updates 表
|
||||
- 触发增量入库（重新向量化变更条款）
|
||||
- 更新 Neo4j 知识图谱
|
||||
│
|
||||
▼
|
||||
[celery: push-worker]
|
||||
- 读取 subscriptions 表
|
||||
- 按域、重要性过滤
|
||||
- 发送推送（Email / Webhook / 飞书）
|
||||
- 标记 is_notified=True
|
||||
```
|
||||
|
||||
### 3.3 关键接口
|
||||
|
||||
```http
|
||||
### 配置监控源
|
||||
POST /api/regulation/sources
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"name": "国家标准全文公开系统",
|
||||
"url": "https://std.samr.gov.cn",
|
||||
"domain": "vehicle_safety",
|
||||
"fetch_interval": 86400,
|
||||
"fetch_config": {
|
||||
"css_selector": ".standard-list .item",
|
||||
"title_selector": ".title",
|
||||
"date_selector": ".date"
|
||||
}
|
||||
}
|
||||
|
||||
### 响应
|
||||
{
|
||||
"id": "uuid-src1",
|
||||
"name": "国家标准全文公开系统",
|
||||
"status": "active",
|
||||
"next_fetch_at": "2026-04-23T02:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
```http
|
||||
### 查看法规变更记录
|
||||
GET /api/regulation/updates?domain=vehicle_safety&limit=10&offset=0
|
||||
|
||||
### 响应
|
||||
{
|
||||
"total": 25,
|
||||
"updates": [
|
||||
{
|
||||
"id": "uuid-upd1",
|
||||
"title": "GB 18384-2022 电动汽车安全要求（修订版）",
|
||||
"url": "https://std.samr.gov.cn/xxxx",
|
||||
"change_type": "revised",
|
||||
"summary": "主要变更：碰撞断电时间由5秒缩短至3秒；新增涉水安全要求",
|
||||
"importance": "high",
|
||||
"fetched_at": "2026-04-22T02:00:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
```http
|
||||
### 手动触发法规源采集（测试用）
|
||||
POST /api/regulation/sources/{source_id}/fetch
|
||||
|
||||
### 响应
|
||||
{
|
||||
"task_id": "uuid-task1",
|
||||
"status": "queued",
|
||||
"source_id": "uuid-src1"
|
||||
}
|
||||
```
|
||||
|
||||
```http
|
||||
### 订阅变更推送
|
||||
POST /api/regulation/subscribe
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"name": "EHS 工程师推送",
|
||||
"channel": "webhook",
|
||||
"target": "https://open.feishu.cn/open-apis/bot/v2/hook/xxxx",
|
||||
"domains": ["ehs", "carbon"],
|
||||
"importance_min": "normal"
|
||||
}
|
||||
```
|
||||
|
||||
### 3.4 内置监控源列表
|
||||
|
||||
| 名称 | URL | 域 |
|
||||
|------|-----|-----|
|
||||
| 国家标准全文公开系统 | https://std.samr.gov.cn | vehicle_safety |
|
||||
| 工信部政策法规 | https://www.miit.gov.cn/jgsj/fgs/zcfg | vehicle_safety |
|
||||
| 应急管理部法规 | https://www.mem.gov.cn/gk/zcfg | ehs |
|
||||
| 生态环境部法规 | https://www.mee.gov.cn/ywgz/fgbz/fl | carbon |
|
||||
| 网信办法规 | https://www.cac.gov.cn/zcfg/index.htm | data_security |
|
||||
|
||||
---
|
||||
|
||||
## 四、接口认证说明（调研版）
|
||||
|
||||
调研版使用简单 API Key 认证（在 `Authorization` 头传入）：
|
||||
|
||||
```http
|
||||
# 所有请求需要携带 API Key
|
||||
Authorization: Bearer <API_SECRET_KEY>
|
||||
```
|
||||
|
||||
> `API_SECRET_KEY` 在 `.env` 中配置，默认值仅供本地调研使用，生产环境必须更换。
|
||||
|
||||
---
|
||||
|
||||
## 五、完整冒烟测试脚本
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
# 完整三条闭环验证
|
||||
API="http://localhost"
|
||||
KEY="your_api_secret_key"
|
||||
HEADER="-H 'Authorization: Bearer $KEY' -H 'Content-Type: application/json'"
|
||||
|
||||
# ── 闭环①测试 ────────────────────────────────
|
||||
echo "=== 测试闭环①：法规入库 → 问答 ==="
|
||||
|
||||
# 1. 创建工作空间
|
||||
WS=$(curl -sf -X POST $API/api/kb/workspaces \
|
||||
-H "Authorization: Bearer $KEY" -H "Content-Type: application/json" \
|
||||
-d '{"name":"测试法规库","domain":"vehicle_safety"}')
|
||||
WS_ID=$(echo $WS | python3 -c "import sys,json; print(json.load(sys.stdin)['id'])")
|
||||
echo "工作空间：$WS_ID"
|
||||
|
||||
# 2. 上传测试 PDF
|
||||
UPLOAD=$(curl -sf -X POST $API/api/kb/files/upload \
|
||||
-H "Authorization: Bearer $KEY" \
|
||||
-F "file=@data/uploads/test_regulation.txt" \
|
||||
-F "workspace_id=$WS_ID")
|
||||
TASK_ID=$(echo $UPLOAD | python3 -c "import sys,json; print(json.load(sys.stdin)['task_id'])")
|
||||
echo "任务ID：$TASK_ID"
|
||||
|
||||
# 3. 等待处理
|
||||
for i in {1..30}; do
|
||||
STATUS=$(curl -sf $API/api/kb/tasks/$TASK_ID -H "Authorization: Bearer $KEY" | \
|
||||
python3 -c "import sys,json; print(json.load(sys.stdin)['status'])")
|
||||
[[ "$STATUS" == "completed" ]] && echo "处理完成" && break
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# 4. 问答测试
|
||||
QA=$(curl -sf -X POST $API/api/kb/qa \
|
||||
-H "Authorization: Bearer $KEY" -H "Content-Type: application/json" \
|
||||
-d "{\"query\":\"碰撞后高压系统要求\",\"workspace_id\":\"$WS_ID\"}")
|
||||
echo "问答结果：$(echo $QA | python3 -c "import sys,json; print(json.load(sys.stdin).get('answer','')[:100])")"
|
||||
|
||||
# ── 闭环②测试 ────────────────────────────────
|
||||
echo ""
|
||||
echo "=== 测试闭环②：合规审查 ==="
|
||||
CHECK=$(curl -sf -X POST $API/api/compliance/check \
|
||||
-H "Authorization: Bearer $KEY" -H "Content-Type: application/json" \
|
||||
-d '{"query":"绝缘电阻50Ω/V","regulation_domains":["vehicle_safety"]}')
|
||||
echo "风险等级：$(echo $CHECK | python3 -c "import sys,json; print(json.load(sys.stdin).get('risk_level','unknown'))")"
|
||||
|
||||
# ── 闭环③测试 ────────────────────────────────
|
||||
echo ""
|
||||
echo "=== 测试闭环③：法规监控 ==="
|
||||
SRC=$(curl -sf -X POST $API/api/regulation/sources \
|
||||
-H "Authorization: Bearer $KEY" -H "Content-Type: application/json" \
|
||||
-d '{"name":"测试源","url":"https://std.samr.gov.cn","domain":"vehicle_safety"}')
|
||||
echo "监控源：$(echo $SRC | python3 -c "import sys,json; print(json.load(sys.stdin).get('id','failed'))")"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 六、数据流示意图（完整版）
|
||||
|
||||
```
|
||||
┌─────────────────────────────────┐
|
||||
│ 用户请求 │
|
||||
│ Web / API / Mobile / Bot │
|
||||
└──────────────┬──────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────┐
|
||||
│ Nginx API Gateway │
|
||||
│ 路由 / 限流 / 认证 │
|
||||
└──────────────┬──────────────────┘
|
||||
│
|
||||
┌────────────────────┼────────────────────┐
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌──────────────┐ ┌──────────────────┐ ┌────────────────┐
|
||||
│ 知识库 │ │ 合规审查 │ │ 法规监控 │
|
||||
│ /api/kb/* │ │ /api/compliance/* │ │/api/regulation/│
|
||||
└──────┬───────┘ └────────┬─────────┘ └───────┬────────┘
|
||||
│ │ │
|
||||
└──────────┬──────────┘ │
|
||||
│ │
|
||||
▼ ▼
|
||||
┌──────────────────┐ ┌──────────────────┐
|
||||
│ compliance- │ │ Celery Beat │
|
||||
│ backend │ │ 定时调度 │
|
||||
└──────┬───────────┘ └────────┬─────────┘
|
||||
│ │
|
||||
┌──────────┼──────────┐ ┌──────────┼──────────┐
|
||||
│ │ │ │ │ │
|
||||
▼ ▼ ▼ ▼ ▼ ▼
|
||||
parse-w vectorize-w compliance-w monitor-w push-w
|
||||
│ │ │ │ │
|
||||
▼ ▼ │ │ ▼
|
||||
mcp-server embedding LLM API 网络抓取 通知推送
|
||||
(MinerU) (BGE-M3) (DeepSeek) (requests) (Email/Bot)
|
||||
│ │
|
||||
└────┬─────┘
|
||||
│
|
||||
┌──────────┼──────────────┐
|
||||
▼ ▼ ▼
|
||||
PostgreSQL Milvus Neo4j
|
||||
(元数据/报告) (向量检索) (知识图谱)
|
||||
```
|
||||
190
README.md
Normal file
190
README.md
Normal file
@@ -0,0 +1,190 @@
|
||||
# AI合规智能中枢 — 调研版部署指南
|
||||
|
||||
面向车企与工厂的全链路合规智能平台，Docker Compose 单机部署版本，用于验证三条业务闭环。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 前置要求
|
||||
|
||||
| 资源 | 最低 | 推荐 |
|
||||
|------|------|------|
|
||||
| CPU | 8核 | 16核+ |
|
||||
| 内存 | 32 GB | 64 GB |
|
||||
| 存储 | 200 GB SSD | 500 GB SSD |
|
||||
| GPU | 无需 | 1× RTX 3090（加速嵌入）|
|
||||
| OS | Ubuntu 22.04 LTS 或 Windows 11 + WSL2 | — |
|
||||
|
||||
### 1. 安装 Docker
|
||||
|
||||
**Ubuntu/Linux：**
|
||||
```bash
|
||||
bash scripts/00_install_docker_ubuntu.sh
|
||||
```
|
||||
|
||||
**Windows（PowerShell 管理员）：**
|
||||
```powershell
|
||||
.\scripts\00_install_docker_windows.ps1
|
||||
```
|
||||
|
||||
### 2. 配置环境变量
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# 编辑 .env，至少填写：
|
||||
# - DEEPSEEK_API_KEY（在 https://platform.deepseek.com 申请）
|
||||
nano .env
|
||||
```
|
||||
|
||||
### 3. 一键启动
|
||||
|
||||
```bash
|
||||
# 拉取镜像（可选，加速首次启动）
|
||||
bash scripts/02_pull_images.sh
|
||||
|
||||
# 分步启动（推荐，含健康等待）
|
||||
bash scripts/06_start_all.sh
|
||||
```
|
||||
|
||||
### 4. 验证部署
|
||||
|
||||
```bash
|
||||
# 检查所有服务状态
|
||||
bash scripts/check_health.sh
|
||||
|
||||
# 运行端到端冒烟测试
|
||||
bash scripts/07_smoke_test.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 服务访问地址
|
||||
|
||||
| 服务 | 地址 | 说明 |
|
||||
|------|------|------|
|
||||
| API 网关 | http://localhost | Nginx 入口，所有 API 请求入口 |
|
||||
| 业务后端 | http://localhost:8000/docs | FastAPI Swagger UI |
|
||||
| Neo4j 浏览器 | http://localhost:7474 | 知识图谱可视化 |
|
||||
| Grafana | http://localhost:3000 | 监控面板（`--profile monitoring` 启动）|
|
||||
| Milvus | localhost:19530 | 向量数据库 gRPC 端口 |
|
||||
|
||||
---
|
||||
|
||||
## 三条业务闭环
|
||||
|
||||
### 闭环①：法规入库 → 检索问答
|
||||
|
||||
```bash
|
||||
# 上传法规PDF
|
||||
curl -X POST http://localhost/api/kb/files/upload \
|
||||
-F "file=@your_regulation.pdf" \
|
||||
-F "workspace_id=auto-regulation"
|
||||
|
||||
# 查询任务状态
|
||||
curl http://localhost/api/kb/tasks/{task_id}
|
||||
|
||||
# 检索问答
|
||||
curl -X POST http://localhost/api/kb/qa \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"query": "GB 18384 电动汽车碰撞安全要求", "top_k": 5}'
|
||||
```
|
||||
|
||||
### 闭环②：文档上传 → 合规审查
|
||||
|
||||
```bash
|
||||
# 上传供应商文件
|
||||
curl -X POST http://localhost/api/compliance/upload \
|
||||
-F "file=@supplier_document.pdf"
|
||||
|
||||
# 触发合规审查
|
||||
curl -X POST http://localhost/api/compliance/check \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"doc_id": "xxx", "regulation_domains": ["vehicle_safety", "data_security"]}'
|
||||
|
||||
# 获取审查报告
|
||||
curl http://localhost/api/compliance/report/{id}
|
||||
```
|
||||
|
||||
### 闭环③：法规监控 → 变更推送
|
||||
|
||||
```bash
|
||||
# 配置监控源
|
||||
curl -X POST http://localhost/api/regulation/sources \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"url": "https://std.samr.gov.cn", "name": "国家标准全文公开"}'
|
||||
|
||||
# 查看变更记录
|
||||
curl http://localhost/api/regulation/updates
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
Depolyment/
|
||||
├── README.md # 本文件
|
||||
├── docker-compose.yml # 全服务编排
|
||||
├── .env.example # 环境变量模板
|
||||
├── scripts/ # 安装与运维脚本
|
||||
├── services/
|
||||
│ ├── embedding/ # BGE-M3 嵌入服务
|
||||
│ ├── mcp-server/ # MinerU 文档解析服务
|
||||
│ └── compliance-backend/ # 核心业务后端
|
||||
├── config/ # Nginx、Prometheus 配置
|
||||
├── init-sql/ # PostgreSQL 初始化 SQL
|
||||
├── data/ # 运行时数据（上传文件、解析结果）
|
||||
├── logs/ # 服务日志
|
||||
└── models/ # AI 模型缓存（BGE-M3、MinerU）
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 常用操作
|
||||
|
||||
```bash
|
||||
# 查看所有服务状态
|
||||
docker compose ps
|
||||
|
||||
# 查看某个服务日志
|
||||
docker compose logs -f compliance-backend
|
||||
|
||||
# 重启某个服务
|
||||
docker compose restart embedding-service
|
||||
|
||||
# 停止所有服务（保留数据）
|
||||
docker compose stop
|
||||
|
||||
# 完全重置（删除所有数据，慎用）
|
||||
bash scripts/reset_all.sh
|
||||
|
||||
# 启动监控面板
|
||||
docker compose --profile monitoring up -d grafana
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## LLM 切换
|
||||
|
||||
默认使用 DeepSeek API，如需切换到 Qwen（阿里云）：
|
||||
|
||||
编辑 `.env`：
|
||||
```bash
|
||||
LLM_PROVIDER=qwen
|
||||
DASHSCOPE_API_KEY=your_key_here
|
||||
QWEN_MODEL=qwen-plus
|
||||
```
|
||||
|
||||
然后重启业务服务：
|
||||
```bash
|
||||
docker compose restart compliance-backend celery-worker
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 生产升级路径
|
||||
|
||||
调研验证通过后，升级要点：
|
||||
1. **LLM**：从 API 切换到本地 vLLM + DeepSeek-V3（需要 4×A100）
|
||||
2. **Milvus**：从 Standalone 升级到分布式集群（加独立 MinIO）
|
||||
3. **编排**：从 Docker Compose 迁移到 Kubernetes（服务配置文件可复用）
|
||||
4. **安全**：启用完整 JWT/RBAC，添加 TLS 证书
|
||||
63
config/nginx.conf
Normal file
63
config/nginx.conf
Normal file
@@ -0,0 +1,63 @@
|
||||
upstream compliance_backend {
|
||||
server compliance-backend:8000;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
# 文件上传大小限制（法规PDF可能较大）
|
||||
client_max_body_size 100M;
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
# 访问日志
|
||||
access_log /var/log/nginx/access.log;
|
||||
error_log /var/log/nginx/error.log;
|
||||
|
||||
# 超时配置（LLM推理可能较慢）
|
||||
proxy_connect_timeout 60s;
|
||||
proxy_send_timeout 300s;
|
||||
proxy_read_timeout 300s;
|
||||
|
||||
# 通用代理头
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
|
||||
# ── 知识库接口 ─────────────────────────────
|
||||
location /api/kb/ {
|
||||
proxy_pass http://compliance_backend/api/kb/;
|
||||
}
|
||||
|
||||
# ── 合规审查接口 ───────────────────────────
|
||||
location /api/compliance/ {
|
||||
proxy_pass http://compliance_backend/api/compliance/;
|
||||
}
|
||||
|
||||
# ── 法规监控接口 ───────────────────────────
|
||||
location /api/regulation/ {
|
||||
proxy_pass http://compliance_backend/api/regulation/;
|
||||
}
|
||||
|
||||
# ── 健康检查 ───────────────────────────────
|
||||
location /health {
|
||||
proxy_pass http://compliance_backend/health;
|
||||
}
|
||||
|
||||
# ── API 文档（开发环境）────────────────────
|
||||
location /docs {
|
||||
proxy_pass http://compliance_backend/docs;
|
||||
}
|
||||
|
||||
location /openapi.json {
|
||||
proxy_pass http://compliance_backend/openapi.json;
|
||||
}
|
||||
|
||||
# ── 根路径 ─────────────────────────────────
|
||||
location / {
|
||||
proxy_pass http://compliance_backend/;
|
||||
}
|
||||
}
|
||||
22
config/prometheus.yml
Normal file
22
config/prometheus.yml
Normal file
@@ -0,0 +1,22 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: "compliance-backend"
|
||||
static_configs:
|
||||
- targets: ["compliance-backend:8000"]
|
||||
metrics_path: /metrics
|
||||
|
||||
- job_name: "milvus"
|
||||
static_configs:
|
||||
- targets: ["milvus:9091"]
|
||||
metrics_path: /metrics
|
||||
|
||||
- job_name: "redis"
|
||||
static_configs:
|
||||
- targets: ["redis:6379"]
|
||||
|
||||
- job_name: "postgres"
|
||||
static_configs:
|
||||
- targets: ["postgres:5432"]
|
||||
380
docker-compose.yml
Normal file
380
docker-compose.yml
Normal file
@@ -0,0 +1,380 @@
|
||||
version: "3.9"
|
||||
|
||||
networks:
|
||||
compliance-net:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
milvus_data:
|
||||
minio_data:
|
||||
neo4j_data:
|
||||
neo4j_logs:
|
||||
|
||||
services:
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# 基础数据层
|
||||
# ═══════════════════════════════════════════════
|
||||
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
container_name: compliance-postgres
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: compliance
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-compliance123}
|
||||
POSTGRES_DB: compliance_db
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- ./init-sql:/docker-entrypoint-initdb.d
|
||||
ports:
|
||||
- "5432:5432"
|
||||
networks: [compliance-net]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U compliance -d compliance_db"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: compliance-redis
|
||||
restart: unless-stopped
|
||||
command: >
|
||||
redis-server
|
||||
--requirepass ${REDIS_PASSWORD:-redis123}
|
||||
--maxmemory 2gb
|
||||
--maxmemory-policy allkeys-lru
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
ports:
|
||||
- "6379:6379"
|
||||
networks: [compliance-net]
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD:-redis123}", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# Milvus 向量数据库（Standalone，含 etcd + minio）
|
||||
# ═══════════════════════════════════════════════
|
||||
|
||||
etcd:
|
||||
image: quay.io/coreos/etcd:v3.5.5
|
||||
container_name: milvus-etcd
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
ETCD_AUTO_COMPACTION_MODE: revision
|
||||
ETCD_AUTO_COMPACTION_RETENTION: "1000"
|
||||
ETCD_QUOTA_BACKEND_BYTES: "4294967296"
|
||||
ETCD_SNAPSHOT_COUNT: "50000"
|
||||
volumes:
|
||||
- milvus_data:/etcd
|
||||
command: >
|
||||
etcd
|
||||
-advertise-client-urls=http://127.0.0.1:2379
|
||||
-listen-client-urls=http://0.0.0.0:2379
|
||||
--data-dir=/etcd
|
||||
networks: [compliance-net]
|
||||
healthcheck:
|
||||
test: ["CMD", "etcdctl", "endpoint", "health"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
minio:
|
||||
image: minio/minio:RELEASE.2023-03-13T19-46-17Z
|
||||
container_name: milvus-minio
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MINIO_ACCESS_KEY: minioadmin
|
||||
MINIO_SECRET_KEY: minioadmin
|
||||
volumes:
|
||||
- minio_data:/minio_data
|
||||
command: minio server /minio_data --console-address ":9001"
|
||||
ports:
|
||||
- "9001:9001" # MinIO 控制台（可选访问）
|
||||
networks: [compliance-net]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
milvus:
|
||||
image: milvusdb/milvus:v2.4.13
|
||||
container_name: compliance-milvus
|
||||
restart: unless-stopped
|
||||
command: ["milvus", "run", "standalone"]
|
||||
environment:
|
||||
ETCD_ENDPOINTS: etcd:2379
|
||||
MINIO_ADDRESS: minio:9000
|
||||
volumes:
|
||||
- milvus_data:/var/lib/milvus
|
||||
ports:
|
||||
- "19530:19530" # gRPC API
|
||||
- "9091:9091" # HTTP API
|
||||
depends_on:
|
||||
etcd:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
networks: [compliance-net]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 10
|
||||
start_period: 60s
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# Neo4j 知识图谱数据库
|
||||
# ═══════════════════════════════════════════════
|
||||
|
||||
neo4j:
|
||||
image: neo4j:5.20-community
|
||||
container_name: compliance-neo4j
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
NEO4J_AUTH: neo4j/${NEO4J_PASSWORD:-neo4j123}
|
||||
NEO4J_PLUGINS: '["apoc"]'
|
||||
NEO4J_dbms_security_procedures_unrestricted: apoc.*
|
||||
NEO4J_dbms_memory_heap_initial__size: 512m
|
||||
NEO4J_dbms_memory_heap_max__size: 2G
|
||||
NEO4J_dbms_memory_pagecache_size: 1G
|
||||
volumes:
|
||||
- neo4j_data:/data
|
||||
- neo4j_logs:/logs
|
||||
ports:
|
||||
- "7474:7474" # Browser UI
|
||||
- "7687:7687" # Bolt 协议
|
||||
networks: [compliance-net]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q --spider http://localhost:7474 || exit 1"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
start_period: 60s
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# AI 模型服务
|
||||
# ═══════════════════════════════════════════════
|
||||
|
||||
embedding-service:
|
||||
build:
|
||||
context: ./services/embedding
|
||||
dockerfile: Dockerfile
|
||||
image: compliance-embedding:latest
|
||||
container_name: compliance-embedding
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
MODEL_NAME: BAAI/bge-m3
|
||||
HF_ENDPOINT: ${HF_ENDPOINT:-https://hf-mirror.com}
|
||||
DEVICE: ${EMBEDDING_DEVICE:-cpu}
|
||||
MAX_BATCH_SIZE: "16"
|
||||
volumes:
|
||||
- ./models:/app/models
|
||||
ports:
|
||||
- "8010:8010"
|
||||
networks: [compliance-net]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 8G
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8010/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 120s # 模型加载需要时间
|
||||
|
||||
mcp-server:
|
||||
build:
|
||||
context: ./services/mcp-server
|
||||
dockerfile: Dockerfile
|
||||
image: compliance-mcp:latest
|
||||
container_name: compliance-mcp
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
DEVICE: ${MCP_DEVICE:-cpu}
|
||||
HF_ENDPOINT: ${HF_ENDPOINT:-https://hf-mirror.com}
|
||||
volumes:
|
||||
- ./models:/app/models
|
||||
- ./data/uploads:/app/uploads
|
||||
- ./data/parsed:/app/parsed
|
||||
ports:
|
||||
- "8011:8011"
|
||||
networks: [compliance-net]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 8G
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8011/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 120s
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# 业务服务层
|
||||
# ═══════════════════════════════════════════════
|
||||
|
||||
compliance-backend:
|
||||
build:
|
||||
context: ./services/compliance-backend
|
||||
dockerfile: Dockerfile
|
||||
image: compliance-backend:latest
|
||||
container_name: compliance-backend
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://compliance:${POSTGRES_PASSWORD:-compliance123}@postgres:5432/compliance_db
|
||||
REDIS_URL: redis://:${REDIS_PASSWORD:-redis123}@redis:6379/0
|
||||
MILVUS_HOST: milvus
|
||||
MILVUS_PORT: "19530"
|
||||
NEO4J_URI: bolt://neo4j:7687
|
||||
NEO4J_USER: neo4j
|
||||
NEO4J_PASSWORD: ${NEO4J_PASSWORD:-neo4j123}
|
||||
EMBEDDING_SERVICE_URL: http://embedding-service:8010
|
||||
MCP_SERVER_URL: http://mcp-server:8011
|
||||
LLM_PROVIDER: ${LLM_PROVIDER:-deepseek}
|
||||
DEEPSEEK_API_KEY: ${DEEPSEEK_API_KEY:-}
|
||||
DEEPSEEK_MODEL: ${DEEPSEEK_MODEL:-deepseek-chat}
|
||||
DASHSCOPE_API_KEY: ${DASHSCOPE_API_KEY:-}
|
||||
QWEN_MODEL: ${QWEN_MODEL:-qwen-plus}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
APP_ENV: ${APP_ENV:-development}
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./logs:/app/logs
|
||||
ports:
|
||||
- "8000:8000"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
milvus:
|
||||
condition: service_healthy
|
||||
embedding-service:
|
||||
condition: service_healthy
|
||||
networks: [compliance-net]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 30s
|
||||
|
||||
celery-worker:
|
||||
build:
|
||||
context: ./services/compliance-backend
|
||||
dockerfile: Dockerfile
|
||||
image: compliance-backend:latest
|
||||
container_name: compliance-worker
|
||||
restart: unless-stopped
|
||||
command: >
|
||||
celery -A app.worker worker
|
||||
--loglevel=info
|
||||
--concurrency=4
|
||||
--queues=default,parse,vectorize,compliance,monitor,push
|
||||
env_file: .env
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://compliance:${POSTGRES_PASSWORD:-compliance123}@postgres:5432/compliance_db
|
||||
REDIS_URL: redis://:${REDIS_PASSWORD:-redis123}@redis:6379/0
|
||||
MILVUS_HOST: milvus
|
||||
MILVUS_PORT: "19530"
|
||||
NEO4J_URI: bolt://neo4j:7687
|
||||
NEO4J_USER: neo4j
|
||||
NEO4J_PASSWORD: ${NEO4J_PASSWORD:-neo4j123}
|
||||
EMBEDDING_SERVICE_URL: http://embedding-service:8010
|
||||
MCP_SERVER_URL: http://mcp-server:8011
|
||||
LLM_PROVIDER: ${LLM_PROVIDER:-deepseek}
|
||||
DEEPSEEK_API_KEY: ${DEEPSEEK_API_KEY:-}
|
||||
DASHSCOPE_API_KEY: ${DASHSCOPE_API_KEY:-}
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./logs:/app/logs
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
compliance-backend:
|
||||
condition: service_healthy
|
||||
networks: [compliance-net]
|
||||
|
||||
celery-beat:
|
||||
build:
|
||||
context: ./services/compliance-backend
|
||||
dockerfile: Dockerfile
|
||||
image: compliance-backend:latest
|
||||
container_name: compliance-beat
|
||||
restart: unless-stopped
|
||||
command: >
|
||||
celery -A app.worker beat
|
||||
--loglevel=info
|
||||
--scheduler celery.beat.PersistentScheduler
|
||||
env_file: .env
|
||||
environment:
|
||||
DATABASE_URL: postgresql+asyncpg://compliance:${POSTGRES_PASSWORD:-compliance123}@postgres:5432/compliance_db
|
||||
REDIS_URL: redis://:${REDIS_PASSWORD:-redis123}@redis:6379/0
|
||||
DEEPSEEK_API_KEY: ${DEEPSEEK_API_KEY:-}
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./logs:/app/logs
|
||||
depends_on:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
networks: [compliance-net]
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# API 网关
|
||||
# ═══════════════════════════════════════════════
|
||||
|
||||
nginx:
|
||||
image: nginx:1.25-alpine
|
||||
container_name: compliance-nginx
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./config/nginx.conf:/etc/nginx/conf.d/default.conf:ro
|
||||
ports:
|
||||
- "80:80"
|
||||
depends_on:
|
||||
compliance-backend:
|
||||
condition: service_healthy
|
||||
networks: [compliance-net]
|
||||
healthcheck:
|
||||
test: ["CMD", "nginx", "-t"]
|
||||
interval: 30s
|
||||
|
||||
# ═══════════════════════════════════════════════
|
||||
# 监控（可选，--profile monitoring 启动）
|
||||
# ═══════════════════════════════════════════════
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.0.0
|
||||
container_name: compliance-grafana
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
volumes:
|
||||
- ./config/prometheus.yml:/etc/grafana/provisioning/datasources/prometheus.yml:ro
|
||||
ports:
|
||||
- "3000:3000"
|
||||
networks: [compliance-net]
|
||||
profiles: [monitoring]
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v2.51.0
|
||||
container_name: compliance-prometheus
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
ports:
|
||||
- "9090:9090"
|
||||
networks: [compliance-net]
|
||||
profiles: [monitoring]
|
||||
192
init-sql/01_init_schema.sql
Normal file
192
init-sql/01_init_schema.sql
Normal file
@@ -0,0 +1,192 @@
|
||||
-- AI合规智能中枢 — PostgreSQL 初始化 Schema
|
||||
-- 执行时机：容器首次启动时自动执行
|
||||
|
||||
-- 启用扩展
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS vector; -- pgvector（pgvector/pgvector:pg16 镜像已内置）
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm; -- 全文检索支持
|
||||
|
||||
-- ══════════════════════════════════════════════════
|
||||
-- 工作空间（知识库）
|
||||
-- ══════════════════════════════════════════════════
|
||||
CREATE TABLE IF NOT EXISTS workspaces (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
domain VARCHAR(100), -- vehicle_safety / data_security / ehs / carbon
|
||||
created_by VARCHAR(255),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- ══════════════════════════════════════════════════
|
||||
-- 文件记录
|
||||
-- ══════════════════════════════════════════════════
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
workspace_id UUID REFERENCES workspaces(id) ON DELETE CASCADE,
|
||||
filename VARCHAR(500) NOT NULL,
|
||||
original_name VARCHAR(500) NOT NULL,
|
||||
file_type VARCHAR(50), -- pdf / docx / xlsx
|
||||
file_size BIGINT,
|
||||
storage_path TEXT, -- data/uploads/相对路径
|
||||
parsed_path TEXT, -- data/parsed/相对路径
|
||||
status VARCHAR(50) DEFAULT 'uploaded', -- uploaded/parsing/parsed/vectorized/failed
|
||||
error_msg TEXT,
|
||||
metadata JSONB DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_files_workspace ON files(workspace_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_files_status ON files(status);
|
||||
|
||||
-- ══════════════════════════════════════════════════
|
||||
-- 异步任务记录
|
||||
-- ══════════════════════════════════════════════════
|
||||
CREATE TABLE IF NOT EXISTS tasks (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
task_type VARCHAR(100) NOT NULL, -- parse / vectorize / compliance_check / regulation_fetch
|
||||
status VARCHAR(50) DEFAULT 'pending', -- pending/running/completed/failed
|
||||
payload JSONB DEFAULT '{}',
|
||||
result JSONB,
|
||||
error_msg TEXT,
|
||||
progress INTEGER DEFAULT 0, -- 0-100
|
||||
file_id UUID REFERENCES files(id),
|
||||
celery_task_id VARCHAR(255),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
completed_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_tasks_type ON tasks(task_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_tasks_file ON tasks(file_id);
|
||||
|
||||
-- ══════════════════════════════════════════════════
|
||||
-- 合规审查报告
|
||||
-- ══════════════════════════════════════════════════
|
||||
CREATE TABLE IF NOT EXISTS compliance_reports (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
file_id UUID REFERENCES files(id),
|
||||
regulation_domains TEXT[], -- 适用法规域
|
||||
overall_risk_level VARCHAR(20), -- high / medium / low
|
||||
risk_score DECIMAL(5,2), -- 0-100
|
||||
findings JSONB DEFAULT '[]', -- 问题列表
|
||||
recommendations JSONB DEFAULT '[]', -- 整改建议
|
||||
report_markdown TEXT, -- 完整报告（Markdown格式）
|
||||
llm_model VARCHAR(100), -- 生成时使用的模型
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_reports_file ON compliance_reports(file_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_reports_risk ON compliance_reports(overall_risk_level);
|
||||
|
||||
-- ══════════════════════════════════════════════════
|
||||
-- 法规监控源
|
||||
-- ══════════════════════════════════════════════════
|
||||
CREATE TABLE IF NOT EXISTS regulation_sources (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
source_type VARCHAR(50) DEFAULT 'webpage', -- webpage / rss / api
|
||||
domain VARCHAR(100), -- vehicle_safety / ehs 等
|
||||
fetch_interval INTEGER DEFAULT 86400, -- 抓取间隔（秒），默认每天
|
||||
is_active BOOLEAN DEFAULT TRUE,
|
||||
last_fetched_at TIMESTAMPTZ,
|
||||
last_hash VARCHAR(64), -- 内容hash，用于变更检测
|
||||
fetch_config JSONB DEFAULT '{}', -- 抓取配置（CSS选择器等）
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sources_active ON regulation_sources(is_active);
|
||||
CREATE INDEX IF NOT EXISTS idx_sources_domain ON regulation_sources(domain);
|
||||
|
||||
-- ══════════════════════════════════════════════════
|
||||
-- 法规变更记录
|
||||
-- ══════════════════════════════════════════════════
|
||||
CREATE TABLE IF NOT EXISTS regulation_updates (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
source_id UUID REFERENCES regulation_sources(id),
|
||||
title VARCHAR(500),
|
||||
url TEXT,
|
||||
change_type VARCHAR(50), -- new / revised / revoked / notice
|
||||
summary TEXT, -- AI生成的变更摘要
|
||||
raw_content TEXT, -- 原始抓取内容
|
||||
diff_content TEXT, -- 与上次内容的差异
|
||||
is_notified BOOLEAN DEFAULT FALSE,
|
||||
importance VARCHAR(20) DEFAULT 'normal', -- high / normal / low
|
||||
fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
published_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_updates_source ON regulation_updates(source_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_updates_notified ON regulation_updates(is_notified);
|
||||
CREATE INDEX IF NOT EXISTS idx_updates_fetched ON regulation_updates(fetched_at DESC);
|
||||
|
||||
-- ══════════════════════════════════════════════════
|
||||
-- 推送订阅
|
||||
-- ══════════════════════════════════════════════════
|
||||
CREATE TABLE IF NOT EXISTS subscriptions (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(255),
|
||||
channel VARCHAR(50) NOT NULL, -- email / webhook / feishu / dingtalk
|
||||
target TEXT NOT NULL, -- 邮件地址 或 Webhook URL
|
||||
domains TEXT[], -- 订阅的法规域，为空则订阅全部
|
||||
importance_min VARCHAR(20) DEFAULT 'normal',
|
||||
is_active BOOLEAN DEFAULT TRUE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- ══════════════════════════════════════════════════
|
||||
-- 全链路审计日志
|
||||
-- ══════════════════════════════════════════════════
|
||||
CREATE TABLE IF NOT EXISTS audit_logs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
action VARCHAR(100) NOT NULL, -- upload / query / compliance_check / etc
|
||||
resource VARCHAR(100),
|
||||
resource_id UUID,
|
||||
user_id VARCHAR(255),
|
||||
ip_address INET,
|
||||
request JSONB,
|
||||
response JSONB,
|
||||
duration_ms INTEGER,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_action ON audit_logs(action);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_created ON audit_logs(created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_user ON audit_logs(user_id);
|
||||
|
||||
-- ══════════════════════════════════════════════════
|
||||
-- 更新时间自动维护
|
||||
-- ══════════════════════════════════════════════════
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER update_workspaces_updated_at
|
||||
BEFORE UPDATE ON workspaces
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
CREATE TRIGGER update_files_updated_at
|
||||
BEFORE UPDATE ON files
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
CREATE TRIGGER update_tasks_updated_at
|
||||
BEFORE UPDATE ON tasks
|
||||
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- ══════════════════════════════════════════════════
|
||||
-- 初始数据：预置监控源
|
||||
-- ══════════════════════════════════════════════════
|
||||
INSERT INTO regulation_sources (name, url, domain, fetch_interval) VALUES
|
||||
('国家标准全文公开系统', 'https://std.samr.gov.cn', 'vehicle_safety', 86400),
|
||||
('工信部政策法规', 'https://www.miit.gov.cn/jgsj/fgs/zcfg/index.html', 'vehicle_safety', 86400),
|
||||
('应急管理部政策法规', 'https://www.mem.gov.cn/gk/zcfg/', 'ehs', 86400),
|
||||
('生态环境部政策法规', 'https://www.mee.gov.cn/ywgz/fgbz/fl/', 'carbon', 86400)
|
||||
ON CONFLICT DO NOTHING;
|
||||
117
scripts/00_install_docker_ubuntu.sh
Normal file
117
scripts/00_install_docker_ubuntu.sh
Normal file
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env bash
|
||||
# ══════════════════════════════════════════════════
|
||||
# 00_install_docker_ubuntu.sh
|
||||
# Ubuntu 22.04 LTS 安装 Docker CE + nvidia-container-toolkit
|
||||
# 用法：bash scripts/00_install_docker_ubuntu.sh
|
||||
# ══════════════════════════════════════════════════
|
||||
set -euo pipefail
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
|
||||
|
||||
# ── 检查 root 权限 ──────────────────────────────
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
error "请以 root 或 sudo 运行：sudo bash scripts/00_install_docker_ubuntu.sh"
|
||||
fi
|
||||
|
||||
# ── 检测 Ubuntu 版本 ────────────────────────────
|
||||
. /etc/os-release
|
||||
info "检测到 OS：$NAME $VERSION_ID"
|
||||
if [[ "$ID" != "ubuntu" ]]; then
|
||||
warn "非 Ubuntu 系统，脚本可能不适用。继续（y/n）？"
|
||||
read -r ans; [[ "$ans" != "y" ]] && exit 0
|
||||
fi
|
||||
|
||||
# ── Step 1：换国内源（可选）──────────────────────
|
||||
info "Step 1/5：配置 APT 源..."
|
||||
if [[ "${USE_MIRROR:-false}" == "true" ]]; then
|
||||
sed -i 's/archive.ubuntu.com/mirrors.aliyun.com/g' /etc/apt/sources.list
|
||||
sed -i 's/security.ubuntu.com/mirrors.aliyun.com/g' /etc/apt/sources.list
|
||||
ok "已切换到阿里云镜像"
|
||||
fi
|
||||
apt-get update -qq
|
||||
|
||||
# ── Step 2：安装依赖 ────────────────────────────
|
||||
info "Step 2/5：安装依赖包..."
|
||||
apt-get install -y -qq \
|
||||
ca-certificates \
|
||||
curl \
|
||||
gnupg \
|
||||
lsb-release \
|
||||
apt-transport-https
|
||||
|
||||
# ── Step 3：安装 Docker CE ──────────────────────
|
||||
info "Step 3/5：安装 Docker CE..."
|
||||
if command -v docker &>/dev/null; then
|
||||
DOCKER_VER=$(docker --version)
|
||||
warn "Docker 已安装：$DOCKER_VER"
|
||||
warn "跳过 Docker 安装。如需重装，请先运行：apt-get remove docker docker-engine docker.io containerd"
|
||||
else
|
||||
# 添加 Docker 官方 GPG 密钥
|
||||
install -m 0755 -d /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \
|
||||
gpg --dearmor -o /etc/apt/keyrings/docker.gpg
|
||||
chmod a+r /etc/apt/keyrings/docker.gpg
|
||||
|
||||
# 添加 Docker 仓库
|
||||
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \
|
||||
https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | \
|
||||
tee /etc/apt/sources.list.d/docker.list > /dev/null
|
||||
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
|
||||
|
||||
# 启动并设置开机自启
|
||||
systemctl enable docker
|
||||
systemctl start docker
|
||||
ok "Docker CE 安装完成"
|
||||
fi
|
||||
|
||||
# 验证
|
||||
docker --version
|
||||
docker compose version
|
||||
|
||||
# ── Step 4：将当前用户加入 docker 组 ────────────
|
||||
info "Step 4/5：配置 Docker 用户组..."
|
||||
CURRENT_USER=${SUDO_USER:-$USER}
|
||||
if [[ -n "$CURRENT_USER" && "$CURRENT_USER" != "root" ]]; then
|
||||
usermod -aG docker "$CURRENT_USER"
|
||||
ok "用户 $CURRENT_USER 已加入 docker 组（重新登录后生效）"
|
||||
fi
|
||||
|
||||
# ── Step 5：安装 nvidia-container-toolkit（可选）─
|
||||
info "Step 5/5：检查 NVIDIA GPU..."
|
||||
if command -v nvidia-smi &>/dev/null; then
|
||||
info "检测到 NVIDIA GPU，安装 nvidia-container-toolkit..."
|
||||
nvidia-smi --query-gpu=name --format=csv,noheader
|
||||
|
||||
# 添加 NVIDIA 仓库
|
||||
curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \
|
||||
gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
|
||||
curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
|
||||
sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
|
||||
tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
|
||||
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq nvidia-container-toolkit
|
||||
nvidia-ctk runtime configure --runtime=docker
|
||||
systemctl restart docker
|
||||
ok "nvidia-container-toolkit 安装完成"
|
||||
else
|
||||
warn "未检测到 NVIDIA GPU，跳过 nvidia-container-toolkit 安装"
|
||||
warn "如有 GPU 请手动安装驱动后重新运行本脚本"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo -e "${GREEN} Docker 安装完成！${NC}"
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
echo " Docker 版本：$(docker --version)"
|
||||
echo " Compose 版本：$(docker compose version)"
|
||||
echo ""
|
||||
echo -e "${YELLOW} 注意：${NC}请重新登录以使 docker 组权限生效"
|
||||
echo " 验证命令：docker run hello-world"
|
||||
105
scripts/00_install_docker_windows.ps1
Normal file
105
scripts/00_install_docker_windows.ps1
Normal file
@@ -0,0 +1,105 @@
|
||||
# ══════════════════════════════════════════════════
|
||||
# 00_install_docker_windows.ps1
|
||||
# Windows 11 安装 Docker Desktop + WSL2 配置
|
||||
# 用法：以管理员身份运行 PowerShell，执行：
|
||||
# .\scripts\00_install_docker_windows.ps1
|
||||
# ══════════════════════════════════════════════════
|
||||
#Requires -RunAsAdministrator
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Write-Info { Write-Host "[INFO] $args" -ForegroundColor Cyan }
|
||||
function Write-Ok { Write-Host "[OK] $args" -ForegroundColor Green }
|
||||
function Write-Warn { Write-Host "[WARN] $args" -ForegroundColor Yellow }
|
||||
function Write-Err { Write-Host "[ERR] $args" -ForegroundColor Red; exit 1 }
|
||||
|
||||
Write-Info "============================================"
|
||||
Write-Info "AI合规智能中枢 — Windows Docker 环境安装"
|
||||
Write-Info "============================================"
|
||||
|
||||
# ── Step 1：启用 WSL2 ──────────────────────────
|
||||
Write-Info "Step 1/4：检查并启用 WSL2..."
|
||||
$wslFeature = Get-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux
|
||||
$vmFeature = Get-WindowsOptionalFeature -Online -FeatureName VirtualMachinePlatform
|
||||
|
||||
if ($wslFeature.State -ne "Enabled") {
|
||||
Write-Info "启用 WSL 功能..."
|
||||
Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Windows-Subsystem-Linux -NoRestart
|
||||
}
|
||||
if ($vmFeature.State -ne "Enabled") {
|
||||
Write-Info "启用虚拟机平台..."
|
||||
Enable-WindowsOptionalFeature -Online -FeatureName VirtualMachinePlatform -NoRestart
|
||||
}
|
||||
|
||||
# 更新 WSL 内核
|
||||
Write-Info "更新 WSL2 内核..."
|
||||
wsl --update
|
||||
wsl --set-default-version 2
|
||||
Write-Ok "WSL2 配置完成"
|
||||
|
||||
# ── Step 2：安装 Ubuntu WSL 发行版 ─────────────
|
||||
Write-Info "Step 2/4：检查 Ubuntu WSL..."
|
||||
$wslList = wsl --list --quiet 2>$null
|
||||
if ($wslList -notmatch "Ubuntu") {
|
||||
Write-Info "安装 Ubuntu 22.04..."
|
||||
wsl --install -d Ubuntu-22.04
|
||||
Write-Ok "Ubuntu 22.04 安装完成（首次运行需要设置用户名和密码）"
|
||||
} else {
|
||||
Write-Ok "Ubuntu WSL 已安装"
|
||||
wsl --list --verbose
|
||||
}
|
||||
|
||||
# ── Step 3：安装 Docker Desktop ────────────────
|
||||
Write-Info "Step 3/4：检查 Docker Desktop..."
|
||||
$dockerCmd = Get-Command docker -ErrorAction SilentlyContinue
|
||||
if ($dockerCmd) {
|
||||
Write-Ok "Docker 已安装：$(docker --version)"
|
||||
} else {
|
||||
# 尝试用 winget 安装
|
||||
$winget = Get-Command winget -ErrorAction SilentlyContinue
|
||||
if ($winget) {
|
||||
Write-Info "通过 winget 安装 Docker Desktop..."
|
||||
winget install -e --id Docker.DockerDesktop --accept-package-agreements --accept-source-agreements
|
||||
Write-Ok "Docker Desktop 安装完成"
|
||||
} else {
|
||||
Write-Warn "未找到 winget，请手动安装 Docker Desktop："
|
||||
Write-Warn "下载地址：https://www.docker.com/products/docker-desktop/"
|
||||
Write-Warn "安装时勾选：Use WSL 2 instead of Hyper-V"
|
||||
Start-Process "https://www.docker.com/products/docker-desktop/"
|
||||
Read-Host "安装完成后按 Enter 继续"
|
||||
}
|
||||
}
|
||||
|
||||
# ── Step 4：配置 Docker Desktop WSL 集成 ───────
|
||||
Write-Info "Step 4/4：提示 Docker Desktop 配置..."
|
||||
Write-Warn ""
|
||||
Write-Warn "请确认 Docker Desktop 已进行以下配置："
|
||||
Write-Warn " 1. Settings → General → 勾选 'Use WSL 2 based engine'"
|
||||
Write-Warn " 2. Settings → Resources → WSL Integration → 开启 Ubuntu-22.04"
|
||||
Write-Warn " 3. 如有 NVIDIA GPU："
|
||||
Write-Warn " Settings → General → 勾选 'Use GPU with WSL 2'"
|
||||
Write-Warn ""
|
||||
|
||||
# ── 验证 ───────────────────────────────────────
|
||||
Write-Info "验证安装..."
|
||||
try {
|
||||
$dockerVer = docker --version
|
||||
$composeVer = docker compose version
|
||||
Write-Ok "Docker: $dockerVer"
|
||||
Write-Ok "Compose: $composeVer"
|
||||
} catch {
|
||||
Write-Warn "Docker 命令不可用，可能需要重启后再验证"
|
||||
Write-Warn "重启后运行：docker run hello-world"
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "============================================" -ForegroundColor Green
|
||||
Write-Host " 安装完成！" -ForegroundColor Green
|
||||
Write-Host "============================================" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
Write-Host "后续步骤（在 WSL2 Ubuntu 中执行）：" -ForegroundColor Yellow
|
||||
Write-Host " 1. 打开 Ubuntu WSL 终端"
|
||||
Write-Host " 2. cd /mnt/c/Projects/AIProjects/AIRegulations/Depolyment"
|
||||
Write-Host " 3. bash scripts/01_setup_project.sh"
|
||||
Write-Host ""
|
||||
Write-Host "如需重启系统请现在重启，然后继续操作。" -ForegroundColor Yellow
|
||||
73
scripts/01_setup_project.sh
Normal file
73
scripts/01_setup_project.sh
Normal file
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
# ══════════════════════════════════════════════════
|
||||
# 01_setup_project.sh
|
||||
# 初始化项目：创建目录、生成 .env 文件
|
||||
# 用法：bash scripts/01_setup_project.sh
|
||||
# ══════════════════════════════════════════════════
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
info "项目目录：$PROJECT_DIR"
|
||||
|
||||
# ── 创建运行时目录 ──────────────────────────────
|
||||
info "创建运行时目录..."
|
||||
mkdir -p data/uploads data/parsed logs models
|
||||
mkdir -p services/embedding services/mcp-server
|
||||
mkdir -p services/compliance-backend/app/{core,api,services,models}
|
||||
ok "目录结构创建完成"
|
||||
|
||||
# ── 复制 .env 文件 ──────────────────────────────
|
||||
if [[ ! -f ".env" ]]; then
|
||||
cp .env.example .env
|
||||
warn "已创建 .env 文件，请编辑并填写必要配置："
|
||||
warn " 必填：DEEPSEEK_API_KEY（或 DASHSCOPE_API_KEY）"
|
||||
warn " 可选：修改各组件密码"
|
||||
echo ""
|
||||
echo -e "${YELLOW}是否现在编辑 .env 文件？(y/n)${NC}"
|
||||
read -r ans
|
||||
if [[ "$ans" == "y" ]]; then
|
||||
${EDITOR:-nano} .env
|
||||
fi
|
||||
else
|
||||
ok ".env 文件已存在，跳过复制"
|
||||
fi
|
||||
|
||||
# ── 验证 .env 关键字段 ──────────────────────────
|
||||
info "验证 .env 配置..."
|
||||
source .env 2>/dev/null || true
|
||||
|
||||
if [[ -z "${DEEPSEEK_API_KEY:-}" && -z "${DASHSCOPE_API_KEY:-}" ]]; then
|
||||
warn "⚠️ 未设置 LLM API Key！"
|
||||
warn " 请在 .env 中设置 DEEPSEEK_API_KEY 或 DASHSCOPE_API_KEY"
|
||||
warn " DeepSeek 申请：https://platform.deepseek.com"
|
||||
else
|
||||
ok "LLM API Key 已配置"
|
||||
fi
|
||||
|
||||
# ── 验证 Docker ─────────────────────────────────
|
||||
info "检查 Docker 环境..."
|
||||
if ! command -v docker &>/dev/null; then
|
||||
warn "Docker 未安装，请先运行：bash scripts/00_install_docker_ubuntu.sh"
|
||||
exit 1
|
||||
fi
|
||||
docker compose version > /dev/null
|
||||
ok "Docker Compose 可用：$(docker compose version)"
|
||||
|
||||
# ── 显示下一步 ──────────────────────────────────
|
||||
echo ""
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo -e "${GREEN} 项目初始化完成！${NC}"
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
echo "下一步操作："
|
||||
echo " 1. 拉取镜像（可选，较慢）：bash scripts/02_pull_images.sh"
|
||||
echo " 2. 启动全部服务： bash scripts/06_start_all.sh"
|
||||
echo " 3. 检查健康状态： bash scripts/check_health.sh"
|
||||
46
scripts/02_pull_images.sh
Normal file
46
scripts/02_pull_images.sh
Normal file
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# ══════════════════════════════════════════════════
|
||||
# 02_pull_images.sh
|
||||
# 预拉取所有 Docker 镜像（离线/弱网环境准备）
|
||||
# 用法：bash scripts/02_pull_images.sh
|
||||
# ══════════════════════════════════════════════════
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
BLUE='\033[0;34m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
|
||||
# 所有基础镜像列表
|
||||
IMAGES=(
|
||||
"pgvector/pgvector:pg16"
|
||||
"redis:7-alpine"
|
||||
"quay.io/coreos/etcd:v3.5.5"
|
||||
"minio/minio:RELEASE.2023-03-13T19-46-17Z"
|
||||
"milvusdb/milvus:v2.4.13"
|
||||
"neo4j:5.20-community"
|
||||
"nginx:1.25-alpine"
|
||||
"grafana/grafana:11.0.0"
|
||||
"prom/prometheus:v2.51.0"
|
||||
)
|
||||
|
||||
info "开始拉取 ${#IMAGES[@]} 个基础镜像..."
|
||||
echo ""
|
||||
|
||||
for img in "${IMAGES[@]}"; do
|
||||
info "拉取：$img"
|
||||
docker pull "$img"
|
||||
ok "完成：$img"
|
||||
echo ""
|
||||
done
|
||||
|
||||
info "所有基础镜像拉取完成"
|
||||
echo ""
|
||||
info "自定义服务镜像（embedding/mcp/backend）将在 build 时自动拉取基础层"
|
||||
echo ""
|
||||
echo -e "${YELLOW}提示：如在国内网络环境下 quay.io 或 milvusdb 拉取慢，${NC}"
|
||||
echo -e "${YELLOW}可配置 Docker 镜像加速器：/etc/docker/daemon.json${NC}"
|
||||
echo ' {"registry-mirrors": ["https://docker.mirrors.ustc.edu.cn"]}'
|
||||
93
scripts/03_start_infra.sh
Normal file
93
scripts/03_start_infra.sh
Normal file
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env bash
|
||||
# ══════════════════════════════════════════════════
|
||||
# 03_start_infra.sh
|
||||
# 分步启动基础设施（含健康等待），顺序：
|
||||
# PostgreSQL + Redis → etcd + MinIO → Milvus → Neo4j
|
||||
# 用法：bash scripts/03_start_infra.sh
|
||||
# ══════════════════════════════════════════════════
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
|
||||
|
||||
# 等待服务健康的函数
|
||||
wait_healthy() {
|
||||
local service=$1
|
||||
local max_wait=${2:-120}
|
||||
local interval=5
|
||||
local elapsed=0
|
||||
|
||||
info "等待 $service 健康就绪..."
|
||||
while [[ $elapsed -lt $max_wait ]]; do
|
||||
local status
|
||||
status=$(docker compose ps --format json "$service" 2>/dev/null | \
|
||||
python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('Health','unknown'))" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [[ "$status" == "healthy" ]]; then
|
||||
ok "$service 已就绪"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo -n "."
|
||||
sleep $interval
|
||||
elapsed=$((elapsed + interval))
|
||||
done
|
||||
echo ""
|
||||
error "$service 等待超时（${max_wait}s），请检查：docker compose logs $service"
|
||||
}
|
||||
|
||||
info "══════════════════════════════════════════"
|
||||
info " 启动基础设施层"
|
||||
info "══════════════════════════════════════════"
|
||||
|
||||
# ── Step 1：PostgreSQL + Redis ──────────────────
|
||||
info "Step 1/4：启动 PostgreSQL 和 Redis..."
|
||||
docker compose up -d postgres redis
|
||||
|
||||
wait_healthy postgres 90
|
||||
wait_healthy redis 30
|
||||
ok "数据层就绪"
|
||||
|
||||
# ── Step 2：etcd + MinIO（Milvus 依赖）─────────
|
||||
info "Step 2/4：启动 etcd 和 MinIO（Milvus 依赖）..."
|
||||
docker compose up -d etcd minio
|
||||
|
||||
wait_healthy etcd 60
|
||||
wait_healthy minio 60
|
||||
ok "对象存储层就绪"
|
||||
|
||||
# ── Step 3：Milvus ──────────────────────────────
|
||||
info "Step 3/4：启动 Milvus（向量数据库）..."
|
||||
docker compose up -d milvus
|
||||
|
||||
info "Milvus 初始化需要约 60 秒，请耐心等待..."
|
||||
wait_healthy milvus 180
|
||||
ok "Milvus 就绪"
|
||||
|
||||
# ── Step 4：Neo4j ───────────────────────────────
|
||||
info "Step 4/4：启动 Neo4j（知识图谱）..."
|
||||
docker compose up -d neo4j
|
||||
|
||||
wait_healthy neo4j 120
|
||||
ok "Neo4j 就绪"
|
||||
|
||||
# ── 汇总 ────────────────────────────────────────
|
||||
echo ""
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo -e "${GREEN} 基础设施启动完成！${NC}"
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
echo " PostgreSQL : localhost:5432"
|
||||
echo " Redis : localhost:6379"
|
||||
echo " Milvus : localhost:19530 (gRPC), localhost:9091 (HTTP)"
|
||||
echo " Neo4j : localhost:7474 (Browser), localhost:7687 (Bolt)"
|
||||
echo " MinIO 控制台: localhost:9001 (admin/minioadmin)"
|
||||
echo ""
|
||||
echo "下一步：bash scripts/04_build_services.sh"
|
||||
59
scripts/04_build_services.sh
Normal file
59
scripts/04_build_services.sh
Normal file
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
# ══════════════════════════════════════════════════
|
||||
# 04_build_services.sh
|
||||
# 构建自定义服务 Docker 镜像
|
||||
# embedding-service / mcp-server / compliance-backend
|
||||
# 用法：bash scripts/04_build_services.sh
|
||||
# ══════════════════════════════════════════════════
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
BLUE='\033[0;34m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
|
||||
info "══════════════════════════════════════════"
|
||||
info " 构建自定义服务镜像"
|
||||
info "══════════════════════════════════════════"
|
||||
warn "首次构建较慢（需下载 Python 依赖 + AI 模型）"
|
||||
warn "BGE-M3 模型约 2.5GB，MinerU 模型约 2GB"
|
||||
echo ""
|
||||
|
||||
# ── 构建嵌入服务 ────────────────────────────────
|
||||
info "构建 embedding-service（BGE-M3）..."
|
||||
START=$(date +%s)
|
||||
docker compose build embedding-service
|
||||
END=$(date +%s)
|
||||
ok "embedding-service 构建完成（$(( END - START ))s）"
|
||||
echo ""
|
||||
|
||||
# ── 构建 MinerU 解析服务 ────────────────────────
|
||||
info "构建 mcp-server（MinerU）..."
|
||||
START=$(date +%s)
|
||||
docker compose build mcp-server
|
||||
END=$(date +%s)
|
||||
ok "mcp-server 构建完成（$(( END - START ))s）"
|
||||
echo ""
|
||||
|
||||
# ── 构建业务后端 ────────────────────────────────
|
||||
info "构建 compliance-backend..."
|
||||
START=$(date +%s)
|
||||
docker compose build compliance-backend
|
||||
END=$(date +%s)
|
||||
ok "compliance-backend 构建完成（$(( END - START ))s）"
|
||||
echo ""
|
||||
|
||||
# ── 列出构建的镜像 ──────────────────────────────
|
||||
info "已构建的镜像："
|
||||
docker images | grep -E "compliance-(embedding|mcp|backend)" || true
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo -e "${GREEN} 所有服务镜像构建完成！${NC}"
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
echo "下一步：bash scripts/05_init_db.sh"
|
||||
124
scripts/05_init_db.sh
Normal file
124
scripts/05_init_db.sh
Normal file
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env bash
|
||||
# ══════════════════════════════════════════════════
|
||||
# 05_init_db.sh
|
||||
# 初始化数据库：PostgreSQL Schema + Milvus Collections + Neo4j Constraints
|
||||
# 用法：bash scripts/05_init_db.sh
|
||||
# 前提：postgres / milvus / neo4j 已运行且健康
|
||||
# ══════════════════════════════════════════════════
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
|
||||
|
||||
source .env 2>/dev/null || true
|
||||
POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-compliance123}
|
||||
NEO4J_PASSWORD=${NEO4J_PASSWORD:-neo4j123}
|
||||
|
||||
# ── Step 1：PostgreSQL Schema ───────────────────
|
||||
info "Step 1/3：初始化 PostgreSQL Schema..."
|
||||
if docker compose ps postgres | grep -q "healthy"; then
|
||||
docker compose exec -T postgres psql \
|
||||
-U compliance -d compliance_db \
|
||||
-f /docker-entrypoint-initdb.d/01_init_schema.sql \
|
||||
2>&1 | tail -5 || warn "SQL 可能部分已存在（IF NOT EXISTS），这是正常的"
|
||||
ok "PostgreSQL Schema 初始化完成"
|
||||
else
|
||||
error "PostgreSQL 未运行，请先执行：bash scripts/03_start_infra.sh"
|
||||
fi
|
||||
|
||||
# ── Step 2：Milvus Collections ──────────────────
|
||||
info "Step 2/3：初始化 Milvus Collections..."
|
||||
if docker compose ps milvus | grep -q "healthy"; then
|
||||
docker compose run --rm --no-deps compliance-backend \
|
||||
python3 -c "
|
||||
import asyncio
|
||||
from pymilvus import connections, Collection, CollectionSchema, FieldSchema, DataType, utility
|
||||
|
||||
connections.connect(host='milvus', port='19530')
|
||||
print('Milvus 连接成功')
|
||||
|
||||
def create_collection(name, description):
|
||||
if utility.has_collection(name):
|
||||
print(f' Collection {name} 已存在，跳过')
|
||||
return
|
||||
|
||||
fields = [
|
||||
FieldSchema(name='id', dtype=DataType.VARCHAR, is_primary=True, max_length=128),
|
||||
FieldSchema(name='file_id', dtype=DataType.VARCHAR, max_length=128),
|
||||
FieldSchema(name='workspace_id', dtype=DataType.VARCHAR, max_length=128),
|
||||
FieldSchema(name='chunk_idx', dtype=DataType.INT64),
|
||||
FieldSchema(name='content', dtype=DataType.VARCHAR, max_length=65535),
|
||||
FieldSchema(name='dense_vec', dtype=DataType.FLOAT_VECTOR, dim=1024), # BGE-M3 dense
|
||||
FieldSchema(name='metadata', dtype=DataType.JSON),
|
||||
]
|
||||
schema = CollectionSchema(fields, description=description)
|
||||
col = Collection(name, schema)
|
||||
|
||||
# 创建向量索引（HNSW，适合调研阶段）
|
||||
index_params = {
|
||||
'metric_type': 'COSINE',
|
||||
'index_type': 'HNSW',
|
||||
'params': {'M': 16, 'efConstruction': 200}
|
||||
}
|
||||
col.create_index('dense_vec', index_params)
|
||||
col.load()
|
||||
print(f' Collection {name} 创建完成')
|
||||
|
||||
create_collection('regulation_chunks', '法规条款向量库')
|
||||
create_collection('doc_chunks', '企业文档向量库')
|
||||
create_collection('case_library', '行业案例库')
|
||||
|
||||
print('Milvus 初始化完成')
|
||||
" 2>&1
|
||||
ok "Milvus Collections 初始化完成"
|
||||
else
|
||||
error "Milvus 未运行，请先执行：bash scripts/03_start_infra.sh"
|
||||
fi
|
||||
|
||||
# ── Step 3：Neo4j 约束和索引 ────────────────────
|
||||
info "Step 3/3：初始化 Neo4j 约束和索引..."
|
||||
sleep 5 # Neo4j 可能还在预热
|
||||
|
||||
docker compose exec -T neo4j cypher-shell \
|
||||
-u neo4j -p "$NEO4J_PASSWORD" \
|
||||
--format plain <<'CYPHER'
|
||||
// 节点约束（唯一性）
|
||||
CREATE CONSTRAINT regulation_id IF NOT EXISTS
|
||||
FOR (r:Regulation) REQUIRE r.id IS UNIQUE;
|
||||
CREATE CONSTRAINT clause_id IF NOT EXISTS
|
||||
FOR (c:Clause) REQUIRE c.id IS UNIQUE;
|
||||
CREATE CONSTRAINT obligation_id IF NOT EXISTS
|
||||
FOR (o:Obligation) REQUIRE o.id IS UNIQUE;
|
||||
|
||||
// 全文索引（模糊查询）
|
||||
CREATE FULLTEXT INDEX regulation_fulltext IF NOT EXISTS
|
||||
FOR (r:Regulation) ON EACH [r.title, r.code, r.domain];
|
||||
CREATE FULLTEXT INDEX clause_fulltext IF NOT EXISTS
|
||||
FOR (c:Clause) ON EACH [c.content, c.title];
|
||||
|
||||
// 插入示例节点（验证连通性）
|
||||
MERGE (d:Domain {name: 'vehicle_safety', label: '车辆安全法规'});
|
||||
MERGE (d:Domain {name: 'data_security', label: '数据安全法规'});
|
||||
MERGE (d:Domain {name: 'ehs', label: 'EHS安全法规'});
|
||||
MERGE (d:Domain {name: 'carbon', label: '碳排放法规'});
|
||||
RETURN '初始化完成' AS result;
|
||||
CYPHER
|
||||
ok "Neo4j 约束和索引初始化完成"
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo -e "${GREEN} 数据库初始化完成！${NC}"
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
echo " PostgreSQL: 所有表已创建"
|
||||
echo " Milvus: regulation_chunks / doc_chunks / case_library"
|
||||
echo " Neo4j: 约束 + 全文索引 + 基础域节点"
|
||||
echo ""
|
||||
echo "下一步：bash scripts/06_start_all.sh"
|
||||
98
scripts/06_start_all.sh
Normal file
98
scripts/06_start_all.sh
Normal file
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env bash
|
||||
# ══════════════════════════════════════════════════
|
||||
# 06_start_all.sh
|
||||
# 一键启动所有服务（完整流程）
|
||||
# 用法：bash scripts/06_start_all.sh
|
||||
# ══════════════════════════════════════════════════
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
|
||||
|
||||
echo ""
|
||||
echo -e "${BLUE}╔══════════════════════════════════════════╗${NC}"
|
||||
echo -e "${BLUE}║ AI合规智能中枢 — 全服务启动 ║${NC}"
|
||||
echo -e "${BLUE}╚══════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
|
||||
# ── 前置检查 ────────────────────────────────────
|
||||
if [[ ! -f ".env" ]]; then
|
||||
error ".env 文件不存在，请先运行：bash scripts/01_setup_project.sh"
|
||||
fi
|
||||
|
||||
source .env 2>/dev/null || true
|
||||
if [[ -z "${DEEPSEEK_API_KEY:-}" && -z "${DASHSCOPE_API_KEY:-}" ]]; then
|
||||
warn "⚠️ 未设置 LLM API Key，LLM 功能将不可用"
|
||||
warn "请在 .env 中设置 DEEPSEEK_API_KEY 或 DASHSCOPE_API_KEY"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# ── Phase 1：基础设施 ────────────────────────────
|
||||
info "Phase 1/4：启动基础设施..."
|
||||
bash "$SCRIPT_DIR/03_start_infra.sh"
|
||||
echo ""
|
||||
|
||||
# ── Phase 2：构建服务镜像 ────────────────────────
|
||||
info "Phase 2/4：构建服务镜像（首次较慢）..."
|
||||
docker compose build embedding-service mcp-server compliance-backend 2>&1 | \
|
||||
grep -E "(Step|Successfully|=>|ERROR)" || true
|
||||
ok "镜像构建完成"
|
||||
echo ""
|
||||
|
||||
# ── Phase 3：初始化数据库 ────────────────────────
|
||||
info "Phase 3/4：初始化数据库..."
|
||||
bash "$SCRIPT_DIR/05_init_db.sh"
|
||||
echo ""
|
||||
|
||||
# ── Phase 4：启动所有服务 ────────────────────────
|
||||
info "Phase 4/4：启动 AI 模型服务和业务服务..."
|
||||
docker compose up -d embedding-service mcp-server
|
||||
info "等待 AI 模型加载（BGE-M3/MinerU 约需 2-3 分钟）..."
|
||||
sleep 30
|
||||
|
||||
# 等待嵌入服务就绪
|
||||
for i in {1..20}; do
|
||||
if curl -sf http://localhost:8010/health > /dev/null 2>&1; then
|
||||
ok "embedding-service 就绪"
|
||||
break
|
||||
fi
|
||||
echo -n "."
|
||||
sleep 10
|
||||
done
|
||||
|
||||
docker compose up -d compliance-backend celery-worker celery-beat nginx
|
||||
info "等待业务服务启动..."
|
||||
sleep 15
|
||||
|
||||
for i in {1..12}; do
|
||||
if curl -sf http://localhost:8000/health > /dev/null 2>&1; then
|
||||
ok "compliance-backend 就绪"
|
||||
break
|
||||
fi
|
||||
echo -n "."
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# ── 最终状态 ────────────────────────────────────
|
||||
echo ""
|
||||
echo -e "${GREEN}╔══════════════════════════════════════════╗${NC}"
|
||||
echo -e "${GREEN}║ 所有服务启动完成！ ║${NC}"
|
||||
echo -e "${GREEN}╚══════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
docker compose ps --format "table {{.Service}}\t{{.Status}}\t{{.Ports}}"
|
||||
echo ""
|
||||
echo -e "${BLUE}访问地址：${NC}"
|
||||
echo " API 网关 : http://localhost"
|
||||
echo " API 文档 : http://localhost/docs"
|
||||
echo " Neo4j 浏览器 : http://localhost:7474"
|
||||
echo " MinIO 控制台 : http://localhost:9001"
|
||||
echo ""
|
||||
echo -e "${YELLOW}运行冒烟测试：${NC}"
|
||||
echo " bash scripts/07_smoke_test.sh"
|
||||
183
scripts/07_smoke_test.sh
Normal file
183
scripts/07_smoke_test.sh
Normal file
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env bash
|
||||
# ══════════════════════════════════════════════════
|
||||
# 07_smoke_test.sh
|
||||
# 端到端冒烟测试：验证三条业务闭环
|
||||
# 用法：bash scripts/07_smoke_test.sh
|
||||
# ══════════════════════════════════════════════════
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[✓]${NC} $*"; }
|
||||
fail() { echo -e "${RED}[✗]${NC} $*"; FAILED=$((FAILED+1)); }
|
||||
warn() { echo -e "${YELLOW}[~]${NC} $*"; }
|
||||
|
||||
FAILED=0
|
||||
API_BASE="http://localhost"
|
||||
|
||||
echo ""
|
||||
echo -e "${BLUE}══════════════════════════════════════════${NC}"
|
||||
echo -e "${BLUE} AI合规智能中枢 端到端冒烟测试${NC}"
|
||||
echo -e "${BLUE}══════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
|
||||
# ── 基础健康检查 ────────────────────────────────
|
||||
info "=== 基础设施健康检查 ==="
|
||||
|
||||
check_service() {
|
||||
local name=$1; local url=$2
|
||||
if curl -sf "$url" > /dev/null 2>&1; then
|
||||
ok "$name"
|
||||
else
|
||||
fail "$name（$url 不可达）"
|
||||
fi
|
||||
}
|
||||
|
||||
check_service "API 网关 (Nginx)" "http://localhost/health"
|
||||
check_service "业务后端 (FastAPI)" "http://localhost:8000/health"
|
||||
check_service "嵌入服务 (BGE-M3)" "http://localhost:8010/health"
|
||||
check_service "解析服务 (MinerU)" "http://localhost:8011/health"
|
||||
check_service "Milvus HTTP" "http://localhost:9091/healthz"
|
||||
check_service "Neo4j Browser" "http://localhost:7474"
|
||||
echo ""
|
||||
|
||||
# ── 嵌入服务测试 ────────────────────────────────
|
||||
info "=== 嵌入服务测试 ==="
|
||||
EMBED_RESP=$(curl -sf -X POST http://localhost:8010/embed \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"texts": ["GB 18384 电动汽车碰撞安全要求"], "batch_size": 1}' 2>/dev/null || echo "{}")
|
||||
|
||||
if echo "$EMBED_RESP" | python3 -c "import sys,json; d=json.load(sys.stdin); assert len(d.get('dense',[])[0])==1024" 2>/dev/null; then
|
||||
ok "BGE-M3 嵌入：返回 1024 维向量"
|
||||
else
|
||||
fail "BGE-M3 嵌入失败，响应：${EMBED_RESP:0:200}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ── 创建测试 PDF ────────────────────────────────
|
||||
info "=== 创建测试文档 ==="
|
||||
TEST_PDF="$PROJECT_DIR/data/uploads/test_regulation.txt"
|
||||
cat > "$TEST_PDF" << 'EOF'
|
||||
GB 18384-2020 电动汽车安全要求
|
||||
|
||||
第一章 总则
|
||||
本标准规定了电动汽车的安全要求，适用于M1类纯电动汽车。
|
||||
|
||||
第二章 电气安全
|
||||
2.1 绝缘电阻要求
|
||||
直流电路绝缘电阻不得低于100Ω/V。
|
||||
2.2 碰撞安全
|
||||
车辆碰撞后，高压电系统应自动断电。
|
||||
碰撞后5秒内，高压系统电压应降至60V以下。
|
||||
|
||||
第三章 防水要求
|
||||
高压系统防护等级应达到IP67。
|
||||
EOF
|
||||
ok "测试文档创建：$TEST_PDF"
|
||||
echo ""
|
||||
|
||||
# ── 闭环①：文件上传 → 向量化 → 问答 ───────────
|
||||
info "=== 闭环①：法规入库 → 检索问答 ==="
|
||||
|
||||
# 创建工作空间
|
||||
WORKSPACE_RESP=$(curl -sf -X POST "$API_BASE/api/kb/workspaces" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "测试法规库", "domain": "vehicle_safety"}' 2>/dev/null || echo "{}")
|
||||
WS_ID=$(echo "$WORKSPACE_RESP" | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -n "$WS_ID" ]]; then
|
||||
ok "工作空间创建：$WS_ID"
|
||||
else
|
||||
warn "工作空间创建失败（可能接口未完全实现），跳过后续上传测试"
|
||||
WS_ID="test-workspace"
|
||||
fi
|
||||
|
||||
# 上传文件
|
||||
UPLOAD_RESP=$(curl -sf -X POST "$API_BASE/api/kb/files/upload" \
|
||||
-F "file=@$TEST_PDF" \
|
||||
-F "workspace_id=$WS_ID" 2>/dev/null || echo "{}")
|
||||
TASK_ID=$(echo "$UPLOAD_RESP" | python3 -c "import sys,json; print(json.load(sys.stdin).get('task_id',''))" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -n "$TASK_ID" ]]; then
|
||||
ok "文件上传任务已创建：$TASK_ID"
|
||||
|
||||
# 轮询任务状态（最多等待120秒）
|
||||
info "等待向量化完成..."
|
||||
for i in {1..24}; do
|
||||
TASK_STATUS=$(curl -sf "$API_BASE/api/kb/tasks/$TASK_ID" 2>/dev/null | \
|
||||
python3 -c "import sys,json; print(json.load(sys.stdin).get('status','unknown'))" 2>/dev/null || echo "unknown")
|
||||
if [[ "$TASK_STATUS" == "completed" ]]; then
|
||||
ok "向量化完成（${i}×5s）"
|
||||
break
|
||||
elif [[ "$TASK_STATUS" == "failed" ]]; then
|
||||
fail "向量化失败"
|
||||
break
|
||||
fi
|
||||
echo -n "."
|
||||
sleep 5
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 检索问答
|
||||
QA_RESP=$(curl -sf -X POST "$API_BASE/api/kb/qa" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"query\": \"碰撞后高压系统电压要求\", \"workspace_id\": \"$WS_ID\", \"top_k\": 3}" 2>/dev/null || echo "{}")
|
||||
ANSWER=$(echo "$QA_RESP" | python3 -c "import sys,json; print(json.load(sys.stdin).get('answer','')[:100])" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -n "$ANSWER" ]]; then
|
||||
ok "问答成功：${ANSWER}..."
|
||||
else
|
||||
warn "问答返回空（LLM API 可能未配置或响应缓慢）"
|
||||
fi
|
||||
else
|
||||
warn "文件上传失败（接口可能未实现）"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ── 闭环②：合规审查 ────────────────────────────
|
||||
info "=== 闭环②：文档上传 → 合规审查 ==="
|
||||
|
||||
CHECK_RESP=$(curl -sf -X POST "$API_BASE/api/compliance/check" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"query": "供应商文件是否符合GB 18384碰撞安全要求", "domains": ["vehicle_safety"]}' 2>/dev/null || echo "{}")
|
||||
RISK=$(echo "$CHECK_RESP" | python3 -c "import sys,json; print(json.load(sys.stdin).get('risk_level','unknown'))" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [[ "$RISK" != "unknown" && -n "$RISK" ]]; then
|
||||
ok "合规审查完成，风险等级：$RISK"
|
||||
else
|
||||
warn "合规审查接口返回空（功能可能未完全实现）"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ── 闭环③：法规监控 ────────────────────────────
|
||||
info "=== 闭环③：法规监控源配置 ==="
|
||||
|
||||
SOURCE_RESP=$(curl -sf -X POST "$API_BASE/api/regulation/sources" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "测试监控源", "url": "https://std.samr.gov.cn", "domain": "vehicle_safety"}' 2>/dev/null || echo "{}")
|
||||
SOURCE_ID=$(echo "$SOURCE_RESP" | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null || echo "")
|
||||
|
||||
if [[ -n "$SOURCE_ID" ]]; then
|
||||
ok "监控源配置成功：$SOURCE_ID"
|
||||
else
|
||||
warn "监控源配置返回空（功能可能未完全实现）"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ── 汇总 ────────────────────────────────────────
|
||||
echo ""
|
||||
echo -e "${BLUE}══════════════════════════════════════════${NC}"
|
||||
if [[ $FAILED -eq 0 ]]; then
|
||||
echo -e "${GREEN} 全部检查通过！${NC}"
|
||||
else
|
||||
echo -e "${YELLOW} 完成，${FAILED} 项失败${NC}（部分功能可能尚未实现）"
|
||||
fi
|
||||
echo -e "${BLUE}══════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
echo "查看服务日志："
|
||||
echo " docker compose logs -f compliance-backend"
|
||||
echo " docker compose logs -f celery-worker"
|
||||
66
scripts/check_health.sh
Normal file
66
scripts/check_health.sh
Normal file
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# ══════════════════════════════════════════════════
|
||||
# check_health.sh
|
||||
# 检查所有服务的健康状态和资源使用
|
||||
# 用法：bash scripts/check_health.sh
|
||||
# ══════════════════════════════════════════════════
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
|
||||
echo ""
|
||||
echo -e "${BLUE}══════════════════════════════════════════${NC}"
|
||||
echo -e "${BLUE} 服务健康检查报告${NC}"
|
||||
echo -e "${BLUE}══════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
|
||||
# Docker 服务状态
|
||||
echo -e "${BLUE}【Docker Compose 服务状态】${NC}"
|
||||
docker compose ps --format "table {{.Service}}\t{{.Status}}\t{{.Ports}}"
|
||||
echo ""
|
||||
|
||||
# HTTP 端点检查
|
||||
echo -e "${BLUE}【HTTP 健康端点】${NC}"
|
||||
check_http() {
|
||||
local name=$1; local url=$2
|
||||
if curl -sf --max-time 5 "$url" > /dev/null 2>&1; then
|
||||
echo -e " ${GREEN}[OK]${NC} $name ($url)"
|
||||
else
|
||||
echo -e " ${RED}[FAIL]${NC} $name ($url)"
|
||||
fi
|
||||
}
|
||||
|
||||
check_http "API 网关" "http://localhost/health"
|
||||
check_http "业务后端" "http://localhost:8000/health"
|
||||
check_http "嵌入服务" "http://localhost:8010/health"
|
||||
check_http "解析服务" "http://localhost:8011/health"
|
||||
check_http "Milvus" "http://localhost:9091/healthz"
|
||||
check_http "Neo4j" "http://localhost:7474"
|
||||
echo ""
|
||||
|
||||
# 资源使用
|
||||
echo -e "${BLUE}【容器资源使用】${NC}"
|
||||
docker stats --no-stream --format \
|
||||
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}" \
|
||||
2>/dev/null | head -15
|
||||
echo ""
|
||||
|
||||
# 磁盘使用
|
||||
echo -e "${BLUE}【磁盘使用】${NC}"
|
||||
df -h . | tail -1 | awk '{print " 项目目录：已用 "$3"，可用 "$4"（" $5 " 使用率）"}'
|
||||
docker system df 2>/dev/null | head -6
|
||||
echo ""
|
||||
|
||||
# LLM 配置检查
|
||||
echo -e "${BLUE}【LLM API 配置】${NC}"
|
||||
source .env 2>/dev/null || true
|
||||
if [[ -n "${DEEPSEEK_API_KEY:-}" ]]; then
|
||||
echo -e " ${GREEN}[OK]${NC} DeepSeek API Key 已配置"
|
||||
elif [[ -n "${DASHSCOPE_API_KEY:-}" ]]; then
|
||||
echo -e " ${GREEN}[OK]${NC} DashScope (Qwen) API Key 已配置"
|
||||
else
|
||||
echo -e " ${YELLOW}[WARN]${NC} 未配置 LLM API Key（LLM 功能不可用）"
|
||||
fi
|
||||
echo ""
|
||||
91
scripts/download_models.sh
Normal file
91
scripts/download_models.sh
Normal file
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
# ══════════════════════════════════════════════════
|
||||
# download_models.sh
|
||||
# 预下载 AI 模型到 ./models 目录（加速容器启动）
|
||||
# 支持 HuggingFace 镜像加速（国内网络）
|
||||
# 用法：bash scripts/download_models.sh
|
||||
# ══════════════════════════════════════════════════
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
BLUE='\033[0;34m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
|
||||
info() { echo -e "${BLUE}[INFO]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[OK]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
|
||||
MODELS_DIR="$PROJECT_DIR/models"
|
||||
mkdir -p "$MODELS_DIR"
|
||||
|
||||
# 设置镜像加速
|
||||
export HF_ENDPOINT="${HF_ENDPOINT:-https://hf-mirror.com}"
|
||||
export HF_HOME="$MODELS_DIR"
|
||||
info "HuggingFace 镜像：$HF_ENDPOINT"
|
||||
info "模型保存路径：$MODELS_DIR"
|
||||
echo ""
|
||||
|
||||
# ── 方法1：通过 huggingface_hub 下载 ────────────
|
||||
download_hf() {
|
||||
local repo=$1; local local_name=$2
|
||||
info "下载 $repo..."
|
||||
if python3 -c "
|
||||
import os
|
||||
os.environ['HF_ENDPOINT'] = '${HF_ENDPOINT}'
|
||||
os.environ['HF_HOME'] = '${MODELS_DIR}'
|
||||
from huggingface_hub import snapshot_download
|
||||
snapshot_download(repo_id='$repo', cache_dir='${MODELS_DIR}')
|
||||
print('下载完成')
|
||||
" 2>&1; then
|
||||
ok "$repo 下载成功"
|
||||
else
|
||||
warn "$repo HuggingFace 下载失败，尝试 ModelScope..."
|
||||
download_modelscope "$repo" "$local_name"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── 方法2：通过 ModelScope 下载（备用）──────────
|
||||
download_modelscope() {
|
||||
local hf_name=$1
|
||||
local ms_name=${2:-$1}
|
||||
python3 -c "
|
||||
try:
|
||||
from modelscope import snapshot_download
|
||||
snapshot_download(model_id='$ms_name', cache_dir='${MODELS_DIR}/modelscope')
|
||||
print('ModelScope 下载完成')
|
||||
except ImportError:
|
||||
print('ModelScope 未安装，跳过')
|
||||
except Exception as e:
|
||||
print(f'ModelScope 下载失败: {e}')
|
||||
" 2>&1 || warn "ModelScope 下载也失败，模型将在容器启动时自动下载"
|
||||
}
|
||||
|
||||
# ── 检查 Python 环境 ────────────────────────────
|
||||
if ! python3 -c "import huggingface_hub" 2>/dev/null; then
|
||||
warn "未安装 huggingface_hub，尝试安装..."
|
||||
pip3 install -q huggingface_hub modelscope 2>/dev/null || \
|
||||
warn "安装失败，模型将在容器首次启动时下载"
|
||||
fi
|
||||
|
||||
# ── 下载模型列表 ────────────────────────────────
|
||||
info "=== 下载 BGE-M3 嵌入模型（约 2.5GB）==="
|
||||
download_hf "BAAI/bge-m3" "BAAI/bge-m3"
|
||||
echo ""
|
||||
|
||||
info "=== 下载 BGE-Reranker 精排模型（约 1.1GB）==="
|
||||
download_hf "BAAI/bge-reranker-v2-m3" "BAAI/bge-reranker-v2-m3"
|
||||
echo ""
|
||||
|
||||
# MinerU 模型通过容器内脚本下载（依赖 magic-pdf 配置）
|
||||
info "=== MinerU 模型说明 ==="
|
||||
warn "MinerU 模型（约 2GB）将在 mcp-server 容器首次启动时自动下载"
|
||||
warn "如需预下载，请在 mcp-server 容器内运行：mineru-models-download"
|
||||
echo ""
|
||||
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo -e "${GREEN} 模型下载完成！${NC}"
|
||||
echo -e "${GREEN}══════════════════════════════════════════${NC}"
|
||||
echo ""
|
||||
echo "已下载到：$MODELS_DIR"
|
||||
du -sh "$MODELS_DIR" 2>/dev/null || true
|
||||
37
scripts/reset_all.sh
Normal file
37
scripts/reset_all.sh
Normal file
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# ══════════════════════════════════════════════════
|
||||
# reset_all.sh
|
||||
# ⚠️ 危险操作：停止所有服务并删除所有数据（慎用！）
|
||||
# 用法：bash scripts/reset_all.sh
|
||||
# ══════════════════════════════════════════════════
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
RED='\033[0;31m'; YELLOW='\033[1;33m'; NC='\033[0m'
|
||||
|
||||
echo ""
|
||||
echo -e "${RED}╔══════════════════════════════════════════╗${NC}"
|
||||
echo -e "${RED}║ ⚠️ 警告：此操作将删除所有数据！ ║${NC}"
|
||||
echo -e "${RED}║ 包括：PostgreSQL / Milvus / Neo4j 数据 ║${NC}"
|
||||
echo -e "${RED}║ 以及所有上传的文件和日志 ║${NC}"
|
||||
echo -e "${RED}╚══════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}确认要重置所有数据吗？（输入 'yes' 确认，其他取消）${NC}"
|
||||
read -r CONFIRM
|
||||
|
||||
if [[ "$CONFIRM" != "yes" ]]; then
|
||||
echo "已取消"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "停止所有服务..."
|
||||
docker compose down --volumes --remove-orphans
|
||||
|
||||
echo "清理数据目录..."
|
||||
rm -rf data/uploads/* data/parsed/* logs/*
|
||||
echo "✓ 数据目录已清空（保留目录结构）"
|
||||
|
||||
echo ""
|
||||
echo -e "${YELLOW}重置完成。重新启动：bash scripts/06_start_all.sh${NC}"
|
||||
24
services/compliance-backend/Dockerfile
Normal file
24
services/compliance-backend/Dockerfile
Normal file
@@ -0,0 +1,24 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 使用 uv 加速依赖安装
|
||||
RUN pip install uv --no-cache-dir
|
||||
|
||||
COPY pyproject.toml .
|
||||
RUN uv pip install --system --no-cache -r pyproject.toml \
|
||||
--index-url https://pypi.tuna.tsinghua.edu.cn/simple \
|
||||
--trusted-host pypi.tuna.tsinghua.edu.cn
|
||||
|
||||
COPY app/ ./app/
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=5 \
|
||||
CMD curl -f http://localhost:8000/health || exit 1
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "2"]
|
||||
0
services/compliance-backend/app/__init__.py
Normal file
0
services/compliance-backend/app/__init__.py
Normal file
0
services/compliance-backend/app/api/__init__.py
Normal file
0
services/compliance-backend/app/api/__init__.py
Normal file
95
services/compliance-backend/app/api/compliance.py
Normal file
95
services/compliance-backend/app/api/compliance.py
Normal file
@@ -0,0 +1,95 @@
|
||||
import uuid
|
||||
import logging
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from langchain.schema import HumanMessage, SystemMessage
|
||||
|
||||
from ..core.llm import get_llm, COMPLIANCE_CHECK_PROMPT
|
||||
from ..services.rag import hybrid_search
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/compliance", tags=["合规审查"])
|
||||
|
||||
|
||||
class ComplianceCheckRequest(BaseModel):
|
||||
query: str
|
||||
regulation_domains: list[str] = ["vehicle_safety"]
|
||||
top_k: int = 5
|
||||
|
||||
|
||||
class ComplianceCheckResponse(BaseModel):
|
||||
risk_level: str
|
||||
risk_score: float
|
||||
findings: list[dict]
|
||||
recommendations: list[str]
|
||||
sources: list[dict]
|
||||
|
||||
|
||||
@router.post("/check", response_model=ComplianceCheckResponse)
|
||||
async def check_compliance(req: ComplianceCheckRequest):
|
||||
"""
|
||||
对输入内容进行合规性检查，与法规库比对后给出风险评估。
|
||||
"""
|
||||
# 检索相关法规（从多个域检索）
|
||||
all_chunks = []
|
||||
for domain in req.regulation_domains:
|
||||
chunks = await hybrid_search(
|
||||
req.query,
|
||||
collection_name="regulation_chunks",
|
||||
top_k=req.top_k,
|
||||
)
|
||||
all_chunks.extend(chunks)
|
||||
|
||||
# 去重 + 按分数排序
|
||||
seen = set()
|
||||
unique_chunks = []
|
||||
for c in sorted(all_chunks, key=lambda x: x["score"], reverse=True):
|
||||
if c["id"] not in seen:
|
||||
seen.add(c["id"])
|
||||
unique_chunks.append(c)
|
||||
top_chunks = unique_chunks[:req.top_k]
|
||||
|
||||
if not top_chunks:
|
||||
return ComplianceCheckResponse(
|
||||
risk_level="unknown",
|
||||
risk_score=0,
|
||||
findings=[{"issue": "未找到相关法规，请先上传法规文档"}],
|
||||
recommendations=["上传相关法规文档到知识库后重试"],
|
||||
sources=[],
|
||||
)
|
||||
|
||||
# 构建法规上下文
|
||||
regulations_text = "\n\n".join(
|
||||
f"[{i+1}] {c['content'][:500]}" for i, c in enumerate(top_chunks)
|
||||
)
|
||||
|
||||
prompt = COMPLIANCE_CHECK_PROMPT.format(
|
||||
content=req.query,
|
||||
regulations=regulations_text,
|
||||
)
|
||||
|
||||
llm = get_llm(temperature=0.0)
|
||||
try:
|
||||
response = await llm.ainvoke([HumanMessage(content=prompt)])
|
||||
analysis = response.content
|
||||
except Exception as e:
|
||||
logger.error(f"LLM 合规分析失败：{e}")
|
||||
analysis = f"LLM 分析失败：{e}"
|
||||
|
||||
# 简单解析 LLM 输出（生产可用结构化输出）
|
||||
risk_level = "medium"
|
||||
risk_score = 50.0
|
||||
if "critical" in analysis.lower() or "严重" in analysis:
|
||||
risk_level, risk_score = "critical", 90.0
|
||||
elif "high" in analysis.lower() or "高风险" in analysis:
|
||||
risk_level, risk_score = "high", 70.0
|
||||
elif "low" in analysis.lower() or "低风险" in analysis:
|
||||
risk_level, risk_score = "low", 20.0
|
||||
|
||||
return ComplianceCheckResponse(
|
||||
risk_level=risk_level,
|
||||
risk_score=risk_score,
|
||||
findings=[{"analysis": analysis}],
|
||||
recommendations=["请参考上述分析进行整改"],
|
||||
sources=[{"content": c["content"][:200], "score": c["score"]} for c in top_chunks],
|
||||
)
|
||||
114
services/compliance-backend/app/api/kb.py
Normal file
114
services/compliance-backend/app/api/kb.py
Normal file
@@ -0,0 +1,114 @@
|
||||
import uuid
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, UploadFile, File, Form, HTTPException, BackgroundTasks
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..core.deps import get_db
|
||||
from ..models.db import Workspace, File as FileRecord, Task
|
||||
from ..services.rag import hybrid_search, rerank, generate_answer
|
||||
from ..worker import process_file_task
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/kb", tags=["知识库"])
|
||||
|
||||
UPLOAD_DIR = Path("/app/data/uploads")
|
||||
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
class WorkspaceCreate(BaseModel):
|
||||
name: str
|
||||
description: str = ""
|
||||
domain: str = "general"
|
||||
|
||||
|
||||
class QARequest(BaseModel):
|
||||
query: str
|
||||
workspace_id: str | None = None
|
||||
top_k: int = 5
|
||||
return_sources: bool = True
|
||||
|
||||
|
||||
@router.post("/workspaces")
|
||||
async def create_workspace(req: WorkspaceCreate, db: AsyncSession = Depends(get_db)):
|
||||
ws = Workspace(name=req.name, description=req.description, domain=req.domain)
|
||||
db.add(ws)
|
||||
await db.flush()
|
||||
return {"id": str(ws.id), "name": ws.name, "domain": ws.domain}
|
||||
|
||||
|
||||
@router.post("/files/upload")
|
||||
async def upload_file(
|
||||
background_tasks: BackgroundTasks,
|
||||
file: UploadFile = File(...),
|
||||
workspace_id: str = Form(default=""),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
content = await file.read()
|
||||
file_id = str(uuid.uuid4())
|
||||
suffix = Path(file.filename or "doc").suffix
|
||||
save_path = UPLOAD_DIR / f"{file_id}{suffix}"
|
||||
save_path.write_bytes(content)
|
||||
|
||||
file_record = FileRecord(
|
||||
id=uuid.UUID(file_id),
|
||||
filename=f"{file_id}{suffix}",
|
||||
original_name=file.filename or "unknown",
|
||||
file_type=suffix.lstrip("."),
|
||||
file_size=len(content),
|
||||
storage_path=str(save_path),
|
||||
workspace_id=uuid.UUID(workspace_id) if workspace_id else None,
|
||||
status="uploaded",
|
||||
)
|
||||
db.add(file_record)
|
||||
|
||||
task = Task(
|
||||
task_type="parse_and_vectorize",
|
||||
status="pending",
|
||||
file_id=uuid.UUID(file_id),
|
||||
payload={"workspace_id": workspace_id},
|
||||
)
|
||||
db.add(task)
|
||||
await db.flush()
|
||||
|
||||
# 异步触发 Celery 任务
|
||||
celery_task = process_file_task.delay(file_id, str(task.id), workspace_id)
|
||||
task.celery_task_id = celery_task.id
|
||||
await db.flush()
|
||||
|
||||
return {"file_id": file_id, "task_id": str(task.id), "status": "processing"}
|
||||
|
||||
|
||||
@router.get("/tasks/{task_id}")
|
||||
async def get_task(task_id: str, db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(select(Task).where(Task.id == uuid.UUID(task_id)))
|
||||
task = result.scalar_one_or_none()
|
||||
if not task:
|
||||
raise HTTPException(status_code=404, detail="任务不存在")
|
||||
return {
|
||||
"task_id": str(task.id),
|
||||
"status": task.status,
|
||||
"progress": task.progress,
|
||||
"file_id": str(task.file_id) if task.file_id else None,
|
||||
"error_msg": task.error_msg,
|
||||
"completed_at": task.completed_at.isoformat() if task.completed_at else None,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/qa")
|
||||
async def qa(req: QARequest):
|
||||
chunks = await hybrid_search(req.query, workspace_id=req.workspace_id, top_k=req.top_k * 2)
|
||||
ranked = await rerank(req.query, chunks, top_k=req.top_k)
|
||||
result = await generate_answer(req.query, ranked)
|
||||
if not req.return_sources:
|
||||
result.pop("sources", None)
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/knowledge/retrieval")
|
||||
async def retrieval(req: QARequest):
|
||||
chunks = await hybrid_search(req.query, workspace_id=req.workspace_id, top_k=req.top_k)
|
||||
return {"chunks": chunks, "total": len(chunks)}
|
||||
111
services/compliance-backend/app/api/regulation.py
Normal file
111
services/compliance-backend/app/api/regulation.py
Normal file
@@ -0,0 +1,111 @@
|
||||
import uuid
|
||||
import logging
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, desc
|
||||
|
||||
from ..core.deps import get_db
|
||||
from ..models.db import RegulationSource, RegulationUpdate
|
||||
from ..worker import fetch_regulation_source
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/regulation", tags=["法规监控"])
|
||||
|
||||
|
||||
class SourceCreate(BaseModel):
|
||||
name: str
|
||||
url: str
|
||||
domain: str = "vehicle_safety"
|
||||
fetch_interval: int = 86400
|
||||
fetch_config: dict = {}
|
||||
|
||||
|
||||
class SubscribeRequest(BaseModel):
|
||||
name: str
|
||||
channel: str # email / webhook / feishu / dingtalk
|
||||
target: str
|
||||
domains: list[str] = []
|
||||
importance_min: str = "normal"
|
||||
|
||||
|
||||
@router.post("/sources")
|
||||
async def create_source(req: SourceCreate, db: AsyncSession = Depends(get_db)):
|
||||
source = RegulationSource(
|
||||
name=req.name,
|
||||
url=req.url,
|
||||
domain=req.domain,
|
||||
fetch_interval=req.fetch_interval,
|
||||
fetch_config=req.fetch_config,
|
||||
)
|
||||
db.add(source)
|
||||
await db.flush()
|
||||
return {
|
||||
"id": str(source.id),
|
||||
"name": source.name,
|
||||
"url": source.url,
|
||||
"domain": source.domain,
|
||||
"status": "active",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
async def list_sources(db: AsyncSession = Depends(get_db)):
|
||||
result = await db.execute(
|
||||
select(RegulationSource).where(RegulationSource.is_active == True)
|
||||
)
|
||||
sources = result.scalars().all()
|
||||
return [{"id": str(s.id), "name": s.name, "url": s.url, "domain": s.domain} for s in sources]
|
||||
|
||||
|
||||
@router.post("/sources/{source_id}/fetch")
|
||||
async def manual_fetch(source_id: str, db: AsyncSession = Depends(get_db)):
|
||||
"""手动触发某个监控源的抓取（测试用）"""
|
||||
result = await db.execute(
|
||||
select(RegulationSource).where(RegulationSource.id == uuid.UUID(source_id))
|
||||
)
|
||||
source = result.scalar_one_or_none()
|
||||
if not source:
|
||||
raise HTTPException(status_code=404, detail="监控源不存在")
|
||||
|
||||
task = fetch_regulation_source.delay(source_id)
|
||||
return {"task_id": task.id, "status": "queued", "source_id": source_id}
|
||||
|
||||
|
||||
@router.get("/updates")
|
||||
async def get_updates(
|
||||
domain: str | None = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
query = select(RegulationUpdate).order_by(desc(RegulationUpdate.fetched_at))
|
||||
result = await db.execute(query.limit(limit).offset(offset))
|
||||
updates = result.scalars().all()
|
||||
return {
|
||||
"updates": [
|
||||
{
|
||||
"id": str(u.id),
|
||||
"title": u.title,
|
||||
"url": u.url,
|
||||
"change_type": u.change_type,
|
||||
"summary": u.summary,
|
||||
"importance": u.importance,
|
||||
"fetched_at": u.fetched_at.isoformat() if u.fetched_at else None,
|
||||
}
|
||||
for u in updates
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@router.post("/subscribe")
|
||||
async def subscribe(req: SubscribeRequest, db: AsyncSession = Depends(get_db)):
|
||||
from ..models.db import Workspace # 借用DB session
|
||||
# 简化版：仅记录订阅（推送逻辑在 push-worker 中实现）
|
||||
return {
|
||||
"id": str(uuid.uuid4()),
|
||||
"name": req.name,
|
||||
"channel": req.channel,
|
||||
"domains": req.domains,
|
||||
"status": "active",
|
||||
}
|
||||
0
services/compliance-backend/app/core/__init__.py
Normal file
0
services/compliance-backend/app/core/__init__.py
Normal file
37
services/compliance-backend/app/core/config.py
Normal file
37
services/compliance-backend/app/core/config.py
Normal file
@@ -0,0 +1,37 @@
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
# 应用
|
||||
app_env: str = "development"
|
||||
log_level: str = "INFO"
|
||||
api_secret_key: str = "change_this_key"
|
||||
|
||||
# 数据库
|
||||
database_url: str = "postgresql+asyncpg://compliance:compliance123@postgres:5432/compliance_db"
|
||||
redis_url: str = "redis://:redis123@redis:6379/0"
|
||||
|
||||
# Milvus
|
||||
milvus_host: str = "milvus"
|
||||
milvus_port: int = 19530
|
||||
|
||||
# Neo4j
|
||||
neo4j_uri: str = "bolt://neo4j:7687"
|
||||
neo4j_user: str = "neo4j"
|
||||
neo4j_password: str = "neo4j123"
|
||||
|
||||
# AI 服务
|
||||
embedding_service_url: str = "http://embedding-service:8010"
|
||||
mcp_server_url: str = "http://mcp-server:8011"
|
||||
|
||||
# LLM
|
||||
llm_provider: str = "deepseek" # deepseek / qwen
|
||||
deepseek_api_key: str = ""
|
||||
deepseek_model: str = "deepseek-chat"
|
||||
dashscope_api_key: str = ""
|
||||
qwen_model: str = "qwen-plus"
|
||||
|
||||
|
||||
settings = Settings()
|
||||
54
services/compliance-backend/app/core/deps.py
Normal file
54
services/compliance-backend/app/core/deps.py
Normal file
@@ -0,0 +1,54 @@
|
||||
from functools import lru_cache
|
||||
from typing import AsyncGenerator
|
||||
|
||||
import httpx
|
||||
from neo4j import AsyncGraphDatabase
|
||||
from pymilvus import connections, Collection
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
||||
|
||||
from .config import settings
|
||||
|
||||
# ── PostgreSQL ──────────────────────────────────
|
||||
engine = create_async_engine(settings.database_url, pool_size=10, max_overflow=20)
|
||||
AsyncSessionLocal = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
|
||||
async def get_db() -> AsyncGenerator[AsyncSession, None]:
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
yield session
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
# ── Milvus ──────────────────────────────────────
|
||||
def get_milvus_collection(name: str) -> Collection:
|
||||
connections.connect(host=settings.milvus_host, port=settings.milvus_port)
|
||||
return Collection(name)
|
||||
|
||||
|
||||
# ── Neo4j ───────────────────────────────────────
|
||||
_neo4j_driver = None
|
||||
|
||||
|
||||
def get_neo4j():
|
||||
global _neo4j_driver
|
||||
if _neo4j_driver is None:
|
||||
_neo4j_driver = AsyncGraphDatabase.driver(
|
||||
settings.neo4j_uri,
|
||||
auth=(settings.neo4j_user, settings.neo4j_password),
|
||||
)
|
||||
return _neo4j_driver
|
||||
|
||||
|
||||
# ── HTTP 客户端（复用连接池）────────────────────
|
||||
_http_client = None
|
||||
|
||||
|
||||
def get_http_client() -> httpx.AsyncClient:
|
||||
global _http_client
|
||||
if _http_client is None:
|
||||
_http_client = httpx.AsyncClient(timeout=120.0)
|
||||
return _http_client
|
||||
56
services/compliance-backend/app/core/llm.py
Normal file
56
services/compliance-backend/app/core/llm.py
Normal file
@@ -0,0 +1,56 @@
|
||||
from langchain_openai import ChatOpenAI
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
from .config import settings
|
||||
|
||||
|
||||
def get_llm(temperature: float = 0.1) -> ChatOpenAI:
|
||||
"""获取 LLM 客户端（DeepSeek 或 Qwen，均兼容 OpenAI API）"""
|
||||
if settings.llm_provider == "deepseek":
|
||||
return ChatOpenAI(
|
||||
model=settings.deepseek_model,
|
||||
api_key=settings.deepseek_api_key,
|
||||
base_url="https://api.deepseek.com/v1",
|
||||
temperature=temperature,
|
||||
max_retries=3,
|
||||
timeout=120,
|
||||
)
|
||||
elif settings.llm_provider == "qwen":
|
||||
return ChatOpenAI(
|
||||
model=settings.qwen_model,
|
||||
api_key=settings.dashscope_api_key,
|
||||
base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
temperature=temperature,
|
||||
max_retries=3,
|
||||
timeout=120,
|
||||
)
|
||||
raise ValueError(f"不支持的 LLM 提供商：{settings.llm_provider}")
|
||||
|
||||
|
||||
RAG_SYSTEM_PROMPT = """你是一位专业的汽车行业合规专家，具备深厚的法规知识（GB标准、UN-ECE、ISO 45001、IATF 16949等）。
|
||||
|
||||
回答规则：
|
||||
1. 仅基于提供的参考文献回答，不添加不在文献中的信息
|
||||
2. 每个关键陈述必须标注来源（格式：[来源：文件名，第X页]）
|
||||
3. 如果参考文献不足以回答问题，明确说明
|
||||
4. 使用专业但清晰的语言，适合工程师和法务人员阅读
|
||||
5. 对于数值要求（如绝缘电阻值、时间限制等），精确引用原文"""
|
||||
|
||||
|
||||
COMPLIANCE_CHECK_PROMPT = """你是一位专业的汽车合规审查专家。
|
||||
|
||||
请对以下内容进行合规性评估：
|
||||
|
||||
【待审查内容】
|
||||
{content}
|
||||
|
||||
【相关法规要求】
|
||||
{regulations}
|
||||
|
||||
请按以下格式输出：
|
||||
1. 整体风险等级：[low/medium/high/critical]
|
||||
2. 风险分数：[0-100]
|
||||
3. 发现的合规问题（逐条列出）：
|
||||
- 问题描述
|
||||
- 违反的具体法规条款
|
||||
- 严重程度
|
||||
4. 整改建议（具体可操作）"""
|
||||
84
services/compliance-backend/app/main.py
Normal file
84
services/compliance-backend/app/main.py
Normal file
@@ -0,0 +1,84 @@
|
||||
import logging
|
||||
import time
|
||||
|
||||
import structlog
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from prometheus_fastapi_instrumentator import Instrumentator
|
||||
|
||||
from .api import kb, compliance, regulation
|
||||
from .core.config import settings
|
||||
|
||||
# 结构化日志配置
|
||||
structlog.configure(
|
||||
wrapper_class=structlog.make_filtering_bound_logger(
|
||||
getattr(logging, settings.log_level.upper(), logging.INFO)
|
||||
)
|
||||
)
|
||||
logger = structlog.get_logger()
|
||||
|
||||
app = FastAPI(
|
||||
title="AI合规智能中枢 API",
|
||||
description="面向车企与工厂的全链路合规智能平台",
|
||||
version="0.1.0",
|
||||
docs_url="/docs",
|
||||
redoc_url="/redoc",
|
||||
)
|
||||
|
||||
# CORS（开发环境）
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"] if settings.app_env == "development" else [],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Prometheus 指标
|
||||
Instrumentator().instrument(app).expose(app)
|
||||
|
||||
# 注册路由
|
||||
app.include_router(kb.router)
|
||||
app.include_router(compliance.router)
|
||||
app.include_router(regulation.router)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def log_requests(request: Request, call_next):
|
||||
start = time.time()
|
||||
response = await call_next(request)
|
||||
duration_ms = int((time.time() - start) * 1000)
|
||||
logger.info(
|
||||
"request",
|
||||
method=request.method,
|
||||
path=request.url.path,
|
||||
status=response.status_code,
|
||||
duration_ms=duration_ms,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
"""健康检查（含依赖服务检测）"""
|
||||
import httpx
|
||||
from .core.config import settings
|
||||
|
||||
checks = {"status": "ok", "services": {}}
|
||||
|
||||
# 检查嵌入服务
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
r = await client.get(f"{settings.embedding_service_url}/health")
|
||||
checks["services"]["embedding"] = "ok" if r.status_code == 200 else "degraded"
|
||||
except Exception:
|
||||
checks["services"]["embedding"] = "unavailable"
|
||||
|
||||
# 检查 MCP Server
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
r = await client.get(f"{settings.mcp_server_url}/health")
|
||||
checks["services"]["mcp"] = "ok" if r.status_code == 200 else "degraded"
|
||||
except Exception:
|
||||
checks["services"]["mcp"] = "unavailable"
|
||||
|
||||
return checks
|
||||
0
services/compliance-backend/app/models/__init__.py
Normal file
0
services/compliance-backend/app/models/__init__.py
Normal file
113
services/compliance-backend/app/models/db.py
Normal file
113
services/compliance-backend/app/models/db.py
Normal file
@@ -0,0 +1,113 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Column, String, Integer, BigInteger, Boolean, Text, ARRAY, Numeric
|
||||
from sqlalchemy import DateTime, ForeignKey, func
|
||||
from sqlalchemy.dialects.postgresql import UUID, JSONB, INET
|
||||
from sqlalchemy.orm import DeclarativeBase, relationship
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class Workspace(Base):
|
||||
__tablename__ = "workspaces"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
name = Column(String(255), nullable=False)
|
||||
description = Column(Text)
|
||||
domain = Column(String(100))
|
||||
created_by = Column(String(255))
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
files = relationship("File", back_populates="workspace")
|
||||
|
||||
|
||||
class File(Base):
|
||||
__tablename__ = "files"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
workspace_id = Column(UUID(as_uuid=True), ForeignKey("workspaces.id", ondelete="CASCADE"))
|
||||
filename = Column(String(500), nullable=False)
|
||||
original_name = Column(String(500), nullable=False)
|
||||
file_type = Column(String(50))
|
||||
file_size = Column(BigInteger)
|
||||
storage_path = Column(Text)
|
||||
parsed_path = Column(Text)
|
||||
status = Column(String(50), default="uploaded")
|
||||
error_msg = Column(Text)
|
||||
metadata = Column(JSONB, default={})
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
workspace = relationship("Workspace", back_populates="files")
|
||||
tasks = relationship("Task", back_populates="file")
|
||||
|
||||
|
||||
class Task(Base):
|
||||
__tablename__ = "tasks"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
task_type = Column(String(100), nullable=False)
|
||||
status = Column(String(50), default="pending")
|
||||
payload = Column(JSONB, default={})
|
||||
result = Column(JSONB)
|
||||
error_msg = Column(Text)
|
||||
progress = Column(Integer, default=0)
|
||||
file_id = Column(UUID(as_uuid=True), ForeignKey("files.id"))
|
||||
celery_task_id = Column(String(255))
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
completed_at = Column(DateTime(timezone=True))
|
||||
|
||||
file = relationship("File", back_populates="tasks")
|
||||
|
||||
|
||||
class ComplianceReport(Base):
|
||||
__tablename__ = "compliance_reports"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
file_id = Column(UUID(as_uuid=True), ForeignKey("files.id"))
|
||||
regulation_domains = Column(ARRAY(Text))
|
||||
overall_risk_level = Column(String(20))
|
||||
risk_score = Column(Numeric(5, 2))
|
||||
findings = Column(JSONB, default=[])
|
||||
recommendations = Column(JSONB, default=[])
|
||||
report_markdown = Column(Text)
|
||||
llm_model = Column(String(100))
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class RegulationSource(Base):
|
||||
__tablename__ = "regulation_sources"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
name = Column(String(255), nullable=False)
|
||||
url = Column(Text, nullable=False)
|
||||
source_type = Column(String(50), default="webpage")
|
||||
domain = Column(String(100))
|
||||
fetch_interval = Column(Integer, default=86400)
|
||||
is_active = Column(Boolean, default=True)
|
||||
last_fetched_at = Column(DateTime(timezone=True))
|
||||
last_hash = Column(String(64))
|
||||
fetch_config = Column(JSONB, default={})
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class RegulationUpdate(Base):
|
||||
__tablename__ = "regulation_updates"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
source_id = Column(UUID(as_uuid=True), ForeignKey("regulation_sources.id"))
|
||||
title = Column(String(500))
|
||||
url = Column(Text)
|
||||
change_type = Column(String(50))
|
||||
summary = Column(Text)
|
||||
raw_content = Column(Text)
|
||||
diff_content = Column(Text)
|
||||
is_notified = Column(Boolean, default=False)
|
||||
importance = Column(String(20), default="normal")
|
||||
fetched_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
published_at = Column(DateTime(timezone=True))
|
||||
21
services/compliance-backend/app/services/embed.py
Normal file
21
services/compliance-backend/app/services/embed.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import httpx
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
from ..core.config import settings
|
||||
|
||||
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(min=1, max=10))
|
||||
async def embed_texts(texts: list[str], batch_size: int = 12) -> dict:
|
||||
"""调用嵌入服务，返回 dense 和 sparse 向量"""
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
resp = await client.post(
|
||||
f"{settings.embedding_service_url}/embed",
|
||||
json={"texts": texts, "batch_size": batch_size},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
async def embed_single(text: str) -> list[float]:
|
||||
"""嵌入单条文本，返回 dense 向量"""
|
||||
result = await embed_texts([text], batch_size=1)
|
||||
return result["dense"][0]
|
||||
65
services/compliance-backend/app/services/graph.py
Normal file
65
services/compliance-backend/app/services/graph.py
Normal file
@@ -0,0 +1,65 @@
|
||||
import logging
|
||||
from ..core.deps import get_neo4j
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def create_regulation_node(regulation: dict) -> str:
|
||||
"""在 Neo4j 中创建法规节点"""
|
||||
driver = get_neo4j()
|
||||
async with driver.session() as session:
|
||||
result = await session.run(
|
||||
"""
|
||||
MERGE (r:Regulation {id: $id})
|
||||
SET r.title = $title,
|
||||
r.domain = $domain,
|
||||
r.version = $version,
|
||||
r.code = $code
|
||||
RETURN r.id as id
|
||||
""",
|
||||
id=regulation.get("id"),
|
||||
title=regulation.get("title", ""),
|
||||
domain=regulation.get("domain", ""),
|
||||
version=regulation.get("version", ""),
|
||||
code=regulation.get("code", ""),
|
||||
)
|
||||
record = await result.single()
|
||||
return record["id"] if record else None
|
||||
|
||||
|
||||
async def create_clause_node(clause: dict, regulation_id: str) -> str:
|
||||
"""创建条款节点并关联到法规"""
|
||||
driver = get_neo4j()
|
||||
async with driver.session() as session:
|
||||
result = await session.run(
|
||||
"""
|
||||
MATCH (r:Regulation {id: $reg_id})
|
||||
MERGE (c:Clause {id: $id})
|
||||
SET c.number = $number,
|
||||
c.content = $content
|
||||
MERGE (r)-[:CONTAINS]->(c)
|
||||
RETURN c.id as id
|
||||
""",
|
||||
reg_id=regulation_id,
|
||||
id=clause.get("id"),
|
||||
number=clause.get("number", ""),
|
||||
content=clause.get("content", "")[:2000],
|
||||
)
|
||||
record = await result.single()
|
||||
return record["id"] if record else None
|
||||
|
||||
|
||||
async def search_related_regulations(domain: str, limit: int = 10) -> list[dict]:
|
||||
"""查询指定域下的所有法规"""
|
||||
driver = get_neo4j()
|
||||
async with driver.session() as session:
|
||||
result = await session.run(
|
||||
"""
|
||||
MATCH (r:Regulation {domain: $domain})
|
||||
RETURN r.id as id, r.title as title, r.code as code, r.version as version
|
||||
LIMIT $limit
|
||||
""",
|
||||
domain=domain,
|
||||
limit=limit,
|
||||
)
|
||||
return [dict(record) async for record in result]
|
||||
59
services/compliance-backend/app/services/monitor.py
Normal file
59
services/compliance-backend/app/services/monitor.py
Normal file
@@ -0,0 +1,59 @@
|
||||
import hashlib
|
||||
import logging
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def fetch_url(url: str, timeout: int = 30) -> str | None:
|
||||
"""抓取 URL 内容"""
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout,
|
||||
headers={"User-Agent": "Mozilla/5.0 (compliance-monitor/1.0)"},
|
||||
follow_redirects=True,
|
||||
) as client:
|
||||
resp = await client.get(url)
|
||||
resp.raise_for_status()
|
||||
return resp.text
|
||||
except Exception as e:
|
||||
logger.warning(f"抓取 {url} 失败：{e}")
|
||||
return None
|
||||
|
||||
|
||||
def extract_text(html: str) -> str:
|
||||
"""提取 HTML 中的主要文本内容"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for tag in soup(["script", "style", "nav", "footer", "header"]):
|
||||
tag.decompose()
|
||||
return soup.get_text(separator="\n", strip=True)
|
||||
|
||||
|
||||
def compute_hash(content: str) -> str:
|
||||
return hashlib.md5(content.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
async def check_source_for_updates(source: dict) -> dict | None:
|
||||
"""
|
||||
检查监控源是否有更新。
|
||||
返回 None 表示无变化，返回 dict 表示有新内容。
|
||||
"""
|
||||
html = await fetch_url(source["url"])
|
||||
if not html:
|
||||
return None
|
||||
|
||||
text = extract_text(html)
|
||||
new_hash = compute_hash(text)
|
||||
|
||||
if source.get("last_hash") == new_hash:
|
||||
logger.info(f"监控源 {source['name']} 无变化")
|
||||
return None
|
||||
|
||||
return {
|
||||
"source_id": source["id"],
|
||||
"raw_content": text[:50000], # 最多保存 50KB
|
||||
"new_hash": new_hash,
|
||||
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
43
services/compliance-backend/app/services/parse.py
Normal file
43
services/compliance-backend/app/services/parse.py
Normal file
@@ -0,0 +1,43 @@
|
||||
import httpx
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
from ..core.config import settings
|
||||
|
||||
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(min=2, max=30))
|
||||
async def parse_document(file_content: bytes, filename: str) -> dict:
|
||||
"""调用 mcp-server 解析文档，返回 Markdown"""
|
||||
async with httpx.AsyncClient(timeout=300.0) as client:
|
||||
resp = await client.post(
|
||||
f"{settings.mcp_server_url}/parse-document",
|
||||
files={"file": (filename, file_content, "application/octet-stream")},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def chunk_text(text: str, chunk_size: int = 512, overlap: int = 64) -> list[dict]:
|
||||
"""将文本按 token 数分块（简单版，按字符数估算）"""
|
||||
chars_per_chunk = chunk_size * 2 # 中文约2字符/token
|
||||
chars_overlap = overlap * 2
|
||||
chunks = []
|
||||
start = 0
|
||||
idx = 0
|
||||
|
||||
while start < len(text):
|
||||
end = min(start + chars_per_chunk, len(text))
|
||||
# 尝试在段落边界截断
|
||||
if end < len(text):
|
||||
for sep in ["\n\n", "\n", "。", ".", " "]:
|
||||
pos = text.rfind(sep, start, end)
|
||||
if pos > start + chars_per_chunk // 2:
|
||||
end = pos + len(sep)
|
||||
break
|
||||
|
||||
chunk_text = text[start:end].strip()
|
||||
if chunk_text:
|
||||
chunks.append({"idx": idx, "content": chunk_text, "start": start, "end": end})
|
||||
idx += 1
|
||||
|
||||
start = max(start + 1, end - chars_overlap)
|
||||
|
||||
return chunks
|
||||
92
services/compliance-backend/app/services/rag.py
Normal file
92
services/compliance-backend/app/services/rag.py
Normal file
@@ -0,0 +1,92 @@
|
||||
import logging
|
||||
from langchain.schema import HumanMessage, SystemMessage
|
||||
from pymilvus import connections, Collection
|
||||
|
||||
from .embed import embed_single, embed_texts
|
||||
from ..core.llm import get_llm, RAG_SYSTEM_PROMPT
|
||||
from ..core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _get_collection(name: str) -> Collection:
|
||||
connections.connect(host=settings.milvus_host, port=settings.milvus_port)
|
||||
return Collection(name)
|
||||
|
||||
|
||||
async def hybrid_search(
|
||||
query: str,
|
||||
collection_name: str = "regulation_chunks",
|
||||
top_k: int = 10,
|
||||
workspace_id: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""混合检索：BGE-M3 向量检索（调研版简化，省去 BM25 融合）"""
|
||||
query_vec = await embed_single(query)
|
||||
|
||||
col = _get_collection(collection_name)
|
||||
|
||||
expr = f'workspace_id == "{workspace_id}"' if workspace_id else None
|
||||
results = col.search(
|
||||
data=[query_vec],
|
||||
anns_field="dense_vec",
|
||||
param={"metric_type": "COSINE", "params": {"ef": 100}},
|
||||
limit=top_k,
|
||||
expr=expr,
|
||||
output_fields=["content", "metadata", "file_id", "chunk_idx"],
|
||||
)
|
||||
|
||||
chunks = []
|
||||
for hits in results:
|
||||
for hit in hits:
|
||||
chunks.append({
|
||||
"id": hit.id,
|
||||
"content": hit.entity.get("content", ""),
|
||||
"score": float(hit.score),
|
||||
"file_id": hit.entity.get("file_id", ""),
|
||||
"chunk_idx": hit.entity.get("chunk_idx", 0),
|
||||
"metadata": hit.entity.get("metadata", {}),
|
||||
})
|
||||
return chunks
|
||||
|
||||
|
||||
async def rerank(query: str, chunks: list[dict], top_k: int = 5) -> list[dict]:
|
||||
"""简化版精排（调研版按 score 直接排序，生产可换 Cross-Encoder）"""
|
||||
return sorted(chunks, key=lambda x: x["score"], reverse=True)[:top_k]
|
||||
|
||||
|
||||
async def generate_answer(query: str, chunks: list[dict]) -> dict:
|
||||
"""基于检索结果，调用 LLM 生成引文锚定的答案"""
|
||||
if not chunks:
|
||||
return {"answer": "未找到相关法规内容，请上传相关法规文档后重试。", "sources": []}
|
||||
|
||||
# 构建 RAG 上下文
|
||||
context_parts = []
|
||||
for i, chunk in enumerate(chunks, 1):
|
||||
meta = chunk.get("metadata", {})
|
||||
source_info = f"[来源 {i}：{meta.get('filename', '未知文件')}，第 {meta.get('page', '?')} 页]"
|
||||
context_parts.append(f"{source_info}\n{chunk['content']}")
|
||||
|
||||
context = "\n\n---\n\n".join(context_parts)
|
||||
user_prompt = f"参考文献：\n\n{context}\n\n问题：{query}\n\n请基于以上参考文献回答，并标注来源。"
|
||||
|
||||
llm = get_llm(temperature=0.1)
|
||||
messages = [SystemMessage(content=RAG_SYSTEM_PROMPT), HumanMessage(content=user_prompt)]
|
||||
|
||||
try:
|
||||
response = await llm.ainvoke(messages)
|
||||
answer = response.content
|
||||
except Exception as e:
|
||||
logger.error(f"LLM 生成失败：{e}")
|
||||
answer = f"LLM 生成失败：{e}。检索到的相关内容：{chunks[0]['content'][:200]}..."
|
||||
|
||||
sources = [
|
||||
{
|
||||
"content": c["content"][:300],
|
||||
"file_id": c.get("file_id", ""),
|
||||
"chunk_idx": c.get("chunk_idx", 0),
|
||||
"score": c.get("score", 0),
|
||||
"metadata": c.get("metadata", {}),
|
||||
}
|
||||
for c in chunks
|
||||
]
|
||||
return {"answer": answer, "sources": sources}
|
||||
212
services/compliance-backend/app/worker.py
Normal file
212
services/compliance-backend/app/worker.py
Normal file
@@ -0,0 +1,212 @@
|
||||
import uuid
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from celery import Celery
|
||||
from celery.schedules import crontab
|
||||
|
||||
from .core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Celery 配置
|
||||
celery_app = Celery(
|
||||
"compliance",
|
||||
broker=settings.redis_url,
|
||||
backend=settings.redis_url,
|
||||
)
|
||||
celery_app.conf.update(
|
||||
task_serializer="json",
|
||||
accept_content=["json"],
|
||||
result_serializer="json",
|
||||
timezone="Asia/Shanghai",
|
||||
task_routes={
|
||||
"app.worker.process_file_task": {"queue": "parse"},
|
||||
"app.worker.fetch_regulation_source": {"queue": "monitor"},
|
||||
"app.worker.send_notifications": {"queue": "push"},
|
||||
},
|
||||
beat_schedule={
|
||||
"daily-regulation-monitor": {
|
||||
"task": "app.worker.run_all_monitors",
|
||||
"schedule": crontab(hour=2, minute=0),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
# ── 文件处理任务（解析 + 向量化）────────────────
|
||||
|
||||
@celery_app.task(name="app.worker.process_file_task", bind=True, max_retries=3)
|
||||
def process_file_task(self, file_id: str, task_id: str, workspace_id: str):
|
||||
"""解析文档并向量化存入 Milvus"""
|
||||
import asyncio
|
||||
asyncio.run(_process_file(file_id, task_id, workspace_id))
|
||||
|
||||
|
||||
async def _process_file(file_id: str, task_id: str, workspace_id: str):
|
||||
from pathlib import Path
|
||||
from sqlalchemy import select
|
||||
from .core.deps import AsyncSessionLocal, get_milvus_collection
|
||||
from .models.db import File, Task
|
||||
from .services.parse import parse_document, chunk_text
|
||||
from .services.embed import embed_texts
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 查找文件记录
|
||||
result = await db.execute(select(File).where(File.id == uuid.UUID(file_id)))
|
||||
file_record = result.scalar_one_or_none()
|
||||
if not file_record:
|
||||
logger.error(f"文件 {file_id} 不存在")
|
||||
return
|
||||
|
||||
task_result = await db.execute(select(Task).where(Task.id == uuid.UUID(task_id)))
|
||||
task = task_result.scalar_one_or_none()
|
||||
|
||||
try:
|
||||
# 更新状态
|
||||
file_record.status = "parsing"
|
||||
if task:
|
||||
task.status = "running"
|
||||
task.progress = 10
|
||||
await db.commit()
|
||||
|
||||
# Step 1：解析文档
|
||||
file_content = Path(file_record.storage_path).read_bytes()
|
||||
parse_result = await parse_document(file_content, file_record.original_name)
|
||||
markdown = parse_result.get("markdown", "")
|
||||
|
||||
if not markdown.strip():
|
||||
raise ValueError("文档解析结果为空")
|
||||
|
||||
file_record.status = "parsed"
|
||||
if task:
|
||||
task.progress = 40
|
||||
await db.commit()
|
||||
|
||||
# Step 2：分块
|
||||
chunks = chunk_text(markdown, chunk_size=512, overlap=64)
|
||||
logger.info(f"文件 {file_id} 分割为 {len(chunks)} 块")
|
||||
|
||||
# Step 3：向量化（分批处理）
|
||||
batch_size = 16
|
||||
col = get_milvus_collection("regulation_chunks")
|
||||
|
||||
for i in range(0, len(chunks), batch_size):
|
||||
batch = chunks[i:i + batch_size]
|
||||
texts = [c["content"] for c in batch]
|
||||
embed_result = await embed_texts(texts, batch_size=batch_size)
|
||||
dense_vecs = embed_result["dense"]
|
||||
|
||||
entities = [
|
||||
[f"{file_id}_{c['idx']}" for c in batch],
|
||||
[file_id] * len(batch),
|
||||
[workspace_id] * len(batch),
|
||||
[c["idx"] for c in batch],
|
||||
[c["content"] for c in batch],
|
||||
dense_vecs,
|
||||
[{"filename": file_record.original_name, "page": c.get("page", 0)} for c in batch],
|
||||
]
|
||||
col.insert(entities)
|
||||
|
||||
if task:
|
||||
task.progress = 40 + int(60 * (i + batch_size) / len(chunks))
|
||||
await db.commit()
|
||||
|
||||
col.flush()
|
||||
|
||||
# 完成
|
||||
file_record.status = "vectorized"
|
||||
if task:
|
||||
task.status = "completed"
|
||||
task.progress = 100
|
||||
task.completed_at = datetime.now(timezone.utc)
|
||||
await db.commit()
|
||||
logger.info(f"文件 {file_id} 处理完成")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"文件 {file_id} 处理失败：{e}")
|
||||
file_record.status = "failed"
|
||||
file_record.error_msg = str(e)
|
||||
if task:
|
||||
task.status = "failed"
|
||||
task.error_msg = str(e)
|
||||
await db.commit()
|
||||
raise
|
||||
|
||||
|
||||
# ── 法规监控任务 ────────────────────────────────
|
||||
|
||||
@celery_app.task(name="app.worker.run_all_monitors")
|
||||
def run_all_monitors():
|
||||
"""定时触发所有活跃监控源"""
|
||||
import asyncio
|
||||
asyncio.run(_run_all_monitors())
|
||||
|
||||
|
||||
async def _run_all_monitors():
|
||||
from sqlalchemy import select
|
||||
from .core.deps import AsyncSessionLocal
|
||||
from .models.db import RegulationSource
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await db.execute(
|
||||
select(RegulationSource).where(RegulationSource.is_active == True)
|
||||
)
|
||||
sources = result.scalars().all()
|
||||
for source in sources:
|
||||
fetch_regulation_source.delay(str(source.id))
|
||||
logger.info(f"触发监控源抓取：{source.name}")
|
||||
|
||||
|
||||
@celery_app.task(name="app.worker.fetch_regulation_source", bind=True, max_retries=2)
|
||||
def fetch_regulation_source(self, source_id: str):
|
||||
import asyncio
|
||||
asyncio.run(_fetch_source(source_id))
|
||||
|
||||
|
||||
async def _fetch_source(source_id: str):
|
||||
import hashlib
|
||||
from sqlalchemy import select
|
||||
from .core.deps import AsyncSessionLocal
|
||||
from .models.db import RegulationSource, RegulationUpdate
|
||||
from .services.monitor import check_source_for_updates
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
result = await db.execute(
|
||||
select(RegulationSource).where(RegulationSource.id == uuid.UUID(source_id))
|
||||
)
|
||||
source = result.scalar_one_or_none()
|
||||
if not source:
|
||||
return
|
||||
|
||||
source_dict = {
|
||||
"id": str(source.id),
|
||||
"name": source.name,
|
||||
"url": source.url,
|
||||
"last_hash": source.last_hash,
|
||||
}
|
||||
update_data = await check_source_for_updates(source_dict)
|
||||
|
||||
if update_data:
|
||||
logger.info(f"检测到变更：{source.name}")
|
||||
source.last_hash = update_data["new_hash"]
|
||||
source.last_fetched_at = datetime.now(timezone.utc)
|
||||
|
||||
update = RegulationUpdate(
|
||||
source_id=uuid.UUID(source_id),
|
||||
change_type="updated",
|
||||
raw_content=update_data["raw_content"][:50000],
|
||||
importance="normal",
|
||||
)
|
||||
db.add(update)
|
||||
await db.commit()
|
||||
else:
|
||||
source.last_fetched_at = datetime.now(timezone.utc)
|
||||
await db.commit()
|
||||
|
||||
|
||||
@celery_app.task(name="app.worker.send_notifications")
|
||||
def send_notifications():
|
||||
logger.info("推送通知任务执行（待实现）")
|
||||
|
||||
|
||||
# 导出供 FastAPI 使用
|
||||
worker = celery_app
|
||||
29
services/compliance-backend/pyproject.toml
Normal file
29
services/compliance-backend/pyproject.toml
Normal file
@@ -0,0 +1,29 @@
|
||||
[project]
|
||||
name = "compliance-backend"
|
||||
version = "0.1.0"
|
||||
description = "AI合规智能中枢 — 业务后端"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"fastapi>=0.115",
|
||||
"uvicorn[standard]>=0.30",
|
||||
"pydantic>=2.7",
|
||||
"pydantic-settings>=2.4",
|
||||
"sqlalchemy[asyncio]>=2.0",
|
||||
"asyncpg>=0.29",
|
||||
"redis[asyncio]>=5.0",
|
||||
"celery[redis]>=5.4",
|
||||
"pymilvus>=2.4",
|
||||
"neo4j>=5.20",
|
||||
"langchain>=0.3",
|
||||
"langchain-openai>=0.2",
|
||||
"langchain-community>=0.3",
|
||||
"llama-index-core>=0.11",
|
||||
"httpx>=0.27",
|
||||
"python-multipart>=0.0.9",
|
||||
"python-jose[cryptography]>=3.3",
|
||||
"structlog>=24.0",
|
||||
"prometheus-fastapi-instrumentator>=7.0",
|
||||
"tenacity>=8.5",
|
||||
"beautifulsoup4>=4.12",
|
||||
"requests>=2.32",
|
||||
]
|
||||
24
services/embedding/Dockerfile
Normal file
24
services/embedding/Dockerfile
Normal file
@@ -0,0 +1,24 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 系统依赖
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Python 依赖（先装，利用构建缓存）
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt \
|
||||
--index-url https://pypi.tuna.tsinghua.edu.cn/simple \
|
||||
--trusted-host pypi.tuna.tsinghua.edu.cn
|
||||
|
||||
COPY main.py .
|
||||
|
||||
# 健康检查
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \
|
||||
CMD curl -f http://localhost:8010/health || exit 1
|
||||
|
||||
EXPOSE 8010
|
||||
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8010", "--workers", "1"]
|
||||
87
services/embedding/main.py
Normal file
87
services/embedding/main.py
Normal file
@@ -0,0 +1,87 @@
|
||||
import os
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MODEL_NAME = os.getenv("MODEL_NAME", "BAAI/bge-m3")
|
||||
MODEL_CACHE = os.getenv("HF_HOME", "/app/models")
|
||||
DEVICE = os.getenv("DEVICE", "cpu")
|
||||
MAX_BATCH = int(os.getenv("MAX_BATCH_SIZE", "16"))
|
||||
|
||||
# 设置 HuggingFace 镜像
|
||||
if os.getenv("HF_ENDPOINT"):
|
||||
os.environ["HF_ENDPOINT"] = os.getenv("HF_ENDPOINT")
|
||||
|
||||
model = None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
global model
|
||||
logger.info(f"加载模型 {MODEL_NAME}，设备：{DEVICE}")
|
||||
try:
|
||||
from FlagEmbedding import BGEM3FlagModel
|
||||
model = BGEM3FlagModel(
|
||||
MODEL_NAME,
|
||||
use_fp16=(DEVICE != "cpu"),
|
||||
cache_dir=MODEL_CACHE,
|
||||
)
|
||||
logger.info("BGE-M3 模型加载完成")
|
||||
except Exception as e:
|
||||
logger.error(f"模型加载失败：{e}")
|
||||
raise
|
||||
yield
|
||||
logger.info("服务关闭")
|
||||
|
||||
|
||||
app = FastAPI(title="BGE-M3 嵌入服务", lifespan=lifespan)
|
||||
|
||||
|
||||
class EmbedRequest(BaseModel):
|
||||
texts: list[str] = Field(..., min_length=1, max_length=100)
|
||||
batch_size: int = Field(default=12, ge=1, le=MAX_BATCH)
|
||||
return_dense: bool = True
|
||||
return_sparse: bool = True
|
||||
|
||||
|
||||
class EmbedResponse(BaseModel):
|
||||
dense: Optional[list[list[float]]] = None
|
||||
sparse: Optional[list[dict]] = None
|
||||
model: str
|
||||
count: int
|
||||
|
||||
|
||||
@app.post("/embed", response_model=EmbedResponse)
|
||||
def embed(req: EmbedRequest) -> EmbedResponse:
|
||||
if model is None:
|
||||
raise HTTPException(status_code=503, detail="模型未就绪")
|
||||
if len(req.texts) > 100:
|
||||
raise HTTPException(status_code=400, detail="单次最多 100 条文本")
|
||||
|
||||
try:
|
||||
output = model.encode(
|
||||
req.texts,
|
||||
batch_size=req.batch_size,
|
||||
return_dense=req.return_dense,
|
||||
return_sparse=req.return_sparse,
|
||||
)
|
||||
return EmbedResponse(
|
||||
dense=output["dense_vecs"].tolist() if req.return_dense else None,
|
||||
sparse=[dict(w) for w in output["lexical_weights"]] if req.return_sparse else None,
|
||||
model=MODEL_NAME,
|
||||
count=len(req.texts),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"嵌入生成失败：{e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok", "model": MODEL_NAME, "device": DEVICE, "ready": model is not None}
|
||||
10
services/embedding/requirements.txt
Normal file
10
services/embedding/requirements.txt
Normal file
@@ -0,0 +1,10 @@
|
||||
fastapi>=0.115
|
||||
uvicorn[standard]>=0.30
|
||||
pydantic>=2.7
|
||||
FlagEmbedding>=1.3
|
||||
# CPU 版本 PyTorch（减小镜像体积）
|
||||
torch>=2.3.0 --index-url https://download.pytorch.org/whl/cpu
|
||||
transformers>=4.44
|
||||
sentence-transformers>=3.0
|
||||
huggingface-hub>=0.24
|
||||
numpy>=1.26
|
||||
38
services/mcp-server/Dockerfile
Normal file
38
services/mcp-server/Dockerfile
Normal file
@@ -0,0 +1,38 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 系统依赖（MinerU 需要 libGL）
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
libgl1-mesa-glx \
|
||||
libglib2.0-0 \
|
||||
libsm6 \
|
||||
libxrender1 \
|
||||
libxext6 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt \
|
||||
--index-url https://pypi.tuna.tsinghua.edu.cn/simple \
|
||||
--trusted-host pypi.tuna.tsinghua.edu.cn
|
||||
|
||||
# 预下载 MinerU 模型（构建时执行，加速启动）
|
||||
RUN python -c "
|
||||
import os
|
||||
os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
|
||||
try:
|
||||
from magic_pdf.model.doc_analyze_by_custom_model import ModelSingleton
|
||||
print('MinerU 模型下载完成')
|
||||
except Exception as e:
|
||||
print(f'模型下载跳过（将在运行时下载）: {e}')
|
||||
" || true
|
||||
|
||||
COPY main.py .
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \
|
||||
CMD curl -f http://localhost:8011/health || exit 1
|
||||
|
||||
EXPOSE 8011
|
||||
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8011", "--workers", "1"]
|
||||
136
services/mcp-server/main.py
Normal file
136
services/mcp-server/main.py
Normal file
@@ -0,0 +1,136 @@
|
||||
import os
|
||||
import tempfile
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, UploadFile, File, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEVICE = os.getenv("DEVICE", "cpu")
|
||||
UPLOAD_DIR = Path(os.getenv("UPLOAD_DIR", "/app/uploads"))
|
||||
PARSED_DIR = Path(os.getenv("PARSED_DIR", "/app/parsed"))
|
||||
|
||||
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
||||
PARSED_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
app = FastAPI(title="MinerU 文档解析服务")
|
||||
|
||||
SUPPORTED_TYPES = {
|
||||
"application/pdf": "pdf",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
|
||||
"application/msword": "doc",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
|
||||
}
|
||||
|
||||
|
||||
def parse_pdf_mineru(pdf_path: str) -> str:
|
||||
"""使用 MinerU 解析 PDF"""
|
||||
try:
|
||||
from magic_pdf.data.data_reader_writer import FileBasedDataWriter
|
||||
from magic_pdf.pipe.UnicodeFormulaPDFPipe import UnicodeFormulaPDFPipe
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
writer = FileBasedDataWriter(tmpdir)
|
||||
pipe = UnicodeFormulaPDFPipe(pdf_path, writer)
|
||||
pipe.pipe_classify()
|
||||
pipe.pipe_analyze()
|
||||
pipe.pipe_parse()
|
||||
md_content = pipe.pipe_mk_uni_format(tmpdir, drop_mode="none")
|
||||
return md_content or ""
|
||||
except Exception as e:
|
||||
logger.warning(f"MinerU 解析失败，降级到 PyMuPDF：{e}")
|
||||
return parse_pdf_pymupdf(pdf_path)
|
||||
|
||||
|
||||
def parse_pdf_pymupdf(pdf_path: str) -> str:
|
||||
"""降级：使用 PyMuPDF 提取文本"""
|
||||
try:
|
||||
import fitz # PyMuPDF
|
||||
doc = fitz.open(pdf_path)
|
||||
pages = []
|
||||
for i, page in enumerate(doc):
|
||||
text = page.get_text()
|
||||
if text.strip():
|
||||
pages.append(f"## 第 {i+1} 页\n\n{text}")
|
||||
return "\n\n".join(pages)
|
||||
except Exception as e:
|
||||
return f"[解析失败：{e}]"
|
||||
|
||||
|
||||
def parse_docx(file_path: str) -> str:
|
||||
"""解析 Word 文档"""
|
||||
try:
|
||||
from docx import Document
|
||||
doc = Document(file_path)
|
||||
parts = []
|
||||
for para in doc.paragraphs:
|
||||
if para.text.strip():
|
||||
style = para.style.name if para.style else ""
|
||||
if "Heading" in style:
|
||||
level = style.replace("Heading ", "").strip()
|
||||
try:
|
||||
prefix = "#" * int(level)
|
||||
except ValueError:
|
||||
prefix = "##"
|
||||
parts.append(f"{prefix} {para.text}")
|
||||
else:
|
||||
parts.append(para.text)
|
||||
for table in doc.tables:
|
||||
rows = []
|
||||
for row in table.rows:
|
||||
rows.append(" | ".join(cell.text.strip() for cell in row.cells))
|
||||
if rows:
|
||||
parts.append("\n".join(rows))
|
||||
return "\n\n".join(parts)
|
||||
except Exception as e:
|
||||
return f"[Word 解析失败：{e}]"
|
||||
|
||||
|
||||
class ParseResponse(BaseModel):
|
||||
filename: str
|
||||
markdown: str
|
||||
page_count: int
|
||||
parser: str
|
||||
|
||||
|
||||
@app.post("/mineru-parse", response_model=ParseResponse)
|
||||
async def mineru_parse(file: UploadFile = File(...)) -> ParseResponse:
|
||||
content = await file.read()
|
||||
suffix = Path(file.filename or "doc.pdf").suffix.lower()
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
|
||||
tmp.write(content)
|
||||
tmp_path = tmp.name
|
||||
|
||||
try:
|
||||
if suffix == ".pdf":
|
||||
markdown = parse_pdf_mineru(tmp_path)
|
||||
parser = "mineru"
|
||||
elif suffix in (".docx", ".doc"):
|
||||
markdown = parse_docx(tmp_path)
|
||||
parser = "python-docx"
|
||||
else:
|
||||
raise HTTPException(status_code=415, detail=f"不支持的文件类型：{suffix}")
|
||||
|
||||
page_count = markdown.count("## 第") if suffix == ".pdf" else markdown.count("\n\n")
|
||||
return ParseResponse(
|
||||
filename=file.filename or "unknown",
|
||||
markdown=markdown,
|
||||
page_count=max(page_count, 1),
|
||||
parser=parser,
|
||||
)
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
|
||||
@app.post("/parse-document", response_model=ParseResponse)
|
||||
async def parse_document(file: UploadFile = File(...)) -> ParseResponse:
|
||||
return await mineru_parse(file)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok", "device": DEVICE}
|
||||
11
services/mcp-server/requirements.txt
Normal file
11
services/mcp-server/requirements.txt
Normal file
@@ -0,0 +1,11 @@
|
||||
fastapi>=0.115
|
||||
uvicorn[standard]>=0.30
|
||||
pydantic>=2.7
|
||||
python-multipart>=0.0.9
|
||||
httpx>=0.27
|
||||
# MinerU 文档解析
|
||||
mineru[pipeline]>=1.0
|
||||
# Word/Excel 降级解析
|
||||
python-docx>=1.1
|
||||
openpyxl>=3.1
|
||||
PyMuPDF>=1.24 # PDF 降级解析
|
||||
Reference in New Issue
Block a user