第一次
This commit is contained in:
ZhuJW
@@ -0,0 +1,676 @@
|
||||
ROOT DB API 使用文档
|
||||
========================
|
||||
|
||||
> 版本：1.0.0
|
||||
|
||||
---
|
||||
|
||||
目录
|
||||
----
|
||||
|
||||
* [Bags](#bags)
|
||||
* [FST](#fst)
|
||||
* [Geometry](#geometry)
|
||||
* [Projects](#projects)
|
||||
* [Tags](#tags)
|
||||
* [Topics](#topics)
|
||||
|
||||
---
|
||||
|
||||
Bags
|
||||
----
|
||||
|
||||
### 1. 获取全部 Bags（按项目分组）
|
||||
```
|
||||
GET /api/bags/all
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/bags/all
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"Project A": ["Bag 1", "Bag 2"],
|
||||
"Uncategorized": ["Bag 3"]
|
||||
}
|
||||
```
|
||||
|
||||
#### 错误响应 500
|
||||
```json
|
||||
{"error": "An error occurred"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. 批量查询 FST 节点
|
||||
```
|
||||
POST /api/bags/fst/nodes
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["a.bag", "b.bag"]
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/fst/nodes \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '["a.bag", "b.bag"]'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"a.bag": [
|
||||
{"name": "node1", "start": 0, "end": 10},
|
||||
{"name": "node2", "start": 11, "end": 20}
|
||||
],
|
||||
"b.bag": [
|
||||
{"name": "node3", "start": 21, "end": 30}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### 错误响应
|
||||
- `400`：Body must be a list of bag names
|
||||
- `500`：An error occurred
|
||||
|
||||
---
|
||||
|
||||
### 3. 获取生命周期信息
|
||||
```
|
||||
POST /api/bags/life_cycle
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
{"bag_names": ["a.bag", "b.bag"]}
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/life_cycle \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"bag_names": ["a.bag", "b.bag"]}'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"a.bag": {
|
||||
"collect_time": "2025-07-16T00:00:00Z",
|
||||
"decode_time": "2025-07-16T01:00:00Z",
|
||||
"fst_index_time": "2025-07-16T02:00:00Z",
|
||||
"mining_time": "2025-07-16T03:00:00Z",
|
||||
"auto_annotate_time": "2025-07-16T04:00:00Z",
|
||||
"manual_annotate_time": "2025-07-16T05:00:00Z",
|
||||
"clone2dev_time": "2025-07-16T06:00:00Z"
|
||||
},
|
||||
"b.bag": { }
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. 批量查询 Minerva 主记录（按 session_id）
|
||||
```
|
||||
POST /api/bags/minerva
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
{"session_ids": ["p1.bag", "p2.bag"]}
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/minerva \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"session_ids": ["p1.bag", "p2.bag"]}'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"p1.bag": [{
|
||||
"session_id": "p1.bag",
|
||||
"vin": "L123456789",
|
||||
"platform": "Platform A",
|
||||
"start_ts": 1626307200,
|
||||
"end_ts": 1626314400,
|
||||
"length": 7200,
|
||||
"datetime": "2025-07-16T00:00:00Z",
|
||||
"path": "/path/to/bag",
|
||||
"converted_path": "/path/to/converted",
|
||||
"gt_path": "/path/to/gt",
|
||||
"reserved_json": {}
|
||||
}],
|
||||
"p2.bag": []
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 5. 批量查询 Minerva 详情（按 bag name）
|
||||
```
|
||||
POST /api/bags/minerva/detail
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["p1.bag", "p2.bag"]
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/minerva/detail \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '["p1.bag", "p2.bag"]'
|
||||
```
|
||||
|
||||
#### 响应格式同 `/api/bags/minerva`
|
||||
|
||||
---
|
||||
|
||||
### 6. 批量查询 Pangu 主记录
|
||||
```
|
||||
POST /api/bags/pangu
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
{"names": ["name1", "name2"]}
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/pangu \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"names": ["name1", "name2"]}'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"name1": [{
|
||||
"name": "Pangu 1",
|
||||
"datetime": "2025-07-16T00:00:00Z",
|
||||
"vehicle": "Vehicle A",
|
||||
"raw_bag_path": "/path/to/bag",
|
||||
"decoded_path": "/path/to/decoded",
|
||||
"reserved_str": "Some reserved string",
|
||||
"reserved_json": {}
|
||||
}],
|
||||
"name2": []
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 7. 批量查询 Pangu 详情（按 bag name）
|
||||
```
|
||||
POST /api/bags/pangu/detail
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["bag1.bag", "bag2.bag"]
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/pangu/detail \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '["bag1.bag", "bag2.bag"]'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"bag1.bag": {"detail": "...", "other_property": "..."},
|
||||
"bag2.bag": {"detail": "...", "other_property": "..."}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 8. 搜索 Main Pangu
|
||||
```
|
||||
POST /api/bags/search/pangu
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
{
|
||||
"name": "Pangu 1",
|
||||
"vehicle": "Vehicle A",
|
||||
"datetime_from": "2025-07-16 00:00:00",
|
||||
"datetime_to": "2025-07-16 23:59:59"
|
||||
}
|
||||
```
|
||||
字段均可选。
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/search/pangu \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "Pangu 1"}'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
[
|
||||
{
|
||||
"name": "Pangu 1",
|
||||
"datetime": "2025-07-16T00:00:00Z",
|
||||
"vehicle": "Vehicle A"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 9. 聚合查询列表
|
||||
```
|
||||
POST /api/bags/search/aggregate?page=1&per_page=20
|
||||
```
|
||||
|
||||
#### 请求参数
|
||||
|
||||
参数来源说明：
|
||||
- collect_start / collect_end：前端日期选择
|
||||
- vehicles：GET /api/bags/vehicles
|
||||
- sw_version / hw_version：GET /api/bags/versions
|
||||
- topics：GET /api/topics/all
|
||||
- tags：GET /api/tags/all
|
||||
- gt_names：GET /api/gt/types
|
||||
- fst_level1：GET /api/fst/print_tree（取 level=1 的节点名；或用 /api/fst/all_nodes 过滤根节点子节点）
|
||||
|
||||
Query 参数：
|
||||
- page: 页码，默认 1
|
||||
- per_page: 每页数量，默认 20，最大 100
|
||||
- debug_sql: 是否打印 SQL（仅日志输出），默认 false
|
||||
|
||||
Body 字段（均可选）：
|
||||
- collect_start: 起始日期，基于 MainPangu.datetime，格式 YYYYMMDD，例如 "20250101"
|
||||
- collect_end: 结束日期，基于 MainPangu.datetime，格式 YYYYMMDD，例如 "20250131"
|
||||
- bag_name: bag 名称模糊匹配（支持 * 通配），例如 "xxx*2025*"
|
||||
- gt_names: GT 名称列表（OR 语义）
|
||||
- fst_level1: FST 一级节点名称（包含其子孙节点）
|
||||
- vehicles: 车号列表（OR 语义，精确匹配）
|
||||
- sw_version: 软件版本，支持 * 通配，例如 "v1.*" 或 "*2024*"
|
||||
- hw_version: 硬件版本，支持 * 通配
|
||||
- topics: Topic 名称列表（OR 语义）
|
||||
- tags: Tag 名称列表（OR 语义）
|
||||
|
||||
#### 请求体示例
|
||||
```json
|
||||
{
|
||||
"collect_start": "20250101",
|
||||
"collect_end": "20250131",
|
||||
"bag_name": "xxx*2025*",
|
||||
"gt_names": ["gt_a", "gt_b"],
|
||||
"fst_level1": "root_fst",
|
||||
"vehicles": ["PL061763"],
|
||||
"sw_version": "v1.*",
|
||||
"hw_version": "h1.*",
|
||||
"topics": ["topic_a", "topic_b"],
|
||||
"tags": ["tag_a", "tag_b"]
|
||||
}
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST "https://<host>/api/bags/search/aggregate?page=1&per_page=20&debug_sql=true" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"collect_start":"20250101","collect_end":"20250131","bag_name":"xxx*2025*","gt_names":["gt_a"],"fst_level1":"root_fst","vehicles":["PL061763"],"sw_version":"v1.*","hw_version":"h1.*","topics":["topic_a"],"tags":["tag_a"]}'
|
||||
```
|
||||
|
||||
#### 响应字段
|
||||
- items: 列表
|
||||
- bag_id: bag_list.id
|
||||
- bag_name: bag 名称
|
||||
- bag_path: 原始 bag 路径（空则返回 ""）
|
||||
- data_path: 解析后数据路径（空则返回 ""）
|
||||
- datetime: MainPangu.datetime（YYYY-MM-DD HH:MM:SS，空则返回 ""）
|
||||
- collect_time: BagLifecycle.collect_time（YYYY-MM-DD HH:MM:SS，空则返回 ""）
|
||||
- vehicle: 车号（空则返回 ""）
|
||||
- sw_version: 软件版本（空则返回 ""）
|
||||
- hw_version: 硬件版本（空则返回 ""）
|
||||
- mviz_link: 可视化链接（预留，当前为空字符串）
|
||||
- mbviz_link: 可视化链接（预留，当前为空字符串）
|
||||
- fst: FST 路径数组（可能多条，level1 对应业务一级节点，level0(root) 不返回）
|
||||
- level1/level2/level3/level4: 对应层级节点对象，缺失则为 {}
|
||||
- gt: 关联 GT 列表（包含 id/name/type/path/comment）
|
||||
- topics: 关联 Topic 列表（包含 id/name/type）
|
||||
- tags: 关联 Tag 列表（包含 id/name/type/creator）
|
||||
- page: 当前页
|
||||
- per_page: 每页数量
|
||||
- total: 总数
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"bag_id": 123,
|
||||
"bag_name": "xxx.bag",
|
||||
"bag_path": "/path/to/bag",
|
||||
"data_path": "/path/to/data",
|
||||
"datetime": "2025-07-16 00:00:00",
|
||||
"collect_time": "2025-07-16 00:00:00",
|
||||
"vehicle": "PL061763",
|
||||
"sw_version": "v1.2.3",
|
||||
"hw_version": "h1.0",
|
||||
"mviz_link": "",
|
||||
"mbviz_link": "",
|
||||
"fst": [
|
||||
{
|
||||
"level1": {"id": 1, "name": "L1"},
|
||||
"level2": {"id": 2, "name": "L2"},
|
||||
"level3": {"id": 3, "name": "L3"},
|
||||
"level4": {"id": 4, "name": "L4"}
|
||||
}
|
||||
],
|
||||
"gt": [
|
||||
{"id": 10, "name": "gt_a", "type": "3DBox", "path": "/path/to/gt", "comment": "备注"}
|
||||
],
|
||||
"topics": [
|
||||
{"id": 20, "name": "topic_a", "type": "sensor_msgs/PointCloud2"}
|
||||
],
|
||||
"tags": [
|
||||
{"id": 30, "name": "tag_a", "type": "user", "creator": "alice"}
|
||||
]
|
||||
}
|
||||
],
|
||||
"page": 1,
|
||||
"per_page": 20,
|
||||
"total": 123
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 10. 获取可用版本列表
|
||||
```
|
||||
GET /api/bags/versions
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/bags/versions
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"sw_versions": ["v1.0", "v1.1", "v2.0"],
|
||||
"hw_versions": ["h1.0", "h2.0"]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
|
||||
### 11. 获取可用车号列表
|
||||
```
|
||||
GET /api/bags/vehicles
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/bags/vehicles
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"vehicles": ["PL061763", "PL061764"]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 12. 批量查询 Tags
|
||||
```
|
||||
POST /api/bags/tags
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["a.bag", "b.bag"]
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"a.bag": [
|
||||
{"name": "tag1", "type": "type1"},
|
||||
{"name": "tag2", "type": "type2"}
|
||||
],
|
||||
"b.bag": [
|
||||
{"name": "tag3", "type": "type3"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 13. 批量查询 Topics
|
||||
```
|
||||
POST /api/bags/topics
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["a.bag", "b.bag"]
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"a.bag": [
|
||||
{"name": "topic1", "type": "type1"},
|
||||
{"name": "topic2", "type": "type2"}
|
||||
],
|
||||
"b.bag": [
|
||||
{"name": "topic3", "type": "type3"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
FST
|
||||
---
|
||||
|
||||
### 1. 批量获取 FST → Bags 映射
|
||||
```
|
||||
POST /api/fst/bags/nodes
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["fst1", "fst2"]
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"fst1": ["Bag1", "Bag2"],
|
||||
"fst2": ["Bag3"]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. 获取指定 FST 及其子节点的 Bags
|
||||
```
|
||||
GET /api/fst/bags/path/{name}
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/fst/bags/path/fst1
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"fst1": {"bags": ["Bag1", "Bag2"]},
|
||||
"fst1_child": {"bags": ["Bag3"]}
|
||||
}
|
||||
```
|
||||
|
||||
#### 错误响应 404：Specified FST not found
|
||||
|
||||
---
|
||||
|
||||
### 3. 获取 FST 树形结构（纯文本）
|
||||
```
|
||||
GET /api/fst/print_tree
|
||||
```
|
||||
#### 成功响应 200
|
||||
```
|
||||
<pre>
|
||||
Your FST tree output here
|
||||
</pre>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. 获取单个 FST 详情
|
||||
```
|
||||
GET /api/fst/{name}
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/fst/fst1
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"name": "fst1",
|
||||
"parent_node": "Parent FST Name",
|
||||
"linked_bags_sum": 5
|
||||
}
|
||||
```
|
||||
|
||||
#### 错误响应 404：Specified FST not found
|
||||
|
||||
---
|
||||
|
||||
Geometry
|
||||
--------
|
||||
|
||||
### 批量获取 Rosbag 的 Geometry 信息
|
||||
```
|
||||
POST /api/geometry/
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
{"rosbag_names": ["rosbag1", "rosbag2"]}
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"rosbag1": [[12.345678, 12.345678, 0.0]],
|
||||
"rosbag2": [[98.765432, 98.765432, 0.0]]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Projects
|
||||
--------
|
||||
|
||||
### 获取所有项目
|
||||
```
|
||||
GET /api/projects/all
|
||||
```
|
||||
#### 成功响应 200
|
||||
```json
|
||||
[
|
||||
{"id": 1, "name": "Project Name"}
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Tags
|
||||
----
|
||||
|
||||
### 1. 获取全部 Tags
|
||||
```
|
||||
GET /api/tags/all
|
||||
```
|
||||
#### 成功响应 200
|
||||
```json
|
||||
[
|
||||
{
|
||||
"name": "Data Quality",
|
||||
"type": "quality",
|
||||
"creator": "alice@example.com"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. 根据 Tag 名称获取关联 Bags
|
||||
```
|
||||
GET /api/tags/bags/{tag_name}
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/tags/bags/Data%20Quality
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{"bags": ["bag_2024_07"]}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 3. 根据 Creator 获取 Tags
|
||||
```
|
||||
GET /api/tags/creators/{creator}
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/tags/creators/alice@example.com
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{"tags": ["Sales", "Data Quality"]}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Topics
|
||||
------
|
||||
|
||||
### 1. 获取全部 Topics
|
||||
```
|
||||
GET /api/topics/all
|
||||
```
|
||||
#### 成功响应 200
|
||||
```json
|
||||
[
|
||||
{"name": "Topic Name", "type": "Topic Type"}
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. 根据 Topic 名称获取关联 Bags
|
||||
```
|
||||
GET /api/topics/bags/{topic_name}
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/topics/bags/topic1
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{"name": ["Bag Name"]}
|
||||
```
|
||||
|
||||
---
|
||||
@@ -0,0 +1,2 @@
|
||||
# Micro Service apps
|
||||
Backend micro service application for fst data production line.
|
||||
@@ -0,0 +1,77 @@
|
||||
# 项目概述：
|
||||
MTA（multi trip aggregation） 管理系统是一个用于MTA数据处理任务的综合性管理系统，它包含了数据索引和下载，数据预处理和瓦片生成，路口轨迹可视化生成，html筛选，数据包下载和分类以及MTA数据处理任务的管理等功能。
|
||||
# 项目结构：
|
||||
mta_manage_system/
|
||||
├── db/
|
||||
│ └── init.sql # 数据库初始化脚本
|
||||
├── docker/
|
||||
│ └── Dockerfile # Docker 镜像构建文件
|
||||
├── models/
|
||||
│ ├── __init__.py # 模型初始化文件
|
||||
│ ├── sessionrecord.py # 会话记录模型
|
||||
│ └── taskstatus.py # 任务状态模型
|
||||
├── services/
|
||||
│ └── long_running_tasks.py # 长时间运行任务服务
|
||||
├── static/
|
||||
│ ├── css/
|
||||
│ │ ├── bootstrap.min.css # Bootstrap CSS 文件
|
||||
│ │ ├── bootstrap-icons.css # Bootstrap Icons CSS 文件
|
||||
│ │ ├── detail.css # 详情页面样式文件
|
||||
│ │ └── sweetalert2.css # SweetAlert2 样式文件
|
||||
│ ├── images/ # 图片资源目录
|
||||
│ └── js/
|
||||
│ ├── bootstrap.bundle.min.js # Bootstrap JavaScript 文件
|
||||
│ ├── detail.js # 详情页面 JavaScript 文件
|
||||
│ ├── injector.js # 注入器 JavaScript 文件
|
||||
│ ├── jquery-3.7.1.min.js # jQuery 库文件
|
||||
│ └── sweetalert2.js # SweetAlert2 库文件
|
||||
├── templates/
|
||||
│ ├── detail.html # 详情页面模板
|
||||
│ └── list.html # 列表页面模板
|
||||
├── utils/ # 工具类
|
||||
│ ├── docker_tool.py # 操作容器工具类
|
||||
│ ├── extract_package_tool.py # 通过瓦片获取包信息
|
||||
│ ├── log_tool.py # 日志工具类
|
||||
│ ├── path_tool.py # 路径处理工具类
|
||||
│ └── task_queue_tool.py # 任务管理工具类
|
||||
├── .env # 环境变量配置文件
|
||||
├── app.py # 应用入口文件
|
||||
├── config.py # 配置文件
|
||||
├── extensions.py # 扩展模块文件
|
||||
├── pyproject.toml # Python 项目配置文件
|
||||
└── README.md # 项目说明文件
|
||||
css 静态资源版本说明：
|
||||
[bootstrap-icons.css](static/css/bootstrap-icons.css) v1.11.0
|
||||
[bootstrap.min.css](static/css/bootstrap.min.css) v4.3.1
|
||||
[sweetalert2.css](static/css/sweetalert2.css) v11.26.3
|
||||
js 静态资源版本说明：
|
||||
[bootstrap.bundle.min.js](static/js/bootstrap.bundle.min.js) v4.3.1
|
||||
[jquery-3.7.1.min.js](static/js/jquery-3.7.1.min.js) v3.7.1
|
||||
[sweetalert2.js](static/js/sweetalert2.js) v11.26.3
|
||||
# 功能模块
|
||||
## 数据库
|
||||
db/目录下包含 init.sql脚本，用于初始化数据库表结构和基础数据。数据库选用了mysql 数据库。
|
||||
## 模型层
|
||||
models/目录定义了与数据库交互的模型类，如 sessionrecord.py和 taskstatus.py分别对应会话记录和任务状态的数据库操作。
|
||||
## 服务层
|
||||
services/目录中的 long_running_tasks.py处理长时间运行的任务逻辑，提供任务调度和管理功能。
|
||||
## 静态资源
|
||||
static/目录存放项目的静态资源，包括 CSS 样式文件、JavaScript 脚本文件和图片资源。
|
||||
## 模板
|
||||
templates/目录包含了前端页面的 HTML 模板文件，如 detail.html和 list.html。
|
||||
## 入口和应用配置
|
||||
app.py是应用的入口点，负责启动 Web 服务。config.py存放项目的配置信息，extensions.py定义和初始化项目所需的扩展模块。
|
||||
|
||||
# 运行指南
|
||||
1，先启动mysql 数据库，执行初始化sql 脚本[init.sql](db/init.sql)
|
||||
2，执行 uv sync
|
||||
3，执行 python3 app.py
|
||||
项目启动会从配置文件[.env](.env)中获取配置参数，其中数据密码为你所使用的数据库账户和密码
|
||||
|
||||
|
||||
# 技术栈
|
||||
|
||||
后端：Python, Flask
|
||||
数据库：MySQL
|
||||
前端：HTML, CSS, JavaScript, Bootstrap, SweetAlert2
|
||||
容器化：Docker
|
||||
@@ -0,0 +1,23 @@
|
||||
from flask import Flask
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from config import config
|
||||
|
||||
|
||||
from extensions import db # 统一导入 db 实例
|
||||
|
||||
|
||||
def create_app():
|
||||
app = Flask(__name__)
|
||||
|
||||
# 配置
|
||||
app.config["SQLALCHEMY_DATABASE_URI"] = config.SQLALCHEMY_DATABASE_URI
|
||||
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
|
||||
|
||||
# 初始化扩展
|
||||
db.init_app(app)
|
||||
|
||||
# 注册蓝图
|
||||
# from routes import main
|
||||
# app.register_blueprint(main)
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,829 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
"""
|
||||
@Project ：bag_filter
|
||||
@File ：app.py
|
||||
@Author ：zuowe
|
||||
@Date ：2025/10/20 10:36
|
||||
@Des ：
|
||||
"""
|
||||
|
||||
import os
|
||||
from flask import Flask, render_template, send_from_directory, request, jsonify
|
||||
from datetime import datetime
|
||||
import json
|
||||
import logging
|
||||
|
||||
from utils.docker_tool import docker_tool
|
||||
from sqlalchemy import select, func
|
||||
from pathlib import Path
|
||||
from config import config, logger
|
||||
from utils.log_tool import setup_logging
|
||||
from utils.extract_package_tool import extract_package_names_from_html
|
||||
from utils.task_queue_tool import task_queue
|
||||
from utils.path_tool import PathTool
|
||||
from extensions import db # 统一导入 db 实例
|
||||
from models.sessionrecord import SessionRecord
|
||||
from models.taskstatus import TaskStatus
|
||||
from services.long_running_tasks import LongRunningTasks
|
||||
from __init__ import create_app
|
||||
|
||||
# 创建应用
|
||||
app = create_app()
|
||||
|
||||
|
||||
# 数据库连接
|
||||
# database_url = config.SQLALCHEMY_DATABASE_URI
|
||||
|
||||
BATH_ROOT = config.BATH_ROOT
|
||||
IMAGE_FOLDER = config.IMAGE_FOLDER
|
||||
PAGES_DIR = config.PAGES_DIR
|
||||
SCRIPT_PATH = config.SCRIPT_PATH
|
||||
HTML_DIR = config.HTML_DIR
|
||||
|
||||
OUTPUT_PIC_PATH = config.OUTPUT_PIC_PATH
|
||||
OUTPUT_PIC_NAME = config.OUTPUT_PIC_NAME
|
||||
OUTPUT_RESULT_PATH = config.OUTPUT_RESULT_PATH
|
||||
#
|
||||
MAX_ALLOW_RUNNING_MTA_TASK = config.MAX_ALLOW_RUNNING_MTA_TASK
|
||||
MTA_IMAGE = config.MTA_IMAGE
|
||||
MTA_IMNAGE_VERSION = config.MTA_IMNAGE_VERSION
|
||||
|
||||
CONTAINER_OUTER_PATH = config.CONTAINER_OUTER_PATH
|
||||
ENV_PARAM_KEY = config.ENV_PARAM_KEY
|
||||
ENV_PARAM_VALUE = config.ENV_PARAM_VALUE
|
||||
MTA_SCRIPT = config.MTA_SCRIPT
|
||||
|
||||
|
||||
# 允许的扩展名
|
||||
ALLOWED_EXTS = config.ALLOWED_EXTS
|
||||
|
||||
# 每页最大条数限制
|
||||
MAX_PER_PAGE = config.MAX_PER_PAGE
|
||||
DEFAULT_PER_PAGE = config.MAX_PER_PAGE
|
||||
|
||||
# 配置日志
|
||||
|
||||
logger = setup_logging("logs/app.log", logging.INFO)
|
||||
|
||||
|
||||
# ===== 数据库配置 - 使用 config 中的统一配置 =====
|
||||
# 在 config.py 中添加数据库配置
|
||||
app.config["SQLALCHEMY_DATABASE_URI"] = config.SQLALCHEMY_DATABASE_URI
|
||||
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def list_pages():
|
||||
return render_template("list.html")
|
||||
|
||||
|
||||
# 列表数据分页查询
|
||||
@app.route("/api/items")
|
||||
def api_items():
|
||||
# 1) 分页参数
|
||||
try:
|
||||
page = int(request.args.get("page", 1))
|
||||
per_page = int(request.args.get("per_page", DEFAULT_PER_PAGE))
|
||||
if page < 1:
|
||||
page = 1
|
||||
if per_page < 1 or per_page > MAX_PER_PAGE:
|
||||
per_page = DEFAULT_PER_PAGE
|
||||
except ValueError:
|
||||
page = 1
|
||||
per_page = DEFAULT_PER_PAGE
|
||||
|
||||
# 2) 过滤参数
|
||||
html_name = request.args.get("html_name", "").strip()
|
||||
central_session = request.args.get("central_session", "").strip()
|
||||
mta_task_status = request.args.get("mta_task_status", "").strip()
|
||||
is_filter_bag = request.args.get("is_filter_bag", "")
|
||||
is_run_mta = request.args.get("is_run_mta", "")
|
||||
logger.info(
|
||||
f"is_filter_bag 值为：{is_filter_bag} ，is_run_mta 值为：{is_run_mta} "
|
||||
)
|
||||
# 将字符串转换为布尔值
|
||||
|
||||
if request.args.get("is_filter_bag", "").strip().lower() == "true":
|
||||
is_filter_bag = True
|
||||
elif request.args.get("is_filter_bag", "").strip().lower() == "false":
|
||||
is_filter_bag = False
|
||||
else:
|
||||
is_filter_bag = request.args.get("is_filter_bag", "").strip()
|
||||
if request.args.get("is_run_mta", "").strip().lower() == "true":
|
||||
is_run_mta = True
|
||||
elif request.args.get("is_run_mta", "").strip().lower() == "false":
|
||||
is_run_mta = False
|
||||
else:
|
||||
is_run_mta = request.args.get("is_run_mta", "").strip()
|
||||
|
||||
# 3) 构建 COUNT 查询（与主查询条件保持一致）
|
||||
stmt_count = select(func.count(SessionRecord.id)).where(
|
||||
(SessionRecord.html_name.like(f"%{html_name}%")) if html_name else True,
|
||||
(SessionRecord.central_session.like(f"%{central_session}%"))
|
||||
if central_session
|
||||
else True,
|
||||
# 处理 is_filter_bag
|
||||
(SessionRecord.is_filter_bag == True)
|
||||
if is_filter_bag == True
|
||||
else (SessionRecord.is_filter_bag == False)
|
||||
if is_filter_bag == False
|
||||
else True,
|
||||
# 处理 is_run_mta
|
||||
(SessionRecord.is_run_mta == True)
|
||||
if is_run_mta == True
|
||||
else (SessionRecord.is_run_mta == False)
|
||||
if is_run_mta == False
|
||||
else True,
|
||||
# 处理 mta_task_status
|
||||
(SessionRecord.mta_task_status == mta_task_status)
|
||||
if mta_task_status and mta_task_status.strip() # 不为空且非空白字符串
|
||||
else True,
|
||||
)
|
||||
total = db.session.scalar(stmt_count) or 0
|
||||
|
||||
# 4) 构建分页数据查询（在 select 上链式调用 offset/limit）
|
||||
stmt_data = (
|
||||
select(SessionRecord)
|
||||
.where(
|
||||
(SessionRecord.html_name.like(f"%{html_name}%")) if html_name else True,
|
||||
(SessionRecord.central_session.like(f"%{central_session}%"))
|
||||
if central_session
|
||||
else True,
|
||||
# 处理 is_filter_bag
|
||||
(SessionRecord.is_filter_bag == True)
|
||||
if is_filter_bag == True
|
||||
else (SessionRecord.is_filter_bag == False)
|
||||
if is_filter_bag == False
|
||||
else True,
|
||||
# 处理 is_run_mta
|
||||
(SessionRecord.is_run_mta == True)
|
||||
if is_run_mta == True
|
||||
else (SessionRecord.is_run_mta == False)
|
||||
if is_run_mta == False
|
||||
else True,
|
||||
# 处理 mta_task_status
|
||||
(SessionRecord.mta_task_status == mta_task_status)
|
||||
if mta_task_status and mta_task_status.strip() # 不为空且非空白字符串
|
||||
else True,
|
||||
)
|
||||
.order_by(SessionRecord.id.asc()) # 建议使用索引字段排序
|
||||
.offset((page - 1) * per_page)
|
||||
.limit(per_page)
|
||||
)
|
||||
|
||||
rows = db.session.execute(stmt_data).scalars().all()
|
||||
logger.info(it.to_dict() for it in rows)
|
||||
|
||||
# 5) 计算分页元信息
|
||||
total_pages = (total + per_page - 1) // per_page
|
||||
has_prev = page > 1
|
||||
has_next = page < total_pages
|
||||
|
||||
# 7) 返回 JSON
|
||||
return jsonify({
|
||||
"items": [it.to_dict() for it in rows],
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"total": total,
|
||||
"total_pages": total_pages,
|
||||
"has_prev": has_prev,
|
||||
"has_next": has_next,
|
||||
})
|
||||
|
||||
|
||||
@app.route("/api/syncdata", methods=["GET"])
|
||||
def sync_data():
|
||||
try:
|
||||
bag_folder = request.args.get("bag_folder")
|
||||
html_folder = request.args.get("html_folder")
|
||||
logger.info(
|
||||
f"获取到的参数：bag_folder = {bag_folder}，html_folder = {html_folder} "
|
||||
)
|
||||
# html_folder ="selected_8to9"
|
||||
# 1. 获取/data/html目录下所有html文件
|
||||
html_dir = PathTool.join(BATH_ROOT, html_folder)
|
||||
if not os.path.exists(html_dir):
|
||||
return jsonify({"error": "HTML directory not found"}), 404
|
||||
|
||||
html_files = [f for f in os.listdir(html_dir) if f.endswith(".html")]
|
||||
# logger.info(html_files)
|
||||
# 2. 获取数据库中已有的html_name列表
|
||||
existing_records = (
|
||||
SessionRecord.query.filter(SessionRecord.html_folder == html_folder)
|
||||
.with_entities(SessionRecord.html_name)
|
||||
.all()
|
||||
)
|
||||
existing_names = {r[0] for r in existing_records}
|
||||
|
||||
# 3. 比对并插入新记录
|
||||
new_count = 0
|
||||
for filename in html_files:
|
||||
if filename not in existing_names:
|
||||
new_record = SessionRecord(
|
||||
html_name=filename,
|
||||
html_folder=html_folder,
|
||||
bag_folder=bag_folder,
|
||||
central_session="", # 默认值
|
||||
query_sessions="", # 默认空JSON数组
|
||||
selected_image_url="null",
|
||||
is_filter_bag=False,
|
||||
is_run_mta=False,
|
||||
run_mta_result="",
|
||||
remark="自动同步添加",
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
)
|
||||
db.session.add(new_record)
|
||||
new_count += 1
|
||||
|
||||
db.session.commit()
|
||||
|
||||
return jsonify({
|
||||
"status": "success",
|
||||
"scanned_files": len(html_files),
|
||||
"new_records_added": new_count,
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
print(e)
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
# 接口：GET /api/items/<id>
|
||||
# =======================
|
||||
@app.route("/api/items/<int:id>", methods=["GET"])
|
||||
def get_item(id: int):
|
||||
# db: Session = SessionLocal()
|
||||
try:
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
if not item:
|
||||
return jsonify({"error": "记录不存在", "id": id}), 404
|
||||
|
||||
data = row2dict(item)
|
||||
return jsonify(status=True, result=data)
|
||||
except Exception as e:
|
||||
app.logger.exception("查询接口异常")
|
||||
return jsonify({"error": "服务器内部错误", "detail": str(e)}), 500
|
||||
|
||||
|
||||
# ======================
|
||||
# 接口：POST /api/items （新增）
|
||||
# PUT /api/items/<id> （更新）
|
||||
# ======================
|
||||
@app.route("/api/saveitems", methods=["POST"])
|
||||
def save_or_update_item(id=None):
|
||||
try:
|
||||
data = request.get_json()
|
||||
if not data:
|
||||
return jsonify({"error": "请求体必须为 JSON"}), 400
|
||||
|
||||
# 1) 获取必填字段校验（可根据需求调整）
|
||||
id = data.get("id")
|
||||
html_name = data.get("html_name")
|
||||
central_session = data.get("central_session")
|
||||
query_session = data.get("query_session")
|
||||
if not html_name or not central_session:
|
||||
return jsonify({"error": "html_name 和 central_session 为必填项"}), 400
|
||||
logger.info(f"查询到的 query_session 为：{query_session}")
|
||||
# 2) 处理布尔字段：转为 Python Boolean 或 None
|
||||
is_filter_bag = str_to_bool(data.get("is_filter_bag"))
|
||||
is_run_mta = str_to_bool(data.get("is_run_mta"))
|
||||
success = str_to_bool(data.get("success"))
|
||||
|
||||
# 3) 判断是新增还是更新
|
||||
if id is None:
|
||||
# 🟢 新增记录
|
||||
item = SessionRecord(
|
||||
html_name=html_name,
|
||||
central_session=central_session,
|
||||
query_sessions=data.get("query_session"), # 可为字符串或后续转 JSON
|
||||
selected_image_url=data.get("selected_image_url"),
|
||||
is_filter_bag=is_filter_bag,
|
||||
is_run_mta=is_run_mta,
|
||||
run_mta_result=data.get("run_mta_result"), # 可为 JSON 字符串或对象
|
||||
remark=data.get("remark"),
|
||||
success=success,
|
||||
)
|
||||
|
||||
db.session.add(item)
|
||||
db.session.flush() # 为了获取自增 id（如有需要）
|
||||
db.session.commit()
|
||||
|
||||
return jsonify({
|
||||
"status": "success",
|
||||
"message": "记录新增成功",
|
||||
"data": {
|
||||
**item.__dict__,
|
||||
"_sa_instance_state": None,
|
||||
}, # 去掉内部属性
|
||||
}), 201
|
||||
else:
|
||||
# 🔵 更新记录
|
||||
item = (
|
||||
db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
)
|
||||
if not item:
|
||||
return jsonify({"error": f"记录不存在，id={id}"}), 404
|
||||
|
||||
# 更新字段
|
||||
item.html_name = html_name
|
||||
item.central_session = central_session
|
||||
item.query_sessions = data.get("query_session")
|
||||
item.selected_image_url = data.get("selected_image_url")
|
||||
item.is_filter_bag = is_filter_bag
|
||||
item.is_run_mta = is_run_mta
|
||||
item.run_mta_result = data.get("run_mta_result")
|
||||
item.remark = data.get("remark")
|
||||
# item.success = success
|
||||
item.updated_at = func.now() # 可选，如果模型没有自动更新
|
||||
|
||||
db.session.commit()
|
||||
return jsonify({"status": "success", "message": "记录更新成功"})
|
||||
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
app.logger.exception("保存/更新接口异常")
|
||||
return jsonify({"error": "服务器内部错误", "detail": str(e)}), 500
|
||||
|
||||
|
||||
@app.route("/detail/<html_name>/<bag_folder>/<html_folder>/<id>")
|
||||
def filterbag_detail(html_name, bag_folder, html_folder, id):
|
||||
# 这里接受url 传参 ，将页面重定向到 详情选择页面 ，并将参数和查询的数据返回到详情页面
|
||||
# file_path = os.path.join(PAGES_DIR, html_name)
|
||||
print(id)
|
||||
# html_folder = "selected_8to9"
|
||||
# html_name = "15_27321_13493_merged_intersections_17_100.0m.html"
|
||||
return render_template(
|
||||
"detail.html",
|
||||
html_name=html_name,
|
||||
bag_folder=bag_folder,
|
||||
html_folder=html_folder,
|
||||
abslute_path=BATH_ROOT,
|
||||
id=id,
|
||||
)
|
||||
|
||||
|
||||
# 获取图片 在页面展示
|
||||
@app.route("/api/getimage/<imagename>/<bagname>/<htmlname>/<bagfolder>")
|
||||
def api_getimages(imagename, bagname, htmlname, bagfolder):
|
||||
# 安全校验：只允许访问 pages 目录下的 HTML
|
||||
# 安全拼接并限制到白名单目录
|
||||
# bagname = "PL162802_event_all_time_event_20250628-131016_0.bag.dir"
|
||||
logger.info(f"image aaa path : imagename: {imagename}")
|
||||
base = PathTool.join(BATH_ROOT, bagfolder, htmlname, bagname, IMAGE_FOLDER)
|
||||
logger.info(f"image aaa path :{base} , bagname: {bagname}")
|
||||
|
||||
# 如果找到图片则返回图片数据
|
||||
if imagename in os.listdir(base):
|
||||
return send_from_directory(base, imagename)
|
||||
|
||||
return jsonify({"error": "Not found or access denied"})
|
||||
|
||||
|
||||
# 获取图片 在页面展示
|
||||
@app.route("/api/getmtaresult/<id>")
|
||||
def api_getmtaresult(id):
|
||||
# 安全校验：只允许访问 pages 目录下的 HTML
|
||||
# 安全拼接并限制到白名单目录
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
bagfolder = item.bag_folder
|
||||
html_name = item.html_name
|
||||
htmlname = html_name.replace(".html", "")
|
||||
bagname = item.central_session
|
||||
imagename = item.run_mta_result
|
||||
|
||||
base = PathTool.join(BATH_ROOT, bagfolder, htmlname, bagname, OUTPUT_PIC_PATH)
|
||||
logger.info(f"image aaa path :{base} , bagname: {bagname}")
|
||||
|
||||
# 如果找到图片则返回图片数据
|
||||
if imagename in os.listdir(base):
|
||||
return send_from_directory(base, OUTPUT_PIC_NAME)
|
||||
|
||||
return jsonify({"error": "Not found or access denied"})
|
||||
|
||||
|
||||
# 获取图片 在页面展示
|
||||
@app.route("/api/getimage")
|
||||
def api_getimage():
|
||||
bagname = request.args.get("bagname")
|
||||
imageIndex = request.args.get("imageIndex")
|
||||
htmlname = request.args.get("htmlname")
|
||||
bagfolder = request.args.get("bagfolder", "")
|
||||
|
||||
logger.info(f"htmlname的地址为：{str(htmlname)},bagfolder :{bagfolder} ")
|
||||
# bagname = "PL162802_event_all_time_event_20250628-131016_0.bag.dir"
|
||||
if not bagname:
|
||||
return jsonify(status=False, message="missing bagname"), 400
|
||||
if imageIndex is None:
|
||||
return jsonify(status=False, message="missing imageIndex"), 400
|
||||
root = Path(BATH_ROOT) / bagfolder / htmlname / bagname / IMAGE_FOLDER
|
||||
image_list = sorted(root.glob("*.jpg"))
|
||||
logger.info(f"拼接后的地址为：{str(root)}")
|
||||
|
||||
imageIndex = int(imageIndex.strip())
|
||||
# 用户使用习惯的索引比 list 索引大1 ，所以这里要减去1
|
||||
imageIndex -= 1
|
||||
# 通过index 获取 list 中选择的图片
|
||||
imagename = str(image_list[imageIndex])
|
||||
payload = {"size": len(image_list), "imagename": imagename}
|
||||
return jsonify(status=True, result=payload)
|
||||
|
||||
|
||||
@app.route("/api/options/<html_name>/<html_folder>")
|
||||
def api_pages(html_name, html_folder):
|
||||
logger.info(f"获取到的参数： html_name = {html_name}，html_folder = {html_folder} ")
|
||||
html_dir = PathTool.join(BATH_ROOT, html_folder)
|
||||
# 可在此处读取html文件 包 内容用于展示
|
||||
file_path = os.path.join(html_dir, html_name)
|
||||
|
||||
# 按“键名全小写”的约定返回，前端可通过 res.ok 与 res.data 访问
|
||||
bagset = extract_package_names_from_html(file_path, logger)
|
||||
|
||||
data = [
|
||||
{"value": k, "label": k}
|
||||
for k in sorted(bagset) # 如需保持原集合无序，可去掉 sorted()
|
||||
]
|
||||
|
||||
return jsonify(status=True, result=data)
|
||||
|
||||
|
||||
@app.route("/api/getimagedata")
|
||||
def get_data():
|
||||
bagname = request.args.get("bagname")
|
||||
htmlname = request.args.get("htmlname", "")
|
||||
bagfolder = request.args.get("bagfolder", "")
|
||||
|
||||
logger.info(f"获取到的参数 htmlname ：{htmlname}")
|
||||
path = PathTool.join(BATH_ROOT, bagfolder, htmlname, bagname, IMAGE_FOLDER)
|
||||
p = Path(path)
|
||||
image_list = sorted(p.glob("*.jpg"))
|
||||
logger.info(len(image_list))
|
||||
imagename = str(image_list[0])
|
||||
# 构造返回：size 为列表大小；first 为第一个元素或 null
|
||||
payload = {
|
||||
"size": len(image_list),
|
||||
"imagename": imagename,
|
||||
"index": str(image_list[0])
|
||||
if image_list
|
||||
else None, # 非 JSON 类型在此统一转 str
|
||||
}
|
||||
return jsonify(status=True, result=payload)
|
||||
|
||||
|
||||
def fmt_date(dt: datetime) -> str:
|
||||
return dt.strftime("%Y-%m-%d %H:%M:%S") if dt else "-"
|
||||
|
||||
|
||||
def to_camel_case(snake: str) -> str:
|
||||
parts = snake.split("_")
|
||||
return parts[0] + "".join(p.title() for p in parts[1:])
|
||||
|
||||
|
||||
def row2dict(row) -> dict:
|
||||
d = {}
|
||||
for col in row.__table__.columns:
|
||||
key = to_camel_case(col.name)
|
||||
val = getattr(row, col.name)
|
||||
if isinstance(val, datetime):
|
||||
d[key] = fmt_date(val)
|
||||
elif col.name in ("querySessions", "runMtaResult") and val is not None:
|
||||
# 若数据库字段为 JSON/Text，这里做一次安全反序列化
|
||||
try:
|
||||
d[key] = json.loads(val) if isinstance(val, str) else val
|
||||
except Exception:
|
||||
d[key] = val
|
||||
else:
|
||||
d[key] = val
|
||||
return d
|
||||
|
||||
|
||||
# ======================
|
||||
# 工具函数：字符串 "true"/"false" 转布尔值
|
||||
# ======================
|
||||
def str_to_bool(s):
|
||||
if s is None:
|
||||
return None
|
||||
if s == "true":
|
||||
return True
|
||||
if s == "false":
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
# ===== 提交接口：写入 MySQL =====
|
||||
@app.route("/api/submit", methods=["POST"])
|
||||
def submit():
|
||||
try:
|
||||
# 1) 解析 JSON
|
||||
data = request.get_json(silent=True)
|
||||
if data is None:
|
||||
return jsonify({"error": "请求体不是合法的 JSON"}), 400
|
||||
|
||||
# 2) 兼容两种可能的前端结构
|
||||
# 结构A（当前表单字段）: central_session, query_sessions, selected_image_url
|
||||
# 结构B（你示例 alert 用到）: tags（数组）, category
|
||||
|
||||
central_session = data.get("category")
|
||||
query_sessions = data.get("tags")
|
||||
bag_folder = data.get("bag_folder")
|
||||
html_name = data.get("html_name")
|
||||
html_name_folder = html_name.replace(".html", "")
|
||||
logger.info(bag_folder)
|
||||
bagpath = PathTool.join(BATH_ROOT, bag_folder, html_name_folder)
|
||||
logger.info("拼接的数据包根路径为：" + bagpath)
|
||||
new_query_sessions = list(
|
||||
map(lambda x: PathTool.join(bagpath, x), query_sessions)
|
||||
)
|
||||
logger.info(f"query_sessions:{query_sessions}")
|
||||
query_sessions = ",".join(query_sessions)
|
||||
|
||||
central_session_str = PathTool.join(bagpath, central_session)
|
||||
query_sessions_str = " ".join(new_query_sessions)
|
||||
logger.info(
|
||||
f"query_sessions_str:{query_sessions_str}, central_session_str : {central_session_str}"
|
||||
)
|
||||
|
||||
# 2) 写入数据库
|
||||
|
||||
id = data.get("id")
|
||||
logger.info(f"查询到的数据ID ： {id}")
|
||||
# 🔵 更新记录
|
||||
# 查询记录
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
|
||||
if not item:
|
||||
return jsonify({"error": f"记录不存在，html_name={html_name}"}), 404
|
||||
|
||||
# 动态更新字段（仅更新传入的字段）
|
||||
# 仅更新指定字段
|
||||
if central_session is not None:
|
||||
item.central_session = central_session
|
||||
if query_sessions is not None:
|
||||
item.query_sessions = query_sessions
|
||||
|
||||
# 更新是否筛选包字段信息 是否跑 mta 需要看弹出框
|
||||
item.is_filter_bag = True
|
||||
|
||||
# item.is_run_mta = True
|
||||
|
||||
# 更新修改时间
|
||||
item.updated_at = func.now()
|
||||
|
||||
db.session.commit()
|
||||
|
||||
payload = {
|
||||
"status": "success",
|
||||
"message": "数据提交成功",
|
||||
"data": {
|
||||
"central_session": central_session_str,
|
||||
"query_sessions": query_sessions_str,
|
||||
},
|
||||
}
|
||||
|
||||
return jsonify(status=True, result=payload)
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
app.logger.exception("提交失败")
|
||||
return jsonify({"error": f"服务器异常：{str(e)}"}), 500
|
||||
|
||||
|
||||
#
|
||||
# 提交 task 方法
|
||||
#
|
||||
|
||||
|
||||
# 查看 容器状态
|
||||
@app.route("/api/gettasksstatus", methods=["GET"])
|
||||
def get_tasks_status():
|
||||
"""查看队列状态"""
|
||||
status = task_queue.get_queue_status()
|
||||
|
||||
print(f"📊 当前队列状态:")
|
||||
print(f" 最大并发任务数: {status['max_concurrent_tasks']}")
|
||||
print(f" 正在执行的任务数: {status['running_count']}")
|
||||
print(f" 等待执行的任务数: {status['pending_count']}")
|
||||
print(f" 已完成的任务数: {status['successful_count']}")
|
||||
print(f" 失败的任务数: {status['failed_count']}")
|
||||
print(f" 取消的任务数: {status['cancelled_count']}")
|
||||
|
||||
return jsonify({
|
||||
"max_concurrent_tasks": status["max_concurrent_tasks"],
|
||||
"running_count": status["running_count"],
|
||||
"pending_count": status["pending_count"],
|
||||
"successful_count": status["successful_count"],
|
||||
"failed_count": status["failed_count"],
|
||||
"cancelled_count": status["cancelled_count"],
|
||||
})
|
||||
|
||||
|
||||
# 查看 容器状态
|
||||
@app.route("/api/stoptask", methods=["GET"])
|
||||
def stop_tasks():
|
||||
"""查看队列状态"""
|
||||
id = request.args.get("id")
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
if not item:
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"data": "",
|
||||
"message": "系统异常，请联系管理员",
|
||||
}), 500
|
||||
# mta_task_id = item.mta_task_id
|
||||
# result = task_queue.cancel_task(mta_task_id)
|
||||
container_name = f"MTA-{id}"
|
||||
stop_result = docker_tool.stop_container_by_name(container_name)
|
||||
item.mta_task_status = TaskStatus.FAILED.value
|
||||
item.mta_task_id = None
|
||||
item.run_mta_result = None
|
||||
item.updated_at = func.now()
|
||||
db.session.commit()
|
||||
return jsonify(stop_result)
|
||||
|
||||
|
||||
@app.route("/api/reloadtask")
|
||||
def reloadTask():
|
||||
id = request.args.get("id")
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
if not item:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"data": "",
|
||||
"message": "系统异常，请联系管理员",
|
||||
}), 200
|
||||
status = task_queue.get_queue_status()
|
||||
running_count = status["running_count"]
|
||||
bagfolder = item.bag_folder
|
||||
|
||||
html_name = item.html_name
|
||||
html_name = html_name.replace(".html", "")
|
||||
|
||||
category = item.central_session
|
||||
query_sessions = item.query_sessions
|
||||
if not category or not query_sessions:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"data": "",
|
||||
"message": "central_session 或者 query_sessions不能为空",
|
||||
}), 200
|
||||
central_session = PathTool.join(BATH_ROOT, html_name, category)
|
||||
querySessionCommand = getQuerySessionCommand(query_sessions, html_name)
|
||||
run_mta_command_result = runmtacommandFun(
|
||||
bagfolder, html_name, central_session, querySessionCommand, category, id
|
||||
)
|
||||
if run_mta_command_result:
|
||||
return jsonify(run_mta_command_result), 200
|
||||
else:
|
||||
return jsonify(run_mta_command_result), 500
|
||||
|
||||
|
||||
@app.route("/api/runmtacommand")
|
||||
def runmtacommand():
|
||||
status = task_queue.get_queue_status()
|
||||
bagfolder = request.args.get("bag_folder")
|
||||
command = request.args.get("command")
|
||||
html_name = request.args.get("html_name", "")
|
||||
html_name = html_name.replace(".html", "")
|
||||
central_session = request.args.get("central_session", "")
|
||||
category = request.args.get("category", "")
|
||||
query_sessions = request.args.get("query_sessions", "")
|
||||
|
||||
run_mta_command_result = runmtacommandFun(
|
||||
bagfolder, html_name, central_session, query_sessions, category, id
|
||||
)
|
||||
if run_mta_command_result:
|
||||
return jsonify(run_mta_command_result), 200
|
||||
else:
|
||||
return jsonify(run_mta_command_result), 500
|
||||
|
||||
|
||||
def getQuerySessionCommand(querySession, html_name):
|
||||
querysessionList = [
|
||||
querySession.strip()
|
||||
for querySession in querySession.split(",")
|
||||
if querySession.strip()
|
||||
]
|
||||
paths = [PathTool.join(BATH_ROOT, html_name, item) for item in querysessionList]
|
||||
return " ".join(paths)
|
||||
|
||||
|
||||
def runmtacommandFun(
|
||||
bagfolder, html_name, central_session, query_sessions, category, id
|
||||
):
|
||||
run_mta_command_result = {"success": True, "data": "", "message": "任务提交成功"}
|
||||
status = task_queue.get_queue_status()
|
||||
running_count = status["running_count"]
|
||||
html_name = html_name.replace(".html", "")
|
||||
result_file = PathTool.join(
|
||||
BATH_ROOT, bagfolder, html_name, category, OUTPUT_RESULT_PATH
|
||||
)
|
||||
command = f"{MTA_SCRIPT} --central_session {central_session} --query_session {query_sessions}"
|
||||
# command = ""
|
||||
id = request.args.get("id", "")
|
||||
data = {
|
||||
"image_name": f"{MTA_IMAGE}:{MTA_IMNAGE_VERSION}",
|
||||
"container_name": f"MTA-{id}",
|
||||
"command": command,
|
||||
"timeout": 3600,
|
||||
"result_file": result_file,
|
||||
"result_file_name": OUTPUT_PIC_NAME,
|
||||
"id": id,
|
||||
}
|
||||
volumes = {CONTAINER_OUTER_PATH: BATH_ROOT}
|
||||
|
||||
environment = {ENV_PARAM_KEY: ENV_PARAM_VALUE}
|
||||
|
||||
def on_task_completion(task_result):
|
||||
print("---------------------- 进入 回调方法 ----------------------")
|
||||
"""任务完成回调"""
|
||||
# 手动创建应用上下文
|
||||
with app.app_context():
|
||||
# 从task_result中获取id，如果不存在则使用闭包变量id
|
||||
|
||||
item = (
|
||||
db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
)
|
||||
if task_result.get("status") == "completed":
|
||||
try:
|
||||
if item:
|
||||
item.is_run_mta = True
|
||||
item.mta_is_running = False
|
||||
item.mta_task_status = TaskStatus.COMPLETED.value
|
||||
item.mta_task_id = None
|
||||
item.run_mta_result = OUTPUT_PIC_NAME
|
||||
item.updated_at = func.now()
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
app.logger.error(f"回调函数数据库操作失败: {e}")
|
||||
db.session.rollback()
|
||||
elif task_result.get("status") == "cancelled":
|
||||
# 失败的情况
|
||||
item.is_run_mta = True
|
||||
item.mta_is_running = True
|
||||
item.mta_task_status = TaskStatus.CANCELLED.value
|
||||
item.mta_task_id = None
|
||||
item.run_mta_result = None
|
||||
item.updated_at = func.now()
|
||||
db.session.commit()
|
||||
else:
|
||||
# 失败的情况
|
||||
item.is_run_mta = True
|
||||
item.mta_is_running = True
|
||||
item.mta_task_status = TaskStatus.FAILED.value
|
||||
item.mta_task_id = None
|
||||
item.run_mta_result = None
|
||||
item.updated_at = func.now()
|
||||
db.session.commit()
|
||||
|
||||
print(f"任务 {id} 数据库更新成功")
|
||||
|
||||
try:
|
||||
"""查看队列状态"""
|
||||
status = task_queue.get_queue_status()
|
||||
|
||||
# 提交任务到队列
|
||||
result = task_queue.submit_task(
|
||||
LongRunningTasks.api_create_container,
|
||||
container_config=data,
|
||||
volumes=volumes,
|
||||
environment=environment,
|
||||
completion_callback=on_task_completion,
|
||||
)
|
||||
run_mta_command_result["data"] = result
|
||||
status = result["status"]
|
||||
task_id = result["BATH_ROOT"]
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
if not item:
|
||||
return jsonify({"error": f"记录不存在，id={id}"}), 404
|
||||
|
||||
# 更新字段
|
||||
item.mta_is_running = True
|
||||
|
||||
item.mta_task_id = task_id
|
||||
item.updated_at = func.now() # 可选，如果模型没有自动更新
|
||||
print(f"任务地位：{id}，task 返回结果为 ：{status}")
|
||||
#
|
||||
if status == "PENDING":
|
||||
item.mta_task_status = TaskStatus.PENDING.value
|
||||
run_mta_command_result["message"] = (
|
||||
f"当前正在执行任务数：{running_count}，新建任务已进入队列"
|
||||
)
|
||||
elif status == "RUNNING":
|
||||
item.mta_task_status = TaskStatus.RUNNING.value
|
||||
run_mta_command_result["message"] = "MTA 任务已经启动"
|
||||
db.session.commit()
|
||||
return run_mta_command_result
|
||||
|
||||
except Exception as e:
|
||||
app.logger.exception("查询接口异常")
|
||||
return {"success": False, "data": e, "message": "任务提交异常"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 本地开发：http://127.0.0.1:5000
|
||||
|
||||
with app.app_context():
|
||||
db.create_all()
|
||||
|
||||
app.run(host="0.0.0.0", port=5000, debug=False)
|
||||
@@ -0,0 +1,105 @@
|
||||
import os
|
||||
import logging
|
||||
from dotenv import load_dotenv
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_environment():
|
||||
"""智能加载环境配置"""
|
||||
env = os.getenv("FLASK_ENV", "development")
|
||||
|
||||
# 加载顺序：基础配置 -> 环境特定配置 -> 本地覆盖
|
||||
env_files = [
|
||||
".env", # 基础配置
|
||||
f".env.{env}", # 环境特定配置
|
||||
".env.local", # 本地覆盖（可选）
|
||||
]
|
||||
|
||||
for env_file in env_files:
|
||||
if Path(env_file).exists():
|
||||
load_dotenv(env_file)
|
||||
logging.info(f"加载环境文件: {env_file}")
|
||||
|
||||
return env
|
||||
|
||||
|
||||
class Config:
|
||||
"""动态配置类"""
|
||||
|
||||
def __init__(self):
|
||||
self.env = load_environment()
|
||||
self._load_config()
|
||||
|
||||
def _load_config(self):
|
||||
"""加载所有配置"""
|
||||
# 路径配置
|
||||
self.ABSOLUTE_HTML_PATH = os.getenv("ABSOLUTE_HTML_PATH")
|
||||
self.BATH_ROOT = os.getenv("BATH_ROOT")
|
||||
self.IMAGE_FOLDER = os.getenv("IMAGE_FOLDER")
|
||||
self.SCRIPT_PATH = os.getenv("SCRIPT_PATH")
|
||||
self.HTML_DIR = os.getenv("HTML_DIR")
|
||||
self.ABSOLUTE_PATH = os.getenv("ABSOLUTE_PATH")
|
||||
|
||||
# 数据库配置
|
||||
self.SQLALCHEMY_DATABASE_URI = os.getenv("DATABASE_URL")
|
||||
self.SQLALCHEMY_TRACK_MODIFICATIONS = False
|
||||
|
||||
# 应用配置
|
||||
self.MAX_PER_PAGE = int(os.getenv("MAX_PER_PAGE", 100))
|
||||
self.DEFAULT_PER_PAGE = int(os.getenv("DEFAULT_PER_PAGE", 10))
|
||||
self.LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
|
||||
self.SECRET_KEY = os.getenv("SECRET_KEY", "dev-secret-key")
|
||||
|
||||
# Flask 配置
|
||||
self.DEBUG = os.getenv("DEBUG", "False").lower() == "true"
|
||||
self.HOST = os.getenv("HOST", "0.0.0.0")
|
||||
self.PORT = int(os.getenv("PORT", 5000))
|
||||
|
||||
# 文件路径
|
||||
self.APP_ROOT = os.path.dirname(os.path.abspath(__file__))
|
||||
self.PAGES_DIR = os.path.join(self.APP_ROOT, "static/pages")
|
||||
self.ALLOWED_EXTS = {".html", ".htm"}
|
||||
|
||||
# 调度器配置
|
||||
self.ENABLE_SCHEDULER = os.getenv("ENABLE_SCHEDULER", "true").lower() == "true"
|
||||
self.SCHEDULER_INTERVAL = int(os.getenv("SCHEDULER_INTERVAL", 5))
|
||||
|
||||
# MTA task 数量
|
||||
self.MAX_ALLOW_RUNNING_MTA_TASK = int(
|
||||
os.getenv("MAX_ALLOW_RUNNING_MTA_TASK", 3)
|
||||
)
|
||||
|
||||
self.OUTPUT_PIC_PATH = os.getenv(
|
||||
"OUTPUT_PIC_PATH", "/mta_middle_proc/global_results/"
|
||||
)
|
||||
self.OUTPUT_PIC_NAME = os.getenv(
|
||||
"OUTPUT_PIC_NAME", "merged_ground_voxel_max_intensity_bev_zfilter.png"
|
||||
)
|
||||
self.OUTPUT_RESULT_PATH = os.getenv(
|
||||
"OUTPUT_RESULT_PATH", "/mta_middle_proc/global_results/slam_lidar_ground/"
|
||||
)
|
||||
self.CONTAINER_INNER_PATH = os.getenv("CONTAINER_INNER_PATH", "/mnt/mta")
|
||||
self.CONTAINER_OUTER_PATH = os.getenv("CONTAINER_OUTER_PATH", "/data/mnt/")
|
||||
self.ENV_PARAM_KEY = os.getenv("ENV_PARAM_KEY", "ROS_MASTER_URL")
|
||||
self.ENV_PARAM_VALUE = os.getenv("ENV_PARAM_VALUE", "http://localhost:11311")
|
||||
self.MTA_SCRIPT = os.getenv("MTA_SCRIPT", "/app/mta_run_param.sh")
|
||||
|
||||
self.MTA_IMAGE = os.getenv("MTA_IMAGE", "/mta_middle_proc/global_results/")
|
||||
self.MTA_IMNAGE_VERSION = os.getenv("MTA_IMNAGE_VERSION", "1.51")
|
||||
self.RESULT_PATH = os.getenv("RESULT_PATH", "/data/mnt/")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Config env={self.env}>"
|
||||
|
||||
|
||||
# 初始化配置
|
||||
config = Config()
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, config.LOG_LEVEL),
|
||||
format="[%(levelname)s] %(asctime)s - %(name)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -0,0 +1,39 @@
|
||||
|
||||
CREATE DATABASE IF NOT EXISTS mta;
|
||||
use mta ;
|
||||
drop TABLE IF EXISTS `session_selection` ;
|
||||
CREATE TABLE IF NOT EXISTS `session_selection` (
|
||||
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '自增主键',
|
||||
`html_name` VARCHAR(255) NOT NULL COMMENT ' html name ',
|
||||
`html_folder` VARCHAR(255) NOT NULL COMMENT ' html 文件存储文件夹 ',
|
||||
`bag_folder` VARCHAR(255) NOT NULL COMMENT ' html 的数据包存储文件夹 ',
|
||||
`central_session` VARCHAR(255) NOT NULL COMMENT '单选：central session',
|
||||
`query_sessions` VARCHAR(1024) NOT NULL COMMENT '多选：query session 列表',
|
||||
`selected_image_url` json DEFAULT NULL COMMENT '页面展示的已选图片URL',
|
||||
`is_filter_bag` boolean DEFAULT NULL COMMENT '是否筛选数据包',
|
||||
`is_run_mta` boolean DEFAULT NULL COMMENT '是否跑了mta 任务',
|
||||
`run_mta_result` VARCHAR(255) DEFAULT NULL COMMENT 'mta 任务结果',
|
||||
`remark` VARCHAR(1024) COMMENT '备注信息',
|
||||
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间',
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
|
||||
|
||||
-- 示例：按 central_session 查询较多时
|
||||
CREATE INDEX idx_central_session ON `session_selection` (`central_session`);
|
||||
-- 示例：按创建时间范围查询
|
||||
CREATE INDEX idx_created_at ON `session_selection` (`created_at`);
|
||||
|
||||
-- 1106
|
||||
-- 添加新字段到 session_selection 表
|
||||
|
||||
ALTER TABLE `session_selection`
|
||||
ADD COLUMN `mta_is_running` BOOLEAN DEFAULT FALSE COMMENT 'mta task 是否在运行';
|
||||
|
||||
ALTER TABLE `session_selection`
|
||||
ADD COLUMN `mta_task_status` VARCHAR(255) DEFAULT 'NOT_STARTED' COMMENT 'mta task 状态:NOT_STARTED, PENDING， RUNNING, FAILED, COMPLETED ,CANCELLED';
|
||||
|
||||
ALTER TABLE `session_selection`
|
||||
ADD COLUMN `mta_task_id` VARCHAR(255) DEFAULT NULL COMMENT 'mta task ID';
|
||||
@@ -0,0 +1,39 @@
|
||||
# 使用官方 Python 基础镜像
|
||||
FROM python:3.12
|
||||
|
||||
# 设置工作目录
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
|
||||
|
||||
# 安装 uv
|
||||
RUN pip install uv
|
||||
|
||||
# 复制 pyproject.toml 和 poetry.lock 文件（如果有的话）
|
||||
COPY pyproject.toml .
|
||||
|
||||
# 使用 uv 安装依赖，使用 --system 参数 可以应用参数 --index-url https://mirrors.aliyun.com/pypi/simple/ 加速
|
||||
RUN uv pip install --system .
|
||||
|
||||
|
||||
# 复制项目文件
|
||||
COPY models/ ./models/
|
||||
COPY services/ ./services/
|
||||
COPY static/ ./static/
|
||||
COPY templates/ ./templates/
|
||||
COPY utils/ ./utils/
|
||||
COPY app.py .
|
||||
|
||||
COPY extensions.py .
|
||||
COPY config.py .
|
||||
COPY .env .
|
||||
COPY .devenv .
|
||||
COPY __init__.py .
|
||||
COPY README.md .
|
||||
|
||||
# 暴露端口（假设你的Flask应用运行在5000端口）
|
||||
EXPOSE 5000
|
||||
|
||||
# 启动命令
|
||||
CMD ["python", "app.py"]
|
||||
@@ -0,0 +1,4 @@
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
|
||||
# 创建全局 db 实例
|
||||
db = SQLAlchemy()
|
||||
@@ -0,0 +1,6 @@
|
||||
# models/__init__.py
|
||||
# 导出模型类，方便从外部导入
|
||||
from .sessionrecord import SessionRecord
|
||||
|
||||
# 可以在这里导入所有模型
|
||||
__all__ = ["SessionRecord"]
|
||||
@@ -0,0 +1,95 @@
|
||||
from typing import List, Set, Dict
|
||||
from datetime import datetime
|
||||
import json
|
||||
from extensions import db
|
||||
|
||||
|
||||
# ===== 数据模型 =====
|
||||
class SessionRecord(db.Model):
|
||||
__tablename__ = "session_selection"
|
||||
|
||||
id = db.Column(
|
||||
db.BigInteger().with_variant(db.Integer, "sqlite"),
|
||||
primary_key=True,
|
||||
autoincrement=True,
|
||||
comment="自增主键",
|
||||
)
|
||||
html_name = db.Column(db.String(255), nullable=False, comment="html name")
|
||||
html_folder = db.Column(db.String(255), nullable=False, comment="html folder")
|
||||
bag_folder = db.Column(db.String(255), nullable=False, comment="bag folder")
|
||||
central_session = db.Column(
|
||||
db.String(255), nullable=False, comment="单选：central session"
|
||||
)
|
||||
query_sessions = db.Column(
|
||||
db.String(1024), nullable=False, comment="多选：query session 列表"
|
||||
)
|
||||
selected_image_url = db.Column(
|
||||
db.Text().with_variant(db.String(1024), "sqlite"), # 兼容 MySQL JSON 与 SQLite
|
||||
nullable=True,
|
||||
comment="页面展示的已选图片URL",
|
||||
)
|
||||
is_filter_bag = db.Column(
|
||||
db.Boolean, nullable=True, default=None, comment="是否筛选数据包"
|
||||
)
|
||||
is_run_mta = db.Column(
|
||||
db.Boolean, nullable=True, default=None, comment="是否跑了mta任务"
|
||||
)
|
||||
mta_is_running = db.Column(
|
||||
db.Boolean, nullable=False, default=False, comment="mta task 是否在运行"
|
||||
)
|
||||
mta_task_status = db.Column(
|
||||
db.String(255),
|
||||
nullable=False,
|
||||
default=False,
|
||||
comment="mta task 状态:NOT_STARTED ,RUNNING ,COMPLETED",
|
||||
)
|
||||
mta_task_id = db.Column(
|
||||
db.String(255), nullable=False, comment="task id：mta_task_id"
|
||||
)
|
||||
run_mta_result = db.Column(db.String(255), nullable=True, comment="mta任务结果")
|
||||
remark = db.Column(db.String(1024), nullable=False, comment="备注信息")
|
||||
created_at = db.Column(
|
||||
db.DateTime, nullable=False, default=datetime.utcnow, comment="创建时间"
|
||||
)
|
||||
updated_at = db.Column( # 修正字段名为 updated_at
|
||||
db.DateTime,
|
||||
nullable=False,
|
||||
default=datetime.utcnow,
|
||||
onupdate=datetime.utcnow,
|
||||
comment="更新时间",
|
||||
)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"id": self.id,
|
||||
"html_name": self.html_name,
|
||||
"html_folder": self.html_folder,
|
||||
"bag_folder": self.bag_folder,
|
||||
"central_session": self.central_session,
|
||||
"query_sessions": self.query_sessions,
|
||||
"selected_image_url": self._parse_json_url(self.selected_image_url),
|
||||
"is_filter_bag": self.is_filter_bag,
|
||||
"is_run_mta": self.is_run_mta,
|
||||
"mta_task_status": self.mta_task_status,
|
||||
"mta_is_running": self.mta_is_running,
|
||||
"mta_task_id": self.mta_task_id,
|
||||
"run_mta_result": self.run_mta_result,
|
||||
"remark": self.remark,
|
||||
"created_at": self.created_at.isoformat(),
|
||||
"updated_at": self.updated_at.isoformat(),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _parse_json_url(value):
|
||||
"""
|
||||
将数据库中的 JSON 字段或字符串安全地反序列化为 Python 对象。
|
||||
MySQL 返回 list/dict；SQLite 可能返回字符串或 None。
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (list, dict)):
|
||||
return value
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return value # 无法解析时原样返回（如旧数据为字符串）
|
||||
@@ -0,0 +1,65 @@
|
||||
from enum import Enum
|
||||
from typing import List, Set, Dict, Any
|
||||
|
||||
|
||||
class TaskStatus(Enum):
|
||||
"""任务状态枚举类"""
|
||||
|
||||
NOT_STARTED = "NOT_STARTED" # 未开始
|
||||
PENDING = "PENDING" # 队列等待中
|
||||
RUNNING = "RUNNING" # 执行中
|
||||
COMPLETED = "COMPLETED" # 已完成
|
||||
FAILED = "FAILED" # 失败
|
||||
CANCELLED = "CANCELLED" # 已取消
|
||||
|
||||
@classmethod
|
||||
def get_active_statuses(cls) -> Set[str]:
|
||||
"""获取活跃状态集合（未完成的状态）"""
|
||||
return {cls.NOT_STARTED.value, cls.PENDING.value, cls.RUNNING.value}
|
||||
|
||||
@classmethod
|
||||
def get_final_statuses(cls) -> Set[str]:
|
||||
"""获取最终状态集合（不会改变的状态）"""
|
||||
return {cls.COMPLETED.value, cls.FAILED.value, cls.CANCELLED.value}
|
||||
|
||||
@classmethod
|
||||
def get_editable_statuses(cls) -> Set[str]:
|
||||
"""获取可编辑状态集合（可以手动改变的状态）"""
|
||||
return {cls.NOT_STARTED.value, cls.PENDING.value}
|
||||
|
||||
@classmethod
|
||||
def is_active(cls, status: str) -> bool:
|
||||
"""检查是否为活跃状态（未完成）"""
|
||||
return status in cls.get_active_statuses()
|
||||
|
||||
@classmethod
|
||||
def is_final(cls, status: str) -> bool:
|
||||
"""检查是否为最终状态（不会改变）"""
|
||||
return status in cls.get_final_statuses()
|
||||
|
||||
@classmethod
|
||||
def is_editable(cls, status: str) -> bool:
|
||||
"""检查状态是否可手动编辑"""
|
||||
return status in cls.get_editable_statuses()
|
||||
|
||||
@classmethod
|
||||
def validate_status(cls, status: str) -> bool:
|
||||
"""验证状态值是否有效"""
|
||||
return any(status == member.value for member in cls)
|
||||
|
||||
@classmethod
|
||||
def get_all_statuses(cls) -> List[str]:
|
||||
"""获取所有状态值列表"""
|
||||
return [member.value for member in cls]
|
||||
|
||||
@classmethod
|
||||
def get_status_choices(cls) -> List[Dict[str, Any]]:
|
||||
"""获取状态选项（用于前端下拉选择）"""
|
||||
return [
|
||||
{"value": cls.NOT_STARTED.value, "label": "未开始", "color": "default"},
|
||||
{"value": cls.PENDING.value, "label": "等待中", "color": "orange"},
|
||||
{"value": cls.RUNNING.value, "label": "执行中", "color": "blue"},
|
||||
{"value": cls.COMPLETED.value, "label": "已完成", "color": "green"},
|
||||
{"value": cls.FAILED.value, "label": "已失败", "color": "red"},
|
||||
{"value": cls.CANCELLED.value, "label": "已取消", "color": "gray"},
|
||||
]
|
||||
@@ -0,0 +1,29 @@
|
||||
[project]
|
||||
name = "mta_management_system"
|
||||
version = "0.1.0"
|
||||
description = "cloud infra pipeline for mb fst data production"
|
||||
authors = [{ name = "MBRDCA", email = "tbd@tbd.com" }]
|
||||
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"flask>=3.1.1",
|
||||
"jinja2==3.1.6",
|
||||
"werkzeug==3.1.3",
|
||||
"click==8.2.1",
|
||||
"xmltodict==0.14.2",
|
||||
|
||||
"python-dotenv==1.1.1",
|
||||
"itsdangerous==2.2.0",
|
||||
"flask_sqlalchemy==3.0.5",
|
||||
"pymysql==1.1.0",
|
||||
"paramiko==3.1.0",
|
||||
"docker==6.1.3",
|
||||
"APScheduler==3.10",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["mta_management_system"]
|
||||
@@ -0,0 +1,228 @@
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from typing import Dict, Any
|
||||
from utils.docker_tool import docker_tool
|
||||
from extensions import db # 统一导入 db 实例
|
||||
from models.sessionrecord import SessionRecord
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LongRunningTasks:
|
||||
"""长时间任务处理类"""
|
||||
|
||||
@staticmethod
|
||||
def api_create_container(
|
||||
task,
|
||||
container_config: Dict[str, Any],
|
||||
volumes: Dict[str, Any],
|
||||
environment: Dict[str, Any],
|
||||
):
|
||||
"""
|
||||
调用API创建容器并等待完成
|
||||
|
||||
Args:
|
||||
container_config: 容器配置，包含：
|
||||
- image_name: 镜像名称
|
||||
- container_name: 容器名称（可选）
|
||||
- command: 启动命令（可选）
|
||||
- timeout: 超时时间（秒，默认3600）
|
||||
|
||||
Returns:
|
||||
Dict: 执行结果
|
||||
"""
|
||||
|
||||
try:
|
||||
# 提取配置参数
|
||||
|
||||
image_name = container_config.get("image_name", "ubuntu:latest")
|
||||
container_name = container_config.get("container_name")
|
||||
command = container_config.get("command")
|
||||
timeout = container_config.get("timeout", 3600)
|
||||
result_file = container_config.get("result_file")
|
||||
id = container_config.get("id")
|
||||
result_file_name = container_config.get("result_file_name")
|
||||
if not image_name:
|
||||
raise ValueError("必须提供image_name参数")
|
||||
logger.info(
|
||||
f"开始创建容器: {container_name or '未命名'}, 镜像: {image_name}"
|
||||
)
|
||||
|
||||
# 调用工具类创建容器
|
||||
# 检查Docker环境
|
||||
if not docker_tool.is_docker_available():
|
||||
return {
|
||||
"success": False,
|
||||
"error": "宿主机Docker环境不可用",
|
||||
"message": "请检查Docker是否正确挂载",
|
||||
}
|
||||
# 在创建容器之前先创建同名容器是否在运行或者已经存在，如果已经在运行则 停止 ，并删除， 如果没有则进行创建
|
||||
containers = docker_tool.get_containers()
|
||||
mta_container = None
|
||||
for container in containers:
|
||||
names = container.get("Names", "")
|
||||
# 精确匹配MTA-1（考虑Docker可能添加/前缀的情况）
|
||||
if names == container_name:
|
||||
mta_container = container
|
||||
break
|
||||
|
||||
if mta_container:
|
||||
container_id = mta_container.get("ID", mta_container.get("Id", ""))
|
||||
print(f"找到MTA容器，ID: {container_id}")
|
||||
# 停止容器
|
||||
stop_result = docker_tool.stop_container(container_id)
|
||||
if not stop_result["success"]:
|
||||
print(f"停止容器失败: {stop_result.get('error', '未知错误')}")
|
||||
return
|
||||
|
||||
print("容器已停止")
|
||||
|
||||
# 删除容器
|
||||
remove_result = docker_tool.remove_container(container_id)
|
||||
if remove_result["success"]:
|
||||
print(f"{container_name}容器已成功删除")
|
||||
else:
|
||||
print(f"删除容器失败: {remove_result.get('error', '未知错误')}")
|
||||
print(
|
||||
f"创建容器参数command: {command} ，volumes 挂载： {volumes}，environment 参数：{environment} "
|
||||
)
|
||||
# 使用工具类创建容器
|
||||
result = docker_tool.create_container(
|
||||
image_name=image_name,
|
||||
container_name=container_name,
|
||||
command=command,
|
||||
volumes=volumes,
|
||||
environment=environment,
|
||||
)
|
||||
if result["success"]:
|
||||
print(" 容器创建成功")
|
||||
container_id = result["container_id"]
|
||||
|
||||
# 根据调用的工具类创建容器 ，根据工具类创建容器的返回值 判断容器是否创建成功
|
||||
completion_file = result_file
|
||||
logger.info(f"等待完成文件: {completion_file}")
|
||||
|
||||
# 轮询MTA 的结果文件否存在
|
||||
start_time = time.time()
|
||||
check_count = 0
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
check_count += 1
|
||||
# 获取 容器运行状态 如果容器停止运行且结果文件不存在则抛出异常
|
||||
container_result = docker_tool.get_container_status_by_id(
|
||||
container_id
|
||||
)
|
||||
state = container_result.get("state", "").lower()
|
||||
if (
|
||||
state != "running"
|
||||
and not LongRunningTasks._check_completion_file(completion_file)
|
||||
):
|
||||
# 确保容器被清理
|
||||
try:
|
||||
docker_tool.stop_container(container_name)
|
||||
docker_tool.remove_container(container_name)
|
||||
except Exception as cleanup_e:
|
||||
logger.error(f"清理容器失败: {cleanup_e}")
|
||||
raise Exception(
|
||||
f"容器状态异常: {state}, 结果文件未生成: {completion_file}"
|
||||
)
|
||||
|
||||
# 检查完成文件是否存在 同时判断 容器是否运行正常， 如果
|
||||
if LongRunningTasks._check_completion_file(completion_file):
|
||||
logger.info(f"MTA 任务执行 完成: {container_name}")
|
||||
# 判断 任务执行完成后 调用工具类 停止容器 然后删除容器
|
||||
docker_tool.stop_container(container_name)
|
||||
docker_tool.remove_container(container_name)
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"container_name": container_name,
|
||||
"image_name": image_name,
|
||||
"id": id, # 添加id到返回结果中，供回调函数使用
|
||||
"command": command,
|
||||
"completion_file": completion_file,
|
||||
"completed_at": datetime.now().isoformat(),
|
||||
"total_wait_seconds": int(time.time() - start_time),
|
||||
"check_count": check_count,
|
||||
}
|
||||
|
||||
# 等待5秒再检查
|
||||
time.sleep(5)
|
||||
print(
|
||||
f"第{check_count}次检查路径：{completion_file}，完成文件尚未出现..."
|
||||
)
|
||||
logger.info(f"第{check_count}次检查，完成文件尚未出现...")
|
||||
else:
|
||||
print(" 容器创建失败")
|
||||
|
||||
logger.info("调用API创建容器...")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"创建容器失败: {e}")
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _check_completion_file(file_path: str) -> bool:
|
||||
import os
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
return False
|
||||
|
||||
for filename in os.listdir(file_path):
|
||||
if filename.endswith(".pcd"):
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def process_with_completion_check(process_config: Dict[str, Any]):
|
||||
"""
|
||||
通用处理任务，使用文件检查判断完成
|
||||
|
||||
Args:
|
||||
process_config: 处理配置，包含：
|
||||
- process_type: 处理类型
|
||||
- completion_file: 完成标志文件路径
|
||||
- timeout: 超时时间（秒）
|
||||
|
||||
Returns:
|
||||
Dict: 执行结果
|
||||
"""
|
||||
try:
|
||||
process_type = process_config.get("process_type", "unknown")
|
||||
completion_file = process_config.get("completion_file")
|
||||
timeout = process_config.get("timeout", 7200)
|
||||
|
||||
if not completion_file:
|
||||
raise ValueError("必须提供completion_file参数")
|
||||
|
||||
logger.info(f"开始处理任务: {process_type}, 完成文件: {completion_file}")
|
||||
|
||||
# 轮询检查完成文件
|
||||
start_time = time.time()
|
||||
check_count = 0
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
check_count += 1
|
||||
|
||||
if LongRunningTasks._check_completion_file(completion_file):
|
||||
logger.info(f"处理任务完成: {process_type}")
|
||||
return {
|
||||
"status": "completed",
|
||||
"process_type": process_type,
|
||||
"completion_file": completion_file,
|
||||
"completed_at": datetime.now().isoformat(),
|
||||
"total_wait_seconds": int(time.time() - start_time),
|
||||
"check_count": check_count,
|
||||
}
|
||||
|
||||
time.sleep(60) # 每分钟检查一次
|
||||
logger.info(f"第{check_count}次检查，任务处理中...")
|
||||
|
||||
raise Exception(f"处理任务超时({timeout}秒)")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理任务失败: {e}")
|
||||
raise
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -0,0 +1,76 @@
|
||||
body { font-family: sans-serif; margin: 1em; background: #f7f9fc; }
|
||||
iframe {
|
||||
width: 95%;
|
||||
max-width: 1300px; /* 单个 iframe 最大宽度 */
|
||||
height: 800px; /* 保持你原有高度 */
|
||||
border: 1px solid #ccc;
|
||||
margin-top: 1em;
|
||||
background: white;
|
||||
box-sizing: border-box; /* 让 padding/border 不额外撑宽 */
|
||||
display: block; /* 去除底部空白缝隙 */
|
||||
margin-left: auto;
|
||||
margin-right: auto;
|
||||
margin-left: 0 ;
|
||||
}
|
||||
.image-box {
|
||||
min-height: 400px; /* 根据你的图片高度设置 */
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
background: #f8f9fa;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
|
||||
#imagePreview {
|
||||
max-width: 100%;
|
||||
max-height: 500px;
|
||||
object-fit: contain; /* 保持比例 */
|
||||
transition: opacity 0.3s ease;
|
||||
}
|
||||
#log { max-height: 200px; overflow-y: auto; background: #222; color: #0f0; padding: 0.5em; font-family: monospace; font-size: 0.9em; }
|
||||
h2 { margin-top: 0.5em; }
|
||||
|
||||
|
||||
.button-container {
|
||||
display: flex;
|
||||
gap: 15px;
|
||||
margin-top: 20px;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.btn {
|
||||
padding: 12px 30px;
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s ease;
|
||||
min-width: 120px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 1px;
|
||||
}
|
||||
|
||||
.btn-primary {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
box-shadow: 0 4px 15px rgba(102, 126, 234, 0.4);
|
||||
}
|
||||
|
||||
.btn-primary:hover {
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 6px 20px rgba(102, 126, 234, 0.6);
|
||||
}
|
||||
|
||||
.btn-secondary {
|
||||
background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
|
||||
color: white;
|
||||
box-shadow: 0 4px 15px rgba(245, 87, 108, 0.4);
|
||||
}
|
||||
|
||||
.btn-secondary:hover {
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 6px 20px rgba(245, 87, 108, 0.6);
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
Binary file not shown.
|
After Width: | Height: | Size: 114 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.0 KiB |
File diff suppressed because one or more lines are too long
@@ -0,0 +1,139 @@
|
||||
'use strict';
|
||||
|
||||
|
||||
// const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
|
||||
// console.log('Bag Folder:', bag_folder);
|
||||
function getJpgFileName(path) {
|
||||
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
const clean = path.replace(/\\/g, '/').split('?')[0].trim();
|
||||
// 判断是否以 .jpg 结尾（忽略大小写）
|
||||
const isJpg = /\.jpg$/i.test(clean);
|
||||
if (!isJpg) return ''; // 不以 .jpg 结尾则返回空
|
||||
|
||||
// 从最后一个斜杠后截取到末尾
|
||||
const idx = clean.lastIndexOf('/');
|
||||
return idx === -1 ? clean : clean.slice(idx + 1);
|
||||
}
|
||||
|
||||
function showImage(imagename,bagname,htmlname,bag_folder) {
|
||||
console.log('htmlname:'+htmlname) ;
|
||||
// alert(bagname)
|
||||
$("#imagediv").empty(); // 或 .html('')
|
||||
|
||||
// 2) 构造要插入的新内容：一个带 class 和 id 的 div，内嵌一张图片
|
||||
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
const newContent = `
|
||||
<img id="imagePreview" src="http://localhost:5000/api/getimage/${encodeURIComponent(imagename)}/${encodeURIComponent(bagname)}/${encodeURIComponent(htmlname)}/${encodeURIComponent(bag_folder)}" alt="图片预览" width="700px">
|
||||
`;
|
||||
|
||||
// 3) 将新内容插入到目标 div 中
|
||||
$("#imagediv").html(newContent);
|
||||
|
||||
|
||||
}
|
||||
|
||||
function getImageByBagAndIndex(bagname,imageIndex,htmlname,bagfolder) {
|
||||
|
||||
var imagename = "";
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
$.ajax({
|
||||
url: '/api/getimage',
|
||||
async: false, // 同步
|
||||
method: 'GET',
|
||||
data: { bagname: bagname,
|
||||
imageIndex: imageIndex,
|
||||
bagfolder: bagfolder,
|
||||
htmlname:htmlname
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
var data = res.result
|
||||
|
||||
console.log(data)
|
||||
var size = data.size
|
||||
imagename = data.imagename
|
||||
}
|
||||
})
|
||||
console.log("imagename===="+imagename)
|
||||
return imagename;
|
||||
}
|
||||
|
||||
function getCentralQuerySession(id) {
|
||||
var data ;
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
$.ajax({
|
||||
url: `/api/items/${id}`,
|
||||
async: false, // 同步
|
||||
method: 'GET',
|
||||
data: { id: id
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
data = res.result
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
console.log("data===="+data)
|
||||
return data;
|
||||
}
|
||||
|
||||
function runMTACommand(command,html_name,central_session ,query_sessions,id,bag_folder,category) {
|
||||
console.log('runMTACommand methon id value :'+id )
|
||||
var imagename = "";
|
||||
console.log(bag_folder)
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
$.ajax({
|
||||
url: '/api/runmtacommand',
|
||||
async: false, // 同步
|
||||
method: 'GET',
|
||||
data: { command: command,
|
||||
html_name: html_name,
|
||||
central_session: central_session,
|
||||
query_sessions:query_sessions,
|
||||
id:id ,
|
||||
bag_folder:bag_folder,
|
||||
category:category
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
console.log(res.status)
|
||||
var checkResult = res.success
|
||||
var msg = res.message
|
||||
showResult(checkResult,msg)
|
||||
|
||||
console.log(res)
|
||||
var data = res.message
|
||||
console.log(data)
|
||||
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
function showResult(result ,msg) {
|
||||
|
||||
if (result ){
|
||||
Swal.fire({
|
||||
title: '任务创建成功！',
|
||||
text: msg,
|
||||
icon: 'success'
|
||||
});
|
||||
} else {
|
||||
Swal.fire({
|
||||
title: '任务创建失败',
|
||||
text: '任务创建失败，请联系管理员',
|
||||
icon: 'error'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 可选：用于确认脚本已加载
|
||||
console.log('[detail.js] loaded, getJpgFileName =', typeof getJpgFileName);
|
||||
@@ -0,0 +1,73 @@
|
||||
(function () {
|
||||
const send = (type, detail = {}) => {
|
||||
try {
|
||||
// 向父窗口发送事件；同源策略下父窗口可接收
|
||||
window.parent.postMessage({ type, timestamp: Date.now(), ...detail }, '*');
|
||||
} catch (e) {
|
||||
console.warn('[injector] postMessage 失败:', e);
|
||||
}
|
||||
};
|
||||
|
||||
// 页面开始加载
|
||||
send('page-start', { href: location.href });
|
||||
|
||||
// 点击
|
||||
document.addEventListener('click', (e) => {
|
||||
send('click', {
|
||||
target: e.target.tagName.toLowerCase() + (e.target.id ? '#' + e.target.id : '') + (e.target.className ? '.' + e.target.className.split(' ').join('.') : ''),
|
||||
href: location.href,
|
||||
x: e.clientX, y: e.clientY,
|
||||
button: e.button, buttons: e.buttons
|
||||
});
|
||||
}, true);
|
||||
|
||||
// 输入/变更（含输入框、文本域、选择框等）
|
||||
['input', 'change', 'focus', 'blur'].forEach(evtName => {
|
||||
document.addEventListener(evtName, (e) => {
|
||||
const tag = e.target.tagName.toLowerCase();
|
||||
if (tag === 'input' || tag === 'textarea' || tag === 'select') {
|
||||
send(evtName, {
|
||||
target: e.target.tagName.toLowerCase() + (e.target.id ? '#' + e.target.id : '') + '.' + e.target.className.split(' ').join('.'),
|
||||
href: location.href,
|
||||
value: e.target.value,
|
||||
name: e.target.name
|
||||
});
|
||||
} else {
|
||||
send(evtName, { target: tag, href: location.href });
|
||||
}
|
||||
}, true);
|
||||
});
|
||||
|
||||
// 滚动
|
||||
let scrollTmr = null;
|
||||
window.addEventListener('scroll', () => {
|
||||
if (scrollTmr) clearTimeout(scrollTmr);
|
||||
scrollTmr = setTimeout(() => {
|
||||
send('scroll', {
|
||||
href: location.href,
|
||||
scrollTop: window.scrollY || document.documentElement.scrollTop,
|
||||
scrollLeft: window.scrollX || document.documentElement.scrollLeft
|
||||
});
|
||||
}, 200);
|
||||
}, true);
|
||||
|
||||
// 键盘（可观测快捷键等）
|
||||
document.addEventListener('keydown', (e) => {
|
||||
send('keydown', {
|
||||
key: e.key, code: e.code,
|
||||
ctrlKey: e.ctrlKey, shiftKey: e.shiftKey, altKey: e.altKey, metaKey: e.metaKey,
|
||||
target: e.target.tagName.toLowerCase(),
|
||||
href: location.href
|
||||
});
|
||||
}, true);
|
||||
|
||||
// 页面可见性变化（切换标签/最小化）
|
||||
document.addEventListener('visibilitychange', () => {
|
||||
send('visibilitychange', { hidden: document.hidden, href: location.href });
|
||||
});
|
||||
|
||||
// 页面卸载前
|
||||
window.addEventListener('beforeunload', () => {
|
||||
send('page-leave', { href: location.href });
|
||||
});
|
||||
})();
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,916 @@
|
||||
<!doctype html>
|
||||
<html lang="zh">
|
||||
<head>
|
||||
<base href="/">
|
||||
<meta charset="UTF-8" />
|
||||
<title>数据包选择页面</title>
|
||||
|
||||
<link rel="stylesheet" href="../static/css/sweetalert2.css">
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/detail.css') }}">
|
||||
|
||||
</head>
|
||||
<body>
|
||||
<h1>数据包选择页面</h1>
|
||||
|
||||
<div id="page" >
|
||||
|
||||
<!-- 使用 Bootstrap 的行和列布局将元素放在同一行 -->
|
||||
<div class="row align-items-center"> <!-- align-items-center 用于垂直居中对齐 -->
|
||||
<!-- 第一列：标题 <h2> -->
|
||||
<div class="col-auto">
|
||||
<h2>{{ html_name }}</h2>
|
||||
</div>
|
||||
|
||||
<!-- 第二列：清空按钮 -->
|
||||
<div class="col-auto">
|
||||
<button type="button" class="btn btn-primary" onclick="clearAll()">清空</button>
|
||||
</div>
|
||||
|
||||
<!-- 第三列：单选组件 -->
|
||||
<div class="col-auto">
|
||||
<fieldset class="form-group">
|
||||
<legend class="col-form-label pt-0">选择类型（单选）</legend>
|
||||
<div id = "selectionType" class="radio-options">
|
||||
<label class="mr-3">
|
||||
<input type="radio" name="selectionType" value="none" checked /> None
|
||||
</label>
|
||||
<label class="mr-3">
|
||||
<input type="radio" name="selectionType" value="centralsession" /> central session
|
||||
</label>
|
||||
<label>
|
||||
<input type="radio" name="selectionType" value="querysession" /> query session
|
||||
</label>
|
||||
</div>
|
||||
</fieldset>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!--<!– <iframe id="frame1" src="" sandbox="allow-scripts allow-same-origin allow-forms allow-popups allow-popups-to-escape-sandbox"></iframe>–>-->
|
||||
<!-- <iframe id="frame12" src="/pages/html2"-->
|
||||
<iframe id="myIframe" src="../static/pages/{{ html_folder }}/{{ html_name }}" sandbox="allow-scripts allow-same-origin allow-forms allow-popups allow-popups-to-escape-sandbox"></iframe>
|
||||
<!-- <iframe id="frame13" src="{{ url_for('static', filename='pages/html1.html') }}" sandbox="allow-scripts allow-same-origin allow-forms allow-popups allow-popups-to-escape-sandbox"></iframe>-->
|
||||
</div>
|
||||
|
||||
<div class="form-row">
|
||||
<!-- 下拉选择框 -->
|
||||
<div class="form-group">
|
||||
<label for="selectKey">选择 数据包：</label>
|
||||
<select id="selectKey" class="form-control">
|
||||
<option value="" disabled selected>请选择</option>
|
||||
</select>
|
||||
</div>
|
||||
<!-- 统计信息显示 -->
|
||||
<!-- <div id="imagecount" style="margin-top:8px;font-size:14px;color:#555;"></div>
|
||||
<div id="imageinfo" style="margin-top:8px;font-size:14px;color:#555;"></div> -->
|
||||
<div style="margin-top:8px; display: flex; align-items: center; gap: 10px; font-size:14px; color:#555;">
|
||||
<div id="imagecount"></div>
|
||||
<div id="imageinfo"></div>
|
||||
</div>
|
||||
<span id="bagname" hidden data-value=""></span>
|
||||
<span id="hiddenImageCount" hidden data-value=""></span>
|
||||
<span id="hiddenHtmlFolder" hidden value={{ html_folder }}></span>
|
||||
<span id="hiddenBagFolder" hidden value={{ bag_folder }}></span>
|
||||
<span id="hiddenAbsilutePath" hidden value={{ abslute_path }}></span>
|
||||
<!--<span id="htmlid" hidden value={{ id }}></span>-->
|
||||
<span id="mtataskid" hidden data-value={{ id }}></span>
|
||||
<!-- 输入框 -->
|
||||
<div class="form-group" style="margin-left: 12px;">
|
||||
<label for="numberInput">输入数字：</label>
|
||||
<input type="text" id="numberInput" class="form-control" placeholder="请输入数字" />
|
||||
</div>
|
||||
<!-- 数字输入组：左侧减、右侧加、中间步长 -->
|
||||
<div class="input-group" style="width: 360px; margin-left: 12px;">
|
||||
<span id="hiddenTag" hidden data-value="1"></span>
|
||||
<!-- 左侧减按钮 -->
|
||||
<button type="button" class=" btn-outline-secondary" id="stepDownBtn" style="border-radius: 0.375rem 0 0 0.375rem;">
|
||||
<img src="{{ url_for('static', filename='images/left.jpg') }}" alt="→" width="16" height="16" style="vertical-align: middle;">
|
||||
</button>
|
||||
<!-- 右侧加按钮 -->
|
||||
<button type="button" class=" btn-outline-secondary" id="stepUpBtn" style="border-radius: 0 0.375rem 0.375rem 0;">
|
||||
<img src="{{ url_for('static', filename='images/right.jpg') }}" alt="→" width="16" height="16" style="vertical-align: middle;">
|
||||
</button>
|
||||
<!-- 步长输入框 -->
|
||||
<div class="input-group" style="width: 100px;">
|
||||
<input type="text" id="stepInput" class="form-control text-center" placeholder="步长" />
|
||||
</div>
|
||||
</div>
|
||||
<!-- 确认按钮 -->
|
||||
<div class="form-group" style="margin-left: 12px;">
|
||||
<button id="confirmBtn" class="btn btn-primary">确认</button>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- 图片容器 -->
|
||||
<div id="image-container">
|
||||
<h2>图片展示</h2>
|
||||
<div class="error" id="error"></div>
|
||||
<div class="image-box" id="imagediv">
|
||||
<img id="imagePreview" src="{{ url_for('static', filename='images/image.jpg') }}" alt="图片预览" width="400px">
|
||||
</div>
|
||||
</div>
|
||||
<!-- 用于展示数字的区域 -->
|
||||
<div id="result" type="hidden" style="margin-top: 12px; font-size: 14px; color: #333;"></div>
|
||||
<img id="preview" alt="预览" style="max-width:100%;height:auto;display:none;margin-top:8px;">
|
||||
|
||||
<!-- 外部展示容器 -->
|
||||
<div id="selected-labels" style="margin-top:8px;color:#333;">未选中</div>
|
||||
<!-- 复选框 -->
|
||||
<form id="myForm">
|
||||
<input type="hidden" id = "html_name" name="html_name" value="{{ html_name }}">
|
||||
<input type="hidden" id = "mtaBagfolder" name="bag_folder" value="{{ bag_folder }}">
|
||||
<input type="hidden" id = "id" name="id" value="{{ id }}">
|
||||
<!-- 单选框组 -->
|
||||
<fieldset>
|
||||
<legend>central session（单选）</legend>
|
||||
<div id="radio-container" class="items"></div>
|
||||
</fieldset>
|
||||
<!-- 用于展示数字的区域 -->
|
||||
<div id="querySession" style="margin-top: 12px; font-size: 14px; color: #333;">选择的数据包：</div>
|
||||
<!-- 复选框组 -->
|
||||
<fieldset>
|
||||
<legend>query session（可多选）</legend>
|
||||
<div id="checkbox-container" class="items"></div>
|
||||
</fieldset>
|
||||
|
||||
|
||||
|
||||
<div class="button-container">
|
||||
<button type="button" class="btn btn-primary" onclick="handleSubmit()">提交</button>
|
||||
<button type="button" class="btn btn-primary" onclick="clearAll()">清空</button>
|
||||
</div>
|
||||
</form>
|
||||
<script src="../static/js/jquery-3.7.1.min.js"></script>
|
||||
<script src="../static/js/detail.js"></script>
|
||||
<script src="../static/js/sweetalert2.js"></script>
|
||||
<script>
|
||||
// 父页面接收消息处理函数
|
||||
window.addEventListener('message', (event) => {
|
||||
// 验证消息来源（可选但推荐）
|
||||
if (event.origin === window.location.origin) {
|
||||
// 检查消息类型
|
||||
if (event.data.type === 'tagClick') {
|
||||
// 弹出alert显示标签名
|
||||
var session_name = event.data.tagName ;
|
||||
console.log("选中的 seesion name:" + session_name)
|
||||
// alert('点击的标签名: ' + event.data.tagName);
|
||||
|
||||
// 获取 name="selectionType" 单选组件的当前选中值
|
||||
var selectedSelectionType = $('input[name="selectionType"]:checked').val();
|
||||
|
||||
console.log("当前选择的 selectionType:", selectedSelectionType);
|
||||
|
||||
if (!selectedSelectionType) {
|
||||
// 如果 selectionType 未选择（即 none），则不执行任何操作
|
||||
console.log("selectionType 未选择（none），不执行任何操作。");
|
||||
return;
|
||||
}
|
||||
|
||||
switch (selectedSelectionType) {
|
||||
case 'centralsession':
|
||||
handleCentralSession(session_name);
|
||||
break;
|
||||
case 'querysession':
|
||||
handleQuerySession(session_name);
|
||||
break;
|
||||
case 'none':
|
||||
// 不执行任何操作
|
||||
console.log("selectionType 为 none，不执行任何操作。");
|
||||
break;
|
||||
default:
|
||||
console.warn("未知的 selectionType:", selectedSelectionType);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理 central session（单选）的逻辑
|
||||
* @param {string} tagName - 点击的标签名
|
||||
*/
|
||||
function handleCentralSession(tagName) {
|
||||
// 查找与 tagName 匹配的单选按钮
|
||||
var radioButton = $('input[name="category"][value="' + tagName + '"]');
|
||||
|
||||
if (radioButton.length) {
|
||||
// 切换选中状态
|
||||
radioButton.prop('checked', !radioButton.prop('checked'));
|
||||
console.log(`central session 单选按钮 "${tagName}" 的选中状态已切换为:`, radioButton.prop('checked'));
|
||||
|
||||
// 更新显示（可选）
|
||||
updateSelectedLabels();
|
||||
} else {
|
||||
console.warn(`未找到与标签名 "${tagName}" 对应的 central session 单选按钮`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理 query session（复选）的逻辑
|
||||
* @param {string} tagName - 点击的标签名
|
||||
*/
|
||||
function handleQuerySession(tagName) {
|
||||
// 查找与 tagName 匹配的复选框
|
||||
var checkbox = $('input[name="tags"][value="' + tagName + '"]');
|
||||
|
||||
if (checkbox.length) {
|
||||
// 切换选中状态
|
||||
checkbox.prop('checked', !checkbox.prop('checked'));
|
||||
console.log(`query session 复选框 "${tagName}" 的选中状态已切换为:`, checkbox.prop('checked'));
|
||||
|
||||
// 更新显示（可选）
|
||||
updateSelectedLabels();
|
||||
} else {
|
||||
console.warn(`未找到与标签名 "${tagName}" 对应的 query session 复选框`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新外部展示容器中的选中标签显示（可选）
|
||||
*/
|
||||
function updateSelectedLabels() {
|
||||
// 更新 central session 选中的标签
|
||||
var selectedCentral = [];
|
||||
$('input[name="category"]:checked').each(function() {
|
||||
var label = $(`label[for="${this.id}"]`).text().replace(/\s*[\u2713\u2611]\s*/, '').trim(); // 去除勾选符号
|
||||
if (label) {
|
||||
selectedCentral.push(label);
|
||||
} else {
|
||||
// 兜底：使用 value
|
||||
selectedCentral.push($(this).val());
|
||||
}
|
||||
});
|
||||
$('#selected-labels').text(selectedCentral.length > 0 ? selectedCentral.join(', ') : '未选中');
|
||||
|
||||
// 更新 query session 选中的标签
|
||||
var selectedQuery = [];
|
||||
$('input[name="tags"]:checked').each(function() {
|
||||
var label = $(`label[for="${this.id}"]`).text().replace(/\s*[\u2713\u2611]\s*/, '').trim(); // 去除勾选符号
|
||||
if (label) {
|
||||
selectedQuery.push(label);
|
||||
} else {
|
||||
// 兜底：使用 value
|
||||
selectedQuery.push($(this).val());
|
||||
}
|
||||
});
|
||||
$('#querySession').text(selectedQuery.length > 0 ? selectedQuery.join(', ') : '未选中');
|
||||
}
|
||||
|
||||
// 初始化时调用一次以设置初始状态
|
||||
updateSelectedLabels();
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
const iframe = document.getElementById('myIframe');
|
||||
iframe.onload = () => {
|
||||
try {
|
||||
const doc = iframe.contentDocument || iframe.contentWindow.document;
|
||||
|
||||
// 向iframe页面注入脚本，添加点击事件监听
|
||||
const script = doc.createElement('script');
|
||||
script.textContent = `
|
||||
// 在iframe页面中添加全局点击事件监听
|
||||
document.addEventListener('click', function(e) {
|
||||
let target = e.target;
|
||||
let tagName = null;
|
||||
|
||||
// 检查元素本身或其父元素是否包含目标格式的文本
|
||||
while (target && target !== document.body) {
|
||||
// 尝试从多种可能的属性中获取标签名
|
||||
const possibleNames = [
|
||||
target.textContent?.trim(),
|
||||
target.innerText?.trim(),
|
||||
target.getAttribute('id'),
|
||||
target.getAttribute('data-name'),
|
||||
target.getAttribute('title')
|
||||
];
|
||||
|
||||
// 检查是否有符合格式的标签名
|
||||
for (const name of possibleNames) {
|
||||
if (name && name.includes('PL') && name.includes('event') && name.includes('.bag.dir')) {
|
||||
tagName = name;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (tagName) break;
|
||||
target = target.parentElement;
|
||||
}
|
||||
|
||||
// 如果找到符合条件的标签名，发送消息给父页面
|
||||
if (tagName) {
|
||||
window.parent.postMessage({
|
||||
type: 'tagClick',
|
||||
tagName: tagName
|
||||
}, '*'); // 使用*允许所有来源，生产环境应指定具体域名
|
||||
}
|
||||
});
|
||||
`;
|
||||
doc.head.appendChild(script);
|
||||
console.log('成功注入点击事件监听器');
|
||||
} catch (error) {
|
||||
console.error('注入脚本时出错:', error);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
$(function () {
|
||||
|
||||
|
||||
|
||||
// 添加调试代码
|
||||
console.log('SweetAlert2 是否加载:', typeof Swal);
|
||||
console.log('Swal.fire 是否存在:', typeof Swal.fire);
|
||||
|
||||
const $result = $("#result");
|
||||
const $img = $("#preview");
|
||||
const $imageinfo = $("#imageinfo");
|
||||
const $imagecount = $("#imagecount");
|
||||
const hiddenImageCount = $("#hiddenImageCount");
|
||||
|
||||
function updateMess(index ,imagename){
|
||||
const cur = `当前展示第 <b>${index}</b> 张 , 文件名为 : <b>${imagename}</b> `;
|
||||
$imageinfo.html( );
|
||||
$imageinfo.html( cur);
|
||||
|
||||
}
|
||||
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
|
||||
console.log('Bag Folder:', bag_folder);
|
||||
$('#selectKey').on('change', function () {
|
||||
const bagname = $(this).val();
|
||||
console.log(bagname)
|
||||
const bagfullname = $('#selectKey option:selected').text();
|
||||
|
||||
$('#bagname').attr('data-value', bagfullname);
|
||||
var html_name = $('#html_name').val();
|
||||
html_name = html_name.replace(".html","")
|
||||
|
||||
|
||||
if (!bagname) return;
|
||||
$('#result').text('加载中...');
|
||||
$.ajax({
|
||||
url: '/api/getimagedata',
|
||||
|
||||
method: 'GET',
|
||||
data: { bagname: bagname ,
|
||||
htmlname: html_name ,
|
||||
bagfolder: bag_folder ,
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
// $('#result').text(res.message ?? JSON.stringify(res, null, 2));
|
||||
var data = res.result
|
||||
// 清空两个输入框
|
||||
$('#numberInput').val('');
|
||||
$('#stepInput').val('');
|
||||
|
||||
console.log(data)
|
||||
var size = data.size
|
||||
var index = data.index
|
||||
var imagename = data.imagename ;
|
||||
// 去除路径
|
||||
imagename = getJpgFileName(imagename) ;
|
||||
// 将包中图片数量更新到隐藏域
|
||||
|
||||
$('#hiddenImageCount').attr('data-value', size);
|
||||
const base = `数据包中共有 <b>${size}</b> 张图片`;
|
||||
const cur = `当前展示第 <b>1</b> 张 , 文件名为 : <b>${imagename}</b> `;
|
||||
$imageinfo.html( cur);
|
||||
$imagecount.html(base );
|
||||
|
||||
// 展示图片
|
||||
|
||||
showImage(imagename,bagname,html_name,bag_folder);
|
||||
|
||||
},
|
||||
error(xhr) {
|
||||
const msg = xhr.responseJSON?.message || xhr.statusText;
|
||||
$('#result').text('加载失败：' + msg);
|
||||
}
|
||||
});
|
||||
})
|
||||
})
|
||||
|
||||
const htmlName = "{{ html_name }}";
|
||||
|
||||
const logBox = document.getElementById('log');
|
||||
|
||||
// function log(text) {
|
||||
// const msg = `[${new Date().toISOString()}] ${text}`;
|
||||
// console.log(msg);
|
||||
// logBox.textContent += msg + '\n';
|
||||
// logBox.scrollTop = logBox.scrollHeight;
|
||||
// }
|
||||
|
||||
window.addEventListener('message', (evt) => {
|
||||
// 仅接收来自本域的 iframe 消息（更安全可校验 evt.origin）
|
||||
// console.debug('收到消息：', evt.data);
|
||||
if (!evt || !evt.data || !evt.data.type) return;
|
||||
const { type, href, target, value, x, y, button, buttons, key, code, ctrlKey, shiftKey, altKey, metaKey, timestamp } = evt.data;
|
||||
const entry = {
|
||||
type, href, target, value,
|
||||
coord: x !== undefined ? { x, y } : null,
|
||||
mouseButtons: buttons !== undefined ? { left: button === 0, middle: button === 1, right: button === 2 } : null,
|
||||
keyInfo: key !== undefined ? { key, code, ctrlKey, shiftKey, altKey, metaKey } : null,
|
||||
ts: new Date(timestamp).toLocaleTimeString()
|
||||
};
|
||||
// log(`${entry.ts} [${type}] ${href || ''} ${target || ''} ${key || ''} ${JSON.stringify(entry.coord || entry.mouseButtons || entry.keyInfo)}`);
|
||||
});
|
||||
|
||||
|
||||
|
||||
// 获取 DOM 元素
|
||||
const selectEl = document.getElementById('selectKey');
|
||||
const inputEl = document.getElementById('numberInput');
|
||||
const btnEl = document.getElementById('confirmBtn');
|
||||
const resultEl = document.getElementById('result');
|
||||
const imageinfo = $("#imageinfo");
|
||||
const hiddenImageCount = $("#hiddenImageCount");
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
const html_folder = $('#hiddenHtmlFolder').attr('value');
|
||||
// 1) 加载下拉选项（默认空）
|
||||
async function loadOptions() {
|
||||
try {
|
||||
const res = await fetch('/api/options/{{ html_name }}/{{ html_folder }}'); // 后台接口：返回 [{value: 'a', label: 'A'}, ...]
|
||||
console.log(res)
|
||||
|
||||
if (!res.ok) throw new Error('加载选项失败');
|
||||
const json = await res.json(); // json 是解析后的 JS 对象
|
||||
console.log(json.status); // true
|
||||
console.log(json.result);
|
||||
const data = await json.result;
|
||||
|
||||
// 清空并填充选项（保留默认占位）
|
||||
selectEl.innerHTML = '<option value="" disabled selected>请选择</option>';
|
||||
data.forEach(item => {
|
||||
const opt = document.createElement('option');
|
||||
opt.value = item.value;
|
||||
opt.textContent = item.label;
|
||||
selectEl.appendChild(opt);
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('加载下拉选项出错:', err);
|
||||
selectEl.innerHTML = '<option value="" disabled selected>加载失败</option>';
|
||||
}
|
||||
}
|
||||
|
||||
// 2) 输入数字后，确认按钮点击事件
|
||||
function get_image() {
|
||||
// 打印输入框的值
|
||||
const inputValue = inputEl.value.trim();
|
||||
const imagecount = document.getElementById('imagecount');
|
||||
const imageinfo = document.getElementById('imageinfo');
|
||||
const hiddenTag = document.getElementById('hiddenTag');
|
||||
|
||||
console.log('输入框的值：', inputValue);
|
||||
var html_name = $('#html_name').val();
|
||||
html_name = html_name.replace(".html","")
|
||||
console.log('html000_name :'+html_name)
|
||||
|
||||
var size = $("#hiddenImageCount").attr('data-value');
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
const n1 = Number(inputValue);
|
||||
const n2 = Number(size);
|
||||
console.log('html111_name :'+html_name)
|
||||
// 在页面显示该数字
|
||||
if (inputValue === '') {
|
||||
resultEl.textContent = '请输入有效数字';
|
||||
resultEl.style.color = '#999';
|
||||
} else if(n1>n2){
|
||||
alert('输入值大于图片总数')
|
||||
}else {
|
||||
// 可按需做数字校验
|
||||
const num = Number(inputValue);
|
||||
if (isNaN(num)) {
|
||||
resultEl.textContent = '输入内容不是有效数字';
|
||||
resultEl.style.color = 'red';
|
||||
} else {
|
||||
// 更新隐藏域的值 为输入值
|
||||
$('#hiddenTag').attr('data-value', inputValue);
|
||||
// 更新页面显示图片信息
|
||||
const base = `数据包中共有 <b>${size}</b> 张图片`;
|
||||
imagecount.innerHTML = '';
|
||||
$(imagecount).html( base);
|
||||
|
||||
|
||||
resultEl.textContent = `数字：${num}`;
|
||||
resultEl.style.color = '#333';
|
||||
|
||||
// 加载图片
|
||||
// 通过包名 调用方法获取 包内图片的数据
|
||||
const bagname = $('#bagname').attr('data-value');
|
||||
console.log('输入数字获取图片：'+html_name)
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
console.log('输入-----------：'+bag_folder)
|
||||
|
||||
|
||||
const imagename = getImageByBagAndIndex(bagname,inputValue,html_name,bag_folder);
|
||||
console.log(imagename)
|
||||
|
||||
var imagename_short = getJpgFileName(imagename);
|
||||
console.log(imagename_short)
|
||||
const cur = `当前展示第 <b>${inputValue}</b> 张 , 文件名为 : <b>${imagename_short}</b> `;
|
||||
|
||||
$(imageinfo).html('');
|
||||
$(imageinfo).html( cur);
|
||||
showImage(imagename_short,bagname,html_name,bag_folder);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- 页面 左右按钮点击图片变更动作 start -->
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
const stepDownBtn = document.getElementById('stepDownBtn');
|
||||
const stepUpBtn = document.getElementById('stepUpBtn');
|
||||
const stepInput = document.getElementById('stepInput');
|
||||
const hiddenTag = document.getElementById('hiddenTag');
|
||||
const resultImg = document.getElementById('resultImg'); // 结果图片容器
|
||||
const $imageinfo = $("#imageinfo");
|
||||
const imagecount = $("#imagecount").attr('data-value');
|
||||
|
||||
// 工具：将字符串转为整数，失败返回 NaN
|
||||
function toInt(v) {
|
||||
const n = Number(v);
|
||||
return Number.isInteger(n) ? n : NaN;
|
||||
}
|
||||
|
||||
// 从输入框或隐藏域读取“当前值”
|
||||
function getCurrentValue() {
|
||||
const raw = stepInput.value.trim();
|
||||
if (raw !== '') {
|
||||
const step = toInt(raw);
|
||||
if (!isNaN(step)) return step;
|
||||
}
|
||||
return toInt(hiddenTag.dataset.value) ?? 1; // 默认 1
|
||||
}
|
||||
|
||||
// 更新隐藏域与输入框显示
|
||||
function updateHidden(newValue) {
|
||||
const n = toInt(newValue);
|
||||
if (!isNaN(n)) {
|
||||
hiddenTag.dataset.value = n;
|
||||
|
||||
return n;
|
||||
}
|
||||
return getCurrentValue(); // 回退
|
||||
}
|
||||
|
||||
// 减步长
|
||||
stepDownBtn.addEventListener('click', async function () {
|
||||
var html_name = $('#html_name').val();
|
||||
html_name = html_name.replace(".html","")
|
||||
console.log('html_name :'+html_name)
|
||||
const base = toInt(hiddenTag.dataset.value)
|
||||
const hide =toInt(hiddenTag.dataset.value)
|
||||
const $imagecount = $("#imagecount");
|
||||
console.log(' hide 值为：'+hide)
|
||||
const raw = stepInput.value.trim() ;
|
||||
const step = raw === "" ? 1 : parseInt(raw, 10);
|
||||
console.log(' step 值为：'+step)
|
||||
const newVal = base - step;
|
||||
|
||||
console.log(' newVal 值为：'+newVal)
|
||||
if (isNaN(newVal) || !Number.isInteger(newVal) || newVal <= 0) {
|
||||
alert('索引必须为大于零整数');
|
||||
return;
|
||||
}
|
||||
console.log('newVal 值为：'+ newVal)
|
||||
const final = updateHidden(newVal);
|
||||
console.log('final 值为：'+ final)
|
||||
|
||||
|
||||
|
||||
|
||||
// 加载图片
|
||||
// 通过包名 调用方法获取 包内图片的数据
|
||||
const bagname = $('#bagname').attr('data-value');
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
|
||||
const imagename = getImageByBagAndIndex(bagname,newVal,html_name,bag_folder);
|
||||
console.log(imagename)
|
||||
|
||||
var imagename_short = getJpgFileName(imagename);
|
||||
console.log(imagename_short)
|
||||
|
||||
const cur = `当前展示第 <b>${final}</b> 张 , 文件名为 : <b>${imagename_short}</b> `;
|
||||
$imageinfo.html( );
|
||||
$imageinfo.html( cur);
|
||||
showImage(imagename_short,bagname,html_name,bag_folder);
|
||||
|
||||
});
|
||||
|
||||
// 加步长
|
||||
stepUpBtn.addEventListener('click', async function () {
|
||||
var html_name = $('#html_name').val();
|
||||
html_name = html_name.replace(".html","")
|
||||
console.log('html_name :'+html_name)
|
||||
const base = toInt(hiddenTag.dataset.value)
|
||||
const hide =toInt(hiddenTag.dataset.value)
|
||||
console.log(' hide 值为：'+hide)
|
||||
const raw = stepInput.value.trim() ;
|
||||
const step = raw === "" ? 1 : parseInt(raw, 10);
|
||||
console.log(' step 值为：'+step)
|
||||
const newVal = base + step;
|
||||
const imagecount = $("#hiddenImageCount").attr('data-value');
|
||||
console.log(' newVal 值为：'+newVal + 'imagecount:' + imagecount)
|
||||
if (isNaN(newVal) || !Number.isInteger(newVal) || newVal <= 0) {
|
||||
alert('索引必须为大于零整数');
|
||||
return;
|
||||
}else if(newVal > imagecount){
|
||||
alert('超出索引最大数值');
|
||||
return;
|
||||
}
|
||||
|
||||
const final = updateHidden(newVal);
|
||||
|
||||
|
||||
|
||||
// 加载图片
|
||||
// 通过包名 调用方法获取 包内图片的数据
|
||||
const bagname = $('#bagname').attr('data-value');
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
|
||||
const imagename = getImageByBagAndIndex(bagname,newVal,html_name,bag_folder);
|
||||
console.log(imagename)
|
||||
|
||||
var imagename_short = getJpgFileName(imagename);
|
||||
console.log(imagename_short)
|
||||
|
||||
const cur = `当前展示第 <b>${final}</b> 张 , 文件名为 : <b>${imagename_short}</b> `;
|
||||
$imageinfo.html( );
|
||||
$imageinfo.html( cur);
|
||||
showImage(imagename_short,bagname,html_name,bag_folder);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
<!-- 页面 左右按钮点击图片变更 end -->
|
||||
|
||||
|
||||
<!-- 单选和复选框 加载数据 js -->
|
||||
// 容器与接口
|
||||
const containerC = document.getElementById('checkbox-container');
|
||||
const containerR = document.getElementById('radio-container');
|
||||
// const html_folder = $('#hiddenHtmlFolder').attr('value');
|
||||
const API = '/api/options/{{ html_name }}/{{ html_folder }}'; // 查询html 中的包数据
|
||||
|
||||
var id = {{ id }} ;
|
||||
var data = getCentralQuerySession(id) ;
|
||||
|
||||
var centralsession = data.centralSession ;
|
||||
var querySessions = data.querySessions ;
|
||||
console.log(centralsession)
|
||||
console.log(querySessions)
|
||||
// 加载并渲染
|
||||
async function loadData() {
|
||||
try {
|
||||
const res = await fetch(API);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const json = await res.json();
|
||||
|
||||
|
||||
const data = Array.isArray(json.result) ? json.result : [];
|
||||
|
||||
// 清空旧项
|
||||
containerC.innerHTML = '';
|
||||
containerR.innerHTML = '';
|
||||
|
||||
// 生成复选框（多选）
|
||||
data.forEach(opt => {
|
||||
const idC = `cb-${opt.value}`;
|
||||
const labelC = document.createElement('label');
|
||||
labelC.className = 'item';
|
||||
labelC.htmlFor = idC;
|
||||
|
||||
const cb = document.createElement('input');
|
||||
cb.type = 'checkbox';
|
||||
cb.id = idC;
|
||||
cb.name = 'tags'; // 多选用同一 name
|
||||
cb.value = opt.value;
|
||||
|
||||
// ✅ 修改位置1: 检查是否在querySessions数组中，如果是则选中
|
||||
if (querySessions.includes(opt.value)) {
|
||||
cb.checked = true;
|
||||
}
|
||||
|
||||
labelC.appendChild(cb);
|
||||
labelC.append(' ', opt.label);
|
||||
containerC.appendChild(labelC);
|
||||
});
|
||||
|
||||
// 生成单选框（单选）
|
||||
data.forEach(opt => {
|
||||
const idR = `rd-${opt.value}`;
|
||||
const labelR = document.createElement('label');
|
||||
labelR.className = 'item';
|
||||
labelR.htmlFor = idR;
|
||||
|
||||
const rd = document.createElement('input');
|
||||
rd.type = 'radio';
|
||||
rd.id = idR;
|
||||
rd.name = 'category'; // 单选用同一 name
|
||||
rd.value = opt.value;
|
||||
|
||||
// ✅ 修改位置2: 检查是否等于centralsession，如果是则选中
|
||||
if (opt.value === centralsession) {
|
||||
rd.checked = true;
|
||||
}
|
||||
|
||||
labelR.appendChild(rd);
|
||||
labelR.append(' ', opt.label);
|
||||
containerR.appendChild(labelR);
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('加载选项失败:', err);
|
||||
containerC.textContent = '加载失败';
|
||||
containerR.textContent = '加载失败';
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// 仅当复选框 A 变化时，打印页面上所有已选中的 label 文本 START
|
||||
function watchCheckboxPrintLabels(cbContainerId, output) {
|
||||
|
||||
const container = typeof output === 'string' ? document.querySelector(output) : output;
|
||||
const cbContainer = document.getElementById(cbContainerId);
|
||||
if (!container || !cbContainer) {
|
||||
console.error('[watchCheckboxPrintLabels] 容器未找到');
|
||||
return;
|
||||
}
|
||||
|
||||
// 使用事件委托监听复选框变化（更稳健，支持动态新增）
|
||||
document.addEventListener('change', (e) => {
|
||||
if (!e.target.matches(`#${cbContainerId} input[type="checkbox"]`)) return;
|
||||
// 收集页面上所有已勾选复选框的 label 文本
|
||||
const labels = Array.from(document.querySelectorAll('input[type="checkbox"]:checked'))
|
||||
.map(cb => {
|
||||
// 优先通过 for 匹配
|
||||
if (cb.id) {
|
||||
const lab = document.querySelector(`label[for="${cb.id}"]`);
|
||||
if (lab) return lab.textContent.trim();
|
||||
}
|
||||
// 兜底：若复选框被 <label><input>...</label> 包裹，取父级文本
|
||||
if (cb.parentElement?.tagName === 'LABEL') {
|
||||
return cb.parentElement.textContent.replace(/\s*[\u2713\u2611]\s*/,'').trim(); // 去除勾选符号
|
||||
}
|
||||
// 再兜底：title、data-label，最后回退 value
|
||||
if (cb.title) return cb.title.trim();
|
||||
if (cb.dataset.label) return cb.dataset.label.trim();
|
||||
return cb.value.trim();
|
||||
});
|
||||
|
||||
// 更新外部 div 显示
|
||||
container.textContent = labels.length > 0 ? labels.join(', ') : '未选中';
|
||||
}, { passive: true });
|
||||
}
|
||||
|
||||
// 调用：传入复选框容器ID 与 外部展示容器（选择器或DOM）
|
||||
watchCheckboxPrintLabels('checkbox-container', '#querySession');
|
||||
|
||||
// 调用：传入复选框容器ID 与 外部展示容器（选择器或DOM）
|
||||
watchCheckboxPrintLabels('radio-container', '#selected-labels');
|
||||
// 仅当复选框 A 变化时，打印页面上所有已选中的 label 文本 END
|
||||
|
||||
// 获取表单数据
|
||||
function getFormData() {
|
||||
var bag_folder = $('#mtaBagfolder').val();
|
||||
console.log(bag_folder)
|
||||
const html_name = $('#html_name').val();
|
||||
var id = $("#mtataskid").attr('data-value');
|
||||
const checkedBoxes = Array.from(
|
||||
document.querySelectorAll('#checkbox-container input:checked')
|
||||
).map(cb => cb.value);
|
||||
|
||||
const selectedRadio = document.querySelector('#radio-container input:checked');
|
||||
return {
|
||||
id:id,
|
||||
html_name:html_name,
|
||||
bag_folder: bag_folder,
|
||||
tags: checkedBoxes,
|
||||
category: selectedRadio ? selectedRadio.value : null
|
||||
};
|
||||
}
|
||||
function showCommitAlert(message, html_name,id, central_session, query_session,bag_folder,category, callback) {
|
||||
Swal.fire({
|
||||
title: 'MTA bag 提交成功！',
|
||||
text: message,
|
||||
icon: 'success',
|
||||
showCancelButton: true, // 显示取消按钮
|
||||
confirmButtonText: '执行命令',
|
||||
cancelButtonText: '返回首页',
|
||||
allowOutsideClick: false,
|
||||
allowEscapeKey: false
|
||||
}).then((result) => {
|
||||
if (result.isConfirmed) {
|
||||
// 点击"执行命令"按钮
|
||||
console.log('🎯 执行命令:', message);
|
||||
console.log('⏰ 执行时间:', new Date().toLocaleString());
|
||||
console.log('执行的html 页面为:', message);
|
||||
console.log('showCommitAlert methon id value :'+id )
|
||||
command = ''
|
||||
// 可以在这里添加实际执行命令的代码
|
||||
console.log(bag_folder)
|
||||
runMTACommand(command,html_name, central_session,query_session,id,bag_folder,category)
|
||||
|
||||
} else if (result.dismiss === Swal.DismissReason.cancel) {
|
||||
// 点击"返回首页"按钮
|
||||
|
||||
window.location.href = "/";
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
// 提交示例
|
||||
function handleSubmit() {
|
||||
const data = getFormData();
|
||||
console.log('提交数据:', data);
|
||||
var category = data.category ;
|
||||
var tags = data.tags ;
|
||||
var bag_folder = data.bag_folder ;
|
||||
var html_name = $('#html_name').val();
|
||||
|
||||
console.log('提交数据:', mtataskid);
|
||||
// 检查 central session
|
||||
if (!category || category.trim() === '') {
|
||||
alert('Central Session 不能为空，请选择或输入有效值');
|
||||
return;
|
||||
}
|
||||
|
||||
// 检查 query session
|
||||
if (!tags || tags.length === 0) {
|
||||
alert('Query Session 不能为空，请至少选择一个标签');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
$.ajax({
|
||||
url: '/api/submit',
|
||||
async: false, // 同步
|
||||
method: 'POST',
|
||||
contentType: 'application/json',
|
||||
data: JSON.stringify(data),
|
||||
success(result) {
|
||||
console.log('✅ bagdata 提交成功:', result);
|
||||
console.log('状态:', result.status);
|
||||
console.log('data:', result.result.data);
|
||||
console.log('Central Session:', result.result.data.central_session);
|
||||
console.log('Query Sessions:', result.result.data.query_sessions);
|
||||
console.log('bag_folder:', bag_folder);
|
||||
console.log('handleSubmit methon id value :'+id )
|
||||
var command = './multi_session_aggregation --central_session '+result.result.data.central_session + ' --query_session '+result.result.data.query_sessions
|
||||
showCommitAlert(command,html_name,id,result.result.data.central_session,result.result.data.query_sessions,bag_folder,category) ;
|
||||
|
||||
// alert(`mta 任务 提交成功 \n 命令为：`+command);
|
||||
|
||||
}
|
||||
})
|
||||
|
||||
} catch (error) {
|
||||
console.error('提交失败:', error);
|
||||
alert(`mta 任务 提交失败: ${error.message}`);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
// 清空
|
||||
function clearAll() {
|
||||
$("#selected-labels").empty();
|
||||
$("#querySession").empty();
|
||||
|
||||
document.querySelectorAll('#checkbox-container input').forEach(cb => cb.checked = false);
|
||||
document.querySelectorAll('#radio-container input').forEach(rd => rd.checked = false);
|
||||
}
|
||||
|
||||
// 页面加载完成后拉取数据
|
||||
loadData();
|
||||
|
||||
// 绑定事件
|
||||
btnEl.addEventListener('click', get_image);
|
||||
|
||||
// 页面加载完成后拉取选项
|
||||
loadOptions();
|
||||
|
||||
// 单选选择项发生变化后页面展示
|
||||
$(document).ready(function() {
|
||||
// 使用事件委托，适用于动态添加的选项
|
||||
$('#radio-container').on('change', 'input[name="category"]', function() {
|
||||
if ($(this).is(':checked')) {
|
||||
// console.log('单选选中的值:', $(this).val());
|
||||
|
||||
$('#selected-labels').text($(this).val());
|
||||
}
|
||||
});
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,697 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>SessionRecord 列表</title>
|
||||
<link href="../static/css/bootstrap.min.css" rel="stylesheet">
|
||||
<link href="../static/css//bootstrap-icons.css" rel="stylesheet">
|
||||
<link rel="stylesheet" href="../static/css/sweetalert2.css">
|
||||
|
||||
<!-- 引入 SweetAlert2 JS -->
|
||||
<script src="../static/js/sweetalert2.js"></script>
|
||||
<style>
|
||||
.text-truncate-custom { max-width: 240px; }
|
||||
.json-viewer { font-size: 0.875rem; background-color: #f8f9fa; padding: 0.5rem; border-radius: 0.25rem; white-space: pre-wrap; margin-top: 0.25rem; }
|
||||
.badge-bool-null { font-size: 0.75em; padding: 0.25em 0.5em; }
|
||||
.badge-bool-true { background-color: #198754; }
|
||||
.badge-bool-false { background-color: #6c757d; }
|
||||
.badge-bool-null { background-color: #6c757d; color: #fff; }
|
||||
.card { box-shadow: 0 0.125rem 0.25rem rgba(0,0,0,0.075); }
|
||||
.table th { width: 140px; }
|
||||
.loading-shade { position: absolute; inset: 0; background: rgba(255,255,255,0.8); display: flex; align-items: center; justify-content: center; z-index: 10; }
|
||||
</style>
|
||||
</head>
|
||||
<body class="p-3">
|
||||
<div class="container-fluid">
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<h5 class="mb-0">SessionRecord 列表</h5>
|
||||
<input type="hidden" id = "page" name="html_name" value="{{ html_name }}">
|
||||
</div>
|
||||
<div class="card-body p-0">
|
||||
<!-- 筛选区 -->
|
||||
<div class="p-3 border-bottom">
|
||||
<div class="row g-2">
|
||||
<div class="col-md-3">
|
||||
<input type="text" class="form-control" id="html_name" placeholder="html_name" value="">
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<input type="text" class="form-control" id="central_session" placeholder="central_session" value="">
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<span>是否筛选过数据包</span>
|
||||
<select class="form-select" id="is_filter_bag">
|
||||
<option value="">全部</option>
|
||||
<option value="true">是</option>
|
||||
<option value="false">否</option>
|
||||
</select>
|
||||
</div>
|
||||
<!-- <div class="col-md-3">-->
|
||||
<!-- <span>是否跑过mta</span>-->
|
||||
<!-- <select class="form-select" id="is_run_mta">-->
|
||||
<!-- <option value="">全部</option>-->
|
||||
<!-- <option value="true">是</option>-->
|
||||
<!-- <option value="false">否</option>-->
|
||||
<!-- </select>-->
|
||||
<!-- </div>-->
|
||||
<div class="col-md-3">
|
||||
<span>mta 任务状态</span>
|
||||
<select class="form-select" id="mta_task_status">
|
||||
<option value="">全部</option>
|
||||
<option value="NOT_STARTED">未启动</option>
|
||||
<option value="RUNNING">在运行</option>
|
||||
<option value="COMPLETED">已完成</option>
|
||||
<option value="FAILED">失败</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="col-md-2">
|
||||
<select class="form-select" id="per_page">
|
||||
<option value="10">10 条/页</option>
|
||||
<option value="20" selected>20 条/页</option>
|
||||
<option value="50">50 条/页</option>
|
||||
<option value="100">100 条/页</option>
|
||||
</select>
|
||||
</div>
|
||||
<!-- jquery 添加一个 同步数据的按钮 -->
|
||||
<div class="col-md-2 ms-auto text-end">
|
||||
<button id="query-data-btn" class="btn btn-primary">
|
||||
<i class="fas fa-sync-alt"></i> 查询
|
||||
</button>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<input type="text" class="" id="html_folder" placeholder="html_folder" value="">
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<input type="text" class="" id="bag_folder" placeholder="bag_folder" value="">
|
||||
</div>
|
||||
<div class="col-md-2 ms-auto text-end">
|
||||
<button id="sync-data-btn" class="btn btn-primary">
|
||||
<i class="fas fa-sync-alt"></i> 同步数据
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 表格 -->
|
||||
<div class="table-responsive" style="position:relative;">
|
||||
<table class="table table-hover mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>HTML Name</th>
|
||||
<th>Central Session</th>
|
||||
<th>Query Sessions</th>
|
||||
<th>Html Folder</th>
|
||||
<th>筛选数据包</th>
|
||||
<!-- <th>MTA 任务</th>-->
|
||||
<th>MTA Task 状态</th>
|
||||
<th>MTA 结果</th>
|
||||
<th>备注</th>
|
||||
<th>创建时间</th>
|
||||
<th>更新时间</th>
|
||||
<th>操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="tbody">
|
||||
<!-- 数据由 jQuery 加载 -->
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- 分页与信息 -->
|
||||
<div class="p-3 d-flex justify-content-between align-items-center">
|
||||
<div id="info" class="text-muted small"></div>
|
||||
<nav aria-label="分页导航">
|
||||
<ul class="pagination mb-0" id="pagination"></ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 编辑弹窗 Modal -->
|
||||
<div class="modal fade" id="editModal" tabindex="-1" aria-labelledby="editModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="editModalLabel">编辑记录</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<!-- 这里放置你要编辑或展示的字段 -->
|
||||
<form id="editForm">
|
||||
<input type="hidden" id="edit_id" name="id">
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="edit_html_name" class="form-label">HTML Name</label>
|
||||
<input type="text" class="form-control" id="edit_html_name" name="html_name" required>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="edit_central_session" class="form-label">Central Session</label>
|
||||
<input type="text" class="form-control" id="edit_central_session" name="central_session" required>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="edit_central_session" class="form-label">Query Sessions</label>
|
||||
<input type="text" class="form-control" id="edit_query_session" name="query_session" required>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="edit_is_filter_bag" class="form-label">筛选数据包</label>
|
||||
<select class="form-select" id="edit_is_filter_bag" name="is_filter_bag">
|
||||
<option value="">请选择</option>
|
||||
<option value="true">是</option>
|
||||
<option value="false">否</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="edit_is_run_mta" class="form-label">是否跑 MTA</label>
|
||||
<select class="form-select" id="edit_is_run_mta" name="is_run_mta">
|
||||
<option value="">请选择</option>
|
||||
<option value="true">是</option>
|
||||
<option value="false">否</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="run_mta_result" class="form-label">是否成功</label>
|
||||
<input type="text" class="form-control" id="edit_run_mta_result" name="run_mta_result" required>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="edit_remark" class="form-label">备注</label>
|
||||
<textarea class="form-control" id="edit_remark" name="remark" rows="3"></textarea>
|
||||
</div>
|
||||
|
||||
<!-- 其它字段按需添加，如 query_sessions、selected_image_url、run_mta_result 等 -->
|
||||
</form>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">取消</button>
|
||||
<button type="button" class="btn btn-primary" id="saveEditBtn" onclick="">保存</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
image
|
||||
<!-- 图片展示弹出框 -->
|
||||
<div class="modal fade" id="imageModal" tabindex="-1" aria-labelledby="imageModal" aria-hidden="true">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="imageModal">MTA 结果展示</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body" id="MTAImageDiv" >
|
||||
<!-- 这里放置你要编辑或展示的字段 -->
|
||||
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
|
||||
<button type="button" class="btn btn-primary" id="closeImage1" onclick="closeImage()">关闭</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 你的项目中已存在 jquery.js，确保已正确引入，如： -->
|
||||
<!-- <script src="/path/to/your/jquery.js"></script> -->
|
||||
<script src="../static/js/jquery-3.7.1.min.js"></script>
|
||||
<script src="../static/js/bootstrap.bundle.min.js"></script>
|
||||
<script>
|
||||
|
||||
// 点击查询数据按钮 start
|
||||
$('#query-data-btn').click(function() {
|
||||
// 显示加载状态
|
||||
const $btn = $(this);
|
||||
|
||||
loadData() ;
|
||||
});
|
||||
|
||||
// 点击同步数据按钮 start
|
||||
$('#sync-data-btn').click(function() {
|
||||
// 显示加载状态
|
||||
const $btn = $(this);
|
||||
$btn.prop('disabled', true);
|
||||
$btn.html('<i class="fas fa-spinner fa-spin"></i> 同步中...');
|
||||
// 同步数据， 先获取输入的 bagfolder 和 htmlfolder 参数值
|
||||
var bag_folder = $('#bag_folder').val();
|
||||
var html_folder = $('#html_folder').val();
|
||||
console.log(bag_folder)
|
||||
console.log(html_folder)
|
||||
|
||||
|
||||
// 调用同步API /api/syncdata
|
||||
$.ajax({
|
||||
url: '/api/syncdata',
|
||||
method: 'GET',
|
||||
data: {
|
||||
bag_folder: bag_folder,
|
||||
html_folder: html_folder
|
||||
},
|
||||
dataType: 'json',
|
||||
success: function(response) {
|
||||
console.log(response)
|
||||
console.log(response.scanned_files)
|
||||
// 同步成功提示
|
||||
Swal.fire({
|
||||
title: '同步成功！',
|
||||
text: 'html文件夹数量：'+response.scanned_files +'，数据库新增数量：'+response.new_records_added,
|
||||
icon: 'success'
|
||||
});
|
||||
|
||||
// 刷新当前页数据（假设你有加载数据的函数）
|
||||
loadData(); // currentPage 是当前页码变量
|
||||
},
|
||||
error: function(xhr) {
|
||||
Swal.fire({
|
||||
title: '同步失败',
|
||||
text: xhr.responseJSON?.message || '服务器错误',
|
||||
icon: 'error'
|
||||
});
|
||||
},
|
||||
complete: function() {
|
||||
// 恢复按钮状态
|
||||
$btn.prop('disabled', false);
|
||||
$btn.html('<i class="fas fa-sync-alt"></i> 同步数据');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
// 点击同步数据按钮 start
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// 保存按钮点击（使用 jQuery AJAX）
|
||||
$('#saveEditBtn').on('click', function () {
|
||||
console.log("进入 保存数据方法 ")
|
||||
const $form = $('#editForm');
|
||||
const $btn = $(this).prop('disabled', true); // 防止重复提交
|
||||
|
||||
|
||||
|
||||
// 2) 获取表单数据并转为普通对象
|
||||
const formData = new FormData($form[0]);
|
||||
const data = Object.fromEntries(formData);
|
||||
console.log(data)
|
||||
|
||||
|
||||
// 4) 判断是新增还是更新
|
||||
const id = data.id;
|
||||
|
||||
|
||||
const url = '/api/saveitems';
|
||||
const method = 'POST';
|
||||
|
||||
// 5) 使用 jQuery $.ajax 发送请求
|
||||
$.ajax({
|
||||
url: url,
|
||||
type: method,
|
||||
contentType: 'application/json',
|
||||
data: JSON.stringify(data),
|
||||
dataType: 'json',
|
||||
success: function (result) {
|
||||
// 成功：隐藏弹窗，重新加载列表
|
||||
$('#editModal').modal('hide');
|
||||
loadData(); // 请确保此方法会重新请求列表并渲染表格
|
||||
},
|
||||
error: function (xhr, status, error) {
|
||||
console.error('保存失败:', status, error);
|
||||
let errorMsg = '保存失败';
|
||||
if (xhr.responseJSON && xhr.responseJSON.error) {
|
||||
errorMsg = xhr.responseJSON.error;
|
||||
} else if (xhr.responseText) {
|
||||
try {
|
||||
const res = JSON.parse(xhr.responseText);
|
||||
if (res.error) errorMsg = res.error;
|
||||
} catch (e) {
|
||||
errorMsg = xhr.responseText.substring(0, 100);
|
||||
}
|
||||
}
|
||||
alert(errorMsg); // 或使用更友好的 toast / modal 提示
|
||||
},
|
||||
complete: function () {
|
||||
$btn.prop('disabled', false); // 恢复按钮
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// 编辑数据
|
||||
function editRow(id){
|
||||
$.get('/api/items/' + id, function (data) {
|
||||
console.log(data.result)
|
||||
console.log("记录数据：", data.result.isFilterBag);
|
||||
console.log("记录数据：", data.result.isRunMta);
|
||||
var isFilterBag = data.result.isFilterBag.toString()
|
||||
var isRunMta = data.result.isRunMta.toString()
|
||||
$('#edit_id').val(data.result.id);
|
||||
$('#edit_html_name').val(data.result.htmlName);
|
||||
$('#edit_central_session').val(data.result.centralSession);
|
||||
$('#edit_query_session').val(data.result.querySessions);
|
||||
$('#edit_is_filter_bag').val(isFilterBag);
|
||||
$('#edit_is_run_mta').val(isRunMta);
|
||||
$('#edit_run_mta_result').val(data.result.runMtaResult);
|
||||
$('#edit_remark').val(data.result.remark);
|
||||
$('#editModal').modal('show');
|
||||
});
|
||||
|
||||
// 直接弹窗，填充）
|
||||
$('#editModal').modal('show');
|
||||
}
|
||||
|
||||
// stop Task
|
||||
function stopTask(id) {
|
||||
$.ajax({
|
||||
url: '/api/stoptask',
|
||||
async: false, // 同步
|
||||
method: 'GET',
|
||||
data: { id: id
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
var msg = res.message
|
||||
var result = res.success
|
||||
if (result ){
|
||||
Swal.fire({
|
||||
title: '任务停止！',
|
||||
text: msg,
|
||||
icon: 'success'
|
||||
});
|
||||
} else {
|
||||
Swal.fire({
|
||||
title: '任务停止失败',
|
||||
text: '任务停止失败，请联系管理员',
|
||||
icon: 'error'
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
loadData();
|
||||
}
|
||||
|
||||
// reload Task
|
||||
function reloadTask(id) {
|
||||
$.ajax({
|
||||
url: '/api/reloadtask',
|
||||
async: false, // 同步
|
||||
method: 'GET',
|
||||
data: { id: id
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
var msg = res.message
|
||||
var result = res.success
|
||||
console.log(msg)
|
||||
console.log(result)
|
||||
if (result ){
|
||||
Swal.fire({
|
||||
title: '任务重跑成功',
|
||||
text: msg,
|
||||
icon: 'success'
|
||||
});
|
||||
} else {
|
||||
Swal.fire({
|
||||
title: '任务重跑失败',
|
||||
text: msg,
|
||||
icon: 'error'
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
loadData();
|
||||
}
|
||||
|
||||
// 图片展示
|
||||
function imageshow(id){
|
||||
|
||||
// 直接弹窗，填充）
|
||||
$('#imageModal').modal('show');
|
||||
|
||||
$("#MTAImageDiv").empty(); // 或 .html('')
|
||||
|
||||
// 2) 构造要插入的新内容：一个带 class 和 id 的 div，内嵌一张图片
|
||||
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
const newContent = `
|
||||
<img id="imagePreview" src="http://localhost:5000/api/getmtaresult/${encodeURIComponent(id)}" alt="图片预览" width="700px">
|
||||
`;
|
||||
|
||||
// 3) 将新内容插入到目标 div 中
|
||||
$("#MTAImageDiv").html(newContent);
|
||||
}
|
||||
function closeImage(){
|
||||
$('#imageModal').modal('hide');
|
||||
}
|
||||
// 工具：对象转查询字符串（自动编码，排除空值）
|
||||
function qs(params) {
|
||||
return Object.keys(params)
|
||||
.filter(function(k) {
|
||||
var v = params[k];
|
||||
return v != null && v !== '';
|
||||
})
|
||||
.map(function(k) {
|
||||
return encodeURIComponent(k) + '=' + encodeURIComponent(params[k]);
|
||||
})
|
||||
.join('&');
|
||||
}
|
||||
|
||||
// 工具：当前查询参数（从表单元素获取）
|
||||
function currentQuery() {
|
||||
console.log('-----5-----'+$('#page').val())
|
||||
return {
|
||||
page: $('#page').val() || 1,
|
||||
per_page: $('#per_page').val() || 20,
|
||||
html_name: $('#html_name').val() || '',
|
||||
central_session: $('#central_session').val() || '',
|
||||
is_filter_bag: $('#is_filter_bag').val() || '',
|
||||
is_run_mta: $('#is_run_mta').val() || '' ,
|
||||
mta_task_status: $('#mta_task_status').val() || ''
|
||||
};
|
||||
}
|
||||
|
||||
// 工具：格式化时间
|
||||
function fmtDate(s) {
|
||||
if (!s) return '-';
|
||||
var d = new Date(s);
|
||||
return d.toISOString().slice(0, 19).replace('T', ' ');
|
||||
}
|
||||
|
||||
// 工具：布尔值徽章
|
||||
function badgeBool(v) {
|
||||
if (v === true) return '<span class="badge bg-success badge-bool-true">是</span>';
|
||||
if (v === false) return '<span class="badge bg-secondary badge-bool-false">否</span>';
|
||||
return '<span class="badge bg-secondary badge-bool-null">-</span>';
|
||||
}
|
||||
|
||||
// 工具：截断文本
|
||||
function trunc(s, len) {
|
||||
if (!s) return '-';
|
||||
s = String(s);
|
||||
return s.length <= len ? s : s.slice(0, len) + '…';
|
||||
}
|
||||
|
||||
// 工具：格式化 JSON（简单处理，可展开查看）
|
||||
function fmtJson(s) {
|
||||
if (!s) return '-';
|
||||
try {
|
||||
var j = JSON.parse(s);
|
||||
return JSON.stringify(j, null, 2);
|
||||
} catch (e) {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
|
||||
// 渲染表格数据
|
||||
function renderRows(items) {
|
||||
var tbody = $('#tbody');
|
||||
if (items.length === 0) {
|
||||
tbody.html('<tr><td colspan="12" class="text-center text-muted">暂无数据</td></tr>');
|
||||
return;
|
||||
}
|
||||
tbody.html($.map(items, function(it) {
|
||||
var qs = it.query_sessions ? it.query_sessions.split(',').map(decodeURIComponent).join(', ') : '-';
|
||||
return '<tr>' +
|
||||
'<td>' + it.id + '</td>' +
|
||||
'<td class="text-truncate-custom" title="' + $('<div>').text(it.html_name).html() + '" > <a href="/detail/' + $('<div>').text(it.html_name).html() + '/'+ $('<div>').text(it.bag_folder).html() + '/'+ $('<div>').text(it.html_folder).html()+ '/'+ $('<div>').text(it.id).html()+'">'+ $('<div>').text(it.html_name).html() +'</a></td>' +
|
||||
'<td>' + $('<div>').text(it.central_session).html() + '</td>' +
|
||||
'<td title="' + $('<div>').text(qs).html() + '">' + qs + '</td>' +
|
||||
'<td>' + $('<div>').text(it.html_folder).html() + '</td>' +
|
||||
'<td>' + badgeBool(it.is_filter_bag) + '</td>' +
|
||||
// '<td>' + badgeBool(it.is_run_mta) + '</td>' +
|
||||
'<td>' + $('<div>').text(it.mta_task_status).html() + '</td>' +
|
||||
'<td class="text-truncate-custom" onclick="imageshow('+ it.id +')" > <a href="javascript:void(0)" > '+ $('<div>').text(it.run_mta_result).html() +'</a></td>' +
|
||||
|
||||
// '</td>' +'<td class="text-truncate-custom" title="' + $('<div>').text(it.run_mta_result).html() + '">' + trunc(it.run_mta_result, 48) + '</td>' +
|
||||
'<td class="text-truncate-custom" title="' + $('<div>').text(it.remark).html() + '">' + trunc(it.remark, 48) + '</td>' +
|
||||
'<td>' + fmtDate(it.created_at) + '</td>' +
|
||||
'<td>' + fmtDate(it.updated_at) + '</td>' +
|
||||
'<td> <button class="btn btn-sm btn-outline-warning" onclick="editRow('+ $('<div>').text(it.id).html() + ')">编辑</button> </td>' +
|
||||
'<td> <button class="btn btn-sm btn-outline-warning" onclick="stopTask('+ $('<div>').text(it.id).html() + ')">停止任务</button> </td>' +
|
||||
'<td> <button class="btn btn-sm btn-outline-warning" onclick="reloadTask('+ $('<div>').text(it.id).html() + ')">重跑任务</button> </td>' +
|
||||
'</tr>';
|
||||
}).join(''));
|
||||
}
|
||||
|
||||
// 渲染分页
|
||||
function renderPagination(meta) {
|
||||
var pagination = $('#pagination');
|
||||
var info = $('#info');
|
||||
var { page, per_page, total, total_pages, has_prev, has_next } = meta;
|
||||
|
||||
// 信息栏
|
||||
var start = (page - 1) * per_page + 1;
|
||||
var end = Math.min(page * per_page, total);
|
||||
info.text('共 ' + total + ' 条记录，第 ' + page + '/' + total_pages + ' 页，每页 ' + per_page + ' 条（显示 ' + start + '-' + end + '）');
|
||||
|
||||
// 分页控件
|
||||
var html = '';
|
||||
var q = currentQuery();
|
||||
|
||||
// 首页
|
||||
q.page = 1;
|
||||
html += '<li class="page-item ' + (page === 1 ? 'disabled' : '') + '">' +
|
||||
'<a class="page-link" href="javascript:;" data-page="1">' + (page === 1 ? '' : '首页') + '</a>' +
|
||||
'</li>';
|
||||
|
||||
// 上一页
|
||||
q.page = page - 1;
|
||||
html += '<li class="page-item ' + (page === 1 ? 'disabled' : '') + '">' +
|
||||
'<a class="page-link" href="javascript:;" data-page="' + (page - 1) + '">' + (page === 1 ? '' : '上一页') + '</a>' +
|
||||
'</li>';
|
||||
|
||||
// 页码逻辑（当前页前后各2页，不足则扩展边界）
|
||||
var startPage = Math.max(1, page - 2);
|
||||
var endPage = Math.min(total_pages, page + 2);
|
||||
if (endPage - startPage < 4) {
|
||||
if (startPage === 1) {
|
||||
endPage = Math.min(total_pages, startPage + 4);
|
||||
} else if (endPage === total_pages) {
|
||||
startPage = Math.max(1, endPage - 4);
|
||||
}
|
||||
}
|
||||
if (startPage > 1) {
|
||||
q.page = 1;
|
||||
html += '<li class="page-item"><a class="page-link" href="javascript:;" data-page="1">1</a></li>';
|
||||
if (startPage > 2) html += '<li class="page-item disabled"><span class="page-link">…</span></li>';
|
||||
}
|
||||
for (var i = startPage; i <= endPage; i++) {
|
||||
q.page = i;
|
||||
html += '<li class="page-item ' + (i === page ? 'active' : '') + '">' +
|
||||
'<a class="page-link" href="javascript:;" data-page="' + i + '">' + i + '</a>' +
|
||||
'</li>';
|
||||
}
|
||||
if (endPage < total_pages) {
|
||||
if (endPage < total_pages - 1) html += '<li class="page-item disabled"><span class="page-link">…</span></li>';
|
||||
q.page = total_pages;
|
||||
html += '<li class="page-item"><a class="page-link" href="javascript:;" data-page="' + total_pages + '">' + total_pages + '</a></li>';
|
||||
}
|
||||
|
||||
// 下一页
|
||||
q.page = page + 1;
|
||||
html += '<li class="page-item ' + (page === total_pages ? 'disabled' : '') + '">' +
|
||||
'<a class="page-link" href="javascript:;" data-page="' + (page + 1) + '">' + (page === total_pages ? '' : '下一页') + '</a>' +
|
||||
'</li>';
|
||||
|
||||
// 尾页
|
||||
q.page = total_pages;
|
||||
html += '<li class="page-item ' + (page === total_pages ? 'disabled' : '') + '">' +
|
||||
'<a class="page-link" href="javascript:;" data-page="' + total_pages + '">' + (page === total_pages ? '' : '尾页') + '</a>' +
|
||||
'</li>';
|
||||
|
||||
pagination.html(html);
|
||||
|
||||
// 绑定页码点击（使用 jQuery 事件委托更佳，这里简化为直接绑定）
|
||||
pagination.find('.page-link[data-page]').off('click').on('click', function() {
|
||||
var p = $(this).data('page');
|
||||
$('#page').val(p);
|
||||
loadData();
|
||||
});
|
||||
}
|
||||
|
||||
// 渲染信息与分页
|
||||
function renderInfoAndPagination(meta) {
|
||||
renderInfo(meta);
|
||||
renderPagination(meta);
|
||||
}
|
||||
|
||||
// 渲染信息
|
||||
function renderInfo(meta) {
|
||||
var { page, per_page, total, total_pages } = meta;
|
||||
var start = (page - 1) * per_page + 1;
|
||||
var end = Math.min(page * per_page, total);
|
||||
$('#info').text('共 ' + total + ' 条记录，第 ' + page + '/' + total_pages + ' 页，每页 ' + per_page + ' 条（显示 ' + start + '-' + end + '）');
|
||||
}
|
||||
|
||||
// 加载数据
|
||||
function loadData() {
|
||||
var q = currentQuery();
|
||||
|
||||
var page = parseInt(q.page) || 1;
|
||||
console.log('-----1------'+parseInt(q.page))
|
||||
var per_page = parseInt(q.per_page) || 20;
|
||||
|
||||
$('#tbody').html('<tr><td colspan="12" class="text-center p-4"><div class="spinner-border text-primary" role="status"><span class="visually-hidden">加载中...</span></div></td></tr>');
|
||||
$('#pagination').html('');
|
||||
|
||||
// 构建查询参数
|
||||
var params = {
|
||||
page: page,
|
||||
per_page: per_page,
|
||||
html_name: q.html_name,
|
||||
central_session: q.central_session,
|
||||
is_filter_bag: q.is_filter_bag,
|
||||
is_run_mta: q.is_run_mta,
|
||||
mta_task_status:q.mta_task_status
|
||||
};
|
||||
console.log(params)
|
||||
$.get('/api/items', params, function(data) {
|
||||
renderRows(data.items);
|
||||
renderInfoAndPagination(data);
|
||||
console.log(data.items)
|
||||
}).fail(function(err) {
|
||||
console.error(err);
|
||||
$('#tbody').html('<tr><td colspan="12" class="text-center text-danger">加载失败</td></tr>');
|
||||
});
|
||||
}
|
||||
|
||||
// 编辑数据
|
||||
|
||||
|
||||
|
||||
|
||||
// 切换 MTA 结果展开/收起
|
||||
window.toggleJson = function(btn, jsonText) {
|
||||
var td = $(btn).closest('td');
|
||||
var viewer = td.find('.json-viewer');
|
||||
if (viewer.length === 0) {
|
||||
viewer = $('<div class="json-viewer mt-1"></div>');
|
||||
td.append(viewer);
|
||||
}
|
||||
if (btn.text() === '收起') {
|
||||
viewer.remove();
|
||||
btn.text('展开');
|
||||
} else {
|
||||
viewer.text(jsonText);
|
||||
btn.text('收起');
|
||||
}
|
||||
};
|
||||
|
||||
// 事件绑定（筛选条件变化时重新加载）
|
||||
$('#html_name, #central_session, #is_filter_bag, #is_run_mta, #per_page').on('input change', loadData);
|
||||
|
||||
|
||||
|
||||
// 首次加载
|
||||
$(function() {
|
||||
loadData();
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,445 @@
|
||||
import subprocess
|
||||
import os
|
||||
import json
|
||||
import socket
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
|
||||
class DockerTool:
|
||||
"""Docker容器管理工具类"""
|
||||
|
||||
def __init__(self):
|
||||
self.docker_available = self._check_docker_env()
|
||||
|
||||
def _check_docker_env(self) -> bool:
|
||||
"""检查宿主机Docker环境"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "info"], capture_output=True, text=True, timeout=5
|
||||
)
|
||||
return result.returncode == 0
|
||||
except Exception as e:
|
||||
print(f"Docker环境检查失败: {e}")
|
||||
return False
|
||||
|
||||
def is_docker_available(self) -> bool:
|
||||
"""返回Docker环境是否可用"""
|
||||
return self.docker_available
|
||||
|
||||
def get_container_detailed_info(self, container_id: str) -> Dict:
|
||||
"""
|
||||
获取容器的详细信息（使用docker inspect命令）
|
||||
|
||||
Args:
|
||||
container_id: 容器ID（完整ID或短ID）
|
||||
|
||||
Returns:
|
||||
包含容器详细信息的字典
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", container_id],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
try:
|
||||
inspect_data = json.loads(result.stdout.strip())
|
||||
if inspect_data:
|
||||
return {
|
||||
"success": True,
|
||||
"inspect_data": inspect_data[0],
|
||||
"message": "获取容器详细信息成功",
|
||||
}
|
||||
except json.JSONDecodeError as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"解析容器信息失败: {e}",
|
||||
"message": "解析容器信息失败",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": f"获取容器 '{container_id}' 详细信息失败",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "获取容器详细信息过程中发生异常",
|
||||
}
|
||||
|
||||
def get_container_status_by_name(self, container_name: str) -> Dict:
|
||||
"""
|
||||
通过容器名称查询容器状态
|
||||
|
||||
Args:
|
||||
container_name: 容器名称
|
||||
|
||||
Returns:
|
||||
包含容器状态信息的字典
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"ps",
|
||||
"-a",
|
||||
"--filter",
|
||||
f"name={container_name}",
|
||||
"--format",
|
||||
"{{json .}}",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"未找到名为 '{container_name}' 的容器",
|
||||
"message": "容器不存在",
|
||||
}
|
||||
|
||||
# 处理多行输出（可能有多个匹配的容器）
|
||||
lines = [line for line in output.split("\n") if line.strip()]
|
||||
containers = [json.loads(line) for line in lines]
|
||||
|
||||
# 精确匹配容器名称（排除名称中包含查询字符串的容器）
|
||||
exact_match = None
|
||||
for container in containers:
|
||||
# Docker返回的名称格式可能是 "/容器名" 或 "容器名"
|
||||
names = container.get("Names", "")
|
||||
if container_name in names:
|
||||
# 检查是否为精确匹配
|
||||
name_list = names.split(",")
|
||||
for name in name_list:
|
||||
clean_name = name.strip("/") # 去除开头的斜杠
|
||||
if clean_name == container_name:
|
||||
exact_match = container
|
||||
break
|
||||
|
||||
if exact_match:
|
||||
break
|
||||
|
||||
if not exact_match and containers:
|
||||
# 如果没有精确匹配，但有多条结果，返回第一个
|
||||
exact_match = containers[0]
|
||||
|
||||
if exact_match:
|
||||
return {
|
||||
"success": True,
|
||||
"container": exact_match,
|
||||
"status": exact_match.get("Status", ""),
|
||||
"state": exact_match.get("State", ""),
|
||||
"message": "获取容器状态成功",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"未找到精确匹配的容器 '{container_name}'",
|
||||
"message": "容器不存在",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": f"查询容器 '{container_name}' 状态失败",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "查询容器状态过程中发生异常",
|
||||
}
|
||||
|
||||
def get_container_status_by_id(self, container_id: str) -> Dict:
|
||||
"""
|
||||
通过容器ID查询容器状态
|
||||
|
||||
Args:
|
||||
container_id: 容器ID（完整ID或短ID）
|
||||
|
||||
Returns:
|
||||
包含容器状态信息的字典
|
||||
"""
|
||||
try:
|
||||
# 使用docker ps命令查询特定容器的状态
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"ps",
|
||||
"-a",
|
||||
"--filter",
|
||||
f"id={container_id}",
|
||||
"--format",
|
||||
"{{json .}}",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"未找到ID为 '{container_id}' 的容器",
|
||||
"message": "容器不存在",
|
||||
}
|
||||
|
||||
# 处理多行输出（理论上应该只有一行，因为ID是唯一的）
|
||||
lines = [line for line in output.split("\n") if line.strip()]
|
||||
if not lines:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"未找到ID为 '{container_id}' 的容器",
|
||||
"message": "容器不存在",
|
||||
}
|
||||
|
||||
# 解析容器信息
|
||||
container_info = json.loads(lines[0])
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"container": container_info,
|
||||
"container_id": container_info.get("ID", ""),
|
||||
"container_name": container_info.get("Names", ""),
|
||||
"status": container_info.get("Status", ""),
|
||||
"state": container_info.get("State", ""),
|
||||
"image": container_info.get("Image", ""),
|
||||
"command": container_info.get("Command", ""),
|
||||
"created": container_info.get("CreatedAt", ""),
|
||||
"ports": container_info.get("Ports", ""),
|
||||
"message": "获取容器状态成功",
|
||||
}
|
||||
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": f"查询容器 '{container_id}' 状态失败",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "查询容器状态过程中发生异常",
|
||||
}
|
||||
|
||||
def stop_container_by_name(self, container_name: str) -> Dict:
|
||||
"""
|
||||
通过容器名称停止容器
|
||||
|
||||
Args:
|
||||
container_name: 容器名称
|
||||
|
||||
Returns:
|
||||
包含停止结果的字典
|
||||
"""
|
||||
try:
|
||||
# 首先通过容器名称查询容器状态
|
||||
status_result = self.get_container_status_by_name(container_name)
|
||||
|
||||
if not status_result["success"]:
|
||||
return {
|
||||
"success": False,
|
||||
"error": status_result.get("error", "查询容器状态失败"),
|
||||
"message": f"无法找到容器 '{container_name}'",
|
||||
}
|
||||
|
||||
# 获取容器ID（可能是短ID或完整ID）
|
||||
container_info = status_result["container"]
|
||||
container_id = container_info.get("ID", "") # Docker ps 返回的可能是短ID
|
||||
|
||||
if not container_id:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "无法获取容器ID",
|
||||
"message": f"从容器信息中无法提取容器ID",
|
||||
}
|
||||
|
||||
# 使用现有的stop_container方法停止容器
|
||||
stop_result = self.stop_container(container_id)
|
||||
|
||||
if stop_result["success"]:
|
||||
return {
|
||||
"success": True,
|
||||
"container_id": container_id,
|
||||
"container_name": container_name,
|
||||
"message": f"容器 '{container_name}' 已成功停止",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": stop_result.get("error", "停止操作失败"),
|
||||
"message": f"停止容器 '{container_name}' 失败",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "通过容器名停止容器过程中发生异常",
|
||||
}
|
||||
|
||||
def create_container(
|
||||
self,
|
||||
image_name: str = "ubuntu:latest",
|
||||
container_name: Optional[str] = None,
|
||||
command: Optional[str] = None,
|
||||
volumes: Optional[Dict[str, str]] = None,
|
||||
environment: Optional[Dict[str, str]] = None,
|
||||
interactive: bool = False,
|
||||
) -> Dict:
|
||||
"""
|
||||
创建新容器
|
||||
|
||||
Args:
|
||||
image_name: 镜像名称
|
||||
container_name: 容器名称
|
||||
command: 启动命令
|
||||
volumes: 路径映射字典，格式为 {宿主机路径: 容器内路径}
|
||||
environment: 环境变量字典，格式为 {变量名: 变量值}
|
||||
interactive: 是否以交互模式运行
|
||||
|
||||
Returns:
|
||||
包含创建结果的字典
|
||||
"""
|
||||
try:
|
||||
# 构建Docker命令
|
||||
docker_command = ["docker", "run", "-d"]
|
||||
# 添加交互模式选项
|
||||
# if interactive:
|
||||
# docker_command.extend(["-it"])
|
||||
|
||||
# 添加容器名称
|
||||
if container_name:
|
||||
docker_command.extend(["--name", container_name])
|
||||
|
||||
# 添加路径映射
|
||||
if volumes:
|
||||
for host_path, container_path in volumes.items():
|
||||
docker_command.extend(["-v", f"{host_path}:{container_path}"])
|
||||
|
||||
# 添加环境变量
|
||||
if environment:
|
||||
for key, value in environment.items():
|
||||
docker_command.extend(["-e", f"{key}={value}"])
|
||||
|
||||
# 添加镜像名称和命令
|
||||
docker_command.append(image_name)
|
||||
if command:
|
||||
docker_command.extend(["/bin/bash", "-c", command])
|
||||
|
||||
# 执行Docker命令
|
||||
result = subprocess.run(
|
||||
docker_command, capture_output=True, text=True, timeout=30
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
container_id = result.stdout.strip()
|
||||
return {
|
||||
"success": True,
|
||||
"container_id": container_id,
|
||||
"message": "容器创建成功",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": "容器创建失败",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "容器创建过程中发生异常",
|
||||
}
|
||||
|
||||
def get_containers(self) -> List[Dict]:
|
||||
"""获取所有容器列表"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "ps", "-a", "--format", "{{json .}}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
containers = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if line:
|
||||
containers.append(json.loads(line))
|
||||
return containers
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
print(f"获取容器列表失败: {e}")
|
||||
return []
|
||||
|
||||
def stop_container(self, container_id: str) -> Dict:
|
||||
"""停止指定容器"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "stop", container_id],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
return {"success": True, "message": f"容器 {container_id} 已停止"}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": f"停止容器 {container_id} 失败",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "停止容器过程中发生异常",
|
||||
}
|
||||
|
||||
def remove_container(self, container_id: str) -> Dict:
|
||||
"""删除指定容器"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "rm", container_id],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
return {"success": True, "message": f"容器 {container_id} 已删除"}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": f"删除容器 {container_id} 失败",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "删除容器过程中发生异常",
|
||||
}
|
||||
|
||||
|
||||
# 创建全局工具实例
|
||||
docker_tool = DockerTool()
|
||||
@@ -0,0 +1,30 @@
|
||||
from typing import List, Set, Dict
|
||||
import re
|
||||
import os
|
||||
|
||||
|
||||
def extract_package_names_from_html(html_file: str, logger) -> Set[str]:
|
||||
"""
|
||||
从单个HTML文件中提取所有legendgroup的包名
|
||||
"""
|
||||
try:
|
||||
with open(html_file, "r", encoding="utf-8") as file:
|
||||
content = file.read()
|
||||
except UnicodeDecodeError:
|
||||
try:
|
||||
with open(html_file, "r", encoding="gbk", errors="ignore") as file:
|
||||
content = file.read()
|
||||
except Exception as e:
|
||||
logger.error(f"读取文件 {html_file} 失败: {e}")
|
||||
return set()
|
||||
|
||||
# 使用正则表达式匹配所有legendgroup中的包名
|
||||
pattern = r'"legendgroup":"([^"]+)"'
|
||||
package_names = re.findall(pattern, content)
|
||||
|
||||
unique_packages = set(package_names)
|
||||
logger.info(
|
||||
f"文件 {os.path.basename(html_file)} 中找到 {len(unique_packages)} 个包名"
|
||||
)
|
||||
|
||||
return unique_packages
|
||||
@@ -0,0 +1,45 @@
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def setup_logging(log_file="app.log", level=logging.INFO):
|
||||
"""配置日志系统
|
||||
|
||||
Args:
|
||||
log_file: 日志文件路径
|
||||
level: 日志级别
|
||||
"""
|
||||
# 创建日志目录（如果不存在）
|
||||
log_path = Path(log_file)
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 创建 logger
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(level)
|
||||
|
||||
# 避免重复配置
|
||||
if logger.handlers:
|
||||
return logger
|
||||
|
||||
# 创建 formatter
|
||||
formatter = logging.Formatter("[%(levelname)s] %(message)s")
|
||||
|
||||
# 文件处理器（追加模式）
|
||||
file_handler = logging.FileHandler(filename=log_file, mode="a", encoding="utf-8")
|
||||
file_handler.setLevel(level)
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
# 控制台处理器
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
console_handler.setLevel(level)
|
||||
console_handler.setFormatter(formatter)
|
||||
|
||||
# 添加处理器
|
||||
logger.addHandler(file_handler)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# 防止重复记录
|
||||
logger.propagate = False
|
||||
|
||||
return logger
|
||||
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
路径拼接工具类
|
||||
提供简单易用的路径操作功能
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Union, List, Optional
|
||||
|
||||
|
||||
class PathTool:
|
||||
"""简单的路径拼接工具类"""
|
||||
|
||||
@staticmethod
|
||||
def join(*paths: Union[str, Path]) -> str:
|
||||
"""
|
||||
拼接多个路径
|
||||
|
||||
Args:
|
||||
*paths: 路径组件，可以是字符串或Path对象
|
||||
|
||||
Returns:
|
||||
str: 拼接后的路径字符串
|
||||
|
||||
Example:
|
||||
>>> PathTool.join("/home", "user", "documents")
|
||||
'/home/user/documents'
|
||||
"""
|
||||
return os.path.join(*paths)
|
||||
|
||||
@staticmethod
|
||||
def join_with_base(base_path: str, *sub_paths: str) -> str:
|
||||
"""
|
||||
基于基础路径拼接子路径
|
||||
|
||||
Args:
|
||||
base_path: 基础路径
|
||||
*sub_paths: 子路径列表
|
||||
|
||||
Returns:
|
||||
str: 完整的路径字符串
|
||||
|
||||
Example:
|
||||
>>> PathTool.join_with_base("/base", "dir1", "dir2", "file.txt")
|
||||
'/base/dir1/dir2/file.txt'
|
||||
"""
|
||||
return PathTool.join(base_path, *sub_paths)
|
||||
|
||||
@staticmethod
|
||||
def normalize(path: str) -> str:
|
||||
"""
|
||||
规范化路径（处理 ../, ./ 等相对路径）
|
||||
|
||||
Args:
|
||||
path: 原始路径
|
||||
|
||||
Returns:
|
||||
str: 规范化后的路径
|
||||
|
||||
Example:
|
||||
>>> PathTool.normalize("/home/user/../user/./documents")
|
||||
'/home/user/documents'
|
||||
"""
|
||||
return os.path.normpath(path)
|
||||
|
||||
@staticmethod
|
||||
def absolute(path: str) -> str:
|
||||
"""
|
||||
获取绝对路径
|
||||
|
||||
Args:
|
||||
path: 相对路径或绝对路径
|
||||
|
||||
Returns:
|
||||
str: 绝对路径
|
||||
|
||||
Example:
|
||||
>>> PathTool.absolute("documents/file.txt")
|
||||
'/current/working/directory/documents/file.txt'
|
||||
"""
|
||||
return os.path.abspath(path)
|
||||
|
||||
@staticmethod
|
||||
def exists(path: str) -> bool:
|
||||
"""
|
||||
检查路径是否存在
|
||||
|
||||
Args:
|
||||
path: 要检查的路径
|
||||
|
||||
Returns:
|
||||
bool: 路径是否存在
|
||||
"""
|
||||
return os.path.exists(path)
|
||||
|
||||
@staticmethod
|
||||
def is_file(path: str) -> bool:
|
||||
"""
|
||||
检查是否为文件
|
||||
|
||||
Args:
|
||||
path: 要检查的路径
|
||||
|
||||
Returns:
|
||||
bool: 是否为文件
|
||||
"""
|
||||
return os.path.isfile(path)
|
||||
|
||||
@staticmethod
|
||||
def is_dir(path: str) -> bool:
|
||||
"""
|
||||
检查是否为目录
|
||||
|
||||
Args:
|
||||
path: 要检查的路径
|
||||
|
||||
Returns:
|
||||
bool: 是否为目录
|
||||
"""
|
||||
return os.path.isdir(path)
|
||||
|
||||
@staticmethod
|
||||
def get_filename(path: str) -> str:
|
||||
"""
|
||||
获取文件名（包含扩展名）
|
||||
|
||||
Args:
|
||||
path: 文件路径
|
||||
|
||||
Returns:
|
||||
str: 文件名
|
||||
|
||||
Example:
|
||||
>>> PathTool.get_filename("/home/user/file.txt")
|
||||
'file.txt'
|
||||
"""
|
||||
return os.path.basename(path)
|
||||
|
||||
@staticmethod
|
||||
def get_parent(path: str) -> str:
|
||||
"""
|
||||
获取父目录路径
|
||||
|
||||
Args:
|
||||
path: 文件或目录路径
|
||||
|
||||
Returns:
|
||||
str: 父目录路径
|
||||
|
||||
Example:
|
||||
>>> PathTool.get_parent("/home/user/file.txt")
|
||||
'/home/user'
|
||||
"""
|
||||
return os.path.dirname(path)
|
||||
|
||||
@staticmethod
|
||||
def get_extension(path: str) -> str:
|
||||
"""
|
||||
获取文件扩展名
|
||||
|
||||
Args:
|
||||
path: 文件路径
|
||||
|
||||
Returns:
|
||||
str: 文件扩展名（包含点号）
|
||||
|
||||
Example:
|
||||
>>> PathTool.get_extension("/home/user/file.txt")
|
||||
'.txt'
|
||||
"""
|
||||
return os.path.splitext(path)[1]
|
||||
|
||||
@staticmethod
|
||||
def change_extension(path: str, new_extension: str) -> str:
|
||||
"""
|
||||
更改文件扩展名
|
||||
|
||||
Args:
|
||||
path: 原文件路径
|
||||
new_extension: 新扩展名（包含点号）
|
||||
|
||||
Returns:
|
||||
str: 更改扩展名后的路径
|
||||
|
||||
Example:
|
||||
>>> PathTool.change_extension("/home/user/file.txt", ".log")
|
||||
'/home/user/file.log'
|
||||
"""
|
||||
name_without_ext = os.path.splitext(path)[0]
|
||||
return name_without_ext + new_extension
|
||||
|
||||
@staticmethod
|
||||
def create_dirs(path: str) -> bool:
|
||||
"""
|
||||
创建目录（包括父目录）
|
||||
|
||||
Args:
|
||||
path: 要创建的目录路径
|
||||
|
||||
Returns:
|
||||
bool: 是否创建成功
|
||||
"""
|
||||
try:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# 创建全局工具实例
|
||||
path_tool = PathTool()
|
||||
|
||||
# 使用示例和测试
|
||||
if __name__ == "__main__":
|
||||
# 测试路径拼接
|
||||
print("=== 路径拼接测试 ===")
|
||||
result1 = PathTool.join("/home", "user/", "documents", "file.txt")
|
||||
print(f"拼接路径: {result1}")
|
||||
|
||||
result2 = PathTool.join_with_base("/base", "dir1", "dir2", "file.txt")
|
||||
print(f"基础路径拼接: {result2}")
|
||||
|
||||
# 测试路径规范化
|
||||
print("\n=== 路径规范化测试 ===")
|
||||
normalized = PathTool.normalize("/home/user/../user/./documents")
|
||||
print(f"规范化路径: {normalized}")
|
||||
|
||||
# 测试文件信息获取
|
||||
print("\n=== 文件信息测试 ===")
|
||||
test_path = "/home/user/document.txt"
|
||||
print(f"文件名: {PathTool.get_filename(test_path)}")
|
||||
print(f"父目录: {PathTool.get_parent(test_path)}")
|
||||
print(f"扩展名: {PathTool.get_extension(test_path)}")
|
||||
|
||||
# 测试扩展名更改
|
||||
new_path = PathTool.change_extension(test_path, ".log")
|
||||
print(f"更改扩展名: {new_path}")
|
||||
|
||||
# 测试路径检查
|
||||
print("\n=== 路径检查测试 ===")
|
||||
print(f"路径是否存在: {PathTool.exists('/tmp')}")
|
||||
print(f"是否为目录: {PathTool.is_dir('/tmp')}")
|
||||
print(f"是否为文件: {PathTool.is_file('/tmp')}")
|
||||
|
||||
# 使用全局实例
|
||||
print("\n=== 全局实例测试 ===")
|
||||
result = path_tool.join("dir1", "dir2", "file.txt")
|
||||
print(f"使用全局实例: {result}")
|
||||
@@ -0,0 +1,398 @@
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import queue
|
||||
from datetime import datetime
|
||||
from threading import Thread
|
||||
from typing import Dict, List, Optional, Callable, Any
|
||||
from enum import Enum
|
||||
import logging
|
||||
|
||||
|
||||
class TaskStatus(Enum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
class SimpleTask:
|
||||
"""简单的任务类"""
|
||||
|
||||
def __init__(self, BATH_ROOT: str, task_func: Callable, *args, **kwargs):
|
||||
self.BATH_ROOT = BATH_ROOT
|
||||
self.task_func = task_func
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
self.status = TaskStatus.PENDING
|
||||
self.progress = 0
|
||||
self.message = ""
|
||||
self.result = None
|
||||
self.error = None
|
||||
self.created_at = datetime.now()
|
||||
self.started_at = None
|
||||
self.completed_at = None
|
||||
self._cancelled = False
|
||||
self._thread = None
|
||||
self.completion_callback = kwargs.pop("completion_callback", None) # 新增回调
|
||||
|
||||
def execute(self):
|
||||
"""执行任务"""
|
||||
try:
|
||||
self.status = TaskStatus.RUNNING
|
||||
self.started_at = datetime.now()
|
||||
|
||||
# 在新线程中执行
|
||||
self._thread = threading.Thread(target=self._run_wrapper, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
except Exception as e:
|
||||
self.status = TaskStatus.FAILED
|
||||
self.error = str(e)
|
||||
self.completed_at = datetime.now()
|
||||
|
||||
def _run_wrapper(self):
|
||||
"""任务执行包装器"""
|
||||
try:
|
||||
# 执行实际的任务函数
|
||||
self.result = self.task_func(self, *self.args, **self.kwargs)
|
||||
|
||||
if self._cancelled:
|
||||
self.status = TaskStatus.CANCELLED
|
||||
else:
|
||||
self.status = TaskStatus.COMPLETED
|
||||
self.progress = 100
|
||||
|
||||
except Exception as e:
|
||||
self.status = TaskStatus.FAILED
|
||||
self.error = str(e)
|
||||
# 确保result包含错误信息，这样回调函数能获取到错误
|
||||
self.result = {"status": "failed", "error": str(e)}
|
||||
finally:
|
||||
self.completed_at = datetime.now()
|
||||
|
||||
# 无论成功还是失败都执行回调
|
||||
if self.completion_callback:
|
||||
try:
|
||||
# 传递任务结果给回调函数
|
||||
callback_result = (
|
||||
self.result if self.result else {"status": self.status.value}
|
||||
)
|
||||
self.completion_callback(callback_result)
|
||||
except Exception as callback_e:
|
||||
print(f"回调函数执行失败: {callback_e}")
|
||||
# 确保logger存在
|
||||
if hasattr(self, "logger"):
|
||||
self.logger.error(f"回调函数执行失败: {callback_e}")
|
||||
else:
|
||||
print(f"回调函数执行失败: {callback_e}")
|
||||
|
||||
def update_progress(self, progress: int, message: str = ""):
|
||||
"""更新任务进度"""
|
||||
if not self._cancelled:
|
||||
self.progress = max(0, min(100, progress))
|
||||
self.message = message
|
||||
|
||||
def cancel(self):
|
||||
"""取消任务"""
|
||||
self._cancelled = True
|
||||
self.status = TaskStatus.CANCELLED
|
||||
self.message = "任务已被取消"
|
||||
|
||||
def is_running(self) -> bool:
|
||||
"""检查任务是否正在运行"""
|
||||
return (
|
||||
self.status == TaskStatus.RUNNING
|
||||
and self._thread
|
||||
and self._thread.is_alive()
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""转换为字典格式"""
|
||||
return {
|
||||
"BATH_ROOT": self.BATH_ROOT,
|
||||
"status": self.status.value,
|
||||
"progress": self.progress,
|
||||
"message": self.message,
|
||||
"result": self.result,
|
||||
"error": self.error,
|
||||
"created_at": self.created_at.isoformat(),
|
||||
"started_at": self.started_at.isoformat() if self.started_at else None,
|
||||
"completed_at": self.completed_at.isoformat()
|
||||
if self.completed_at
|
||||
else None,
|
||||
"is_running": self.is_running(),
|
||||
}
|
||||
|
||||
|
||||
class SimpleTaskQueue:
|
||||
"""简单的任务队列管理器"""
|
||||
|
||||
def __init__(self, max_concurrent_tasks: int = 3):
|
||||
"""
|
||||
初始化任务队列
|
||||
|
||||
Args:
|
||||
max_concurrent_tasks: 最大并发任务数
|
||||
"""
|
||||
self.max_concurrent_tasks = max_concurrent_tasks
|
||||
self.pending_queue = queue.Queue() # 等待队列
|
||||
self.running_tasks: Dict[str, SimpleTask] = {} # 运行中的任务
|
||||
self.completed_tasks: Dict[str, SimpleTask] = {} # 已完成的任务
|
||||
self.cancelled_tasks: Dict[str, SimpleTask] = {} # 已取消的任务
|
||||
self.lock = threading.RLock()
|
||||
self.is_running = True
|
||||
|
||||
# 启动调度器线程
|
||||
self.scheduler_thread = threading.Thread(
|
||||
target=self._scheduler_loop, daemon=True
|
||||
)
|
||||
self.scheduler_thread.start()
|
||||
|
||||
# 配置日志
|
||||
self.logger = self._setup_logger()
|
||||
self.logger.info(f"简单任务队列已启动，最大并发数: {max_concurrent_tasks}")
|
||||
|
||||
def _setup_logger(self) -> logging.Logger:
|
||||
"""设置日志"""
|
||||
logger = logging.getLogger("SimpleTaskQueue")
|
||||
logger.setLevel(logging.INFO)
|
||||
if not logger.handlers:
|
||||
handler = logging.StreamHandler()
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
return logger
|
||||
|
||||
def _scheduler_loop(self):
|
||||
"""调度器循环 - 自动管理任务执行"""
|
||||
while self.is_running:
|
||||
try:
|
||||
with self.lock:
|
||||
current_running = len(self.running_tasks)
|
||||
|
||||
# 检查是否可以启动新任务
|
||||
if (
|
||||
current_running < self.max_concurrent_tasks
|
||||
and not self.pending_queue.empty()
|
||||
):
|
||||
available_slots = self.max_concurrent_tasks - current_running
|
||||
|
||||
for _ in range(
|
||||
min(available_slots, self.pending_queue.qsize())
|
||||
):
|
||||
if not self.pending_queue.empty():
|
||||
task = self.pending_queue.get_nowait()
|
||||
self._start_task(task)
|
||||
self.logger.info(
|
||||
f"从等待队列启动任务: {task.BATH_ROOT}"
|
||||
)
|
||||
|
||||
# 清理已完成的任务
|
||||
self._cleanup_completed_tasks()
|
||||
|
||||
# 短暂休眠
|
||||
time.sleep(1)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"调度器错误: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
def _cleanup_completed_tasks(self):
|
||||
"""清理已完成的任务"""
|
||||
completed_ids = []
|
||||
with self.lock:
|
||||
for BATH_ROOT, task in list(self.running_tasks.items()):
|
||||
if not task.is_running() and task.status in [
|
||||
TaskStatus.COMPLETED,
|
||||
TaskStatus.FAILED,
|
||||
TaskStatus.CANCELLED,
|
||||
]:
|
||||
completed_ids.append(BATH_ROOT)
|
||||
if (
|
||||
task.status == TaskStatus.COMPLETED
|
||||
or task.status == TaskStatus.FAILED
|
||||
):
|
||||
self.completed_tasks[BATH_ROOT] = task
|
||||
elif task.status == TaskStatus.CANCELLED:
|
||||
self.cancelled_tasks[BATH_ROOT] = task
|
||||
self.logger.info(
|
||||
f"任务完成: {BATH_ROOT}, 状态: {task.status.value}"
|
||||
)
|
||||
|
||||
# 从运行列表中移除已完成的任务
|
||||
for BATH_ROOT in completed_ids:
|
||||
if BATH_ROOT in self.running_tasks:
|
||||
self.running_tasks.pop(BATH_ROOT)
|
||||
|
||||
def _start_task(self, task: SimpleTask):
|
||||
"""启动任务"""
|
||||
with self.lock:
|
||||
self.running_tasks[task.BATH_ROOT] = task
|
||||
task.execute()
|
||||
self.logger.info(f"任务开始执行: {task.BATH_ROOT}")
|
||||
|
||||
def submit_task(self, task_func: Callable, *args, **kwargs) -> Dict[str, Any]:
|
||||
"""
|
||||
提交新任务到队列
|
||||
|
||||
Args:
|
||||
task_func: 要执行的任务函数
|
||||
*args: 任务函数参数
|
||||
**kwargs: 任务函数关键字参数
|
||||
|
||||
Returns:
|
||||
任务提交结果
|
||||
"""
|
||||
BATH_ROOT = str(uuid.uuid4())[:8]
|
||||
|
||||
# 创建任务对象
|
||||
task = SimpleTask(BATH_ROOT, task_func, *args, **kwargs)
|
||||
|
||||
with self.lock:
|
||||
current_running = len(self.running_tasks)
|
||||
|
||||
if current_running < self.max_concurrent_tasks:
|
||||
# 直接启动任务
|
||||
self._start_task(task)
|
||||
status = "RUNNING"
|
||||
message = "任务已立即启动"
|
||||
immediate_start = True
|
||||
else:
|
||||
# 添加到等待队列
|
||||
self.pending_queue.put(task)
|
||||
status = "PENDING"
|
||||
message = f"任务已加入等待队列，当前位置: {self.pending_queue.qsize()}"
|
||||
immediate_start = False
|
||||
|
||||
self.logger.info(f"提交任务: {BATH_ROOT}, 立即执行: {immediate_start}")
|
||||
|
||||
return {
|
||||
"BATH_ROOT": BATH_ROOT,
|
||||
"status": status,
|
||||
"immediate_start": immediate_start,
|
||||
"message": message,
|
||||
"running_count": current_running,
|
||||
"pending_count": self.pending_queue.qsize(),
|
||||
"max_concurrent": self.max_concurrent_tasks,
|
||||
"submitted_at": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
def get_task_status(self, BATH_ROOT: str) -> Optional[Dict[str, Any]]:
|
||||
"""获取任务状态"""
|
||||
with self.lock:
|
||||
# 检查运行中的任务
|
||||
if BATH_ROOT in self.running_tasks:
|
||||
return self.running_tasks[BATH_ROOT].to_dict()
|
||||
|
||||
# 检查已完成的任务
|
||||
if BATH_ROOT in self.completed_tasks:
|
||||
return self.completed_tasks[BATH_ROOT].to_dict()
|
||||
|
||||
# 检查等待队列中的任务
|
||||
for task in list(self.pending_queue.queue):
|
||||
if task.BATH_ROOT == BATH_ROOT:
|
||||
return task.to_dict()
|
||||
|
||||
return None
|
||||
|
||||
def cancel_task(self, BATH_ROOT: str) -> Dict[str, Any]:
|
||||
"""取消任务"""
|
||||
with self.lock:
|
||||
# 检查运行中的任务
|
||||
if BATH_ROOT in self.running_tasks:
|
||||
task = self.running_tasks[BATH_ROOT]
|
||||
task.cancel()
|
||||
self.cancelled_tasks[BATH_ROOT] = task
|
||||
if BATH_ROOT in self.running_tasks:
|
||||
self.running_tasks.pop(BATH_ROOT)
|
||||
self.logger.info(f"取消运行中的任务: {BATH_ROOT}")
|
||||
return {"success": True, "message": "任务取消请求已发送"}
|
||||
|
||||
# 检查等待队列中的任务
|
||||
temp_queue = queue.Queue()
|
||||
found = False
|
||||
while not self.pending_queue.empty():
|
||||
task = self.pending_queue.get()
|
||||
if task.task_id == BATH_ROOT:
|
||||
found = True
|
||||
task.cancel()
|
||||
self.cancelled_tasks[BATH_ROOT] = task
|
||||
self.logger.info(f"取消等待队列中的任务: {BATH_ROOT}")
|
||||
else:
|
||||
temp_queue.put(task)
|
||||
|
||||
# 将剩余任务放回原队列
|
||||
while not temp_queue.empty():
|
||||
self.pending_queue.put(temp_queue.get())
|
||||
|
||||
if found:
|
||||
return {"success": True, "message": "任务已从等待队列中取消"}
|
||||
if BATH_ROOT in self.cancelled_tasks:
|
||||
return {"success": True, "message": "任务已被取消"}
|
||||
return {"success": False, "error": "任务不存在"}
|
||||
|
||||
def get_queue_status(self) -> Dict[str, Any]:
|
||||
"""获取队列状态"""
|
||||
with self.lock:
|
||||
# 获取运行中任务信息
|
||||
running_tasks = []
|
||||
for task in self.running_tasks.values():
|
||||
task_info = task.to_dict()
|
||||
running_tasks.append(task_info)
|
||||
|
||||
# 获取等待任务信息
|
||||
pending_tasks = []
|
||||
temp_queue = queue.Queue()
|
||||
while not self.pending_queue.empty():
|
||||
task = self.pending_queue.get()
|
||||
pending_tasks.append(task.to_dict())
|
||||
temp_queue.put(task)
|
||||
|
||||
# 恢复队列
|
||||
while not temp_queue.empty():
|
||||
self.pending_queue.put(temp_queue.get())
|
||||
|
||||
# 分别统计不同状态的任务数量
|
||||
successful_count = len([
|
||||
t
|
||||
for t in self.completed_tasks.values()
|
||||
if t.status == TaskStatus.COMPLETED
|
||||
])
|
||||
failed_count = len([
|
||||
t
|
||||
for t in self.completed_tasks.values()
|
||||
if t.status == TaskStatus.FAILED
|
||||
])
|
||||
cancelled_count = len(self.cancelled_tasks)
|
||||
|
||||
# 获取已取消任务信息
|
||||
cancelled_tasks = []
|
||||
for task in self.cancelled_tasks.values():
|
||||
task_info = task.to_dict()
|
||||
cancelled_tasks.append(task_info)
|
||||
|
||||
return {
|
||||
"max_concurrent_tasks": self.max_concurrent_tasks,
|
||||
"pending_count": self.pending_queue.qsize(),
|
||||
"running_count": len(self.running_tasks),
|
||||
"successful_count": successful_count,
|
||||
"failed_count": failed_count,
|
||||
"cancelled_count": cancelled_count, # 更新统计
|
||||
"cancelled_tasks": cancelled_tasks, # 添加已取消任务列表
|
||||
"running_tasks": running_tasks,
|
||||
"pending_tasks": pending_tasks,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
def stop(self):
|
||||
"""停止任务队列"""
|
||||
self.is_running = False
|
||||
self.logger.info("任务队列已停止")
|
||||
|
||||
|
||||
# 全局任务队列实例
|
||||
task_queue = SimpleTaskQueue(max_concurrent_tasks=3)
|
||||
@@ -0,0 +1 @@
|
||||
/.idea/
|
||||
@@ -0,0 +1,446 @@
|
||||
# ROOT DB API - 数据库字段文档
|
||||
|
||||
本文档详细说明了 ROOT DB API 数据库中所有可用的数据字段，按功能模块分类。
|
||||
|
||||
## 🗂️ 数据库架构概览
|
||||
|
||||
数据库包含 **18 个核心业务表** + **1 个审计日志表**，支持：
|
||||
- 🚗 **自动驾驶数据管理** (Pangu/Minerva)
|
||||
- 📦 **ROS Bag 文件管理**
|
||||
- 🌳 **FST (File System Tree) 分类**
|
||||
- 🏷️ **标签和主题管理**
|
||||
- 📍 **地理空间数据支持**
|
||||
- 🎯 **真值数据管理**
|
||||
|
||||
---
|
||||
|
||||
## 📊 1. 基础配置模块
|
||||
|
||||
### 1.1 项目表 (`project`)
|
||||
管理数据来源项目信息
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 项目自增主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL UNIQUE | 项目名称，全局唯一 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 最近一次更新时间 |
|
||||
|
||||
**说明**: 数据来源标识，1：DFDI，2：CFDI
|
||||
|
||||
---
|
||||
|
||||
## 📚 2. 字典数据模块
|
||||
|
||||
### 2.1 驾驶模式字典 (`drive_mode`)
|
||||
自动驾驶模式分类字典
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `type` | VARCHAR(100) | NOT NULL | 驾驶模式大类 |
|
||||
| `sub_type` | VARCHAR(100) | | 驾驶模式子类 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
| `comment` | TEXT | | 备注 |
|
||||
|
||||
**驾驶模式类型**:
|
||||
- **AD_DRIVE_OFF**: OFF, PASSIVE, STANDBY
|
||||
- **AD_DRIVE_ACC**: ACC
|
||||
- **AD_DRIVE_CP**: CP
|
||||
- **AD_DRIVE_NP**: HDMAP_HNP, DDLD_HNP, HDMAP_HNP_PLUS, DDLD_HNP_PLUS, HDMAP_UNP, DDLD_UNP
|
||||
- **AD_PARK_APA**: 各种 APA 停车模式 (20+ 种)
|
||||
- **AD_PARK_MPA**: 停车场地图相关模式
|
||||
|
||||
### 2.2 Topic 字典 (`topic_list`)
|
||||
ROS Topic 信息管理
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL UNIQUE | Topic 名称 (如 /lidar/points) |
|
||||
| `type` | VARCHAR(255) | | Topic 类型 (如 sensor_msgs/PointCloud2) |
|
||||
| `key_data` | TEXT | | 关键数据摘要 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 2.3 标签字典 (`reserved_tag_list`)
|
||||
数据标签管理
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL | 标签名称 |
|
||||
| `type` | VARCHAR(50) | | 标签分类 |
|
||||
| `creator` | VARCHAR(255) | | 创建人 |
|
||||
| `comments` | TEXT | | 备注 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `is_deleted` | BOOLEAN | DEFAULT FALSE | 逻辑删除标记 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 2.4 真值元数据 (`gt_meta`)
|
||||
Ground Truth 数据元信息
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL | 真值名称 |
|
||||
| `type` | VARCHAR(100) | NOT NULL | 真值类型 (如 3DBox/Lane) |
|
||||
| `path` | TEXT | NOT NULL | 真值文件存储路径 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
| `comment` | TEXT | | 备注 |
|
||||
|
||||
---
|
||||
|
||||
## 📦 3. Bag 文件管理模块
|
||||
|
||||
### 3.1 Bag 主表 (`bag_list`)
|
||||
ROS Bag 文件核心信息
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL UNIQUE | Bag 文件名称，全局唯一 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `project_id` | INT | FK → project(id) | 数据来源项目 |
|
||||
| `tile_id` | VARCHAR(255) | | 瓦片 ID |
|
||||
| `is_decoded` | BOOLEAN | DEFAULT FALSE | 是否已解码 |
|
||||
| `is_deleted` | BOOLEAN | DEFAULT FALSE | 逻辑删除 |
|
||||
| `od_annotated` | INT | | 目标检测标注状态 |
|
||||
| `ld_annotated` | INT | | 车道线标注状态 |
|
||||
| `fst_indexed` | BOOLEAN | DEFAULT FALSE | 是否已建立 FST 索引 |
|
||||
| `is_active_data` | BOOLEAN | DEFAULT TRUE | 是否为活跃数据 |
|
||||
| `reserved_str` | VARCHAR(255) | | 预留字符串 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
| `drive_mode` | SMALLINT | FK → drive_mode(id) | 驾驶模式字典 ID |
|
||||
| `sw_version` | VARCHAR(100) | | 软件版本号 |
|
||||
| `hw_version` | VARCHAR(100) | | 硬件版本号 |
|
||||
|
||||
### 3.2 Bag 生命周期 (`bag_lifecycle`)
|
||||
Bag 文件处理流程时间戳
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_name` | VARCHAR(255) | NOT NULL UNIQUE, FK → bag_list(name) | Bag 名称 |
|
||||
| `collect_time` | TIMESTAMPTZ | | 采集完成时间 |
|
||||
| `clone2dev_time` | TIMESTAMPTZ | | 拷贝到bucket时间 |
|
||||
| `decode_time` | TIMESTAMPTZ | | 解码完成时间 |
|
||||
| `mining_time` | TIMESTAMPTZ | | 数据挖掘完成时间 |
|
||||
| `auto_annotate_time` | TIMESTAMPTZ | | 自动标注完成时间 |
|
||||
| `manual_annotate_time` | TIMESTAMPTZ | | 人工标注完成时间 |
|
||||
| `fst_index_time` | TIMESTAMPTZ | | FST 索引完成时间 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 记录更新时间 |
|
||||
|
||||
---
|
||||
|
||||
## 🚗 4. Pangu 数据管理模块
|
||||
|
||||
### 4.1 Pangu 主表 (`main_pangu`)
|
||||
Pangu 原始数据信息
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL UNIQUE | 名称，全局唯一 |
|
||||
| `vehicle` | VARCHAR(255) | | 车辆编号 |
|
||||
| `datetime` | TIMESTAMPTZ | | rosbag的生成时间 |
|
||||
| `bag_path` | TEXT | | 原始 Bag 路径 |
|
||||
| `data_path` | TEXT | | 解析后数据路径 |
|
||||
| `reserved_str` | VARCHAR(255) | | 预留字符串 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 4.2 Pangu 融合表 (`joined_pangu`)
|
||||
融合后的 Pangu 数据
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL UNIQUE | 融合后名称 |
|
||||
| `data_path` | TEXT | | 融合后数据路径 |
|
||||
| `reserved_str` | VARCHAR(255) | | 预留字符串 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 4.3 Bag 融合关系 (`joined_bags`)
|
||||
记录 Bag 文件融合关系
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `parent_id` | INT | NOT NULL | 父 Bag ID |
|
||||
| `child_id` | INT | NOT NULL UNIQUE | 子 Bag ID，唯一 |
|
||||
| `reserved_str` | VARCHAR(255) | | 预留字符串 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 4.4 Pangu 详细路径 (`secondary_pangu`)
|
||||
Pangu 数据文件路径详情
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_id` | INT | NOT NULL UNIQUE, FK → bag_list(id) | 关联 bag_list.id |
|
||||
| `lidar_gt_pandar128` | TEXT | | 激光真值路径 |
|
||||
| `object_lidar_gt_pandar128_manual` | TEXT | | 人工真值路径 |
|
||||
| `lidar_fd_multi_scan_raw` | TEXT | | 原始雷达路径 |
|
||||
| `camera_fisheye_left` | TEXT | | 左鱼眼相机路径 |
|
||||
| `camera_fisheye_right` | TEXT | | 右鱼眼相机路径 |
|
||||
| `camera_front_wide` | TEXT | | 前广角相机路径 |
|
||||
| `raw_gps` | TEXT | | 原始 GPS 路径 |
|
||||
| `raw_imu` | TEXT | | 原始 IMU 路径 |
|
||||
| `ego_motion` | TEXT | | 自车运动路径 |
|
||||
| `vehicle_wheel` | TEXT | | 轮速路径 |
|
||||
| `calibration` | TEXT | | 标定文件路径 |
|
||||
| `sdmap` | TEXT | | SD 地图路径或地图版本标识 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
---
|
||||
|
||||
## 🅰️ 5. Minerva 数据管理模块
|
||||
|
||||
### 5.1 Minerva 主表 (`main_minerva`)
|
||||
Minerva 数据会话信息
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `session_id` | VARCHAR(255) | UNIQUE | 会话 ID，唯一 |
|
||||
| `start_ts` | TIMESTAMPTZ | | 开始时间戳 |
|
||||
| `end_ts` | TIMESTAMPTZ | | 结束时间戳 |
|
||||
| `length` | INT | | 时长（秒） |
|
||||
| `datetime` | TIMESTAMPTZ | | 数据日期 |
|
||||
| `vin` | VARCHAR(255) | | 车辆识别码 |
|
||||
| `platform` | VARCHAR(255) | | 平台名称 |
|
||||
| `mapped` | BOOLEAN | DEFAULT FALSE | 是否已映射 |
|
||||
| `path` | TEXT | | 原始数据路径 |
|
||||
| `converted_path` | TEXT | | 转换后路径 |
|
||||
| `gt_path` | TEXT | | 真值路径 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 5.2 Minerva 详细路径 (`secondary_minerva`)
|
||||
Minerva 数据文件路径详情
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_id` | INT | NOT NULL UNIQUE, FK → bag_list(id) | 关联 bag_list.id |
|
||||
| `lidar_gt_top_p128` | TEXT | | 顶部Lidar真值路径 |
|
||||
| `lidar_parking_gt_front_p128` | TEXT | | 前停车Lidar真值路径 |
|
||||
| `lidar_parking_gt_left_p128` | TEXT | | 左停车Lidar真值路径 |
|
||||
| `lidar_parking_gt_right_p128` | TEXT | | 右停车Lidar真值路径 |
|
||||
| `lidar_parking_gt_rear_p128` | TEXT | | 后停车Lidar真值路径 |
|
||||
| `camera_fisheye_left_200fov` | TEXT | | 左 200° 鱼眼路径 |
|
||||
| `camera_fisheye_right_200fov` | TEXT | | 右 200° 鱼眼路径 |
|
||||
| `camera_fisheye_rear_200fov` | TEXT | | 后 200° 鱼眼路径 |
|
||||
| `camera_fisheye_front_200fov` | TEXT | | 前 200° 鱼眼路径 |
|
||||
| `camera_front_wide_120fov` | TEXT | | 前 120° 广角路径 |
|
||||
| `camera_front_tele_30fov` | TEXT | | 前 30° 长焦路径 |
|
||||
| `camera_rear_right_70fov` | TEXT | | 右后 70° 相机路径 |
|
||||
| `camera_rear_left_70fov` | TEXT | | 左后 70° 相机路径 |
|
||||
| `raw_gps` | TEXT | | 原始 GPS 路径 |
|
||||
| `raw_imu` | TEXT | | 原始 IMU 路径 |
|
||||
| `ego_motion` | TEXT | | 自车运动路径 |
|
||||
| `calibration` | TEXT | | 标定文件路径 |
|
||||
| `rig` | TEXT | | rig 文件路径 |
|
||||
| `fst_new` | TEXT | | 新 FST 路径 |
|
||||
| `fst_old` | TEXT | | 旧 FST 路径 |
|
||||
| `comments` | TEXT | | 备注 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
---
|
||||
|
||||
## 🌳 6. FST (File System Tree) 模块
|
||||
|
||||
### 6.1 FST 节点表 (`fst`)
|
||||
文件系统树分类结构
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL | 节点名称 |
|
||||
| `parent_id` | INT | FK → fst(id) | 父节点 ID |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
| `bag_sum` | INT | DEFAULT 0 | 子树 Bag 数量 |
|
||||
|
||||
### 6.2 FST-Bag 关联 (`fst_bag`)
|
||||
FST 节点与 Bag 文件的关联关系
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_id` | INT | NOT NULL, FK → bag_list(id) | Bag ID |
|
||||
| `fst_node_id` | INT | NOT NULL, FK → fst(id) | FST 节点 ID |
|
||||
| `fst_node_level` | INT | NOT NULL | 节点层级 |
|
||||
| `event_start_time` | INT | | 事件开始时间（ms） |
|
||||
| `event_end_time` | INT | | 事件结束时间（ms） |
|
||||
| `img_url` | VARCHAR(255) | | 事件截图 URL |
|
||||
| `video_url` | VARCHAR(255) | | 事件视频 URL |
|
||||
| `comments` | VARCHAR(255) | | 备注 |
|
||||
|
||||
---
|
||||
|
||||
## 🗺️ 7. 地理空间数据模块
|
||||
|
||||
### 7.1 几何信息表 (`geometry_info`)
|
||||
Bag 文件的地理空间轨迹数据
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `rosbag_name` | VARCHAR(255) | NOT NULL | Rosbag 名称 |
|
||||
| `gnss_downsampled_points` | geometry[] | | 降采样 GNSS 轨迹点 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `is_overlapped` | BOOLEAN | DEFAULT FALSE | 是否与其他 Bag 重叠 |
|
||||
|
||||
---
|
||||
|
||||
## 🔗 8. 关联关系模块
|
||||
|
||||
### 8.1 Bag-Topic 关联 (`bag_topic`)
|
||||
Bag 文件与 ROS Topic 的多对多关系
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_id` | INT | NOT NULL, FK → bag_list(id) | Bag ID |
|
||||
| `topic_id` | INT | NOT NULL, FK → topic_list(id) | Topic ID |
|
||||
|
||||
### 8.2 Bag-标签 关联 (`bag_reserved_tag`)
|
||||
Bag 文件与标签的多对多关系
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_id` | INT | NOT NULL, FK → bag_list(id) | Bag ID |
|
||||
| `tag_id` | INT | NOT NULL, FK → reserved_tag_list(id) | 标签 ID |
|
||||
|
||||
### 8.3 Bag-真值 关联 (`bag_gt`)
|
||||
Bag 文件与真值数据的多对多关系
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `gt_id` | INT | NOT NULL, FK → gt_meta(id) | 真值元数据 ID |
|
||||
| `bag_id` | INT | NOT NULL, FK → bag_list(id) | Bag ID |
|
||||
| `comment` | TEXT | | 备注 |
|
||||
|
||||
---
|
||||
|
||||
## 📝 9. 审计日志模块
|
||||
|
||||
### 9.1 操作历史 (`ops_history`)
|
||||
通用审计日志，记录所有数据变更
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | BIGSERIAL | PRIMARY KEY | 审计日志自增主键 |
|
||||
| `table_name` | REGCLASS | NOT NULL | 被审计表 |
|
||||
| `row_pk` | TEXT | NOT NULL | 行主键值 |
|
||||
| `column_name` | TEXT | NOT NULL | 被修改列名 |
|
||||
| `old_value` | TEXT | | 旧值 |
|
||||
| `new_value` | TEXT | | 新值 |
|
||||
| `changed_by` | NAME | NOT NULL DEFAULT CURRENT_USER | 数据库用户 |
|
||||
| `changed_at` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 变更时间 |
|
||||
| `comment` | TEXT | | 可读描述 |
|
||||
| `app_module` | TEXT | | 应用模块 |
|
||||
| `trace_id` | TEXT | | 链路追踪 ID |
|
||||
|
||||
---
|
||||
|
||||
## 📊 10. 数据统计信息
|
||||
|
||||
### 📈 表数据规模
|
||||
根据字段设计和索引配置，预估的数据规模：
|
||||
|
||||
| 表名 | 预估记录数 | 主要用途 | 关键字段 |
|
||||
|------|------------|----------|----------|
|
||||
| `bag_list` | 10K+ | Bag文件管理 | name, project_id, is_decoded |
|
||||
| `main_pangu` | 10K+ | Pangu数据 | name, vehicle, datetime |
|
||||
| `main_minerva` | 5K+ | Minerva会话 | session_id, vin, platform |
|
||||
| `fst` | 1K+ | FST分类树 | name, parent_id, bag_sum |
|
||||
| `topic_list` | 500+ | ROS Topic字典 | name, type |
|
||||
| `reserved_tag_list` | 1K+ | 标签字典 | name, type, creator |
|
||||
| `bag_topic` | 100K+ | Bag-Topic关联 | bag_id, topic_id |
|
||||
| `fst_bag` | 50K+ | FST-Bag关联 | bag_id, fst_node_id |
|
||||
| `geometry_info` | 10K+ | 地理轨迹 | rosbag_name, gnss_points |
|
||||
|
||||
### 🔍 关键索引
|
||||
|
||||
**性能优化索引**:
|
||||
- `idx_bag_list_project_id` - 按项目查询
|
||||
- `idx_main_pangu_datetime` - 按时间范围查询
|
||||
- `idx_main_minerva_session_id` - 会话ID查询
|
||||
- `idx_fst_name` - FST节点名称查询
|
||||
- `idx_bag_list_json_gin` - JSON字段GIN索引
|
||||
|
||||
**复合索引**:
|
||||
- `idx_bag_list_project_active` - 活跃项目数据
|
||||
- `uq_taglist_name_not_deleted` - 未删除标签唯一性
|
||||
- `idx_lifecycle_*_time` - 生命周期时间BRIN索引
|
||||
|
||||
---
|
||||
|
||||
## 🚀 API 数据访问
|
||||
|
||||
### 常用查询模式
|
||||
|
||||
#### 1. 获取项目下所有Bag
|
||||
```sql
|
||||
SELECT bl.*, p.name as project_name
|
||||
FROM bag_list bl
|
||||
JOIN project p ON bl.project_id = p.id
|
||||
WHERE p.name = 'project_name' AND bl.is_deleted = FALSE;
|
||||
```
|
||||
|
||||
#### 2. 获取Bag的完整信息
|
||||
```sql
|
||||
SELECT
|
||||
bl.*,
|
||||
sp.camera_front_wide,
|
||||
sp.lidar_gt_pandar128,
|
||||
mp.vehicle,
|
||||
mp.datetime
|
||||
FROM bag_list bl
|
||||
LEFT JOIN secondary_pangu sp ON bl.id = sp.bag_id
|
||||
LEFT JOIN main_pangu mp ON bl.name = mp.name
|
||||
WHERE bl.name = 'bag_name';
|
||||
```
|
||||
|
||||
#### 3. 获取FST树结构
|
||||
```sql
|
||||
WITH RECURSIVE fst_tree AS (
|
||||
SELECT id, name, parent_id, 0 as level
|
||||
FROM fst WHERE parent_id IS NULL
|
||||
UNION ALL
|
||||
SELECT f.id, f.name, f.parent_id, ft.level + 1
|
||||
FROM fst f
|
||||
JOIN fst_tree ft ON f.parent_id = ft.id
|
||||
)
|
||||
SELECT * FROM fst_tree ORDER BY level, name;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📚 扩展信息
|
||||
|
||||
### JSON 扩展字段
|
||||
多个表包含 `reserved_json` 字段，支持：
|
||||
- **动态扩展**: 无需修改表结构添加新字段
|
||||
- **复杂数据**: 存储嵌套或数组类型数据
|
||||
- **高效查询**: 支持 GIN 索引的JSON查询
|
||||
- **版本兼容**: 向后兼容的数据扩展
|
||||
|
||||
### PostGIS 地理支持
|
||||
- **空间扩展**: PostGIS + UUID-OSSP 扩展
|
||||
- **几何类型**: 支持 geometry[] 数组类型
|
||||
- **轨迹数据**: GNSS 降采样轨迹点存储
|
||||
- **空间查询**: 支持地理范围和重叠检测
|
||||
|
||||
### 数据一致性
|
||||
- **外键约束**: 保证数据完整性
|
||||
- **唯一约束**: 防止重复数据
|
||||
- **级联删除**: 自动清理关联数据
|
||||
- **软删除**: 逻辑删除保留数据历史
|
||||
|
||||
---
|
||||
|
||||
**文档版本**: v1.0 | **更新时间**: 2024-01-14 | **数据库版本**: PostgreSQL 13+ with PostGIS
|
||||
@@ -0,0 +1,393 @@
|
||||
# fst_stash 设计与接口说明
|
||||
|
||||
本文档记录与 `fst_stash` 相关的表结构、核心服务函数以及 API 接口，方便前后端统一理解“导入/草稿版 FST 树”的读写方式。
|
||||
|
||||
---
|
||||
|
||||
## 一、表设计：fst_stash
|
||||
|
||||
**表名**：`fst_stash`
|
||||
|
||||
- `id`：`bigint`，主键，自增
|
||||
- `fst_versions`：`varchar(255)`，非空
|
||||
- 外键，关联 `version_list.version`
|
||||
- 表示该条 stash 对应的版本号（例如 `MB Driving FST V1`）
|
||||
- `content`：`jsonb`，可空
|
||||
- 存放完整的 FST JSON 内容（通常包含 `meta` 和 `nodes`）
|
||||
- `created_time`：`timestamptz`，非空，默认当前时间
|
||||
- `updated_time`：`timestamptz`，非空，默认当前时间，更新时自动刷新
|
||||
|
||||
约束与关系：
|
||||
|
||||
- `fst_versions` 外键指向 `version_list.version`，`ON DELETE CASCADE`
|
||||
删除某个版本时，对应的 stash 会自动被删除。
|
||||
|
||||
内容约定（非数据库约束）：
|
||||
|
||||
- 推荐结构：
|
||||
|
||||
```json
|
||||
{
|
||||
"meta": {
|
||||
"desc": "MB Driving FST V1",
|
||||
"creator": "xxx",
|
||||
"created_at": "2026-02-25T10:00:00Z"
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "ROOT",
|
||||
"parentId": null,
|
||||
"treeLevel": 1,
|
||||
"node_version": 1
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "CHILD",
|
||||
"parentId": 1,
|
||||
"treeLevel": 2,
|
||||
"node_version": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- 接口侧只要求 `nodes` 为数组，并且每个节点至少包含：
|
||||
- `id`：用于 parentId 关联
|
||||
- `name`：节点名称
|
||||
- `parentId`：父节点 id（根节点为 `null`）
|
||||
- 节点级乐观锁字段：
|
||||
- `node_version`：节点版本号，默认值 `1`
|
||||
- 写入时若缺失会自动补齐为 `1`
|
||||
- `/fst/stash/update` 默认开启乐观锁校验，不一致返回 `409`
|
||||
|
||||
---
|
||||
|
||||
## 二、核心服务函数（service.py）
|
||||
|
||||
文件位置：`src/core/service.py`
|
||||
|
||||
### 1. `upsert_fst_stash`
|
||||
|
||||
```python
|
||||
def upsert_fst_stash(
|
||||
db: Session,
|
||||
*,
|
||||
version: str,
|
||||
content: Optional[Dict[str, Any]],
|
||||
use_node_optimistic_lock: bool = False,
|
||||
) -> "FstStash":
|
||||
"""
|
||||
Insert or update FST stash record for a version.
|
||||
If a record exists for the given version, update its content.
|
||||
If not, create a new record.
|
||||
"""
|
||||
```
|
||||
|
||||
行为说明：
|
||||
|
||||
- 先检查 `version_list` 中是否存在 `version`：
|
||||
- 若不存在，抛出 `ValueError("Version '<version>' not found in version_list")`
|
||||
- 再按 `fst_versions = version` 查询 `fst_stash`：
|
||||
- 若已存在：更新其 `content`（仅当传入 `content` 非 `None` 时）
|
||||
- 若不存在：插入新行，写入 `fst_versions` 和 `content`
|
||||
- 节点版本规则：
|
||||
- 遍历 `nodes` 及其 `children`，自动补齐 `node_version`（默认 `1`）
|
||||
- 更新时若节点内容变更，该节点 `node_version` 自动 `+1`
|
||||
- 当 `use_node_optimistic_lock=True` 时，校验传入 `node_version` 与库内一致，否则抛出冲突异常
|
||||
- `db.flush()` 后返回对应的 `FstStash` 对象
|
||||
|
||||
### 2. `get_fst_stash`
|
||||
|
||||
```python
|
||||
def get_fst_stash(db: Session, version: str) -> Optional["FstStash"]:
|
||||
"""Get FST stash record by version."""
|
||||
```
|
||||
|
||||
行为说明：
|
||||
|
||||
- 直接根据 `fst_versions = version` 查询并返回 `FstStash` 实体
|
||||
- 找不到则返回 `None`
|
||||
|
||||
---
|
||||
|
||||
## 三、API 接口列表（fst.py）
|
||||
|
||||
文件位置：`src/api/fst.py`
|
||||
|
||||
### 1. POST `/fst/stash` —— 保存 stash 内容
|
||||
|
||||
对应函数：`save_fst_stash`
|
||||
|
||||
用途：
|
||||
|
||||
- 通用的保存接口，根据 `version` 写入 `fst_stash.content`。
|
||||
- 适合“导入 JSON 后一次性保存”的场景。
|
||||
|
||||
请求：
|
||||
|
||||
```json
|
||||
POST /fst/stash
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"version": "MB Driving FST V1",
|
||||
"content": {
|
||||
"meta": { "...": "..." },
|
||||
"nodes": [ { "id": 1, "name": "ROOT", "parentId": null }, ... ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
返回（200）：
|
||||
|
||||
```json
|
||||
{
|
||||
"id": 1,
|
||||
"version": "MB Driving FST V1",
|
||||
"updated_time": "2026-02-25T10:00:00+00:00"
|
||||
}
|
||||
```
|
||||
|
||||
错误：
|
||||
|
||||
- `400`：缺少 `version` 或版本在 `version_list` 中不存在
|
||||
- `500`：其他内部错误
|
||||
|
||||
内部实现：
|
||||
|
||||
- 调用 `upsert_fst_stash(db, version=version, content=content)`
|
||||
|
||||
### 2. GET `/fst/stash/<version>` —— 根据版本获取原始 stash 内容
|
||||
|
||||
对应函数：`get_fst_stash_by_version`
|
||||
|
||||
用途：
|
||||
|
||||
- 按版本直接返回 `fst_stash` 的原始存储内容；
|
||||
- 适合需要拿到完整 JSON（含 `meta`、`nodes` 等原始结构）的场景。
|
||||
|
||||
请求示例：
|
||||
|
||||
```http
|
||||
GET /fst/stash/MB%20Driving%20FST%20V1
|
||||
```
|
||||
|
||||
返回（200）：
|
||||
|
||||
```json
|
||||
{
|
||||
"id": 1,
|
||||
"version": "MB Driving FST V1",
|
||||
"content": {
|
||||
"meta": { "...": "..." },
|
||||
"nodes": [ ... ]
|
||||
},
|
||||
"updated_time": "2026-02-25T10:00:00+00:00"
|
||||
}
|
||||
```
|
||||
|
||||
错误：
|
||||
|
||||
- `404`：指定版本在 `fst_stash` 中不存在
|
||||
- `500`：其他内部错误
|
||||
|
||||
### 3. GET `/fst/stash/print_tree` —— 基于 stash 构建 FST 树
|
||||
|
||||
对应函数：`print_tree_from_stash`
|
||||
|
||||
用途：
|
||||
|
||||
- 根据 stash 中的 `nodes` 数组，构建与 `/fst/print_tree` 相同结构的树；
|
||||
- 方便前端直接复用现有 Tree 渲染逻辑，但数据来源于 `fst_stash`。
|
||||
|
||||
请求示例：
|
||||
|
||||
```http
|
||||
GET /fst/stash/print_tree?version=MB%20Driving%20FST%20V1
|
||||
```
|
||||
|
||||
处理逻辑简要：
|
||||
|
||||
1. 使用 `get_fst_stash(db, version)` 读取 stash
|
||||
- 若不存在或 `content` 为空 → `404`
|
||||
2. 读取 `content["nodes"]`，要求为数组，否则 `400`
|
||||
3. 遍历 `nodes`：
|
||||
- 按 `id` 建立节点映射
|
||||
- 将每个节点转换为：
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "<name>",
|
||||
"label": "<name>",
|
||||
"level": 0,
|
||||
"children": []
|
||||
}
|
||||
```
|
||||
|
||||
4. 根据 `parentId` 将子节点挂到父节点的 `children` 中
|
||||
5. 从所有节点中找出根节点（`parentId` 为空或找不到父节点）作为返回数组
|
||||
6. 递归设置 `level`：根节点为 0，子节点在父节点基础上 +1
|
||||
|
||||
返回（200）：
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": "ROOT",
|
||||
"label": "ROOT",
|
||||
"level": 0,
|
||||
"children": [
|
||||
{
|
||||
"id": "CHILD",
|
||||
"label": "CHILD",
|
||||
"level": 1,
|
||||
"children": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
错误：
|
||||
|
||||
- `400`：`version` 缺失，或 `content` 中不含合法的 `nodes` 数组
|
||||
- `404`：stash 不存在或内容为空
|
||||
- `500`：其他内部错误
|
||||
|
||||
### 4. POST `/fst/stash/update` —— 按版本更新 stash（对齐 /fst/update 风格）
|
||||
|
||||
对应函数：`upsert_fst_stash_api`
|
||||
|
||||
用途：
|
||||
|
||||
- 参考 `/fst/update` 的风格进行设计；
|
||||
- 也是对 `fst_stash` 做 UPSERT，只是路径和语义更偏向“编辑当前版本的 stash 内容”；
|
||||
- 前端可统一通过 `/fst/stash/update` 写入当前版本的 JSON。
|
||||
- 该接口默认开启节点级乐观锁校验。
|
||||
|
||||
请求：
|
||||
|
||||
```json
|
||||
POST /fst/stash/update
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"version": "MB Driving FST V1",
|
||||
"content": {
|
||||
"meta": { "...": "..." },
|
||||
"nodes": [ ... ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
返回（200）：
|
||||
|
||||
```json
|
||||
{
|
||||
"id": 1,
|
||||
"version": "MB Driving FST V1",
|
||||
"updated_time": "2026-02-25T10:00:00+00:00"
|
||||
}
|
||||
```
|
||||
|
||||
错误：
|
||||
|
||||
- `400`：
|
||||
- 缺少 `version`
|
||||
- `version` 不在 `version_list` 中（`upsert_fst_stash` 抛出的 `ValueError`）
|
||||
- `409`：
|
||||
- 节点级乐观锁冲突（返回 `STASH_NODE_VERSION_CONFLICT` 和冲突节点列表）
|
||||
- `500`：其他内部错误
|
||||
|
||||
内部实现：
|
||||
|
||||
- 和 `POST /fst/stash` 一样，最终都调用 `upsert_fst_stash`，只是路由风格与语义更贴近 `/fst/update`。
|
||||
|
||||
---
|
||||
|
||||
## 四、与正式 FST 表的关系
|
||||
|
||||
- 正式 FST 表读取接口：
|
||||
- `GET /fst/print_tree`：直接从 `fst` 表读取，构建树结构并返回。
|
||||
- Stash 相关接口：
|
||||
- `POST /fst/stash`、`POST /fst/stash/update`：将某个版本的 JSON 写入 `fst_stash`
|
||||
- `GET /fst/stash/<version>`：按版本拿回原始 JSON
|
||||
- `GET /fst/stash/print_tree`：基于 stash 的 `nodes` 构建树，结构与 `/fst/print_tree` 相同
|
||||
|
||||
当前阶段设计原则：
|
||||
|
||||
- **正式 FST 表**：仍然通过 `/fst/update` 等接口直接修改；
|
||||
- **fst_stash**：作为“导入/草稿/预览版本”的 JSON 存储，方便前端导入导出、预览树结构以及后续一键发布功能的扩展。
|
||||
|
||||
---
|
||||
|
||||
## 五、后续新增：Feishu 写入（版本生效）
|
||||
|
||||
本节记录后续新增的“版本生效并同步到飞书多维表格”能力。
|
||||
|
||||
### 1. 触发接口
|
||||
|
||||
文件位置：`src/api/versions.py`
|
||||
|
||||
- `POST /versions/stash/<version_id>/activate`
|
||||
- `POST /versions/fst/stash/activate/<version_id>`
|
||||
- `POST /versions/<version_id>/activate`
|
||||
|
||||
说明：
|
||||
|
||||
- 三个路由映射到同一个函数 `activate_version_api`。
|
||||
- 请求体可选传入 `content`：
|
||||
- 传了：直接使用该 JSON 作为飞书写入内容；
|
||||
- 不传：回退读取 `fst_stash.content`。
|
||||
- 若最终拿不到 JSON，返回 `404`。
|
||||
|
||||
### 2. 执行方式（异步任务）
|
||||
|
||||
`activate_version_api` 不阻塞等待飞书完成，而是：
|
||||
|
||||
1. 创建 `task_id` 并写入内存任务表（`ACTIVATE_TASKS`），初始状态 `queued`；
|
||||
2. 启动后台线程执行 `_run_activate_task(task_id, version_id, content)`；
|
||||
3. 立即返回 `202 Accepted`。
|
||||
|
||||
任务结果查询接口：
|
||||
|
||||
- `GET /versions/stash/<version_id>/activate/result?task_id=...`
|
||||
- `GET /versions/<version_id>/activate/result?task_id=...`
|
||||
|
||||
如果不传 `task_id`，会返回该版本最近一次任务结果。
|
||||
|
||||
### 3. 与飞书 SDK 的对接
|
||||
|
||||
后台线程 `_run_activate_task` 的关键流程：
|
||||
|
||||
1. 将 `version_list.feishu_sync_status` 置为 `syncing`；
|
||||
2. 调用 `create_bitable_for_version(version_obj.version, content)`；
|
||||
3. 成功后：
|
||||
- `feishu_sync_status = synced`
|
||||
- `status = active`
|
||||
4. 失败时：
|
||||
- 回滚后将 `feishu_sync_status = failed`
|
||||
- 任务状态记为 `failed` 并返回错误信息。
|
||||
|
||||
### 4. Feishu 写入目标与结构（feishu_bitable_sdk.py）
|
||||
|
||||
文件位置：`src/core/feishu_bitable_sdk.py`
|
||||
|
||||
- 主多维表标题固定为 `Fst_Editor`（常量 `FST_EDITOR_BITABLE_TITLE`）。
|
||||
- 目前新增了父文档定位能力：
|
||||
- 优先使用 `PARENT_NODE_TOKEN`；
|
||||
- 若为空，则按 `FST_EDITOR_PARENT_DOC_TITLE`（当前值为 `Content generated automatically by bot`）在空间根目录解析父文档；
|
||||
- 在该父文档下复用或创建 `Fst_Editor` 多维表。
|
||||
- 保留“只保留当前版本表”的策略：目标版本表不存在则创建，其他版本表会尝试删除。
|
||||
- 当内容可解析为 `nodes`/字段结构时，按业务列写入；否则回退写入 JSON 文本列。
|
||||
|
||||
### 5. 子文档同步（可开关）
|
||||
|
||||
`create_bitable_for_version` 在主表写入前后会根据开关触发一级节点子文档同步：
|
||||
|
||||
- `ENABLE_WIKI_CHILD_DOC_SYNC`
|
||||
- `ENABLE_WIKI_CHILD_DOC_SYNC_ASYNC`
|
||||
|
||||
默认配置下为异步触发，不阻塞主流程；结果会体现在返回体 `wiki_child_docs_result` 字段中。
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE version_list
|
||||
ADD COLUMN IF NOT EXISTS feishu_sync_status VARCHAR(50) NOT NULL DEFAULT 'unsynced';
|
||||
|
||||
COMMENT ON COLUMN version_list.feishu_sync_status IS 'Feishu sync status: unsynced/synced';
|
||||
@@ -0,0 +1,702 @@
|
||||
-- ======================================================
|
||||
-- 0 扩展
|
||||
-- ======================================================
|
||||
CREATE EXTENSION IF NOT EXISTS postgis;
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
|
||||
COMMENT ON EXTENSION postgis IS 'PostGIS 空间数据库扩展';
|
||||
COMMENT ON EXTENSION "uuid-ossp" IS 'UUID 生成扩展';
|
||||
|
||||
-- ======================================================
|
||||
-- ① 基础准备
|
||||
-- ======================================================
|
||||
CREATE TABLE IF NOT EXISTS project (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
COMMENT ON TABLE project IS '数据来源 1：DFDI 2：CFDI';
|
||||
COMMENT ON COLUMN project.id IS '项目自增主键';
|
||||
COMMENT ON COLUMN project.name IS '项目名称，全局唯一';
|
||||
COMMENT ON COLUMN project.update_time IS '最近一次更新时间';
|
||||
|
||||
-- ======================================================
|
||||
-- ② 字典 & 主数据（无外部依赖）
|
||||
-- ======================================================
|
||||
CREATE TABLE IF NOT EXISTS drive_mode (
|
||||
id SERIAL PRIMARY KEY,
|
||||
type VARCHAR(100) NOT NULL,
|
||||
sub_type VARCHAR(100),
|
||||
reserved_json JSONB,
|
||||
comment TEXT,
|
||||
UNIQUE (type, sub_type)
|
||||
);
|
||||
COMMENT ON TABLE drive_mode IS '驾驶模式字典表';
|
||||
COMMENT ON COLUMN drive_mode.id IS '主键';
|
||||
COMMENT ON COLUMN drive_mode.type IS '驾驶模式大类，AD Drive/AD PARK(APA)/AD PARK(MPA)';
|
||||
COMMENT ON COLUMN drive_mode.sub_type IS '驾驶模式子类，如ACC/CP/NP';
|
||||
COMMENT ON COLUMN drive_mode.reserved_json IS '预留扩展 JSON';
|
||||
COMMENT ON COLUMN drive_mode.comment IS '备注';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS topic_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
type VARCHAR(255),
|
||||
key_data TEXT,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE topic_list IS 'Topic表';
|
||||
COMMENT ON COLUMN topic_list.id IS '主键';
|
||||
COMMENT ON COLUMN topic_list.name IS 'Topic 名称，如 /lidar/points';
|
||||
COMMENT ON COLUMN topic_list.type IS 'Topic 类型，如 sensor_msgs/PointCloud2';
|
||||
COMMENT ON COLUMN topic_list.key_data IS '关键数据摘要';
|
||||
COMMENT ON COLUMN topic_list.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN topic_list.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS reserved_tag_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
type VARCHAR(50),
|
||||
creator VARCHAR(255),
|
||||
comments TEXT,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
is_deleted BOOLEAN DEFAULT FALSE,
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE reserved_tag_list IS '预留标签字典表';
|
||||
COMMENT ON COLUMN reserved_tag_list.id IS '主键';
|
||||
COMMENT ON COLUMN reserved_tag_list.name IS '标签名称';
|
||||
COMMENT ON COLUMN reserved_tag_list.type IS '标签分类';
|
||||
COMMENT ON COLUMN reserved_tag_list.creator IS '创建人';
|
||||
COMMENT ON COLUMN reserved_tag_list.comments IS '备注';
|
||||
COMMENT ON COLUMN reserved_tag_list.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN reserved_tag_list.is_deleted IS '逻辑删除标记';
|
||||
COMMENT ON COLUMN reserved_tag_list.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS gt_meta (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
type VARCHAR(100) NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
reserved_json JSONB,
|
||||
comment TEXT,
|
||||
CONSTRAINT gt_meta_name_type_unique UNIQUE (name, type)
|
||||
);
|
||||
COMMENT ON TABLE gt_meta IS '真值元数据表';
|
||||
COMMENT ON COLUMN gt_meta.id IS '主键';
|
||||
COMMENT ON COLUMN gt_meta.name IS '真值名称';
|
||||
COMMENT ON COLUMN gt_meta.type IS '真值类型，如 3DBox/Lane';
|
||||
COMMENT ON COLUMN gt_meta.path IS '真值文件存储路径';
|
||||
COMMENT ON COLUMN gt_meta.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN gt_meta.reserved_json IS '预留扩展 JSON';
|
||||
COMMENT ON COLUMN gt_meta.comment IS '备注';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS version_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
version VARCHAR(255) NOT NULL UNIQUE,
|
||||
type VARCHAR(50),
|
||||
description TEXT,
|
||||
release_date TIMESTAMPTZ,
|
||||
created_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
status VARCHAR(50) NOT NULL DEFAULT 'available',
|
||||
feishu_sync_status VARCHAR(50) NOT NULL DEFAULT 'unsynced',
|
||||
reserved_json JSONB
|
||||
);
|
||||
|
||||
-- ======================================================
|
||||
-- ③ 核心业务表（依赖 ①②）
|
||||
-- ======================================================
|
||||
CREATE TABLE IF NOT EXISTS bag_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
project_id INT,
|
||||
tile_id VARCHAR(255),
|
||||
is_decoded BOOLEAN DEFAULT FALSE,
|
||||
is_deleted BOOLEAN DEFAULT FALSE,
|
||||
od_annotated INT,
|
||||
ld_annotated INT,
|
||||
fst_indexed BOOLEAN DEFAULT FALSE,
|
||||
is_active_data BOOLEAN DEFAULT TRUE,
|
||||
reserved_str VARCHAR(255),
|
||||
reserved_json JSONB,
|
||||
drive_mode SMALLINT,
|
||||
sw_version VARCHAR(100),
|
||||
hw_version VARCHAR(100),
|
||||
FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (drive_mode) REFERENCES drive_mode(id) ON DELETE SET NULL
|
||||
);
|
||||
COMMENT ON TABLE bag_list IS 'Rosbag 主表';
|
||||
COMMENT ON COLUMN bag_list.id IS '主键';
|
||||
COMMENT ON COLUMN bag_list.name IS 'Bag 文件名称，全局唯一';
|
||||
COMMENT ON COLUMN bag_list.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN bag_list.project_id IS '数据来源';
|
||||
COMMENT ON COLUMN bag_list.tile_id IS '瓦片 ID';
|
||||
COMMENT ON COLUMN bag_list.is_decoded IS '是否已解码';
|
||||
COMMENT ON COLUMN bag_list.is_deleted IS '逻辑删除';
|
||||
COMMENT ON COLUMN bag_list.od_annotated IS '是否目标检测标注';
|
||||
COMMENT ON COLUMN bag_list.ld_annotated IS '是否车道线标注';
|
||||
COMMENT ON COLUMN bag_list.fst_indexed IS '是否已建立 FST 索引';
|
||||
COMMENT ON COLUMN bag_list.is_active_data IS '是否为活跃数据';
|
||||
COMMENT ON COLUMN bag_list.reserved_str IS '预留字符串';
|
||||
COMMENT ON COLUMN bag_list.reserved_json IS '预留扩展 JSON';
|
||||
COMMENT ON COLUMN bag_list.drive_mode IS '驾驶模式字典 ID';
|
||||
COMMENT ON COLUMN bag_list.sw_version IS '软件版本号（Software Version）';
|
||||
COMMENT ON COLUMN bag_list.hw_version IS '硬件版本号（Hardware Version）';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bag_lifecycle (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_name VARCHAR(255) NOT NULL UNIQUE,
|
||||
collect_time TIMESTAMPTZ,
|
||||
clone2dev_time TIMESTAMPTZ,
|
||||
decode_time TIMESTAMPTZ,
|
||||
mining_time TIMESTAMPTZ,
|
||||
auto_annotate_time TIMESTAMPTZ,
|
||||
manual_annotate_time TIMESTAMPTZ,
|
||||
fst_index_time TIMESTAMPTZ,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
FOREIGN KEY (bag_name) REFERENCES bag_list(name) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE bag_lifecycle IS 'Bag 生命周期时间表';
|
||||
COMMENT ON COLUMN bag_lifecycle.id IS '主键';
|
||||
COMMENT ON COLUMN bag_lifecycle.bag_name IS 'Bag 名称，关联 bag_list.name';
|
||||
COMMENT ON COLUMN bag_lifecycle.collect_time IS '采集完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.clone2dev_time IS '拷贝到bucket时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.decode_time IS '解码完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.mining_time IS '数据挖掘完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.auto_annotate_time IS '自动标注完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.manual_annotate_time IS '人工标注完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.fst_index_time IS 'FST 索引完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.update_time IS '记录更新时间';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS main_pangu (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
vehicle VARCHAR(255),
|
||||
datetime TIMESTAMPTZ,
|
||||
bag_path TEXT,
|
||||
data_path TEXT,
|
||||
reserved_str VARCHAR(255),
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE main_pangu IS 'Pangu 主表（原始数据）';
|
||||
COMMENT ON COLUMN main_pangu.id IS '主键';
|
||||
COMMENT ON COLUMN main_pangu.name IS '名称，全局唯一';
|
||||
COMMENT ON COLUMN main_pangu.vehicle IS '车辆编号';
|
||||
COMMENT ON COLUMN main_pangu.datetime IS 'rosbag的生成时间';
|
||||
COMMENT ON COLUMN main_pangu.bag_path IS '原始 Bag 路径';
|
||||
COMMENT ON COLUMN main_pangu.data_path IS '解析后数据路径';
|
||||
COMMENT ON COLUMN main_pangu.reserved_str IS '预留字符串';
|
||||
COMMENT ON COLUMN main_pangu.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS joined_pangu (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
data_path TEXT,
|
||||
reserved_str VARCHAR(255),
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE joined_pangu IS 'Pangu rosbag融合表';
|
||||
COMMENT ON COLUMN joined_pangu.id IS '主键';
|
||||
COMMENT ON COLUMN joined_pangu.name IS '融合后名称';
|
||||
COMMENT ON COLUMN joined_pangu.data_path IS '融合后数据路径';
|
||||
COMMENT ON COLUMN joined_pangu.reserved_str IS '预留字符串';
|
||||
COMMENT ON COLUMN joined_pangu.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS joined_bags (
|
||||
id SERIAL PRIMARY KEY,
|
||||
parent_id INT NOT NULL,
|
||||
child_id INT NOT NULL UNIQUE,
|
||||
reserved_str VARCHAR(255),
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE joined_bags IS 'Bag 融合关系表';
|
||||
COMMENT ON COLUMN joined_bags.id IS '主键';
|
||||
COMMENT ON COLUMN joined_bags.parent_id IS '父 Bag ID';
|
||||
COMMENT ON COLUMN joined_bags.child_id IS '子 Bag ID，唯一';
|
||||
COMMENT ON COLUMN joined_bags.reserved_str IS '预留字符串';
|
||||
COMMENT ON COLUMN joined_bags.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS main_minerva (
|
||||
id SERIAL PRIMARY KEY,
|
||||
session_id VARCHAR(255) UNIQUE,
|
||||
start_ts TIMESTAMPTZ,
|
||||
end_ts TIMESTAMPTZ,
|
||||
length INT,
|
||||
datetime TIMESTAMPTZ,
|
||||
vin VARCHAR(255),
|
||||
platform VARCHAR(255),
|
||||
mapped BOOLEAN DEFAULT FALSE,
|
||||
path TEXT,
|
||||
converted_path TEXT,
|
||||
gt_path TEXT,
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE main_minerva IS 'Minerva 主表';
|
||||
COMMENT ON COLUMN main_minerva.id IS '主键';
|
||||
COMMENT ON COLUMN main_minerva.session_id IS '会话 ID，唯一';
|
||||
COMMENT ON COLUMN main_minerva.start_ts IS '开始时间戳';
|
||||
COMMENT ON COLUMN main_minerva.end_ts IS '结束时间戳';
|
||||
COMMENT ON COLUMN main_minerva.length IS '时长（秒）';
|
||||
COMMENT ON COLUMN main_minerva.datetime IS '数据日期';
|
||||
COMMENT ON COLUMN main_minerva.vin IS '车辆识别码';
|
||||
COMMENT ON COLUMN main_minerva.platform IS '平台名称';
|
||||
COMMENT ON COLUMN main_minerva.mapped IS '是否已映射';
|
||||
COMMENT ON COLUMN main_minerva.path IS '原始数据路径';
|
||||
COMMENT ON COLUMN main_minerva.converted_path IS '转换后路径';
|
||||
COMMENT ON COLUMN main_minerva.gt_path IS '真值路径';
|
||||
COMMENT ON COLUMN main_minerva.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
-- ======================================================
|
||||
-- ④ 子表 & 关联表（依赖 bag_list.id）
|
||||
-- ======================================================
|
||||
CREATE TABLE IF NOT EXISTS secondary_pangu (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL UNIQUE,
|
||||
lidar_gt_pandar128 TEXT,
|
||||
object_lidar_gt_pandar128_manual TEXT,
|
||||
lidar_fd_multi_scan_raw TEXT,
|
||||
camera_fisheye_left TEXT,
|
||||
camera_fisheye_right TEXT,
|
||||
camera_front_wide TEXT,
|
||||
raw_gps TEXT,
|
||||
raw_imu TEXT,
|
||||
ego_motion TEXT,
|
||||
vehicle_wheel TEXT,
|
||||
calibration TEXT,
|
||||
sdmap TEXT,
|
||||
reserved_json JSONB,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE secondary_pangu IS 'Pangu 二级数据路径表';
|
||||
COMMENT ON COLUMN secondary_pangu.id IS '主键';
|
||||
COMMENT ON COLUMN secondary_pangu.bag_id IS '关联 bag_list.id';
|
||||
COMMENT ON COLUMN secondary_pangu.lidar_gt_pandar128 IS '激光真值路径';
|
||||
COMMENT ON COLUMN secondary_pangu.object_lidar_gt_pandar128_manual IS '人工真值路径';
|
||||
COMMENT ON COLUMN secondary_pangu.lidar_fd_multi_scan_raw IS '原始雷达路径';
|
||||
COMMENT ON COLUMN secondary_pangu.camera_fisheye_left IS '左鱼眼相机路径';
|
||||
COMMENT ON COLUMN secondary_pangu.camera_fisheye_right IS '右鱼眼相机路径';
|
||||
COMMENT ON COLUMN secondary_pangu.camera_front_wide IS '前广角相机路径';
|
||||
COMMENT ON COLUMN secondary_pangu.raw_gps IS '原始 GPS 路径';
|
||||
COMMENT ON COLUMN secondary_pangu.raw_imu IS '原始 IMU 路径';
|
||||
COMMENT ON COLUMN secondary_pangu.ego_motion IS '自车运动路径';
|
||||
COMMENT ON COLUMN secondary_pangu.vehicle_wheel IS '轮速路径';
|
||||
COMMENT ON COLUMN secondary_pangu.calibration IS '标定文件路径';
|
||||
COMMENT ON COLUMN secondary_pangu.sdmap IS 'SD 地图路径或地图版本标识';
|
||||
COMMENT ON COLUMN secondary_pangu.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS secondary_minerva (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL UNIQUE,
|
||||
lidar_gt_top_p128 TEXT,
|
||||
lidar_parking_gt_front_p128 TEXT,
|
||||
lidar_parking_gt_left_p128 TEXT,
|
||||
lidar_parking_gt_right_p128 TEXT,
|
||||
lidar_parking_gt_rear_p128 TEXT,
|
||||
camera_fisheye_left_200fov TEXT,
|
||||
camera_fisheye_right_200fov TEXT,
|
||||
camera_fisheye_rear_200fov TEXT,
|
||||
camera_fisheye_front_200fov TEXT,
|
||||
camera_front_wide_120fov TEXT,
|
||||
camera_front_tele_30fov TEXT,
|
||||
camera_rear_right_70fov TEXT,
|
||||
camera_rear_left_70fov TEXT,
|
||||
raw_gps TEXT,
|
||||
raw_imu TEXT,
|
||||
ego_motion TEXT,
|
||||
calibration TEXT,
|
||||
rig TEXT,
|
||||
fst_new TEXT,
|
||||
fst_old TEXT,
|
||||
comments TEXT,
|
||||
reserved_json JSONB,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE secondary_minerva IS 'Minerva 二级数据路径表';
|
||||
COMMENT ON COLUMN secondary_minerva.id IS '主键';
|
||||
COMMENT ON COLUMN secondary_minerva.bag_id IS '关联 bag_list.id';
|
||||
COMMENT ON COLUMN secondary_minerva.lidar_gt_top_p128 IS '顶部Lidar真值路径';
|
||||
COMMENT ON COLUMN secondary_minerva.lidar_parking_gt_front_p128 IS '前停车Lidar真值路径';
|
||||
COMMENT ON COLUMN secondary_minerva.lidar_parking_gt_left_p128 IS '左停车Lidar真值路径';
|
||||
COMMENT ON COLUMN secondary_minerva.lidar_parking_gt_right_p128 IS '右停车Lidar真值路径';
|
||||
COMMENT ON COLUMN secondary_minerva.lidar_parking_gt_rear_p128 IS '后停车Lidar真值路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_fisheye_left_200fov IS '左 200° 鱼眼路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_fisheye_right_200fov IS '右 200° 鱼眼路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_fisheye_rear_200fov IS '后 200° 鱼眼路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_fisheye_front_200fov IS '前 200° 鱼眼路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_front_wide_120fov IS '前 120° 广角路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_front_tele_30fov IS '前 30° 长焦路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_rear_right_70fov IS '右后 70° 相机路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_rear_left_70fov IS '左后 70° 相机路径';
|
||||
COMMENT ON COLUMN secondary_minerva.raw_gps IS '原始 GPS 路径';
|
||||
COMMENT ON COLUMN secondary_minerva.raw_imu IS '原始 IMU 路径';
|
||||
COMMENT ON COLUMN secondary_minerva.ego_motion IS '自车运动路径';
|
||||
COMMENT ON COLUMN secondary_minerva.calibration IS '标定文件路径';
|
||||
COMMENT ON COLUMN secondary_minerva.rig IS 'rig 文件路径';
|
||||
COMMENT ON COLUMN secondary_minerva.fst_new IS '新 FST 路径';
|
||||
COMMENT ON COLUMN secondary_minerva.fst_old IS '旧 FST 路径';
|
||||
COMMENT ON COLUMN secondary_minerva.comments IS '备注';
|
||||
COMMENT ON COLUMN secondary_minerva.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fst (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
parent_id INT,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
reserved_json JSONB,
|
||||
bag_sum INT DEFAULT 0,
|
||||
FOREIGN KEY (parent_id) REFERENCES fst(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE fst IS 'FST 节点表';
|
||||
COMMENT ON COLUMN fst.id IS '主键';
|
||||
COMMENT ON COLUMN fst.name IS '节点名称';
|
||||
COMMENT ON COLUMN fst.parent_id IS '父节点 ID';
|
||||
COMMENT ON COLUMN fst.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN fst.reserved_json IS '预留扩展 JSON';
|
||||
COMMENT ON COLUMN fst.bag_sum IS '子树 Bag 数量';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS geometry_info (
|
||||
id SERIAL PRIMARY KEY,
|
||||
rosbag_name VARCHAR(255) NOT NULL,
|
||||
gnss_downsampled_points geometry[],
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
is_overlapped BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
COMMENT ON TABLE geometry_info IS 'Bag 几何信息表（GNSS 轨迹）';
|
||||
COMMENT ON COLUMN geometry_info.id IS '主键';
|
||||
COMMENT ON COLUMN geometry_info.rosbag_name IS 'Rosbag 名称';
|
||||
COMMENT ON COLUMN geometry_info.gnss_downsampled_points IS '降采样 GNSS 轨迹点';
|
||||
COMMENT ON COLUMN geometry_info.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN geometry_info.is_overlapped IS '是否与其他 Bag 重叠';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bag_topic (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL,
|
||||
topic_id INT NOT NULL,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (topic_id) REFERENCES topic_list(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE bag_topic IS 'Bag-Topic 多对多关系';
|
||||
COMMENT ON COLUMN bag_topic.id IS '主键';
|
||||
COMMENT ON COLUMN bag_topic.bag_id IS 'Bag ID';
|
||||
COMMENT ON COLUMN bag_topic.topic_id IS 'Topic ID';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bag_reserved_tag (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL,
|
||||
tag_id INT NOT NULL,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (tag_id) REFERENCES reserved_tag_list(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE bag_reserved_tag IS 'Bag-预留标签多对多关系';
|
||||
COMMENT ON COLUMN bag_reserved_tag.id IS '主键';
|
||||
COMMENT ON COLUMN bag_reserved_tag.bag_id IS 'Bag ID';
|
||||
COMMENT ON COLUMN bag_reserved_tag.tag_id IS '标签 ID';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bag_gt (
|
||||
id SERIAL PRIMARY KEY,
|
||||
gt_id INT NOT NULL,
|
||||
bag_id INT NOT NULL,
|
||||
comment TEXT,
|
||||
UNIQUE (gt_id, bag_id),
|
||||
FOREIGN KEY (gt_id) REFERENCES gt_meta(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE bag_gt IS 'Bag-真值多对多关系';
|
||||
COMMENT ON COLUMN bag_gt.id IS '主键';
|
||||
COMMENT ON COLUMN bag_gt.gt_id IS '真值元数据 ID';
|
||||
COMMENT ON COLUMN bag_gt.bag_id IS 'Bag ID';
|
||||
COMMENT ON COLUMN bag_gt.comment IS '备注';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fst_bag (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL,
|
||||
fst_node_id INT NOT NULL,
|
||||
fst_node_level INT NOT NULL,
|
||||
event_start_time INT,
|
||||
event_end_time INT,
|
||||
img_url VARCHAR(255),
|
||||
video_url VARCHAR(255),
|
||||
comments VARCHAR(255),
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (fst_node_id) REFERENCES fst(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE fst_bag IS 'FST 节点与 Bag 的关联表';
|
||||
COMMENT ON COLUMN fst_bag.id IS '主键';
|
||||
COMMENT ON COLUMN fst_bag.bag_id IS 'Bag ID';
|
||||
COMMENT ON COLUMN fst_bag.fst_node_id IS 'FST 节点 ID';
|
||||
COMMENT ON COLUMN fst_bag.fst_node_level IS '节点层级';
|
||||
COMMENT ON COLUMN fst_bag.event_start_time IS '事件开始时间（ms）';
|
||||
COMMENT ON COLUMN fst_bag.event_end_time IS '事件结束时间（ms）';
|
||||
COMMENT ON COLUMN fst_bag.img_url IS '事件截图 URL';
|
||||
COMMENT ON COLUMN fst_bag.video_url IS '事件视频 URL';
|
||||
COMMENT ON COLUMN fst_bag.comments IS '备注';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS recompute_result (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL,
|
||||
recompute_version VARCHAR(100) NOT NULL,
|
||||
result_type VARCHAR(50) NOT NULL,
|
||||
storage_path VARCHAR(500) NOT NULL,
|
||||
status VARCHAR(50) NOT NULL DEFAULT 'available',
|
||||
created_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
reserved_json JSONB,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE,
|
||||
UNIQUE (bag_id, recompute_version, result_type)
|
||||
);
|
||||
COMMENT ON TABLE recompute_result IS 'Recompute 结果存储表';
|
||||
COMMENT ON COLUMN recompute_result.id IS '主键';
|
||||
COMMENT ON COLUMN recompute_result.bag_id IS '关联 bag_list.id';
|
||||
COMMENT ON COLUMN recompute_result.recompute_version IS 'Recompute版本';
|
||||
COMMENT ON COLUMN recompute_result.result_type IS '结果类型 (如: perception, planning, etc)';
|
||||
COMMENT ON COLUMN recompute_result.storage_path IS '结果存储路径';
|
||||
COMMENT ON COLUMN recompute_result.status IS '结果状态: available, archived, deleted';
|
||||
COMMENT ON COLUMN recompute_result.created_time IS 'Recompute完成时间';
|
||||
COMMENT ON COLUMN recompute_result.updated_time IS '更新时间';
|
||||
COMMENT ON COLUMN recompute_result.reserved_json IS '预留JSON字段';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fst_stash (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
fst_versions VARCHAR(255) NOT NULL,
|
||||
content JSONB,
|
||||
created_time TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
updated_time TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
CONSTRAINT fk_fst_stash_version FOREIGN KEY (fst_versions) REFERENCES version_list (version) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE fst_stash IS 'FST 暂存表';
|
||||
COMMENT ON COLUMN fst_stash.id IS '主键';
|
||||
COMMENT ON COLUMN fst_stash.fst_versions IS '关联 version_list.version';
|
||||
COMMENT ON COLUMN fst_stash.content IS '暂存内容 (JSON)';
|
||||
COMMENT ON COLUMN fst_stash.created_time IS '创建时间';
|
||||
COMMENT ON COLUMN fst_stash.updated_time IS '更新时间';
|
||||
|
||||
-- ======================================================
|
||||
-- 审计日志表
|
||||
-- ======================================================
|
||||
CREATE TABLE IF NOT EXISTS ops_history (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
table_name REGCLASS NOT NULL,
|
||||
row_pk TEXT NOT NULL,
|
||||
column_name TEXT NOT NULL,
|
||||
old_value TEXT,
|
||||
new_value TEXT,
|
||||
changed_by NAME NOT NULL DEFAULT CURRENT_USER,
|
||||
changed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
comment TEXT,
|
||||
app_module TEXT,
|
||||
trace_id TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ops_history_changed_at ON ops_history (changed_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_ops_history_table_name ON ops_history (table_name);
|
||||
COMMENT ON TABLE ops_history IS '通用审计日志：记录 DML 变更';
|
||||
COMMENT ON COLUMN ops_history.id IS '审计日志自增主键';
|
||||
COMMENT ON COLUMN ops_history.table_name IS '被审计表（系统元类型 REGCLASS）';
|
||||
COMMENT ON COLUMN ops_history.row_pk IS '行主键值；批量更新时形如 *col*rows*';
|
||||
COMMENT ON COLUMN ops_history.column_name IS '被修改列名；批量更新时为 *batch*';
|
||||
COMMENT ON COLUMN ops_history.old_value IS '旧值';
|
||||
COMMENT ON COLUMN ops_history.new_value IS '新值';
|
||||
COMMENT ON COLUMN ops_history.changed_by IS '数据库用户';
|
||||
COMMENT ON COLUMN ops_history.changed_at IS '变更时间';
|
||||
COMMENT ON COLUMN ops_history.comment IS '可读描述（自动生成或人工补充）';
|
||||
COMMENT ON COLUMN ops_history.app_module IS '应用模块（通过 SET audit.app_module = ''xxx''）';
|
||||
COMMENT ON COLUMN ops_history.trace_id IS '链路追踪 ID（通过 SET audit.trace_id = ''xxx''）';
|
||||
|
||||
-- ======================================================
|
||||
-- ⑤ 索引
|
||||
-- ======================================================
|
||||
CREATE INDEX IF NOT EXISTS idx_project_name ON project(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_list_project_id ON bag_list(project_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_pangu_datetime ON main_pangu(datetime);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_pangu_name ON main_pangu(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_minerva_session_id ON main_minerva(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_minerva_start_ts ON main_minerva(start_ts);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_minerva_end_ts ON main_minerva(end_ts);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_minerva_vin ON main_minerva(vin);
|
||||
CREATE INDEX IF NOT EXISTS idx_secondary_pangu_bag_id ON secondary_pangu(bag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_secondary_minerva_bag_id ON secondary_minerva(bag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_fst_name ON fst(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_topic_list_name ON topic_list(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_topic_bag_id ON bag_topic(bag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_topic_topic_id ON bag_topic(topic_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_tag_list_name ON reserved_tag_list(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_tag_bag_id ON bag_reserved_tag(bag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_tag_tag_id ON bag_reserved_tag(tag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_gt_meta_name ON gt_meta(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_gt_meta_type ON gt_meta(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_gt_gt_id ON bag_gt(gt_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_gt_bag_id ON bag_gt(bag_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_list_project_active
|
||||
ON bag_list(project_id) WHERE is_deleted = FALSE;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_taglist_name_not_deleted
|
||||
ON reserved_tag_list (name) WHERE is_deleted = FALSE;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_list_json_gin
|
||||
ON bag_list USING GIN (reserved_json);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_recompute_result_version ON recompute_result(recompute_version);
|
||||
CREATE INDEX IF NOT EXISTS idx_recompute_result_type ON recompute_result(result_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_recompute_result_status ON recompute_result(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_recompute_result_created_time ON recompute_result(created_time);
|
||||
CREATE INDEX IF NOT EXISTS idx_recompute_result_storage_path ON recompute_result(storage_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_fst_stash_fst_versions ON fst_stash(fst_versions);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_lifecycle_decode_time ON bag_lifecycle USING BRIN (decode_time);
|
||||
CREATE INDEX IF NOT EXISTS idx_lifecycle_mining_time ON bag_lifecycle USING BRIN (mining_time);
|
||||
CREATE INDEX IF NOT EXISTS idx_lifecycle_manual_annotate_time ON bag_lifecycle USING BRIN (manual_annotate_time);
|
||||
|
||||
-- ======================================================
|
||||
-- ⑥ 触发器函数
|
||||
-- ======================================================
|
||||
CREATE OR REPLACE FUNCTION set_update_time() RETURNS TRIGGER AS
|
||||
$$
|
||||
BEGIN
|
||||
NEW.update_time := NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
COMMENT ON FUNCTION set_update_time() IS '通用：更新行时自动刷新 update_time';
|
||||
|
||||
CREATE OR REPLACE FUNCTION audit_changes() RETURNS TRIGGER AS $$
|
||||
DECLARE
|
||||
ch_col text; -- 变化列名（字符串） -- FIX
|
||||
row_cnt int; -- 变化列总数（整数） -- FIX
|
||||
oldv text;
|
||||
newv text;
|
||||
pkval text;
|
||||
tmpl text;
|
||||
BEGIN
|
||||
/* 构造主键字符串 */
|
||||
SELECT string_agg(
|
||||
coalesce(nullif(to_jsonb(NEW)->>attname, ''), 'null'), ',')
|
||||
INTO pkval
|
||||
FROM pg_index i
|
||||
JOIN pg_attribute a ON a.attrelid = i.indrelid
|
||||
AND a.attnum = ANY(i.indkey)
|
||||
WHERE i.indrelid = TG_RELID AND i.indisprimary;
|
||||
|
||||
IF TG_OP = 'UPDATE' THEN
|
||||
/* 1. 变化列总数 */
|
||||
SELECT count(*)::int INTO row_cnt -- FIX：用独立变量
|
||||
FROM pg_attribute
|
||||
WHERE attrelid = TG_RELID
|
||||
AND attnum > 0 AND NOT attisdropped
|
||||
AND to_jsonb(OLD)->>attname IS DISTINCT FROM to_jsonb(NEW)->>attname;
|
||||
|
||||
/* 2. 任意一个变化列的细节 */
|
||||
SELECT attname::text,
|
||||
nullif(to_jsonb(OLD)->>attname, ''),
|
||||
nullif(to_jsonb(NEW)->>attname, '')
|
||||
INTO ch_col, oldv, newv -- FIX：列名放 ch_col
|
||||
FROM pg_attribute
|
||||
WHERE attrelid = TG_RELID
|
||||
AND attnum > 0 AND NOT attisdropped
|
||||
AND to_jsonb(OLD)->>attname IS DISTINCT FROM to_jsonb(NEW)->>attname
|
||||
LIMIT 1;
|
||||
|
||||
IF FOUND THEN
|
||||
tmpl := format('批量更新 %s.%s: %s → %s （共 %s 行）',
|
||||
TG_TABLE_NAME, ch_col, oldv, newv, row_cnt);
|
||||
|
||||
INSERT INTO ops_history(table_name, row_pk, column_name,
|
||||
old_value, new_value, changed_by, comment,
|
||||
app_module, trace_id)
|
||||
VALUES (TG_RELID::REGCLASS,
|
||||
'*'||row_cnt||'rows*', -- FIX：用整数变量
|
||||
'*batch*',
|
||||
oldv,
|
||||
jsonb_build_object('new_val', newv, 'rows', row_cnt), -- FIX
|
||||
CURRENT_USER,
|
||||
tmpl,
|
||||
current_setting('audit.app_module', true),
|
||||
current_setting('audit.trace_id', true));
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
/* 以下逐列记录逻辑未改动，故省略 … */
|
||||
FOR ch_col IN
|
||||
SELECT attname::text
|
||||
FROM pg_attribute
|
||||
WHERE attrelid = TG_RELID
|
||||
AND attnum > 0 AND NOT attisdropped
|
||||
AND to_jsonb(OLD)->>attname IS DISTINCT FROM to_jsonb(NEW)->>attname
|
||||
LOOP
|
||||
oldv := nullif(to_jsonb(OLD)->>ch_col, '');
|
||||
newv := nullif(to_jsonb(NEW)->>ch_col, '');
|
||||
|
||||
tmpl := format('%s %s.%s: %s → %s',
|
||||
TG_OP, TG_TABLE_NAME, ch_col,
|
||||
coalesce(oldv,'(null)'), coalesce(newv,'(null)'));
|
||||
|
||||
INSERT INTO ops_history(table_name, row_pk, column_name,
|
||||
old_value, new_value, changed_by, comment,
|
||||
app_module, trace_id)
|
||||
VALUES (TG_RELID::REGCLASS,
|
||||
pkval,
|
||||
ch_col,
|
||||
oldv,
|
||||
newv,
|
||||
CURRENT_USER,
|
||||
tmpl,
|
||||
current_setting('audit.app_module', true),
|
||||
current_setting('audit.trace_id', true));
|
||||
END LOOP;
|
||||
|
||||
RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- ======================================================
|
||||
-- ⑦ 为每张业务表挂触发器
|
||||
-- ======================================================
|
||||
-- 7.1 自动刷新 update_time
|
||||
DO $$
|
||||
DECLARE
|
||||
t text;
|
||||
BEGIN
|
||||
FOREACH t IN ARRAY ARRAY[
|
||||
'project','bag_list','main_minerva','fst',
|
||||
'topic_list','reserved_tag_list','gt_meta',
|
||||
'bag_lifecycle','main_pangu','joined_pangu','joined_bags',
|
||||
'secondary_pangu','secondary_minerva','geometry_info',
|
||||
'bag_topic','bag_reserved_tag','bag_gt','fst_bag',
|
||||
'fst_stash'
|
||||
] LOOP
|
||||
EXECUTE format($f$
|
||||
DROP TRIGGER IF EXISTS trg_%I_update ON %I;
|
||||
CREATE TRIGGER trg_%I_update
|
||||
BEFORE UPDATE ON %I
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION set_update_time();
|
||||
$f$, t, t, t, t);
|
||||
END LOOP;
|
||||
END $$;
|
||||
|
||||
-- 7.2 审计触发器（所有表）
|
||||
DO $$
|
||||
DECLARE
|
||||
tbl TEXT;
|
||||
BEGIN
|
||||
FOREACH tbl IN ARRAY ARRAY[
|
||||
'project','bag_list','main_minerva','fst',
|
||||
'topic_list','reserved_tag_list','gt_meta',
|
||||
'bag_lifecycle','main_pangu','joined_pangu','joined_bags',
|
||||
'secondary_pangu','secondary_minerva','geometry_info',
|
||||
'bag_topic','bag_reserved_tag','bag_gt','fst_bag',
|
||||
'fst_stash'
|
||||
] LOOP
|
||||
EXECUTE format($f$
|
||||
DROP TRIGGER IF EXISTS trg_audit_%I ON %I;
|
||||
CREATE TRIGGER trg_audit_%I
|
||||
AFTER INSERT OR UPDATE OR DELETE ON %I
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION audit_changes();
|
||||
$f$, tbl, tbl, tbl, tbl);
|
||||
END LOOP;
|
||||
END $$;
|
||||
@@ -0,0 +1,679 @@
|
||||
{
|
||||
"info": {
|
||||
"_postman_id": "a24cba51-2034-4777-a911-85ce7240d1ac",
|
||||
"name": "ROOT API",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
|
||||
"_exporter_id": "17291944"
|
||||
},
|
||||
"item": [
|
||||
{
|
||||
"name": "projects",
|
||||
"item": [
|
||||
{
|
||||
"name": "All projects",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/projects/all",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"projects",
|
||||
"all"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "tags",
|
||||
"item": [
|
||||
{
|
||||
"name": "All tags",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/tags/all",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"tags",
|
||||
"all"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_by_creator",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/tags/creators/admin",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"tags",
|
||||
"creators",
|
||||
"admin"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "tag2bag",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/tags/bags?tags=driving,city&op=and",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"tags",
|
||||
"bags"
|
||||
],
|
||||
"query": [
|
||||
{
|
||||
"key": "tags",
|
||||
"value": "driving,city"
|
||||
},
|
||||
{
|
||||
"key": "op",
|
||||
"value": "and"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "bags",
|
||||
"item": [
|
||||
{
|
||||
"name": "All bags",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/all",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"all"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "bag2topic",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"001.bag\",\"002.bag\"]\n",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/topics",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"topics"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "bag2fst node",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"001.bag\",\"002.bag\"]\n",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/fst/nodes",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"fst",
|
||||
"nodes"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "bag2tags",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"001.bag\",\"002.bag\",\"test21.bag\"]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/tags",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"tags"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "bags_life_cycle",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"bag_names\":[\"001.bag\",\"002.bag\"]}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/life_cycle",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"life_cycle"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_pangu",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"names\":[\"p1.bag\",\"p2.bag\"]}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/pangu",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"pangu"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_minerva",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"session_ids\":[\"p1.bag\",\"p2.bag\"]}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/minerva",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"minerva"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "search pangu",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"name\":\"001.bag\"}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/search/pangu",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"search",
|
||||
"pangu"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_pangu_detail",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"p1.bag\",\"p2.bag\",\"test21.bag\"]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/pangu/detail",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"pangu",
|
||||
"detail"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_minerva_detail",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"p1.bag\",\"p2.bag\"]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/minerva/detail",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"minerva",
|
||||
"detail"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "topics",
|
||||
"item": [
|
||||
{
|
||||
"name": "All topics",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/topics/all",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"topics",
|
||||
"all"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "topic2bag",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/topics/bags/topic1",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"topics",
|
||||
"bags",
|
||||
"topic1"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "geometry",
|
||||
"item": [
|
||||
{
|
||||
"name": "rosbags",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"rosbag_names\":[\"001.bag\"]}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/geometry",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"geometry"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "fst",
|
||||
"item": [
|
||||
{
|
||||
"name": "get_by_bags_fst_node",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"fst1\",\"fst1_fst2_fst3\",\"fst1_fst2\",\"fst1_fst4\"]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/bags/nodes",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"bags",
|
||||
"nodes"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "add_bag_to_fst",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\n {\n \"bag_name\": \"test21.bag\",\n \"nodes\": [\"fst1\", \"fst1_fst2\", \"fst1_fst2_fst3\", \"\"],\n \"start_time\": 5,\n \"end_time\": 15,\n \"comments\":\"test\",\n \"tags\":[\"driving\",\"city\",\"highway\"]\n }\n]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/bags/update",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"bags",
|
||||
"update"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "update_fst_node",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"name\": \"fst-qqq-123\",\n \"parent_name\":\"fst1\"\n}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/update",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"update"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "delete_fst_nold",
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\n {\n \"bag_name\": \"test21.bag\",\n \"nodes\": [\"fst1\", \"fst1_fst2\", \"fst1_fst2_fst3\", \"\"],\n \"start_time\": 5,\n \"end_time\": 15,\n \"comments\":\"test\",\n \"tags\":[\"driving\",\"city\",\"highway\"]\n }\n]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/fst-qqq-123",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"fst-qqq-123"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_by_bags_fst_path",
|
||||
"protocolProfileBehavior": {
|
||||
"disableBodyPruning": true
|
||||
},
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"fst1\",\"fst2\"]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/bags/path/fst1",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"bags",
|
||||
"path",
|
||||
"fst1"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_fst_detail_by_name",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/fst1_fst2_fst3",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"fst1_fst2_fst3"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "print_tree",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/print_tree",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"print_tree"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_all_linked_bags",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/baglist",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"baglist"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"variable": [
|
||||
{
|
||||
"key": "baseURL",
|
||||
"value": "http://127.0.0.1:5232",
|
||||
"type": "default"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
[project]
|
||||
name = "root_db_api"
|
||||
version = "0.5.0"
|
||||
description = "A core API service for database operations."
|
||||
readme = "README.md"
|
||||
authors = [{ name = "Cheng Li", email = "tbd@tbd.com" }]
|
||||
requires-python = ">=3.12"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 核心依赖项 (Direct Dependencies Only)
|
||||
# ----------------------------------------------------
|
||||
dependencies = [
|
||||
# Web 框架 / 服务
|
||||
"flask>=3.1.1",
|
||||
"flasgger==0.9.7b2",
|
||||
"geoalchemy2==0.17.1",
|
||||
"shapely>=2.1.1",
|
||||
# 实用工具 / 缓存 / 监控
|
||||
"numpy==2.3.1",
|
||||
"folium==0.20.0",
|
||||
"tenacity==9.1.2",
|
||||
"tqdm==4.67.1",
|
||||
"pytz==2025.2",
|
||||
"prometheus-client==0.22.1",
|
||||
"flask-caching>=2.3.1",
|
||||
"redis>=6.4.0",
|
||||
# 配置 / 外部服务
|
||||
"cos-python-sdk-v5==1.9.37",
|
||||
"python-dotenv==1.1.1",
|
||||
"requests==2.32.4",
|
||||
"pyyaml==6.0.2",
|
||||
"pydantic==2.11.7",
|
||||
"sqlalchemy==2.0.41",
|
||||
"psycopg2-binary==2.9.10",
|
||||
"openpyxl>=3.1.0",
|
||||
"gunicorn>=23.0.0",
|
||||
"pytest==8.4.1",
|
||||
"pytest-cov==6.2.0",
|
||||
"pytest-mock==3.14.0",
|
||||
"flask-sqlalchemy>=3.1.1",
|
||||
]
|
||||
# ----------------------------------------------------
|
||||
# 标准构建系统配置（使用 Setuptools）
|
||||
# ----------------------------------------------------
|
||||
[build-system]
|
||||
requires = ["setuptools>=61.0.0"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
|
||||
# ----------------------------------------------------
|
||||
# uv 工具配置（用于 Workspaces 或 PyPI 镜像）
|
||||
# ----------------------------------------------------
|
||||
[[tool.uv.index]]
|
||||
url = "https://pypi.mirrors.ustc.edu.cn/simple"
|
||||
default = true
|
||||
@@ -0,0 +1,23 @@
|
||||
from flask import Blueprint
|
||||
|
||||
from .bags import bp as bags_bp
|
||||
from .fst import bp as fst_bp
|
||||
from .geometry import bp as geometry_bp
|
||||
from .gt import bp as gt_bp
|
||||
from .projects import bp as projects_bp
|
||||
from .recompute import bp as recompute_bp
|
||||
from .tags import bp as tags_bp
|
||||
from .topics import bp as topics_bp
|
||||
from .versions import bp as versions_bp
|
||||
|
||||
api_bp = Blueprint("api", __name__)
|
||||
|
||||
api_bp.register_blueprint(bags_bp, url_prefix="/bags")
|
||||
api_bp.register_blueprint(fst_bp, url_prefix="/fst")
|
||||
api_bp.register_blueprint(projects_bp, url_prefix="/projects")
|
||||
api_bp.register_blueprint(tags_bp, url_prefix="/tags")
|
||||
api_bp.register_blueprint(topics_bp, url_prefix="/topics")
|
||||
api_bp.register_blueprint(geometry_bp, url_prefix="/geometry")
|
||||
api_bp.register_blueprint(gt_bp, url_prefix="/gt")
|
||||
api_bp.register_blueprint(recompute_bp, url_prefix="/recompute")
|
||||
api_bp.register_blueprint(versions_bp, url_prefix="/versions")
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,97 @@
|
||||
from flasgger import swag_from
|
||||
from flask import Blueprint, jsonify, request
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.sql import text
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("geometry", __name__, url_prefix="/geometry")
|
||||
|
||||
|
||||
# 根据多个 rosbag 名称获取几何信息
|
||||
@bp.route("/", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["几何数据管理"],
|
||||
"summary": "根据rosbag名称获取几何信息",
|
||||
"description": "根据多个rosbag名称查询对应的几何轨迹信息",
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["rosbag_names"],
|
||||
"properties": {
|
||||
"rosbag_names": {
|
||||
"type": "array",
|
||||
"description": "要查询的rosbag名称列表",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"example": "bag1.bag",
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "返回指定rosbag名称的几何轨迹信息",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"minItems": 3,
|
||||
"maxItems": 3,
|
||||
"items": {"type": "number"},
|
||||
"example": [12.345678, 12.345678, 0.0],
|
||||
"description": "经度、纬度、高度坐标",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {"description": "缺少rosbag_names参数"},
|
||||
"404": {"description": "未找到指定名称的rosbag"},
|
||||
},
|
||||
})
|
||||
def get_geometry_info_by_rosbags():
|
||||
"""Get geometry info by multiple rosbag names""" # 根据多个 rosbag 名称获取几何信息
|
||||
db: Session = SessionLocal()
|
||||
rosbag_names = request.json.get("rosbag_names", [])
|
||||
|
||||
try:
|
||||
if not rosbag_names:
|
||||
return jsonify({}), 400 # 如果没有提供rosbag名称，返回400错误
|
||||
|
||||
# 查询多个rosbag名称对应的几何信息
|
||||
sql = text("""
|
||||
SELECT rosbag_name,
|
||||
ARRAY(
|
||||
SELECT ARRAY[
|
||||
ST_X(pt.geom), ST_Y(pt.geom),
|
||||
COALESCE(ST_Z(pt.geom), 0.0) ]
|
||||
FROM unnest(gnss_downsampled_points) AS pt(geom)
|
||||
) AS lon_lat_alt
|
||||
FROM geometry_info
|
||||
WHERE rosbag_name IN :rosbag_names
|
||||
""")
|
||||
|
||||
# 使用execute并传入多个rosbag名称
|
||||
result = db.execute(sql, {"rosbag_names": tuple(rosbag_names)}).fetchall()
|
||||
|
||||
if not result:
|
||||
return jsonify({}), 404 # 如果未找到结果，返回404
|
||||
|
||||
# 构建以键值形式返回的结果
|
||||
response = {row.rosbag_name: row.lon_lat_alt for row in result}
|
||||
return jsonify(response)
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,403 @@
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from flasgger import swag_from
|
||||
from flask import Blueprint, jsonify, request
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import (
|
||||
get_all_gt_types,
|
||||
get_bags_by_gt_name,
|
||||
link_bags_to_gt_with_stat,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("gt", __name__, url_prefix="/gt")
|
||||
|
||||
|
||||
# ------------------------- 1. 查询所有 GT 种类 -------------------------
|
||||
@bp.route("/types", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["真值数据管理"],
|
||||
"summary": "获取所有 GT 种类",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "GT 类型列表",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"name": {"type": "string"},
|
||||
"type": {"type": "string"},
|
||||
"path": {"type": "string"},
|
||||
"comment": {"type": "string"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
})
|
||||
def list_gt_types():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
data = get_all_gt_types(db)
|
||||
finally:
|
||||
db.close()
|
||||
return jsonify(data)
|
||||
|
||||
|
||||
# ------------------------- 2. 根据 GT 名称查询关联的 bag -------------------------
|
||||
@bp.route("/<string:gt_name>/bags", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["真值数据管理"],
|
||||
"summary": "根据 GT 名称获取关联的 bag 列表",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "gt_name",
|
||||
"in": "path",
|
||||
"type": "string",
|
||||
"required": True,
|
||||
"description": "GT 唯一名称",
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "bag 列表",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"name": {"type": "string"},
|
||||
"tile_id": {"type": "string"},
|
||||
"is_decoded": {"type": "boolean"},
|
||||
"project_id": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"404": {
|
||||
"description": "GT 不存在",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"error": {"type": "string", "example": "GT not found"},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def list_bags_by_gt_name(gt_name: str):
|
||||
db = SessionLocal()
|
||||
try:
|
||||
bags = get_bags_by_gt_name(db, gt_name)
|
||||
if bags is None:
|
||||
return jsonify({"error": "GT not found"}), 404
|
||||
finally:
|
||||
db.close()
|
||||
return jsonify(bags)
|
||||
|
||||
|
||||
# ------------------------- 3. 批量关联 bag 到指定 GT（纯路径+body）-------------------------
|
||||
@bp.route("/<string:gt_name>/bags", methods=["POST"])
|
||||
@swag_from({
|
||||
"openapi": "3.0.0",
|
||||
"tags": ["真值数据管理"],
|
||||
"summary": "批量将 bag 列表关联到指定 GT",
|
||||
"description": "路径传入 GT 名称，body 传入 bag 列表，完成关联后返回统计结果。",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "gt_name",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "GT 唯一名称",
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["bag_names"],
|
||||
"properties": {
|
||||
"bag_names": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "要关联的 bag 名列表",
|
||||
"example": ["a.bag", "b.bag"],
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "关联完成",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"gt_name": {"type": "string"},
|
||||
"total": {"type": "integer"},
|
||||
"succeeded": {"type": "integer"},
|
||||
"skipped": {"type": "integer"},
|
||||
"failed": {"type": "integer"},
|
||||
"details": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"bag_name": {"type": "string"},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"success",
|
||||
"skipped",
|
||||
"failed",
|
||||
],
|
||||
},
|
||||
"message": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {"description": "参数缺失或空列表"},
|
||||
"404": {"description": "GT 不存在"},
|
||||
},
|
||||
})
|
||||
def batch_link_bags_to_gt(gt_name: str):
|
||||
"""批量给 bag 关联指定 GT（仅 gt_name + body）"""
|
||||
body = request.get_json(silent=True)
|
||||
if not body:
|
||||
return jsonify({"error": "JSON body required"}), 400
|
||||
|
||||
bag_names = body.get("bag_names")
|
||||
if not isinstance(bag_names, list) or len(bag_names) == 0:
|
||||
return jsonify({
|
||||
"error": "bag_names(list) is required and cannot be empty"
|
||||
}), 400
|
||||
db = SessionLocal()
|
||||
try:
|
||||
# 复用刚才的 service，去掉 creator 参数
|
||||
stat = link_bags_to_gt_with_stat(db, gt_name, bag_names)
|
||||
if stat is None:
|
||||
return jsonify({"error": "GT not found"}), 404
|
||||
return jsonify(stat), 201
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# ---------- 1. 任务类型列表 ----------
|
||||
@bp.route("/tasks/types", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["真值数据管理"],
|
||||
"summary": "获取任务类型列表",
|
||||
"parameters": [
|
||||
{"name": "repo", "in": "query", "type": "string", "default": "repo"},
|
||||
{"name": "branch", "in": "query", "type": "string", "default": "main"},
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "成功",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"task_types": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
"branch": {"type": "string"},
|
||||
"repo": {"type": "string"},
|
||||
"total_count": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"404": {
|
||||
"description": "分支不存在",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
"error": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {"type": "string"},
|
||||
"message": {"type": "string"},
|
||||
"details": {"type": "object"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def list_task_types():
|
||||
repo = request.args.get("repo", "repo")
|
||||
branch = request.args.get("branch", "main")
|
||||
|
||||
# ------ mock 逻辑 ------
|
||||
if branch == "non-existent-branch":
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": {
|
||||
"code": "BRANCH_NOT_FOUND",
|
||||
"message": "指定的分支不存在",
|
||||
"details": {"branch": branch, "repo": repo},
|
||||
},
|
||||
}), 404
|
||||
|
||||
mock_types = [
|
||||
"tsa",
|
||||
"lane_change",
|
||||
]
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"data": {
|
||||
"task_types": mock_types,
|
||||
"branch": branch,
|
||||
"repo": repo,
|
||||
"total_count": len(mock_types),
|
||||
},
|
||||
}), 200
|
||||
|
||||
|
||||
# ---------- 2. 任务真值路径 ----------
|
||||
@bp.route("/tasks/<string:task_type>/truth-path", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["真值数据管理"],
|
||||
"summary": "根据任务类型获取真值路径",
|
||||
"parameters": [
|
||||
{"name": "task_type", "in": "path", "required": True, "type": "string"},
|
||||
{"name": "repo", "in": "query", "type": "string", "default": "repo"},
|
||||
{"name": "branch", "in": "query", "type": "string", "default": "main"},
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "成功",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"task_type": {"type": "string"},
|
||||
"truth_path": {"type": "string"},
|
||||
"branch": {"type": "string"},
|
||||
"repo": {"type": "string"},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file_size": {"type": "integer"},
|
||||
"last_modified": {"type": "string"},
|
||||
"format": {"type": "string"},
|
||||
"encoding": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"404": {
|
||||
"description": "任务类型不存在",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
"error": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {"type": "string"},
|
||||
"message": {"type": "string"},
|
||||
"details": {"type": "object"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def task_truth_path(task_type: str):
|
||||
repo = request.args.get("repo", "repo")
|
||||
branch = request.args.get("branch", "main")
|
||||
|
||||
# ------ mock 逻辑 ------
|
||||
valid_tasks = [
|
||||
"tsa",
|
||||
"lane_change",
|
||||
]
|
||||
if task_type not in valid_tasks:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": {
|
||||
"code": "TASK_TYPE_NOT_FOUND",
|
||||
"message": "指定的任务类型不存在",
|
||||
"details": {"task_type": task_type, "branch": branch, "repo": repo},
|
||||
},
|
||||
}), 404
|
||||
|
||||
mock_path = f"/data1/MB/shared/gt/{task_type}/"
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"data": {
|
||||
"task_type": task_type,
|
||||
"truth_path": mock_path,
|
||||
"branch": branch,
|
||||
"repo": repo,
|
||||
"metadata": {
|
||||
"file_size": 2048576,
|
||||
"last_modified": "2024-12-01T10:30:00Z",
|
||||
"format": "json",
|
||||
"encoding": "utf-8",
|
||||
},
|
||||
},
|
||||
}), 200
|
||||
@@ -0,0 +1,58 @@
|
||||
from flasgger import Swagger, swag_from
|
||||
from flask import Blueprint, jsonify
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import get_projects
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("projects", __name__, url_prefix="/projects")
|
||||
|
||||
# Initialize Flasgger
|
||||
swagger = Swagger()
|
||||
|
||||
|
||||
# 获取所有项目
|
||||
@bp.route("/all", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["项目管理"],
|
||||
"summary": "获取所有项目",
|
||||
"description": "获取系统中所有项目的列表信息",
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "项目列表",
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"example": 1,
|
||||
"description": "项目ID",
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"example": "自动驾驶数据集",
|
||||
"description": "项目名称",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
500: {
|
||||
"description": "服务器内部错误",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"error": {"type": "string", "example": "发生错误"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def read_projects():
|
||||
"""Get all projects"""
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
projects = get_projects(db) # 现在返回 list[dict]
|
||||
finally:
|
||||
db.close()
|
||||
return jsonify(projects)
|
||||
@@ -0,0 +1,244 @@
|
||||
from flasgger import swag_from
|
||||
from flask import Blueprint, request, jsonify
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import (
|
||||
get_bag_recompute_versions,
|
||||
get_storage_paths_batch,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("recompute", __name__, url_prefix="/recompute")
|
||||
|
||||
|
||||
# 接口1: 批量获取指定bag列表的所有recompute版本
|
||||
@bp.route("/versions", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["重计算管理"],
|
||||
"summary": "批量获取bag的重计算版本",
|
||||
"description": "根据bag名称列表，返回每个bag对应的所有重计算版本",
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["bag_names"],
|
||||
"properties": {
|
||||
"bag_names": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "bag名称列表",
|
||||
"example": ["bag1.bag", "bag2.bag", "bag3.bag"],
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Bag versions mapping",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
"example": {
|
||||
"bag1.bag": ["v1.0", "v1.1", "v2.0"],
|
||||
"bag2.bag": ["v1.0", "v2.0"],
|
||||
"bag3.bag": [],
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad request",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"error": {"type": "string"}},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"error": {"type": "string"}},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def get_bag_versions():
|
||||
"""接口1: 给定指定的rosbag name的列表，返回每个包对应的recompute的全部版本"""
|
||||
try:
|
||||
data = request.get_json()
|
||||
if not data or "bag_names" not in data:
|
||||
return jsonify({"error": "bag_names list required"}), 400
|
||||
|
||||
bag_names = data["bag_names"]
|
||||
if not isinstance(bag_names, list) or len(bag_names) == 0:
|
||||
return jsonify({"error": "bag_names must be a non-empty list"}), 400
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
bag_versions = get_bag_recompute_versions(db, bag_names)
|
||||
return jsonify(bag_versions), 200
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
# 接口2: 批量获取指定bag和版本的文件位置
|
||||
@bp.route("/storage", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["重计算管理"],
|
||||
"summary": "批量获取bag和版本的存储路径",
|
||||
"description": "根据bag名称和重计算版本，返回对应的文件存储位置",
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["requests"],
|
||||
"properties": {
|
||||
"requests": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["bag_name", "recompute_version"],
|
||||
"properties": {
|
||||
"bag_name": {
|
||||
"type": "string",
|
||||
"description": "Bag name",
|
||||
},
|
||||
"recompute_version": {
|
||||
"type": "string",
|
||||
"description": "Recompute version",
|
||||
},
|
||||
},
|
||||
},
|
||||
"description": "List of bag name and version pairs",
|
||||
"example": [
|
||||
{
|
||||
"bag_name": "bag1.bag",
|
||||
"recompute_version": "v1.0",
|
||||
},
|
||||
{
|
||||
"bag_name": "bag2.bag",
|
||||
"recompute_version": "v1.1",
|
||||
},
|
||||
],
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Storage paths for each bag and version pair",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"bag_name": {"type": "string"},
|
||||
"recompute_version": {"type": "string"},
|
||||
"results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"result_id": {"type": "integer"},
|
||||
"bag_name": {"type": "string"},
|
||||
"recompute_version": {"type": "string"},
|
||||
"result_type": {"type": "string"},
|
||||
"storage_path": {"type": "string"},
|
||||
"status": {"type": "string"},
|
||||
"created_time": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
},
|
||||
"reserved_json": {"type": "object"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad request",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"error": {"type": "string"}},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"error": {"type": "string"}},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def get_storage_paths():
|
||||
"""接口2: 给定指定的rosbag name和recompute版本列表，批量查询数据库返回对应的文件位置"""
|
||||
try:
|
||||
data = request.get_json()
|
||||
if not data or "requests" not in data:
|
||||
return jsonify({"error": "requests list required"}), 400
|
||||
|
||||
requests = data["requests"]
|
||||
if not isinstance(requests, list) or len(requests) == 0:
|
||||
return jsonify({"error": "requests must be a non-empty list"}), 400
|
||||
|
||||
# 验证每个请求的格式
|
||||
for req in requests:
|
||||
if (
|
||||
not isinstance(req, dict)
|
||||
or "bag_name" not in req
|
||||
or "recompute_version" not in req
|
||||
):
|
||||
return jsonify({
|
||||
"error": "Each request must contain bag_name and recompute_version"
|
||||
}), 400
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
results = get_storage_paths_batch(db, requests)
|
||||
return jsonify(results), 200
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
@@ -0,0 +1,274 @@
|
||||
from flasgger import Swagger, swag_from
|
||||
from flask import Blueprint, jsonify, request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import (
|
||||
get_tags,
|
||||
get_tags_by_creator,
|
||||
query_bags_by_tags,
|
||||
apply_reserved_tag_with_creator,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("tags", __name__, url_prefix="/tags")
|
||||
|
||||
# Initialize Flasgger
|
||||
swagger = Swagger()
|
||||
|
||||
|
||||
# Get all tags
|
||||
@bp.route("/all", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["标签管理"],
|
||||
"summary": "获取所有标签",
|
||||
"description": "获取所有标签，包含名称、类型和创建者信息",
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "标签列表",
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"example": "数据质量",
|
||||
"description": "标签名称",
|
||||
},
|
||||
"type": {
|
||||
"type": "string",
|
||||
"example": "质量",
|
||||
"description": "标签类型",
|
||||
},
|
||||
"creator": {
|
||||
"type": "string",
|
||||
"example": "admin@example.com",
|
||||
"description": "创建者",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
})
|
||||
def read_tags():
|
||||
"""Fetch all tags""" # Get all tags
|
||||
db: Session = SessionLocal()
|
||||
tags = get_tags(db)
|
||||
db.close()
|
||||
return jsonify(tags)
|
||||
|
||||
|
||||
@bp.route("/bags", methods=["GET"])
|
||||
@swag_from({
|
||||
"openapi": "3.0.0",
|
||||
"tags": ["标签管理"],
|
||||
"summary": "根据标签名称获取bag文件",
|
||||
"description": "支持交集(and)或并集(or)查询rosbag",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "tags",
|
||||
"in": "query",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "逗号分隔的标签名，如 'A,B,C'",
|
||||
},
|
||||
{
|
||||
"name": "op",
|
||||
"in": "query",
|
||||
"required": False,
|
||||
"schema": {"type": "string", "enum": ["and", "or"], "default": "and"},
|
||||
"description": "and=交集，or=并集",
|
||||
},
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "bag文件列表",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"bags": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"name": {"type": "string"},
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {"description": "参数错误"},
|
||||
},
|
||||
})
|
||||
def get_bags_by_tags():
|
||||
tag_str = request.args.get("tags", "").strip()
|
||||
op = request.args.get("op", "and").strip().lower()
|
||||
|
||||
if not tag_str:
|
||||
return jsonify({"bags": []}), 400
|
||||
tag_list = [t.strip() for t in tag_str.split(",") if t.strip()]
|
||||
if op not in {"and", "or"}:
|
||||
return jsonify({"bags": []}), 400
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
bags = query_bags_by_tags(db, tag_list=tag_list, op=op)
|
||||
return jsonify({"bags": bags})
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# Get tags by creator
|
||||
@bp.route("/creators/<string:creator>", methods=["GET"])
|
||||
@swag_from({
|
||||
"openapi": "3.0.0", # 新增
|
||||
"tags": ["标签管理"],
|
||||
"summary": "根据创建者获取标签",
|
||||
"description": "获取指定创建者创建的所有标签",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "creator",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "创建者标识符（如用户名或邮箱）",
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "List of tag names",
|
||||
"content": { # 新增：用 content 包装
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": {"type": "string", "example": "Sales"},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"404": {"description": "No tags found for the specified creator"},
|
||||
},
|
||||
})
|
||||
def get_tags_by_creator_name(creator):
|
||||
"""Fetch tags by creator""" # Get tags by creator
|
||||
db: Session = SessionLocal()
|
||||
tags = get_tags_by_creator(db, creator=creator)
|
||||
db.close()
|
||||
return jsonify({
|
||||
"tags": [t["name"] for t in tags]
|
||||
}) # Changed from "creators" to "tags" for clarity
|
||||
|
||||
|
||||
# --------------- 5. 批量打标签（含 creator 参数）----------------
|
||||
@bp.route("/<string:tag_name>/bags", methods=["POST"])
|
||||
@swag_from({
|
||||
"openapi": "3.0.0",
|
||||
"tags": ["标签管理"],
|
||||
"summary": "批量给 bag 打指定标签（含 creator）",
|
||||
"description": "路径传入标签名，body 传入 bag 列表，query 传入 creator，完成打标签后返回统计结果。",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "tag_name",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "标签名称",
|
||||
},
|
||||
{
|
||||
"name": "creator",
|
||||
"in": "query",
|
||||
"required": False,
|
||||
"schema": {"type": "string", "default": "system"},
|
||||
"description": "操作人标识（如邮箱）",
|
||||
},
|
||||
],
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["bag_names"],
|
||||
"properties": {
|
||||
"bag_names": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "待打标签的 bag 名列表",
|
||||
"example": ["a.bag", "b.bag"],
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "打标签完成",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tag_name": {"type": "string"},
|
||||
"creator": {"type": "string"},
|
||||
"total": {"type": "integer"},
|
||||
"succeeded": {"type": "integer"},
|
||||
"skipped": {"type": "integer"},
|
||||
"failed": {"type": "integer"},
|
||||
"details": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"bag_name": {"type": "string"},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"success",
|
||||
"skipped",
|
||||
"failed",
|
||||
],
|
||||
},
|
||||
"message": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {"description": "参数缺失或空列表"},
|
||||
},
|
||||
})
|
||||
def batch_tag_bags_with_creator(tag_name: str):
|
||||
"""批量给 bag 打指定标签（含 creator 参数）"""
|
||||
creator = request.args.get("creator", "system").strip()
|
||||
body = request.get_json(silent=True)
|
||||
if not body:
|
||||
return jsonify({"error": "JSON body required"}), 400
|
||||
|
||||
bag_names = body.get("bag_names")
|
||||
if not isinstance(bag_names, list) or len(bag_names) == 0:
|
||||
return jsonify({
|
||||
"error": "bag_names(list) is required and cannot be empty"
|
||||
}), 400
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
stat = apply_reserved_tag_with_creator(db, tag_name, bag_names, creator)
|
||||
return jsonify(stat), 201
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,94 @@
|
||||
from flasgger import Swagger, swag_from
|
||||
from flask import jsonify, Blueprint
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import (
|
||||
get_all_topics,
|
||||
get_bags_by_topic_name,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("topics", __name__, url_prefix="/topics")
|
||||
|
||||
# Initialize Flasgger
|
||||
swagger = Swagger()
|
||||
|
||||
|
||||
# Get all topics
|
||||
@bp.route("/all", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["主题管理"],
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "获取所有主题列表",
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"example": "/camera/front/image_raw",
|
||||
"description": "主题名称",
|
||||
},
|
||||
"type": {
|
||||
"type": "string",
|
||||
"example": "sensor_msgs/Image",
|
||||
"description": "消息类型",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
})
|
||||
def read_topics():
|
||||
"""Fetch all topics""" # Get all topics
|
||||
db: Session = SessionLocal()
|
||||
topics = get_all_topics(db)
|
||||
db.close()
|
||||
return jsonify(topics)
|
||||
|
||||
|
||||
# Get bags by topic name
|
||||
@bp.route("/bags/<path:topic_name>", methods=["GET"])
|
||||
@swag_from({
|
||||
"openapi": "3.0.0", # 新增
|
||||
"tags": ["主题管理"],
|
||||
"summary": "Fetch bags by topic name",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "topic_name",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "The name of the topic",
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Get bags related to the specified topic",
|
||||
"content": { # 新增：用 content 包装
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "array",
|
||||
"items": {"type": "string", "example": "Bag Name"},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"404": {"description": "No related topic found"},
|
||||
},
|
||||
})
|
||||
def get_bags_by_topic(topic_name):
|
||||
"""Fetch bags by topic name""" # Get bags by topic name
|
||||
db: Session = SessionLocal()
|
||||
topic_name = "/" + topic_name
|
||||
bags = get_bags_by_topic_name(db, topic_name=topic_name)
|
||||
db.close()
|
||||
return jsonify({"name": [b["name"] for b in bags]})
|
||||
@@ -0,0 +1,618 @@
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
import threading
|
||||
from typing import Any, Dict, List
|
||||
import uuid
|
||||
|
||||
from flasgger import swag_from
|
||||
from flask import Blueprint, Response, jsonify, request
|
||||
from sqlalchemy.orm import Session
|
||||
from openpyxl import Workbook
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import (
|
||||
create_version,
|
||||
list_versions,
|
||||
update_version_status,
|
||||
save_version_snapshot,
|
||||
get_version_snapshot,
|
||||
delete_version,
|
||||
get_fst_stash,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.feishu_bitable_sdk import (
|
||||
create_bitable_for_version,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.feishu_api_constants import (
|
||||
ENABLE_FEISHU_SYNC,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.models import VersionList
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("versions", __name__, url_prefix="/versions")
|
||||
|
||||
ACTIVATE_TASKS: Dict[str, Dict[str, Any]] = {}
|
||||
ACTIVATE_TASKS_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _activate_now_iso() -> str:
|
||||
return datetime.utcnow().isoformat()
|
||||
|
||||
|
||||
def _put_activate_task(task_id: str, task: Dict[str, Any]) -> None:
|
||||
with ACTIVATE_TASKS_LOCK:
|
||||
ACTIVATE_TASKS[task_id] = task
|
||||
|
||||
|
||||
def _patch_activate_task(task_id: str, patch: Dict[str, Any]) -> None:
|
||||
with ACTIVATE_TASKS_LOCK:
|
||||
current = dict(ACTIVATE_TASKS.get(task_id) or {})
|
||||
current.update(patch)
|
||||
current["updated_at"] = _activate_now_iso()
|
||||
ACTIVATE_TASKS[task_id] = current
|
||||
|
||||
|
||||
def _get_activate_task(task_id: str) -> Dict[str, Any]:
|
||||
with ACTIVATE_TASKS_LOCK:
|
||||
task = ACTIVATE_TASKS.get(task_id) or {}
|
||||
return dict(task)
|
||||
|
||||
|
||||
def _run_activate_task(task_id: str, version_id: int, content: Any) -> None:
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
_patch_activate_task(task_id, {"status": "running"})
|
||||
|
||||
version_obj = db.get(VersionList, version_id)
|
||||
if version_obj is None:
|
||||
_patch_activate_task(task_id, {
|
||||
"status": "failed",
|
||||
"error": "version not found",
|
||||
})
|
||||
return
|
||||
|
||||
# 当后端飞书同步开关关闭时，只生效版本，不触发外部飞书写入。
|
||||
if not ENABLE_FEISHU_SYNC:
|
||||
version_obj.feishu_sync_status = "unsynced"
|
||||
version_obj.status = "active"
|
||||
db.commit()
|
||||
|
||||
feishu_result = {
|
||||
"skipped": True,
|
||||
"reason": "feishu_sync_disabled",
|
||||
}
|
||||
_patch_activate_task(task_id, {
|
||||
"status": "completed",
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"result": {
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"status": version_obj.status,
|
||||
"feishu_sync_status": version_obj.feishu_sync_status,
|
||||
"feishu_result": feishu_result,
|
||||
},
|
||||
})
|
||||
return
|
||||
|
||||
version_obj.feishu_sync_status = "syncing"
|
||||
db.commit()
|
||||
|
||||
feishu_result = create_bitable_for_version(version_obj.version, content)
|
||||
|
||||
version_obj.feishu_sync_status = "synced"
|
||||
version_obj.status = "active"
|
||||
db.commit()
|
||||
|
||||
_patch_activate_task(task_id, {
|
||||
"status": "completed",
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"result": {
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"status": version_obj.status,
|
||||
"feishu_sync_status": version_obj.feishu_sync_status,
|
||||
"feishu_result": feishu_result,
|
||||
},
|
||||
})
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
version_obj = db.get(VersionList, version_id)
|
||||
if version_obj is not None:
|
||||
version_obj.feishu_sync_status = "failed"
|
||||
db.commit()
|
||||
_patch_activate_task(task_id, {
|
||||
"status": "failed",
|
||||
"error": str(exc),
|
||||
})
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "获取版本列表",
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "版本列表",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"version": {"type": "string"},
|
||||
"type": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"release_date": {"type": "string"},
|
||||
"created_time": {"type": "string"},
|
||||
"update_time": {"type": "string"},
|
||||
"status": {"type": "string"},
|
||||
"feishu_sync_status": {"type": "string"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
})
|
||||
def get_version_list():
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
data = list_versions(db)
|
||||
return jsonify(data)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "新增版本",
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["version"],
|
||||
"properties": {
|
||||
"version": {"type": "string", "example": "v1.0.0"},
|
||||
"type": {"type": "string", "example": "software"},
|
||||
"description": {"type": "string"},
|
||||
"release_date": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
},
|
||||
"status": {"type": "string", "example": "available"},
|
||||
"feishu_sync_status": {
|
||||
"type": "string",
|
||||
"example": "unsynced",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
200: {"description": "创建成功"},
|
||||
400: {"description": "请求参数错误"},
|
||||
},
|
||||
})
|
||||
def create_version_item():
|
||||
payload = request.get_json(silent=True) or {}
|
||||
version = payload.get("version")
|
||||
if not version:
|
||||
return jsonify({"error": "version 必填"}), 400
|
||||
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
exists = (
|
||||
db.query(VersionList.id)
|
||||
.filter(VersionList.version == version)
|
||||
.first()
|
||||
)
|
||||
if exists:
|
||||
return jsonify({"message": "version already exists"}), 409
|
||||
|
||||
obj = create_version(
|
||||
db,
|
||||
version=version,
|
||||
type=payload.get("type"),
|
||||
description=payload.get("description"),
|
||||
release_date=payload.get("release_date"),
|
||||
status=payload.get("status"),
|
||||
feishu_sync_status=payload.get("feishu_sync_status"),
|
||||
)
|
||||
db.commit()
|
||||
return jsonify({
|
||||
"id": obj.id,
|
||||
"version": obj.version,
|
||||
"type": obj.type,
|
||||
"description": obj.description,
|
||||
"release_date": obj.release_date,
|
||||
"created_time": obj.created_time,
|
||||
"update_time": obj.update_time,
|
||||
"status": obj.status,
|
||||
"feishu_sync_status": obj.feishu_sync_status,
|
||||
})
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("/export", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "按版本导出 FST/Bag 快照",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "version_id",
|
||||
"in": "query",
|
||||
"required": False,
|
||||
"schema": {"type": "string"},
|
||||
"description": "版本号，对应 version_list.version；不传则导出版本列表",
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "Excel 文件",
|
||||
"content": {
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": {
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"format": "binary",
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
400: {"description": "参数错误"},
|
||||
404: {"description": "版本不存在"},
|
||||
},
|
||||
})
|
||||
def export_versions():
|
||||
version_token = request.args.get("version_id")
|
||||
if not version_token:
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
data = list_versions(db)
|
||||
finally:
|
||||
db.close()
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "Versions"
|
||||
ws.append(["Version Name", "Create Time", "Update Time", "Status"])
|
||||
for item in data:
|
||||
created = item.get("created_time")
|
||||
updated = item.get("update_time")
|
||||
created_str = created.isoformat() if isinstance(created, datetime) else str(created) if created is not None else ""
|
||||
updated_str = updated.isoformat() if isinstance(updated, datetime) else str(updated) if updated is not None else ""
|
||||
ws.append([
|
||||
item.get("version") or "",
|
||||
created_str,
|
||||
updated_str,
|
||||
item.get("status") or "",
|
||||
])
|
||||
stream = BytesIO()
|
||||
wb.save(stream)
|
||||
stream.seek(0)
|
||||
filename = f"versions_{datetime.now().strftime('%Y%m%d%H%M%S')}.xlsx"
|
||||
return Response(
|
||||
stream.getvalue(),
|
||||
mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
headers={"Content-Disposition": f"attachment; filename={filename}"},
|
||||
)
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
version_obj = (
|
||||
db.query(VersionList)
|
||||
.filter(VersionList.version == version_token)
|
||||
.first()
|
||||
)
|
||||
if version_obj is None and version_token.isdigit():
|
||||
version_obj = db.get(VersionList, int(version_token))
|
||||
if version_obj is None:
|
||||
return jsonify({"error": "版本不存在"}), 404
|
||||
version_pk = version_obj.id
|
||||
version_str = version_obj.version
|
||||
snapshot = get_version_snapshot(db, version_pk)
|
||||
if snapshot is None:
|
||||
snapshot = save_version_snapshot(db, version_pk)
|
||||
if snapshot is None:
|
||||
return jsonify({"error": "版本不存在"}), 404
|
||||
finally:
|
||||
db.close()
|
||||
tree = snapshot or []
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "FST Tree Snapshot"
|
||||
ws.append(["Version", "Node ID", "Label", "Level", "Parent Node ID"])
|
||||
|
||||
def _walk(nodes, parent_id=None):
|
||||
for node in nodes:
|
||||
node_id = node.get("id")
|
||||
label = node.get("label")
|
||||
level = node.get("level")
|
||||
ws.append([
|
||||
version_str,
|
||||
node_id or "",
|
||||
label or "",
|
||||
level if level is not None else "",
|
||||
parent_id or "",
|
||||
])
|
||||
children = node.get("children") or []
|
||||
if children:
|
||||
_walk(children, node_id or parent_id)
|
||||
|
||||
_walk(tree)
|
||||
stream = BytesIO()
|
||||
wb.save(stream)
|
||||
stream.seek(0)
|
||||
filename = f"version_{version_str}_fst_snapshot_{datetime.now().strftime('%Y%m%d%H%M%S')}.xlsx"
|
||||
return Response(
|
||||
stream.getvalue(),
|
||||
mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
headers={"Content-Disposition": f"attachment; filename={filename}"},
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/<int:version_id>/snapshot", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "生成并保存指定版本的 FST/Bag 快照",
|
||||
"responses": {
|
||||
200: {"description": "生成成功"},
|
||||
404: {"description": "版本不存在"},
|
||||
},
|
||||
})
|
||||
def create_version_snapshot(version_id: int):
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
snapshot = save_version_snapshot(db, version_id)
|
||||
if snapshot is None:
|
||||
return jsonify({"error": "版本不存在"}), 404
|
||||
return jsonify({"version_id": version_id, "snapshot": snapshot}), 200
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("/<int:version_id>/status", methods=["PATCH"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "更新版本状态与描述",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "version_id",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"type": "integer",
|
||||
"description": "版本 ID",
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["status"],
|
||||
"properties": {
|
||||
"status": {
|
||||
"type": "string",
|
||||
"example": "已生效",
|
||||
},
|
||||
"description": {
|
||||
"type": "string",
|
||||
"example": "版本说明，可选",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
200: {"description": "更新成功"},
|
||||
400: {"description": "请求参数错误"},
|
||||
404: {"description": "未找到版本"},
|
||||
},
|
||||
})
|
||||
def update_version_status_api(version_id: int):
|
||||
payload = request.get_json(silent=True) or {}
|
||||
status = payload.get("status")
|
||||
if not status:
|
||||
return jsonify({"error": "status 必填"}), 400
|
||||
description = payload.get("description")
|
||||
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
obj = update_version_status(
|
||||
db,
|
||||
version_id=version_id,
|
||||
status=status,
|
||||
description=description,
|
||||
)
|
||||
if obj is None:
|
||||
return jsonify({"error": "version not found"}), 404
|
||||
db.commit()
|
||||
return jsonify({
|
||||
"id": obj.id,
|
||||
"version": obj.version,
|
||||
"status": obj.status,
|
||||
"description": obj.description,
|
||||
})
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("/<int:version_id>", methods=["DELETE"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "删除指定版本",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "version_id",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "integer"},
|
||||
"description": "版本 ID（version_list.id）",
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "删除成功",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
404: {
|
||||
"description": "未找到版本",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"error": {"type": "string"},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def delete_version_api(version_id: int):
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
ok = delete_version(db, version_id)
|
||||
if not ok:
|
||||
return jsonify({"error": "version not found"}), 404
|
||||
db.commit()
|
||||
return jsonify({"success": True}), 200
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("/stash/<int:version_id>/activate", methods=["POST"])
|
||||
@bp.route("/fst/stash/activate/<int:version_id>", methods=["POST"])
|
||||
@bp.route("/<int:version_id>/activate", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "生效版本并创建飞书多维表格（fst/stash 命名空间）",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "version_id",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "integer"},
|
||||
"description": "版本 ID（version_list.id）",
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": False,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {
|
||||
"type": "object",
|
||||
"description": "可选，直接上传的 JSON；不传则读取 fst_stash.content",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
200: {"description": "生效并创建飞书多维表格成功"},
|
||||
400: {"description": "参数错误"},
|
||||
404: {"description": "版本或 JSON 不存在"},
|
||||
500: {"description": "飞书调用或服务异常"},
|
||||
},
|
||||
})
|
||||
def activate_version_api(version_id: int):
|
||||
payload = request.get_json(silent=True) or {}
|
||||
input_content = payload.get("content")
|
||||
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
version_obj = db.get(VersionList, version_id)
|
||||
if version_obj is None:
|
||||
return jsonify({"error": "version not found"}), 404
|
||||
|
||||
content = input_content
|
||||
if content is None:
|
||||
stash = get_fst_stash(db, version=version_obj.version)
|
||||
if stash is not None:
|
||||
content = stash.content
|
||||
|
||||
if content is None:
|
||||
return jsonify({"error": "no json content found for this version"}), 404
|
||||
|
||||
task_id = uuid.uuid4().hex
|
||||
now = _activate_now_iso()
|
||||
_put_activate_task(task_id, {
|
||||
"task_id": task_id,
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"status": "queued",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
})
|
||||
|
||||
threading.Thread(
|
||||
target=_run_activate_task,
|
||||
args=(task_id, version_obj.id, content),
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
return jsonify({
|
||||
"accepted": True,
|
||||
"task_id": task_id,
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"status": "queued",
|
||||
}), 202
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("/stash/<int:version_id>/activate/result", methods=["GET"])
|
||||
@bp.route("/<int:version_id>/activate/result", methods=["GET"])
|
||||
def activate_version_result_api(version_id: int):
|
||||
task_id = (request.args.get("task_id") or "").strip()
|
||||
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
version_obj = db.get(VersionList, version_id)
|
||||
if version_obj is None:
|
||||
return jsonify({"error": "version not found"}), 404
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if task_id:
|
||||
task = _get_activate_task(task_id)
|
||||
if not task:
|
||||
return jsonify({"error": "task not found", "task_id": task_id}), 404
|
||||
return jsonify(task), 200
|
||||
|
||||
with ACTIVATE_TASKS_LOCK:
|
||||
candidates = [
|
||||
dict(item)
|
||||
for item in ACTIVATE_TASKS.values()
|
||||
if isinstance(item, dict) and item.get("version_id") == version_id
|
||||
]
|
||||
|
||||
if not candidates:
|
||||
return jsonify({"error": "task not found for version", "version_id": version_id}), 404
|
||||
|
||||
candidates.sort(key=lambda x: str(x.get("updated_at") or ""), reverse=True)
|
||||
return jsonify(candidates[0]), 200
|
||||
@@ -0,0 +1,34 @@
|
||||
# run.py
|
||||
from flasgger import Swagger
|
||||
from flask import Flask, render_template
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api import api_bp
|
||||
|
||||
app = Flask(__name__, template_folder="../templates")
|
||||
|
||||
# 注册蓝图
|
||||
app.register_blueprint(api_bp, url_prefix="/api")
|
||||
|
||||
# Swagger 配置
|
||||
app.config["SWAGGER"] = {
|
||||
"title": "ROOT DB API",
|
||||
"version": "1.0.0",
|
||||
"uiversion": 3,
|
||||
"openapi": "3.0.0",
|
||||
}
|
||||
Swagger(app)
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def hello_world():
|
||||
return "Hello World!"
|
||||
|
||||
|
||||
@app.route("/data-browser")
|
||||
def data_browser():
|
||||
"""数据浏览器前端界面"""
|
||||
return render_template("data_browser.html")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(debug=False, host="0.0.0.0", port=5232)
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Shared Feishu Open API endpoints, common constants, and reusable helpers."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import requests
|
||||
|
||||
WIKI_NODES_URL_TMPL = "https://open.feishu.cn/open-apis/wiki/v2/spaces/{space_id}/nodes"
|
||||
WIKI_GET_NODE_URL = "https://open.feishu.cn/open-apis/wiki/v2/spaces/get_node"
|
||||
WIKI_NODE_URL_TMPL = "https://open.feishu.cn/open-apis/wiki/v2/spaces/{space_id}/nodes/{node_token}"
|
||||
|
||||
DOCX_BLOCKS_URL_TMPL = "https://open.feishu.cn/open-apis/docx/v1/documents/{document_id}/blocks"
|
||||
DOCX_BLOCK_URL_TMPL = "https://open.feishu.cn/open-apis/docx/v1/documents/{document_id}/blocks/{block_id}"
|
||||
DOCX_BLOCK_CHILDREN_URL_TMPL = (
|
||||
"https://open.feishu.cn/open-apis/docx/v1/documents/{document_id}/blocks/{block_id}/children"
|
||||
)
|
||||
|
||||
BITABLE_TABLES_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables"
|
||||
BITABLE_TABLE_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}"
|
||||
BITABLE_FIELDS_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/fields"
|
||||
BITABLE_FIELD_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/fields/{field_id}"
|
||||
BITABLE_RECORDS_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records"
|
||||
BITABLE_RECORD_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/{record_id}"
|
||||
BITABLE_RECORDS_BATCH_CREATE_URL_TMPL = (
|
||||
"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/batch_create"
|
||||
)
|
||||
BITABLE_RECORDS_BATCH_UPDATE_URL_TMPL = (
|
||||
"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/batch_update"
|
||||
)
|
||||
BITABLE_VIEWS_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/views"
|
||||
BITABLE_VIEW_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/views/{view_id}"
|
||||
|
||||
DOC_PAGE_SIZE = 50
|
||||
WIKI_NODE_PAGE_SIZE = 50
|
||||
MAX_PAGE_LOOP = 200
|
||||
RECORD_PAGE_SIZE = 500
|
||||
RECORD_BATCH_SIZE = 200
|
||||
|
||||
TEXT_FIELD_TYPE = 1
|
||||
LINK_FIELD_TYPE = 18
|
||||
PRIMARY_FIELD_TARGET_NAME = "Name"
|
||||
PARENT_LINK_FIELD_NAME = "Parent Record Link"
|
||||
TARGET_TEXT_FIELDS = ["Name", "Scene", "Level", "Describe"]
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool = False) -> bool:
|
||||
raw = os.getenv(name)
|
||||
if raw is None:
|
||||
return default
|
||||
# Accept common true values and tolerate `tru` typo from env settings.
|
||||
return raw.strip().lower() in {"1", "true", "tru", "yes", "on"}
|
||||
|
||||
|
||||
# Backend Feishu sync master switch (default off).
|
||||
ENABLE_FEISHU_SYNC = _env_bool("ENABLE_FEISHU_SYNC", default=False)
|
||||
|
||||
|
||||
def build_auth_headers(tenant_access_token: str) -> Dict[str, str]:
|
||||
return {
|
||||
"Authorization": f"Bearer {tenant_access_token}",
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
}
|
||||
|
||||
|
||||
def raise_for_business_error(result: Dict[str, Any], action: str) -> None:
|
||||
code = result.get("code", -1)
|
||||
if code != 0:
|
||||
msg = result.get("msg", "unknown error")
|
||||
raise RuntimeError(f"{action} failed: code={code}, msg={msg}, raw={json.dumps(result, ensure_ascii=False)}")
|
||||
|
||||
|
||||
def request_json_with_retry(
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
headers: Dict[str, str],
|
||||
action: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
payload: Optional[Dict[str, Any]] = None,
|
||||
timeout: int = 30,
|
||||
max_attempts: int = 3,
|
||||
backoff_seconds: int = 1,
|
||||
) -> Dict[str, Any]:
|
||||
last_error: Optional[str] = None
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
response = requests.request(
|
||||
method=method,
|
||||
url=url,
|
||||
headers=headers,
|
||||
params=params,
|
||||
json=payload,
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
raise_for_business_error(result, action)
|
||||
return result
|
||||
except Exception as exc:
|
||||
last_error = str(exc)
|
||||
|
||||
if attempt < max_attempts:
|
||||
time.sleep(backoff_seconds * attempt)
|
||||
|
||||
raise RuntimeError(f"{action} failed after retries: {last_error}")
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,835 @@
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.feishu_api_constants import (
|
||||
DOC_PAGE_SIZE,
|
||||
MAX_PAGE_LOOP,
|
||||
WIKI_GET_NODE_URL,
|
||||
WIKI_NODES_URL_TMPL,
|
||||
build_auth_headers as _auth_headers,
|
||||
raise_for_business_error as _raise_for_business_error,
|
||||
request_json_with_retry as _shared_request_json_with_retry,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.feishu_doc_bitable_block_sdk import (
|
||||
ensure_bitable_block_for_document,
|
||||
sync_document_bitable_rows,
|
||||
)
|
||||
|
||||
RETRY_MAX_ATTEMPTS = 3
|
||||
RETRY_BACKOFF_SECONDS = 1
|
||||
FST_EDITOR_DOC_TITLE = "Fst_Editor"
|
||||
DOC_TREE_MAX_DEPTH = 5
|
||||
CONTENT_WRAP_KEYS = ("content", "data", "payload", "body", "result")
|
||||
|
||||
|
||||
def _request_json_with_retry(
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
headers: Dict[str, str],
|
||||
action: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
payload: Optional[Dict[str, Any]] = None,
|
||||
timeout: int = 30,
|
||||
) -> Dict[str, Any]:
|
||||
return _shared_request_json_with_retry(
|
||||
method=method,
|
||||
url=url,
|
||||
headers=headers,
|
||||
action=action,
|
||||
params=params,
|
||||
payload=payload,
|
||||
timeout=timeout,
|
||||
max_attempts=RETRY_MAX_ATTEMPTS,
|
||||
backoff_seconds=RETRY_BACKOFF_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def _extract_node_title(node: Dict[str, Any]) -> str:
|
||||
title = node.get("title")
|
||||
if isinstance(title, str):
|
||||
return title
|
||||
|
||||
obj_create_info = node.get("obj_create_info")
|
||||
if isinstance(obj_create_info, dict):
|
||||
inner_title = obj_create_info.get("title")
|
||||
if isinstance(inner_title, str):
|
||||
return inner_title
|
||||
inner_obj_title = obj_create_info.get("obj_title")
|
||||
if isinstance(inner_obj_title, str):
|
||||
return inner_obj_title
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_content_object(content: Any) -> Any:
|
||||
if isinstance(content, str):
|
||||
try:
|
||||
return json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
return content
|
||||
|
||||
|
||||
def _resolve_nodes_owner(parsed: Any, depth: int = 0) -> Optional[Dict[str, Any]]:
|
||||
if depth > 6:
|
||||
return None
|
||||
|
||||
if isinstance(parsed, dict):
|
||||
nodes = parsed.get("nodes")
|
||||
if isinstance(nodes, list):
|
||||
return parsed
|
||||
|
||||
for key in CONTENT_WRAP_KEYS:
|
||||
inner = parsed.get(key)
|
||||
owner = _resolve_nodes_owner(inner, depth + 1)
|
||||
if isinstance(owner, dict):
|
||||
return owner
|
||||
|
||||
if isinstance(parsed, list):
|
||||
return {"nodes": parsed}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _extract_first_level_names(content: Any) -> List[str]:
|
||||
parsed = _parse_content_object(content)
|
||||
owner = _resolve_nodes_owner(parsed)
|
||||
if not isinstance(owner, dict):
|
||||
return []
|
||||
|
||||
nodes = owner.get("nodes")
|
||||
if not isinstance(nodes, list):
|
||||
return []
|
||||
|
||||
has_tree_level = any(isinstance(item, dict) and item.get("treeLevel") is not None for item in nodes)
|
||||
|
||||
names: List[str] = []
|
||||
seen: Set[str] = set()
|
||||
for item in nodes:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if _is_deleted_node(item):
|
||||
continue
|
||||
if has_tree_level and item.get("treeLevel") != 1:
|
||||
continue
|
||||
for key in ("name", "Name", "label", "id"):
|
||||
value = item.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
text = str(value).strip()
|
||||
if text and text not in seen:
|
||||
seen.add(text)
|
||||
names.append(text)
|
||||
break
|
||||
return names
|
||||
|
||||
|
||||
def debug_extract_first_level_names(content: Any) -> Dict[str, Any]:
|
||||
"""Return diagnostic details for first-level name extraction."""
|
||||
parsed = content
|
||||
parse_error: Optional[str] = None
|
||||
if isinstance(content, str):
|
||||
try:
|
||||
parsed = json.loads(content)
|
||||
except json.JSONDecodeError as exc:
|
||||
parse_error = str(exc)
|
||||
parsed = None
|
||||
|
||||
owner = _resolve_nodes_owner(parsed)
|
||||
node_count = 0
|
||||
tree_level_counts: Dict[str, int] = {}
|
||||
root_keys: List[str] = list(parsed.keys()) if isinstance(parsed, dict) else []
|
||||
owner_keys: List[str] = list(owner.keys()) if isinstance(owner, dict) else []
|
||||
if isinstance(owner, dict):
|
||||
nodes = owner.get("nodes")
|
||||
if isinstance(nodes, list):
|
||||
node_count = len(nodes)
|
||||
for item in nodes:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
level = item.get("treeLevel")
|
||||
level_key = "null" if level is None else str(level)
|
||||
tree_level_counts[level_key] = tree_level_counts.get(level_key, 0) + 1
|
||||
|
||||
names = _extract_first_level_names(content)
|
||||
return {
|
||||
"names": names,
|
||||
"name_count": len(names),
|
||||
"node_count": node_count,
|
||||
"tree_level_counts": tree_level_counts,
|
||||
"parse_error": parse_error,
|
||||
"resolved_nodes_owner": bool(isinstance(owner, dict)),
|
||||
"root_keys": root_keys,
|
||||
"owner_keys": owner_keys,
|
||||
}
|
||||
|
||||
|
||||
def _extract_item_title(item: Dict[str, Any]) -> str:
|
||||
for key in ("title", "name", "Name", "label", "id"):
|
||||
value = item.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
text = str(value).strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _is_deleted_node(item: Dict[str, Any]) -> bool:
|
||||
deleted = item.get("deleted")
|
||||
if isinstance(deleted, bool):
|
||||
return deleted
|
||||
if isinstance(deleted, int):
|
||||
return deleted == 1
|
||||
if isinstance(deleted, str):
|
||||
return deleted.strip() in {"1", "true", "True"}
|
||||
return False
|
||||
|
||||
|
||||
def _stringify_value(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, (dict, list)):
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
return str(value)
|
||||
|
||||
|
||||
def _extract_item_describe(item: Dict[str, Any]) -> str:
|
||||
for key in ("Description", "description", "label", "name", "Name"):
|
||||
value = item.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
text = _stringify_value(value).strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_item_scene(item: Dict[str, Any]) -> str:
|
||||
return _stringify_value(item.get("scene")).strip()
|
||||
|
||||
|
||||
def _build_rows_by_first_level(content: Any) -> Dict[str, List[Dict[str, Any]]]:
|
||||
parsed = _parse_content_object(content)
|
||||
owner = _resolve_nodes_owner(parsed)
|
||||
if not isinstance(owner, dict):
|
||||
return {}
|
||||
|
||||
nodes = owner.get("nodes")
|
||||
if not isinstance(nodes, list) or not nodes:
|
||||
return {}
|
||||
|
||||
result: Dict[str, List[Dict[str, Any]]] = {}
|
||||
|
||||
# Tree payload: each level-1 node has nested children.
|
||||
if any(isinstance(node, dict) and isinstance(node.get("children"), list) for node in nodes):
|
||||
def _walk_tree(
|
||||
node: Dict[str, Any],
|
||||
*,
|
||||
root_name: str,
|
||||
parent_name: str,
|
||||
fallback_level: int,
|
||||
) -> None:
|
||||
if _is_deleted_node(node):
|
||||
return
|
||||
|
||||
name = _extract_item_title(node)
|
||||
if not name:
|
||||
return
|
||||
|
||||
level_value = node.get("treeLevel")
|
||||
level_text = _stringify_value(level_value if level_value is not None else fallback_level)
|
||||
result.setdefault(root_name, []).append(
|
||||
{
|
||||
"Name": name,
|
||||
"Scene": _extract_item_scene(node),
|
||||
"Level": level_text,
|
||||
"Describe": _extract_item_describe(node),
|
||||
"__parent_name__": parent_name,
|
||||
}
|
||||
)
|
||||
|
||||
children = node.get("children")
|
||||
if isinstance(children, list):
|
||||
for child in children:
|
||||
if isinstance(child, dict):
|
||||
_walk_tree(
|
||||
child,
|
||||
root_name=root_name,
|
||||
parent_name=name,
|
||||
fallback_level=fallback_level + 1,
|
||||
)
|
||||
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
if _is_deleted_node(node):
|
||||
continue
|
||||
root_name = _extract_item_title(node)
|
||||
if not root_name:
|
||||
continue
|
||||
_walk_tree(
|
||||
node,
|
||||
root_name=root_name,
|
||||
parent_name="",
|
||||
fallback_level=1,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
# Flat payload with treeLevel + parentId.
|
||||
id_to_item: Dict[str, Dict[str, Any]] = {}
|
||||
id_to_name: Dict[str, str] = {}
|
||||
first_level_ids: Set[str] = set()
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
if _is_deleted_node(node):
|
||||
continue
|
||||
node_id = _stringify_value(node.get("id")).strip()
|
||||
node_name = _extract_item_title(node)
|
||||
if not node_id or not node_name:
|
||||
continue
|
||||
id_to_item[node_id] = node
|
||||
id_to_name[node_id] = node_name
|
||||
if node.get("treeLevel") == 1:
|
||||
first_level_ids.add(node_id)
|
||||
|
||||
for node_id, node in id_to_item.items():
|
||||
if _is_deleted_node(node):
|
||||
continue
|
||||
current_id = node_id
|
||||
parent_id = _stringify_value(node.get("parentId")).strip()
|
||||
root_id = current_id if current_id in first_level_ids else ""
|
||||
visited: Set[str] = set()
|
||||
|
||||
while parent_id:
|
||||
if parent_id in visited:
|
||||
break
|
||||
visited.add(parent_id)
|
||||
if parent_id in first_level_ids:
|
||||
root_id = parent_id
|
||||
break
|
||||
parent_node = id_to_item.get(parent_id)
|
||||
if not isinstance(parent_node, dict):
|
||||
break
|
||||
parent_id = _stringify_value(parent_node.get("parentId")).strip()
|
||||
|
||||
if not root_id:
|
||||
continue
|
||||
|
||||
root_name = id_to_name.get(root_id)
|
||||
if not isinstance(root_name, str) or not root_name:
|
||||
continue
|
||||
|
||||
parent_id_text = _stringify_value(node.get("parentId")).strip()
|
||||
level_value = node.get("treeLevel")
|
||||
level_text = _stringify_value(level_value if level_value is not None else "")
|
||||
result.setdefault(root_name, []).append(
|
||||
{
|
||||
# Keep flat payload behavior consistent with feishu_bitable_sdk:
|
||||
# Name uses node id and parent link key uses parent id.
|
||||
"Name": node_id,
|
||||
"Scene": _extract_item_scene(node),
|
||||
"Level": level_text,
|
||||
"Describe": _extract_item_describe(node),
|
||||
"__parent_name__": parent_id_text,
|
||||
}
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _extract_doc_structure(content: Any) -> List[Dict[str, Any]]:
|
||||
parsed = _parse_content_object(content)
|
||||
owner = _resolve_nodes_owner(parsed)
|
||||
if not isinstance(owner, dict):
|
||||
return []
|
||||
|
||||
raw_nodes = owner.get("nodes")
|
||||
if not isinstance(raw_nodes, list):
|
||||
return []
|
||||
|
||||
# Flat payload with treeLevel should only create top-level docs.
|
||||
if all(isinstance(node, dict) and "children" not in node for node in raw_nodes):
|
||||
first_level_names = _extract_first_level_names(owner)
|
||||
return [{"title": title, "children": []} for title in first_level_names]
|
||||
|
||||
def _normalize_nodes(nodes: List[Any], depth: int) -> List[Dict[str, Any]]:
|
||||
if depth >= DOC_TREE_MAX_DEPTH:
|
||||
return []
|
||||
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
title = _extract_item_title(node)
|
||||
if not title:
|
||||
continue
|
||||
|
||||
children = node.get("children")
|
||||
child_docs = _normalize_nodes(children, depth + 1) if isinstance(children, list) else []
|
||||
normalized.append(
|
||||
{
|
||||
"title": title,
|
||||
"children": child_docs,
|
||||
}
|
||||
)
|
||||
return normalized
|
||||
|
||||
return _normalize_nodes(raw_nodes, 0)
|
||||
|
||||
|
||||
def get_wiki_node_info_by_token(tenant_access_token: str, token: str) -> Dict[str, Any]:
|
||||
result = _request_json_with_retry(
|
||||
"GET",
|
||||
WIKI_GET_NODE_URL,
|
||||
headers=_auth_headers(tenant_access_token),
|
||||
params={"token": token},
|
||||
action="get_wiki_node_info_by_token",
|
||||
)
|
||||
node = ((result.get("data") or {}).get("node") or {})
|
||||
if not isinstance(node, dict) or not node:
|
||||
raise RuntimeError(f"get_wiki_node_info_by_token failed: node missing, raw={json.dumps(result, ensure_ascii=False)}")
|
||||
return node
|
||||
|
||||
|
||||
def list_wiki_child_nodes(
|
||||
tenant_access_token: str,
|
||||
*,
|
||||
space_id: str,
|
||||
parent_node_token: Optional[str],
|
||||
) -> List[Dict[str, Any]]:
|
||||
url = WIKI_NODES_URL_TMPL.format(space_id=space_id)
|
||||
headers = _auth_headers(tenant_access_token)
|
||||
page_token = ""
|
||||
items: List[Dict[str, Any]] = []
|
||||
seen_tokens: Set[str] = set()
|
||||
loop_count = 0
|
||||
|
||||
while True:
|
||||
loop_count += 1
|
||||
if loop_count > MAX_PAGE_LOOP:
|
||||
raise RuntimeError("list_wiki_child_nodes exceeded max pagination loops")
|
||||
|
||||
params: Dict[str, Any] = {
|
||||
"page_size": DOC_PAGE_SIZE,
|
||||
}
|
||||
if parent_node_token:
|
||||
params["parent_node_token"] = parent_node_token
|
||||
if page_token:
|
||||
params["page_token"] = page_token
|
||||
|
||||
result = _request_json_with_retry(
|
||||
"GET",
|
||||
url,
|
||||
headers=headers,
|
||||
params=params,
|
||||
action="list_wiki_child_nodes",
|
||||
)
|
||||
data = result.get("data") or {}
|
||||
batch = data.get("items") or []
|
||||
for item in batch:
|
||||
if isinstance(item, dict):
|
||||
items.append(item)
|
||||
|
||||
if not bool(data.get("has_more")):
|
||||
break
|
||||
|
||||
next_page_token = str(data.get("page_token") or "")
|
||||
if not next_page_token or next_page_token == page_token or next_page_token in seen_tokens:
|
||||
break
|
||||
seen_tokens.add(next_page_token)
|
||||
page_token = next_page_token
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def get_all_descendant_nodes(
|
||||
tenant_access_token: str,
|
||||
*,
|
||||
space_id: str,
|
||||
parent_node_token: str,
|
||||
depth: int = 0,
|
||||
max_depth: int = 20,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Recursively list all descendant nodes under a parent node."""
|
||||
if depth > max_depth:
|
||||
raise RuntimeError("get_all_descendant_nodes exceeded max_depth")
|
||||
|
||||
descendants: List[Dict[str, Any]] = []
|
||||
child_nodes = list_wiki_child_nodes(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=parent_node_token,
|
||||
)
|
||||
|
||||
for node in child_nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
|
||||
node_with_depth = dict(node)
|
||||
node_with_depth["depth"] = depth
|
||||
node_with_depth["parent_token"] = parent_node_token
|
||||
descendants.append(node_with_depth)
|
||||
|
||||
child_token = node.get("node_token")
|
||||
if not isinstance(child_token, str) or not child_token:
|
||||
continue
|
||||
|
||||
has_child = bool(node.get("has_child", False))
|
||||
if not has_child:
|
||||
continue
|
||||
|
||||
sub_descendants = get_all_descendant_nodes(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=child_token,
|
||||
depth=depth + 1,
|
||||
max_depth=max_depth,
|
||||
)
|
||||
descendants.extend(sub_descendants)
|
||||
|
||||
return descendants
|
||||
|
||||
|
||||
def create_child_docx(
|
||||
tenant_access_token: str,
|
||||
*,
|
||||
space_id: str,
|
||||
parent_node_token: str,
|
||||
title: str,
|
||||
) -> Dict[str, Any]:
|
||||
url = WIKI_NODES_URL_TMPL.format(space_id=space_id)
|
||||
payload = {
|
||||
"obj_type": "docx",
|
||||
"node_type": "origin",
|
||||
"parent_node_token": parent_node_token,
|
||||
"title": title,
|
||||
}
|
||||
result = _request_json_with_retry(
|
||||
"POST",
|
||||
url,
|
||||
headers=_auth_headers(tenant_access_token),
|
||||
payload=payload,
|
||||
action="create_child_docx",
|
||||
)
|
||||
node = ((result.get("data") or {}).get("node") or {})
|
||||
if not isinstance(node, dict) or not node:
|
||||
raise RuntimeError(f"create_child_docx failed: node missing, raw={json.dumps(result, ensure_ascii=False)}")
|
||||
return node
|
||||
|
||||
|
||||
def _ensure_doc_tree_under_parent(
|
||||
tenant_access_token: str,
|
||||
*,
|
||||
space_id: str,
|
||||
parent_node_token: str,
|
||||
doc_items: List[Dict[str, Any]],
|
||||
current_depth: int = 0,
|
||||
max_depth: int = DOC_TREE_MAX_DEPTH,
|
||||
) -> Dict[str, Any]:
|
||||
if current_depth >= max_depth:
|
||||
return {
|
||||
"created": [],
|
||||
"existing": [],
|
||||
"failed": [],
|
||||
"max_depth_reached": True,
|
||||
}
|
||||
|
||||
direct_children = list_wiki_child_nodes(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=parent_node_token,
|
||||
)
|
||||
direct_title_to_node = {
|
||||
_extract_node_title(node).strip(): node
|
||||
for node in direct_children
|
||||
if isinstance(node, dict) and _extract_node_title(node).strip()
|
||||
}
|
||||
|
||||
created: List[Dict[str, Any]] = []
|
||||
existing: List[Dict[str, Any]] = []
|
||||
failed: List[Dict[str, Any]] = []
|
||||
max_depth_reached = False
|
||||
|
||||
for item in doc_items:
|
||||
title = _extract_item_title(item)
|
||||
if not title:
|
||||
failed.append(
|
||||
{
|
||||
"title": "",
|
||||
"parent_token": parent_node_token,
|
||||
"depth": current_depth,
|
||||
"error": "empty_title",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
target_node = direct_title_to_node.get(title)
|
||||
source = "existing"
|
||||
if not isinstance(target_node, dict):
|
||||
try:
|
||||
target_node = create_child_docx(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=parent_node_token,
|
||||
title=title,
|
||||
)
|
||||
source = "created"
|
||||
direct_title_to_node[title] = target_node
|
||||
except Exception as exc:
|
||||
failed.append(
|
||||
{
|
||||
"title": title,
|
||||
"parent_token": parent_node_token,
|
||||
"depth": current_depth,
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
entry = {
|
||||
"title": title,
|
||||
"node_token": target_node.get("node_token"),
|
||||
"obj_token": target_node.get("obj_token"),
|
||||
"parent_token": parent_node_token,
|
||||
"depth": current_depth,
|
||||
}
|
||||
if source == "created":
|
||||
created.append(entry)
|
||||
else:
|
||||
existing.append(entry)
|
||||
|
||||
children = item.get("children")
|
||||
if isinstance(children, list) and children:
|
||||
child_token = target_node.get("node_token")
|
||||
if isinstance(child_token, str) and child_token:
|
||||
child_result = _ensure_doc_tree_under_parent(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=child_token,
|
||||
doc_items=children,
|
||||
current_depth=current_depth + 1,
|
||||
max_depth=max_depth,
|
||||
)
|
||||
created.extend(child_result.get("created", []))
|
||||
existing.extend(child_result.get("existing", []))
|
||||
failed.extend(child_result.get("failed", []))
|
||||
max_depth_reached = max_depth_reached or bool(child_result.get("max_depth_reached"))
|
||||
|
||||
return {
|
||||
"created": created,
|
||||
"existing": existing,
|
||||
"failed": failed,
|
||||
"max_depth_reached": max_depth_reached,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_doc_token(
|
||||
tenant_access_token: str,
|
||||
node: Dict[str, Any],
|
||||
) -> Optional[str]:
|
||||
obj_token = node.get("obj_token")
|
||||
if isinstance(obj_token, str) and obj_token:
|
||||
return obj_token
|
||||
|
||||
node_token = node.get("node_token")
|
||||
if not isinstance(node_token, str) or not node_token:
|
||||
return None
|
||||
|
||||
try:
|
||||
detail = get_wiki_node_info_by_token(tenant_access_token, node_token)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
detail_obj_token = detail.get("obj_token")
|
||||
if isinstance(detail_obj_token, str) and detail_obj_token:
|
||||
return detail_obj_token
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_fst_editor_parent_node(
|
||||
tenant_access_token: str,
|
||||
*,
|
||||
fst_editor_token: Optional[str],
|
||||
space_id: Optional[str],
|
||||
) -> Dict[str, Any]:
|
||||
if fst_editor_token:
|
||||
try:
|
||||
return get_wiki_node_info_by_token(tenant_access_token, fst_editor_token)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if space_id:
|
||||
root_nodes = list_wiki_child_nodes(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=None,
|
||||
)
|
||||
target = next(
|
||||
(
|
||||
node
|
||||
for node in root_nodes
|
||||
if isinstance(node, dict)
|
||||
and _extract_node_title(node).strip().lower() == FST_EDITOR_DOC_TITLE.lower()
|
||||
and str(node.get("obj_type") or "") in {"docx", "doc"}
|
||||
),
|
||||
None,
|
||||
)
|
||||
if isinstance(target, dict):
|
||||
return target
|
||||
|
||||
raise RuntimeError("cannot resolve fst_editor parent doc node by token/title")
|
||||
|
||||
|
||||
def ensure_child_docs_for_first_level_names(
|
||||
tenant_access_token: str,
|
||||
content: Any,
|
||||
*,
|
||||
fst_editor_token: Optional[str] = None,
|
||||
bitable_app_token: Optional[str] = None,
|
||||
space_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
extraction_debug = debug_extract_first_level_names(content)
|
||||
parent_node = _resolve_fst_editor_parent_node(
|
||||
tenant_access_token,
|
||||
fst_editor_token=fst_editor_token,
|
||||
space_id=space_id,
|
||||
)
|
||||
space_id = str(parent_node.get("space_id") or "")
|
||||
parent_node_token = str(parent_node.get("node_token") or "")
|
||||
if not space_id or not parent_node_token:
|
||||
raise RuntimeError("fst_editor parent node missing space_id or node_token")
|
||||
|
||||
# Only create level-1 documents.
|
||||
target_names = _extract_first_level_names(content)
|
||||
rows_by_first_level = _build_rows_by_first_level(content)
|
||||
if not target_names and rows_by_first_level:
|
||||
target_names = list(rows_by_first_level.keys())
|
||||
|
||||
# Keep deterministic order while allowing fallback names from grouped rows.
|
||||
seen_names: Set[str] = set()
|
||||
merged_target_names: List[str] = []
|
||||
for name in target_names + list(rows_by_first_level.keys()):
|
||||
if not isinstance(name, str):
|
||||
continue
|
||||
clean_name = name.strip()
|
||||
if not clean_name or clean_name in seen_names:
|
||||
continue
|
||||
seen_names.add(clean_name)
|
||||
merged_target_names.append(clean_name)
|
||||
target_names = merged_target_names
|
||||
|
||||
doc_structure = [{"title": title, "children": []} for title in target_names]
|
||||
|
||||
tree_sync_result = _ensure_doc_tree_under_parent(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=parent_node_token,
|
||||
doc_items=doc_structure,
|
||||
current_depth=0,
|
||||
max_depth=DOC_TREE_MAX_DEPTH,
|
||||
)
|
||||
|
||||
existing_children = list_wiki_child_nodes(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=parent_node_token,
|
||||
)
|
||||
# Keep creation scope to direct children only; avoid deep traversal blocking main sync flow.
|
||||
existing_descendants = existing_children
|
||||
existing_title_to_node = {
|
||||
_extract_node_title(node).strip(): node
|
||||
for node in existing_children
|
||||
if isinstance(node, dict) and _extract_node_title(node).strip()
|
||||
}
|
||||
|
||||
synced_title_to_doc_token = {
|
||||
str(item.get("title")).strip(): str(item.get("obj_token") or "")
|
||||
for item in (tree_sync_result.get("created", []) + tree_sync_result.get("existing", []))
|
||||
if isinstance(item, dict)
|
||||
and isinstance(item.get("title"), str)
|
||||
and str(item.get("title")).strip()
|
||||
}
|
||||
|
||||
created_docs: List[Dict[str, Any]] = [
|
||||
item for item in tree_sync_result.get("created", []) if item.get("depth") == 0
|
||||
]
|
||||
existing_docs: List[Dict[str, Any]] = [
|
||||
item for item in tree_sync_result.get("existing", []) if item.get("depth") == 0
|
||||
]
|
||||
insertion_results: List[Dict[str, Any]] = []
|
||||
missing_names: List[str] = []
|
||||
top_level_created_names = {item.get("title") for item in created_docs if isinstance(item.get("title"), str)}
|
||||
|
||||
for name in target_names:
|
||||
existing = existing_title_to_node.get(name)
|
||||
if existing:
|
||||
target_node = existing
|
||||
document_id = _resolve_doc_token(tenant_access_token, target_node)
|
||||
if (not isinstance(document_id, str) or not document_id) and synced_title_to_doc_token.get(name):
|
||||
document_id = synced_title_to_doc_token.get(name)
|
||||
|
||||
if isinstance(document_id, str) and document_id:
|
||||
try:
|
||||
top_bitable_result = ensure_bitable_block_for_document(
|
||||
tenant_access_token,
|
||||
document_id=document_id,
|
||||
)
|
||||
write_result = sync_document_bitable_rows(
|
||||
tenant_access_token,
|
||||
document_id=document_id,
|
||||
rows=rows_by_first_level.get(name, []),
|
||||
bitable_app_token=bitable_app_token,
|
||||
)
|
||||
insertion_results.append({
|
||||
"title": name,
|
||||
"document_id": document_id,
|
||||
"inserted": True,
|
||||
"mode": top_bitable_result.get("mode"),
|
||||
"row_count": len(rows_by_first_level.get(name, [])),
|
||||
"bitable_write_result": write_result,
|
||||
"source": "created" if name in top_level_created_names else "existing",
|
||||
})
|
||||
except Exception as exc:
|
||||
insertion_results.append({
|
||||
"title": name,
|
||||
"document_id": document_id,
|
||||
"inserted": False,
|
||||
"source": "created" if name in top_level_created_names else "existing",
|
||||
"error": str(exc),
|
||||
})
|
||||
else:
|
||||
insertion_results.append({
|
||||
"title": name,
|
||||
"inserted": False,
|
||||
"source": "created" if name in top_level_created_names else "existing",
|
||||
"error": "document_id_missing",
|
||||
})
|
||||
continue
|
||||
|
||||
missing_names.append(name)
|
||||
|
||||
return {
|
||||
"fst_editor": {
|
||||
"title": _extract_node_title(parent_node),
|
||||
"space_id": space_id,
|
||||
"node_token": parent_node_token,
|
||||
"obj_type": parent_node.get("obj_type"),
|
||||
"obj_token": parent_node.get("obj_token"),
|
||||
},
|
||||
"target_names": target_names,
|
||||
"rows_by_first_level_count": {
|
||||
key: len(value)
|
||||
for key, value in rows_by_first_level.items()
|
||||
},
|
||||
"extraction_debug": extraction_debug,
|
||||
"existing_child_count": len(existing_children),
|
||||
"existing_descendant_count": len(existing_descendants),
|
||||
"existing_docs": existing_docs,
|
||||
"created_docs": created_docs,
|
||||
"tree_sync_result": tree_sync_result,
|
||||
"insertion_results": insertion_results,
|
||||
"missing_names": missing_names,
|
||||
}
|
||||
@@ -0,0 +1,514 @@
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
Integer,
|
||||
String,
|
||||
Boolean,
|
||||
ForeignKey,
|
||||
TIMESTAMP,
|
||||
BigInteger,
|
||||
Text,
|
||||
SmallInteger,
|
||||
UniqueConstraint,
|
||||
Index,
|
||||
JSON,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB, ARRAY
|
||||
from sqlalchemy.orm import declarative_base
|
||||
from sqlalchemy.orm import relationship, backref
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
# ============================================================
|
||||
# ① 基础准备
|
||||
# ============================================================
|
||||
|
||||
|
||||
class Project(Base):
|
||||
__tablename__ = "project"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False, unique=True)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
comment="last update time",
|
||||
)
|
||||
|
||||
bags = relationship(
|
||||
"BagList", back_populates="project", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ② 字典 & 主数据（无外部依赖）
|
||||
# ============================================================
|
||||
|
||||
|
||||
class DriveMode(Base):
|
||||
__tablename__ = "drive_mode"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
type = Column(String(100), nullable=False)
|
||||
sub_type = Column(String(100))
|
||||
reserved_json = Column(JSONB)
|
||||
comment = Column(Text)
|
||||
|
||||
__table_args__ = (UniqueConstraint("type", "sub_type"),)
|
||||
|
||||
|
||||
class TopicList(Base):
|
||||
__tablename__ = "topic_list"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False, unique=True)
|
||||
type = Column(String(255))
|
||||
key_data = Column(Text)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
class ReservedTagList(Base):
|
||||
__tablename__ = "reserved_tag_list"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False)
|
||||
type = Column(String(50))
|
||||
creator = Column(String(255))
|
||||
comments = Column(Text)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
is_deleted = Column(Boolean, default=False)
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"uq_reserved_tag_list_name_not_deleted",
|
||||
"name",
|
||||
unique=True,
|
||||
postgresql_where=(is_deleted.is_(False)),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class GTMeta(Base):
|
||||
__tablename__ = "gt_meta"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False)
|
||||
type = Column(String(100), nullable=False)
|
||||
path = Column(Text, nullable=False)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
reserved_json = Column(JSON)
|
||||
comment = Column(Text)
|
||||
|
||||
# 复合唯一约束
|
||||
__table_args__ = (UniqueConstraint("name", "type", name="uq_gt_meta_name_type"),)
|
||||
|
||||
|
||||
class VersionList(Base):
|
||||
__tablename__ = "version_list"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
version = Column(String(255), nullable=False, unique=True)
|
||||
type = Column(String(50))
|
||||
description = Column(Text)
|
||||
release_date = Column(TIMESTAMP(timezone=True))
|
||||
created_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
status = Column(String(50), nullable=False, default="available")
|
||||
feishu_sync_status = Column(String(50), nullable=False, default="unsynced")
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ③ 核心业务表（依赖 ①②）
|
||||
# ============================================================
|
||||
|
||||
|
||||
class BagList(Base):
|
||||
__tablename__ = "bag_list"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False, unique=True)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
project_id = Column(BigInteger, ForeignKey("project.id", ondelete="CASCADE"))
|
||||
tile_id = Column(String(255))
|
||||
is_decoded = Column(Boolean, default=False)
|
||||
is_deleted = Column(Boolean, default=False)
|
||||
od_annotated = Column(Integer)
|
||||
ld_annotated = Column(Integer)
|
||||
fst_indexed = Column(Boolean, default=False)
|
||||
is_active_data = Column(Boolean, default=True)
|
||||
reserved_str = Column(String(255))
|
||||
reserved_json = Column(JSONB)
|
||||
drive_mode = Column(SmallInteger, ForeignKey("drive_mode.id", ondelete="SET NULL"))
|
||||
sw_version = Column(String(255))
|
||||
hw_version = Column(String(255))
|
||||
|
||||
project = relationship("Project", back_populates="bags")
|
||||
lifecycle = relationship(
|
||||
"BagLifecycle",
|
||||
back_populates="bag",
|
||||
uselist=False,
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
secondary_pangu = relationship(
|
||||
"SecondaryPangu",
|
||||
back_populates="bag",
|
||||
uselist=False,
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
secondary_minerva = relationship(
|
||||
"SecondaryMinerva",
|
||||
back_populates="bag",
|
||||
uselist=False,
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
|
||||
class BagLifecycle(Base):
|
||||
__tablename__ = "bag_lifecycle"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_name = Column(
|
||||
String(255),
|
||||
ForeignKey("bag_list.name", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
)
|
||||
collect_time = Column(TIMESTAMP(timezone=True))
|
||||
clone2dev_time = Column(TIMESTAMP(timezone=True))
|
||||
decode_time = Column(TIMESTAMP(timezone=True))
|
||||
mining_time = Column(TIMESTAMP(timezone=True))
|
||||
auto_annotate_time = Column(TIMESTAMP(timezone=True))
|
||||
manual_annotate_time = Column(TIMESTAMP(timezone=True))
|
||||
fst_index_time = Column(TIMESTAMP(timezone=True))
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
bag = relationship("BagList", back_populates="lifecycle")
|
||||
|
||||
|
||||
class MainPangu(Base):
|
||||
__tablename__ = "main_pangu"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False, unique=True)
|
||||
vehicle = Column(String(255))
|
||||
datetime = Column(TIMESTAMP(timezone=True))
|
||||
bag_path = Column(Text)
|
||||
data_path = Column(Text)
|
||||
reserved_str = Column(String(255))
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
class JoinedPangu(Base):
|
||||
__tablename__ = "joined_pangu"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False, unique=True)
|
||||
data_path = Column(Text)
|
||||
reserved_str = Column(String(255))
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
class JoinedBags(Base):
|
||||
__tablename__ = "joined_bags"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
parent_id = Column(BigInteger, nullable=False)
|
||||
child_id = Column(BigInteger, nullable=False, unique=True)
|
||||
reserved_str = Column(String(255))
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
class MainMinerva(Base):
|
||||
__tablename__ = "main_minerva"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
session_id = Column(String(255), unique=True)
|
||||
start_ts = Column(TIMESTAMP(timezone=True))
|
||||
end_ts = Column(TIMESTAMP(timezone=True))
|
||||
length = Column(Integer)
|
||||
datetime = Column(TIMESTAMP(timezone=True))
|
||||
vin = Column(String(255))
|
||||
platform = Column(String(255))
|
||||
mapped = Column(Boolean, default=False)
|
||||
path = Column(Text)
|
||||
converted_path = Column(Text)
|
||||
gt_path = Column(Text)
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ④ 子表 & 关联表（依赖 bag_list.id）
|
||||
# ============================================================
|
||||
|
||||
|
||||
class SecondaryPangu(Base):
|
||||
__tablename__ = "secondary_pangu"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
)
|
||||
lidar_gt_pandar128 = Column(Text)
|
||||
object_lidar_gt_pandar128_manual = Column(Text)
|
||||
lidar_fd_multi_scan_raw = Column(Text)
|
||||
camera_fisheye_left = Column(Text)
|
||||
camera_fisheye_right = Column(Text)
|
||||
camera_front_wide = Column(Text)
|
||||
raw_gps = Column(Text)
|
||||
raw_imu = Column(Text)
|
||||
ego_motion = Column(Text)
|
||||
vehicle_wheel = Column(Text)
|
||||
calibration = Column(Text)
|
||||
sdmap = Column(Text)
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
bag = relationship("BagList", back_populates="secondary_pangu")
|
||||
|
||||
|
||||
class SecondaryMinerva(Base):
|
||||
__tablename__ = "secondary_minerva"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
)
|
||||
lidar_gt_top_p128 = Column(Text)
|
||||
lidar_parking_gt_front_p128 = Column(Text)
|
||||
lidar_parking_gt_left_p128 = Column(Text)
|
||||
lidar_parking_gt_right_p128 = Column(Text)
|
||||
lidar_parking_gt_rear_p128 = Column(Text)
|
||||
camera_fisheye_left_200fov = Column(Text)
|
||||
camera_fisheye_right_200fov = Column(Text)
|
||||
camera_fisheye_rear_200fov = Column(Text)
|
||||
camera_fisheye_front_200fov = Column(Text)
|
||||
camera_front_wide_120fov = Column(Text)
|
||||
camera_front_tele_30fov = Column(Text)
|
||||
camera_rear_right_70fov = Column(Text)
|
||||
camera_rear_left_70fov = Column(Text)
|
||||
raw_gps = Column(Text)
|
||||
raw_imu = Column(Text)
|
||||
ego_motion = Column(Text)
|
||||
calibration = Column(Text)
|
||||
rig = Column(Text)
|
||||
fst_new = Column(Text)
|
||||
fst_old = Column(Text)
|
||||
comments = Column(Text)
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
bag = relationship("BagList", back_populates="secondary_minerva")
|
||||
|
||||
|
||||
class FST(Base):
|
||||
__tablename__ = "fst"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False)
|
||||
parent_id = Column(BigInteger, ForeignKey("fst.id", ondelete="CASCADE"))
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
reserved_json = Column(JSONB)
|
||||
bag_sum = Column(Integer, default=0)
|
||||
is_delete = Column(SmallInteger, default=0)
|
||||
|
||||
parent = relationship(
|
||||
"FST",
|
||||
remote_side=[id],
|
||||
backref=backref("children", cascade="all, delete-orphan"),
|
||||
)
|
||||
|
||||
|
||||
class GeometryInfo(Base):
|
||||
__tablename__ = "geometry_info"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
rosbag_name = Column(String(255), nullable=False)
|
||||
gnss_downsampled_points = Column(ARRAY(String)) # 或 geoalchemy2 Geometry
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
is_overlapped = Column(Boolean, default=False)
|
||||
|
||||
|
||||
class BagTopic(Base):
|
||||
__tablename__ = "bag_topic"
|
||||
__table_args__ = (UniqueConstraint("bag_id", "topic_id"),)
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger, ForeignKey("bag_list.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
topic_id = Column(
|
||||
BigInteger, ForeignKey("topic_list.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
|
||||
bag = relationship("BagList")
|
||||
topic = relationship("TopicList")
|
||||
|
||||
|
||||
class BagReservedTag(Base):
|
||||
__tablename__ = "bag_reserved_tag"
|
||||
__table_args__ = (UniqueConstraint("bag_id", "tag_id"),)
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
tag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("reserved_tag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
bag = relationship("BagList")
|
||||
tag = relationship("ReservedTagList")
|
||||
|
||||
|
||||
class BagGT(Base):
|
||||
__tablename__ = "bag_gt"
|
||||
__table_args__ = (UniqueConstraint("gt_id", "bag_id"),)
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
gt_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("gt_meta.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
comment = Column(Text)
|
||||
|
||||
|
||||
class FSTBag(Base):
|
||||
__tablename__ = "fst_bag"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
fst_node_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("fst.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
fst_node_level = Column(Integer, nullable=False)
|
||||
event_start_time = Column(Integer)
|
||||
event_end_time = Column(Integer)
|
||||
img_url = Column(String(255))
|
||||
video_url = Column(String(255))
|
||||
comments = Column(String(255))
|
||||
|
||||
bag = relationship("BagList", backref="fst_links")
|
||||
fst_node = relationship("FST", backref="bag_links")
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Recompute 结果存储表
|
||||
# ============================================================
|
||||
|
||||
|
||||
class RecomputeResult(Base):
|
||||
__tablename__ = "recompute_result"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
comment="Bag ID",
|
||||
)
|
||||
recompute_version = Column(String(100), nullable=False, comment="Recompute版本")
|
||||
result_type = Column(
|
||||
String(50), nullable=False, comment="结果类型 (如: perception, planning, etc)"
|
||||
)
|
||||
storage_path = Column(String(500), nullable=False, comment="结果存储路径")
|
||||
status = Column(
|
||||
String(50),
|
||||
nullable=False,
|
||||
default="available",
|
||||
comment="结果状态: available, archived, deleted",
|
||||
)
|
||||
created_time = Column(
|
||||
TIMESTAMP(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
comment="Recompute完成时间",
|
||||
)
|
||||
updated_time = Column(
|
||||
TIMESTAMP(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
comment="更新时间",
|
||||
)
|
||||
reserved_json = Column(JSONB, comment="预留JSON字段")
|
||||
|
||||
# 与BagList的关系
|
||||
bag = relationship("BagList", backref="recompute_results")
|
||||
|
||||
# 索引和约束
|
||||
__table_args__ = (
|
||||
# 同一个bag的同一版本同一类型的recompute结果应该是唯一的
|
||||
UniqueConstraint(
|
||||
"bag_id", "recompute_version", "result_type", name="uq_recompute_result"
|
||||
),
|
||||
Index("idx_recompute_result_version", "recompute_version"),
|
||||
Index("idx_recompute_result_type", "result_type"),
|
||||
Index("idx_recompute_result_status", "status"),
|
||||
Index("idx_recompute_result_created_time", "created_time"),
|
||||
Index("idx_recompute_result_storage_path", "storage_path"),
|
||||
)
|
||||
|
||||
|
||||
class FstStash(Base):
|
||||
__tablename__ = "fst_stash"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
fst_versions = Column(
|
||||
String(255),
|
||||
ForeignKey("version_list.version", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
content = Column(JSONB, nullable=True)
|
||||
created_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_time = Column(
|
||||
TIMESTAMP(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
)
|
||||
|
||||
version_rel = relationship("VersionList", backref=backref("fst_stashes", cascade="all, delete-orphan"))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,36 @@
|
||||
# fst_data_pipeline/apps/root_db_api/src/db/memory_cache.py
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
class _MemoryCache:
|
||||
"""极简带过期时间的线程安全内存缓存"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._data: dict[str, tuple[Any, float]] = {}
|
||||
|
||||
def get(self, key: str) -> Optional[Any]:
|
||||
with self._lock:
|
||||
item = self._data.get(key)
|
||||
if item is None:
|
||||
return None
|
||||
value, expire = item
|
||||
if expire > 0 and time.time() > expire:
|
||||
self._data.pop(key, None)
|
||||
return None
|
||||
return value
|
||||
|
||||
def set(self, key: str, value: Any, timeout: int = 600) -> None:
|
||||
expire = time.time() + timeout if timeout > 0 else 0
|
||||
with self._lock:
|
||||
self._data[key] = (value, expire)
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
with self._lock:
|
||||
self._data.pop(key, None)
|
||||
|
||||
|
||||
# 全局单例
|
||||
memory_cache = _MemoryCache()
|
||||
@@ -0,0 +1,30 @@
|
||||
import os
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from contextlib import contextmanager
|
||||
|
||||
DB_USER = os.getenv("DB_USER")
|
||||
DB_PASSWORD = os.getenv("DB_PASSWORD")
|
||||
DB_BASE_URL = os.getenv("DB_BASE_URL")
|
||||
DB_URL = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_BASE_URL}"
|
||||
|
||||
engine = create_engine(
|
||||
DB_URL,
|
||||
pool_size=20,
|
||||
max_overflow=40,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=3600,
|
||||
connect_args={"options": "-c search_path=public"},
|
||||
echo=True,
|
||||
)
|
||||
|
||||
SessionLocal = sessionmaker(bind=engine, expire_on_commit=False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_db_session():
|
||||
session = SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
session.close()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,40 @@
|
||||
README.md
|
||||
pyproject.toml
|
||||
src/__init__.py
|
||||
src/app.py
|
||||
src/api/__init__.py
|
||||
src/api/bags.py
|
||||
src/api/fst.py
|
||||
src/api/geometry.py
|
||||
src/api/gt.py
|
||||
src/api/projects.py
|
||||
src/api/recompute.py
|
||||
src/api/tags.py
|
||||
src/api/topics.py
|
||||
src/api/versions.py
|
||||
src/core/__init__.py
|
||||
src/core/feishu_api_constants.py
|
||||
src/core/feishu_bitable_sdk.py
|
||||
src/core/feishu_doc_bitable_block_sdk.py
|
||||
src/core/feishu_wiki_doc_sdk.py
|
||||
src/core/models.py
|
||||
src/core/service.py
|
||||
src/db/__init__.py
|
||||
src/db/cache.py
|
||||
src/db/connection.py
|
||||
src/root_db_api.egg-info/PKG-INFO
|
||||
src/root_db_api.egg-info/SOURCES.txt
|
||||
src/root_db_api.egg-info/dependency_links.txt
|
||||
src/root_db_api.egg-info/requires.txt
|
||||
src/root_db_api.egg-info/top_level.txt
|
||||
src/test/__init__.py
|
||||
src/test/api/__init__.py
|
||||
src/test/api/test_bags.py
|
||||
src/test/api/test_fst.py
|
||||
src/test/api/test_geometry.py
|
||||
src/test/api/test_projects.py
|
||||
src/test/api/test_recompute.py
|
||||
src/test/api/test_tags.py
|
||||
src/test/api/test_topics.py
|
||||
src/test/service/__init__.py
|
||||
src/test/service/test_service.py
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
flask>=3.1.1
|
||||
flasgger==0.9.7b2
|
||||
geoalchemy2==0.17.1
|
||||
shapely>=2.1.1
|
||||
numpy==2.3.1
|
||||
folium==0.20.0
|
||||
tenacity==9.1.2
|
||||
tqdm==4.67.1
|
||||
pytz==2025.2
|
||||
prometheus-client==0.22.1
|
||||
flask-caching>=2.3.1
|
||||
redis>=6.4.0
|
||||
cos-python-sdk-v5==1.9.37
|
||||
python-dotenv==1.1.1
|
||||
requests==2.32.4
|
||||
pyyaml==6.0.2
|
||||
pydantic==2.11.7
|
||||
sqlalchemy==2.0.41
|
||||
psycopg2-binary==2.9.10
|
||||
openpyxl>=3.1.0
|
||||
gunicorn>=23.0.0
|
||||
pytest==8.4.1
|
||||
pytest-cov==6.2.0
|
||||
pytest-mock==3.14.0
|
||||
flask-sqlalchemy>=3.1.1
|
||||
@@ -0,0 +1,6 @@
|
||||
__init__
|
||||
api
|
||||
app
|
||||
core
|
||||
db
|
||||
test
|
||||
@@ -0,0 +1,343 @@
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api import bags_bp
|
||||
|
||||
|
||||
# ------------------ Flask test client ------------------
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
"""Flask test client with bags blueprint registered."""
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(bags_bp, url_prefix="/bags")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
# ------------------ 单元测试用例 ------------------
|
||||
def test_get_all_bags_200(client, mocker):
|
||||
bag1 = MagicMock()
|
||||
bag1.name = "bag1"
|
||||
bag1.project.name = "ProjectX"
|
||||
|
||||
bag2 = MagicMock()
|
||||
bag2.name = "bag2"
|
||||
bag2.project = None
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_bags",
|
||||
return_value=[bag1, bag2],
|
||||
)
|
||||
|
||||
resp = client.get("/bags/all")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"ProjectX": ["bag1"], "Uncategorized": ["bag2"]}
|
||||
|
||||
|
||||
def test_search_main_pangu_by_names_200(client, mocker):
|
||||
p = MagicMock()
|
||||
p.name = "p1"
|
||||
p.vehicle = "v1"
|
||||
p.datetime = "2021-01-01T00:00:00Z"
|
||||
p.bag_path = "/raw"
|
||||
p.data_path = "/dec"
|
||||
p.reserved_str = "rs"
|
||||
p.reserved_json = {}
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_main_pangu_by_name",
|
||||
return_value=[p],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/pangu", json={"names": ["p1"]})
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["p1"][0]["vehicle"] == "v1"
|
||||
|
||||
|
||||
def test_get_pangu_details_batch_200(client, mocker):
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_pangu_details_by_bag_name",
|
||||
return_value={"detail": "ok"},
|
||||
)
|
||||
|
||||
resp = client.post("/bags/pangu/detail", json=["a.bag", "b.bag"])
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["a.bag"]["detail"] == "ok"
|
||||
|
||||
|
||||
def test_get_pangu_details_batch_400(client):
|
||||
resp = client.post("/bags/pangu/detail", json={"wrong": "format"})
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_search_main_minerva_by_session_ids_200(client, mocker):
|
||||
m = MagicMock()
|
||||
m.session_id = "s1"
|
||||
m.vin = "VIN1"
|
||||
m.start_ts = 0
|
||||
m.end_ts = 1
|
||||
m.platform = "p"
|
||||
m.path = "/p"
|
||||
m.converted_path = "/c"
|
||||
m.gt_path = "/g"
|
||||
m.datetime = "now"
|
||||
m.length = 10
|
||||
m.reserved_json = {}
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_main_minerva_by_session_id",
|
||||
return_value=[m],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/minerva", json={"session_ids": ["s1"]})
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["s1"][0]["vin"] == "VIN1"
|
||||
|
||||
|
||||
def test_get_minerva_details_batch_200(client, mocker):
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_minerva_details_by_bag_name",
|
||||
return_value={"session_id": "s1"},
|
||||
)
|
||||
|
||||
resp = client.post("/bags/minerva/detail", json=["a.bag"])
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["a.bag"]["session_id"] == "s1"
|
||||
|
||||
|
||||
def test_get_topics_batch_200(client, mocker):
|
||||
t = MagicMock()
|
||||
t.name = "t1"
|
||||
t.type = "std_msgs/String"
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_topics_by_bag_name",
|
||||
return_value=[t],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/topics", json=["a.bag"])
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"a.bag": [{"name": "t1", "type": "std_msgs/String"}]}
|
||||
|
||||
|
||||
def test_get_tags_batch_200(client, mocker):
|
||||
tag = MagicMock()
|
||||
tag.name = "tag1"
|
||||
tag.type = "scenario"
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_tags_by_bag_name",
|
||||
return_value=[tag],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/tags", json=["a.bag"])
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["a.bag"][0]["name"] == "tag1"
|
||||
|
||||
|
||||
def test_get_fst_nodes_batch_200(client, mocker):
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_fst_nodes_by_bag_name",
|
||||
return_value=[{"name": "n1", "start": 0, "end": 10}],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/fst/nodes", json=["a.bag"])
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["a.bag"][0]["name"] == "n1"
|
||||
|
||||
|
||||
def test_get_bag_lifecycle_200(client, mocker):
|
||||
lc = MagicMock()
|
||||
lc.bag_name = "a.bag"
|
||||
lc.collect_time = "2021-01-01T00:00:00Z"
|
||||
lc.clone2dev_time = "2021-01-01T01:00:00Z"
|
||||
lc.decode_time = "2021-01-01T02:00:00Z"
|
||||
lc.mining_time = "2021-01-01T03:00:00Z"
|
||||
lc.auto_annotate_time = "2021-01-01T04:00:00Z"
|
||||
lc.manual_annotate_time = "2021-01-01T05:00:00Z"
|
||||
lc.fst_index_time = "2021-01-01T06:00:00Z"
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_bag_lifecycle_by_bag_names",
|
||||
return_value=[lc],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/life_cycle", json={"bag_names": ["a.bag"]})
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["a.bag"]["collect_time"] == "2021-01-01T00:00:00Z"
|
||||
|
||||
|
||||
# ------------------ 400 分支示例 ------------------
|
||||
@pytest.mark.parametrize(
|
||||
"url,bad_body",
|
||||
[
|
||||
("/bags/pangu/detail", {"not": "list"}),
|
||||
("/bags/minerva/detail", {"not": "list"}),
|
||||
("/bags/topics", {"bag_names": "not_list"}),
|
||||
("/bags/tags", {"bag_names": "not_list"}),
|
||||
("/bags/fst/nodes", "not_list"),
|
||||
],
|
||||
)
|
||||
def test_bad_body_400(client, url, bad_body):
|
||||
resp = client.post(url, json=bad_body)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_search_pangu_basic_200(client, mocker):
|
||||
"""测试基本的pangu搜索功能"""
|
||||
p = MagicMock()
|
||||
p.name = "test_bag.bag"
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_main_pangu_by_conditions",
|
||||
return_value=[p],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/search/pangu", json={})
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert isinstance(data, list)
|
||||
assert data == ["test_bag.bag"]
|
||||
|
||||
|
||||
def test_search_pangu_with_all_params_200(client, mocker):
|
||||
"""测试带所有参数的pangu搜索"""
|
||||
p1 = MagicMock()
|
||||
p1.name = "test_bag1.bag"
|
||||
p2 = MagicMock()
|
||||
p2.name = "test_bag2.bag"
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_main_pangu_by_conditions",
|
||||
return_value=[p1, p2],
|
||||
)
|
||||
|
||||
request_data = {
|
||||
"starttime": "20240101",
|
||||
"endtime": "20241231",
|
||||
"vehicle": "test_vehicle",
|
||||
"keyword": "highway",
|
||||
"topics": ["/camera/front", "/lidar/points"],
|
||||
"fst": "highway_scenarios",
|
||||
}
|
||||
|
||||
resp = client.post("/bags/search/pangu", json=request_data)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data == ["test_bag1.bag", "test_bag2.bag"]
|
||||
|
||||
|
||||
def test_search_pangu_invalid_date_format_400(client):
|
||||
"""测试日期格式错误"""
|
||||
request_data = {"starttime": "invalid_date"}
|
||||
|
||||
resp = client.post("/bags/search/pangu", json=request_data)
|
||||
assert resp.status_code == 400
|
||||
data = resp.get_json()
|
||||
assert "error" in data
|
||||
|
||||
|
||||
def test_search_pangu_invalid_topics_400(client):
|
||||
"""测试topics参数格式错误"""
|
||||
request_data = {"topics": "not_a_list"}
|
||||
|
||||
resp = client.post("/bags/search/pangu", json=request_data)
|
||||
assert resp.status_code == 400
|
||||
data = resp.get_json()
|
||||
assert "topics must be an array" in data["error"]
|
||||
|
||||
|
||||
def test_search_pangu_empty_result_200(client, mocker):
|
||||
"""测试搜索无结果"""
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_main_pangu_by_conditions",
|
||||
return_value=[],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/search/pangu", json={"keyword": "nonexistent"})
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data == []
|
||||
|
||||
|
||||
def test_merge_bags_query_specific_bags_200(client, mocker):
|
||||
"""测试查询指定bag的合并关系"""
|
||||
mock_result = {
|
||||
"parent1.bag": ["child1.bag", "child2.bag"],
|
||||
"parent2.bag": ["child3.bag"],
|
||||
}
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.query_existed_joined_bag",
|
||||
return_value=mock_result,
|
||||
)
|
||||
|
||||
request_data = {"bag_names": ["parent1.bag", "parent2.bag"]}
|
||||
|
||||
resp = client.post("/bags/joined/query", json=request_data)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data == mock_result
|
||||
|
||||
|
||||
def test_merge_bags_query_wildcard_200(client, mocker):
|
||||
"""测试通配符查询所有合并关系"""
|
||||
mock_result = {
|
||||
"parent1.bag": ["child1.bag", "child2.bag"],
|
||||
"parent2.bag": ["child3.bag", "child4.bag"],
|
||||
"parent3.bag": ["child5.bag"],
|
||||
}
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.query_all_joined_bags",
|
||||
return_value=mock_result,
|
||||
)
|
||||
|
||||
request_data = {"bag_names": ["*"]}
|
||||
|
||||
resp = client.post("/bags/joined/query", json=request_data)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data == mock_result
|
||||
|
||||
|
||||
def test_merge_bags_query_empty_bag_names_400(client):
|
||||
"""测试空的bag_names参数"""
|
||||
resp = client.post("/bags/joined/query", json={})
|
||||
assert resp.status_code == 400
|
||||
data = resp.get_json()
|
||||
assert "bag_names must be a list" in data["error"]
|
||||
|
||||
|
||||
def test_merge_bags_query_invalid_bag_names_400(client):
|
||||
"""测试无效的bag_names参数格式"""
|
||||
resp = client.post("/bags/joined/query", json={"bag_names": "not_a_list"})
|
||||
assert resp.status_code == 400
|
||||
data = resp.get_json()
|
||||
assert "bag_names must be a list" in data["error"]
|
||||
|
||||
|
||||
def test_merge_bags_query_empty_list_400(client):
|
||||
"""测试空列表的bag_names参数 - 现在应该返回200因为代码已经改变"""
|
||||
resp = client.post("/bags/joined/query", json={"bag_names": []})
|
||||
assert resp.status_code == 400
|
||||
data = resp.get_json()
|
||||
assert "bag_names must be a list" in data["error"]
|
||||
|
||||
|
||||
def test_merge_bags_query_exception_500(client, mocker):
|
||||
"""测试内部异常处理"""
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.query_existed_joined_bag",
|
||||
side_effect=Exception("Database error"),
|
||||
)
|
||||
|
||||
request_data = {"bag_names": ["test.bag"]}
|
||||
|
||||
resp = client.post("/bags/joined/query", json=request_data)
|
||||
assert resp.status_code == 500
|
||||
data = resp.get_json()
|
||||
assert "Internal server error" in data["error"]
|
||||
@@ -0,0 +1,233 @@
|
||||
# tests/test_fst.py
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api import fst_bp
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def app():
|
||||
_app = Flask(__name__)
|
||||
_app.register_blueprint(fst_bp, url_prefix="/fst")
|
||||
_app.config["TESTING"] = True
|
||||
return _app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(app):
|
||||
return app.test_client()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_db_session():
|
||||
"""
|
||||
全局自动 mock SessionLocal，避免每个测试都手动 patch。
|
||||
返回一个 mock session 实例，可以在测试中进一步断言。
|
||||
"""
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.SessionLocal",
|
||||
return_value=MagicMock(),
|
||||
) as mocked:
|
||||
yield mocked
|
||||
|
||||
|
||||
# ----------------- 1. 打印树 -----------------
|
||||
def test_print_tree_200(client: pytest.fixture):
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.print_fst_tree",
|
||||
return_value="fake_tree",
|
||||
):
|
||||
resp = client.get("/fst/print_tree")
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.content_type == "application/json"
|
||||
assert resp.get_json() == "fake_tree"
|
||||
|
||||
|
||||
# ----------------- 2. 批量根据 FST 名称查 Bags -----------------
|
||||
def test_get_bags_by_fst_nodes_batch_200(client: pytest.fixture):
|
||||
# 构造 Bag 假对象
|
||||
Bag = type("Bag", (), {"name": None})
|
||||
fst1_bags = [Bag(), Bag()]
|
||||
fst1_bags[0].name, fst1_bags[1].name = "b1", "b2"
|
||||
fst2_bags = [Bag()]
|
||||
fst2_bags[0].name = "b3"
|
||||
|
||||
side_effect = {"fst1": fst1_bags, "fst2": fst2_bags}
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_bags_by_fst_name",
|
||||
side_effect=lambda db, name: side_effect[name],
|
||||
):
|
||||
# 默认第 1 页，每页 20
|
||||
resp = client.post("/fst/bags/nodes", json=["fst1", "fst2"])
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
# 期望 3 条记录，不分页
|
||||
assert data["page"] == 1
|
||||
assert data["per_page"] == 20
|
||||
assert data["total"] == 3
|
||||
|
||||
# 顺序与插入顺序一致
|
||||
assert data["items"] == [
|
||||
{"bag_name": "b1", "sts": "fst1"},
|
||||
{"bag_name": "b2", "sts": "fst1"},
|
||||
{"bag_name": "b3", "sts": "fst2"},
|
||||
]
|
||||
|
||||
|
||||
def test_get_bags_by_fst_nodes_batch_400(client: pytest.fixture):
|
||||
resp = client.post("/fst/bags/nodes", json={"wrong": "format"})
|
||||
assert resp.status_code == 400
|
||||
assert "Body must be a list" in resp.get_json()["error"]
|
||||
|
||||
|
||||
# ----------------- 3. 根据路径查 Bags -----------------
|
||||
def test_get_bags_by_fst_path_200(client: pytest.fixture):
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_bags_by_fst_paths",
|
||||
return_value={"fst": {"bags": ["bagA", "bagB"]}},
|
||||
):
|
||||
resp = client.get("/fst/bags/path/fst")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"fst": {"bags": ["bagA", "bagB"]}}
|
||||
|
||||
|
||||
# ----------------- 4. 根据名称查 FST -----------------
|
||||
def test_get_fst_node_by_name_200(client: pytest.fixture):
|
||||
fst_mock = MagicMock()
|
||||
fst_mock.name = "node"
|
||||
fst_mock.parent_id = 123
|
||||
|
||||
parent_mock = MagicMock()
|
||||
parent_mock.name = "parent_node"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_fst",
|
||||
return_value=fst_mock,
|
||||
),
|
||||
patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_fst_by_id",
|
||||
return_value=parent_mock,
|
||||
),
|
||||
patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_total_bag_sum",
|
||||
return_value=42,
|
||||
),
|
||||
):
|
||||
resp = client.get("/fst/node")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {
|
||||
"name": "node",
|
||||
"parent_node": "parent_node",
|
||||
"linked_bags_sum": 42,
|
||||
}
|
||||
|
||||
|
||||
def test_get_fst_node_by_name_404(client: pytest.fixture):
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_fst", return_value=None
|
||||
):
|
||||
resp = client.get("/fst/nonexistent")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
def test_update_fst_bag__invalid_json(client: pytest.fixture):
|
||||
resp = client.post("/fst/bags/update", json={"not": "list"})
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
# ----------------- 7. 创建/更新 FST 节点 -----------------
|
||||
def test_upsert_fst__create_success(client: pytest.fixture):
|
||||
node_mock = MagicMock()
|
||||
node_mock.id = 99
|
||||
node_mock.name = "new_node"
|
||||
node_mock.parent_id = None
|
||||
node_mock.reserved_json = {"foo": "bar"}
|
||||
node_mock.bag_sum = 0
|
||||
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.upsert_fst_node",
|
||||
return_value=node_mock,
|
||||
):
|
||||
resp = client.post(
|
||||
"/fst/update",
|
||||
json={
|
||||
"name": "new_node",
|
||||
"parent_name": None,
|
||||
"reserved_json": {"foo": "bar"},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["name"] == "new_node"
|
||||
|
||||
|
||||
def test_upsert_fst__400_missing_name(client: pytest.fixture):
|
||||
resp = client.post("/fst/update", json={"parent_name": "root"})
|
||||
assert resp.status_code == 400
|
||||
assert "Field 'name' is required" in resp.get_json()["error"]
|
||||
|
||||
|
||||
# ----------------- 8. 删除 FST 节点 -----------------
|
||||
def test_delete_fst_by_name__success(client: pytest.fixture):
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.delete_fst_by_fst_name"
|
||||
) as mock_delete:
|
||||
resp = client.delete("/fst/some_node")
|
||||
mock_delete.assert_called_once()
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"ok": True}
|
||||
|
||||
|
||||
def test_delete_fst_by_name__404(client: pytest.fixture):
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.delete_fst_by_fst_name",
|
||||
side_effect=ValueError("not found"),
|
||||
):
|
||||
resp = client.delete("/fst/nonexistent")
|
||||
assert resp.status_code == 404
|
||||
assert "not found" in resp.get_json()["error"]
|
||||
|
||||
|
||||
# ----------------- 9. 获取 FST 树形结构与 bag 详情 -----------------
|
||||
def test_list_bag_names_with_fst__with_bags(client: pytest.fixture):
|
||||
# Mock返回数据库行格式 - 每行的第一列是bag名称
|
||||
mock_rows = [
|
||||
("sample_03.bag", "other_data"),
|
||||
("sample_01.bag", "other_data"),
|
||||
("sample_02.bag", "other_data"),
|
||||
("sample_01.bag", "other_data"), # 重复项，测试去重功能
|
||||
]
|
||||
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_bag_names_linked_to_fst",
|
||||
return_value=mock_rows,
|
||||
):
|
||||
resp = client.get("/fst/baglist")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
# 验证返回的是排序后的bag名称列表（去重）
|
||||
assert isinstance(data, list)
|
||||
assert data == ["sample_01.bag", "sample_02.bag", "sample_03.bag"]
|
||||
|
||||
|
||||
def test_list_bag_names_with_fst__empty_list(client: pytest.fixture):
|
||||
# Mock返回空的数据库结果
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_bag_names_linked_to_fst",
|
||||
return_value=[],
|
||||
):
|
||||
resp = client.get("/fst/baglist")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
# 验证返回空列表
|
||||
assert isinstance(data, list)
|
||||
assert data == []
|
||||
@@ -0,0 +1,61 @@
|
||||
# tests/test_geometry.py
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api import geometry_bp
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
"""Flask test client with geometry blueprint registered."""
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(geometry_bp, url_prefix="/geometry")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
def test_get_geometry_info_by_rosbags_200(client, mocker):
|
||||
"""200 OK：查询到多条几何记录"""
|
||||
# 1. 伪造查询结果
|
||||
fake_rows = [
|
||||
MagicMock(
|
||||
rosbag_name="bag1", lon_lat_alt=[[12.34, 56.78, 0.0], [12.35, 56.79, 1.2]]
|
||||
),
|
||||
MagicMock(rosbag_name="bag2", lon_lat_alt=[[98.76, 54.32, 3.3]]),
|
||||
]
|
||||
|
||||
# 2. 拦截 SessionLocal 返回的 session 对象
|
||||
mock_session = MagicMock()
|
||||
mock_session.execute.return_value.fetchall.return_value = fake_rows
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.geometry.SessionLocal",
|
||||
return_value=mock_session,
|
||||
)
|
||||
|
||||
resp = client.post("/geometry/", json={"rosbag_names": ["bag1", "bag2"]})
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["bag1"] == [[12.34, 56.78, 0.0], [12.35, 56.79, 1.2]]
|
||||
assert data["bag2"] == [[98.76, 54.32, 3.3]]
|
||||
|
||||
|
||||
def test_get_geometry_info_by_rosbags_400(client, mocker):
|
||||
"""400 空列表或格式错误"""
|
||||
resp = client.post("/geometry/", json={"rosbag_names": []})
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_get_geometry_info_by_rosbags_404(client, mocker):
|
||||
"""404 找不到任何记录"""
|
||||
mock_session = MagicMock()
|
||||
mock_session.execute.return_value.fetchall.return_value = []
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.geometry.SessionLocal",
|
||||
return_value=mock_session,
|
||||
)
|
||||
|
||||
resp = client.post("/geometry/", json={"rosbag_names": ["nonexistent"]})
|
||||
assert resp.status_code == 404
|
||||
@@ -0,0 +1,27 @@
|
||||
# tests/test_projects.py
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
from fst_data_pipeline.apps.root_db_api.src.api import api_bp
|
||||
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(api_bp, url_prefix="/api")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
# tests/test_projects.py
|
||||
def test_read_projects_200(client, mocker):
|
||||
fake = [{"id": 1, "name": "Alpha"}, {"id": 2, "name": "Beta"}]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.projects.get_projects",
|
||||
return_value=fake,
|
||||
)
|
||||
|
||||
resp = client.get("/api/projects/all")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == [{"id": 1, "name": "Alpha"}, {"id": 2, "name": "Beta"}]
|
||||
@@ -0,0 +1,185 @@
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api.recompute import bp as recompute_bp
|
||||
|
||||
|
||||
# ------------------ Flask test client ------------------
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
"""Flask test client with recompute blueprint registered."""
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(recompute_bp, url_prefix="/recompute")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
# ------------------ 单元测试用例 ------------------
|
||||
def test_get_bag_versions_200(client, mocker):
|
||||
"""测试批量获取bag版本列表 - 成功场景"""
|
||||
mock_result = {
|
||||
"bag1.bag": ["v1.0", "v1.1", "v2.0"],
|
||||
"bag2.bag": ["v1.0", "v2.0"],
|
||||
"bag3.bag": [],
|
||||
}
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.recompute.get_bag_recompute_versions",
|
||||
return_value=mock_result,
|
||||
)
|
||||
mocker.patch("fst_data_pipeline.apps.root_db_api.src.api.recompute.SessionLocal")
|
||||
|
||||
resp = client.post(
|
||||
"/recompute/versions", json={"bag_names": ["bag1.bag", "bag2.bag", "bag3.bag"]}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["bag1.bag"] == ["v1.0", "v1.1", "v2.0"]
|
||||
assert data["bag2.bag"] == ["v1.0", "v2.0"]
|
||||
assert data["bag3.bag"] == []
|
||||
|
||||
|
||||
def test_get_bag_versions_400_no_bag_names(client):
|
||||
"""测试批量获取bag版本列表 - 缺少bag_names"""
|
||||
resp = client.post("/recompute/versions", json={})
|
||||
assert resp.status_code == 400
|
||||
assert "bag_names list required" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_bag_versions_400_empty_bag_names(client):
|
||||
"""测试批量获取bag版本列表 - bag_names为空列表"""
|
||||
resp = client.post("/recompute/versions", json={"bag_names": []})
|
||||
assert resp.status_code == 400
|
||||
assert "bag_names must be a non-empty list" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_bag_versions_400_invalid_bag_names(client):
|
||||
"""测试批量获取bag版本列表 - bag_names不是列表"""
|
||||
resp = client.post("/recompute/versions", json={"bag_names": "not_a_list"})
|
||||
assert resp.status_code == 400
|
||||
assert "bag_names must be a non-empty list" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_bag_versions_500_exception(client, mocker):
|
||||
"""测试批量获取bag版本列表 - 服务器异常"""
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.recompute.get_bag_recompute_versions",
|
||||
side_effect=Exception("Database error"),
|
||||
)
|
||||
mocker.patch("fst_data_pipeline.apps.root_db_api.src.api.recompute.SessionLocal")
|
||||
|
||||
resp = client.post("/recompute/versions", json={"bag_names": ["bag1.bag"]})
|
||||
assert resp.status_code == 500
|
||||
assert "Database error" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_storage_paths_200(client, mocker):
|
||||
"""测试批量获取存储路径 - 成功场景"""
|
||||
mock_result = [
|
||||
{
|
||||
"bag_name": "bag1.bag",
|
||||
"recompute_version": "v1.0",
|
||||
"results": [
|
||||
{
|
||||
"result_id": 1,
|
||||
"bag_name": "bag1.bag",
|
||||
"recompute_version": "v1.0",
|
||||
"result_type": "perception",
|
||||
"storage_path": "/path/to/result1",
|
||||
"status": "available",
|
||||
"created_time": "2024-01-16T10:30:00Z",
|
||||
"reserved_json": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.recompute.get_storage_paths_batch",
|
||||
return_value=mock_result,
|
||||
)
|
||||
mocker.patch("fst_data_pipeline.apps.root_db_api.src.api.recompute.SessionLocal")
|
||||
|
||||
requests = [{"bag_name": "bag1.bag", "recompute_version": "v1.0"}]
|
||||
resp = client.post("/recompute/storage", json={"requests": requests})
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert len(data) == 1
|
||||
assert data[0]["bag_name"] == "bag1.bag"
|
||||
assert data[0]["recompute_version"] == "v1.0"
|
||||
assert len(data[0]["results"]) == 1
|
||||
assert data[0]["results"][0]["result_type"] == "perception"
|
||||
|
||||
|
||||
def test_get_storage_paths_400_no_requests(client):
|
||||
"""测试批量获取存储路径 - 缺少requests"""
|
||||
resp = client.post("/recompute/storage", json={})
|
||||
assert resp.status_code == 400
|
||||
assert "requests list required" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_storage_paths_400_empty_requests(client):
|
||||
"""测试批量获取存储路径 - requests为空列表"""
|
||||
resp = client.post("/recompute/storage", json={"requests": []})
|
||||
assert resp.status_code == 400
|
||||
assert "requests must be a non-empty list" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_storage_paths_400_invalid_requests(client):
|
||||
"""测试批量获取存储路径 - requests不是列表"""
|
||||
resp = client.post("/recompute/storage", json={"requests": "not_a_list"})
|
||||
assert resp.status_code == 400
|
||||
assert "requests must be a non-empty list" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_storage_paths_400_invalid_request_format(client):
|
||||
"""测试批量获取存储路径 - 单个request格式错误"""
|
||||
invalid_requests = [
|
||||
{"bag_name": "bag1.bag"}, # 缺少recompute_version
|
||||
{"recompute_version": "v1.0"}, # 缺少bag_name
|
||||
{"wrong": "format"}, # 完全错误的格式
|
||||
]
|
||||
|
||||
for invalid_request in invalid_requests:
|
||||
resp = client.post("/recompute/storage", json={"requests": [invalid_request]})
|
||||
assert resp.status_code == 400
|
||||
assert (
|
||||
"Each request must contain bag_name and recompute_version"
|
||||
in resp.get_json()["error"]
|
||||
)
|
||||
|
||||
|
||||
def test_get_storage_paths_500_exception(client, mocker):
|
||||
"""测试批量获取存储路径 - 服务器异常"""
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.recompute.get_storage_paths_batch",
|
||||
side_effect=Exception("Service error"),
|
||||
)
|
||||
mocker.patch("fst_data_pipeline.apps.root_db_api.src.api.recompute.SessionLocal")
|
||||
|
||||
requests = [{"bag_name": "bag1.bag", "recompute_version": "v1.0"}]
|
||||
resp = client.post("/recompute/storage", json={"requests": requests})
|
||||
assert resp.status_code == 500
|
||||
assert "Service error" in resp.get_json()["error"]
|
||||
|
||||
|
||||
# ------------------ 参数化测试 ------------------
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint,bad_body",
|
||||
[
|
||||
("/recompute/versions", {"bag_names": "not_list"}),
|
||||
("/recompute/versions", {"bag_names": []}),
|
||||
("/recompute/versions", {}),
|
||||
("/recompute/storage", {"requests": "not_list"}),
|
||||
("/recompute/storage", {"requests": []}),
|
||||
("/recompute/storage", {}),
|
||||
("/recompute/storage", {"requests": [{"invalid": "format"}]}),
|
||||
],
|
||||
)
|
||||
def test_bad_requests_400(client, endpoint, bad_body):
|
||||
"""参数化测试：各种错误请求格式"""
|
||||
resp = client.post(endpoint, json=bad_body)
|
||||
assert resp.status_code == 400
|
||||
@@ -0,0 +1,113 @@
|
||||
# tests/test_tags.py
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api.tags import bp as tags_bp
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
"""Flask test client with tags blueprint registered."""
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(tags_bp, url_prefix="/tags")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
# ---------- /tags/all ----------
|
||||
def test_read_tags_200(client, mocker):
|
||||
fake_tags = [
|
||||
{"name": "tag1", "type": "scenario", "creator": "alice"},
|
||||
{"name": "tag2", "type": "label", "creator": "bob"},
|
||||
]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.get_tags",
|
||||
return_value=fake_tags,
|
||||
)
|
||||
|
||||
resp = client.get("/tags/all")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == fake_tags
|
||||
|
||||
|
||||
# ---------- /tags/bags/<tag_name> ----------
|
||||
def test_get_bags_by_tags_and_200(client, mocker):
|
||||
fake_bags = [{"id": 1, "name": "foo.bag"}, {"id": 2, "name": "bar.bag"}]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.query_bags_by_tags",
|
||||
return_value=fake_bags,
|
||||
)
|
||||
|
||||
resp = client.get("/tags/bags?tags=foo,bar&op=and")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"bags": fake_bags}
|
||||
|
||||
|
||||
def test_get_bags_by_tags_or_200(client, mocker):
|
||||
fake_bags = [{"id": 3, "name": "baz.bag"}]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.query_bags_by_tags",
|
||||
return_value=fake_bags,
|
||||
)
|
||||
|
||||
resp = client.get("/tags/bags?tags=baz&op=or")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"bags": fake_bags}
|
||||
|
||||
|
||||
def test_get_bags_by_tags_default_op_and(client, mocker):
|
||||
"""不传 op 时默认按 and 处理"""
|
||||
fake_bags = [{"id": 4, "name": "default.bag"}]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.query_bags_by_tags",
|
||||
return_value=fake_bags,
|
||||
)
|
||||
|
||||
resp = client.get("/tags/bags?tags=default") # 不带 op 参数
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"bags": fake_bags}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad_tags", ["", " "])
|
||||
def test_get_bags_by_tags_missing_tags_param(client, bad_tags):
|
||||
resp = client.get(f"/tags/bags?tags={bad_tags}")
|
||||
assert resp.status_code == 400
|
||||
assert resp.get_json() == {"bags": []}
|
||||
|
||||
|
||||
def test_get_bags_by_tags_invalid_op(client):
|
||||
resp = client.get("/tags/bags?tags=A,B&op=invalid")
|
||||
assert resp.status_code == 400
|
||||
assert resp.get_json() == {"bags": []}
|
||||
|
||||
|
||||
# ---------- /tags/creators/<creator> ----------
|
||||
def test_get_tags_by_creator_200(client, mocker):
|
||||
fake_tags = [
|
||||
{"name": "tagA", "type": "quality", "creator": "alice"},
|
||||
{"name": "tagB", "type": "label", "creator": "alice"},
|
||||
]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.get_tags_by_creator",
|
||||
return_value=fake_tags,
|
||||
)
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.SessionLocal",
|
||||
return_value=mocker.MagicMock(),
|
||||
)
|
||||
|
||||
resp = client.get("/tags/creators/alice")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"tags": ["tagA", "tagB"]}
|
||||
|
||||
|
||||
def test_get_tags_by_creator_404(client, mocker):
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.get_tags_by_creator",
|
||||
return_value=[],
|
||||
)
|
||||
|
||||
resp = client.get("/tags/creators/nobody")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"tags": []}
|
||||
@@ -0,0 +1,53 @@
|
||||
# tests/test_topics.py
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api.topics import bp as topics_bp
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
"""Flask test client with topics blueprint registered."""
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(topics_bp, url_prefix="/topics")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
# ---------- /topics/all ----------
|
||||
def test_read_topics_200(client, mocker):
|
||||
fake_topics = [
|
||||
{"name": "topic1", "type": "scenario"},
|
||||
{"name": "topic2", "type": "label"},
|
||||
]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.topics.get_all_topics",
|
||||
return_value=fake_topics,
|
||||
)
|
||||
resp = client.get("/topics/all")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == fake_topics
|
||||
|
||||
|
||||
# ---------- /topics/bags/<topic_name> ----------
|
||||
def test_get_bags_by_topic_200(client, mocker):
|
||||
fake_bags = [{"name": "bagA"}, {"name": "bagB"}]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.topics.get_bags_by_topic_name",
|
||||
return_value=fake_bags,
|
||||
)
|
||||
resp = client.get("/topics/bags/highway")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"name": ["bagA", "bagB"]}
|
||||
|
||||
|
||||
def test_get_bags_by_topic_empty(client, mocker):
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.topics.get_bags_by_topic_name",
|
||||
return_value=[],
|
||||
)
|
||||
|
||||
resp = client.get("/topics/bags/unknown")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"name": []}
|
||||
@@ -0,0 +1 @@
|
||||
# Test service package
|
||||
@@ -0,0 +1,703 @@
|
||||
# tests/test_services_mock_only.py
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# 把要测试的 service 函数全部一次性 import
|
||||
import fst_data_pipeline.apps.root_db_api.src.core.service as srv
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db():
|
||||
"""返回一个 mock Session，所有测试函数共用"""
|
||||
return MagicMock()
|
||||
|
||||
|
||||
# ---------- Project ----------
|
||||
def test_get_projects(db):
|
||||
p1 = MagicMock()
|
||||
p1.id, p1.name = 1, "proj1"
|
||||
|
||||
p2 = MagicMock()
|
||||
p2.id, p2.name = 2, "proj2"
|
||||
|
||||
db.query().all.return_value = [p1, p2]
|
||||
assert srv.get_projects(db) == [
|
||||
{"id": 1, "name": "proj1"},
|
||||
{"id": 2, "name": "proj2"},
|
||||
]
|
||||
|
||||
|
||||
# ---------- Bag ----------
|
||||
def test_get_bags(db):
|
||||
bag = MagicMock()
|
||||
bag.name = "bag1"
|
||||
bag.project = MagicMock()
|
||||
bag.project.name = "p1"
|
||||
|
||||
db.query().options().all.return_value = [bag]
|
||||
assert srv.get_bags(db) == [bag]
|
||||
|
||||
|
||||
def test_get_bags_by_conditions(db):
|
||||
q = db.query.return_value
|
||||
q.filter.return_value = q
|
||||
bag = MagicMock()
|
||||
bag.name = "foo"
|
||||
q.all.return_value = [bag]
|
||||
|
||||
res = srv.get_bags_by_conditions(db, name="bar", project_id=1)
|
||||
assert res == [bag]
|
||||
# 只要 filter 被调用了两次即可
|
||||
assert q.filter.call_count == 2
|
||||
|
||||
|
||||
# ---------- Topic ----------
|
||||
def test_get_all_topics(db):
|
||||
t = MagicMock()
|
||||
t.name, t.type = "/imu", "sensor_msgs/Imu"
|
||||
db.query().all.return_value = [t]
|
||||
assert srv.get_all_topics(db) == [{"name": "/imu", "type": "sensor_msgs/Imu"}]
|
||||
|
||||
|
||||
def test_get_bags_by_topic_name(db):
|
||||
bag = MagicMock()
|
||||
bag.name = "bag1"
|
||||
db.query().join().join().filter().all.return_value = [bag]
|
||||
assert srv.get_bags_by_topic_name(db, "imu") == [{"name": "bag1"}]
|
||||
|
||||
|
||||
def test_get_topics_by_bag_name(db):
|
||||
topic = MagicMock()
|
||||
topic.name = "/gps"
|
||||
topic.type = "sensor_msgs/NavSatFix"
|
||||
db.query().join().join().filter().all.return_value = [topic]
|
||||
assert srv.get_topics_by_bag_name(db, "bag1") == [topic]
|
||||
|
||||
|
||||
# ---------- Tag ----------
|
||||
def test_get_tags(db):
|
||||
tag = MagicMock()
|
||||
tag.name, tag.type, tag.creator = "tag1", "bool", "Alice"
|
||||
db.query().all.return_value = [tag]
|
||||
assert srv.get_tags(db) == [{"name": "tag1", "type": "bool", "creator": "Alice"}]
|
||||
|
||||
|
||||
def test_get_tags_by_creator(db):
|
||||
tag = MagicMock()
|
||||
tag.name = "t1"
|
||||
tag.type = "bool"
|
||||
tag.creator = "alice"
|
||||
db.query().filter().all.return_value = [tag]
|
||||
assert srv.get_tags_by_creator(db, "alice") == [
|
||||
{"name": "t1", "type": "bool", "creator": "alice"},
|
||||
]
|
||||
|
||||
|
||||
def test_get_bags_by_tag_name(db):
|
||||
bag = MagicMock()
|
||||
bag.name = "b1"
|
||||
db.query().join().join().filter().all.return_value = [bag]
|
||||
assert srv.get_bags_by_tag_name(db, "hot") == [{"name": "b1"}]
|
||||
|
||||
|
||||
def test_get_tags_by_bag_name(db):
|
||||
tag = MagicMock()
|
||||
tag.name = "urgent"
|
||||
db.query().join().join().filter().all.return_value = [tag]
|
||||
assert srv.get_tags_by_bag_name(db, "bag99") == [tag]
|
||||
|
||||
|
||||
# ---------- MainPangu ----------
|
||||
def test_get_main_pangu_by_conditions(db):
|
||||
from datetime import date
|
||||
|
||||
q = db.query.return_value
|
||||
q.filter.return_value = q
|
||||
q.join.return_value = q # Add join mock for BagList join
|
||||
q.limit.return_value = q
|
||||
rec = MagicMock()
|
||||
rec.keyword = "pangu1"
|
||||
rec.vehicle = "Lexus"
|
||||
rec.start_time = "20250101"
|
||||
rec.end_time = "20250102"
|
||||
q.all.return_value = [rec]
|
||||
|
||||
res = srv.get_main_pangu_by_conditions(
|
||||
db,
|
||||
start_time=date(2025, 1, 1), # Use date objects
|
||||
end_time=date(2025, 1, 2),
|
||||
vehicle="Lexus",
|
||||
keyword="pangu1",
|
||||
topics=None, # Add new parameters
|
||||
fst=None,
|
||||
limit=1000,
|
||||
)
|
||||
assert res == [rec]
|
||||
|
||||
|
||||
def test_get_main_pangu_by_name(db):
|
||||
rec = MagicMock()
|
||||
rec.name = "pangu_2024"
|
||||
db.query().filter().all.return_value = [rec]
|
||||
assert srv.get_main_pangu_by_name(db, "2024") == [rec]
|
||||
|
||||
|
||||
def test_get_pangu_details_by_bag_name(db):
|
||||
main = MagicMock()
|
||||
main.name = "bag1"
|
||||
main.vehicle = "Lexus"
|
||||
|
||||
bag = MagicMock()
|
||||
bag.id = 7
|
||||
bag.foo = "bar"
|
||||
|
||||
sec = MagicMock()
|
||||
sec.extra = "meta"
|
||||
|
||||
db.query().filter().first.side_effect = [main, bag, sec]
|
||||
res = srv.get_pangu_details_by_bag_name(db, "bag1")
|
||||
assert res["bag_name"] == "bag1"
|
||||
assert res["vehicle"] == "Lexus"
|
||||
assert res["foo"] == "bar"
|
||||
assert res["extra"] == "meta"
|
||||
|
||||
|
||||
# ---------- MainMinerva ----------
|
||||
def test_get_main_minerva_by_session_id(db):
|
||||
rec = MagicMock()
|
||||
rec.session_id = "session123"
|
||||
db.query().filter().all.return_value = [rec]
|
||||
assert srv.get_main_minerva_by_session_id(db, "123") == [rec]
|
||||
|
||||
|
||||
def test_get_minerva_details_by_bag_name(db):
|
||||
main = MagicMock()
|
||||
main.session_id = "s1"
|
||||
main.vin = "VIN001"
|
||||
|
||||
bag = MagicMock()
|
||||
bag.id = 5
|
||||
|
||||
sec = MagicMock()
|
||||
sec.meta = "data"
|
||||
|
||||
db.query().filter().first.side_effect = [main, bag, sec]
|
||||
res = srv.get_minerva_details_by_bag_name(db, "s1")
|
||||
assert res["bag_name"] == "s1"
|
||||
assert res["vin"] == "VIN001"
|
||||
assert res["meta"] == "data"
|
||||
|
||||
|
||||
# ---------- BagLifecycle ----------
|
||||
def test_get_bag_lifecycle_by_bag_names(db):
|
||||
lc = MagicMock()
|
||||
lc.bag_name = "b1"
|
||||
db.query().filter().all.return_value = [lc]
|
||||
assert srv.get_bag_lifecycle_by_bag_names(db, ["b1"]) == [lc]
|
||||
|
||||
|
||||
# ---------- GeometryInfo ----------
|
||||
def test_get_geometry_info(db):
|
||||
geom = MagicMock()
|
||||
db.query().all.return_value = [geom]
|
||||
assert srv.get_geometry_info(db) == [geom]
|
||||
|
||||
|
||||
def test_get_geometry_info_by_rosbag_name(db):
|
||||
db.query().filter().first.return_value = "geom_found"
|
||||
assert srv.get_geometry_info_by_rosbag_name(db, "rosbag1") == "geom_found"
|
||||
|
||||
|
||||
# ---------- FST ----------
|
||||
def test_get_fst(db):
|
||||
fst = MagicMock()
|
||||
fst.name = "root"
|
||||
db.query().filter().first.return_value = fst
|
||||
assert srv.get_fst(db, "root").name == "root"
|
||||
|
||||
|
||||
# ---------- FST ----------
|
||||
def test_get_total_bag_sum(db):
|
||||
q = MagicMock()
|
||||
db.query.return_value = q
|
||||
q.filter.return_value = q
|
||||
q.scalar.return_value = 42
|
||||
assert srv.get_total_bag_sum(db, "root") == 42
|
||||
|
||||
|
||||
def test_get_fst_by_id(db):
|
||||
fst = MagicMock()
|
||||
fst.id = 99
|
||||
db.query().filter().first.return_value = fst
|
||||
assert srv.get_fst_by_id(db, 99).id == 99
|
||||
|
||||
|
||||
def test_get_fst_nodes_by_bag_name(db):
|
||||
row = MagicMock()
|
||||
row.name, row.start, row.end, row.comments, row.video_url, row.img_url = (
|
||||
"fstA",
|
||||
10.0,
|
||||
20.0,
|
||||
"test",
|
||||
"video",
|
||||
"img",
|
||||
)
|
||||
db.query().select_from().join().join().filter().all.return_value = [row]
|
||||
res = srv.get_fst_nodes_by_bag_name(db, "bag1")
|
||||
assert res == [
|
||||
{
|
||||
"name": "fstA",
|
||||
"start": "10.0s",
|
||||
"end": "20.0s",
|
||||
"comments": "test",
|
||||
"video_url": "video",
|
||||
"img_url": "img",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_get_fst_path_by_bag_name(db):
|
||||
# 链：child -> parent -> root
|
||||
child = MagicMock()
|
||||
child.id, child.name, child.parent_id = 3, "child", 2
|
||||
|
||||
parent = MagicMock()
|
||||
parent.id, parent.name, parent.parent_id = 2, "parent", 1
|
||||
|
||||
root = MagicMock()
|
||||
root.id, root.name, root.parent_id = 1, "root", None
|
||||
|
||||
bag = MagicMock()
|
||||
bag.fst_node_id = 3
|
||||
|
||||
db.query().filter().first.side_effect = [bag, child, parent, root, None]
|
||||
path = srv.get_fst_path_by_bag_name(db, "bag1")
|
||||
expected = [
|
||||
{"id": 3, "name": "child"},
|
||||
{"id": 2, "name": "parent"},
|
||||
{"id": 1, "name": "root"},
|
||||
]
|
||||
assert path == expected
|
||||
|
||||
|
||||
def test_get_bags_by_fst_name(db):
|
||||
bag = MagicMock()
|
||||
bag.name = "bag_fst"
|
||||
db.query().join().join().filter().all.return_value = [bag]
|
||||
assert srv.get_bags_by_fst_name(db, "fst") == [bag]
|
||||
|
||||
|
||||
# ---------- query_bags_by_tags ----------
|
||||
def test_query_bags_by_tags_empty_tag_list(db):
|
||||
"""空标签列表立即返回空列表"""
|
||||
res = srv.query_bags_by_tags(db, [])
|
||||
assert res == []
|
||||
# 确保没有任何查询被调用
|
||||
db.query.assert_not_called()
|
||||
|
||||
|
||||
def test_query_bags_by_tags_and_op(db):
|
||||
"""AND 逻辑：bag 必须同时关联全部标签"""
|
||||
# 1. 先 mock 标签查询：假设前端传了 ["t1", "t2"]，库里有这两个标签
|
||||
tag1, tag2 = MagicMock(), MagicMock()
|
||||
tag1.id, tag2.id = 11, 22
|
||||
db.query.return_value.filter.return_value.all.return_value = [(11,), (22,)]
|
||||
|
||||
# 2. 再 mock bag 查询：只返回一个 bag 名 "bag_ok"
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.group_by.return_value = query_mock
|
||||
query_mock.having.return_value = query_mock
|
||||
query_mock.all.return_value = [("bag_ok",)]
|
||||
|
||||
res = srv.query_bags_by_tags(db, ["t1", "t2"], op="and")
|
||||
assert res == ["bag_ok"]
|
||||
|
||||
having_call = query_mock.having.call_args[0][0]
|
||||
assert query_mock.having.call_count == 1
|
||||
|
||||
|
||||
def test_query_bags_by_tags_or_op(db):
|
||||
"""OR 逻辑：bag 只要关联任一标签即可"""
|
||||
# 1. 标签查询同上
|
||||
db.query.return_value.filter.return_value.all.return_value = [(11,), (22,)]
|
||||
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.distinct.return_value = query_mock
|
||||
query_mock.all.return_value = [("bag_a",), ("bag_b",)]
|
||||
|
||||
res = srv.query_bags_by_tags(db, ["t1", "t2"], op="or")
|
||||
assert res == ["bag_a", "bag_b"]
|
||||
|
||||
# 验证调用了 distinct
|
||||
assert query_mock.distinct.call_count == 1
|
||||
|
||||
# ---------- get_bag_names_linked_to_fst ----------
|
||||
def test_get_bag_names_linked_to_fst_ok(db):
|
||||
"""成功场景：返回去重且按名字排序的 bag 名列表"""
|
||||
# 模拟 query 链式调用
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.order_by.return_value = query_mock
|
||||
# distinct 返回的是子查询对象，继续链式
|
||||
query_mock.distinct.return_value = query_mock
|
||||
# 最终 all() 给出结果
|
||||
query_mock.all.return_value = [("bag_z",), ("bag_a",)]
|
||||
|
||||
res = srv.get_bag_names_linked_to_fst(db)
|
||||
# 期望按 order_by 排好序
|
||||
assert res == [("bag_z",), ("bag_a",)]
|
||||
|
||||
# 验证调用了 distinct 和 order_by
|
||||
assert query_mock.distinct.call_count == 1
|
||||
query_mock.order_by.assert_called_once()
|
||||
|
||||
def test_get_bag_names_linked_to_fst_empty(db):
|
||||
"""结果为空时返回空列表"""
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.order_by.return_value = query_mock
|
||||
query_mock.distinct.return_value = query_mock
|
||||
query_mock.all.return_value = []
|
||||
|
||||
res = srv.get_bag_names_linked_to_fst(db)
|
||||
assert res == []
|
||||
|
||||
# ---------- print_fst_tree ----------
|
||||
def test_print_fst_tree_single_tree(db):
|
||||
"""单棵树：层级与字段正确"""
|
||||
db.query.return_value.all.return_value = [
|
||||
("1", "root", None),
|
||||
("2", "child", "1"),
|
||||
("3", "grand", "2"),
|
||||
]
|
||||
trees = srv.print_fst_tree(db)
|
||||
assert len(trees) == 1
|
||||
root = trees[0]
|
||||
assert root["id"] == "root"
|
||||
assert root["level"] == 0
|
||||
child = root["children"][0]
|
||||
assert child["id"] == "child"
|
||||
assert child["level"] == 1
|
||||
grand = child["children"][0]
|
||||
assert grand["id"] == "grand"
|
||||
assert grand["level"] == 2
|
||||
assert grand["children"] == []
|
||||
|
||||
def test_print_fst_tree_multi_trees(db):
|
||||
"""多棵树：返回顺序与根节点数量"""
|
||||
db.query.return_value.all.return_value = [
|
||||
("10", "treeA", None),
|
||||
("20", "treeB", None),
|
||||
("30", "subA", "10"),
|
||||
]
|
||||
trees = srv.print_fst_tree(db)
|
||||
assert len(trees) == 2
|
||||
assert [t["id"] for t in trees] == ["treeA", "treeB"]
|
||||
assert trees[0]["children"][0]["id"] == "subA"
|
||||
|
||||
def test_print_fst_tree_skip_dirty_data(db):
|
||||
"""脏数据（非数字 id/pid）自动跳过"""
|
||||
db.query.return_value.all.return_value = [
|
||||
("abc", "bad1", "1"), # 无效 id
|
||||
("1", "root", "xyz"), # 无效 pid，当根处理
|
||||
("2", "child", "1"), # 正常
|
||||
]
|
||||
trees = srv.print_fst_tree(db)
|
||||
# 仅保留有效节点，且 root 当成新根
|
||||
assert len(trees) == 1
|
||||
assert trees[0]["id"] == "root"
|
||||
assert trees[0]["children"][0]["id"] == "child"
|
||||
|
||||
|
||||
# ---------- batch_upsert_fst_bag ----------
|
||||
def test_batch_upsert_fst_bag_empty_items(db):
|
||||
"""空列表直接返回空"""
|
||||
res = srv.batch_upsert_fst_bag(db, [])
|
||||
assert res == []
|
||||
db.query.assert_not_called()
|
||||
|
||||
|
||||
def test_batch_upsert_fst_bag_normal(db, monkeypatch):
|
||||
"""正常流程：预取、建标签、去重、逐条 upsert"""
|
||||
# 预取阶段
|
||||
node_name2id = {"root": 1, "child": 2}
|
||||
tag_name2id = {"hot": 11}
|
||||
bag_name2id = {"bag1": 101}
|
||||
monkeypatch.setattr(
|
||||
srv, "_prefetch", lambda _db, _items: (node_name2id, tag_name2id, bag_name2id)
|
||||
)
|
||||
monkeypatch.setattr(srv, "_ensure_tags", lambda _db, _names, _m: None)
|
||||
|
||||
# 已存在关系
|
||||
db.query.return_value.filter.return_value.all.return_value = []
|
||||
|
||||
# 逐条 upsert mock
|
||||
fake_result = MagicMock()
|
||||
monkeypatch.setattr(srv, "_upsert_one_item", lambda *_args: fake_result)
|
||||
|
||||
items = [MagicMock(bag_name="bag1", nodes=["root", "child"], tags=["hot"])]
|
||||
res = srv.batch_upsert_fst_bag(db, items)
|
||||
assert res == [fake_result]
|
||||
|
||||
|
||||
# ---------- delete_fst_by_fst_name ----------
|
||||
def test_delete_fst_by_fst_name_not_found(db):
|
||||
"""节点不存在抛 ValueError"""
|
||||
db.query.return_value.filter.return_value.first.return_value = None
|
||||
with pytest.raises(ValueError, match="FST node 'no' not found"):
|
||||
srv.delete_fst_by_fst_name(db, "no")
|
||||
|
||||
|
||||
def test_delete_fst_by_fst_name_ok(db):
|
||||
"""存在节点：子节点变根，再删除"""
|
||||
node = MagicMock(id=99)
|
||||
db.query.return_value.filter.return_value.first.return_value = node
|
||||
db.execute = MagicMock()
|
||||
db.delete = MagicMock()
|
||||
|
||||
srv.delete_fst_by_fst_name(db, "to_del")
|
||||
|
||||
# 子节点 parent_id 被置空
|
||||
db.execute.assert_called_once()
|
||||
call = db.execute.call_args[0][0]
|
||||
assert "parent_id" in str(call)
|
||||
db.delete.assert_called_once_with(node)
|
||||
|
||||
|
||||
# ---------- upsert_fst_node ----------
|
||||
def test_upsert_fst_node_insert_root(db):
|
||||
"""新建根节点"""
|
||||
db.query.return_value.filter.return_value.first.return_value = None
|
||||
new_node = MagicMock()
|
||||
db.add = MagicMock()
|
||||
db.flush = MagicMock()
|
||||
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.core.service.FST", return_value=new_node
|
||||
):
|
||||
node = srv.upsert_fst_node(db, name="root")
|
||||
assert node is new_node
|
||||
db.add.assert_called_once_with(new_node)
|
||||
db.flush.assert_called_once()
|
||||
|
||||
|
||||
def test_upsert_fst_node_update_with_parent(db):
|
||||
"""更新已有节点，同时换父节点"""
|
||||
old_parent = MagicMock(id=10)
|
||||
new_parent = MagicMock(id=20)
|
||||
node = MagicMock(parent_id=10)
|
||||
db.query.return_value.filter.return_value.first.side_effect = [new_parent, node]
|
||||
db.flush = MagicMock()
|
||||
|
||||
res = srv.upsert_fst_node(db, name="n1", parent_name="new_p", bag_sum=5)
|
||||
assert res.parent_id == 20
|
||||
assert res.bag_sum == 5
|
||||
db.flush.assert_called_once()
|
||||
|
||||
|
||||
def test_upsert_fst_node_parent_not_found(db):
|
||||
"""指定父节点不存在抛 ValueError"""
|
||||
db.query.return_value.filter.return_value.first.return_value = None
|
||||
with pytest.raises(ValueError, match="Parent FST node 'no' not found"):
|
||||
srv.upsert_fst_node(db, name="n1", parent_name="no")
|
||||
|
||||
|
||||
# ---------- Recompute ----------
|
||||
def test_get_bag_recompute_versions(db):
|
||||
"""测试获取bag的recompute版本列表"""
|
||||
# Mock join查询结果 - 返回bag_name和recompute_version对
|
||||
result1 = MagicMock()
|
||||
result1.bag_name = "bag1.bag"
|
||||
result1.recompute_version = "v1.0"
|
||||
|
||||
result2 = MagicMock()
|
||||
result2.bag_name = "bag1.bag"
|
||||
result2.recompute_version = "v1.1"
|
||||
|
||||
result3 = MagicMock()
|
||||
result3.bag_name = "bag2.bag"
|
||||
result3.recompute_version = "v1.0"
|
||||
|
||||
result4 = MagicMock()
|
||||
result4.bag_name = "bag2.bag"
|
||||
result4.recompute_version = "v2.0"
|
||||
|
||||
# Mock query chain
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.all.return_value = [result1, result2, result3, result4]
|
||||
|
||||
result = srv.get_bag_recompute_versions(db, ["bag1.bag", "bag2.bag"])
|
||||
|
||||
expected = {"bag1.bag": ["v1.0", "v1.1"], "bag2.bag": ["v1.0", "v2.0"]}
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_get_bag_recompute_versions_empty_bags(db):
|
||||
"""测试空bag列表的处理"""
|
||||
# Mock query chain - 空bag列表仍会执行查询，但filter为空
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.all.return_value = []
|
||||
|
||||
result = srv.get_bag_recompute_versions(db, [])
|
||||
assert result == {}
|
||||
|
||||
|
||||
def test_get_bag_recompute_versions_bag_not_found(db):
|
||||
"""测试bag不存在的情况"""
|
||||
# Mock 空的查询结果
|
||||
db.query.return_value.filter.return_value.all.return_value = []
|
||||
|
||||
result = srv.get_bag_recompute_versions(db, ["nonexistent.bag"])
|
||||
assert result == {"nonexistent.bag": []}
|
||||
|
||||
|
||||
def test_get_storage_paths_batch(db):
|
||||
"""测试批量获取存储路径"""
|
||||
from datetime import datetime
|
||||
|
||||
# Mock join查询结果 - 使用实际查询中select的字段名
|
||||
result1 = MagicMock()
|
||||
result1.result_id = 1
|
||||
result1.bag_name = "bag1.bag"
|
||||
result1.recompute_version = "v1.0"
|
||||
result1.result_type = "perception"
|
||||
result1.storage_path = "/path/to/result1"
|
||||
result1.status = "available"
|
||||
result1.created_time = datetime.fromisoformat("2024-01-16T10:30:00")
|
||||
result1.reserved_json = {}
|
||||
|
||||
# Mock query chain
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.order_by.return_value = query_mock
|
||||
query_mock.all.return_value = [result1]
|
||||
|
||||
requests = [{"bag_name": "bag1.bag", "recompute_version": "v1.0"}]
|
||||
results = srv.get_storage_paths_batch(db, requests)
|
||||
|
||||
expected = [
|
||||
{
|
||||
"bag_name": "bag1.bag",
|
||||
"recompute_version": "v1.0",
|
||||
"results": [
|
||||
{
|
||||
"result_id": 1,
|
||||
"bag_name": "bag1.bag",
|
||||
"recompute_version": "v1.0",
|
||||
"result_type": "perception",
|
||||
"storage_path": "/path/to/result1",
|
||||
"status": "available",
|
||||
"created_time": "2024-01-16T10:30:00",
|
||||
"reserved_json": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
assert results == expected
|
||||
|
||||
|
||||
def test_get_storage_paths_batch_empty_requests(db):
|
||||
"""测试空请求列表"""
|
||||
result = srv.get_storage_paths_batch(db, [])
|
||||
assert result == []
|
||||
# 确保没有数据库查询
|
||||
db.query.assert_not_called()
|
||||
|
||||
|
||||
def test_get_storage_paths_batch_no_results(db):
|
||||
"""测试没有找到结果的情况"""
|
||||
# Mock 空的查询结果
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.order_by.return_value = query_mock
|
||||
query_mock.all.return_value = []
|
||||
|
||||
requests = [{"bag_name": "nonexistent.bag", "recompute_version": "v1.0"}]
|
||||
results = srv.get_storage_paths_batch(db, requests)
|
||||
|
||||
expected = [
|
||||
{"bag_name": "nonexistent.bag", "recompute_version": "v1.0", "results": []}
|
||||
]
|
||||
assert results == expected
|
||||
|
||||
|
||||
def test_query_all_joined_bags(db):
|
||||
"""测试查询所有合并bags的parent-children关系"""
|
||||
# 直接Mock三个查询的返回值，避免复杂的side_effect逻辑
|
||||
|
||||
# Mock第一个查询：获取所有parent_id
|
||||
first_query = MagicMock()
|
||||
first_query.distinct.return_value.scalar_subquery.return_value = [1, 2]
|
||||
|
||||
# Mock第二个查询：获取parent_id到children_name的映射
|
||||
second_query = MagicMock()
|
||||
second_query.join.return_value = second_query
|
||||
second_query.filter.return_value = second_query
|
||||
second_query.order_by.return_value = second_query
|
||||
second_query.all.return_value = [
|
||||
(1, "child1.bag"),
|
||||
(1, "child2.bag"),
|
||||
(2, "child3.bag"),
|
||||
]
|
||||
|
||||
# Mock第三个查询：获取parent_id到parent_name的映射
|
||||
third_query = MagicMock()
|
||||
third_query.filter.return_value = third_query
|
||||
third_query.all.return_value = [
|
||||
(1, "parent1.bag"),
|
||||
(2, "parent2.bag"),
|
||||
]
|
||||
|
||||
# 使用side_effect返回不同的mock对象
|
||||
db.query.side_effect = [first_query, second_query, third_query]
|
||||
|
||||
result = srv.query_all_joined_bags(db)
|
||||
|
||||
# 验证结果结构
|
||||
assert isinstance(result, dict)
|
||||
assert "parent1.bag" in result
|
||||
assert "parent2.bag" in result
|
||||
assert result["parent1.bag"] == ["child1.bag", "child2.bag"]
|
||||
assert result["parent2.bag"] == ["child3.bag"]
|
||||
|
||||
|
||||
def test_query_all_joined_bags_empty(db):
|
||||
"""测试查询所有合并bags - 空结果"""
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.models import JoinedBags
|
||||
|
||||
# Mock空结果
|
||||
db.query.return_value.distinct.return_value.scalar_subquery.return_value = (
|
||||
MagicMock()
|
||||
)
|
||||
|
||||
joined_query_mock = MagicMock()
|
||||
joined_query_mock.join.return_value = joined_query_mock
|
||||
joined_query_mock.filter.return_value = joined_query_mock
|
||||
joined_query_mock.order_by.return_value = joined_query_mock
|
||||
joined_query_mock.all.return_value = []
|
||||
|
||||
def query_side_effect(*args, **kwargs):
|
||||
if len(args) == 1 and args[0] == JoinedBags.parent_id:
|
||||
return db.query.return_value
|
||||
else:
|
||||
return joined_query_mock
|
||||
|
||||
db.query.side_effect = query_side_effect
|
||||
|
||||
result = srv.query_all_joined_bags(db)
|
||||
|
||||
# 验证空结果
|
||||
assert result == {}
|
||||
+1154
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
# Core module
|
||||
|
||||
Shared lib and utils in fst_data_pipeline project
|
||||
@@ -0,0 +1,100 @@
|
||||
import os
|
||||
from dotenv import load_dotenv, find_dotenv
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
"""
|
||||
可重用的配置管理工具类，支持多环境配置和敏感信息管理
|
||||
使用示例：
|
||||
config = ConfigManager(env="production")
|
||||
db_url = config.get("DATABASE_URL")
|
||||
debug_mode = config.get_bool("DEBUG_MODE", default=False)
|
||||
"""
|
||||
|
||||
def __init__(self, env=None, env_file_path=None):
|
||||
"""
|
||||
初始化配置管理器
|
||||
:param env: 环境名称（如'development', 'production'）
|
||||
:param env_file_path: 自定义.env文件路径
|
||||
"""
|
||||
self._loaded = False
|
||||
self.env = env or os.getenv("APP_ENV", "development")
|
||||
self._load_config(env_file_path)
|
||||
|
||||
def _load_config(self, custom_path=None):
|
||||
"""加载环境变量配置"""
|
||||
if self._loaded:
|
||||
return
|
||||
# 确定要加载的.env文件
|
||||
env_file = custom_path or f".env.{self.env}"
|
||||
|
||||
# 尝试加载指定环境的文件，不存在则加载默认.env
|
||||
if not os.path.exists(env_file) and not custom_path:
|
||||
env_file = "../.env"
|
||||
|
||||
# 使用dotenv加载配置
|
||||
if os.path.exists(env_file):
|
||||
load_dotenv(env_file, override=True)
|
||||
else:
|
||||
# 尝试自动发现.env文件
|
||||
found_dotenv = find_dotenv(usecwd=True)
|
||||
if found_dotenv:
|
||||
load_dotenv(found_dotenv, override=True)
|
||||
|
||||
self._loaded = True
|
||||
|
||||
def get(self, key, default=None):
|
||||
"""
|
||||
获取字符串配置值
|
||||
:param key: 配置键名
|
||||
:param default: 默认值（未找到时返回）
|
||||
"""
|
||||
return os.getenv(key, default)
|
||||
|
||||
def get_int(self, key, default=None):
|
||||
"""
|
||||
获取整数配置值
|
||||
:param key: 配置键名
|
||||
:param default: 默认值（未找到或转换失败时返回）
|
||||
"""
|
||||
value = self.get(key)
|
||||
try:
|
||||
return int(value) if value is not None else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
def get_bool(self, key, default=False):
|
||||
"""
|
||||
获取布尔值配置
|
||||
:param key: 配置键名
|
||||
:param default: 默认值（未找到或转换失败时返回）
|
||||
"""
|
||||
value = self.get(key, "").lower()
|
||||
if value in ["true", "1", "yes", "y"]:
|
||||
return True
|
||||
elif value in ["false", "0", "no", "n"]:
|
||||
return False
|
||||
return default
|
||||
|
||||
def get_float(self, key, default=None):
|
||||
"""
|
||||
获取浮点数配置值
|
||||
:param key: 配置键名
|
||||
:param default: 默认值（未找到或转换失败时返回）
|
||||
"""
|
||||
value = self.get(key)
|
||||
try:
|
||||
return float(value) if value is not None else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
def require(self, key):
|
||||
"""
|
||||
获取必须存在的配置值，不存在则抛出异常
|
||||
:param key: 配置键名
|
||||
:raises EnvironmentError: 当配置不存在时
|
||||
"""
|
||||
value = self.get(key)
|
||||
if not value:
|
||||
raise EnvironmentError(f"必需配置项缺失: {key}")
|
||||
return value
|
||||
@@ -0,0 +1,2 @@
|
||||
# fst data production automation pipeline
|
||||
all automation scripts to enable fst data production line.
|
||||
@@ -0,0 +1,439 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
import threading
|
||||
import queue
|
||||
import subprocess
|
||||
import shutil
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from prometheus_client import start_http_server, Counter, Gauge, Summary, Histogram
|
||||
|
||||
# —— 常量 & 路径 —— #
|
||||
BASE = os.getcwd()
|
||||
INPUT_ROOT = os.path.join(BASE, "input")
|
||||
OUTPUT_ROOT = os.path.join(BASE, "output")
|
||||
EMPTY_DIR = os.path.join(BASE, "empty")
|
||||
LOG_DIR = os.path.join(BASE, "logs")
|
||||
|
||||
EXCLUDE_SUBDIRS = ["struct_infos/*"]
|
||||
DOCKER_IMAGE = (
|
||||
"artifact.swfcn.i.mercedes-benz.com/swfcn_docker/perception-dnn/od-net:prod_v0.2"
|
||||
)
|
||||
|
||||
BATCH_SIZE = 10
|
||||
MAX_LOCAL = 20
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAY_S = 2
|
||||
METRICS_PORT = 8001
|
||||
SENTINEL = (None, None)
|
||||
|
||||
# —— 日志配置 —— #
|
||||
os.makedirs(LOG_DIR, exist_ok=True)
|
||||
logger = logging.getLogger("pipeline")
|
||||
logger.setLevel(logging.INFO)
|
||||
h_info = logging.FileHandler(os.path.join(LOG_DIR, "pipeline.log"), encoding="utf-8")
|
||||
h_err = logging.FileHandler(os.path.join(LOG_DIR, "error_tasks.log"), encoding="utf-8")
|
||||
fmt = logging.Formatter("%(asctime)s %(levelname)s [%(threadName)s] %(message)s")
|
||||
h_info.setFormatter(fmt)
|
||||
h_err.setFormatter(fmt)
|
||||
h_err.setLevel(logging.ERROR)
|
||||
logger.addHandler(h_info)
|
||||
logger.addHandler(h_err)
|
||||
|
||||
# —— Prometheus 指标 —— #
|
||||
DL_TOTAL = Counter("pipeline_download_total", "下载尝试总数")
|
||||
DL_FAIL = Counter("pipeline_download_failures", "下载失败总数")
|
||||
DL_RETRY = Counter("pipeline_download_retries", "下载重试总数")
|
||||
PR_TOTAL = Counter("pipeline_process_total", "处理尝试总数")
|
||||
PR_FAIL = Counter("pipeline_process_failures", "处理失败总数")
|
||||
PR_RETRY = Counter("pipeline_process_retries", "处理重试总数")
|
||||
UP_TOTAL = Counter("pipeline_upload_total", "上传尝试总数")
|
||||
UP_FAIL = Counter("pipeline_upload_failures", "上传失败总数")
|
||||
UP_RETRY = Counter("pipeline_upload_retries", "上传重试总数")
|
||||
|
||||
DL_DUR = Summary("pipeline_download_duration_seconds", "单批下载耗时秒")
|
||||
PR_DUR = Summary("pipeline_process_duration_seconds", "单批处理耗时秒")
|
||||
UP_DUR = Summary("pipeline_upload_duration_seconds", "单批上传耗时秒")
|
||||
|
||||
BATCH_SIZE_HIST = Histogram(
|
||||
"pipeline_batch_size",
|
||||
"单批任务中文件夹数量分布",
|
||||
buckets=[1, 10, 20, 50, 100, 200, 500],
|
||||
)
|
||||
FILE_DL_DUR = Histogram(
|
||||
"pipeline_file_download_duration_seconds", "单文件/单目录下载耗时分布"
|
||||
)
|
||||
BATCH_OUT_FILES = Gauge(
|
||||
"pipeline_batch_output_subfolder_count", "单批处理后 output 下指定子文件夹数"
|
||||
)
|
||||
|
||||
Q_BATCH = Gauge("pipeline_queue_batches", "待下载批次数")
|
||||
Q_PROC = Gauge("pipeline_queue_processing", "待处理批次数")
|
||||
Q_UP = Gauge("pipeline_queue_uploading", "待上传批次数")
|
||||
LOCAL_COUNT = Gauge("pipeline_local_subdir_count", "当前本地 input 子文件夹总数")
|
||||
|
||||
# —— 队列 & 控制 —— #
|
||||
batch_q = queue.Queue()
|
||||
proc_q = queue.Queue()
|
||||
up_q = queue.Queue()
|
||||
|
||||
# —— 全局计数 & 锁，用于减少磁盘扫描 —— #
|
||||
_local_counter = 0
|
||||
_counter_lock = threading.Lock()
|
||||
_downloaded_per_batch = {} # batch_id -> 成功下载的子目录数量
|
||||
|
||||
|
||||
def incr_local(n=1, batch_id=None):
|
||||
global _local_counter
|
||||
with _counter_lock:
|
||||
_local_counter += n
|
||||
if batch_id:
|
||||
_downloaded_per_batch.setdefault(batch_id, 0)
|
||||
_downloaded_per_batch[batch_id] += n
|
||||
|
||||
|
||||
def decr_local_batch(batch_id):
|
||||
global _local_counter
|
||||
with _counter_lock:
|
||||
n = _downloaded_per_batch.pop(batch_id, 0)
|
||||
_local_counter -= n
|
||||
if _local_counter < 0:
|
||||
_local_counter = 0
|
||||
|
||||
|
||||
def get_local_count():
|
||||
with _counter_lock:
|
||||
return _local_counter
|
||||
|
||||
|
||||
# —— 子目录限流 —— #
|
||||
def count_local_subdirs_in_batch(batch_dir):
|
||||
# 只统计当前 batch 下直接子目录
|
||||
if not os.path.isdir(batch_dir):
|
||||
return 0
|
||||
return sum(
|
||||
1 for e in os.listdir(batch_dir) if os.path.isdir(os.path.join(batch_dir, e))
|
||||
)
|
||||
|
||||
|
||||
# —— 统一 subprocess + 重试 —— #
|
||||
def run(cmd, timeout=None):
|
||||
"""
|
||||
统一调用 subprocess，记录日志，并返回 (code, timed_out, output).
|
||||
"""
|
||||
logger.info("RUN: %s", " ".join(cmd))
|
||||
try:
|
||||
p = subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
|
||||
)
|
||||
start = time.time()
|
||||
output = []
|
||||
timed_out = False
|
||||
for line in p.stdout:
|
||||
output.append(line)
|
||||
if timeout and (time.time() - start) > timeout:
|
||||
p.kill()
|
||||
timed_out = True
|
||||
break
|
||||
code = p.wait()
|
||||
out_str = "".join(output)
|
||||
if code != 0:
|
||||
logger.warning("CMD 返回非零(%d): %s", code, out_str.strip())
|
||||
return code, timed_out, out_str
|
||||
except Exception as e:
|
||||
logger.exception("RUN 异常: %s", e)
|
||||
return -1, False, ""
|
||||
|
||||
|
||||
def with_retry(tag, func, *args):
|
||||
"""
|
||||
重试包装，func 必须返回 (code, timed_out, output)
|
||||
"""
|
||||
for i in range(1, MAX_RETRIES + 1):
|
||||
code, timed_out, out = func(*args)
|
||||
if code == 0:
|
||||
return True
|
||||
if timed_out:
|
||||
logger.error("%s 超时，不再重试", tag)
|
||||
break
|
||||
# 统计重试
|
||||
if tag.startswith("DL["):
|
||||
DL_RETRY.inc()
|
||||
if tag.startswith("PR["):
|
||||
PR_RETRY.inc()
|
||||
if tag.startswith("UP["):
|
||||
UP_RETRY.inc()
|
||||
logger.warning("%s 重试 %d/%d, output: %s", tag, i, MAX_RETRIES, out.strip())
|
||||
time.sleep(RETRY_DELAY_S)
|
||||
logger.error("%s 最终失败", tag)
|
||||
return False
|
||||
|
||||
|
||||
# —— 删除软连接 —— #
|
||||
def delete_symlinks(root_path):
|
||||
"""
|
||||
调用 find 一次性删除所有软链接，效率更高。
|
||||
"""
|
||||
logger.info("删除软连接: %s", root_path)
|
||||
code, _, out = run(
|
||||
["sudo", "find", root_path, "-type", "l", "-delete"], timeout=120
|
||||
)
|
||||
if code != 0:
|
||||
logger.warning("删除软连接失败: %s", out.strip())
|
||||
|
||||
|
||||
# —— 下载阶段 —— #
|
||||
@DL_DUR.time()
|
||||
def do_download(batch_id, remote_paths, batch_timeout):
|
||||
if batch_id is None:
|
||||
proc_q.put(SENTINEL)
|
||||
return
|
||||
|
||||
DL_TOTAL.inc()
|
||||
start = time.time()
|
||||
in_dir = os.path.join(INPUT_ROOT, batch_id)
|
||||
os.makedirs(in_dir, exist_ok=True)
|
||||
logger.info("DL[%s] 开始, targets=%s", batch_id, remote_paths)
|
||||
|
||||
# 本地同 batch 子目录限流
|
||||
while _local_counter >= MAX_LOCAL:
|
||||
logger.warning("DL[%s] 本地子目录≥%d，sleep 5min", batch_id, MAX_LOCAL)
|
||||
time.sleep(300)
|
||||
|
||||
exclude_flags = []
|
||||
for sub in EXCLUDE_SUBDIRS:
|
||||
exclude_flags += ["--ignore", f"'{sub}'"]
|
||||
|
||||
success_count = 0
|
||||
for remote in remote_paths:
|
||||
# 检查 lidar_gt_pandar128
|
||||
lidar_sub = remote.rstrip("/") + "/lidar_gt_pandar128"
|
||||
code, _, out = run(["coscmd", "list", lidar_sub])
|
||||
if code != 0 or not out.strip():
|
||||
logger.info("DL[%s] 跳过无数据: %s", batch_id, lidar_sub)
|
||||
continue
|
||||
|
||||
elapsed = time.time() - start
|
||||
if elapsed > batch_timeout:
|
||||
logger.error("DL[%s] 超时，停止下载", batch_id)
|
||||
DL_FAIL.inc()
|
||||
break
|
||||
|
||||
f_start = time.time()
|
||||
local_bag = os.path.join(in_dir, os.path.basename(remote))
|
||||
cmd = ["coscmd", "-s", "download", "-r", remote, local_bag] + exclude_flags
|
||||
ok = with_retry(
|
||||
f"DL[{batch_id}]", lambda c: run(c, timeout=batch_timeout - elapsed), cmd
|
||||
)
|
||||
FILE_DL_DUR.observe(time.time() - f_start)
|
||||
if ok:
|
||||
success_count += 1
|
||||
incr_local(1, batch_id) # 成功下载一个子目录
|
||||
else:
|
||||
DL_FAIL.inc()
|
||||
|
||||
logger.info("DL[%s] 完成，成功 %d", batch_id, success_count)
|
||||
proc_q.put((batch_id, (in_dir, remote_paths)))
|
||||
|
||||
|
||||
# —— 处理阶段 —— #
|
||||
@PR_DUR.time()
|
||||
def do_process(batch_id, data, batch_timeout):
|
||||
if batch_id is None:
|
||||
up_q.put(SENTINEL)
|
||||
return
|
||||
|
||||
in_dir, remote_paths = data
|
||||
PR_TOTAL.inc()
|
||||
start = time.time()
|
||||
out_dir = os.path.join(OUTPUT_ROOT, batch_id)
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
logger.info("PR[%s] 开始", batch_id)
|
||||
|
||||
# 统一用 run() 运行 Docker
|
||||
docker_cmd = [
|
||||
"docker",
|
||||
"run",
|
||||
"--rm",
|
||||
"--gpus",
|
||||
"all",
|
||||
"--shm-size=16g",
|
||||
"-v",
|
||||
f"{in_dir}:/input",
|
||||
"-v",
|
||||
f"{out_dir}:/output",
|
||||
DOCKER_IMAGE,
|
||||
"bash",
|
||||
"-c",
|
||||
"cd /code/projects/od_net/lidarnet/ && "
|
||||
"source ../.venv/bin/activate && "
|
||||
"python pipeline.py --input_dir=/input --output_dir=/output",
|
||||
]
|
||||
ok = with_retry(
|
||||
f"PR[{batch_id}]", lambda c: run(c, timeout=batch_timeout), docker_cmd
|
||||
)
|
||||
elapsed = time.time() - start
|
||||
if ok:
|
||||
logger.info("PR[%s] 成功 耗时 %.1fs", batch_id, elapsed)
|
||||
else:
|
||||
PR_FAIL.inc()
|
||||
logger.error("PR[%s] 失败 耗时 %.1fs", batch_id, elapsed)
|
||||
|
||||
# 统计并清理输出子目录
|
||||
existing = []
|
||||
for name in os.listdir(out_dir):
|
||||
path = os.path.join(out_dir, name)
|
||||
if os.path.isdir(path) and name.endswith(".dir"):
|
||||
existing.append(path)
|
||||
BATCH_OUT_FILES.set(len(existing))
|
||||
logger.info("PR[%s] 输出子目录: %s", batch_id, existing)
|
||||
|
||||
# for folder in existing:
|
||||
# for item in os.listdir(folder):
|
||||
# p = os.path.join(folder, item)
|
||||
# if item != "object_tracking":
|
||||
# # 非 object_tracking 全删
|
||||
# if os.path.isdir(p):
|
||||
# shutil.rmtree(p, ignore_errors=True)
|
||||
# else:
|
||||
# os.remove(p)
|
||||
# else:
|
||||
# # 重命名 object_tracking
|
||||
# newp = os.path.join(folder, "object_auto_labeling")
|
||||
# os.rename(p, newp)
|
||||
|
||||
shutil.rmtree(in_dir, ignore_errors=True)
|
||||
up_q.put((batch_id, (out_dir, remote_paths)))
|
||||
|
||||
|
||||
# —— 上传阶段 —— #
|
||||
@UP_DUR.time()
|
||||
def do_upload(batch_id, data, batch_timeout):
|
||||
if batch_id is None:
|
||||
return
|
||||
|
||||
out_dir, remote_paths = data
|
||||
UP_TOTAL.inc()
|
||||
start = time.time()
|
||||
logger.info("UP[%s] 开始", batch_id)
|
||||
|
||||
# 删除中间产物
|
||||
for sub in ("logs", "intermedia_products"):
|
||||
shutil.rmtree(os.path.join(out_dir, sub), ignore_errors=True)
|
||||
|
||||
# 删除所有软连接
|
||||
delete_symlinks(out_dir)
|
||||
|
||||
# 只上传 object_tracking -> remote/aaa
|
||||
for remote in remote_paths:
|
||||
base = remote.rstrip("/")
|
||||
dest = f"{base}/derived/object_auto_labeling"
|
||||
for root, dirs, _ in os.walk(out_dir):
|
||||
if "object_tracking" in dirs:
|
||||
src = os.path.join(root, "object_tracking")
|
||||
elapsed = time.time() - start
|
||||
if elapsed > batch_timeout:
|
||||
logger.error("UP[%s] 超时", batch_id)
|
||||
UP_FAIL.inc()
|
||||
return
|
||||
ok = with_retry(
|
||||
f"UP[{batch_id}]",
|
||||
lambda c: run(c, timeout=batch_timeout - elapsed),
|
||||
["coscmd", "-s", "upload", "-r", src, dest],
|
||||
)
|
||||
if not ok:
|
||||
UP_FAIL.inc()
|
||||
logger.info("UP[%s] 上传完成", batch_id)
|
||||
|
||||
# 清理本地输出 & 输入目录
|
||||
for cmd in [
|
||||
["sudo", "rsync", "-a", "--delete", f"{EMPTY_DIR}/", f"{out_dir}/"],
|
||||
]:
|
||||
code, _, _ = run(cmd, timeout=60)
|
||||
if code != 0:
|
||||
logger.warning("UP[%s] rsync 失败: %s", batch_id, cmd)
|
||||
shutil.rmtree(out_dir, ignore_errors=True)
|
||||
# 新增：删除 input 下对应 batch，释放磁盘
|
||||
in_dir = os.path.join(INPUT_ROOT, batch_id)
|
||||
shutil.rmtree(in_dir, ignore_errors=True)
|
||||
decr_local_batch(batch_id)
|
||||
|
||||
logger.info("UP[%s] 本地清理完成", batch_id)
|
||||
|
||||
|
||||
# —— Worker & 主流程 —— #
|
||||
def worker(q, fn, timeout):
|
||||
while True:
|
||||
bid, data = q.get()
|
||||
try:
|
||||
fn(bid, data, timeout)
|
||||
except Exception:
|
||||
logger.exception("阶段异常，batch=%s", bid)
|
||||
finally:
|
||||
q.task_done()
|
||||
if bid is None:
|
||||
break
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--tasks-file", required=True, help="每行一个 COS 目录路径")
|
||||
p.add_argument("--batch-size", type=int, default=BATCH_SIZE)
|
||||
p.add_argument("--batch-timeout", type=int, default=3600)
|
||||
args = p.parse_args()
|
||||
T = args.batch_timeout
|
||||
|
||||
# 确保基础目录
|
||||
for d in (INPUT_ROOT, OUTPUT_ROOT, EMPTY_DIR, LOG_DIR):
|
||||
os.makedirs(d, exist_ok=True)
|
||||
|
||||
# 读取并分批
|
||||
lines = [
|
||||
line.strip() for line in open(args.tasks_file, encoding="utf-8") if line.strip()
|
||||
]
|
||||
for idx in range(0, len(lines), args.batch_size):
|
||||
blk = lines[idx : idx + args.batch_size]
|
||||
BATCH_SIZE_HIST.observe(len(blk))
|
||||
bid = (
|
||||
datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
|
||||
+ f"_{idx // args.batch_size + 1}"
|
||||
)
|
||||
batch_q.put((bid, blk))
|
||||
batch_q.put(SENTINEL)
|
||||
|
||||
# 启动监控
|
||||
start_http_server(METRICS_PORT)
|
||||
logger.info("Metrics 服务启动，端口 %d", METRICS_PORT)
|
||||
|
||||
# 启动线程
|
||||
threads = [
|
||||
threading.Thread(
|
||||
target=worker, args=(batch_q, do_download, T), name="DL-Worker"
|
||||
),
|
||||
threading.Thread(target=worker, args=(proc_q, do_process, T), name="PR-Worker"),
|
||||
threading.Thread(target=worker, args=(up_q, do_upload, T), name="UP-Worker"),
|
||||
]
|
||||
for t in threads:
|
||||
t.start()
|
||||
|
||||
# 主线程仅更新队列长度和本地目录计数，无全盘扫描
|
||||
while batch_q.unfinished_tasks or proc_q.unfinished_tasks or up_q.unfinished_tasks:
|
||||
Q_BATCH.set(batch_q.qsize())
|
||||
Q_PROC.set(proc_q.qsize())
|
||||
Q_UP.set(up_q.qsize())
|
||||
LOCAL_COUNT.set(get_local_count())
|
||||
time.sleep(1)
|
||||
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
logger.info("所有批次完成")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
# cleanup_bags_psql.py
|
||||
"""
|
||||
批量清理 bag_list 中 fst_indexed=FALSE 且日期早于 N 个月的记录：
|
||||
1. 删除 COS 目录
|
||||
2. 删除关联 6 张表
|
||||
3. 标记 bag_list.is_deleted=TRUE
|
||||
4. 失败自动重试 3 次，失败明细写 delete_failed.log
|
||||
5. 最终生成 CSV 并打印统计
|
||||
"""
|
||||
|
||||
import csv
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import psycopg2
|
||||
import requests
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
from tenacity import (
|
||||
retry,
|
||||
stop_after_attempt,
|
||||
wait_exponential,
|
||||
retry_if_exception_type,
|
||||
)
|
||||
|
||||
# 如果原来用 ConfigManager，保持不动；这里为了单文件可跑，直接用 os.environ 兜底
|
||||
import os
|
||||
|
||||
# ===================== 配置 =====================
|
||||
DB_HOST = os.getenv("ROOT_DB_HOST", "localhost")
|
||||
DB_PORT = int(os.getenv("ROOT_DB_PORT", 5432))
|
||||
DB_NAME = os.getenv("ROOT_DB_NAME", "default_dbname")
|
||||
DB_USER = os.getenv("ROOT_DB_USER", "default_user")
|
||||
DB_PASSWORD = os.getenv("ROOT_DB_PASSWD", "default_password")
|
||||
|
||||
COS_REGION = os.getenv("COS_CLIENT_REGION", "ap-guangzhou")
|
||||
COS_SECRET_ID = os.getenv("COS_CLIENT_SECRET_ID", "default_id")
|
||||
COS_SECRET_KEY = os.getenv("COS_CLIENT_SECRET_KEY", "default_key")
|
||||
COS_BUCKET = os.getenv("COS_BUCKET", "b-perception-e2e-1318950322")
|
||||
|
||||
BASE_URL = os.getenv("ROOT_DB_API", "http://localhost")
|
||||
PROJECT_IDS = (1, 2)
|
||||
MONTHS = 6
|
||||
MAX_WORKERS = 10
|
||||
RETRY_TIMES = 3
|
||||
|
||||
CSV_FILE = f"deleted_bags_{datetime.now():%Y%m%d_%H%M%S}.csv"
|
||||
|
||||
DB_CONF = dict(
|
||||
host=DB_HOST, port=DB_PORT, user=DB_USER, password=DB_PASSWORD, dbname=DB_NAME
|
||||
)
|
||||
|
||||
# ===================== 日志 =====================
|
||||
logging.basicConfig(
|
||||
filename=f"cleanup_{datetime.now():%Y%m%d}.log",
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
)
|
||||
|
||||
# 失败明细日志
|
||||
failure_handler = logging.FileHandler("delete_failed.log", encoding="utf-8")
|
||||
failure_handler.setFormatter(logging.Formatter("%(asctime)s,%(message)s"))
|
||||
logger_failure = logging.getLogger("failure")
|
||||
logger_failure.setLevel(logging.INFO)
|
||||
logger_failure.addHandler(failure_handler)
|
||||
|
||||
# ===================== COS 客户端 =====================
|
||||
cos = CosS3Client(
|
||||
CosConfig(Region=COS_REGION, SecretId=COS_SECRET_ID, SecretKey=COS_SECRET_KEY)
|
||||
)
|
||||
|
||||
# ===================== 统计 =====================
|
||||
stats_lock = threading.Lock()
|
||||
stats = {"total": 0, "success": 0, "fail": 0}
|
||||
|
||||
|
||||
def inc_stat(key: str):
|
||||
with stats_lock:
|
||||
stats[key] += 1
|
||||
|
||||
|
||||
# ===================== 数据库工具 =====================
|
||||
def get_conn():
|
||||
return psycopg2.connect(**DB_CONF)
|
||||
|
||||
|
||||
def fetch_candidates() -> list[tuple[int, str]]:
|
||||
"""返回 [(id, name), ...]"""
|
||||
sql = """
|
||||
SELECT id, name
|
||||
FROM bag_list
|
||||
WHERE project_id = ANY(%s)
|
||||
AND fst_indexed = FALSE
|
||||
AND is_deleted = FALSE
|
||||
"""
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql, (PROJECT_IDS,))
|
||||
return cur.fetchall()
|
||||
|
||||
|
||||
# ===================== 日期过滤 =====================
|
||||
DATE_RE = re.compile(r"_(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})_")
|
||||
|
||||
|
||||
def need_delete(name: str) -> bool:
|
||||
days = 30 * MONTHS
|
||||
m = DATE_RE.search(name)
|
||||
if not m:
|
||||
return False
|
||||
dt = datetime.strptime("".join(m.groups()), "%Y%m%d%H%M%S")
|
||||
return dt < datetime.utcnow() - timedelta(days=days)
|
||||
|
||||
|
||||
# ===================== 调接口拿 COS 路径 =====================
|
||||
def get_pangu_detail(bag_name: str):
|
||||
url = f"{BASE_URL}/api/bags/pangu/detail"
|
||||
resp = requests.get(url, params={"bagName": bag_name}, timeout=10)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
return data["dataPath"], data["rawPath"]
|
||||
|
||||
|
||||
# ===================== COS 批量删前缀 =====================
|
||||
def cos_delete_prefix(prefix: str):
|
||||
if not prefix:
|
||||
return
|
||||
marker = ""
|
||||
while True:
|
||||
resp = cos.list_objects(Bucket=COS_BUCKET, Prefix=prefix, Marker=marker)
|
||||
contents = resp.get("Contents", [])
|
||||
if not contents:
|
||||
break
|
||||
keys = [{"Key": obj["Key"]} for obj in contents]
|
||||
cos.delete_objects(Bucket=COS_BUCKET, Delete={"Objects": keys})
|
||||
if resp.get("IsTruncated") == "false":
|
||||
break
|
||||
marker = resp["NextMarker"]
|
||||
|
||||
|
||||
# ===================== 重试包装 =====================
|
||||
@retry(
|
||||
stop=stop_after_attempt(RETRY_TIMES),
|
||||
wait=wait_exponential(multiplier=2, min=4, max=30),
|
||||
retry=retry_if_exception_type((
|
||||
requests.exceptions.RequestException,
|
||||
psycopg2.OperationalError,
|
||||
)),
|
||||
reraise=True,
|
||||
)
|
||||
def _do_cleanup(bag_id: int, bag_name: str):
|
||||
# 1. 拿路径并删 COS
|
||||
data_path, raw_path = get_pangu_detail(bag_name)
|
||||
cos_delete_prefix(data_path)
|
||||
cos_delete_prefix(raw_path)
|
||||
|
||||
# 2. 一个事务删关联表
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
DELETE FROM bag_lifecycle WHERE bag_name = %s;
|
||||
DELETE FROM secondary_pangu WHERE bag_id = %s;
|
||||
DELETE FROM secondary_minerva WHERE bag_id = %s;
|
||||
DELETE FROM bag_topic WHERE bag_id = %s;
|
||||
DELETE FROM bag_reserved_tag WHERE bag_id = %s;
|
||||
DELETE FROM fst_bag WHERE bag_id = %s;
|
||||
UPDATE bag_list SET is_deleted = TRUE WHERE id = %s;
|
||||
""",
|
||||
(bag_name, bag_id, bag_id, bag_id, bag_id, bag_id, bag_id),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def process_one(bag_id: int, bag_name: str) -> tuple[bool, int, str]:
|
||||
"""外层统一入口：负责统计、重试、失败落盘"""
|
||||
inc_stat("total")
|
||||
try:
|
||||
_do_cleanup(bag_id, bag_name)
|
||||
inc_stat("success")
|
||||
logging.info("Cleaned bag_id=%d name=%s", bag_id, bag_name)
|
||||
return True, bag_id, bag_name
|
||||
except Exception as e:
|
||||
inc_stat("fail")
|
||||
# 失败明细落盘
|
||||
logger_failure.info("%d,%s,%s", bag_id, bag_name, str(e).replace(",", ";"))
|
||||
logging.error("Failed bag_id=%d name=%s error=%s", bag_id, bag_name, e)
|
||||
return False, bag_id, bag_name
|
||||
|
||||
|
||||
# ===================== 主流程 =====================
|
||||
def main():
|
||||
candidates = fetch_candidates()
|
||||
logging.info("Fetched %d candidate bags", len(candidates))
|
||||
|
||||
to_delete = [(bid, nm) for bid, nm in candidates if need_delete(nm)]
|
||||
logging.info("Need delete %d bags", len(to_delete))
|
||||
if not to_delete:
|
||||
logging.info("Nothing to delete, exit.")
|
||||
return
|
||||
|
||||
results = []
|
||||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
|
||||
futures = [pool.submit(process_one, bid, nm) for bid, nm in to_delete]
|
||||
for f in as_completed(futures):
|
||||
results.append(f.result())
|
||||
|
||||
# 写 CSV
|
||||
with open(CSV_FILE, "w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.writer(f)
|
||||
writer.writerow(["id", "name", "success"])
|
||||
for ok, bid, nm in results:
|
||||
writer.writerow([bid, nm, ok])
|
||||
|
||||
# 最终汇总
|
||||
logging.info(
|
||||
"=== Done: Total=%d Success=%d Fail=%d. CSV=%s FailureDetail=delete_failed.log ===",
|
||||
stats["total"],
|
||||
stats["success"],
|
||||
stats["fail"],
|
||||
CSV_FILE,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,270 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import csv
|
||||
import sys
|
||||
import shutil
|
||||
import logging
|
||||
from logging.handlers import TimedRotatingFileHandler
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import argparse
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
from tenacity import retry, stop_after_attempt, wait_fixed, retry_if_exception_type
|
||||
import psycopg2
|
||||
from psycopg2 import OperationalError
|
||||
from subprocess import CalledProcessError
|
||||
from tqdm import tqdm
|
||||
from fst_data_pipeline.core.config_manager import ConfigManager
|
||||
|
||||
# ==== 1. 全局目录管理 ====
|
||||
ROOT_DIR = Path(__file__).parent.resolve()
|
||||
LOG_DIR = ROOT_DIR / "logs"
|
||||
DOWNLOAD_DIR = ROOT_DIR / "downloads"
|
||||
VIDEO_DIR = ROOT_DIR / "videos"
|
||||
for d in (LOG_DIR, DOWNLOAD_DIR, VIDEO_DIR):
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ==== 2. 日志配置 ====
|
||||
logger = logging.getLogger("bag_pipeline")
|
||||
logger.setLevel(logging.DEBUG)
|
||||
ch = logging.StreamHandler(sys.stdout)
|
||||
ch.setLevel(logging.INFO)
|
||||
ch.setFormatter(logging.Formatter("[%(levelname)s] %(message)s"))
|
||||
fh = TimedRotatingFileHandler(
|
||||
LOG_DIR / "pipeline.log", when="midnight", backupCount=7, encoding="utf-8"
|
||||
)
|
||||
fh.setLevel(logging.DEBUG)
|
||||
fh.setFormatter(
|
||||
logging.Formatter("%(asctime)s %(levelname)s [%(processName)s] %(message)s")
|
||||
)
|
||||
logger.addHandler(ch)
|
||||
logger.addHandler(fh)
|
||||
|
||||
# ==== 3. 数据库连接工厂 ====
|
||||
config = ConfigManager()
|
||||
DB_HOST = config.get("ROOT_DB_HOST", "localhost")
|
||||
DB_PORT = config.get_int("ROOT_DB_PORT", 5432)
|
||||
DB_NAME = config.get("ROOT_DB_NAME", "default_dbname")
|
||||
DB_USER = config.get("ROOT_DB_USER", "default_user")
|
||||
DB_PASSWORD = config.get("ROOT_DB_PASSWD", "default_password")
|
||||
|
||||
DB_CFG = dict(
|
||||
host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD
|
||||
)
|
||||
|
||||
|
||||
def get_conn():
|
||||
return psycopg2.connect(**DB_CFG)
|
||||
|
||||
|
||||
def read_and_validate_csv(path):
|
||||
required = {"bag name", "FST 1st node", "FST 2nd node", "FST 3rd node"}
|
||||
records = []
|
||||
seen_bags = set()
|
||||
with open(path, encoding="utf-8", newline="") as f:
|
||||
reader = csv.DictReader(f)
|
||||
headers = set(reader.fieldnames or [])
|
||||
if not required.issubset(headers):
|
||||
missing = required - headers
|
||||
raise ValueError(f"CSV 缺少必要列: {missing}")
|
||||
for i, row in enumerate(reader, start=2):
|
||||
bag = row["bag name"].strip()
|
||||
if not bag:
|
||||
raise ValueError(f"第{i}行 bag name 为空")
|
||||
if bag in seen_bags:
|
||||
raise ValueError(f'第{i}行 bag 重复: "{bag}"')
|
||||
seen_bags.add(bag)
|
||||
nodes = []
|
||||
for col in ("FST 1st node", "FST 2nd node", "FST 3rd node"):
|
||||
v = row.get(col, "").strip()
|
||||
if v:
|
||||
nodes.append(v)
|
||||
if not nodes:
|
||||
raise ValueError(f"第{i}行没有任何 FST 节点")
|
||||
records.append({"bag": bag, "nodes": nodes, "first": nodes[0]})
|
||||
return records
|
||||
|
||||
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_fixed(5),
|
||||
retry=retry_if_exception_type((OperationalError, CalledProcessError)),
|
||||
)
|
||||
def process_record(rec) -> bool:
|
||||
bag, first = rec["bag"], rec["first"]
|
||||
logger.info(f"[{bag}] 开始处理")
|
||||
|
||||
# 5.1 查 data_path
|
||||
try:
|
||||
conn = get_conn()
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT data_path FROM main_pangu WHERE bag_path LIKE %s LIMIT 1",
|
||||
(f"%{bag}%",),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
logger.error(f"[{bag}] 查询 data_path 出错: {e}")
|
||||
raise
|
||||
if not row:
|
||||
logger.error(f"[{bag}] main_pangu 未找到记录，跳过")
|
||||
return False
|
||||
data_path = row[0]
|
||||
|
||||
# 5.2 下载图片
|
||||
dst_dir = DOWNLOAD_DIR / bag / "camera_front_wide"
|
||||
dst_dir.mkdir(parents=True, exist_ok=True)
|
||||
src_path = data_path.partition("/")[2] + "/camera_front_wide/"
|
||||
cmd_dl = ["coscmd", "download", "-r", src_path, str(dst_dir)]
|
||||
try:
|
||||
subprocess.run(cmd_dl, check=True, stdout=subprocess.DEVNULL)
|
||||
except CalledProcessError as e:
|
||||
logger.error(f"[{bag}] 下载失败: {e}")
|
||||
raise
|
||||
|
||||
# 5.3 生成 list.txt
|
||||
imgs = sorted(
|
||||
p.name for p in dst_dir.iterdir() if p.suffix.lower() in (".jpg", ".png")
|
||||
)
|
||||
sampled = imgs[::3]
|
||||
if not imgs:
|
||||
logger.error(f"[{bag}] 未找到任何图片，跳过")
|
||||
shutil.rmtree(DOWNLOAD_DIR / bag, ignore_errors=True)
|
||||
return False
|
||||
(dst_dir / "list.txt").write_text(
|
||||
"\n".join(f'file "{fn}"' for fn in sampled), encoding="utf-8"
|
||||
)
|
||||
|
||||
# 5.4 ffmpeg 合成
|
||||
out_dir = VIDEO_DIR / first
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_mp4 = out_dir / f"{bag}.mp4"
|
||||
cmd_ff = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-f",
|
||||
"concat",
|
||||
"-safe",
|
||||
"0",
|
||||
"-i",
|
||||
"list.txt",
|
||||
"-vf",
|
||||
"scale=640:360",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
str(out_mp4),
|
||||
]
|
||||
try:
|
||||
subprocess.run(cmd_ff, cwd=dst_dir, check=True)
|
||||
except CalledProcessError as e:
|
||||
logger.error(f"[{bag}] 合成失败: {e}")
|
||||
raise
|
||||
|
||||
# 5.5 更新数据库
|
||||
try:
|
||||
conn = get_conn()
|
||||
with conn:
|
||||
with conn.cursor() as cur:
|
||||
# 5.5.1 bag_list
|
||||
cur.execute(
|
||||
"SELECT id,fst_indexed FROM bag_list "
|
||||
"WHERE name=%s AND is_deleted=FALSE AND is_active_data=TRUE",
|
||||
(bag,),
|
||||
)
|
||||
r = cur.fetchone()
|
||||
if not r:
|
||||
logger.error(f"[{bag}] bag_list 未找到，跳过更新")
|
||||
conn.rollback()
|
||||
return False
|
||||
bag_id, fst_indexed = r
|
||||
if not fst_indexed:
|
||||
cur.execute(
|
||||
"UPDATE bag_list SET fst_indexed=TRUE WHERE id=%s", (bag_id,)
|
||||
)
|
||||
# 5.5.2 逐级关联 fst
|
||||
parent = None
|
||||
for name in rec["nodes"]:
|
||||
cur.execute("SELECT id,parent_id FROM fst WHERE name=%s", (name,))
|
||||
fr = cur.fetchone()
|
||||
if not fr:
|
||||
logger.error(f"[{bag}] FST 节点不存在 '{name}'，跳过")
|
||||
conn.rollback()
|
||||
return False
|
||||
fid, actual_parent = fr
|
||||
if parent is not None and actual_parent != parent:
|
||||
logger.error(
|
||||
f"[{bag}] 节点 '{name}' 父级校验失败 "
|
||||
f"(actual={actual_parent}!=expect={parent})，跳过"
|
||||
)
|
||||
conn.rollback()
|
||||
return False
|
||||
|
||||
# ---- 确保没插入过才 update bag_sum ----
|
||||
cur.execute(
|
||||
"SELECT 1 FROM fst_bag WHERE bag_id=%s AND fst_node_id=%s",
|
||||
(bag_id, fid),
|
||||
)
|
||||
if not cur.fetchone():
|
||||
cur.execute(
|
||||
"INSERT INTO fst_bag(bag_id,fst_node_id) VALUES(%s,%s)",
|
||||
(bag_id, fid),
|
||||
)
|
||||
cur.execute(
|
||||
"UPDATE fst SET bag_sum = bag_sum + 1 WHERE id=%s", (fid,)
|
||||
)
|
||||
parent = fid
|
||||
except OperationalError as e:
|
||||
logger.error(f"[{bag}] 更新DB连接错误: {e}")
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception(f"[{bag}] 更新DB时未知错误，跳过")
|
||||
return False
|
||||
finally:
|
||||
if conn:
|
||||
conn.close()
|
||||
|
||||
# 5.6 清理 & 成功日志
|
||||
shutil.rmtree(DOWNLOAD_DIR / bag, ignore_errors=True)
|
||||
logger.info(f"[{bag}] 处理成功")
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="批量下载→合成→更新DB")
|
||||
parser.add_argument("csv_path", help="CSV 文件路径")
|
||||
parser.add_argument("-w", "--workers", type=int, default=4, help="并行 worker 数")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
records = read_and_validate_csv(args.csv_path)
|
||||
except Exception as e:
|
||||
logger.error(f"CSV 校验失败: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
total = len(records)
|
||||
logger.info(f"共 {total} 条，启动 {args.workers} 并行 worker")
|
||||
|
||||
success = skipped = failed = 0
|
||||
with ProcessPoolExecutor(max_workers=args.workers) as exe:
|
||||
fut2bag = {exe.submit(process_record, rec): rec["bag"] for rec in records}
|
||||
for fut in tqdm(as_completed(fut2bag), total=total, desc="进度"):
|
||||
bag = fut2bag[fut]
|
||||
try:
|
||||
ok = fut.result()
|
||||
if ok:
|
||||
success += 1
|
||||
else:
|
||||
skipped += 1
|
||||
except Exception as e:
|
||||
logger.error(f"[{bag}] 最终失败: {e}")
|
||||
failed += 1
|
||||
|
||||
logger.info(f"结束：{success} 成功，{skipped} 跳过，{failed} 失败，共 {total}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
-----------------------------------------------------------------------------------
|
||||
功能概述:
|
||||
把数据库里的rosbag的经纬度降采样, 提取, 画图生成每周采集覆盖度在SD map上的分布情况
|
||||
|
||||
使用示例:
|
||||
|
||||
|
||||
配置说明:
|
||||
-----------------------------------------------------------------------------------
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import csv
|
||||
import time
|
||||
import math
|
||||
import logging
|
||||
import threading
|
||||
import argparse
|
||||
from io import StringIO
|
||||
from http.server import HTTPServer, SimpleHTTPRequestHandler
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
import numpy as np
|
||||
import psycopg2
|
||||
from shapely import wkb
|
||||
import folium
|
||||
from folium import TileLayer, FeatureGroup, LayerControl
|
||||
from branca.element import Element
|
||||
from branca.colormap import linear
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
from tqdm import tqdm
|
||||
from fst_data_pipeline.core.config_manager import ConfigManager
|
||||
|
||||
|
||||
# ====== 配置区 ======
|
||||
config = ConfigManager()
|
||||
DB_HOST = config.get("ROOT_DB_HOST", "localhost")
|
||||
DB_PORT = config.get_int("ROOT_DB_PORT", 5432)
|
||||
DB_NAME = config.get("ROOT_DB_NAME", "default_dbname")
|
||||
DB_USER = config.get("ROOT_DB_USER", "default_user")
|
||||
DB_PASSWORD = config.get("ROOT_DB_PASSWD", "default_password")
|
||||
|
||||
COS_REGION = config.get("COS_CLIENT_REGION", "default_region")
|
||||
COS_SECRET_ID = config.get("COS_CLIENT_SECRET_ID", "default_id")
|
||||
COS_SECRET_KEY = config.get("COS_CLIENT_SECRET_KEY", "default_key")
|
||||
|
||||
# ===== 日志配置 =====
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
format="%(asctime)s %(levelname)s [%(threadName)s] %(message)s",
|
||||
datefmt="%Y-%m-%dT%H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger("gnss_map")
|
||||
|
||||
|
||||
# ===== 静态文件服务 (带 CORS) =====
|
||||
class CORSHandler(SimpleHTTPRequestHandler):
|
||||
def end_headers(self):
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
super().end_headers()
|
||||
|
||||
|
||||
def serve_map(port: int, web_dir: str):
|
||||
os.chdir(web_dir)
|
||||
logger.info("Serving static files from %s", web_dir)
|
||||
HTTPServer(("0.0.0.0", port), CORSHandler).serve_forever()
|
||||
|
||||
|
||||
# ===== 工具：haversine 计算距离 =====
|
||||
def haversine(lat1, lon1, lat2, lon2):
|
||||
R = 6371000.0
|
||||
φ1, φ2 = math.radians(lat1), math.radians(lat2)
|
||||
dφ = math.radians(lat2 - lat1)
|
||||
dλ = math.radians(lon2 - lon1)
|
||||
a = math.sin(dφ / 2) ** 2 + math.cos(φ1) * math.cos(φ2) * math.sin(dλ / 2) ** 2
|
||||
return 2 * R * math.asin(math.sqrt(a))
|
||||
|
||||
|
||||
# ===== 单 bag 处理：下载→去重→下采样 → 返回 (name, pts_wkt) =====
|
||||
def process_bag(name, cos, bucket, prefix):
|
||||
logger.debug("Start processing bag %s", name)
|
||||
key = f"{prefix}/{name}.dir/raw_gnss.csv"
|
||||
text = None
|
||||
for attempt in range(1, 4):
|
||||
try:
|
||||
resp = cos.get_object(Bucket=bucket, Key=key)
|
||||
text = resp["Body"].get_raw_stream().read().decode("utf-8")
|
||||
logger.debug("[%s] downloaded %d bytes", name, len(text))
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning("[%s] download failed (attempt %d): %s", name, attempt, e)
|
||||
time.sleep(1)
|
||||
if not text:
|
||||
logger.error("[%s] all download attempts failed → skip", name)
|
||||
return None
|
||||
|
||||
rows = list(csv.DictReader(StringIO(text)))
|
||||
unique = []
|
||||
for r in rows:
|
||||
try:
|
||||
lat, lon, alt = float(r["lat"]), float(r["lon"]), float(r.get("alt", 0))
|
||||
except AttributeError:
|
||||
continue
|
||||
if not any(haversine(lat, lon, plat, plon) < 1.0 for plat, plon, _ in unique):
|
||||
unique.append((lat, lon, alt))
|
||||
logger.debug("[%s] unique points: %d", name, len(unique))
|
||||
if not unique:
|
||||
return None
|
||||
|
||||
# 下采样最多30点
|
||||
if len(unique) > 30:
|
||||
idx = np.linspace(0, len(unique) - 1, 30, dtype=int)
|
||||
sampled = [unique[i] for i in idx]
|
||||
else:
|
||||
sampled = unique
|
||||
logger.info("[%s] sampled %d points", name, len(sampled))
|
||||
|
||||
# 仅拼接 ST_MakePoint(...) 逗号列表，留给 SQL 拼 ARRAY[]
|
||||
pts_wkt = ",".join(f"ST_MakePoint({lon},{lat},{alt})" for lat, lon, alt in sampled)
|
||||
return name, pts_wkt
|
||||
|
||||
|
||||
# ===== 主流程 =====
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("-p", "--map-port", type=int, default=8000)
|
||||
p.add_argument("-u", "--tile-url", required=True)
|
||||
p.add_argument("-w", "--workers", type=int, default=8)
|
||||
args = p.parse_args()
|
||||
|
||||
# COS client
|
||||
cos = CosS3Client(
|
||||
CosConfig(
|
||||
Region=COS_REGION,
|
||||
SecretId=COS_SECRET_ID,
|
||||
SecretKey=COS_SECRET_KEY,
|
||||
)
|
||||
)
|
||||
BUCKET = config.get("bucket", "b-perception-e2e-1318950322")
|
||||
PREFIX = config.get("prefix", "mb_raw_rosbag_decode_dirs")
|
||||
|
||||
# DB 配置
|
||||
DB_CONF = dict(
|
||||
host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD
|
||||
)
|
||||
|
||||
# 1. 取待处理 bag
|
||||
logger.info("Fetch unprocessed bags from DB")
|
||||
with psycopg2.connect(**DB_CONF) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT bl.name
|
||||
FROM bag_list bl
|
||||
LEFT JOIN geometry_info gi ON gi.rosbag_name=bl.name
|
||||
WHERE bl.is_decoded=TRUE
|
||||
AND gi.rosbag_name IS NULL
|
||||
"""
|
||||
)
|
||||
names = [r[0] for r in cur.fetchall()]
|
||||
logger.info("To process: %d bags", len(names))
|
||||
|
||||
# 2. 并行下载 & 处理
|
||||
results = []
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as ex:
|
||||
futures = {ex.submit(process_bag, n, cos, BUCKET, PREFIX): n for n in names}
|
||||
for f in tqdm(as_completed(futures), total=len(names), desc="Processing"):
|
||||
n = futures[f]
|
||||
try:
|
||||
r = f.result()
|
||||
if r:
|
||||
results.append(r)
|
||||
except Exception as e:
|
||||
logger.error("[%s] error: %s", n, e)
|
||||
logger.info("Valid processed bags: %d", len(results))
|
||||
if not results:
|
||||
logger.warning("No new data → skip insertion, proceed to visualization")
|
||||
|
||||
# 3. 批量插入 geometry_info
|
||||
if results:
|
||||
logger.info("Batch inserting into geometry_info")
|
||||
with psycopg2.connect(**DB_CONF) as conn:
|
||||
with conn.cursor() as cur:
|
||||
batch = 50
|
||||
for i in range(0, len(results), batch):
|
||||
chunk = results[i : i + batch]
|
||||
# 手工拼 VALUES 列表
|
||||
vals = []
|
||||
for name, pts in chunk:
|
||||
esc = name.replace("'", "''")
|
||||
vals.append(f"('{esc}', ARRAY[{pts}]::geometry(PointZ)[])")
|
||||
sql = f"""
|
||||
INSERT INTO geometry_info(rosbag_name,gnss_downsampled_points)
|
||||
VALUES {",".join(vals)}
|
||||
ON CONFLICT(rosbag_name) DO NOTHING
|
||||
"""
|
||||
cur.execute(sql)
|
||||
conn.commit()
|
||||
logger.info(
|
||||
"Inserted batch %d~%d", i + 1, min(i + batch, len(results))
|
||||
)
|
||||
logger.info("Batch insert done, total: %d", len(results))
|
||||
|
||||
# 4. 从 DB 读取所有 geometry
|
||||
logger.info("Load all geometries from DB")
|
||||
with psycopg2.connect(**DB_CONF) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"SELECT rosbag_name, gnss_downsampled_points FROM geometry_info"
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
|
||||
by_bag, all_pts = {}, []
|
||||
for name, arr in tqdm(rows, desc="Loading geometry"):
|
||||
pts = []
|
||||
for hw in arr.strip("{}").split(","):
|
||||
h = re.sub(r"[^0-9A-Fa-f]", "", hw)
|
||||
p = wkb.loads(bytes.fromhex(h))
|
||||
pts.append((p.y, p.x))
|
||||
all_pts.append((p.y, p.x))
|
||||
if pts:
|
||||
by_bag[name] = pts
|
||||
|
||||
if all_pts:
|
||||
lats = [p[0] for p in all_pts]
|
||||
lons = [p[1] for p in all_pts]
|
||||
center = (sum(lats) / len(lats), sum(lons) / len(lons))
|
||||
else:
|
||||
logger.warning("No points in DB → default center (0,0)")
|
||||
center = (0.0, 0.0)
|
||||
logger.info("Map center: %s", center)
|
||||
|
||||
# 5. Folium 地图 & 两个图层（视图）
|
||||
m = folium.Map(
|
||||
location=center,
|
||||
zoom_start=6,
|
||||
tiles=None,
|
||||
prefer_canvas=True,
|
||||
control_scale=True,
|
||||
zoom_control=True,
|
||||
)
|
||||
TileLayer(tiles=args.tile_url, overlay=True, control=False, attr=" ").add_to(m)
|
||||
|
||||
header = m.get_root().header
|
||||
for tag in [
|
||||
'<link rel="stylesheet" href="leaflet.css"/>',
|
||||
'<link rel="stylesheet" href="MarkerCluster.css"/>',
|
||||
'<link rel="stylesheet" href="MarkerCluster.Default.css"/>',
|
||||
'<link rel="stylesheet" href="leaflet.awesome-markers.css"/>',
|
||||
'<link rel="stylesheet" href="leaflet.awesome.rotate.min.css"/>',
|
||||
'<link rel="stylesheet" href="bootstrap.min.css"/>',
|
||||
'<link rel="stylesheet" href="all.min.css"/>',
|
||||
'<script src="jquery-1.12.4.min.js"></script>',
|
||||
'<script src="bootstrap.bundle.min.js"></script>',
|
||||
'<script src="leaflet.js"></script>',
|
||||
'<script src="leaflet.markercluster.js"></script>',
|
||||
'<script src="leaflet.awesome-markers.js"></script>',
|
||||
]:
|
||||
header.add_child(Element(tag))
|
||||
|
||||
# 轨迹视图
|
||||
fg1 = FeatureGroup(name="Trajectory", overlay=False, show=True)
|
||||
for pts in by_bag.values():
|
||||
folium.PolyLine(pts, color="#3388ff", weight=1, opacity=0.7).add_to(fg1)
|
||||
m.add_child(fg1)
|
||||
|
||||
# 聚类点视图
|
||||
cnt = {}
|
||||
for lat, lon in all_pts:
|
||||
cnt[(lat, lon)] = cnt.get((lat, lon), 0) + 1
|
||||
if cnt:
|
||||
mi, ma = min(cnt.values()), max(cnt.values())
|
||||
cmap = linear.YlOrRd_09.scale(mi, ma)
|
||||
cmap.caption = "Point Count"
|
||||
else:
|
||||
cmap = None
|
||||
fg2 = FeatureGroup(name="Cluster", overlay=False, show=False)
|
||||
for (lat, lon), c in cnt.items():
|
||||
color = cmap(c) if cmap else "#000000"
|
||||
folium.CircleMarker(
|
||||
location=(lat, lon),
|
||||
radius=1 + min(c, 4),
|
||||
color=color,
|
||||
fill=True,
|
||||
fill_color=color,
|
||||
fill_opacity=0.7,
|
||||
popup=f"count: {c}",
|
||||
).add_to(fg2)
|
||||
m.add_child(fg2)
|
||||
if cmap:
|
||||
m.add_child(cmap)
|
||||
|
||||
LayerControl(collapsed=False).add_to(m)
|
||||
|
||||
# 6. 保存并启动静态服务
|
||||
script_dir = os.path.dirname(__file__)
|
||||
html = os.path.join(script_dir, "map.html")
|
||||
m.save(html)
|
||||
logger.info("Saved map → %s", html)
|
||||
|
||||
threading.Thread(
|
||||
target=serve_map,
|
||||
args=(args.map_port, script_dir),
|
||||
daemon=True,
|
||||
name="HTTP-Server",
|
||||
).start()
|
||||
logger.info("Server running at http://127.0.0.1:%d/map.html", args.map_port)
|
||||
threading.Event().wait()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,293 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""_summary_
|
||||
✅ 批量读取 ROS bag 文件列表(一个 txt 文件里每行是文件路径)
|
||||
✅ 根据文件名解析出车辆、时间、bag 名称
|
||||
✅ 自动:
|
||||
在数据库里插入 / 更新 bag_list(记录名称与 project_id)
|
||||
插入 / 更新 main_pangu(记录 bag 的元信息)
|
||||
插入 / 更新 secondary_pangu(记录 COS 上的文件、目录存在性及链接)
|
||||
读取 meta_info.txt 文件里的 topic 列表, 更新 topic_list 与 bag_topic 关联表
|
||||
✅ 使用腾讯云 COS (对象存储) API 检查文件、目录是否存在, 并提供文件访问链接
|
||||
✅ 支持多线程并发处理, 加快大量 bag 的同步速度
|
||||
✅ 自动维护数据库连接池 (psycopg2 ThreadedConnectionPool)
|
||||
✅ 日志文件记录所有操作, 包括成功、跳过、失败, 便于后续排查
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import logging
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from psycopg2.pool import ThreadedConnectionPool
|
||||
from tqdm import tqdm
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
from qcloud_cos.cos_exception import CosServiceError
|
||||
from fst_data_pipeline.core.config_manager import ConfigManager
|
||||
|
||||
# ====== 配置区 ======
|
||||
config = ConfigManager()
|
||||
DB_HOST = config.get("ROOT_DB_HOST", "localhost")
|
||||
DB_PORT = config.get_int("ROOT_DB_PORT", 5432)
|
||||
DB_NAME = config.get("ROOT_DB_NAME", "default_dbname")
|
||||
DB_USER = config.get("ROOT_DB_USER", "default_user")
|
||||
DB_PASSWORD = config.get("ROOT_DB_PASSWD", "default_password")
|
||||
|
||||
COS_REGION = config.get("COS_CLIENT_REGION", "default_region")
|
||||
COS_SECRET_ID = config.get("COS_CLIENT_SECRET_ID", "default_id")
|
||||
COS_SECRET_KEY = config.get("COS_CLIENT_SECRET_KEY", "default_key")
|
||||
|
||||
BUCKET = config.get("COS_BUCKET", "b-perception-e2e-1318950322")
|
||||
config.require("COS_PREFIX_DECODE")
|
||||
PREFIX = config.get("COS_PREFIX_DECODE", "mb_raw_rosbag_decode_dirs")
|
||||
DOMAIN = f"https://{BUCKET}.cos.{COS_REGION}.myqcloud.com"
|
||||
|
||||
MIN_CONN, MAX_CONN = 1, 10
|
||||
MAX_WORKERS = 8 # 并发线程数, 可调
|
||||
|
||||
# ====== 日志 ======
|
||||
LOGFILE = f"{datetime.now():%Y-%m-%d}.log"
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
handlers=[logging.StreamHandler(), logging.FileHandler(LOGFILE, mode="a")],
|
||||
)
|
||||
logger = logging.getLogger()
|
||||
|
||||
# ====== COS 客户端 ======
|
||||
cos = CosS3Client(
|
||||
CosConfig(Region=COS_REGION, SecretId=COS_SECRET_ID, SecretKey=COS_SECRET_KEY)
|
||||
)
|
||||
|
||||
# ====== 全局 DB 连接池 ======
|
||||
pool = ThreadedConnectionPool(
|
||||
MIN_CONN, MAX_CONN, host=DB_HOST, dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD
|
||||
)
|
||||
|
||||
# ====== 工具函数 ======
|
||||
|
||||
|
||||
def get_db():
|
||||
"""从池里取一个 conn, 设置 autocommit=True"""
|
||||
conn = pool.getconn()
|
||||
conn.autocommit = True
|
||||
return conn, conn.cursor()
|
||||
|
||||
|
||||
def release_db(conn, cur):
|
||||
cur.close()
|
||||
pool.putconn(conn)
|
||||
|
||||
|
||||
def parse_rosbag_line(line):
|
||||
raw = line.strip()
|
||||
name = os.path.basename(raw).split("/")[0]
|
||||
if not name.endswith(".bag"):
|
||||
name += ".bag"
|
||||
m = re.search(r"_(\d{4})(\d{2})(\d{2})-(\d{2})(\d{2})(\d{2})_", name)
|
||||
if not m:
|
||||
raise ValueError(f"无法解析 rosbag 名称/时间: {name}")
|
||||
yy, mm, dd, HH, MN, SS = m.groups()
|
||||
v = name[:8]
|
||||
dt = datetime.strptime(f"{yy}-{mm}-{dd} {HH}:{MN}:{SS}", "%Y-%m-%d %H:%M:%S")
|
||||
return name, {
|
||||
"name": name,
|
||||
"vehicle": v,
|
||||
"datetime": dt,
|
||||
"path": f"{DOMAIN}/mb_cuct_data_collection/{name}",
|
||||
"data_path": f"{DOMAIN}/{PREFIX}{name}.dir",
|
||||
}
|
||||
|
||||
|
||||
def download_txt(name):
|
||||
key = f"{name}.meta_info.txt"
|
||||
try:
|
||||
obj = cos.get_object(Bucket=BUCKET, Key=key)
|
||||
except CosServiceError:
|
||||
return None
|
||||
fd, tmp = tempfile.mkstemp(prefix=name + "_", suffix=".txt")
|
||||
os.close(fd)
|
||||
with open(tmp, "wb") as f:
|
||||
for chunk in obj["Body"].get_raw_stream():
|
||||
f.write(chunk)
|
||||
return tmp
|
||||
|
||||
|
||||
def parse_rosbag_info(fp):
|
||||
tops = []
|
||||
on = False
|
||||
with open(fp, "r", encoding="utf-8") as f:
|
||||
for L in f:
|
||||
if L.startswith("topics:"):
|
||||
on = True
|
||||
continue
|
||||
if on:
|
||||
s = L.strip()
|
||||
if not s:
|
||||
break
|
||||
p = s.split()
|
||||
if len(p) > 3:
|
||||
tops.append((p[0], " ".join(p[3:]).rstrip(":")))
|
||||
return tops
|
||||
|
||||
|
||||
# ==== 读取已成功处理的 bag_name 列表 ====
|
||||
|
||||
|
||||
def load_done(logfile):
|
||||
done = set()
|
||||
if not os.path.exists(logfile):
|
||||
return done
|
||||
pat = re.compile(r"SUCCESS\s+(.+\.bag)\b")
|
||||
with open(logfile, "r", encoding="utf-8") as f:
|
||||
for L in f:
|
||||
m = pat.search(L)
|
||||
if m:
|
||||
done.add(m.group(1))
|
||||
return done
|
||||
|
||||
|
||||
# ==== 单条处理逻辑 ====
|
||||
|
||||
|
||||
def process_one(line):
|
||||
try:
|
||||
bag_name, ent = parse_rosbag_line(line)
|
||||
except ValueError as e:
|
||||
logger.error(e)
|
||||
return
|
||||
|
||||
conn, cur = get_db()
|
||||
try:
|
||||
# 1) bag_list
|
||||
cur.execute(
|
||||
"INSERT INTO bag_list(name,project_id) VALUES(%s,%s) "
|
||||
"ON CONFLICT(name) DO UPDATE SET update_time=NOW() "
|
||||
"RETURNING id",
|
||||
(bag_name, 1),
|
||||
)
|
||||
bag_id = cur.fetchone()[0]
|
||||
|
||||
# 2) main_pangu
|
||||
cur.execute(
|
||||
"INSERT INTO main_pangu(bag_id,name,vehicle,datetime,path,data_path) "
|
||||
"VALUES(%s,%s,%s,%s,%s,%s) "
|
||||
"ON CONFLICT(bag_id) DO UPDATE SET "
|
||||
"name=COALESCE(main_pangu.name,EXCLUDED.name),"
|
||||
"vehicle=COALESCE(main_pangu.vehicle,EXCLUDED.vehicle),"
|
||||
"datetime=COALESCE(main_pangu.datetime,EXCLUDED.datetime),"
|
||||
"path=COALESCE(main_pangu.path,EXCLUDED.path),"
|
||||
"data_path=COALESCE(main_pangu.data_path,EXCLUDED.data_path)",
|
||||
(
|
||||
bag_id,
|
||||
ent["name"],
|
||||
ent["vehicle"],
|
||||
ent["datetime"],
|
||||
ent["path"],
|
||||
ent["data_path"],
|
||||
),
|
||||
)
|
||||
|
||||
# 3) secondary_pangu
|
||||
file_fs = ["raw_gnss.csv", "raw_imu.csv", "ego_motion.csv", "vehicle_wheel.csv"]
|
||||
dir_fs = [
|
||||
"camera_front_wide",
|
||||
"lidar_gt_pandar128",
|
||||
"object_lidar_gt_pandar128_manual",
|
||||
"lidar_fd_multi_scan_raw",
|
||||
"camera_fisheye_left",
|
||||
"camera_fisheye_right",
|
||||
"calibration",
|
||||
]
|
||||
data = {}
|
||||
for f in file_fs:
|
||||
key = f"{PREFIX}{bag_name}.dir/{f}"
|
||||
try:
|
||||
cos.head_object(Bucket=BUCKET, Key=key)
|
||||
data[f] = f"{DOMAIN}/{key}"
|
||||
except CosServiceError:
|
||||
data[f] = None
|
||||
for d in dir_fs:
|
||||
pfx = f"{PREFIX}{bag_name}.dir/{d}/"
|
||||
try:
|
||||
r = cos.list_objects_v2(Bucket=BUCKET, Prefix=pfx, MaxKeys=1)
|
||||
data[d] = f"{DOMAIN}/{pfx}" if r.get("Contents") else None
|
||||
except CosServiceError:
|
||||
data[d] = None
|
||||
data["opt"] = {}
|
||||
cols = ",".join(["bag_id"] + file_fs + dir_fs + ["opt"])
|
||||
vals = [bag_id] + [data[x] for x in file_fs + dir_fs] + [data["opt"]]
|
||||
upd = (
|
||||
",".join(
|
||||
f"{x}=COALESCE(secondary_pangu.{x},EXCLUDED.{x})"
|
||||
for x in file_fs + dir_fs
|
||||
)
|
||||
+ ",opt=EXCLUDED.opt"
|
||||
)
|
||||
cur.execute(
|
||||
f"INSERT INTO secondary_pangu({cols}) VALUES({','.join(['%s'] * len(vals))}) "
|
||||
f"ON CONFLICT(bag_id) DO UPDATE SET {upd}",
|
||||
vals,
|
||||
)
|
||||
|
||||
# 4) topics + 关联
|
||||
txt = download_txt(bag_name)
|
||||
if txt:
|
||||
tops = parse_rosbag_info(txt)
|
||||
os.remove(txt)
|
||||
for nm, ty in tops:
|
||||
cur.execute(
|
||||
"INSERT INTO topic_list(name,type) VALUES(%s,%s) "
|
||||
"ON CONFLICT(name) DO UPDATE "
|
||||
"SET type=COALESCE(topic_list.type,EXCLUDED.type),update_time=NOW() "
|
||||
"RETURNING id",
|
||||
(nm, ty),
|
||||
)
|
||||
tid = cur.fetchone()[0]
|
||||
cur.execute(
|
||||
"INSERT INTO bag_topic(bag_id,topic_id) VALUES(%s,%s) "
|
||||
"ON CONFLICT DO NOTHING",
|
||||
(bag_id, tid),
|
||||
)
|
||||
|
||||
logger.info("SUCCESS %s", bag_name)
|
||||
|
||||
except Exception:
|
||||
logger.exception("FAIL %s", bag_name)
|
||||
finally:
|
||||
release_db(conn, cur)
|
||||
|
||||
|
||||
# ==== 主入口 ====
|
||||
|
||||
|
||||
def main(txtfile, threads):
|
||||
done = load_done(LOGFILE)
|
||||
lines = [L for L in open(txtfile, "r") if L.strip()]
|
||||
tasks = []
|
||||
with ThreadPoolExecutor(max_workers=threads) as ex:
|
||||
for L in lines:
|
||||
try:
|
||||
nm = os.path.basename(L.strip())
|
||||
if not nm.endswith(".bag"):
|
||||
nm += ".bag"
|
||||
except FileExistsError:
|
||||
continue
|
||||
if nm in done:
|
||||
logger.info("SKIP %s", nm)
|
||||
continue
|
||||
tasks.append(ex.submit(process_one, L))
|
||||
|
||||
for _ in tqdm(as_completed(tasks), total=len(tasks), desc="Bags"):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("file", help="ROS bag 列表 txt")
|
||||
p.add_argument("--threads", type=int, default=MAX_WORKERS)
|
||||
args = p.parse_args()
|
||||
main(args.file, args.threads)
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
|
||||
import psycopg2
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
from fst_data_pipeline.core.config_manager import ConfigManager
|
||||
|
||||
# ====== 配置区 ======
|
||||
|
||||
config = ConfigManager()
|
||||
DB_HOST = config.get("ROOT_DB_HOST", "localhost")
|
||||
DB_PORT = config.get_int("ROOT_DB_PORT", 5432)
|
||||
DB_NAME = config.get("ROOT_DB_NAME", "default_dbname")
|
||||
DB_USER = config.get("ROOT_DB_USER", "default_user")
|
||||
DB_PASSWORD = config.get("ROOT_DB_PASSWD", "default_password")
|
||||
|
||||
COS_REGION = config.get("COS_CLIENT_REGION", "default_region")
|
||||
COS_SECRET_ID = config.get("COS_CLIENT_SECRET_ID", "default_id")
|
||||
COS_SECRET_KEY = config.get("COS_CLIENT_SECRET_KEY", "default_key")
|
||||
|
||||
BUCKET = config.get("COS_BUCKET", "b-perception-e2e-1318950322")
|
||||
config.require("COS_PREFIX_DECODE")
|
||||
PREFIX = config.get("COS_PREFIX_DECODE", "mb_raw_rosbag_decode_dirs")
|
||||
DOMAIN = f"https://{BUCKET}.cos.{COS_REGION}.myqcloud.com"
|
||||
|
||||
|
||||
# 输出缺失列表的文件
|
||||
MISSING_FILE = "missing.txt"
|
||||
|
||||
|
||||
def fetch_paths():
|
||||
conn = psycopg2.connect(
|
||||
host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD
|
||||
)
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT data_path FROM bag_list;")
|
||||
paths = [row[0] for row in cur.fetchall()]
|
||||
conn.close()
|
||||
return paths
|
||||
|
||||
|
||||
def check_dirs(paths):
|
||||
cfg = CosConfig(
|
||||
Secret_id=COS_SECRET_ID, Secret_key=COS_SECRET_KEY, Region=COS_REGION
|
||||
)
|
||||
client = CosS3Client(cfg)
|
||||
exist, missing = [], []
|
||||
for p in paths:
|
||||
prefix = p.rstrip("/") + "/lidar_gt_pandar128/"
|
||||
resp = client.list_objects(Bucket=BUCKET, Prefix=prefix, MaxKeys=1)
|
||||
if resp.get("Contents"):
|
||||
exist.append(p)
|
||||
else:
|
||||
missing.append(p)
|
||||
return exist, missing
|
||||
|
||||
|
||||
def write_missing(missing, filepath):
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
for p in missing:
|
||||
f.write(p + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
paths = fetch_paths()
|
||||
exist, missing = check_dirs(paths)
|
||||
|
||||
print(f"总路径数: {len(paths)}")
|
||||
print(f"存在 lidar_gt_pandar128 的: {len(exist)}")
|
||||
for p in exist:
|
||||
print(" [OK] ", p)
|
||||
|
||||
print(f"\n缺失 lidar_gt_pandar128 的: {len(missing)}")
|
||||
for p in missing:
|
||||
print(" [MISS]", p)
|
||||
|
||||
# 将缺失列表写入文件
|
||||
write_missing(missing, MISSING_FILE)
|
||||
print(f"\n已将 {len(missing)} 条缺失记录写入 `{MISSING_FILE}`")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,354 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
四种互斥模式：
|
||||
(默认) —— 全量扫描
|
||||
file --tasks-file—— 指定文件
|
||||
service —— HTTP 服务
|
||||
check —— fst_bag 关联并推送缺失
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple, Dict, Any
|
||||
|
||||
import psycopg2
|
||||
import requests
|
||||
from flask import Flask, jsonify, request
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
|
||||
from fst_data_pipeline.core.config_manager import ConfigManager
|
||||
|
||||
# ---------- 配置 ----------
|
||||
cfg = ConfigManager()
|
||||
|
||||
DB = {
|
||||
"host": cfg.get("ROOT_DB_HOST"),
|
||||
"port": cfg.get_int("ROOT_DB_PORT"),
|
||||
"dbname": cfg.get("ROOT_DB_NAME"),
|
||||
"user": cfg.get("ROOT_DB_USER"),
|
||||
"password": cfg.get("ROOT_DB_PASSWD"),
|
||||
}
|
||||
|
||||
COS_CFG = CosConfig(
|
||||
Region=cfg.get("COS_CLIENT_REGION"),
|
||||
SecretId=cfg.get("COS_CLIENT_SECRET"),
|
||||
SecretKey=cfg.get("COS_CLIENT_KEY"),
|
||||
)
|
||||
BUCKET = cfg.get("COS_BUCKET")
|
||||
|
||||
LD_READY = cfg.get("LD_READY_URL", "http://ld-checker/ready")
|
||||
OD_READY = cfg.get("OD_READY_URL", "http://od-checker/ready")
|
||||
LD_NOTIFY = cfg.get("LD_NOTIFY_URL", "http://ld-checker/notify")
|
||||
OD_NOTIFY = cfg.get("OD_NOTIFY_URL", "http://od-checker/notify")
|
||||
|
||||
LOCAL_API_HOST = cfg.get("ROOT_DB_API", "http://10.0.240.4:5232")
|
||||
|
||||
MAX_WORKERS = 16
|
||||
logging.basicConfig(level=logging.INFO, format="[%(asctime)s] %(message)s")
|
||||
log = logging.getLogger("bag")
|
||||
|
||||
# ---------- 类型 ----------
|
||||
BagRow = Tuple[
|
||||
int, str, int, bool, int, int
|
||||
] # id, name, project_id, is_decoded, od_old, ld_old
|
||||
|
||||
|
||||
# ---------- 工具 ----------
|
||||
def pg_conn():
|
||||
conn = psycopg2.connect(**DB)
|
||||
conn.autocommit = True
|
||||
return conn, conn.cursor()
|
||||
|
||||
|
||||
def cos_client() -> CosS3Client:
|
||||
return CosS3Client(COS_CFG)
|
||||
|
||||
|
||||
def cos_exists(cli: CosS3Client, prefix: str) -> bool:
|
||||
try:
|
||||
return bool(
|
||||
cli.list_objects(Bucket=BUCKET, Prefix=prefix, MaxKeys=1).get("Contents")
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# ---------- HTTP ----------
|
||||
def _get_all_fst_bags() -> List[str]:
|
||||
url = f"{LOCAL_API_HOST}/api/fst/baglist"
|
||||
try:
|
||||
r = requests.get(url, timeout=10)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except Exception as e:
|
||||
log.error("获取 fst bag list 失败: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
def _resolve_paths(bags: List[str]) -> Dict[str, str]:
|
||||
url = f"{LOCAL_API_HOST}/api/bags/pangu/detail"
|
||||
if not bags:
|
||||
return {}
|
||||
try:
|
||||
r = requests.post(url, json=bags, timeout=30)
|
||||
r.raise_for_status()
|
||||
return {
|
||||
name: info["data_path"].split("/", 1)[-1]
|
||||
for name, info in r.json().items()
|
||||
if info.get("data_path")
|
||||
}
|
||||
except Exception as e:
|
||||
log.error("解析 bag path 失败: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
# ---------- 推送 ----------
|
||||
MISSING_LD: List[str] = []
|
||||
MISSING_OD: List[str] = []
|
||||
|
||||
|
||||
def push_list(ready_url: str, notify_url: str, bag_list: List[str], kind: str) -> None:
|
||||
if not bag_list:
|
||||
return
|
||||
path_map = _resolve_paths(bag_list)
|
||||
if not path_map:
|
||||
log.warning("无可推送的 %s 路径", kind)
|
||||
return
|
||||
paths = [path_map[name] for name in bag_list if name in path_map]
|
||||
|
||||
log.info("%s: %d bags → pushing %d paths", kind, len(bag_list), len(paths))
|
||||
while True:
|
||||
try:
|
||||
if requests.get(ready_url, timeout=5).json().get("ready"):
|
||||
break
|
||||
except Exception as e:
|
||||
log.warning("ready check failed: %s", e)
|
||||
time.sleep(2)
|
||||
try:
|
||||
resp = requests.post(notify_url, json=paths, timeout=30)
|
||||
resp.raise_for_status()
|
||||
log.info("%s pushed %d paths, status=%s", kind, len(paths), resp.status_code)
|
||||
except Exception as e:
|
||||
log.error("%s push failed: %s", kind, e)
|
||||
|
||||
|
||||
# ---------- 核心处理 ----------
|
||||
def process_row(row: BagRow, collect_missing: bool = False) -> None:
|
||||
bag_id, name, project_id, _, od_old, ld_old = row
|
||||
|
||||
# 提前检查是否已“完美”
|
||||
conn, cur = pg_conn()
|
||||
try:
|
||||
cur.execute("SELECT reserved_str FROM main_pangu WHERE name=%s", (name,))
|
||||
reserved_str = (cur.fetchone() or ("",))[0]
|
||||
has_md5 = reserved_str.startswith("md5:")
|
||||
if ld_old == 3 and od_old == 3 and has_md5:
|
||||
log.debug("[%s] 已完整，跳过", name)
|
||||
return
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
# 需要真正检查 COS
|
||||
conn, cur = pg_conn()
|
||||
cli = cos_client()
|
||||
try:
|
||||
base = "mb" if project_id == 1 else "mmt"
|
||||
derived = f"{base}_raw_rosbag_decode_dirs/{name}.dir/derived/"
|
||||
|
||||
ld_auto = cos_exists(cli, derived + "LDGT/")
|
||||
ld_man = cos_exists(cli, derived + "LD_manual/")
|
||||
od_auto = cos_exists(cli, derived + "object_auto_labeling/")
|
||||
od_man = cos_exists(cli, derived + "object_manual_labeling/")
|
||||
|
||||
def calc_flag(a: bool, m: bool) -> int:
|
||||
return 3 if a and m else 2 if m else 1 if a else 0
|
||||
|
||||
ld_new = calc_flag(ld_auto, ld_man)
|
||||
od_new = calc_flag(od_auto, od_man)
|
||||
|
||||
# 更新状态
|
||||
for field, old, new in (
|
||||
("ld_annotated", ld_old, ld_new),
|
||||
("od_annotated", od_old, od_new),
|
||||
):
|
||||
if old != new:
|
||||
cur.execute(
|
||||
f"UPDATE bag_list SET {field}=%s WHERE id=%s", (new, bag_id)
|
||||
)
|
||||
log.info("[%s] %s %s→%s", name, field, old, new)
|
||||
|
||||
# 处理 md5
|
||||
if not has_md5:
|
||||
cur.execute("SELECT bag_path FROM main_pangu WHERE name=%s", (name,))
|
||||
bag_path = (cur.fetchone() or ("",))[0]
|
||||
if bag_path:
|
||||
cos_key = bag_path.lstrip("/")
|
||||
tmp_path = f"/tmp/{name}.bag"
|
||||
try:
|
||||
cli.download_file(Bucket=BUCKET, Key=cos_key, DestFilePath=tmp_path)
|
||||
md5 = subprocess.check_output(
|
||||
["md5sum", tmp_path], text=True
|
||||
).split()[0]
|
||||
os.remove(tmp_path)
|
||||
except Exception as e:
|
||||
log.warning("[%s] 下载/计算 md5 失败: %s", name, e)
|
||||
md5 = None
|
||||
|
||||
if md5:
|
||||
cur.execute(
|
||||
"UPDATE main_pangu SET reserved_str='md5:'||%s WHERE name=%s",
|
||||
(md5, name),
|
||||
)
|
||||
cur.execute(
|
||||
"SELECT reserved_json FROM main_pangu WHERE name=%s", (name,)
|
||||
)
|
||||
reserved_json: Dict[str, Any] = json.loads(
|
||||
cur.fetchone()[0] or "{}"
|
||||
)
|
||||
reserved_json.update({
|
||||
"bag_meta": f"https://cla-dev.ca4ad.com/cdi/data/{md5}",
|
||||
"bag_player": f"https://mviz-dev.ca4ad.com/player/v4/?bag_md5={md5}",
|
||||
})
|
||||
cur.execute(
|
||||
"UPDATE main_pangu SET reserved_json=COALESCE(reserved_json,'{}'::jsonb)||%s WHERE name=%s",
|
||||
(json.dumps(reserved_json, ensure_ascii=False), name),
|
||||
)
|
||||
log.info("[%s] 写入 md5:%s", name, md5)
|
||||
|
||||
# 收集缺失
|
||||
if collect_missing:
|
||||
if not ld_auto:
|
||||
MISSING_LD.append(name)
|
||||
if not od_auto:
|
||||
MISSING_OD.append(name)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------- 数据获取 ----------
|
||||
def fetch_rows_from_file(path: Path) -> List[BagRow]:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
names = [l.strip() for l in f if l.strip()]
|
||||
if not names:
|
||||
return []
|
||||
conn, cur = pg_conn()
|
||||
cur.execute("SELECT * FROM bag_list WHERE name = ANY(%s)", (names,))
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
return rows
|
||||
|
||||
|
||||
def fetch_rows_from_all() -> List[BagRow]:
|
||||
conn, cur = pg_conn()
|
||||
cur.execute("SELECT * FROM bag_list")
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
return rows
|
||||
|
||||
|
||||
def fetch_rows_from_fst() -> List[BagRow]:
|
||||
bag_names = _get_all_fst_bags()
|
||||
if not bag_names:
|
||||
return []
|
||||
conn, cur = pg_conn()
|
||||
cur.execute("SELECT * FROM bag_list WHERE name = ANY(%s)", (bag_names,))
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
log.info("fst_bag 关联 %d 条", len(rows))
|
||||
return rows
|
||||
|
||||
|
||||
# ---------- 模式封装 ----------
|
||||
def _run(rows: List[BagRow], collect_missing: bool):
|
||||
MISSING_LD.clear()
|
||||
MISSING_OD.clear()
|
||||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex:
|
||||
list(ex.map(lambda r: process_row(r, collect_missing), rows))
|
||||
if collect_missing:
|
||||
push_list(LD_READY, LD_NOTIFY, MISSING_LD, "LD")
|
||||
push_list(OD_READY, OD_NOTIFY, MISSING_OD, "OD")
|
||||
|
||||
|
||||
def run_all():
|
||||
rows = fetch_rows_from_all()
|
||||
_run(rows, collect_missing=False)
|
||||
log.info("全量扫描完成")
|
||||
|
||||
|
||||
def run_file(path: Path):
|
||||
rows = fetch_rows_from_file(path)
|
||||
_run(rows, collect_missing=False)
|
||||
log.info("file 模式完成")
|
||||
|
||||
|
||||
def run_service(host: str, port: int):
|
||||
app = Flask(__name__)
|
||||
|
||||
@app.route("/ready")
|
||||
def ready():
|
||||
return jsonify(ready=True)
|
||||
|
||||
def handle_notify(bags: List[str]):
|
||||
rows = []
|
||||
if bags:
|
||||
conn, cur = pg_conn()
|
||||
cur.execute("SELECT * FROM bag_list WHERE name = ANY(%s)", (bags,))
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
for r in rows:
|
||||
process_row(r, collect_missing=False)
|
||||
|
||||
@app.route("/notify_ld", methods=["POST"])
|
||||
def notify_ld():
|
||||
bags = request.get_json(force=True)
|
||||
if not isinstance(bags, list):
|
||||
return jsonify(error="need list"), 400
|
||||
handle_notify(bags)
|
||||
return jsonify(status="accepted"), 202
|
||||
|
||||
@app.route("/notify_od", methods=["POST"])
|
||||
def notify_od():
|
||||
bags = request.get_json(force=True)
|
||||
if not isinstance(bags, list):
|
||||
return jsonify(error="need list"), 400
|
||||
handle_notify(bags)
|
||||
return jsonify(status="accepted"), 202
|
||||
|
||||
app.run(host=host, port=port, threaded=True)
|
||||
|
||||
|
||||
def run_check():
|
||||
rows = fetch_rows_from_fst()
|
||||
_run(rows, collect_missing=True)
|
||||
log.info("check 模式完成")
|
||||
|
||||
|
||||
# ---------- CLI ----------
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers(dest="mode", help="运行模式")
|
||||
parser.set_defaults(mode="all")
|
||||
f = sub.add_parser("file", help="读取 bag 列表文件")
|
||||
f.add_argument("--tasks-file", required=True)
|
||||
s = sub.add_parser("service", help="启动 HTTP 服务")
|
||||
s.add_argument("--host", default="0.0.0.0")
|
||||
s.add_argument("--port", type=int, default=8000)
|
||||
sub.add_parser("check", help="仅扫描 fst_bag 关联 bag 并推送缺失")
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.mode == "file":
|
||||
run_file(Path(args.tasks_file))
|
||||
elif args.mode == "service":
|
||||
run_service(args.host, args.port)
|
||||
elif args.mode == "check":
|
||||
run_check()
|
||||
else:
|
||||
run_all()
|
||||
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
-----------------------------------------------------------------------------------
|
||||
功能概述:
|
||||
该脚本用于同步腾讯云 COS(对象存储)上的原始 .bag 文件与 PostgreSQL 中的 bag_list 表。
|
||||
同时根据 COS 上解码输出目录 b/ 下的 .dir 文件, 更新数据库中对应记录的解码状态。
|
||||
支持在数据库同步完成后, 根据 --notify 参数轮询后端服务是否就绪(通过 /ready 接口),
|
||||
并在就绪后将未解码文件列表 POST 到 /notify 接口, 驱动后续解码或处理流程。
|
||||
|
||||
主要流程:
|
||||
1. 连接腾讯云 COS, 列出 a/ 前缀下所有 .bag 文件, 插入数据库(跳过已存在)。
|
||||
2. 查询数据库中未解码且未删除的记录。
|
||||
3. 列出 b/ 前缀下的 .dir 文件, 推断已解码的 .bag 文件, 批量更新数据库标记 is_decoded=TRUE。
|
||||
4. 如果指定 --notify:
|
||||
- 轮询 /ready 接口直到返回 { "ready": true }
|
||||
- 再将剩余未解码文件列表(带前缀)POST 到 /notify 接口。
|
||||
|
||||
依赖环境:
|
||||
- Python 3.6+
|
||||
- psycopg2 (PostgreSQL 驱动)
|
||||
- requests (HTTP 请求)
|
||||
- qcloud_cos (腾讯云 COS Python SDK)
|
||||
|
||||
使用示例:
|
||||
python3 cos_pg_sync.py
|
||||
python3 cos_pg_sync.py --notify
|
||||
|
||||
配置说明:
|
||||
可在脚本开头直接修改:
|
||||
COS_SECRET_ID / COS_SECRET_KEY / COS_REGION / COS_BUCKET
|
||||
PostgreSQL 连接信息 PG_HOST 等
|
||||
GATE_URL / NOTIFY_URL 等服务端接口地址
|
||||
-----------------------------------------------------------------------------------
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import argparse
|
||||
import logging
|
||||
import psycopg2
|
||||
from psycopg2.extras import execute_values
|
||||
import requests
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
from fst_data_pipeline.core.config_manager import ConfigManager
|
||||
|
||||
# ====== 配置区 ======
|
||||
config = ConfigManager()
|
||||
DB_HOST = config.get("ROOT_DB_HOST", "localhost")
|
||||
DB_PORT = config.get_int("ROOT_DB_PORT", 5432)
|
||||
DB_NAME = config.get("ROOT_DB_NAME", "default_dbname")
|
||||
DB_USER = config.get("ROOT_DB_USER", "default_user")
|
||||
DB_PASSWORD = config.get("ROOT_DB_PASSWD", "default_password")
|
||||
|
||||
COS_REGION = config.get("COS_CLIENT_REGION", "default_region")
|
||||
COS_SECRET_ID = config.get("COS_CLIENT_SECRET_ID", "default_id")
|
||||
COS_SECRET_KEY = config.get("COS_CLIENT_SECRET_KEY", "default_key")
|
||||
BUCKET = config.get("COS_BUCKET", "b-perception-e2e-1318950322")
|
||||
|
||||
config.require("COS_PREFIX_DECODE")
|
||||
DECODED_PREFIX = config.get("COS_PREFIX_DECODE", "mb_raw_rosbag_decode_dirs")
|
||||
RAW_BAG_PREFIX = config.get("COS_PREFIX_BAG", "mb_cuct_data_collection")
|
||||
|
||||
DOMAIN = f"https://{BUCKET}.cos.{COS_REGION}.myqcloud.com"
|
||||
|
||||
|
||||
# Service 模式的 HTTP 接口
|
||||
GATE_URL = "https://example.com/ready"
|
||||
NOTIFY_URL = "https://example.com/notify"
|
||||
LOG_FILE = "sync.log"
|
||||
|
||||
|
||||
def init_logger():
|
||||
fmt = "%(asctime)s [%(levelname)s] %(message)s"
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format=fmt,
|
||||
handlers=[
|
||||
logging.StreamHandler(),
|
||||
logging.FileHandler(LOG_FILE, encoding="utf-8"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def get_cos_client():
|
||||
cfg = CosConfig(Region=COS_REGION, SecretId=COS_SECRET_ID, SecretKey=COS_SECRET_KEY)
|
||||
return CosS3Client(cfg)
|
||||
|
||||
|
||||
def list_objs(cos, prefix):
|
||||
"""分页列出 COS 上指定前缀下的所有对象 key"""
|
||||
keys = []
|
||||
token = None
|
||||
while True:
|
||||
params = {
|
||||
"Bucket": BUCKET,
|
||||
"Prefix": prefix,
|
||||
"MaxKeys": 1000,
|
||||
}
|
||||
if token:
|
||||
params["ContinuationToken"] = token
|
||||
resp = cos.list_objects_v2(**params)
|
||||
for obj in resp.get("Contents", []):
|
||||
keys.append(obj["Key"])
|
||||
if resp.get("IsTruncated"):
|
||||
token = resp["NextContinuationToken"]
|
||||
else:
|
||||
break
|
||||
return keys
|
||||
|
||||
|
||||
def wait_for_ready():
|
||||
"""轮询 /ready, 直到返回 JSON { "ready": true }"""
|
||||
logging.info(f"开始轮询 Gate URL: {GATE_URL}")
|
||||
while True:
|
||||
try:
|
||||
r = requests.get(GATE_URL, timeout=10)
|
||||
if r.status_code == 200:
|
||||
data = r.json()
|
||||
if isinstance(data, dict) and data.get("ready") is True:
|
||||
logging.info("Gate 就绪, 开始通知流程")
|
||||
return
|
||||
logging.info(
|
||||
"Gate 未就绪(HTTP %d 或 ready=false), 10 分钟后重试", r.status_code
|
||||
)
|
||||
except Exception as e:
|
||||
logging.warning(f"轮询 Gate 时发生异常: {e}")
|
||||
time.sleep(600) # 10 分钟后重试
|
||||
|
||||
|
||||
def main():
|
||||
init_logger()
|
||||
|
||||
p = argparse.ArgumentParser(
|
||||
description="同步 COS 与 bag_list, 并可选通知未解码列表"
|
||||
)
|
||||
p.add_argument(
|
||||
"--notify",
|
||||
"-n",
|
||||
action="store_true",
|
||||
help="轮询 /ready, 再 POST 剩余未解码列表(带前缀)",
|
||||
)
|
||||
args = p.parse_args()
|
||||
|
||||
# 1. 初始化 COS 客户端和数据库连接
|
||||
cos = get_cos_client()
|
||||
conn = psycopg2.connect(
|
||||
host=DB_HOST, port=DB_PORT, user=DB_USER, password=DB_PASSWORD, dbname=DB_NAME
|
||||
)
|
||||
cur = conn.cursor()
|
||||
|
||||
# —— STEP1：同步 a/ 下的 .bag 到 bag_list(name, project_id) ——
|
||||
keys_a = list_objs(cos, RAW_BAG_PREFIX)
|
||||
# 提取所有 .bag 的名字(不含前缀)
|
||||
names = [os.path.basename(k) for k in keys_a if k.lower().endswith(".bag")]
|
||||
if names:
|
||||
# 默认 project_id=1
|
||||
records = [(n, 1) for n in names]
|
||||
execute_values(
|
||||
cur,
|
||||
"INSERT INTO bag_list(name, project_id) VALUES %s "
|
||||
"ON CONFLICT(name) DO NOTHING",
|
||||
records,
|
||||
)
|
||||
logging.info(f"STEP1: 尝试插入 {len(records)} 条记录, 冲突则跳过")
|
||||
else:
|
||||
logging.info("STEP1: 未发现 .bag 文件, 无需插入")
|
||||
|
||||
# —— STEP2：查询所有 is_decoded=FALSE 且 is_deleted=FALSE 的 name ——
|
||||
cur.execute("SELECT name FROM bag_list WHERE is_decoded=FALSE AND is_deleted=FALSE")
|
||||
undecoded = [row[0] for row in cur.fetchall()]
|
||||
logging.info(f"STEP2: 当前未解码记录数：{len(undecoded)}")
|
||||
|
||||
# —— STEP3：扫描 b/ 下 .dir, 并批量更新 is_decoded ——
|
||||
keys_b = list_objs(cos, DECODED_PREFIX)
|
||||
# available 是所有已解码的 bag 名字(带 .bag 后缀)
|
||||
available = {
|
||||
os.path.basename(k)[:-4] + ".bag" for k in keys_b if k.lower().endswith(".dir")
|
||||
}
|
||||
to_update = list(set(undecoded) & available)
|
||||
if to_update:
|
||||
cur.execute(
|
||||
"UPDATE bag_list "
|
||||
"SET is_decoded=TRUE, update_time=CURRENT_TIMESTAMP "
|
||||
"WHERE name = ANY(%s)",
|
||||
(to_update,),
|
||||
)
|
||||
logging.info(f"STEP3: 标记已解码 {len(to_update)} 条记录")
|
||||
else:
|
||||
logging.info("STEP3: 无需更新解码状态")
|
||||
|
||||
conn.commit()
|
||||
|
||||
# —— 可选通知流程 ——
|
||||
if args.notify:
|
||||
# 1) 等待服务端就绪
|
||||
wait_for_ready()
|
||||
|
||||
# 2) 再次查询剩余未解码名称
|
||||
cur.execute(
|
||||
"SELECT name FROM bag_list WHERE is_decoded=FALSE AND is_deleted=FALSE"
|
||||
)
|
||||
remaining = [row[0] for row in cur.fetchall()]
|
||||
logging.info(f"通知前剩余未解码 {len(remaining)} 条: {remaining}")
|
||||
|
||||
# 3) 补上前缀, POST 给 /notify
|
||||
paths = [RAW_BAG_PREFIX + name for name in remaining]
|
||||
try:
|
||||
resp = requests.post(NOTIFY_URL, json=paths, timeout=10)
|
||||
logging.info(f"POST /notify 返回 HTTP {resp.status_code}: {resp.text}")
|
||||
except Exception as e:
|
||||
logging.error(f"通知失败: {e}")
|
||||
|
||||
# 清理资源
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user