第一次
This commit is contained in:
ZhuJW
676
fst_data_pipeline/apps/API.md
Normal file
676
fst_data_pipeline/apps/API.md
Normal file
@@ -0,0 +1,676 @@
|
||||
ROOT DB API 使用文档
|
||||
========================
|
||||
|
||||
> 版本：1.0.0
|
||||
|
||||
---
|
||||
|
||||
目录
|
||||
----
|
||||
|
||||
* [Bags](#bags)
|
||||
* [FST](#fst)
|
||||
* [Geometry](#geometry)
|
||||
* [Projects](#projects)
|
||||
* [Tags](#tags)
|
||||
* [Topics](#topics)
|
||||
|
||||
---
|
||||
|
||||
Bags
|
||||
----
|
||||
|
||||
### 1. 获取全部 Bags（按项目分组）
|
||||
```
|
||||
GET /api/bags/all
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/bags/all
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"Project A": ["Bag 1", "Bag 2"],
|
||||
"Uncategorized": ["Bag 3"]
|
||||
}
|
||||
```
|
||||
|
||||
#### 错误响应 500
|
||||
```json
|
||||
{"error": "An error occurred"}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. 批量查询 FST 节点
|
||||
```
|
||||
POST /api/bags/fst/nodes
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["a.bag", "b.bag"]
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/fst/nodes \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '["a.bag", "b.bag"]'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"a.bag": [
|
||||
{"name": "node1", "start": 0, "end": 10},
|
||||
{"name": "node2", "start": 11, "end": 20}
|
||||
],
|
||||
"b.bag": [
|
||||
{"name": "node3", "start": 21, "end": 30}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### 错误响应
|
||||
- `400`：Body must be a list of bag names
|
||||
- `500`：An error occurred
|
||||
|
||||
---
|
||||
|
||||
### 3. 获取生命周期信息
|
||||
```
|
||||
POST /api/bags/life_cycle
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
{"bag_names": ["a.bag", "b.bag"]}
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/life_cycle \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"bag_names": ["a.bag", "b.bag"]}'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"a.bag": {
|
||||
"collect_time": "2025-07-16T00:00:00Z",
|
||||
"decode_time": "2025-07-16T01:00:00Z",
|
||||
"fst_index_time": "2025-07-16T02:00:00Z",
|
||||
"mining_time": "2025-07-16T03:00:00Z",
|
||||
"auto_annotate_time": "2025-07-16T04:00:00Z",
|
||||
"manual_annotate_time": "2025-07-16T05:00:00Z",
|
||||
"clone2dev_time": "2025-07-16T06:00:00Z"
|
||||
},
|
||||
"b.bag": { }
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. 批量查询 Minerva 主记录（按 session_id）
|
||||
```
|
||||
POST /api/bags/minerva
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
{"session_ids": ["p1.bag", "p2.bag"]}
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/minerva \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"session_ids": ["p1.bag", "p2.bag"]}'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"p1.bag": [{
|
||||
"session_id": "p1.bag",
|
||||
"vin": "L123456789",
|
||||
"platform": "Platform A",
|
||||
"start_ts": 1626307200,
|
||||
"end_ts": 1626314400,
|
||||
"length": 7200,
|
||||
"datetime": "2025-07-16T00:00:00Z",
|
||||
"path": "/path/to/bag",
|
||||
"converted_path": "/path/to/converted",
|
||||
"gt_path": "/path/to/gt",
|
||||
"reserved_json": {}
|
||||
}],
|
||||
"p2.bag": []
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 5. 批量查询 Minerva 详情（按 bag name）
|
||||
```
|
||||
POST /api/bags/minerva/detail
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["p1.bag", "p2.bag"]
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/minerva/detail \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '["p1.bag", "p2.bag"]'
|
||||
```
|
||||
|
||||
#### 响应格式同 `/api/bags/minerva`
|
||||
|
||||
---
|
||||
|
||||
### 6. 批量查询 Pangu 主记录
|
||||
```
|
||||
POST /api/bags/pangu
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
{"names": ["name1", "name2"]}
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/pangu \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"names": ["name1", "name2"]}'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"name1": [{
|
||||
"name": "Pangu 1",
|
||||
"datetime": "2025-07-16T00:00:00Z",
|
||||
"vehicle": "Vehicle A",
|
||||
"raw_bag_path": "/path/to/bag",
|
||||
"decoded_path": "/path/to/decoded",
|
||||
"reserved_str": "Some reserved string",
|
||||
"reserved_json": {}
|
||||
}],
|
||||
"name2": []
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 7. 批量查询 Pangu 详情（按 bag name）
|
||||
```
|
||||
POST /api/bags/pangu/detail
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["bag1.bag", "bag2.bag"]
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/pangu/detail \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '["bag1.bag", "bag2.bag"]'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"bag1.bag": {"detail": "...", "other_property": "..."},
|
||||
"bag2.bag": {"detail": "...", "other_property": "..."}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 8. 搜索 Main Pangu
|
||||
```
|
||||
POST /api/bags/search/pangu
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
{
|
||||
"name": "Pangu 1",
|
||||
"vehicle": "Vehicle A",
|
||||
"datetime_from": "2025-07-16 00:00:00",
|
||||
"datetime_to": "2025-07-16 23:59:59"
|
||||
}
|
||||
```
|
||||
字段均可选。
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST https://<host>/api/bags/search/pangu \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "Pangu 1"}'
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
[
|
||||
{
|
||||
"name": "Pangu 1",
|
||||
"datetime": "2025-07-16T00:00:00Z",
|
||||
"vehicle": "Vehicle A"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 9. 聚合查询列表
|
||||
```
|
||||
POST /api/bags/search/aggregate?page=1&per_page=20
|
||||
```
|
||||
|
||||
#### 请求参数
|
||||
|
||||
参数来源说明：
|
||||
- collect_start / collect_end：前端日期选择
|
||||
- vehicles：GET /api/bags/vehicles
|
||||
- sw_version / hw_version：GET /api/bags/versions
|
||||
- topics：GET /api/topics/all
|
||||
- tags：GET /api/tags/all
|
||||
- gt_names：GET /api/gt/types
|
||||
- fst_level1：GET /api/fst/print_tree（取 level=1 的节点名；或用 /api/fst/all_nodes 过滤根节点子节点）
|
||||
|
||||
Query 参数：
|
||||
- page: 页码，默认 1
|
||||
- per_page: 每页数量，默认 20，最大 100
|
||||
- debug_sql: 是否打印 SQL（仅日志输出），默认 false
|
||||
|
||||
Body 字段（均可选）：
|
||||
- collect_start: 起始日期，基于 MainPangu.datetime，格式 YYYYMMDD，例如 "20250101"
|
||||
- collect_end: 结束日期，基于 MainPangu.datetime，格式 YYYYMMDD，例如 "20250131"
|
||||
- bag_name: bag 名称模糊匹配（支持 * 通配），例如 "xxx*2025*"
|
||||
- gt_names: GT 名称列表（OR 语义）
|
||||
- fst_level1: FST 一级节点名称（包含其子孙节点）
|
||||
- vehicles: 车号列表（OR 语义，精确匹配）
|
||||
- sw_version: 软件版本，支持 * 通配，例如 "v1.*" 或 "*2024*"
|
||||
- hw_version: 硬件版本，支持 * 通配
|
||||
- topics: Topic 名称列表（OR 语义）
|
||||
- tags: Tag 名称列表（OR 语义）
|
||||
|
||||
#### 请求体示例
|
||||
```json
|
||||
{
|
||||
"collect_start": "20250101",
|
||||
"collect_end": "20250131",
|
||||
"bag_name": "xxx*2025*",
|
||||
"gt_names": ["gt_a", "gt_b"],
|
||||
"fst_level1": "root_fst",
|
||||
"vehicles": ["PL061763"],
|
||||
"sw_version": "v1.*",
|
||||
"hw_version": "h1.*",
|
||||
"topics": ["topic_a", "topic_b"],
|
||||
"tags": ["tag_a", "tag_b"]
|
||||
}
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl -X POST "https://<host>/api/bags/search/aggregate?page=1&per_page=20&debug_sql=true" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"collect_start":"20250101","collect_end":"20250131","bag_name":"xxx*2025*","gt_names":["gt_a"],"fst_level1":"root_fst","vehicles":["PL061763"],"sw_version":"v1.*","hw_version":"h1.*","topics":["topic_a"],"tags":["tag_a"]}'
|
||||
```
|
||||
|
||||
#### 响应字段
|
||||
- items: 列表
|
||||
- bag_id: bag_list.id
|
||||
- bag_name: bag 名称
|
||||
- bag_path: 原始 bag 路径（空则返回 ""）
|
||||
- data_path: 解析后数据路径（空则返回 ""）
|
||||
- datetime: MainPangu.datetime（YYYY-MM-DD HH:MM:SS，空则返回 ""）
|
||||
- collect_time: BagLifecycle.collect_time（YYYY-MM-DD HH:MM:SS，空则返回 ""）
|
||||
- vehicle: 车号（空则返回 ""）
|
||||
- sw_version: 软件版本（空则返回 ""）
|
||||
- hw_version: 硬件版本（空则返回 ""）
|
||||
- mviz_link: 可视化链接（预留，当前为空字符串）
|
||||
- mbviz_link: 可视化链接（预留，当前为空字符串）
|
||||
- fst: FST 路径数组（可能多条，level1 对应业务一级节点，level0(root) 不返回）
|
||||
- level1/level2/level3/level4: 对应层级节点对象，缺失则为 {}
|
||||
- gt: 关联 GT 列表（包含 id/name/type/path/comment）
|
||||
- topics: 关联 Topic 列表（包含 id/name/type）
|
||||
- tags: 关联 Tag 列表（包含 id/name/type/creator）
|
||||
- page: 当前页
|
||||
- per_page: 每页数量
|
||||
- total: 总数
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"items": [
|
||||
{
|
||||
"bag_id": 123,
|
||||
"bag_name": "xxx.bag",
|
||||
"bag_path": "/path/to/bag",
|
||||
"data_path": "/path/to/data",
|
||||
"datetime": "2025-07-16 00:00:00",
|
||||
"collect_time": "2025-07-16 00:00:00",
|
||||
"vehicle": "PL061763",
|
||||
"sw_version": "v1.2.3",
|
||||
"hw_version": "h1.0",
|
||||
"mviz_link": "",
|
||||
"mbviz_link": "",
|
||||
"fst": [
|
||||
{
|
||||
"level1": {"id": 1, "name": "L1"},
|
||||
"level2": {"id": 2, "name": "L2"},
|
||||
"level3": {"id": 3, "name": "L3"},
|
||||
"level4": {"id": 4, "name": "L4"}
|
||||
}
|
||||
],
|
||||
"gt": [
|
||||
{"id": 10, "name": "gt_a", "type": "3DBox", "path": "/path/to/gt", "comment": "备注"}
|
||||
],
|
||||
"topics": [
|
||||
{"id": 20, "name": "topic_a", "type": "sensor_msgs/PointCloud2"}
|
||||
],
|
||||
"tags": [
|
||||
{"id": 30, "name": "tag_a", "type": "user", "creator": "alice"}
|
||||
]
|
||||
}
|
||||
],
|
||||
"page": 1,
|
||||
"per_page": 20,
|
||||
"total": 123
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 10. 获取可用版本列表
|
||||
```
|
||||
GET /api/bags/versions
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/bags/versions
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"sw_versions": ["v1.0", "v1.1", "v2.0"],
|
||||
"hw_versions": ["h1.0", "h2.0"]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
|
||||
### 11. 获取可用车号列表
|
||||
```
|
||||
GET /api/bags/vehicles
|
||||
```
|
||||
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/bags/vehicles
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"vehicles": ["PL061763", "PL061764"]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 12. 批量查询 Tags
|
||||
```
|
||||
POST /api/bags/tags
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["a.bag", "b.bag"]
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"a.bag": [
|
||||
{"name": "tag1", "type": "type1"},
|
||||
{"name": "tag2", "type": "type2"}
|
||||
],
|
||||
"b.bag": [
|
||||
{"name": "tag3", "type": "type3"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 13. 批量查询 Topics
|
||||
```
|
||||
POST /api/bags/topics
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["a.bag", "b.bag"]
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"a.bag": [
|
||||
{"name": "topic1", "type": "type1"},
|
||||
{"name": "topic2", "type": "type2"}
|
||||
],
|
||||
"b.bag": [
|
||||
{"name": "topic3", "type": "type3"}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
FST
|
||||
---
|
||||
|
||||
### 1. 批量获取 FST → Bags 映射
|
||||
```
|
||||
POST /api/fst/bags/nodes
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
["fst1", "fst2"]
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"fst1": ["Bag1", "Bag2"],
|
||||
"fst2": ["Bag3"]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. 获取指定 FST 及其子节点的 Bags
|
||||
```
|
||||
GET /api/fst/bags/path/{name}
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/fst/bags/path/fst1
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"fst1": {"bags": ["Bag1", "Bag2"]},
|
||||
"fst1_child": {"bags": ["Bag3"]}
|
||||
}
|
||||
```
|
||||
|
||||
#### 错误响应 404：Specified FST not found
|
||||
|
||||
---
|
||||
|
||||
### 3. 获取 FST 树形结构（纯文本）
|
||||
```
|
||||
GET /api/fst/print_tree
|
||||
```
|
||||
#### 成功响应 200
|
||||
```
|
||||
<pre>
|
||||
Your FST tree output here
|
||||
</pre>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. 获取单个 FST 详情
|
||||
```
|
||||
GET /api/fst/{name}
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/fst/fst1
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"name": "fst1",
|
||||
"parent_node": "Parent FST Name",
|
||||
"linked_bags_sum": 5
|
||||
}
|
||||
```
|
||||
|
||||
#### 错误响应 404：Specified FST not found
|
||||
|
||||
---
|
||||
|
||||
Geometry
|
||||
--------
|
||||
|
||||
### 批量获取 Rosbag 的 Geometry 信息
|
||||
```
|
||||
POST /api/geometry/
|
||||
```
|
||||
#### 请求体
|
||||
```json
|
||||
{"rosbag_names": ["rosbag1", "rosbag2"]}
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{
|
||||
"rosbag1": [[12.345678, 12.345678, 0.0]],
|
||||
"rosbag2": [[98.765432, 98.765432, 0.0]]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Projects
|
||||
--------
|
||||
|
||||
### 获取所有项目
|
||||
```
|
||||
GET /api/projects/all
|
||||
```
|
||||
#### 成功响应 200
|
||||
```json
|
||||
[
|
||||
{"id": 1, "name": "Project Name"}
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Tags
|
||||
----
|
||||
|
||||
### 1. 获取全部 Tags
|
||||
```
|
||||
GET /api/tags/all
|
||||
```
|
||||
#### 成功响应 200
|
||||
```json
|
||||
[
|
||||
{
|
||||
"name": "Data Quality",
|
||||
"type": "quality",
|
||||
"creator": "alice@example.com"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. 根据 Tag 名称获取关联 Bags
|
||||
```
|
||||
GET /api/tags/bags/{tag_name}
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/tags/bags/Data%20Quality
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{"bags": ["bag_2024_07"]}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 3. 根据 Creator 获取 Tags
|
||||
```
|
||||
GET /api/tags/creators/{creator}
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/tags/creators/alice@example.com
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{"tags": ["Sales", "Data Quality"]}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Topics
|
||||
------
|
||||
|
||||
### 1. 获取全部 Topics
|
||||
```
|
||||
GET /api/topics/all
|
||||
```
|
||||
#### 成功响应 200
|
||||
```json
|
||||
[
|
||||
{"name": "Topic Name", "type": "Topic Type"}
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. 根据 Topic 名称获取关联 Bags
|
||||
```
|
||||
GET /api/topics/bags/{topic_name}
|
||||
```
|
||||
#### 请求示例
|
||||
```
|
||||
curl https://<host>/api/topics/bags/topic1
|
||||
```
|
||||
|
||||
#### 成功响应 200
|
||||
```json
|
||||
{"name": ["Bag Name"]}
|
||||
```
|
||||
|
||||
---
|
||||
2
fst_data_pipeline/apps/README.md
Normal file
2
fst_data_pipeline/apps/README.md
Normal file
@@ -0,0 +1,2 @@
|
||||
# Micro Service apps
|
||||
Backend micro service application for fst data production line.
|
||||
0
fst_data_pipeline/apps/__init__.py
Normal file
0
fst_data_pipeline/apps/__init__.py
Normal file
77
fst_data_pipeline/apps/mta_manage_system/README.md
Normal file
77
fst_data_pipeline/apps/mta_manage_system/README.md
Normal file
@@ -0,0 +1,77 @@
|
||||
# 项目概述：
|
||||
MTA（multi trip aggregation） 管理系统是一个用于MTA数据处理任务的综合性管理系统，它包含了数据索引和下载，数据预处理和瓦片生成，路口轨迹可视化生成，html筛选，数据包下载和分类以及MTA数据处理任务的管理等功能。
|
||||
# 项目结构：
|
||||
mta_manage_system/
|
||||
├── db/
|
||||
│ └── init.sql # 数据库初始化脚本
|
||||
├── docker/
|
||||
│ └── Dockerfile # Docker 镜像构建文件
|
||||
├── models/
|
||||
│ ├── __init__.py # 模型初始化文件
|
||||
│ ├── sessionrecord.py # 会话记录模型
|
||||
│ └── taskstatus.py # 任务状态模型
|
||||
├── services/
|
||||
│ └── long_running_tasks.py # 长时间运行任务服务
|
||||
├── static/
|
||||
│ ├── css/
|
||||
│ │ ├── bootstrap.min.css # Bootstrap CSS 文件
|
||||
│ │ ├── bootstrap-icons.css # Bootstrap Icons CSS 文件
|
||||
│ │ ├── detail.css # 详情页面样式文件
|
||||
│ │ └── sweetalert2.css # SweetAlert2 样式文件
|
||||
│ ├── images/ # 图片资源目录
|
||||
│ └── js/
|
||||
│ ├── bootstrap.bundle.min.js # Bootstrap JavaScript 文件
|
||||
│ ├── detail.js # 详情页面 JavaScript 文件
|
||||
│ ├── injector.js # 注入器 JavaScript 文件
|
||||
│ ├── jquery-3.7.1.min.js # jQuery 库文件
|
||||
│ └── sweetalert2.js # SweetAlert2 库文件
|
||||
├── templates/
|
||||
│ ├── detail.html # 详情页面模板
|
||||
│ └── list.html # 列表页面模板
|
||||
├── utils/ # 工具类
|
||||
│ ├── docker_tool.py # 操作容器工具类
|
||||
│ ├── extract_package_tool.py # 通过瓦片获取包信息
|
||||
│ ├── log_tool.py # 日志工具类
|
||||
│ ├── path_tool.py # 路径处理工具类
|
||||
│ └── task_queue_tool.py # 任务管理工具类
|
||||
├── .env # 环境变量配置文件
|
||||
├── app.py # 应用入口文件
|
||||
├── config.py # 配置文件
|
||||
├── extensions.py # 扩展模块文件
|
||||
├── pyproject.toml # Python 项目配置文件
|
||||
└── README.md # 项目说明文件
|
||||
css 静态资源版本说明：
|
||||
[bootstrap-icons.css](static/css/bootstrap-icons.css) v1.11.0
|
||||
[bootstrap.min.css](static/css/bootstrap.min.css) v4.3.1
|
||||
[sweetalert2.css](static/css/sweetalert2.css) v11.26.3
|
||||
js 静态资源版本说明：
|
||||
[bootstrap.bundle.min.js](static/js/bootstrap.bundle.min.js) v4.3.1
|
||||
[jquery-3.7.1.min.js](static/js/jquery-3.7.1.min.js) v3.7.1
|
||||
[sweetalert2.js](static/js/sweetalert2.js) v11.26.3
|
||||
# 功能模块
|
||||
## 数据库
|
||||
db/目录下包含 init.sql脚本，用于初始化数据库表结构和基础数据。数据库选用了mysql 数据库。
|
||||
## 模型层
|
||||
models/目录定义了与数据库交互的模型类，如 sessionrecord.py和 taskstatus.py分别对应会话记录和任务状态的数据库操作。
|
||||
## 服务层
|
||||
services/目录中的 long_running_tasks.py处理长时间运行的任务逻辑，提供任务调度和管理功能。
|
||||
## 静态资源
|
||||
static/目录存放项目的静态资源，包括 CSS 样式文件、JavaScript 脚本文件和图片资源。
|
||||
## 模板
|
||||
templates/目录包含了前端页面的 HTML 模板文件，如 detail.html和 list.html。
|
||||
## 入口和应用配置
|
||||
app.py是应用的入口点，负责启动 Web 服务。config.py存放项目的配置信息，extensions.py定义和初始化项目所需的扩展模块。
|
||||
|
||||
# 运行指南
|
||||
1，先启动mysql 数据库，执行初始化sql 脚本[init.sql](db/init.sql)
|
||||
2，执行 uv sync
|
||||
3，执行 python3 app.py
|
||||
项目启动会从配置文件[.env](.env)中获取配置参数，其中数据密码为你所使用的数据库账户和密码
|
||||
|
||||
|
||||
# 技术栈
|
||||
|
||||
后端：Python, Flask
|
||||
数据库：MySQL
|
||||
前端：HTML, CSS, JavaScript, Bootstrap, SweetAlert2
|
||||
容器化：Docker
|
||||
23
fst_data_pipeline/apps/mta_manage_system/__init__.py
Normal file
23
fst_data_pipeline/apps/mta_manage_system/__init__.py
Normal file
@@ -0,0 +1,23 @@
|
||||
from flask import Flask
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from config import config
|
||||
|
||||
|
||||
from extensions import db # 统一导入 db 实例
|
||||
|
||||
|
||||
def create_app():
|
||||
app = Flask(__name__)
|
||||
|
||||
# 配置
|
||||
app.config["SQLALCHEMY_DATABASE_URI"] = config.SQLALCHEMY_DATABASE_URI
|
||||
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
|
||||
|
||||
# 初始化扩展
|
||||
db.init_app(app)
|
||||
|
||||
# 注册蓝图
|
||||
# from routes import main
|
||||
# app.register_blueprint(main)
|
||||
|
||||
return app
|
||||
829
fst_data_pipeline/apps/mta_manage_system/app.py
Normal file
829
fst_data_pipeline/apps/mta_manage_system/app.py
Normal file
@@ -0,0 +1,829 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
"""
|
||||
@Project ：bag_filter
|
||||
@File ：app.py
|
||||
@Author ：zuowe
|
||||
@Date ：2025/10/20 10:36
|
||||
@Des ：
|
||||
"""
|
||||
|
||||
import os
|
||||
from flask import Flask, render_template, send_from_directory, request, jsonify
|
||||
from datetime import datetime
|
||||
import json
|
||||
import logging
|
||||
|
||||
from utils.docker_tool import docker_tool
|
||||
from sqlalchemy import select, func
|
||||
from pathlib import Path
|
||||
from config import config, logger
|
||||
from utils.log_tool import setup_logging
|
||||
from utils.extract_package_tool import extract_package_names_from_html
|
||||
from utils.task_queue_tool import task_queue
|
||||
from utils.path_tool import PathTool
|
||||
from extensions import db # 统一导入 db 实例
|
||||
from models.sessionrecord import SessionRecord
|
||||
from models.taskstatus import TaskStatus
|
||||
from services.long_running_tasks import LongRunningTasks
|
||||
from __init__ import create_app
|
||||
|
||||
# 创建应用
|
||||
app = create_app()
|
||||
|
||||
|
||||
# 数据库连接
|
||||
# database_url = config.SQLALCHEMY_DATABASE_URI
|
||||
|
||||
BATH_ROOT = config.BATH_ROOT
|
||||
IMAGE_FOLDER = config.IMAGE_FOLDER
|
||||
PAGES_DIR = config.PAGES_DIR
|
||||
SCRIPT_PATH = config.SCRIPT_PATH
|
||||
HTML_DIR = config.HTML_DIR
|
||||
|
||||
OUTPUT_PIC_PATH = config.OUTPUT_PIC_PATH
|
||||
OUTPUT_PIC_NAME = config.OUTPUT_PIC_NAME
|
||||
OUTPUT_RESULT_PATH = config.OUTPUT_RESULT_PATH
|
||||
#
|
||||
MAX_ALLOW_RUNNING_MTA_TASK = config.MAX_ALLOW_RUNNING_MTA_TASK
|
||||
MTA_IMAGE = config.MTA_IMAGE
|
||||
MTA_IMNAGE_VERSION = config.MTA_IMNAGE_VERSION
|
||||
|
||||
CONTAINER_OUTER_PATH = config.CONTAINER_OUTER_PATH
|
||||
ENV_PARAM_KEY = config.ENV_PARAM_KEY
|
||||
ENV_PARAM_VALUE = config.ENV_PARAM_VALUE
|
||||
MTA_SCRIPT = config.MTA_SCRIPT
|
||||
|
||||
|
||||
# 允许的扩展名
|
||||
ALLOWED_EXTS = config.ALLOWED_EXTS
|
||||
|
||||
# 每页最大条数限制
|
||||
MAX_PER_PAGE = config.MAX_PER_PAGE
|
||||
DEFAULT_PER_PAGE = config.MAX_PER_PAGE
|
||||
|
||||
# 配置日志
|
||||
|
||||
logger = setup_logging("logs/app.log", logging.INFO)
|
||||
|
||||
|
||||
# ===== 数据库配置 - 使用 config 中的统一配置 =====
|
||||
# 在 config.py 中添加数据库配置
|
||||
app.config["SQLALCHEMY_DATABASE_URI"] = config.SQLALCHEMY_DATABASE_URI
|
||||
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def list_pages():
|
||||
return render_template("list.html")
|
||||
|
||||
|
||||
# 列表数据分页查询
|
||||
@app.route("/api/items")
|
||||
def api_items():
|
||||
# 1) 分页参数
|
||||
try:
|
||||
page = int(request.args.get("page", 1))
|
||||
per_page = int(request.args.get("per_page", DEFAULT_PER_PAGE))
|
||||
if page < 1:
|
||||
page = 1
|
||||
if per_page < 1 or per_page > MAX_PER_PAGE:
|
||||
per_page = DEFAULT_PER_PAGE
|
||||
except ValueError:
|
||||
page = 1
|
||||
per_page = DEFAULT_PER_PAGE
|
||||
|
||||
# 2) 过滤参数
|
||||
html_name = request.args.get("html_name", "").strip()
|
||||
central_session = request.args.get("central_session", "").strip()
|
||||
mta_task_status = request.args.get("mta_task_status", "").strip()
|
||||
is_filter_bag = request.args.get("is_filter_bag", "")
|
||||
is_run_mta = request.args.get("is_run_mta", "")
|
||||
logger.info(
|
||||
f"is_filter_bag 值为：{is_filter_bag} ，is_run_mta 值为：{is_run_mta} "
|
||||
)
|
||||
# 将字符串转换为布尔值
|
||||
|
||||
if request.args.get("is_filter_bag", "").strip().lower() == "true":
|
||||
is_filter_bag = True
|
||||
elif request.args.get("is_filter_bag", "").strip().lower() == "false":
|
||||
is_filter_bag = False
|
||||
else:
|
||||
is_filter_bag = request.args.get("is_filter_bag", "").strip()
|
||||
if request.args.get("is_run_mta", "").strip().lower() == "true":
|
||||
is_run_mta = True
|
||||
elif request.args.get("is_run_mta", "").strip().lower() == "false":
|
||||
is_run_mta = False
|
||||
else:
|
||||
is_run_mta = request.args.get("is_run_mta", "").strip()
|
||||
|
||||
# 3) 构建 COUNT 查询（与主查询条件保持一致）
|
||||
stmt_count = select(func.count(SessionRecord.id)).where(
|
||||
(SessionRecord.html_name.like(f"%{html_name}%")) if html_name else True,
|
||||
(SessionRecord.central_session.like(f"%{central_session}%"))
|
||||
if central_session
|
||||
else True,
|
||||
# 处理 is_filter_bag
|
||||
(SessionRecord.is_filter_bag == True)
|
||||
if is_filter_bag == True
|
||||
else (SessionRecord.is_filter_bag == False)
|
||||
if is_filter_bag == False
|
||||
else True,
|
||||
# 处理 is_run_mta
|
||||
(SessionRecord.is_run_mta == True)
|
||||
if is_run_mta == True
|
||||
else (SessionRecord.is_run_mta == False)
|
||||
if is_run_mta == False
|
||||
else True,
|
||||
# 处理 mta_task_status
|
||||
(SessionRecord.mta_task_status == mta_task_status)
|
||||
if mta_task_status and mta_task_status.strip() # 不为空且非空白字符串
|
||||
else True,
|
||||
)
|
||||
total = db.session.scalar(stmt_count) or 0
|
||||
|
||||
# 4) 构建分页数据查询（在 select 上链式调用 offset/limit）
|
||||
stmt_data = (
|
||||
select(SessionRecord)
|
||||
.where(
|
||||
(SessionRecord.html_name.like(f"%{html_name}%")) if html_name else True,
|
||||
(SessionRecord.central_session.like(f"%{central_session}%"))
|
||||
if central_session
|
||||
else True,
|
||||
# 处理 is_filter_bag
|
||||
(SessionRecord.is_filter_bag == True)
|
||||
if is_filter_bag == True
|
||||
else (SessionRecord.is_filter_bag == False)
|
||||
if is_filter_bag == False
|
||||
else True,
|
||||
# 处理 is_run_mta
|
||||
(SessionRecord.is_run_mta == True)
|
||||
if is_run_mta == True
|
||||
else (SessionRecord.is_run_mta == False)
|
||||
if is_run_mta == False
|
||||
else True,
|
||||
# 处理 mta_task_status
|
||||
(SessionRecord.mta_task_status == mta_task_status)
|
||||
if mta_task_status and mta_task_status.strip() # 不为空且非空白字符串
|
||||
else True,
|
||||
)
|
||||
.order_by(SessionRecord.id.asc()) # 建议使用索引字段排序
|
||||
.offset((page - 1) * per_page)
|
||||
.limit(per_page)
|
||||
)
|
||||
|
||||
rows = db.session.execute(stmt_data).scalars().all()
|
||||
logger.info(it.to_dict() for it in rows)
|
||||
|
||||
# 5) 计算分页元信息
|
||||
total_pages = (total + per_page - 1) // per_page
|
||||
has_prev = page > 1
|
||||
has_next = page < total_pages
|
||||
|
||||
# 7) 返回 JSON
|
||||
return jsonify({
|
||||
"items": [it.to_dict() for it in rows],
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"total": total,
|
||||
"total_pages": total_pages,
|
||||
"has_prev": has_prev,
|
||||
"has_next": has_next,
|
||||
})
|
||||
|
||||
|
||||
@app.route("/api/syncdata", methods=["GET"])
|
||||
def sync_data():
|
||||
try:
|
||||
bag_folder = request.args.get("bag_folder")
|
||||
html_folder = request.args.get("html_folder")
|
||||
logger.info(
|
||||
f"获取到的参数：bag_folder = {bag_folder}，html_folder = {html_folder} "
|
||||
)
|
||||
# html_folder ="selected_8to9"
|
||||
# 1. 获取/data/html目录下所有html文件
|
||||
html_dir = PathTool.join(BATH_ROOT, html_folder)
|
||||
if not os.path.exists(html_dir):
|
||||
return jsonify({"error": "HTML directory not found"}), 404
|
||||
|
||||
html_files = [f for f in os.listdir(html_dir) if f.endswith(".html")]
|
||||
# logger.info(html_files)
|
||||
# 2. 获取数据库中已有的html_name列表
|
||||
existing_records = (
|
||||
SessionRecord.query.filter(SessionRecord.html_folder == html_folder)
|
||||
.with_entities(SessionRecord.html_name)
|
||||
.all()
|
||||
)
|
||||
existing_names = {r[0] for r in existing_records}
|
||||
|
||||
# 3. 比对并插入新记录
|
||||
new_count = 0
|
||||
for filename in html_files:
|
||||
if filename not in existing_names:
|
||||
new_record = SessionRecord(
|
||||
html_name=filename,
|
||||
html_folder=html_folder,
|
||||
bag_folder=bag_folder,
|
||||
central_session="", # 默认值
|
||||
query_sessions="", # 默认空JSON数组
|
||||
selected_image_url="null",
|
||||
is_filter_bag=False,
|
||||
is_run_mta=False,
|
||||
run_mta_result="",
|
||||
remark="自动同步添加",
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
)
|
||||
db.session.add(new_record)
|
||||
new_count += 1
|
||||
|
||||
db.session.commit()
|
||||
|
||||
return jsonify({
|
||||
"status": "success",
|
||||
"scanned_files": len(html_files),
|
||||
"new_records_added": new_count,
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
print(e)
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
# 接口：GET /api/items/<id>
|
||||
# =======================
|
||||
@app.route("/api/items/<int:id>", methods=["GET"])
|
||||
def get_item(id: int):
|
||||
# db: Session = SessionLocal()
|
||||
try:
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
if not item:
|
||||
return jsonify({"error": "记录不存在", "id": id}), 404
|
||||
|
||||
data = row2dict(item)
|
||||
return jsonify(status=True, result=data)
|
||||
except Exception as e:
|
||||
app.logger.exception("查询接口异常")
|
||||
return jsonify({"error": "服务器内部错误", "detail": str(e)}), 500
|
||||
|
||||
|
||||
# ======================
|
||||
# 接口：POST /api/items （新增）
|
||||
# PUT /api/items/<id> （更新）
|
||||
# ======================
|
||||
@app.route("/api/saveitems", methods=["POST"])
|
||||
def save_or_update_item(id=None):
|
||||
try:
|
||||
data = request.get_json()
|
||||
if not data:
|
||||
return jsonify({"error": "请求体必须为 JSON"}), 400
|
||||
|
||||
# 1) 获取必填字段校验（可根据需求调整）
|
||||
id = data.get("id")
|
||||
html_name = data.get("html_name")
|
||||
central_session = data.get("central_session")
|
||||
query_session = data.get("query_session")
|
||||
if not html_name or not central_session:
|
||||
return jsonify({"error": "html_name 和 central_session 为必填项"}), 400
|
||||
logger.info(f"查询到的 query_session 为：{query_session}")
|
||||
# 2) 处理布尔字段：转为 Python Boolean 或 None
|
||||
is_filter_bag = str_to_bool(data.get("is_filter_bag"))
|
||||
is_run_mta = str_to_bool(data.get("is_run_mta"))
|
||||
success = str_to_bool(data.get("success"))
|
||||
|
||||
# 3) 判断是新增还是更新
|
||||
if id is None:
|
||||
# 🟢 新增记录
|
||||
item = SessionRecord(
|
||||
html_name=html_name,
|
||||
central_session=central_session,
|
||||
query_sessions=data.get("query_session"), # 可为字符串或后续转 JSON
|
||||
selected_image_url=data.get("selected_image_url"),
|
||||
is_filter_bag=is_filter_bag,
|
||||
is_run_mta=is_run_mta,
|
||||
run_mta_result=data.get("run_mta_result"), # 可为 JSON 字符串或对象
|
||||
remark=data.get("remark"),
|
||||
success=success,
|
||||
)
|
||||
|
||||
db.session.add(item)
|
||||
db.session.flush() # 为了获取自增 id（如有需要）
|
||||
db.session.commit()
|
||||
|
||||
return jsonify({
|
||||
"status": "success",
|
||||
"message": "记录新增成功",
|
||||
"data": {
|
||||
**item.__dict__,
|
||||
"_sa_instance_state": None,
|
||||
}, # 去掉内部属性
|
||||
}), 201
|
||||
else:
|
||||
# 🔵 更新记录
|
||||
item = (
|
||||
db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
)
|
||||
if not item:
|
||||
return jsonify({"error": f"记录不存在，id={id}"}), 404
|
||||
|
||||
# 更新字段
|
||||
item.html_name = html_name
|
||||
item.central_session = central_session
|
||||
item.query_sessions = data.get("query_session")
|
||||
item.selected_image_url = data.get("selected_image_url")
|
||||
item.is_filter_bag = is_filter_bag
|
||||
item.is_run_mta = is_run_mta
|
||||
item.run_mta_result = data.get("run_mta_result")
|
||||
item.remark = data.get("remark")
|
||||
# item.success = success
|
||||
item.updated_at = func.now() # 可选，如果模型没有自动更新
|
||||
|
||||
db.session.commit()
|
||||
return jsonify({"status": "success", "message": "记录更新成功"})
|
||||
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
app.logger.exception("保存/更新接口异常")
|
||||
return jsonify({"error": "服务器内部错误", "detail": str(e)}), 500
|
||||
|
||||
|
||||
@app.route("/detail/<html_name>/<bag_folder>/<html_folder>/<id>")
|
||||
def filterbag_detail(html_name, bag_folder, html_folder, id):
|
||||
# 这里接受url 传参 ，将页面重定向到 详情选择页面 ，并将参数和查询的数据返回到详情页面
|
||||
# file_path = os.path.join(PAGES_DIR, html_name)
|
||||
print(id)
|
||||
# html_folder = "selected_8to9"
|
||||
# html_name = "15_27321_13493_merged_intersections_17_100.0m.html"
|
||||
return render_template(
|
||||
"detail.html",
|
||||
html_name=html_name,
|
||||
bag_folder=bag_folder,
|
||||
html_folder=html_folder,
|
||||
abslute_path=BATH_ROOT,
|
||||
id=id,
|
||||
)
|
||||
|
||||
|
||||
# 获取图片 在页面展示
|
||||
@app.route("/api/getimage/<imagename>/<bagname>/<htmlname>/<bagfolder>")
|
||||
def api_getimages(imagename, bagname, htmlname, bagfolder):
|
||||
# 安全校验：只允许访问 pages 目录下的 HTML
|
||||
# 安全拼接并限制到白名单目录
|
||||
# bagname = "PL162802_event_all_time_event_20250628-131016_0.bag.dir"
|
||||
logger.info(f"image aaa path : imagename: {imagename}")
|
||||
base = PathTool.join(BATH_ROOT, bagfolder, htmlname, bagname, IMAGE_FOLDER)
|
||||
logger.info(f"image aaa path :{base} , bagname: {bagname}")
|
||||
|
||||
# 如果找到图片则返回图片数据
|
||||
if imagename in os.listdir(base):
|
||||
return send_from_directory(base, imagename)
|
||||
|
||||
return jsonify({"error": "Not found or access denied"})
|
||||
|
||||
|
||||
# 获取图片 在页面展示
|
||||
@app.route("/api/getmtaresult/<id>")
|
||||
def api_getmtaresult(id):
|
||||
# 安全校验：只允许访问 pages 目录下的 HTML
|
||||
# 安全拼接并限制到白名单目录
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
bagfolder = item.bag_folder
|
||||
html_name = item.html_name
|
||||
htmlname = html_name.replace(".html", "")
|
||||
bagname = item.central_session
|
||||
imagename = item.run_mta_result
|
||||
|
||||
base = PathTool.join(BATH_ROOT, bagfolder, htmlname, bagname, OUTPUT_PIC_PATH)
|
||||
logger.info(f"image aaa path :{base} , bagname: {bagname}")
|
||||
|
||||
# 如果找到图片则返回图片数据
|
||||
if imagename in os.listdir(base):
|
||||
return send_from_directory(base, OUTPUT_PIC_NAME)
|
||||
|
||||
return jsonify({"error": "Not found or access denied"})
|
||||
|
||||
|
||||
# 获取图片 在页面展示
|
||||
@app.route("/api/getimage")
|
||||
def api_getimage():
|
||||
bagname = request.args.get("bagname")
|
||||
imageIndex = request.args.get("imageIndex")
|
||||
htmlname = request.args.get("htmlname")
|
||||
bagfolder = request.args.get("bagfolder", "")
|
||||
|
||||
logger.info(f"htmlname的地址为：{str(htmlname)},bagfolder :{bagfolder} ")
|
||||
# bagname = "PL162802_event_all_time_event_20250628-131016_0.bag.dir"
|
||||
if not bagname:
|
||||
return jsonify(status=False, message="missing bagname"), 400
|
||||
if imageIndex is None:
|
||||
return jsonify(status=False, message="missing imageIndex"), 400
|
||||
root = Path(BATH_ROOT) / bagfolder / htmlname / bagname / IMAGE_FOLDER
|
||||
image_list = sorted(root.glob("*.jpg"))
|
||||
logger.info(f"拼接后的地址为：{str(root)}")
|
||||
|
||||
imageIndex = int(imageIndex.strip())
|
||||
# 用户使用习惯的索引比 list 索引大1 ，所以这里要减去1
|
||||
imageIndex -= 1
|
||||
# 通过index 获取 list 中选择的图片
|
||||
imagename = str(image_list[imageIndex])
|
||||
payload = {"size": len(image_list), "imagename": imagename}
|
||||
return jsonify(status=True, result=payload)
|
||||
|
||||
|
||||
@app.route("/api/options/<html_name>/<html_folder>")
|
||||
def api_pages(html_name, html_folder):
|
||||
logger.info(f"获取到的参数： html_name = {html_name}，html_folder = {html_folder} ")
|
||||
html_dir = PathTool.join(BATH_ROOT, html_folder)
|
||||
# 可在此处读取html文件 包 内容用于展示
|
||||
file_path = os.path.join(html_dir, html_name)
|
||||
|
||||
# 按“键名全小写”的约定返回，前端可通过 res.ok 与 res.data 访问
|
||||
bagset = extract_package_names_from_html(file_path, logger)
|
||||
|
||||
data = [
|
||||
{"value": k, "label": k}
|
||||
for k in sorted(bagset) # 如需保持原集合无序，可去掉 sorted()
|
||||
]
|
||||
|
||||
return jsonify(status=True, result=data)
|
||||
|
||||
|
||||
@app.route("/api/getimagedata")
|
||||
def get_data():
|
||||
bagname = request.args.get("bagname")
|
||||
htmlname = request.args.get("htmlname", "")
|
||||
bagfolder = request.args.get("bagfolder", "")
|
||||
|
||||
logger.info(f"获取到的参数 htmlname ：{htmlname}")
|
||||
path = PathTool.join(BATH_ROOT, bagfolder, htmlname, bagname, IMAGE_FOLDER)
|
||||
p = Path(path)
|
||||
image_list = sorted(p.glob("*.jpg"))
|
||||
logger.info(len(image_list))
|
||||
imagename = str(image_list[0])
|
||||
# 构造返回：size 为列表大小；first 为第一个元素或 null
|
||||
payload = {
|
||||
"size": len(image_list),
|
||||
"imagename": imagename,
|
||||
"index": str(image_list[0])
|
||||
if image_list
|
||||
else None, # 非 JSON 类型在此统一转 str
|
||||
}
|
||||
return jsonify(status=True, result=payload)
|
||||
|
||||
|
||||
def fmt_date(dt: datetime) -> str:
|
||||
return dt.strftime("%Y-%m-%d %H:%M:%S") if dt else "-"
|
||||
|
||||
|
||||
def to_camel_case(snake: str) -> str:
|
||||
parts = snake.split("_")
|
||||
return parts[0] + "".join(p.title() for p in parts[1:])
|
||||
|
||||
|
||||
def row2dict(row) -> dict:
|
||||
d = {}
|
||||
for col in row.__table__.columns:
|
||||
key = to_camel_case(col.name)
|
||||
val = getattr(row, col.name)
|
||||
if isinstance(val, datetime):
|
||||
d[key] = fmt_date(val)
|
||||
elif col.name in ("querySessions", "runMtaResult") and val is not None:
|
||||
# 若数据库字段为 JSON/Text，这里做一次安全反序列化
|
||||
try:
|
||||
d[key] = json.loads(val) if isinstance(val, str) else val
|
||||
except Exception:
|
||||
d[key] = val
|
||||
else:
|
||||
d[key] = val
|
||||
return d
|
||||
|
||||
|
||||
# ======================
|
||||
# 工具函数：字符串 "true"/"false" 转布尔值
|
||||
# ======================
|
||||
def str_to_bool(s):
|
||||
if s is None:
|
||||
return None
|
||||
if s == "true":
|
||||
return True
|
||||
if s == "false":
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
# ===== 提交接口：写入 MySQL =====
|
||||
@app.route("/api/submit", methods=["POST"])
|
||||
def submit():
|
||||
try:
|
||||
# 1) 解析 JSON
|
||||
data = request.get_json(silent=True)
|
||||
if data is None:
|
||||
return jsonify({"error": "请求体不是合法的 JSON"}), 400
|
||||
|
||||
# 2) 兼容两种可能的前端结构
|
||||
# 结构A（当前表单字段）: central_session, query_sessions, selected_image_url
|
||||
# 结构B（你示例 alert 用到）: tags（数组）, category
|
||||
|
||||
central_session = data.get("category")
|
||||
query_sessions = data.get("tags")
|
||||
bag_folder = data.get("bag_folder")
|
||||
html_name = data.get("html_name")
|
||||
html_name_folder = html_name.replace(".html", "")
|
||||
logger.info(bag_folder)
|
||||
bagpath = PathTool.join(BATH_ROOT, bag_folder, html_name_folder)
|
||||
logger.info("拼接的数据包根路径为：" + bagpath)
|
||||
new_query_sessions = list(
|
||||
map(lambda x: PathTool.join(bagpath, x), query_sessions)
|
||||
)
|
||||
logger.info(f"query_sessions:{query_sessions}")
|
||||
query_sessions = ",".join(query_sessions)
|
||||
|
||||
central_session_str = PathTool.join(bagpath, central_session)
|
||||
query_sessions_str = " ".join(new_query_sessions)
|
||||
logger.info(
|
||||
f"query_sessions_str:{query_sessions_str}, central_session_str : {central_session_str}"
|
||||
)
|
||||
|
||||
# 2) 写入数据库
|
||||
|
||||
id = data.get("id")
|
||||
logger.info(f"查询到的数据ID ： {id}")
|
||||
# 🔵 更新记录
|
||||
# 查询记录
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
|
||||
if not item:
|
||||
return jsonify({"error": f"记录不存在，html_name={html_name}"}), 404
|
||||
|
||||
# 动态更新字段（仅更新传入的字段）
|
||||
# 仅更新指定字段
|
||||
if central_session is not None:
|
||||
item.central_session = central_session
|
||||
if query_sessions is not None:
|
||||
item.query_sessions = query_sessions
|
||||
|
||||
# 更新是否筛选包字段信息 是否跑 mta 需要看弹出框
|
||||
item.is_filter_bag = True
|
||||
|
||||
# item.is_run_mta = True
|
||||
|
||||
# 更新修改时间
|
||||
item.updated_at = func.now()
|
||||
|
||||
db.session.commit()
|
||||
|
||||
payload = {
|
||||
"status": "success",
|
||||
"message": "数据提交成功",
|
||||
"data": {
|
||||
"central_session": central_session_str,
|
||||
"query_sessions": query_sessions_str,
|
||||
},
|
||||
}
|
||||
|
||||
return jsonify(status=True, result=payload)
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
app.logger.exception("提交失败")
|
||||
return jsonify({"error": f"服务器异常：{str(e)}"}), 500
|
||||
|
||||
|
||||
#
|
||||
# 提交 task 方法
|
||||
#
|
||||
|
||||
|
||||
# 查看 容器状态
|
||||
@app.route("/api/gettasksstatus", methods=["GET"])
|
||||
def get_tasks_status():
|
||||
"""查看队列状态"""
|
||||
status = task_queue.get_queue_status()
|
||||
|
||||
print(f"📊 当前队列状态:")
|
||||
print(f" 最大并发任务数: {status['max_concurrent_tasks']}")
|
||||
print(f" 正在执行的任务数: {status['running_count']}")
|
||||
print(f" 等待执行的任务数: {status['pending_count']}")
|
||||
print(f" 已完成的任务数: {status['successful_count']}")
|
||||
print(f" 失败的任务数: {status['failed_count']}")
|
||||
print(f" 取消的任务数: {status['cancelled_count']}")
|
||||
|
||||
return jsonify({
|
||||
"max_concurrent_tasks": status["max_concurrent_tasks"],
|
||||
"running_count": status["running_count"],
|
||||
"pending_count": status["pending_count"],
|
||||
"successful_count": status["successful_count"],
|
||||
"failed_count": status["failed_count"],
|
||||
"cancelled_count": status["cancelled_count"],
|
||||
})
|
||||
|
||||
|
||||
# 查看 容器状态
|
||||
@app.route("/api/stoptask", methods=["GET"])
|
||||
def stop_tasks():
|
||||
"""查看队列状态"""
|
||||
id = request.args.get("id")
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
if not item:
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"data": "",
|
||||
"message": "系统异常，请联系管理员",
|
||||
}), 500
|
||||
# mta_task_id = item.mta_task_id
|
||||
# result = task_queue.cancel_task(mta_task_id)
|
||||
container_name = f"MTA-{id}"
|
||||
stop_result = docker_tool.stop_container_by_name(container_name)
|
||||
item.mta_task_status = TaskStatus.FAILED.value
|
||||
item.mta_task_id = None
|
||||
item.run_mta_result = None
|
||||
item.updated_at = func.now()
|
||||
db.session.commit()
|
||||
return jsonify(stop_result)
|
||||
|
||||
|
||||
@app.route("/api/reloadtask")
|
||||
def reloadTask():
|
||||
id = request.args.get("id")
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
if not item:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"data": "",
|
||||
"message": "系统异常，请联系管理员",
|
||||
}), 200
|
||||
status = task_queue.get_queue_status()
|
||||
running_count = status["running_count"]
|
||||
bagfolder = item.bag_folder
|
||||
|
||||
html_name = item.html_name
|
||||
html_name = html_name.replace(".html", "")
|
||||
|
||||
category = item.central_session
|
||||
query_sessions = item.query_sessions
|
||||
if not category or not query_sessions:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"data": "",
|
||||
"message": "central_session 或者 query_sessions不能为空",
|
||||
}), 200
|
||||
central_session = PathTool.join(BATH_ROOT, html_name, category)
|
||||
querySessionCommand = getQuerySessionCommand(query_sessions, html_name)
|
||||
run_mta_command_result = runmtacommandFun(
|
||||
bagfolder, html_name, central_session, querySessionCommand, category, id
|
||||
)
|
||||
if run_mta_command_result:
|
||||
return jsonify(run_mta_command_result), 200
|
||||
else:
|
||||
return jsonify(run_mta_command_result), 500
|
||||
|
||||
|
||||
@app.route("/api/runmtacommand")
|
||||
def runmtacommand():
|
||||
status = task_queue.get_queue_status()
|
||||
bagfolder = request.args.get("bag_folder")
|
||||
command = request.args.get("command")
|
||||
html_name = request.args.get("html_name", "")
|
||||
html_name = html_name.replace(".html", "")
|
||||
central_session = request.args.get("central_session", "")
|
||||
category = request.args.get("category", "")
|
||||
query_sessions = request.args.get("query_sessions", "")
|
||||
|
||||
run_mta_command_result = runmtacommandFun(
|
||||
bagfolder, html_name, central_session, query_sessions, category, id
|
||||
)
|
||||
if run_mta_command_result:
|
||||
return jsonify(run_mta_command_result), 200
|
||||
else:
|
||||
return jsonify(run_mta_command_result), 500
|
||||
|
||||
|
||||
def getQuerySessionCommand(querySession, html_name):
|
||||
querysessionList = [
|
||||
querySession.strip()
|
||||
for querySession in querySession.split(",")
|
||||
if querySession.strip()
|
||||
]
|
||||
paths = [PathTool.join(BATH_ROOT, html_name, item) for item in querysessionList]
|
||||
return " ".join(paths)
|
||||
|
||||
|
||||
def runmtacommandFun(
|
||||
bagfolder, html_name, central_session, query_sessions, category, id
|
||||
):
|
||||
run_mta_command_result = {"success": True, "data": "", "message": "任务提交成功"}
|
||||
status = task_queue.get_queue_status()
|
||||
running_count = status["running_count"]
|
||||
html_name = html_name.replace(".html", "")
|
||||
result_file = PathTool.join(
|
||||
BATH_ROOT, bagfolder, html_name, category, OUTPUT_RESULT_PATH
|
||||
)
|
||||
command = f"{MTA_SCRIPT} --central_session {central_session} --query_session {query_sessions}"
|
||||
# command = ""
|
||||
id = request.args.get("id", "")
|
||||
data = {
|
||||
"image_name": f"{MTA_IMAGE}:{MTA_IMNAGE_VERSION}",
|
||||
"container_name": f"MTA-{id}",
|
||||
"command": command,
|
||||
"timeout": 3600,
|
||||
"result_file": result_file,
|
||||
"result_file_name": OUTPUT_PIC_NAME,
|
||||
"id": id,
|
||||
}
|
||||
volumes = {CONTAINER_OUTER_PATH: BATH_ROOT}
|
||||
|
||||
environment = {ENV_PARAM_KEY: ENV_PARAM_VALUE}
|
||||
|
||||
def on_task_completion(task_result):
|
||||
print("---------------------- 进入 回调方法 ----------------------")
|
||||
"""任务完成回调"""
|
||||
# 手动创建应用上下文
|
||||
with app.app_context():
|
||||
# 从task_result中获取id，如果不存在则使用闭包变量id
|
||||
|
||||
item = (
|
||||
db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
)
|
||||
if task_result.get("status") == "completed":
|
||||
try:
|
||||
if item:
|
||||
item.is_run_mta = True
|
||||
item.mta_is_running = False
|
||||
item.mta_task_status = TaskStatus.COMPLETED.value
|
||||
item.mta_task_id = None
|
||||
item.run_mta_result = OUTPUT_PIC_NAME
|
||||
item.updated_at = func.now()
|
||||
db.session.commit()
|
||||
except Exception as e:
|
||||
app.logger.error(f"回调函数数据库操作失败: {e}")
|
||||
db.session.rollback()
|
||||
elif task_result.get("status") == "cancelled":
|
||||
# 失败的情况
|
||||
item.is_run_mta = True
|
||||
item.mta_is_running = True
|
||||
item.mta_task_status = TaskStatus.CANCELLED.value
|
||||
item.mta_task_id = None
|
||||
item.run_mta_result = None
|
||||
item.updated_at = func.now()
|
||||
db.session.commit()
|
||||
else:
|
||||
# 失败的情况
|
||||
item.is_run_mta = True
|
||||
item.mta_is_running = True
|
||||
item.mta_task_status = TaskStatus.FAILED.value
|
||||
item.mta_task_id = None
|
||||
item.run_mta_result = None
|
||||
item.updated_at = func.now()
|
||||
db.session.commit()
|
||||
|
||||
print(f"任务 {id} 数据库更新成功")
|
||||
|
||||
try:
|
||||
"""查看队列状态"""
|
||||
status = task_queue.get_queue_status()
|
||||
|
||||
# 提交任务到队列
|
||||
result = task_queue.submit_task(
|
||||
LongRunningTasks.api_create_container,
|
||||
container_config=data,
|
||||
volumes=volumes,
|
||||
environment=environment,
|
||||
completion_callback=on_task_completion,
|
||||
)
|
||||
run_mta_command_result["data"] = result
|
||||
status = result["status"]
|
||||
task_id = result["BATH_ROOT"]
|
||||
item = db.session.query(SessionRecord).filter(SessionRecord.id == id).first()
|
||||
if not item:
|
||||
return jsonify({"error": f"记录不存在，id={id}"}), 404
|
||||
|
||||
# 更新字段
|
||||
item.mta_is_running = True
|
||||
|
||||
item.mta_task_id = task_id
|
||||
item.updated_at = func.now() # 可选，如果模型没有自动更新
|
||||
print(f"任务地位：{id}，task 返回结果为 ：{status}")
|
||||
#
|
||||
if status == "PENDING":
|
||||
item.mta_task_status = TaskStatus.PENDING.value
|
||||
run_mta_command_result["message"] = (
|
||||
f"当前正在执行任务数：{running_count}，新建任务已进入队列"
|
||||
)
|
||||
elif status == "RUNNING":
|
||||
item.mta_task_status = TaskStatus.RUNNING.value
|
||||
run_mta_command_result["message"] = "MTA 任务已经启动"
|
||||
db.session.commit()
|
||||
return run_mta_command_result
|
||||
|
||||
except Exception as e:
|
||||
app.logger.exception("查询接口异常")
|
||||
return {"success": False, "data": e, "message": "任务提交异常"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 本地开发：http://127.0.0.1:5000
|
||||
|
||||
with app.app_context():
|
||||
db.create_all()
|
||||
|
||||
app.run(host="0.0.0.0", port=5000, debug=False)
|
||||
105
fst_data_pipeline/apps/mta_manage_system/config.py
Normal file
105
fst_data_pipeline/apps/mta_manage_system/config.py
Normal file
@@ -0,0 +1,105 @@
|
||||
import os
|
||||
import logging
|
||||
from dotenv import load_dotenv
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_environment():
|
||||
"""智能加载环境配置"""
|
||||
env = os.getenv("FLASK_ENV", "development")
|
||||
|
||||
# 加载顺序：基础配置 -> 环境特定配置 -> 本地覆盖
|
||||
env_files = [
|
||||
".env", # 基础配置
|
||||
f".env.{env}", # 环境特定配置
|
||||
".env.local", # 本地覆盖（可选）
|
||||
]
|
||||
|
||||
for env_file in env_files:
|
||||
if Path(env_file).exists():
|
||||
load_dotenv(env_file)
|
||||
logging.info(f"加载环境文件: {env_file}")
|
||||
|
||||
return env
|
||||
|
||||
|
||||
class Config:
|
||||
"""动态配置类"""
|
||||
|
||||
def __init__(self):
|
||||
self.env = load_environment()
|
||||
self._load_config()
|
||||
|
||||
def _load_config(self):
|
||||
"""加载所有配置"""
|
||||
# 路径配置
|
||||
self.ABSOLUTE_HTML_PATH = os.getenv("ABSOLUTE_HTML_PATH")
|
||||
self.BATH_ROOT = os.getenv("BATH_ROOT")
|
||||
self.IMAGE_FOLDER = os.getenv("IMAGE_FOLDER")
|
||||
self.SCRIPT_PATH = os.getenv("SCRIPT_PATH")
|
||||
self.HTML_DIR = os.getenv("HTML_DIR")
|
||||
self.ABSOLUTE_PATH = os.getenv("ABSOLUTE_PATH")
|
||||
|
||||
# 数据库配置
|
||||
self.SQLALCHEMY_DATABASE_URI = os.getenv("DATABASE_URL")
|
||||
self.SQLALCHEMY_TRACK_MODIFICATIONS = False
|
||||
|
||||
# 应用配置
|
||||
self.MAX_PER_PAGE = int(os.getenv("MAX_PER_PAGE", 100))
|
||||
self.DEFAULT_PER_PAGE = int(os.getenv("DEFAULT_PER_PAGE", 10))
|
||||
self.LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
|
||||
self.SECRET_KEY = os.getenv("SECRET_KEY", "dev-secret-key")
|
||||
|
||||
# Flask 配置
|
||||
self.DEBUG = os.getenv("DEBUG", "False").lower() == "true"
|
||||
self.HOST = os.getenv("HOST", "0.0.0.0")
|
||||
self.PORT = int(os.getenv("PORT", 5000))
|
||||
|
||||
# 文件路径
|
||||
self.APP_ROOT = os.path.dirname(os.path.abspath(__file__))
|
||||
self.PAGES_DIR = os.path.join(self.APP_ROOT, "static/pages")
|
||||
self.ALLOWED_EXTS = {".html", ".htm"}
|
||||
|
||||
# 调度器配置
|
||||
self.ENABLE_SCHEDULER = os.getenv("ENABLE_SCHEDULER", "true").lower() == "true"
|
||||
self.SCHEDULER_INTERVAL = int(os.getenv("SCHEDULER_INTERVAL", 5))
|
||||
|
||||
# MTA task 数量
|
||||
self.MAX_ALLOW_RUNNING_MTA_TASK = int(
|
||||
os.getenv("MAX_ALLOW_RUNNING_MTA_TASK", 3)
|
||||
)
|
||||
|
||||
self.OUTPUT_PIC_PATH = os.getenv(
|
||||
"OUTPUT_PIC_PATH", "/mta_middle_proc/global_results/"
|
||||
)
|
||||
self.OUTPUT_PIC_NAME = os.getenv(
|
||||
"OUTPUT_PIC_NAME", "merged_ground_voxel_max_intensity_bev_zfilter.png"
|
||||
)
|
||||
self.OUTPUT_RESULT_PATH = os.getenv(
|
||||
"OUTPUT_RESULT_PATH", "/mta_middle_proc/global_results/slam_lidar_ground/"
|
||||
)
|
||||
self.CONTAINER_INNER_PATH = os.getenv("CONTAINER_INNER_PATH", "/mnt/mta")
|
||||
self.CONTAINER_OUTER_PATH = os.getenv("CONTAINER_OUTER_PATH", "/data/mnt/")
|
||||
self.ENV_PARAM_KEY = os.getenv("ENV_PARAM_KEY", "ROS_MASTER_URL")
|
||||
self.ENV_PARAM_VALUE = os.getenv("ENV_PARAM_VALUE", "http://localhost:11311")
|
||||
self.MTA_SCRIPT = os.getenv("MTA_SCRIPT", "/app/mta_run_param.sh")
|
||||
|
||||
self.MTA_IMAGE = os.getenv("MTA_IMAGE", "/mta_middle_proc/global_results/")
|
||||
self.MTA_IMNAGE_VERSION = os.getenv("MTA_IMNAGE_VERSION", "1.51")
|
||||
self.RESULT_PATH = os.getenv("RESULT_PATH", "/data/mnt/")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Config env={self.env}>"
|
||||
|
||||
|
||||
# 初始化配置
|
||||
config = Config()
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, config.LOG_LEVEL),
|
||||
format="[%(levelname)s] %(asctime)s - %(name)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
39
fst_data_pipeline/apps/mta_manage_system/db/init.sql
Normal file
39
fst_data_pipeline/apps/mta_manage_system/db/init.sql
Normal file
@@ -0,0 +1,39 @@
|
||||
|
||||
CREATE DATABASE IF NOT EXISTS mta;
|
||||
use mta ;
|
||||
drop TABLE IF EXISTS `session_selection` ;
|
||||
CREATE TABLE IF NOT EXISTS `session_selection` (
|
||||
`id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '自增主键',
|
||||
`html_name` VARCHAR(255) NOT NULL COMMENT ' html name ',
|
||||
`html_folder` VARCHAR(255) NOT NULL COMMENT ' html 文件存储文件夹 ',
|
||||
`bag_folder` VARCHAR(255) NOT NULL COMMENT ' html 的数据包存储文件夹 ',
|
||||
`central_session` VARCHAR(255) NOT NULL COMMENT '单选：central session',
|
||||
`query_sessions` VARCHAR(1024) NOT NULL COMMENT '多选：query session 列表',
|
||||
`selected_image_url` json DEFAULT NULL COMMENT '页面展示的已选图片URL',
|
||||
`is_filter_bag` boolean DEFAULT NULL COMMENT '是否筛选数据包',
|
||||
`is_run_mta` boolean DEFAULT NULL COMMENT '是否跑了mta 任务',
|
||||
`run_mta_result` VARCHAR(255) DEFAULT NULL COMMENT 'mta 任务结果',
|
||||
`remark` VARCHAR(1024) COMMENT '备注信息',
|
||||
`created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||
`updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间',
|
||||
PRIMARY KEY (`id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
|
||||
|
||||
|
||||
|
||||
-- 示例：按 central_session 查询较多时
|
||||
CREATE INDEX idx_central_session ON `session_selection` (`central_session`);
|
||||
-- 示例：按创建时间范围查询
|
||||
CREATE INDEX idx_created_at ON `session_selection` (`created_at`);
|
||||
|
||||
-- 1106
|
||||
-- 添加新字段到 session_selection 表
|
||||
|
||||
ALTER TABLE `session_selection`
|
||||
ADD COLUMN `mta_is_running` BOOLEAN DEFAULT FALSE COMMENT 'mta task 是否在运行';
|
||||
|
||||
ALTER TABLE `session_selection`
|
||||
ADD COLUMN `mta_task_status` VARCHAR(255) DEFAULT 'NOT_STARTED' COMMENT 'mta task 状态:NOT_STARTED, PENDING， RUNNING, FAILED, COMPLETED ,CANCELLED';
|
||||
|
||||
ALTER TABLE `session_selection`
|
||||
ADD COLUMN `mta_task_id` VARCHAR(255) DEFAULT NULL COMMENT 'mta task ID';
|
||||
39
fst_data_pipeline/apps/mta_manage_system/docker/Dockerfile
Normal file
39
fst_data_pipeline/apps/mta_manage_system/docker/Dockerfile
Normal file
@@ -0,0 +1,39 @@
|
||||
# 使用官方 Python 基础镜像
|
||||
FROM python:3.12
|
||||
|
||||
# 设置工作目录
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
|
||||
|
||||
# 安装 uv
|
||||
RUN pip install uv
|
||||
|
||||
# 复制 pyproject.toml 和 poetry.lock 文件（如果有的话）
|
||||
COPY pyproject.toml .
|
||||
|
||||
# 使用 uv 安装依赖，使用 --system 参数 可以应用参数 --index-url https://mirrors.aliyun.com/pypi/simple/ 加速
|
||||
RUN uv pip install --system .
|
||||
|
||||
|
||||
# 复制项目文件
|
||||
COPY models/ ./models/
|
||||
COPY services/ ./services/
|
||||
COPY static/ ./static/
|
||||
COPY templates/ ./templates/
|
||||
COPY utils/ ./utils/
|
||||
COPY app.py .
|
||||
|
||||
COPY extensions.py .
|
||||
COPY config.py .
|
||||
COPY .env .
|
||||
COPY .devenv .
|
||||
COPY __init__.py .
|
||||
COPY README.md .
|
||||
|
||||
# 暴露端口（假设你的Flask应用运行在5000端口）
|
||||
EXPOSE 5000
|
||||
|
||||
# 启动命令
|
||||
CMD ["python", "app.py"]
|
||||
4
fst_data_pipeline/apps/mta_manage_system/extensions.py
Normal file
4
fst_data_pipeline/apps/mta_manage_system/extensions.py
Normal file
@@ -0,0 +1,4 @@
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
|
||||
# 创建全局 db 实例
|
||||
db = SQLAlchemy()
|
||||
@@ -0,0 +1,6 @@
|
||||
# models/__init__.py
|
||||
# 导出模型类，方便从外部导入
|
||||
from .sessionrecord import SessionRecord
|
||||
|
||||
# 可以在这里导入所有模型
|
||||
__all__ = ["SessionRecord"]
|
||||
@@ -0,0 +1,95 @@
|
||||
from typing import List, Set, Dict
|
||||
from datetime import datetime
|
||||
import json
|
||||
from extensions import db
|
||||
|
||||
|
||||
# ===== 数据模型 =====
|
||||
class SessionRecord(db.Model):
|
||||
__tablename__ = "session_selection"
|
||||
|
||||
id = db.Column(
|
||||
db.BigInteger().with_variant(db.Integer, "sqlite"),
|
||||
primary_key=True,
|
||||
autoincrement=True,
|
||||
comment="自增主键",
|
||||
)
|
||||
html_name = db.Column(db.String(255), nullable=False, comment="html name")
|
||||
html_folder = db.Column(db.String(255), nullable=False, comment="html folder")
|
||||
bag_folder = db.Column(db.String(255), nullable=False, comment="bag folder")
|
||||
central_session = db.Column(
|
||||
db.String(255), nullable=False, comment="单选：central session"
|
||||
)
|
||||
query_sessions = db.Column(
|
||||
db.String(1024), nullable=False, comment="多选：query session 列表"
|
||||
)
|
||||
selected_image_url = db.Column(
|
||||
db.Text().with_variant(db.String(1024), "sqlite"), # 兼容 MySQL JSON 与 SQLite
|
||||
nullable=True,
|
||||
comment="页面展示的已选图片URL",
|
||||
)
|
||||
is_filter_bag = db.Column(
|
||||
db.Boolean, nullable=True, default=None, comment="是否筛选数据包"
|
||||
)
|
||||
is_run_mta = db.Column(
|
||||
db.Boolean, nullable=True, default=None, comment="是否跑了mta任务"
|
||||
)
|
||||
mta_is_running = db.Column(
|
||||
db.Boolean, nullable=False, default=False, comment="mta task 是否在运行"
|
||||
)
|
||||
mta_task_status = db.Column(
|
||||
db.String(255),
|
||||
nullable=False,
|
||||
default=False,
|
||||
comment="mta task 状态:NOT_STARTED ,RUNNING ,COMPLETED",
|
||||
)
|
||||
mta_task_id = db.Column(
|
||||
db.String(255), nullable=False, comment="task id：mta_task_id"
|
||||
)
|
||||
run_mta_result = db.Column(db.String(255), nullable=True, comment="mta任务结果")
|
||||
remark = db.Column(db.String(1024), nullable=False, comment="备注信息")
|
||||
created_at = db.Column(
|
||||
db.DateTime, nullable=False, default=datetime.utcnow, comment="创建时间"
|
||||
)
|
||||
updated_at = db.Column( # 修正字段名为 updated_at
|
||||
db.DateTime,
|
||||
nullable=False,
|
||||
default=datetime.utcnow,
|
||||
onupdate=datetime.utcnow,
|
||||
comment="更新时间",
|
||||
)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"id": self.id,
|
||||
"html_name": self.html_name,
|
||||
"html_folder": self.html_folder,
|
||||
"bag_folder": self.bag_folder,
|
||||
"central_session": self.central_session,
|
||||
"query_sessions": self.query_sessions,
|
||||
"selected_image_url": self._parse_json_url(self.selected_image_url),
|
||||
"is_filter_bag": self.is_filter_bag,
|
||||
"is_run_mta": self.is_run_mta,
|
||||
"mta_task_status": self.mta_task_status,
|
||||
"mta_is_running": self.mta_is_running,
|
||||
"mta_task_id": self.mta_task_id,
|
||||
"run_mta_result": self.run_mta_result,
|
||||
"remark": self.remark,
|
||||
"created_at": self.created_at.isoformat(),
|
||||
"updated_at": self.updated_at.isoformat(),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _parse_json_url(value):
|
||||
"""
|
||||
将数据库中的 JSON 字段或字符串安全地反序列化为 Python 对象。
|
||||
MySQL 返回 list/dict；SQLite 可能返回字符串或 None。
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (list, dict)):
|
||||
return value
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return value # 无法解析时原样返回（如旧数据为字符串）
|
||||
@@ -0,0 +1,65 @@
|
||||
from enum import Enum
|
||||
from typing import List, Set, Dict, Any
|
||||
|
||||
|
||||
class TaskStatus(Enum):
|
||||
"""任务状态枚举类"""
|
||||
|
||||
NOT_STARTED = "NOT_STARTED" # 未开始
|
||||
PENDING = "PENDING" # 队列等待中
|
||||
RUNNING = "RUNNING" # 执行中
|
||||
COMPLETED = "COMPLETED" # 已完成
|
||||
FAILED = "FAILED" # 失败
|
||||
CANCELLED = "CANCELLED" # 已取消
|
||||
|
||||
@classmethod
|
||||
def get_active_statuses(cls) -> Set[str]:
|
||||
"""获取活跃状态集合（未完成的状态）"""
|
||||
return {cls.NOT_STARTED.value, cls.PENDING.value, cls.RUNNING.value}
|
||||
|
||||
@classmethod
|
||||
def get_final_statuses(cls) -> Set[str]:
|
||||
"""获取最终状态集合（不会改变的状态）"""
|
||||
return {cls.COMPLETED.value, cls.FAILED.value, cls.CANCELLED.value}
|
||||
|
||||
@classmethod
|
||||
def get_editable_statuses(cls) -> Set[str]:
|
||||
"""获取可编辑状态集合（可以手动改变的状态）"""
|
||||
return {cls.NOT_STARTED.value, cls.PENDING.value}
|
||||
|
||||
@classmethod
|
||||
def is_active(cls, status: str) -> bool:
|
||||
"""检查是否为活跃状态（未完成）"""
|
||||
return status in cls.get_active_statuses()
|
||||
|
||||
@classmethod
|
||||
def is_final(cls, status: str) -> bool:
|
||||
"""检查是否为最终状态（不会改变）"""
|
||||
return status in cls.get_final_statuses()
|
||||
|
||||
@classmethod
|
||||
def is_editable(cls, status: str) -> bool:
|
||||
"""检查状态是否可手动编辑"""
|
||||
return status in cls.get_editable_statuses()
|
||||
|
||||
@classmethod
|
||||
def validate_status(cls, status: str) -> bool:
|
||||
"""验证状态值是否有效"""
|
||||
return any(status == member.value for member in cls)
|
||||
|
||||
@classmethod
|
||||
def get_all_statuses(cls) -> List[str]:
|
||||
"""获取所有状态值列表"""
|
||||
return [member.value for member in cls]
|
||||
|
||||
@classmethod
|
||||
def get_status_choices(cls) -> List[Dict[str, Any]]:
|
||||
"""获取状态选项（用于前端下拉选择）"""
|
||||
return [
|
||||
{"value": cls.NOT_STARTED.value, "label": "未开始", "color": "default"},
|
||||
{"value": cls.PENDING.value, "label": "等待中", "color": "orange"},
|
||||
{"value": cls.RUNNING.value, "label": "执行中", "color": "blue"},
|
||||
{"value": cls.COMPLETED.value, "label": "已完成", "color": "green"},
|
||||
{"value": cls.FAILED.value, "label": "已失败", "color": "red"},
|
||||
{"value": cls.CANCELLED.value, "label": "已取消", "color": "gray"},
|
||||
]
|
||||
29
fst_data_pipeline/apps/mta_manage_system/pyproject.toml
Normal file
29
fst_data_pipeline/apps/mta_manage_system/pyproject.toml
Normal file
@@ -0,0 +1,29 @@
|
||||
[project]
|
||||
name = "mta_management_system"
|
||||
version = "0.1.0"
|
||||
description = "cloud infra pipeline for mb fst data production"
|
||||
authors = [{ name = "MBRDCA", email = "tbd@tbd.com" }]
|
||||
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"flask>=3.1.1",
|
||||
"jinja2==3.1.6",
|
||||
"werkzeug==3.1.3",
|
||||
"click==8.2.1",
|
||||
"xmltodict==0.14.2",
|
||||
|
||||
"python-dotenv==1.1.1",
|
||||
"itsdangerous==2.2.0",
|
||||
"flask_sqlalchemy==3.0.5",
|
||||
"pymysql==1.1.0",
|
||||
"paramiko==3.1.0",
|
||||
"docker==6.1.3",
|
||||
"APScheduler==3.10",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["mta_management_system"]
|
||||
@@ -0,0 +1,228 @@
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from typing import Dict, Any
|
||||
from utils.docker_tool import docker_tool
|
||||
from extensions import db # 统一导入 db 实例
|
||||
from models.sessionrecord import SessionRecord
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LongRunningTasks:
|
||||
"""长时间任务处理类"""
|
||||
|
||||
@staticmethod
|
||||
def api_create_container(
|
||||
task,
|
||||
container_config: Dict[str, Any],
|
||||
volumes: Dict[str, Any],
|
||||
environment: Dict[str, Any],
|
||||
):
|
||||
"""
|
||||
调用API创建容器并等待完成
|
||||
|
||||
Args:
|
||||
container_config: 容器配置，包含：
|
||||
- image_name: 镜像名称
|
||||
- container_name: 容器名称（可选）
|
||||
- command: 启动命令（可选）
|
||||
- timeout: 超时时间（秒，默认3600）
|
||||
|
||||
Returns:
|
||||
Dict: 执行结果
|
||||
"""
|
||||
|
||||
try:
|
||||
# 提取配置参数
|
||||
|
||||
image_name = container_config.get("image_name", "ubuntu:latest")
|
||||
container_name = container_config.get("container_name")
|
||||
command = container_config.get("command")
|
||||
timeout = container_config.get("timeout", 3600)
|
||||
result_file = container_config.get("result_file")
|
||||
id = container_config.get("id")
|
||||
result_file_name = container_config.get("result_file_name")
|
||||
if not image_name:
|
||||
raise ValueError("必须提供image_name参数")
|
||||
logger.info(
|
||||
f"开始创建容器: {container_name or '未命名'}, 镜像: {image_name}"
|
||||
)
|
||||
|
||||
# 调用工具类创建容器
|
||||
# 检查Docker环境
|
||||
if not docker_tool.is_docker_available():
|
||||
return {
|
||||
"success": False,
|
||||
"error": "宿主机Docker环境不可用",
|
||||
"message": "请检查Docker是否正确挂载",
|
||||
}
|
||||
# 在创建容器之前先创建同名容器是否在运行或者已经存在，如果已经在运行则 停止 ，并删除， 如果没有则进行创建
|
||||
containers = docker_tool.get_containers()
|
||||
mta_container = None
|
||||
for container in containers:
|
||||
names = container.get("Names", "")
|
||||
# 精确匹配MTA-1（考虑Docker可能添加/前缀的情况）
|
||||
if names == container_name:
|
||||
mta_container = container
|
||||
break
|
||||
|
||||
if mta_container:
|
||||
container_id = mta_container.get("ID", mta_container.get("Id", ""))
|
||||
print(f"找到MTA容器，ID: {container_id}")
|
||||
# 停止容器
|
||||
stop_result = docker_tool.stop_container(container_id)
|
||||
if not stop_result["success"]:
|
||||
print(f"停止容器失败: {stop_result.get('error', '未知错误')}")
|
||||
return
|
||||
|
||||
print("容器已停止")
|
||||
|
||||
# 删除容器
|
||||
remove_result = docker_tool.remove_container(container_id)
|
||||
if remove_result["success"]:
|
||||
print(f"{container_name}容器已成功删除")
|
||||
else:
|
||||
print(f"删除容器失败: {remove_result.get('error', '未知错误')}")
|
||||
print(
|
||||
f"创建容器参数command: {command} ，volumes 挂载： {volumes}，environment 参数：{environment} "
|
||||
)
|
||||
# 使用工具类创建容器
|
||||
result = docker_tool.create_container(
|
||||
image_name=image_name,
|
||||
container_name=container_name,
|
||||
command=command,
|
||||
volumes=volumes,
|
||||
environment=environment,
|
||||
)
|
||||
if result["success"]:
|
||||
print(" 容器创建成功")
|
||||
container_id = result["container_id"]
|
||||
|
||||
# 根据调用的工具类创建容器 ，根据工具类创建容器的返回值 判断容器是否创建成功
|
||||
completion_file = result_file
|
||||
logger.info(f"等待完成文件: {completion_file}")
|
||||
|
||||
# 轮询MTA 的结果文件否存在
|
||||
start_time = time.time()
|
||||
check_count = 0
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
check_count += 1
|
||||
# 获取 容器运行状态 如果容器停止运行且结果文件不存在则抛出异常
|
||||
container_result = docker_tool.get_container_status_by_id(
|
||||
container_id
|
||||
)
|
||||
state = container_result.get("state", "").lower()
|
||||
if (
|
||||
state != "running"
|
||||
and not LongRunningTasks._check_completion_file(completion_file)
|
||||
):
|
||||
# 确保容器被清理
|
||||
try:
|
||||
docker_tool.stop_container(container_name)
|
||||
docker_tool.remove_container(container_name)
|
||||
except Exception as cleanup_e:
|
||||
logger.error(f"清理容器失败: {cleanup_e}")
|
||||
raise Exception(
|
||||
f"容器状态异常: {state}, 结果文件未生成: {completion_file}"
|
||||
)
|
||||
|
||||
# 检查完成文件是否存在 同时判断 容器是否运行正常， 如果
|
||||
if LongRunningTasks._check_completion_file(completion_file):
|
||||
logger.info(f"MTA 任务执行 完成: {container_name}")
|
||||
# 判断 任务执行完成后 调用工具类 停止容器 然后删除容器
|
||||
docker_tool.stop_container(container_name)
|
||||
docker_tool.remove_container(container_name)
|
||||
|
||||
return {
|
||||
"status": "completed",
|
||||
"container_name": container_name,
|
||||
"image_name": image_name,
|
||||
"id": id, # 添加id到返回结果中，供回调函数使用
|
||||
"command": command,
|
||||
"completion_file": completion_file,
|
||||
"completed_at": datetime.now().isoformat(),
|
||||
"total_wait_seconds": int(time.time() - start_time),
|
||||
"check_count": check_count,
|
||||
}
|
||||
|
||||
# 等待5秒再检查
|
||||
time.sleep(5)
|
||||
print(
|
||||
f"第{check_count}次检查路径：{completion_file}，完成文件尚未出现..."
|
||||
)
|
||||
logger.info(f"第{check_count}次检查，完成文件尚未出现...")
|
||||
else:
|
||||
print(" 容器创建失败")
|
||||
|
||||
logger.info("调用API创建容器...")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"创建容器失败: {e}")
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _check_completion_file(file_path: str) -> bool:
|
||||
import os
|
||||
|
||||
if not os.path.exists(file_path):
|
||||
return False
|
||||
|
||||
for filename in os.listdir(file_path):
|
||||
if filename.endswith(".pcd"):
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def process_with_completion_check(process_config: Dict[str, Any]):
|
||||
"""
|
||||
通用处理任务，使用文件检查判断完成
|
||||
|
||||
Args:
|
||||
process_config: 处理配置，包含：
|
||||
- process_type: 处理类型
|
||||
- completion_file: 完成标志文件路径
|
||||
- timeout: 超时时间（秒）
|
||||
|
||||
Returns:
|
||||
Dict: 执行结果
|
||||
"""
|
||||
try:
|
||||
process_type = process_config.get("process_type", "unknown")
|
||||
completion_file = process_config.get("completion_file")
|
||||
timeout = process_config.get("timeout", 7200)
|
||||
|
||||
if not completion_file:
|
||||
raise ValueError("必须提供completion_file参数")
|
||||
|
||||
logger.info(f"开始处理任务: {process_type}, 完成文件: {completion_file}")
|
||||
|
||||
# 轮询检查完成文件
|
||||
start_time = time.time()
|
||||
check_count = 0
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
check_count += 1
|
||||
|
||||
if LongRunningTasks._check_completion_file(completion_file):
|
||||
logger.info(f"处理任务完成: {process_type}")
|
||||
return {
|
||||
"status": "completed",
|
||||
"process_type": process_type,
|
||||
"completion_file": completion_file,
|
||||
"completed_at": datetime.now().isoformat(),
|
||||
"total_wait_seconds": int(time.time() - start_time),
|
||||
"check_count": check_count,
|
||||
}
|
||||
|
||||
time.sleep(60) # 每分钟检查一次
|
||||
logger.info(f"第{check_count}次检查，任务处理中...")
|
||||
|
||||
raise Exception(f"处理任务超时({timeout}秒)")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"处理任务失败: {e}")
|
||||
raise
|
||||
2079
fst_data_pipeline/apps/mta_manage_system/static/css/bootstrap-icons.css
vendored
Normal file
2079
fst_data_pipeline/apps/mta_manage_system/static/css/bootstrap-icons.css
vendored
Normal file
File diff suppressed because it is too large
Load Diff
7
fst_data_pipeline/apps/mta_manage_system/static/css/bootstrap.min.css
vendored
Normal file
7
fst_data_pipeline/apps/mta_manage_system/static/css/bootstrap.min.css
vendored
Normal file
File diff suppressed because one or more lines are too long
@@ -0,0 +1,76 @@
|
||||
body { font-family: sans-serif; margin: 1em; background: #f7f9fc; }
|
||||
iframe {
|
||||
width: 95%;
|
||||
max-width: 1300px; /* 单个 iframe 最大宽度 */
|
||||
height: 800px; /* 保持你原有高度 */
|
||||
border: 1px solid #ccc;
|
||||
margin-top: 1em;
|
||||
background: white;
|
||||
box-sizing: border-box; /* 让 padding/border 不额外撑宽 */
|
||||
display: block; /* 去除底部空白缝隙 */
|
||||
margin-left: auto;
|
||||
margin-right: auto;
|
||||
margin-left: 0 ;
|
||||
}
|
||||
.image-box {
|
||||
min-height: 400px; /* 根据你的图片高度设置 */
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
background: #f8f9fa;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
|
||||
#imagePreview {
|
||||
max-width: 100%;
|
||||
max-height: 500px;
|
||||
object-fit: contain; /* 保持比例 */
|
||||
transition: opacity 0.3s ease;
|
||||
}
|
||||
#log { max-height: 200px; overflow-y: auto; background: #222; color: #0f0; padding: 0.5em; font-family: monospace; font-size: 0.9em; }
|
||||
h2 { margin-top: 0.5em; }
|
||||
|
||||
|
||||
.button-container {
|
||||
display: flex;
|
||||
gap: 15px;
|
||||
margin-top: 20px;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.btn {
|
||||
padding: 12px 30px;
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s ease;
|
||||
min-width: 120px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 1px;
|
||||
}
|
||||
|
||||
.btn-primary {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
box-shadow: 0 4px 15px rgba(102, 126, 234, 0.4);
|
||||
}
|
||||
|
||||
.btn-primary:hover {
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 6px 20px rgba(102, 126, 234, 0.6);
|
||||
}
|
||||
|
||||
.btn-secondary {
|
||||
background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
|
||||
color: white;
|
||||
box-shadow: 0 4px 15px rgba(245, 87, 108, 0.4);
|
||||
}
|
||||
|
||||
.btn-secondary:hover {
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 6px 20px rgba(245, 87, 108, 0.6);
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
BIN
fst_data_pipeline/apps/mta_manage_system/static/images/image.jpg
Normal file
BIN
fst_data_pipeline/apps/mta_manage_system/static/images/image.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 114 KiB |
BIN
fst_data_pipeline/apps/mta_manage_system/static/images/left.jpg
Normal file
BIN
fst_data_pipeline/apps/mta_manage_system/static/images/left.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 5.5 KiB |
BIN
fst_data_pipeline/apps/mta_manage_system/static/images/right.jpg
Normal file
BIN
fst_data_pipeline/apps/mta_manage_system/static/images/right.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 5.0 KiB |
7
fst_data_pipeline/apps/mta_manage_system/static/js/bootstrap.bundle.min.js
vendored
Normal file
7
fst_data_pipeline/apps/mta_manage_system/static/js/bootstrap.bundle.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
139
fst_data_pipeline/apps/mta_manage_system/static/js/detail.js
Normal file
139
fst_data_pipeline/apps/mta_manage_system/static/js/detail.js
Normal file
@@ -0,0 +1,139 @@
|
||||
'use strict';
|
||||
|
||||
|
||||
// const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
|
||||
// console.log('Bag Folder:', bag_folder);
|
||||
function getJpgFileName(path) {
|
||||
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
const clean = path.replace(/\\/g, '/').split('?')[0].trim();
|
||||
// 判断是否以 .jpg 结尾（忽略大小写）
|
||||
const isJpg = /\.jpg$/i.test(clean);
|
||||
if (!isJpg) return ''; // 不以 .jpg 结尾则返回空
|
||||
|
||||
// 从最后一个斜杠后截取到末尾
|
||||
const idx = clean.lastIndexOf('/');
|
||||
return idx === -1 ? clean : clean.slice(idx + 1);
|
||||
}
|
||||
|
||||
function showImage(imagename,bagname,htmlname,bag_folder) {
|
||||
console.log('htmlname:'+htmlname) ;
|
||||
// alert(bagname)
|
||||
$("#imagediv").empty(); // 或 .html('')
|
||||
|
||||
// 2) 构造要插入的新内容：一个带 class 和 id 的 div，内嵌一张图片
|
||||
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
const newContent = `
|
||||
<img id="imagePreview" src="http://localhost:5000/api/getimage/${encodeURIComponent(imagename)}/${encodeURIComponent(bagname)}/${encodeURIComponent(htmlname)}/${encodeURIComponent(bag_folder)}" alt="图片预览" width="700px">
|
||||
`;
|
||||
|
||||
// 3) 将新内容插入到目标 div 中
|
||||
$("#imagediv").html(newContent);
|
||||
|
||||
|
||||
}
|
||||
|
||||
function getImageByBagAndIndex(bagname,imageIndex,htmlname,bagfolder) {
|
||||
|
||||
var imagename = "";
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
$.ajax({
|
||||
url: '/api/getimage',
|
||||
async: false, // 同步
|
||||
method: 'GET',
|
||||
data: { bagname: bagname,
|
||||
imageIndex: imageIndex,
|
||||
bagfolder: bagfolder,
|
||||
htmlname:htmlname
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
var data = res.result
|
||||
|
||||
console.log(data)
|
||||
var size = data.size
|
||||
imagename = data.imagename
|
||||
}
|
||||
})
|
||||
console.log("imagename===="+imagename)
|
||||
return imagename;
|
||||
}
|
||||
|
||||
function getCentralQuerySession(id) {
|
||||
var data ;
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
$.ajax({
|
||||
url: `/api/items/${id}`,
|
||||
async: false, // 同步
|
||||
method: 'GET',
|
||||
data: { id: id
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
data = res.result
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
console.log("data===="+data)
|
||||
return data;
|
||||
}
|
||||
|
||||
function runMTACommand(command,html_name,central_session ,query_sessions,id,bag_folder,category) {
|
||||
console.log('runMTACommand methon id value :'+id )
|
||||
var imagename = "";
|
||||
console.log(bag_folder)
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
$.ajax({
|
||||
url: '/api/runmtacommand',
|
||||
async: false, // 同步
|
||||
method: 'GET',
|
||||
data: { command: command,
|
||||
html_name: html_name,
|
||||
central_session: central_session,
|
||||
query_sessions:query_sessions,
|
||||
id:id ,
|
||||
bag_folder:bag_folder,
|
||||
category:category
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
console.log(res.status)
|
||||
var checkResult = res.success
|
||||
var msg = res.message
|
||||
showResult(checkResult,msg)
|
||||
|
||||
console.log(res)
|
||||
var data = res.message
|
||||
console.log(data)
|
||||
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
function showResult(result ,msg) {
|
||||
|
||||
if (result ){
|
||||
Swal.fire({
|
||||
title: '任务创建成功！',
|
||||
text: msg,
|
||||
icon: 'success'
|
||||
});
|
||||
} else {
|
||||
Swal.fire({
|
||||
title: '任务创建失败',
|
||||
text: '任务创建失败，请联系管理员',
|
||||
icon: 'error'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 可选：用于确认脚本已加载
|
||||
console.log('[detail.js] loaded, getJpgFileName =', typeof getJpgFileName);
|
||||
@@ -0,0 +1,73 @@
|
||||
(function () {
|
||||
const send = (type, detail = {}) => {
|
||||
try {
|
||||
// 向父窗口发送事件；同源策略下父窗口可接收
|
||||
window.parent.postMessage({ type, timestamp: Date.now(), ...detail }, '*');
|
||||
} catch (e) {
|
||||
console.warn('[injector] postMessage 失败:', e);
|
||||
}
|
||||
};
|
||||
|
||||
// 页面开始加载
|
||||
send('page-start', { href: location.href });
|
||||
|
||||
// 点击
|
||||
document.addEventListener('click', (e) => {
|
||||
send('click', {
|
||||
target: e.target.tagName.toLowerCase() + (e.target.id ? '#' + e.target.id : '') + (e.target.className ? '.' + e.target.className.split(' ').join('.') : ''),
|
||||
href: location.href,
|
||||
x: e.clientX, y: e.clientY,
|
||||
button: e.button, buttons: e.buttons
|
||||
});
|
||||
}, true);
|
||||
|
||||
// 输入/变更（含输入框、文本域、选择框等）
|
||||
['input', 'change', 'focus', 'blur'].forEach(evtName => {
|
||||
document.addEventListener(evtName, (e) => {
|
||||
const tag = e.target.tagName.toLowerCase();
|
||||
if (tag === 'input' || tag === 'textarea' || tag === 'select') {
|
||||
send(evtName, {
|
||||
target: e.target.tagName.toLowerCase() + (e.target.id ? '#' + e.target.id : '') + '.' + e.target.className.split(' ').join('.'),
|
||||
href: location.href,
|
||||
value: e.target.value,
|
||||
name: e.target.name
|
||||
});
|
||||
} else {
|
||||
send(evtName, { target: tag, href: location.href });
|
||||
}
|
||||
}, true);
|
||||
});
|
||||
|
||||
// 滚动
|
||||
let scrollTmr = null;
|
||||
window.addEventListener('scroll', () => {
|
||||
if (scrollTmr) clearTimeout(scrollTmr);
|
||||
scrollTmr = setTimeout(() => {
|
||||
send('scroll', {
|
||||
href: location.href,
|
||||
scrollTop: window.scrollY || document.documentElement.scrollTop,
|
||||
scrollLeft: window.scrollX || document.documentElement.scrollLeft
|
||||
});
|
||||
}, 200);
|
||||
}, true);
|
||||
|
||||
// 键盘（可观测快捷键等）
|
||||
document.addEventListener('keydown', (e) => {
|
||||
send('keydown', {
|
||||
key: e.key, code: e.code,
|
||||
ctrlKey: e.ctrlKey, shiftKey: e.shiftKey, altKey: e.altKey, metaKey: e.metaKey,
|
||||
target: e.target.tagName.toLowerCase(),
|
||||
href: location.href
|
||||
});
|
||||
}, true);
|
||||
|
||||
// 页面可见性变化（切换标签/最小化）
|
||||
document.addEventListener('visibilitychange', () => {
|
||||
send('visibilitychange', { hidden: document.hidden, href: location.href });
|
||||
});
|
||||
|
||||
// 页面卸载前
|
||||
window.addEventListener('beforeunload', () => {
|
||||
send('page-leave', { href: location.href });
|
||||
});
|
||||
})();
|
||||
2
fst_data_pipeline/apps/mta_manage_system/static/js/jquery-3.7.1.min.js
vendored
Normal file
2
fst_data_pipeline/apps/mta_manage_system/static/js/jquery-3.7.1.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
916
fst_data_pipeline/apps/mta_manage_system/templates/detail.html
Normal file
916
fst_data_pipeline/apps/mta_manage_system/templates/detail.html
Normal file
@@ -0,0 +1,916 @@
|
||||
<!doctype html>
|
||||
<html lang="zh">
|
||||
<head>
|
||||
<base href="/">
|
||||
<meta charset="UTF-8" />
|
||||
<title>数据包选择页面</title>
|
||||
|
||||
<link rel="stylesheet" href="../static/css/sweetalert2.css">
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/detail.css') }}">
|
||||
|
||||
</head>
|
||||
<body>
|
||||
<h1>数据包选择页面</h1>
|
||||
|
||||
<div id="page" >
|
||||
|
||||
<!-- 使用 Bootstrap 的行和列布局将元素放在同一行 -->
|
||||
<div class="row align-items-center"> <!-- align-items-center 用于垂直居中对齐 -->
|
||||
<!-- 第一列：标题 <h2> -->
|
||||
<div class="col-auto">
|
||||
<h2>{{ html_name }}</h2>
|
||||
</div>
|
||||
|
||||
<!-- 第二列：清空按钮 -->
|
||||
<div class="col-auto">
|
||||
<button type="button" class="btn btn-primary" onclick="clearAll()">清空</button>
|
||||
</div>
|
||||
|
||||
<!-- 第三列：单选组件 -->
|
||||
<div class="col-auto">
|
||||
<fieldset class="form-group">
|
||||
<legend class="col-form-label pt-0">选择类型（单选）</legend>
|
||||
<div id = "selectionType" class="radio-options">
|
||||
<label class="mr-3">
|
||||
<input type="radio" name="selectionType" value="none" checked /> None
|
||||
</label>
|
||||
<label class="mr-3">
|
||||
<input type="radio" name="selectionType" value="centralsession" /> central session
|
||||
</label>
|
||||
<label>
|
||||
<input type="radio" name="selectionType" value="querysession" /> query session
|
||||
</label>
|
||||
</div>
|
||||
</fieldset>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!--<!– <iframe id="frame1" src="" sandbox="allow-scripts allow-same-origin allow-forms allow-popups allow-popups-to-escape-sandbox"></iframe>–>-->
|
||||
<!-- <iframe id="frame12" src="/pages/html2"-->
|
||||
<iframe id="myIframe" src="../static/pages/{{ html_folder }}/{{ html_name }}" sandbox="allow-scripts allow-same-origin allow-forms allow-popups allow-popups-to-escape-sandbox"></iframe>
|
||||
<!-- <iframe id="frame13" src="{{ url_for('static', filename='pages/html1.html') }}" sandbox="allow-scripts allow-same-origin allow-forms allow-popups allow-popups-to-escape-sandbox"></iframe>-->
|
||||
</div>
|
||||
|
||||
<div class="form-row">
|
||||
<!-- 下拉选择框 -->
|
||||
<div class="form-group">
|
||||
<label for="selectKey">选择 数据包：</label>
|
||||
<select id="selectKey" class="form-control">
|
||||
<option value="" disabled selected>请选择</option>
|
||||
</select>
|
||||
</div>
|
||||
<!-- 统计信息显示 -->
|
||||
<!-- <div id="imagecount" style="margin-top:8px;font-size:14px;color:#555;"></div>
|
||||
<div id="imageinfo" style="margin-top:8px;font-size:14px;color:#555;"></div> -->
|
||||
<div style="margin-top:8px; display: flex; align-items: center; gap: 10px; font-size:14px; color:#555;">
|
||||
<div id="imagecount"></div>
|
||||
<div id="imageinfo"></div>
|
||||
</div>
|
||||
<span id="bagname" hidden data-value=""></span>
|
||||
<span id="hiddenImageCount" hidden data-value=""></span>
|
||||
<span id="hiddenHtmlFolder" hidden value={{ html_folder }}></span>
|
||||
<span id="hiddenBagFolder" hidden value={{ bag_folder }}></span>
|
||||
<span id="hiddenAbsilutePath" hidden value={{ abslute_path }}></span>
|
||||
<!--<span id="htmlid" hidden value={{ id }}></span>-->
|
||||
<span id="mtataskid" hidden data-value={{ id }}></span>
|
||||
<!-- 输入框 -->
|
||||
<div class="form-group" style="margin-left: 12px;">
|
||||
<label for="numberInput">输入数字：</label>
|
||||
<input type="text" id="numberInput" class="form-control" placeholder="请输入数字" />
|
||||
</div>
|
||||
<!-- 数字输入组：左侧减、右侧加、中间步长 -->
|
||||
<div class="input-group" style="width: 360px; margin-left: 12px;">
|
||||
<span id="hiddenTag" hidden data-value="1"></span>
|
||||
<!-- 左侧减按钮 -->
|
||||
<button type="button" class=" btn-outline-secondary" id="stepDownBtn" style="border-radius: 0.375rem 0 0 0.375rem;">
|
||||
<img src="{{ url_for('static', filename='images/left.jpg') }}" alt="→" width="16" height="16" style="vertical-align: middle;">
|
||||
</button>
|
||||
<!-- 右侧加按钮 -->
|
||||
<button type="button" class=" btn-outline-secondary" id="stepUpBtn" style="border-radius: 0 0.375rem 0.375rem 0;">
|
||||
<img src="{{ url_for('static', filename='images/right.jpg') }}" alt="→" width="16" height="16" style="vertical-align: middle;">
|
||||
</button>
|
||||
<!-- 步长输入框 -->
|
||||
<div class="input-group" style="width: 100px;">
|
||||
<input type="text" id="stepInput" class="form-control text-center" placeholder="步长" />
|
||||
</div>
|
||||
</div>
|
||||
<!-- 确认按钮 -->
|
||||
<div class="form-group" style="margin-left: 12px;">
|
||||
<button id="confirmBtn" class="btn btn-primary">确认</button>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- 图片容器 -->
|
||||
<div id="image-container">
|
||||
<h2>图片展示</h2>
|
||||
<div class="error" id="error"></div>
|
||||
<div class="image-box" id="imagediv">
|
||||
<img id="imagePreview" src="{{ url_for('static', filename='images/image.jpg') }}" alt="图片预览" width="400px">
|
||||
</div>
|
||||
</div>
|
||||
<!-- 用于展示数字的区域 -->
|
||||
<div id="result" type="hidden" style="margin-top: 12px; font-size: 14px; color: #333;"></div>
|
||||
<img id="preview" alt="预览" style="max-width:100%;height:auto;display:none;margin-top:8px;">
|
||||
|
||||
<!-- 外部展示容器 -->
|
||||
<div id="selected-labels" style="margin-top:8px;color:#333;">未选中</div>
|
||||
<!-- 复选框 -->
|
||||
<form id="myForm">
|
||||
<input type="hidden" id = "html_name" name="html_name" value="{{ html_name }}">
|
||||
<input type="hidden" id = "mtaBagfolder" name="bag_folder" value="{{ bag_folder }}">
|
||||
<input type="hidden" id = "id" name="id" value="{{ id }}">
|
||||
<!-- 单选框组 -->
|
||||
<fieldset>
|
||||
<legend>central session（单选）</legend>
|
||||
<div id="radio-container" class="items"></div>
|
||||
</fieldset>
|
||||
<!-- 用于展示数字的区域 -->
|
||||
<div id="querySession" style="margin-top: 12px; font-size: 14px; color: #333;">选择的数据包：</div>
|
||||
<!-- 复选框组 -->
|
||||
<fieldset>
|
||||
<legend>query session（可多选）</legend>
|
||||
<div id="checkbox-container" class="items"></div>
|
||||
</fieldset>
|
||||
|
||||
|
||||
|
||||
<div class="button-container">
|
||||
<button type="button" class="btn btn-primary" onclick="handleSubmit()">提交</button>
|
||||
<button type="button" class="btn btn-primary" onclick="clearAll()">清空</button>
|
||||
</div>
|
||||
</form>
|
||||
<script src="../static/js/jquery-3.7.1.min.js"></script>
|
||||
<script src="../static/js/detail.js"></script>
|
||||
<script src="../static/js/sweetalert2.js"></script>
|
||||
<script>
|
||||
// 父页面接收消息处理函数
|
||||
window.addEventListener('message', (event) => {
|
||||
// 验证消息来源（可选但推荐）
|
||||
if (event.origin === window.location.origin) {
|
||||
// 检查消息类型
|
||||
if (event.data.type === 'tagClick') {
|
||||
// 弹出alert显示标签名
|
||||
var session_name = event.data.tagName ;
|
||||
console.log("选中的 seesion name:" + session_name)
|
||||
// alert('点击的标签名: ' + event.data.tagName);
|
||||
|
||||
// 获取 name="selectionType" 单选组件的当前选中值
|
||||
var selectedSelectionType = $('input[name="selectionType"]:checked').val();
|
||||
|
||||
console.log("当前选择的 selectionType:", selectedSelectionType);
|
||||
|
||||
if (!selectedSelectionType) {
|
||||
// 如果 selectionType 未选择（即 none），则不执行任何操作
|
||||
console.log("selectionType 未选择（none），不执行任何操作。");
|
||||
return;
|
||||
}
|
||||
|
||||
switch (selectedSelectionType) {
|
||||
case 'centralsession':
|
||||
handleCentralSession(session_name);
|
||||
break;
|
||||
case 'querysession':
|
||||
handleQuerySession(session_name);
|
||||
break;
|
||||
case 'none':
|
||||
// 不执行任何操作
|
||||
console.log("selectionType 为 none，不执行任何操作。");
|
||||
break;
|
||||
default:
|
||||
console.warn("未知的 selectionType:", selectedSelectionType);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理 central session（单选）的逻辑
|
||||
* @param {string} tagName - 点击的标签名
|
||||
*/
|
||||
function handleCentralSession(tagName) {
|
||||
// 查找与 tagName 匹配的单选按钮
|
||||
var radioButton = $('input[name="category"][value="' + tagName + '"]');
|
||||
|
||||
if (radioButton.length) {
|
||||
// 切换选中状态
|
||||
radioButton.prop('checked', !radioButton.prop('checked'));
|
||||
console.log(`central session 单选按钮 "${tagName}" 的选中状态已切换为:`, radioButton.prop('checked'));
|
||||
|
||||
// 更新显示（可选）
|
||||
updateSelectedLabels();
|
||||
} else {
|
||||
console.warn(`未找到与标签名 "${tagName}" 对应的 central session 单选按钮`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理 query session（复选）的逻辑
|
||||
* @param {string} tagName - 点击的标签名
|
||||
*/
|
||||
function handleQuerySession(tagName) {
|
||||
// 查找与 tagName 匹配的复选框
|
||||
var checkbox = $('input[name="tags"][value="' + tagName + '"]');
|
||||
|
||||
if (checkbox.length) {
|
||||
// 切换选中状态
|
||||
checkbox.prop('checked', !checkbox.prop('checked'));
|
||||
console.log(`query session 复选框 "${tagName}" 的选中状态已切换为:`, checkbox.prop('checked'));
|
||||
|
||||
// 更新显示（可选）
|
||||
updateSelectedLabels();
|
||||
} else {
|
||||
console.warn(`未找到与标签名 "${tagName}" 对应的 query session 复选框`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新外部展示容器中的选中标签显示（可选）
|
||||
*/
|
||||
function updateSelectedLabels() {
|
||||
// 更新 central session 选中的标签
|
||||
var selectedCentral = [];
|
||||
$('input[name="category"]:checked').each(function() {
|
||||
var label = $(`label[for="${this.id}"]`).text().replace(/\s*[\u2713\u2611]\s*/, '').trim(); // 去除勾选符号
|
||||
if (label) {
|
||||
selectedCentral.push(label);
|
||||
} else {
|
||||
// 兜底：使用 value
|
||||
selectedCentral.push($(this).val());
|
||||
}
|
||||
});
|
||||
$('#selected-labels').text(selectedCentral.length > 0 ? selectedCentral.join(', ') : '未选中');
|
||||
|
||||
// 更新 query session 选中的标签
|
||||
var selectedQuery = [];
|
||||
$('input[name="tags"]:checked').each(function() {
|
||||
var label = $(`label[for="${this.id}"]`).text().replace(/\s*[\u2713\u2611]\s*/, '').trim(); // 去除勾选符号
|
||||
if (label) {
|
||||
selectedQuery.push(label);
|
||||
} else {
|
||||
// 兜底：使用 value
|
||||
selectedQuery.push($(this).val());
|
||||
}
|
||||
});
|
||||
$('#querySession').text(selectedQuery.length > 0 ? selectedQuery.join(', ') : '未选中');
|
||||
}
|
||||
|
||||
// 初始化时调用一次以设置初始状态
|
||||
updateSelectedLabels();
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
const iframe = document.getElementById('myIframe');
|
||||
iframe.onload = () => {
|
||||
try {
|
||||
const doc = iframe.contentDocument || iframe.contentWindow.document;
|
||||
|
||||
// 向iframe页面注入脚本，添加点击事件监听
|
||||
const script = doc.createElement('script');
|
||||
script.textContent = `
|
||||
// 在iframe页面中添加全局点击事件监听
|
||||
document.addEventListener('click', function(e) {
|
||||
let target = e.target;
|
||||
let tagName = null;
|
||||
|
||||
// 检查元素本身或其父元素是否包含目标格式的文本
|
||||
while (target && target !== document.body) {
|
||||
// 尝试从多种可能的属性中获取标签名
|
||||
const possibleNames = [
|
||||
target.textContent?.trim(),
|
||||
target.innerText?.trim(),
|
||||
target.getAttribute('id'),
|
||||
target.getAttribute('data-name'),
|
||||
target.getAttribute('title')
|
||||
];
|
||||
|
||||
// 检查是否有符合格式的标签名
|
||||
for (const name of possibleNames) {
|
||||
if (name && name.includes('PL') && name.includes('event') && name.includes('.bag.dir')) {
|
||||
tagName = name;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (tagName) break;
|
||||
target = target.parentElement;
|
||||
}
|
||||
|
||||
// 如果找到符合条件的标签名，发送消息给父页面
|
||||
if (tagName) {
|
||||
window.parent.postMessage({
|
||||
type: 'tagClick',
|
||||
tagName: tagName
|
||||
}, '*'); // 使用*允许所有来源，生产环境应指定具体域名
|
||||
}
|
||||
});
|
||||
`;
|
||||
doc.head.appendChild(script);
|
||||
console.log('成功注入点击事件监听器');
|
||||
} catch (error) {
|
||||
console.error('注入脚本时出错:', error);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
$(function () {
|
||||
|
||||
|
||||
|
||||
// 添加调试代码
|
||||
console.log('SweetAlert2 是否加载:', typeof Swal);
|
||||
console.log('Swal.fire 是否存在:', typeof Swal.fire);
|
||||
|
||||
const $result = $("#result");
|
||||
const $img = $("#preview");
|
||||
const $imageinfo = $("#imageinfo");
|
||||
const $imagecount = $("#imagecount");
|
||||
const hiddenImageCount = $("#hiddenImageCount");
|
||||
|
||||
function updateMess(index ,imagename){
|
||||
const cur = `当前展示第 <b>${index}</b> 张 , 文件名为 : <b>${imagename}</b> `;
|
||||
$imageinfo.html( );
|
||||
$imageinfo.html( cur);
|
||||
|
||||
}
|
||||
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
|
||||
console.log('Bag Folder:', bag_folder);
|
||||
$('#selectKey').on('change', function () {
|
||||
const bagname = $(this).val();
|
||||
console.log(bagname)
|
||||
const bagfullname = $('#selectKey option:selected').text();
|
||||
|
||||
$('#bagname').attr('data-value', bagfullname);
|
||||
var html_name = $('#html_name').val();
|
||||
html_name = html_name.replace(".html","")
|
||||
|
||||
|
||||
if (!bagname) return;
|
||||
$('#result').text('加载中...');
|
||||
$.ajax({
|
||||
url: '/api/getimagedata',
|
||||
|
||||
method: 'GET',
|
||||
data: { bagname: bagname ,
|
||||
htmlname: html_name ,
|
||||
bagfolder: bag_folder ,
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
// $('#result').text(res.message ?? JSON.stringify(res, null, 2));
|
||||
var data = res.result
|
||||
// 清空两个输入框
|
||||
$('#numberInput').val('');
|
||||
$('#stepInput').val('');
|
||||
|
||||
console.log(data)
|
||||
var size = data.size
|
||||
var index = data.index
|
||||
var imagename = data.imagename ;
|
||||
// 去除路径
|
||||
imagename = getJpgFileName(imagename) ;
|
||||
// 将包中图片数量更新到隐藏域
|
||||
|
||||
$('#hiddenImageCount').attr('data-value', size);
|
||||
const base = `数据包中共有 <b>${size}</b> 张图片`;
|
||||
const cur = `当前展示第 <b>1</b> 张 , 文件名为 : <b>${imagename}</b> `;
|
||||
$imageinfo.html( cur);
|
||||
$imagecount.html(base );
|
||||
|
||||
// 展示图片
|
||||
|
||||
showImage(imagename,bagname,html_name,bag_folder);
|
||||
|
||||
},
|
||||
error(xhr) {
|
||||
const msg = xhr.responseJSON?.message || xhr.statusText;
|
||||
$('#result').text('加载失败：' + msg);
|
||||
}
|
||||
});
|
||||
})
|
||||
})
|
||||
|
||||
const htmlName = "{{ html_name }}";
|
||||
|
||||
const logBox = document.getElementById('log');
|
||||
|
||||
// function log(text) {
|
||||
// const msg = `[${new Date().toISOString()}] ${text}`;
|
||||
// console.log(msg);
|
||||
// logBox.textContent += msg + '\n';
|
||||
// logBox.scrollTop = logBox.scrollHeight;
|
||||
// }
|
||||
|
||||
window.addEventListener('message', (evt) => {
|
||||
// 仅接收来自本域的 iframe 消息（更安全可校验 evt.origin）
|
||||
// console.debug('收到消息：', evt.data);
|
||||
if (!evt || !evt.data || !evt.data.type) return;
|
||||
const { type, href, target, value, x, y, button, buttons, key, code, ctrlKey, shiftKey, altKey, metaKey, timestamp } = evt.data;
|
||||
const entry = {
|
||||
type, href, target, value,
|
||||
coord: x !== undefined ? { x, y } : null,
|
||||
mouseButtons: buttons !== undefined ? { left: button === 0, middle: button === 1, right: button === 2 } : null,
|
||||
keyInfo: key !== undefined ? { key, code, ctrlKey, shiftKey, altKey, metaKey } : null,
|
||||
ts: new Date(timestamp).toLocaleTimeString()
|
||||
};
|
||||
// log(`${entry.ts} [${type}] ${href || ''} ${target || ''} ${key || ''} ${JSON.stringify(entry.coord || entry.mouseButtons || entry.keyInfo)}`);
|
||||
});
|
||||
|
||||
|
||||
|
||||
// 获取 DOM 元素
|
||||
const selectEl = document.getElementById('selectKey');
|
||||
const inputEl = document.getElementById('numberInput');
|
||||
const btnEl = document.getElementById('confirmBtn');
|
||||
const resultEl = document.getElementById('result');
|
||||
const imageinfo = $("#imageinfo");
|
||||
const hiddenImageCount = $("#hiddenImageCount");
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
const html_folder = $('#hiddenHtmlFolder').attr('value');
|
||||
// 1) 加载下拉选项（默认空）
|
||||
async function loadOptions() {
|
||||
try {
|
||||
const res = await fetch('/api/options/{{ html_name }}/{{ html_folder }}'); // 后台接口：返回 [{value: 'a', label: 'A'}, ...]
|
||||
console.log(res)
|
||||
|
||||
if (!res.ok) throw new Error('加载选项失败');
|
||||
const json = await res.json(); // json 是解析后的 JS 对象
|
||||
console.log(json.status); // true
|
||||
console.log(json.result);
|
||||
const data = await json.result;
|
||||
|
||||
// 清空并填充选项（保留默认占位）
|
||||
selectEl.innerHTML = '<option value="" disabled selected>请选择</option>';
|
||||
data.forEach(item => {
|
||||
const opt = document.createElement('option');
|
||||
opt.value = item.value;
|
||||
opt.textContent = item.label;
|
||||
selectEl.appendChild(opt);
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('加载下拉选项出错:', err);
|
||||
selectEl.innerHTML = '<option value="" disabled selected>加载失败</option>';
|
||||
}
|
||||
}
|
||||
|
||||
// 2) 输入数字后，确认按钮点击事件
|
||||
function get_image() {
|
||||
// 打印输入框的值
|
||||
const inputValue = inputEl.value.trim();
|
||||
const imagecount = document.getElementById('imagecount');
|
||||
const imageinfo = document.getElementById('imageinfo');
|
||||
const hiddenTag = document.getElementById('hiddenTag');
|
||||
|
||||
console.log('输入框的值：', inputValue);
|
||||
var html_name = $('#html_name').val();
|
||||
html_name = html_name.replace(".html","")
|
||||
console.log('html000_name :'+html_name)
|
||||
|
||||
var size = $("#hiddenImageCount").attr('data-value');
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
const n1 = Number(inputValue);
|
||||
const n2 = Number(size);
|
||||
console.log('html111_name :'+html_name)
|
||||
// 在页面显示该数字
|
||||
if (inputValue === '') {
|
||||
resultEl.textContent = '请输入有效数字';
|
||||
resultEl.style.color = '#999';
|
||||
} else if(n1>n2){
|
||||
alert('输入值大于图片总数')
|
||||
}else {
|
||||
// 可按需做数字校验
|
||||
const num = Number(inputValue);
|
||||
if (isNaN(num)) {
|
||||
resultEl.textContent = '输入内容不是有效数字';
|
||||
resultEl.style.color = 'red';
|
||||
} else {
|
||||
// 更新隐藏域的值 为输入值
|
||||
$('#hiddenTag').attr('data-value', inputValue);
|
||||
// 更新页面显示图片信息
|
||||
const base = `数据包中共有 <b>${size}</b> 张图片`;
|
||||
imagecount.innerHTML = '';
|
||||
$(imagecount).html( base);
|
||||
|
||||
|
||||
resultEl.textContent = `数字：${num}`;
|
||||
resultEl.style.color = '#333';
|
||||
|
||||
// 加载图片
|
||||
// 通过包名 调用方法获取 包内图片的数据
|
||||
const bagname = $('#bagname').attr('data-value');
|
||||
console.log('输入数字获取图片：'+html_name)
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
console.log('输入-----------：'+bag_folder)
|
||||
|
||||
|
||||
const imagename = getImageByBagAndIndex(bagname,inputValue,html_name,bag_folder);
|
||||
console.log(imagename)
|
||||
|
||||
var imagename_short = getJpgFileName(imagename);
|
||||
console.log(imagename_short)
|
||||
const cur = `当前展示第 <b>${inputValue}</b> 张 , 文件名为 : <b>${imagename_short}</b> `;
|
||||
|
||||
$(imageinfo).html('');
|
||||
$(imageinfo).html( cur);
|
||||
showImage(imagename_short,bagname,html_name,bag_folder);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- 页面 左右按钮点击图片变更动作 start -->
|
||||
document.addEventListener('DOMContentLoaded', function () {
|
||||
const stepDownBtn = document.getElementById('stepDownBtn');
|
||||
const stepUpBtn = document.getElementById('stepUpBtn');
|
||||
const stepInput = document.getElementById('stepInput');
|
||||
const hiddenTag = document.getElementById('hiddenTag');
|
||||
const resultImg = document.getElementById('resultImg'); // 结果图片容器
|
||||
const $imageinfo = $("#imageinfo");
|
||||
const imagecount = $("#imagecount").attr('data-value');
|
||||
|
||||
// 工具：将字符串转为整数，失败返回 NaN
|
||||
function toInt(v) {
|
||||
const n = Number(v);
|
||||
return Number.isInteger(n) ? n : NaN;
|
||||
}
|
||||
|
||||
// 从输入框或隐藏域读取“当前值”
|
||||
function getCurrentValue() {
|
||||
const raw = stepInput.value.trim();
|
||||
if (raw !== '') {
|
||||
const step = toInt(raw);
|
||||
if (!isNaN(step)) return step;
|
||||
}
|
||||
return toInt(hiddenTag.dataset.value) ?? 1; // 默认 1
|
||||
}
|
||||
|
||||
// 更新隐藏域与输入框显示
|
||||
function updateHidden(newValue) {
|
||||
const n = toInt(newValue);
|
||||
if (!isNaN(n)) {
|
||||
hiddenTag.dataset.value = n;
|
||||
|
||||
return n;
|
||||
}
|
||||
return getCurrentValue(); // 回退
|
||||
}
|
||||
|
||||
// 减步长
|
||||
stepDownBtn.addEventListener('click', async function () {
|
||||
var html_name = $('#html_name').val();
|
||||
html_name = html_name.replace(".html","")
|
||||
console.log('html_name :'+html_name)
|
||||
const base = toInt(hiddenTag.dataset.value)
|
||||
const hide =toInt(hiddenTag.dataset.value)
|
||||
const $imagecount = $("#imagecount");
|
||||
console.log(' hide 值为：'+hide)
|
||||
const raw = stepInput.value.trim() ;
|
||||
const step = raw === "" ? 1 : parseInt(raw, 10);
|
||||
console.log(' step 值为：'+step)
|
||||
const newVal = base - step;
|
||||
|
||||
console.log(' newVal 值为：'+newVal)
|
||||
if (isNaN(newVal) || !Number.isInteger(newVal) || newVal <= 0) {
|
||||
alert('索引必须为大于零整数');
|
||||
return;
|
||||
}
|
||||
console.log('newVal 值为：'+ newVal)
|
||||
const final = updateHidden(newVal);
|
||||
console.log('final 值为：'+ final)
|
||||
|
||||
|
||||
|
||||
|
||||
// 加载图片
|
||||
// 通过包名 调用方法获取 包内图片的数据
|
||||
const bagname = $('#bagname').attr('data-value');
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
|
||||
const imagename = getImageByBagAndIndex(bagname,newVal,html_name,bag_folder);
|
||||
console.log(imagename)
|
||||
|
||||
var imagename_short = getJpgFileName(imagename);
|
||||
console.log(imagename_short)
|
||||
|
||||
const cur = `当前展示第 <b>${final}</b> 张 , 文件名为 : <b>${imagename_short}</b> `;
|
||||
$imageinfo.html( );
|
||||
$imageinfo.html( cur);
|
||||
showImage(imagename_short,bagname,html_name,bag_folder);
|
||||
|
||||
});
|
||||
|
||||
// 加步长
|
||||
stepUpBtn.addEventListener('click', async function () {
|
||||
var html_name = $('#html_name').val();
|
||||
html_name = html_name.replace(".html","")
|
||||
console.log('html_name :'+html_name)
|
||||
const base = toInt(hiddenTag.dataset.value)
|
||||
const hide =toInt(hiddenTag.dataset.value)
|
||||
console.log(' hide 值为：'+hide)
|
||||
const raw = stepInput.value.trim() ;
|
||||
const step = raw === "" ? 1 : parseInt(raw, 10);
|
||||
console.log(' step 值为：'+step)
|
||||
const newVal = base + step;
|
||||
const imagecount = $("#hiddenImageCount").attr('data-value');
|
||||
console.log(' newVal 值为：'+newVal + 'imagecount:' + imagecount)
|
||||
if (isNaN(newVal) || !Number.isInteger(newVal) || newVal <= 0) {
|
||||
alert('索引必须为大于零整数');
|
||||
return;
|
||||
}else if(newVal > imagecount){
|
||||
alert('超出索引最大数值');
|
||||
return;
|
||||
}
|
||||
|
||||
const final = updateHidden(newVal);
|
||||
|
||||
|
||||
|
||||
// 加载图片
|
||||
// 通过包名 调用方法获取 包内图片的数据
|
||||
const bagname = $('#bagname').attr('data-value');
|
||||
const bag_folder = $('#hiddenBagFolder').attr('value');
|
||||
|
||||
const imagename = getImageByBagAndIndex(bagname,newVal,html_name,bag_folder);
|
||||
console.log(imagename)
|
||||
|
||||
var imagename_short = getJpgFileName(imagename);
|
||||
console.log(imagename_short)
|
||||
|
||||
const cur = `当前展示第 <b>${final}</b> 张 , 文件名为 : <b>${imagename_short}</b> `;
|
||||
$imageinfo.html( );
|
||||
$imageinfo.html( cur);
|
||||
showImage(imagename_short,bagname,html_name,bag_folder);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
<!-- 页面 左右按钮点击图片变更 end -->
|
||||
|
||||
|
||||
<!-- 单选和复选框 加载数据 js -->
|
||||
// 容器与接口
|
||||
const containerC = document.getElementById('checkbox-container');
|
||||
const containerR = document.getElementById('radio-container');
|
||||
// const html_folder = $('#hiddenHtmlFolder').attr('value');
|
||||
const API = '/api/options/{{ html_name }}/{{ html_folder }}'; // 查询html 中的包数据
|
||||
|
||||
var id = {{ id }} ;
|
||||
var data = getCentralQuerySession(id) ;
|
||||
|
||||
var centralsession = data.centralSession ;
|
||||
var querySessions = data.querySessions ;
|
||||
console.log(centralsession)
|
||||
console.log(querySessions)
|
||||
// 加载并渲染
|
||||
async function loadData() {
|
||||
try {
|
||||
const res = await fetch(API);
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const json = await res.json();
|
||||
|
||||
|
||||
const data = Array.isArray(json.result) ? json.result : [];
|
||||
|
||||
// 清空旧项
|
||||
containerC.innerHTML = '';
|
||||
containerR.innerHTML = '';
|
||||
|
||||
// 生成复选框（多选）
|
||||
data.forEach(opt => {
|
||||
const idC = `cb-${opt.value}`;
|
||||
const labelC = document.createElement('label');
|
||||
labelC.className = 'item';
|
||||
labelC.htmlFor = idC;
|
||||
|
||||
const cb = document.createElement('input');
|
||||
cb.type = 'checkbox';
|
||||
cb.id = idC;
|
||||
cb.name = 'tags'; // 多选用同一 name
|
||||
cb.value = opt.value;
|
||||
|
||||
// ✅ 修改位置1: 检查是否在querySessions数组中，如果是则选中
|
||||
if (querySessions.includes(opt.value)) {
|
||||
cb.checked = true;
|
||||
}
|
||||
|
||||
labelC.appendChild(cb);
|
||||
labelC.append(' ', opt.label);
|
||||
containerC.appendChild(labelC);
|
||||
});
|
||||
|
||||
// 生成单选框（单选）
|
||||
data.forEach(opt => {
|
||||
const idR = `rd-${opt.value}`;
|
||||
const labelR = document.createElement('label');
|
||||
labelR.className = 'item';
|
||||
labelR.htmlFor = idR;
|
||||
|
||||
const rd = document.createElement('input');
|
||||
rd.type = 'radio';
|
||||
rd.id = idR;
|
||||
rd.name = 'category'; // 单选用同一 name
|
||||
rd.value = opt.value;
|
||||
|
||||
// ✅ 修改位置2: 检查是否等于centralsession，如果是则选中
|
||||
if (opt.value === centralsession) {
|
||||
rd.checked = true;
|
||||
}
|
||||
|
||||
labelR.appendChild(rd);
|
||||
labelR.append(' ', opt.label);
|
||||
containerR.appendChild(labelR);
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('加载选项失败:', err);
|
||||
containerC.textContent = '加载失败';
|
||||
containerR.textContent = '加载失败';
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// 仅当复选框 A 变化时，打印页面上所有已选中的 label 文本 START
|
||||
function watchCheckboxPrintLabels(cbContainerId, output) {
|
||||
|
||||
const container = typeof output === 'string' ? document.querySelector(output) : output;
|
||||
const cbContainer = document.getElementById(cbContainerId);
|
||||
if (!container || !cbContainer) {
|
||||
console.error('[watchCheckboxPrintLabels] 容器未找到');
|
||||
return;
|
||||
}
|
||||
|
||||
// 使用事件委托监听复选框变化（更稳健，支持动态新增）
|
||||
document.addEventListener('change', (e) => {
|
||||
if (!e.target.matches(`#${cbContainerId} input[type="checkbox"]`)) return;
|
||||
// 收集页面上所有已勾选复选框的 label 文本
|
||||
const labels = Array.from(document.querySelectorAll('input[type="checkbox"]:checked'))
|
||||
.map(cb => {
|
||||
// 优先通过 for 匹配
|
||||
if (cb.id) {
|
||||
const lab = document.querySelector(`label[for="${cb.id}"]`);
|
||||
if (lab) return lab.textContent.trim();
|
||||
}
|
||||
// 兜底：若复选框被 <label><input>...</label> 包裹，取父级文本
|
||||
if (cb.parentElement?.tagName === 'LABEL') {
|
||||
return cb.parentElement.textContent.replace(/\s*[\u2713\u2611]\s*/,'').trim(); // 去除勾选符号
|
||||
}
|
||||
// 再兜底：title、data-label，最后回退 value
|
||||
if (cb.title) return cb.title.trim();
|
||||
if (cb.dataset.label) return cb.dataset.label.trim();
|
||||
return cb.value.trim();
|
||||
});
|
||||
|
||||
// 更新外部 div 显示
|
||||
container.textContent = labels.length > 0 ? labels.join(', ') : '未选中';
|
||||
}, { passive: true });
|
||||
}
|
||||
|
||||
// 调用：传入复选框容器ID 与 外部展示容器（选择器或DOM）
|
||||
watchCheckboxPrintLabels('checkbox-container', '#querySession');
|
||||
|
||||
// 调用：传入复选框容器ID 与 外部展示容器（选择器或DOM）
|
||||
watchCheckboxPrintLabels('radio-container', '#selected-labels');
|
||||
// 仅当复选框 A 变化时，打印页面上所有已选中的 label 文本 END
|
||||
|
||||
// 获取表单数据
|
||||
function getFormData() {
|
||||
var bag_folder = $('#mtaBagfolder').val();
|
||||
console.log(bag_folder)
|
||||
const html_name = $('#html_name').val();
|
||||
var id = $("#mtataskid").attr('data-value');
|
||||
const checkedBoxes = Array.from(
|
||||
document.querySelectorAll('#checkbox-container input:checked')
|
||||
).map(cb => cb.value);
|
||||
|
||||
const selectedRadio = document.querySelector('#radio-container input:checked');
|
||||
return {
|
||||
id:id,
|
||||
html_name:html_name,
|
||||
bag_folder: bag_folder,
|
||||
tags: checkedBoxes,
|
||||
category: selectedRadio ? selectedRadio.value : null
|
||||
};
|
||||
}
|
||||
function showCommitAlert(message, html_name,id, central_session, query_session,bag_folder,category, callback) {
|
||||
Swal.fire({
|
||||
title: 'MTA bag 提交成功！',
|
||||
text: message,
|
||||
icon: 'success',
|
||||
showCancelButton: true, // 显示取消按钮
|
||||
confirmButtonText: '执行命令',
|
||||
cancelButtonText: '返回首页',
|
||||
allowOutsideClick: false,
|
||||
allowEscapeKey: false
|
||||
}).then((result) => {
|
||||
if (result.isConfirmed) {
|
||||
// 点击"执行命令"按钮
|
||||
console.log('🎯 执行命令:', message);
|
||||
console.log('⏰ 执行时间:', new Date().toLocaleString());
|
||||
console.log('执行的html 页面为:', message);
|
||||
console.log('showCommitAlert methon id value :'+id )
|
||||
command = ''
|
||||
// 可以在这里添加实际执行命令的代码
|
||||
console.log(bag_folder)
|
||||
runMTACommand(command,html_name, central_session,query_session,id,bag_folder,category)
|
||||
|
||||
} else if (result.dismiss === Swal.DismissReason.cancel) {
|
||||
// 点击"返回首页"按钮
|
||||
|
||||
window.location.href = "/";
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
// 提交示例
|
||||
function handleSubmit() {
|
||||
const data = getFormData();
|
||||
console.log('提交数据:', data);
|
||||
var category = data.category ;
|
||||
var tags = data.tags ;
|
||||
var bag_folder = data.bag_folder ;
|
||||
var html_name = $('#html_name').val();
|
||||
|
||||
console.log('提交数据:', mtataskid);
|
||||
// 检查 central session
|
||||
if (!category || category.trim() === '') {
|
||||
alert('Central Session 不能为空，请选择或输入有效值');
|
||||
return;
|
||||
}
|
||||
|
||||
// 检查 query session
|
||||
if (!tags || tags.length === 0) {
|
||||
alert('Query Session 不能为空，请至少选择一个标签');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
$.ajax({
|
||||
url: '/api/submit',
|
||||
async: false, // 同步
|
||||
method: 'POST',
|
||||
contentType: 'application/json',
|
||||
data: JSON.stringify(data),
|
||||
success(result) {
|
||||
console.log('✅ bagdata 提交成功:', result);
|
||||
console.log('状态:', result.status);
|
||||
console.log('data:', result.result.data);
|
||||
console.log('Central Session:', result.result.data.central_session);
|
||||
console.log('Query Sessions:', result.result.data.query_sessions);
|
||||
console.log('bag_folder:', bag_folder);
|
||||
console.log('handleSubmit methon id value :'+id )
|
||||
var command = './multi_session_aggregation --central_session '+result.result.data.central_session + ' --query_session '+result.result.data.query_sessions
|
||||
showCommitAlert(command,html_name,id,result.result.data.central_session,result.result.data.query_sessions,bag_folder,category) ;
|
||||
|
||||
// alert(`mta 任务 提交成功 \n 命令为：`+command);
|
||||
|
||||
}
|
||||
})
|
||||
|
||||
} catch (error) {
|
||||
console.error('提交失败:', error);
|
||||
alert(`mta 任务 提交失败: ${error.message}`);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
// 清空
|
||||
function clearAll() {
|
||||
$("#selected-labels").empty();
|
||||
$("#querySession").empty();
|
||||
|
||||
document.querySelectorAll('#checkbox-container input').forEach(cb => cb.checked = false);
|
||||
document.querySelectorAll('#radio-container input').forEach(rd => rd.checked = false);
|
||||
}
|
||||
|
||||
// 页面加载完成后拉取数据
|
||||
loadData();
|
||||
|
||||
// 绑定事件
|
||||
btnEl.addEventListener('click', get_image);
|
||||
|
||||
// 页面加载完成后拉取选项
|
||||
loadOptions();
|
||||
|
||||
// 单选选择项发生变化后页面展示
|
||||
$(document).ready(function() {
|
||||
// 使用事件委托，适用于动态添加的选项
|
||||
$('#radio-container').on('change', 'input[name="category"]', function() {
|
||||
if ($(this).is(':checked')) {
|
||||
// console.log('单选选中的值:', $(this).val());
|
||||
|
||||
$('#selected-labels').text($(this).val());
|
||||
}
|
||||
});
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
697
fst_data_pipeline/apps/mta_manage_system/templates/list.html
Normal file
697
fst_data_pipeline/apps/mta_manage_system/templates/list.html
Normal file
@@ -0,0 +1,697 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>SessionRecord 列表</title>
|
||||
<link href="../static/css/bootstrap.min.css" rel="stylesheet">
|
||||
<link href="../static/css//bootstrap-icons.css" rel="stylesheet">
|
||||
<link rel="stylesheet" href="../static/css/sweetalert2.css">
|
||||
|
||||
<!-- 引入 SweetAlert2 JS -->
|
||||
<script src="../static/js/sweetalert2.js"></script>
|
||||
<style>
|
||||
.text-truncate-custom { max-width: 240px; }
|
||||
.json-viewer { font-size: 0.875rem; background-color: #f8f9fa; padding: 0.5rem; border-radius: 0.25rem; white-space: pre-wrap; margin-top: 0.25rem; }
|
||||
.badge-bool-null { font-size: 0.75em; padding: 0.25em 0.5em; }
|
||||
.badge-bool-true { background-color: #198754; }
|
||||
.badge-bool-false { background-color: #6c757d; }
|
||||
.badge-bool-null { background-color: #6c757d; color: #fff; }
|
||||
.card { box-shadow: 0 0.125rem 0.25rem rgba(0,0,0,0.075); }
|
||||
.table th { width: 140px; }
|
||||
.loading-shade { position: absolute; inset: 0; background: rgba(255,255,255,0.8); display: flex; align-items: center; justify-content: center; z-index: 10; }
|
||||
</style>
|
||||
</head>
|
||||
<body class="p-3">
|
||||
<div class="container-fluid">
|
||||
<div class="card">
|
||||
<div class="card-header">
|
||||
<h5 class="mb-0">SessionRecord 列表</h5>
|
||||
<input type="hidden" id = "page" name="html_name" value="{{ html_name }}">
|
||||
</div>
|
||||
<div class="card-body p-0">
|
||||
<!-- 筛选区 -->
|
||||
<div class="p-3 border-bottom">
|
||||
<div class="row g-2">
|
||||
<div class="col-md-3">
|
||||
<input type="text" class="form-control" id="html_name" placeholder="html_name" value="">
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<input type="text" class="form-control" id="central_session" placeholder="central_session" value="">
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<span>是否筛选过数据包</span>
|
||||
<select class="form-select" id="is_filter_bag">
|
||||
<option value="">全部</option>
|
||||
<option value="true">是</option>
|
||||
<option value="false">否</option>
|
||||
</select>
|
||||
</div>
|
||||
<!-- <div class="col-md-3">-->
|
||||
<!-- <span>是否跑过mta</span>-->
|
||||
<!-- <select class="form-select" id="is_run_mta">-->
|
||||
<!-- <option value="">全部</option>-->
|
||||
<!-- <option value="true">是</option>-->
|
||||
<!-- <option value="false">否</option>-->
|
||||
<!-- </select>-->
|
||||
<!-- </div>-->
|
||||
<div class="col-md-3">
|
||||
<span>mta 任务状态</span>
|
||||
<select class="form-select" id="mta_task_status">
|
||||
<option value="">全部</option>
|
||||
<option value="NOT_STARTED">未启动</option>
|
||||
<option value="RUNNING">在运行</option>
|
||||
<option value="COMPLETED">已完成</option>
|
||||
<option value="FAILED">失败</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="col-md-2">
|
||||
<select class="form-select" id="per_page">
|
||||
<option value="10">10 条/页</option>
|
||||
<option value="20" selected>20 条/页</option>
|
||||
<option value="50">50 条/页</option>
|
||||
<option value="100">100 条/页</option>
|
||||
</select>
|
||||
</div>
|
||||
<!-- jquery 添加一个 同步数据的按钮 -->
|
||||
<div class="col-md-2 ms-auto text-end">
|
||||
<button id="query-data-btn" class="btn btn-primary">
|
||||
<i class="fas fa-sync-alt"></i> 查询
|
||||
</button>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<input type="text" class="" id="html_folder" placeholder="html_folder" value="">
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<input type="text" class="" id="bag_folder" placeholder="bag_folder" value="">
|
||||
</div>
|
||||
<div class="col-md-2 ms-auto text-end">
|
||||
<button id="sync-data-btn" class="btn btn-primary">
|
||||
<i class="fas fa-sync-alt"></i> 同步数据
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 表格 -->
|
||||
<div class="table-responsive" style="position:relative;">
|
||||
<table class="table table-hover mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>HTML Name</th>
|
||||
<th>Central Session</th>
|
||||
<th>Query Sessions</th>
|
||||
<th>Html Folder</th>
|
||||
<th>筛选数据包</th>
|
||||
<!-- <th>MTA 任务</th>-->
|
||||
<th>MTA Task 状态</th>
|
||||
<th>MTA 结果</th>
|
||||
<th>备注</th>
|
||||
<th>创建时间</th>
|
||||
<th>更新时间</th>
|
||||
<th>操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="tbody">
|
||||
<!-- 数据由 jQuery 加载 -->
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- 分页与信息 -->
|
||||
<div class="p-3 d-flex justify-content-between align-items-center">
|
||||
<div id="info" class="text-muted small"></div>
|
||||
<nav aria-label="分页导航">
|
||||
<ul class="pagination mb-0" id="pagination"></ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 编辑弹窗 Modal -->
|
||||
<div class="modal fade" id="editModal" tabindex="-1" aria-labelledby="editModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="editModalLabel">编辑记录</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<!-- 这里放置你要编辑或展示的字段 -->
|
||||
<form id="editForm">
|
||||
<input type="hidden" id="edit_id" name="id">
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="edit_html_name" class="form-label">HTML Name</label>
|
||||
<input type="text" class="form-control" id="edit_html_name" name="html_name" required>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="edit_central_session" class="form-label">Central Session</label>
|
||||
<input type="text" class="form-control" id="edit_central_session" name="central_session" required>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="edit_central_session" class="form-label">Query Sessions</label>
|
||||
<input type="text" class="form-control" id="edit_query_session" name="query_session" required>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="edit_is_filter_bag" class="form-label">筛选数据包</label>
|
||||
<select class="form-select" id="edit_is_filter_bag" name="is_filter_bag">
|
||||
<option value="">请选择</option>
|
||||
<option value="true">是</option>
|
||||
<option value="false">否</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="edit_is_run_mta" class="form-label">是否跑 MTA</label>
|
||||
<select class="form-select" id="edit_is_run_mta" name="is_run_mta">
|
||||
<option value="">请选择</option>
|
||||
<option value="true">是</option>
|
||||
<option value="false">否</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="run_mta_result" class="form-label">是否成功</label>
|
||||
<input type="text" class="form-control" id="edit_run_mta_result" name="run_mta_result" required>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="edit_remark" class="form-label">备注</label>
|
||||
<textarea class="form-control" id="edit_remark" name="remark" rows="3"></textarea>
|
||||
</div>
|
||||
|
||||
<!-- 其它字段按需添加，如 query_sessions、selected_image_url、run_mta_result 等 -->
|
||||
</form>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">取消</button>
|
||||
<button type="button" class="btn btn-primary" id="saveEditBtn" onclick="">保存</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
image
|
||||
<!-- 图片展示弹出框 -->
|
||||
<div class="modal fade" id="imageModal" tabindex="-1" aria-labelledby="imageModal" aria-hidden="true">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="imageModal">MTA 结果展示</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body" id="MTAImageDiv" >
|
||||
<!-- 这里放置你要编辑或展示的字段 -->
|
||||
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
|
||||
<button type="button" class="btn btn-primary" id="closeImage1" onclick="closeImage()">关闭</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 你的项目中已存在 jquery.js，确保已正确引入，如： -->
|
||||
<!-- <script src="/path/to/your/jquery.js"></script> -->
|
||||
<script src="../static/js/jquery-3.7.1.min.js"></script>
|
||||
<script src="../static/js/bootstrap.bundle.min.js"></script>
|
||||
<script>
|
||||
|
||||
// 点击查询数据按钮 start
|
||||
$('#query-data-btn').click(function() {
|
||||
// 显示加载状态
|
||||
const $btn = $(this);
|
||||
|
||||
loadData() ;
|
||||
});
|
||||
|
||||
// 点击同步数据按钮 start
|
||||
$('#sync-data-btn').click(function() {
|
||||
// 显示加载状态
|
||||
const $btn = $(this);
|
||||
$btn.prop('disabled', true);
|
||||
$btn.html('<i class="fas fa-spinner fa-spin"></i> 同步中...');
|
||||
// 同步数据， 先获取输入的 bagfolder 和 htmlfolder 参数值
|
||||
var bag_folder = $('#bag_folder').val();
|
||||
var html_folder = $('#html_folder').val();
|
||||
console.log(bag_folder)
|
||||
console.log(html_folder)
|
||||
|
||||
|
||||
// 调用同步API /api/syncdata
|
||||
$.ajax({
|
||||
url: '/api/syncdata',
|
||||
method: 'GET',
|
||||
data: {
|
||||
bag_folder: bag_folder,
|
||||
html_folder: html_folder
|
||||
},
|
||||
dataType: 'json',
|
||||
success: function(response) {
|
||||
console.log(response)
|
||||
console.log(response.scanned_files)
|
||||
// 同步成功提示
|
||||
Swal.fire({
|
||||
title: '同步成功！',
|
||||
text: 'html文件夹数量：'+response.scanned_files +'，数据库新增数量：'+response.new_records_added,
|
||||
icon: 'success'
|
||||
});
|
||||
|
||||
// 刷新当前页数据（假设你有加载数据的函数）
|
||||
loadData(); // currentPage 是当前页码变量
|
||||
},
|
||||
error: function(xhr) {
|
||||
Swal.fire({
|
||||
title: '同步失败',
|
||||
text: xhr.responseJSON?.message || '服务器错误',
|
||||
icon: 'error'
|
||||
});
|
||||
},
|
||||
complete: function() {
|
||||
// 恢复按钮状态
|
||||
$btn.prop('disabled', false);
|
||||
$btn.html('<i class="fas fa-sync-alt"></i> 同步数据');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
// 点击同步数据按钮 start
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// 保存按钮点击（使用 jQuery AJAX）
|
||||
$('#saveEditBtn').on('click', function () {
|
||||
console.log("进入 保存数据方法 ")
|
||||
const $form = $('#editForm');
|
||||
const $btn = $(this).prop('disabled', true); // 防止重复提交
|
||||
|
||||
|
||||
|
||||
// 2) 获取表单数据并转为普通对象
|
||||
const formData = new FormData($form[0]);
|
||||
const data = Object.fromEntries(formData);
|
||||
console.log(data)
|
||||
|
||||
|
||||
// 4) 判断是新增还是更新
|
||||
const id = data.id;
|
||||
|
||||
|
||||
const url = '/api/saveitems';
|
||||
const method = 'POST';
|
||||
|
||||
// 5) 使用 jQuery $.ajax 发送请求
|
||||
$.ajax({
|
||||
url: url,
|
||||
type: method,
|
||||
contentType: 'application/json',
|
||||
data: JSON.stringify(data),
|
||||
dataType: 'json',
|
||||
success: function (result) {
|
||||
// 成功：隐藏弹窗，重新加载列表
|
||||
$('#editModal').modal('hide');
|
||||
loadData(); // 请确保此方法会重新请求列表并渲染表格
|
||||
},
|
||||
error: function (xhr, status, error) {
|
||||
console.error('保存失败:', status, error);
|
||||
let errorMsg = '保存失败';
|
||||
if (xhr.responseJSON && xhr.responseJSON.error) {
|
||||
errorMsg = xhr.responseJSON.error;
|
||||
} else if (xhr.responseText) {
|
||||
try {
|
||||
const res = JSON.parse(xhr.responseText);
|
||||
if (res.error) errorMsg = res.error;
|
||||
} catch (e) {
|
||||
errorMsg = xhr.responseText.substring(0, 100);
|
||||
}
|
||||
}
|
||||
alert(errorMsg); // 或使用更友好的 toast / modal 提示
|
||||
},
|
||||
complete: function () {
|
||||
$btn.prop('disabled', false); // 恢复按钮
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// 编辑数据
|
||||
function editRow(id){
|
||||
$.get('/api/items/' + id, function (data) {
|
||||
console.log(data.result)
|
||||
console.log("记录数据：", data.result.isFilterBag);
|
||||
console.log("记录数据：", data.result.isRunMta);
|
||||
var isFilterBag = data.result.isFilterBag.toString()
|
||||
var isRunMta = data.result.isRunMta.toString()
|
||||
$('#edit_id').val(data.result.id);
|
||||
$('#edit_html_name').val(data.result.htmlName);
|
||||
$('#edit_central_session').val(data.result.centralSession);
|
||||
$('#edit_query_session').val(data.result.querySessions);
|
||||
$('#edit_is_filter_bag').val(isFilterBag);
|
||||
$('#edit_is_run_mta').val(isRunMta);
|
||||
$('#edit_run_mta_result').val(data.result.runMtaResult);
|
||||
$('#edit_remark').val(data.result.remark);
|
||||
$('#editModal').modal('show');
|
||||
});
|
||||
|
||||
// 直接弹窗，填充）
|
||||
$('#editModal').modal('show');
|
||||
}
|
||||
|
||||
// stop Task
|
||||
function stopTask(id) {
|
||||
$.ajax({
|
||||
url: '/api/stoptask',
|
||||
async: false, // 同步
|
||||
method: 'GET',
|
||||
data: { id: id
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
var msg = res.message
|
||||
var result = res.success
|
||||
if (result ){
|
||||
Swal.fire({
|
||||
title: '任务停止！',
|
||||
text: msg,
|
||||
icon: 'success'
|
||||
});
|
||||
} else {
|
||||
Swal.fire({
|
||||
title: '任务停止失败',
|
||||
text: '任务停止失败，请联系管理员',
|
||||
icon: 'error'
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
loadData();
|
||||
}
|
||||
|
||||
// reload Task
|
||||
function reloadTask(id) {
|
||||
$.ajax({
|
||||
url: '/api/reloadtask',
|
||||
async: false, // 同步
|
||||
method: 'GET',
|
||||
data: { id: id
|
||||
},
|
||||
dataType: 'json',
|
||||
success(res) {
|
||||
var msg = res.message
|
||||
var result = res.success
|
||||
console.log(msg)
|
||||
console.log(result)
|
||||
if (result ){
|
||||
Swal.fire({
|
||||
title: '任务重跑成功',
|
||||
text: msg,
|
||||
icon: 'success'
|
||||
});
|
||||
} else {
|
||||
Swal.fire({
|
||||
title: '任务重跑失败',
|
||||
text: msg,
|
||||
icon: 'error'
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
loadData();
|
||||
}
|
||||
|
||||
// 图片展示
|
||||
function imageshow(id){
|
||||
|
||||
// 直接弹窗，填充）
|
||||
$('#imageModal').modal('show');
|
||||
|
||||
$("#MTAImageDiv").empty(); // 或 .html('')
|
||||
|
||||
// 2) 构造要插入的新内容：一个带 class 和 id 的 div，内嵌一张图片
|
||||
|
||||
// 统一斜杠并去除末尾可能的查询参数
|
||||
const newContent = `
|
||||
<img id="imagePreview" src="http://localhost:5000/api/getmtaresult/${encodeURIComponent(id)}" alt="图片预览" width="700px">
|
||||
`;
|
||||
|
||||
// 3) 将新内容插入到目标 div 中
|
||||
$("#MTAImageDiv").html(newContent);
|
||||
}
|
||||
function closeImage(){
|
||||
$('#imageModal').modal('hide');
|
||||
}
|
||||
// 工具：对象转查询字符串（自动编码，排除空值）
|
||||
function qs(params) {
|
||||
return Object.keys(params)
|
||||
.filter(function(k) {
|
||||
var v = params[k];
|
||||
return v != null && v !== '';
|
||||
})
|
||||
.map(function(k) {
|
||||
return encodeURIComponent(k) + '=' + encodeURIComponent(params[k]);
|
||||
})
|
||||
.join('&');
|
||||
}
|
||||
|
||||
// 工具：当前查询参数（从表单元素获取）
|
||||
function currentQuery() {
|
||||
console.log('-----5-----'+$('#page').val())
|
||||
return {
|
||||
page: $('#page').val() || 1,
|
||||
per_page: $('#per_page').val() || 20,
|
||||
html_name: $('#html_name').val() || '',
|
||||
central_session: $('#central_session').val() || '',
|
||||
is_filter_bag: $('#is_filter_bag').val() || '',
|
||||
is_run_mta: $('#is_run_mta').val() || '' ,
|
||||
mta_task_status: $('#mta_task_status').val() || ''
|
||||
};
|
||||
}
|
||||
|
||||
// 工具：格式化时间
|
||||
function fmtDate(s) {
|
||||
if (!s) return '-';
|
||||
var d = new Date(s);
|
||||
return d.toISOString().slice(0, 19).replace('T', ' ');
|
||||
}
|
||||
|
||||
// 工具：布尔值徽章
|
||||
function badgeBool(v) {
|
||||
if (v === true) return '<span class="badge bg-success badge-bool-true">是</span>';
|
||||
if (v === false) return '<span class="badge bg-secondary badge-bool-false">否</span>';
|
||||
return '<span class="badge bg-secondary badge-bool-null">-</span>';
|
||||
}
|
||||
|
||||
// 工具：截断文本
|
||||
function trunc(s, len) {
|
||||
if (!s) return '-';
|
||||
s = String(s);
|
||||
return s.length <= len ? s : s.slice(0, len) + '…';
|
||||
}
|
||||
|
||||
// 工具：格式化 JSON（简单处理，可展开查看）
|
||||
function fmtJson(s) {
|
||||
if (!s) return '-';
|
||||
try {
|
||||
var j = JSON.parse(s);
|
||||
return JSON.stringify(j, null, 2);
|
||||
} catch (e) {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
|
||||
// 渲染表格数据
|
||||
function renderRows(items) {
|
||||
var tbody = $('#tbody');
|
||||
if (items.length === 0) {
|
||||
tbody.html('<tr><td colspan="12" class="text-center text-muted">暂无数据</td></tr>');
|
||||
return;
|
||||
}
|
||||
tbody.html($.map(items, function(it) {
|
||||
var qs = it.query_sessions ? it.query_sessions.split(',').map(decodeURIComponent).join(', ') : '-';
|
||||
return '<tr>' +
|
||||
'<td>' + it.id + '</td>' +
|
||||
'<td class="text-truncate-custom" title="' + $('<div>').text(it.html_name).html() + '" > <a href="/detail/' + $('<div>').text(it.html_name).html() + '/'+ $('<div>').text(it.bag_folder).html() + '/'+ $('<div>').text(it.html_folder).html()+ '/'+ $('<div>').text(it.id).html()+'">'+ $('<div>').text(it.html_name).html() +'</a></td>' +
|
||||
'<td>' + $('<div>').text(it.central_session).html() + '</td>' +
|
||||
'<td title="' + $('<div>').text(qs).html() + '">' + qs + '</td>' +
|
||||
'<td>' + $('<div>').text(it.html_folder).html() + '</td>' +
|
||||
'<td>' + badgeBool(it.is_filter_bag) + '</td>' +
|
||||
// '<td>' + badgeBool(it.is_run_mta) + '</td>' +
|
||||
'<td>' + $('<div>').text(it.mta_task_status).html() + '</td>' +
|
||||
'<td class="text-truncate-custom" onclick="imageshow('+ it.id +')" > <a href="javascript:void(0)" > '+ $('<div>').text(it.run_mta_result).html() +'</a></td>' +
|
||||
|
||||
// '</td>' +'<td class="text-truncate-custom" title="' + $('<div>').text(it.run_mta_result).html() + '">' + trunc(it.run_mta_result, 48) + '</td>' +
|
||||
'<td class="text-truncate-custom" title="' + $('<div>').text(it.remark).html() + '">' + trunc(it.remark, 48) + '</td>' +
|
||||
'<td>' + fmtDate(it.created_at) + '</td>' +
|
||||
'<td>' + fmtDate(it.updated_at) + '</td>' +
|
||||
'<td> <button class="btn btn-sm btn-outline-warning" onclick="editRow('+ $('<div>').text(it.id).html() + ')">编辑</button> </td>' +
|
||||
'<td> <button class="btn btn-sm btn-outline-warning" onclick="stopTask('+ $('<div>').text(it.id).html() + ')">停止任务</button> </td>' +
|
||||
'<td> <button class="btn btn-sm btn-outline-warning" onclick="reloadTask('+ $('<div>').text(it.id).html() + ')">重跑任务</button> </td>' +
|
||||
'</tr>';
|
||||
}).join(''));
|
||||
}
|
||||
|
||||
// 渲染分页
|
||||
function renderPagination(meta) {
|
||||
var pagination = $('#pagination');
|
||||
var info = $('#info');
|
||||
var { page, per_page, total, total_pages, has_prev, has_next } = meta;
|
||||
|
||||
// 信息栏
|
||||
var start = (page - 1) * per_page + 1;
|
||||
var end = Math.min(page * per_page, total);
|
||||
info.text('共 ' + total + ' 条记录，第 ' + page + '/' + total_pages + ' 页，每页 ' + per_page + ' 条（显示 ' + start + '-' + end + '）');
|
||||
|
||||
// 分页控件
|
||||
var html = '';
|
||||
var q = currentQuery();
|
||||
|
||||
// 首页
|
||||
q.page = 1;
|
||||
html += '<li class="page-item ' + (page === 1 ? 'disabled' : '') + '">' +
|
||||
'<a class="page-link" href="javascript:;" data-page="1">' + (page === 1 ? '' : '首页') + '</a>' +
|
||||
'</li>';
|
||||
|
||||
// 上一页
|
||||
q.page = page - 1;
|
||||
html += '<li class="page-item ' + (page === 1 ? 'disabled' : '') + '">' +
|
||||
'<a class="page-link" href="javascript:;" data-page="' + (page - 1) + '">' + (page === 1 ? '' : '上一页') + '</a>' +
|
||||
'</li>';
|
||||
|
||||
// 页码逻辑（当前页前后各2页，不足则扩展边界）
|
||||
var startPage = Math.max(1, page - 2);
|
||||
var endPage = Math.min(total_pages, page + 2);
|
||||
if (endPage - startPage < 4) {
|
||||
if (startPage === 1) {
|
||||
endPage = Math.min(total_pages, startPage + 4);
|
||||
} else if (endPage === total_pages) {
|
||||
startPage = Math.max(1, endPage - 4);
|
||||
}
|
||||
}
|
||||
if (startPage > 1) {
|
||||
q.page = 1;
|
||||
html += '<li class="page-item"><a class="page-link" href="javascript:;" data-page="1">1</a></li>';
|
||||
if (startPage > 2) html += '<li class="page-item disabled"><span class="page-link">…</span></li>';
|
||||
}
|
||||
for (var i = startPage; i <= endPage; i++) {
|
||||
q.page = i;
|
||||
html += '<li class="page-item ' + (i === page ? 'active' : '') + '">' +
|
||||
'<a class="page-link" href="javascript:;" data-page="' + i + '">' + i + '</a>' +
|
||||
'</li>';
|
||||
}
|
||||
if (endPage < total_pages) {
|
||||
if (endPage < total_pages - 1) html += '<li class="page-item disabled"><span class="page-link">…</span></li>';
|
||||
q.page = total_pages;
|
||||
html += '<li class="page-item"><a class="page-link" href="javascript:;" data-page="' + total_pages + '">' + total_pages + '</a></li>';
|
||||
}
|
||||
|
||||
// 下一页
|
||||
q.page = page + 1;
|
||||
html += '<li class="page-item ' + (page === total_pages ? 'disabled' : '') + '">' +
|
||||
'<a class="page-link" href="javascript:;" data-page="' + (page + 1) + '">' + (page === total_pages ? '' : '下一页') + '</a>' +
|
||||
'</li>';
|
||||
|
||||
// 尾页
|
||||
q.page = total_pages;
|
||||
html += '<li class="page-item ' + (page === total_pages ? 'disabled' : '') + '">' +
|
||||
'<a class="page-link" href="javascript:;" data-page="' + total_pages + '">' + (page === total_pages ? '' : '尾页') + '</a>' +
|
||||
'</li>';
|
||||
|
||||
pagination.html(html);
|
||||
|
||||
// 绑定页码点击（使用 jQuery 事件委托更佳，这里简化为直接绑定）
|
||||
pagination.find('.page-link[data-page]').off('click').on('click', function() {
|
||||
var p = $(this).data('page');
|
||||
$('#page').val(p);
|
||||
loadData();
|
||||
});
|
||||
}
|
||||
|
||||
// 渲染信息与分页
|
||||
function renderInfoAndPagination(meta) {
|
||||
renderInfo(meta);
|
||||
renderPagination(meta);
|
||||
}
|
||||
|
||||
// 渲染信息
|
||||
function renderInfo(meta) {
|
||||
var { page, per_page, total, total_pages } = meta;
|
||||
var start = (page - 1) * per_page + 1;
|
||||
var end = Math.min(page * per_page, total);
|
||||
$('#info').text('共 ' + total + ' 条记录，第 ' + page + '/' + total_pages + ' 页，每页 ' + per_page + ' 条（显示 ' + start + '-' + end + '）');
|
||||
}
|
||||
|
||||
// 加载数据
|
||||
function loadData() {
|
||||
var q = currentQuery();
|
||||
|
||||
var page = parseInt(q.page) || 1;
|
||||
console.log('-----1------'+parseInt(q.page))
|
||||
var per_page = parseInt(q.per_page) || 20;
|
||||
|
||||
$('#tbody').html('<tr><td colspan="12" class="text-center p-4"><div class="spinner-border text-primary" role="status"><span class="visually-hidden">加载中...</span></div></td></tr>');
|
||||
$('#pagination').html('');
|
||||
|
||||
// 构建查询参数
|
||||
var params = {
|
||||
page: page,
|
||||
per_page: per_page,
|
||||
html_name: q.html_name,
|
||||
central_session: q.central_session,
|
||||
is_filter_bag: q.is_filter_bag,
|
||||
is_run_mta: q.is_run_mta,
|
||||
mta_task_status:q.mta_task_status
|
||||
};
|
||||
console.log(params)
|
||||
$.get('/api/items', params, function(data) {
|
||||
renderRows(data.items);
|
||||
renderInfoAndPagination(data);
|
||||
console.log(data.items)
|
||||
}).fail(function(err) {
|
||||
console.error(err);
|
||||
$('#tbody').html('<tr><td colspan="12" class="text-center text-danger">加载失败</td></tr>');
|
||||
});
|
||||
}
|
||||
|
||||
// 编辑数据
|
||||
|
||||
|
||||
|
||||
|
||||
// 切换 MTA 结果展开/收起
|
||||
window.toggleJson = function(btn, jsonText) {
|
||||
var td = $(btn).closest('td');
|
||||
var viewer = td.find('.json-viewer');
|
||||
if (viewer.length === 0) {
|
||||
viewer = $('<div class="json-viewer mt-1"></div>');
|
||||
td.append(viewer);
|
||||
}
|
||||
if (btn.text() === '收起') {
|
||||
viewer.remove();
|
||||
btn.text('展开');
|
||||
} else {
|
||||
viewer.text(jsonText);
|
||||
btn.text('收起');
|
||||
}
|
||||
};
|
||||
|
||||
// 事件绑定（筛选条件变化时重新加载）
|
||||
$('#html_name, #central_session, #is_filter_bag, #is_run_mta, #per_page').on('input change', loadData);
|
||||
|
||||
|
||||
|
||||
// 首次加载
|
||||
$(function() {
|
||||
loadData();
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
445
fst_data_pipeline/apps/mta_manage_system/utils/docker_tool.py
Normal file
445
fst_data_pipeline/apps/mta_manage_system/utils/docker_tool.py
Normal file
@@ -0,0 +1,445 @@
|
||||
import subprocess
|
||||
import os
|
||||
import json
|
||||
import socket
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
|
||||
class DockerTool:
|
||||
"""Docker容器管理工具类"""
|
||||
|
||||
def __init__(self):
|
||||
self.docker_available = self._check_docker_env()
|
||||
|
||||
def _check_docker_env(self) -> bool:
|
||||
"""检查宿主机Docker环境"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "info"], capture_output=True, text=True, timeout=5
|
||||
)
|
||||
return result.returncode == 0
|
||||
except Exception as e:
|
||||
print(f"Docker环境检查失败: {e}")
|
||||
return False
|
||||
|
||||
def is_docker_available(self) -> bool:
|
||||
"""返回Docker环境是否可用"""
|
||||
return self.docker_available
|
||||
|
||||
def get_container_detailed_info(self, container_id: str) -> Dict:
|
||||
"""
|
||||
获取容器的详细信息（使用docker inspect命令）
|
||||
|
||||
Args:
|
||||
container_id: 容器ID（完整ID或短ID）
|
||||
|
||||
Returns:
|
||||
包含容器详细信息的字典
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "inspect", container_id],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
try:
|
||||
inspect_data = json.loads(result.stdout.strip())
|
||||
if inspect_data:
|
||||
return {
|
||||
"success": True,
|
||||
"inspect_data": inspect_data[0],
|
||||
"message": "获取容器详细信息成功",
|
||||
}
|
||||
except json.JSONDecodeError as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"解析容器信息失败: {e}",
|
||||
"message": "解析容器信息失败",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": f"获取容器 '{container_id}' 详细信息失败",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "获取容器详细信息过程中发生异常",
|
||||
}
|
||||
|
||||
def get_container_status_by_name(self, container_name: str) -> Dict:
|
||||
"""
|
||||
通过容器名称查询容器状态
|
||||
|
||||
Args:
|
||||
container_name: 容器名称
|
||||
|
||||
Returns:
|
||||
包含容器状态信息的字典
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"ps",
|
||||
"-a",
|
||||
"--filter",
|
||||
f"name={container_name}",
|
||||
"--format",
|
||||
"{{json .}}",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"未找到名为 '{container_name}' 的容器",
|
||||
"message": "容器不存在",
|
||||
}
|
||||
|
||||
# 处理多行输出（可能有多个匹配的容器）
|
||||
lines = [line for line in output.split("\n") if line.strip()]
|
||||
containers = [json.loads(line) for line in lines]
|
||||
|
||||
# 精确匹配容器名称（排除名称中包含查询字符串的容器）
|
||||
exact_match = None
|
||||
for container in containers:
|
||||
# Docker返回的名称格式可能是 "/容器名" 或 "容器名"
|
||||
names = container.get("Names", "")
|
||||
if container_name in names:
|
||||
# 检查是否为精确匹配
|
||||
name_list = names.split(",")
|
||||
for name in name_list:
|
||||
clean_name = name.strip("/") # 去除开头的斜杠
|
||||
if clean_name == container_name:
|
||||
exact_match = container
|
||||
break
|
||||
|
||||
if exact_match:
|
||||
break
|
||||
|
||||
if not exact_match and containers:
|
||||
# 如果没有精确匹配，但有多条结果，返回第一个
|
||||
exact_match = containers[0]
|
||||
|
||||
if exact_match:
|
||||
return {
|
||||
"success": True,
|
||||
"container": exact_match,
|
||||
"status": exact_match.get("Status", ""),
|
||||
"state": exact_match.get("State", ""),
|
||||
"message": "获取容器状态成功",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"未找到精确匹配的容器 '{container_name}'",
|
||||
"message": "容器不存在",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": f"查询容器 '{container_name}' 状态失败",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "查询容器状态过程中发生异常",
|
||||
}
|
||||
|
||||
def get_container_status_by_id(self, container_id: str) -> Dict:
|
||||
"""
|
||||
通过容器ID查询容器状态
|
||||
|
||||
Args:
|
||||
container_id: 容器ID（完整ID或短ID）
|
||||
|
||||
Returns:
|
||||
包含容器状态信息的字典
|
||||
"""
|
||||
try:
|
||||
# 使用docker ps命令查询特定容器的状态
|
||||
result = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"ps",
|
||||
"-a",
|
||||
"--filter",
|
||||
f"id={container_id}",
|
||||
"--format",
|
||||
"{{json .}}",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
output = result.stdout.strip()
|
||||
if not output:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"未找到ID为 '{container_id}' 的容器",
|
||||
"message": "容器不存在",
|
||||
}
|
||||
|
||||
# 处理多行输出（理论上应该只有一行，因为ID是唯一的）
|
||||
lines = [line for line in output.split("\n") if line.strip()]
|
||||
if not lines:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"未找到ID为 '{container_id}' 的容器",
|
||||
"message": "容器不存在",
|
||||
}
|
||||
|
||||
# 解析容器信息
|
||||
container_info = json.loads(lines[0])
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"container": container_info,
|
||||
"container_id": container_info.get("ID", ""),
|
||||
"container_name": container_info.get("Names", ""),
|
||||
"status": container_info.get("Status", ""),
|
||||
"state": container_info.get("State", ""),
|
||||
"image": container_info.get("Image", ""),
|
||||
"command": container_info.get("Command", ""),
|
||||
"created": container_info.get("CreatedAt", ""),
|
||||
"ports": container_info.get("Ports", ""),
|
||||
"message": "获取容器状态成功",
|
||||
}
|
||||
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": f"查询容器 '{container_id}' 状态失败",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "查询容器状态过程中发生异常",
|
||||
}
|
||||
|
||||
def stop_container_by_name(self, container_name: str) -> Dict:
|
||||
"""
|
||||
通过容器名称停止容器
|
||||
|
||||
Args:
|
||||
container_name: 容器名称
|
||||
|
||||
Returns:
|
||||
包含停止结果的字典
|
||||
"""
|
||||
try:
|
||||
# 首先通过容器名称查询容器状态
|
||||
status_result = self.get_container_status_by_name(container_name)
|
||||
|
||||
if not status_result["success"]:
|
||||
return {
|
||||
"success": False,
|
||||
"error": status_result.get("error", "查询容器状态失败"),
|
||||
"message": f"无法找到容器 '{container_name}'",
|
||||
}
|
||||
|
||||
# 获取容器ID（可能是短ID或完整ID）
|
||||
container_info = status_result["container"]
|
||||
container_id = container_info.get("ID", "") # Docker ps 返回的可能是短ID
|
||||
|
||||
if not container_id:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "无法获取容器ID",
|
||||
"message": f"从容器信息中无法提取容器ID",
|
||||
}
|
||||
|
||||
# 使用现有的stop_container方法停止容器
|
||||
stop_result = self.stop_container(container_id)
|
||||
|
||||
if stop_result["success"]:
|
||||
return {
|
||||
"success": True,
|
||||
"container_id": container_id,
|
||||
"container_name": container_name,
|
||||
"message": f"容器 '{container_name}' 已成功停止",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": stop_result.get("error", "停止操作失败"),
|
||||
"message": f"停止容器 '{container_name}' 失败",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "通过容器名停止容器过程中发生异常",
|
||||
}
|
||||
|
||||
def create_container(
|
||||
self,
|
||||
image_name: str = "ubuntu:latest",
|
||||
container_name: Optional[str] = None,
|
||||
command: Optional[str] = None,
|
||||
volumes: Optional[Dict[str, str]] = None,
|
||||
environment: Optional[Dict[str, str]] = None,
|
||||
interactive: bool = False,
|
||||
) -> Dict:
|
||||
"""
|
||||
创建新容器
|
||||
|
||||
Args:
|
||||
image_name: 镜像名称
|
||||
container_name: 容器名称
|
||||
command: 启动命令
|
||||
volumes: 路径映射字典，格式为 {宿主机路径: 容器内路径}
|
||||
environment: 环境变量字典，格式为 {变量名: 变量值}
|
||||
interactive: 是否以交互模式运行
|
||||
|
||||
Returns:
|
||||
包含创建结果的字典
|
||||
"""
|
||||
try:
|
||||
# 构建Docker命令
|
||||
docker_command = ["docker", "run", "-d"]
|
||||
# 添加交互模式选项
|
||||
# if interactive:
|
||||
# docker_command.extend(["-it"])
|
||||
|
||||
# 添加容器名称
|
||||
if container_name:
|
||||
docker_command.extend(["--name", container_name])
|
||||
|
||||
# 添加路径映射
|
||||
if volumes:
|
||||
for host_path, container_path in volumes.items():
|
||||
docker_command.extend(["-v", f"{host_path}:{container_path}"])
|
||||
|
||||
# 添加环境变量
|
||||
if environment:
|
||||
for key, value in environment.items():
|
||||
docker_command.extend(["-e", f"{key}={value}"])
|
||||
|
||||
# 添加镜像名称和命令
|
||||
docker_command.append(image_name)
|
||||
if command:
|
||||
docker_command.extend(["/bin/bash", "-c", command])
|
||||
|
||||
# 执行Docker命令
|
||||
result = subprocess.run(
|
||||
docker_command, capture_output=True, text=True, timeout=30
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
container_id = result.stdout.strip()
|
||||
return {
|
||||
"success": True,
|
||||
"container_id": container_id,
|
||||
"message": "容器创建成功",
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": "容器创建失败",
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "容器创建过程中发生异常",
|
||||
}
|
||||
|
||||
def get_containers(self) -> List[Dict]:
|
||||
"""获取所有容器列表"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "ps", "-a", "--format", "{{json .}}"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
containers = []
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if line:
|
||||
containers.append(json.loads(line))
|
||||
return containers
|
||||
return []
|
||||
|
||||
except Exception as e:
|
||||
print(f"获取容器列表失败: {e}")
|
||||
return []
|
||||
|
||||
def stop_container(self, container_id: str) -> Dict:
|
||||
"""停止指定容器"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "stop", container_id],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
return {"success": True, "message": f"容器 {container_id} 已停止"}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": f"停止容器 {container_id} 失败",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "停止容器过程中发生异常",
|
||||
}
|
||||
|
||||
def remove_container(self, container_id: str) -> Dict:
|
||||
"""删除指定容器"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "rm", container_id],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
if result.returncode == 0:
|
||||
return {"success": True, "message": f"容器 {container_id} 已删除"}
|
||||
else:
|
||||
return {
|
||||
"success": False,
|
||||
"error": result.stderr.strip(),
|
||||
"message": f"删除容器 {container_id} 失败",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"message": "删除容器过程中发生异常",
|
||||
}
|
||||
|
||||
|
||||
# 创建全局工具实例
|
||||
docker_tool = DockerTool()
|
||||
@@ -0,0 +1,30 @@
|
||||
from typing import List, Set, Dict
|
||||
import re
|
||||
import os
|
||||
|
||||
|
||||
def extract_package_names_from_html(html_file: str, logger) -> Set[str]:
|
||||
"""
|
||||
从单个HTML文件中提取所有legendgroup的包名
|
||||
"""
|
||||
try:
|
||||
with open(html_file, "r", encoding="utf-8") as file:
|
||||
content = file.read()
|
||||
except UnicodeDecodeError:
|
||||
try:
|
||||
with open(html_file, "r", encoding="gbk", errors="ignore") as file:
|
||||
content = file.read()
|
||||
except Exception as e:
|
||||
logger.error(f"读取文件 {html_file} 失败: {e}")
|
||||
return set()
|
||||
|
||||
# 使用正则表达式匹配所有legendgroup中的包名
|
||||
pattern = r'"legendgroup":"([^"]+)"'
|
||||
package_names = re.findall(pattern, content)
|
||||
|
||||
unique_packages = set(package_names)
|
||||
logger.info(
|
||||
f"文件 {os.path.basename(html_file)} 中找到 {len(unique_packages)} 个包名"
|
||||
)
|
||||
|
||||
return unique_packages
|
||||
45
fst_data_pipeline/apps/mta_manage_system/utils/log_tool.py
Normal file
45
fst_data_pipeline/apps/mta_manage_system/utils/log_tool.py
Normal file
@@ -0,0 +1,45 @@
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def setup_logging(log_file="app.log", level=logging.INFO):
|
||||
"""配置日志系统
|
||||
|
||||
Args:
|
||||
log_file: 日志文件路径
|
||||
level: 日志级别
|
||||
"""
|
||||
# 创建日志目录（如果不存在）
|
||||
log_path = Path(log_file)
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 创建 logger
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(level)
|
||||
|
||||
# 避免重复配置
|
||||
if logger.handlers:
|
||||
return logger
|
||||
|
||||
# 创建 formatter
|
||||
formatter = logging.Formatter("[%(levelname)s] %(message)s")
|
||||
|
||||
# 文件处理器（追加模式）
|
||||
file_handler = logging.FileHandler(filename=log_file, mode="a", encoding="utf-8")
|
||||
file_handler.setLevel(level)
|
||||
file_handler.setFormatter(formatter)
|
||||
|
||||
# 控制台处理器
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
console_handler.setLevel(level)
|
||||
console_handler.setFormatter(formatter)
|
||||
|
||||
# 添加处理器
|
||||
logger.addHandler(file_handler)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# 防止重复记录
|
||||
logger.propagate = False
|
||||
|
||||
return logger
|
||||
249
fst_data_pipeline/apps/mta_manage_system/utils/path_tool.py
Normal file
249
fst_data_pipeline/apps/mta_manage_system/utils/path_tool.py
Normal file
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
路径拼接工具类
|
||||
提供简单易用的路径操作功能
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Union, List, Optional
|
||||
|
||||
|
||||
class PathTool:
|
||||
"""简单的路径拼接工具类"""
|
||||
|
||||
@staticmethod
|
||||
def join(*paths: Union[str, Path]) -> str:
|
||||
"""
|
||||
拼接多个路径
|
||||
|
||||
Args:
|
||||
*paths: 路径组件，可以是字符串或Path对象
|
||||
|
||||
Returns:
|
||||
str: 拼接后的路径字符串
|
||||
|
||||
Example:
|
||||
>>> PathTool.join("/home", "user", "documents")
|
||||
'/home/user/documents'
|
||||
"""
|
||||
return os.path.join(*paths)
|
||||
|
||||
@staticmethod
|
||||
def join_with_base(base_path: str, *sub_paths: str) -> str:
|
||||
"""
|
||||
基于基础路径拼接子路径
|
||||
|
||||
Args:
|
||||
base_path: 基础路径
|
||||
*sub_paths: 子路径列表
|
||||
|
||||
Returns:
|
||||
str: 完整的路径字符串
|
||||
|
||||
Example:
|
||||
>>> PathTool.join_with_base("/base", "dir1", "dir2", "file.txt")
|
||||
'/base/dir1/dir2/file.txt'
|
||||
"""
|
||||
return PathTool.join(base_path, *sub_paths)
|
||||
|
||||
@staticmethod
|
||||
def normalize(path: str) -> str:
|
||||
"""
|
||||
规范化路径（处理 ../, ./ 等相对路径）
|
||||
|
||||
Args:
|
||||
path: 原始路径
|
||||
|
||||
Returns:
|
||||
str: 规范化后的路径
|
||||
|
||||
Example:
|
||||
>>> PathTool.normalize("/home/user/../user/./documents")
|
||||
'/home/user/documents'
|
||||
"""
|
||||
return os.path.normpath(path)
|
||||
|
||||
@staticmethod
|
||||
def absolute(path: str) -> str:
|
||||
"""
|
||||
获取绝对路径
|
||||
|
||||
Args:
|
||||
path: 相对路径或绝对路径
|
||||
|
||||
Returns:
|
||||
str: 绝对路径
|
||||
|
||||
Example:
|
||||
>>> PathTool.absolute("documents/file.txt")
|
||||
'/current/working/directory/documents/file.txt'
|
||||
"""
|
||||
return os.path.abspath(path)
|
||||
|
||||
@staticmethod
|
||||
def exists(path: str) -> bool:
|
||||
"""
|
||||
检查路径是否存在
|
||||
|
||||
Args:
|
||||
path: 要检查的路径
|
||||
|
||||
Returns:
|
||||
bool: 路径是否存在
|
||||
"""
|
||||
return os.path.exists(path)
|
||||
|
||||
@staticmethod
|
||||
def is_file(path: str) -> bool:
|
||||
"""
|
||||
检查是否为文件
|
||||
|
||||
Args:
|
||||
path: 要检查的路径
|
||||
|
||||
Returns:
|
||||
bool: 是否为文件
|
||||
"""
|
||||
return os.path.isfile(path)
|
||||
|
||||
@staticmethod
|
||||
def is_dir(path: str) -> bool:
|
||||
"""
|
||||
检查是否为目录
|
||||
|
||||
Args:
|
||||
path: 要检查的路径
|
||||
|
||||
Returns:
|
||||
bool: 是否为目录
|
||||
"""
|
||||
return os.path.isdir(path)
|
||||
|
||||
@staticmethod
|
||||
def get_filename(path: str) -> str:
|
||||
"""
|
||||
获取文件名（包含扩展名）
|
||||
|
||||
Args:
|
||||
path: 文件路径
|
||||
|
||||
Returns:
|
||||
str: 文件名
|
||||
|
||||
Example:
|
||||
>>> PathTool.get_filename("/home/user/file.txt")
|
||||
'file.txt'
|
||||
"""
|
||||
return os.path.basename(path)
|
||||
|
||||
@staticmethod
|
||||
def get_parent(path: str) -> str:
|
||||
"""
|
||||
获取父目录路径
|
||||
|
||||
Args:
|
||||
path: 文件或目录路径
|
||||
|
||||
Returns:
|
||||
str: 父目录路径
|
||||
|
||||
Example:
|
||||
>>> PathTool.get_parent("/home/user/file.txt")
|
||||
'/home/user'
|
||||
"""
|
||||
return os.path.dirname(path)
|
||||
|
||||
@staticmethod
|
||||
def get_extension(path: str) -> str:
|
||||
"""
|
||||
获取文件扩展名
|
||||
|
||||
Args:
|
||||
path: 文件路径
|
||||
|
||||
Returns:
|
||||
str: 文件扩展名（包含点号）
|
||||
|
||||
Example:
|
||||
>>> PathTool.get_extension("/home/user/file.txt")
|
||||
'.txt'
|
||||
"""
|
||||
return os.path.splitext(path)[1]
|
||||
|
||||
@staticmethod
|
||||
def change_extension(path: str, new_extension: str) -> str:
|
||||
"""
|
||||
更改文件扩展名
|
||||
|
||||
Args:
|
||||
path: 原文件路径
|
||||
new_extension: 新扩展名（包含点号）
|
||||
|
||||
Returns:
|
||||
str: 更改扩展名后的路径
|
||||
|
||||
Example:
|
||||
>>> PathTool.change_extension("/home/user/file.txt", ".log")
|
||||
'/home/user/file.log'
|
||||
"""
|
||||
name_without_ext = os.path.splitext(path)[0]
|
||||
return name_without_ext + new_extension
|
||||
|
||||
@staticmethod
|
||||
def create_dirs(path: str) -> bool:
|
||||
"""
|
||||
创建目录（包括父目录）
|
||||
|
||||
Args:
|
||||
path: 要创建的目录路径
|
||||
|
||||
Returns:
|
||||
bool: 是否创建成功
|
||||
"""
|
||||
try:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# 创建全局工具实例
|
||||
path_tool = PathTool()
|
||||
|
||||
# 使用示例和测试
|
||||
if __name__ == "__main__":
|
||||
# 测试路径拼接
|
||||
print("=== 路径拼接测试 ===")
|
||||
result1 = PathTool.join("/home", "user/", "documents", "file.txt")
|
||||
print(f"拼接路径: {result1}")
|
||||
|
||||
result2 = PathTool.join_with_base("/base", "dir1", "dir2", "file.txt")
|
||||
print(f"基础路径拼接: {result2}")
|
||||
|
||||
# 测试路径规范化
|
||||
print("\n=== 路径规范化测试 ===")
|
||||
normalized = PathTool.normalize("/home/user/../user/./documents")
|
||||
print(f"规范化路径: {normalized}")
|
||||
|
||||
# 测试文件信息获取
|
||||
print("\n=== 文件信息测试 ===")
|
||||
test_path = "/home/user/document.txt"
|
||||
print(f"文件名: {PathTool.get_filename(test_path)}")
|
||||
print(f"父目录: {PathTool.get_parent(test_path)}")
|
||||
print(f"扩展名: {PathTool.get_extension(test_path)}")
|
||||
|
||||
# 测试扩展名更改
|
||||
new_path = PathTool.change_extension(test_path, ".log")
|
||||
print(f"更改扩展名: {new_path}")
|
||||
|
||||
# 测试路径检查
|
||||
print("\n=== 路径检查测试 ===")
|
||||
print(f"路径是否存在: {PathTool.exists('/tmp')}")
|
||||
print(f"是否为目录: {PathTool.is_dir('/tmp')}")
|
||||
print(f"是否为文件: {PathTool.is_file('/tmp')}")
|
||||
|
||||
# 使用全局实例
|
||||
print("\n=== 全局实例测试 ===")
|
||||
result = path_tool.join("dir1", "dir2", "file.txt")
|
||||
print(f"使用全局实例: {result}")
|
||||
@@ -0,0 +1,398 @@
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import queue
|
||||
from datetime import datetime
|
||||
from threading import Thread
|
||||
from typing import Dict, List, Optional, Callable, Any
|
||||
from enum import Enum
|
||||
import logging
|
||||
|
||||
|
||||
class TaskStatus(Enum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
class SimpleTask:
|
||||
"""简单的任务类"""
|
||||
|
||||
def __init__(self, BATH_ROOT: str, task_func: Callable, *args, **kwargs):
|
||||
self.BATH_ROOT = BATH_ROOT
|
||||
self.task_func = task_func
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
self.status = TaskStatus.PENDING
|
||||
self.progress = 0
|
||||
self.message = ""
|
||||
self.result = None
|
||||
self.error = None
|
||||
self.created_at = datetime.now()
|
||||
self.started_at = None
|
||||
self.completed_at = None
|
||||
self._cancelled = False
|
||||
self._thread = None
|
||||
self.completion_callback = kwargs.pop("completion_callback", None) # 新增回调
|
||||
|
||||
def execute(self):
|
||||
"""执行任务"""
|
||||
try:
|
||||
self.status = TaskStatus.RUNNING
|
||||
self.started_at = datetime.now()
|
||||
|
||||
# 在新线程中执行
|
||||
self._thread = threading.Thread(target=self._run_wrapper, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
except Exception as e:
|
||||
self.status = TaskStatus.FAILED
|
||||
self.error = str(e)
|
||||
self.completed_at = datetime.now()
|
||||
|
||||
def _run_wrapper(self):
|
||||
"""任务执行包装器"""
|
||||
try:
|
||||
# 执行实际的任务函数
|
||||
self.result = self.task_func(self, *self.args, **self.kwargs)
|
||||
|
||||
if self._cancelled:
|
||||
self.status = TaskStatus.CANCELLED
|
||||
else:
|
||||
self.status = TaskStatus.COMPLETED
|
||||
self.progress = 100
|
||||
|
||||
except Exception as e:
|
||||
self.status = TaskStatus.FAILED
|
||||
self.error = str(e)
|
||||
# 确保result包含错误信息，这样回调函数能获取到错误
|
||||
self.result = {"status": "failed", "error": str(e)}
|
||||
finally:
|
||||
self.completed_at = datetime.now()
|
||||
|
||||
# 无论成功还是失败都执行回调
|
||||
if self.completion_callback:
|
||||
try:
|
||||
# 传递任务结果给回调函数
|
||||
callback_result = (
|
||||
self.result if self.result else {"status": self.status.value}
|
||||
)
|
||||
self.completion_callback(callback_result)
|
||||
except Exception as callback_e:
|
||||
print(f"回调函数执行失败: {callback_e}")
|
||||
# 确保logger存在
|
||||
if hasattr(self, "logger"):
|
||||
self.logger.error(f"回调函数执行失败: {callback_e}")
|
||||
else:
|
||||
print(f"回调函数执行失败: {callback_e}")
|
||||
|
||||
def update_progress(self, progress: int, message: str = ""):
|
||||
"""更新任务进度"""
|
||||
if not self._cancelled:
|
||||
self.progress = max(0, min(100, progress))
|
||||
self.message = message
|
||||
|
||||
def cancel(self):
|
||||
"""取消任务"""
|
||||
self._cancelled = True
|
||||
self.status = TaskStatus.CANCELLED
|
||||
self.message = "任务已被取消"
|
||||
|
||||
def is_running(self) -> bool:
|
||||
"""检查任务是否正在运行"""
|
||||
return (
|
||||
self.status == TaskStatus.RUNNING
|
||||
and self._thread
|
||||
and self._thread.is_alive()
|
||||
)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""转换为字典格式"""
|
||||
return {
|
||||
"BATH_ROOT": self.BATH_ROOT,
|
||||
"status": self.status.value,
|
||||
"progress": self.progress,
|
||||
"message": self.message,
|
||||
"result": self.result,
|
||||
"error": self.error,
|
||||
"created_at": self.created_at.isoformat(),
|
||||
"started_at": self.started_at.isoformat() if self.started_at else None,
|
||||
"completed_at": self.completed_at.isoformat()
|
||||
if self.completed_at
|
||||
else None,
|
||||
"is_running": self.is_running(),
|
||||
}
|
||||
|
||||
|
||||
class SimpleTaskQueue:
|
||||
"""简单的任务队列管理器"""
|
||||
|
||||
def __init__(self, max_concurrent_tasks: int = 3):
|
||||
"""
|
||||
初始化任务队列
|
||||
|
||||
Args:
|
||||
max_concurrent_tasks: 最大并发任务数
|
||||
"""
|
||||
self.max_concurrent_tasks = max_concurrent_tasks
|
||||
self.pending_queue = queue.Queue() # 等待队列
|
||||
self.running_tasks: Dict[str, SimpleTask] = {} # 运行中的任务
|
||||
self.completed_tasks: Dict[str, SimpleTask] = {} # 已完成的任务
|
||||
self.cancelled_tasks: Dict[str, SimpleTask] = {} # 已取消的任务
|
||||
self.lock = threading.RLock()
|
||||
self.is_running = True
|
||||
|
||||
# 启动调度器线程
|
||||
self.scheduler_thread = threading.Thread(
|
||||
target=self._scheduler_loop, daemon=True
|
||||
)
|
||||
self.scheduler_thread.start()
|
||||
|
||||
# 配置日志
|
||||
self.logger = self._setup_logger()
|
||||
self.logger.info(f"简单任务队列已启动，最大并发数: {max_concurrent_tasks}")
|
||||
|
||||
def _setup_logger(self) -> logging.Logger:
|
||||
"""设置日志"""
|
||||
logger = logging.getLogger("SimpleTaskQueue")
|
||||
logger.setLevel(logging.INFO)
|
||||
if not logger.handlers:
|
||||
handler = logging.StreamHandler()
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
return logger
|
||||
|
||||
def _scheduler_loop(self):
|
||||
"""调度器循环 - 自动管理任务执行"""
|
||||
while self.is_running:
|
||||
try:
|
||||
with self.lock:
|
||||
current_running = len(self.running_tasks)
|
||||
|
||||
# 检查是否可以启动新任务
|
||||
if (
|
||||
current_running < self.max_concurrent_tasks
|
||||
and not self.pending_queue.empty()
|
||||
):
|
||||
available_slots = self.max_concurrent_tasks - current_running
|
||||
|
||||
for _ in range(
|
||||
min(available_slots, self.pending_queue.qsize())
|
||||
):
|
||||
if not self.pending_queue.empty():
|
||||
task = self.pending_queue.get_nowait()
|
||||
self._start_task(task)
|
||||
self.logger.info(
|
||||
f"从等待队列启动任务: {task.BATH_ROOT}"
|
||||
)
|
||||
|
||||
# 清理已完成的任务
|
||||
self._cleanup_completed_tasks()
|
||||
|
||||
# 短暂休眠
|
||||
time.sleep(1)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"调度器错误: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
def _cleanup_completed_tasks(self):
|
||||
"""清理已完成的任务"""
|
||||
completed_ids = []
|
||||
with self.lock:
|
||||
for BATH_ROOT, task in list(self.running_tasks.items()):
|
||||
if not task.is_running() and task.status in [
|
||||
TaskStatus.COMPLETED,
|
||||
TaskStatus.FAILED,
|
||||
TaskStatus.CANCELLED,
|
||||
]:
|
||||
completed_ids.append(BATH_ROOT)
|
||||
if (
|
||||
task.status == TaskStatus.COMPLETED
|
||||
or task.status == TaskStatus.FAILED
|
||||
):
|
||||
self.completed_tasks[BATH_ROOT] = task
|
||||
elif task.status == TaskStatus.CANCELLED:
|
||||
self.cancelled_tasks[BATH_ROOT] = task
|
||||
self.logger.info(
|
||||
f"任务完成: {BATH_ROOT}, 状态: {task.status.value}"
|
||||
)
|
||||
|
||||
# 从运行列表中移除已完成的任务
|
||||
for BATH_ROOT in completed_ids:
|
||||
if BATH_ROOT in self.running_tasks:
|
||||
self.running_tasks.pop(BATH_ROOT)
|
||||
|
||||
def _start_task(self, task: SimpleTask):
|
||||
"""启动任务"""
|
||||
with self.lock:
|
||||
self.running_tasks[task.BATH_ROOT] = task
|
||||
task.execute()
|
||||
self.logger.info(f"任务开始执行: {task.BATH_ROOT}")
|
||||
|
||||
def submit_task(self, task_func: Callable, *args, **kwargs) -> Dict[str, Any]:
|
||||
"""
|
||||
提交新任务到队列
|
||||
|
||||
Args:
|
||||
task_func: 要执行的任务函数
|
||||
*args: 任务函数参数
|
||||
**kwargs: 任务函数关键字参数
|
||||
|
||||
Returns:
|
||||
任务提交结果
|
||||
"""
|
||||
BATH_ROOT = str(uuid.uuid4())[:8]
|
||||
|
||||
# 创建任务对象
|
||||
task = SimpleTask(BATH_ROOT, task_func, *args, **kwargs)
|
||||
|
||||
with self.lock:
|
||||
current_running = len(self.running_tasks)
|
||||
|
||||
if current_running < self.max_concurrent_tasks:
|
||||
# 直接启动任务
|
||||
self._start_task(task)
|
||||
status = "RUNNING"
|
||||
message = "任务已立即启动"
|
||||
immediate_start = True
|
||||
else:
|
||||
# 添加到等待队列
|
||||
self.pending_queue.put(task)
|
||||
status = "PENDING"
|
||||
message = f"任务已加入等待队列，当前位置: {self.pending_queue.qsize()}"
|
||||
immediate_start = False
|
||||
|
||||
self.logger.info(f"提交任务: {BATH_ROOT}, 立即执行: {immediate_start}")
|
||||
|
||||
return {
|
||||
"BATH_ROOT": BATH_ROOT,
|
||||
"status": status,
|
||||
"immediate_start": immediate_start,
|
||||
"message": message,
|
||||
"running_count": current_running,
|
||||
"pending_count": self.pending_queue.qsize(),
|
||||
"max_concurrent": self.max_concurrent_tasks,
|
||||
"submitted_at": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
def get_task_status(self, BATH_ROOT: str) -> Optional[Dict[str, Any]]:
|
||||
"""获取任务状态"""
|
||||
with self.lock:
|
||||
# 检查运行中的任务
|
||||
if BATH_ROOT in self.running_tasks:
|
||||
return self.running_tasks[BATH_ROOT].to_dict()
|
||||
|
||||
# 检查已完成的任务
|
||||
if BATH_ROOT in self.completed_tasks:
|
||||
return self.completed_tasks[BATH_ROOT].to_dict()
|
||||
|
||||
# 检查等待队列中的任务
|
||||
for task in list(self.pending_queue.queue):
|
||||
if task.BATH_ROOT == BATH_ROOT:
|
||||
return task.to_dict()
|
||||
|
||||
return None
|
||||
|
||||
def cancel_task(self, BATH_ROOT: str) -> Dict[str, Any]:
|
||||
"""取消任务"""
|
||||
with self.lock:
|
||||
# 检查运行中的任务
|
||||
if BATH_ROOT in self.running_tasks:
|
||||
task = self.running_tasks[BATH_ROOT]
|
||||
task.cancel()
|
||||
self.cancelled_tasks[BATH_ROOT] = task
|
||||
if BATH_ROOT in self.running_tasks:
|
||||
self.running_tasks.pop(BATH_ROOT)
|
||||
self.logger.info(f"取消运行中的任务: {BATH_ROOT}")
|
||||
return {"success": True, "message": "任务取消请求已发送"}
|
||||
|
||||
# 检查等待队列中的任务
|
||||
temp_queue = queue.Queue()
|
||||
found = False
|
||||
while not self.pending_queue.empty():
|
||||
task = self.pending_queue.get()
|
||||
if task.task_id == BATH_ROOT:
|
||||
found = True
|
||||
task.cancel()
|
||||
self.cancelled_tasks[BATH_ROOT] = task
|
||||
self.logger.info(f"取消等待队列中的任务: {BATH_ROOT}")
|
||||
else:
|
||||
temp_queue.put(task)
|
||||
|
||||
# 将剩余任务放回原队列
|
||||
while not temp_queue.empty():
|
||||
self.pending_queue.put(temp_queue.get())
|
||||
|
||||
if found:
|
||||
return {"success": True, "message": "任务已从等待队列中取消"}
|
||||
if BATH_ROOT in self.cancelled_tasks:
|
||||
return {"success": True, "message": "任务已被取消"}
|
||||
return {"success": False, "error": "任务不存在"}
|
||||
|
||||
def get_queue_status(self) -> Dict[str, Any]:
|
||||
"""获取队列状态"""
|
||||
with self.lock:
|
||||
# 获取运行中任务信息
|
||||
running_tasks = []
|
||||
for task in self.running_tasks.values():
|
||||
task_info = task.to_dict()
|
||||
running_tasks.append(task_info)
|
||||
|
||||
# 获取等待任务信息
|
||||
pending_tasks = []
|
||||
temp_queue = queue.Queue()
|
||||
while not self.pending_queue.empty():
|
||||
task = self.pending_queue.get()
|
||||
pending_tasks.append(task.to_dict())
|
||||
temp_queue.put(task)
|
||||
|
||||
# 恢复队列
|
||||
while not temp_queue.empty():
|
||||
self.pending_queue.put(temp_queue.get())
|
||||
|
||||
# 分别统计不同状态的任务数量
|
||||
successful_count = len([
|
||||
t
|
||||
for t in self.completed_tasks.values()
|
||||
if t.status == TaskStatus.COMPLETED
|
||||
])
|
||||
failed_count = len([
|
||||
t
|
||||
for t in self.completed_tasks.values()
|
||||
if t.status == TaskStatus.FAILED
|
||||
])
|
||||
cancelled_count = len(self.cancelled_tasks)
|
||||
|
||||
# 获取已取消任务信息
|
||||
cancelled_tasks = []
|
||||
for task in self.cancelled_tasks.values():
|
||||
task_info = task.to_dict()
|
||||
cancelled_tasks.append(task_info)
|
||||
|
||||
return {
|
||||
"max_concurrent_tasks": self.max_concurrent_tasks,
|
||||
"pending_count": self.pending_queue.qsize(),
|
||||
"running_count": len(self.running_tasks),
|
||||
"successful_count": successful_count,
|
||||
"failed_count": failed_count,
|
||||
"cancelled_count": cancelled_count, # 更新统计
|
||||
"cancelled_tasks": cancelled_tasks, # 添加已取消任务列表
|
||||
"running_tasks": running_tasks,
|
||||
"pending_tasks": pending_tasks,
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
def stop(self):
|
||||
"""停止任务队列"""
|
||||
self.is_running = False
|
||||
self.logger.info("任务队列已停止")
|
||||
|
||||
|
||||
# 全局任务队列实例
|
||||
task_queue = SimpleTaskQueue(max_concurrent_tasks=3)
|
||||
1
fst_data_pipeline/apps/root_db_api/.gitignore
vendored
Normal file
1
fst_data_pipeline/apps/root_db_api/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
/.idea/
|
||||
446
fst_data_pipeline/apps/root_db_api/DATABASE_FIELDS.md
Normal file
446
fst_data_pipeline/apps/root_db_api/DATABASE_FIELDS.md
Normal file
@@ -0,0 +1,446 @@
|
||||
# ROOT DB API - 数据库字段文档
|
||||
|
||||
本文档详细说明了 ROOT DB API 数据库中所有可用的数据字段，按功能模块分类。
|
||||
|
||||
## 🗂️ 数据库架构概览
|
||||
|
||||
数据库包含 **18 个核心业务表** + **1 个审计日志表**，支持：
|
||||
- 🚗 **自动驾驶数据管理** (Pangu/Minerva)
|
||||
- 📦 **ROS Bag 文件管理**
|
||||
- 🌳 **FST (File System Tree) 分类**
|
||||
- 🏷️ **标签和主题管理**
|
||||
- 📍 **地理空间数据支持**
|
||||
- 🎯 **真值数据管理**
|
||||
|
||||
---
|
||||
|
||||
## 📊 1. 基础配置模块
|
||||
|
||||
### 1.1 项目表 (`project`)
|
||||
管理数据来源项目信息
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 项目自增主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL UNIQUE | 项目名称，全局唯一 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 最近一次更新时间 |
|
||||
|
||||
**说明**: 数据来源标识，1：DFDI，2：CFDI
|
||||
|
||||
---
|
||||
|
||||
## 📚 2. 字典数据模块
|
||||
|
||||
### 2.1 驾驶模式字典 (`drive_mode`)
|
||||
自动驾驶模式分类字典
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `type` | VARCHAR(100) | NOT NULL | 驾驶模式大类 |
|
||||
| `sub_type` | VARCHAR(100) | | 驾驶模式子类 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
| `comment` | TEXT | | 备注 |
|
||||
|
||||
**驾驶模式类型**:
|
||||
- **AD_DRIVE_OFF**: OFF, PASSIVE, STANDBY
|
||||
- **AD_DRIVE_ACC**: ACC
|
||||
- **AD_DRIVE_CP**: CP
|
||||
- **AD_DRIVE_NP**: HDMAP_HNP, DDLD_HNP, HDMAP_HNP_PLUS, DDLD_HNP_PLUS, HDMAP_UNP, DDLD_UNP
|
||||
- **AD_PARK_APA**: 各种 APA 停车模式 (20+ 种)
|
||||
- **AD_PARK_MPA**: 停车场地图相关模式
|
||||
|
||||
### 2.2 Topic 字典 (`topic_list`)
|
||||
ROS Topic 信息管理
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL UNIQUE | Topic 名称 (如 /lidar/points) |
|
||||
| `type` | VARCHAR(255) | | Topic 类型 (如 sensor_msgs/PointCloud2) |
|
||||
| `key_data` | TEXT | | 关键数据摘要 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 2.3 标签字典 (`reserved_tag_list`)
|
||||
数据标签管理
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL | 标签名称 |
|
||||
| `type` | VARCHAR(50) | | 标签分类 |
|
||||
| `creator` | VARCHAR(255) | | 创建人 |
|
||||
| `comments` | TEXT | | 备注 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `is_deleted` | BOOLEAN | DEFAULT FALSE | 逻辑删除标记 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 2.4 真值元数据 (`gt_meta`)
|
||||
Ground Truth 数据元信息
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL | 真值名称 |
|
||||
| `type` | VARCHAR(100) | NOT NULL | 真值类型 (如 3DBox/Lane) |
|
||||
| `path` | TEXT | NOT NULL | 真值文件存储路径 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
| `comment` | TEXT | | 备注 |
|
||||
|
||||
---
|
||||
|
||||
## 📦 3. Bag 文件管理模块
|
||||
|
||||
### 3.1 Bag 主表 (`bag_list`)
|
||||
ROS Bag 文件核心信息
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL UNIQUE | Bag 文件名称，全局唯一 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `project_id` | INT | FK → project(id) | 数据来源项目 |
|
||||
| `tile_id` | VARCHAR(255) | | 瓦片 ID |
|
||||
| `is_decoded` | BOOLEAN | DEFAULT FALSE | 是否已解码 |
|
||||
| `is_deleted` | BOOLEAN | DEFAULT FALSE | 逻辑删除 |
|
||||
| `od_annotated` | INT | | 目标检测标注状态 |
|
||||
| `ld_annotated` | INT | | 车道线标注状态 |
|
||||
| `fst_indexed` | BOOLEAN | DEFAULT FALSE | 是否已建立 FST 索引 |
|
||||
| `is_active_data` | BOOLEAN | DEFAULT TRUE | 是否为活跃数据 |
|
||||
| `reserved_str` | VARCHAR(255) | | 预留字符串 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
| `drive_mode` | SMALLINT | FK → drive_mode(id) | 驾驶模式字典 ID |
|
||||
| `sw_version` | VARCHAR(100) | | 软件版本号 |
|
||||
| `hw_version` | VARCHAR(100) | | 硬件版本号 |
|
||||
|
||||
### 3.2 Bag 生命周期 (`bag_lifecycle`)
|
||||
Bag 文件处理流程时间戳
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_name` | VARCHAR(255) | NOT NULL UNIQUE, FK → bag_list(name) | Bag 名称 |
|
||||
| `collect_time` | TIMESTAMPTZ | | 采集完成时间 |
|
||||
| `clone2dev_time` | TIMESTAMPTZ | | 拷贝到bucket时间 |
|
||||
| `decode_time` | TIMESTAMPTZ | | 解码完成时间 |
|
||||
| `mining_time` | TIMESTAMPTZ | | 数据挖掘完成时间 |
|
||||
| `auto_annotate_time` | TIMESTAMPTZ | | 自动标注完成时间 |
|
||||
| `manual_annotate_time` | TIMESTAMPTZ | | 人工标注完成时间 |
|
||||
| `fst_index_time` | TIMESTAMPTZ | | FST 索引完成时间 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 记录更新时间 |
|
||||
|
||||
---
|
||||
|
||||
## 🚗 4. Pangu 数据管理模块
|
||||
|
||||
### 4.1 Pangu 主表 (`main_pangu`)
|
||||
Pangu 原始数据信息
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL UNIQUE | 名称，全局唯一 |
|
||||
| `vehicle` | VARCHAR(255) | | 车辆编号 |
|
||||
| `datetime` | TIMESTAMPTZ | | rosbag的生成时间 |
|
||||
| `bag_path` | TEXT | | 原始 Bag 路径 |
|
||||
| `data_path` | TEXT | | 解析后数据路径 |
|
||||
| `reserved_str` | VARCHAR(255) | | 预留字符串 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 4.2 Pangu 融合表 (`joined_pangu`)
|
||||
融合后的 Pangu 数据
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL UNIQUE | 融合后名称 |
|
||||
| `data_path` | TEXT | | 融合后数据路径 |
|
||||
| `reserved_str` | VARCHAR(255) | | 预留字符串 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 4.3 Bag 融合关系 (`joined_bags`)
|
||||
记录 Bag 文件融合关系
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `parent_id` | INT | NOT NULL | 父 Bag ID |
|
||||
| `child_id` | INT | NOT NULL UNIQUE | 子 Bag ID，唯一 |
|
||||
| `reserved_str` | VARCHAR(255) | | 预留字符串 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 4.4 Pangu 详细路径 (`secondary_pangu`)
|
||||
Pangu 数据文件路径详情
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_id` | INT | NOT NULL UNIQUE, FK → bag_list(id) | 关联 bag_list.id |
|
||||
| `lidar_gt_pandar128` | TEXT | | 激光真值路径 |
|
||||
| `object_lidar_gt_pandar128_manual` | TEXT | | 人工真值路径 |
|
||||
| `lidar_fd_multi_scan_raw` | TEXT | | 原始雷达路径 |
|
||||
| `camera_fisheye_left` | TEXT | | 左鱼眼相机路径 |
|
||||
| `camera_fisheye_right` | TEXT | | 右鱼眼相机路径 |
|
||||
| `camera_front_wide` | TEXT | | 前广角相机路径 |
|
||||
| `raw_gps` | TEXT | | 原始 GPS 路径 |
|
||||
| `raw_imu` | TEXT | | 原始 IMU 路径 |
|
||||
| `ego_motion` | TEXT | | 自车运动路径 |
|
||||
| `vehicle_wheel` | TEXT | | 轮速路径 |
|
||||
| `calibration` | TEXT | | 标定文件路径 |
|
||||
| `sdmap` | TEXT | | SD 地图路径或地图版本标识 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
---
|
||||
|
||||
## 🅰️ 5. Minerva 数据管理模块
|
||||
|
||||
### 5.1 Minerva 主表 (`main_minerva`)
|
||||
Minerva 数据会话信息
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `session_id` | VARCHAR(255) | UNIQUE | 会话 ID，唯一 |
|
||||
| `start_ts` | TIMESTAMPTZ | | 开始时间戳 |
|
||||
| `end_ts` | TIMESTAMPTZ | | 结束时间戳 |
|
||||
| `length` | INT | | 时长（秒） |
|
||||
| `datetime` | TIMESTAMPTZ | | 数据日期 |
|
||||
| `vin` | VARCHAR(255) | | 车辆识别码 |
|
||||
| `platform` | VARCHAR(255) | | 平台名称 |
|
||||
| `mapped` | BOOLEAN | DEFAULT FALSE | 是否已映射 |
|
||||
| `path` | TEXT | | 原始数据路径 |
|
||||
| `converted_path` | TEXT | | 转换后路径 |
|
||||
| `gt_path` | TEXT | | 真值路径 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
### 5.2 Minerva 详细路径 (`secondary_minerva`)
|
||||
Minerva 数据文件路径详情
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_id` | INT | NOT NULL UNIQUE, FK → bag_list(id) | 关联 bag_list.id |
|
||||
| `lidar_gt_top_p128` | TEXT | | 顶部Lidar真值路径 |
|
||||
| `lidar_parking_gt_front_p128` | TEXT | | 前停车Lidar真值路径 |
|
||||
| `lidar_parking_gt_left_p128` | TEXT | | 左停车Lidar真值路径 |
|
||||
| `lidar_parking_gt_right_p128` | TEXT | | 右停车Lidar真值路径 |
|
||||
| `lidar_parking_gt_rear_p128` | TEXT | | 后停车Lidar真值路径 |
|
||||
| `camera_fisheye_left_200fov` | TEXT | | 左 200° 鱼眼路径 |
|
||||
| `camera_fisheye_right_200fov` | TEXT | | 右 200° 鱼眼路径 |
|
||||
| `camera_fisheye_rear_200fov` | TEXT | | 后 200° 鱼眼路径 |
|
||||
| `camera_fisheye_front_200fov` | TEXT | | 前 200° 鱼眼路径 |
|
||||
| `camera_front_wide_120fov` | TEXT | | 前 120° 广角路径 |
|
||||
| `camera_front_tele_30fov` | TEXT | | 前 30° 长焦路径 |
|
||||
| `camera_rear_right_70fov` | TEXT | | 右后 70° 相机路径 |
|
||||
| `camera_rear_left_70fov` | TEXT | | 左后 70° 相机路径 |
|
||||
| `raw_gps` | TEXT | | 原始 GPS 路径 |
|
||||
| `raw_imu` | TEXT | | 原始 IMU 路径 |
|
||||
| `ego_motion` | TEXT | | 自车运动路径 |
|
||||
| `calibration` | TEXT | | 标定文件路径 |
|
||||
| `rig` | TEXT | | rig 文件路径 |
|
||||
| `fst_new` | TEXT | | 新 FST 路径 |
|
||||
| `fst_old` | TEXT | | 旧 FST 路径 |
|
||||
| `comments` | TEXT | | 备注 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
|
||||
---
|
||||
|
||||
## 🌳 6. FST (File System Tree) 模块
|
||||
|
||||
### 6.1 FST 节点表 (`fst`)
|
||||
文件系统树分类结构
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `name` | VARCHAR(255) | NOT NULL | 节点名称 |
|
||||
| `parent_id` | INT | FK → fst(id) | 父节点 ID |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `reserved_json` | JSONB | | 预留扩展 JSON |
|
||||
| `bag_sum` | INT | DEFAULT 0 | 子树 Bag 数量 |
|
||||
|
||||
### 6.2 FST-Bag 关联 (`fst_bag`)
|
||||
FST 节点与 Bag 文件的关联关系
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_id` | INT | NOT NULL, FK → bag_list(id) | Bag ID |
|
||||
| `fst_node_id` | INT | NOT NULL, FK → fst(id) | FST 节点 ID |
|
||||
| `fst_node_level` | INT | NOT NULL | 节点层级 |
|
||||
| `event_start_time` | INT | | 事件开始时间（ms） |
|
||||
| `event_end_time` | INT | | 事件结束时间（ms） |
|
||||
| `img_url` | VARCHAR(255) | | 事件截图 URL |
|
||||
| `video_url` | VARCHAR(255) | | 事件视频 URL |
|
||||
| `comments` | VARCHAR(255) | | 备注 |
|
||||
|
||||
---
|
||||
|
||||
## 🗺️ 7. 地理空间数据模块
|
||||
|
||||
### 7.1 几何信息表 (`geometry_info`)
|
||||
Bag 文件的地理空间轨迹数据
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `rosbag_name` | VARCHAR(255) | NOT NULL | Rosbag 名称 |
|
||||
| `gnss_downsampled_points` | geometry[] | | 降采样 GNSS 轨迹点 |
|
||||
| `update_time` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 更新时间 |
|
||||
| `is_overlapped` | BOOLEAN | DEFAULT FALSE | 是否与其他 Bag 重叠 |
|
||||
|
||||
---
|
||||
|
||||
## 🔗 8. 关联关系模块
|
||||
|
||||
### 8.1 Bag-Topic 关联 (`bag_topic`)
|
||||
Bag 文件与 ROS Topic 的多对多关系
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_id` | INT | NOT NULL, FK → bag_list(id) | Bag ID |
|
||||
| `topic_id` | INT | NOT NULL, FK → topic_list(id) | Topic ID |
|
||||
|
||||
### 8.2 Bag-标签 关联 (`bag_reserved_tag`)
|
||||
Bag 文件与标签的多对多关系
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `bag_id` | INT | NOT NULL, FK → bag_list(id) | Bag ID |
|
||||
| `tag_id` | INT | NOT NULL, FK → reserved_tag_list(id) | 标签 ID |
|
||||
|
||||
### 8.3 Bag-真值 关联 (`bag_gt`)
|
||||
Bag 文件与真值数据的多对多关系
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | SERIAL | PRIMARY KEY | 主键 |
|
||||
| `gt_id` | INT | NOT NULL, FK → gt_meta(id) | 真值元数据 ID |
|
||||
| `bag_id` | INT | NOT NULL, FK → bag_list(id) | Bag ID |
|
||||
| `comment` | TEXT | | 备注 |
|
||||
|
||||
---
|
||||
|
||||
## 📝 9. 审计日志模块
|
||||
|
||||
### 9.1 操作历史 (`ops_history`)
|
||||
通用审计日志，记录所有数据变更
|
||||
|
||||
| 字段名 | 类型 | 约束 | 说明 |
|
||||
|--------|------|------|------|
|
||||
| `id` | BIGSERIAL | PRIMARY KEY | 审计日志自增主键 |
|
||||
| `table_name` | REGCLASS | NOT NULL | 被审计表 |
|
||||
| `row_pk` | TEXT | NOT NULL | 行主键值 |
|
||||
| `column_name` | TEXT | NOT NULL | 被修改列名 |
|
||||
| `old_value` | TEXT | | 旧值 |
|
||||
| `new_value` | TEXT | | 新值 |
|
||||
| `changed_by` | NAME | NOT NULL DEFAULT CURRENT_USER | 数据库用户 |
|
||||
| `changed_at` | TIMESTAMPTZ | NOT NULL DEFAULT NOW() | 变更时间 |
|
||||
| `comment` | TEXT | | 可读描述 |
|
||||
| `app_module` | TEXT | | 应用模块 |
|
||||
| `trace_id` | TEXT | | 链路追踪 ID |
|
||||
|
||||
---
|
||||
|
||||
## 📊 10. 数据统计信息
|
||||
|
||||
### 📈 表数据规模
|
||||
根据字段设计和索引配置，预估的数据规模：
|
||||
|
||||
| 表名 | 预估记录数 | 主要用途 | 关键字段 |
|
||||
|------|------------|----------|----------|
|
||||
| `bag_list` | 10K+ | Bag文件管理 | name, project_id, is_decoded |
|
||||
| `main_pangu` | 10K+ | Pangu数据 | name, vehicle, datetime |
|
||||
| `main_minerva` | 5K+ | Minerva会话 | session_id, vin, platform |
|
||||
| `fst` | 1K+ | FST分类树 | name, parent_id, bag_sum |
|
||||
| `topic_list` | 500+ | ROS Topic字典 | name, type |
|
||||
| `reserved_tag_list` | 1K+ | 标签字典 | name, type, creator |
|
||||
| `bag_topic` | 100K+ | Bag-Topic关联 | bag_id, topic_id |
|
||||
| `fst_bag` | 50K+ | FST-Bag关联 | bag_id, fst_node_id |
|
||||
| `geometry_info` | 10K+ | 地理轨迹 | rosbag_name, gnss_points |
|
||||
|
||||
### 🔍 关键索引
|
||||
|
||||
**性能优化索引**:
|
||||
- `idx_bag_list_project_id` - 按项目查询
|
||||
- `idx_main_pangu_datetime` - 按时间范围查询
|
||||
- `idx_main_minerva_session_id` - 会话ID查询
|
||||
- `idx_fst_name` - FST节点名称查询
|
||||
- `idx_bag_list_json_gin` - JSON字段GIN索引
|
||||
|
||||
**复合索引**:
|
||||
- `idx_bag_list_project_active` - 活跃项目数据
|
||||
- `uq_taglist_name_not_deleted` - 未删除标签唯一性
|
||||
- `idx_lifecycle_*_time` - 生命周期时间BRIN索引
|
||||
|
||||
---
|
||||
|
||||
## 🚀 API 数据访问
|
||||
|
||||
### 常用查询模式
|
||||
|
||||
#### 1. 获取项目下所有Bag
|
||||
```sql
|
||||
SELECT bl.*, p.name as project_name
|
||||
FROM bag_list bl
|
||||
JOIN project p ON bl.project_id = p.id
|
||||
WHERE p.name = 'project_name' AND bl.is_deleted = FALSE;
|
||||
```
|
||||
|
||||
#### 2. 获取Bag的完整信息
|
||||
```sql
|
||||
SELECT
|
||||
bl.*,
|
||||
sp.camera_front_wide,
|
||||
sp.lidar_gt_pandar128,
|
||||
mp.vehicle,
|
||||
mp.datetime
|
||||
FROM bag_list bl
|
||||
LEFT JOIN secondary_pangu sp ON bl.id = sp.bag_id
|
||||
LEFT JOIN main_pangu mp ON bl.name = mp.name
|
||||
WHERE bl.name = 'bag_name';
|
||||
```
|
||||
|
||||
#### 3. 获取FST树结构
|
||||
```sql
|
||||
WITH RECURSIVE fst_tree AS (
|
||||
SELECT id, name, parent_id, 0 as level
|
||||
FROM fst WHERE parent_id IS NULL
|
||||
UNION ALL
|
||||
SELECT f.id, f.name, f.parent_id, ft.level + 1
|
||||
FROM fst f
|
||||
JOIN fst_tree ft ON f.parent_id = ft.id
|
||||
)
|
||||
SELECT * FROM fst_tree ORDER BY level, name;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📚 扩展信息
|
||||
|
||||
### JSON 扩展字段
|
||||
多个表包含 `reserved_json` 字段，支持：
|
||||
- **动态扩展**: 无需修改表结构添加新字段
|
||||
- **复杂数据**: 存储嵌套或数组类型数据
|
||||
- **高效查询**: 支持 GIN 索引的JSON查询
|
||||
- **版本兼容**: 向后兼容的数据扩展
|
||||
|
||||
### PostGIS 地理支持
|
||||
- **空间扩展**: PostGIS + UUID-OSSP 扩展
|
||||
- **几何类型**: 支持 geometry[] 数组类型
|
||||
- **轨迹数据**: GNSS 降采样轨迹点存储
|
||||
- **空间查询**: 支持地理范围和重叠检测
|
||||
|
||||
### 数据一致性
|
||||
- **外键约束**: 保证数据完整性
|
||||
- **唯一约束**: 防止重复数据
|
||||
- **级联删除**: 自动清理关联数据
|
||||
- **软删除**: 逻辑删除保留数据历史
|
||||
|
||||
---
|
||||
|
||||
**文档版本**: v1.0 | **更新时间**: 2024-01-14 | **数据库版本**: PostgreSQL 13+ with PostGIS
|
||||
393
fst_data_pipeline/apps/root_db_api/FST_fst_stash_changes.md
Normal file
393
fst_data_pipeline/apps/root_db_api/FST_fst_stash_changes.md
Normal file
@@ -0,0 +1,393 @@
|
||||
# fst_stash 设计与接口说明
|
||||
|
||||
本文档记录与 `fst_stash` 相关的表结构、核心服务函数以及 API 接口，方便前后端统一理解“导入/草稿版 FST 树”的读写方式。
|
||||
|
||||
---
|
||||
|
||||
## 一、表设计：fst_stash
|
||||
|
||||
**表名**：`fst_stash`
|
||||
|
||||
- `id`：`bigint`，主键，自增
|
||||
- `fst_versions`：`varchar(255)`，非空
|
||||
- 外键，关联 `version_list.version`
|
||||
- 表示该条 stash 对应的版本号（例如 `MB Driving FST V1`）
|
||||
- `content`：`jsonb`，可空
|
||||
- 存放完整的 FST JSON 内容（通常包含 `meta` 和 `nodes`）
|
||||
- `created_time`：`timestamptz`，非空，默认当前时间
|
||||
- `updated_time`：`timestamptz`，非空，默认当前时间，更新时自动刷新
|
||||
|
||||
约束与关系：
|
||||
|
||||
- `fst_versions` 外键指向 `version_list.version`，`ON DELETE CASCADE`
|
||||
删除某个版本时，对应的 stash 会自动被删除。
|
||||
|
||||
内容约定（非数据库约束）：
|
||||
|
||||
- 推荐结构：
|
||||
|
||||
```json
|
||||
{
|
||||
"meta": {
|
||||
"desc": "MB Driving FST V1",
|
||||
"creator": "xxx",
|
||||
"created_at": "2026-02-25T10:00:00Z"
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "ROOT",
|
||||
"parentId": null,
|
||||
"treeLevel": 1,
|
||||
"node_version": 1
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"name": "CHILD",
|
||||
"parentId": 1,
|
||||
"treeLevel": 2,
|
||||
"node_version": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
- 接口侧只要求 `nodes` 为数组，并且每个节点至少包含：
|
||||
- `id`：用于 parentId 关联
|
||||
- `name`：节点名称
|
||||
- `parentId`：父节点 id（根节点为 `null`）
|
||||
- 节点级乐观锁字段：
|
||||
- `node_version`：节点版本号，默认值 `1`
|
||||
- 写入时若缺失会自动补齐为 `1`
|
||||
- `/fst/stash/update` 默认开启乐观锁校验，不一致返回 `409`
|
||||
|
||||
---
|
||||
|
||||
## 二、核心服务函数（service.py）
|
||||
|
||||
文件位置：`src/core/service.py`
|
||||
|
||||
### 1. `upsert_fst_stash`
|
||||
|
||||
```python
|
||||
def upsert_fst_stash(
|
||||
db: Session,
|
||||
*,
|
||||
version: str,
|
||||
content: Optional[Dict[str, Any]],
|
||||
use_node_optimistic_lock: bool = False,
|
||||
) -> "FstStash":
|
||||
"""
|
||||
Insert or update FST stash record for a version.
|
||||
If a record exists for the given version, update its content.
|
||||
If not, create a new record.
|
||||
"""
|
||||
```
|
||||
|
||||
行为说明：
|
||||
|
||||
- 先检查 `version_list` 中是否存在 `version`：
|
||||
- 若不存在，抛出 `ValueError("Version '<version>' not found in version_list")`
|
||||
- 再按 `fst_versions = version` 查询 `fst_stash`：
|
||||
- 若已存在：更新其 `content`（仅当传入 `content` 非 `None` 时）
|
||||
- 若不存在：插入新行，写入 `fst_versions` 和 `content`
|
||||
- 节点版本规则：
|
||||
- 遍历 `nodes` 及其 `children`，自动补齐 `node_version`（默认 `1`）
|
||||
- 更新时若节点内容变更，该节点 `node_version` 自动 `+1`
|
||||
- 当 `use_node_optimistic_lock=True` 时，校验传入 `node_version` 与库内一致，否则抛出冲突异常
|
||||
- `db.flush()` 后返回对应的 `FstStash` 对象
|
||||
|
||||
### 2. `get_fst_stash`
|
||||
|
||||
```python
|
||||
def get_fst_stash(db: Session, version: str) -> Optional["FstStash"]:
|
||||
"""Get FST stash record by version."""
|
||||
```
|
||||
|
||||
行为说明：
|
||||
|
||||
- 直接根据 `fst_versions = version` 查询并返回 `FstStash` 实体
|
||||
- 找不到则返回 `None`
|
||||
|
||||
---
|
||||
|
||||
## 三、API 接口列表（fst.py）
|
||||
|
||||
文件位置：`src/api/fst.py`
|
||||
|
||||
### 1. POST `/fst/stash` —— 保存 stash 内容
|
||||
|
||||
对应函数：`save_fst_stash`
|
||||
|
||||
用途：
|
||||
|
||||
- 通用的保存接口，根据 `version` 写入 `fst_stash.content`。
|
||||
- 适合“导入 JSON 后一次性保存”的场景。
|
||||
|
||||
请求：
|
||||
|
||||
```json
|
||||
POST /fst/stash
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"version": "MB Driving FST V1",
|
||||
"content": {
|
||||
"meta": { "...": "..." },
|
||||
"nodes": [ { "id": 1, "name": "ROOT", "parentId": null }, ... ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
返回（200）：
|
||||
|
||||
```json
|
||||
{
|
||||
"id": 1,
|
||||
"version": "MB Driving FST V1",
|
||||
"updated_time": "2026-02-25T10:00:00+00:00"
|
||||
}
|
||||
```
|
||||
|
||||
错误：
|
||||
|
||||
- `400`：缺少 `version` 或版本在 `version_list` 中不存在
|
||||
- `500`：其他内部错误
|
||||
|
||||
内部实现：
|
||||
|
||||
- 调用 `upsert_fst_stash(db, version=version, content=content)`
|
||||
|
||||
### 2. GET `/fst/stash/<version>` —— 根据版本获取原始 stash 内容
|
||||
|
||||
对应函数：`get_fst_stash_by_version`
|
||||
|
||||
用途：
|
||||
|
||||
- 按版本直接返回 `fst_stash` 的原始存储内容；
|
||||
- 适合需要拿到完整 JSON（含 `meta`、`nodes` 等原始结构）的场景。
|
||||
|
||||
请求示例：
|
||||
|
||||
```http
|
||||
GET /fst/stash/MB%20Driving%20FST%20V1
|
||||
```
|
||||
|
||||
返回（200）：
|
||||
|
||||
```json
|
||||
{
|
||||
"id": 1,
|
||||
"version": "MB Driving FST V1",
|
||||
"content": {
|
||||
"meta": { "...": "..." },
|
||||
"nodes": [ ... ]
|
||||
},
|
||||
"updated_time": "2026-02-25T10:00:00+00:00"
|
||||
}
|
||||
```
|
||||
|
||||
错误：
|
||||
|
||||
- `404`：指定版本在 `fst_stash` 中不存在
|
||||
- `500`：其他内部错误
|
||||
|
||||
### 3. GET `/fst/stash/print_tree` —— 基于 stash 构建 FST 树
|
||||
|
||||
对应函数：`print_tree_from_stash`
|
||||
|
||||
用途：
|
||||
|
||||
- 根据 stash 中的 `nodes` 数组，构建与 `/fst/print_tree` 相同结构的树；
|
||||
- 方便前端直接复用现有 Tree 渲染逻辑，但数据来源于 `fst_stash`。
|
||||
|
||||
请求示例：
|
||||
|
||||
```http
|
||||
GET /fst/stash/print_tree?version=MB%20Driving%20FST%20V1
|
||||
```
|
||||
|
||||
处理逻辑简要：
|
||||
|
||||
1. 使用 `get_fst_stash(db, version)` 读取 stash
|
||||
- 若不存在或 `content` 为空 → `404`
|
||||
2. 读取 `content["nodes"]`，要求为数组，否则 `400`
|
||||
3. 遍历 `nodes`：
|
||||
- 按 `id` 建立节点映射
|
||||
- 将每个节点转换为：
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "<name>",
|
||||
"label": "<name>",
|
||||
"level": 0,
|
||||
"children": []
|
||||
}
|
||||
```
|
||||
|
||||
4. 根据 `parentId` 将子节点挂到父节点的 `children` 中
|
||||
5. 从所有节点中找出根节点（`parentId` 为空或找不到父节点）作为返回数组
|
||||
6. 递归设置 `level`：根节点为 0，子节点在父节点基础上 +1
|
||||
|
||||
返回（200）：
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id": "ROOT",
|
||||
"label": "ROOT",
|
||||
"level": 0,
|
||||
"children": [
|
||||
{
|
||||
"id": "CHILD",
|
||||
"label": "CHILD",
|
||||
"level": 1,
|
||||
"children": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
错误：
|
||||
|
||||
- `400`：`version` 缺失，或 `content` 中不含合法的 `nodes` 数组
|
||||
- `404`：stash 不存在或内容为空
|
||||
- `500`：其他内部错误
|
||||
|
||||
### 4. POST `/fst/stash/update` —— 按版本更新 stash（对齐 /fst/update 风格）
|
||||
|
||||
对应函数：`upsert_fst_stash_api`
|
||||
|
||||
用途：
|
||||
|
||||
- 参考 `/fst/update` 的风格进行设计；
|
||||
- 也是对 `fst_stash` 做 UPSERT，只是路径和语义更偏向“编辑当前版本的 stash 内容”；
|
||||
- 前端可统一通过 `/fst/stash/update` 写入当前版本的 JSON。
|
||||
- 该接口默认开启节点级乐观锁校验。
|
||||
|
||||
请求：
|
||||
|
||||
```json
|
||||
POST /fst/stash/update
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"version": "MB Driving FST V1",
|
||||
"content": {
|
||||
"meta": { "...": "..." },
|
||||
"nodes": [ ... ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
返回（200）：
|
||||
|
||||
```json
|
||||
{
|
||||
"id": 1,
|
||||
"version": "MB Driving FST V1",
|
||||
"updated_time": "2026-02-25T10:00:00+00:00"
|
||||
}
|
||||
```
|
||||
|
||||
错误：
|
||||
|
||||
- `400`：
|
||||
- 缺少 `version`
|
||||
- `version` 不在 `version_list` 中（`upsert_fst_stash` 抛出的 `ValueError`）
|
||||
- `409`：
|
||||
- 节点级乐观锁冲突（返回 `STASH_NODE_VERSION_CONFLICT` 和冲突节点列表）
|
||||
- `500`：其他内部错误
|
||||
|
||||
内部实现：
|
||||
|
||||
- 和 `POST /fst/stash` 一样，最终都调用 `upsert_fst_stash`，只是路由风格与语义更贴近 `/fst/update`。
|
||||
|
||||
---
|
||||
|
||||
## 四、与正式 FST 表的关系
|
||||
|
||||
- 正式 FST 表读取接口：
|
||||
- `GET /fst/print_tree`：直接从 `fst` 表读取，构建树结构并返回。
|
||||
- Stash 相关接口：
|
||||
- `POST /fst/stash`、`POST /fst/stash/update`：将某个版本的 JSON 写入 `fst_stash`
|
||||
- `GET /fst/stash/<version>`：按版本拿回原始 JSON
|
||||
- `GET /fst/stash/print_tree`：基于 stash 的 `nodes` 构建树，结构与 `/fst/print_tree` 相同
|
||||
|
||||
当前阶段设计原则：
|
||||
|
||||
- **正式 FST 表**：仍然通过 `/fst/update` 等接口直接修改；
|
||||
- **fst_stash**：作为“导入/草稿/预览版本”的 JSON 存储，方便前端导入导出、预览树结构以及后续一键发布功能的扩展。
|
||||
|
||||
---
|
||||
|
||||
## 五、后续新增：Feishu 写入（版本生效）
|
||||
|
||||
本节记录后续新增的“版本生效并同步到飞书多维表格”能力。
|
||||
|
||||
### 1. 触发接口
|
||||
|
||||
文件位置：`src/api/versions.py`
|
||||
|
||||
- `POST /versions/stash/<version_id>/activate`
|
||||
- `POST /versions/fst/stash/activate/<version_id>`
|
||||
- `POST /versions/<version_id>/activate`
|
||||
|
||||
说明：
|
||||
|
||||
- 三个路由映射到同一个函数 `activate_version_api`。
|
||||
- 请求体可选传入 `content`：
|
||||
- 传了：直接使用该 JSON 作为飞书写入内容；
|
||||
- 不传：回退读取 `fst_stash.content`。
|
||||
- 若最终拿不到 JSON，返回 `404`。
|
||||
|
||||
### 2. 执行方式（异步任务）
|
||||
|
||||
`activate_version_api` 不阻塞等待飞书完成，而是：
|
||||
|
||||
1. 创建 `task_id` 并写入内存任务表（`ACTIVATE_TASKS`），初始状态 `queued`；
|
||||
2. 启动后台线程执行 `_run_activate_task(task_id, version_id, content)`；
|
||||
3. 立即返回 `202 Accepted`。
|
||||
|
||||
任务结果查询接口：
|
||||
|
||||
- `GET /versions/stash/<version_id>/activate/result?task_id=...`
|
||||
- `GET /versions/<version_id>/activate/result?task_id=...`
|
||||
|
||||
如果不传 `task_id`，会返回该版本最近一次任务结果。
|
||||
|
||||
### 3. 与飞书 SDK 的对接
|
||||
|
||||
后台线程 `_run_activate_task` 的关键流程：
|
||||
|
||||
1. 将 `version_list.feishu_sync_status` 置为 `syncing`；
|
||||
2. 调用 `create_bitable_for_version(version_obj.version, content)`；
|
||||
3. 成功后：
|
||||
- `feishu_sync_status = synced`
|
||||
- `status = active`
|
||||
4. 失败时：
|
||||
- 回滚后将 `feishu_sync_status = failed`
|
||||
- 任务状态记为 `failed` 并返回错误信息。
|
||||
|
||||
### 4. Feishu 写入目标与结构（feishu_bitable_sdk.py）
|
||||
|
||||
文件位置：`src/core/feishu_bitable_sdk.py`
|
||||
|
||||
- 主多维表标题固定为 `Fst_Editor`（常量 `FST_EDITOR_BITABLE_TITLE`）。
|
||||
- 目前新增了父文档定位能力：
|
||||
- 优先使用 `PARENT_NODE_TOKEN`；
|
||||
- 若为空，则按 `FST_EDITOR_PARENT_DOC_TITLE`（当前值为 `Content generated automatically by bot`）在空间根目录解析父文档；
|
||||
- 在该父文档下复用或创建 `Fst_Editor` 多维表。
|
||||
- 保留“只保留当前版本表”的策略：目标版本表不存在则创建，其他版本表会尝试删除。
|
||||
- 当内容可解析为 `nodes`/字段结构时，按业务列写入；否则回退写入 JSON 文本列。
|
||||
|
||||
### 5. 子文档同步（可开关）
|
||||
|
||||
`create_bitable_for_version` 在主表写入前后会根据开关触发一级节点子文档同步：
|
||||
|
||||
- `ENABLE_WIKI_CHILD_DOC_SYNC`
|
||||
- `ENABLE_WIKI_CHILD_DOC_SYNC_ASYNC`
|
||||
|
||||
默认配置下为异步触发，不阻塞主流程；结果会体现在返回体 `wiki_child_docs_result` 字段中。
|
||||
|
||||
1669
fst_data_pipeline/apps/root_db_api/README.md
Normal file
1669
fst_data_pipeline/apps/root_db_api/README.md
Normal file
File diff suppressed because it is too large
Load Diff
0
fst_data_pipeline/apps/root_db_api/__init__.py
Normal file
0
fst_data_pipeline/apps/root_db_api/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
ALTER TABLE version_list
|
||||
ADD COLUMN IF NOT EXISTS feishu_sync_status VARCHAR(50) NOT NULL DEFAULT 'unsynced';
|
||||
|
||||
COMMENT ON COLUMN version_list.feishu_sync_status IS 'Feishu sync status: unsynced/synced';
|
||||
702
fst_data_pipeline/apps/root_db_api/dev/db/init.sql
Normal file
702
fst_data_pipeline/apps/root_db_api/dev/db/init.sql
Normal file
@@ -0,0 +1,702 @@
|
||||
-- ======================================================
|
||||
-- 0 扩展
|
||||
-- ======================================================
|
||||
CREATE EXTENSION IF NOT EXISTS postgis;
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
|
||||
COMMENT ON EXTENSION postgis IS 'PostGIS 空间数据库扩展';
|
||||
COMMENT ON EXTENSION "uuid-ossp" IS 'UUID 生成扩展';
|
||||
|
||||
-- ======================================================
|
||||
-- ① 基础准备
|
||||
-- ======================================================
|
||||
CREATE TABLE IF NOT EXISTS project (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
COMMENT ON TABLE project IS '数据来源 1：DFDI 2：CFDI';
|
||||
COMMENT ON COLUMN project.id IS '项目自增主键';
|
||||
COMMENT ON COLUMN project.name IS '项目名称，全局唯一';
|
||||
COMMENT ON COLUMN project.update_time IS '最近一次更新时间';
|
||||
|
||||
-- ======================================================
|
||||
-- ② 字典 & 主数据（无外部依赖）
|
||||
-- ======================================================
|
||||
CREATE TABLE IF NOT EXISTS drive_mode (
|
||||
id SERIAL PRIMARY KEY,
|
||||
type VARCHAR(100) NOT NULL,
|
||||
sub_type VARCHAR(100),
|
||||
reserved_json JSONB,
|
||||
comment TEXT,
|
||||
UNIQUE (type, sub_type)
|
||||
);
|
||||
COMMENT ON TABLE drive_mode IS '驾驶模式字典表';
|
||||
COMMENT ON COLUMN drive_mode.id IS '主键';
|
||||
COMMENT ON COLUMN drive_mode.type IS '驾驶模式大类，AD Drive/AD PARK(APA)/AD PARK(MPA)';
|
||||
COMMENT ON COLUMN drive_mode.sub_type IS '驾驶模式子类，如ACC/CP/NP';
|
||||
COMMENT ON COLUMN drive_mode.reserved_json IS '预留扩展 JSON';
|
||||
COMMENT ON COLUMN drive_mode.comment IS '备注';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS topic_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
type VARCHAR(255),
|
||||
key_data TEXT,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE topic_list IS 'Topic表';
|
||||
COMMENT ON COLUMN topic_list.id IS '主键';
|
||||
COMMENT ON COLUMN topic_list.name IS 'Topic 名称，如 /lidar/points';
|
||||
COMMENT ON COLUMN topic_list.type IS 'Topic 类型，如 sensor_msgs/PointCloud2';
|
||||
COMMENT ON COLUMN topic_list.key_data IS '关键数据摘要';
|
||||
COMMENT ON COLUMN topic_list.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN topic_list.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS reserved_tag_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
type VARCHAR(50),
|
||||
creator VARCHAR(255),
|
||||
comments TEXT,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
is_deleted BOOLEAN DEFAULT FALSE,
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE reserved_tag_list IS '预留标签字典表';
|
||||
COMMENT ON COLUMN reserved_tag_list.id IS '主键';
|
||||
COMMENT ON COLUMN reserved_tag_list.name IS '标签名称';
|
||||
COMMENT ON COLUMN reserved_tag_list.type IS '标签分类';
|
||||
COMMENT ON COLUMN reserved_tag_list.creator IS '创建人';
|
||||
COMMENT ON COLUMN reserved_tag_list.comments IS '备注';
|
||||
COMMENT ON COLUMN reserved_tag_list.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN reserved_tag_list.is_deleted IS '逻辑删除标记';
|
||||
COMMENT ON COLUMN reserved_tag_list.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS gt_meta (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
type VARCHAR(100) NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
reserved_json JSONB,
|
||||
comment TEXT,
|
||||
CONSTRAINT gt_meta_name_type_unique UNIQUE (name, type)
|
||||
);
|
||||
COMMENT ON TABLE gt_meta IS '真值元数据表';
|
||||
COMMENT ON COLUMN gt_meta.id IS '主键';
|
||||
COMMENT ON COLUMN gt_meta.name IS '真值名称';
|
||||
COMMENT ON COLUMN gt_meta.type IS '真值类型，如 3DBox/Lane';
|
||||
COMMENT ON COLUMN gt_meta.path IS '真值文件存储路径';
|
||||
COMMENT ON COLUMN gt_meta.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN gt_meta.reserved_json IS '预留扩展 JSON';
|
||||
COMMENT ON COLUMN gt_meta.comment IS '备注';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS version_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
version VARCHAR(255) NOT NULL UNIQUE,
|
||||
type VARCHAR(50),
|
||||
description TEXT,
|
||||
release_date TIMESTAMPTZ,
|
||||
created_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
status VARCHAR(50) NOT NULL DEFAULT 'available',
|
||||
feishu_sync_status VARCHAR(50) NOT NULL DEFAULT 'unsynced',
|
||||
reserved_json JSONB
|
||||
);
|
||||
|
||||
-- ======================================================
|
||||
-- ③ 核心业务表（依赖 ①②）
|
||||
-- ======================================================
|
||||
CREATE TABLE IF NOT EXISTS bag_list (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
project_id INT,
|
||||
tile_id VARCHAR(255),
|
||||
is_decoded BOOLEAN DEFAULT FALSE,
|
||||
is_deleted BOOLEAN DEFAULT FALSE,
|
||||
od_annotated INT,
|
||||
ld_annotated INT,
|
||||
fst_indexed BOOLEAN DEFAULT FALSE,
|
||||
is_active_data BOOLEAN DEFAULT TRUE,
|
||||
reserved_str VARCHAR(255),
|
||||
reserved_json JSONB,
|
||||
drive_mode SMALLINT,
|
||||
sw_version VARCHAR(100),
|
||||
hw_version VARCHAR(100),
|
||||
FOREIGN KEY (project_id) REFERENCES project(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (drive_mode) REFERENCES drive_mode(id) ON DELETE SET NULL
|
||||
);
|
||||
COMMENT ON TABLE bag_list IS 'Rosbag 主表';
|
||||
COMMENT ON COLUMN bag_list.id IS '主键';
|
||||
COMMENT ON COLUMN bag_list.name IS 'Bag 文件名称，全局唯一';
|
||||
COMMENT ON COLUMN bag_list.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN bag_list.project_id IS '数据来源';
|
||||
COMMENT ON COLUMN bag_list.tile_id IS '瓦片 ID';
|
||||
COMMENT ON COLUMN bag_list.is_decoded IS '是否已解码';
|
||||
COMMENT ON COLUMN bag_list.is_deleted IS '逻辑删除';
|
||||
COMMENT ON COLUMN bag_list.od_annotated IS '是否目标检测标注';
|
||||
COMMENT ON COLUMN bag_list.ld_annotated IS '是否车道线标注';
|
||||
COMMENT ON COLUMN bag_list.fst_indexed IS '是否已建立 FST 索引';
|
||||
COMMENT ON COLUMN bag_list.is_active_data IS '是否为活跃数据';
|
||||
COMMENT ON COLUMN bag_list.reserved_str IS '预留字符串';
|
||||
COMMENT ON COLUMN bag_list.reserved_json IS '预留扩展 JSON';
|
||||
COMMENT ON COLUMN bag_list.drive_mode IS '驾驶模式字典 ID';
|
||||
COMMENT ON COLUMN bag_list.sw_version IS '软件版本号（Software Version）';
|
||||
COMMENT ON COLUMN bag_list.hw_version IS '硬件版本号（Hardware Version）';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bag_lifecycle (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_name VARCHAR(255) NOT NULL UNIQUE,
|
||||
collect_time TIMESTAMPTZ,
|
||||
clone2dev_time TIMESTAMPTZ,
|
||||
decode_time TIMESTAMPTZ,
|
||||
mining_time TIMESTAMPTZ,
|
||||
auto_annotate_time TIMESTAMPTZ,
|
||||
manual_annotate_time TIMESTAMPTZ,
|
||||
fst_index_time TIMESTAMPTZ,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
FOREIGN KEY (bag_name) REFERENCES bag_list(name) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE bag_lifecycle IS 'Bag 生命周期时间表';
|
||||
COMMENT ON COLUMN bag_lifecycle.id IS '主键';
|
||||
COMMENT ON COLUMN bag_lifecycle.bag_name IS 'Bag 名称，关联 bag_list.name';
|
||||
COMMENT ON COLUMN bag_lifecycle.collect_time IS '采集完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.clone2dev_time IS '拷贝到bucket时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.decode_time IS '解码完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.mining_time IS '数据挖掘完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.auto_annotate_time IS '自动标注完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.manual_annotate_time IS '人工标注完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.fst_index_time IS 'FST 索引完成时间';
|
||||
COMMENT ON COLUMN bag_lifecycle.update_time IS '记录更新时间';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS main_pangu (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
vehicle VARCHAR(255),
|
||||
datetime TIMESTAMPTZ,
|
||||
bag_path TEXT,
|
||||
data_path TEXT,
|
||||
reserved_str VARCHAR(255),
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE main_pangu IS 'Pangu 主表（原始数据）';
|
||||
COMMENT ON COLUMN main_pangu.id IS '主键';
|
||||
COMMENT ON COLUMN main_pangu.name IS '名称，全局唯一';
|
||||
COMMENT ON COLUMN main_pangu.vehicle IS '车辆编号';
|
||||
COMMENT ON COLUMN main_pangu.datetime IS 'rosbag的生成时间';
|
||||
COMMENT ON COLUMN main_pangu.bag_path IS '原始 Bag 路径';
|
||||
COMMENT ON COLUMN main_pangu.data_path IS '解析后数据路径';
|
||||
COMMENT ON COLUMN main_pangu.reserved_str IS '预留字符串';
|
||||
COMMENT ON COLUMN main_pangu.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS joined_pangu (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
data_path TEXT,
|
||||
reserved_str VARCHAR(255),
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE joined_pangu IS 'Pangu rosbag融合表';
|
||||
COMMENT ON COLUMN joined_pangu.id IS '主键';
|
||||
COMMENT ON COLUMN joined_pangu.name IS '融合后名称';
|
||||
COMMENT ON COLUMN joined_pangu.data_path IS '融合后数据路径';
|
||||
COMMENT ON COLUMN joined_pangu.reserved_str IS '预留字符串';
|
||||
COMMENT ON COLUMN joined_pangu.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS joined_bags (
|
||||
id SERIAL PRIMARY KEY,
|
||||
parent_id INT NOT NULL,
|
||||
child_id INT NOT NULL UNIQUE,
|
||||
reserved_str VARCHAR(255),
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE joined_bags IS 'Bag 融合关系表';
|
||||
COMMENT ON COLUMN joined_bags.id IS '主键';
|
||||
COMMENT ON COLUMN joined_bags.parent_id IS '父 Bag ID';
|
||||
COMMENT ON COLUMN joined_bags.child_id IS '子 Bag ID，唯一';
|
||||
COMMENT ON COLUMN joined_bags.reserved_str IS '预留字符串';
|
||||
COMMENT ON COLUMN joined_bags.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS main_minerva (
|
||||
id SERIAL PRIMARY KEY,
|
||||
session_id VARCHAR(255) UNIQUE,
|
||||
start_ts TIMESTAMPTZ,
|
||||
end_ts TIMESTAMPTZ,
|
||||
length INT,
|
||||
datetime TIMESTAMPTZ,
|
||||
vin VARCHAR(255),
|
||||
platform VARCHAR(255),
|
||||
mapped BOOLEAN DEFAULT FALSE,
|
||||
path TEXT,
|
||||
converted_path TEXT,
|
||||
gt_path TEXT,
|
||||
reserved_json JSONB
|
||||
);
|
||||
COMMENT ON TABLE main_minerva IS 'Minerva 主表';
|
||||
COMMENT ON COLUMN main_minerva.id IS '主键';
|
||||
COMMENT ON COLUMN main_minerva.session_id IS '会话 ID，唯一';
|
||||
COMMENT ON COLUMN main_minerva.start_ts IS '开始时间戳';
|
||||
COMMENT ON COLUMN main_minerva.end_ts IS '结束时间戳';
|
||||
COMMENT ON COLUMN main_minerva.length IS '时长（秒）';
|
||||
COMMENT ON COLUMN main_minerva.datetime IS '数据日期';
|
||||
COMMENT ON COLUMN main_minerva.vin IS '车辆识别码';
|
||||
COMMENT ON COLUMN main_minerva.platform IS '平台名称';
|
||||
COMMENT ON COLUMN main_minerva.mapped IS '是否已映射';
|
||||
COMMENT ON COLUMN main_minerva.path IS '原始数据路径';
|
||||
COMMENT ON COLUMN main_minerva.converted_path IS '转换后路径';
|
||||
COMMENT ON COLUMN main_minerva.gt_path IS '真值路径';
|
||||
COMMENT ON COLUMN main_minerva.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
-- ======================================================
|
||||
-- ④ 子表 & 关联表（依赖 bag_list.id）
|
||||
-- ======================================================
|
||||
CREATE TABLE IF NOT EXISTS secondary_pangu (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL UNIQUE,
|
||||
lidar_gt_pandar128 TEXT,
|
||||
object_lidar_gt_pandar128_manual TEXT,
|
||||
lidar_fd_multi_scan_raw TEXT,
|
||||
camera_fisheye_left TEXT,
|
||||
camera_fisheye_right TEXT,
|
||||
camera_front_wide TEXT,
|
||||
raw_gps TEXT,
|
||||
raw_imu TEXT,
|
||||
ego_motion TEXT,
|
||||
vehicle_wheel TEXT,
|
||||
calibration TEXT,
|
||||
sdmap TEXT,
|
||||
reserved_json JSONB,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE secondary_pangu IS 'Pangu 二级数据路径表';
|
||||
COMMENT ON COLUMN secondary_pangu.id IS '主键';
|
||||
COMMENT ON COLUMN secondary_pangu.bag_id IS '关联 bag_list.id';
|
||||
COMMENT ON COLUMN secondary_pangu.lidar_gt_pandar128 IS '激光真值路径';
|
||||
COMMENT ON COLUMN secondary_pangu.object_lidar_gt_pandar128_manual IS '人工真值路径';
|
||||
COMMENT ON COLUMN secondary_pangu.lidar_fd_multi_scan_raw IS '原始雷达路径';
|
||||
COMMENT ON COLUMN secondary_pangu.camera_fisheye_left IS '左鱼眼相机路径';
|
||||
COMMENT ON COLUMN secondary_pangu.camera_fisheye_right IS '右鱼眼相机路径';
|
||||
COMMENT ON COLUMN secondary_pangu.camera_front_wide IS '前广角相机路径';
|
||||
COMMENT ON COLUMN secondary_pangu.raw_gps IS '原始 GPS 路径';
|
||||
COMMENT ON COLUMN secondary_pangu.raw_imu IS '原始 IMU 路径';
|
||||
COMMENT ON COLUMN secondary_pangu.ego_motion IS '自车运动路径';
|
||||
COMMENT ON COLUMN secondary_pangu.vehicle_wheel IS '轮速路径';
|
||||
COMMENT ON COLUMN secondary_pangu.calibration IS '标定文件路径';
|
||||
COMMENT ON COLUMN secondary_pangu.sdmap IS 'SD 地图路径或地图版本标识';
|
||||
COMMENT ON COLUMN secondary_pangu.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS secondary_minerva (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL UNIQUE,
|
||||
lidar_gt_top_p128 TEXT,
|
||||
lidar_parking_gt_front_p128 TEXT,
|
||||
lidar_parking_gt_left_p128 TEXT,
|
||||
lidar_parking_gt_right_p128 TEXT,
|
||||
lidar_parking_gt_rear_p128 TEXT,
|
||||
camera_fisheye_left_200fov TEXT,
|
||||
camera_fisheye_right_200fov TEXT,
|
||||
camera_fisheye_rear_200fov TEXT,
|
||||
camera_fisheye_front_200fov TEXT,
|
||||
camera_front_wide_120fov TEXT,
|
||||
camera_front_tele_30fov TEXT,
|
||||
camera_rear_right_70fov TEXT,
|
||||
camera_rear_left_70fov TEXT,
|
||||
raw_gps TEXT,
|
||||
raw_imu TEXT,
|
||||
ego_motion TEXT,
|
||||
calibration TEXT,
|
||||
rig TEXT,
|
||||
fst_new TEXT,
|
||||
fst_old TEXT,
|
||||
comments TEXT,
|
||||
reserved_json JSONB,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE secondary_minerva IS 'Minerva 二级数据路径表';
|
||||
COMMENT ON COLUMN secondary_minerva.id IS '主键';
|
||||
COMMENT ON COLUMN secondary_minerva.bag_id IS '关联 bag_list.id';
|
||||
COMMENT ON COLUMN secondary_minerva.lidar_gt_top_p128 IS '顶部Lidar真值路径';
|
||||
COMMENT ON COLUMN secondary_minerva.lidar_parking_gt_front_p128 IS '前停车Lidar真值路径';
|
||||
COMMENT ON COLUMN secondary_minerva.lidar_parking_gt_left_p128 IS '左停车Lidar真值路径';
|
||||
COMMENT ON COLUMN secondary_minerva.lidar_parking_gt_right_p128 IS '右停车Lidar真值路径';
|
||||
COMMENT ON COLUMN secondary_minerva.lidar_parking_gt_rear_p128 IS '后停车Lidar真值路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_fisheye_left_200fov IS '左 200° 鱼眼路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_fisheye_right_200fov IS '右 200° 鱼眼路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_fisheye_rear_200fov IS '后 200° 鱼眼路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_fisheye_front_200fov IS '前 200° 鱼眼路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_front_wide_120fov IS '前 120° 广角路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_front_tele_30fov IS '前 30° 长焦路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_rear_right_70fov IS '右后 70° 相机路径';
|
||||
COMMENT ON COLUMN secondary_minerva.camera_rear_left_70fov IS '左后 70° 相机路径';
|
||||
COMMENT ON COLUMN secondary_minerva.raw_gps IS '原始 GPS 路径';
|
||||
COMMENT ON COLUMN secondary_minerva.raw_imu IS '原始 IMU 路径';
|
||||
COMMENT ON COLUMN secondary_minerva.ego_motion IS '自车运动路径';
|
||||
COMMENT ON COLUMN secondary_minerva.calibration IS '标定文件路径';
|
||||
COMMENT ON COLUMN secondary_minerva.rig IS 'rig 文件路径';
|
||||
COMMENT ON COLUMN secondary_minerva.fst_new IS '新 FST 路径';
|
||||
COMMENT ON COLUMN secondary_minerva.fst_old IS '旧 FST 路径';
|
||||
COMMENT ON COLUMN secondary_minerva.comments IS '备注';
|
||||
COMMENT ON COLUMN secondary_minerva.reserved_json IS '预留扩展 JSON';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fst (
|
||||
id SERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
parent_id INT,
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
reserved_json JSONB,
|
||||
bag_sum INT DEFAULT 0,
|
||||
FOREIGN KEY (parent_id) REFERENCES fst(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE fst IS 'FST 节点表';
|
||||
COMMENT ON COLUMN fst.id IS '主键';
|
||||
COMMENT ON COLUMN fst.name IS '节点名称';
|
||||
COMMENT ON COLUMN fst.parent_id IS '父节点 ID';
|
||||
COMMENT ON COLUMN fst.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN fst.reserved_json IS '预留扩展 JSON';
|
||||
COMMENT ON COLUMN fst.bag_sum IS '子树 Bag 数量';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS geometry_info (
|
||||
id SERIAL PRIMARY KEY,
|
||||
rosbag_name VARCHAR(255) NOT NULL,
|
||||
gnss_downsampled_points geometry[],
|
||||
update_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
is_overlapped BOOLEAN DEFAULT FALSE
|
||||
);
|
||||
COMMENT ON TABLE geometry_info IS 'Bag 几何信息表（GNSS 轨迹）';
|
||||
COMMENT ON COLUMN geometry_info.id IS '主键';
|
||||
COMMENT ON COLUMN geometry_info.rosbag_name IS 'Rosbag 名称';
|
||||
COMMENT ON COLUMN geometry_info.gnss_downsampled_points IS '降采样 GNSS 轨迹点';
|
||||
COMMENT ON COLUMN geometry_info.update_time IS '更新时间';
|
||||
COMMENT ON COLUMN geometry_info.is_overlapped IS '是否与其他 Bag 重叠';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bag_topic (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL,
|
||||
topic_id INT NOT NULL,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (topic_id) REFERENCES topic_list(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE bag_topic IS 'Bag-Topic 多对多关系';
|
||||
COMMENT ON COLUMN bag_topic.id IS '主键';
|
||||
COMMENT ON COLUMN bag_topic.bag_id IS 'Bag ID';
|
||||
COMMENT ON COLUMN bag_topic.topic_id IS 'Topic ID';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bag_reserved_tag (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL,
|
||||
tag_id INT NOT NULL,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (tag_id) REFERENCES reserved_tag_list(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE bag_reserved_tag IS 'Bag-预留标签多对多关系';
|
||||
COMMENT ON COLUMN bag_reserved_tag.id IS '主键';
|
||||
COMMENT ON COLUMN bag_reserved_tag.bag_id IS 'Bag ID';
|
||||
COMMENT ON COLUMN bag_reserved_tag.tag_id IS '标签 ID';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS bag_gt (
|
||||
id SERIAL PRIMARY KEY,
|
||||
gt_id INT NOT NULL,
|
||||
bag_id INT NOT NULL,
|
||||
comment TEXT,
|
||||
UNIQUE (gt_id, bag_id),
|
||||
FOREIGN KEY (gt_id) REFERENCES gt_meta(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE bag_gt IS 'Bag-真值多对多关系';
|
||||
COMMENT ON COLUMN bag_gt.id IS '主键';
|
||||
COMMENT ON COLUMN bag_gt.gt_id IS '真值元数据 ID';
|
||||
COMMENT ON COLUMN bag_gt.bag_id IS 'Bag ID';
|
||||
COMMENT ON COLUMN bag_gt.comment IS '备注';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fst_bag (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL,
|
||||
fst_node_id INT NOT NULL,
|
||||
fst_node_level INT NOT NULL,
|
||||
event_start_time INT,
|
||||
event_end_time INT,
|
||||
img_url VARCHAR(255),
|
||||
video_url VARCHAR(255),
|
||||
comments VARCHAR(255),
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (fst_node_id) REFERENCES fst(id) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE fst_bag IS 'FST 节点与 Bag 的关联表';
|
||||
COMMENT ON COLUMN fst_bag.id IS '主键';
|
||||
COMMENT ON COLUMN fst_bag.bag_id IS 'Bag ID';
|
||||
COMMENT ON COLUMN fst_bag.fst_node_id IS 'FST 节点 ID';
|
||||
COMMENT ON COLUMN fst_bag.fst_node_level IS '节点层级';
|
||||
COMMENT ON COLUMN fst_bag.event_start_time IS '事件开始时间（ms）';
|
||||
COMMENT ON COLUMN fst_bag.event_end_time IS '事件结束时间（ms）';
|
||||
COMMENT ON COLUMN fst_bag.img_url IS '事件截图 URL';
|
||||
COMMENT ON COLUMN fst_bag.video_url IS '事件视频 URL';
|
||||
COMMENT ON COLUMN fst_bag.comments IS '备注';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS recompute_result (
|
||||
id SERIAL PRIMARY KEY,
|
||||
bag_id INT NOT NULL,
|
||||
recompute_version VARCHAR(100) NOT NULL,
|
||||
result_type VARCHAR(50) NOT NULL,
|
||||
storage_path VARCHAR(500) NOT NULL,
|
||||
status VARCHAR(50) NOT NULL DEFAULT 'available',
|
||||
created_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
reserved_json JSONB,
|
||||
FOREIGN KEY (bag_id) REFERENCES bag_list(id) ON DELETE CASCADE,
|
||||
UNIQUE (bag_id, recompute_version, result_type)
|
||||
);
|
||||
COMMENT ON TABLE recompute_result IS 'Recompute 结果存储表';
|
||||
COMMENT ON COLUMN recompute_result.id IS '主键';
|
||||
COMMENT ON COLUMN recompute_result.bag_id IS '关联 bag_list.id';
|
||||
COMMENT ON COLUMN recompute_result.recompute_version IS 'Recompute版本';
|
||||
COMMENT ON COLUMN recompute_result.result_type IS '结果类型 (如: perception, planning, etc)';
|
||||
COMMENT ON COLUMN recompute_result.storage_path IS '结果存储路径';
|
||||
COMMENT ON COLUMN recompute_result.status IS '结果状态: available, archived, deleted';
|
||||
COMMENT ON COLUMN recompute_result.created_time IS 'Recompute完成时间';
|
||||
COMMENT ON COLUMN recompute_result.updated_time IS '更新时间';
|
||||
COMMENT ON COLUMN recompute_result.reserved_json IS '预留JSON字段';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fst_stash (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
fst_versions VARCHAR(255) NOT NULL,
|
||||
content JSONB,
|
||||
created_time TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
updated_time TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
CONSTRAINT fk_fst_stash_version FOREIGN KEY (fst_versions) REFERENCES version_list (version) ON DELETE CASCADE
|
||||
);
|
||||
COMMENT ON TABLE fst_stash IS 'FST 暂存表';
|
||||
COMMENT ON COLUMN fst_stash.id IS '主键';
|
||||
COMMENT ON COLUMN fst_stash.fst_versions IS '关联 version_list.version';
|
||||
COMMENT ON COLUMN fst_stash.content IS '暂存内容 (JSON)';
|
||||
COMMENT ON COLUMN fst_stash.created_time IS '创建时间';
|
||||
COMMENT ON COLUMN fst_stash.updated_time IS '更新时间';
|
||||
|
||||
-- ======================================================
|
||||
-- 审计日志表
|
||||
-- ======================================================
|
||||
CREATE TABLE IF NOT EXISTS ops_history (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
table_name REGCLASS NOT NULL,
|
||||
row_pk TEXT NOT NULL,
|
||||
column_name TEXT NOT NULL,
|
||||
old_value TEXT,
|
||||
new_value TEXT,
|
||||
changed_by NAME NOT NULL DEFAULT CURRENT_USER,
|
||||
changed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
comment TEXT,
|
||||
app_module TEXT,
|
||||
trace_id TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_ops_history_changed_at ON ops_history (changed_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_ops_history_table_name ON ops_history (table_name);
|
||||
COMMENT ON TABLE ops_history IS '通用审计日志：记录 DML 变更';
|
||||
COMMENT ON COLUMN ops_history.id IS '审计日志自增主键';
|
||||
COMMENT ON COLUMN ops_history.table_name IS '被审计表（系统元类型 REGCLASS）';
|
||||
COMMENT ON COLUMN ops_history.row_pk IS '行主键值；批量更新时形如 *col*rows*';
|
||||
COMMENT ON COLUMN ops_history.column_name IS '被修改列名；批量更新时为 *batch*';
|
||||
COMMENT ON COLUMN ops_history.old_value IS '旧值';
|
||||
COMMENT ON COLUMN ops_history.new_value IS '新值';
|
||||
COMMENT ON COLUMN ops_history.changed_by IS '数据库用户';
|
||||
COMMENT ON COLUMN ops_history.changed_at IS '变更时间';
|
||||
COMMENT ON COLUMN ops_history.comment IS '可读描述（自动生成或人工补充）';
|
||||
COMMENT ON COLUMN ops_history.app_module IS '应用模块（通过 SET audit.app_module = ''xxx''）';
|
||||
COMMENT ON COLUMN ops_history.trace_id IS '链路追踪 ID（通过 SET audit.trace_id = ''xxx''）';
|
||||
|
||||
-- ======================================================
|
||||
-- ⑤ 索引
|
||||
-- ======================================================
|
||||
CREATE INDEX IF NOT EXISTS idx_project_name ON project(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_list_project_id ON bag_list(project_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_pangu_datetime ON main_pangu(datetime);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_pangu_name ON main_pangu(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_minerva_session_id ON main_minerva(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_minerva_start_ts ON main_minerva(start_ts);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_minerva_end_ts ON main_minerva(end_ts);
|
||||
CREATE INDEX IF NOT EXISTS idx_main_minerva_vin ON main_minerva(vin);
|
||||
CREATE INDEX IF NOT EXISTS idx_secondary_pangu_bag_id ON secondary_pangu(bag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_secondary_minerva_bag_id ON secondary_minerva(bag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_fst_name ON fst(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_topic_list_name ON topic_list(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_topic_bag_id ON bag_topic(bag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_topic_topic_id ON bag_topic(topic_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_tag_list_name ON reserved_tag_list(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_tag_bag_id ON bag_reserved_tag(bag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_tag_tag_id ON bag_reserved_tag(tag_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_gt_meta_name ON gt_meta(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_gt_meta_type ON gt_meta(type);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_gt_gt_id ON bag_gt(gt_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_gt_bag_id ON bag_gt(bag_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_list_project_active
|
||||
ON bag_list(project_id) WHERE is_deleted = FALSE;
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_taglist_name_not_deleted
|
||||
ON reserved_tag_list (name) WHERE is_deleted = FALSE;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bag_list_json_gin
|
||||
ON bag_list USING GIN (reserved_json);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_recompute_result_version ON recompute_result(recompute_version);
|
||||
CREATE INDEX IF NOT EXISTS idx_recompute_result_type ON recompute_result(result_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_recompute_result_status ON recompute_result(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_recompute_result_created_time ON recompute_result(created_time);
|
||||
CREATE INDEX IF NOT EXISTS idx_recompute_result_storage_path ON recompute_result(storage_path);
|
||||
CREATE INDEX IF NOT EXISTS idx_fst_stash_fst_versions ON fst_stash(fst_versions);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_lifecycle_decode_time ON bag_lifecycle USING BRIN (decode_time);
|
||||
CREATE INDEX IF NOT EXISTS idx_lifecycle_mining_time ON bag_lifecycle USING BRIN (mining_time);
|
||||
CREATE INDEX IF NOT EXISTS idx_lifecycle_manual_annotate_time ON bag_lifecycle USING BRIN (manual_annotate_time);
|
||||
|
||||
-- ======================================================
|
||||
-- ⑥ 触发器函数
|
||||
-- ======================================================
|
||||
CREATE OR REPLACE FUNCTION set_update_time() RETURNS TRIGGER AS
|
||||
$$
|
||||
BEGIN
|
||||
NEW.update_time := NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
COMMENT ON FUNCTION set_update_time() IS '通用：更新行时自动刷新 update_time';
|
||||
|
||||
CREATE OR REPLACE FUNCTION audit_changes() RETURNS TRIGGER AS $$
|
||||
DECLARE
|
||||
ch_col text; -- 变化列名（字符串） -- FIX
|
||||
row_cnt int; -- 变化列总数（整数） -- FIX
|
||||
oldv text;
|
||||
newv text;
|
||||
pkval text;
|
||||
tmpl text;
|
||||
BEGIN
|
||||
/* 构造主键字符串 */
|
||||
SELECT string_agg(
|
||||
coalesce(nullif(to_jsonb(NEW)->>attname, ''), 'null'), ',')
|
||||
INTO pkval
|
||||
FROM pg_index i
|
||||
JOIN pg_attribute a ON a.attrelid = i.indrelid
|
||||
AND a.attnum = ANY(i.indkey)
|
||||
WHERE i.indrelid = TG_RELID AND i.indisprimary;
|
||||
|
||||
IF TG_OP = 'UPDATE' THEN
|
||||
/* 1. 变化列总数 */
|
||||
SELECT count(*)::int INTO row_cnt -- FIX：用独立变量
|
||||
FROM pg_attribute
|
||||
WHERE attrelid = TG_RELID
|
||||
AND attnum > 0 AND NOT attisdropped
|
||||
AND to_jsonb(OLD)->>attname IS DISTINCT FROM to_jsonb(NEW)->>attname;
|
||||
|
||||
/* 2. 任意一个变化列的细节 */
|
||||
SELECT attname::text,
|
||||
nullif(to_jsonb(OLD)->>attname, ''),
|
||||
nullif(to_jsonb(NEW)->>attname, '')
|
||||
INTO ch_col, oldv, newv -- FIX：列名放 ch_col
|
||||
FROM pg_attribute
|
||||
WHERE attrelid = TG_RELID
|
||||
AND attnum > 0 AND NOT attisdropped
|
||||
AND to_jsonb(OLD)->>attname IS DISTINCT FROM to_jsonb(NEW)->>attname
|
||||
LIMIT 1;
|
||||
|
||||
IF FOUND THEN
|
||||
tmpl := format('批量更新 %s.%s: %s → %s （共 %s 行）',
|
||||
TG_TABLE_NAME, ch_col, oldv, newv, row_cnt);
|
||||
|
||||
INSERT INTO ops_history(table_name, row_pk, column_name,
|
||||
old_value, new_value, changed_by, comment,
|
||||
app_module, trace_id)
|
||||
VALUES (TG_RELID::REGCLASS,
|
||||
'*'||row_cnt||'rows*', -- FIX：用整数变量
|
||||
'*batch*',
|
||||
oldv,
|
||||
jsonb_build_object('new_val', newv, 'rows', row_cnt), -- FIX
|
||||
CURRENT_USER,
|
||||
tmpl,
|
||||
current_setting('audit.app_module', true),
|
||||
current_setting('audit.trace_id', true));
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
/* 以下逐列记录逻辑未改动，故省略 … */
|
||||
FOR ch_col IN
|
||||
SELECT attname::text
|
||||
FROM pg_attribute
|
||||
WHERE attrelid = TG_RELID
|
||||
AND attnum > 0 AND NOT attisdropped
|
||||
AND to_jsonb(OLD)->>attname IS DISTINCT FROM to_jsonb(NEW)->>attname
|
||||
LOOP
|
||||
oldv := nullif(to_jsonb(OLD)->>ch_col, '');
|
||||
newv := nullif(to_jsonb(NEW)->>ch_col, '');
|
||||
|
||||
tmpl := format('%s %s.%s: %s → %s',
|
||||
TG_OP, TG_TABLE_NAME, ch_col,
|
||||
coalesce(oldv,'(null)'), coalesce(newv,'(null)'));
|
||||
|
||||
INSERT INTO ops_history(table_name, row_pk, column_name,
|
||||
old_value, new_value, changed_by, comment,
|
||||
app_module, trace_id)
|
||||
VALUES (TG_RELID::REGCLASS,
|
||||
pkval,
|
||||
ch_col,
|
||||
oldv,
|
||||
newv,
|
||||
CURRENT_USER,
|
||||
tmpl,
|
||||
current_setting('audit.app_module', true),
|
||||
current_setting('audit.trace_id', true));
|
||||
END LOOP;
|
||||
|
||||
RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- ======================================================
|
||||
-- ⑦ 为每张业务表挂触发器
|
||||
-- ======================================================
|
||||
-- 7.1 自动刷新 update_time
|
||||
DO $$
|
||||
DECLARE
|
||||
t text;
|
||||
BEGIN
|
||||
FOREACH t IN ARRAY ARRAY[
|
||||
'project','bag_list','main_minerva','fst',
|
||||
'topic_list','reserved_tag_list','gt_meta',
|
||||
'bag_lifecycle','main_pangu','joined_pangu','joined_bags',
|
||||
'secondary_pangu','secondary_minerva','geometry_info',
|
||||
'bag_topic','bag_reserved_tag','bag_gt','fst_bag',
|
||||
'fst_stash'
|
||||
] LOOP
|
||||
EXECUTE format($f$
|
||||
DROP TRIGGER IF EXISTS trg_%I_update ON %I;
|
||||
CREATE TRIGGER trg_%I_update
|
||||
BEFORE UPDATE ON %I
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION set_update_time();
|
||||
$f$, t, t, t, t);
|
||||
END LOOP;
|
||||
END $$;
|
||||
|
||||
-- 7.2 审计触发器（所有表）
|
||||
DO $$
|
||||
DECLARE
|
||||
tbl TEXT;
|
||||
BEGIN
|
||||
FOREACH tbl IN ARRAY ARRAY[
|
||||
'project','bag_list','main_minerva','fst',
|
||||
'topic_list','reserved_tag_list','gt_meta',
|
||||
'bag_lifecycle','main_pangu','joined_pangu','joined_bags',
|
||||
'secondary_pangu','secondary_minerva','geometry_info',
|
||||
'bag_topic','bag_reserved_tag','bag_gt','fst_bag',
|
||||
'fst_stash'
|
||||
] LOOP
|
||||
EXECUTE format($f$
|
||||
DROP TRIGGER IF EXISTS trg_audit_%I ON %I;
|
||||
CREATE TRIGGER trg_audit_%I
|
||||
AFTER INSERT OR UPDATE OR DELETE ON %I
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION audit_changes();
|
||||
$f$, tbl, tbl, tbl, tbl);
|
||||
END LOOP;
|
||||
END $$;
|
||||
@@ -0,0 +1,679 @@
|
||||
{
|
||||
"info": {
|
||||
"_postman_id": "a24cba51-2034-4777-a911-85ce7240d1ac",
|
||||
"name": "ROOT API",
|
||||
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
|
||||
"_exporter_id": "17291944"
|
||||
},
|
||||
"item": [
|
||||
{
|
||||
"name": "projects",
|
||||
"item": [
|
||||
{
|
||||
"name": "All projects",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/projects/all",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"projects",
|
||||
"all"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "tags",
|
||||
"item": [
|
||||
{
|
||||
"name": "All tags",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/tags/all",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"tags",
|
||||
"all"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_by_creator",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/tags/creators/admin",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"tags",
|
||||
"creators",
|
||||
"admin"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "tag2bag",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/tags/bags?tags=driving,city&op=and",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"tags",
|
||||
"bags"
|
||||
],
|
||||
"query": [
|
||||
{
|
||||
"key": "tags",
|
||||
"value": "driving,city"
|
||||
},
|
||||
{
|
||||
"key": "op",
|
||||
"value": "and"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "bags",
|
||||
"item": [
|
||||
{
|
||||
"name": "All bags",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/all",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"all"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "bag2topic",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"001.bag\",\"002.bag\"]\n",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/topics",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"topics"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "bag2fst node",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"001.bag\",\"002.bag\"]\n",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/fst/nodes",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"fst",
|
||||
"nodes"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "bag2tags",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"001.bag\",\"002.bag\",\"test21.bag\"]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/tags",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"tags"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "bags_life_cycle",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"bag_names\":[\"001.bag\",\"002.bag\"]}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/life_cycle",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"life_cycle"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_pangu",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"names\":[\"p1.bag\",\"p2.bag\"]}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/pangu",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"pangu"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_minerva",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"session_ids\":[\"p1.bag\",\"p2.bag\"]}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/minerva",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"minerva"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "search pangu",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"name\":\"001.bag\"}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/search/pangu",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"search",
|
||||
"pangu"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_pangu_detail",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"p1.bag\",\"p2.bag\",\"test21.bag\"]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/pangu/detail",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"pangu",
|
||||
"detail"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_minerva_detail",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"p1.bag\",\"p2.bag\"]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/bags/minerva/detail",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"bags",
|
||||
"minerva",
|
||||
"detail"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "topics",
|
||||
"item": [
|
||||
{
|
||||
"name": "All topics",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/topics/all",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"topics",
|
||||
"all"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "topic2bag",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/topics/bags/topic1",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"topics",
|
||||
"bags",
|
||||
"topic1"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "geometry",
|
||||
"item": [
|
||||
{
|
||||
"name": "rosbags",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\"rosbag_names\":[\"001.bag\"]}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/geometry",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"geometry"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "fst",
|
||||
"item": [
|
||||
{
|
||||
"name": "get_by_bags_fst_node",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"fst1\",\"fst1_fst2_fst3\",\"fst1_fst2\",\"fst1_fst4\"]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/bags/nodes",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"bags",
|
||||
"nodes"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "add_bag_to_fst",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\n {\n \"bag_name\": \"test21.bag\",\n \"nodes\": [\"fst1\", \"fst1_fst2\", \"fst1_fst2_fst3\", \"\"],\n \"start_time\": 5,\n \"end_time\": 15,\n \"comments\":\"test\",\n \"tags\":[\"driving\",\"city\",\"highway\"]\n }\n]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/bags/update",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"bags",
|
||||
"update"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "update_fst_node",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "{\n \"name\": \"fst-qqq-123\",\n \"parent_name\":\"fst1\"\n}",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/update",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"update"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "delete_fst_nold",
|
||||
"request": {
|
||||
"method": "DELETE",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\n {\n \"bag_name\": \"test21.bag\",\n \"nodes\": [\"fst1\", \"fst1_fst2\", \"fst1_fst2_fst3\", \"\"],\n \"start_time\": 5,\n \"end_time\": 15,\n \"comments\":\"test\",\n \"tags\":[\"driving\",\"city\",\"highway\"]\n }\n]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/fst-qqq-123",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"fst-qqq-123"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_by_bags_fst_path",
|
||||
"protocolProfileBehavior": {
|
||||
"disableBodyPruning": true
|
||||
},
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"body": {
|
||||
"mode": "raw",
|
||||
"raw": "[\"fst1\",\"fst2\"]",
|
||||
"options": {
|
||||
"raw": {
|
||||
"language": "json"
|
||||
}
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/bags/path/fst1",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"bags",
|
||||
"path",
|
||||
"fst1"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_fst_detail_by_name",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/fst1_fst2_fst3",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"fst1_fst2_fst3"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "print_tree",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/print_tree",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"print_tree"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
},
|
||||
{
|
||||
"name": "get_all_linked_bags",
|
||||
"request": {
|
||||
"method": "GET",
|
||||
"header": [],
|
||||
"url": {
|
||||
"raw": "{{baseURL}}/api/fst/baglist",
|
||||
"host": [
|
||||
"{{baseURL}}"
|
||||
],
|
||||
"path": [
|
||||
"api",
|
||||
"fst",
|
||||
"baglist"
|
||||
]
|
||||
}
|
||||
},
|
||||
"response": []
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"variable": [
|
||||
{
|
||||
"key": "baseURL",
|
||||
"value": "http://127.0.0.1:5232",
|
||||
"type": "default"
|
||||
}
|
||||
]
|
||||
}
|
||||
55
fst_data_pipeline/apps/root_db_api/pyproject.toml
Normal file
55
fst_data_pipeline/apps/root_db_api/pyproject.toml
Normal file
@@ -0,0 +1,55 @@
|
||||
[project]
|
||||
name = "root_db_api"
|
||||
version = "0.5.0"
|
||||
description = "A core API service for database operations."
|
||||
readme = "README.md"
|
||||
authors = [{ name = "Cheng Li", email = "tbd@tbd.com" }]
|
||||
requires-python = ">=3.12"
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 核心依赖项 (Direct Dependencies Only)
|
||||
# ----------------------------------------------------
|
||||
dependencies = [
|
||||
# Web 框架 / 服务
|
||||
"flask>=3.1.1",
|
||||
"flasgger==0.9.7b2",
|
||||
"geoalchemy2==0.17.1",
|
||||
"shapely>=2.1.1",
|
||||
# 实用工具 / 缓存 / 监控
|
||||
"numpy==2.3.1",
|
||||
"folium==0.20.0",
|
||||
"tenacity==9.1.2",
|
||||
"tqdm==4.67.1",
|
||||
"pytz==2025.2",
|
||||
"prometheus-client==0.22.1",
|
||||
"flask-caching>=2.3.1",
|
||||
"redis>=6.4.0",
|
||||
# 配置 / 外部服务
|
||||
"cos-python-sdk-v5==1.9.37",
|
||||
"python-dotenv==1.1.1",
|
||||
"requests==2.32.4",
|
||||
"pyyaml==6.0.2",
|
||||
"pydantic==2.11.7",
|
||||
"sqlalchemy==2.0.41",
|
||||
"psycopg2-binary==2.9.10",
|
||||
"openpyxl>=3.1.0",
|
||||
"gunicorn>=23.0.0",
|
||||
"pytest==8.4.1",
|
||||
"pytest-cov==6.2.0",
|
||||
"pytest-mock==3.14.0",
|
||||
"flask-sqlalchemy>=3.1.1",
|
||||
]
|
||||
# ----------------------------------------------------
|
||||
# 标准构建系统配置（使用 Setuptools）
|
||||
# ----------------------------------------------------
|
||||
[build-system]
|
||||
requires = ["setuptools>=61.0.0"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
|
||||
# ----------------------------------------------------
|
||||
# uv 工具配置（用于 Workspaces 或 PyPI 镜像）
|
||||
# ----------------------------------------------------
|
||||
[[tool.uv.index]]
|
||||
url = "https://pypi.mirrors.ustc.edu.cn/simple"
|
||||
default = true
|
||||
0
fst_data_pipeline/apps/root_db_api/src/__init__.py
Normal file
0
fst_data_pipeline/apps/root_db_api/src/__init__.py
Normal file
23
fst_data_pipeline/apps/root_db_api/src/api/__init__.py
Normal file
23
fst_data_pipeline/apps/root_db_api/src/api/__init__.py
Normal file
@@ -0,0 +1,23 @@
|
||||
from flask import Blueprint
|
||||
|
||||
from .bags import bp as bags_bp
|
||||
from .fst import bp as fst_bp
|
||||
from .geometry import bp as geometry_bp
|
||||
from .gt import bp as gt_bp
|
||||
from .projects import bp as projects_bp
|
||||
from .recompute import bp as recompute_bp
|
||||
from .tags import bp as tags_bp
|
||||
from .topics import bp as topics_bp
|
||||
from .versions import bp as versions_bp
|
||||
|
||||
api_bp = Blueprint("api", __name__)
|
||||
|
||||
api_bp.register_blueprint(bags_bp, url_prefix="/bags")
|
||||
api_bp.register_blueprint(fst_bp, url_prefix="/fst")
|
||||
api_bp.register_blueprint(projects_bp, url_prefix="/projects")
|
||||
api_bp.register_blueprint(tags_bp, url_prefix="/tags")
|
||||
api_bp.register_blueprint(topics_bp, url_prefix="/topics")
|
||||
api_bp.register_blueprint(geometry_bp, url_prefix="/geometry")
|
||||
api_bp.register_blueprint(gt_bp, url_prefix="/gt")
|
||||
api_bp.register_blueprint(recompute_bp, url_prefix="/recompute")
|
||||
api_bp.register_blueprint(versions_bp, url_prefix="/versions")
|
||||
1723
fst_data_pipeline/apps/root_db_api/src/api/bags.py
Normal file
1723
fst_data_pipeline/apps/root_db_api/src/api/bags.py
Normal file
File diff suppressed because it is too large
Load Diff
1295
fst_data_pipeline/apps/root_db_api/src/api/fst.py
Normal file
1295
fst_data_pipeline/apps/root_db_api/src/api/fst.py
Normal file
File diff suppressed because it is too large
Load Diff
97
fst_data_pipeline/apps/root_db_api/src/api/geometry.py
Normal file
97
fst_data_pipeline/apps/root_db_api/src/api/geometry.py
Normal file
@@ -0,0 +1,97 @@
|
||||
from flasgger import swag_from
|
||||
from flask import Blueprint, jsonify, request
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.sql import text
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("geometry", __name__, url_prefix="/geometry")
|
||||
|
||||
|
||||
# 根据多个 rosbag 名称获取几何信息
|
||||
@bp.route("/", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["几何数据管理"],
|
||||
"summary": "根据rosbag名称获取几何信息",
|
||||
"description": "根据多个rosbag名称查询对应的几何轨迹信息",
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["rosbag_names"],
|
||||
"properties": {
|
||||
"rosbag_names": {
|
||||
"type": "array",
|
||||
"description": "要查询的rosbag名称列表",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"example": "bag1.bag",
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "返回指定rosbag名称的几何轨迹信息",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "array",
|
||||
"minItems": 3,
|
||||
"maxItems": 3,
|
||||
"items": {"type": "number"},
|
||||
"example": [12.345678, 12.345678, 0.0],
|
||||
"description": "经度、纬度、高度坐标",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {"description": "缺少rosbag_names参数"},
|
||||
"404": {"description": "未找到指定名称的rosbag"},
|
||||
},
|
||||
})
|
||||
def get_geometry_info_by_rosbags():
|
||||
"""Get geometry info by multiple rosbag names""" # 根据多个 rosbag 名称获取几何信息
|
||||
db: Session = SessionLocal()
|
||||
rosbag_names = request.json.get("rosbag_names", [])
|
||||
|
||||
try:
|
||||
if not rosbag_names:
|
||||
return jsonify({}), 400 # 如果没有提供rosbag名称，返回400错误
|
||||
|
||||
# 查询多个rosbag名称对应的几何信息
|
||||
sql = text("""
|
||||
SELECT rosbag_name,
|
||||
ARRAY(
|
||||
SELECT ARRAY[
|
||||
ST_X(pt.geom), ST_Y(pt.geom),
|
||||
COALESCE(ST_Z(pt.geom), 0.0) ]
|
||||
FROM unnest(gnss_downsampled_points) AS pt(geom)
|
||||
) AS lon_lat_alt
|
||||
FROM geometry_info
|
||||
WHERE rosbag_name IN :rosbag_names
|
||||
""")
|
||||
|
||||
# 使用execute并传入多个rosbag名称
|
||||
result = db.execute(sql, {"rosbag_names": tuple(rosbag_names)}).fetchall()
|
||||
|
||||
if not result:
|
||||
return jsonify({}), 404 # 如果未找到结果，返回404
|
||||
|
||||
# 构建以键值形式返回的结果
|
||||
response = {row.rosbag_name: row.lon_lat_alt for row in result}
|
||||
return jsonify(response)
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
403
fst_data_pipeline/apps/root_db_api/src/api/gt.py
Normal file
403
fst_data_pipeline/apps/root_db_api/src/api/gt.py
Normal file
@@ -0,0 +1,403 @@
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from flasgger import swag_from
|
||||
from flask import Blueprint, jsonify, request
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import (
|
||||
get_all_gt_types,
|
||||
get_bags_by_gt_name,
|
||||
link_bags_to_gt_with_stat,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("gt", __name__, url_prefix="/gt")
|
||||
|
||||
|
||||
# ------------------------- 1. 查询所有 GT 种类 -------------------------
|
||||
@bp.route("/types", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["真值数据管理"],
|
||||
"summary": "获取所有 GT 种类",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "GT 类型列表",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"name": {"type": "string"},
|
||||
"type": {"type": "string"},
|
||||
"path": {"type": "string"},
|
||||
"comment": {"type": "string"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
})
|
||||
def list_gt_types():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
data = get_all_gt_types(db)
|
||||
finally:
|
||||
db.close()
|
||||
return jsonify(data)
|
||||
|
||||
|
||||
# ------------------------- 2. 根据 GT 名称查询关联的 bag -------------------------
|
||||
@bp.route("/<string:gt_name>/bags", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["真值数据管理"],
|
||||
"summary": "根据 GT 名称获取关联的 bag 列表",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "gt_name",
|
||||
"in": "path",
|
||||
"type": "string",
|
||||
"required": True,
|
||||
"description": "GT 唯一名称",
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "bag 列表",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"name": {"type": "string"},
|
||||
"tile_id": {"type": "string"},
|
||||
"is_decoded": {"type": "boolean"},
|
||||
"project_id": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"404": {
|
||||
"description": "GT 不存在",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"error": {"type": "string", "example": "GT not found"},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def list_bags_by_gt_name(gt_name: str):
|
||||
db = SessionLocal()
|
||||
try:
|
||||
bags = get_bags_by_gt_name(db, gt_name)
|
||||
if bags is None:
|
||||
return jsonify({"error": "GT not found"}), 404
|
||||
finally:
|
||||
db.close()
|
||||
return jsonify(bags)
|
||||
|
||||
|
||||
# ------------------------- 3. 批量关联 bag 到指定 GT（纯路径+body）-------------------------
|
||||
@bp.route("/<string:gt_name>/bags", methods=["POST"])
|
||||
@swag_from({
|
||||
"openapi": "3.0.0",
|
||||
"tags": ["真值数据管理"],
|
||||
"summary": "批量将 bag 列表关联到指定 GT",
|
||||
"description": "路径传入 GT 名称，body 传入 bag 列表，完成关联后返回统计结果。",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "gt_name",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "GT 唯一名称",
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["bag_names"],
|
||||
"properties": {
|
||||
"bag_names": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "要关联的 bag 名列表",
|
||||
"example": ["a.bag", "b.bag"],
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "关联完成",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"gt_name": {"type": "string"},
|
||||
"total": {"type": "integer"},
|
||||
"succeeded": {"type": "integer"},
|
||||
"skipped": {"type": "integer"},
|
||||
"failed": {"type": "integer"},
|
||||
"details": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"bag_name": {"type": "string"},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"success",
|
||||
"skipped",
|
||||
"failed",
|
||||
],
|
||||
},
|
||||
"message": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {"description": "参数缺失或空列表"},
|
||||
"404": {"description": "GT 不存在"},
|
||||
},
|
||||
})
|
||||
def batch_link_bags_to_gt(gt_name: str):
|
||||
"""批量给 bag 关联指定 GT（仅 gt_name + body）"""
|
||||
body = request.get_json(silent=True)
|
||||
if not body:
|
||||
return jsonify({"error": "JSON body required"}), 400
|
||||
|
||||
bag_names = body.get("bag_names")
|
||||
if not isinstance(bag_names, list) or len(bag_names) == 0:
|
||||
return jsonify({
|
||||
"error": "bag_names(list) is required and cannot be empty"
|
||||
}), 400
|
||||
db = SessionLocal()
|
||||
try:
|
||||
# 复用刚才的 service，去掉 creator 参数
|
||||
stat = link_bags_to_gt_with_stat(db, gt_name, bag_names)
|
||||
if stat is None:
|
||||
return jsonify({"error": "GT not found"}), 404
|
||||
return jsonify(stat), 201
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# ---------- 1. 任务类型列表 ----------
|
||||
@bp.route("/tasks/types", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["真值数据管理"],
|
||||
"summary": "获取任务类型列表",
|
||||
"parameters": [
|
||||
{"name": "repo", "in": "query", "type": "string", "default": "repo"},
|
||||
{"name": "branch", "in": "query", "type": "string", "default": "main"},
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "成功",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"task_types": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
"branch": {"type": "string"},
|
||||
"repo": {"type": "string"},
|
||||
"total_count": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"404": {
|
||||
"description": "分支不存在",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
"error": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {"type": "string"},
|
||||
"message": {"type": "string"},
|
||||
"details": {"type": "object"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def list_task_types():
|
||||
repo = request.args.get("repo", "repo")
|
||||
branch = request.args.get("branch", "main")
|
||||
|
||||
# ------ mock 逻辑 ------
|
||||
if branch == "non-existent-branch":
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": {
|
||||
"code": "BRANCH_NOT_FOUND",
|
||||
"message": "指定的分支不存在",
|
||||
"details": {"branch": branch, "repo": repo},
|
||||
},
|
||||
}), 404
|
||||
|
||||
mock_types = [
|
||||
"tsa",
|
||||
"lane_change",
|
||||
]
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"data": {
|
||||
"task_types": mock_types,
|
||||
"branch": branch,
|
||||
"repo": repo,
|
||||
"total_count": len(mock_types),
|
||||
},
|
||||
}), 200
|
||||
|
||||
|
||||
# ---------- 2. 任务真值路径 ----------
|
||||
@bp.route("/tasks/<string:task_type>/truth-path", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["真值数据管理"],
|
||||
"summary": "根据任务类型获取真值路径",
|
||||
"parameters": [
|
||||
{"name": "task_type", "in": "path", "required": True, "type": "string"},
|
||||
{"name": "repo", "in": "query", "type": "string", "default": "repo"},
|
||||
{"name": "branch", "in": "query", "type": "string", "default": "main"},
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "成功",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"task_type": {"type": "string"},
|
||||
"truth_path": {"type": "string"},
|
||||
"branch": {"type": "string"},
|
||||
"repo": {"type": "string"},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file_size": {"type": "integer"},
|
||||
"last_modified": {"type": "string"},
|
||||
"format": {"type": "string"},
|
||||
"encoding": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"404": {
|
||||
"description": "任务类型不存在",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
"error": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {"type": "string"},
|
||||
"message": {"type": "string"},
|
||||
"details": {"type": "object"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def task_truth_path(task_type: str):
|
||||
repo = request.args.get("repo", "repo")
|
||||
branch = request.args.get("branch", "main")
|
||||
|
||||
# ------ mock 逻辑 ------
|
||||
valid_tasks = [
|
||||
"tsa",
|
||||
"lane_change",
|
||||
]
|
||||
if task_type not in valid_tasks:
|
||||
return jsonify({
|
||||
"success": False,
|
||||
"error": {
|
||||
"code": "TASK_TYPE_NOT_FOUND",
|
||||
"message": "指定的任务类型不存在",
|
||||
"details": {"task_type": task_type, "branch": branch, "repo": repo},
|
||||
},
|
||||
}), 404
|
||||
|
||||
mock_path = f"/data1/MB/shared/gt/{task_type}/"
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"data": {
|
||||
"task_type": task_type,
|
||||
"truth_path": mock_path,
|
||||
"branch": branch,
|
||||
"repo": repo,
|
||||
"metadata": {
|
||||
"file_size": 2048576,
|
||||
"last_modified": "2024-12-01T10:30:00Z",
|
||||
"format": "json",
|
||||
"encoding": "utf-8",
|
||||
},
|
||||
},
|
||||
}), 200
|
||||
58
fst_data_pipeline/apps/root_db_api/src/api/projects.py
Normal file
58
fst_data_pipeline/apps/root_db_api/src/api/projects.py
Normal file
@@ -0,0 +1,58 @@
|
||||
from flasgger import Swagger, swag_from
|
||||
from flask import Blueprint, jsonify
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import get_projects
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("projects", __name__, url_prefix="/projects")
|
||||
|
||||
# Initialize Flasgger
|
||||
swagger = Swagger()
|
||||
|
||||
|
||||
# 获取所有项目
|
||||
@bp.route("/all", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["项目管理"],
|
||||
"summary": "获取所有项目",
|
||||
"description": "获取系统中所有项目的列表信息",
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "项目列表",
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"example": 1,
|
||||
"description": "项目ID",
|
||||
},
|
||||
"name": {
|
||||
"type": "string",
|
||||
"example": "自动驾驶数据集",
|
||||
"description": "项目名称",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
500: {
|
||||
"description": "服务器内部错误",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"error": {"type": "string", "example": "发生错误"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def read_projects():
|
||||
"""Get all projects"""
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
projects = get_projects(db) # 现在返回 list[dict]
|
||||
finally:
|
||||
db.close()
|
||||
return jsonify(projects)
|
||||
244
fst_data_pipeline/apps/root_db_api/src/api/recompute.py
Normal file
244
fst_data_pipeline/apps/root_db_api/src/api/recompute.py
Normal file
@@ -0,0 +1,244 @@
|
||||
from flasgger import swag_from
|
||||
from flask import Blueprint, request, jsonify
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import (
|
||||
get_bag_recompute_versions,
|
||||
get_storage_paths_batch,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("recompute", __name__, url_prefix="/recompute")
|
||||
|
||||
|
||||
# 接口1: 批量获取指定bag列表的所有recompute版本
|
||||
@bp.route("/versions", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["重计算管理"],
|
||||
"summary": "批量获取bag的重计算版本",
|
||||
"description": "根据bag名称列表，返回每个bag对应的所有重计算版本",
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["bag_names"],
|
||||
"properties": {
|
||||
"bag_names": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "bag名称列表",
|
||||
"example": ["bag1.bag", "bag2.bag", "bag3.bag"],
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Bag versions mapping",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
"example": {
|
||||
"bag1.bag": ["v1.0", "v1.1", "v2.0"],
|
||||
"bag2.bag": ["v1.0", "v2.0"],
|
||||
"bag3.bag": [],
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad request",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"error": {"type": "string"}},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"error": {"type": "string"}},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def get_bag_versions():
|
||||
"""接口1: 给定指定的rosbag name的列表，返回每个包对应的recompute的全部版本"""
|
||||
try:
|
||||
data = request.get_json()
|
||||
if not data or "bag_names" not in data:
|
||||
return jsonify({"error": "bag_names list required"}), 400
|
||||
|
||||
bag_names = data["bag_names"]
|
||||
if not isinstance(bag_names, list) or len(bag_names) == 0:
|
||||
return jsonify({"error": "bag_names must be a non-empty list"}), 400
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
bag_versions = get_bag_recompute_versions(db, bag_names)
|
||||
return jsonify(bag_versions), 200
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
|
||||
# 接口2: 批量获取指定bag和版本的文件位置
|
||||
@bp.route("/storage", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["重计算管理"],
|
||||
"summary": "批量获取bag和版本的存储路径",
|
||||
"description": "根据bag名称和重计算版本，返回对应的文件存储位置",
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["requests"],
|
||||
"properties": {
|
||||
"requests": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["bag_name", "recompute_version"],
|
||||
"properties": {
|
||||
"bag_name": {
|
||||
"type": "string",
|
||||
"description": "Bag name",
|
||||
},
|
||||
"recompute_version": {
|
||||
"type": "string",
|
||||
"description": "Recompute version",
|
||||
},
|
||||
},
|
||||
},
|
||||
"description": "List of bag name and version pairs",
|
||||
"example": [
|
||||
{
|
||||
"bag_name": "bag1.bag",
|
||||
"recompute_version": "v1.0",
|
||||
},
|
||||
{
|
||||
"bag_name": "bag2.bag",
|
||||
"recompute_version": "v1.1",
|
||||
},
|
||||
],
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Storage paths for each bag and version pair",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"bag_name": {"type": "string"},
|
||||
"recompute_version": {"type": "string"},
|
||||
"results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"result_id": {"type": "integer"},
|
||||
"bag_name": {"type": "string"},
|
||||
"recompute_version": {"type": "string"},
|
||||
"result_type": {"type": "string"},
|
||||
"storage_path": {"type": "string"},
|
||||
"status": {"type": "string"},
|
||||
"created_time": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
},
|
||||
"reserved_json": {"type": "object"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad request",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"error": {"type": "string"}},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"error": {"type": "string"}},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def get_storage_paths():
|
||||
"""接口2: 给定指定的rosbag name和recompute版本列表，批量查询数据库返回对应的文件位置"""
|
||||
try:
|
||||
data = request.get_json()
|
||||
if not data or "requests" not in data:
|
||||
return jsonify({"error": "requests list required"}), 400
|
||||
|
||||
requests = data["requests"]
|
||||
if not isinstance(requests, list) or len(requests) == 0:
|
||||
return jsonify({"error": "requests must be a non-empty list"}), 400
|
||||
|
||||
# 验证每个请求的格式
|
||||
for req in requests:
|
||||
if (
|
||||
not isinstance(req, dict)
|
||||
or "bag_name" not in req
|
||||
or "recompute_version" not in req
|
||||
):
|
||||
return jsonify({
|
||||
"error": "Each request must contain bag_name and recompute_version"
|
||||
}), 400
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
results = get_storage_paths_batch(db, requests)
|
||||
return jsonify(results), 200
|
||||
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
274
fst_data_pipeline/apps/root_db_api/src/api/tags.py
Normal file
274
fst_data_pipeline/apps/root_db_api/src/api/tags.py
Normal file
@@ -0,0 +1,274 @@
|
||||
from flasgger import Swagger, swag_from
|
||||
from flask import Blueprint, jsonify, request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import (
|
||||
get_tags,
|
||||
get_tags_by_creator,
|
||||
query_bags_by_tags,
|
||||
apply_reserved_tag_with_creator,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("tags", __name__, url_prefix="/tags")
|
||||
|
||||
# Initialize Flasgger
|
||||
swagger = Swagger()
|
||||
|
||||
|
||||
# Get all tags
|
||||
@bp.route("/all", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["标签管理"],
|
||||
"summary": "获取所有标签",
|
||||
"description": "获取所有标签，包含名称、类型和创建者信息",
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "标签列表",
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"example": "数据质量",
|
||||
"description": "标签名称",
|
||||
},
|
||||
"type": {
|
||||
"type": "string",
|
||||
"example": "质量",
|
||||
"description": "标签类型",
|
||||
},
|
||||
"creator": {
|
||||
"type": "string",
|
||||
"example": "admin@example.com",
|
||||
"description": "创建者",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
})
|
||||
def read_tags():
|
||||
"""Fetch all tags""" # Get all tags
|
||||
db: Session = SessionLocal()
|
||||
tags = get_tags(db)
|
||||
db.close()
|
||||
return jsonify(tags)
|
||||
|
||||
|
||||
@bp.route("/bags", methods=["GET"])
|
||||
@swag_from({
|
||||
"openapi": "3.0.0",
|
||||
"tags": ["标签管理"],
|
||||
"summary": "根据标签名称获取bag文件",
|
||||
"description": "支持交集(and)或并集(or)查询rosbag",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "tags",
|
||||
"in": "query",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "逗号分隔的标签名，如 'A,B,C'",
|
||||
},
|
||||
{
|
||||
"name": "op",
|
||||
"in": "query",
|
||||
"required": False,
|
||||
"schema": {"type": "string", "enum": ["and", "or"], "default": "and"},
|
||||
"description": "and=交集，or=并集",
|
||||
},
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "bag文件列表",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"bags": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"name": {"type": "string"},
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {"description": "参数错误"},
|
||||
},
|
||||
})
|
||||
def get_bags_by_tags():
|
||||
tag_str = request.args.get("tags", "").strip()
|
||||
op = request.args.get("op", "and").strip().lower()
|
||||
|
||||
if not tag_str:
|
||||
return jsonify({"bags": []}), 400
|
||||
tag_list = [t.strip() for t in tag_str.split(",") if t.strip()]
|
||||
if op not in {"and", "or"}:
|
||||
return jsonify({"bags": []}), 400
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
bags = query_bags_by_tags(db, tag_list=tag_list, op=op)
|
||||
return jsonify({"bags": bags})
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
# Get tags by creator
|
||||
@bp.route("/creators/<string:creator>", methods=["GET"])
|
||||
@swag_from({
|
||||
"openapi": "3.0.0", # 新增
|
||||
"tags": ["标签管理"],
|
||||
"summary": "根据创建者获取标签",
|
||||
"description": "获取指定创建者创建的所有标签",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "creator",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "创建者标识符（如用户名或邮箱）",
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "List of tag names",
|
||||
"content": { # 新增：用 content 包装
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tags": {
|
||||
"type": "array",
|
||||
"items": {"type": "string", "example": "Sales"},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"404": {"description": "No tags found for the specified creator"},
|
||||
},
|
||||
})
|
||||
def get_tags_by_creator_name(creator):
|
||||
"""Fetch tags by creator""" # Get tags by creator
|
||||
db: Session = SessionLocal()
|
||||
tags = get_tags_by_creator(db, creator=creator)
|
||||
db.close()
|
||||
return jsonify({
|
||||
"tags": [t["name"] for t in tags]
|
||||
}) # Changed from "creators" to "tags" for clarity
|
||||
|
||||
|
||||
# --------------- 5. 批量打标签（含 creator 参数）----------------
|
||||
@bp.route("/<string:tag_name>/bags", methods=["POST"])
|
||||
@swag_from({
|
||||
"openapi": "3.0.0",
|
||||
"tags": ["标签管理"],
|
||||
"summary": "批量给 bag 打指定标签（含 creator）",
|
||||
"description": "路径传入标签名，body 传入 bag 列表，query 传入 creator，完成打标签后返回统计结果。",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "tag_name",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "标签名称",
|
||||
},
|
||||
{
|
||||
"name": "creator",
|
||||
"in": "query",
|
||||
"required": False,
|
||||
"schema": {"type": "string", "default": "system"},
|
||||
"description": "操作人标识（如邮箱）",
|
||||
},
|
||||
],
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["bag_names"],
|
||||
"properties": {
|
||||
"bag_names": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "待打标签的 bag 名列表",
|
||||
"example": ["a.bag", "b.bag"],
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "打标签完成",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"tag_name": {"type": "string"},
|
||||
"creator": {"type": "string"},
|
||||
"total": {"type": "integer"},
|
||||
"succeeded": {"type": "integer"},
|
||||
"skipped": {"type": "integer"},
|
||||
"failed": {"type": "integer"},
|
||||
"details": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"bag_name": {"type": "string"},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"success",
|
||||
"skipped",
|
||||
"failed",
|
||||
],
|
||||
},
|
||||
"message": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"400": {"description": "参数缺失或空列表"},
|
||||
},
|
||||
})
|
||||
def batch_tag_bags_with_creator(tag_name: str):
|
||||
"""批量给 bag 打指定标签（含 creator 参数）"""
|
||||
creator = request.args.get("creator", "system").strip()
|
||||
body = request.get_json(silent=True)
|
||||
if not body:
|
||||
return jsonify({"error": "JSON body required"}), 400
|
||||
|
||||
bag_names = body.get("bag_names")
|
||||
if not isinstance(bag_names, list) or len(bag_names) == 0:
|
||||
return jsonify({
|
||||
"error": "bag_names(list) is required and cannot be empty"
|
||||
}), 400
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
stat = apply_reserved_tag_with_creator(db, tag_name, bag_names, creator)
|
||||
return jsonify(stat), 201
|
||||
finally:
|
||||
db.close()
|
||||
94
fst_data_pipeline/apps/root_db_api/src/api/topics.py
Normal file
94
fst_data_pipeline/apps/root_db_api/src/api/topics.py
Normal file
@@ -0,0 +1,94 @@
|
||||
from flasgger import Swagger, swag_from
|
||||
from flask import jsonify, Blueprint
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import (
|
||||
get_all_topics,
|
||||
get_bags_by_topic_name,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("topics", __name__, url_prefix="/topics")
|
||||
|
||||
# Initialize Flasgger
|
||||
swagger = Swagger()
|
||||
|
||||
|
||||
# Get all topics
|
||||
@bp.route("/all", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["主题管理"],
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "获取所有主题列表",
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"example": "/camera/front/image_raw",
|
||||
"description": "主题名称",
|
||||
},
|
||||
"type": {
|
||||
"type": "string",
|
||||
"example": "sensor_msgs/Image",
|
||||
"description": "消息类型",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
})
|
||||
def read_topics():
|
||||
"""Fetch all topics""" # Get all topics
|
||||
db: Session = SessionLocal()
|
||||
topics = get_all_topics(db)
|
||||
db.close()
|
||||
return jsonify(topics)
|
||||
|
||||
|
||||
# Get bags by topic name
|
||||
@bp.route("/bags/<path:topic_name>", methods=["GET"])
|
||||
@swag_from({
|
||||
"openapi": "3.0.0", # 新增
|
||||
"tags": ["主题管理"],
|
||||
"summary": "Fetch bags by topic name",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "topic_name",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
"description": "The name of the topic",
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Get bags related to the specified topic",
|
||||
"content": { # 新增：用 content 包装
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "array",
|
||||
"items": {"type": "string", "example": "Bag Name"},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"404": {"description": "No related topic found"},
|
||||
},
|
||||
})
|
||||
def get_bags_by_topic(topic_name):
|
||||
"""Fetch bags by topic name""" # Get bags by topic name
|
||||
db: Session = SessionLocal()
|
||||
topic_name = "/" + topic_name
|
||||
bags = get_bags_by_topic_name(db, topic_name=topic_name)
|
||||
db.close()
|
||||
return jsonify({"name": [b["name"] for b in bags]})
|
||||
618
fst_data_pipeline/apps/root_db_api/src/api/versions.py
Normal file
618
fst_data_pipeline/apps/root_db_api/src/api/versions.py
Normal file
@@ -0,0 +1,618 @@
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
import threading
|
||||
from typing import Any, Dict, List
|
||||
import uuid
|
||||
|
||||
from flasgger import swag_from
|
||||
from flask import Blueprint, Response, jsonify, request
|
||||
from sqlalchemy.orm import Session
|
||||
from openpyxl import Workbook
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.service import (
|
||||
create_version,
|
||||
list_versions,
|
||||
update_version_status,
|
||||
save_version_snapshot,
|
||||
get_version_snapshot,
|
||||
delete_version,
|
||||
get_fst_stash,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.feishu_bitable_sdk import (
|
||||
create_bitable_for_version,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.feishu_api_constants import (
|
||||
ENABLE_FEISHU_SYNC,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.models import VersionList
|
||||
from fst_data_pipeline.apps.root_db_api.src.db.connection import SessionLocal
|
||||
|
||||
bp = Blueprint("versions", __name__, url_prefix="/versions")
|
||||
|
||||
ACTIVATE_TASKS: Dict[str, Dict[str, Any]] = {}
|
||||
ACTIVATE_TASKS_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _activate_now_iso() -> str:
|
||||
return datetime.utcnow().isoformat()
|
||||
|
||||
|
||||
def _put_activate_task(task_id: str, task: Dict[str, Any]) -> None:
|
||||
with ACTIVATE_TASKS_LOCK:
|
||||
ACTIVATE_TASKS[task_id] = task
|
||||
|
||||
|
||||
def _patch_activate_task(task_id: str, patch: Dict[str, Any]) -> None:
|
||||
with ACTIVATE_TASKS_LOCK:
|
||||
current = dict(ACTIVATE_TASKS.get(task_id) or {})
|
||||
current.update(patch)
|
||||
current["updated_at"] = _activate_now_iso()
|
||||
ACTIVATE_TASKS[task_id] = current
|
||||
|
||||
|
||||
def _get_activate_task(task_id: str) -> Dict[str, Any]:
|
||||
with ACTIVATE_TASKS_LOCK:
|
||||
task = ACTIVATE_TASKS.get(task_id) or {}
|
||||
return dict(task)
|
||||
|
||||
|
||||
def _run_activate_task(task_id: str, version_id: int, content: Any) -> None:
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
_patch_activate_task(task_id, {"status": "running"})
|
||||
|
||||
version_obj = db.get(VersionList, version_id)
|
||||
if version_obj is None:
|
||||
_patch_activate_task(task_id, {
|
||||
"status": "failed",
|
||||
"error": "version not found",
|
||||
})
|
||||
return
|
||||
|
||||
# 当后端飞书同步开关关闭时，只生效版本，不触发外部飞书写入。
|
||||
if not ENABLE_FEISHU_SYNC:
|
||||
version_obj.feishu_sync_status = "unsynced"
|
||||
version_obj.status = "active"
|
||||
db.commit()
|
||||
|
||||
feishu_result = {
|
||||
"skipped": True,
|
||||
"reason": "feishu_sync_disabled",
|
||||
}
|
||||
_patch_activate_task(task_id, {
|
||||
"status": "completed",
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"result": {
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"status": version_obj.status,
|
||||
"feishu_sync_status": version_obj.feishu_sync_status,
|
||||
"feishu_result": feishu_result,
|
||||
},
|
||||
})
|
||||
return
|
||||
|
||||
version_obj.feishu_sync_status = "syncing"
|
||||
db.commit()
|
||||
|
||||
feishu_result = create_bitable_for_version(version_obj.version, content)
|
||||
|
||||
version_obj.feishu_sync_status = "synced"
|
||||
version_obj.status = "active"
|
||||
db.commit()
|
||||
|
||||
_patch_activate_task(task_id, {
|
||||
"status": "completed",
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"result": {
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"status": version_obj.status,
|
||||
"feishu_sync_status": version_obj.feishu_sync_status,
|
||||
"feishu_result": feishu_result,
|
||||
},
|
||||
})
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
version_obj = db.get(VersionList, version_id)
|
||||
if version_obj is not None:
|
||||
version_obj.feishu_sync_status = "failed"
|
||||
db.commit()
|
||||
_patch_activate_task(task_id, {
|
||||
"status": "failed",
|
||||
"error": str(exc),
|
||||
})
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "获取版本列表",
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "版本列表",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"version": {"type": "string"},
|
||||
"type": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"release_date": {"type": "string"},
|
||||
"created_time": {"type": "string"},
|
||||
"update_time": {"type": "string"},
|
||||
"status": {"type": "string"},
|
||||
"feishu_sync_status": {"type": "string"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
})
|
||||
def get_version_list():
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
data = list_versions(db)
|
||||
return jsonify(data)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "新增版本",
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["version"],
|
||||
"properties": {
|
||||
"version": {"type": "string", "example": "v1.0.0"},
|
||||
"type": {"type": "string", "example": "software"},
|
||||
"description": {"type": "string"},
|
||||
"release_date": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
},
|
||||
"status": {"type": "string", "example": "available"},
|
||||
"feishu_sync_status": {
|
||||
"type": "string",
|
||||
"example": "unsynced",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
200: {"description": "创建成功"},
|
||||
400: {"description": "请求参数错误"},
|
||||
},
|
||||
})
|
||||
def create_version_item():
|
||||
payload = request.get_json(silent=True) or {}
|
||||
version = payload.get("version")
|
||||
if not version:
|
||||
return jsonify({"error": "version 必填"}), 400
|
||||
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
exists = (
|
||||
db.query(VersionList.id)
|
||||
.filter(VersionList.version == version)
|
||||
.first()
|
||||
)
|
||||
if exists:
|
||||
return jsonify({"message": "version already exists"}), 409
|
||||
|
||||
obj = create_version(
|
||||
db,
|
||||
version=version,
|
||||
type=payload.get("type"),
|
||||
description=payload.get("description"),
|
||||
release_date=payload.get("release_date"),
|
||||
status=payload.get("status"),
|
||||
feishu_sync_status=payload.get("feishu_sync_status"),
|
||||
)
|
||||
db.commit()
|
||||
return jsonify({
|
||||
"id": obj.id,
|
||||
"version": obj.version,
|
||||
"type": obj.type,
|
||||
"description": obj.description,
|
||||
"release_date": obj.release_date,
|
||||
"created_time": obj.created_time,
|
||||
"update_time": obj.update_time,
|
||||
"status": obj.status,
|
||||
"feishu_sync_status": obj.feishu_sync_status,
|
||||
})
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("/export", methods=["GET"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "按版本导出 FST/Bag 快照",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "version_id",
|
||||
"in": "query",
|
||||
"required": False,
|
||||
"schema": {"type": "string"},
|
||||
"description": "版本号，对应 version_list.version；不传则导出版本列表",
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "Excel 文件",
|
||||
"content": {
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": {
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"format": "binary",
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
400: {"description": "参数错误"},
|
||||
404: {"description": "版本不存在"},
|
||||
},
|
||||
})
|
||||
def export_versions():
|
||||
version_token = request.args.get("version_id")
|
||||
if not version_token:
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
data = list_versions(db)
|
||||
finally:
|
||||
db.close()
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "Versions"
|
||||
ws.append(["Version Name", "Create Time", "Update Time", "Status"])
|
||||
for item in data:
|
||||
created = item.get("created_time")
|
||||
updated = item.get("update_time")
|
||||
created_str = created.isoformat() if isinstance(created, datetime) else str(created) if created is not None else ""
|
||||
updated_str = updated.isoformat() if isinstance(updated, datetime) else str(updated) if updated is not None else ""
|
||||
ws.append([
|
||||
item.get("version") or "",
|
||||
created_str,
|
||||
updated_str,
|
||||
item.get("status") or "",
|
||||
])
|
||||
stream = BytesIO()
|
||||
wb.save(stream)
|
||||
stream.seek(0)
|
||||
filename = f"versions_{datetime.now().strftime('%Y%m%d%H%M%S')}.xlsx"
|
||||
return Response(
|
||||
stream.getvalue(),
|
||||
mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
headers={"Content-Disposition": f"attachment; filename={filename}"},
|
||||
)
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
version_obj = (
|
||||
db.query(VersionList)
|
||||
.filter(VersionList.version == version_token)
|
||||
.first()
|
||||
)
|
||||
if version_obj is None and version_token.isdigit():
|
||||
version_obj = db.get(VersionList, int(version_token))
|
||||
if version_obj is None:
|
||||
return jsonify({"error": "版本不存在"}), 404
|
||||
version_pk = version_obj.id
|
||||
version_str = version_obj.version
|
||||
snapshot = get_version_snapshot(db, version_pk)
|
||||
if snapshot is None:
|
||||
snapshot = save_version_snapshot(db, version_pk)
|
||||
if snapshot is None:
|
||||
return jsonify({"error": "版本不存在"}), 404
|
||||
finally:
|
||||
db.close()
|
||||
tree = snapshot or []
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "FST Tree Snapshot"
|
||||
ws.append(["Version", "Node ID", "Label", "Level", "Parent Node ID"])
|
||||
|
||||
def _walk(nodes, parent_id=None):
|
||||
for node in nodes:
|
||||
node_id = node.get("id")
|
||||
label = node.get("label")
|
||||
level = node.get("level")
|
||||
ws.append([
|
||||
version_str,
|
||||
node_id or "",
|
||||
label or "",
|
||||
level if level is not None else "",
|
||||
parent_id or "",
|
||||
])
|
||||
children = node.get("children") or []
|
||||
if children:
|
||||
_walk(children, node_id or parent_id)
|
||||
|
||||
_walk(tree)
|
||||
stream = BytesIO()
|
||||
wb.save(stream)
|
||||
stream.seek(0)
|
||||
filename = f"version_{version_str}_fst_snapshot_{datetime.now().strftime('%Y%m%d%H%M%S')}.xlsx"
|
||||
return Response(
|
||||
stream.getvalue(),
|
||||
mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
headers={"Content-Disposition": f"attachment; filename={filename}"},
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/<int:version_id>/snapshot", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "生成并保存指定版本的 FST/Bag 快照",
|
||||
"responses": {
|
||||
200: {"description": "生成成功"},
|
||||
404: {"description": "版本不存在"},
|
||||
},
|
||||
})
|
||||
def create_version_snapshot(version_id: int):
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
snapshot = save_version_snapshot(db, version_id)
|
||||
if snapshot is None:
|
||||
return jsonify({"error": "版本不存在"}), 404
|
||||
return jsonify({"version_id": version_id, "snapshot": snapshot}), 200
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("/<int:version_id>/status", methods=["PATCH"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "更新版本状态与描述",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "version_id",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"type": "integer",
|
||||
"description": "版本 ID",
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": ["status"],
|
||||
"properties": {
|
||||
"status": {
|
||||
"type": "string",
|
||||
"example": "已生效",
|
||||
},
|
||||
"description": {
|
||||
"type": "string",
|
||||
"example": "版本说明，可选",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
200: {"description": "更新成功"},
|
||||
400: {"description": "请求参数错误"},
|
||||
404: {"description": "未找到版本"},
|
||||
},
|
||||
})
|
||||
def update_version_status_api(version_id: int):
|
||||
payload = request.get_json(silent=True) or {}
|
||||
status = payload.get("status")
|
||||
if not status:
|
||||
return jsonify({"error": "status 必填"}), 400
|
||||
description = payload.get("description")
|
||||
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
obj = update_version_status(
|
||||
db,
|
||||
version_id=version_id,
|
||||
status=status,
|
||||
description=description,
|
||||
)
|
||||
if obj is None:
|
||||
return jsonify({"error": "version not found"}), 404
|
||||
db.commit()
|
||||
return jsonify({
|
||||
"id": obj.id,
|
||||
"version": obj.version,
|
||||
"status": obj.status,
|
||||
"description": obj.description,
|
||||
})
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("/<int:version_id>", methods=["DELETE"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "删除指定版本",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "version_id",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "integer"},
|
||||
"description": "版本 ID（version_list.id）",
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
200: {
|
||||
"description": "删除成功",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {"type": "boolean"},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
404: {
|
||||
"description": "未找到版本",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"error": {"type": "string"},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
def delete_version_api(version_id: int):
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
ok = delete_version(db, version_id)
|
||||
if not ok:
|
||||
return jsonify({"error": "version not found"}), 404
|
||||
db.commit()
|
||||
return jsonify({"success": True}), 200
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("/stash/<int:version_id>/activate", methods=["POST"])
|
||||
@bp.route("/fst/stash/activate/<int:version_id>", methods=["POST"])
|
||||
@bp.route("/<int:version_id>/activate", methods=["POST"])
|
||||
@swag_from({
|
||||
"tags": ["versions"],
|
||||
"summary": "生效版本并创建飞书多维表格（fst/stash 命名空间）",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "version_id",
|
||||
"in": "path",
|
||||
"required": True,
|
||||
"schema": {"type": "integer"},
|
||||
"description": "版本 ID（version_list.id）",
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": False,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {
|
||||
"type": "object",
|
||||
"description": "可选，直接上传的 JSON；不传则读取 fst_stash.content",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"responses": {
|
||||
200: {"description": "生效并创建飞书多维表格成功"},
|
||||
400: {"description": "参数错误"},
|
||||
404: {"description": "版本或 JSON 不存在"},
|
||||
500: {"description": "飞书调用或服务异常"},
|
||||
},
|
||||
})
|
||||
def activate_version_api(version_id: int):
|
||||
payload = request.get_json(silent=True) or {}
|
||||
input_content = payload.get("content")
|
||||
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
version_obj = db.get(VersionList, version_id)
|
||||
if version_obj is None:
|
||||
return jsonify({"error": "version not found"}), 404
|
||||
|
||||
content = input_content
|
||||
if content is None:
|
||||
stash = get_fst_stash(db, version=version_obj.version)
|
||||
if stash is not None:
|
||||
content = stash.content
|
||||
|
||||
if content is None:
|
||||
return jsonify({"error": "no json content found for this version"}), 404
|
||||
|
||||
task_id = uuid.uuid4().hex
|
||||
now = _activate_now_iso()
|
||||
_put_activate_task(task_id, {
|
||||
"task_id": task_id,
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"status": "queued",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
})
|
||||
|
||||
threading.Thread(
|
||||
target=_run_activate_task,
|
||||
args=(task_id, version_obj.id, content),
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
return jsonify({
|
||||
"accepted": True,
|
||||
"task_id": task_id,
|
||||
"version_id": version_obj.id,
|
||||
"version": version_obj.version,
|
||||
"status": "queued",
|
||||
}), 202
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@bp.route("/stash/<int:version_id>/activate/result", methods=["GET"])
|
||||
@bp.route("/<int:version_id>/activate/result", methods=["GET"])
|
||||
def activate_version_result_api(version_id: int):
|
||||
task_id = (request.args.get("task_id") or "").strip()
|
||||
|
||||
db: Session = SessionLocal()
|
||||
try:
|
||||
version_obj = db.get(VersionList, version_id)
|
||||
if version_obj is None:
|
||||
return jsonify({"error": "version not found"}), 404
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if task_id:
|
||||
task = _get_activate_task(task_id)
|
||||
if not task:
|
||||
return jsonify({"error": "task not found", "task_id": task_id}), 404
|
||||
return jsonify(task), 200
|
||||
|
||||
with ACTIVATE_TASKS_LOCK:
|
||||
candidates = [
|
||||
dict(item)
|
||||
for item in ACTIVATE_TASKS.values()
|
||||
if isinstance(item, dict) and item.get("version_id") == version_id
|
||||
]
|
||||
|
||||
if not candidates:
|
||||
return jsonify({"error": "task not found for version", "version_id": version_id}), 404
|
||||
|
||||
candidates.sort(key=lambda x: str(x.get("updated_at") or ""), reverse=True)
|
||||
return jsonify(candidates[0]), 200
|
||||
34
fst_data_pipeline/apps/root_db_api/src/app.py
Normal file
34
fst_data_pipeline/apps/root_db_api/src/app.py
Normal file
@@ -0,0 +1,34 @@
|
||||
# run.py
|
||||
from flasgger import Swagger
|
||||
from flask import Flask, render_template
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api import api_bp
|
||||
|
||||
app = Flask(__name__, template_folder="../templates")
|
||||
|
||||
# 注册蓝图
|
||||
app.register_blueprint(api_bp, url_prefix="/api")
|
||||
|
||||
# Swagger 配置
|
||||
app.config["SWAGGER"] = {
|
||||
"title": "ROOT DB API",
|
||||
"version": "1.0.0",
|
||||
"uiversion": 3,
|
||||
"openapi": "3.0.0",
|
||||
}
|
||||
Swagger(app)
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def hello_world():
|
||||
return "Hello World!"
|
||||
|
||||
|
||||
@app.route("/data-browser")
|
||||
def data_browser():
|
||||
"""数据浏览器前端界面"""
|
||||
return render_template("data_browser.html")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(debug=False, host="0.0.0.0", port=5232)
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Shared Feishu Open API endpoints, common constants, and reusable helpers."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import requests
|
||||
|
||||
WIKI_NODES_URL_TMPL = "https://open.feishu.cn/open-apis/wiki/v2/spaces/{space_id}/nodes"
|
||||
WIKI_GET_NODE_URL = "https://open.feishu.cn/open-apis/wiki/v2/spaces/get_node"
|
||||
WIKI_NODE_URL_TMPL = "https://open.feishu.cn/open-apis/wiki/v2/spaces/{space_id}/nodes/{node_token}"
|
||||
|
||||
DOCX_BLOCKS_URL_TMPL = "https://open.feishu.cn/open-apis/docx/v1/documents/{document_id}/blocks"
|
||||
DOCX_BLOCK_URL_TMPL = "https://open.feishu.cn/open-apis/docx/v1/documents/{document_id}/blocks/{block_id}"
|
||||
DOCX_BLOCK_CHILDREN_URL_TMPL = (
|
||||
"https://open.feishu.cn/open-apis/docx/v1/documents/{document_id}/blocks/{block_id}/children"
|
||||
)
|
||||
|
||||
BITABLE_TABLES_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables"
|
||||
BITABLE_TABLE_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}"
|
||||
BITABLE_FIELDS_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/fields"
|
||||
BITABLE_FIELD_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/fields/{field_id}"
|
||||
BITABLE_RECORDS_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records"
|
||||
BITABLE_RECORD_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/{record_id}"
|
||||
BITABLE_RECORDS_BATCH_CREATE_URL_TMPL = (
|
||||
"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/batch_create"
|
||||
)
|
||||
BITABLE_RECORDS_BATCH_UPDATE_URL_TMPL = (
|
||||
"https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/records/batch_update"
|
||||
)
|
||||
BITABLE_VIEWS_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/views"
|
||||
BITABLE_VIEW_URL_TMPL = "https://open.feishu.cn/open-apis/bitable/v1/apps/{app_token}/tables/{table_id}/views/{view_id}"
|
||||
|
||||
DOC_PAGE_SIZE = 50
|
||||
WIKI_NODE_PAGE_SIZE = 50
|
||||
MAX_PAGE_LOOP = 200
|
||||
RECORD_PAGE_SIZE = 500
|
||||
RECORD_BATCH_SIZE = 200
|
||||
|
||||
TEXT_FIELD_TYPE = 1
|
||||
LINK_FIELD_TYPE = 18
|
||||
PRIMARY_FIELD_TARGET_NAME = "Name"
|
||||
PARENT_LINK_FIELD_NAME = "Parent Record Link"
|
||||
TARGET_TEXT_FIELDS = ["Name", "Scene", "Level", "Describe"]
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool = False) -> bool:
|
||||
raw = os.getenv(name)
|
||||
if raw is None:
|
||||
return default
|
||||
# Accept common true values and tolerate `tru` typo from env settings.
|
||||
return raw.strip().lower() in {"1", "true", "tru", "yes", "on"}
|
||||
|
||||
|
||||
# Backend Feishu sync master switch (default off).
|
||||
ENABLE_FEISHU_SYNC = _env_bool("ENABLE_FEISHU_SYNC", default=False)
|
||||
|
||||
|
||||
def build_auth_headers(tenant_access_token: str) -> Dict[str, str]:
|
||||
return {
|
||||
"Authorization": f"Bearer {tenant_access_token}",
|
||||
"Content-Type": "application/json; charset=utf-8",
|
||||
}
|
||||
|
||||
|
||||
def raise_for_business_error(result: Dict[str, Any], action: str) -> None:
|
||||
code = result.get("code", -1)
|
||||
if code != 0:
|
||||
msg = result.get("msg", "unknown error")
|
||||
raise RuntimeError(f"{action} failed: code={code}, msg={msg}, raw={json.dumps(result, ensure_ascii=False)}")
|
||||
|
||||
|
||||
def request_json_with_retry(
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
headers: Dict[str, str],
|
||||
action: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
payload: Optional[Dict[str, Any]] = None,
|
||||
timeout: int = 30,
|
||||
max_attempts: int = 3,
|
||||
backoff_seconds: int = 1,
|
||||
) -> Dict[str, Any]:
|
||||
last_error: Optional[str] = None
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
response = requests.request(
|
||||
method=method,
|
||||
url=url,
|
||||
headers=headers,
|
||||
params=params,
|
||||
json=payload,
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = response.json()
|
||||
raise_for_business_error(result, action)
|
||||
return result
|
||||
except Exception as exc:
|
||||
last_error = str(exc)
|
||||
|
||||
if attempt < max_attempts:
|
||||
time.sleep(backoff_seconds * attempt)
|
||||
|
||||
raise RuntimeError(f"{action} failed after retries: {last_error}")
|
||||
1768
fst_data_pipeline/apps/root_db_api/src/core/feishu_bitable_sdk.py
Normal file
1768
fst_data_pipeline/apps/root_db_api/src/core/feishu_bitable_sdk.py
Normal file
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,835 @@
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.feishu_api_constants import (
|
||||
DOC_PAGE_SIZE,
|
||||
MAX_PAGE_LOOP,
|
||||
WIKI_GET_NODE_URL,
|
||||
WIKI_NODES_URL_TMPL,
|
||||
build_auth_headers as _auth_headers,
|
||||
raise_for_business_error as _raise_for_business_error,
|
||||
request_json_with_retry as _shared_request_json_with_retry,
|
||||
)
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.feishu_doc_bitable_block_sdk import (
|
||||
ensure_bitable_block_for_document,
|
||||
sync_document_bitable_rows,
|
||||
)
|
||||
|
||||
RETRY_MAX_ATTEMPTS = 3
|
||||
RETRY_BACKOFF_SECONDS = 1
|
||||
FST_EDITOR_DOC_TITLE = "Fst_Editor"
|
||||
DOC_TREE_MAX_DEPTH = 5
|
||||
CONTENT_WRAP_KEYS = ("content", "data", "payload", "body", "result")
|
||||
|
||||
|
||||
def _request_json_with_retry(
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
headers: Dict[str, str],
|
||||
action: str,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
payload: Optional[Dict[str, Any]] = None,
|
||||
timeout: int = 30,
|
||||
) -> Dict[str, Any]:
|
||||
return _shared_request_json_with_retry(
|
||||
method=method,
|
||||
url=url,
|
||||
headers=headers,
|
||||
action=action,
|
||||
params=params,
|
||||
payload=payload,
|
||||
timeout=timeout,
|
||||
max_attempts=RETRY_MAX_ATTEMPTS,
|
||||
backoff_seconds=RETRY_BACKOFF_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def _extract_node_title(node: Dict[str, Any]) -> str:
|
||||
title = node.get("title")
|
||||
if isinstance(title, str):
|
||||
return title
|
||||
|
||||
obj_create_info = node.get("obj_create_info")
|
||||
if isinstance(obj_create_info, dict):
|
||||
inner_title = obj_create_info.get("title")
|
||||
if isinstance(inner_title, str):
|
||||
return inner_title
|
||||
inner_obj_title = obj_create_info.get("obj_title")
|
||||
if isinstance(inner_obj_title, str):
|
||||
return inner_obj_title
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_content_object(content: Any) -> Any:
|
||||
if isinstance(content, str):
|
||||
try:
|
||||
return json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
return content
|
||||
|
||||
|
||||
def _resolve_nodes_owner(parsed: Any, depth: int = 0) -> Optional[Dict[str, Any]]:
|
||||
if depth > 6:
|
||||
return None
|
||||
|
||||
if isinstance(parsed, dict):
|
||||
nodes = parsed.get("nodes")
|
||||
if isinstance(nodes, list):
|
||||
return parsed
|
||||
|
||||
for key in CONTENT_WRAP_KEYS:
|
||||
inner = parsed.get(key)
|
||||
owner = _resolve_nodes_owner(inner, depth + 1)
|
||||
if isinstance(owner, dict):
|
||||
return owner
|
||||
|
||||
if isinstance(parsed, list):
|
||||
return {"nodes": parsed}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _extract_first_level_names(content: Any) -> List[str]:
|
||||
parsed = _parse_content_object(content)
|
||||
owner = _resolve_nodes_owner(parsed)
|
||||
if not isinstance(owner, dict):
|
||||
return []
|
||||
|
||||
nodes = owner.get("nodes")
|
||||
if not isinstance(nodes, list):
|
||||
return []
|
||||
|
||||
has_tree_level = any(isinstance(item, dict) and item.get("treeLevel") is not None for item in nodes)
|
||||
|
||||
names: List[str] = []
|
||||
seen: Set[str] = set()
|
||||
for item in nodes:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if _is_deleted_node(item):
|
||||
continue
|
||||
if has_tree_level and item.get("treeLevel") != 1:
|
||||
continue
|
||||
for key in ("name", "Name", "label", "id"):
|
||||
value = item.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
text = str(value).strip()
|
||||
if text and text not in seen:
|
||||
seen.add(text)
|
||||
names.append(text)
|
||||
break
|
||||
return names
|
||||
|
||||
|
||||
def debug_extract_first_level_names(content: Any) -> Dict[str, Any]:
|
||||
"""Return diagnostic details for first-level name extraction."""
|
||||
parsed = content
|
||||
parse_error: Optional[str] = None
|
||||
if isinstance(content, str):
|
||||
try:
|
||||
parsed = json.loads(content)
|
||||
except json.JSONDecodeError as exc:
|
||||
parse_error = str(exc)
|
||||
parsed = None
|
||||
|
||||
owner = _resolve_nodes_owner(parsed)
|
||||
node_count = 0
|
||||
tree_level_counts: Dict[str, int] = {}
|
||||
root_keys: List[str] = list(parsed.keys()) if isinstance(parsed, dict) else []
|
||||
owner_keys: List[str] = list(owner.keys()) if isinstance(owner, dict) else []
|
||||
if isinstance(owner, dict):
|
||||
nodes = owner.get("nodes")
|
||||
if isinstance(nodes, list):
|
||||
node_count = len(nodes)
|
||||
for item in nodes:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
level = item.get("treeLevel")
|
||||
level_key = "null" if level is None else str(level)
|
||||
tree_level_counts[level_key] = tree_level_counts.get(level_key, 0) + 1
|
||||
|
||||
names = _extract_first_level_names(content)
|
||||
return {
|
||||
"names": names,
|
||||
"name_count": len(names),
|
||||
"node_count": node_count,
|
||||
"tree_level_counts": tree_level_counts,
|
||||
"parse_error": parse_error,
|
||||
"resolved_nodes_owner": bool(isinstance(owner, dict)),
|
||||
"root_keys": root_keys,
|
||||
"owner_keys": owner_keys,
|
||||
}
|
||||
|
||||
|
||||
def _extract_item_title(item: Dict[str, Any]) -> str:
|
||||
for key in ("title", "name", "Name", "label", "id"):
|
||||
value = item.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
text = str(value).strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _is_deleted_node(item: Dict[str, Any]) -> bool:
|
||||
deleted = item.get("deleted")
|
||||
if isinstance(deleted, bool):
|
||||
return deleted
|
||||
if isinstance(deleted, int):
|
||||
return deleted == 1
|
||||
if isinstance(deleted, str):
|
||||
return deleted.strip() in {"1", "true", "True"}
|
||||
return False
|
||||
|
||||
|
||||
def _stringify_value(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, (dict, list)):
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
return str(value)
|
||||
|
||||
|
||||
def _extract_item_describe(item: Dict[str, Any]) -> str:
|
||||
for key in ("Description", "description", "label", "name", "Name"):
|
||||
value = item.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
text = _stringify_value(value).strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_item_scene(item: Dict[str, Any]) -> str:
|
||||
return _stringify_value(item.get("scene")).strip()
|
||||
|
||||
|
||||
def _build_rows_by_first_level(content: Any) -> Dict[str, List[Dict[str, Any]]]:
|
||||
parsed = _parse_content_object(content)
|
||||
owner = _resolve_nodes_owner(parsed)
|
||||
if not isinstance(owner, dict):
|
||||
return {}
|
||||
|
||||
nodes = owner.get("nodes")
|
||||
if not isinstance(nodes, list) or not nodes:
|
||||
return {}
|
||||
|
||||
result: Dict[str, List[Dict[str, Any]]] = {}
|
||||
|
||||
# Tree payload: each level-1 node has nested children.
|
||||
if any(isinstance(node, dict) and isinstance(node.get("children"), list) for node in nodes):
|
||||
def _walk_tree(
|
||||
node: Dict[str, Any],
|
||||
*,
|
||||
root_name: str,
|
||||
parent_name: str,
|
||||
fallback_level: int,
|
||||
) -> None:
|
||||
if _is_deleted_node(node):
|
||||
return
|
||||
|
||||
name = _extract_item_title(node)
|
||||
if not name:
|
||||
return
|
||||
|
||||
level_value = node.get("treeLevel")
|
||||
level_text = _stringify_value(level_value if level_value is not None else fallback_level)
|
||||
result.setdefault(root_name, []).append(
|
||||
{
|
||||
"Name": name,
|
||||
"Scene": _extract_item_scene(node),
|
||||
"Level": level_text,
|
||||
"Describe": _extract_item_describe(node),
|
||||
"__parent_name__": parent_name,
|
||||
}
|
||||
)
|
||||
|
||||
children = node.get("children")
|
||||
if isinstance(children, list):
|
||||
for child in children:
|
||||
if isinstance(child, dict):
|
||||
_walk_tree(
|
||||
child,
|
||||
root_name=root_name,
|
||||
parent_name=name,
|
||||
fallback_level=fallback_level + 1,
|
||||
)
|
||||
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
if _is_deleted_node(node):
|
||||
continue
|
||||
root_name = _extract_item_title(node)
|
||||
if not root_name:
|
||||
continue
|
||||
_walk_tree(
|
||||
node,
|
||||
root_name=root_name,
|
||||
parent_name="",
|
||||
fallback_level=1,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
# Flat payload with treeLevel + parentId.
|
||||
id_to_item: Dict[str, Dict[str, Any]] = {}
|
||||
id_to_name: Dict[str, str] = {}
|
||||
first_level_ids: Set[str] = set()
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
if _is_deleted_node(node):
|
||||
continue
|
||||
node_id = _stringify_value(node.get("id")).strip()
|
||||
node_name = _extract_item_title(node)
|
||||
if not node_id or not node_name:
|
||||
continue
|
||||
id_to_item[node_id] = node
|
||||
id_to_name[node_id] = node_name
|
||||
if node.get("treeLevel") == 1:
|
||||
first_level_ids.add(node_id)
|
||||
|
||||
for node_id, node in id_to_item.items():
|
||||
if _is_deleted_node(node):
|
||||
continue
|
||||
current_id = node_id
|
||||
parent_id = _stringify_value(node.get("parentId")).strip()
|
||||
root_id = current_id if current_id in first_level_ids else ""
|
||||
visited: Set[str] = set()
|
||||
|
||||
while parent_id:
|
||||
if parent_id in visited:
|
||||
break
|
||||
visited.add(parent_id)
|
||||
if parent_id in first_level_ids:
|
||||
root_id = parent_id
|
||||
break
|
||||
parent_node = id_to_item.get(parent_id)
|
||||
if not isinstance(parent_node, dict):
|
||||
break
|
||||
parent_id = _stringify_value(parent_node.get("parentId")).strip()
|
||||
|
||||
if not root_id:
|
||||
continue
|
||||
|
||||
root_name = id_to_name.get(root_id)
|
||||
if not isinstance(root_name, str) or not root_name:
|
||||
continue
|
||||
|
||||
parent_id_text = _stringify_value(node.get("parentId")).strip()
|
||||
level_value = node.get("treeLevel")
|
||||
level_text = _stringify_value(level_value if level_value is not None else "")
|
||||
result.setdefault(root_name, []).append(
|
||||
{
|
||||
# Keep flat payload behavior consistent with feishu_bitable_sdk:
|
||||
# Name uses node id and parent link key uses parent id.
|
||||
"Name": node_id,
|
||||
"Scene": _extract_item_scene(node),
|
||||
"Level": level_text,
|
||||
"Describe": _extract_item_describe(node),
|
||||
"__parent_name__": parent_id_text,
|
||||
}
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _extract_doc_structure(content: Any) -> List[Dict[str, Any]]:
|
||||
parsed = _parse_content_object(content)
|
||||
owner = _resolve_nodes_owner(parsed)
|
||||
if not isinstance(owner, dict):
|
||||
return []
|
||||
|
||||
raw_nodes = owner.get("nodes")
|
||||
if not isinstance(raw_nodes, list):
|
||||
return []
|
||||
|
||||
# Flat payload with treeLevel should only create top-level docs.
|
||||
if all(isinstance(node, dict) and "children" not in node for node in raw_nodes):
|
||||
first_level_names = _extract_first_level_names(owner)
|
||||
return [{"title": title, "children": []} for title in first_level_names]
|
||||
|
||||
def _normalize_nodes(nodes: List[Any], depth: int) -> List[Dict[str, Any]]:
|
||||
if depth >= DOC_TREE_MAX_DEPTH:
|
||||
return []
|
||||
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
title = _extract_item_title(node)
|
||||
if not title:
|
||||
continue
|
||||
|
||||
children = node.get("children")
|
||||
child_docs = _normalize_nodes(children, depth + 1) if isinstance(children, list) else []
|
||||
normalized.append(
|
||||
{
|
||||
"title": title,
|
||||
"children": child_docs,
|
||||
}
|
||||
)
|
||||
return normalized
|
||||
|
||||
return _normalize_nodes(raw_nodes, 0)
|
||||
|
||||
|
||||
def get_wiki_node_info_by_token(tenant_access_token: str, token: str) -> Dict[str, Any]:
|
||||
result = _request_json_with_retry(
|
||||
"GET",
|
||||
WIKI_GET_NODE_URL,
|
||||
headers=_auth_headers(tenant_access_token),
|
||||
params={"token": token},
|
||||
action="get_wiki_node_info_by_token",
|
||||
)
|
||||
node = ((result.get("data") or {}).get("node") or {})
|
||||
if not isinstance(node, dict) or not node:
|
||||
raise RuntimeError(f"get_wiki_node_info_by_token failed: node missing, raw={json.dumps(result, ensure_ascii=False)}")
|
||||
return node
|
||||
|
||||
|
||||
def list_wiki_child_nodes(
|
||||
tenant_access_token: str,
|
||||
*,
|
||||
space_id: str,
|
||||
parent_node_token: Optional[str],
|
||||
) -> List[Dict[str, Any]]:
|
||||
url = WIKI_NODES_URL_TMPL.format(space_id=space_id)
|
||||
headers = _auth_headers(tenant_access_token)
|
||||
page_token = ""
|
||||
items: List[Dict[str, Any]] = []
|
||||
seen_tokens: Set[str] = set()
|
||||
loop_count = 0
|
||||
|
||||
while True:
|
||||
loop_count += 1
|
||||
if loop_count > MAX_PAGE_LOOP:
|
||||
raise RuntimeError("list_wiki_child_nodes exceeded max pagination loops")
|
||||
|
||||
params: Dict[str, Any] = {
|
||||
"page_size": DOC_PAGE_SIZE,
|
||||
}
|
||||
if parent_node_token:
|
||||
params["parent_node_token"] = parent_node_token
|
||||
if page_token:
|
||||
params["page_token"] = page_token
|
||||
|
||||
result = _request_json_with_retry(
|
||||
"GET",
|
||||
url,
|
||||
headers=headers,
|
||||
params=params,
|
||||
action="list_wiki_child_nodes",
|
||||
)
|
||||
data = result.get("data") or {}
|
||||
batch = data.get("items") or []
|
||||
for item in batch:
|
||||
if isinstance(item, dict):
|
||||
items.append(item)
|
||||
|
||||
if not bool(data.get("has_more")):
|
||||
break
|
||||
|
||||
next_page_token = str(data.get("page_token") or "")
|
||||
if not next_page_token or next_page_token == page_token or next_page_token in seen_tokens:
|
||||
break
|
||||
seen_tokens.add(next_page_token)
|
||||
page_token = next_page_token
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def get_all_descendant_nodes(
|
||||
tenant_access_token: str,
|
||||
*,
|
||||
space_id: str,
|
||||
parent_node_token: str,
|
||||
depth: int = 0,
|
||||
max_depth: int = 20,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Recursively list all descendant nodes under a parent node."""
|
||||
if depth > max_depth:
|
||||
raise RuntimeError("get_all_descendant_nodes exceeded max_depth")
|
||||
|
||||
descendants: List[Dict[str, Any]] = []
|
||||
child_nodes = list_wiki_child_nodes(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=parent_node_token,
|
||||
)
|
||||
|
||||
for node in child_nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
|
||||
node_with_depth = dict(node)
|
||||
node_with_depth["depth"] = depth
|
||||
node_with_depth["parent_token"] = parent_node_token
|
||||
descendants.append(node_with_depth)
|
||||
|
||||
child_token = node.get("node_token")
|
||||
if not isinstance(child_token, str) or not child_token:
|
||||
continue
|
||||
|
||||
has_child = bool(node.get("has_child", False))
|
||||
if not has_child:
|
||||
continue
|
||||
|
||||
sub_descendants = get_all_descendant_nodes(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=child_token,
|
||||
depth=depth + 1,
|
||||
max_depth=max_depth,
|
||||
)
|
||||
descendants.extend(sub_descendants)
|
||||
|
||||
return descendants
|
||||
|
||||
|
||||
def create_child_docx(
|
||||
tenant_access_token: str,
|
||||
*,
|
||||
space_id: str,
|
||||
parent_node_token: str,
|
||||
title: str,
|
||||
) -> Dict[str, Any]:
|
||||
url = WIKI_NODES_URL_TMPL.format(space_id=space_id)
|
||||
payload = {
|
||||
"obj_type": "docx",
|
||||
"node_type": "origin",
|
||||
"parent_node_token": parent_node_token,
|
||||
"title": title,
|
||||
}
|
||||
result = _request_json_with_retry(
|
||||
"POST",
|
||||
url,
|
||||
headers=_auth_headers(tenant_access_token),
|
||||
payload=payload,
|
||||
action="create_child_docx",
|
||||
)
|
||||
node = ((result.get("data") or {}).get("node") or {})
|
||||
if not isinstance(node, dict) or not node:
|
||||
raise RuntimeError(f"create_child_docx failed: node missing, raw={json.dumps(result, ensure_ascii=False)}")
|
||||
return node
|
||||
|
||||
|
||||
def _ensure_doc_tree_under_parent(
|
||||
tenant_access_token: str,
|
||||
*,
|
||||
space_id: str,
|
||||
parent_node_token: str,
|
||||
doc_items: List[Dict[str, Any]],
|
||||
current_depth: int = 0,
|
||||
max_depth: int = DOC_TREE_MAX_DEPTH,
|
||||
) -> Dict[str, Any]:
|
||||
if current_depth >= max_depth:
|
||||
return {
|
||||
"created": [],
|
||||
"existing": [],
|
||||
"failed": [],
|
||||
"max_depth_reached": True,
|
||||
}
|
||||
|
||||
direct_children = list_wiki_child_nodes(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=parent_node_token,
|
||||
)
|
||||
direct_title_to_node = {
|
||||
_extract_node_title(node).strip(): node
|
||||
for node in direct_children
|
||||
if isinstance(node, dict) and _extract_node_title(node).strip()
|
||||
}
|
||||
|
||||
created: List[Dict[str, Any]] = []
|
||||
existing: List[Dict[str, Any]] = []
|
||||
failed: List[Dict[str, Any]] = []
|
||||
max_depth_reached = False
|
||||
|
||||
for item in doc_items:
|
||||
title = _extract_item_title(item)
|
||||
if not title:
|
||||
failed.append(
|
||||
{
|
||||
"title": "",
|
||||
"parent_token": parent_node_token,
|
||||
"depth": current_depth,
|
||||
"error": "empty_title",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
target_node = direct_title_to_node.get(title)
|
||||
source = "existing"
|
||||
if not isinstance(target_node, dict):
|
||||
try:
|
||||
target_node = create_child_docx(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=parent_node_token,
|
||||
title=title,
|
||||
)
|
||||
source = "created"
|
||||
direct_title_to_node[title] = target_node
|
||||
except Exception as exc:
|
||||
failed.append(
|
||||
{
|
||||
"title": title,
|
||||
"parent_token": parent_node_token,
|
||||
"depth": current_depth,
|
||||
"error": str(exc),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
entry = {
|
||||
"title": title,
|
||||
"node_token": target_node.get("node_token"),
|
||||
"obj_token": target_node.get("obj_token"),
|
||||
"parent_token": parent_node_token,
|
||||
"depth": current_depth,
|
||||
}
|
||||
if source == "created":
|
||||
created.append(entry)
|
||||
else:
|
||||
existing.append(entry)
|
||||
|
||||
children = item.get("children")
|
||||
if isinstance(children, list) and children:
|
||||
child_token = target_node.get("node_token")
|
||||
if isinstance(child_token, str) and child_token:
|
||||
child_result = _ensure_doc_tree_under_parent(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=child_token,
|
||||
doc_items=children,
|
||||
current_depth=current_depth + 1,
|
||||
max_depth=max_depth,
|
||||
)
|
||||
created.extend(child_result.get("created", []))
|
||||
existing.extend(child_result.get("existing", []))
|
||||
failed.extend(child_result.get("failed", []))
|
||||
max_depth_reached = max_depth_reached or bool(child_result.get("max_depth_reached"))
|
||||
|
||||
return {
|
||||
"created": created,
|
||||
"existing": existing,
|
||||
"failed": failed,
|
||||
"max_depth_reached": max_depth_reached,
|
||||
}
|
||||
|
||||
|
||||
def _resolve_doc_token(
|
||||
tenant_access_token: str,
|
||||
node: Dict[str, Any],
|
||||
) -> Optional[str]:
|
||||
obj_token = node.get("obj_token")
|
||||
if isinstance(obj_token, str) and obj_token:
|
||||
return obj_token
|
||||
|
||||
node_token = node.get("node_token")
|
||||
if not isinstance(node_token, str) or not node_token:
|
||||
return None
|
||||
|
||||
try:
|
||||
detail = get_wiki_node_info_by_token(tenant_access_token, node_token)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
detail_obj_token = detail.get("obj_token")
|
||||
if isinstance(detail_obj_token, str) and detail_obj_token:
|
||||
return detail_obj_token
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_fst_editor_parent_node(
|
||||
tenant_access_token: str,
|
||||
*,
|
||||
fst_editor_token: Optional[str],
|
||||
space_id: Optional[str],
|
||||
) -> Dict[str, Any]:
|
||||
if fst_editor_token:
|
||||
try:
|
||||
return get_wiki_node_info_by_token(tenant_access_token, fst_editor_token)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if space_id:
|
||||
root_nodes = list_wiki_child_nodes(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=None,
|
||||
)
|
||||
target = next(
|
||||
(
|
||||
node
|
||||
for node in root_nodes
|
||||
if isinstance(node, dict)
|
||||
and _extract_node_title(node).strip().lower() == FST_EDITOR_DOC_TITLE.lower()
|
||||
and str(node.get("obj_type") or "") in {"docx", "doc"}
|
||||
),
|
||||
None,
|
||||
)
|
||||
if isinstance(target, dict):
|
||||
return target
|
||||
|
||||
raise RuntimeError("cannot resolve fst_editor parent doc node by token/title")
|
||||
|
||||
|
||||
def ensure_child_docs_for_first_level_names(
|
||||
tenant_access_token: str,
|
||||
content: Any,
|
||||
*,
|
||||
fst_editor_token: Optional[str] = None,
|
||||
bitable_app_token: Optional[str] = None,
|
||||
space_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
extraction_debug = debug_extract_first_level_names(content)
|
||||
parent_node = _resolve_fst_editor_parent_node(
|
||||
tenant_access_token,
|
||||
fst_editor_token=fst_editor_token,
|
||||
space_id=space_id,
|
||||
)
|
||||
space_id = str(parent_node.get("space_id") or "")
|
||||
parent_node_token = str(parent_node.get("node_token") or "")
|
||||
if not space_id or not parent_node_token:
|
||||
raise RuntimeError("fst_editor parent node missing space_id or node_token")
|
||||
|
||||
# Only create level-1 documents.
|
||||
target_names = _extract_first_level_names(content)
|
||||
rows_by_first_level = _build_rows_by_first_level(content)
|
||||
if not target_names and rows_by_first_level:
|
||||
target_names = list(rows_by_first_level.keys())
|
||||
|
||||
# Keep deterministic order while allowing fallback names from grouped rows.
|
||||
seen_names: Set[str] = set()
|
||||
merged_target_names: List[str] = []
|
||||
for name in target_names + list(rows_by_first_level.keys()):
|
||||
if not isinstance(name, str):
|
||||
continue
|
||||
clean_name = name.strip()
|
||||
if not clean_name or clean_name in seen_names:
|
||||
continue
|
||||
seen_names.add(clean_name)
|
||||
merged_target_names.append(clean_name)
|
||||
target_names = merged_target_names
|
||||
|
||||
doc_structure = [{"title": title, "children": []} for title in target_names]
|
||||
|
||||
tree_sync_result = _ensure_doc_tree_under_parent(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=parent_node_token,
|
||||
doc_items=doc_structure,
|
||||
current_depth=0,
|
||||
max_depth=DOC_TREE_MAX_DEPTH,
|
||||
)
|
||||
|
||||
existing_children = list_wiki_child_nodes(
|
||||
tenant_access_token,
|
||||
space_id=space_id,
|
||||
parent_node_token=parent_node_token,
|
||||
)
|
||||
# Keep creation scope to direct children only; avoid deep traversal blocking main sync flow.
|
||||
existing_descendants = existing_children
|
||||
existing_title_to_node = {
|
||||
_extract_node_title(node).strip(): node
|
||||
for node in existing_children
|
||||
if isinstance(node, dict) and _extract_node_title(node).strip()
|
||||
}
|
||||
|
||||
synced_title_to_doc_token = {
|
||||
str(item.get("title")).strip(): str(item.get("obj_token") or "")
|
||||
for item in (tree_sync_result.get("created", []) + tree_sync_result.get("existing", []))
|
||||
if isinstance(item, dict)
|
||||
and isinstance(item.get("title"), str)
|
||||
and str(item.get("title")).strip()
|
||||
}
|
||||
|
||||
created_docs: List[Dict[str, Any]] = [
|
||||
item for item in tree_sync_result.get("created", []) if item.get("depth") == 0
|
||||
]
|
||||
existing_docs: List[Dict[str, Any]] = [
|
||||
item for item in tree_sync_result.get("existing", []) if item.get("depth") == 0
|
||||
]
|
||||
insertion_results: List[Dict[str, Any]] = []
|
||||
missing_names: List[str] = []
|
||||
top_level_created_names = {item.get("title") for item in created_docs if isinstance(item.get("title"), str)}
|
||||
|
||||
for name in target_names:
|
||||
existing = existing_title_to_node.get(name)
|
||||
if existing:
|
||||
target_node = existing
|
||||
document_id = _resolve_doc_token(tenant_access_token, target_node)
|
||||
if (not isinstance(document_id, str) or not document_id) and synced_title_to_doc_token.get(name):
|
||||
document_id = synced_title_to_doc_token.get(name)
|
||||
|
||||
if isinstance(document_id, str) and document_id:
|
||||
try:
|
||||
top_bitable_result = ensure_bitable_block_for_document(
|
||||
tenant_access_token,
|
||||
document_id=document_id,
|
||||
)
|
||||
write_result = sync_document_bitable_rows(
|
||||
tenant_access_token,
|
||||
document_id=document_id,
|
||||
rows=rows_by_first_level.get(name, []),
|
||||
bitable_app_token=bitable_app_token,
|
||||
)
|
||||
insertion_results.append({
|
||||
"title": name,
|
||||
"document_id": document_id,
|
||||
"inserted": True,
|
||||
"mode": top_bitable_result.get("mode"),
|
||||
"row_count": len(rows_by_first_level.get(name, [])),
|
||||
"bitable_write_result": write_result,
|
||||
"source": "created" if name in top_level_created_names else "existing",
|
||||
})
|
||||
except Exception as exc:
|
||||
insertion_results.append({
|
||||
"title": name,
|
||||
"document_id": document_id,
|
||||
"inserted": False,
|
||||
"source": "created" if name in top_level_created_names else "existing",
|
||||
"error": str(exc),
|
||||
})
|
||||
else:
|
||||
insertion_results.append({
|
||||
"title": name,
|
||||
"inserted": False,
|
||||
"source": "created" if name in top_level_created_names else "existing",
|
||||
"error": "document_id_missing",
|
||||
})
|
||||
continue
|
||||
|
||||
missing_names.append(name)
|
||||
|
||||
return {
|
||||
"fst_editor": {
|
||||
"title": _extract_node_title(parent_node),
|
||||
"space_id": space_id,
|
||||
"node_token": parent_node_token,
|
||||
"obj_type": parent_node.get("obj_type"),
|
||||
"obj_token": parent_node.get("obj_token"),
|
||||
},
|
||||
"target_names": target_names,
|
||||
"rows_by_first_level_count": {
|
||||
key: len(value)
|
||||
for key, value in rows_by_first_level.items()
|
||||
},
|
||||
"extraction_debug": extraction_debug,
|
||||
"existing_child_count": len(existing_children),
|
||||
"existing_descendant_count": len(existing_descendants),
|
||||
"existing_docs": existing_docs,
|
||||
"created_docs": created_docs,
|
||||
"tree_sync_result": tree_sync_result,
|
||||
"insertion_results": insertion_results,
|
||||
"missing_names": missing_names,
|
||||
}
|
||||
514
fst_data_pipeline/apps/root_db_api/src/core/models.py
Normal file
514
fst_data_pipeline/apps/root_db_api/src/core/models.py
Normal file
@@ -0,0 +1,514 @@
|
||||
from sqlalchemy import (
|
||||
Column,
|
||||
Integer,
|
||||
String,
|
||||
Boolean,
|
||||
ForeignKey,
|
||||
TIMESTAMP,
|
||||
BigInteger,
|
||||
Text,
|
||||
SmallInteger,
|
||||
UniqueConstraint,
|
||||
Index,
|
||||
JSON,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB, ARRAY
|
||||
from sqlalchemy.orm import declarative_base
|
||||
from sqlalchemy.orm import relationship, backref
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
# ============================================================
|
||||
# ① 基础准备
|
||||
# ============================================================
|
||||
|
||||
|
||||
class Project(Base):
|
||||
__tablename__ = "project"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False, unique=True)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
comment="last update time",
|
||||
)
|
||||
|
||||
bags = relationship(
|
||||
"BagList", back_populates="project", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ② 字典 & 主数据（无外部依赖）
|
||||
# ============================================================
|
||||
|
||||
|
||||
class DriveMode(Base):
|
||||
__tablename__ = "drive_mode"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
type = Column(String(100), nullable=False)
|
||||
sub_type = Column(String(100))
|
||||
reserved_json = Column(JSONB)
|
||||
comment = Column(Text)
|
||||
|
||||
__table_args__ = (UniqueConstraint("type", "sub_type"),)
|
||||
|
||||
|
||||
class TopicList(Base):
|
||||
__tablename__ = "topic_list"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False, unique=True)
|
||||
type = Column(String(255))
|
||||
key_data = Column(Text)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
class ReservedTagList(Base):
|
||||
__tablename__ = "reserved_tag_list"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False)
|
||||
type = Column(String(50))
|
||||
creator = Column(String(255))
|
||||
comments = Column(Text)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
is_deleted = Column(Boolean, default=False)
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"uq_reserved_tag_list_name_not_deleted",
|
||||
"name",
|
||||
unique=True,
|
||||
postgresql_where=(is_deleted.is_(False)),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class GTMeta(Base):
|
||||
__tablename__ = "gt_meta"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False)
|
||||
type = Column(String(100), nullable=False)
|
||||
path = Column(Text, nullable=False)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
reserved_json = Column(JSON)
|
||||
comment = Column(Text)
|
||||
|
||||
# 复合唯一约束
|
||||
__table_args__ = (UniqueConstraint("name", "type", name="uq_gt_meta_name_type"),)
|
||||
|
||||
|
||||
class VersionList(Base):
|
||||
__tablename__ = "version_list"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
version = Column(String(255), nullable=False, unique=True)
|
||||
type = Column(String(50))
|
||||
description = Column(Text)
|
||||
release_date = Column(TIMESTAMP(timezone=True))
|
||||
created_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
status = Column(String(50), nullable=False, default="available")
|
||||
feishu_sync_status = Column(String(50), nullable=False, default="unsynced")
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ③ 核心业务表（依赖 ①②）
|
||||
# ============================================================
|
||||
|
||||
|
||||
class BagList(Base):
|
||||
__tablename__ = "bag_list"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False, unique=True)
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
project_id = Column(BigInteger, ForeignKey("project.id", ondelete="CASCADE"))
|
||||
tile_id = Column(String(255))
|
||||
is_decoded = Column(Boolean, default=False)
|
||||
is_deleted = Column(Boolean, default=False)
|
||||
od_annotated = Column(Integer)
|
||||
ld_annotated = Column(Integer)
|
||||
fst_indexed = Column(Boolean, default=False)
|
||||
is_active_data = Column(Boolean, default=True)
|
||||
reserved_str = Column(String(255))
|
||||
reserved_json = Column(JSONB)
|
||||
drive_mode = Column(SmallInteger, ForeignKey("drive_mode.id", ondelete="SET NULL"))
|
||||
sw_version = Column(String(255))
|
||||
hw_version = Column(String(255))
|
||||
|
||||
project = relationship("Project", back_populates="bags")
|
||||
lifecycle = relationship(
|
||||
"BagLifecycle",
|
||||
back_populates="bag",
|
||||
uselist=False,
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
secondary_pangu = relationship(
|
||||
"SecondaryPangu",
|
||||
back_populates="bag",
|
||||
uselist=False,
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
secondary_minerva = relationship(
|
||||
"SecondaryMinerva",
|
||||
back_populates="bag",
|
||||
uselist=False,
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
|
||||
class BagLifecycle(Base):
|
||||
__tablename__ = "bag_lifecycle"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_name = Column(
|
||||
String(255),
|
||||
ForeignKey("bag_list.name", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
)
|
||||
collect_time = Column(TIMESTAMP(timezone=True))
|
||||
clone2dev_time = Column(TIMESTAMP(timezone=True))
|
||||
decode_time = Column(TIMESTAMP(timezone=True))
|
||||
mining_time = Column(TIMESTAMP(timezone=True))
|
||||
auto_annotate_time = Column(TIMESTAMP(timezone=True))
|
||||
manual_annotate_time = Column(TIMESTAMP(timezone=True))
|
||||
fst_index_time = Column(TIMESTAMP(timezone=True))
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
bag = relationship("BagList", back_populates="lifecycle")
|
||||
|
||||
|
||||
class MainPangu(Base):
|
||||
__tablename__ = "main_pangu"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False, unique=True)
|
||||
vehicle = Column(String(255))
|
||||
datetime = Column(TIMESTAMP(timezone=True))
|
||||
bag_path = Column(Text)
|
||||
data_path = Column(Text)
|
||||
reserved_str = Column(String(255))
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
class JoinedPangu(Base):
|
||||
__tablename__ = "joined_pangu"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False, unique=True)
|
||||
data_path = Column(Text)
|
||||
reserved_str = Column(String(255))
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
class JoinedBags(Base):
|
||||
__tablename__ = "joined_bags"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
parent_id = Column(BigInteger, nullable=False)
|
||||
child_id = Column(BigInteger, nullable=False, unique=True)
|
||||
reserved_str = Column(String(255))
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
class MainMinerva(Base):
|
||||
__tablename__ = "main_minerva"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
session_id = Column(String(255), unique=True)
|
||||
start_ts = Column(TIMESTAMP(timezone=True))
|
||||
end_ts = Column(TIMESTAMP(timezone=True))
|
||||
length = Column(Integer)
|
||||
datetime = Column(TIMESTAMP(timezone=True))
|
||||
vin = Column(String(255))
|
||||
platform = Column(String(255))
|
||||
mapped = Column(Boolean, default=False)
|
||||
path = Column(Text)
|
||||
converted_path = Column(Text)
|
||||
gt_path = Column(Text)
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# ④ 子表 & 关联表（依赖 bag_list.id）
|
||||
# ============================================================
|
||||
|
||||
|
||||
class SecondaryPangu(Base):
|
||||
__tablename__ = "secondary_pangu"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
)
|
||||
lidar_gt_pandar128 = Column(Text)
|
||||
object_lidar_gt_pandar128_manual = Column(Text)
|
||||
lidar_fd_multi_scan_raw = Column(Text)
|
||||
camera_fisheye_left = Column(Text)
|
||||
camera_fisheye_right = Column(Text)
|
||||
camera_front_wide = Column(Text)
|
||||
raw_gps = Column(Text)
|
||||
raw_imu = Column(Text)
|
||||
ego_motion = Column(Text)
|
||||
vehicle_wheel = Column(Text)
|
||||
calibration = Column(Text)
|
||||
sdmap = Column(Text)
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
bag = relationship("BagList", back_populates="secondary_pangu")
|
||||
|
||||
|
||||
class SecondaryMinerva(Base):
|
||||
__tablename__ = "secondary_minerva"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
)
|
||||
lidar_gt_top_p128 = Column(Text)
|
||||
lidar_parking_gt_front_p128 = Column(Text)
|
||||
lidar_parking_gt_left_p128 = Column(Text)
|
||||
lidar_parking_gt_right_p128 = Column(Text)
|
||||
lidar_parking_gt_rear_p128 = Column(Text)
|
||||
camera_fisheye_left_200fov = Column(Text)
|
||||
camera_fisheye_right_200fov = Column(Text)
|
||||
camera_fisheye_rear_200fov = Column(Text)
|
||||
camera_fisheye_front_200fov = Column(Text)
|
||||
camera_front_wide_120fov = Column(Text)
|
||||
camera_front_tele_30fov = Column(Text)
|
||||
camera_rear_right_70fov = Column(Text)
|
||||
camera_rear_left_70fov = Column(Text)
|
||||
raw_gps = Column(Text)
|
||||
raw_imu = Column(Text)
|
||||
ego_motion = Column(Text)
|
||||
calibration = Column(Text)
|
||||
rig = Column(Text)
|
||||
fst_new = Column(Text)
|
||||
fst_old = Column(Text)
|
||||
comments = Column(Text)
|
||||
reserved_json = Column(JSONB)
|
||||
|
||||
bag = relationship("BagList", back_populates="secondary_minerva")
|
||||
|
||||
|
||||
class FST(Base):
|
||||
__tablename__ = "fst"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False)
|
||||
parent_id = Column(BigInteger, ForeignKey("fst.id", ondelete="CASCADE"))
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
reserved_json = Column(JSONB)
|
||||
bag_sum = Column(Integer, default=0)
|
||||
is_delete = Column(SmallInteger, default=0)
|
||||
|
||||
parent = relationship(
|
||||
"FST",
|
||||
remote_side=[id],
|
||||
backref=backref("children", cascade="all, delete-orphan"),
|
||||
)
|
||||
|
||||
|
||||
class GeometryInfo(Base):
|
||||
__tablename__ = "geometry_info"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
rosbag_name = Column(String(255), nullable=False)
|
||||
gnss_downsampled_points = Column(ARRAY(String)) # 或 geoalchemy2 Geometry
|
||||
update_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
is_overlapped = Column(Boolean, default=False)
|
||||
|
||||
|
||||
class BagTopic(Base):
|
||||
__tablename__ = "bag_topic"
|
||||
__table_args__ = (UniqueConstraint("bag_id", "topic_id"),)
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger, ForeignKey("bag_list.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
topic_id = Column(
|
||||
BigInteger, ForeignKey("topic_list.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
|
||||
bag = relationship("BagList")
|
||||
topic = relationship("TopicList")
|
||||
|
||||
|
||||
class BagReservedTag(Base):
|
||||
__tablename__ = "bag_reserved_tag"
|
||||
__table_args__ = (UniqueConstraint("bag_id", "tag_id"),)
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
tag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("reserved_tag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
bag = relationship("BagList")
|
||||
tag = relationship("ReservedTagList")
|
||||
|
||||
|
||||
class BagGT(Base):
|
||||
__tablename__ = "bag_gt"
|
||||
__table_args__ = (UniqueConstraint("gt_id", "bag_id"),)
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
gt_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("gt_meta.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
comment = Column(Text)
|
||||
|
||||
|
||||
class FSTBag(Base):
|
||||
__tablename__ = "fst_bag"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
fst_node_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("fst.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
fst_node_level = Column(Integer, nullable=False)
|
||||
event_start_time = Column(Integer)
|
||||
event_end_time = Column(Integer)
|
||||
img_url = Column(String(255))
|
||||
video_url = Column(String(255))
|
||||
comments = Column(String(255))
|
||||
|
||||
bag = relationship("BagList", backref="fst_links")
|
||||
fst_node = relationship("FST", backref="bag_links")
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Recompute 结果存储表
|
||||
# ============================================================
|
||||
|
||||
|
||||
class RecomputeResult(Base):
|
||||
__tablename__ = "recompute_result"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
bag_id = Column(
|
||||
BigInteger,
|
||||
ForeignKey("bag_list.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
comment="Bag ID",
|
||||
)
|
||||
recompute_version = Column(String(100), nullable=False, comment="Recompute版本")
|
||||
result_type = Column(
|
||||
String(50), nullable=False, comment="结果类型 (如: perception, planning, etc)"
|
||||
)
|
||||
storage_path = Column(String(500), nullable=False, comment="结果存储路径")
|
||||
status = Column(
|
||||
String(50),
|
||||
nullable=False,
|
||||
default="available",
|
||||
comment="结果状态: available, archived, deleted",
|
||||
)
|
||||
created_time = Column(
|
||||
TIMESTAMP(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
comment="Recompute完成时间",
|
||||
)
|
||||
updated_time = Column(
|
||||
TIMESTAMP(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
comment="更新时间",
|
||||
)
|
||||
reserved_json = Column(JSONB, comment="预留JSON字段")
|
||||
|
||||
# 与BagList的关系
|
||||
bag = relationship("BagList", backref="recompute_results")
|
||||
|
||||
# 索引和约束
|
||||
__table_args__ = (
|
||||
# 同一个bag的同一版本同一类型的recompute结果应该是唯一的
|
||||
UniqueConstraint(
|
||||
"bag_id", "recompute_version", "result_type", name="uq_recompute_result"
|
||||
),
|
||||
Index("idx_recompute_result_version", "recompute_version"),
|
||||
Index("idx_recompute_result_type", "result_type"),
|
||||
Index("idx_recompute_result_status", "status"),
|
||||
Index("idx_recompute_result_created_time", "created_time"),
|
||||
Index("idx_recompute_result_storage_path", "storage_path"),
|
||||
)
|
||||
|
||||
|
||||
class FstStash(Base):
|
||||
__tablename__ = "fst_stash"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
fst_versions = Column(
|
||||
String(255),
|
||||
ForeignKey("version_list.version", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
content = Column(JSONB, nullable=True)
|
||||
created_time = Column(
|
||||
TIMESTAMP(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
updated_time = Column(
|
||||
TIMESTAMP(timezone=True),
|
||||
nullable=False,
|
||||
server_default=func.now(),
|
||||
onupdate=func.now(),
|
||||
)
|
||||
|
||||
version_rel = relationship("VersionList", backref=backref("fst_stashes", cascade="all, delete-orphan"))
|
||||
2530
fst_data_pipeline/apps/root_db_api/src/core/service.py
Normal file
2530
fst_data_pipeline/apps/root_db_api/src/core/service.py
Normal file
File diff suppressed because it is too large
Load Diff
36
fst_data_pipeline/apps/root_db_api/src/db/cache.py
Normal file
36
fst_data_pipeline/apps/root_db_api/src/db/cache.py
Normal file
@@ -0,0 +1,36 @@
|
||||
# fst_data_pipeline/apps/root_db_api/src/db/memory_cache.py
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
class _MemoryCache:
|
||||
"""极简带过期时间的线程安全内存缓存"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._data: dict[str, tuple[Any, float]] = {}
|
||||
|
||||
def get(self, key: str) -> Optional[Any]:
|
||||
with self._lock:
|
||||
item = self._data.get(key)
|
||||
if item is None:
|
||||
return None
|
||||
value, expire = item
|
||||
if expire > 0 and time.time() > expire:
|
||||
self._data.pop(key, None)
|
||||
return None
|
||||
return value
|
||||
|
||||
def set(self, key: str, value: Any, timeout: int = 600) -> None:
|
||||
expire = time.time() + timeout if timeout > 0 else 0
|
||||
with self._lock:
|
||||
self._data[key] = (value, expire)
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
with self._lock:
|
||||
self._data.pop(key, None)
|
||||
|
||||
|
||||
# 全局单例
|
||||
memory_cache = _MemoryCache()
|
||||
30
fst_data_pipeline/apps/root_db_api/src/db/connection.py
Normal file
30
fst_data_pipeline/apps/root_db_api/src/db/connection.py
Normal file
@@ -0,0 +1,30 @@
|
||||
import os
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from contextlib import contextmanager
|
||||
|
||||
DB_USER = os.getenv("DB_USER")
|
||||
DB_PASSWORD = os.getenv("DB_PASSWORD")
|
||||
DB_BASE_URL = os.getenv("DB_BASE_URL")
|
||||
DB_URL = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_BASE_URL}"
|
||||
|
||||
engine = create_engine(
|
||||
DB_URL,
|
||||
pool_size=20,
|
||||
max_overflow=40,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=3600,
|
||||
connect_args={"options": "-c search_path=public"},
|
||||
echo=True,
|
||||
)
|
||||
|
||||
SessionLocal = sessionmaker(bind=engine, expire_on_commit=False)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_db_session():
|
||||
session = SessionLocal()
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
session.close()
|
||||
1702
fst_data_pipeline/apps/root_db_api/src/root_db_api.egg-info/PKG-INFO
Normal file
1702
fst_data_pipeline/apps/root_db_api/src/root_db_api.egg-info/PKG-INFO
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,40 @@
|
||||
README.md
|
||||
pyproject.toml
|
||||
src/__init__.py
|
||||
src/app.py
|
||||
src/api/__init__.py
|
||||
src/api/bags.py
|
||||
src/api/fst.py
|
||||
src/api/geometry.py
|
||||
src/api/gt.py
|
||||
src/api/projects.py
|
||||
src/api/recompute.py
|
||||
src/api/tags.py
|
||||
src/api/topics.py
|
||||
src/api/versions.py
|
||||
src/core/__init__.py
|
||||
src/core/feishu_api_constants.py
|
||||
src/core/feishu_bitable_sdk.py
|
||||
src/core/feishu_doc_bitable_block_sdk.py
|
||||
src/core/feishu_wiki_doc_sdk.py
|
||||
src/core/models.py
|
||||
src/core/service.py
|
||||
src/db/__init__.py
|
||||
src/db/cache.py
|
||||
src/db/connection.py
|
||||
src/root_db_api.egg-info/PKG-INFO
|
||||
src/root_db_api.egg-info/SOURCES.txt
|
||||
src/root_db_api.egg-info/dependency_links.txt
|
||||
src/root_db_api.egg-info/requires.txt
|
||||
src/root_db_api.egg-info/top_level.txt
|
||||
src/test/__init__.py
|
||||
src/test/api/__init__.py
|
||||
src/test/api/test_bags.py
|
||||
src/test/api/test_fst.py
|
||||
src/test/api/test_geometry.py
|
||||
src/test/api/test_projects.py
|
||||
src/test/api/test_recompute.py
|
||||
src/test/api/test_tags.py
|
||||
src/test/api/test_topics.py
|
||||
src/test/service/__init__.py
|
||||
src/test/service/test_service.py
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
flask>=3.1.1
|
||||
flasgger==0.9.7b2
|
||||
geoalchemy2==0.17.1
|
||||
shapely>=2.1.1
|
||||
numpy==2.3.1
|
||||
folium==0.20.0
|
||||
tenacity==9.1.2
|
||||
tqdm==4.67.1
|
||||
pytz==2025.2
|
||||
prometheus-client==0.22.1
|
||||
flask-caching>=2.3.1
|
||||
redis>=6.4.0
|
||||
cos-python-sdk-v5==1.9.37
|
||||
python-dotenv==1.1.1
|
||||
requests==2.32.4
|
||||
pyyaml==6.0.2
|
||||
pydantic==2.11.7
|
||||
sqlalchemy==2.0.41
|
||||
psycopg2-binary==2.9.10
|
||||
openpyxl>=3.1.0
|
||||
gunicorn>=23.0.0
|
||||
pytest==8.4.1
|
||||
pytest-cov==6.2.0
|
||||
pytest-mock==3.14.0
|
||||
flask-sqlalchemy>=3.1.1
|
||||
@@ -0,0 +1,6 @@
|
||||
__init__
|
||||
api
|
||||
app
|
||||
core
|
||||
db
|
||||
test
|
||||
343
fst_data_pipeline/apps/root_db_api/src/test/api/test_bags.py
Normal file
343
fst_data_pipeline/apps/root_db_api/src/test/api/test_bags.py
Normal file
@@ -0,0 +1,343 @@
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api import bags_bp
|
||||
|
||||
|
||||
# ------------------ Flask test client ------------------
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
"""Flask test client with bags blueprint registered."""
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(bags_bp, url_prefix="/bags")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
# ------------------ 单元测试用例 ------------------
|
||||
def test_get_all_bags_200(client, mocker):
|
||||
bag1 = MagicMock()
|
||||
bag1.name = "bag1"
|
||||
bag1.project.name = "ProjectX"
|
||||
|
||||
bag2 = MagicMock()
|
||||
bag2.name = "bag2"
|
||||
bag2.project = None
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_bags",
|
||||
return_value=[bag1, bag2],
|
||||
)
|
||||
|
||||
resp = client.get("/bags/all")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"ProjectX": ["bag1"], "Uncategorized": ["bag2"]}
|
||||
|
||||
|
||||
def test_search_main_pangu_by_names_200(client, mocker):
|
||||
p = MagicMock()
|
||||
p.name = "p1"
|
||||
p.vehicle = "v1"
|
||||
p.datetime = "2021-01-01T00:00:00Z"
|
||||
p.bag_path = "/raw"
|
||||
p.data_path = "/dec"
|
||||
p.reserved_str = "rs"
|
||||
p.reserved_json = {}
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_main_pangu_by_name",
|
||||
return_value=[p],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/pangu", json={"names": ["p1"]})
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["p1"][0]["vehicle"] == "v1"
|
||||
|
||||
|
||||
def test_get_pangu_details_batch_200(client, mocker):
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_pangu_details_by_bag_name",
|
||||
return_value={"detail": "ok"},
|
||||
)
|
||||
|
||||
resp = client.post("/bags/pangu/detail", json=["a.bag", "b.bag"])
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["a.bag"]["detail"] == "ok"
|
||||
|
||||
|
||||
def test_get_pangu_details_batch_400(client):
|
||||
resp = client.post("/bags/pangu/detail", json={"wrong": "format"})
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_search_main_minerva_by_session_ids_200(client, mocker):
|
||||
m = MagicMock()
|
||||
m.session_id = "s1"
|
||||
m.vin = "VIN1"
|
||||
m.start_ts = 0
|
||||
m.end_ts = 1
|
||||
m.platform = "p"
|
||||
m.path = "/p"
|
||||
m.converted_path = "/c"
|
||||
m.gt_path = "/g"
|
||||
m.datetime = "now"
|
||||
m.length = 10
|
||||
m.reserved_json = {}
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_main_minerva_by_session_id",
|
||||
return_value=[m],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/minerva", json={"session_ids": ["s1"]})
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["s1"][0]["vin"] == "VIN1"
|
||||
|
||||
|
||||
def test_get_minerva_details_batch_200(client, mocker):
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_minerva_details_by_bag_name",
|
||||
return_value={"session_id": "s1"},
|
||||
)
|
||||
|
||||
resp = client.post("/bags/minerva/detail", json=["a.bag"])
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["a.bag"]["session_id"] == "s1"
|
||||
|
||||
|
||||
def test_get_topics_batch_200(client, mocker):
|
||||
t = MagicMock()
|
||||
t.name = "t1"
|
||||
t.type = "std_msgs/String"
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_topics_by_bag_name",
|
||||
return_value=[t],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/topics", json=["a.bag"])
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"a.bag": [{"name": "t1", "type": "std_msgs/String"}]}
|
||||
|
||||
|
||||
def test_get_tags_batch_200(client, mocker):
|
||||
tag = MagicMock()
|
||||
tag.name = "tag1"
|
||||
tag.type = "scenario"
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_tags_by_bag_name",
|
||||
return_value=[tag],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/tags", json=["a.bag"])
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["a.bag"][0]["name"] == "tag1"
|
||||
|
||||
|
||||
def test_get_fst_nodes_batch_200(client, mocker):
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_fst_nodes_by_bag_name",
|
||||
return_value=[{"name": "n1", "start": 0, "end": 10}],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/fst/nodes", json=["a.bag"])
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["a.bag"][0]["name"] == "n1"
|
||||
|
||||
|
||||
def test_get_bag_lifecycle_200(client, mocker):
|
||||
lc = MagicMock()
|
||||
lc.bag_name = "a.bag"
|
||||
lc.collect_time = "2021-01-01T00:00:00Z"
|
||||
lc.clone2dev_time = "2021-01-01T01:00:00Z"
|
||||
lc.decode_time = "2021-01-01T02:00:00Z"
|
||||
lc.mining_time = "2021-01-01T03:00:00Z"
|
||||
lc.auto_annotate_time = "2021-01-01T04:00:00Z"
|
||||
lc.manual_annotate_time = "2021-01-01T05:00:00Z"
|
||||
lc.fst_index_time = "2021-01-01T06:00:00Z"
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_bag_lifecycle_by_bag_names",
|
||||
return_value=[lc],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/life_cycle", json={"bag_names": ["a.bag"]})
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["a.bag"]["collect_time"] == "2021-01-01T00:00:00Z"
|
||||
|
||||
|
||||
# ------------------ 400 分支示例 ------------------
|
||||
@pytest.mark.parametrize(
|
||||
"url,bad_body",
|
||||
[
|
||||
("/bags/pangu/detail", {"not": "list"}),
|
||||
("/bags/minerva/detail", {"not": "list"}),
|
||||
("/bags/topics", {"bag_names": "not_list"}),
|
||||
("/bags/tags", {"bag_names": "not_list"}),
|
||||
("/bags/fst/nodes", "not_list"),
|
||||
],
|
||||
)
|
||||
def test_bad_body_400(client, url, bad_body):
|
||||
resp = client.post(url, json=bad_body)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_search_pangu_basic_200(client, mocker):
|
||||
"""测试基本的pangu搜索功能"""
|
||||
p = MagicMock()
|
||||
p.name = "test_bag.bag"
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_main_pangu_by_conditions",
|
||||
return_value=[p],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/search/pangu", json={})
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert isinstance(data, list)
|
||||
assert data == ["test_bag.bag"]
|
||||
|
||||
|
||||
def test_search_pangu_with_all_params_200(client, mocker):
|
||||
"""测试带所有参数的pangu搜索"""
|
||||
p1 = MagicMock()
|
||||
p1.name = "test_bag1.bag"
|
||||
p2 = MagicMock()
|
||||
p2.name = "test_bag2.bag"
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_main_pangu_by_conditions",
|
||||
return_value=[p1, p2],
|
||||
)
|
||||
|
||||
request_data = {
|
||||
"starttime": "20240101",
|
||||
"endtime": "20241231",
|
||||
"vehicle": "test_vehicle",
|
||||
"keyword": "highway",
|
||||
"topics": ["/camera/front", "/lidar/points"],
|
||||
"fst": "highway_scenarios",
|
||||
}
|
||||
|
||||
resp = client.post("/bags/search/pangu", json=request_data)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data == ["test_bag1.bag", "test_bag2.bag"]
|
||||
|
||||
|
||||
def test_search_pangu_invalid_date_format_400(client):
|
||||
"""测试日期格式错误"""
|
||||
request_data = {"starttime": "invalid_date"}
|
||||
|
||||
resp = client.post("/bags/search/pangu", json=request_data)
|
||||
assert resp.status_code == 400
|
||||
data = resp.get_json()
|
||||
assert "error" in data
|
||||
|
||||
|
||||
def test_search_pangu_invalid_topics_400(client):
|
||||
"""测试topics参数格式错误"""
|
||||
request_data = {"topics": "not_a_list"}
|
||||
|
||||
resp = client.post("/bags/search/pangu", json=request_data)
|
||||
assert resp.status_code == 400
|
||||
data = resp.get_json()
|
||||
assert "topics must be an array" in data["error"]
|
||||
|
||||
|
||||
def test_search_pangu_empty_result_200(client, mocker):
|
||||
"""测试搜索无结果"""
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.get_main_pangu_by_conditions",
|
||||
return_value=[],
|
||||
)
|
||||
|
||||
resp = client.post("/bags/search/pangu", json={"keyword": "nonexistent"})
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data == []
|
||||
|
||||
|
||||
def test_merge_bags_query_specific_bags_200(client, mocker):
|
||||
"""测试查询指定bag的合并关系"""
|
||||
mock_result = {
|
||||
"parent1.bag": ["child1.bag", "child2.bag"],
|
||||
"parent2.bag": ["child3.bag"],
|
||||
}
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.query_existed_joined_bag",
|
||||
return_value=mock_result,
|
||||
)
|
||||
|
||||
request_data = {"bag_names": ["parent1.bag", "parent2.bag"]}
|
||||
|
||||
resp = client.post("/bags/joined/query", json=request_data)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data == mock_result
|
||||
|
||||
|
||||
def test_merge_bags_query_wildcard_200(client, mocker):
|
||||
"""测试通配符查询所有合并关系"""
|
||||
mock_result = {
|
||||
"parent1.bag": ["child1.bag", "child2.bag"],
|
||||
"parent2.bag": ["child3.bag", "child4.bag"],
|
||||
"parent3.bag": ["child5.bag"],
|
||||
}
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.query_all_joined_bags",
|
||||
return_value=mock_result,
|
||||
)
|
||||
|
||||
request_data = {"bag_names": ["*"]}
|
||||
|
||||
resp = client.post("/bags/joined/query", json=request_data)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data == mock_result
|
||||
|
||||
|
||||
def test_merge_bags_query_empty_bag_names_400(client):
|
||||
"""测试空的bag_names参数"""
|
||||
resp = client.post("/bags/joined/query", json={})
|
||||
assert resp.status_code == 400
|
||||
data = resp.get_json()
|
||||
assert "bag_names must be a list" in data["error"]
|
||||
|
||||
|
||||
def test_merge_bags_query_invalid_bag_names_400(client):
|
||||
"""测试无效的bag_names参数格式"""
|
||||
resp = client.post("/bags/joined/query", json={"bag_names": "not_a_list"})
|
||||
assert resp.status_code == 400
|
||||
data = resp.get_json()
|
||||
assert "bag_names must be a list" in data["error"]
|
||||
|
||||
|
||||
def test_merge_bags_query_empty_list_400(client):
|
||||
"""测试空列表的bag_names参数 - 现在应该返回200因为代码已经改变"""
|
||||
resp = client.post("/bags/joined/query", json={"bag_names": []})
|
||||
assert resp.status_code == 400
|
||||
data = resp.get_json()
|
||||
assert "bag_names must be a list" in data["error"]
|
||||
|
||||
|
||||
def test_merge_bags_query_exception_500(client, mocker):
|
||||
"""测试内部异常处理"""
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.bags.query_existed_joined_bag",
|
||||
side_effect=Exception("Database error"),
|
||||
)
|
||||
|
||||
request_data = {"bag_names": ["test.bag"]}
|
||||
|
||||
resp = client.post("/bags/joined/query", json=request_data)
|
||||
assert resp.status_code == 500
|
||||
data = resp.get_json()
|
||||
assert "Internal server error" in data["error"]
|
||||
233
fst_data_pipeline/apps/root_db_api/src/test/api/test_fst.py
Normal file
233
fst_data_pipeline/apps/root_db_api/src/test/api/test_fst.py
Normal file
@@ -0,0 +1,233 @@
|
||||
# tests/test_fst.py
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api import fst_bp
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def app():
|
||||
_app = Flask(__name__)
|
||||
_app.register_blueprint(fst_bp, url_prefix="/fst")
|
||||
_app.config["TESTING"] = True
|
||||
return _app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(app):
|
||||
return app.test_client()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_db_session():
|
||||
"""
|
||||
全局自动 mock SessionLocal，避免每个测试都手动 patch。
|
||||
返回一个 mock session 实例，可以在测试中进一步断言。
|
||||
"""
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.SessionLocal",
|
||||
return_value=MagicMock(),
|
||||
) as mocked:
|
||||
yield mocked
|
||||
|
||||
|
||||
# ----------------- 1. 打印树 -----------------
|
||||
def test_print_tree_200(client: pytest.fixture):
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.print_fst_tree",
|
||||
return_value="fake_tree",
|
||||
):
|
||||
resp = client.get("/fst/print_tree")
|
||||
|
||||
assert resp.status_code == 200
|
||||
assert resp.content_type == "application/json"
|
||||
assert resp.get_json() == "fake_tree"
|
||||
|
||||
|
||||
# ----------------- 2. 批量根据 FST 名称查 Bags -----------------
|
||||
def test_get_bags_by_fst_nodes_batch_200(client: pytest.fixture):
|
||||
# 构造 Bag 假对象
|
||||
Bag = type("Bag", (), {"name": None})
|
||||
fst1_bags = [Bag(), Bag()]
|
||||
fst1_bags[0].name, fst1_bags[1].name = "b1", "b2"
|
||||
fst2_bags = [Bag()]
|
||||
fst2_bags[0].name = "b3"
|
||||
|
||||
side_effect = {"fst1": fst1_bags, "fst2": fst2_bags}
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_bags_by_fst_name",
|
||||
side_effect=lambda db, name: side_effect[name],
|
||||
):
|
||||
# 默认第 1 页，每页 20
|
||||
resp = client.post("/fst/bags/nodes", json=["fst1", "fst2"])
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
# 期望 3 条记录，不分页
|
||||
assert data["page"] == 1
|
||||
assert data["per_page"] == 20
|
||||
assert data["total"] == 3
|
||||
|
||||
# 顺序与插入顺序一致
|
||||
assert data["items"] == [
|
||||
{"bag_name": "b1", "sts": "fst1"},
|
||||
{"bag_name": "b2", "sts": "fst1"},
|
||||
{"bag_name": "b3", "sts": "fst2"},
|
||||
]
|
||||
|
||||
|
||||
def test_get_bags_by_fst_nodes_batch_400(client: pytest.fixture):
|
||||
resp = client.post("/fst/bags/nodes", json={"wrong": "format"})
|
||||
assert resp.status_code == 400
|
||||
assert "Body must be a list" in resp.get_json()["error"]
|
||||
|
||||
|
||||
# ----------------- 3. 根据路径查 Bags -----------------
|
||||
def test_get_bags_by_fst_path_200(client: pytest.fixture):
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_bags_by_fst_paths",
|
||||
return_value={"fst": {"bags": ["bagA", "bagB"]}},
|
||||
):
|
||||
resp = client.get("/fst/bags/path/fst")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"fst": {"bags": ["bagA", "bagB"]}}
|
||||
|
||||
|
||||
# ----------------- 4. 根据名称查 FST -----------------
|
||||
def test_get_fst_node_by_name_200(client: pytest.fixture):
|
||||
fst_mock = MagicMock()
|
||||
fst_mock.name = "node"
|
||||
fst_mock.parent_id = 123
|
||||
|
||||
parent_mock = MagicMock()
|
||||
parent_mock.name = "parent_node"
|
||||
|
||||
with (
|
||||
patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_fst",
|
||||
return_value=fst_mock,
|
||||
),
|
||||
patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_fst_by_id",
|
||||
return_value=parent_mock,
|
||||
),
|
||||
patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_total_bag_sum",
|
||||
return_value=42,
|
||||
),
|
||||
):
|
||||
resp = client.get("/fst/node")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {
|
||||
"name": "node",
|
||||
"parent_node": "parent_node",
|
||||
"linked_bags_sum": 42,
|
||||
}
|
||||
|
||||
|
||||
def test_get_fst_node_by_name_404(client: pytest.fixture):
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_fst", return_value=None
|
||||
):
|
||||
resp = client.get("/fst/nonexistent")
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
def test_update_fst_bag__invalid_json(client: pytest.fixture):
|
||||
resp = client.post("/fst/bags/update", json={"not": "list"})
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
# ----------------- 7. 创建/更新 FST 节点 -----------------
|
||||
def test_upsert_fst__create_success(client: pytest.fixture):
|
||||
node_mock = MagicMock()
|
||||
node_mock.id = 99
|
||||
node_mock.name = "new_node"
|
||||
node_mock.parent_id = None
|
||||
node_mock.reserved_json = {"foo": "bar"}
|
||||
node_mock.bag_sum = 0
|
||||
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.upsert_fst_node",
|
||||
return_value=node_mock,
|
||||
):
|
||||
resp = client.post(
|
||||
"/fst/update",
|
||||
json={
|
||||
"name": "new_node",
|
||||
"parent_name": None,
|
||||
"reserved_json": {"foo": "bar"},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["name"] == "new_node"
|
||||
|
||||
|
||||
def test_upsert_fst__400_missing_name(client: pytest.fixture):
|
||||
resp = client.post("/fst/update", json={"parent_name": "root"})
|
||||
assert resp.status_code == 400
|
||||
assert "Field 'name' is required" in resp.get_json()["error"]
|
||||
|
||||
|
||||
# ----------------- 8. 删除 FST 节点 -----------------
|
||||
def test_delete_fst_by_name__success(client: pytest.fixture):
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.delete_fst_by_fst_name"
|
||||
) as mock_delete:
|
||||
resp = client.delete("/fst/some_node")
|
||||
mock_delete.assert_called_once()
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"ok": True}
|
||||
|
||||
|
||||
def test_delete_fst_by_name__404(client: pytest.fixture):
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.delete_fst_by_fst_name",
|
||||
side_effect=ValueError("not found"),
|
||||
):
|
||||
resp = client.delete("/fst/nonexistent")
|
||||
assert resp.status_code == 404
|
||||
assert "not found" in resp.get_json()["error"]
|
||||
|
||||
|
||||
# ----------------- 9. 获取 FST 树形结构与 bag 详情 -----------------
|
||||
def test_list_bag_names_with_fst__with_bags(client: pytest.fixture):
|
||||
# Mock返回数据库行格式 - 每行的第一列是bag名称
|
||||
mock_rows = [
|
||||
("sample_03.bag", "other_data"),
|
||||
("sample_01.bag", "other_data"),
|
||||
("sample_02.bag", "other_data"),
|
||||
("sample_01.bag", "other_data"), # 重复项，测试去重功能
|
||||
]
|
||||
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_bag_names_linked_to_fst",
|
||||
return_value=mock_rows,
|
||||
):
|
||||
resp = client.get("/fst/baglist")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
# 验证返回的是排序后的bag名称列表（去重）
|
||||
assert isinstance(data, list)
|
||||
assert data == ["sample_01.bag", "sample_02.bag", "sample_03.bag"]
|
||||
|
||||
|
||||
def test_list_bag_names_with_fst__empty_list(client: pytest.fixture):
|
||||
# Mock返回空的数据库结果
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.fst.get_bag_names_linked_to_fst",
|
||||
return_value=[],
|
||||
):
|
||||
resp = client.get("/fst/baglist")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
# 验证返回空列表
|
||||
assert isinstance(data, list)
|
||||
assert data == []
|
||||
@@ -0,0 +1,61 @@
|
||||
# tests/test_geometry.py
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api import geometry_bp
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
"""Flask test client with geometry blueprint registered."""
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(geometry_bp, url_prefix="/geometry")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
def test_get_geometry_info_by_rosbags_200(client, mocker):
|
||||
"""200 OK：查询到多条几何记录"""
|
||||
# 1. 伪造查询结果
|
||||
fake_rows = [
|
||||
MagicMock(
|
||||
rosbag_name="bag1", lon_lat_alt=[[12.34, 56.78, 0.0], [12.35, 56.79, 1.2]]
|
||||
),
|
||||
MagicMock(rosbag_name="bag2", lon_lat_alt=[[98.76, 54.32, 3.3]]),
|
||||
]
|
||||
|
||||
# 2. 拦截 SessionLocal 返回的 session 对象
|
||||
mock_session = MagicMock()
|
||||
mock_session.execute.return_value.fetchall.return_value = fake_rows
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.geometry.SessionLocal",
|
||||
return_value=mock_session,
|
||||
)
|
||||
|
||||
resp = client.post("/geometry/", json={"rosbag_names": ["bag1", "bag2"]})
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["bag1"] == [[12.34, 56.78, 0.0], [12.35, 56.79, 1.2]]
|
||||
assert data["bag2"] == [[98.76, 54.32, 3.3]]
|
||||
|
||||
|
||||
def test_get_geometry_info_by_rosbags_400(client, mocker):
|
||||
"""400 空列表或格式错误"""
|
||||
resp = client.post("/geometry/", json={"rosbag_names": []})
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
def test_get_geometry_info_by_rosbags_404(client, mocker):
|
||||
"""404 找不到任何记录"""
|
||||
mock_session = MagicMock()
|
||||
mock_session.execute.return_value.fetchall.return_value = []
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.geometry.SessionLocal",
|
||||
return_value=mock_session,
|
||||
)
|
||||
|
||||
resp = client.post("/geometry/", json={"rosbag_names": ["nonexistent"]})
|
||||
assert resp.status_code == 404
|
||||
@@ -0,0 +1,27 @@
|
||||
# tests/test_projects.py
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
from fst_data_pipeline.apps.root_db_api.src.api import api_bp
|
||||
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(api_bp, url_prefix="/api")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
# tests/test_projects.py
|
||||
def test_read_projects_200(client, mocker):
|
||||
fake = [{"id": 1, "name": "Alpha"}, {"id": 2, "name": "Beta"}]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.projects.get_projects",
|
||||
return_value=fake,
|
||||
)
|
||||
|
||||
resp = client.get("/api/projects/all")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == [{"id": 1, "name": "Alpha"}, {"id": 2, "name": "Beta"}]
|
||||
@@ -0,0 +1,185 @@
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api.recompute import bp as recompute_bp
|
||||
|
||||
|
||||
# ------------------ Flask test client ------------------
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
"""Flask test client with recompute blueprint registered."""
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(recompute_bp, url_prefix="/recompute")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
# ------------------ 单元测试用例 ------------------
|
||||
def test_get_bag_versions_200(client, mocker):
|
||||
"""测试批量获取bag版本列表 - 成功场景"""
|
||||
mock_result = {
|
||||
"bag1.bag": ["v1.0", "v1.1", "v2.0"],
|
||||
"bag2.bag": ["v1.0", "v2.0"],
|
||||
"bag3.bag": [],
|
||||
}
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.recompute.get_bag_recompute_versions",
|
||||
return_value=mock_result,
|
||||
)
|
||||
mocker.patch("fst_data_pipeline.apps.root_db_api.src.api.recompute.SessionLocal")
|
||||
|
||||
resp = client.post(
|
||||
"/recompute/versions", json={"bag_names": ["bag1.bag", "bag2.bag", "bag3.bag"]}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert data["bag1.bag"] == ["v1.0", "v1.1", "v2.0"]
|
||||
assert data["bag2.bag"] == ["v1.0", "v2.0"]
|
||||
assert data["bag3.bag"] == []
|
||||
|
||||
|
||||
def test_get_bag_versions_400_no_bag_names(client):
|
||||
"""测试批量获取bag版本列表 - 缺少bag_names"""
|
||||
resp = client.post("/recompute/versions", json={})
|
||||
assert resp.status_code == 400
|
||||
assert "bag_names list required" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_bag_versions_400_empty_bag_names(client):
|
||||
"""测试批量获取bag版本列表 - bag_names为空列表"""
|
||||
resp = client.post("/recompute/versions", json={"bag_names": []})
|
||||
assert resp.status_code == 400
|
||||
assert "bag_names must be a non-empty list" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_bag_versions_400_invalid_bag_names(client):
|
||||
"""测试批量获取bag版本列表 - bag_names不是列表"""
|
||||
resp = client.post("/recompute/versions", json={"bag_names": "not_a_list"})
|
||||
assert resp.status_code == 400
|
||||
assert "bag_names must be a non-empty list" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_bag_versions_500_exception(client, mocker):
|
||||
"""测试批量获取bag版本列表 - 服务器异常"""
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.recompute.get_bag_recompute_versions",
|
||||
side_effect=Exception("Database error"),
|
||||
)
|
||||
mocker.patch("fst_data_pipeline.apps.root_db_api.src.api.recompute.SessionLocal")
|
||||
|
||||
resp = client.post("/recompute/versions", json={"bag_names": ["bag1.bag"]})
|
||||
assert resp.status_code == 500
|
||||
assert "Database error" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_storage_paths_200(client, mocker):
|
||||
"""测试批量获取存储路径 - 成功场景"""
|
||||
mock_result = [
|
||||
{
|
||||
"bag_name": "bag1.bag",
|
||||
"recompute_version": "v1.0",
|
||||
"results": [
|
||||
{
|
||||
"result_id": 1,
|
||||
"bag_name": "bag1.bag",
|
||||
"recompute_version": "v1.0",
|
||||
"result_type": "perception",
|
||||
"storage_path": "/path/to/result1",
|
||||
"status": "available",
|
||||
"created_time": "2024-01-16T10:30:00Z",
|
||||
"reserved_json": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.recompute.get_storage_paths_batch",
|
||||
return_value=mock_result,
|
||||
)
|
||||
mocker.patch("fst_data_pipeline.apps.root_db_api.src.api.recompute.SessionLocal")
|
||||
|
||||
requests = [{"bag_name": "bag1.bag", "recompute_version": "v1.0"}]
|
||||
resp = client.post("/recompute/storage", json={"requests": requests})
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
assert len(data) == 1
|
||||
assert data[0]["bag_name"] == "bag1.bag"
|
||||
assert data[0]["recompute_version"] == "v1.0"
|
||||
assert len(data[0]["results"]) == 1
|
||||
assert data[0]["results"][0]["result_type"] == "perception"
|
||||
|
||||
|
||||
def test_get_storage_paths_400_no_requests(client):
|
||||
"""测试批量获取存储路径 - 缺少requests"""
|
||||
resp = client.post("/recompute/storage", json={})
|
||||
assert resp.status_code == 400
|
||||
assert "requests list required" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_storage_paths_400_empty_requests(client):
|
||||
"""测试批量获取存储路径 - requests为空列表"""
|
||||
resp = client.post("/recompute/storage", json={"requests": []})
|
||||
assert resp.status_code == 400
|
||||
assert "requests must be a non-empty list" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_storage_paths_400_invalid_requests(client):
|
||||
"""测试批量获取存储路径 - requests不是列表"""
|
||||
resp = client.post("/recompute/storage", json={"requests": "not_a_list"})
|
||||
assert resp.status_code == 400
|
||||
assert "requests must be a non-empty list" in resp.get_json()["error"]
|
||||
|
||||
|
||||
def test_get_storage_paths_400_invalid_request_format(client):
|
||||
"""测试批量获取存储路径 - 单个request格式错误"""
|
||||
invalid_requests = [
|
||||
{"bag_name": "bag1.bag"}, # 缺少recompute_version
|
||||
{"recompute_version": "v1.0"}, # 缺少bag_name
|
||||
{"wrong": "format"}, # 完全错误的格式
|
||||
]
|
||||
|
||||
for invalid_request in invalid_requests:
|
||||
resp = client.post("/recompute/storage", json={"requests": [invalid_request]})
|
||||
assert resp.status_code == 400
|
||||
assert (
|
||||
"Each request must contain bag_name and recompute_version"
|
||||
in resp.get_json()["error"]
|
||||
)
|
||||
|
||||
|
||||
def test_get_storage_paths_500_exception(client, mocker):
|
||||
"""测试批量获取存储路径 - 服务器异常"""
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.recompute.get_storage_paths_batch",
|
||||
side_effect=Exception("Service error"),
|
||||
)
|
||||
mocker.patch("fst_data_pipeline.apps.root_db_api.src.api.recompute.SessionLocal")
|
||||
|
||||
requests = [{"bag_name": "bag1.bag", "recompute_version": "v1.0"}]
|
||||
resp = client.post("/recompute/storage", json={"requests": requests})
|
||||
assert resp.status_code == 500
|
||||
assert "Service error" in resp.get_json()["error"]
|
||||
|
||||
|
||||
# ------------------ 参数化测试 ------------------
|
||||
@pytest.mark.parametrize(
|
||||
"endpoint,bad_body",
|
||||
[
|
||||
("/recompute/versions", {"bag_names": "not_list"}),
|
||||
("/recompute/versions", {"bag_names": []}),
|
||||
("/recompute/versions", {}),
|
||||
("/recompute/storage", {"requests": "not_list"}),
|
||||
("/recompute/storage", {"requests": []}),
|
||||
("/recompute/storage", {}),
|
||||
("/recompute/storage", {"requests": [{"invalid": "format"}]}),
|
||||
],
|
||||
)
|
||||
def test_bad_requests_400(client, endpoint, bad_body):
|
||||
"""参数化测试：各种错误请求格式"""
|
||||
resp = client.post(endpoint, json=bad_body)
|
||||
assert resp.status_code == 400
|
||||
113
fst_data_pipeline/apps/root_db_api/src/test/api/test_tags.py
Normal file
113
fst_data_pipeline/apps/root_db_api/src/test/api/test_tags.py
Normal file
@@ -0,0 +1,113 @@
|
||||
# tests/test_tags.py
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api.tags import bp as tags_bp
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
"""Flask test client with tags blueprint registered."""
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(tags_bp, url_prefix="/tags")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
# ---------- /tags/all ----------
|
||||
def test_read_tags_200(client, mocker):
|
||||
fake_tags = [
|
||||
{"name": "tag1", "type": "scenario", "creator": "alice"},
|
||||
{"name": "tag2", "type": "label", "creator": "bob"},
|
||||
]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.get_tags",
|
||||
return_value=fake_tags,
|
||||
)
|
||||
|
||||
resp = client.get("/tags/all")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == fake_tags
|
||||
|
||||
|
||||
# ---------- /tags/bags/<tag_name> ----------
|
||||
def test_get_bags_by_tags_and_200(client, mocker):
|
||||
fake_bags = [{"id": 1, "name": "foo.bag"}, {"id": 2, "name": "bar.bag"}]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.query_bags_by_tags",
|
||||
return_value=fake_bags,
|
||||
)
|
||||
|
||||
resp = client.get("/tags/bags?tags=foo,bar&op=and")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"bags": fake_bags}
|
||||
|
||||
|
||||
def test_get_bags_by_tags_or_200(client, mocker):
|
||||
fake_bags = [{"id": 3, "name": "baz.bag"}]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.query_bags_by_tags",
|
||||
return_value=fake_bags,
|
||||
)
|
||||
|
||||
resp = client.get("/tags/bags?tags=baz&op=or")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"bags": fake_bags}
|
||||
|
||||
|
||||
def test_get_bags_by_tags_default_op_and(client, mocker):
|
||||
"""不传 op 时默认按 and 处理"""
|
||||
fake_bags = [{"id": 4, "name": "default.bag"}]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.query_bags_by_tags",
|
||||
return_value=fake_bags,
|
||||
)
|
||||
|
||||
resp = client.get("/tags/bags?tags=default") # 不带 op 参数
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"bags": fake_bags}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad_tags", ["", " "])
|
||||
def test_get_bags_by_tags_missing_tags_param(client, bad_tags):
|
||||
resp = client.get(f"/tags/bags?tags={bad_tags}")
|
||||
assert resp.status_code == 400
|
||||
assert resp.get_json() == {"bags": []}
|
||||
|
||||
|
||||
def test_get_bags_by_tags_invalid_op(client):
|
||||
resp = client.get("/tags/bags?tags=A,B&op=invalid")
|
||||
assert resp.status_code == 400
|
||||
assert resp.get_json() == {"bags": []}
|
||||
|
||||
|
||||
# ---------- /tags/creators/<creator> ----------
|
||||
def test_get_tags_by_creator_200(client, mocker):
|
||||
fake_tags = [
|
||||
{"name": "tagA", "type": "quality", "creator": "alice"},
|
||||
{"name": "tagB", "type": "label", "creator": "alice"},
|
||||
]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.get_tags_by_creator",
|
||||
return_value=fake_tags,
|
||||
)
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.SessionLocal",
|
||||
return_value=mocker.MagicMock(),
|
||||
)
|
||||
|
||||
resp = client.get("/tags/creators/alice")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"tags": ["tagA", "tagB"]}
|
||||
|
||||
|
||||
def test_get_tags_by_creator_404(client, mocker):
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.tags.get_tags_by_creator",
|
||||
return_value=[],
|
||||
)
|
||||
|
||||
resp = client.get("/tags/creators/nobody")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"tags": []}
|
||||
@@ -0,0 +1,53 @@
|
||||
# tests/test_topics.py
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from fst_data_pipeline.apps.root_db_api.src.api.topics import bp as topics_bp
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
"""Flask test client with topics blueprint registered."""
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(topics_bp, url_prefix="/topics")
|
||||
app.config["TESTING"] = True
|
||||
with app.test_client() as c:
|
||||
yield c
|
||||
|
||||
|
||||
# ---------- /topics/all ----------
|
||||
def test_read_topics_200(client, mocker):
|
||||
fake_topics = [
|
||||
{"name": "topic1", "type": "scenario"},
|
||||
{"name": "topic2", "type": "label"},
|
||||
]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.topics.get_all_topics",
|
||||
return_value=fake_topics,
|
||||
)
|
||||
resp = client.get("/topics/all")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == fake_topics
|
||||
|
||||
|
||||
# ---------- /topics/bags/<topic_name> ----------
|
||||
def test_get_bags_by_topic_200(client, mocker):
|
||||
fake_bags = [{"name": "bagA"}, {"name": "bagB"}]
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.topics.get_bags_by_topic_name",
|
||||
return_value=fake_bags,
|
||||
)
|
||||
resp = client.get("/topics/bags/highway")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"name": ["bagA", "bagB"]}
|
||||
|
||||
|
||||
def test_get_bags_by_topic_empty(client, mocker):
|
||||
mocker.patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.api.topics.get_bags_by_topic_name",
|
||||
return_value=[],
|
||||
)
|
||||
|
||||
resp = client.get("/topics/bags/unknown")
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {"name": []}
|
||||
@@ -0,0 +1 @@
|
||||
# Test service package
|
||||
@@ -0,0 +1,703 @@
|
||||
# tests/test_services_mock_only.py
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# 把要测试的 service 函数全部一次性 import
|
||||
import fst_data_pipeline.apps.root_db_api.src.core.service as srv
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db():
|
||||
"""返回一个 mock Session，所有测试函数共用"""
|
||||
return MagicMock()
|
||||
|
||||
|
||||
# ---------- Project ----------
|
||||
def test_get_projects(db):
|
||||
p1 = MagicMock()
|
||||
p1.id, p1.name = 1, "proj1"
|
||||
|
||||
p2 = MagicMock()
|
||||
p2.id, p2.name = 2, "proj2"
|
||||
|
||||
db.query().all.return_value = [p1, p2]
|
||||
assert srv.get_projects(db) == [
|
||||
{"id": 1, "name": "proj1"},
|
||||
{"id": 2, "name": "proj2"},
|
||||
]
|
||||
|
||||
|
||||
# ---------- Bag ----------
|
||||
def test_get_bags(db):
|
||||
bag = MagicMock()
|
||||
bag.name = "bag1"
|
||||
bag.project = MagicMock()
|
||||
bag.project.name = "p1"
|
||||
|
||||
db.query().options().all.return_value = [bag]
|
||||
assert srv.get_bags(db) == [bag]
|
||||
|
||||
|
||||
def test_get_bags_by_conditions(db):
|
||||
q = db.query.return_value
|
||||
q.filter.return_value = q
|
||||
bag = MagicMock()
|
||||
bag.name = "foo"
|
||||
q.all.return_value = [bag]
|
||||
|
||||
res = srv.get_bags_by_conditions(db, name="bar", project_id=1)
|
||||
assert res == [bag]
|
||||
# 只要 filter 被调用了两次即可
|
||||
assert q.filter.call_count == 2
|
||||
|
||||
|
||||
# ---------- Topic ----------
|
||||
def test_get_all_topics(db):
|
||||
t = MagicMock()
|
||||
t.name, t.type = "/imu", "sensor_msgs/Imu"
|
||||
db.query().all.return_value = [t]
|
||||
assert srv.get_all_topics(db) == [{"name": "/imu", "type": "sensor_msgs/Imu"}]
|
||||
|
||||
|
||||
def test_get_bags_by_topic_name(db):
|
||||
bag = MagicMock()
|
||||
bag.name = "bag1"
|
||||
db.query().join().join().filter().all.return_value = [bag]
|
||||
assert srv.get_bags_by_topic_name(db, "imu") == [{"name": "bag1"}]
|
||||
|
||||
|
||||
def test_get_topics_by_bag_name(db):
|
||||
topic = MagicMock()
|
||||
topic.name = "/gps"
|
||||
topic.type = "sensor_msgs/NavSatFix"
|
||||
db.query().join().join().filter().all.return_value = [topic]
|
||||
assert srv.get_topics_by_bag_name(db, "bag1") == [topic]
|
||||
|
||||
|
||||
# ---------- Tag ----------
|
||||
def test_get_tags(db):
|
||||
tag = MagicMock()
|
||||
tag.name, tag.type, tag.creator = "tag1", "bool", "Alice"
|
||||
db.query().all.return_value = [tag]
|
||||
assert srv.get_tags(db) == [{"name": "tag1", "type": "bool", "creator": "Alice"}]
|
||||
|
||||
|
||||
def test_get_tags_by_creator(db):
|
||||
tag = MagicMock()
|
||||
tag.name = "t1"
|
||||
tag.type = "bool"
|
||||
tag.creator = "alice"
|
||||
db.query().filter().all.return_value = [tag]
|
||||
assert srv.get_tags_by_creator(db, "alice") == [
|
||||
{"name": "t1", "type": "bool", "creator": "alice"},
|
||||
]
|
||||
|
||||
|
||||
def test_get_bags_by_tag_name(db):
|
||||
bag = MagicMock()
|
||||
bag.name = "b1"
|
||||
db.query().join().join().filter().all.return_value = [bag]
|
||||
assert srv.get_bags_by_tag_name(db, "hot") == [{"name": "b1"}]
|
||||
|
||||
|
||||
def test_get_tags_by_bag_name(db):
|
||||
tag = MagicMock()
|
||||
tag.name = "urgent"
|
||||
db.query().join().join().filter().all.return_value = [tag]
|
||||
assert srv.get_tags_by_bag_name(db, "bag99") == [tag]
|
||||
|
||||
|
||||
# ---------- MainPangu ----------
|
||||
def test_get_main_pangu_by_conditions(db):
|
||||
from datetime import date
|
||||
|
||||
q = db.query.return_value
|
||||
q.filter.return_value = q
|
||||
q.join.return_value = q # Add join mock for BagList join
|
||||
q.limit.return_value = q
|
||||
rec = MagicMock()
|
||||
rec.keyword = "pangu1"
|
||||
rec.vehicle = "Lexus"
|
||||
rec.start_time = "20250101"
|
||||
rec.end_time = "20250102"
|
||||
q.all.return_value = [rec]
|
||||
|
||||
res = srv.get_main_pangu_by_conditions(
|
||||
db,
|
||||
start_time=date(2025, 1, 1), # Use date objects
|
||||
end_time=date(2025, 1, 2),
|
||||
vehicle="Lexus",
|
||||
keyword="pangu1",
|
||||
topics=None, # Add new parameters
|
||||
fst=None,
|
||||
limit=1000,
|
||||
)
|
||||
assert res == [rec]
|
||||
|
||||
|
||||
def test_get_main_pangu_by_name(db):
|
||||
rec = MagicMock()
|
||||
rec.name = "pangu_2024"
|
||||
db.query().filter().all.return_value = [rec]
|
||||
assert srv.get_main_pangu_by_name(db, "2024") == [rec]
|
||||
|
||||
|
||||
def test_get_pangu_details_by_bag_name(db):
|
||||
main = MagicMock()
|
||||
main.name = "bag1"
|
||||
main.vehicle = "Lexus"
|
||||
|
||||
bag = MagicMock()
|
||||
bag.id = 7
|
||||
bag.foo = "bar"
|
||||
|
||||
sec = MagicMock()
|
||||
sec.extra = "meta"
|
||||
|
||||
db.query().filter().first.side_effect = [main, bag, sec]
|
||||
res = srv.get_pangu_details_by_bag_name(db, "bag1")
|
||||
assert res["bag_name"] == "bag1"
|
||||
assert res["vehicle"] == "Lexus"
|
||||
assert res["foo"] == "bar"
|
||||
assert res["extra"] == "meta"
|
||||
|
||||
|
||||
# ---------- MainMinerva ----------
|
||||
def test_get_main_minerva_by_session_id(db):
|
||||
rec = MagicMock()
|
||||
rec.session_id = "session123"
|
||||
db.query().filter().all.return_value = [rec]
|
||||
assert srv.get_main_minerva_by_session_id(db, "123") == [rec]
|
||||
|
||||
|
||||
def test_get_minerva_details_by_bag_name(db):
|
||||
main = MagicMock()
|
||||
main.session_id = "s1"
|
||||
main.vin = "VIN001"
|
||||
|
||||
bag = MagicMock()
|
||||
bag.id = 5
|
||||
|
||||
sec = MagicMock()
|
||||
sec.meta = "data"
|
||||
|
||||
db.query().filter().first.side_effect = [main, bag, sec]
|
||||
res = srv.get_minerva_details_by_bag_name(db, "s1")
|
||||
assert res["bag_name"] == "s1"
|
||||
assert res["vin"] == "VIN001"
|
||||
assert res["meta"] == "data"
|
||||
|
||||
|
||||
# ---------- BagLifecycle ----------
|
||||
def test_get_bag_lifecycle_by_bag_names(db):
|
||||
lc = MagicMock()
|
||||
lc.bag_name = "b1"
|
||||
db.query().filter().all.return_value = [lc]
|
||||
assert srv.get_bag_lifecycle_by_bag_names(db, ["b1"]) == [lc]
|
||||
|
||||
|
||||
# ---------- GeometryInfo ----------
|
||||
def test_get_geometry_info(db):
|
||||
geom = MagicMock()
|
||||
db.query().all.return_value = [geom]
|
||||
assert srv.get_geometry_info(db) == [geom]
|
||||
|
||||
|
||||
def test_get_geometry_info_by_rosbag_name(db):
|
||||
db.query().filter().first.return_value = "geom_found"
|
||||
assert srv.get_geometry_info_by_rosbag_name(db, "rosbag1") == "geom_found"
|
||||
|
||||
|
||||
# ---------- FST ----------
|
||||
def test_get_fst(db):
|
||||
fst = MagicMock()
|
||||
fst.name = "root"
|
||||
db.query().filter().first.return_value = fst
|
||||
assert srv.get_fst(db, "root").name == "root"
|
||||
|
||||
|
||||
# ---------- FST ----------
|
||||
def test_get_total_bag_sum(db):
|
||||
q = MagicMock()
|
||||
db.query.return_value = q
|
||||
q.filter.return_value = q
|
||||
q.scalar.return_value = 42
|
||||
assert srv.get_total_bag_sum(db, "root") == 42
|
||||
|
||||
|
||||
def test_get_fst_by_id(db):
|
||||
fst = MagicMock()
|
||||
fst.id = 99
|
||||
db.query().filter().first.return_value = fst
|
||||
assert srv.get_fst_by_id(db, 99).id == 99
|
||||
|
||||
|
||||
def test_get_fst_nodes_by_bag_name(db):
|
||||
row = MagicMock()
|
||||
row.name, row.start, row.end, row.comments, row.video_url, row.img_url = (
|
||||
"fstA",
|
||||
10.0,
|
||||
20.0,
|
||||
"test",
|
||||
"video",
|
||||
"img",
|
||||
)
|
||||
db.query().select_from().join().join().filter().all.return_value = [row]
|
||||
res = srv.get_fst_nodes_by_bag_name(db, "bag1")
|
||||
assert res == [
|
||||
{
|
||||
"name": "fstA",
|
||||
"start": "10.0s",
|
||||
"end": "20.0s",
|
||||
"comments": "test",
|
||||
"video_url": "video",
|
||||
"img_url": "img",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_get_fst_path_by_bag_name(db):
|
||||
# 链：child -> parent -> root
|
||||
child = MagicMock()
|
||||
child.id, child.name, child.parent_id = 3, "child", 2
|
||||
|
||||
parent = MagicMock()
|
||||
parent.id, parent.name, parent.parent_id = 2, "parent", 1
|
||||
|
||||
root = MagicMock()
|
||||
root.id, root.name, root.parent_id = 1, "root", None
|
||||
|
||||
bag = MagicMock()
|
||||
bag.fst_node_id = 3
|
||||
|
||||
db.query().filter().first.side_effect = [bag, child, parent, root, None]
|
||||
path = srv.get_fst_path_by_bag_name(db, "bag1")
|
||||
expected = [
|
||||
{"id": 3, "name": "child"},
|
||||
{"id": 2, "name": "parent"},
|
||||
{"id": 1, "name": "root"},
|
||||
]
|
||||
assert path == expected
|
||||
|
||||
|
||||
def test_get_bags_by_fst_name(db):
|
||||
bag = MagicMock()
|
||||
bag.name = "bag_fst"
|
||||
db.query().join().join().filter().all.return_value = [bag]
|
||||
assert srv.get_bags_by_fst_name(db, "fst") == [bag]
|
||||
|
||||
|
||||
# ---------- query_bags_by_tags ----------
|
||||
def test_query_bags_by_tags_empty_tag_list(db):
|
||||
"""空标签列表立即返回空列表"""
|
||||
res = srv.query_bags_by_tags(db, [])
|
||||
assert res == []
|
||||
# 确保没有任何查询被调用
|
||||
db.query.assert_not_called()
|
||||
|
||||
|
||||
def test_query_bags_by_tags_and_op(db):
|
||||
"""AND 逻辑：bag 必须同时关联全部标签"""
|
||||
# 1. 先 mock 标签查询：假设前端传了 ["t1", "t2"]，库里有这两个标签
|
||||
tag1, tag2 = MagicMock(), MagicMock()
|
||||
tag1.id, tag2.id = 11, 22
|
||||
db.query.return_value.filter.return_value.all.return_value = [(11,), (22,)]
|
||||
|
||||
# 2. 再 mock bag 查询：只返回一个 bag 名 "bag_ok"
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.group_by.return_value = query_mock
|
||||
query_mock.having.return_value = query_mock
|
||||
query_mock.all.return_value = [("bag_ok",)]
|
||||
|
||||
res = srv.query_bags_by_tags(db, ["t1", "t2"], op="and")
|
||||
assert res == ["bag_ok"]
|
||||
|
||||
having_call = query_mock.having.call_args[0][0]
|
||||
assert query_mock.having.call_count == 1
|
||||
|
||||
|
||||
def test_query_bags_by_tags_or_op(db):
|
||||
"""OR 逻辑：bag 只要关联任一标签即可"""
|
||||
# 1. 标签查询同上
|
||||
db.query.return_value.filter.return_value.all.return_value = [(11,), (22,)]
|
||||
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.distinct.return_value = query_mock
|
||||
query_mock.all.return_value = [("bag_a",), ("bag_b",)]
|
||||
|
||||
res = srv.query_bags_by_tags(db, ["t1", "t2"], op="or")
|
||||
assert res == ["bag_a", "bag_b"]
|
||||
|
||||
# 验证调用了 distinct
|
||||
assert query_mock.distinct.call_count == 1
|
||||
|
||||
# ---------- get_bag_names_linked_to_fst ----------
|
||||
def test_get_bag_names_linked_to_fst_ok(db):
|
||||
"""成功场景：返回去重且按名字排序的 bag 名列表"""
|
||||
# 模拟 query 链式调用
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.order_by.return_value = query_mock
|
||||
# distinct 返回的是子查询对象，继续链式
|
||||
query_mock.distinct.return_value = query_mock
|
||||
# 最终 all() 给出结果
|
||||
query_mock.all.return_value = [("bag_z",), ("bag_a",)]
|
||||
|
||||
res = srv.get_bag_names_linked_to_fst(db)
|
||||
# 期望按 order_by 排好序
|
||||
assert res == [("bag_z",), ("bag_a",)]
|
||||
|
||||
# 验证调用了 distinct 和 order_by
|
||||
assert query_mock.distinct.call_count == 1
|
||||
query_mock.order_by.assert_called_once()
|
||||
|
||||
def test_get_bag_names_linked_to_fst_empty(db):
|
||||
"""结果为空时返回空列表"""
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.order_by.return_value = query_mock
|
||||
query_mock.distinct.return_value = query_mock
|
||||
query_mock.all.return_value = []
|
||||
|
||||
res = srv.get_bag_names_linked_to_fst(db)
|
||||
assert res == []
|
||||
|
||||
# ---------- print_fst_tree ----------
|
||||
def test_print_fst_tree_single_tree(db):
|
||||
"""单棵树：层级与字段正确"""
|
||||
db.query.return_value.all.return_value = [
|
||||
("1", "root", None),
|
||||
("2", "child", "1"),
|
||||
("3", "grand", "2"),
|
||||
]
|
||||
trees = srv.print_fst_tree(db)
|
||||
assert len(trees) == 1
|
||||
root = trees[0]
|
||||
assert root["id"] == "root"
|
||||
assert root["level"] == 0
|
||||
child = root["children"][0]
|
||||
assert child["id"] == "child"
|
||||
assert child["level"] == 1
|
||||
grand = child["children"][0]
|
||||
assert grand["id"] == "grand"
|
||||
assert grand["level"] == 2
|
||||
assert grand["children"] == []
|
||||
|
||||
def test_print_fst_tree_multi_trees(db):
|
||||
"""多棵树：返回顺序与根节点数量"""
|
||||
db.query.return_value.all.return_value = [
|
||||
("10", "treeA", None),
|
||||
("20", "treeB", None),
|
||||
("30", "subA", "10"),
|
||||
]
|
||||
trees = srv.print_fst_tree(db)
|
||||
assert len(trees) == 2
|
||||
assert [t["id"] for t in trees] == ["treeA", "treeB"]
|
||||
assert trees[0]["children"][0]["id"] == "subA"
|
||||
|
||||
def test_print_fst_tree_skip_dirty_data(db):
|
||||
"""脏数据（非数字 id/pid）自动跳过"""
|
||||
db.query.return_value.all.return_value = [
|
||||
("abc", "bad1", "1"), # 无效 id
|
||||
("1", "root", "xyz"), # 无效 pid，当根处理
|
||||
("2", "child", "1"), # 正常
|
||||
]
|
||||
trees = srv.print_fst_tree(db)
|
||||
# 仅保留有效节点，且 root 当成新根
|
||||
assert len(trees) == 1
|
||||
assert trees[0]["id"] == "root"
|
||||
assert trees[0]["children"][0]["id"] == "child"
|
||||
|
||||
|
||||
# ---------- batch_upsert_fst_bag ----------
|
||||
def test_batch_upsert_fst_bag_empty_items(db):
|
||||
"""空列表直接返回空"""
|
||||
res = srv.batch_upsert_fst_bag(db, [])
|
||||
assert res == []
|
||||
db.query.assert_not_called()
|
||||
|
||||
|
||||
def test_batch_upsert_fst_bag_normal(db, monkeypatch):
|
||||
"""正常流程：预取、建标签、去重、逐条 upsert"""
|
||||
# 预取阶段
|
||||
node_name2id = {"root": 1, "child": 2}
|
||||
tag_name2id = {"hot": 11}
|
||||
bag_name2id = {"bag1": 101}
|
||||
monkeypatch.setattr(
|
||||
srv, "_prefetch", lambda _db, _items: (node_name2id, tag_name2id, bag_name2id)
|
||||
)
|
||||
monkeypatch.setattr(srv, "_ensure_tags", lambda _db, _names, _m: None)
|
||||
|
||||
# 已存在关系
|
||||
db.query.return_value.filter.return_value.all.return_value = []
|
||||
|
||||
# 逐条 upsert mock
|
||||
fake_result = MagicMock()
|
||||
monkeypatch.setattr(srv, "_upsert_one_item", lambda *_args: fake_result)
|
||||
|
||||
items = [MagicMock(bag_name="bag1", nodes=["root", "child"], tags=["hot"])]
|
||||
res = srv.batch_upsert_fst_bag(db, items)
|
||||
assert res == [fake_result]
|
||||
|
||||
|
||||
# ---------- delete_fst_by_fst_name ----------
|
||||
def test_delete_fst_by_fst_name_not_found(db):
|
||||
"""节点不存在抛 ValueError"""
|
||||
db.query.return_value.filter.return_value.first.return_value = None
|
||||
with pytest.raises(ValueError, match="FST node 'no' not found"):
|
||||
srv.delete_fst_by_fst_name(db, "no")
|
||||
|
||||
|
||||
def test_delete_fst_by_fst_name_ok(db):
|
||||
"""存在节点：子节点变根，再删除"""
|
||||
node = MagicMock(id=99)
|
||||
db.query.return_value.filter.return_value.first.return_value = node
|
||||
db.execute = MagicMock()
|
||||
db.delete = MagicMock()
|
||||
|
||||
srv.delete_fst_by_fst_name(db, "to_del")
|
||||
|
||||
# 子节点 parent_id 被置空
|
||||
db.execute.assert_called_once()
|
||||
call = db.execute.call_args[0][0]
|
||||
assert "parent_id" in str(call)
|
||||
db.delete.assert_called_once_with(node)
|
||||
|
||||
|
||||
# ---------- upsert_fst_node ----------
|
||||
def test_upsert_fst_node_insert_root(db):
|
||||
"""新建根节点"""
|
||||
db.query.return_value.filter.return_value.first.return_value = None
|
||||
new_node = MagicMock()
|
||||
db.add = MagicMock()
|
||||
db.flush = MagicMock()
|
||||
|
||||
with patch(
|
||||
"fst_data_pipeline.apps.root_db_api.src.core.service.FST", return_value=new_node
|
||||
):
|
||||
node = srv.upsert_fst_node(db, name="root")
|
||||
assert node is new_node
|
||||
db.add.assert_called_once_with(new_node)
|
||||
db.flush.assert_called_once()
|
||||
|
||||
|
||||
def test_upsert_fst_node_update_with_parent(db):
|
||||
"""更新已有节点，同时换父节点"""
|
||||
old_parent = MagicMock(id=10)
|
||||
new_parent = MagicMock(id=20)
|
||||
node = MagicMock(parent_id=10)
|
||||
db.query.return_value.filter.return_value.first.side_effect = [new_parent, node]
|
||||
db.flush = MagicMock()
|
||||
|
||||
res = srv.upsert_fst_node(db, name="n1", parent_name="new_p", bag_sum=5)
|
||||
assert res.parent_id == 20
|
||||
assert res.bag_sum == 5
|
||||
db.flush.assert_called_once()
|
||||
|
||||
|
||||
def test_upsert_fst_node_parent_not_found(db):
|
||||
"""指定父节点不存在抛 ValueError"""
|
||||
db.query.return_value.filter.return_value.first.return_value = None
|
||||
with pytest.raises(ValueError, match="Parent FST node 'no' not found"):
|
||||
srv.upsert_fst_node(db, name="n1", parent_name="no")
|
||||
|
||||
|
||||
# ---------- Recompute ----------
|
||||
def test_get_bag_recompute_versions(db):
|
||||
"""测试获取bag的recompute版本列表"""
|
||||
# Mock join查询结果 - 返回bag_name和recompute_version对
|
||||
result1 = MagicMock()
|
||||
result1.bag_name = "bag1.bag"
|
||||
result1.recompute_version = "v1.0"
|
||||
|
||||
result2 = MagicMock()
|
||||
result2.bag_name = "bag1.bag"
|
||||
result2.recompute_version = "v1.1"
|
||||
|
||||
result3 = MagicMock()
|
||||
result3.bag_name = "bag2.bag"
|
||||
result3.recompute_version = "v1.0"
|
||||
|
||||
result4 = MagicMock()
|
||||
result4.bag_name = "bag2.bag"
|
||||
result4.recompute_version = "v2.0"
|
||||
|
||||
# Mock query chain
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.all.return_value = [result1, result2, result3, result4]
|
||||
|
||||
result = srv.get_bag_recompute_versions(db, ["bag1.bag", "bag2.bag"])
|
||||
|
||||
expected = {"bag1.bag": ["v1.0", "v1.1"], "bag2.bag": ["v1.0", "v2.0"]}
|
||||
assert result == expected
|
||||
|
||||
|
||||
def test_get_bag_recompute_versions_empty_bags(db):
|
||||
"""测试空bag列表的处理"""
|
||||
# Mock query chain - 空bag列表仍会执行查询，但filter为空
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.all.return_value = []
|
||||
|
||||
result = srv.get_bag_recompute_versions(db, [])
|
||||
assert result == {}
|
||||
|
||||
|
||||
def test_get_bag_recompute_versions_bag_not_found(db):
|
||||
"""测试bag不存在的情况"""
|
||||
# Mock 空的查询结果
|
||||
db.query.return_value.filter.return_value.all.return_value = []
|
||||
|
||||
result = srv.get_bag_recompute_versions(db, ["nonexistent.bag"])
|
||||
assert result == {"nonexistent.bag": []}
|
||||
|
||||
|
||||
def test_get_storage_paths_batch(db):
|
||||
"""测试批量获取存储路径"""
|
||||
from datetime import datetime
|
||||
|
||||
# Mock join查询结果 - 使用实际查询中select的字段名
|
||||
result1 = MagicMock()
|
||||
result1.result_id = 1
|
||||
result1.bag_name = "bag1.bag"
|
||||
result1.recompute_version = "v1.0"
|
||||
result1.result_type = "perception"
|
||||
result1.storage_path = "/path/to/result1"
|
||||
result1.status = "available"
|
||||
result1.created_time = datetime.fromisoformat("2024-01-16T10:30:00")
|
||||
result1.reserved_json = {}
|
||||
|
||||
# Mock query chain
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.order_by.return_value = query_mock
|
||||
query_mock.all.return_value = [result1]
|
||||
|
||||
requests = [{"bag_name": "bag1.bag", "recompute_version": "v1.0"}]
|
||||
results = srv.get_storage_paths_batch(db, requests)
|
||||
|
||||
expected = [
|
||||
{
|
||||
"bag_name": "bag1.bag",
|
||||
"recompute_version": "v1.0",
|
||||
"results": [
|
||||
{
|
||||
"result_id": 1,
|
||||
"bag_name": "bag1.bag",
|
||||
"recompute_version": "v1.0",
|
||||
"result_type": "perception",
|
||||
"storage_path": "/path/to/result1",
|
||||
"status": "available",
|
||||
"created_time": "2024-01-16T10:30:00",
|
||||
"reserved_json": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
assert results == expected
|
||||
|
||||
|
||||
def test_get_storage_paths_batch_empty_requests(db):
|
||||
"""测试空请求列表"""
|
||||
result = srv.get_storage_paths_batch(db, [])
|
||||
assert result == []
|
||||
# 确保没有数据库查询
|
||||
db.query.assert_not_called()
|
||||
|
||||
|
||||
def test_get_storage_paths_batch_no_results(db):
|
||||
"""测试没有找到结果的情况"""
|
||||
# Mock 空的查询结果
|
||||
query_mock = db.query.return_value
|
||||
query_mock.join.return_value = query_mock
|
||||
query_mock.filter.return_value = query_mock
|
||||
query_mock.order_by.return_value = query_mock
|
||||
query_mock.all.return_value = []
|
||||
|
||||
requests = [{"bag_name": "nonexistent.bag", "recompute_version": "v1.0"}]
|
||||
results = srv.get_storage_paths_batch(db, requests)
|
||||
|
||||
expected = [
|
||||
{"bag_name": "nonexistent.bag", "recompute_version": "v1.0", "results": []}
|
||||
]
|
||||
assert results == expected
|
||||
|
||||
|
||||
def test_query_all_joined_bags(db):
|
||||
"""测试查询所有合并bags的parent-children关系"""
|
||||
# 直接Mock三个查询的返回值，避免复杂的side_effect逻辑
|
||||
|
||||
# Mock第一个查询：获取所有parent_id
|
||||
first_query = MagicMock()
|
||||
first_query.distinct.return_value.scalar_subquery.return_value = [1, 2]
|
||||
|
||||
# Mock第二个查询：获取parent_id到children_name的映射
|
||||
second_query = MagicMock()
|
||||
second_query.join.return_value = second_query
|
||||
second_query.filter.return_value = second_query
|
||||
second_query.order_by.return_value = second_query
|
||||
second_query.all.return_value = [
|
||||
(1, "child1.bag"),
|
||||
(1, "child2.bag"),
|
||||
(2, "child3.bag"),
|
||||
]
|
||||
|
||||
# Mock第三个查询：获取parent_id到parent_name的映射
|
||||
third_query = MagicMock()
|
||||
third_query.filter.return_value = third_query
|
||||
third_query.all.return_value = [
|
||||
(1, "parent1.bag"),
|
||||
(2, "parent2.bag"),
|
||||
]
|
||||
|
||||
# 使用side_effect返回不同的mock对象
|
||||
db.query.side_effect = [first_query, second_query, third_query]
|
||||
|
||||
result = srv.query_all_joined_bags(db)
|
||||
|
||||
# 验证结果结构
|
||||
assert isinstance(result, dict)
|
||||
assert "parent1.bag" in result
|
||||
assert "parent2.bag" in result
|
||||
assert result["parent1.bag"] == ["child1.bag", "child2.bag"]
|
||||
assert result["parent2.bag"] == ["child3.bag"]
|
||||
|
||||
|
||||
def test_query_all_joined_bags_empty(db):
|
||||
"""测试查询所有合并bags - 空结果"""
|
||||
from fst_data_pipeline.apps.root_db_api.src.core.models import JoinedBags
|
||||
|
||||
# Mock空结果
|
||||
db.query.return_value.distinct.return_value.scalar_subquery.return_value = (
|
||||
MagicMock()
|
||||
)
|
||||
|
||||
joined_query_mock = MagicMock()
|
||||
joined_query_mock.join.return_value = joined_query_mock
|
||||
joined_query_mock.filter.return_value = joined_query_mock
|
||||
joined_query_mock.order_by.return_value = joined_query_mock
|
||||
joined_query_mock.all.return_value = []
|
||||
|
||||
def query_side_effect(*args, **kwargs):
|
||||
if len(args) == 1 and args[0] == JoinedBags.parent_id:
|
||||
return db.query.return_value
|
||||
else:
|
||||
return joined_query_mock
|
||||
|
||||
db.query.side_effect = query_side_effect
|
||||
|
||||
result = srv.query_all_joined_bags(db)
|
||||
|
||||
# 验证空结果
|
||||
assert result == {}
|
||||
1154
fst_data_pipeline/apps/root_db_api/uv.lock
generated
Normal file
1154
fst_data_pipeline/apps/root_db_api/uv.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user