407 lines
15 KiB
Python
407 lines
15 KiB
Python
|
"""
|
||
|
|
全国标准信息公共服务平台 - 行业标准(汽车)数据采集脚本
|
||
|
|
数据来源: https://std.samr.gov.cn/hb/hbQuery?initnode=QC%20%E6%B1%BD%E8%BD%A6
|
||
|
|
API: GET https://std.samr.gov.cn/hb/search/hbPage
|
||
|
|
|
||
|
|
功能:
|
||
|
|
1. 全量采集 QC 汽车行业标准数据 (共约 990 条)
|
||
|
|
2. 采集字段: 标准号、标准名称、发布日期、实施日期、所属行业、标准状态、
|
||
|
|
标准性质、标准类别、归口单位、发布文号、CCS分类、ICS分类、制修定、备案号 等
|
||
|
|
3. 支持关键词搜索
|
||
|
|
4. 支持断点续采
|
||
|
|
5. 导出为格式化的 Excel 文件 (含统计 Sheet)
|
||
|
|
|
||
|
|
用法:
|
||
|
|
python samr_qc_scraper.py # 全量采集 QC 汽车行业标准
|
||
|
|
python samr_qc_scraper.py --search "制动" # 搜索关键词
|
||
|
|
python samr_qc_scraper.py --industry "QC 汽车" # 指定行业 (默认 QC 汽车)
|
||
|
|
python samr_qc_scraper.py --resume # 断点续采
|
||
|
|
python samr_qc_scraper.py --page-size 50 # 每页50条
|
||
|
|
python samr_qc_scraper.py --output 自定义名称 # 自定义输出文件名
|
||
|
|
"""
|
||
|
|
|
||
|
|
import sys
|
||
|
|
import io
|
||
|
|
import os
|
||
|
|
import json
|
||
|
|
import time
|
||
|
|
import argparse
|
||
|
|
from datetime import datetime
|
||
|
|
|
||
|
|
import requests
|
||
|
|
from openpyxl import Workbook
|
||
|
|
from openpyxl.styles import Font, Alignment, PatternFill, Border, Side
|
||
|
|
from openpyxl.utils import get_column_letter
|
||
|
|
|
||
|
|
# ─── Windows 控制台中文输出修复 ─────────────────────────
|
||
|
|
if sys.platform == "win32":
|
||
|
|
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
|
||
|
|
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")
|
||
|
|
|
||
|
|
# ─── 配置 ───────────────────────────────────────────────
|
||
|
|
API_URL = "https://std.samr.gov.cn/hb/search/hbPage"
|
||
|
|
DETAIL_URL = "https://std.samr.gov.cn/hb/search/stdHBDetailed?id={id}"
|
||
|
|
SOURCE_PAGE = "https://std.samr.gov.cn/hb/hbQuery?initnode=QC%20%E6%B1%BD%E8%BD%A6"
|
||
|
|
DEFAULT_PAGE_SIZE = 50
|
||
|
|
MAX_RETRIES = 5
|
||
|
|
RETRY_DELAY = 3
|
||
|
|
REQUEST_TIMEOUT = 30
|
||
|
|
CACHE_FILE = ".samr_qc_cache.json"
|
||
|
|
|
||
|
|
HEADERS = {
|
||
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||
|
|
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||
|
|
"Accept": "application/json, text/plain, */*",
|
||
|
|
"Referer": "https://std.samr.gov.cn/hb/hbQuery?initnode=QC%20%E6%B1%BD%E8%BD%A6",
|
||
|
|
"X-Requested-With": "XMLHttpRequest",
|
||
|
|
}
|
||
|
|
|
||
|
|
# 字段映射: API 字段 → 中文显示名
|
||
|
|
FIELD_MAP = {
|
||
|
|
"C_STD_CODE": "标准号",
|
||
|
|
"C_NAME": "标准名称",
|
||
|
|
"ISSUE_DATE": "发布日期",
|
||
|
|
"ACT_DATE": "实施日期",
|
||
|
|
"TRADE_DEPT": "所属行业",
|
||
|
|
"STATE": "标准状态",
|
||
|
|
"STD_NATURE": "标准性质",
|
||
|
|
"STD_CATEGORY": "标准类别",
|
||
|
|
"CHARGE_DEPT": "归口单位",
|
||
|
|
"NOTICE_NO": "发布文号",
|
||
|
|
"CCS": "CCS分类",
|
||
|
|
"ICS": "ICS分类",
|
||
|
|
"ICS_NAME1_1": "ICS分类名称",
|
||
|
|
"STD_ZXD": "制修定",
|
||
|
|
"RECORD_NO": "备案号",
|
||
|
|
"STD_LEVEL": "标准层级",
|
||
|
|
"STD_DOMAIN": "标准领域",
|
||
|
|
"id": "标准ID",
|
||
|
|
}
|
||
|
|
|
||
|
|
# Excel 输出列定义 (列名, 列宽)
|
||
|
|
OUTPUT_COLUMNS = [
|
||
|
|
("标准号", 20),
|
||
|
|
("标准名称", 52),
|
||
|
|
("发布日期", 13),
|
||
|
|
("实施日期", 13),
|
||
|
|
("所属行业", 12),
|
||
|
|
("标准状态", 12),
|
||
|
|
("标准性质", 10),
|
||
|
|
("标准类别", 12),
|
||
|
|
("归口单位", 20),
|
||
|
|
("发布文号", 18),
|
||
|
|
("CCS分类", 10),
|
||
|
|
("ICS分类", 10),
|
||
|
|
("ICS分类名称", 20),
|
||
|
|
("制修定", 8),
|
||
|
|
("备案号", 16),
|
||
|
|
("标准层级", 10),
|
||
|
|
("标准领域", 10),
|
||
|
|
("详情链接", 55),
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
# ─── 网络请求 ───────────────────────────────────────────
|
||
|
|
def fetch_page(session, page_num, page_size, industry="QC 汽车"):
|
||
|
|
"""请求单页数据"""
|
||
|
|
params = {
|
||
|
|
"op": industry,
|
||
|
|
"ISSUE_DATE": "",
|
||
|
|
"pageNumber": page_num,
|
||
|
|
"pageSize": page_size,
|
||
|
|
}
|
||
|
|
for attempt in range(1, MAX_RETRIES + 1):
|
||
|
|
try:
|
||
|
|
resp = session.get(API_URL, params=params, headers=HEADERS, timeout=REQUEST_TIMEOUT)
|
||
|
|
resp.raise_for_status()
|
||
|
|
data = json.loads(resp.content.decode("utf-8"))
|
||
|
|
return data
|
||
|
|
except Exception as e:
|
||
|
|
print(f" [!] 第 {page_num} 页请求失败 (第 {attempt}/{MAX_RETRIES} 次): {e}")
|
||
|
|
if attempt < MAX_RETRIES:
|
||
|
|
time.sleep(RETRY_DELAY * attempt)
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def normalize_record(record):
|
||
|
|
"""标准化单条记录"""
|
||
|
|
result = {}
|
||
|
|
for api_key, cn_name in FIELD_MAP.items():
|
||
|
|
val = record.get(api_key, "")
|
||
|
|
if isinstance(val, str):
|
||
|
|
val = val.strip()
|
||
|
|
result[cn_name] = val if val else ""
|
||
|
|
# 补充详情链接
|
||
|
|
std_id = record.get("id", "")
|
||
|
|
result["详情链接"] = DETAIL_URL.format(id=std_id) if std_id else ""
|
||
|
|
# 清理 ICS 分类名称 (格式: "43_道路车辆工程" → "道路车辆工程")
|
||
|
|
ics_name = result.get("ICS分类名称", "")
|
||
|
|
if "_" in ics_name:
|
||
|
|
result["ICS分类名称"] = ics_name.split("_")[-1] or ics_name
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
# ─── 缓存 ───────────────────────────────────────────────
|
||
|
|
def load_cache():
|
||
|
|
if os.path.exists(CACHE_FILE):
|
||
|
|
try:
|
||
|
|
with open(CACHE_FILE, "r", encoding="utf-8") as f:
|
||
|
|
return json.load(f)
|
||
|
|
except (json.JSONDecodeError, IOError):
|
||
|
|
pass
|
||
|
|
return {"records": [], "last_page": 0, "industry": "", "total": 0}
|
||
|
|
|
||
|
|
|
||
|
|
def save_cache(cache):
|
||
|
|
with open(CACHE_FILE, "w", encoding="utf-8") as f:
|
||
|
|
json.dump(cache, f, ensure_ascii=False)
|
||
|
|
|
||
|
|
|
||
|
|
# ─── 主采集流程 ─────────────────────────────────────────
|
||
|
|
def scrape_all(page_size, industry="QC 汽车", search_keyword="", resume=False):
|
||
|
|
"""采集所有数据"""
|
||
|
|
session = requests.Session()
|
||
|
|
all_records = []
|
||
|
|
start_page = 1
|
||
|
|
total = 0
|
||
|
|
|
||
|
|
if resume:
|
||
|
|
cache = load_cache()
|
||
|
|
if cache["records"] and cache["industry"] == industry:
|
||
|
|
all_records = cache["records"]
|
||
|
|
start_page = cache["last_page"] + 1
|
||
|
|
total = cache["total"]
|
||
|
|
print(f" [*] 从缓存恢复: 已有 {len(all_records)} 条, 从第 {start_page} 页继续")
|
||
|
|
else:
|
||
|
|
print(" [*] 缓存不匹配, 从头开始采集")
|
||
|
|
|
||
|
|
# 首次请求, 获取总数
|
||
|
|
first_data = fetch_page(session, start_page, page_size, industry)
|
||
|
|
if not first_data:
|
||
|
|
print(" [!] 无法获取数据, 请检查网络")
|
||
|
|
return all_records
|
||
|
|
|
||
|
|
total = first_data.get("total", 0)
|
||
|
|
rows = first_data.get("rows") or []
|
||
|
|
total_pages = (total + page_size - 1) // page_size
|
||
|
|
|
||
|
|
records = [normalize_record(r) for r in rows]
|
||
|
|
all_records.extend(records)
|
||
|
|
|
||
|
|
print(f" 总计: {total} 条标准, 共 {total_pages} 页, 每页 {page_size} 条")
|
||
|
|
print(f" 已采集: {len(all_records)}/{total}", end="\r")
|
||
|
|
|
||
|
|
cache = {"records": all_records, "last_page": start_page, "industry": industry, "total": total}
|
||
|
|
save_cache(cache)
|
||
|
|
|
||
|
|
for page_num in range(start_page + 1, total_pages + 1):
|
||
|
|
data = fetch_page(session, page_num, page_size, industry)
|
||
|
|
if data is None:
|
||
|
|
print(f"\n [!] 第 {page_num} 页失败, 保存进度退出")
|
||
|
|
save_cache(cache)
|
||
|
|
break
|
||
|
|
|
||
|
|
rows = data.get("rows") or []
|
||
|
|
records = [normalize_record(r) for r in rows]
|
||
|
|
all_records.extend(records)
|
||
|
|
|
||
|
|
cache["records"] = all_records
|
||
|
|
cache["last_page"] = page_num
|
||
|
|
save_cache(cache)
|
||
|
|
|
||
|
|
print(f" 已采集: {len(all_records)}/{total} (第 {page_num}/{total_pages} 页)", end="\r")
|
||
|
|
time.sleep(0.3)
|
||
|
|
|
||
|
|
print()
|
||
|
|
return all_records
|
||
|
|
|
||
|
|
|
||
|
|
# ─── Excel 导出 ─────────────────────────────────────────
|
||
|
|
def export_to_excel(records, output_path):
|
||
|
|
wb = Workbook()
|
||
|
|
ws = wb.active
|
||
|
|
ws.title = "汽车行业标准"
|
||
|
|
|
||
|
|
# ── 样式定义 ──
|
||
|
|
hdr_font = Font(name="微软雅黑", bold=True, color="FFFFFF", size=11)
|
||
|
|
hdr_fill = PatternFill(start_color="2F5496", end_color="2F5496", fill_type="solid")
|
||
|
|
hdr_align = Alignment(horizontal="center", vertical="center", wrap_text=True)
|
||
|
|
dat_font = Font(name="微软雅黑", size=10)
|
||
|
|
dat_align = Alignment(vertical="center", wrap_text=True)
|
||
|
|
even_fill = PatternFill(start_color="D6E4F0", end_color="D6E4F0", fill_type="solid")
|
||
|
|
border = Border(
|
||
|
|
left=Side(style="thin", color="B4C6E7"),
|
||
|
|
right=Side(style="thin", color="B4C6E7"),
|
||
|
|
top=Side(style="thin", color="B4C6E7"),
|
||
|
|
bottom=Side(style="thin", color="B4C6E7"),
|
||
|
|
)
|
||
|
|
|
||
|
|
# ── 标题行 ──
|
||
|
|
col_names = [c[0] for c in OUTPUT_COLUMNS]
|
||
|
|
for ci, name in enumerate(col_names, 1):
|
||
|
|
cell = ws.cell(row=1, column=ci, value=name)
|
||
|
|
cell.font = hdr_font
|
||
|
|
cell.fill = hdr_fill
|
||
|
|
cell.alignment = hdr_align
|
||
|
|
cell.border = border
|
||
|
|
|
||
|
|
# ── 数据行 ──
|
||
|
|
for ri, rec in enumerate(records, 2):
|
||
|
|
for ci, (col_name, _) in enumerate(OUTPUT_COLUMNS, 1):
|
||
|
|
val = rec.get(col_name, "")
|
||
|
|
cell = ws.cell(row=ri, column=ci, value=val)
|
||
|
|
cell.font = dat_font
|
||
|
|
cell.alignment = dat_align
|
||
|
|
cell.border = border
|
||
|
|
if ri % 2 == 0:
|
||
|
|
cell.fill = even_fill
|
||
|
|
|
||
|
|
# ── 列宽 ──
|
||
|
|
for ci, (_, w) in enumerate(OUTPUT_COLUMNS, 1):
|
||
|
|
ws.column_dimensions[get_column_letter(ci)].width = w
|
||
|
|
|
||
|
|
# ── 冻结 & 筛选 ──
|
||
|
|
ws.freeze_panes = "A2"
|
||
|
|
ws.auto_filter.ref = f"A1:{get_column_letter(len(col_names))}{len(records) + 1}"
|
||
|
|
|
||
|
|
# ── 统计信息 Sheet ──
|
||
|
|
ws_stat = wb.create_sheet("统计信息")
|
||
|
|
|
||
|
|
status_count = {}
|
||
|
|
nature_count = {}
|
||
|
|
category_count = {}
|
||
|
|
zxd_count = {}
|
||
|
|
year_count = {}
|
||
|
|
|
||
|
|
for r in records:
|
||
|
|
for field, target in [
|
||
|
|
("标准状态", status_count),
|
||
|
|
("标准性质", nature_count),
|
||
|
|
("标准类别", category_count),
|
||
|
|
("制修定", zxd_count),
|
||
|
|
]:
|
||
|
|
v = r.get(field, "未知") or "未知"
|
||
|
|
target[v] = target.get(v, 0) + 1
|
||
|
|
|
||
|
|
issue = r.get("发布日期", "") or ""
|
||
|
|
year = issue[:4] if len(issue) >= 4 else "未知"
|
||
|
|
year_count[year] = year_count.get(year, 0) + 1
|
||
|
|
|
||
|
|
stat_rows = [
|
||
|
|
("采集时间", datetime.now().strftime("%Y-%m-%d %H:%M:%S")),
|
||
|
|
("数据来源", "全国标准信息公共服务平台 (std.samr.gov.cn)"),
|
||
|
|
("所属行业", "QC 汽车"),
|
||
|
|
("标准总数", len(records)),
|
||
|
|
("", ""),
|
||
|
|
("── 标准状态分布 ──", ""),
|
||
|
|
("状态", "数量"),
|
||
|
|
]
|
||
|
|
for k, v in sorted(status_count.items(), key=lambda x: -x[1]):
|
||
|
|
stat_rows.append((k, v))
|
||
|
|
|
||
|
|
stat_rows += [("", ""), ("── 标准性质分布 ──", ""), ("性质", "数量")]
|
||
|
|
for k, v in sorted(nature_count.items(), key=lambda x: -x[1]):
|
||
|
|
stat_rows.append((k, v))
|
||
|
|
|
||
|
|
stat_rows += [("", ""), ("── 标准类别分布 ──", ""), ("类别", "数量")]
|
||
|
|
for k, v in sorted(category_count.items(), key=lambda x: -x[1]):
|
||
|
|
stat_rows.append((k, v))
|
||
|
|
|
||
|
|
stat_rows += [("", ""), ("── 制修定分布 ──", ""), ("类型", "数量")]
|
||
|
|
for k, v in sorted(zxd_count.items(), key=lambda x: -x[1]):
|
||
|
|
stat_rows.append((k, v))
|
||
|
|
|
||
|
|
stat_rows += [("", ""), ("── 按发布年份分布 ──", ""), ("年份", "数量")]
|
||
|
|
for y, c in sorted(year_count.items(), reverse=True):
|
||
|
|
stat_rows.append((y, c))
|
||
|
|
|
||
|
|
for ri, (a, b) in enumerate(stat_rows, 1):
|
||
|
|
ca = ws_stat.cell(row=ri, column=1, value=a)
|
||
|
|
cb = ws_stat.cell(row=ri, column=2, value=b)
|
||
|
|
if a.startswith("──"):
|
||
|
|
ca.font = Font(name="微软雅黑", bold=True, size=11)
|
||
|
|
else:
|
||
|
|
ca.font = Font(name="微软雅黑", size=10)
|
||
|
|
cb.font = Font(name="微软雅黑", size=10)
|
||
|
|
|
||
|
|
ws_stat.column_dimensions["A"].width = 28
|
||
|
|
ws_stat.column_dimensions["B"].width = 50
|
||
|
|
|
||
|
|
wb.save(output_path)
|
||
|
|
|
||
|
|
|
||
|
|
# ─── 入口 ───────────────────────────────────────────────
|
||
|
|
def main():
|
||
|
|
parser = argparse.ArgumentParser(
|
||
|
|
description="全国标准信息公共服务平台 — 行业标准(汽车)数据采集工具",
|
||
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||
|
|
epilog="""
|
||
|
|
示例:
|
||
|
|
python samr_qc_scraper.py # 全量采集
|
||
|
|
python samr_qc_scraper.py --resume # 断点续采
|
||
|
|
python samr_qc_scraper.py --page-size 50 # 每页50条
|
||
|
|
python samr_qc_scraper.py --output QC汽车标准 # 自定义文件名
|
||
|
|
""",
|
||
|
|
)
|
||
|
|
parser.add_argument("--industry", "-i", default="QC 汽车", help="行业筛选 (默认: QC 汽车)")
|
||
|
|
parser.add_argument("--resume", "-r", action="store_true", help="断点续采")
|
||
|
|
parser.add_argument("--page-size", "-p", type=int, default=DEFAULT_PAGE_SIZE, help="每页条数 (默认50)")
|
||
|
|
parser.add_argument("--output", "-o", default=None, help="输出文件名 (不含扩展名)")
|
||
|
|
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
if args.output:
|
||
|
|
output_name = args.output
|
||
|
|
else:
|
||
|
|
output_name = f"行业标准_QC汽车_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
||
|
|
output_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), f"{output_name}.xlsx")
|
||
|
|
|
||
|
|
print("=" * 60)
|
||
|
|
print(" 全国标准信息公共服务平台 — 行业标准数据采集工具")
|
||
|
|
print("=" * 60)
|
||
|
|
print(f" 数据来源: std.samr.gov.cn")
|
||
|
|
print(f" 所属行业: {args.industry}")
|
||
|
|
print(f" 每页条数: {args.page_size}")
|
||
|
|
print(f" 输出文件: {output_path}")
|
||
|
|
print("-" * 60)
|
||
|
|
|
||
|
|
start_time = time.time()
|
||
|
|
records = scrape_all(args.page_size, args.industry, resume=args.resume)
|
||
|
|
elapsed = time.time() - start_time
|
||
|
|
|
||
|
|
if not records:
|
||
|
|
print(" 未获取到任何数据")
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
# 去重
|
||
|
|
seen = set()
|
||
|
|
unique = []
|
||
|
|
for r in records:
|
||
|
|
code = r.get("标准号", "")
|
||
|
|
if code not in seen:
|
||
|
|
seen.add(code)
|
||
|
|
unique.append(r)
|
||
|
|
dup = len(records) - len(unique)
|
||
|
|
|
||
|
|
print("-" * 60)
|
||
|
|
print(f" 采集完成! 用时 {elapsed:.1f} 秒")
|
||
|
|
print(f" 获取 {len(records)} 条, 去重后 {len(unique)} 条", end="")
|
||
|
|
print(f" (移除 {dup} 条重复)" if dup else "")
|
||
|
|
|
||
|
|
print(" 生成 Excel 文件...")
|
||
|
|
export_to_excel(unique, output_path)
|
||
|
|
size = os.path.getsize(output_path) / 1024
|
||
|
|
|
||
|
|
print(f"\n {'=' * 50}")
|
||
|
|
print(f" 导出完成: {output_path}")
|
||
|
|
print(f" 文件大小: {size:.1f} KB")
|
||
|
|
print(f" 标准总数: {len(unique)}")
|
||
|
|
print(f" {'=' * 50}")
|
||
|
|
|
||
|
|
if not args.resume and os.path.exists(CACHE_FILE):
|
||
|
|
os.remove(CACHE_FILE)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|