392 lines
16 KiB
Python
392 lines
16 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
将 jsons/compressed_group_time_issues.json 导入 MySQL。
|
||
|
||
只导入以下四类，并把同一 row 中二一教育和学科网的数据合并为同一条记录：
|
||
- single_files
|
||
- same_child_archive_time
|
||
- zxxk_child_earlier
|
||
- twole_child_earlier
|
||
"""
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Dict, Iterable, List, Optional, Tuple
|
||
|
||
import mysql.connector
|
||
from mysql.connector import Error
|
||
|
||
|
||
DB_CONFIG = {
|
||
'host': '127.0.0.1',
|
||
'port': 3306,
|
||
'user': 'root',
|
||
'password': 'root123456',
|
||
'database': 'testdb',
|
||
'charset': 'utf8mb4'
|
||
}
|
||
|
||
BASE_DIR = Path(__file__).resolve().parent
|
||
DEFAULT_JSON_FILE_PATH = BASE_DIR / "jsons" / "compressed_group_time_issues.json"
|
||
DEFAULT_TABLE_NAME = "compressed_group_time_records"
|
||
SELECTED_CATEGORIES = (
|
||
"single_files",
|
||
"same_child_archive_time",
|
||
"zxxk_child_earlier",
|
||
"twole_child_earlier",
|
||
)
|
||
|
||
|
||
CREATE_TABLE_TEMPLATE = """
|
||
CREATE TABLE IF NOT EXISTS `{table_name}` (
|
||
id BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键',
|
||
category VARCHAR(64) NOT NULL COMMENT '分类',
|
||
source_folder VARCHAR(512) NOT NULL COMMENT '来源文件夹',
|
||
word_doc VARCHAR(512) NOT NULL COMMENT '关联 Word 文档',
|
||
row_index INT NOT NULL COMMENT 'Word 内容行索引',
|
||
|
||
earlier_site VARCHAR(64) COMMENT '更早的网站中文名',
|
||
earlier_site_key VARCHAR(32) COMMENT '更早的网站 key: twole/zxxk',
|
||
child_time_diff_seconds DOUBLE COMMENT '学科网子文件时间 - 二一教育子文件时间',
|
||
|
||
twole_title TEXT COMMENT '二一教育标题',
|
||
twole_url TEXT COMMENT '二一教育 URL',
|
||
twole_source_time VARCHAR(100) COMMENT 'Word 中二一教育时间',
|
||
twole_id VARCHAR(100) COMMENT '二一教育 URL ID',
|
||
twole_file_kind VARCHAR(32) COMMENT '二一教育文件类型',
|
||
twole_file_name VARCHAR(1024) COMMENT '二一教育文件名',
|
||
twole_file_mtime DOUBLE COMMENT '二一教育文件 mtime',
|
||
twole_file_datetime DATETIME COMMENT '二一教育文件时间',
|
||
twole_archive_mtime DOUBLE COMMENT '二一教育压缩包 mtime',
|
||
twole_archive_datetime DATETIME COMMENT '二一教育压缩包时间',
|
||
twole_latest_child_mtime DOUBLE COMMENT '二一教育压缩包内最新子文件 mtime',
|
||
twole_latest_child_datetime DATETIME COMMENT '二一教育压缩包内最新子文件时间',
|
||
twole_child_archive_time_relation VARCHAR(32) COMMENT '二一教育子文件与压缩包时间关系',
|
||
twole_child_archive_time_diff_seconds DOUBLE COMMENT '二一教育子文件与压缩包时间差',
|
||
|
||
zxxk_title TEXT COMMENT '学科网标题',
|
||
zxxk_url TEXT COMMENT '学科网 URL',
|
||
zxxk_source_time VARCHAR(100) COMMENT 'Word 中学科网时间',
|
||
zxxk_id VARCHAR(100) COMMENT '学科网 URL ID',
|
||
zxxk_file_kind VARCHAR(32) COMMENT '学科网文件类型',
|
||
zxxk_file_name VARCHAR(1024) COMMENT '学科网文件名',
|
||
zxxk_file_mtime DOUBLE COMMENT '学科网文件 mtime',
|
||
zxxk_file_datetime DATETIME COMMENT '学科网文件时间',
|
||
zxxk_archive_mtime DOUBLE COMMENT '学科网压缩包 mtime',
|
||
zxxk_archive_datetime DATETIME COMMENT '学科网压缩包时间',
|
||
zxxk_latest_child_mtime DOUBLE COMMENT '学科网压缩包内最新子文件 mtime',
|
||
zxxk_latest_child_datetime DATETIME COMMENT '学科网压缩包内最新子文件时间',
|
||
zxxk_child_archive_time_relation VARCHAR(32) COMMENT '学科网子文件与压缩包时间关系',
|
||
zxxk_child_archive_time_diff_seconds DOUBLE COMMENT '学科网子文件与压缩包时间差',
|
||
|
||
twole_json JSON COMMENT '二一教育原始 JSON',
|
||
zxxk_json JSON COMMENT '学科网原始 JSON',
|
||
raw_json JSON NOT NULL COMMENT '合并前原始 JSON',
|
||
imported_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '导入时间',
|
||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
|
||
|
||
UNIQUE KEY uniq_category_row (category, source_folder(191), word_doc(191), row_index),
|
||
KEY idx_category (category),
|
||
KEY idx_earlier_site_key (earlier_site_key),
|
||
KEY idx_source_row (source_folder(191), word_doc(191), row_index),
|
||
KEY idx_twole_id (twole_id),
|
||
KEY idx_zxxk_id (zxxk_id),
|
||
KEY idx_twole_latest_child_mtime (twole_latest_child_mtime),
|
||
KEY idx_zxxk_latest_child_mtime (zxxk_latest_child_mtime)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='压缩包时间分类记录';
|
||
"""
|
||
|
||
|
||
INSERT_COLUMNS = (
|
||
"category", "source_folder", "word_doc", "row_index",
|
||
"earlier_site", "earlier_site_key", "child_time_diff_seconds",
|
||
"twole_title", "twole_url", "twole_source_time", "twole_id", "twole_file_kind",
|
||
"twole_file_name", "twole_file_mtime", "twole_file_datetime",
|
||
"twole_archive_mtime", "twole_archive_datetime",
|
||
"twole_latest_child_mtime", "twole_latest_child_datetime",
|
||
"twole_child_archive_time_relation", "twole_child_archive_time_diff_seconds",
|
||
"zxxk_title", "zxxk_url", "zxxk_source_time", "zxxk_id", "zxxk_file_kind",
|
||
"zxxk_file_name", "zxxk_file_mtime", "zxxk_file_datetime",
|
||
"zxxk_archive_mtime", "zxxk_archive_datetime",
|
||
"zxxk_latest_child_mtime", "zxxk_latest_child_datetime",
|
||
"zxxk_child_archive_time_relation", "zxxk_child_archive_time_diff_seconds",
|
||
"twole_json", "zxxk_json", "raw_json",
|
||
)
|
||
|
||
|
||
def mysql_identifier(name: str) -> str:
|
||
if not re.match(r"^[A-Za-z0-9_]+$", name):
|
||
raise ValueError(f"非法表名：{name}")
|
||
return name
|
||
|
||
|
||
def epoch_to_datetime(value) -> Optional[str]:
|
||
if value in (None, ""):
|
||
return None
|
||
return datetime.fromtimestamp(float(value)).strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
|
||
def normalize_datetime(value) -> Optional[str]:
|
||
if not value:
|
||
return None
|
||
if isinstance(value, str) and re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$", value):
|
||
return value
|
||
return None
|
||
|
||
|
||
def json_dumps(value) -> str:
|
||
return json.dumps(value or {}, ensure_ascii=False)
|
||
|
||
|
||
def extract_url_id(url: Optional[str], site: str) -> Optional[str]:
|
||
if not url:
|
||
return None
|
||
if site == "twole":
|
||
match = re.search(r"/(\d+)\.shtml(?:[?#].*)?$", url)
|
||
if match:
|
||
return match.group(1)
|
||
if site == "zxxk":
|
||
match = re.search(r"/(?:soft/)?(\d+)\.html(?:[?#].*)?$", url)
|
||
if match:
|
||
return match.group(1)
|
||
numbers = re.findall(r"\d+", url)
|
||
return numbers[-1] if numbers else None
|
||
|
||
|
||
def first_file_from_group(group: Dict) -> Dict:
|
||
files = group.get("files") or []
|
||
return files[0] if files else {}
|
||
|
||
|
||
def build_side_from_individual(record: Dict) -> Dict:
|
||
file_info = record.get("file") or {}
|
||
site = record.get("site") or ""
|
||
return {
|
||
"title": record.get("title"),
|
||
"url": record.get("url"),
|
||
"source_time": record.get("time"),
|
||
"id": extract_url_id(record.get("url"), site),
|
||
"file_kind": record.get("file_kind"),
|
||
"file_name": file_info.get("name"),
|
||
"file_mtime": file_info.get("mtime"),
|
||
"file_datetime": normalize_datetime(file_info.get("datetime")) or epoch_to_datetime(file_info.get("mtime")),
|
||
"archive_mtime": record.get("archive_mtime"),
|
||
"archive_datetime": normalize_datetime(record.get("archive_datetime")) or epoch_to_datetime(record.get("archive_mtime")),
|
||
"latest_child_mtime": record.get("latest_child_mtime"),
|
||
"latest_child_datetime": normalize_datetime(record.get("latest_child_datetime")) or epoch_to_datetime(record.get("latest_child_mtime")),
|
||
"child_archive_time_relation": record.get("child_archive_time_relation"),
|
||
"child_archive_time_diff_seconds": record.get("child_archive_time_diff_seconds"),
|
||
"json": record,
|
||
}
|
||
|
||
|
||
def build_side_from_pair(group: Dict, site: str, pair_record: Dict) -> Dict:
|
||
file_info = first_file_from_group(group)
|
||
return {
|
||
"title": group.get("title"),
|
||
"url": group.get("url"),
|
||
"source_time": group.get("time"),
|
||
"id": extract_url_id(group.get("url"), site),
|
||
"file_kind": "archive",
|
||
"file_name": file_info.get("name"),
|
||
"file_mtime": file_info.get("mtime"),
|
||
"file_datetime": normalize_datetime(file_info.get("datetime")) or epoch_to_datetime(file_info.get("mtime")),
|
||
"archive_mtime": file_info.get("mtime"),
|
||
"archive_datetime": normalize_datetime(file_info.get("datetime")) or epoch_to_datetime(file_info.get("mtime")),
|
||
"latest_child_mtime": pair_record.get(f"{site}_latest_child_mtime"),
|
||
"latest_child_datetime": normalize_datetime(pair_record.get(f"{site}_latest_child_datetime")) or epoch_to_datetime(pair_record.get(f"{site}_latest_child_mtime")),
|
||
"child_archive_time_relation": "different",
|
||
"child_archive_time_diff_seconds": None,
|
||
"json": group,
|
||
}
|
||
|
||
|
||
def empty_import_record(category: str, source_folder: str, word_doc: str, row_index: int, raw_json) -> Dict:
|
||
record = {column: None for column in INSERT_COLUMNS}
|
||
record.update({
|
||
"category": category,
|
||
"source_folder": source_folder,
|
||
"word_doc": word_doc,
|
||
"row_index": row_index,
|
||
"raw_json": json_dumps(raw_json),
|
||
})
|
||
return record
|
||
|
||
|
||
def apply_side(record: Dict, site: str, side: Dict) -> None:
|
||
prefix = "twole" if site == "twole" else "zxxk"
|
||
for key in (
|
||
"title", "url", "source_time", "id", "file_kind", "file_name", "file_mtime", "file_datetime",
|
||
"archive_mtime", "archive_datetime", "latest_child_mtime", "latest_child_datetime",
|
||
"child_archive_time_relation", "child_archive_time_diff_seconds",
|
||
):
|
||
record[f"{prefix}_{key}"] = side.get(key)
|
||
record[f"{prefix}_json"] = json_dumps(side.get("json"))
|
||
|
||
|
||
def row_key(record: Dict) -> Tuple[str, str, int]:
|
||
return (record.get("source_folder") or "", record.get("word_doc") or "", int(record.get("row_index") or 0))
|
||
|
||
|
||
def normalize_individual_category(category: str, records: Iterable[Dict]) -> List[Dict]:
|
||
grouped: Dict[Tuple[str, str, int], Dict] = {}
|
||
raw_records: Dict[Tuple[str, str, int], List[Dict]] = {}
|
||
|
||
for source_record in records:
|
||
key = row_key(source_record)
|
||
raw_records.setdefault(key, []).append(source_record)
|
||
if key not in grouped:
|
||
grouped[key] = empty_import_record(category, key[0], key[1], key[2], raw_records[key])
|
||
grouped[key]["raw_json"] = json_dumps(raw_records[key])
|
||
|
||
site = source_record.get("site")
|
||
if site in ("twole", "zxxk"):
|
||
apply_side(grouped[key], site, build_side_from_individual(source_record))
|
||
|
||
return [grouped[key] for key in sorted(grouped)]
|
||
|
||
|
||
def normalize_pair_category(category: str, records: Iterable[Dict]) -> List[Dict]:
|
||
normalized = []
|
||
for source_record in records:
|
||
key = row_key(source_record)
|
||
record = empty_import_record(category, key[0], key[1], key[2], source_record)
|
||
record["earlier_site"] = source_record.get("earlier_site")
|
||
record["earlier_site_key"] = source_record.get("earlier_site_key")
|
||
record["child_time_diff_seconds"] = source_record.get("child_time_diff_seconds")
|
||
apply_side(record, "twole", build_side_from_pair(source_record.get("twole") or {}, "twole", source_record))
|
||
apply_side(record, "zxxk", build_side_from_pair(source_record.get("zxxk") or {}, "zxxk", source_record))
|
||
normalized.append(record)
|
||
return normalized
|
||
|
||
|
||
def normalize_records_for_import(data: Dict) -> List[Dict]:
|
||
normalized: List[Dict] = []
|
||
normalized.extend(normalize_individual_category("single_files", data.get("single_files", [])))
|
||
normalized.extend(normalize_individual_category("same_child_archive_time", data.get("same_child_archive_time", [])))
|
||
normalized.extend(normalize_pair_category("zxxk_child_earlier", data.get("zxxk_child_earlier", [])))
|
||
normalized.extend(normalize_pair_category("twole_child_earlier", data.get("twole_child_earlier", [])))
|
||
return normalized
|
||
|
||
|
||
def load_json_data(file_path: Path) -> Dict:
|
||
with file_path.open("r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
if not isinstance(data, dict):
|
||
raise ValueError("当前导入脚本只支持新版分类字典格式 JSON")
|
||
return data
|
||
|
||
|
||
def create_connection():
|
||
return mysql.connector.connect(**DB_CONFIG)
|
||
|
||
|
||
def create_table(connection, table_name: str) -> None:
|
||
cursor = connection.cursor()
|
||
try:
|
||
cursor.execute(CREATE_TABLE_TEMPLATE.format(table_name=mysql_identifier(table_name)))
|
||
connection.commit()
|
||
finally:
|
||
cursor.close()
|
||
|
||
|
||
def truncate_table(connection, table_name: str) -> None:
|
||
cursor = connection.cursor()
|
||
try:
|
||
cursor.execute(f"TRUNCATE TABLE `{mysql_identifier(table_name)}`")
|
||
connection.commit()
|
||
finally:
|
||
cursor.close()
|
||
|
||
|
||
def build_insert_sql(table_name: str) -> str:
|
||
columns = ", ".join(f"`{column}`" for column in INSERT_COLUMNS)
|
||
placeholders = ", ".join(["%s"] * len(INSERT_COLUMNS))
|
||
updates = ", ".join(
|
||
f"`{column}` = VALUES(`{column}`)"
|
||
for column in INSERT_COLUMNS
|
||
if column not in {"category", "source_folder", "word_doc", "row_index"}
|
||
)
|
||
return f"""
|
||
INSERT INTO `{mysql_identifier(table_name)}` ({columns})
|
||
VALUES ({placeholders})
|
||
ON DUPLICATE KEY UPDATE {updates}
|
||
"""
|
||
|
||
|
||
def insert_records(connection, table_name: str, records: List[Dict], batch_size: int = 500) -> int:
|
||
if not records:
|
||
return 0
|
||
|
||
insert_sql = build_insert_sql(table_name)
|
||
cursor = connection.cursor()
|
||
inserted = 0
|
||
try:
|
||
for start in range(0, len(records), batch_size):
|
||
batch = records[start:start + batch_size]
|
||
values = [tuple(record.get(column) for column in INSERT_COLUMNS) for record in batch]
|
||
cursor.executemany(insert_sql, values)
|
||
connection.commit()
|
||
inserted += len(batch)
|
||
print(f"已写入/更新 {inserted}/{len(records)} 条记录")
|
||
finally:
|
||
cursor.close()
|
||
return inserted
|
||
|
||
|
||
def print_category_summary(records: List[Dict]) -> None:
|
||
counts = {category: 0 for category in SELECTED_CATEGORIES}
|
||
for record in records:
|
||
counts[record["category"]] = counts.get(record["category"], 0) + 1
|
||
print("待导入分类统计：")
|
||
for category in SELECTED_CATEGORIES:
|
||
print(f" {category}: {counts.get(category, 0)}")
|
||
print(f" total: {len(records)}")
|
||
|
||
|
||
def parse_args():
|
||
parser = argparse.ArgumentParser(description="导入 compressed_group_time_issues.json 到 MySQL")
|
||
parser.add_argument("--json-file", type=Path, default=DEFAULT_JSON_FILE_PATH, help="JSON 文件路径")
|
||
parser.add_argument("--table", default=DEFAULT_TABLE_NAME, help="MySQL 表名")
|
||
parser.add_argument("--truncate", action="store_true", help="导入前清空目标表")
|
||
parser.add_argument("--dry-run", action="store_true", help="只解析和统计，不连接 MySQL")
|
||
return parser.parse_args()
|
||
|
||
|
||
def main():
|
||
args = parse_args()
|
||
data = load_json_data(args.json_file)
|
||
records = normalize_records_for_import(data)
|
||
print_category_summary(records)
|
||
|
||
if args.dry_run:
|
||
print("dry-run 模式：未连接 MySQL，未写入数据")
|
||
return
|
||
|
||
connection = None
|
||
try:
|
||
connection = create_connection()
|
||
print("成功连接到 MySQL")
|
||
create_table(connection, args.table)
|
||
print(f"已确认目标表：{args.table}")
|
||
if args.truncate:
|
||
truncate_table(connection, args.table)
|
||
print(f"已清空目标表：{args.table}")
|
||
inserted = insert_records(connection, args.table, records)
|
||
print(f"导入完成：写入/更新 {inserted} 条记录")
|
||
except Error as exc:
|
||
print(f"MySQL 操作失败：{exc}")
|
||
raise
|
||
finally:
|
||
if connection and connection.is_connected():
|
||
connection.close()
|
||
print("MySQL 连接已关闭")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|