338 lines
11 KiB
Python
338 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
将 compressed_group_time_issues.json 数据导入到 MySQL 数据库
|
||
"""
|
||
|
||
import json
|
||
import mysql.connector
|
||
from mysql.connector import Error
|
||
import re
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
|
||
# MySQL 连接配置
|
||
DB_CONFIG = {
|
||
'host': '127.0.0.1',
|
||
'port': 3306,
|
||
'user': 'root',
|
||
'password': 'root123456',
|
||
'database': 'testdb',
|
||
'charset': 'utf8mb4'
|
||
}
|
||
|
||
# JSON 文件路径
|
||
JSON_FILE_PATH = 'server/jsons/compressed_group_time_issues.json'
|
||
|
||
# 建表 SQL
|
||
CREATE_TABLE_SQL = """
|
||
CREATE TABLE IF NOT EXISTS compressed_group_time_issues (
|
||
id INT AUTO_INCREMENT PRIMARY KEY COMMENT '主键',
|
||
row_index INT COMMENT '行索引',
|
||
|
||
-- Group1 字段
|
||
group1_latest_extracted_mtime DOUBLE COMMENT 'group1 最新提取文件的 mtime',
|
||
group1_title VARCHAR(1000) COMMENT 'group1 标题',
|
||
group1_url VARCHAR(1000) COMMENT 'group1 URL',
|
||
group1_time VARCHAR(100) COMMENT 'group1 时间',
|
||
group1_file_name VARCHAR(1000) COMMENT 'group1 第一个文件名',
|
||
group1_file_mtime DOUBLE COMMENT 'group1 第一个文件 mtime',
|
||
group1_file_datetime VARCHAR(50) COMMENT 'group1 第一个文件 datetime',
|
||
|
||
-- Group2 字段
|
||
group2_latest_extracted_mtime DOUBLE COMMENT 'group2 最新提取文件的 mtime',
|
||
group2_title VARCHAR(1000) COMMENT 'group2 标题',
|
||
group2_url VARCHAR(1000) COMMENT 'group2 URL',
|
||
group2_time VARCHAR(100) COMMENT 'group2 时间',
|
||
group2_file_name VARCHAR(1000) COMMENT 'group2 第一个文件名',
|
||
group2_file_mtime DOUBLE COMMENT 'group2 第一个文件 mtime',
|
||
group2_file_datetime VARCHAR(50) COMMENT 'group2 第一个文件 datetime',
|
||
|
||
-- 来源信息
|
||
source_folder VARCHAR(500) COMMENT '来源文件夹',
|
||
word_doc VARCHAR(500) COMMENT '关联的 word 文档',
|
||
|
||
-- 新增字段
|
||
21cn_son_file_datetime VARCHAR(50) COMMENT 'group1_latest_extracted_mtime 的日期时间格式',
|
||
xkw_son_file_datetime VARCHAR(50) COMMENT 'group2_latest_extracted_mtime 的日期时间格式',
|
||
21ID VARCHAR(100) COMMENT 'group1_url 中的数字部分',
|
||
xkwID VARCHAR(100) COMMENT 'group2_url 中的数字部分',
|
||
|
||
-- 索引
|
||
INDEX idx_row_index (row_index),
|
||
INDEX idx_source_folder (source_folder)
|
||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='压缩组时间问题数据表';
|
||
"""
|
||
|
||
|
||
def create_connection():
|
||
"""创建数据库连接"""
|
||
try:
|
||
connection = mysql.connector.connect(**DB_CONFIG)
|
||
if connection.is_connected():
|
||
print(f"成功连接到 MySQL 数据库")
|
||
print(f"数据库版本：{connection.get_server_info()}")
|
||
return connection
|
||
except Error as e:
|
||
print(f"连接 MySQL 失败：{e}")
|
||
return None
|
||
|
||
|
||
def create_table(connection):
|
||
"""创建表"""
|
||
try:
|
||
cursor = connection.cursor()
|
||
cursor.execute(CREATE_TABLE_SQL)
|
||
connection.commit()
|
||
print("表 'compressed_group_time_issues' 创建成功!")
|
||
cursor.close()
|
||
except Error as e:
|
||
print(f"创建表失败：{e}")
|
||
|
||
|
||
def load_json_data(file_path):
|
||
"""加载 JSON 数据"""
|
||
try:
|
||
with open(file_path, 'r', encoding='utf-8') as f:
|
||
data = json.load(f)
|
||
print(f"成功加载 JSON 文件，共 {len(data)} 条记录")
|
||
return data
|
||
except FileNotFoundError:
|
||
print(f"文件不存在：{file_path}")
|
||
return None
|
||
except json.JSONDecodeError as e:
|
||
print(f"JSON 解析错误：{e}")
|
||
return None
|
||
|
||
|
||
def flatten_record(record):
|
||
"""
|
||
扁平化记录
|
||
"""
|
||
# 获取 group1 的第一个文件
|
||
group1_files = record.get('group1', {}).get('files', [])
|
||
group1_first_file = group1_files[0] if group1_files else {}
|
||
|
||
# 获取 group2 的第一个文件
|
||
group2_files = record.get('group2', {}).get('files', [])
|
||
group2_first_file = group2_files[0] if group2_files else {}
|
||
|
||
# 计算新字段
|
||
group1_mtime = record.get('group1_latest_extracted_mtime')
|
||
group1_datetime = datetime.fromtimestamp(group1_mtime).strftime('%Y-%m-%d %H:%M:%S') if group1_mtime else None
|
||
|
||
group2_mtime = record.get('group2_latest_extracted_mtime')
|
||
group2_datetime = datetime.fromtimestamp(group2_mtime).strftime('%Y-%m-%d %H:%M:%S') if group2_mtime else None
|
||
|
||
group1_url = record.get('group1', {}).get('url', '')
|
||
group1_id = None
|
||
if group1_url:
|
||
match = re.search(r'/(\d+)\.shtml$', group1_url)
|
||
if match:
|
||
group1_id = match.group(1)
|
||
|
||
group2_url = record.get('group2', {}).get('url', '')
|
||
group2_id = None
|
||
if group2_url:
|
||
match = re.search(r'/(\d+)\.html$', group2_url)
|
||
if match:
|
||
group2_id = match.group(1)
|
||
|
||
return {
|
||
'row_index': record.get('row_index'),
|
||
|
||
# Group1 字段
|
||
'group1_latest_extracted_mtime': record.get('group1_latest_extracted_mtime'),
|
||
'group1_title': record.get('group1', {}).get('title'),
|
||
'group1_url': record.get('group1', {}).get('url'),
|
||
'group1_time': record.get('group1', {}).get('time'),
|
||
'group1_file_name': group1_first_file.get('name'),
|
||
'group1_file_mtime': group1_first_file.get('mtime'),
|
||
'group1_file_datetime': group1_first_file.get('datetime'),
|
||
|
||
# Group2 字段
|
||
'group2_latest_extracted_mtime': record.get('group2_latest_extracted_mtime'),
|
||
'group2_title': record.get('group2', {}).get('title'),
|
||
'group2_url': record.get('group2', {}).get('url'),
|
||
'group2_time': record.get('group2', {}).get('time'),
|
||
'group2_file_name': group2_first_file.get('name'),
|
||
'group2_file_mtime': group2_first_file.get('mtime'),
|
||
'group2_file_datetime': group2_first_file.get('datetime'),
|
||
|
||
# 来源信息
|
||
'source_folder': record.get('source_folder'),
|
||
'word_doc': record.get('word_doc'),
|
||
|
||
# 新增字段
|
||
'21cn_son_file_datetime': group1_datetime,
|
||
'xkw_son_file_datetime': group2_datetime,
|
||
'21ID': group1_id,
|
||
'xkwID': group2_id
|
||
}
|
||
|
||
|
||
def insert_data(connection, data):
|
||
"""插入数据"""
|
||
insert_sql = """
|
||
INSERT INTO compressed_group_time_issues (
|
||
row_index,
|
||
group1_latest_extracted_mtime, group1_title, group1_url, group1_time,
|
||
group1_file_name, group1_file_mtime, group1_file_datetime,
|
||
group2_latest_extracted_mtime, group2_title, group2_url, group2_time,
|
||
group2_file_name, group2_file_mtime, group2_file_datetime,
|
||
source_folder, word_doc,
|
||
21cn_son_file_datetime, xkw_son_file_datetime, 21ID, xkwID
|
||
) VALUES (
|
||
%s, %s, %s, %s, %s, %s, %s, %s,
|
||
%s, %s, %s, %s, %s, %s, %s, %s, %s,
|
||
%s, %s, %s, %s
|
||
)
|
||
"""
|
||
|
||
try:
|
||
cursor = connection.cursor()
|
||
inserted_count = 0
|
||
|
||
for record in data:
|
||
flattened = flatten_record(record)
|
||
|
||
values = (
|
||
flattened['row_index'],
|
||
flattened['group1_latest_extracted_mtime'],
|
||
flattened['group1_title'],
|
||
flattened['group1_url'],
|
||
flattened['group1_time'],
|
||
flattened['group1_file_name'],
|
||
flattened['group1_file_mtime'],
|
||
flattened['group1_file_datetime'],
|
||
flattened['group2_latest_extracted_mtime'],
|
||
flattened['group2_title'],
|
||
flattened['group2_url'],
|
||
flattened['group2_time'],
|
||
flattened['group2_file_name'],
|
||
flattened['group2_file_mtime'],
|
||
flattened['group2_file_datetime'],
|
||
flattened['source_folder'],
|
||
flattened['word_doc'],
|
||
flattened['21cn_son_file_datetime'],
|
||
flattened['xkw_son_file_datetime'],
|
||
flattened['21ID'],
|
||
flattened['xkwID']
|
||
)
|
||
|
||
cursor.execute(insert_sql, values)
|
||
inserted_count += 1
|
||
|
||
if inserted_count % 50 == 0:
|
||
print(f"已插入 {inserted_count} 条记录...")
|
||
|
||
connection.commit()
|
||
print(f"成功插入 {inserted_count} 条记录!")
|
||
cursor.close()
|
||
return inserted_count
|
||
except Error as e:
|
||
print(f"插入数据失败：{e}")
|
||
connection.rollback()
|
||
return 0
|
||
|
||
|
||
def verify_data(connection):
|
||
"""验证导入的数据"""
|
||
try:
|
||
cursor = connection.cursor()
|
||
|
||
# 查询总记录数
|
||
cursor.execute("SELECT COUNT(*) FROM compressed_group_time_issues")
|
||
count = cursor.fetchone()[0]
|
||
print(f"\n验证：表中总记录数 = {count}")
|
||
|
||
# 查询前 2 条记录作为示例
|
||
cursor.execute("""
|
||
SELECT row_index, group1_title, group2_title, source_folder,
|
||
21cn_son_file_datetime, xkw_son_file_datetime, 21ID, xkwID
|
||
FROM compressed_group_time_issues
|
||
LIMIT 2
|
||
""")
|
||
rows = cursor.fetchall()
|
||
print("\n示例数据:")
|
||
for row in rows:
|
||
print(f" row_index={row[0]}")
|
||
print(f" group1_title={row[1][:50]}...")
|
||
print(f" group2_title={row[2][:50]}...")
|
||
print(f" source_folder={row[3]}")
|
||
print(f" 21cn_son_file_datetime={row[4]}")
|
||
print(f" xkw_son_file_datetime={row[5]}")
|
||
print(f" 21ID={row[6]}")
|
||
print(f" xkwID={row[7]}")
|
||
print("-" * 50)
|
||
|
||
cursor.close()
|
||
except Error as e:
|
||
print(f"验证数据失败：{e}")
|
||
|
||
|
||
def export_table_to_excel(connection, output_path: Path):
|
||
"""将数据库表导出为 Excel 文件"""
|
||
try:
|
||
import pandas as pd
|
||
except ImportError:
|
||
print("请先安装 pandas 和 openpyxl：pip install pandas openpyxl")
|
||
return False
|
||
|
||
try:
|
||
cursor = connection.cursor()
|
||
cursor.execute("SELECT * FROM compressed_group_time_issues")
|
||
rows = cursor.fetchall()
|
||
columns = cursor.column_names
|
||
cursor.close()
|
||
|
||
df = pd.DataFrame(rows, columns=columns)
|
||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||
df.to_excel(output_path, index=False)
|
||
print(f"成功导出 Excel 文件：{output_path}")
|
||
return True
|
||
except Exception as e:
|
||
print(f"导出 Excel 失败：{e}")
|
||
return False
|
||
|
||
|
||
def main():
|
||
"""主函数"""
|
||
print("=" * 60)
|
||
print("开始导入 compressed_group_time_issues.json 到 MySQL")
|
||
print("=" * 60)
|
||
|
||
# 1. 创建数据库连接
|
||
connection = create_connection()
|
||
if not connection:
|
||
return
|
||
|
||
# 2. 创建表
|
||
create_table(connection)
|
||
|
||
# 3. 加载 JSON 数据
|
||
data = load_json_data(JSON_FILE_PATH)
|
||
if not data:
|
||
connection.close()
|
||
return
|
||
|
||
# 4. 插入数据
|
||
insert_data(connection, data)
|
||
|
||
# 5. 验证数据
|
||
verify_data(connection)
|
||
# 6. 导出 Excel
|
||
excel_path = Path('server/jsons/compressed_group_time_issues.xlsx')
|
||
export_table_to_excel(connection, excel_path)
|
||
# 关闭连接
|
||
connection.close()
|
||
print("\n数据库连接已关闭")
|
||
print("=" * 60)
|
||
print("导入完成!")
|
||
print("=" * 60)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|