优化一些程序

server/import_json_to_mysql.py

@@ -7,6 +7,9 @@
 import json
 import mysql.connector
 from mysql.connector import Error
+import re
+from datetime import datetime
+from pathlib import Path
 
 
 # MySQL 连接配置
@@ -50,6 +53,12 @@ CREATE TABLE IF NOT EXISTS compressed_group_time_issues (
     source_folder VARCHAR(500) COMMENT '来源文件夹',
     word_doc VARCHAR(500) COMMENT '关联的 word 文档',
 
+    -- 新增字段
+    21cn_son_file_datetime VARCHAR(50) COMMENT 'group1_latest_extracted_mtime 的日期时间格式',
+    xkw_son_file_datetime VARCHAR(50) COMMENT 'group2_latest_extracted_mtime 的日期时间格式',
+    21ID VARCHAR(100) COMMENT 'group1_url 中的数字部分',
+    xkwID VARCHAR(100) COMMENT 'group2_url 中的数字部分',
+
     -- 索引
     INDEX idx_row_index (row_index),
     INDEX idx_source_folder (source_folder)
@@ -109,6 +118,27 @@ def flatten_record(record):
     group2_files = record.get('group2', {}).get('files', [])
     group2_first_file = group2_files[0] if group2_files else {}
 
+    # 计算新字段
+    group1_mtime = record.get('group1_latest_extracted_mtime')
+    group1_datetime = datetime.fromtimestamp(group1_mtime).strftime('%Y-%m-%d %H:%M:%S') if group1_mtime else None
+
+    group2_mtime = record.get('group2_latest_extracted_mtime')
+    group2_datetime = datetime.fromtimestamp(group2_mtime).strftime('%Y-%m-%d %H:%M:%S') if group2_mtime else None
+
+    group1_url = record.get('group1', {}).get('url', '')
+    group1_id = None
+    if group1_url:
+        match = re.search(r'/(\d+)\.shtml$', group1_url)
+        if match:
+            group1_id = match.group(1)
+
+    group2_url = record.get('group2', {}).get('url', '')
+    group2_id = None
+    if group2_url:
+        match = re.search(r'/(\d+)\.html$', group2_url)
+        if match:
+            group2_id = match.group(1)
+
     return {
         'row_index': record.get('row_index'),
 
@@ -132,7 +162,13 @@ def flatten_record(record):
 
         # 来源信息
         'source_folder': record.get('source_folder'),
-        'word_doc': record.get('word_doc')
+        'word_doc': record.get('word_doc'),
+
+        # 新增字段
+        '21cn_son_file_datetime': group1_datetime,
+        'xkw_son_file_datetime': group2_datetime,
+        '21ID': group1_id,
+        'xkwID': group2_id
     }
 
 
@@ -145,10 +181,12 @@ def insert_data(connection, data):
         group1_file_name, group1_file_mtime, group1_file_datetime,
         group2_latest_extracted_mtime, group2_title, group2_url, group2_time,
         group2_file_name, group2_file_mtime, group2_file_datetime,
-        source_folder, word_doc
+        source_folder, word_doc,
+        21cn_son_file_datetime, xkw_son_file_datetime, 21ID, xkwID
     ) VALUES (
         %s, %s, %s, %s, %s, %s, %s, %s,
-        %s, %s, %s, %s, %s, %s, %s, %s, %s
+        %s, %s, %s, %s, %s, %s, %s, %s, %s,
+        %s, %s, %s, %s
     )
     """
 
@@ -176,7 +214,11 @@ def insert_data(connection, data):
                 flattened['group2_file_mtime'],
                 flattened['group2_file_datetime'],
                 flattened['source_folder'],
-                flattened['word_doc']
+                flattened['word_doc'],
+                flattened['21cn_son_file_datetime'],
+                flattened['xkw_son_file_datetime'],
+                flattened['21ID'],
+                flattened['xkwID']
             )
 
             cursor.execute(insert_sql, values)
@@ -207,7 +249,8 @@ def verify_data(connection):
 
         # 查询前 2 条记录作为示例
         cursor.execute("""
-            SELECT row_index, group1_title, group2_title, source_folder
+            SELECT row_index, group1_title, group2_title, source_folder,
+                   21cn_son_file_datetime, xkw_son_file_datetime, 21ID, xkwID
             FROM compressed_group_time_issues
             LIMIT 2
         """)
@@ -218,6 +261,10 @@ def verify_data(connection):
             print(f"  group1_title={row[1][:50]}...")
             print(f"  group2_title={row[2][:50]}...")
             print(f"  source_folder={row[3]}")
+            print(f"  21cn_son_file_datetime={row[4]}")
+            print(f"  xkw_son_file_datetime={row[5]}")
+            print(f"  21ID={row[6]}")
+            print(f"  xkwID={row[7]}")
             print("-" * 50)
 
         cursor.close()
@@ -225,6 +272,31 @@ def verify_data(connection):
         print(f"验证数据失败：{e}")
 
 
+def export_table_to_excel(connection, output_path: Path):
+    """将数据库表导出为 Excel 文件"""
+    try:
+        import pandas as pd
+    except ImportError:
+        print("请先安装 pandas 和 openpyxl：pip install pandas openpyxl")
+        return False
+
+    try:
+        cursor = connection.cursor()
+        cursor.execute("SELECT * FROM compressed_group_time_issues")
+        rows = cursor.fetchall()
+        columns = cursor.column_names
+        cursor.close()
+
+        df = pd.DataFrame(rows, columns=columns)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        df.to_excel(output_path, index=False)
+        print(f"成功导出 Excel 文件：{output_path}")
+        return True
+    except Exception as e:
+        print(f"导出 Excel 失败：{e}")
+        return False
+
+
 def main():
     """主函数"""
     print("=" * 60)
@@ -250,7 +322,9 @@ def main():
 
     # 5. 验证数据
     verify_data(connection)
-
+    # 6. 导出 Excel
+    excel_path = Path('server/jsons/compressed_group_time_issues.xlsx')
+    export_table_to_excel(connection, excel_path)
     # 关闭连接
     connection.close()
     print("\n数据库连接已关闭")