diff --git a/server/association_detail.py b/server/association_detail.py index 9fcccb9..09d9c97 100644 --- a/server/association_detail.py +++ b/server/association_detail.py @@ -791,6 +791,33 @@ def main(): print(f"成功:{success_count}, 失败:{fail_count}") print(f"问题组报告已保存到:{report_file}") print(len(all_problem_groups)) + + # 将所有涉及的原始文件夹从 cc 目录复制到 ff 目录 + script_dir = Path(__file__).parent + cc_dir = script_dir / "cc" + ff_dir = script_dir / "ff" + ff_dir.mkdir(exist_ok=True) + + # 收集所有需要复制的文件夹(去重) + folders_to_copy = set() + for group in all_problem_groups: + source_folder = group.get("source_folder") + if source_folder: + folders_to_copy.add(source_folder) + + print(f"\n准备复制 {len(folders_to_copy)} 个文件夹到 ff 目录...") + for folder_name in folders_to_copy: + src_folder = cc_dir / folder_name + if src_folder.exists() and src_folder.is_dir(): + dest_folder = ff_dir / folder_name + if not dest_folder.exists(): + print(f" 复制:{folder_name}") + shutil.copytree(str(src_folder), str(dest_folder)) + else: + print(f" 已存在,跳过:{folder_name}") + else: + print(f" 未找到源文件夹:{folder_name} (cc/{folder_name})") + return if __name__ == "__main__": diff --git a/server/association_detail2.py b/server/association_detail2.py new file mode 100644 index 0000000..4db3c68 --- /dev/null +++ b/server/association_detail2.py @@ -0,0 +1,863 @@ +# -*- coding: utf-8 -*- +""" +数据文件关联处理程序 +功能:解压压缩文件,读取 Word 文档,将文件与 Word 内容关联 +""" + +import os +import sys +import zipfile +import json +import argparse +from pathlib import Path +from typing import Optional, List, Dict, Tuple, Union +from datetime import datetime +import time +import re +import shutil +import requests +import math + +OLLAMA_EMBED_URL = os.environ.get('OLLAMA_EMBED_URL', 'http://localhost:11434/api/embeddings') +OLLAMA_EMBED_MODEL = os.environ.get('OLLAMA_EMBED_MODEL', 'nomic/text-embedding-3-large') + + +def clear_directory(dir_path: Path) -> None: + """清空目录中的所有文件和子目录""" + if dir_path.exists(): + for item in dir_path.iterdir(): + if item.is_file(): + item.unlink() + elif item.is_dir(): + shutil.rmtree(item) + + +def get_embedding(text: str) -> List[float]: + """使用 Ollama 获取文本的 embedding""" + try: + response = requests.post( + OLLAMA_EMBED_URL, + json={ + 'model': OLLAMA_EMBED_MODEL, + 'input': text, + }, + timeout=10, + ) + response.raise_for_status() + data = response.json() + if isinstance(data, dict): + if 'embedding' in data: + return data['embedding'] + if 'data' in data and isinstance(data['data'], list) and data['data']: + return data['data'][0].get('embedding', []) or [] + return [] + except Exception as e: + print(f"获取 embedding 失败: {e}") + return [] + + +def cosine_similarity(vec1: List[float], vec2: List[float]) -> float: + """计算两个向量的余弦相似度""" + if not vec1 or not vec2: + return 0.0 + dot = sum(a * b for a, b in zip(vec1, vec2)) + norm1 = math.sqrt(sum(a * a for a in vec1)) + norm2 = math.sqrt(sum(b * b for b in vec2)) + return dot / (norm1 * norm2) if norm1 and norm2 else 0.0 + + +def normalize_for_match(text: str) -> str: + """规范化文本,去掉路径和扩展名后的匹配用字符串""" + text = str(text).lower() + text = re.sub(r'[\u0020-\u002f\u003a-\u0040\u005b-\u0060\u007b-\u007e]+', ' ', text) + text = re.sub(r'\s+', ' ', text).strip() + return text + + +def token_overlap_score(a: str, b: str) -> float: + tokens_a = set(re.findall(r'[\u4e00-\u9fff]+|\d+|[a-z]+', a)) + tokens_b = set(re.findall(r'[\u4e00-\u9fff]+|\d+|[a-z]+', b)) + if not tokens_a or not tokens_b: + return 0.0 + return len(tokens_a & tokens_b) / max(1, len(tokens_b)) + + +def find_best_match(title: str, candidates: List[Tuple[Path, datetime]]) -> Tuple[Path, datetime]: + """从候选文件中找到与 title 最相似的文件""" + if not candidates: + return None + + title_norm = normalize_for_match(title) + exact_candidates = [] + for file_path, ctime in candidates: + stem_norm = normalize_for_match(file_path.stem) + if stem_norm and (stem_norm == title_norm or stem_norm in title_norm or title_norm in stem_norm): + exact_candidates.append((file_path, ctime, stem_norm)) + + if exact_candidates: + # 直接返回最精确的“包含匹配”候选项 + return max(exact_candidates, key=lambda item: len(item[2]))[:2] + + title_emb = get_embedding(title) + best_match = None + best_score = -1.0 + + for file_path, ctime in candidates: + stem_norm = normalize_for_match(file_path.stem) + overlap = token_overlap_score(title_norm, stem_norm) + score = overlap * 0.3 + + if title_emb: + stem_emb = get_embedding(file_path.stem) + if stem_emb: + score += cosine_similarity(title_emb, stem_emb) * 0.7 + + if score > best_score: + best_score = score + best_match = (file_path, ctime) + + return best_match or candidates[0] + + +def extract_all_zips( + source_dir: Union[str, Path] = "cc", + output_dir: Union[str, Path] = "data", + delete_original: bool = True +) -> List[Path]: + """ + 解压指定目录中的所有 zip 文件 + + Args: + source_dir: 包含压缩文件的目录 + output_dir: 输出目录 + delete_original: 解压后删除原文件 + + Returns: + 所有解压后的目录路径列表 + """ + source_dir = Path(source_dir) + output_dir = Path(output_dir) + + # 清空 output_dir + clear_directory(output_dir) + + zip_files = list(source_dir.glob("*.zip")) + + if not zip_files: + print(f"在 {source_dir} 中未发现 zip 文件") + return [] + + print(f"找到 {len(zip_files)} 个压缩文件,开始解压到 {output_dir}...") + results = [] + for zip_file in zip_files: + # 解压到指定目录 + extract_dir = output_dir / zip_file.stem + extract_dir.mkdir(parents=True, exist_ok=True) + + with zipfile.ZipFile(zip_file, 'r') as zip_ref: + # 解压所有文件并保留原始时间戳 + for member in zip_ref.infolist(): + # 解压文件 + zip_ref.extract(member, extract_dir) + + # 获取解压后的文件路径 + member_path = extract_dir / member.filename + + # 如果文件存在,设置时间戳 + if member_path.exists(): + # 将 ZipInfo 的日期时间转换为时间戳 + date_time = datetime(*member.date_time[:5]) + timestamp = date_time.timestamp() + # 设置访问时间和修改时间 + os.utime(member_path, (timestamp, timestamp)) + + # 检查是否多了一层目录(zip 内部只有一个同名文件夹) + inner_dir = extract_dir / zip_file.stem + if inner_dir.is_dir() and len(list(extract_dir.iterdir())) == 1: + # 移动内部文件夹所有内容到外层 + for item in inner_dir.iterdir(): + item.rename(extract_dir / item.name) + # 删除空的内层文件夹 + inner_dir.rmdir() + print(f"✓ 已解压:{zip_file.name} (移除了多余目录层)") + else: + print(f"✓ 已解压:{zip_file.name}") + + results.append(extract_dir) + + # 删除原压缩包 + if delete_original: + zip_file.unlink() + + return results + + +def read_word_content(doc_path: Path) -> List[str]: + """读取 Word 文档内容 (.docx 或 .doc),提取文本行""" + if doc_path.suffix.lower() == '.docx': + return read_docx_content(doc_path) + elif doc_path.suffix.lower() == '.doc': + return read_doc_content(doc_path) + return [] + + +def read_docx_content(doc_path: Path) -> List[str]: + """读取 .docx 格式 Word 文档内容""" + def is_date_time(line: str) -> bool: + """判断一行是否是日期时间格式""" + import re + # 匹配 YYYY-MM-DD, YYYY/MM/DD, YYYY-MM-DD HH:MM:SS, YYYY/MM/DD H:MM 等格式 + pattern = r'^\d{4}[-/]\d{1,2}[-/]\d{1,2}( \d{1,2}:\d{1,2}(:\d{1,2})?)?$' + return bool(re.match(pattern, line.strip())) + + try: + with zipfile.ZipFile(doc_path, 'r') as z: + content = z.read('word/document.xml').decode('utf-8') + # 提取所有 ... 中的内容,保留原始空白字符 + texts = re.findall(r']*>(.*?)', content, re.DOTALL) + + # 去除 HTML 标签,不替换内部的空白字符 + processed = [] + for text in texts: + text = re.sub(r'<[^>]+>', '', text).strip() + if text: + processed.append(text) + + # 删除超链接格式:HYPERLINK "url" url -> 只保留 url 部分 + def clean_hyperlink(line): + """清理超链接格式""" + # 匹配 HYPERLINK "url" url 格式(HTML 实体编码) + # 也兼容普通引号的情况 + hl_pattern = r'HYPERLINK\s+"([^&]+)"\s+(\S+)' + match = re.match(hl_pattern, line) + if match: + return match.group(2) # 返回第二个 url + return line + + processed = [clean_hyperlink(line) for line in processed] + + # 按行处理 + lines = [line.strip() for line in processed if line.strip()] + + # 检测是否每行都是单字段(没有\t) + has_tab_in_first_rows = any('\t' in line for line in lines[:6]) + + if not has_tab_in_first_rows and len(lines) >= 6: + # 新格式:按网站分组,然后将对应位置的资料配对 + + # 收集 21 世纪教育的资料 + twole_titles = [] + twole_urls = [] + twole_dates = [] + in_twole = False + + # 收集学科网的资料 + zxxk_titles = [] + zxxk_urls = [] + zxxk_dates = [] + in_zxxk = False + + i = 0 + while i < len(lines): + line = lines[i] + + # 判断是否为题目标记 + if line == '21世纪教育': + in_twole = True + in_zxxk = False + i += 1 + elif line == '学科网': + in_twole = False + in_zxxk = True + i += 1 + elif line.strip() == '......': + # 分隔符,根据当前位置决定归属 + i += 1 + elif in_twole: + # 21 世纪教育的数据行:标题在一行,url 在下一行,可选日期在第三行 + title = line + if i + 1 < len(lines) and lines[i + 1].startswith('https://www.21cnjy.com/'): + url = lines[i + 1] + if i + 2 < len(lines) and is_date_time(lines[i + 2]): + date = lines[i + 2] + twole_titles.append(title) + twole_urls.append(url) + twole_dates.append(date) + i += 3 + else: + twole_titles.append(title) + twole_urls.append(url) + twole_dates.append('1970-1-1') + i += 2 + else: + twole_titles.append(title) + twole_urls.append('') + twole_dates.append('1970-1-1') + i += 1 + elif in_zxxk: + # 学科网的数据行:标题在一行,url 在下一行,可选日期在第三行 + title = line + if i + 1 < len(lines) and lines[i + 1].startswith('https://www.zxxk.com/'): + url = lines[i + 1] + if i + 2 < len(lines) and is_date_time(lines[i + 2]): + date = lines[i + 2] + zxxk_titles.append(title) + zxxk_urls.append(url) + zxxk_dates.append(date) + i += 3 + else: + zxxk_titles.append(title) + zxxk_urls.append(url) + zxxk_dates.append('1970-1-1') + i += 2 + else: + zxxk_titles.append(title) + zxxk_urls.append('') + zxxk_dates.append('1970-1-1') + i += 1 + else: + i += 1 + + # 配对生成结果:每对包含 6 个元素(标题 1, url1, 日期 1, 标题 2, url2, 日期 2) + result = [] + # 取两者中较小长度,避免索引越界 + count = min(len(twole_titles), len(zxxk_titles)) + for j in range(count): + # 构造 6 个字段 + parts = [ + twole_titles[j], # 21 世纪教育标题 + twole_urls[j], # 21 世纪教育 url + twole_dates[j], # 21 世纪教育日期 + zxxk_titles[j], # 学科网标题 + zxxk_urls[j], # 学科网 url + zxxk_dates[j] # 学科网日期 + ] + result.append('\t'.join(parts)) + + return result + else: + # 每行已经有\t 分割,直接返回 + return lines + + except Exception as e: + print(f"读取 .docx 失败 ({doc_path.name}): {e}") + return [] + + +def read_doc_content(doc_path: Path) -> List[str]: + """读取 .doc 格式 Word 文档内容(使用 LibreOffice)""" + import subprocess + import tempfile + + soffice_paths = [ + '/Applications/LibreOffice.app/Contents/MacOS/soffice', + '/Applications/LibreOffice.app/Contents/MacOS/libreoffice', + 'soffice', + ] + + print(f"🔍 正在查找 LibreOffice...") + soffice_cmd = None + for path in soffice_paths: + print(f" 检查:{path}") + try: + import os + if os.path.isfile(path): + print(f" ✓ 文件存在") + result = subprocess.run([path, '--version'], capture_output=True, timeout=10) + print(f" ✓ 版本检查返回码:{result.returncode}") + print(f" 输出:{result.stdout.decode()[:100]}") + if result.returncode == 0: + soffice_cmd = path + print(f"✓ 使用 LibreOffice: {path}") + break + except Exception as e: + print(f" ✗ 错误:{e}") + continue + + if not soffice_cmd: + print("❌ LibreOffice 未安装或无法访问") + return [] + + print(f"📄 开始转换文件:{doc_path}") + try: + with tempfile.TemporaryDirectory() as tmpdir: + print(f"📁 临时目录:{tmpdir}") + result = subprocess.run( + [soffice_cmd, '--headless', '--convert-to', 'txt', '--outdir', tmpdir, str(doc_path)], + capture_output=True, + timeout=60 + ) + print(f"🔄 转换返回码:{result.returncode}") + if result.returncode != 0: + print(f"✗ 转换失败:{result.stderr.decode()[:300]}") + return [] + txt_file = Path(tmpdir) / (doc_path.stem + '.txt') + if txt_file.exists(): + with open(txt_file, 'r', encoding='utf-8', errors='ignore') as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + print(f"✓ 成功读取 {len(lines)} 行内容") + return lines + else: + print(f"✗ 转换后的文本文件不存在") + except Exception as e: + print(f"✗ LibreOffice 读取失败:{e}") + + print(f"⚠️ 无法读取 .doc 文件 ({doc_path.name})") + return [] + + +def find_word_doc_recursive(base_dir: Path) -> Optional[Path]: + """递归查找 base_dir 下符合目标的 Word 文档 + + 目标目录条件:包含一个 Word 文件 (.docx 或 .doc) 以及两个子目录 + 如果当前目录只有一个子目录且无 Word 文件,则继续深入查找 + """ + def find_target_dir(current_dir: Path) -> Optional[Path]: + items = list(current_dir.iterdir()) + subdirs = [item for item in items if item.is_dir()] + word_files = [item for item in items if item.suffix.lower() in ('.docx', '.doc')] + + # 目标条件:恰好一个 Word 文件且至少两个子目录 + if len(word_files) == 1 and len(subdirs) >= 2: + return word_files[0] + + # 只有一个子目录且没有 Word 文件,继续深入 + if len(subdirs) == 1 and len(word_files) == 0: + return find_target_dir(subdirs[0]) + + return None + + return find_target_dir(base_dir) + + +def get_files_with_times(folder: Path) -> List[Tuple[Path, datetime]]: + """获取文件夹中所有文件及其创建时间""" + files = [] + for f in folder.iterdir(): + if f.is_file(): + stat_info = f.stat() + # macOS 使用 st_birthtime 作为创建时间,其他系统使用 st_ctime + try: + ctime = datetime.fromtimestamp(stat_info.st_birthtime) + except AttributeError: + ctime = datetime.fromtimestamp(stat_info.st_ctime) + files.append((f, ctime)) + files.sort(key=lambda x: x[1]) + return files + + +def extract_zip_recursive(zip_path: Path, extract_dir: Path, clear_dir: bool = True) -> List[Path]: + """递归解压 zip 文件,如果解压后的文件还有 zip,继续解压""" + # 先清空 extract_dir 中的内容 + if clear_dir: + for item in extract_dir.iterdir(): + if item.is_file(): + item.unlink() + elif item.is_dir(): + shutil.rmtree(item) + + # 解压当前层并保留原始时间戳 + with zipfile.ZipFile(zip_path, 'r') as zip_ref: + for member in zip_ref.infolist(): + zip_ref.extract(member, extract_dir) + + member_path = extract_dir / member.filename + if member_path.exists(): + date_time = datetime(*member.date_time[:5]) + timestamp = date_time.timestamp() + os.utime(member_path, (timestamp, timestamp)) + + # 查找解压后的所有 zip 文件,递归解压 + new_zips = list(extract_dir.glob("*.zip")) + while new_zips: + for z in new_zips: + with zipfile.ZipFile(z, 'r') as zip_ref: + for member in zip_ref.infolist(): + zip_ref.extract(member, extract_dir) + + member_path = extract_dir / member.filename + if member_path.exists(): + date_time = datetime(*member.date_time[:5]) + timestamp = date_time.timestamp() + os.utime(member_path, (timestamp, timestamp)) + z.unlink() # 删除已解压的 zip 文件 + new_zips = list(extract_dir.glob("*.zip")) + + # 递归收集所有解压后的文件(包括子目录) + def collect_all_files(dir_path: Path) -> List[Path]: + files = [] + for item in dir_path.iterdir(): + if item.is_file(): + files.append(item) + elif item.is_dir(): + files.extend(collect_all_files(item)) + return files + + extracted_files = collect_all_files(extract_dir) + return extracted_files + + +def has_audio_video_files(files_list: List[Dict[str, Union[str, float]]]) -> bool: + """检查文件列表中是否包含音视频文件""" + audio_video_exts = {'.mp3', '.mp4', '.avi', '.mkv', '.wav', '.flac', '.aac', '.ogg', '.wma', '.mov', '.wmv'} + for file_entry in files_list: + name = file_entry.get('name', '') + if any(name.lower().endswith(ext) for ext in audio_video_exts): + return True + return False + + +def build_file_associations( + word_content: List[str], + twole_folder: Path, + zxxk_folder: Path, +) -> Dict: + """构建文件关联关系""" + result = [] + problematic_groups = [] + + twole_files = get_files_with_times(twole_folder) + zxxk_files = get_files_with_times(zxxk_folder) + + for i, content in enumerate(word_content): + # 按 \t 分割成段 + parts = content.split('\t') + if len(parts) < 6: + continue + + # 提取两组数据:标题、网址、时间 + title1, url1, time1 = parts[0].strip(), parts[1].strip(), parts[2].strip() + title2, url2, time2 = parts[3].strip(), parts[4].strip(), parts[5].strip() + + # 辅助函数:处理文件路径,如果是 zip 则递归解压 + def process_file(file_path: Path) -> List[Dict[str, Union[str, float]]]: + def format_datetime(mtime: float) -> str: + """将时间戳格式化为可读的日期时间字符串""" + return datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S") + + files = [] + # 检查是否是压缩文件 + if file_path.suffix.lower() in ('.zip', '.rar', '.7z'): + # 先添加压缩文件本身 + mtime = file_path.stat().st_mtime + files.append({"name": file_path.name, "mtime": mtime, "datetime": format_datetime(mtime)}) + + tmp_dir = Path(__file__).parent / "tmp" + tmp_dir.mkdir(exist_ok=True) + extract_target = tmp_dir / file_path.stem + # 清空 extract_target 目录 + clear_directory(extract_target) + extract_target.mkdir(parents=True, exist_ok=True) + + # 根据压缩格式选择解压方式 + if file_path.suffix.lower() == '.zip': + extracted = extract_zip_recursive(file_path, extract_target) + else: + # 其他压缩格式使用 subprocess + import subprocess + subprocess.run(['unzip', '-o', str(file_path), '-d', str(extract_target)], + capture_output=True) + extracted = list(extract_target.iterdir()) + + # 将解压后的文件名和修改时间添加到列表 + for f in extracted: + if f.is_file(): + mtime = f.stat().st_mtime + files.append({"name": f.name, "mtime": mtime, "datetime": format_datetime(mtime)}) + else: + mtime = file_path.stat().st_mtime + files.append({"name": file_path.name, "mtime": mtime, "datetime": format_datetime(mtime)}) + return files + + def get_latest_extracted_mtime(file_entries: List[Dict[str, Union[str, float]]], archive_name: str) -> Optional[float]: + extracted_times = [ + entry["mtime"] + for entry in file_entries + if entry.get("name") != archive_name and "mtime" in entry + ] + if not extracted_times: + return None + + # 找出压缩文件本身的时间 + archive_mtime = None + for entry in file_entries: + if entry.get("name") == archive_name and "mtime" in entry: + archive_mtime = entry["mtime"] + break + + latest_extracted = max(extracted_times) + + # 如果压缩文件时间和解压文件最晚时间差小于 30 秒,返回 None + if archive_mtime is not None and abs(latest_extracted - archive_mtime) < 300: + return None + + return latest_extracted + + # 初始化条目 + entry = {} + group1_files = [] + group2_files = [] + group1_latest = None + group2_latest = None + + # 第一组:根据网址匹配文件 + if 'www.21cnjy.com' in url1 and twole_files: + candidates = twole_files + best_match = find_best_match(title1, candidates) + if best_match: + twole_files.remove(best_match) + file_path, ctime = best_match + group1_files = process_file(file_path) + entry["group1"] = { + "files": group1_files, + "title": title1, + "url": url1, + "time": time1 + } + if file_path.suffix.lower() in ('.zip', '.rar', '.7z'): + group1_latest = get_latest_extracted_mtime(group1_files, file_path.name) + elif 'www.zxxk.com' in url1 and zxxk_files: + candidates = zxxk_files + best_match = find_best_match(title1, candidates) + if best_match: + zxxk_files.remove(best_match) + file_path, ctime = best_match + group1_files = process_file(file_path) + entry["group1"] = { + "files": group1_files, + "title": title1, + "url": url1, + "time": time1 + } + if file_path.suffix.lower() in ('.zip', '.rar', '.7z'): + group1_latest = get_latest_extracted_mtime(group1_files, file_path.name) + + # 第二组:根据网址匹配文件 + if 'www.21cnjy.com' in url2 and twole_files: + candidates = twole_files + best_match = find_best_match(title2, candidates) + if best_match: + twole_files.remove(best_match) + file_path, ctime = best_match + group2_files = process_file(file_path) + entry["group2"] = { + "files": group2_files, + "title": title2, + "url": url2, + "time": time2 + } + if file_path.suffix.lower() in ('.zip', '.rar', '.7z'): + group2_latest = get_latest_extracted_mtime(group2_files, file_path.name) + elif 'www.zxxk.com' in url2 and zxxk_files: + candidates = zxxk_files + best_match = find_best_match(title2, candidates) + if best_match: + zxxk_files.remove(best_match) + file_path, ctime = best_match + group2_files = process_file(file_path) + entry["group2"] = { + "files": group2_files, + "title": title2, + "url": url2, + "time": time2 + } + if file_path.suffix.lower() in ('.zip', '.rar', '.7z'): + group2_latest = get_latest_extracted_mtime(group2_files, file_path.name) + + # 如果 group1 和 group2 都是压缩文件,并且 group1 的解压文件最新时间早于 group2(时间差超过阈值),则记录到问题列表 + # 或者如果解压出来的文件中包含音视频文件,也记录到问题列表 + TIME_DIFF_THRESHOLD = 30 # 时间差阈值(秒),避免将 zip 文件和解压文件时间相近的误判为问题组 + time_diff_condition = ( + group1_latest is not None + and group2_latest is not None + and group1_latest < group2_latest + and (group2_latest - group1_latest) > TIME_DIFF_THRESHOLD + ) + if time_diff_condition: + problematic_groups.append({ + "row_index": i, + "group1_latest_extracted_mtime": group1_latest, + "group2_latest_extracted_mtime": group2_latest, + "group1": entry.get("group1"), + "group2": entry.get("group2") + }) + + # 如果有匹配的条目,添加到结果列表 + if entry: + result.append(entry) + + return {"items": result, "problematic_groups": problematic_groups} + + +def main(): + parser = argparse.ArgumentParser(description='数据文件关联处理程序') + parser.add_argument('--extract', action='store_true', help='解压压缩文件') + parser.add_argument('--associate', action='store_true', help='建立文件关联') + parser.add_argument('--batch', action='store_true', help='批量处理所有 zip 文件') + parser.add_argument('--base-dir', type=str, help='基础目录') + parser.add_argument('--delete-original', action='store_true', help='解压后删除原文件') + + args = parser.parse_args() + + # 解压压缩文件 + if args.extract: + source_dir = Path(args.base_dir) if args.base_dir else Path("cc") + extract_all_zips(source_dir, "data", delete_original=args.delete_original) + return + + # 批量处理所有文件夹(已从 cc 目录解压好) + if True or args.batch: + source_dir = Path(args.base_dir) if args.base_dir else Path("cc") + data_dir = Path("data") + server_jsons_dir = Path("server") / "jsons" + server_jsons_dir.mkdir(parents=True, exist_ok=True) + + # 获取 cc 目录下的所有子文件夹(已解压好的文件夹) + folders = [f for f in source_dir.iterdir() if f.is_dir()] + + if not folders: + print(f"在 {source_dir} 中未发现文件夹") + return + + print(f"找到 {len(folders)} 个文件夹,开始批量处理...") + success_count = 0 + fail_count = 0 + all_problem_groups = [] + + for folder in folders: + print(f"\n【开始处理】{folder.name}") + # 清空 data 目录 + clear_directory(data_dir) + + try: + # 拷贝文件夹到 data 目录 + extract_dir = data_dir / folder.name + shutil.copytree(str(folder), str(extract_dir)) + # 更新 folder 变量,避免后续移动时路径不正确 + folder = extract_dir + + # 查找 Word 文档 + word_doc = find_word_doc_recursive(extract_dir) + if not word_doc: + print(f"⚠️ 在 {folder.name} 中未找到 Word 文档") + fail_count += 1 + continue + + # 读取 Word 内容 + word_content = read_word_content(word_doc) + if not word_content: + print(f"⚠️ 无法读取 Word 文档内容") + fail_count += 1 + continue + + base_dir = word_doc.parent + print(f"✓ 找到 Word 文档:{word_doc}") + print(f" 关联目录:{base_dir}") + + # 建立文件关联 + twole_folder = base_dir / "21世纪教育网" + zxxk_folder = base_dir / "学科网" + + if not zxxk_folder.exists(): + print("❌ 文件夹不存在,请先解压文件") + # 将失败的文件移动到本程序所在目录的 ee 子目录 + script_dir = Path(__file__).parent + ee_dir = script_dir / "ee" + ee_dir.mkdir(exist_ok=True) + dest_file = ee_dir / folder.name + shutil.move(str(folder), str(dest_file)) + print(f" 已移动失败文件到:{dest_file}") + fail_count += 1 + continue + if not twole_folder.exists(): + twole_folder = base_dir / "二一教育" + if not twole_folder.exists(): + twole_folder = base_dir / "21世纪教育" + if not twole_folder.exists(): + twole_folder = base_dir / "二一世纪教育" + + if not twole_folder.exists(): + print("❌ 文件夹不存在,请先解压文件") + # 将失败的文件移动到本程序所在目录的 ee 子目录 + script_dir = Path(__file__).parent + ee_dir = script_dir / "ee" + ee_dir.mkdir(exist_ok=True) + dest_file = ee_dir + print(str(folder), str(dest_file)) + shutil.move(str(folder), str(dest_file)) + print(f" 已移动失败文件到:{dest_file}") + fail_count += 1 + continue + + associations = build_file_associations(word_content, twole_folder, zxxk_folder) + for group in associations.get("problematic_groups", []): + group["source_folder"] = folder.name + group["word_doc"] = word_doc.name + all_problem_groups.extend(associations.get("problematic_groups", [])) + + if not associations["items"]: + # 移动到ee目录 + script_dir = Path(__file__).parent + ee_dir = script_dir / "ee" + ee_dir.mkdir(exist_ok=True) + dest_file = ee_dir / folder.name + shutil.move(str(folder), str(dest_file)) + print(f" 已移动空关联文件夹到:{dest_file}") + fail_count += 1 + continue + + # 保存 JSON 文件 + output_file = server_jsons_dir / f"{word_doc.stem}.json" + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(associations, f, ensure_ascii=False, indent=2) + + print(f"✓ JSON 文件已保存到:{output_file}") + + # 删除原文件夹 + if args.delete_original: + shutil.rmtree(folder) + + success_count += 1 + + except Exception as e: + print(f"❌ 处理失败:{e}") + fail_count += 1 + + report_file = server_jsons_dir / "compressed_group_time_issues.json" + with open(report_file, 'w', encoding='utf-8') as f: + json.dump(all_problem_groups, f, ensure_ascii=False, indent=2) + print(f"\n===== 批量处理完成 =====") + print(f"成功:{success_count}, 失败:{fail_count}") + print(f"问题组报告已保存到:{report_file}") + print(len(all_problem_groups)) + + # 将所有涉及的原始文件夹从 cc 目录复制到 ff 目录 + script_dir = Path(__file__).parent + cc_dir = script_dir / "cc" + ff_dir = script_dir / "ff" + ff_dir.mkdir(exist_ok=True) + + # 收集所有需要复制的文件夹(去重) + folders_to_copy = set() + for group in all_problem_groups: + source_folder = group.get("source_folder") + if source_folder: + folders_to_copy.add(source_folder) + + print(f"\n准备复制 {len(folders_to_copy)} 个文件夹到 ff 目录...") + for folder_name in folders_to_copy: + src_folder = cc_dir / folder_name + if src_folder.exists() and src_folder.is_dir(): + dest_folder = ff_dir / folder_name + if not dest_folder.exists(): + print(f" 复制:{folder_name}") + shutil.copytree(str(src_folder), str(dest_folder)) + else: + print(f" 已存在,跳过:{folder_name}") + else: + print(f" 未找到源文件夹:{folder_name} (cc/{folder_name})") + + return + +if __name__ == "__main__": + main() diff --git a/server/backup_files.py b/server/backup_files.py new file mode 100644 index 0000000..18e0c02 --- /dev/null +++ b/server/backup_files.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +根据 compressed_group_time_issues.json 的信息,将文件备份到 bak 目录 +""" + +import json +import shutil +from pathlib import Path + + +# 配置路径 +JSON_FILE_PATH = 'server/jsons/2.json' +CC_DIR = Path('cc') +BAK_DIR = Path('bak') + + +def find_subfolder(base_dir: Path, folder_name: str) -> Path | None: + """ + 在 base_dir 下查找名为 folder_name 的子文件夹 + """ + target = base_dir / folder_name + if target.exists() and target.is_dir(): + return target + return None + + +def find_21cnjy_folder(base_dir: Path) -> Path | None: + """ + 查找二一教育(21 世纪教育网)文件夹 + 参照 association_detail.py 第 733-738 行的逻辑 + """ + candidates = [ + "21世纪教育网", + "二一教育", + "21世纪教育", + "二一世纪教育" + ] + for candidate in candidates: + folder = base_dir / candidate + if folder.exists() and folder.is_dir(): + return folder + return None + + +def find_zxxk_folder(base_dir: Path) -> Path | None: + """ + 查找学科网文件夹 + """ + return base_dir / "学科网" + + +def copy_file_with_path(src_file: Path, dst_dir: Path) -> None: + """ + 复制文件及其 2 级路径(source_folder 和 二一教育/学科网)到目标目录 + """ + if not src_file.exists(): + return + + source_folder_name = src_file.parent.parent.name + platform_folder_name = src_file.parent.name + + # 构建目标路径:bak/source_folder/platform_folder/ + dst_file = dst_dir / source_folder_name / platform_folder_name / src_file.name + + # 创建目标目录 + dst_file.parent.mkdir(parents=True, exist_ok=True) + + # 复制文件 + shutil.copy2(str(src_file), str(dst_file)) + + +def process_record(record: dict, cc_dir: Path, bak_dir: Path) -> dict: + """ + 处理一条记录,返回处理结果 + """ + result = { + 'row_index': record.get('row_index'), + 'source_folder': record.get('source_folder'), + 'group1_copied': False, + 'group2_copied': False, + 'group1_file': None, + 'group2_file': None, + 'error': None + } + + source_folder_name = record.get('source_folder') + if not source_folder_name: + result['error'] = 'Missing source_folder' + return result + + # 在 cc 目录下查找 source_folder + source_dir = find_subfolder(cc_dir, source_folder_name) + if not source_dir: + result['error'] = f'Source folder not found: {source_folder_name}' + return result + + # 查找二一教育和学科网文件夹 + group1_folder = find_21cnjy_folder(source_dir) + group2_folder = find_zxxk_folder(source_dir) + + if not group2_folder: + result['error'] = f'学科网 folder not found in {source_folder_name}' + return result + + if not group1_folder: + result['error'] = f'二一教育 folder not found in {source_folder_name}' + return result + + # 获取 group1 和 group2 的第一个文件(zip 文件) + group1_files = record.get('group1', {}).get('files', []) + group2_files = record.get('group2', {}).get('files', []) + + if not group1_files or not group2_files: + result['error'] = 'Missing files in group1 or group2' + return result + + group1_zip_name = group1_files[0].get('name') + group2_zip_name = group2_files[0].get('name') + + # 构建源文件路径 + group1_src_file = group1_folder / group1_zip_name + group2_src_file = group2_folder / group2_zip_name + + # 复制 group1 文件(二一教育) + if group1_src_file.exists(): + copy_file_with_path(group1_src_file, bak_dir) + result['group1_copied'] = True + result['group1_file'] = group1_zip_name + else: + result['error'] = f'Group1 file not found: {group1_src_file}' + return result + + # 复制 group2 文件(学科网) + if group2_src_file.exists(): + copy_file_with_path(group2_src_file, bak_dir) + result['group2_copied'] = True + result['group2_file'] = group2_zip_name + else: + result['error'] = f'Group2 file not found: {group2_src_file}' + return result + + return result + + +def main(): + """主函数""" + print("=" * 60) + print("开始备份文件到 bak 目录") + print("=" * 60) + + # 加载 JSON 数据 + with open(JSON_FILE_PATH, 'r', encoding='utf-8') as f: + data = json.load(f) + + print(f"共 {len(data)} 条记录需要处理\n") + + success_count = 0 + fail_count = 0 + + for record in data: + row_index = record.get('row_index', 'unknown') + source_folder = record.get('source_folder', 'unknown') + + result = process_record(record, CC_DIR, BAK_DIR) + + if result['group1_copied'] and result['group2_copied']: + success_count += 1 + print(f"✓ [{row_index}] {source_folder}") + print(f" 二一教育:{result['group1_file']}") + print(f" 学科网:{result['group2_file']}") + else: + fail_count += 1 + print(f"✗ [{row_index}] {source_folder}") + print(f" 错误:{result['error']}") + + print("\n" + "=" * 60) + print(f"备份完成!成功:{success_count} 条,失败:{fail_count} 条") + print("=" * 60) + + +if __name__ == '__main__': + main() diff --git a/server/import_json_to_mysql.py b/server/import_json_to_mysql.py new file mode 100644 index 0000000..a5a053d --- /dev/null +++ b/server/import_json_to_mysql.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +将 compressed_group_time_issues.json 数据导入到 MySQL 数据库 +""" + +import json +import mysql.connector +from mysql.connector import Error + + +# MySQL 连接配置 +DB_CONFIG = { + 'host': '127.0.0.1', + 'port': 3306, + 'user': 'root', + 'password': 'root123456', + 'database': 'testdb', + 'charset': 'utf8mb4' +} + +# JSON 文件路径 +JSON_FILE_PATH = 'server/jsons/compressed_group_time_issues.json' + +# 建表 SQL +CREATE_TABLE_SQL = """ +CREATE TABLE IF NOT EXISTS compressed_group_time_issues ( + id INT AUTO_INCREMENT PRIMARY KEY COMMENT '主键', + row_index INT COMMENT '行索引', + + -- Group1 字段 + group1_latest_extracted_mtime DOUBLE COMMENT 'group1 最新提取文件的 mtime', + group1_title VARCHAR(1000) COMMENT 'group1 标题', + group1_url VARCHAR(1000) COMMENT 'group1 URL', + group1_time VARCHAR(100) COMMENT 'group1 时间', + group1_file_name VARCHAR(1000) COMMENT 'group1 第一个文件名', + group1_file_mtime DOUBLE COMMENT 'group1 第一个文件 mtime', + group1_file_datetime VARCHAR(50) COMMENT 'group1 第一个文件 datetime', + + -- Group2 字段 + group2_latest_extracted_mtime DOUBLE COMMENT 'group2 最新提取文件的 mtime', + group2_title VARCHAR(1000) COMMENT 'group2 标题', + group2_url VARCHAR(1000) COMMENT 'group2 URL', + group2_time VARCHAR(100) COMMENT 'group2 时间', + group2_file_name VARCHAR(1000) COMMENT 'group2 第一个文件名', + group2_file_mtime DOUBLE COMMENT 'group2 第一个文件 mtime', + group2_file_datetime VARCHAR(50) COMMENT 'group2 第一个文件 datetime', + + -- 来源信息 + source_folder VARCHAR(500) COMMENT '来源文件夹', + word_doc VARCHAR(500) COMMENT '关联的 word 文档', + + -- 索引 + INDEX idx_row_index (row_index), + INDEX idx_source_folder (source_folder) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='压缩组时间问题数据表'; +""" + + +def create_connection(): + """创建数据库连接""" + try: + connection = mysql.connector.connect(**DB_CONFIG) + if connection.is_connected(): + print(f"成功连接到 MySQL 数据库") + print(f"数据库版本:{connection.get_server_info()}") + return connection + except Error as e: + print(f"连接 MySQL 失败:{e}") + return None + + +def create_table(connection): + """创建表""" + try: + cursor = connection.cursor() + cursor.execute(CREATE_TABLE_SQL) + connection.commit() + print("表 'compressed_group_time_issues' 创建成功!") + cursor.close() + except Error as e: + print(f"创建表失败:{e}") + + +def load_json_data(file_path): + """加载 JSON 数据""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + print(f"成功加载 JSON 文件,共 {len(data)} 条记录") + return data + except FileNotFoundError: + print(f"文件不存在:{file_path}") + return None + except json.JSONDecodeError as e: + print(f"JSON 解析错误:{e}") + return None + + +def flatten_record(record): + """ + 扁平化记录 + """ + # 获取 group1 的第一个文件 + group1_files = record.get('group1', {}).get('files', []) + group1_first_file = group1_files[0] if group1_files else {} + + # 获取 group2 的第一个文件 + group2_files = record.get('group2', {}).get('files', []) + group2_first_file = group2_files[0] if group2_files else {} + + return { + 'row_index': record.get('row_index'), + + # Group1 字段 + 'group1_latest_extracted_mtime': record.get('group1_latest_extracted_mtime'), + 'group1_title': record.get('group1', {}).get('title'), + 'group1_url': record.get('group1', {}).get('url'), + 'group1_time': record.get('group1', {}).get('time'), + 'group1_file_name': group1_first_file.get('name'), + 'group1_file_mtime': group1_first_file.get('mtime'), + 'group1_file_datetime': group1_first_file.get('datetime'), + + # Group2 字段 + 'group2_latest_extracted_mtime': record.get('group2_latest_extracted_mtime'), + 'group2_title': record.get('group2', {}).get('title'), + 'group2_url': record.get('group2', {}).get('url'), + 'group2_time': record.get('group2', {}).get('time'), + 'group2_file_name': group2_first_file.get('name'), + 'group2_file_mtime': group2_first_file.get('mtime'), + 'group2_file_datetime': group2_first_file.get('datetime'), + + # 来源信息 + 'source_folder': record.get('source_folder'), + 'word_doc': record.get('word_doc') + } + + +def insert_data(connection, data): + """插入数据""" + insert_sql = """ + INSERT INTO compressed_group_time_issues ( + row_index, + group1_latest_extracted_mtime, group1_title, group1_url, group1_time, + group1_file_name, group1_file_mtime, group1_file_datetime, + group2_latest_extracted_mtime, group2_title, group2_url, group2_time, + group2_file_name, group2_file_mtime, group2_file_datetime, + source_folder, word_doc + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, %s, %s + ) + """ + + try: + cursor = connection.cursor() + inserted_count = 0 + + for record in data: + flattened = flatten_record(record) + + values = ( + flattened['row_index'], + flattened['group1_latest_extracted_mtime'], + flattened['group1_title'], + flattened['group1_url'], + flattened['group1_time'], + flattened['group1_file_name'], + flattened['group1_file_mtime'], + flattened['group1_file_datetime'], + flattened['group2_latest_extracted_mtime'], + flattened['group2_title'], + flattened['group2_url'], + flattened['group2_time'], + flattened['group2_file_name'], + flattened['group2_file_mtime'], + flattened['group2_file_datetime'], + flattened['source_folder'], + flattened['word_doc'] + ) + + cursor.execute(insert_sql, values) + inserted_count += 1 + + if inserted_count % 50 == 0: + print(f"已插入 {inserted_count} 条记录...") + + connection.commit() + print(f"成功插入 {inserted_count} 条记录!") + cursor.close() + return inserted_count + except Error as e: + print(f"插入数据失败:{e}") + connection.rollback() + return 0 + + +def verify_data(connection): + """验证导入的数据""" + try: + cursor = connection.cursor() + + # 查询总记录数 + cursor.execute("SELECT COUNT(*) FROM compressed_group_time_issues") + count = cursor.fetchone()[0] + print(f"\n验证:表中总记录数 = {count}") + + # 查询前 2 条记录作为示例 + cursor.execute(""" + SELECT row_index, group1_title, group2_title, source_folder + FROM compressed_group_time_issues + LIMIT 2 + """) + rows = cursor.fetchall() + print("\n示例数据:") + for row in rows: + print(f" row_index={row[0]}") + print(f" group1_title={row[1][:50]}...") + print(f" group2_title={row[2][:50]}...") + print(f" source_folder={row[3]}") + print("-" * 50) + + cursor.close() + except Error as e: + print(f"验证数据失败:{e}") + + +def main(): + """主函数""" + print("=" * 60) + print("开始导入 compressed_group_time_issues.json 到 MySQL") + print("=" * 60) + + # 1. 创建数据库连接 + connection = create_connection() + if not connection: + return + + # 2. 创建表 + create_table(connection) + + # 3. 加载 JSON 数据 + data = load_json_data(JSON_FILE_PATH) + if not data: + connection.close() + return + + # 4. 插入数据 + insert_data(connection, data) + + # 5. 验证数据 + verify_data(connection) + + # 关闭连接 + connection.close() + print("\n数据库连接已关闭") + print("=" * 60) + print("导入完成!") + print("=" * 60) + + +if __name__ == '__main__': + main() diff --git a/server/pbassociation_detail.py b/server/pbassociation_detail.py new file mode 100644 index 0000000..57768c9 --- /dev/null +++ b/server/pbassociation_detail.py @@ -0,0 +1,730 @@ +# -*- coding: utf-8 -*- +""" +数据文件关联处理程序 +功能:解压压缩文件,读取 Word 文档,将文件与 Word 内容关联 +""" + +import os +import sys +import zipfile +import json +import argparse +from pathlib import Path +from typing import Optional, List, Dict, Tuple, Union +from datetime import datetime +import time +import re +import shutil +import requests +import math + +OLLAMA_EMBED_URL = os.environ.get('OLLAMA_EMBED_URL', 'http://localhost:11434/api/embeddings') +OLLAMA_EMBED_MODEL = os.environ.get('OLLAMA_EMBED_MODEL', 'nomic/text-embedding-3-large') + + +def clear_directory(dir_path: Path) -> None: + """清空目录中的所有文件和子目录""" + if dir_path.exists(): + for item in dir_path.iterdir(): + if item.is_file(): + item.unlink() + elif item.is_dir(): + shutil.rmtree(item) + + +def get_embedding(text: str) -> List[float]: + """使用 Ollama 获取文本的 embedding""" + try: + response = requests.post( + OLLAMA_EMBED_URL, + json={ + 'model': OLLAMA_EMBED_MODEL, + 'input': text, + }, + timeout=10, + ) + response.raise_for_status() + data = response.json() + if isinstance(data, dict): + if 'embedding' in data: + return data['embedding'] + if 'data' in data and isinstance(data['data'], list) and data['data']: + return data['data'][0].get('embedding', []) or [] + return [] + except Exception as e: + print(f"获取 embedding 失败: {e}") + return [] + + +def cosine_similarity(vec1: List[float], vec2: List[float]) -> float: + """计算两个向量的余弦相似度""" + if not vec1 or not vec2: + return 0.0 + dot = sum(a * b for a, b in zip(vec1, vec2)) + norm1 = math.sqrt(sum(a * a for a in vec1)) + norm2 = math.sqrt(sum(b * b for b in vec2)) + return dot / (norm1 * norm2) if norm1 and norm2 else 0.0 + + +def normalize_for_match(text: str) -> str: + """规范化文本,去掉路径和扩展名后的匹配用字符串""" + text = str(text).lower() + text = re.sub(r'[\u0020-\u002f\u003a-\u0040\u005b-\u0060\u007b-\u007e]+', ' ', text) + text = re.sub(r'\s+', ' ', text).strip() + return text + + +def token_overlap_score(a: str, b: str) -> float: + tokens_a = set(re.findall(r'[\u4e00-\u9fff]+|\d+|[a-z]+', a)) + tokens_b = set(re.findall(r'[\u4e00-\u9fff]+|\d+|[a-z]+', b)) + if not tokens_a or not tokens_b: + return 0.0 + return len(tokens_a & tokens_b) / max(1, len(tokens_b)) + + +def find_best_match(title: str, candidates: List[Tuple[Path, datetime]]) -> Tuple[Path, datetime]: + """从候选文件中找到与 title 最相似的文件""" + if not candidates: + return None + + title_norm = normalize_for_match(title) + exact_candidates = [] + for file_path, ctime in candidates: + stem_norm = normalize_for_match(file_path.stem) + if stem_norm and (stem_norm == title_norm or stem_norm in title_norm or title_norm in stem_norm): + exact_candidates.append((file_path, ctime, stem_norm)) + + if exact_candidates: + # 直接返回最精确的“包含匹配”候选项 + return max(exact_candidates, key=lambda item: len(item[2]))[:2] + + title_emb = get_embedding(title) + best_match = None + best_score = -1.0 + + for file_path, ctime in candidates: + stem_norm = normalize_for_match(file_path.stem) + overlap = token_overlap_score(title_norm, stem_norm) + score = overlap * 0.3 + + if title_emb: + stem_emb = get_embedding(file_path.stem) + if stem_emb: + score += cosine_similarity(title_emb, stem_emb) * 0.7 + + if score > best_score: + best_score = score + best_match = (file_path, ctime) + + return best_match or candidates[0] + + +def read_word_content(doc_path: Path) -> List[str]: + """读取 Word 文档内容 (.docx 或 .doc),提取文本行""" + if doc_path.suffix.lower() == '.docx': + return read_docx_content(doc_path) + elif doc_path.suffix.lower() == '.doc': + return read_doc_content(doc_path) + return [] + + +def read_docx_content(doc_path: Path) -> List[str]: + """读取 .docx 格式 Word 文档内容""" + try: + with zipfile.ZipFile(doc_path, 'r') as z: + content = z.read('word/document.xml').decode('utf-8') + # 提取所有 ... 中的内容,保留原始空白字符 + texts = re.findall(r']*>(.*?)', content, re.DOTALL) + + # 去除 HTML 标签,不替换内部的空白字符 + processed = [] + for text in texts: + text = re.sub(r'<[^>]+>', '', text).strip() + if text: + processed.append(text) + + # 拼接所有文本,保留 \t 和 \n + full_text = '\n'.join(processed) + + # 智能检测:判断是使用 \t 分割 6 段,还是每行一个字段 + lines = full_text.split('\n') + lines = [line.strip() for line in lines if line.strip()] + + # 检测是否每行都是单字段(没有\t) + # 如果前几行都没有\t,可能是每行一个字段 + has_tab_in_first_rows = any('\t' in line for line in lines[:6]) + + if not has_tab_in_first_rows and len(lines) >= 6: + # 判断是否是 6 行一组的模式:第二行是网址且第三行是日期/时间 + def is_url_line(line: str) -> bool: + """判断一行是否是网址""" + return bool(re.match(r'^https?://', line.strip())) + + def is_datetime_line(line: str) -> bool: + """判断一行是否是日期或时间""" + datetime_pattern = r'^\d{4}[-/]\d{1,2}[-/]\d{1,2}\s+\d{1,2}:\d{2}' + date_pattern = r'^\d{4}[-/]\d{1,2}[-/]\d{1,2}' + return bool(re.match(datetime_pattern, line.strip())) or \ + bool(re.match(date_pattern, line.strip())) + + # 检查当前起始位置是否是 6 行一组模式 + def is_six_row_group(lines: list, start_idx: int) -> bool: + """检查从 start_idx 开始是否是 6 行一组""" + if start_idx + 5 < len(lines): + return is_url_line(lines[start_idx + 1]) and \ + is_datetime_line(lines[start_idx + 2]) and \ + is_url_line(lines[start_idx + 4]) and \ + is_datetime_line(lines[start_idx + 5]) + return False + + # 检查当前起始位置是否是 5 行一组模式 + def is_five_row_group(lines: list, start_idx: int) -> bool: + """检查从 start_idx 开始是否是 5 行一组""" + if start_idx + 4 < len(lines): + return is_url_line(lines[start_idx + 1]) and \ + is_datetime_line(lines[start_idx + 2]) and \ + is_url_line(lines[start_idx + 4]) + return False + + # 按组处理,每组独立判断 offset + # 规则:优先检查 6 行一组,如果是则合并 6 行;否则检查 5 行一组,如果是则合并 5 行并补充时间;否则丢弃第一行,继续判断 + result = [] + i = 0 + while i + 4 <= len(lines): + if i + 5 < len(lines) and is_six_row_group(lines, i): + # 是 6 行一组,合并这 6 行 + combined = '\t'.join(lines[i:i+6]) + # 检查并补充缺失的时间信息 + parts = combined.split('\t') + if len(parts) >= 3 and not is_datetime_line(parts[2]): + parts[2] = '1970-1-1' + if len(parts) >= 6 and not is_datetime_line(parts[5]): + parts[5] = '1970-1-1' + combined = '\t'.join(parts) + result.append(combined) + i += 6 + elif is_five_row_group(lines, i): + # 是 5 行一组,合并这 5 行并补充时间2 + combined = '\t'.join(lines[i:i+5]) + # 检查并补充缺失的时间信息 + parts = combined.split('\t') + if len(parts) >= 3 and not is_datetime_line(parts[2]): + parts[2] = '1970-1-1' + # 为第二组补充时间 + if len(parts) >= 5: + parts.append('1970-1-1') + combined = '\t'.join(parts) + result.append(combined) + i += 5 + else: + # 不是有效组,丢弃当前行(相当于 offset+1) + i += 1 + # 处理剩余行(不足 6 行的) + while i < len(lines): + result.append(lines[i]) + i += 1 + return result + else: + # 每行已经有\t分割,直接返回 + return lines + except Exception as e: + print(f"读取 .docx 失败 ({doc_path.name}): {e}") + return [] + + +def read_doc_content(doc_path: Path) -> List[str]: + """读取 .doc 格式 Word 文档内容(使用 LibreOffice)""" + import subprocess + import tempfile + + soffice_paths = [ + '/Applications/LibreOffice.app/Contents/MacOS/soffice', + '/Applications/LibreOffice.app/Contents/MacOS/libreoffice', + 'soffice', + ] + + print(f"🔍 正在查找 LibreOffice...") + soffice_cmd = None + for path in soffice_paths: + print(f" 检查:{path}") + try: + import os + if os.path.isfile(path): + print(f" ✓ 文件存在") + result = subprocess.run([path, '--version'], capture_output=True, timeout=10) + print(f" ✓ 版本检查返回码:{result.returncode}") + print(f" 输出:{result.stdout.decode()[:100]}") + if result.returncode == 0: + soffice_cmd = path + print(f"✓ 使用 LibreOffice: {path}") + break + except Exception as e: + print(f" ✗ 错误:{e}") + continue + + if not soffice_cmd: + print("❌ LibreOffice 未安装或无法访问") + return [] + + print(f"📄 开始转换文件:{doc_path}") + try: + with tempfile.TemporaryDirectory() as tmpdir: + print(f"📁 临时目录:{tmpdir}") + result = subprocess.run( + [soffice_cmd, '--headless', '--convert-to', 'txt', '--outdir', tmpdir, str(doc_path)], + capture_output=True, + timeout=60 + ) + print(f"🔄 转换返回码:{result.returncode}") + if result.returncode != 0: + print(f"✗ 转换失败:{result.stderr.decode()[:300]}") + return [] + txt_file = Path(tmpdir) / (doc_path.stem + '.txt') + if txt_file.exists(): + with open(txt_file, 'r', encoding='utf-8', errors='ignore') as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + print(f"✓ 成功读取 {len(lines)} 行内容") + return lines + else: + print(f"✗ 转换后的文本文件不存在") + except Exception as e: + print(f"✗ LibreOffice 读取失败:{e}") + + print(f"⚠️ 无法读取 .doc 文件 ({doc_path.name})") + return [] + + +def find_word_doc_recursive(base_dir: Path) -> Optional[Path]: + """递归查找 base_dir 下符合目标的 Word 文档 + + 目标目录条件:包含一个 Word 文件 (.docx 或 .doc) 以及两个子目录 + 如果当前目录只有一个子目录且无 Word 文件,则继续深入查找 + """ + def find_target_dir(current_dir: Path) -> Optional[Path]: + items = list(current_dir.iterdir()) + subdirs = [item for item in items if item.is_dir()] + word_files = [item for item in items if item.suffix.lower() in ('.docx', '.doc')] + + # 目标条件:恰好一个 Word 文件且至少两个子目录 + if len(word_files) == 1 and len(subdirs) >= 2: + return word_files[0] + + # 只有一个子目录且没有 Word 文件,继续深入 + if len(subdirs) == 1 and len(word_files) == 0: + return find_target_dir(subdirs[0]) + + return None + + return find_target_dir(base_dir) + + +def get_files_with_times(folder: Path) -> List[Tuple[Path, datetime]]: + """获取文件夹中所有文件及其创建时间""" + files = [] + for f in folder.iterdir(): + if f.is_file(): + stat_info = f.stat() + # macOS 使用 st_birthtime 作为创建时间,其他系统使用 st_ctime + try: + ctime = datetime.fromtimestamp(stat_info.st_birthtime) + except AttributeError: + ctime = datetime.fromtimestamp(stat_info.st_ctime) + files.append((f, ctime)) + files.sort(key=lambda x: x[1]) + return files + + +def extract_zip_recursive(zip_path: Path, extract_dir: Path, clear_dir: bool = True) -> List[Path]: + """解压 zip 文件 + + 使用 macOS 的 open 命令调用 Archive Utility 解压, + 先将 zip 文件移动到 extract_dir 里面,然后在 extract_dir 中解压, + 解压后会生成一个与压缩文件同名的文件夹。 + 不再递归处理解压出来的 zip 文件。 + """ + # 使用 open 命令调用 Archive Utility 解压 + # Archive Utility 会在 zip 文件所在目录生成同名文件夹 + extract_dir= extract_dir.parent + if clear_dir: + for item in extract_dir.iterdir(): + if item.is_file(): + item.unlink() + elif item.is_dir(): + shutil.rmtree(item) + + # 先将 zip 文件移动到 extract_dir 里面 + moved_zip_path = extract_dir / zip_path.name + shutil.move(str(zip_path), str(moved_zip_path)) + + import time + os.system(f'open -W "{moved_zip_path}"') + #time.sleep(0.5) # 等待 Archive Utility 完成解压 + + # 解压后的文件夹名称与 zip 文件名一致(不含扩展名) + # 文件夹位于 extract_dir 中(因为 zip 已经移动到这里) + extracted_folder = extract_dir / moved_zip_path.stem + + # 递归收集解压文件夹中的所有文件(包括子目录) + def collect_all_files(dir_path: Path) -> List[Path]: + files = [] + for item in dir_path.iterdir(): + if item.is_file(): + files.append(item) + elif item.is_dir(): + files.extend(collect_all_files(item)) + return files + + if extracted_folder.exists(): + extracted_files = collect_all_files(extracted_folder) + else: + # 如果文件夹名有差异,查找可能存在的文件夹 + extracted_files = [] + for item in extract_dir.iterdir(): + if item.is_dir() and not item.name.endswith('.app'): + extracted_files = collect_all_files(item) + break + + # 如果还没有收集到文件,直接收集 extract_dir 中的所有文件(排除 zip 文件本身) + if len(extracted_files) == 0: + extracted_files = [ + item for item in extract_dir.iterdir() + if item.is_file() and item != moved_zip_path + ] + if extracted_files: + print(f"⚠️ 解压完成,未找到解压后的文件夹,但直接从 extract_dir 中收集了 {len(extracted_files)} 个文件。") + + return extracted_files + + +def has_audio_video_files(files_list: List[Dict[str, Union[str, float]]]) -> bool: + """检查文件列表中是否包含音视频文件""" + audio_video_exts = {'.mp3', '.mp4', '.avi', '.mkv', '.wav', '.flac', '.aac', '.ogg', '.wma', '.mov', '.wmv'} + for file_entry in files_list: + name = file_entry.get('name', '') + if any(name.lower().endswith(ext) for ext in audio_video_exts): + return True + return False + + +def build_file_associations( + word_content: List[str], + twole_folder: Path, + zxxk_folder: Path, +) -> Dict: + """构建文件关联关系""" + result = [] + problematic_groups = [] + + twole_files = get_files_with_times(twole_folder) + zxxk_files = get_files_with_times(zxxk_folder) + + for i, content in enumerate(word_content): + # 按 \t 分割成段 + parts = content.split('\t') + if len(parts) < 6: + continue + + # 提取两组数据:标题、网址、时间 + title1, url1, time1 = parts[0].strip(), parts[1].strip(), parts[2].strip() + title2, url2, time2 = parts[3].strip(), parts[4].strip(), parts[5].strip() + + # 辅助函数:处理文件路径,如果是 zip 则递归解压 + def process_file(file_path: Path) -> List[Dict[str, Union[str, float]]]: + def format_datetime(mtime: float) -> str: + """将时间戳格式化为可读的日期时间字符串""" + return datetime.fromtimestamp(mtime).strftime("%Y-%m-%d %H:%M:%S") + + files = [] + # 检查是否是压缩文件 + if file_path.suffix.lower() in ('.zip', '.rar', '.7z'): + # 先添加压缩文件本身 + mtime = file_path.stat().st_mtime + files.append({"name": file_path.name, "mtime": mtime, "datetime": format_datetime(mtime)}) + + tmp_dir = Path(__file__).parent / "tmp" + tmp_dir.mkdir(exist_ok=True) + extract_target = tmp_dir / file_path.stem + # 清空 extract_target 目录 + clear_directory(extract_target) + extract_target.mkdir(parents=True, exist_ok=True) + + # 根据压缩格式选择解压方式 + if file_path.suffix.lower() == '.zip': + extracted = extract_zip_recursive(file_path, extract_target) + else: + # 其他压缩格式使用 subprocess + import subprocess + subprocess.run(['unzip', '-o', str(file_path), '-d', str(extract_target)], + capture_output=True) + extracted = list(extract_target.iterdir()) + + # 将解压后的文件名和修改时间添加到列表 + for f in extracted: + if f.is_file(): + mtime = f.stat().st_mtime + files.append({"name": f.name, "mtime": mtime, "datetime": format_datetime(mtime)}) + else: + mtime = file_path.stat().st_mtime + files.append({"name": file_path.name, "mtime": mtime, "datetime": format_datetime(mtime)}) + return files + + def get_latest_extracted_mtime(file_entries: List[Dict[str, Union[str, float]]], archive_name: str) -> Optional[float]: + extracted_times = [ + entry["mtime"] + for entry in file_entries + if entry.get("name") != archive_name and "mtime" in entry + ] + if not extracted_times: + return None + + # 找出压缩文件本身的时间 + archive_mtime = None + for entry in file_entries: + if entry.get("name") == archive_name and "mtime" in entry: + archive_mtime = entry["mtime"] + break + + latest_extracted = max(extracted_times) + + # 如果压缩文件时间和解压文件最晚时间差小于 30 秒,返回 None + if archive_mtime is not None and abs(latest_extracted - archive_mtime) < 300: + return None + + return latest_extracted + + # 初始化条目 + entry = {} + group1_files = [] + group2_files = [] + group1_latest = None + group2_latest = None + + # 第一组:根据网址匹配文件 + if 'www.21cnjy.com' in url1 and twole_files: + candidates = twole_files + best_match = find_best_match(title1, candidates) + if best_match: + twole_files.remove(best_match) + file_path, ctime = best_match + group1_files = process_file(file_path) + entry["group1"] = { + "files": group1_files, + "title": title1, + "url": url1, + "time": time1 + } + if file_path.suffix.lower() in ('.zip', '.rar', '.7z'): + group1_latest = get_latest_extracted_mtime(group1_files, file_path.name) + elif 'www.zxxk.com' in url1 and zxxk_files: + candidates = zxxk_files + best_match = find_best_match(title1, candidates) + if best_match: + zxxk_files.remove(best_match) + file_path, ctime = best_match + group1_files = process_file(file_path) + entry["group1"] = { + "files": group1_files, + "title": title1, + "url": url1, + "time": time1 + } + if file_path.suffix.lower() in ('.zip', '.rar', '.7z'): + group1_latest = get_latest_extracted_mtime(group1_files, file_path.name) + + # 第二组:根据网址匹配文件 + if 'www.21cnjy.com' in url2 and twole_files: + candidates = twole_files + best_match = find_best_match(title2, candidates) + if best_match: + twole_files.remove(best_match) + file_path, ctime = best_match + group2_files = process_file(file_path) + entry["group2"] = { + "files": group2_files, + "title": title2, + "url": url2, + "time": time2 + } + if file_path.suffix.lower() in ('.zip', '.rar', '.7z'): + group2_latest = get_latest_extracted_mtime(group2_files, file_path.name) + elif 'www.zxxk.com' in url2 and zxxk_files: + candidates = zxxk_files + best_match = find_best_match(title2, candidates) + if best_match: + zxxk_files.remove(best_match) + file_path, ctime = best_match + group2_files = process_file(file_path) + entry["group2"] = { + "files": group2_files, + "title": title2, + "url": url2, + "time": time2 + } + if file_path.suffix.lower() in ('.zip', '.rar', '.7z'): + group2_latest = get_latest_extracted_mtime(group2_files, file_path.name) + + # 如果 group1 和 group2 都是压缩文件,并且 group1 的解压文件最新时间早于 group2(时间差超过阈值),则记录到问题列表 + # 或者如果解压出来的文件中包含音视频文件,也记录到问题列表 + TIME_DIFF_THRESHOLD = 30 # 时间差阈值(秒),避免将 zip 文件和解压文件时间相近的误判为问题组 + time_diff_condition = ( + group1_latest is not None + and group2_latest is not None + and group1_latest < group2_latest + and (group2_latest - group1_latest) > TIME_DIFF_THRESHOLD + ) + if time_diff_condition: + problematic_groups.append({ + "row_index": i, + "group1_latest_extracted_mtime": group1_latest, + "group2_latest_extracted_mtime": group2_latest, + "group1": entry.get("group1"), + "group2": entry.get("group2") + }) + + # 如果有匹配的条目,添加到结果列表 + if entry: + result.append(entry) + + return {"items": result, "problematic_groups": problematic_groups} + + +def main(): + parser = argparse.ArgumentParser(description='数据文件关联处理程序') + parser.add_argument('--extract', action='store_true', help='解压压缩文件') + parser.add_argument('--associate', action='store_true', help='建立文件关联') + parser.add_argument('--batch', action='store_true', help='批量处理所有 zip 文件') + parser.add_argument('--base-dir', type=str, help='基础目录') + parser.add_argument('--delete-original', action='store_true', help='解压后删除原文件') + + args = parser.parse_args() + + + # 批量处理所有文件夹(已从 cc 目录解压好) + if True or args.batch: + source_dir = Path(args.base_dir) if args.base_dir else Path("cc") + data_dir = Path("data") + server_jsons_dir = Path("server") / "jsons" + server_jsons_dir.mkdir(parents=True, exist_ok=True) + + # 获取 cc 目录下的所有子文件夹(已解压好的文件夹) + folders = [f for f in source_dir.iterdir() if f.is_dir()] + + if not folders: + print(f"在 {source_dir} 中未发现文件夹") + return + + print(f"找到 {len(folders)} 个文件夹,开始批量处理...") + success_count = 0 + fail_count = 0 + all_problem_groups = [] + + for folder in folders: + print(f"\n【开始处理】{folder.name}") + # 清空 data 目录 + clear_directory(data_dir) + + try: + # 拷贝文件夹到 data 目录 + extract_dir = data_dir / folder.name + shutil.copytree(str(folder), str(extract_dir)) + # 更新 folder 变量,避免后续移动时路径不正确 + folder = extract_dir + + # 查找 Word 文档 + word_doc = find_word_doc_recursive(extract_dir) + if not word_doc: + print(f"⚠️ 在 {folder.name} 中未找到 Word 文档") + fail_count += 1 + continue + + # 读取 Word 内容 + word_content = read_word_content(word_doc) + if not word_content: + print(f"⚠️ 无法读取 Word 文档内容") + fail_count += 1 + continue + + base_dir = word_doc.parent + print(f"✓ 找到 Word 文档:{word_doc}") + print(f" 关联目录:{base_dir}") + + # 建立文件关联 + twole_folder = base_dir / "21世纪教育网" + zxxk_folder = base_dir / "学科网" + + if not zxxk_folder.exists(): + print("❌ 文件夹不存在,请先解压文件") + # 将失败的文件移动到本程序所在目录的 ee 子目录 + script_dir = Path(__file__).parent + ee_dir = script_dir / "ee" + ee_dir.mkdir(exist_ok=True) + dest_file = ee_dir / folder.name + shutil.move(str(folder), str(dest_file)) + print(f" 已移动失败文件到:{dest_file}") + fail_count += 1 + continue + if not twole_folder.exists(): + twole_folder = base_dir / "二一教育" + if not twole_folder.exists(): + twole_folder = base_dir / "21世纪教育" + if not twole_folder.exists(): + twole_folder = base_dir / "二一世纪教育" + + if not twole_folder.exists(): + print("❌ 文件夹不存在,请先解压文件") + # 将失败的文件移动到本程序所在目录的 ee 子目录 + script_dir = Path(__file__).parent + ee_dir = script_dir / "ee" + ee_dir.mkdir(exist_ok=True) + dest_file = ee_dir + print(str(folder), str(dest_file)) + shutil.move(str(folder), str(dest_file)) + print(f" 已移动失败文件到:{dest_file}") + fail_count += 1 + continue + + associations = build_file_associations(word_content, twole_folder, zxxk_folder) + for group in associations.get("problematic_groups", []): + group["source_folder"] = folder.name + group["word_doc"] = word_doc.name + all_problem_groups.extend(associations.get("problematic_groups", [])) + + if not associations["items"]: + # 移动到ee目录 + script_dir = Path(__file__).parent + ee_dir = script_dir / "ee" + ee_dir.mkdir(exist_ok=True) + dest_file = ee_dir / folder.name + shutil.move(str(folder), str(dest_file)) + print(f" 已移动空关联文件夹到:{dest_file}") + fail_count += 1 + continue + + # 保存 JSON 文件 + output_file = server_jsons_dir / f"{word_doc.stem}.json" + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(associations, f, ensure_ascii=False, indent=2) + + print(f"✓ JSON 文件已保存到:{output_file}") + + # 删除原文件夹 + if args.delete_original: + shutil.rmtree(folder) + + success_count += 1 + + except Exception as e: + print(f"❌ 处理失败:{e}") + fail_count += 1 + + report_file = server_jsons_dir / "compressed_group_time_issues.json" + with open(report_file, 'w', encoding='utf-8') as f: + json.dump(all_problem_groups, f, ensure_ascii=False, indent=2) + print(f"\n===== 批量处理完成 =====") + print(f"成功:{success_count}, 失败:{fail_count}") + print(f"问题组报告已保存到:{report_file}") + print(len(all_problem_groups)) + return + +if __name__ == "__main__": + main()