add scripts
This commit is contained in:
825
server/classify_zxxk_materials.py
Normal file
825
server/classify_zxxk_materials.py
Normal file
@@ -0,0 +1,825 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
学科网资料时间分类程序。
|
||||
|
||||
按资料文件本身和压缩包内最新子文件的时间关系，把学科网资料分为四类。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent
|
||||
ARCHIVE_EXTENSIONS = {".zip", ".rar", ".7z"}
|
||||
FIVE_MINUTES_SECONDS = 5 * 60
|
||||
DB_CONFIG = {
|
||||
"host": os.environ.get("XKW_DB_HOST", "192.168.0.164"),
|
||||
"port": int(os.environ.get("XKW_DB_PORT", "3307")),
|
||||
"user": os.environ.get("XKW_DB_USER", "root"),
|
||||
"password": os.environ.get("XKW_DB_PASSWORD", "myP#ssw0rd"),
|
||||
"database": os.environ.get("XKW_DB_NAME", "xkw"),
|
||||
"charset": "utf8mb4",
|
||||
}
|
||||
EVIDENCE_TABLE_NAME = os.environ.get("XKW_EVIDENCE_TABLE", "证据10总表")
|
||||
|
||||
SINGLE_FILE = "single_file"
|
||||
ARCHIVE_WITHIN_5_MINUTES = "archive_within_5_minutes"
|
||||
ARCHIVE_CHILD_LATER = "archive_child_later"
|
||||
ARCHIVE_CHILD_EARLIER = "archive_child_earlier"
|
||||
ARCHIVE_REQUIRES_SOURCE_TIME = "archive_requires_source_time"
|
||||
UNSUPPORTED_ARCHIVE = "unsupported_archive"
|
||||
EMPTY_ARCHIVE = "empty_archive"
|
||||
FAILED_ARCHIVE = "failed_archive"
|
||||
|
||||
CLASSIFICATION_CATEGORIES = (
|
||||
SINGLE_FILE,
|
||||
ARCHIVE_WITHIN_5_MINUTES,
|
||||
ARCHIVE_CHILD_LATER,
|
||||
ARCHIVE_CHILD_EARLIER,
|
||||
)
|
||||
|
||||
MATCH_KEYWORDS = [
|
||||
"语文", "数学", "英语", "物理", "化学", "生物", "历史", "地理", "政治", "科学",
|
||||
"中考", "高考", "会考", "学考", "一模", "二模", "三模", "模拟", "月考", "期中",
|
||||
"期末", "联考", "调研", "测试", "试卷", "试题", "课件", "导学案", "教案",
|
||||
]
|
||||
|
||||
|
||||
def format_datetime(ts: Optional[float]) -> Optional[str]:
|
||||
if ts is None:
|
||||
return None
|
||||
return datetime.fromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def clear_directory(dir_path: Path) -> None:
|
||||
if not dir_path.exists():
|
||||
return
|
||||
for item in dir_path.iterdir():
|
||||
if item.is_dir():
|
||||
shutil.rmtree(item)
|
||||
else:
|
||||
item.unlink()
|
||||
|
||||
|
||||
def is_archive(path: Path) -> bool:
|
||||
return path.suffix.lower() in ARCHIVE_EXTENSIONS
|
||||
|
||||
|
||||
def build_file_record(path: Path, source_root: Optional[Path] = None) -> Dict:
|
||||
stat_info = path.stat()
|
||||
record = {
|
||||
"file_name": path.name,
|
||||
"path": str(path.resolve()),
|
||||
"mtime": stat_info.st_mtime,
|
||||
"datetime": format_datetime(stat_info.st_mtime),
|
||||
}
|
||||
if source_root:
|
||||
try:
|
||||
record["relative_path"] = str(path.resolve().relative_to(source_root.resolve()))
|
||||
except ValueError:
|
||||
record["relative_path"] = path.name
|
||||
return record
|
||||
|
||||
|
||||
def extract_zip_with_member_times(zip_path: Path, extract_dir: Path) -> List[Path]:
|
||||
clear_directory(extract_dir)
|
||||
extract_dir.mkdir(parents=True, exist_ok=True)
|
||||
extracted_files: List[Path] = []
|
||||
|
||||
with zipfile.ZipFile(zip_path, "r") as zip_file:
|
||||
for member in zip_file.infolist():
|
||||
extracted_path = zip_file.extract(member, extract_dir)
|
||||
member_path = Path(extracted_path)
|
||||
if not member_path.exists():
|
||||
continue
|
||||
|
||||
timestamp = datetime(*member.date_time[:6]).timestamp()
|
||||
os.utime(member_path, (timestamp, timestamp))
|
||||
if member_path.is_file():
|
||||
extracted_files.append(member_path)
|
||||
|
||||
return extracted_files
|
||||
|
||||
|
||||
def collect_files(base_dir: Path) -> List[Path]:
|
||||
return sorted((path for path in base_dir.rglob("*") if path.is_file()), key=lambda path: str(path).lower())
|
||||
|
||||
|
||||
def extract_external_archive(archive_path: Path, extract_dir: Path) -> List[Path]:
|
||||
clear_directory(extract_dir)
|
||||
extract_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
seven_zip = shutil.which("7z")
|
||||
if not seven_zip:
|
||||
raise RuntimeError(f"需要安装 7z 才能解压 {archive_path.suffix} 文件")
|
||||
|
||||
result = subprocess.run(
|
||||
[seven_zip, "x", "-y", f"-o{extract_dir}", str(archive_path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
error_text = (result.stderr or result.stdout or "").strip()
|
||||
raise RuntimeError(error_text or f"7z 解压失败：{archive_path.name}")
|
||||
|
||||
return collect_files(extract_dir)
|
||||
|
||||
|
||||
def extract_archive(archive_path: Path, extract_dir: Path) -> List[Path]:
|
||||
if archive_path.suffix.lower() == ".zip":
|
||||
return extract_zip_with_member_times(archive_path, extract_dir)
|
||||
return extract_external_archive(archive_path, extract_dir)
|
||||
|
||||
|
||||
def latest_child_mtime_from_files(files: List[Path]) -> Optional[float]:
|
||||
child_times = [file_path.stat().st_mtime for file_path in files if file_path.is_file()]
|
||||
return max(child_times) if child_times else None
|
||||
|
||||
|
||||
def preliminary_archive_category(archive_mtime: float, child_mtime: float) -> str:
|
||||
diff_seconds = child_mtime - archive_mtime
|
||||
if abs(diff_seconds) <= FIVE_MINUTES_SECONDS:
|
||||
return ARCHIVE_WITHIN_5_MINUTES
|
||||
return ARCHIVE_REQUIRES_SOURCE_TIME
|
||||
|
||||
|
||||
def parse_database_datetime(value) -> Tuple[Optional[str], Optional[float]]:
|
||||
if value in (None, ""):
|
||||
return None, None
|
||||
if isinstance(value, datetime):
|
||||
dt = value
|
||||
return dt.strftime("%Y-%m-%d %H:%M:%S"), dt.timestamp()
|
||||
|
||||
text = str(value).strip()
|
||||
formats = (
|
||||
"%d/%m/%y %H:%M:%S",
|
||||
"%d/%m/%y %H:%M",
|
||||
"%d/%m/%Y %H:%M:%S",
|
||||
"%d/%m/%Y %H:%M",
|
||||
"%Y/%m/%d %H:%M:%S",
|
||||
"%Y/%m/%d %H:%M",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M",
|
||||
"%Y/%m/%d",
|
||||
"%Y-%m-%d",
|
||||
)
|
||||
for fmt in formats:
|
||||
try:
|
||||
dt = datetime.strptime(text, fmt)
|
||||
return dt.strftime("%Y-%m-%d %H:%M:%S"), dt.timestamp()
|
||||
except ValueError:
|
||||
continue
|
||||
return text, None
|
||||
|
||||
|
||||
def normalize_for_match(text: str) -> str:
|
||||
text = str(text).lower()
|
||||
text = re.sub(r"[\u0020-\u002f\u003a-\u0040\u005b-\u0060\u007b-\u007e]+", " ", text)
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
return text
|
||||
|
||||
|
||||
def tokenize_for_match(text: str) -> Dict[str, float]:
|
||||
normalized = normalize_for_match(text)
|
||||
compact = re.sub(r"[^0-9a-z\u4e00-\u9fff]+", "", normalized)
|
||||
tokens: Dict[str, float] = {}
|
||||
|
||||
def add_token(token: str, weight: float) -> None:
|
||||
token = re.sub(r"^[年上下前后新旧本]+", "", token)
|
||||
if token:
|
||||
tokens[token] = max(tokens.get(token, 0.0), weight)
|
||||
|
||||
for keyword in MATCH_KEYWORDS:
|
||||
if keyword in compact:
|
||||
add_token(keyword, 1.8)
|
||||
|
||||
for number in re.findall(r"\d{4}|\d+", normalized):
|
||||
add_token(number, 0.6 if len(number) == 4 else 0.3)
|
||||
|
||||
for alpha in re.findall(r"[a-z]+", normalized):
|
||||
add_token(alpha, 0.2)
|
||||
|
||||
for chunk in re.findall(r"[\u4e00-\u9fff]+", compact):
|
||||
for n, weight in ((2, 0.25), (3, 0.7), (4, 1.1)):
|
||||
if len(chunk) < n:
|
||||
continue
|
||||
for index in range(len(chunk) - n + 1):
|
||||
add_token(chunk[index:index + n], weight)
|
||||
|
||||
return tokens
|
||||
|
||||
|
||||
def token_overlap_score(a: str, b: str) -> float:
|
||||
tokens_a = tokenize_for_match(a)
|
||||
tokens_b = tokenize_for_match(b)
|
||||
if not tokens_a or not tokens_b:
|
||||
return 0.0
|
||||
overlap_weight = sum(min(weight, tokens_b.get(token, 0.0)) for token, weight in tokens_a.items() if token in tokens_b)
|
||||
total_weight = sum(tokens_a.values())
|
||||
return overlap_weight / total_weight if total_weight else 0.0
|
||||
|
||||
|
||||
def parse_word_content_lines(raw_lines: List[str]) -> List[str]:
|
||||
def normalize_line(line: str) -> str:
|
||||
return line.replace("\ufeff", "").strip()
|
||||
|
||||
def is_url_line(line: str) -> bool:
|
||||
return bool(re.match(r"^https?://", normalize_line(line)))
|
||||
|
||||
def is_index_line(line: str) -> bool:
|
||||
return bool(re.match(r"^\d+[\.、]?$", normalize_line(line)))
|
||||
|
||||
def is_twole_label(line: str) -> bool:
|
||||
return bool(re.match(r"^(二一教育|21 世纪教育|21 世纪教育网|二一世纪教育)\s*[:：]?$", normalize_line(line)))
|
||||
|
||||
def is_zxxk_label(line: str) -> bool:
|
||||
return bool(re.match(r"^学科网\s*[:：]?$", normalize_line(line)))
|
||||
|
||||
def is_date_time(line: str) -> bool:
|
||||
pattern = r"^\d{4}[-/]\d{1,2}[-/]\d{1,2}(\s+\d{1,2}:\d{1,2}(:\d{1,2})?(\s*(AM|PM|上午 | 下午))?)?$"
|
||||
return bool(re.match(pattern, normalize_line(line)))
|
||||
|
||||
def is_skip_line(line: str) -> bool:
|
||||
return is_index_line(line) or is_twole_label(line) or is_zxxk_label(line) or not normalize_line(line)
|
||||
|
||||
def is_url_fragment(line: str) -> bool:
|
||||
return bool(re.match(r"^[A-Za-z0-9._~:/?#\[\]@!$&'()*+,;=%-]+$", normalize_line(line)))
|
||||
|
||||
def consume_url(lines: List[str], start: int) -> Optional[Tuple[str, int]]:
|
||||
i = start
|
||||
while i < len(lines) and is_skip_line(lines[i]):
|
||||
i += 1
|
||||
if i >= len(lines):
|
||||
return None
|
||||
first_part = normalize_line(lines[i])
|
||||
if not first_part.startswith("http"):
|
||||
return None
|
||||
url_parts = [first_part]
|
||||
i += 1
|
||||
while i < len(lines):
|
||||
line = normalize_line(lines[i])
|
||||
if is_date_time(line):
|
||||
break
|
||||
if is_skip_line(line):
|
||||
i += 1
|
||||
continue
|
||||
if not is_url_fragment(line):
|
||||
break
|
||||
url_parts.append(line)
|
||||
i += 1
|
||||
return "".join(url_parts), i
|
||||
|
||||
def parse_content_item(lines: List[str], start: int) -> Optional[Tuple[str, str, str, int]]:
|
||||
i = start
|
||||
while i < len(lines) and is_skip_line(lines[i]):
|
||||
i += 1
|
||||
title_parts = []
|
||||
while i < len(lines) and len(title_parts) < 8:
|
||||
line = normalize_line(lines[i])
|
||||
if is_url_line(line):
|
||||
break
|
||||
if is_date_time(line):
|
||||
return None
|
||||
if is_skip_line(line):
|
||||
i += 1
|
||||
continue
|
||||
title_parts.append(line)
|
||||
i += 1
|
||||
while i < len(lines) and is_skip_line(lines[i]):
|
||||
i += 1
|
||||
url_result = consume_url(lines, i)
|
||||
if not title_parts or not url_result:
|
||||
return None
|
||||
title = "".join(title_parts)
|
||||
url, i = url_result
|
||||
date = "1970-1-1"
|
||||
if i < len(lines) and is_date_time(lines[i]):
|
||||
date = normalize_line(lines[i])
|
||||
i += 1
|
||||
return title, url, date, i
|
||||
|
||||
def get_site_from_url(url: str) -> Optional[str]:
|
||||
normalized_url = normalize_line(url).lower()
|
||||
if "21cnjy.com" in normalized_url:
|
||||
return "twole"
|
||||
if "zxxk.com" in normalized_url:
|
||||
return "zxxk"
|
||||
return None
|
||||
|
||||
def extract_content_items(lines: List[str]) -> List[Tuple[str, str, str, str]]:
|
||||
items = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
item = parse_content_item(lines, i)
|
||||
if not item:
|
||||
i += 1
|
||||
continue
|
||||
title, url, date, next_i = item
|
||||
site = get_site_from_url(url)
|
||||
if site in ("twole", "zxxk"):
|
||||
items.append((title, url, date, site))
|
||||
i = next_i
|
||||
return items
|
||||
|
||||
def build_grouped_rows(items: List[Tuple[str, str, str, str]]) -> List[str]:
|
||||
twole_items = [(title, url, date) for title, url, date, site in items if site == "twole"]
|
||||
zxxk_items = [(title, url, date) for title, url, date, site in items if site == "zxxk"]
|
||||
count = min(len(twole_items), len(zxxk_items))
|
||||
return ["\t".join([twole_items[index][0], twole_items[index][1], twole_items[index][2],
|
||||
zxxk_items[index][0], zxxk_items[index][1], zxxk_items[index][2]]) for index in range(count)]
|
||||
|
||||
def parse_site_items(lines: List[str]) -> List[str]:
|
||||
items = extract_content_items(lines)
|
||||
sites = [site for _, _, _, site in items]
|
||||
if len(sites) < 2 or len(set(sites)) < 2:
|
||||
return []
|
||||
return build_grouped_rows(items)
|
||||
|
||||
def parse_grouped_site_rows(lines: List[str]) -> List[str]:
|
||||
twole_items = []
|
||||
zxxk_items = []
|
||||
current_site = None
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
line = normalize_line(lines[i])
|
||||
if is_twole_label(line):
|
||||
current_site = "twole"
|
||||
i += 1
|
||||
continue
|
||||
if is_zxxk_label(line):
|
||||
current_site = "zxxk"
|
||||
i += 1
|
||||
continue
|
||||
if not line or line in ("......", "……") or line.startswith("其它内容") or line.startswith("其他内容"):
|
||||
i += 1
|
||||
continue
|
||||
if current_site not in ("twole", "zxxk"):
|
||||
i += 1
|
||||
continue
|
||||
title = line
|
||||
url = ""
|
||||
date = "1970-1-1"
|
||||
url_result = consume_url(lines, i + 1)
|
||||
if url_result:
|
||||
url, i = url_result
|
||||
if i < len(lines) and is_date_time(lines[i]):
|
||||
date = normalize_line(lines[i])
|
||||
i += 1
|
||||
else:
|
||||
i += 1
|
||||
if current_site == "twole":
|
||||
twole_items.append((title, url, date))
|
||||
else:
|
||||
zxxk_items.append((title, url, date))
|
||||
count = min(len(twole_items), len(zxxk_items))
|
||||
return ["\t".join([twole_items[index][0], twole_items[index][1], twole_items[index][2],
|
||||
zxxk_items[index][0], zxxk_items[index][1], zxxk_items[index][2]]) for index in range(count)]
|
||||
|
||||
lines = [normalize_line(line) for line in raw_lines if normalize_line(line)]
|
||||
has_tab_in_first_rows = any("\t" in line for line in lines[:6])
|
||||
split_lines = []
|
||||
for line in lines:
|
||||
split_lines.extend(normalize_line(part) for part in line.split("\t") if normalize_line(part))
|
||||
|
||||
site_result = parse_site_items(split_lines)
|
||||
if site_result:
|
||||
return site_result
|
||||
has_site_labels = any(is_twole_label(line) or is_zxxk_label(line) for line in split_lines)
|
||||
if has_site_labels:
|
||||
grouped_result = parse_grouped_site_rows(split_lines)
|
||||
if grouped_result:
|
||||
return grouped_result
|
||||
if has_tab_in_first_rows or len(split_lines) < 6:
|
||||
return lines
|
||||
return lines
|
||||
|
||||
|
||||
def read_docx_content(doc_path: Path) -> List[str]:
|
||||
def clean_hyperlink(line: str) -> str:
|
||||
patterns = [r"HYPERLINK\s+"([^&]+)"\s+(\S+)", r"HYPERLINK\s+\"([^\"]+)\"\s+(\S+)"]
|
||||
for pattern in patterns:
|
||||
match = re.match(pattern, line)
|
||||
if match:
|
||||
return match.group(2)
|
||||
return line
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(doc_path, "r") as z:
|
||||
content = z.read("word/document.xml").decode("utf-8")
|
||||
texts = re.findall(r"<w:t[^>]*>(.*?)</w:t>", content, re.DOTALL)
|
||||
processed = []
|
||||
for text in texts:
|
||||
text = re.sub(r"<[^>]+>", "", text).strip()
|
||||
if text:
|
||||
processed.append(clean_hyperlink(text))
|
||||
return parse_word_content_lines(processed)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def read_doc_content_old(doc_path: Path) -> List[str]:
|
||||
for soffice_cmd in ["/Applications/LibreOffice.app/Contents/MacOS/soffice",
|
||||
"/Applications/LibreOffice.app/Contents/MacOS/libreoffice", "soffice"]:
|
||||
if os.path.isfile(soffice_cmd) and subprocess.run([soffice_cmd, "--version"], capture_output=True).returncode == 0:
|
||||
break
|
||||
else:
|
||||
return []
|
||||
|
||||
import tempfile
|
||||
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
result = subprocess.run(
|
||||
[soffice_cmd, "--headless", "--convert-to", "txt", "--outdir", tmpdir, str(doc_path)],
|
||||
capture_output=True,
|
||||
timeout=60,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return []
|
||||
txt_file = Path(tmpdir) / (doc_path.stem + ".txt")
|
||||
if txt_file.exists():
|
||||
lines = [line.strip() for line in txt_file.read_text(encoding="utf-8", errors="ignore").splitlines() if line.strip()]
|
||||
return parse_word_content_lines(lines)
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def read_word_content(doc_path: Path) -> List[str]:
|
||||
if doc_path.suffix.lower() == ".docx":
|
||||
return read_docx_content(doc_path)
|
||||
if doc_path.suffix.lower() == ".doc":
|
||||
return read_doc_content_old(doc_path)
|
||||
return []
|
||||
|
||||
|
||||
def find_word_doc_for_zxxk_dir(zxxk_dir: Path) -> Optional[Path]:
|
||||
parent = zxxk_dir.parent
|
||||
word_files = sorted(
|
||||
(item for item in parent.iterdir() if item.is_file() and item.suffix.lower() in (".docx", ".doc")),
|
||||
key=lambda item: item.name.lower(),
|
||||
)
|
||||
return word_files[0] if word_files else None
|
||||
|
||||
|
||||
def extract_zxxk_id(url: str) -> Optional[str]:
|
||||
match = re.search(r"/(?:soft/)?(\d+)\.html(?:[?#].*)?$", url or "")
|
||||
if match:
|
||||
return match.group(1)
|
||||
numbers = re.findall(r"\d+", url or "")
|
||||
return numbers[-1] if numbers else None
|
||||
|
||||
|
||||
def extract_zxxk_metadata_rows(word_doc: Optional[Path]) -> List[Dict]:
|
||||
if not word_doc:
|
||||
return []
|
||||
rows = []
|
||||
for row_index, content in enumerate(read_word_content(word_doc)):
|
||||
parts = [part.strip() for part in content.split("\t")]
|
||||
if len(parts) < 6:
|
||||
continue
|
||||
candidates = [(parts[0], parts[1], parts[2]), (parts[3], parts[4], parts[5])]
|
||||
for title, url, source_time in candidates:
|
||||
if "zxxk.com" not in url.lower():
|
||||
continue
|
||||
rows.append({
|
||||
"title": title,
|
||||
"url": url,
|
||||
"source_time": source_time,
|
||||
"word_source_time": source_time,
|
||||
"zxxk_id": extract_zxxk_id(url),
|
||||
"word_doc": word_doc.name,
|
||||
"word_doc_path": str(word_doc.resolve()),
|
||||
"word_row_index": row_index,
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def match_metadata_for_file(material_path: Path, metadata_rows: List[Dict]) -> Optional[Dict]:
|
||||
if not metadata_rows:
|
||||
return None
|
||||
file_name = material_path.name
|
||||
file_stem = material_path.stem
|
||||
file_norm = normalize_for_match(file_stem)
|
||||
exact_matches = []
|
||||
for row in metadata_rows:
|
||||
title_norm = normalize_for_match(row["title"])
|
||||
if title_norm and (title_norm == file_norm or title_norm in file_norm or file_norm in title_norm):
|
||||
exact_matches.append(row)
|
||||
if exact_matches:
|
||||
return max(exact_matches, key=lambda row: len(normalize_for_match(row["title"])))
|
||||
|
||||
scored_rows = [
|
||||
(token_overlap_score(row["title"], file_name), row)
|
||||
for row in metadata_rows
|
||||
]
|
||||
best_score, best_row = max(scored_rows, key=lambda item: item[0])
|
||||
return best_row if best_score > 0 else None
|
||||
|
||||
|
||||
def enrich_record_with_metadata(record: Dict, material_path: Path, metadata_rows: List[Dict]) -> None:
|
||||
metadata = match_metadata_for_file(material_path, metadata_rows)
|
||||
record["metadata_found"] = metadata is not None
|
||||
if not metadata:
|
||||
record.update({
|
||||
"title": None,
|
||||
"url": None,
|
||||
"source_time": None,
|
||||
"source_time_raw": None,
|
||||
"source_time_mtime": None,
|
||||
"word_source_time": None,
|
||||
"source_time_source": None,
|
||||
"classification_time_basis": None,
|
||||
"child_source_time_diff_seconds": None,
|
||||
"zxxk_id": None,
|
||||
"word_doc": None,
|
||||
"word_doc_path": None,
|
||||
"word_row_index": None,
|
||||
})
|
||||
return
|
||||
record.update(metadata)
|
||||
record["source_time_source"] = "word"
|
||||
|
||||
|
||||
class MysqlUploadTimeLookup:
|
||||
def __init__(self, config: Optional[Dict] = None, table_name: str = EVIDENCE_TABLE_NAME):
|
||||
self.config = dict(config or DB_CONFIG)
|
||||
self.table_name = table_name
|
||||
self.conn = None
|
||||
self.cache: Dict[Tuple[str, str], Optional[str]] = {}
|
||||
|
||||
def close(self) -> None:
|
||||
if self.conn:
|
||||
self.conn.close()
|
||||
self.conn = None
|
||||
|
||||
def connect(self):
|
||||
if self.conn:
|
||||
return self.conn
|
||||
import mysql.connector
|
||||
|
||||
self.conn = mysql.connector.connect(**self.config)
|
||||
return self.conn
|
||||
|
||||
def __call__(self, metadata: Dict) -> Optional[str]:
|
||||
zxxk_id = metadata.get("zxxk_id") or ""
|
||||
url = metadata.get("url") or ""
|
||||
cache_key = (zxxk_id, url)
|
||||
if cache_key not in self.cache:
|
||||
self.cache[cache_key] = self.fetch_upload_time(zxxk_id, url)
|
||||
return self.cache[cache_key]
|
||||
|
||||
def fetch_upload_time(self, zxxk_id: str, url: str) -> Optional[str]:
|
||||
if not zxxk_id and not url:
|
||||
return None
|
||||
|
||||
conn = self.connect()
|
||||
cursor = conn.cursor()
|
||||
try:
|
||||
if zxxk_id:
|
||||
cursor.execute(
|
||||
f"SELECT `学科网上传日期` FROM `{self.table_name}` "
|
||||
"WHERE `xkwID` = %s AND `学科网上传日期` IS NOT NULL AND `学科网上传日期` <> '' LIMIT 1",
|
||||
(zxxk_id,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row and row[0]:
|
||||
return str(row[0])
|
||||
|
||||
if url:
|
||||
cursor.execute(
|
||||
f"SELECT `学科网上传日期` FROM `{self.table_name}` "
|
||||
"WHERE `学科网资料链接` = %s AND `学科网上传日期` IS NOT NULL AND `学科网上传日期` <> '' LIMIT 1",
|
||||
(url,),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if row and row[0]:
|
||||
return str(row[0])
|
||||
finally:
|
||||
cursor.close()
|
||||
return None
|
||||
|
||||
|
||||
def apply_upload_time_lookup(record: Dict, upload_time_lookup) -> None:
|
||||
if not record.get("metadata_found") or not upload_time_lookup:
|
||||
return
|
||||
upload_time = upload_time_lookup(record)
|
||||
if not upload_time:
|
||||
return
|
||||
normalized_time, timestamp = parse_database_datetime(upload_time)
|
||||
record["source_time_raw"] = str(upload_time)
|
||||
record["source_time"] = normalized_time
|
||||
record["source_time_mtime"] = timestamp
|
||||
record["source_time_source"] = "mysql"
|
||||
|
||||
|
||||
def reclassify_archive_with_source_time(record: Dict) -> None:
|
||||
if record.get("file_kind") != "archive" or record.get("latest_child_mtime") is None:
|
||||
return
|
||||
|
||||
source_time_mtime = record.get("source_time_mtime")
|
||||
if source_time_mtime is not None:
|
||||
record["child_source_time_diff_seconds"] = record["latest_child_mtime"] - source_time_mtime
|
||||
|
||||
archive_diff_seconds = record.get("child_archive_time_diff_seconds")
|
||||
if archive_diff_seconds is not None and abs(archive_diff_seconds) <= FIVE_MINUTES_SECONDS:
|
||||
record["category"] = ARCHIVE_WITHIN_5_MINUTES
|
||||
record["classification_time_basis"] = "archive_mtime_within_5_minutes"
|
||||
return
|
||||
|
||||
if source_time_mtime is None:
|
||||
record["category"] = ARCHIVE_REQUIRES_SOURCE_TIME
|
||||
record["classification_time_basis"] = "requires_mysql_upload_time"
|
||||
return
|
||||
|
||||
diff_seconds = record["child_source_time_diff_seconds"]
|
||||
record["classification_time_basis"] = "mysql_upload_time"
|
||||
if diff_seconds > 6000:
|
||||
record["category"] = ARCHIVE_CHILD_LATER
|
||||
else:
|
||||
record["category"] = ARCHIVE_CHILD_EARLIER
|
||||
|
||||
|
||||
def classify_material(material_path: Path, work_dir: Path, source_root: Optional[Path] = None) -> Dict:
|
||||
material_path = Path(material_path)
|
||||
file_record = build_file_record(material_path, source_root=source_root)
|
||||
|
||||
result = {
|
||||
**file_record,
|
||||
"category": SINGLE_FILE,
|
||||
"file_kind": "single_file",
|
||||
"archive_mtime": None,
|
||||
"archive_datetime": None,
|
||||
"latest_child_mtime": None,
|
||||
"latest_child_datetime": None,
|
||||
"child_archive_time_diff_seconds": None,
|
||||
"child_source_time_diff_seconds": None,
|
||||
"classification_time_basis": None,
|
||||
"child_count": 0,
|
||||
}
|
||||
|
||||
if not is_archive(material_path):
|
||||
return result
|
||||
|
||||
result.update({
|
||||
"file_kind": "archive",
|
||||
"archive_mtime": file_record["mtime"],
|
||||
"archive_datetime": file_record["datetime"],
|
||||
})
|
||||
|
||||
extract_dir = Path(work_dir) / material_path.stem
|
||||
child_mtime = None
|
||||
child_count = 0
|
||||
try:
|
||||
extracted_files = extract_archive(material_path, extract_dir)
|
||||
child_count = len(extracted_files)
|
||||
child_mtime = latest_child_mtime_from_files(extracted_files)
|
||||
except Exception as exc:
|
||||
result["category"] = FAILED_ARCHIVE
|
||||
result["error"] = str(exc)
|
||||
return result
|
||||
finally:
|
||||
if extract_dir.exists():
|
||||
shutil.rmtree(extract_dir)
|
||||
|
||||
result["child_count"] = child_count
|
||||
if child_mtime is None:
|
||||
result["category"] = EMPTY_ARCHIVE
|
||||
return result
|
||||
|
||||
diff_seconds = child_mtime - file_record["mtime"]
|
||||
result.update({
|
||||
"category": preliminary_archive_category(file_record["mtime"], child_mtime),
|
||||
"latest_child_mtime": child_mtime,
|
||||
"latest_child_datetime": format_datetime(child_mtime),
|
||||
"child_archive_time_diff_seconds": diff_seconds,
|
||||
"classification_time_basis": "archive_mtime_initial",
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
def find_zxxk_dirs(source_dir: Path) -> List[Path]:
|
||||
source_dir = Path(source_dir)
|
||||
if source_dir.name == "学科网" and source_dir.is_dir():
|
||||
return [source_dir]
|
||||
return sorted(
|
||||
(path for path in source_dir.rglob("学科网") if path.is_dir()),
|
||||
key=lambda path: str(path).lower(),
|
||||
)
|
||||
|
||||
|
||||
def iter_materials(zxxk_dir: Path) -> List[Path]:
|
||||
return sorted(
|
||||
(item for item in zxxk_dir.iterdir() if item.is_file()),
|
||||
key=lambda item: item.name.lower(),
|
||||
)
|
||||
|
||||
|
||||
def empty_result(source_dir: Path) -> Dict:
|
||||
return {
|
||||
"source_dir": str(Path(source_dir).resolve()),
|
||||
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"total_zxxk_dirs": 0,
|
||||
"total_materials": 0,
|
||||
"summary": {category: 0 for category in CLASSIFICATION_CATEGORIES},
|
||||
"categories": {category: [] for category in CLASSIFICATION_CATEGORIES},
|
||||
"other_categories": {},
|
||||
"warnings": [],
|
||||
}
|
||||
|
||||
|
||||
def add_classification(result: Dict, record: Dict) -> None:
|
||||
category = record["category"]
|
||||
if category in result["categories"]:
|
||||
result["categories"][category].append(record)
|
||||
result["summary"][category] += 1
|
||||
else:
|
||||
result["other_categories"].setdefault(category, []).append(record)
|
||||
result["total_materials"] += 1
|
||||
|
||||
|
||||
def classify_zxxk_materials(source_dir: Path, work_dir: Path, upload_time_lookup=None) -> Dict:
|
||||
source_dir = Path(source_dir)
|
||||
work_dir = Path(work_dir)
|
||||
result = empty_result(source_dir)
|
||||
zxxk_dirs = find_zxxk_dirs(source_dir)
|
||||
result["total_zxxk_dirs"] = len(zxxk_dirs)
|
||||
|
||||
for zxxk_dir in zxxk_dirs:
|
||||
word_doc = find_word_doc_for_zxxk_dir(zxxk_dir)
|
||||
metadata_rows = extract_zxxk_metadata_rows(word_doc)
|
||||
for material_path in iter_materials(zxxk_dir):
|
||||
archive_key = hashlib.sha1(str(material_path.resolve()).encode("utf-8")).hexdigest()[:16]
|
||||
relative_work_dir = work_dir / archive_key
|
||||
record = classify_material(material_path, relative_work_dir, source_root=source_dir)
|
||||
enrich_record_with_metadata(record, material_path, metadata_rows)
|
||||
try:
|
||||
apply_upload_time_lookup(record, upload_time_lookup)
|
||||
except Exception as exc:
|
||||
result["warnings"].append(f"查询学科网上传日期失败：{exc}")
|
||||
upload_time_lookup = None
|
||||
reclassify_archive_with_source_time(record)
|
||||
if relative_work_dir.exists() and not any(relative_work_dir.iterdir()):
|
||||
relative_work_dir.rmdir()
|
||||
try:
|
||||
record["zxxk_dir"] = str(zxxk_dir.resolve().relative_to(source_dir.resolve()))
|
||||
except ValueError:
|
||||
record["zxxk_dir"] = str(zxxk_dir.resolve())
|
||||
add_classification(result, record)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def write_json(data: Dict, output_path: Path) -> None:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with output_path.open("w", encoding="utf-8") as f:
|
||||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def print_summary(result: Dict, output_path: Path) -> None:
|
||||
print("===== 学科网资料分类完成 =====")
|
||||
print(f"学科网目录数：{result['total_zxxk_dirs']}")
|
||||
print(f"资料总数：{result['total_materials']}")
|
||||
print(f"1、单文件：{result['summary'][SINGLE_FILE]}")
|
||||
print(f"2、压缩文件，子文件最后时间与压缩包时间在5分钟之内：{result['summary'][ARCHIVE_WITHIN_5_MINUTES]}")
|
||||
print(f"3、剩余压缩文件，子文件最后时间比数据库发布时间更晚：{result['summary'][ARCHIVE_CHILD_LATER]}")
|
||||
print(f"4、剩余压缩文件，子文件最后时间比数据库发布时间更早：{result['summary'][ARCHIVE_CHILD_EARLIER]}")
|
||||
if result["other_categories"]:
|
||||
other_total = sum(len(items) for items in result["other_categories"].values())
|
||||
print(f"其他/异常：{other_total}")
|
||||
print(f"分类结果：{output_path}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="学科网资料时间分类程序")
|
||||
parser.add_argument("--source-dir", type=str, default=str(BASE_DIR / "ccold"), help="待扫描目录，默认 server/cc")
|
||||
parser.add_argument("--work-dir", type=str, default=str(BASE_DIR / "tmp" / "zxxk_material_classify"), help="解压临时目录")
|
||||
parser.add_argument("--output", type=str, default=str(BASE_DIR / "jsons" / "zxxk_material_time_categories.json"), help="分类 JSON 输出路径")
|
||||
parser.add_argument("--skip-db-time", action="store_true", help="不从 MySQL 查询学科网上传日期")
|
||||
args = parser.parse_args()
|
||||
|
||||
source_dir = Path(args.source_dir).expanduser().resolve()
|
||||
work_dir = Path(args.work_dir).expanduser().resolve()
|
||||
output_path = Path(args.output).expanduser().resolve()
|
||||
|
||||
upload_time_lookup = None if args.skip_db_time else MysqlUploadTimeLookup()
|
||||
try:
|
||||
result = classify_zxxk_materials(source_dir, work_dir, upload_time_lookup=upload_time_lookup)
|
||||
finally:
|
||||
if upload_time_lookup:
|
||||
upload_time_lookup.close()
|
||||
write_json(result, output_path)
|
||||
print_summary(result, output_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user