优化

server/association_detail_smart.py

@@ -19,7 +19,13 @@ import requests
 import math
 
 OLLAMA_EMBED_URL = os.environ.get('OLLAMA_EMBED_URL', 'http://localhost:11434/api/embeddings')
-OLLAMA_EMBED_MODEL = os.environ.get('OLLAMA_EMBED_MODEL', 'nomic-embed-text:latest')
+OLLAMA_EMBED_MODEL = os.environ.get('OLLAMA_EMBED_MODEL', 'qwen3-embedding')
+
+MATCH_KEYWORDS = [
+    '语文', '数学', '英语', '物理', '化学', '生物', '历史', '地理', '政治', '科学',
+    '中考', '高考', '会考', '学考', '一模', '二模', '三模', '模拟', '月考', '期中',
+    '期末', '联考', '调研', '测试', '试卷', '试题', '课件', '导学案', '教案'
+]
 
 
 def clear_directory(dir_path: Path) -> None:
@@ -74,12 +80,50 @@ def normalize_for_match(text: str) -> str:
     return text
 
 
+def tokenize_for_match(text: str) -> Dict[str, float]:
+    """为中文匹配生成更细粒度的 token 权重。"""
+    normalized = normalize_for_match(text)
+    compact = re.sub(r'[^0-9a-z\u4e00-\u9fff]+', '', normalized)
+    tokens: Dict[str, float] = {}
+
+    def add_token(token: str, weight: float) -> None:
+        token = re.sub(r'^[年上下前后新旧本]+', '', token)
+        if token:
+            tokens[token] = max(tokens.get(token, 0.0), weight)
+
+    for keyword in MATCH_KEYWORDS:
+        if keyword in compact:
+            add_token(keyword, 1.8)
+
+    for number in re.findall(r'\d{4}|\d+', normalized):
+        add_token(number, 0.6 if len(number) == 4 else 0.3)
+
+    for alpha in re.findall(r'[a-z]+', normalized):
+        add_token(alpha, 0.2)
+
+    for chunk in re.findall(r'[\u4e00-\u9fff]+', compact):
+        for n, weight in ((2, 0.25), (3, 0.7), (4, 1.1)):
+            if len(chunk) < n:
+                continue
+            for index in range(len(chunk) - n + 1):
+                add_token(chunk[index:index + n], weight)
+
+    return tokens
+
+
 def token_overlap_score(a: str, b: str) -> float:
-    tokens_a = set(re.findall(r'[\u4e00-\u9fff]+|\d+|[a-z]+', a))
-    tokens_b = set(re.findall(r'[\u4e00-\u9fff]+|\d+|[a-z]+', b))
+    tokens_a = tokenize_for_match(a)
+    tokens_b = tokenize_for_match(b)
     if not tokens_a or not tokens_b:
         return 0.0
-    return len(tokens_a & tokens_b) / max(1, len(tokens_b))
+
+    overlap_weight = sum(
+        min(weight, tokens_b.get(token, 0.0))
+        for token, weight in tokens_a.items()
+        if token in tokens_b
+    )
+    total_weight = sum(tokens_a.values())
+    return overlap_weight / total_weight if total_weight else 0.0
 
 
 def find_best_match(title: str, candidates: List[Tuple[Path, datetime]]) -> Tuple[Path, datetime]:
@@ -201,34 +245,173 @@ def read_word_content(doc_path: Path) -> List[str]:
     return []
 
 
-def read_docx_content(doc_path: Path) -> List[str]:
-    """读取 .docx 格式 Word 文档内容，并智能兼容分组格式和混排格式。"""
+def parse_word_content_lines(raw_lines: List[str]) -> List[str]:
+    """智能兼容 Word 内容中的分组格式和混排格式。"""
+    def normalize_line(line: str) -> str:
+        return line.replace('\ufeff', '').strip()
+
     def is_url_line(line: str) -> bool:
-        return bool(re.match(r'^https?://', line.strip()))
+        return bool(re.match(r'^https?://', normalize_line(line)))
 
     def is_index_line(line: str) -> bool:
-        return bool(re.match(r'^\d+[\.、]?$', line.strip()))
+        return bool(re.match(r'^\d+[\.、]?$', normalize_line(line)))
 
     def is_twole_label(line: str) -> bool:
-        return bool(re.match(r'^(二一教育|21世纪教育|21世纪教育网|二一世纪教育)[:：]?$', line.strip()))
+        return bool(re.match(r'^(二一教育|21世纪教育|21世纪教育网|二一世纪教育)[:：]?$', normalize_line(line)))
 
     def is_zxxk_label(line: str) -> bool:
-        return bool(re.match(r'^学科网[:：]?$', line.strip()))
+        return bool(re.match(r'^学科网[:：]?$', normalize_line(line)))
 
     def is_date_time(line: str) -> bool:
-        pattern = r'^\d{4}[-/]\d{1,2}[-/]\d{1,2}(\s+\d{1,2}:\d{1,2}(:\d{1,2})?)?$'
-        return bool(re.match(pattern, line.strip()))
+        pattern = r'^\d{4}[-/]\d{1,2}[-/]\d{1,2}(\s+\d{1,2}:\d{1,2}(:\d{1,2})?(\s*(AM|PM|上午|下午))?)?$'
+        return bool(re.match(pattern, normalize_line(line)))
 
-    def clean_hyperlink(line: str) -> str:
-        patterns = [
-            r'HYPERLINK\s+&quot;([^&]+)&quot;\s+(\S+)',
-            r'HYPERLINK\s+"([^"]+)"\s+(\S+)',
+    def is_skip_line(line: str) -> bool:
+        return is_index_line(line) or is_twole_label(line) or is_zxxk_label(line) or not normalize_line(line)
+
+    def is_url_fragment(line: str) -> bool:
+        return bool(re.match(r"^[A-Za-z0-9._~:/?#\[\]@!$&'()*+,;=%-]+$", normalize_line(line)))
+
+    def consume_url(lines: List[str], start: int) -> Optional[Tuple[str, int]]:
+        i = start
+        while i < len(lines) and is_skip_line(lines[i]):
+            i += 1
+
+        if i >= len(lines):
+            return None
+
+        first_part = normalize_line(lines[i])
+        if not first_part.startswith('http'):
+            return None
+
+        url_parts = [first_part]
+        i += 1
+        while i < len(lines):
+            line = normalize_line(lines[i])
+            if is_date_time(line):
+                break
+            if is_skip_line(line):
+                i += 1
+                continue
+            if not is_url_fragment(line):
+                break
+            url_parts.append(line)
+            i += 1
+
+        return ''.join(url_parts), i
+
+    def parse_content_item(lines: List[str], start: int) -> Optional[Tuple[str, str, str, int]]:
+        i = start
+        while i < len(lines) and is_skip_line(lines[i]):
+            i += 1
+
+        title_parts = []
+        while i < len(lines) and len(title_parts) < 8:
+            line = normalize_line(lines[i])
+            if is_url_line(line):
+                break
+            if is_date_time(line):
+                return None
+            if is_skip_line(line):
+                i += 1
+                continue
+            title_parts.append(line)
+            i += 1
+
+        while i < len(lines) and is_skip_line(lines[i]):
+            i += 1
+
+        url_result = consume_url(lines, i)
+        if not title_parts or not url_result:
+            return None
+
+        title = ''.join(title_parts)
+        url, i = url_result
+        date = '1970-1-1'
+        if i < len(lines) and is_date_time(lines[i]):
+            date = normalize_line(lines[i])
+            i += 1
+
+        return title, url, date, i
+
+    def get_site_from_url(url: str) -> Optional[str]:
+        normalized_url = normalize_line(url).lower()
+        if '21cnjy.com' in normalized_url:
+            return 'twole'
+        if 'zxxk.com' in normalized_url:
+            return 'zxxk'
+        return None
+
+    def extract_content_items(lines: List[str]) -> List[Tuple[str, str, str, str]]:
+        items = []
+        i = 0
+        while i < len(lines):
+            item = parse_content_item(lines, i)
+            if not item:
+                i += 1
+                continue
+
+            title, url, date, next_i = item
+            site = get_site_from_url(url)
+            if site in ('twole', 'zxxk'):
+                items.append((title, url, date, site))
+            i = next_i
+        return items
+
+    def build_grouped_rows(items: List[Tuple[str, str, str, str]]) -> List[str]:
+        twole_items = [(title, url, date) for title, url, date, site in items if site == 'twole']
+        zxxk_items = [(title, url, date) for title, url, date, site in items if site == 'zxxk']
+        count = min(len(twole_items), len(zxxk_items))
+        return [
+            '\t'.join([
+                twole_items[index][0],
+                twole_items[index][1],
+                twole_items[index][2],
+                zxxk_items[index][0],
+                zxxk_items[index][1],
+                zxxk_items[index][2],
+            ])
+            for index in range(count)
         ]
-        for pattern in patterns:
-            match = re.match(pattern, line)
-            if match:
-                return match.group(2)
-        return line
+
+    def build_mixed_rows(items: List[Tuple[str, str, str, str]]) -> List[str]:
+        result = []
+        i = 0
+        while i + 1 < len(items):
+            title1, url1, time1, site1 = items[i]
+            title2, url2, time2, site2 = items[i + 1]
+            if site1 == site2:
+                i += 1
+                continue
+
+            if site1 == 'twole':
+                twole_item = (title1, url1, time1)
+                zxxk_item = (title2, url2, time2)
+            else:
+                twole_item = (title2, url2, time2)
+                zxxk_item = (title1, url1, time1)
+
+            result.append('\t'.join([
+                twole_item[0],
+                twole_item[1],
+                twole_item[2],
+                zxxk_item[0],
+                zxxk_item[1],
+                zxxk_item[2],
+            ]))
+            i += 2
+        return result
+
+    def parse_site_items(lines: List[str]) -> List[str]:
+        items = extract_content_items(lines)
+        sites = [site for _, _, _, site in items]
+        if len(sites) < 2 or len(set(sites)) < 2:
+            return []
+
+        switches = sum(1 for current, next_site in zip(sites, sites[1:]) if current != next_site)
+        if switches <= 1:
+            return build_grouped_rows(items)
+        return build_mixed_rows(items)
 
     def parse_grouped_site_rows(lines: List[str]) -> List[str]:
         twole_items = []
@@ -237,12 +420,12 @@ def read_docx_content(doc_path: Path) -> List[str]:
         i = 0
 
         while i < len(lines):
-            line = lines[i].strip()
-            if line in ('21世纪教育', '21世纪教育网', '二一教育', '二一世纪教育'):
+            line = normalize_line(lines[i])
+            if is_twole_label(line):
                 current_site = 'twole'
                 i += 1
                 continue
-            if line == '学科网':
+            if is_zxxk_label(line):
                 current_site = 'zxxk'
                 i += 1
                 continue
@@ -257,11 +440,11 @@ def read_docx_content(doc_path: Path) -> List[str]:
             title = line
             url = ''
             date = '1970-1-1'
-            if i + 1 < len(lines) and is_url_line(lines[i + 1]):
-                url = lines[i + 1].strip()
-                i += 2
+            url_result = consume_url(lines, i + 1)
+            if url_result:
+                url, i = url_result
                 if i < len(lines) and is_date_time(lines[i]):
-                    date = lines[i].strip()
+                    date = normalize_line(lines[i])
                     i += 1
             else:
                 i += 1
@@ -289,48 +472,22 @@ def read_docx_content(doc_path: Path) -> List[str]:
         i = 0
 
         while i < len(lines):
-            if is_index_line(lines[i]):
-                i += 1
-                continue
-
-            if not is_twole_label(lines[i]):
-                i += 1
-                continue
-
-            start = i
-            if i + 2 >= len(lines):
+            if i + 3 >= len(lines):
                 break
 
-            title1 = lines[i + 1].strip()
-            url1 = lines[i + 2].strip()
-            if not is_url_line(url1):
+            start = i
+            item1 = parse_content_item(lines, i)
+            if not item1:
+                i += 1
+                continue
+
+            title1, url1, time1, next_i = item1
+            item2 = parse_content_item(lines, next_i)
+            if not item2:
                 i = start + 1
                 continue
 
-            i += 3
-            time1 = '1970-1-1'
-            if i < len(lines) and is_date_time(lines[i]):
-                time1 = lines[i].strip()
-                i += 1
-
-            while i < len(lines) and (is_index_line(lines[i]) or not lines[i].strip()):
-                i += 1
-
-            if i + 2 >= len(lines) or not is_zxxk_label(lines[i]):
-                i = start + 1
-                continue
-
-            title2 = lines[i + 1].strip()
-            url2 = lines[i + 2].strip()
-            if not is_url_line(url2):
-                i = start + 1
-                continue
-
-            i += 3
-            time2 = '1970-1-1'
-            if i < len(lines) and is_date_time(lines[i]):
-                time2 = lines[i].strip()
-                i += 1
+            title2, url2, time2, i = item2
 
             result.append('\t'.join([title1, url1, time1, title2, url2, time2]))
 
@@ -351,6 +508,51 @@ def read_docx_content(doc_path: Path) -> List[str]:
                 i += 1
         return result
 
+    lines = [normalize_line(line) for line in raw_lines if normalize_line(line)]
+    has_tab_in_first_rows = any('\t' in line for line in lines[:6])
+
+    split_lines = []
+    for line in lines:
+        split_lines.extend(normalize_line(part) for part in line.split('\t') if normalize_line(part))
+
+    site_result = parse_site_items(split_lines)
+    if site_result:
+        return site_result
+
+    has_site_labels = any(is_twole_label(line) or is_zxxk_label(line) for line in split_lines)
+
+    if has_site_labels:
+        grouped_result = parse_grouped_site_rows(split_lines)
+        if grouped_result:
+            return grouped_result
+
+    labeled_mixed_result = parse_labeled_mixed_rows(split_lines)
+    if labeled_mixed_result:
+        return labeled_mixed_result
+
+    if has_tab_in_first_rows or len(split_lines) < 6:
+        return lines
+
+    mixed_result = parse_mixed_rows(split_lines)
+    if mixed_result:
+        return mixed_result
+
+    return lines
+
+
+def read_docx_content(doc_path: Path) -> List[str]:
+    """读取 .docx 格式 Word 文档内容，并智能兼容分组格式和混排格式。"""
+    def clean_hyperlink(line: str) -> str:
+        patterns = [
+            r'HYPERLINK\s+&quot;([^&]+)&quot;\s+(\S+)',
+            r'HYPERLINK\s+"([^"]+)"\s+(\S+)',
+        ]
+        for pattern in patterns:
+            match = re.match(pattern, line)
+            if match:
+                return match.group(2)
+        return line
+
     try:
         with zipfile.ZipFile(doc_path, 'r') as z:
             content = z.read('word/document.xml').decode('utf-8')
@@ -362,30 +564,7 @@ def read_docx_content(doc_path: Path) -> List[str]:
                 if text:
                     processed.append(clean_hyperlink(text))
 
-            lines = [line.strip() for line in processed if line.strip()]
-            has_tab_in_first_rows = any('\t' in line for line in lines[:6])
-
-            split_lines = []
-            for line in lines:
-                split_lines.extend(part.strip() for part in line.split('\t') if part.strip())
-
-            labeled_mixed_result = parse_labeled_mixed_rows(split_lines)
-            if labeled_mixed_result:
-                return labeled_mixed_result
-
-            if has_tab_in_first_rows or len(split_lines) < 6:
-                return lines
-
-            if any(line in ('21世纪教育', '21世纪教育网', '二一教育', '二一世纪教育', '学科网') for line in split_lines):
-                grouped_result = parse_grouped_site_rows(split_lines)
-                if grouped_result:
-                    return grouped_result
-
-            mixed_result = parse_mixed_rows(split_lines)
-            if mixed_result:
-                return mixed_result
-
-            return lines
+            return parse_word_content_lines(processed)
 
     except Exception as e:
         print(f"读取 .docx 失败 ({doc_path.name}): {e}")
@@ -443,7 +622,9 @@ def read_doc_content(doc_path: Path) -> List[str]:
                 with open(txt_file, 'r', encoding='utf-8', errors='ignore') as f:
                     lines = [line.strip() for line in f.readlines() if line.strip()]
                 print(f"✓ 成功读取 {len(lines)} 行内容")
-                return lines
+                parsed_lines = parse_word_content_lines(lines)
+                print(f"✓ 智能解析后得到 {len(parsed_lines)} 行内容")
+                return parsed_lines
             else:
                 print(f"✗ 转换后的文本文件不存在")
     except Exception as e:
@@ -560,6 +741,14 @@ def build_file_associations(
     zxxk_folder: Path,
 ) -> Dict:
     """构建文件关联关系"""
+    def get_url_site(url: str) -> Optional[str]:
+        normalized_url = url.lower()
+        if '21cnjy.com' in normalized_url:
+            return 'twole'
+        if 'zxxk.com' in normalized_url:
+            return 'zxxk'
+        return None
+
     result = []
     problematic_groups = []
 
@@ -697,6 +886,8 @@ def build_file_associations(
                     group2_latest = get_latest_extracted_mtime(group2_files, file_path.name)
         elif 'www.zxxk.com' in url2 and zxxk_files:
             candidates = zxxk_files
+            if title2 == "河北省秦皇岛市青龙满族自治县青龙实验中学、青龙满族自治县第二中学2022-2023学年高三下学期4月月考英语试题":
+                print("fdafs")
             best_match = find_best_match(title2, candidates)
             if best_match:
                 zxxk_files.remove(best_match)
@@ -714,7 +905,14 @@ def build_file_associations(
         # 如果 group1 和 group2 都是压缩文件，并且 group1 的解压文件最新时间早于 group2（时间差超过阈值），则记录到问题列表
         # 或者如果解压出来的文件中包含音视频文件，也记录到问题列表
         TIME_DIFF_THRESHOLD = 30  # 时间差阈值（秒），避免将 zip 文件和解压文件时间相近的误判为问题组
+        pair_sites = {
+            get_url_site(entry.get("group1", {}).get("url", "")),
+            get_url_site(entry.get("group2", {}).get("url", "")),
+        }
+        is_valid_cross_site_pair = pair_sites == {'twole', 'zxxk'}
         time_diff_condition = (
+            is_valid_cross_site_pair
+            and
             group1_latest is not None
             and group2_latest is not None
             and group1_latest < group2_latest