match_unmatched.py是针对未匹配xkw_code的进一步宽松匹配处理，如果还匹配不到输出至未匹配填充0_仍未匹配.json

zq/问题数据/match_unmatched.py

@@ -0,0 +1,166 @@
+"""
+读取未匹配填充0.json，对每个json对象：
+1. 取filename去掉后缀名，jieba分词后与结果.xlsx中"学科网资料名"进行分词包含匹配
+2. 从url中提取数字ID，写入url_id字段
+3. 在匹配到的行中，用url_id与"暗码"精准匹配：匹配则xkw_code=暗码值，否则xkw_code=0
+4. xkw_code与url_id一致则"一致性"=1，否则=0
+"""
+
+import json
+import re
+import os
+import jieba
+
+import openpyxl
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+INPUT_JSON = os.path.join(SCRIPT_DIR, "未匹配填充0.json")
+EXCEL_PATH = os.path.join(SCRIPT_DIR, "..", "结果.xlsx")
+STILL_UNMATCHED_JSON = os.path.join(SCRIPT_DIR, "未匹配填充0_仍未匹配.json")
+
+# 过滤掉太短或无意义的分词token
+MIN_TOKEN_LEN = 2
+
+
+def strip_ext(name: str) -> str:
+    """去掉文件后缀名"""
+    base, _ = os.path.splitext(name)
+    return base
+
+
+def is_meaningful_token(t: str) -> bool:
+    """token中必须包含至少一个汉字或字母数字字符，否则视为纯标点无意义"""
+    return any(c.isalnum() or '\u4e00' <= c <= '\u9fff' for c in t)
+
+
+def tokenize(text: str) -> list[str]:
+    """使用jieba分词，过滤短词和纯标点/符号"""
+    result = []
+    for t in jieba.cut(text, cut_all=False):
+        t = t.strip()
+        if len(t) >= MIN_TOKEN_LEN and is_meaningful_token(t):
+            result.append(t)
+    return result
+
+
+def extract_url_id(url: str) -> str:
+    """从url中提取数字ID，例如 https://www.zxxk.com/soft/33839713.html -> 33839713"""
+    m = re.search(r'/(\d+)(?:\.\w+)?$', url)
+    if m:
+        return m.group(1)
+    # fallback: 取url中最后一段数字
+    nums = re.findall(r'\d+', url)
+    return nums[-1] if nums else ""
+
+
+def build_excel_index(excel_path: str):
+    """
+    读取Excel，返回：
+    - rows: list of dict，每行包含 学科网资料名(无后缀)、暗码
+    - anma_index: dict[暗码str -> 学科网资料名str]，用于url_id精准查找
+    """
+    wb = openpyxl.load_workbook(excel_path, read_only=True, data_only=True)
+    ws = wb.active
+    headers = [cell.value for cell in next(ws.iter_rows(min_row=1, max_row=1))]
+
+    idx_xkw_name = headers.index("学科网资料名")
+    idx_anma = headers.index("暗码")
+
+    rows = []
+    anma_index = {}  # 暗码 -> 学科网资料名（无后缀），学科网资料名为空时存 ""
+    for row in ws.iter_rows(min_row=2, values_only=True):
+        xkw_name = row[idx_xkw_name]
+        anma = row[idx_anma]
+        anma_str = str(anma) if anma is not None else ""
+
+        # anma_index：只要暗码有值就记录，学科网资料名可以为空
+        if anma_str and anma_str not in anma_index:
+            xkw_name_noext = strip_ext(str(xkw_name)) if xkw_name else ""
+            anma_index[anma_str] = xkw_name_noext
+
+        # rows（用于分词匹配）：学科网资料名为空则跳过
+        if not xkw_name:
+            continue
+        xkw_name_noext = strip_ext(str(xkw_name))
+        rows.append({
+            "xkw_name": xkw_name_noext,
+            "anma": anma_str,
+        })
+    wb.close()
+    return rows, anma_index
+
+
+def tokens_match(tokens: list[str], target: str) -> bool:
+    """判断target是否包含所有token"""
+    return all(t in target for t in tokens)
+
+
+def process(input_json: str, excel_path: str, still_unmatched_json: str):
+    print("加载Excel...")
+    excel_rows, anma_index = build_excel_index(excel_path)
+    print(f"Excel共 {len(excel_rows)} 行数据")
+
+    print("加载JSON...")
+    with open(input_json, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    print(f"JSON共 {len(data)} 条记录")
+
+    for i, obj in enumerate(data):
+        filename = obj.get("filename", "")
+        url = obj.get("url", "")
+
+        # 1. 去掉后缀名并分词
+        name_noext = strip_ext(filename)
+        tokens = tokenize(name_noext)
+
+        # 2. 提取url_id
+        url_id = extract_url_id(url)
+        obj["url_id"] = url_id
+
+        # 3. 分词包含匹配，找出所有匹配行
+        matched_rows = [r for r in excel_rows if tokens_match(tokens, r["xkw_name"])]
+
+        # 4. 在匹配行中用url_id与暗码精准匹配
+        xkw_code = "0"
+        if matched_rows:
+            exact = [r for r in matched_rows if r["anma"] == url_id]
+            if exact:
+                xkw_code = exact[0]["anma"]
+
+        obj["xkw_code"] = xkw_code
+        obj.pop("一致性", None)  # 清除旧字段
+
+        # 5. xkw_code不为0但与url_id不一致
+        obj["xwk_code不为0但xkw_code与url_id不一致"] = 1 if xkw_code != "0" and xkw_code != url_id else 0
+
+        # 6. 用url_id去暗码字段精准查找
+        xkw_name_by_id = anma_index.get(url_id)
+        obj["xlsx中的暗码"] = url_id if xkw_name_by_id is not None else "0"
+        obj["学科网资料名"] = xkw_name_by_id if xkw_name_by_id is not None else ""
+
+        # 7. xlsx中的暗码有值但学科网资料名为空
+        xlsx_anma = obj["xlsx中的暗码"]
+        xkw_name_val = obj["学科网资料名"]
+        obj["xlsx中的暗码有值学科网资料名为空"] = 1 if xlsx_anma != "0" and not xkw_name_val else 0
+
+        if (i + 1) % 100 == 0:
+            print(f"  已处理 {i+1}/{len(data)}")
+
+    # 直接写回原文件
+    print(f"写回原文件 {input_json}")
+    with open(input_json, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2)
+
+    # xkw_code为0的输出到新文件
+    still_unmatched = [obj for obj in data if obj["xkw_code"] == "0"]
+    print(f"写出仍未匹配记录到 {still_unmatched_json}")
+    with open(still_unmatched_json, "w", encoding="utf-8") as f:
+        json.dump(still_unmatched, f, ensure_ascii=False, indent=2)
+
+    matched_count = sum(1 for obj in data if obj["xkw_code"] != "0")
+    inconsistent_count = sum(1 for obj in data if obj["xwk_code不为0但xkw_code与url_id不一致"] == 1)
+    print(f"完成：共{len(data)}条，xkw_code非0共{matched_count}条，其中与url_id不一致{inconsistent_count}条，仍未匹配{len(still_unmatched)}条")
+
+
+if __name__ == "__main__":
+    process(INPUT_JSON, EXCEL_PATH, STILL_UNMATCHED_JSON)