167 lines
6.0 KiB
Python
167 lines
6.0 KiB
Python
"""
|
||
读取未匹配填充0.json，对每个json对象：
|
||
1. 取filename去掉后缀名，jieba分词后与结果.xlsx中"学科网资料名"进行分词包含匹配
|
||
2. 从url中提取数字ID，写入url_id字段
|
||
3. 在匹配到的行中，用url_id与"暗码"精准匹配：匹配则xkw_code=暗码值，否则xkw_code=0
|
||
4. xkw_code与url_id一致则"一致性"=1，否则=0
|
||
"""
|
||
|
||
import json
|
||
import re
|
||
import os
|
||
import jieba
|
||
|
||
import openpyxl
|
||
|
||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
INPUT_JSON = os.path.join(SCRIPT_DIR, "未匹配填充0.json")
|
||
EXCEL_PATH = os.path.join(SCRIPT_DIR, "..", "结果.xlsx")
|
||
STILL_UNMATCHED_JSON = os.path.join(SCRIPT_DIR, "未匹配填充0_仍未匹配.json")
|
||
|
||
# 过滤掉太短或无意义的分词token
|
||
MIN_TOKEN_LEN = 2
|
||
|
||
|
||
def strip_ext(name: str) -> str:
|
||
"""去掉文件后缀名"""
|
||
base, _ = os.path.splitext(name)
|
||
return base
|
||
|
||
|
||
def is_meaningful_token(t: str) -> bool:
|
||
"""token中必须包含至少一个汉字或字母数字字符，否则视为纯标点无意义"""
|
||
return any(c.isalnum() or '\u4e00' <= c <= '\u9fff' for c in t)
|
||
|
||
|
||
def tokenize(text: str) -> list[str]:
|
||
"""使用jieba分词，过滤短词和纯标点/符号"""
|
||
result = []
|
||
for t in jieba.cut(text, cut_all=False):
|
||
t = t.strip()
|
||
if len(t) >= MIN_TOKEN_LEN and is_meaningful_token(t):
|
||
result.append(t)
|
||
return result
|
||
|
||
|
||
def extract_url_id(url: str) -> str:
|
||
"""从url中提取数字ID，例如 https://www.zxxk.com/soft/33839713.html -> 33839713"""
|
||
m = re.search(r'/(\d+)(?:\.\w+)?$', url)
|
||
if m:
|
||
return m.group(1)
|
||
# fallback: 取url中最后一段数字
|
||
nums = re.findall(r'\d+', url)
|
||
return nums[-1] if nums else ""
|
||
|
||
|
||
def build_excel_index(excel_path: str):
|
||
"""
|
||
读取Excel，返回：
|
||
- rows: list of dict，每行包含 学科网资料名(无后缀)、暗码
|
||
- anma_index: dict[暗码str -> 学科网资料名str]，用于url_id精准查找
|
||
"""
|
||
wb = openpyxl.load_workbook(excel_path, read_only=True, data_only=True)
|
||
ws = wb.active
|
||
headers = [cell.value for cell in next(ws.iter_rows(min_row=1, max_row=1))]
|
||
|
||
idx_xkw_name = headers.index("学科网资料名")
|
||
idx_anma = headers.index("暗码")
|
||
|
||
rows = []
|
||
anma_index = {} # 暗码 -> 学科网资料名（无后缀），学科网资料名为空时存 ""
|
||
for row in ws.iter_rows(min_row=2, values_only=True):
|
||
xkw_name = row[idx_xkw_name]
|
||
anma = row[idx_anma]
|
||
anma_str = str(anma) if anma is not None else ""
|
||
|
||
# anma_index：只要暗码有值就记录，学科网资料名可以为空
|
||
if anma_str and anma_str not in anma_index:
|
||
xkw_name_noext = strip_ext(str(xkw_name)) if xkw_name else ""
|
||
anma_index[anma_str] = xkw_name_noext
|
||
|
||
# rows（用于分词匹配）：学科网资料名为空则跳过
|
||
if not xkw_name:
|
||
continue
|
||
xkw_name_noext = strip_ext(str(xkw_name))
|
||
rows.append({
|
||
"xkw_name": xkw_name_noext,
|
||
"anma": anma_str,
|
||
})
|
||
wb.close()
|
||
return rows, anma_index
|
||
|
||
|
||
def tokens_match(tokens: list[str], target: str) -> bool:
|
||
"""判断target是否包含所有token"""
|
||
return all(t in target for t in tokens)
|
||
|
||
|
||
def process(input_json: str, excel_path: str, still_unmatched_json: str):
|
||
print("加载Excel...")
|
||
excel_rows, anma_index = build_excel_index(excel_path)
|
||
print(f"Excel共 {len(excel_rows)} 行数据")
|
||
|
||
print("加载JSON...")
|
||
with open(input_json, "r", encoding="utf-8") as f:
|
||
data = json.load(f)
|
||
print(f"JSON共 {len(data)} 条记录")
|
||
|
||
for i, obj in enumerate(data):
|
||
filename = obj.get("filename", "")
|
||
url = obj.get("url", "")
|
||
|
||
# 1. 去掉后缀名并分词
|
||
name_noext = strip_ext(filename)
|
||
tokens = tokenize(name_noext)
|
||
|
||
# 2. 提取url_id
|
||
url_id = extract_url_id(url)
|
||
obj["url_id"] = url_id
|
||
|
||
# 3. 分词包含匹配，找出所有匹配行
|
||
matched_rows = [r for r in excel_rows if tokens_match(tokens, r["xkw_name"])]
|
||
|
||
# 4. 在匹配行中用url_id与暗码精准匹配
|
||
xkw_code = "0"
|
||
if matched_rows:
|
||
exact = [r for r in matched_rows if r["anma"] == url_id]
|
||
if exact:
|
||
xkw_code = exact[0]["anma"]
|
||
|
||
obj["xkw_code"] = xkw_code
|
||
obj.pop("一致性", None) # 清除旧字段
|
||
|
||
# 5. xkw_code不为0但与url_id不一致
|
||
obj["xwk_code不为0但xkw_code与url_id不一致"] = 1 if xkw_code != "0" and xkw_code != url_id else 0
|
||
|
||
# 6. 用url_id去暗码字段精准查找
|
||
xkw_name_by_id = anma_index.get(url_id)
|
||
obj["xlsx中的暗码"] = url_id if xkw_name_by_id is not None else "0"
|
||
obj["学科网资料名"] = xkw_name_by_id if xkw_name_by_id is not None else ""
|
||
|
||
# 7. xlsx中的暗码有值但学科网资料名为空
|
||
xlsx_anma = obj["xlsx中的暗码"]
|
||
xkw_name_val = obj["学科网资料名"]
|
||
obj["xlsx中的暗码有值学科网资料名为空"] = 1 if xlsx_anma != "0" and not xkw_name_val else 0
|
||
|
||
if (i + 1) % 100 == 0:
|
||
print(f" 已处理 {i+1}/{len(data)}")
|
||
|
||
# 直接写回原文件
|
||
print(f"写回原文件 {input_json}")
|
||
with open(input_json, "w", encoding="utf-8") as f:
|
||
json.dump(data, f, ensure_ascii=False, indent=2)
|
||
|
||
# xkw_code为0的输出到新文件
|
||
still_unmatched = [obj for obj in data if obj["xkw_code"] == "0"]
|
||
print(f"写出仍未匹配记录到 {still_unmatched_json}")
|
||
with open(still_unmatched_json, "w", encoding="utf-8") as f:
|
||
json.dump(still_unmatched, f, ensure_ascii=False, indent=2)
|
||
|
||
matched_count = sum(1 for obj in data if obj["xkw_code"] != "0")
|
||
inconsistent_count = sum(1 for obj in data if obj["xwk_code不为0但xkw_code与url_id不一致"] == 1)
|
||
print(f"完成：共{len(data)}条，xkw_code非0共{matched_count}条，其中与url_id不一致{inconsistent_count}条，仍未匹配{len(still_unmatched)}条")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
process(INPUT_JSON, EXCEL_PATH, STILL_UNMATCHED_JSON)
|