fix: preserve data granularity and recover tasks safely

This commit is contained in:
Codex
2026-10-11 14:18:56 +08:00
parent 23f3a15a15
commit 603184549b
29 changed files with 4557 additions and 418 deletions
+27
View File
@@ -4,11 +4,27 @@ import json
import re
from datetime import datetime, timezone, timedelta
from typing import Any
from pathlib import Path
from functools import lru_cache
from urllib.parse import urlencode
PROVINCES = {"11":"北京","12":"天津","13":"河北","14":"山西","15":"内蒙古","21":"辽宁","22":"吉林","23":"黑龙江","31":"上海","32":"江苏","33":"浙江","34":"安徽","35":"福建","36":"江西","37":"山东","41":"河南","42":"湖北","43":"湖南","44":"广东","45":"广西","46":"海南","50":"重庆","51":"四川","52":"贵州","53":"云南","54":"西藏","61":"陕西","62":"甘肃","63":"青海","64":"宁夏","65":"新疆"}
@lru_cache(maxsize=1)
def school_metadata():
path = Path(__file__).resolve().parent.parent / 'data/school_levels.json'
data = json.loads(path.read_text(encoding='utf-8')) if path.exists() else {}
return {k: {normalized_name(n) for n in data.get(k, [])} for k in ('985','211','double_first_class_2022_known')}
def school_level(name, existing=''):
metadata = school_metadata()
name = normalized_name(name)
levels = [label for key, label in [('985','985'),('211','211'),('double_first_class_2022_known','双一流')] if name in metadata[key] or label in str(existing).split('/')]
return '/'.join(levels) or existing
def now() -> str:
return datetime.now(timezone(timedelta(hours=8))).isoformat(timespec="seconds")
@@ -17,6 +33,16 @@ def normalized_name(value: str) -> str:
return str(value or "").replace("(", "(").replace(")", ")").replace(" ", "").strip()
def is_special_score(note):
text = str(note or '')
tokens = ('专项', '少干', '少骨', '少数民族骨干', '退役大学生士兵', '退役士兵', '强军', '援藏', '单独考试')
for token in tokens:
cleaned = re.sub(r'(?:不含|不包括|除外)\s*' + re.escape(token), '', text)
if token in cleaned:
return True
return False
def integer(value: Any) -> int | None:
if value is None or isinstance(value, bool):
return None
@@ -47,6 +73,7 @@ def record_key(record: dict) -> str:
def finish_record(record: dict) -> dict:
defaults = {"school_code":"", "school_name":"", "province":"", "major_code":"", "major_name":"", "department_code":"", "department_name":"", "direction_code":"", "direction":"", "study_mode":"", "degree_type":"", "exam_type":"", "advisor":"", "quota_text":"", "quota":None, "quota_scope":"unknown", "subjects":[], "special_plans":[], "level":"", "remarks":"", "source":"", "source_url":"", "fetched_at":now(), "year_verified":False}
row = {**defaults, **record}
row['level'] = school_level(row['school_name'], row['level'])
if not row["school_code"]:
row["school_code"] = "name:" + normalized_name(row["school_name"])
row["record_key"] = record_key(row)