feat: implement unified data sources, proxy routing and local API

This commit is contained in:
Codex
2026-10-11 13:49:07 +08:00
parent 25cccd4372
commit 389b2aa3a5
20 changed files with 21601 additions and 4 deletions
+111
View File
@@ -0,0 +1,111 @@
"""招生目录与分数线规范化;来源口径和未知值始终保留。"""
import hashlib
import json
import re
from datetime import datetime, timezone, timedelta
from typing import Any
from urllib.parse import urlencode
PROVINCES = {"11":"北京","12":"天津","13":"河北","14":"山西","15":"内蒙古","21":"辽宁","22":"吉林","23":"黑龙江","31":"上海","32":"江苏","33":"浙江","34":"安徽","35":"福建","36":"江西","37":"山东","41":"河南","42":"湖北","43":"湖南","44":"广东","45":"广西","46":"海南","50":"重庆","51":"四川","52":"贵州","53":"云南","54":"西藏","61":"陕西","62":"甘肃","63":"青海","64":"宁夏","65":"新疆"}
def now() -> str:
return datetime.now(timezone(timedelta(hours=8))).isoformat(timespec="seconds")
def normalized_name(value: str) -> str:
return str(value or "").replace("(", "(").replace(")", ")").replace(" ", "").strip()
def integer(value: Any) -> int | None:
if value is None or isinstance(value, bool):
return None
text = str(value).strip()
return int(text) if re.fullmatch(r"\d+", text) else None
def parse_quota(text: str) -> tuple[int | None, str]:
text = str(text or "").strip()
match = re.search(r"专业\s*[::]\s*(\d+)", text)
if match and not re.match(r"\s*[-—~至或]", text[match.end():]):
return int(match.group(1)), "major"
if "学院" in text or "院系" in text:
return None, "college"
if "方向" in text:
return None, "direction"
if re.fullmatch(r"\d+", text):
return int(text), "unspecified"
return None, "unknown"
def record_key(record: dict) -> str:
fields = ["year","school_code","school_name","major_code","department_code","department_name","direction_code","direction","study_mode","exam_type","subjects","special_plans","source"]
serialized = json.dumps([record.get(key, "") for key in fields], ensure_ascii=False, sort_keys=True)
return hashlib.sha256(serialized.encode()).hexdigest()
def finish_record(record: dict) -> dict:
defaults = {"school_code":"", "school_name":"", "province":"", "major_code":"", "major_name":"", "department_code":"", "department_name":"", "direction_code":"", "direction":"", "study_mode":"", "degree_type":"", "exam_type":"", "advisor":"", "quota_text":"", "quota":None, "quota_scope":"unknown", "subjects":[], "special_plans":[], "level":"", "remarks":"", "source":"", "source_url":"", "fetched_at":now(), "year_verified":False}
row = {**defaults, **record}
if not row["school_code"]:
row["school_code"] = "name:" + normalized_name(row["school_name"])
row["record_key"] = record_key(row)
return row
def normalize_catalog(item: dict, year: int, school: dict | None = None, major: dict | None = None) -> list[dict]:
item = {**(school or {}), **(major or {}), **item}
quota, scope = parse_quota(item.get("nzsrsstr", ""))
if not item.get("nzsrsstr") and integer(item.get("nzsrs")) not in (None, 0):
quota, scope = integer(item["nzsrs"]), "unspecified"
labels = []
if str(item.get("syl", "")) == "1":
labels.append("双一流")
base = {
"year": year, "school_code": str(item.get("dwdm", "")), "school_name": item.get("dwmc", ""),
"province": item.get("szss") or PROVINCES.get(str(item.get("ssdm", item.get("szssm", ""))), ""),
"major_code": str(item.get("zydm", "")), "major_name": item.get("zymc", ""),
"department_code": str(item.get("yxsdm", "")), "department_name": item.get("yxsmc", ""),
"direction_code": str(item.get("yjfxdm", "")), "direction": item.get("yjfxmc", ""),
"study_mode": {"1":"全日制", "2":"非全日制"}.get(str(item.get("xxfs", "")), ""),
"degree_type": item.get("xwlxmc", ""), "exam_type": item.get("ksfsmc", ""),
"advisor": item.get("zdjs", ""), "quota_text": item.get("nzsrsstr", ""), "quota": quota, "quota_scope": scope,
"special_plans": [label for key,label in [("tydxs","退役大学生士兵"),("jsggjh","少数民族骨干")] if str(item.get(key, "")) == "1"],
"level": "/".join(labels), "remarks": item.get("bz", item.get("zybz", "")),
"source":"研招网", "source_url":"https://yz.chsi.com.cn/zsml/zydetail.do?" + urlencode({"zydm": item.get("zydm", ""), "zymc": item.get("zymc", "")}),
}
combinations = item.get("kskmz") or [{}]
result = []
for combination in combinations:
subjects = []
for number in range(1, 5):
subject = combination.get(f"km{number}Vo") or {}
name = str(subject.get("kskmmc", "") or "")
code = str(subject.get("kskmdm", "") or "")
subjects.append((code if code and not name.startswith(code) else "") + name)
result.append(finish_record({**base, "subjects":subjects}))
return result
def select_score_lines(items: list[dict], major_code: str, school_code: str, school_name: str, year: int) -> list[dict]:
levels = {6:[], 4:[], 2:[]}
for item in items:
code = str(item.get("code", "")).strip()
if len(code) in levels and (code == major_code or len(code) < 6 and major_code.startswith(code)):
levels[len(code)].append(item)
result = []
for size in (6, 4, 2):
if not levels[size]:
continue
for item in levels[size]:
result.append({
"school_code":school_code, "school_name":school_name, "major_code":major_code, "year":year,
"department_id":str(item.get("depart_id", "") or ""), "department_name":item.get("depart_name", "") or "",
"scope":{6:"major",4:"discipline",2:"category"}[size], "raw_code":str(item.get("code", "")),
"total":integer(item.get("total")), "politics":integer(item.get("politics")), "english":integer(item.get("english")),
"special_one":integer(item.get("special_one")), "special_two":integer(item.get("special_two")),
"note":item.get("note", "") or "", "source":"掌上考研", "source_url":"https://www.kaoyan.cn/", "fetched_at":now(), "raw_evidence":item,
})
break
return result