feat: implement unified data sources, proxy routing and local API
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
"""招生目录与分数线规范化;来源口径和未知值始终保留。"""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
PROVINCES = {"11":"北京","12":"天津","13":"河北","14":"山西","15":"内蒙古","21":"辽宁","22":"吉林","23":"黑龙江","31":"上海","32":"江苏","33":"浙江","34":"安徽","35":"福建","36":"江西","37":"山东","41":"河南","42":"湖北","43":"湖南","44":"广东","45":"广西","46":"海南","50":"重庆","51":"四川","52":"贵州","53":"云南","54":"西藏","61":"陕西","62":"甘肃","63":"青海","64":"宁夏","65":"新疆"}
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return datetime.now(timezone(timedelta(hours=8))).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def normalized_name(value: str) -> str:
|
||||
return str(value or "").replace("(", "(").replace(")", ")").replace(" ", "").strip()
|
||||
|
||||
|
||||
def integer(value: Any) -> int | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return int(text) if re.fullmatch(r"\d+", text) else None
|
||||
|
||||
|
||||
def parse_quota(text: str) -> tuple[int | None, str]:
|
||||
text = str(text or "").strip()
|
||||
match = re.search(r"专业\s*[::]\s*(\d+)", text)
|
||||
if match and not re.match(r"\s*[-—~至或]", text[match.end():]):
|
||||
return int(match.group(1)), "major"
|
||||
if "学院" in text or "院系" in text:
|
||||
return None, "college"
|
||||
if "方向" in text:
|
||||
return None, "direction"
|
||||
if re.fullmatch(r"\d+", text):
|
||||
return int(text), "unspecified"
|
||||
return None, "unknown"
|
||||
|
||||
|
||||
def record_key(record: dict) -> str:
|
||||
fields = ["year","school_code","school_name","major_code","department_code","department_name","direction_code","direction","study_mode","exam_type","subjects","special_plans","source"]
|
||||
serialized = json.dumps([record.get(key, "") for key in fields], ensure_ascii=False, sort_keys=True)
|
||||
return hashlib.sha256(serialized.encode()).hexdigest()
|
||||
|
||||
|
||||
def finish_record(record: dict) -> dict:
|
||||
defaults = {"school_code":"", "school_name":"", "province":"", "major_code":"", "major_name":"", "department_code":"", "department_name":"", "direction_code":"", "direction":"", "study_mode":"", "degree_type":"", "exam_type":"", "advisor":"", "quota_text":"", "quota":None, "quota_scope":"unknown", "subjects":[], "special_plans":[], "level":"", "remarks":"", "source":"", "source_url":"", "fetched_at":now(), "year_verified":False}
|
||||
row = {**defaults, **record}
|
||||
if not row["school_code"]:
|
||||
row["school_code"] = "name:" + normalized_name(row["school_name"])
|
||||
row["record_key"] = record_key(row)
|
||||
return row
|
||||
|
||||
|
||||
def normalize_catalog(item: dict, year: int, school: dict | None = None, major: dict | None = None) -> list[dict]:
|
||||
item = {**(school or {}), **(major or {}), **item}
|
||||
quota, scope = parse_quota(item.get("nzsrsstr", ""))
|
||||
if not item.get("nzsrsstr") and integer(item.get("nzsrs")) not in (None, 0):
|
||||
quota, scope = integer(item["nzsrs"]), "unspecified"
|
||||
labels = []
|
||||
if str(item.get("syl", "")) == "1":
|
||||
labels.append("双一流")
|
||||
base = {
|
||||
"year": year, "school_code": str(item.get("dwdm", "")), "school_name": item.get("dwmc", ""),
|
||||
"province": item.get("szss") or PROVINCES.get(str(item.get("ssdm", item.get("szssm", ""))), ""),
|
||||
"major_code": str(item.get("zydm", "")), "major_name": item.get("zymc", ""),
|
||||
"department_code": str(item.get("yxsdm", "")), "department_name": item.get("yxsmc", ""),
|
||||
"direction_code": str(item.get("yjfxdm", "")), "direction": item.get("yjfxmc", ""),
|
||||
"study_mode": {"1":"全日制", "2":"非全日制"}.get(str(item.get("xxfs", "")), ""),
|
||||
"degree_type": item.get("xwlxmc", ""), "exam_type": item.get("ksfsmc", ""),
|
||||
"advisor": item.get("zdjs", ""), "quota_text": item.get("nzsrsstr", ""), "quota": quota, "quota_scope": scope,
|
||||
"special_plans": [label for key,label in [("tydxs","退役大学生士兵"),("jsggjh","少数民族骨干")] if str(item.get(key, "")) == "1"],
|
||||
"level": "/".join(labels), "remarks": item.get("bz", item.get("zybz", "")),
|
||||
"source":"研招网", "source_url":"https://yz.chsi.com.cn/zsml/zydetail.do?" + urlencode({"zydm": item.get("zydm", ""), "zymc": item.get("zymc", "")}),
|
||||
}
|
||||
combinations = item.get("kskmz") or [{}]
|
||||
result = []
|
||||
for combination in combinations:
|
||||
subjects = []
|
||||
for number in range(1, 5):
|
||||
subject = combination.get(f"km{number}Vo") or {}
|
||||
name = str(subject.get("kskmmc", "") or "")
|
||||
code = str(subject.get("kskmdm", "") or "")
|
||||
subjects.append((code if code and not name.startswith(code) else "") + name)
|
||||
result.append(finish_record({**base, "subjects":subjects}))
|
||||
return result
|
||||
|
||||
|
||||
def select_score_lines(items: list[dict], major_code: str, school_code: str, school_name: str, year: int) -> list[dict]:
|
||||
levels = {6:[], 4:[], 2:[]}
|
||||
for item in items:
|
||||
code = str(item.get("code", "")).strip()
|
||||
if len(code) in levels and (code == major_code or len(code) < 6 and major_code.startswith(code)):
|
||||
levels[len(code)].append(item)
|
||||
result = []
|
||||
for size in (6, 4, 2):
|
||||
if not levels[size]:
|
||||
continue
|
||||
for item in levels[size]:
|
||||
result.append({
|
||||
"school_code":school_code, "school_name":school_name, "major_code":major_code, "year":year,
|
||||
"department_id":str(item.get("depart_id", "") or ""), "department_name":item.get("depart_name", "") or "",
|
||||
"scope":{6:"major",4:"discipline",2:"category"}[size], "raw_code":str(item.get("code", "")),
|
||||
"total":integer(item.get("total")), "politics":integer(item.get("politics")), "english":integer(item.get("english")),
|
||||
"special_one":integer(item.get("special_one")), "special_two":integer(item.get("special_two")),
|
||||
"note":item.get("note", "") or "", "source":"掌上考研", "source_url":"https://www.kaoyan.cn/", "fetched_at":now(), "raw_evidence":item,
|
||||
})
|
||||
break
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user