Files
kaoyan/studio/connectors.py
T

134 lines
7.5 KiB
Python

"""Independent adapters for the two projects' public data sources."""
import json
import re
from studio.domain import finish_record, integer, normalized_name, select_score_lines
from studio.network import RemoteUnavailable
class LoginRequired(RuntimeError):
pass
class SourceYearMismatch(RuntimeError):
pass
class CatalogClient:
BASE = "https://yz.chsi.com.cn/zsml/"
def __init__(self, transport):
self.transport = transport
def verify_year(self, year):
response = self.transport.request("GET", self.BASE)
text = re.sub("<[^>]*>", " ", response.text)
matches = re.findall(r"(20\d{2})\s*年.{0,30}(?:硕士|招生|专业目录)", text)
years = set(map(int, matches))
if not years:
raise SourceYearMismatch("未能核实研招网当前目录年份,已停止写入;请检查登录及目录开放状态")
current = max(years)
if int(year) != current:
raise SourceYearMismatch(f"研招网当前公开目录为 {current} 年,任务选择了 {year} 年;历史目录请导入已有快照")
return current
def pages(self, endpoint, payload, page_size=50):
seen = set()
for page in range(1, 10001):
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
try:
body = response.json()
except ValueError:
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
if body.get("flag") is not True:
message = str(body.get("msg", ""))
if any(token in message for token in ("登录", "登陆", "认证")):
raise LoginRequired("研招网要求登录,请更新 Cookie 后恢复任务")
if any(token in message for token in ("频繁", "稍后", "验证码")):
raise RemoteUnavailable("研招网访问受限,请增加请求间隔并完成人工验证后恢复")
raise RemoteUnavailable("研招网目录接口未成功返回数据")
data = body.get("msg") or {}
if not isinstance(data, dict) or not isinstance(data.get("list"), list):
raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据")
rows = data["list"]
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen:
raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页")
seen.add(signature)
yield from rows
more = data.get("nextPageAvailable")
total = integer(data.get("totalCount"))
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and page * page_size >= total) or len(rows) < page_size):
return
raise RemoteUnavailable("目录分页超过安全上限")
def schools(self, province, keyword=""):
return self.pages("dws", {"ssdm": province, "dwmc": keyword, "dwdm": "", "xxfs": "", "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"})
def majors(self, school, major_code="", study_mode=""):
return self.pages("dwzys", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major_code, "zymc": "", "mldm": "", "yjxkdm": "", "xwlx": "", "xxfs": study_mode})
def directions(self, school, major, study_mode=""):
return self.pages("yjfxs", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major["zydm"], "zymc": major.get("zymc", ""), "xxfs": study_mode, "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"})
ALIASES = {"湖州师范学院": "湖州师范大学", "闽江学院": "闽江大学", "皖南医学院": "皖南医科大学", "中国航空研究院(613所)": "中国航空研究院", "中国电子科技集团有限公司研究生院": "中国电子科技集团", "湖南理工学院": "湖南理工大学"}
class ScoreClient:
BASE = "https://api.kaoyan.cn/pc"
def __init__(self, transport):
self.transport = transport
self.cache = {}
def post(self, endpoint, payload):
response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"})
try:
body = response.json()
except ValueError:
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
if str(body.get("code")) != "0000":
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
return body.get("data") or {}
def pages(self, endpoint, payload, limit=100):
seen = set()
for page in range(1, 10001):
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data
if not isinstance(rows, list):
raise RemoteUnavailable("掌上考研响应结构已变化")
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen:
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
seen.add(signature)
yield from rows
total = integer(data.get("total")) if isinstance(data, dict) else None
last = integer(data.get("last_page")) if isinstance(data, dict) else None
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total:
return
raise RemoteUnavailable("历史数据分页超过安全上限")
def school_id(self, name):
key = normalized_name(name)
if key in self.cache:
return self.cache[key]
for query in dict.fromkeys([name, ALIASES.get(key, name)]):
matches = [row for row in self.pages("/school/schoolList", {"school_name": query}, limit=10) if normalized_name(row.get("school_name", "")) == normalized_name(query)]
if len(matches) == 1 and matches[0].get("school_id"):
self.cache[key] = matches[0]["school_id"]
return self.cache[key]
return None
def score_lines(self, school, major_code, year, school_id):
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
def plans(self, school, major_code, year, school_id):
result = []
for row in self.pages("/school/planListV2", {"school_id": school_id, "year": year, "keyword": major_code}):
if str(row.get("special_code", "")) != major_code:
continue
result.append(finish_record({**school, "year": year, "major_code": major_code, "major_name": row.get("special_name", ""), "department_code": str(row.get("depart_id", "") or ""), "department_name": row.get("depart_name", ""), "direction_code": str(row.get("spe_id", "") or row.get("plan_id", "")), "direction": row.get("research_direction", ""), "quota": integer(row.get("recruit_number")), "quota_text": str(row.get("recruit_number", "") or ""), "quota_scope": "major", "subjects": [], "remarks": row.get("remark", ""), "reference_books": [], "source": "掌上考研", "source_url": "https://www.kaoyan.cn/", "year_verified": True, "raw_evidence": row}))
return result