135 lines
7.6 KiB
Python
135 lines
7.6 KiB
Python
"""Independent adapters for the two projects' public data sources."""
|
|
import json
|
|
import re
|
|
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
|
from studio.network import RemoteUnavailable
|
|
|
|
|
|
class LoginRequired(RuntimeError):
|
|
pass
|
|
|
|
|
|
class SourceYearMismatch(RuntimeError):
|
|
pass
|
|
|
|
|
|
class CatalogClient:
|
|
BASE = "https://yz.chsi.com.cn/zsml/"
|
|
|
|
def __init__(self, transport):
|
|
self.transport = transport
|
|
|
|
def verify_year(self, year):
|
|
response = self.transport.request("GET", self.BASE)
|
|
text = re.sub("<[^>]*>", " ", response.text)
|
|
matches = re.findall(r"(20\d{2})\s*年.{0,30}(?:硕士|招生|专业目录)", text)
|
|
matches.extend(re.findall(r"\bcurYear\s*:\s*['\"](20\d{2})['\"]", response.text))
|
|
years = set(map(int, matches))
|
|
if not years:
|
|
raise SourceYearMismatch("未能核实研招网当前目录年份,已停止写入;请检查登录及目录开放状态")
|
|
current = max(years)
|
|
if int(year) != current:
|
|
raise SourceYearMismatch(f"研招网当前公开目录为 {current} 年,任务选择了 {year} 年;历史目录请导入已有快照")
|
|
return current
|
|
|
|
def pages(self, endpoint, payload, page_size=50):
|
|
seen = set()
|
|
for page in range(1, 10001):
|
|
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
|
|
try:
|
|
body = response.json()
|
|
except ValueError:
|
|
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
|
|
if body.get("flag") is not True:
|
|
message = str(body.get("msg", ""))
|
|
if any(token in message for token in ("登录", "登陆", "认证")):
|
|
raise LoginRequired("研招网要求登录,请更新 Cookie 后恢复任务")
|
|
if any(token in message for token in ("频繁", "稍后", "验证码")):
|
|
raise RemoteUnavailable("研招网访问受限,请增加请求间隔并完成人工验证后恢复")
|
|
raise RemoteUnavailable("研招网目录接口未成功返回数据")
|
|
data = body.get("msg") or {}
|
|
if not isinstance(data, dict) or not isinstance(data.get("list"), list):
|
|
raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据")
|
|
rows = data["list"]
|
|
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
|
if rows and signature in seen:
|
|
raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页")
|
|
seen.add(signature)
|
|
yield from rows
|
|
more = data.get("nextPageAvailable")
|
|
total = integer(data.get("totalCount"))
|
|
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and page * page_size >= total) or len(rows) < page_size):
|
|
return
|
|
raise RemoteUnavailable("目录分页超过安全上限")
|
|
|
|
def schools(self, province, keyword=""):
|
|
return self.pages("dws", {"ssdm": province, "dwmc": keyword, "dwdm": "", "xxfs": "", "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"})
|
|
|
|
def majors(self, school, major_code="", study_mode=""):
|
|
return self.pages("dwzys", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major_code, "zymc": "", "mldm": "", "yjxkdm": "", "xwlx": "", "xxfs": study_mode})
|
|
|
|
def directions(self, school, major, study_mode=""):
|
|
return self.pages("yjfxs", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major["zydm"], "zymc": major.get("zymc", ""), "xxfs": study_mode, "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"})
|
|
|
|
|
|
ALIASES = {"湖州师范学院": "湖州师范大学", "闽江学院": "闽江大学", "皖南医学院": "皖南医科大学", "中国航空研究院(613所)": "中国航空研究院", "中国电子科技集团有限公司研究生院": "中国电子科技集团", "湖南理工学院": "湖南理工大学"}
|
|
|
|
|
|
class ScoreClient:
|
|
BASE = "https://api.kaoyan.cn/pc"
|
|
|
|
def __init__(self, transport):
|
|
self.transport = transport
|
|
self.cache = {}
|
|
|
|
def post(self, endpoint, payload):
|
|
response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"})
|
|
try:
|
|
body = response.json()
|
|
except ValueError:
|
|
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
|
|
if str(body.get("code")) != "0000":
|
|
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
|
|
return body.get("data") or {}
|
|
|
|
def pages(self, endpoint, payload, limit=100):
|
|
seen = set()
|
|
for page in range(1, 10001):
|
|
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
|
|
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data
|
|
if not isinstance(rows, list):
|
|
raise RemoteUnavailable("掌上考研响应结构已变化")
|
|
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
|
if rows and signature in seen:
|
|
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
|
|
seen.add(signature)
|
|
yield from rows
|
|
total = integer(data.get("total")) if isinstance(data, dict) else None
|
|
last = integer(data.get("last_page")) if isinstance(data, dict) else None
|
|
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total:
|
|
return
|
|
raise RemoteUnavailable("历史数据分页超过安全上限")
|
|
|
|
def school_id(self, name):
|
|
key = normalized_name(name)
|
|
if key in self.cache:
|
|
return self.cache[key]
|
|
for query in dict.fromkeys([name, ALIASES.get(key, name)]):
|
|
matches = [row for row in self.pages("/school/schoolList", {"school_name": query}, limit=10) if normalized_name(row.get("school_name", "")) == normalized_name(query)]
|
|
if len(matches) == 1 and matches[0].get("school_id"):
|
|
self.cache[key] = matches[0]["school_id"]
|
|
return self.cache[key]
|
|
return None
|
|
|
|
def score_lines(self, school, major_code, year, school_id):
|
|
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
|
|
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
|
|
|
|
def plans(self, school, major_code, year, school_id):
|
|
result = []
|
|
for row in self.pages("/school/planListV2", {"school_id": school_id, "year": year, "keyword": major_code}):
|
|
if str(row.get("special_code", "")) != major_code:
|
|
continue
|
|
result.append(finish_record({**school, "year": year, "major_code": major_code, "major_name": row.get("special_name", ""), "department_code": str(row.get("depart_id", "") or ""), "department_name": row.get("depart_name", ""), "direction_code": str(row.get("spe_id", "") or row.get("plan_id", "")), "direction": row.get("research_direction", ""), "quota": integer(row.get("recruit_number")), "quota_text": str(row.get("recruit_number", "") or ""), "quota_scope": "major", "subjects": [], "remarks": row.get("remark", ""), "reference_books": [], "source": "掌上考研", "source_url": "https://www.kaoyan.cn/", "year_verified": True, "raw_evidence": row}))
|
|
return result
|