"""Independent adapters for the two projects' public data sources.""" import json import re from studio.domain import finish_record, integer, normalized_name, select_score_lines from studio.network import RemoteUnavailable class LoginRequired(RuntimeError): pass class SourceYearMismatch(RuntimeError): pass class CatalogClient: BASE = "https://yz.chsi.com.cn/zsml/" def __init__(self, transport): self.transport = transport def verify_year(self, year): response = self.transport.request("GET", self.BASE) text = re.sub("<[^>]*>", " ", response.text) matches = re.findall(r"(20\d{2})\s*年.{0,30}(?:硕士|招生|专业目录)", text) years = set(map(int, matches)) if not years: raise SourceYearMismatch("未能核实研招网当前目录年份,已停止写入;请检查登录及目录开放状态") current = max(years) if int(year) != current: raise SourceYearMismatch(f"研招网当前公开目录为 {current} 年,任务选择了 {year} 年;历史目录请导入已有快照") return current def pages(self, endpoint, payload, page_size=50): seen = set() for page in range(1, 10001): response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size}) try: body = response.json() except ValueError: raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None if body.get("flag") is not True: message = str(body.get("msg", "")) if any(token in message for token in ("登录", "登陆", "认证")): raise LoginRequired("研招网要求登录,请更新 Cookie 后恢复任务") if any(token in message for token in ("频繁", "稍后", "验证码")): raise RemoteUnavailable("研招网访问受限,请增加请求间隔并完成人工验证后恢复") raise RemoteUnavailable("研招网目录接口未成功返回数据") data = body.get("msg") or {} if not isinstance(data, dict) or not isinstance(data.get("list"), list): raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据") rows = data["list"] signature = json.dumps(rows, ensure_ascii=False, sort_keys=True) if rows and signature in seen: raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页") seen.add(signature) yield from rows more = data.get("nextPageAvailable") total = integer(data.get("totalCount")) if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and page * page_size >= total) or len(rows) < page_size): return raise RemoteUnavailable("目录分页超过安全上限") def schools(self, province, keyword=""): return self.pages("dws", {"ssdm": province, "dwmc": keyword, "dwdm": "", "xxfs": "", "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"}) def majors(self, school, major_code="", study_mode=""): return self.pages("dwzys", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major_code, "zymc": "", "mldm": "", "yjxkdm": "", "xwlx": "", "xxfs": study_mode}) def directions(self, school, major, study_mode=""): return self.pages("yjfxs", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major["zydm"], "zymc": major.get("zymc", ""), "xxfs": study_mode, "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"}) ALIASES = {"湖州师范学院": "湖州师范大学", "闽江学院": "闽江大学", "皖南医学院": "皖南医科大学", "中国航空研究院(613所)": "中国航空研究院", "中国电子科技集团有限公司研究生院": "中国电子科技集团", "湖南理工学院": "湖南理工大学"} class ScoreClient: BASE = "https://api.kaoyan.cn/pc" def __init__(self, transport): self.transport = transport self.cache = {} def post(self, endpoint, payload): response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"}) try: body = response.json() except ValueError: raise RemoteUnavailable("掌上考研未返回有效 JSON") from None if str(body.get("code")) != "0000": raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试") return body.get("data") or {} def pages(self, endpoint, payload, limit=100): seen = set() for page in range(1, 10001): data = self.post(endpoint, {**payload, "page": page, "limit": limit}) rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data if not isinstance(rows, list): raise RemoteUnavailable("掌上考研响应结构已变化") signature = json.dumps(rows, ensure_ascii=False, sort_keys=True) if rows and signature in seen: raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页") seen.add(signature) yield from rows total = integer(data.get("total")) if isinstance(data, dict) else None last = integer(data.get("last_page")) if isinstance(data, dict) else None if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total: return raise RemoteUnavailable("历史数据分页超过安全上限") def school_id(self, name): key = normalized_name(name) if key in self.cache: return self.cache[key] for query in dict.fromkeys([name, ALIASES.get(key, name)]): matches = [row for row in self.pages("/school/schoolList", {"school_name": query}, limit=10) if normalized_name(row.get("school_name", "")) == normalized_name(query)] if len(matches) == 1 and matches[0].get("school_id"): self.cache[key] = matches[0]["school_id"] return self.cache[key] return None def score_lines(self, school, major_code, year, school_id): rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year})) return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year) def plans(self, school, major_code, year, school_id): result = [] for row in self.pages("/school/planListV2", {"school_id": school_id, "year": year, "keyword": major_code}): if str(row.get("special_code", "")) != major_code: continue result.append(finish_record({**school, "year": year, "major_code": major_code, "major_name": row.get("special_name", ""), "department_code": str(row.get("depart_id", "") or ""), "department_name": row.get("depart_name", ""), "direction_code": str(row.get("spe_id", "") or row.get("plan_id", "")), "direction": row.get("research_direction", ""), "quota": integer(row.get("recruit_number")), "quota_text": str(row.get("recruit_number", "") or ""), "quota_scope": "major", "subjects": [], "remarks": row.get("remark", ""), "reference_books": [], "source": "掌上考研", "source_url": "https://www.kaoyan.cn/", "year_verified": True, "raw_evidence": row})) return result