"""Independent adapters for the two projects' public data sources.""" import json import re import hashlib from studio.domain import finish_record, integer, normalized_name, select_score_lines from studio.network import RemoteUnavailable, RateLimited class LoginRequired(RuntimeError): def __init__(self, message, reason="login"): super().__init__(message) self.reason = reason class SourceYearMismatch(RuntimeError): pass class CatalogClient: BASE = "https://yz.chsi.com.cn/zsml/" def __init__(self, transport, auth=None, page_cache=None, save_page=None, on_request=None): self.transport, self.auth = transport, auth self.page_cache, self.save_page, self.on_request = page_cache, save_page, on_request def verify_year(self, year): response = self.transport.request("GET", self.BASE) text = re.sub("<[^>]*>", " ", response.text) matches = re.findall(r"(20\d{2})\s*年.{0,30}(?:硕士|招生|专业目录)", text) matches.extend(re.findall(r"\bcurYear\s*:\s*['\"](20\d{2})['\"]", response.text)) years = set(map(int, matches)) if not years: raise SourceYearMismatch("未能核实研招网当前目录年份,已停止写入;请检查登录及目录开放状态") current = max(years) if int(year) != current: raise SourceYearMismatch(f"研招网当前公开目录为 {current} 年,任务选择了 {year} 年;历史目录请导入已有快照") return current def _fetch_page(self, endpoint, payload): for attempt in range(3): response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data=payload) try: body = response.json() except ValueError: if '验证码' in response.text: raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复', reason="captcha") from None if re.search(r"passport/login|j_spring_cas_security_check|name=[\"']password[\"']", response.text) or '/passport/login' in response.url: raise LoginRequired("研招网返回登录页面,请在浏览器完成登录后恢复") from None raise RemoteUnavailable("研招目录接口未返回有效 JSON,请稍后重试") from None if not isinstance(body, dict): raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据") message = str(body.get('msg', '')) if body.get('flag') is not True and '验证码' in message: raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复', reason="captcha") if body.get('flag') is not True and any(token in message for token in ('频繁', '稍后')): if attempt == 2: raise RateLimited('研招网业务限流持续出现,已暂停;请增加请求间隔后恢复') self.transport.sleep(5 * 2 ** attempt) continue if body.get("flag") is not True: if any(token in message for token in ("登录", "登陆", "认证")): raise LoginRequired("研招网要求登录,请重新验证或在浏览器登录后恢复任务") raise RemoteUnavailable("研招网目录接口未成功返回数据") data = body.get("msg") if not isinstance(data, dict) or not isinstance(data.get("list"), list): raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据") return data def fetch_page(self, endpoint, payload): # Lists are a snapshot for this task. Direction records and login proofs # remain live requests; failed/partial responses never enter the cache. cacheable = self.page_cache is not None and endpoint in ('dws', 'dwzys') key = self._page_key(endpoint, payload) if cacheable else None if cacheable and key in self.page_cache: return self.page_cache[key] if self.on_request: self.on_request(endpoint, payload) try: data = self._fetch_page(endpoint, payload) except LoginRequired as exc: if not self.auth or exc.reason == "captcha": raise # A CAS cookie may still be valid while the catalogue session has expired. # Refresh once, then retry exactly the same page; never skip its records. with self.transport.router.hold_route(): status = self.auth.check() if not status["valid"]: raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None data = self._fetch_page(endpoint, payload) return data @staticmethod def _page_key(endpoint, payload): return hashlib.sha256(json.dumps([endpoint, payload], sort_keys=True, ensure_ascii=False).encode()).hexdigest() def _cache_page(self, endpoint, payload, data): if self.page_cache is None or endpoint not in ('dws', 'dwzys'): return key = self._page_key(endpoint, payload) if key not in self.page_cache: if self.save_page: self.save_page(key, data) self.page_cache[key] = data def pages(self, endpoint, payload, page_size=10): seen, start = set(), 0 for page in range(1, 10001): page_payload = {**payload, "curPage": page, "start": start, "pageSize": page_size} data = self.fetch_page(endpoint, page_payload) rows = data["list"] identifier = {'dws': 'dwdm', 'dwzys': 'zydm'}.get(endpoint) if identifier: for row in rows: code = row.get(identifier) if isinstance(row, dict) else None if isinstance(code, bool) or not isinstance(code, (str, int)) or not str(code).strip(): label = '院校' if endpoint == 'dws' else '专业' raise RemoteUnavailable(f'研招目录{label}代码缺失或结构异常,已停止以避免缓存不完整数据') signature = json.dumps(rows, ensure_ascii=False, sort_keys=True) if rows and signature in seen: raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页") seen.add(signature) more = data.get("nextPageAvailable") total = integer(data.get("totalCount")) size = integer(data.get("size")) or page_size terminal = not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size) next_start = integer(data.get("startOfNextPage")) next_start = start + len(rows) if next_start is None else next_start if not terminal and next_start <= start: raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页") self._cache_page(endpoint, page_payload, data) yield from rows if terminal: return start, page_size = next_start, size raise RemoteUnavailable("目录分页超过安全上限") def schools(self, province, keyword=""): return self.pages("dws", {"ssdm": province, "dwmc": keyword, "dwdm": "", "xxfs": "", "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"}) def majors(self, school, major_code="", study_mode=""): return self.pages("dwzys", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major_code, "zymc": "", "mldm": "", "yjxkdm": "", "xwlx": "", "xxfs": study_mode}) def directions(self, school, major, study_mode=""): return self.pages("yjfxs", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major["zydm"], "zymc": major.get("zymc", ""), "xxfs": study_mode, "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"}) ALIASES = {"湖州师范学院": "湖州师范大学", "闽江学院": "闽江大学", "皖南医学院": "皖南医科大学", "中国航空研究院(613所)": "中国航空研究院", "中国电子科技集团有限公司研究生院": "中国电子科技集团", "湖南理工学院": "湖南理工大学"} class ScoreClient: BASE = "https://api.kaoyan.cn/pc" def __init__(self, transport): self.transport = transport self.cache = {} def post(self, endpoint, payload): for attempt in range(3): response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"}) try: body = response.json() except ValueError: raise RemoteUnavailable("掌上考研未返回有效 JSON") from None if str(body.get("code")) == "0000": return body.get("data") or {} message = str(body.get('message', body.get('msg', ''))) if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')): if attempt == 2: raise RateLimited('掌上考研业务限流持续出现,请增加请求间隔后恢复') self.transport.sleep(5 * 2 ** attempt) else: raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试") def pages(self, endpoint, payload, limit=100): seen = set() for page in range(1, 10001): data = self.post(endpoint, {**payload, "page": page, "limit": limit}) rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data if not isinstance(rows, list): raise RemoteUnavailable("掌上考研响应结构已变化") signature = json.dumps(rows, ensure_ascii=False, sort_keys=True) if rows and signature in seen: raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页") seen.add(signature) yield from rows total = integer(data.get("total")) if isinstance(data, dict) else None last = integer(data.get("last_page")) if isinstance(data, dict) else None if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total: return raise RemoteUnavailable("历史数据分页超过安全上限") def school_id(self, name): key = normalized_name(name) if key in self.cache: return self.cache[key] for query in dict.fromkeys([name, ALIASES.get(key, name)]): matches = [row for row in self.pages("/school/schoolList", {"school_name": query}, limit=10) if normalized_name(row.get("school_name", "")) == normalized_name(query)] if len(matches) == 1 and matches[0].get("school_id"): self.cache[key] = matches[0]["school_id"] return self.cache[key] return None def score_lines(self, school, major_code, year, school_id): rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year})) return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year) def plans(self, school, major_code, year, school_id): result = [] for row in self.pages("/school/planListV2", {"school_id": school_id, "year": year, "keyword": major_code}): if str(row.get("special_code", "")) != major_code: continue result.append(finish_record({**school, "year": year, "major_code": major_code, "major_name": row.get("special_name", ""), "department_code": str(row.get("depart_id", "") or ""), "department_name": row.get("depart_name", ""), "direction_code": str(row.get("spe_id", "") or row.get("plan_id", "")), "direction": row.get("research_direction", ""), "quota": integer(row.get("recruit_number")), "quota_text": str(row.get("recruit_number", "") or ""), "quota_scope": "major", "subjects": [], "remarks": row.get("remark", ""), "reference_books": [], "source": "掌上考研", "source_url": "https://www.kaoyan.cn/", "year_verified": True, "raw_evidence": row})) return result