Fix CHSI catalogue session verification and pagination

This commit is contained in:
Codex
2026-10-11 15:28:14 +08:00
parent 8001cd86b1
commit 2e3439dbb9
11 changed files with 496 additions and 85 deletions
+54 -25
View File
@@ -6,7 +6,9 @@ from studio.network import RemoteUnavailable, RateLimited
class LoginRequired(RuntimeError):
pass
def __init__(self, message, reason="login"):
super().__init__(message)
self.reason = reason
class SourceYearMismatch(RuntimeError):
@@ -16,8 +18,8 @@ class SourceYearMismatch(RuntimeError):
class CatalogClient:
BASE = "https://yz.chsi.com.cn/zsml/"
def __init__(self, transport):
self.transport = transport
def __init__(self, transport, auth=None):
self.transport, self.auth = transport, auth
def verify_year(self, year):
response = self.transport.request("GET", self.BASE)
@@ -32,32 +34,53 @@ class CatalogClient:
raise SourceYearMismatch(f"研招网当前公开目录为 {current} 年,任务选择了 {year} 年;历史目录请导入已有快照")
return current
def pages(self, endpoint, payload, page_size=50):
seen = set()
for page in range(1, 10001):
for attempt in range(3):
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
try:
body = response.json()
except ValueError:
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
message = str(body.get('msg', ''))
if body.get('flag') is not True and '验证码' in message:
raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复')
if body.get('flag') is not True and any(token in message for token in ('频繁', '稍后')):
if attempt == 2:
raise RateLimited('研招网业务限流持续出现,已暂停;请增加请求间隔后恢复')
self.transport.sleep(5 * 2 ** attempt)
continue
break
def _fetch_page(self, endpoint, payload):
for attempt in range(3):
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data=payload)
try:
body = response.json()
except ValueError:
if '验证码' in response.text:
raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复', reason="captcha") from None
if re.search(r"passport/login|j_spring_cas_security_check|name=[\"']password[\"']", response.text) or '/passport/login' in response.url:
raise LoginRequired("研招网返回登录页面,请在浏览器完成登录后恢复") from None
raise RemoteUnavailable("研招目录接口未返回有效 JSON,请稍后重试") from None
if not isinstance(body, dict):
raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据")
message = str(body.get('msg', ''))
if body.get('flag') is not True and '验证码' in message:
raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复', reason="captcha")
if body.get('flag') is not True and any(token in message for token in ('频繁', '稍后')):
if attempt == 2:
raise RateLimited('研招网业务限流持续出现,已暂停;请增加请求间隔后恢复')
self.transport.sleep(5 * 2 ** attempt)
continue
if body.get("flag") is not True:
message = str(body.get("msg", ""))
if any(token in message for token in ("登录", "登陆", "认证")):
raise LoginRequired("研招网要求登录,请更新 Cookie 后恢复任务")
raise LoginRequired("研招网要求登录,请重新验证或在浏览器登录后恢复任务")
raise RemoteUnavailable("研招网目录接口未成功返回数据")
data = body.get("msg") or {}
data = body.get("msg")
if not isinstance(data, dict) or not isinstance(data.get("list"), list):
raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据")
return data
def fetch_page(self, endpoint, payload):
try:
return self._fetch_page(endpoint, payload)
except LoginRequired as exc:
if not self.auth or exc.reason == "captcha":
raise
# A CAS cookie may still be valid while the catalogue session has expired.
# Refresh once, then retry exactly the same page; never skip its records.
status = self.auth.check()
if not status["valid"]:
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None
return self._fetch_page(endpoint, payload)
def pages(self, endpoint, payload, page_size=10):
seen, start = set(), 0
for page in range(1, 10001):
data = self.fetch_page(endpoint, {**payload, "curPage": page, "start": start, "pageSize": page_size})
rows = data["list"]
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen:
@@ -66,8 +89,14 @@ class CatalogClient:
yield from rows
more = data.get("nextPageAvailable")
total = integer(data.get("totalCount"))
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and page * page_size >= total) or len(rows) < page_size):
size = integer(data.get("size")) or page_size
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size):
return
next_start = integer(data.get("startOfNextPage"))
next_start = start + len(rows) if next_start is None else next_start
if next_start <= start:
raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页")
start, page_size = next_start, size
raise RemoteUnavailable("目录分页超过安全上限")
def schools(self, province, keyword=""):