fix: preserve data granularity and recover tasks safely

This commit is contained in:
Codex
2026-10-11 14:18:56 +08:00
parent 23f3a15a15
commit 603184549b
29 changed files with 4557 additions and 418 deletions
+31 -16
View File
@@ -2,7 +2,7 @@
import json
import re
from studio.domain import finish_record, integer, normalized_name, select_score_lines
from studio.network import RemoteUnavailable
from studio.network import RemoteUnavailable, RateLimited
class LoginRequired(RuntimeError):
@@ -35,17 +35,25 @@ class CatalogClient:
def pages(self, endpoint, payload, page_size=50):
seen = set()
for page in range(1, 10001):
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
try:
body = response.json()
except ValueError:
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
for attempt in range(3):
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
try:
body = response.json()
except ValueError:
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
message = str(body.get('msg', ''))
if body.get('flag') is not True and '验证码' in message:
raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复')
if body.get('flag') is not True and any(token in message for token in ('频繁', '稍后')):
if attempt == 2:
raise RateLimited('研招网业务限流持续出现,已暂停;请增加请求间隔后恢复')
self.transport.sleep(5 * 2 ** attempt)
continue
break
if body.get("flag") is not True:
message = str(body.get("msg", ""))
if any(token in message for token in ("登录", "登陆", "认证")):
raise LoginRequired("研招网要求登录,请更新 Cookie 后恢复任务")
if any(token in message for token in ("频繁", "稍后", "验证码")):
raise RemoteUnavailable("研招网访问受限,请增加请求间隔并完成人工验证后恢复")
raise RemoteUnavailable("研招网目录接口未成功返回数据")
data = body.get("msg") or {}
if not isinstance(data, dict) or not isinstance(data.get("list"), list):
@@ -83,14 +91,21 @@ class ScoreClient:
self.cache = {}
def post(self, endpoint, payload):
response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"})
try:
body = response.json()
except ValueError:
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
if str(body.get("code")) != "0000":
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
return body.get("data") or {}
for attempt in range(3):
response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"})
try:
body = response.json()
except ValueError:
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
if str(body.get("code")) == "0000":
return body.get("data") or {}
message = str(body.get('message', body.get('msg', '')))
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
if attempt == 2:
raise RateLimited('掌上考研业务限流持续出现,请增加请求间隔后恢复')
self.transport.sleep(5 * 2 ** attempt)
else:
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
def pages(self, endpoint, payload, limit=100):
seen = set()