fix: preserve data granularity and recover tasks safely
This commit is contained in:
+31
-16
@@ -2,7 +2,7 @@
|
||||
import json
|
||||
import re
|
||||
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
||||
from studio.network import RemoteUnavailable
|
||||
from studio.network import RemoteUnavailable, RateLimited
|
||||
|
||||
|
||||
class LoginRequired(RuntimeError):
|
||||
@@ -35,17 +35,25 @@ class CatalogClient:
|
||||
def pages(self, endpoint, payload, page_size=50):
|
||||
seen = set()
|
||||
for page in range(1, 10001):
|
||||
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
|
||||
for attempt in range(3):
|
||||
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
|
||||
message = str(body.get('msg', ''))
|
||||
if body.get('flag') is not True and '验证码' in message:
|
||||
raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复')
|
||||
if body.get('flag') is not True and any(token in message for token in ('频繁', '稍后')):
|
||||
if attempt == 2:
|
||||
raise RateLimited('研招网业务限流持续出现,已暂停;请增加请求间隔后恢复')
|
||||
self.transport.sleep(5 * 2 ** attempt)
|
||||
continue
|
||||
break
|
||||
if body.get("flag") is not True:
|
||||
message = str(body.get("msg", ""))
|
||||
if any(token in message for token in ("登录", "登陆", "认证")):
|
||||
raise LoginRequired("研招网要求登录,请更新 Cookie 后恢复任务")
|
||||
if any(token in message for token in ("频繁", "稍后", "验证码")):
|
||||
raise RemoteUnavailable("研招网访问受限,请增加请求间隔并完成人工验证后恢复")
|
||||
raise RemoteUnavailable("研招网目录接口未成功返回数据")
|
||||
data = body.get("msg") or {}
|
||||
if not isinstance(data, dict) or not isinstance(data.get("list"), list):
|
||||
@@ -83,14 +91,21 @@ class ScoreClient:
|
||||
self.cache = {}
|
||||
|
||||
def post(self, endpoint, payload):
|
||||
response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
|
||||
if str(body.get("code")) != "0000":
|
||||
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
|
||||
return body.get("data") or {}
|
||||
for attempt in range(3):
|
||||
response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
|
||||
if str(body.get("code")) == "0000":
|
||||
return body.get("data") or {}
|
||||
message = str(body.get('message', body.get('msg', '')))
|
||||
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
|
||||
if attempt == 2:
|
||||
raise RateLimited('掌上考研业务限流持续出现,请增加请求间隔后恢复')
|
||||
self.transport.sleep(5 * 2 ** attempt)
|
||||
else:
|
||||
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
|
||||
|
||||
def pages(self, endpoint, payload, limit=100):
|
||||
seen = set()
|
||||
|
||||
Reference in New Issue
Block a user