Fix historical pagination and bound export and proxy memory

This commit is contained in:
2026-10-13 00:31:34 +08:00
parent 4c510759fb
commit ec2c897752
20 changed files with 1050 additions and 87 deletions
+43 -8
View File
@@ -2,6 +2,7 @@
import json
import re
import hashlib
from collections import OrderedDict
from studio.domain import finish_record, integer, normalized_name, select_score_lines
from studio.network import RemoteUnavailable, RateLimited
@@ -153,6 +154,7 @@ class ScoreClient:
def __init__(self, transport):
self.transport = transport
self.cache = {}
self.score_cache = OrderedDict()
def post(self, endpoint, payload):
for attempt in range(3):
@@ -161,8 +163,13 @@ class ScoreClient:
body = response.json()
except ValueError:
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
if not isinstance(body, dict):
raise RemoteUnavailable("掌上考研响应结构已变化")
if str(body.get("code")) == "0000":
return body.get("data") or {}
data = body.get("data")
if not isinstance(data, (dict, list)):
raise RemoteUnavailable("掌上考研缺少有效数据,未写入不完整结果")
return data
message = str(body.get('message', body.get('msg', '')))
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
if attempt == 2:
@@ -172,20 +179,40 @@ class ScoreClient:
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
def pages(self, endpoint, payload, limit=100):
seen = set()
seen, consumed = set(), 0
for page in range(1, 10001):
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data
if not isinstance(rows, list):
full_list = isinstance(data, list)
rows = data.get("data", data.get("list")) if isinstance(data, dict) else data
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
raise RemoteUnavailable("掌上考研响应结构已变化")
total = integer(data.get("total")) if isinstance(data, dict) else None
last = integer(data.get("last_page")) if isinstance(data, dict) else None
if isinstance(data, dict):
for field, value in (('total', total), ('last_page', last)):
if data.get(field) is not None and (value is None or value < 0 or isinstance(data[field], bool)):
raise RemoteUnavailable("掌上考研分页信息无效,未写入不完整结果")
if not rows and (total is not None and consumed < total or last is not None and page < last):
raise RemoteUnavailable("掌上考研提前返回空页,未写入不完整结果")
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen:
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
seen.add(signature)
consumed += len(rows)
yield from rows
total = integer(data.get("total")) if isinstance(data, dict) else None
last = integer(data.get("last_page")) if isinstance(data, dict) else None
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total:
# Bare arrays are complete responses; wrapped pages use their actual
# metadata even when the server ignores the requested page size.
terminal = full_list or not rows
if not terminal:
if last is not None:
terminal = page >= last
if terminal and total is not None and consumed < total:
raise RemoteUnavailable("掌上考研分页未返回全部记录")
elif total is not None:
terminal = consumed >= total
else:
terminal = len(rows) < limit
if terminal:
return
raise RemoteUnavailable("历史数据分页超过安全上限")
@@ -198,10 +225,18 @@ class ScoreClient:
if len(matches) == 1 and matches[0].get("school_id"):
self.cache[key] = matches[0]["school_id"]
return self.cache[key]
self.cache[key] = None
return None
def score_lines(self, school, major_code, year, school_id):
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
key = (str(school_id), int(year))
if key not in self.score_cache:
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
self.score_cache[key] = rows
if len(self.score_cache) > 8:
self.score_cache.popitem(last=False)
self.score_cache.move_to_end(key)
rows = self.score_cache[key]
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
def plans(self, school, major_code, year, school_id):