Fix historical pagination and bound export and proxy memory
This commit is contained in:
+43
-8
@@ -2,6 +2,7 @@
|
||||
import json
|
||||
import re
|
||||
import hashlib
|
||||
from collections import OrderedDict
|
||||
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
||||
from studio.network import RemoteUnavailable, RateLimited
|
||||
|
||||
@@ -153,6 +154,7 @@ class ScoreClient:
|
||||
def __init__(self, transport):
|
||||
self.transport = transport
|
||||
self.cache = {}
|
||||
self.score_cache = OrderedDict()
|
||||
|
||||
def post(self, endpoint, payload):
|
||||
for attempt in range(3):
|
||||
@@ -161,8 +163,13 @@ class ScoreClient:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
|
||||
if not isinstance(body, dict):
|
||||
raise RemoteUnavailable("掌上考研响应结构已变化")
|
||||
if str(body.get("code")) == "0000":
|
||||
return body.get("data") or {}
|
||||
data = body.get("data")
|
||||
if not isinstance(data, (dict, list)):
|
||||
raise RemoteUnavailable("掌上考研缺少有效数据,未写入不完整结果")
|
||||
return data
|
||||
message = str(body.get('message', body.get('msg', '')))
|
||||
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
|
||||
if attempt == 2:
|
||||
@@ -172,20 +179,40 @@ class ScoreClient:
|
||||
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
|
||||
|
||||
def pages(self, endpoint, payload, limit=100):
|
||||
seen = set()
|
||||
seen, consumed = set(), 0
|
||||
for page in range(1, 10001):
|
||||
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
|
||||
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data
|
||||
if not isinstance(rows, list):
|
||||
full_list = isinstance(data, list)
|
||||
rows = data.get("data", data.get("list")) if isinstance(data, dict) else data
|
||||
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
|
||||
raise RemoteUnavailable("掌上考研响应结构已变化")
|
||||
total = integer(data.get("total")) if isinstance(data, dict) else None
|
||||
last = integer(data.get("last_page")) if isinstance(data, dict) else None
|
||||
if isinstance(data, dict):
|
||||
for field, value in (('total', total), ('last_page', last)):
|
||||
if data.get(field) is not None and (value is None or value < 0 or isinstance(data[field], bool)):
|
||||
raise RemoteUnavailable("掌上考研分页信息无效,未写入不完整结果")
|
||||
if not rows and (total is not None and consumed < total or last is not None and page < last):
|
||||
raise RemoteUnavailable("掌上考研提前返回空页,未写入不完整结果")
|
||||
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
||||
if rows and signature in seen:
|
||||
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
|
||||
seen.add(signature)
|
||||
consumed += len(rows)
|
||||
yield from rows
|
||||
total = integer(data.get("total")) if isinstance(data, dict) else None
|
||||
last = integer(data.get("last_page")) if isinstance(data, dict) else None
|
||||
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total:
|
||||
# Bare arrays are complete responses; wrapped pages use their actual
|
||||
# metadata even when the server ignores the requested page size.
|
||||
terminal = full_list or not rows
|
||||
if not terminal:
|
||||
if last is not None:
|
||||
terminal = page >= last
|
||||
if terminal and total is not None and consumed < total:
|
||||
raise RemoteUnavailable("掌上考研分页未返回全部记录")
|
||||
elif total is not None:
|
||||
terminal = consumed >= total
|
||||
else:
|
||||
terminal = len(rows) < limit
|
||||
if terminal:
|
||||
return
|
||||
raise RemoteUnavailable("历史数据分页超过安全上限")
|
||||
|
||||
@@ -198,10 +225,18 @@ class ScoreClient:
|
||||
if len(matches) == 1 and matches[0].get("school_id"):
|
||||
self.cache[key] = matches[0]["school_id"]
|
||||
return self.cache[key]
|
||||
self.cache[key] = None
|
||||
return None
|
||||
|
||||
def score_lines(self, school, major_code, year, school_id):
|
||||
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
|
||||
key = (str(school_id), int(year))
|
||||
if key not in self.score_cache:
|
||||
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
|
||||
self.score_cache[key] = rows
|
||||
if len(self.score_cache) > 8:
|
||||
self.score_cache.popitem(last=False)
|
||||
self.score_cache.move_to_end(key)
|
||||
rows = self.score_cache[key]
|
||||
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
|
||||
|
||||
def plans(self, school, major_code, year, school_id):
|
||||
|
||||
Reference in New Issue
Block a user