fix: resume catalog jobs without repeated checkpoint queries
This commit is contained in:
+42
-8
@@ -1,6 +1,7 @@
|
||||
"""Independent adapters for the two projects' public data sources."""
|
||||
import json
|
||||
import re
|
||||
import hashlib
|
||||
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
||||
from studio.network import RemoteUnavailable, RateLimited
|
||||
|
||||
@@ -18,8 +19,9 @@ class SourceYearMismatch(RuntimeError):
|
||||
class CatalogClient:
|
||||
BASE = "https://yz.chsi.com.cn/zsml/"
|
||||
|
||||
def __init__(self, transport, auth=None):
|
||||
def __init__(self, transport, auth=None, page_cache=None, save_page=None, on_request=None):
|
||||
self.transport, self.auth = transport, auth
|
||||
self.page_cache, self.save_page, self.on_request = page_cache, save_page, on_request
|
||||
|
||||
def verify_year(self, year):
|
||||
response = self.transport.request("GET", self.BASE)
|
||||
@@ -65,8 +67,16 @@ class CatalogClient:
|
||||
return data
|
||||
|
||||
def fetch_page(self, endpoint, payload):
|
||||
# Lists are a snapshot for this task. Direction records and login proofs
|
||||
# remain live requests; failed/partial responses never enter the cache.
|
||||
cacheable = self.page_cache is not None and endpoint in ('dws', 'dwzys')
|
||||
key = self._page_key(endpoint, payload) if cacheable else None
|
||||
if cacheable and key in self.page_cache:
|
||||
return self.page_cache[key]
|
||||
if self.on_request:
|
||||
self.on_request(endpoint, payload)
|
||||
try:
|
||||
return self._fetch_page(endpoint, payload)
|
||||
data = self._fetch_page(endpoint, payload)
|
||||
except LoginRequired as exc:
|
||||
if not self.auth or exc.reason == "captcha":
|
||||
raise
|
||||
@@ -76,27 +86,51 @@ class CatalogClient:
|
||||
status = self.auth.check()
|
||||
if not status["valid"]:
|
||||
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None
|
||||
return self._fetch_page(endpoint, payload)
|
||||
data = self._fetch_page(endpoint, payload)
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _page_key(endpoint, payload):
|
||||
return hashlib.sha256(json.dumps([endpoint, payload], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
|
||||
def _cache_page(self, endpoint, payload, data):
|
||||
if self.page_cache is None or endpoint not in ('dws', 'dwzys'):
|
||||
return
|
||||
key = self._page_key(endpoint, payload)
|
||||
if key not in self.page_cache:
|
||||
if self.save_page:
|
||||
self.save_page(key, data)
|
||||
self.page_cache[key] = data
|
||||
|
||||
def pages(self, endpoint, payload, page_size=10):
|
||||
seen, start = set(), 0
|
||||
for page in range(1, 10001):
|
||||
data = self.fetch_page(endpoint, {**payload, "curPage": page, "start": start, "pageSize": page_size})
|
||||
page_payload = {**payload, "curPage": page, "start": start, "pageSize": page_size}
|
||||
data = self.fetch_page(endpoint, page_payload)
|
||||
rows = data["list"]
|
||||
identifier = {'dws': 'dwdm', 'dwzys': 'zydm'}.get(endpoint)
|
||||
if identifier:
|
||||
for row in rows:
|
||||
code = row.get(identifier) if isinstance(row, dict) else None
|
||||
if isinstance(code, bool) or not isinstance(code, (str, int)) or not str(code).strip():
|
||||
label = '院校' if endpoint == 'dws' else '专业'
|
||||
raise RemoteUnavailable(f'研招目录{label}代码缺失或结构异常,已停止以避免缓存不完整数据')
|
||||
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
||||
if rows and signature in seen:
|
||||
raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页")
|
||||
seen.add(signature)
|
||||
yield from rows
|
||||
more = data.get("nextPageAvailable")
|
||||
total = integer(data.get("totalCount"))
|
||||
size = integer(data.get("size")) or page_size
|
||||
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size):
|
||||
return
|
||||
terminal = not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size)
|
||||
next_start = integer(data.get("startOfNextPage"))
|
||||
next_start = start + len(rows) if next_start is None else next_start
|
||||
if next_start <= start:
|
||||
if not terminal and next_start <= start:
|
||||
raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页")
|
||||
self._cache_page(endpoint, page_payload, data)
|
||||
yield from rows
|
||||
if terminal:
|
||||
return
|
||||
start, page_size = next_start, size
|
||||
raise RemoteUnavailable("目录分页超过安全上限")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user