fix: resume catalog jobs without repeated checkpoint queries

This commit is contained in:
2026-10-11 21:18:49 +08:00
parent c4a71e88a1
commit 4c510759fb
9 changed files with 309 additions and 13 deletions
+42 -8
View File
@@ -1,6 +1,7 @@
"""Independent adapters for the two projects' public data sources."""
import json
import re
import hashlib
from studio.domain import finish_record, integer, normalized_name, select_score_lines
from studio.network import RemoteUnavailable, RateLimited
@@ -18,8 +19,9 @@ class SourceYearMismatch(RuntimeError):
class CatalogClient:
BASE = "https://yz.chsi.com.cn/zsml/"
def __init__(self, transport, auth=None):
def __init__(self, transport, auth=None, page_cache=None, save_page=None, on_request=None):
self.transport, self.auth = transport, auth
self.page_cache, self.save_page, self.on_request = page_cache, save_page, on_request
def verify_year(self, year):
response = self.transport.request("GET", self.BASE)
@@ -65,8 +67,16 @@ class CatalogClient:
return data
def fetch_page(self, endpoint, payload):
# Lists are a snapshot for this task. Direction records and login proofs
# remain live requests; failed/partial responses never enter the cache.
cacheable = self.page_cache is not None and endpoint in ('dws', 'dwzys')
key = self._page_key(endpoint, payload) if cacheable else None
if cacheable and key in self.page_cache:
return self.page_cache[key]
if self.on_request:
self.on_request(endpoint, payload)
try:
return self._fetch_page(endpoint, payload)
data = self._fetch_page(endpoint, payload)
except LoginRequired as exc:
if not self.auth or exc.reason == "captcha":
raise
@@ -76,27 +86,51 @@ class CatalogClient:
status = self.auth.check()
if not status["valid"]:
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None
return self._fetch_page(endpoint, payload)
data = self._fetch_page(endpoint, payload)
return data
@staticmethod
def _page_key(endpoint, payload):
return hashlib.sha256(json.dumps([endpoint, payload], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
def _cache_page(self, endpoint, payload, data):
if self.page_cache is None or endpoint not in ('dws', 'dwzys'):
return
key = self._page_key(endpoint, payload)
if key not in self.page_cache:
if self.save_page:
self.save_page(key, data)
self.page_cache[key] = data
def pages(self, endpoint, payload, page_size=10):
seen, start = set(), 0
for page in range(1, 10001):
data = self.fetch_page(endpoint, {**payload, "curPage": page, "start": start, "pageSize": page_size})
page_payload = {**payload, "curPage": page, "start": start, "pageSize": page_size}
data = self.fetch_page(endpoint, page_payload)
rows = data["list"]
identifier = {'dws': 'dwdm', 'dwzys': 'zydm'}.get(endpoint)
if identifier:
for row in rows:
code = row.get(identifier) if isinstance(row, dict) else None
if isinstance(code, bool) or not isinstance(code, (str, int)) or not str(code).strip():
label = '院校' if endpoint == 'dws' else '专业'
raise RemoteUnavailable(f'研招目录{label}代码缺失或结构异常,已停止以避免缓存不完整数据')
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen:
raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页")
seen.add(signature)
yield from rows
more = data.get("nextPageAvailable")
total = integer(data.get("totalCount"))
size = integer(data.get("size")) or page_size
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size):
return
terminal = not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size)
next_start = integer(data.get("startOfNextPage"))
next_start = start + len(rows) if next_start is None else next_start
if next_start <= start:
if not terminal and next_start <= start:
raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页")
self._cache_page(endpoint, page_payload, data)
yield from rows
if terminal:
return
start, page_size = next_start, size
raise RemoteUnavailable("目录分页超过安全上限")