fix: resume catalog jobs without repeated checkpoint queries
This commit is contained in:
+20
-2
@@ -189,7 +189,13 @@ class JobManager:
|
||||
self._scores(jid, params)
|
||||
|
||||
def _catalog(self, jid, params):
|
||||
client = CatalogClient(self.transport, auth=self.auth)
|
||||
completed = self.db.completed_units(jid)
|
||||
location = '研招目录'
|
||||
def on_request(endpoint, payload):
|
||||
phase = {'dws': '院校列表', 'dwzys': '专业列表', 'yjfxs': '研究方向'}[endpoint]
|
||||
self.db.job_update(jid, message=f"{location}:正在请求{phase}第 {payload.get('curPage', 1)} 页")
|
||||
client = CatalogClient(self.transport, auth=self.auth, page_cache=self.db.catalog_pages(jid),
|
||||
save_page=lambda key, data: self.db.cache_catalog_page(jid, key, data), on_request=on_request)
|
||||
client.verify_year(params["year"])
|
||||
if self.auth:
|
||||
status = self.auth.check()
|
||||
@@ -198,18 +204,27 @@ class JobManager:
|
||||
self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功")
|
||||
for province in params["provinces"]:
|
||||
self.checkpoint(jid)
|
||||
location = PROVINCES[province]
|
||||
self.db.job_update(jid, message=location + ':正在核对院校列表')
|
||||
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
|
||||
for school in client.schools(province, params.get("school_keyword", "")):
|
||||
self.checkpoint(jid)
|
||||
school = {"ssdm": province, **school}
|
||||
location = f"{PROVINCES[province]} · {school.get('dwmc', school['dwdm'])}"
|
||||
self.db.job_update(jid, message=location + ':正在核对专业列表与已完成断点')
|
||||
self.db.log(jid, 'info', location + ':核对专业列表与断点')
|
||||
skipped = 0
|
||||
for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")):
|
||||
self.checkpoint(jid)
|
||||
if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]):
|
||||
continue
|
||||
unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}"
|
||||
if self.unit_done(jid, unit):
|
||||
if unit in completed:
|
||||
skipped += 1
|
||||
continue
|
||||
self.checkpoint(jid)
|
||||
label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}"
|
||||
location = label
|
||||
self.db.job_update(jid, message=label)
|
||||
try:
|
||||
records = []
|
||||
@@ -217,9 +232,12 @@ class JobManager:
|
||||
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
|
||||
self._save(jid, records)
|
||||
self.complete_unit(jid, unit)
|
||||
completed.add(unit)
|
||||
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
|
||||
except RemoteUnavailable as exc:
|
||||
self._error(jid, f"{label}:{exc}")
|
||||
if skipped:
|
||||
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
|
||||
|
||||
def _scores(self, jid, params):
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
|
||||
|
||||
Reference in New Issue
Block a user