fix: preserve data granularity and recover tasks safely

This commit is contained in:
Codex
2026-10-11 14:18:56 +08:00
parent 23f3a15a15
commit 603184549b
29 changed files with 4557 additions and 418 deletions
+27 -7
View File
@@ -3,13 +3,17 @@ import threading
import uuid
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
from studio.domain import PROVINCES, normalize_catalog
from studio.network import ProxyUnavailable, RemoteUnavailable
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
class Cancelled(Exception):
pass
class Shutdown(Exception):
pass
class JobManager:
def __init__(self, database, settings, transport=None, runner=None):
self.db, self.settings, self.transport = database, settings, transport
@@ -54,7 +58,7 @@ class JobManager:
def _launch(self, jid):
condition = threading.Condition()
control = {"condition": condition, "paused": False, "cancelled": False}
control = {"condition": condition, "paused": False, "cancelled": False, "shutdown": False}
thread = threading.Thread(target=self._run, args=(jid,), daemon=True, name="collector-" + jid[:8])
control["thread"] = thread
self.controls[jid] = control
@@ -71,6 +75,8 @@ class JobManager:
self.checkpoint(jid)
errors = self.get(jid)["errors"]
self.db.job_update(jid, status="partial" if errors else "completed", message="部分单元失败,可恢复重试" if errors else "采集完成")
except Shutdown:
self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复")
except Cancelled:
self.db.job_update(jid, status="cancelled", message="已取消,已保存数据保留")
except LoginRequired as exc:
@@ -79,6 +85,9 @@ class JobManager:
except ProxyUnavailable as exc:
self.db.job_update(jid, status="waiting_proxy", message=str(exc))
self.db.log(jid, "warning", str(exc))
except RateLimited as exc:
self.db.job_update(jid, status="waiting_source", message=str(exc))
self.db.log(jid, "warning", str(exc))
except (RemoteUnavailable, SourceYearMismatch, ValueError) as exc:
self.db.job_update(jid, status="failed", message=str(exc))
self.db.log(jid, "error", str(exc))
@@ -88,16 +97,20 @@ class JobManager:
finally:
if self.transport:
self.transport.local.guard = None
if self.controls[jid]["shutdown"] and self.get(jid)["status"] != 'cancelled':
self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复")
def checkpoint(self, jid):
control = self.controls.get(jid)
if not control:
return
with control["condition"]:
while control["paused"] and not control["cancelled"]:
while control["paused"] and not control["cancelled"] and not control["shutdown"]:
control["condition"].wait(.25)
if control["cancelled"]:
raise Cancelled()
if control["shutdown"]:
raise Shutdown()
def unit_done(self, jid, unit):
return self.db.unit_done(jid, unit)
@@ -117,16 +130,20 @@ class JobManager:
def resume(self, jid):
with self.lock:
job = self.get(jid)
if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy"):
if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy", "waiting_source"):
raise ValueError("此任务当前不能恢复")
self._ensure_available(jid)
control = self.controls.get(jid)
if control and control["thread"].is_alive():
if control and control["thread"].is_alive() and control["paused"] and job['status'] == 'paused':
with control["condition"]:
self.db.job_update(jid, status="running", message="正在继续采集")
control["paused"] = False
control["condition"].notify_all()
else:
if control and control['thread'].is_alive():
control['thread'].join(timeout=1)
if control['thread'].is_alive():
raise ValueError("旧任务正在结束,请稍后点击恢复")
self._launch(jid)
return self.get(jid)
@@ -147,7 +164,10 @@ class JobManager:
self.closed = True
for jid, control in list(self.controls.items()):
if control["thread"].is_alive():
self.cancel(jid)
with control['condition']:
control['shutdown'] = True
self.db.job_update(jid, status='paused', message='服务停止,任务可从断点恢复')
control['condition'].notify_all()
for control in list(self.controls.values()):
control["thread"].join(timeout=1)
@@ -194,7 +214,7 @@ class JobManager:
self._error(jid, f"{label}:{exc}")
def _scores(self, jid, params):
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "keyword": params.get("school_keyword", "")}
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
records = self.db.all_catalog(filters)
targets = {}
for row in records: