fix: preserve data granularity and recover tasks safely
This commit is contained in:
+27
-7
@@ -3,13 +3,17 @@ import threading
|
||||
import uuid
|
||||
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
|
||||
from studio.domain import PROVINCES, normalize_catalog
|
||||
from studio.network import ProxyUnavailable, RemoteUnavailable
|
||||
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||
|
||||
|
||||
class Cancelled(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Shutdown(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class JobManager:
|
||||
def __init__(self, database, settings, transport=None, runner=None):
|
||||
self.db, self.settings, self.transport = database, settings, transport
|
||||
@@ -54,7 +58,7 @@ class JobManager:
|
||||
|
||||
def _launch(self, jid):
|
||||
condition = threading.Condition()
|
||||
control = {"condition": condition, "paused": False, "cancelled": False}
|
||||
control = {"condition": condition, "paused": False, "cancelled": False, "shutdown": False}
|
||||
thread = threading.Thread(target=self._run, args=(jid,), daemon=True, name="collector-" + jid[:8])
|
||||
control["thread"] = thread
|
||||
self.controls[jid] = control
|
||||
@@ -71,6 +75,8 @@ class JobManager:
|
||||
self.checkpoint(jid)
|
||||
errors = self.get(jid)["errors"]
|
||||
self.db.job_update(jid, status="partial" if errors else "completed", message="部分单元失败,可恢复重试" if errors else "采集完成")
|
||||
except Shutdown:
|
||||
self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复")
|
||||
except Cancelled:
|
||||
self.db.job_update(jid, status="cancelled", message="已取消,已保存数据保留")
|
||||
except LoginRequired as exc:
|
||||
@@ -79,6 +85,9 @@ class JobManager:
|
||||
except ProxyUnavailable as exc:
|
||||
self.db.job_update(jid, status="waiting_proxy", message=str(exc))
|
||||
self.db.log(jid, "warning", str(exc))
|
||||
except RateLimited as exc:
|
||||
self.db.job_update(jid, status="waiting_source", message=str(exc))
|
||||
self.db.log(jid, "warning", str(exc))
|
||||
except (RemoteUnavailable, SourceYearMismatch, ValueError) as exc:
|
||||
self.db.job_update(jid, status="failed", message=str(exc))
|
||||
self.db.log(jid, "error", str(exc))
|
||||
@@ -88,16 +97,20 @@ class JobManager:
|
||||
finally:
|
||||
if self.transport:
|
||||
self.transport.local.guard = None
|
||||
if self.controls[jid]["shutdown"] and self.get(jid)["status"] != 'cancelled':
|
||||
self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复")
|
||||
|
||||
def checkpoint(self, jid):
|
||||
control = self.controls.get(jid)
|
||||
if not control:
|
||||
return
|
||||
with control["condition"]:
|
||||
while control["paused"] and not control["cancelled"]:
|
||||
while control["paused"] and not control["cancelled"] and not control["shutdown"]:
|
||||
control["condition"].wait(.25)
|
||||
if control["cancelled"]:
|
||||
raise Cancelled()
|
||||
if control["shutdown"]:
|
||||
raise Shutdown()
|
||||
|
||||
def unit_done(self, jid, unit):
|
||||
return self.db.unit_done(jid, unit)
|
||||
@@ -117,16 +130,20 @@ class JobManager:
|
||||
def resume(self, jid):
|
||||
with self.lock:
|
||||
job = self.get(jid)
|
||||
if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy"):
|
||||
if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy", "waiting_source"):
|
||||
raise ValueError("此任务当前不能恢复")
|
||||
self._ensure_available(jid)
|
||||
control = self.controls.get(jid)
|
||||
if control and control["thread"].is_alive():
|
||||
if control and control["thread"].is_alive() and control["paused"] and job['status'] == 'paused':
|
||||
with control["condition"]:
|
||||
self.db.job_update(jid, status="running", message="正在继续采集")
|
||||
control["paused"] = False
|
||||
control["condition"].notify_all()
|
||||
else:
|
||||
if control and control['thread'].is_alive():
|
||||
control['thread'].join(timeout=1)
|
||||
if control['thread'].is_alive():
|
||||
raise ValueError("旧任务正在结束,请稍后点击恢复")
|
||||
self._launch(jid)
|
||||
return self.get(jid)
|
||||
|
||||
@@ -147,7 +164,10 @@ class JobManager:
|
||||
self.closed = True
|
||||
for jid, control in list(self.controls.items()):
|
||||
if control["thread"].is_alive():
|
||||
self.cancel(jid)
|
||||
with control['condition']:
|
||||
control['shutdown'] = True
|
||||
self.db.job_update(jid, status='paused', message='服务停止,任务可从断点恢复')
|
||||
control['condition'].notify_all()
|
||||
for control in list(self.controls.values()):
|
||||
control["thread"].join(timeout=1)
|
||||
|
||||
@@ -194,7 +214,7 @@ class JobManager:
|
||||
self._error(jid, f"{label}:{exc}")
|
||||
|
||||
def _scores(self, jid, params):
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "keyword": params.get("school_keyword", "")}
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
|
||||
records = self.db.all_catalog(filters)
|
||||
targets = {}
|
||||
for row in records:
|
||||
|
||||
Reference in New Issue
Block a user