280 lines
14 KiB
Python
280 lines
14 KiB
Python
"""Persistent units and cooperative pause/cancel for one local worker."""
|
||
import threading
|
||
import uuid
|
||
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
|
||
from studio.domain import PROVINCES, normalize_catalog
|
||
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
|
||
|
||
|
||
class Cancelled(Exception):
|
||
pass
|
||
|
||
|
||
class Shutdown(Exception):
|
||
pass
|
||
|
||
|
||
class JobManager:
|
||
def __init__(self, database, settings, transport=None, runner=None, auth=None):
|
||
self.db, self.settings, self.transport = database, settings, transport
|
||
self.auth = auth
|
||
self.runner = runner or self._collect
|
||
self.controls = {}
|
||
self.lock = threading.RLock()
|
||
self.closed = False
|
||
|
||
def get(self, jid=None):
|
||
return self.db.job_get(jid)
|
||
|
||
def start(self, params):
|
||
params = {**params}
|
||
if params.get("mode") not in ("catalog", "scores", "combined"):
|
||
raise ValueError("请选择有效的采集模式")
|
||
if not 2000 <= int(params.get("year", 0)) <= 2100:
|
||
raise ValueError("请选择目录年份")
|
||
if params.get("major_code") and (not str(params["major_code"]).isdigit() or len(str(params["major_code"])) != 6):
|
||
raise ValueError("专业代码应为 6 位数字")
|
||
provinces = params.get("provinces") or list(PROVINCES)
|
||
if any(str(p) not in PROVINCES for p in provinces):
|
||
raise ValueError("省份代码无效")
|
||
params["provinces"] = list(map(str, provinces))
|
||
params["score_years"] = sorted(set(int(y) for y in params.get("score_years", [])))
|
||
if any(not 2000 <= y <= 2100 for y in params["score_years"]):
|
||
raise ValueError("历史年份无效")
|
||
if params["mode"] in ("combined", "scores") and not params["score_years"]:
|
||
raise ValueError("请至少选择一个分数线年份")
|
||
with self.lock:
|
||
self._ensure_available()
|
||
jid = uuid.uuid4().hex
|
||
self.db.job_create(jid, params)
|
||
self._launch(jid)
|
||
return self.get(jid)
|
||
|
||
def _ensure_available(self, same=None):
|
||
if self.closed:
|
||
raise ValueError("任务服务正在关闭")
|
||
for jid, control in self.controls.items():
|
||
if jid != same and control["thread"].is_alive():
|
||
raise ValueError("已有正在运行或暂停的任务,请完成或取消后再新建")
|
||
|
||
def _launch(self, jid):
|
||
condition = threading.Condition()
|
||
control = {"condition": condition, "paused": False, "cancelled": False, "shutdown": False}
|
||
thread = threading.Thread(target=self._run, args=(jid,), daemon=True, name="collector-" + jid[:8])
|
||
control["thread"] = thread
|
||
self.controls[jid] = control
|
||
thread.start()
|
||
|
||
def _run(self, jid):
|
||
job = self.get(jid)
|
||
self.db.job_update(jid, status="running", errors=0, resume_pending=0, message="正在采集")
|
||
self.db.log(jid, "info", "任务开始/恢复,将跳过已完成单元")
|
||
if self.transport:
|
||
self.transport.local.guard = lambda: self.checkpoint(jid)
|
||
try:
|
||
self.runner(jid, job["params"], self)
|
||
self.checkpoint(jid)
|
||
errors = self.get(jid)["errors"]
|
||
self.db.job_update(jid, status="partial" if errors else "completed", message="部分单元失败,可恢复重试" if errors else "采集完成")
|
||
except Shutdown:
|
||
self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复")
|
||
except Cancelled:
|
||
self.db.job_update(jid, status="cancelled", message="已取消,已保存数据保留")
|
||
except LoginRequired as exc:
|
||
if self.auth:
|
||
self.auth.invalidate(str(exc), state="captcha_required" if exc.reason == "captcha" else "login_required")
|
||
self.db.job_update(jid, status="waiting_login", message=str(exc))
|
||
self.db.log(jid, "warning", str(exc))
|
||
except ProxyUnavailable as exc:
|
||
self.db.job_update(jid, status="waiting_proxy", message=str(exc))
|
||
self.db.log(jid, "warning", str(exc))
|
||
except RateLimited as exc:
|
||
self.db.job_update(jid, status="waiting_source", message=str(exc))
|
||
self.db.log(jid, "warning", str(exc))
|
||
except (RemoteUnavailable, SourceYearMismatch, ValueError) as exc:
|
||
self.db.job_update(jid, status="failed", message=str(exc))
|
||
self.db.log(jid, "error", str(exc))
|
||
except Exception as exc:
|
||
self.db.job_update(jid, status="failed", message=f"采集异常({type(exc).__name__}),已保存断点")
|
||
self.db.log(jid, "error", f"采集异常类型:{type(exc).__name__}")
|
||
finally:
|
||
if self.transport:
|
||
self.transport.local.guard = None
|
||
if self.controls[jid]["shutdown"] and self.get(jid)["status"] != 'cancelled':
|
||
self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复")
|
||
|
||
def checkpoint(self, jid):
|
||
control = self.controls.get(jid)
|
||
if not control:
|
||
return
|
||
with control["condition"]:
|
||
while control["paused"] and not control["cancelled"] and not control["shutdown"]:
|
||
control["condition"].wait(.25)
|
||
if control["cancelled"]:
|
||
raise Cancelled()
|
||
if control["shutdown"]:
|
||
raise Shutdown()
|
||
|
||
def unit_done(self, jid, unit):
|
||
return self.db.unit_done(jid, unit)
|
||
|
||
def complete_unit(self, jid, unit):
|
||
self.db.complete_unit(jid, unit)
|
||
|
||
def pause(self, jid):
|
||
control = self.controls.get(jid)
|
||
if not control or not control["thread"].is_alive():
|
||
raise ValueError("任务当前不在运行")
|
||
with control["condition"]:
|
||
control["paused"] = True
|
||
self.db.job_update(jid, status="paused", resume_pending=0, message="已暂停,当前请求结束后停止")
|
||
return self.get(jid)
|
||
|
||
def resume(self, jid):
|
||
with self.lock:
|
||
job = self.get(jid)
|
||
if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy", "waiting_source"):
|
||
raise ValueError("此任务当前不能恢复")
|
||
self._ensure_available(jid)
|
||
control = self.controls.get(jid)
|
||
if control and control["thread"].is_alive() and control["paused"] and job['status'] == 'paused':
|
||
with control["condition"]:
|
||
self.db.job_update(jid, status="running", message="正在继续采集")
|
||
control["paused"] = False
|
||
control["condition"].notify_all()
|
||
else:
|
||
if control and control['thread'].is_alive():
|
||
control['thread'].join(timeout=1)
|
||
if control['thread'].is_alive():
|
||
raise ValueError("旧任务正在结束,请稍后点击恢复")
|
||
self._launch(jid)
|
||
return self.get(jid)
|
||
|
||
def cancel(self, jid):
|
||
if not self.get(jid):
|
||
raise ValueError("任务不存在")
|
||
control = self.controls.get(jid)
|
||
if control and control["thread"].is_alive():
|
||
with control["condition"]:
|
||
control["cancelled"] = True
|
||
control["condition"].notify_all()
|
||
self.db.job_update(jid, message="正在取消,等待当前请求结束")
|
||
else:
|
||
self.db.job_update(jid, status="cancelled", message="已取消")
|
||
return self.get(jid)
|
||
|
||
def close(self):
|
||
self.closed = True
|
||
for jid, control in list(self.controls.items()):
|
||
if control["thread"].is_alive():
|
||
with control['condition']:
|
||
control['shutdown'] = True
|
||
self.db.job_update(jid, status='paused', resume_pending=int(not control['paused']), message='服务停止,任务可从断点恢复')
|
||
control['condition'].notify_all()
|
||
for control in list(self.controls.values()):
|
||
control["thread"].join(timeout=1)
|
||
|
||
def _save(self, jid, records):
|
||
self.db.upsert_catalog(records)
|
||
self.db.job_update(jid, records=self.get(jid)["records"] + len(records))
|
||
|
||
def _error(self, jid, message):
|
||
self.db.job_update(jid, errors=self.get(jid)["errors"] + 1)
|
||
self.db.log(jid, "error", message)
|
||
|
||
def _collect(self, jid, params, manager):
|
||
if params["mode"] in ("catalog", "combined"):
|
||
self._catalog(jid, params)
|
||
if params["mode"] in ("scores", "combined"):
|
||
self._scores(jid, params)
|
||
|
||
def _catalog(self, jid, params):
|
||
completed = self.db.completed_units(jid)
|
||
location = '研招目录'
|
||
def on_request(endpoint, payload):
|
||
phase = {'dws': '院校列表', 'dwzys': '专业列表', 'yjfxs': '研究方向'}[endpoint]
|
||
self.db.job_update(jid, message=f"{location}:正在请求{phase}第 {payload.get('curPage', 1)} 页")
|
||
client = CatalogClient(self.transport, auth=self.auth, page_cache=self.db.catalog_pages(jid),
|
||
save_page=lambda key, data: self.db.cache_catalog_page(jid, key, data), on_request=on_request)
|
||
client.verify_year(params["year"])
|
||
if self.auth:
|
||
status = self.auth.check()
|
||
if not status["valid"]:
|
||
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login")
|
||
self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功")
|
||
for province in params["provinces"]:
|
||
self.checkpoint(jid)
|
||
location = PROVINCES[province]
|
||
self.db.job_update(jid, message=location + ':正在核对院校列表')
|
||
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
|
||
for school in client.schools(province, params.get("school_keyword", "")):
|
||
self.checkpoint(jid)
|
||
school = {"ssdm": province, **school}
|
||
location = f"{PROVINCES[province]} · {school.get('dwmc', school['dwdm'])}"
|
||
self.db.job_update(jid, message=location + ':正在核对专业列表与已完成断点')
|
||
self.db.log(jid, 'info', location + ':核对专业列表与断点')
|
||
skipped = 0
|
||
for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")):
|
||
self.checkpoint(jid)
|
||
if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]):
|
||
continue
|
||
unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}"
|
||
if unit in completed:
|
||
skipped += 1
|
||
continue
|
||
self.checkpoint(jid)
|
||
label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}"
|
||
location = label
|
||
self.db.job_update(jid, message=label)
|
||
try:
|
||
records = []
|
||
for row in client.directions(school, major, params.get("study_mode", "")):
|
||
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
|
||
self._save(jid, records)
|
||
self.complete_unit(jid, unit)
|
||
completed.add(unit)
|
||
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
|
||
except RemoteUnavailable as exc:
|
||
self._error(jid, f"{label}:{exc}")
|
||
if skipped:
|
||
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
|
||
|
||
def _scores(self, jid, params):
|
||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
|
||
records = self.db.all_catalog(filters)
|
||
targets = {}
|
||
for row in records:
|
||
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]:
|
||
continue
|
||
targets[(row["school_code"], row["major_code"])] = row
|
||
if not targets and params.get("major_code"):
|
||
for row in self.db.schools(params.get("school_keyword", "")):
|
||
if row["province"] in [PROVINCES[p] for p in params["provinces"]]:
|
||
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
|
||
if not targets:
|
||
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
|
||
client = ScoreClient(self.transport)
|
||
for (_, code), record in targets.items():
|
||
self.checkpoint(jid)
|
||
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
|
||
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")]
|
||
if not years:
|
||
continue
|
||
school_id = client.school_id(school["school_name"])
|
||
if school_id is None:
|
||
self._error(jid, school["school_name"] + ":掌上考研未找到精确匹配,已跳过")
|
||
continue
|
||
for year in years:
|
||
self.checkpoint(jid)
|
||
label = f"{school['school_name']} · {code} · {year} 年"
|
||
self.db.job_update(jid, message=label)
|
||
try:
|
||
scores = client.score_lines(school, code, year, school_id)
|
||
self.db.upsert_scores(scores)
|
||
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
|
||
self._save(jid, plans)
|
||
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}")
|
||
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
|
||
except RemoteUnavailable as exc:
|
||
self._error(jid, f"{label}:{exc}")
|