"""Persistent units and cooperative pause/cancel for one local worker.""" import threading import uuid from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch from studio.domain import PROVINCES, normalize_catalog from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited class Cancelled(Exception): pass class Shutdown(Exception): pass class JobManager: def __init__(self, database, settings, transport=None, runner=None, auth=None): self.db, self.settings, self.transport = database, settings, transport self.auth = auth self.runner = runner or self._collect self.controls = {} self.lock = threading.RLock() self.closed = False def get(self, jid=None): return self.db.job_get(jid) def start(self, params): params = {**params} if params.get("mode") not in ("catalog", "scores", "combined"): raise ValueError("请选择有效的采集模式") if not 2000 <= int(params.get("year", 0)) <= 2100: raise ValueError("请选择目录年份") if params.get("major_code") and (not str(params["major_code"]).isdigit() or len(str(params["major_code"])) != 6): raise ValueError("专业代码应为 6 位数字") provinces = params.get("provinces") or list(PROVINCES) if any(str(p) not in PROVINCES for p in provinces): raise ValueError("省份代码无效") params["provinces"] = list(map(str, provinces)) params["score_years"] = sorted(set(int(y) for y in params.get("score_years", []))) if any(not 2000 <= y <= 2100 for y in params["score_years"]): raise ValueError("历史年份无效") if params["mode"] in ("combined", "scores") and not params["score_years"]: raise ValueError("请至少选择一个分数线年份") with self.lock: self._ensure_available() jid = uuid.uuid4().hex self.db.job_create(jid, params) self._launch(jid) return self.get(jid) def _ensure_available(self, same=None): if self.closed: raise ValueError("任务服务正在关闭") for jid, control in self.controls.items(): if jid != same and control["thread"].is_alive(): raise ValueError("已有正在运行或暂停的任务,请完成或取消后再新建") def _launch(self, jid): condition = threading.Condition() control = {"condition": condition, "paused": False, "cancelled": False, "shutdown": False} thread = threading.Thread(target=self._run, args=(jid,), daemon=True, name="collector-" + jid[:8]) control["thread"] = thread self.controls[jid] = control thread.start() def _run(self, jid): job = self.get(jid) self.db.job_update(jid, status="running", errors=0, resume_pending=0, message="正在采集") self.db.log(jid, "info", "任务开始/恢复,将跳过已完成单元") if self.transport: self.transport.local.guard = lambda: self.checkpoint(jid) try: self.runner(jid, job["params"], self) self.checkpoint(jid) errors = self.get(jid)["errors"] self.db.job_update(jid, status="partial" if errors else "completed", message="部分单元失败,可恢复重试" if errors else "采集完成") except Shutdown: self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复") except Cancelled: self.db.job_update(jid, status="cancelled", message="已取消,已保存数据保留") except LoginRequired as exc: if self.auth: self.auth.invalidate(str(exc), state="captcha_required" if exc.reason == "captcha" else "login_required") self.db.job_update(jid, status="waiting_login", message=str(exc)) self.db.log(jid, "warning", str(exc)) except ProxyUnavailable as exc: self.db.job_update(jid, status="waiting_proxy", message=str(exc)) self.db.log(jid, "warning", str(exc)) except RateLimited as exc: self.db.job_update(jid, status="waiting_source", message=str(exc)) self.db.log(jid, "warning", str(exc)) except (RemoteUnavailable, SourceYearMismatch, ValueError) as exc: self.db.job_update(jid, status="failed", message=str(exc)) self.db.log(jid, "error", str(exc)) except Exception as exc: self.db.job_update(jid, status="failed", message=f"采集异常({type(exc).__name__}),已保存断点") self.db.log(jid, "error", f"采集异常类型:{type(exc).__name__}") finally: if self.transport: self.transport.local.guard = None if self.controls[jid]["shutdown"] and self.get(jid)["status"] != 'cancelled': self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复") def checkpoint(self, jid): control = self.controls.get(jid) if not control: return with control["condition"]: while control["paused"] and not control["cancelled"] and not control["shutdown"]: control["condition"].wait(.25) if control["cancelled"]: raise Cancelled() if control["shutdown"]: raise Shutdown() def unit_done(self, jid, unit): return self.db.unit_done(jid, unit) def complete_unit(self, jid, unit): self.db.complete_unit(jid, unit) def pause(self, jid): control = self.controls.get(jid) if not control or not control["thread"].is_alive(): raise ValueError("任务当前不在运行") with control["condition"]: control["paused"] = True self.db.job_update(jid, status="paused", resume_pending=0, message="已暂停,当前请求结束后停止") return self.get(jid) def resume(self, jid): with self.lock: job = self.get(jid) if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy", "waiting_source"): raise ValueError("此任务当前不能恢复") self._ensure_available(jid) control = self.controls.get(jid) if control and control["thread"].is_alive() and control["paused"] and job['status'] == 'paused': with control["condition"]: self.db.job_update(jid, status="running", message="正在继续采集") control["paused"] = False control["condition"].notify_all() else: if control and control['thread'].is_alive(): control['thread'].join(timeout=1) if control['thread'].is_alive(): raise ValueError("旧任务正在结束,请稍后点击恢复") self._launch(jid) return self.get(jid) def cancel(self, jid): if not self.get(jid): raise ValueError("任务不存在") control = self.controls.get(jid) if control and control["thread"].is_alive(): with control["condition"]: control["cancelled"] = True control["condition"].notify_all() self.db.job_update(jid, message="正在取消,等待当前请求结束") else: self.db.job_update(jid, status="cancelled", message="已取消") return self.get(jid) def close(self): self.closed = True for jid, control in list(self.controls.items()): if control["thread"].is_alive(): with control['condition']: control['shutdown'] = True self.db.job_update(jid, status='paused', resume_pending=int(not control['paused']), message='服务停止,任务可从断点恢复') control['condition'].notify_all() for control in list(self.controls.values()): control["thread"].join(timeout=1) def _save(self, jid, records): self.db.upsert_catalog(records) self.db.job_update(jid, records=self.get(jid)["records"] + len(records)) def _error(self, jid, message): self.db.job_update(jid, errors=self.get(jid)["errors"] + 1) self.db.log(jid, "error", message) def _collect(self, jid, params, manager): if params["mode"] in ("catalog", "combined"): self._catalog(jid, params) if params["mode"] in ("scores", "combined"): self._scores(jid, params) def _catalog(self, jid, params): completed = self.db.completed_units(jid) location = '研招目录' def on_request(endpoint, payload): phase = {'dws': '院校列表', 'dwzys': '专业列表', 'yjfxs': '研究方向'}[endpoint] self.db.job_update(jid, message=f"{location}:正在请求{phase}第 {payload.get('curPage', 1)} 页") client = CatalogClient(self.transport, auth=self.auth, page_cache=self.db.catalog_pages(jid), save_page=lambda key, data: self.db.cache_catalog_page(jid, key, data), on_request=on_request) client.verify_year(params["year"]) if self.auth: status = self.auth.check() if not status["valid"]: raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功") for province in params["provinces"]: self.checkpoint(jid) location = PROVINCES[province] self.db.job_update(jid, message=location + ':正在核对院校列表') self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校") for school in client.schools(province, params.get("school_keyword", "")): self.checkpoint(jid) school = {"ssdm": province, **school} location = f"{PROVINCES[province]} · {school.get('dwmc', school['dwdm'])}" self.db.job_update(jid, message=location + ':正在核对专业列表与已完成断点') self.db.log(jid, 'info', location + ':核对专业列表与断点') skipped = 0 for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")): self.checkpoint(jid) if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]): continue unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}" if unit in completed: skipped += 1 continue self.checkpoint(jid) label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}" location = label self.db.job_update(jid, message=label) try: records = [] for row in client.directions(school, major, params.get("study_mode", "")): records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major)) self._save(jid, records) self.complete_unit(jid, unit) completed.add(unit) self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合") except RemoteUnavailable as exc: self._error(jid, f"{label}:{exc}") if skipped: self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业") def _scores(self, jid, params): filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")} records = self.db.all_catalog(filters) targets = {} for row in records: if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]: continue targets[(row["school_code"], row["major_code"])] = row if not targets and params.get("major_code"): for row in self.db.schools(params.get("school_keyword", "")): if row["province"] in [PROVINCES[p] for p in params["provinces"]]: targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]} if not targets: raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引") client = ScoreClient(self.transport) for (_, code), record in targets.items(): self.checkpoint(jid) school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")} years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")] if not years: continue school_id = client.school_id(school["school_name"]) if school_id is None: self._error(jid, school["school_name"] + ":掌上考研未找到精确匹配,已跳过") continue for year in years: self.checkpoint(jid) label = f"{school['school_name']} · {code} · {year} 年" self.db.job_update(jid, message=label) try: scores = client.score_lines(school, code, year, school_id) self.db.upsert_scores(scores) plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else [] self._save(jid, plans) self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}") self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条") except RemoteUnavailable as exc: self._error(jid, f"{label}:{exc}")