Files
kaoyan/studio/jobs.py
T

281 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Persistent units and cooperative pause/cancel for one local worker."""
import threading
import uuid
from itertools import chain
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
from studio.domain import PROVINCES, normalize_catalog
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
class Cancelled(Exception):
pass
class Shutdown(Exception):
pass
class JobManager:
def __init__(self, database, settings, transport=None, runner=None, auth=None):
self.db, self.settings, self.transport = database, settings, transport
self.auth = auth
self.runner = runner or self._collect
self.controls = {}
self.lock = threading.RLock()
self.closed = False
def get(self, jid=None):
return self.db.job_get(jid)
def start(self, params):
params = {**params}
if params.get("mode") not in ("catalog", "scores", "combined"):
raise ValueError("请选择有效的采集模式")
if not 2000 <= int(params.get("year", 0)) <= 2100:
raise ValueError("请选择目录年份")
if params.get("major_code") and (not str(params["major_code"]).isdigit() or len(str(params["major_code"])) != 6):
raise ValueError("专业代码应为 6 位数字")
provinces = params.get("provinces") or list(PROVINCES)
if any(str(p) not in PROVINCES for p in provinces):
raise ValueError("省份代码无效")
params["provinces"] = list(map(str, provinces))
params["score_years"] = sorted(set(int(y) for y in params.get("score_years", [])))
if any(not 2000 <= y <= 2100 for y in params["score_years"]):
raise ValueError("历史年份无效")
if params["mode"] in ("combined", "scores") and not params["score_years"]:
raise ValueError("请至少选择一个分数线年份")
with self.lock:
self._ensure_available()
jid = uuid.uuid4().hex
self.db.job_create(jid, params)
self._launch(jid)
return self.get(jid)
def _ensure_available(self, same=None):
if self.closed:
raise ValueError("任务服务正在关闭")
for jid, control in self.controls.items():
if jid != same and control["thread"].is_alive():
raise ValueError("已有正在运行或暂停的任务,请完成或取消后再新建")
def _launch(self, jid):
condition = threading.Condition()
control = {"condition": condition, "paused": False, "cancelled": False, "shutdown": False}
thread = threading.Thread(target=self._run, args=(jid,), daemon=True, name="collector-" + jid[:8])
control["thread"] = thread
self.controls[jid] = control
thread.start()
def _run(self, jid):
job = self.get(jid)
self.db.job_update(jid, status="running", errors=0, resume_pending=0, message="正在采集")
self.db.log(jid, "info", "任务开始/恢复,将跳过已完成单元")
if self.transport:
self.transport.local.guard = lambda: self.checkpoint(jid)
try:
self.runner(jid, job["params"], self)
self.checkpoint(jid)
errors = self.get(jid)["errors"]
self.db.job_update(jid, status="partial" if errors else "completed", message="部分单元失败,可恢复重试" if errors else "采集完成")
except Shutdown:
self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复")
except Cancelled:
self.db.job_update(jid, status="cancelled", message="已取消,已保存数据保留")
except LoginRequired as exc:
if self.auth:
self.auth.invalidate(str(exc), state="captcha_required" if exc.reason == "captcha" else "login_required")
self.db.job_update(jid, status="waiting_login", message=str(exc))
self.db.log(jid, "warning", str(exc))
except ProxyUnavailable as exc:
self.db.job_update(jid, status="waiting_proxy", message=str(exc))
self.db.log(jid, "warning", str(exc))
except RateLimited as exc:
self.db.job_update(jid, status="waiting_source", message=str(exc))
self.db.log(jid, "warning", str(exc))
except (RemoteUnavailable, SourceYearMismatch, ValueError) as exc:
self.db.job_update(jid, status="failed", message=str(exc))
self.db.log(jid, "error", str(exc))
except Exception as exc:
self.db.job_update(jid, status="failed", message=f"采集异常({type(exc).__name__}),已保存断点")
self.db.log(jid, "error", f"采集异常类型:{type(exc).__name__}")
finally:
if self.transport:
self.transport.local.guard = None
if self.controls[jid]["shutdown"] and self.get(jid)["status"] != 'cancelled':
self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复")
def checkpoint(self, jid):
control = self.controls.get(jid)
if not control:
return
with control["condition"]:
while control["paused"] and not control["cancelled"] and not control["shutdown"]:
control["condition"].wait(.25)
if control["cancelled"]:
raise Cancelled()
if control["shutdown"]:
raise Shutdown()
def unit_done(self, jid, unit):
return self.db.unit_done(jid, unit)
def complete_unit(self, jid, unit):
self.db.complete_unit(jid, unit)
def pause(self, jid):
control = self.controls.get(jid)
if not control or not control["thread"].is_alive():
raise ValueError("任务当前不在运行")
with control["condition"]:
control["paused"] = True
self.db.job_update(jid, status="paused", resume_pending=0, message="已暂停,当前请求结束后停止")
return self.get(jid)
def resume(self, jid):
with self.lock:
job = self.get(jid)
if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy", "waiting_source"):
raise ValueError("此任务当前不能恢复")
self._ensure_available(jid)
control = self.controls.get(jid)
if control and control["thread"].is_alive() and control["paused"] and job['status'] == 'paused':
with control["condition"]:
self.db.job_update(jid, status="running", message="正在继续采集")
control["paused"] = False
control["condition"].notify_all()
else:
if control and control['thread'].is_alive():
control['thread'].join(timeout=1)
if control['thread'].is_alive():
raise ValueError("旧任务正在结束,请稍后点击恢复")
self._launch(jid)
return self.get(jid)
def cancel(self, jid):
if not self.get(jid):
raise ValueError("任务不存在")
control = self.controls.get(jid)
if control and control["thread"].is_alive():
with control["condition"]:
control["cancelled"] = True
control["condition"].notify_all()
self.db.job_update(jid, message="正在取消,等待当前请求结束")
else:
self.db.job_update(jid, status="cancelled", message="已取消")
return self.get(jid)
def close(self):
self.closed = True
for jid, control in list(self.controls.items()):
if control["thread"].is_alive():
with control['condition']:
control['shutdown'] = True
self.db.job_update(jid, status='paused', resume_pending=int(not control['paused']), message='服务停止,任务可从断点恢复')
control['condition'].notify_all()
for control in list(self.controls.values()):
control["thread"].join(timeout=1)
def _save(self, jid, records):
self.db.upsert_catalog(records)
self.db.job_update(jid, records=self.get(jid)["records"] + len(records))
def _error(self, jid, message):
self.db.job_update(jid, errors=self.get(jid)["errors"] + 1)
self.db.log(jid, "error", message)
def _collect(self, jid, params, manager):
if params["mode"] in ("catalog", "combined"):
self._catalog(jid, params)
if params["mode"] in ("scores", "combined"):
self._scores(jid, params)
def _catalog(self, jid, params):
completed = self.db.completed_units(jid)
location = '研招目录'
def on_request(endpoint, payload):
phase = {'dws': '院校列表', 'dwzys': '专业列表', 'yjfxs': '研究方向'}[endpoint]
self.db.job_update(jid, message=f"{location}:正在请求{phase}第 {payload.get('curPage', 1)} 页")
client = CatalogClient(self.transport, auth=self.auth, page_cache=self.db.catalog_pages(jid),
save_page=lambda key, data: self.db.cache_catalog_page(jid, key, data), on_request=on_request)
client.verify_year(params["year"])
if self.auth:
status = self.auth.check()
if not status["valid"]:
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login")
self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功")
for province in params["provinces"]:
self.checkpoint(jid)
location = PROVINCES[province]
self.db.job_update(jid, message=location + ':正在核对院校列表')
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
for school in client.schools(province, params.get("school_keyword", "")):
self.checkpoint(jid)
school = {"ssdm": province, **school}
location = f"{PROVINCES[province]} · {school.get('dwmc', school['dwdm'])}"
self.db.job_update(jid, message=location + ':正在核对专业列表与已完成断点')
self.db.log(jid, 'info', location + ':核对专业列表与断点')
skipped = 0
for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")):
self.checkpoint(jid)
if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]):
continue
unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}"
if unit in completed:
skipped += 1
continue
self.checkpoint(jid)
label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}"
location = label
self.db.job_update(jid, message=label)
try:
records = []
for row in client.directions(school, major, params.get("study_mode", "")):
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
self.db.save_unit(jid, unit, records)
completed.add(unit)
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}")
if skipped:
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
def _scores(self, jid, params):
provinces = [PROVINCES[p] for p in params['provinces']]
filters = {"year": params["year"], "major_code": params.get("major_code", ""),
"school_keyword": params.get("school_keyword", ""), 'provinces': provinces}
targets = iter(self.db.score_targets(filters))
first = next(targets, None)
if first is None and params.get('major_code'):
targets = ({**row, 'major_code': params['major_code']}
for row in self.db.schools(params.get('school_keyword', '')) if row['province'] in provinces)
first = next(targets, None)
if first is None:
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
completed = self.db.completed_units(jid)
self.db.log(jid, 'info', '开始补充分数线和历史计划,分批读取专业目标并复用院校年份分数线')
client = ScoreClient(self.transport)
for record in chain((first,), targets):
self.checkpoint(jid)
code = record['major_code']
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
years = [y for y in params["score_years"] if f"scores:{school['school_code']}:{code}:{y}" not in completed]
if not years:
continue
school_id = client.school_id(school["school_name"])
if school_id is None:
self._error(jid, school["school_name"] + ":掌上考研未找到精确匹配,已跳过")
continue
for year in years:
self.checkpoint(jid)
label = f"{school['school_name']} · {code} · {year} 年"
self.db.job_update(jid, message=label)
try:
scores = client.score_lines(school, code, year, school_id)
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
unit = f"scores:{school['school_code']}:{code}:{year}"
self.db.save_unit(jid, unit, plans, scores=scores)
completed.add(unit)
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}")