feat: implement unified data sources, proxy routing and local API

This commit is contained in:
Codex
2026-10-11 13:49:07 +08:00
parent 25cccd4372
commit 389b2aa3a5
20 changed files with 21601 additions and 4 deletions
+233
View File
@@ -0,0 +1,233 @@
"""Persistent units and cooperative pause/cancel for one local worker."""
import threading
import uuid
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
from studio.domain import PROVINCES, normalize_catalog
from studio.network import ProxyUnavailable, RemoteUnavailable
class Cancelled(Exception):
pass
class JobManager:
def __init__(self, database, settings, transport=None, runner=None):
self.db, self.settings, self.transport = database, settings, transport
self.runner = runner or self._collect
self.controls = {}
self.lock = threading.RLock()
self.closed = False
def get(self, jid=None):
return self.db.job_get(jid)
def start(self, params):
params = {**params}
if params.get("mode") not in ("catalog", "scores", "combined"):
raise ValueError("请选择有效的采集模式")
if not 2000 <= int(params.get("year", 0)) <= 2100:
raise ValueError("请选择目录年份")
if params.get("major_code") and (not str(params["major_code"]).isdigit() or len(str(params["major_code"])) != 6):
raise ValueError("专业代码应为 6 位数字")
provinces = params.get("provinces") or list(PROVINCES)
if any(str(p) not in PROVINCES for p in provinces):
raise ValueError("省份代码无效")
params["provinces"] = list(map(str, provinces))
params["score_years"] = sorted(set(int(y) for y in params.get("score_years", [])))
if any(not 2000 <= y <= 2100 for y in params["score_years"]):
raise ValueError("历史年份无效")
if params["mode"] in ("combined", "scores") and not params["score_years"]:
raise ValueError("请至少选择一个分数线年份")
with self.lock:
self._ensure_available()
jid = uuid.uuid4().hex
self.db.job_create(jid, params)
self._launch(jid)
return self.get(jid)
def _ensure_available(self, same=None):
if self.closed:
raise ValueError("任务服务正在关闭")
for jid, control in self.controls.items():
if jid != same and control["thread"].is_alive():
raise ValueError("已有正在运行或暂停的任务,请完成或取消后再新建")
def _launch(self, jid):
condition = threading.Condition()
control = {"condition": condition, "paused": False, "cancelled": False}
thread = threading.Thread(target=self._run, args=(jid,), daemon=True, name="collector-" + jid[:8])
control["thread"] = thread
self.controls[jid] = control
thread.start()
def _run(self, jid):
job = self.get(jid)
self.db.job_update(jid, status="running", errors=0, message="正在采集")
self.db.log(jid, "info", "任务开始/恢复,将跳过已完成单元")
if self.transport:
self.transport.local.guard = lambda: self.checkpoint(jid)
try:
self.runner(jid, job["params"], self)
self.checkpoint(jid)
errors = self.get(jid)["errors"]
self.db.job_update(jid, status="partial" if errors else "completed", message="部分单元失败,可恢复重试" if errors else "采集完成")
except Cancelled:
self.db.job_update(jid, status="cancelled", message="已取消,已保存数据保留")
except LoginRequired as exc:
self.db.job_update(jid, status="waiting_login", message=str(exc))
self.db.log(jid, "warning", str(exc))
except ProxyUnavailable as exc:
self.db.job_update(jid, status="waiting_proxy", message=str(exc))
self.db.log(jid, "warning", str(exc))
except (RemoteUnavailable, SourceYearMismatch, ValueError) as exc:
self.db.job_update(jid, status="failed", message=str(exc))
self.db.log(jid, "error", str(exc))
except Exception as exc:
self.db.job_update(jid, status="failed", message=f"采集异常({type(exc).__name__}),已保存断点")
self.db.log(jid, "error", f"采集异常类型:{type(exc).__name__}")
finally:
if self.transport:
self.transport.local.guard = None
def checkpoint(self, jid):
control = self.controls.get(jid)
if not control:
return
with control["condition"]:
while control["paused"] and not control["cancelled"]:
control["condition"].wait(.25)
if control["cancelled"]:
raise Cancelled()
def unit_done(self, jid, unit):
return self.db.unit_done(jid, unit)
def complete_unit(self, jid, unit):
self.db.complete_unit(jid, unit)
def pause(self, jid):
control = self.controls.get(jid)
if not control or not control["thread"].is_alive():
raise ValueError("任务当前不在运行")
with control["condition"]:
control["paused"] = True
self.db.job_update(jid, status="paused", message="已暂停,当前请求结束后停止")
return self.get(jid)
def resume(self, jid):
with self.lock:
job = self.get(jid)
if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy"):
raise ValueError("此任务当前不能恢复")
self._ensure_available(jid)
control = self.controls.get(jid)
if control and control["thread"].is_alive():
with control["condition"]:
self.db.job_update(jid, status="running", message="正在继续采集")
control["paused"] = False
control["condition"].notify_all()
else:
self._launch(jid)
return self.get(jid)
def cancel(self, jid):
if not self.get(jid):
raise ValueError("任务不存在")
control = self.controls.get(jid)
if control and control["thread"].is_alive():
with control["condition"]:
control["cancelled"] = True
control["condition"].notify_all()
self.db.job_update(jid, message="正在取消,等待当前请求结束")
else:
self.db.job_update(jid, status="cancelled", message="已取消")
return self.get(jid)
def close(self):
self.closed = True
for jid, control in list(self.controls.items()):
if control["thread"].is_alive():
self.cancel(jid)
for control in list(self.controls.values()):
control["thread"].join(timeout=1)
def _save(self, jid, records):
self.db.upsert_catalog(records)
self.db.job_update(jid, records=self.get(jid)["records"] + len(records))
def _error(self, jid, message):
self.db.job_update(jid, errors=self.get(jid)["errors"] + 1)
self.db.log(jid, "error", message)
def _collect(self, jid, params, manager):
if params["mode"] in ("catalog", "combined"):
self._catalog(jid, params)
if params["mode"] in ("scores", "combined"):
self._scores(jid, params)
def _catalog(self, jid, params):
client = CatalogClient(self.transport)
client.verify_year(params["year"])
for province in params["provinces"]:
self.checkpoint(jid)
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
for school in client.schools(province, params.get("school_keyword", "")):
self.checkpoint(jid)
school = {"ssdm": province, **school}
for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")):
if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]):
continue
unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}"
if self.unit_done(jid, unit):
continue
self.checkpoint(jid)
label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}"
self.db.job_update(jid, message=label)
try:
records = []
for row in client.directions(school, major, params.get("study_mode", "")):
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
self._save(jid, records)
self.complete_unit(jid, unit)
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}")
def _scores(self, jid, params):
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "keyword": params.get("school_keyword", "")}
records = self.db.all_catalog(filters)
targets = {}
for row in records:
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]:
continue
targets[(row["school_code"], row["major_code"])] = row
if not targets and params.get("major_code"):
for row in self.db.schools(params.get("school_keyword", "")):
if row["province"] in [PROVINCES[p] for p in params["provinces"]]:
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
if not targets:
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
client = ScoreClient(self.transport)
for (_, code), record in targets.items():
self.checkpoint(jid)
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")]
if not years:
continue
school_id = client.school_id(school["school_name"])
if school_id is None:
self._error(jid, school["school_name"] + ":掌上考研未找到精确匹配,已跳过")
continue
for year in years:
self.checkpoint(jid)
label = f"{school['school_name']} · {code} · {year} 年"
self.db.job_update(jid, message=label)
try:
scores = client.score_lines(school, code, year, school_id)
self.db.upsert_scores(scores)
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
self._save(jid, plans)
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}")
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}")