feat: implement unified data sources, proxy routing and local API
This commit is contained in:
@@ -7,4 +7,6 @@ frontend/node_modules/
|
||||
frontend/dist/
|
||||
frontend/*.tsbuildinfo
|
||||
.superpowers/
|
||||
*.egg-info/
|
||||
*-install.log
|
||||
|
||||
|
||||
+15787
File diff suppressed because it is too large
Load Diff
+2762
File diff suppressed because it is too large
Load Diff
+6
-4
@@ -3,10 +3,12 @@
|
||||
用户已明确要求创建项目,并选择本地网页;依照会话授权直接推进开发。
|
||||
原目录检查:YAM 无本地修改;yzw 有本地修改;代理池运行中。新独立仓库隔离开发,不修改以上目录。
|
||||
Pre-flight:数据库/任务/网络/API 的数据接口按设计共享。图形界面通过 HTTP 使用这些接口,不依赖原项目环境。
|
||||
Task 1: pending
|
||||
Task 2: pending
|
||||
Task 3: pending
|
||||
Task 4: pending
|
||||
Task 1: complete — 5 tests green; cross-year records, alternatives, conservative quota, score granularity.
|
||||
Task 2: complete — 5 tests green; shared proxy, strict route, fallback, rotation, DPAPI vault.
|
||||
Task 3: complete — 5 tests green; pagination, business login status, year mismatch, exact campus, pause/resume.
|
||||
Task 4: complete — 5 tests green; file import/export, YAM SQLite, favorites, origin guard, formula-safe exports.
|
||||
Task 5: pending
|
||||
Task 6: pending
|
||||
|
||||
Public index: 2,255 major entries and 460 unique schools extracted from YAM public data files. Seed index contains no admissions snapshots, cookies or signed request tokens.
|
||||
|
||||
|
||||
Generated
+1337
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,2 @@
|
||||
"""考研数据工作台。"""
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import uvicorn
|
||||
from studio.api import create_app
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="考研数据工作台")
|
||||
parser.add_argument("--port", type=int, default=8765)
|
||||
parser.add_argument("--data-dir", type=Path)
|
||||
args = parser.parse_args()
|
||||
uvicorn.run(create_app(args.data_dir), host="127.0.0.1", port=args.port, log_level="info")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+230
@@ -0,0 +1,230 @@
|
||||
"""Local-only HTTP workbench and production SPA host."""
|
||||
import json
|
||||
import subprocess
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
from fastapi import FastAPI, HTTPException, Request, UploadFile, File, Form
|
||||
from fastapi.responses import FileResponse, JSONResponse, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from studio.auth import AuthService, CookieVault
|
||||
from studio.database import Database
|
||||
from studio.domain import PROVINCES
|
||||
from studio.imports import import_file, export_sheet
|
||||
from studio.jobs import JobManager
|
||||
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable
|
||||
from studio.settings import SettingsStore
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def create_app(data_dir=None):
|
||||
directory = Path(data_dir or ROOT / ".runtime")
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
db = Database(directory / "studio.db")
|
||||
settings = SettingsStore(directory)
|
||||
router = ProxyRouter(settings)
|
||||
transport = Transport(settings, router)
|
||||
auth = AuthService(transport, CookieVault(directory))
|
||||
jobs = JobManager(db, settings, transport)
|
||||
if (ROOT / "data/schools.json").exists():
|
||||
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
yield
|
||||
jobs.close()
|
||||
|
||||
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
|
||||
app.state.db, app.state.jobs = db, jobs
|
||||
app.state.settings, app.state.router = settings, router
|
||||
|
||||
@app.middleware("http")
|
||||
async def local_access(request: Request, call_next):
|
||||
host = request.url.hostname
|
||||
if host not in ("127.0.0.1", "localhost", "::1"):
|
||||
return JSONResponse({"detail": "仅允许本机访问"}, status_code=403)
|
||||
if request.method not in ("GET", "HEAD", "OPTIONS"):
|
||||
origin = request.headers.get("origin")
|
||||
if origin:
|
||||
parsed = urlparse(origin)
|
||||
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
|
||||
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
|
||||
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
|
||||
return await call_next(request)
|
||||
|
||||
@app.exception_handler(ValueError)
|
||||
async def invalid(request, exc):
|
||||
return JSONResponse({"detail": str(exc)}, status_code=400)
|
||||
|
||||
@app.exception_handler(ProxyUnavailable)
|
||||
@app.exception_handler(RemoteUnavailable)
|
||||
async def remote_error(request, exc):
|
||||
return JSONResponse({"detail": str(exc)}, status_code=503)
|
||||
|
||||
def filters(request):
|
||||
values = dict(request.query_params)
|
||||
for key in ("year", "min_quota", "max_score"):
|
||||
values[key] = int(values[key]) if values.get(key) else None
|
||||
values["favorites"] = values.get("favorites") in ("true", "1")
|
||||
return values
|
||||
|
||||
@app.get("/api/health")
|
||||
def health():
|
||||
return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve())}
|
||||
|
||||
@app.get("/api/overview")
|
||||
def overview():
|
||||
return {**db.overview(), "provinces_list": [{"code": k, "name": v} for k, v in PROVINCES.items()], "sources": [{"name": "研招网", "role": "全国院校 · 专业 · 方向 · 科目", "url": "https://yz.chsi.com.cn/zsml/"}, {"name": "掌上考研", "role": "历年分数线 · 招生计划", "url": "https://www.kaoyan.cn/"}]}
|
||||
|
||||
@app.get("/api/majors")
|
||||
def majors():
|
||||
path = ROOT / "data/majors.json"
|
||||
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else []
|
||||
|
||||
@app.get("/api/schools")
|
||||
def schools(keyword: str = "", province: str = ""):
|
||||
return db.schools(keyword, province)
|
||||
|
||||
@app.get("/api/catalog")
|
||||
def catalog(request: Request, page: int = 1, page_size: int = 30):
|
||||
return db.query_catalog(filters(request), page, page_size)
|
||||
|
||||
@app.get("/api/catalog/{rid}")
|
||||
def detail(rid: int):
|
||||
result = db.history(rid)
|
||||
if result is None:
|
||||
raise HTTPException(404, "记录不存在")
|
||||
return result
|
||||
|
||||
@app.put("/api/catalog/{rid}/favorite")
|
||||
def favorite(rid: int, body: dict):
|
||||
if not db.favorite(rid, bool(body.get("enabled"))):
|
||||
raise HTTPException(404, "记录不存在")
|
||||
return {"ok": True}
|
||||
|
||||
@app.get("/api/jobs")
|
||||
def list_jobs():
|
||||
return jobs.get()
|
||||
|
||||
@app.post("/api/jobs")
|
||||
def start_job(body: dict):
|
||||
return jobs.start(body)
|
||||
|
||||
@app.get("/api/jobs/{jid}/logs")
|
||||
def job_logs(jid: str, after: int = 0):
|
||||
return db.logs(jid, after)
|
||||
|
||||
@app.post("/api/jobs/{jid}/{action}")
|
||||
def job_action(jid: str, action: str):
|
||||
if action not in ("pause", "resume", "cancel"):
|
||||
raise HTTPException(404, "操作不存在")
|
||||
return getattr(jobs, action)(jid)
|
||||
|
||||
@app.get("/api/settings")
|
||||
def get_settings():
|
||||
return settings.read()
|
||||
|
||||
@app.put("/api/settings")
|
||||
def put_settings(body: dict):
|
||||
result = settings.update(body)
|
||||
router.reset()
|
||||
return result
|
||||
|
||||
@app.get("/api/auth")
|
||||
def auth_status():
|
||||
return auth.status()
|
||||
|
||||
@app.post("/api/auth/{action}")
|
||||
def auth_action(action: str, body: dict = {}):
|
||||
if action == "cookie":
|
||||
return auth.cookie(body.get("cookie", ""))
|
||||
if action == "login":
|
||||
return auth.login(body.get("username", ""), body.get("password", ""))
|
||||
if action in ("check", "browser", "clear"):
|
||||
return getattr(auth, action)()
|
||||
raise HTTPException(404, "操作不存在")
|
||||
|
||||
@app.get("/api/proxy")
|
||||
def proxy_status(refresh: bool = False):
|
||||
if refresh:
|
||||
router.refresh()
|
||||
try:
|
||||
router.resolve()
|
||||
except ProxyUnavailable:
|
||||
pass
|
||||
return router.status()
|
||||
|
||||
@app.post("/api/proxy/rotate")
|
||||
def rotate():
|
||||
router.rotate()
|
||||
return router.status()
|
||||
|
||||
@app.post("/api/proxy/manage")
|
||||
def manage(body: dict):
|
||||
action = body.get("action")
|
||||
if action not in ("start", "stop", "status"):
|
||||
raise ValueError("代理池操作无效")
|
||||
script = Path(settings.read()["pool_project"]) / "manage.ps1"
|
||||
if not script.is_file():
|
||||
raise ValueError("代理池目录没有 manage.ps1,请检查配置;也可手动启动原代理池")
|
||||
result = subprocess.run(["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", str(script), action], capture_output=True, timeout=55, creationflags=subprocess.CREATE_NO_WINDOW)
|
||||
output = (result.stdout + result.stderr).decode("utf-8", errors="replace")[-5000:]
|
||||
router.reset()
|
||||
return {"ok": result.returncode == 0, "output": output, "action": action}
|
||||
|
||||
@app.post("/api/import")
|
||||
def upload(file: UploadFile = File(...), year: int | None = Form(None)):
|
||||
suffix = Path(file.filename or "").suffix.lower()
|
||||
folder = directory / "imports"
|
||||
folder.mkdir(exist_ok=True)
|
||||
temporary = folder / (uuid.uuid4().hex + suffix)
|
||||
try:
|
||||
total = 0
|
||||
with temporary.open("wb") as stream:
|
||||
while chunk := file.file.read(1024 * 1024):
|
||||
total += len(chunk)
|
||||
if total > 100 * 1024 * 1024:
|
||||
raise ValueError("一次最多上传 100 MB,请拆分文件")
|
||||
stream.write(chunk)
|
||||
records, scores = import_file(temporary, year)
|
||||
db.upsert_catalog(records)
|
||||
db.upsert_scores(scores)
|
||||
return {"records": len(records), "scores": len(scores), "message": "导入完成,相同记录按年份更新"}
|
||||
except (UnicodeError, json.JSONDecodeError, KeyError) as exc:
|
||||
raise ValueError(f"文件编码或结构无法识别({type(exc).__name__})") from None
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
file.file.close()
|
||||
|
||||
@app.get("/api/export")
|
||||
def export(request: Request, format: str = "csv"):
|
||||
if format not in ("csv", "xlsx", "json"):
|
||||
raise ValueError("导出格式无效")
|
||||
rows = db.all_catalog(filters(request))
|
||||
if format == "json":
|
||||
pairs = {(r["school_code"], r["major_code"]) for r in rows}
|
||||
with db.connection() as conn:
|
||||
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs]
|
||||
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode()
|
||||
mime = "application/json"
|
||||
else:
|
||||
payload = export_sheet(rows, format)
|
||||
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'})
|
||||
|
||||
dist = ROOT / "frontend/dist"
|
||||
if (dist / "assets").exists():
|
||||
app.mount("/assets", StaticFiles(directory=dist / "assets"), name="assets")
|
||||
|
||||
@app.get("/{path:path}")
|
||||
def frontend(path: str):
|
||||
if path.startswith("api/"):
|
||||
raise HTTPException(404, "API 不存在")
|
||||
index = dist / "index.html"
|
||||
if index.exists():
|
||||
return FileResponse(index)
|
||||
return Response("请先运行 frontend 的 npm run build,再启动工作台。", media_type="text/plain", status_code=503)
|
||||
|
||||
return app
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
"""Windows DPAPI protects local CHSI cookies; passwords are never stored."""
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
LOGIN_URL = "https://account.chsi.com.cn/passport/login?entrytype=yzgr&service=https%3A%2F%2Fyz.chsi.com.cn%2Fj_spring_cas_security_check"
|
||||
|
||||
|
||||
class Blob(ctypes.Structure):
|
||||
_fields_ = [("size", ctypes.c_ulong), ("data", ctypes.POINTER(ctypes.c_ubyte))]
|
||||
|
||||
|
||||
def protect(payload, decrypt=False):
|
||||
if os.name != "nt":
|
||||
raise RuntimeError("凭据加密存储需要 Windows DPAPI")
|
||||
memory = ctypes.create_string_buffer(payload)
|
||||
source = Blob(len(payload), ctypes.cast(memory, ctypes.POINTER(ctypes.c_ubyte)))
|
||||
result = Blob()
|
||||
crypt = ctypes.windll.crypt32
|
||||
fn = crypt.CryptUnprotectData if decrypt else crypt.CryptProtectData
|
||||
# UI_FORBIDDEN prevents a modal system prompt in a background job.
|
||||
if not fn(ctypes.byref(source), None, None, None, None, 1, ctypes.byref(result)):
|
||||
raise RuntimeError("Windows 凭据加密失败,请使用当前 Windows 用户重新登录")
|
||||
try:
|
||||
return ctypes.string_at(result.data, result.size)
|
||||
finally:
|
||||
ctypes.windll.kernel32.LocalFree(result.data)
|
||||
|
||||
|
||||
class CookieVault:
|
||||
def __init__(self, directory):
|
||||
self.path = Path(directory) / "chsi-cookie.enc"
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def save(self, cookies):
|
||||
temporary = self.path.with_suffix(".tmp")
|
||||
temporary.write_bytes(protect(json.dumps(cookies, ensure_ascii=False).encode()))
|
||||
temporary.replace(self.path)
|
||||
|
||||
def load(self):
|
||||
if not self.path.exists():
|
||||
return {}
|
||||
return json.loads(protect(self.path.read_bytes(), decrypt=True))
|
||||
|
||||
def clear(self):
|
||||
self.path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
class AuthService:
|
||||
def __init__(self, transport, vault):
|
||||
self.transport, self.vault = transport, vault
|
||||
self.message = "尚未验证登录状态"
|
||||
self.valid = False
|
||||
self.browser_state = "idle"
|
||||
self.browser_message = ""
|
||||
self.browser_thread = None
|
||||
try:
|
||||
self._load(self.vault.load())
|
||||
except RuntimeError:
|
||||
self.message = "保存的 Cookie 无法解密,请重新登录"
|
||||
|
||||
def _load(self, cookies):
|
||||
with self.transport.lock:
|
||||
self.transport.session.cookies.clear()
|
||||
for name, value in cookies.items():
|
||||
self.transport.session.cookies.set(name, value, domain=".chsi.com.cn", path="/")
|
||||
|
||||
def status(self):
|
||||
return {"saved": self.vault.path.exists(), "valid": self.valid, "message": self.message, "browser_state": self.browser_state, "browser_message": self.browser_message}
|
||||
|
||||
def cookie(self, text):
|
||||
if not text.strip() or "\n" in text or "\r" in text:
|
||||
raise ValueError("请粘贴单行 Cookie 请求头")
|
||||
text = text.removeprefix("Cookie:").strip()
|
||||
cookies = {}
|
||||
for token in text.split(";"):
|
||||
if "=" in token:
|
||||
name, value = token.strip().split("=", 1)
|
||||
if name and value:
|
||||
cookies[name] = value
|
||||
if not cookies:
|
||||
raise ValueError("未识别到 Cookie 键值")
|
||||
self._load(cookies)
|
||||
self.vault.save(cookies)
|
||||
self.valid = False
|
||||
self.message = "Cookie 已加密保存,点击验证确认目录访问权限"
|
||||
return self.status()
|
||||
|
||||
def check(self):
|
||||
response = self.transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/dws.do", data={"ssdm": "11", "curPage": 1, "start": 0, "pageSize": 1, "dwlxs[0]": "all"})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
self.valid = False
|
||||
self.message = "目录接口未返回有效数据,可能需要登录或验证码"
|
||||
return self.status()
|
||||
self.valid = body.get("flag") is True and isinstance(body.get("msg"), dict)
|
||||
self.message = "研招目录接口验证成功" if self.valid else "登录未生效或数据源限流,请在浏览器完成登录后重试"
|
||||
if self.valid:
|
||||
self.vault.save(self.transport.session.cookies.get_dict())
|
||||
return self.status()
|
||||
|
||||
def login(self, username, password):
|
||||
if not username or not password:
|
||||
raise ValueError("请输入学信网账号和密码")
|
||||
response = self.transport.request("GET", LOGIN_URL)
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
form = {node.get("name"): node.get("value", "") for node in soup.select("input[type=hidden][name]")}
|
||||
if "lt" not in form or "execution" not in form:
|
||||
raise ValueError("登录页面需要人工验证,请使用浏览器登录")
|
||||
form.update(username=username, password=password, _eventId="submit")
|
||||
self.transport.request("POST", LOGIN_URL, data=form)
|
||||
return self.check()
|
||||
|
||||
def clear(self):
|
||||
self._load({})
|
||||
self.vault.clear()
|
||||
self.valid = False
|
||||
self.message = "已清除本项目保存的登录凭据"
|
||||
return self.status()
|
||||
|
||||
def browser(self):
|
||||
if self.browser_thread and self.browser_thread.is_alive():
|
||||
return self.status()
|
||||
proxy = self.transport.router.resolve()
|
||||
self.browser_state = "opening"
|
||||
self.browser_message = "正在打开独立 Edge 登录窗口,完成登录后可关闭窗口"
|
||||
|
||||
def run():
|
||||
from playwright.sync_api import sync_playwright
|
||||
try:
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(channel="msedge", headless=False, proxy={"server": proxy} if proxy else None)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
page.goto(LOGIN_URL, wait_until="domcontentloaded", timeout=60000)
|
||||
self.browser_state = "waiting"
|
||||
self.browser_message = "请在 Edge 完成学信网登录和验证码;窗口保持打开,工作台自动检测"
|
||||
deadline = time.monotonic() + 300
|
||||
while time.monotonic() < deadline:
|
||||
page.wait_for_timeout(2000)
|
||||
if "yz.chsi.com.cn" in page.url:
|
||||
cookies = {c["name"]: c["value"] for c in context.cookies() if "chsi.com.cn" in c["domain"]}
|
||||
if cookies:
|
||||
self._load(cookies)
|
||||
if self.check()["valid"]:
|
||||
self.browser_state = "completed"
|
||||
self.browser_message = "已获取登录 Cookie,目录接口验证成功"
|
||||
browser.close()
|
||||
return
|
||||
browser.close()
|
||||
self.browser_state = "expired"
|
||||
self.browser_message = "登录等待超时,可再次打开浏览器"
|
||||
except Exception:
|
||||
self.browser_state = "failed"
|
||||
self.browser_message = "登录窗口已关闭或连接失败,可重试或粘贴 Cookie"
|
||||
self.browser_thread = threading.Thread(target=run, daemon=True, name="browser-login")
|
||||
self.browser_thread.start()
|
||||
return self.status()
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Independent adapters for the two projects' public data sources."""
|
||||
import json
|
||||
import re
|
||||
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
||||
from studio.network import RemoteUnavailable
|
||||
|
||||
|
||||
class LoginRequired(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class SourceYearMismatch(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class CatalogClient:
|
||||
BASE = "https://yz.chsi.com.cn/zsml/"
|
||||
|
||||
def __init__(self, transport):
|
||||
self.transport = transport
|
||||
|
||||
def verify_year(self, year):
|
||||
response = self.transport.request("GET", self.BASE)
|
||||
text = re.sub("<[^>]*>", " ", response.text)
|
||||
matches = re.findall(r"(20\d{2})\s*年.{0,30}(?:硕士|招生|专业目录)", text)
|
||||
years = set(map(int, matches))
|
||||
if not years:
|
||||
raise SourceYearMismatch("未能核实研招网当前目录年份,已停止写入;请检查登录及目录开放状态")
|
||||
current = max(years)
|
||||
if int(year) != current:
|
||||
raise SourceYearMismatch(f"研招网当前公开目录为 {current} 年,任务选择了 {year} 年;历史目录请导入已有快照")
|
||||
return current
|
||||
|
||||
def pages(self, endpoint, payload, page_size=50):
|
||||
seen = set()
|
||||
for page in range(1, 10001):
|
||||
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
|
||||
if body.get("flag") is not True:
|
||||
message = str(body.get("msg", ""))
|
||||
if any(token in message for token in ("登录", "登陆", "认证")):
|
||||
raise LoginRequired("研招网要求登录,请更新 Cookie 后恢复任务")
|
||||
if any(token in message for token in ("频繁", "稍后", "验证码")):
|
||||
raise RemoteUnavailable("研招网访问受限,请增加请求间隔并完成人工验证后恢复")
|
||||
raise RemoteUnavailable("研招网目录接口未成功返回数据")
|
||||
data = body.get("msg") or {}
|
||||
if not isinstance(data, dict) or not isinstance(data.get("list"), list):
|
||||
raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据")
|
||||
rows = data["list"]
|
||||
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
||||
if rows and signature in seen:
|
||||
raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页")
|
||||
seen.add(signature)
|
||||
yield from rows
|
||||
more = data.get("nextPageAvailable")
|
||||
total = integer(data.get("totalCount"))
|
||||
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and page * page_size >= total) or len(rows) < page_size):
|
||||
return
|
||||
raise RemoteUnavailable("目录分页超过安全上限")
|
||||
|
||||
def schools(self, province, keyword=""):
|
||||
return self.pages("dws", {"ssdm": province, "dwmc": keyword, "dwdm": "", "xxfs": "", "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"})
|
||||
|
||||
def majors(self, school, major_code="", study_mode=""):
|
||||
return self.pages("dwzys", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major_code, "zymc": "", "mldm": "", "yjxkdm": "", "xwlx": "", "xxfs": study_mode})
|
||||
|
||||
def directions(self, school, major, study_mode=""):
|
||||
return self.pages("yjfxs", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major["zydm"], "zymc": major.get("zymc", ""), "xxfs": study_mode, "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"})
|
||||
|
||||
|
||||
ALIASES = {"湖州师范学院": "湖州师范大学", "闽江学院": "闽江大学", "皖南医学院": "皖南医科大学", "中国航空研究院(613所)": "中国航空研究院", "中国电子科技集团有限公司研究生院": "中国电子科技集团", "湖南理工学院": "湖南理工大学"}
|
||||
|
||||
|
||||
class ScoreClient:
|
||||
BASE = "https://api.kaoyan.cn/pc"
|
||||
|
||||
def __init__(self, transport):
|
||||
self.transport = transport
|
||||
self.cache = {}
|
||||
|
||||
def post(self, endpoint, payload):
|
||||
response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
|
||||
if str(body.get("code")) != "0000":
|
||||
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
|
||||
return body.get("data") or {}
|
||||
|
||||
def pages(self, endpoint, payload, limit=100):
|
||||
seen = set()
|
||||
for page in range(1, 10001):
|
||||
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
|
||||
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data
|
||||
if not isinstance(rows, list):
|
||||
raise RemoteUnavailable("掌上考研响应结构已变化")
|
||||
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
||||
if rows and signature in seen:
|
||||
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
|
||||
seen.add(signature)
|
||||
yield from rows
|
||||
total = integer(data.get("total")) if isinstance(data, dict) else None
|
||||
last = integer(data.get("last_page")) if isinstance(data, dict) else None
|
||||
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total:
|
||||
return
|
||||
raise RemoteUnavailable("历史数据分页超过安全上限")
|
||||
|
||||
def school_id(self, name):
|
||||
key = normalized_name(name)
|
||||
if key in self.cache:
|
||||
return self.cache[key]
|
||||
for query in dict.fromkeys([name, ALIASES.get(key, name)]):
|
||||
matches = [row for row in self.pages("/school/schoolList", {"school_name": query}, limit=10) if normalized_name(row.get("school_name", "")) == normalized_name(query)]
|
||||
if len(matches) == 1 and matches[0].get("school_id"):
|
||||
self.cache[key] = matches[0]["school_id"]
|
||||
return self.cache[key]
|
||||
return None
|
||||
|
||||
def score_lines(self, school, major_code, year, school_id):
|
||||
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
|
||||
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
|
||||
|
||||
def plans(self, school, major_code, year, school_id):
|
||||
result = []
|
||||
for row in self.pages("/school/planListV2", {"school_id": school_id, "year": year, "keyword": major_code}):
|
||||
if str(row.get("special_code", "")) != major_code:
|
||||
continue
|
||||
result.append(finish_record({**school, "year": year, "major_code": major_code, "major_name": row.get("special_name", ""), "department_code": str(row.get("depart_id", "") or ""), "department_name": row.get("depart_name", ""), "direction_code": str(row.get("spe_id", "") or row.get("plan_id", "")), "direction": row.get("research_direction", ""), "quota": integer(row.get("recruit_number")), "quota_text": str(row.get("recruit_number", "") or ""), "quota_scope": "major", "subjects": [], "remarks": row.get("remark", ""), "reference_books": [], "source": "掌上考研", "source_url": "https://www.kaoyan.cn/", "year_verified": True, "raw_evidence": row}))
|
||||
return result
|
||||
@@ -0,0 +1,196 @@
|
||||
"""按年份保存招生目录、来源证据、分数线和任务断点。"""
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from studio.domain import now, normalized_name
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS catalog(
|
||||
id INTEGER PRIMARY KEY, record_key TEXT NOT NULL UNIQUE, year INTEGER NOT NULL,
|
||||
school_code TEXT, school_name TEXT, province TEXT, major_code TEXT, major_name TEXT,
|
||||
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
|
||||
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
|
||||
CREATE TABLE IF NOT EXISTS scores(
|
||||
id INTEGER PRIMARY KEY, school_code TEXT, major_code TEXT, year INTEGER,
|
||||
department_id TEXT, department_name TEXT, scope TEXT, total INTEGER, source TEXT, data TEXT NOT NULL,
|
||||
UNIQUE(school_code,major_code,year,department_id,department_name,scope,source));
|
||||
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
|
||||
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
||||
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
|
||||
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
|
||||
"""
|
||||
|
||||
|
||||
def encode(value):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self, path: Path):
|
||||
self.path = Path(path)
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.connection() as conn:
|
||||
conn.executescript(SCHEMA)
|
||||
conn.execute("UPDATE jobs SET status='paused',message='服务重启,任务可从断点恢复' WHERE status IN ('running','queued')")
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
conn = sqlite3.connect(self.path, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.create_function("norm_name", 1, normalized_name, deterministic=True)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
except BaseException:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def upsert_catalog(self, records: list[dict]):
|
||||
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
|
||||
values = []
|
||||
for record in records:
|
||||
row = {**record, "subjects":" | ".join(record.get("subjects", [])), "data":encode(record)}
|
||||
values.append([row.get(field) for field in fields])
|
||||
if not values:
|
||||
return
|
||||
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
|
||||
def _school(self, conn, record):
|
||||
conn.execute("INSERT INTO schools VALUES(?,?,?,?,?) ON CONFLICT(school_code) DO UPDATE SET school_name=excluded.school_name,province=CASE WHEN excluded.province<>'' THEN excluded.province ELSE schools.province END,level=CASE WHEN excluded.level<>'' THEN excluded.level ELSE schools.level END", (record["school_code"],record["school_name"],record.get("province", ""),record.get("level", ""),encode(record)))
|
||||
|
||||
def index_schools(self, rows):
|
||||
with self.connection() as conn:
|
||||
for row in rows:
|
||||
self._school(conn, row)
|
||||
|
||||
def upsert_scores(self, records):
|
||||
with self.connection() as conn:
|
||||
for row in records:
|
||||
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", ""),row.get("department_name", ""),row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row)))
|
||||
|
||||
def _query(self, filters: dict):
|
||||
conditions, params = ["1=1"], []
|
||||
for field in ["year","province","major_code","study_mode","degree_type","school_code"]:
|
||||
if filters.get(field) not in (None, "", 0):
|
||||
conditions.append(f"c.{field}=?")
|
||||
params.append(filters[field])
|
||||
for field in ["level","subjects","department_name"]:
|
||||
value = filters.get(field) or (filters.get("subject") if field == "subjects" else "")
|
||||
if value:
|
||||
conditions.append(f"c.{field} LIKE ?")
|
||||
params.append(f"%{value}%")
|
||||
if filters.get("direction"):
|
||||
conditions.append("json_extract(c.data,'$.direction') LIKE ?")
|
||||
params.append(f"%{filters['direction']}%")
|
||||
if filters.get("keyword"):
|
||||
conditions.append("(c.school_name LIKE ? OR c.major_name LIKE ? OR c.department_name LIKE ? OR c.major_code LIKE ?)")
|
||||
params.extend([f"%{filters['keyword']}%"] * 4)
|
||||
if filters.get("favorites"):
|
||||
conditions.append("c.favorite=1")
|
||||
if filters.get("min_quota") is not None:
|
||||
conditions.append("c.quota>=?")
|
||||
params.append(filters["min_quota"])
|
||||
for field, index in [("english", 1), ("math", 2)]:
|
||||
if filters.get(field):
|
||||
conditions.append(f"json_extract(c.data,'$.subjects[{index}]') LIKE ?")
|
||||
params.append(str(filters[field])+"%")
|
||||
select = """SELECT c.*,s.data AS score_data,s.total AS score_total,s.year AS score_year,s.scope AS score_scope FROM catalog c LEFT JOIN scores s ON s.id=(SELECT x.id FROM scores x WHERE x.school_code=c.school_code AND x.major_code=c.major_code AND x.year<=c.year AND (x.department_name='' AND x.department_id='' OR x.department_id<>'' AND x.department_id=c.department_code OR x.department_name<>'' AND norm_name(x.department_name)=norm_name(c.department_name)) AND coalesce(json_extract(x.data,'$.note'),'') NOT LIKE '%专项%' AND coalesce(json_extract(x.data,'$.note'),'') NOT LIKE '%少干%' ORDER BY x.year DESC,CASE x.scope WHEN 'major' THEN 0 WHEN 'discipline' THEN 1 ELSE 2 END,x.id DESC LIMIT 1) WHERE """ + " AND ".join(conditions)
|
||||
if filters.get("max_score") is not None:
|
||||
select = "SELECT * FROM ("+select+") q WHERE q.score_total<=?"
|
||||
params.append(filters["max_score"])
|
||||
return select, params
|
||||
|
||||
def query_catalog(self, filters: dict, page=1, page_size=50):
|
||||
select, params = self._query(filters)
|
||||
page, page_size = max(1,int(page)), max(1,min(int(page_size),200))
|
||||
with self.connection() as conn:
|
||||
total = conn.execute("SELECT count(*) FROM ("+select+")", params).fetchone()[0]
|
||||
order = {"school":"school_name,major_code,department_name,id", "quota":"quota IS NULL,quota DESC,school_name,id", "score":"score_total IS NULL,score_total,school_name,id", "year":"year DESC,school_name,major_code,department_name,id"}.get(filters.get("sort"), "year DESC,school_name,major_code,department_name,id")
|
||||
rows = conn.execute(select+" ORDER BY "+order+" LIMIT ? OFFSET ?",[*params,page_size,(page-1)*page_size]).fetchall()
|
||||
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
|
||||
|
||||
def all_catalog(self, filters):
|
||||
select, params = self._query(filters)
|
||||
with self.connection() as conn:
|
||||
return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)]
|
||||
|
||||
def _catalog_row(self, row):
|
||||
data = json.loads(row["data"])
|
||||
return {**data,"id":row["id"],"favorite":bool(row["favorite"]),"score_total":row["score_total"],"score_year":row["score_year"],"score_scope":row["score_scope"],"score":json.loads(row["score_data"]) if row["score_data"] else None}
|
||||
|
||||
def record(self, rid):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute("SELECT data,favorite FROM catalog WHERE id=?",(rid,)).fetchone()
|
||||
return {**json.loads(row["data"]),"id":rid,"favorite":bool(row["favorite"])} if row else None
|
||||
|
||||
def favorite(self, rid, enabled):
|
||||
with self.connection() as conn:
|
||||
return conn.execute("UPDATE catalog SET favorite=? WHERE id=?",(int(enabled),rid)).rowcount > 0
|
||||
|
||||
def history(self, rid):
|
||||
record = self.record(rid)
|
||||
if record is None:
|
||||
return None
|
||||
with self.connection() as conn:
|
||||
scores = conn.execute("SELECT data FROM scores WHERE school_code=? AND major_code=? ORDER BY year,scope,department_name",(record["school_code"],record["major_code"])).fetchall()
|
||||
plans = conn.execute("SELECT data FROM catalog WHERE school_code=? AND major_code=? AND norm_name(department_name)=norm_name(?) ORDER BY year,id",(record["school_code"],record["major_code"],record["department_name"])).fetchall()
|
||||
def matches(row):
|
||||
return not row.get("department_name") and not row.get("department_id") or row.get("department_id") and row["department_id"] == record["department_code"] or row.get("department_name") and normalized_name(row["department_name"]) == normalized_name(record["department_name"])
|
||||
return {"record":record,"scores":[row for entry in scores if matches(row:=json.loads(entry["data"]))],"plans":[json.loads(row["data"]) for row in plans]}
|
||||
|
||||
def overview(self):
|
||||
with self.connection() as conn:
|
||||
totals = dict(conn.execute("SELECT count(*) AS records,count(DISTINCT school_code) AS schools,count(DISTINCT major_code) AS majors,sum(favorite) AS favorites FROM catalog").fetchone())
|
||||
totals["indexed_schools"] = conn.execute("SELECT count(*) FROM schools").fetchone()[0]
|
||||
totals["scores"] = conn.execute("SELECT count(*) FROM scores").fetchone()[0]
|
||||
totals["years"] = [row[0] for row in conn.execute("SELECT DISTINCT year FROM catalog ORDER BY year DESC")]
|
||||
totals["provinces"] = [dict(row) for row in conn.execute("SELECT province,count(DISTINCT school_code) AS count FROM catalog WHERE province<>'' GROUP BY province ORDER BY count DESC LIMIT 10")]
|
||||
return totals
|
||||
|
||||
def schools(self, keyword="", province=""):
|
||||
with self.connection() as conn:
|
||||
return [dict(row) for row in conn.execute("SELECT school_code,school_name,province,level FROM schools WHERE school_name LIKE ? AND (?='' OR province=?) ORDER BY school_name LIMIT 2000",(f"%{keyword}%",province,province))]
|
||||
|
||||
def job_create(self, jid, params):
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT INTO jobs(id,params,status,created_at,updated_at,message) VALUES(?,?,'queued',?,?,?)",(jid,encode(params),now(),now(),"等待开始"))
|
||||
|
||||
def job_update(self, jid, **changes):
|
||||
allowed = {"status","message","errors","records"}
|
||||
changes = {key:value for key,value in changes.items() if key in allowed}
|
||||
changes["updated_at"] = now()
|
||||
with self.connection() as conn:
|
||||
conn.execute("UPDATE jobs SET "+",".join(f"{key}=?" for key in changes)+" WHERE id=?",[*changes.values(),jid])
|
||||
|
||||
def job_get(self, jid=None):
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute("SELECT j.*,(SELECT count(*) FROM job_units u WHERE u.job_id=j.id) AS completed_units FROM jobs j "+("WHERE j.id=?" if jid else "ORDER BY j.created_at DESC LIMIT 100"), (jid,) if jid else ()).fetchall()
|
||||
result = [{**dict(row),"params":json.loads(row["params"])} for row in rows]
|
||||
return (result[0] if result else None) if jid else result
|
||||
|
||||
def unit_done(self, jid, unit):
|
||||
with self.connection() as conn:
|
||||
return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None
|
||||
|
||||
def complete_unit(self, jid, unit):
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
|
||||
|
||||
def log(self, jid, level, message):
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
|
||||
|
||||
def logs(self, jid, after=0):
|
||||
with self.connection() as conn:
|
||||
return [dict(row) for row in conn.execute("SELECT * FROM job_logs WHERE job_id=? AND id>? ORDER BY id DESC LIMIT 300",(jid,after))][::-1]
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
"""招生目录与分数线规范化;来源口径和未知值始终保留。"""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
PROVINCES = {"11":"北京","12":"天津","13":"河北","14":"山西","15":"内蒙古","21":"辽宁","22":"吉林","23":"黑龙江","31":"上海","32":"江苏","33":"浙江","34":"安徽","35":"福建","36":"江西","37":"山东","41":"河南","42":"湖北","43":"湖南","44":"广东","45":"广西","46":"海南","50":"重庆","51":"四川","52":"贵州","53":"云南","54":"西藏","61":"陕西","62":"甘肃","63":"青海","64":"宁夏","65":"新疆"}
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return datetime.now(timezone(timedelta(hours=8))).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def normalized_name(value: str) -> str:
|
||||
return str(value or "").replace("(", "(").replace(")", ")").replace(" ", "").strip()
|
||||
|
||||
|
||||
def integer(value: Any) -> int | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return int(text) if re.fullmatch(r"\d+", text) else None
|
||||
|
||||
|
||||
def parse_quota(text: str) -> tuple[int | None, str]:
|
||||
text = str(text or "").strip()
|
||||
match = re.search(r"专业\s*[::]\s*(\d+)", text)
|
||||
if match and not re.match(r"\s*[-—~至或]", text[match.end():]):
|
||||
return int(match.group(1)), "major"
|
||||
if "学院" in text or "院系" in text:
|
||||
return None, "college"
|
||||
if "方向" in text:
|
||||
return None, "direction"
|
||||
if re.fullmatch(r"\d+", text):
|
||||
return int(text), "unspecified"
|
||||
return None, "unknown"
|
||||
|
||||
|
||||
def record_key(record: dict) -> str:
|
||||
fields = ["year","school_code","school_name","major_code","department_code","department_name","direction_code","direction","study_mode","exam_type","subjects","special_plans","source"]
|
||||
serialized = json.dumps([record.get(key, "") for key in fields], ensure_ascii=False, sort_keys=True)
|
||||
return hashlib.sha256(serialized.encode()).hexdigest()
|
||||
|
||||
|
||||
def finish_record(record: dict) -> dict:
|
||||
defaults = {"school_code":"", "school_name":"", "province":"", "major_code":"", "major_name":"", "department_code":"", "department_name":"", "direction_code":"", "direction":"", "study_mode":"", "degree_type":"", "exam_type":"", "advisor":"", "quota_text":"", "quota":None, "quota_scope":"unknown", "subjects":[], "special_plans":[], "level":"", "remarks":"", "source":"", "source_url":"", "fetched_at":now(), "year_verified":False}
|
||||
row = {**defaults, **record}
|
||||
if not row["school_code"]:
|
||||
row["school_code"] = "name:" + normalized_name(row["school_name"])
|
||||
row["record_key"] = record_key(row)
|
||||
return row
|
||||
|
||||
|
||||
def normalize_catalog(item: dict, year: int, school: dict | None = None, major: dict | None = None) -> list[dict]:
|
||||
item = {**(school or {}), **(major or {}), **item}
|
||||
quota, scope = parse_quota(item.get("nzsrsstr", ""))
|
||||
if not item.get("nzsrsstr") and integer(item.get("nzsrs")) not in (None, 0):
|
||||
quota, scope = integer(item["nzsrs"]), "unspecified"
|
||||
labels = []
|
||||
if str(item.get("syl", "")) == "1":
|
||||
labels.append("双一流")
|
||||
base = {
|
||||
"year": year, "school_code": str(item.get("dwdm", "")), "school_name": item.get("dwmc", ""),
|
||||
"province": item.get("szss") or PROVINCES.get(str(item.get("ssdm", item.get("szssm", ""))), ""),
|
||||
"major_code": str(item.get("zydm", "")), "major_name": item.get("zymc", ""),
|
||||
"department_code": str(item.get("yxsdm", "")), "department_name": item.get("yxsmc", ""),
|
||||
"direction_code": str(item.get("yjfxdm", "")), "direction": item.get("yjfxmc", ""),
|
||||
"study_mode": {"1":"全日制", "2":"非全日制"}.get(str(item.get("xxfs", "")), ""),
|
||||
"degree_type": item.get("xwlxmc", ""), "exam_type": item.get("ksfsmc", ""),
|
||||
"advisor": item.get("zdjs", ""), "quota_text": item.get("nzsrsstr", ""), "quota": quota, "quota_scope": scope,
|
||||
"special_plans": [label for key,label in [("tydxs","退役大学生士兵"),("jsggjh","少数民族骨干")] if str(item.get(key, "")) == "1"],
|
||||
"level": "/".join(labels), "remarks": item.get("bz", item.get("zybz", "")),
|
||||
"source":"研招网", "source_url":"https://yz.chsi.com.cn/zsml/zydetail.do?" + urlencode({"zydm": item.get("zydm", ""), "zymc": item.get("zymc", "")}),
|
||||
}
|
||||
combinations = item.get("kskmz") or [{}]
|
||||
result = []
|
||||
for combination in combinations:
|
||||
subjects = []
|
||||
for number in range(1, 5):
|
||||
subject = combination.get(f"km{number}Vo") or {}
|
||||
name = str(subject.get("kskmmc", "") or "")
|
||||
code = str(subject.get("kskmdm", "") or "")
|
||||
subjects.append((code if code and not name.startswith(code) else "") + name)
|
||||
result.append(finish_record({**base, "subjects":subjects}))
|
||||
return result
|
||||
|
||||
|
||||
def select_score_lines(items: list[dict], major_code: str, school_code: str, school_name: str, year: int) -> list[dict]:
|
||||
levels = {6:[], 4:[], 2:[]}
|
||||
for item in items:
|
||||
code = str(item.get("code", "")).strip()
|
||||
if len(code) in levels and (code == major_code or len(code) < 6 and major_code.startswith(code)):
|
||||
levels[len(code)].append(item)
|
||||
result = []
|
||||
for size in (6, 4, 2):
|
||||
if not levels[size]:
|
||||
continue
|
||||
for item in levels[size]:
|
||||
result.append({
|
||||
"school_code":school_code, "school_name":school_name, "major_code":major_code, "year":year,
|
||||
"department_id":str(item.get("depart_id", "") or ""), "department_name":item.get("depart_name", "") or "",
|
||||
"scope":{6:"major",4:"discipline",2:"category"}[size], "raw_code":str(item.get("code", "")),
|
||||
"total":integer(item.get("total")), "politics":integer(item.get("politics")), "english":integer(item.get("english")),
|
||||
"special_one":integer(item.get("special_one")), "special_two":integer(item.get("special_two")),
|
||||
"note":item.get("note", "") or "", "source":"掌上考研", "source_url":"https://www.kaoyan.cn/", "fetched_at":now(), "raw_evidence":item,
|
||||
})
|
||||
break
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Exchange formats for yzw CSV, YAM SQLite and studio snapshots."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from openpyxl import load_workbook, Workbook
|
||||
from studio.domain import finish_record, integer, parse_quota
|
||||
|
||||
LABELS = {"year": "年份", "school_name": "院校", "school_code": "院校代码", "province": "省份", "level": "院校层次", "major_code": "专业代码", "major_name": "专业", "degree_type": "学位类型", "department_name": "院系", "direction": "研究方向", "study_mode": "学习方式", "exam_type": "考试方式", "advisor": "导师", "quota_text": "招生人数原文", "quota": "明确人数", "quota_scope": "人数口径", "subjects": "考试科目", "special_plans": "专项计划", "score_year": "参考线年份", "score_total": "参考线总分", "score_scope": "分数线口径", "remarks": "备注", "source": "来源", "source_url": "来源链接", "year_verified": "来源年份已核实", "fetched_at": "采集时间"}
|
||||
ALIASES = {"院校名称": "school_name", "学校名称": "school_name", "专业名称": "major_name", "院系所": "department_name", "拟招生人数": "quota_text", "planned_enrollment": "quota_text", "nzsrsstr": "quota_text", "school_level": "level", "department": "department_name", "research_direction": "direction", "enrollment_text": "quota_text", "enrollment_count": "quota", "exam_subjects": "subjects"}
|
||||
ALIASES.update({v: k for k, v in LABELS.items()})
|
||||
|
||||
|
||||
def list_field(value):
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
if not value:
|
||||
return []
|
||||
if str(value).startswith("["):
|
||||
try:
|
||||
decoded = json.loads(value)
|
||||
if isinstance(decoded, list):
|
||||
return decoded
|
||||
except ValueError:
|
||||
pass
|
||||
return str(value).split(" | ")
|
||||
|
||||
|
||||
def row_record(raw, year=None, source="文件导入"):
|
||||
row = {ALIASES.get(str(k).strip(), str(k).strip()): (v if v is not None else "") for k, v in raw.items() if k is not None}
|
||||
explicit_year = integer(row.get("year")) or integer(year)
|
||||
if explicit_year is None:
|
||||
raise ValueError("文件未包含年份,请明确指定这份数据的招生年份")
|
||||
if not 2000 <= explicit_year <= 2100:
|
||||
raise ValueError("文件年份无效")
|
||||
if not row.get("school_name") or not row.get("major_code"):
|
||||
raise ValueError("文件需包含院校名称和专业代码")
|
||||
row["major_code"] = str(row["major_code"]).strip().zfill(6)
|
||||
if not row["major_code"].isdigit() or len(row["major_code"]) != 6:
|
||||
raise ValueError("文件专业代码应为 6 位数字")
|
||||
row["school_code"] = str(row.get("school_code", "")).strip()
|
||||
subjects = list_field(row.get("subjects"))
|
||||
if not subjects:
|
||||
subjects = [str(row.get(f"exam_subject{i}") or row.get(f"subject{i}") or row.get(f"考试科目{i}") or "") for i in range(1, 5)]
|
||||
quota, scope = parse_quota(row.get("quota_text", ""))
|
||||
if quota is None and row.get("quota") != "":
|
||||
quota = integer(row.get("quota"))
|
||||
scope = row.get("quota_scope") or "unspecified"
|
||||
plans = list_field(row.get("special_plans"))
|
||||
if row.get("veteran_program") in ("是", "1", 1):
|
||||
plans.append("退役大学生士兵")
|
||||
if row.get("shaogu_program") in ("是", "1", 1):
|
||||
plans.append("少数民族骨干")
|
||||
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
|
||||
|
||||
|
||||
def import_file(path, year=None):
|
||||
path = Path(path)
|
||||
suffix = path.suffix.lower()
|
||||
scores = []
|
||||
if suffix in (".db", ".sqlite", ".sqlite3"):
|
||||
return import_yam(path, year)
|
||||
if suffix == ".json":
|
||||
content = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if isinstance(content, dict):
|
||||
rows = content.get("catalog", content.get("records", []))
|
||||
scores = content.get("scores", [])
|
||||
else:
|
||||
rows = content
|
||||
if not isinstance(rows, list) or not isinstance(scores, list):
|
||||
raise ValueError("JSON 应是记录数组或包含 catalog/scores 的工作台快照")
|
||||
elif suffix == ".csv":
|
||||
payload = path.read_bytes()
|
||||
try:
|
||||
text = payload.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
text = payload.decode("gb18030")
|
||||
rows = list(csv.DictReader(io.StringIO(text)))
|
||||
elif suffix == ".xlsx":
|
||||
book = load_workbook(path, read_only=True, data_only=True)
|
||||
try:
|
||||
sheet = book.active
|
||||
if sheet.max_row > 300001 or sheet.max_column > 100:
|
||||
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
|
||||
values = sheet.iter_rows(values_only=True)
|
||||
headers = next(values, [])
|
||||
rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)]
|
||||
finally:
|
||||
book.close()
|
||||
else:
|
||||
raise ValueError("支持 CSV、XLSX、JSON 和 YAM SQLite 文件")
|
||||
if len(rows) > 300000:
|
||||
raise ValueError("一次最多导入 30 万条,请拆分文件")
|
||||
records = [row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入") for row in rows]
|
||||
for score in scores:
|
||||
if not all(k in score for k in ("school_code", "major_code", "year", "scope")):
|
||||
raise ValueError("快照分数线缺少院校/专业/年份/口径")
|
||||
score["year"] = int(score["year"])
|
||||
score["total"] = integer(score.get("total"))
|
||||
score.setdefault("note", "")
|
||||
return records, scores
|
||||
|
||||
|
||||
def import_yam(path, year):
|
||||
conn = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
conn.execute("PRAGMA query_only=ON")
|
||||
tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
if not {"schools", "departments"}.issubset(tables):
|
||||
raise ValueError("此 SQLite 不包含 YAM schools/departments 表;请从原软件导出 CSV/JSON")
|
||||
schools = [dict(row) for row in conn.execute("SELECT * FROM schools")]
|
||||
lookup = {(str(r["school_id"]), str(r.get("major_code", ""))): r for r in schools}
|
||||
records, departments = [], {}
|
||||
for entry in conn.execute("SELECT * FROM departments"):
|
||||
row = dict(entry)
|
||||
school = lookup.get((str(row["school_id"]), str(row.get("major_code", "")))) or next((r for r in schools if str(r["school_id"]) == str(row["school_id"])), {})
|
||||
department = str(row.get("department_id", ""))
|
||||
departments[(str(row["school_id"]), department)] = row.get("name", "")
|
||||
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "level": school.get("level", ""), "department_code": department, "department_name": row.get("name", ""), "source": row.get("source") or "YAM 导入"}, year))
|
||||
scores = []
|
||||
if "score_lines" in tables:
|
||||
for entry in conn.execute("SELECT * FROM score_lines"):
|
||||
row = dict(entry)
|
||||
if not integer(row.get("year")):
|
||||
continue
|
||||
raw = str(row.get("raw_code", ""))
|
||||
scope = row.get("match_scope") or ({2: "category", 4: "discipline", 6: "major"}.get(len(raw)) if raw else "unknown")
|
||||
scores.append({**row, "school_code": str(row["school_id"]), "major_code": str(row.get("major_code", "")), "department_id": str(row.get("department_id", "") or ""), "department_name": row.get("raw_department_name") or departments.get((str(row["school_id"]), str(row.get("department_id", ""))), ""), "scope": scope, "note": row.get("note") or "", "source": row.get("source") or "YAM 导入"})
|
||||
if "admission_plans" in tables:
|
||||
for entry in conn.execute("SELECT * FROM admission_plans"):
|
||||
row = dict(entry)
|
||||
school = lookup.get((str(row["school_id"]), str(row.get("major_code", ""))), {})
|
||||
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "direction": row.get("research_direction", ""), "direction_code": str(row.get("spe_id") or row.get("plan_id") or ""), "department_code": str(row.get("department_id", "") or ""), "source": row.get("source") or "YAM 导入", "remarks": row.get("note", "")}, year))
|
||||
return records, scores
|
||||
except sqlite3.DatabaseError:
|
||||
raise ValueError("SQLite 文件无法读取,请确认是完整导出的数据库") from None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def spreadsheet_value(value):
|
||||
if isinstance(value, (list, dict)):
|
||||
value = " | ".join(str(v) for v in value) if isinstance(value, list) else json.dumps(value, ensure_ascii=False)
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str) and value.lstrip().startswith(("=", "+", "-", "@", "\t", "\r")):
|
||||
return "'" + value
|
||||
return value
|
||||
|
||||
|
||||
def export_sheet(records, kind):
|
||||
rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records]
|
||||
if kind == "csv":
|
||||
stream = io.StringIO(newline="")
|
||||
writer = csv.writer(stream)
|
||||
writer.writerow(list(LABELS.values()))
|
||||
writer.writerows(rows)
|
||||
return stream.getvalue().encode("utf-8-sig")
|
||||
book = Workbook()
|
||||
sheet = book.active
|
||||
sheet.title = "院校专业"
|
||||
sheet.append(list(LABELS.values()))
|
||||
for row in rows:
|
||||
sheet.append(row)
|
||||
sheet.freeze_panes = "A2"
|
||||
sheet.auto_filter.ref = sheet.dimensions
|
||||
from openpyxl.styles import Font, PatternFill, Alignment
|
||||
for cell in sheet[1]:
|
||||
cell.font = Font(color="FFFFFF", bold=True)
|
||||
cell.fill = PatternFill("solid", fgColor="183447")
|
||||
cell.alignment = Alignment(vertical="center")
|
||||
for col in sheet.columns:
|
||||
sheet.column_dimensions[col[0].column_letter].width = 24
|
||||
output = io.BytesIO()
|
||||
book.save(output)
|
||||
return output.getvalue()
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
"""Persistent units and cooperative pause/cancel for one local worker."""
|
||||
import threading
|
||||
import uuid
|
||||
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
|
||||
from studio.domain import PROVINCES, normalize_catalog
|
||||
from studio.network import ProxyUnavailable, RemoteUnavailable
|
||||
|
||||
|
||||
class Cancelled(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class JobManager:
|
||||
def __init__(self, database, settings, transport=None, runner=None):
|
||||
self.db, self.settings, self.transport = database, settings, transport
|
||||
self.runner = runner or self._collect
|
||||
self.controls = {}
|
||||
self.lock = threading.RLock()
|
||||
self.closed = False
|
||||
|
||||
def get(self, jid=None):
|
||||
return self.db.job_get(jid)
|
||||
|
||||
def start(self, params):
|
||||
params = {**params}
|
||||
if params.get("mode") not in ("catalog", "scores", "combined"):
|
||||
raise ValueError("请选择有效的采集模式")
|
||||
if not 2000 <= int(params.get("year", 0)) <= 2100:
|
||||
raise ValueError("请选择目录年份")
|
||||
if params.get("major_code") and (not str(params["major_code"]).isdigit() or len(str(params["major_code"])) != 6):
|
||||
raise ValueError("专业代码应为 6 位数字")
|
||||
provinces = params.get("provinces") or list(PROVINCES)
|
||||
if any(str(p) not in PROVINCES for p in provinces):
|
||||
raise ValueError("省份代码无效")
|
||||
params["provinces"] = list(map(str, provinces))
|
||||
params["score_years"] = sorted(set(int(y) for y in params.get("score_years", [])))
|
||||
if any(not 2000 <= y <= 2100 for y in params["score_years"]):
|
||||
raise ValueError("历史年份无效")
|
||||
if params["mode"] in ("combined", "scores") and not params["score_years"]:
|
||||
raise ValueError("请至少选择一个分数线年份")
|
||||
with self.lock:
|
||||
self._ensure_available()
|
||||
jid = uuid.uuid4().hex
|
||||
self.db.job_create(jid, params)
|
||||
self._launch(jid)
|
||||
return self.get(jid)
|
||||
|
||||
def _ensure_available(self, same=None):
|
||||
if self.closed:
|
||||
raise ValueError("任务服务正在关闭")
|
||||
for jid, control in self.controls.items():
|
||||
if jid != same and control["thread"].is_alive():
|
||||
raise ValueError("已有正在运行或暂停的任务,请完成或取消后再新建")
|
||||
|
||||
def _launch(self, jid):
|
||||
condition = threading.Condition()
|
||||
control = {"condition": condition, "paused": False, "cancelled": False}
|
||||
thread = threading.Thread(target=self._run, args=(jid,), daemon=True, name="collector-" + jid[:8])
|
||||
control["thread"] = thread
|
||||
self.controls[jid] = control
|
||||
thread.start()
|
||||
|
||||
def _run(self, jid):
|
||||
job = self.get(jid)
|
||||
self.db.job_update(jid, status="running", errors=0, message="正在采集")
|
||||
self.db.log(jid, "info", "任务开始/恢复,将跳过已完成单元")
|
||||
if self.transport:
|
||||
self.transport.local.guard = lambda: self.checkpoint(jid)
|
||||
try:
|
||||
self.runner(jid, job["params"], self)
|
||||
self.checkpoint(jid)
|
||||
errors = self.get(jid)["errors"]
|
||||
self.db.job_update(jid, status="partial" if errors else "completed", message="部分单元失败,可恢复重试" if errors else "采集完成")
|
||||
except Cancelled:
|
||||
self.db.job_update(jid, status="cancelled", message="已取消,已保存数据保留")
|
||||
except LoginRequired as exc:
|
||||
self.db.job_update(jid, status="waiting_login", message=str(exc))
|
||||
self.db.log(jid, "warning", str(exc))
|
||||
except ProxyUnavailable as exc:
|
||||
self.db.job_update(jid, status="waiting_proxy", message=str(exc))
|
||||
self.db.log(jid, "warning", str(exc))
|
||||
except (RemoteUnavailable, SourceYearMismatch, ValueError) as exc:
|
||||
self.db.job_update(jid, status="failed", message=str(exc))
|
||||
self.db.log(jid, "error", str(exc))
|
||||
except Exception as exc:
|
||||
self.db.job_update(jid, status="failed", message=f"采集异常({type(exc).__name__}),已保存断点")
|
||||
self.db.log(jid, "error", f"采集异常类型:{type(exc).__name__}")
|
||||
finally:
|
||||
if self.transport:
|
||||
self.transport.local.guard = None
|
||||
|
||||
def checkpoint(self, jid):
|
||||
control = self.controls.get(jid)
|
||||
if not control:
|
||||
return
|
||||
with control["condition"]:
|
||||
while control["paused"] and not control["cancelled"]:
|
||||
control["condition"].wait(.25)
|
||||
if control["cancelled"]:
|
||||
raise Cancelled()
|
||||
|
||||
def unit_done(self, jid, unit):
|
||||
return self.db.unit_done(jid, unit)
|
||||
|
||||
def complete_unit(self, jid, unit):
|
||||
self.db.complete_unit(jid, unit)
|
||||
|
||||
def pause(self, jid):
|
||||
control = self.controls.get(jid)
|
||||
if not control or not control["thread"].is_alive():
|
||||
raise ValueError("任务当前不在运行")
|
||||
with control["condition"]:
|
||||
control["paused"] = True
|
||||
self.db.job_update(jid, status="paused", message="已暂停,当前请求结束后停止")
|
||||
return self.get(jid)
|
||||
|
||||
def resume(self, jid):
|
||||
with self.lock:
|
||||
job = self.get(jid)
|
||||
if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy"):
|
||||
raise ValueError("此任务当前不能恢复")
|
||||
self._ensure_available(jid)
|
||||
control = self.controls.get(jid)
|
||||
if control and control["thread"].is_alive():
|
||||
with control["condition"]:
|
||||
self.db.job_update(jid, status="running", message="正在继续采集")
|
||||
control["paused"] = False
|
||||
control["condition"].notify_all()
|
||||
else:
|
||||
self._launch(jid)
|
||||
return self.get(jid)
|
||||
|
||||
def cancel(self, jid):
|
||||
if not self.get(jid):
|
||||
raise ValueError("任务不存在")
|
||||
control = self.controls.get(jid)
|
||||
if control and control["thread"].is_alive():
|
||||
with control["condition"]:
|
||||
control["cancelled"] = True
|
||||
control["condition"].notify_all()
|
||||
self.db.job_update(jid, message="正在取消,等待当前请求结束")
|
||||
else:
|
||||
self.db.job_update(jid, status="cancelled", message="已取消")
|
||||
return self.get(jid)
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
for jid, control in list(self.controls.items()):
|
||||
if control["thread"].is_alive():
|
||||
self.cancel(jid)
|
||||
for control in list(self.controls.values()):
|
||||
control["thread"].join(timeout=1)
|
||||
|
||||
def _save(self, jid, records):
|
||||
self.db.upsert_catalog(records)
|
||||
self.db.job_update(jid, records=self.get(jid)["records"] + len(records))
|
||||
|
||||
def _error(self, jid, message):
|
||||
self.db.job_update(jid, errors=self.get(jid)["errors"] + 1)
|
||||
self.db.log(jid, "error", message)
|
||||
|
||||
def _collect(self, jid, params, manager):
|
||||
if params["mode"] in ("catalog", "combined"):
|
||||
self._catalog(jid, params)
|
||||
if params["mode"] in ("scores", "combined"):
|
||||
self._scores(jid, params)
|
||||
|
||||
def _catalog(self, jid, params):
|
||||
client = CatalogClient(self.transport)
|
||||
client.verify_year(params["year"])
|
||||
for province in params["provinces"]:
|
||||
self.checkpoint(jid)
|
||||
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
|
||||
for school in client.schools(province, params.get("school_keyword", "")):
|
||||
self.checkpoint(jid)
|
||||
school = {"ssdm": province, **school}
|
||||
for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")):
|
||||
if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]):
|
||||
continue
|
||||
unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}"
|
||||
if self.unit_done(jid, unit):
|
||||
continue
|
||||
self.checkpoint(jid)
|
||||
label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}"
|
||||
self.db.job_update(jid, message=label)
|
||||
try:
|
||||
records = []
|
||||
for row in client.directions(school, major, params.get("study_mode", "")):
|
||||
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
|
||||
self._save(jid, records)
|
||||
self.complete_unit(jid, unit)
|
||||
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
|
||||
except RemoteUnavailable as exc:
|
||||
self._error(jid, f"{label}:{exc}")
|
||||
|
||||
def _scores(self, jid, params):
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "keyword": params.get("school_keyword", "")}
|
||||
records = self.db.all_catalog(filters)
|
||||
targets = {}
|
||||
for row in records:
|
||||
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]:
|
||||
continue
|
||||
targets[(row["school_code"], row["major_code"])] = row
|
||||
if not targets and params.get("major_code"):
|
||||
for row in self.db.schools(params.get("school_keyword", "")):
|
||||
if row["province"] in [PROVINCES[p] for p in params["provinces"]]:
|
||||
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
|
||||
if not targets:
|
||||
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
|
||||
client = ScoreClient(self.transport)
|
||||
for (_, code), record in targets.items():
|
||||
self.checkpoint(jid)
|
||||
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
|
||||
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")]
|
||||
if not years:
|
||||
continue
|
||||
school_id = client.school_id(school["school_name"])
|
||||
if school_id is None:
|
||||
self._error(jid, school["school_name"] + ":掌上考研未找到精确匹配,已跳过")
|
||||
continue
|
||||
for year in years:
|
||||
self.checkpoint(jid)
|
||||
label = f"{school['school_name']} · {code} · {year} 年"
|
||||
self.db.job_update(jid, message=label)
|
||||
try:
|
||||
scores = client.score_lines(school, code, year, school_id)
|
||||
self.db.upsert_scores(scores)
|
||||
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
|
||||
self._save(jid, plans)
|
||||
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}")
|
||||
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
|
||||
except RemoteUnavailable as exc:
|
||||
self._error(jid, f"{label}:{exc}")
|
||||
@@ -0,0 +1,149 @@
|
||||
"""One explicit route for CHSI login, catalog and historical data."""
|
||||
import threading
|
||||
import time
|
||||
import requests
|
||||
from studio.settings import proxy_url
|
||||
|
||||
|
||||
class ProxyUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class RemoteUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ProxyRouter:
|
||||
def __init__(self, settings, session=None):
|
||||
self.settings = settings
|
||||
self.session = session or requests.Session()
|
||||
self.session.trust_env = False
|
||||
self.active = None
|
||||
self.failed = set()
|
||||
self.candidates = []
|
||||
self.last_error = ""
|
||||
self.counts = {}
|
||||
self.fallback = False
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def refresh(self):
|
||||
with self.lock:
|
||||
config = self.settings.read()
|
||||
try:
|
||||
response = self.session.get(config["pool_url"] + "/all/", params={"type": "https"}, timeout=5)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if not isinstance(payload, list):
|
||||
raise ValueError("代理池返回格式不正确")
|
||||
proxies = []
|
||||
for entry in payload:
|
||||
# The pool already filters HTTPS CONNECT-capable entries.
|
||||
value = entry.get("proxy") if isinstance(entry, dict) else entry
|
||||
if value:
|
||||
try:
|
||||
proxies.append(proxy_url(value))
|
||||
except ValueError:
|
||||
continue
|
||||
self.last_error = ""
|
||||
try:
|
||||
self.counts = self.session.get(config["pool_url"] + "/count/", timeout=5).json()
|
||||
except (requests.RequestException, ValueError):
|
||||
self.counts = {"https_available": len(proxies)}
|
||||
except (requests.RequestException, ValueError):
|
||||
proxies = []
|
||||
self.last_error = "无法连接代理池 API,请检查地址和服务状态"
|
||||
self.counts = {}
|
||||
self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"]))
|
||||
return self.status()
|
||||
|
||||
def resolve(self):
|
||||
with self.lock:
|
||||
config = self.settings.read()
|
||||
if config["network_mode"] == "direct":
|
||||
self.fallback = False
|
||||
return None
|
||||
if self.active and self.active not in self.failed:
|
||||
self.fallback = False
|
||||
return self.active
|
||||
available = [p for p in self.candidates if p not in self.failed]
|
||||
if not available:
|
||||
self.refresh()
|
||||
available = [p for p in self.candidates if p not in self.failed]
|
||||
if available:
|
||||
self.active = available[0]
|
||||
self.fallback = False
|
||||
return self.active
|
||||
self.active = None
|
||||
if config["network_mode"] == "proxy_only":
|
||||
self.fallback = False
|
||||
raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务")
|
||||
self.fallback = True
|
||||
return None
|
||||
|
||||
def rotate(self, failed=None):
|
||||
with self.lock:
|
||||
old = self.active
|
||||
if failed:
|
||||
self.failed.add(failed)
|
||||
self.active = None
|
||||
if old in self.candidates:
|
||||
self.candidates.remove(old)
|
||||
self.candidates.append(old)
|
||||
return self.resolve()
|
||||
|
||||
def reset(self):
|
||||
with self.lock:
|
||||
self.active = None
|
||||
self.failed.clear()
|
||||
self.candidates = []
|
||||
self.fallback = False
|
||||
|
||||
def status(self):
|
||||
config = self.settings.read()
|
||||
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
|
||||
|
||||
|
||||
class Transport:
|
||||
def __init__(self, settings, router, session=None, wait=None):
|
||||
self.settings, self.router = settings, router
|
||||
self.session = session or requests.Session()
|
||||
self.session.trust_env = False
|
||||
self.session.headers = getattr(self.session, "headers", {})
|
||||
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
|
||||
self.wait = wait or time.sleep
|
||||
self.local = threading.local()
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def sleep(self, seconds):
|
||||
remaining = seconds
|
||||
while remaining > 0:
|
||||
if guard := getattr(self.local, "guard", None):
|
||||
guard()
|
||||
step = min(.25, remaining)
|
||||
self.wait(step)
|
||||
remaining -= step
|
||||
|
||||
def request(self, method, url, **kwargs):
|
||||
config = self.settings.read()
|
||||
for attempt in range(config["retries"]):
|
||||
self.sleep(config["request_interval"] if attempt == 0 else min(20, 2 ** attempt))
|
||||
proxy = self.router.resolve()
|
||||
kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {}
|
||||
kwargs.setdefault("timeout", config["timeout"])
|
||||
try:
|
||||
with self.lock:
|
||||
response = self.session.request(method, url, **kwargs)
|
||||
if response.status_code in (429, 503):
|
||||
# Rate limiting does not prove an IP is broken.
|
||||
self.sleep(min(30, 5 * (attempt + 1)))
|
||||
continue
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
if proxy:
|
||||
self.router.rotate(failed=proxy)
|
||||
if attempt + 1 == config["retries"]:
|
||||
raise RemoteUnavailable("网络连接失败,已尝试切换代理;请检查网络或恢复任务") from None
|
||||
except requests.HTTPError as exc:
|
||||
raise RemoteUnavailable(f"数据源返回 HTTP {exc.response.status_code}") from None
|
||||
raise RemoteUnavailable("数据源限流,请降低请求频率后重试")
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Project-local, non-secret configuration."""
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
DEFAULTS = {
|
||||
"network_mode": "prefer_proxy", "pool_url": "http://127.0.0.1:5010",
|
||||
"pool_project": r"C:\Users\Jayson\Desktop\proxy_pool",
|
||||
"backup_proxies": [], "request_interval": 1.2, "timeout": 30, "retries": 3,
|
||||
}
|
||||
|
||||
|
||||
def proxy_url(value):
|
||||
value = str(value).strip()
|
||||
if not value:
|
||||
return ""
|
||||
value = value if "://" in value else "http://" + value
|
||||
parsed = urlparse(value)
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError:
|
||||
port = None
|
||||
if parsed.scheme not in ("http", "https", "socks5") or not parsed.hostname or not port:
|
||||
raise ValueError("代理格式应为 IP:端口或 http://IP:端口")
|
||||
if parsed.username or parsed.password:
|
||||
raise ValueError("此版本请使用不带账号密码的代理地址")
|
||||
return value
|
||||
|
||||
|
||||
class SettingsStore:
|
||||
def __init__(self, directory):
|
||||
self.path = Path(directory) / "settings.json"
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def read(self):
|
||||
with self.lock:
|
||||
values = json.loads(self.path.read_text(encoding="utf-8")) if self.path.exists() else {}
|
||||
return {**DEFAULTS, **{k: v for k, v in values.items() if k in DEFAULTS}}
|
||||
|
||||
def update(self, values):
|
||||
with self.lock:
|
||||
values = {**self.read(), **{k: v for k, v in values.items() if k in DEFAULTS}}
|
||||
if values["network_mode"] not in ("direct", "prefer_proxy", "proxy_only"):
|
||||
raise ValueError("未知网络策略")
|
||||
parsed = urlparse(values["pool_url"])
|
||||
if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:
|
||||
raise ValueError("代理池 API 地址无效")
|
||||
values["pool_url"] = values["pool_url"].rstrip("/")
|
||||
for key, low, high in [("request_interval", .3, 60), ("timeout", 5, 120), ("retries", 1, 6)]:
|
||||
values[key] = float(values[key]) if key == "request_interval" else int(values[key])
|
||||
if not low <= values[key] <= high:
|
||||
raise ValueError(f"{key} 应在 {low} 到 {high} 之间")
|
||||
proxies = values["backup_proxies"]
|
||||
if isinstance(proxies, str):
|
||||
proxies = proxies.splitlines()
|
||||
values["backup_proxies"] = list(dict.fromkeys(proxy_url(p) for p in proxies if str(p).strip()))
|
||||
temporary = self.path.with_suffix(".tmp")
|
||||
temporary.write_text(json.dumps(values, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
temporary.replace(self.path)
|
||||
return values
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import io
|
||||
import json
|
||||
import sqlite3
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from studio.api import create_app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(tmp_path):
|
||||
with TestClient(create_app(tmp_path), base_url="http://127.0.0.1:8765") as client:
|
||||
yield client
|
||||
|
||||
|
||||
def csv_upload(client, year=2026):
|
||||
text = 'school_name,school_code,major_code,major_name,department_name,nzsrsstr,subject1,subject2,subject3,subject4\n北京大学,10001,081200,计算机科学与技术,信息学院,专业:6(不含推免),101思想政治理论,201英语一,301数学一,408计算机学科专业基础\n'
|
||||
return client.post("/api/import", files={"file": ("yzw.csv", text.encode("utf-8-sig"))}, data={"year": year})
|
||||
|
||||
|
||||
def test_csv_upload_filter_favorite_and_export(client):
|
||||
assert csv_upload(client).json()["records"] == 1
|
||||
rows = client.get("/api/catalog", params={"year": 2026, "subject": "408"}).json()
|
||||
assert rows["total"] == 1
|
||||
record = rows["items"][0]
|
||||
assert record["quota"] == 6
|
||||
assert client.put(f'/api/catalog/{record["id"]}/favorite', json={"enabled": True}).status_code == 200
|
||||
assert client.get("/api/catalog", params={"favorites": True}).json()["total"] == 1
|
||||
exported = client.get("/api/export", params={"format": "json", "year": 2026}).json()
|
||||
assert exported["catalog"][0]["year"] == 2026
|
||||
assert client.get("/api/export", params={"format": "xlsx"}).content.startswith(b"PK")
|
||||
|
||||
|
||||
def test_import_does_not_invent_year_when_csv_has_no_year(client):
|
||||
content = b"school_name,major_code\nTest,081200\n"
|
||||
response = client.post("/api/import", files={"file": ("data.csv", content)})
|
||||
assert response.status_code == 400
|
||||
assert "年份" in response.json()["detail"]
|
||||
|
||||
|
||||
def test_cross_origin_write_is_blocked_and_secrets_are_not_in_settings(client):
|
||||
response = client.put("/api/settings", json={"cookie": "secret", "network_mode": "direct"}, headers={"Origin": "https://untrusted.example"})
|
||||
assert response.status_code == 403
|
||||
assert "cookie" not in client.get("/api/settings").json()
|
||||
|
||||
|
||||
def test_export_prevents_spreadsheet_formula_execution(client):
|
||||
data = {"catalog": [{"year": 2026, "school_name": "=1+1", "major_code": "081200", "major_name": "test"}]}
|
||||
response = client.post("/api/import", files={"file": ("studio.json", json.dumps(data).encode())})
|
||||
assert response.status_code == 200
|
||||
response = client.get("/api/export", params={"format": "csv"})
|
||||
assert "'=1+1" in response.content.decode("utf-8-sig")
|
||||
|
||||
|
||||
def test_import_yam_sqlite_keeps_year_and_score_scope(client, tmp_path):
|
||||
path = tmp_path / "yam.db"
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript('CREATE TABLE schools(school_id TEXT,major_code TEXT,name TEXT,province TEXT); CREATE TABLE departments(department_id TEXT,school_id TEXT,major_code TEXT,name TEXT,research_direction TEXT,exam_subjects TEXT,enrollment_text TEXT); CREATE TABLE score_lines(school_id TEXT,department_id TEXT,major_code TEXT,year INTEGER,total INTEGER,match_scope TEXT,raw_code TEXT);')
|
||||
conn.execute("INSERT INTO schools VALUES('10001','081200','北京大学','北京')")
|
||||
conn.execute("INSERT INTO departments VALUES('001','10001','081200','信息学院','人工智能','[\"201英语一\"]','专业:6')")
|
||||
conn.execute("INSERT INTO score_lines VALUES('10001','001','081200',2025,310,'discipline','0812')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
response = client.post("/api/import", files={"file": ("yam.db", path.read_bytes())}, data={"year": 2026})
|
||||
assert response.status_code == 200, response.text
|
||||
rows = client.get("/api/catalog").json()["items"]
|
||||
assert rows[0]["year"] == 2026
|
||||
assert rows[0]["score_year"] == 2025
|
||||
assert rows[0]["score_scope"] == "discipline"
|
||||
@@ -0,0 +1,74 @@
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
|
||||
from studio.database import Database
|
||||
from studio.jobs import JobManager
|
||||
from studio.settings import SettingsStore
|
||||
from tests.test_network import Response
|
||||
|
||||
|
||||
class CatalogTransport:
|
||||
def request(self, method, url, **kwargs):
|
||||
if method == "GET":
|
||||
return Response({}, text="2027年硕士专业目录查询")
|
||||
page = int(kwargs["data"].get("curPage", 1))
|
||||
rows = [{"dwdm": "10001", "dwmc": "北京大学"}] if page == 1 else [{"dwdm": "10002", "dwmc": "中国人民大学"}]
|
||||
return Response({"flag": True, "msg": {"list": rows, "nextPageAvailable": page == 1, "totalCount": 2}})
|
||||
|
||||
|
||||
def test_catalog_pagination_consumes_all_pages():
|
||||
client = CatalogClient(CatalogTransport())
|
||||
assert [row["dwdm"] for row in client.schools("11")] == ["10001", "10002"]
|
||||
|
||||
|
||||
def test_false_login_business_status_cannot_be_success():
|
||||
class Unauthenticated(CatalogTransport):
|
||||
def request(self, *args, **kwargs):
|
||||
return Response({"flag": False, "msg": "请登录"})
|
||||
with pytest.raises(LoginRequired):
|
||||
list(CatalogClient(Unauthenticated()).schools("11"))
|
||||
|
||||
|
||||
def test_source_year_mismatch_prevents_wrongly_dated_catalog():
|
||||
with pytest.raises(SourceYearMismatch):
|
||||
CatalogClient(CatalogTransport()).verify_year(2026)
|
||||
|
||||
|
||||
def test_school_lookup_rejects_ambiguous_or_wrong_campus():
|
||||
class SearchTransport:
|
||||
def request(self, *args, **kwargs):
|
||||
return Response({"code": "0000", "data": {"data": [{"school_name": "中国地质大学(武汉)", "school_id": 10}]}})
|
||||
assert ScoreClient(SearchTransport()).school_id("中国地质大学(北京)") is None
|
||||
|
||||
|
||||
def test_task_pause_resume_reuses_persisted_work_without_duplicate_entries(tmp_path):
|
||||
db = Database(tmp_path / "studio.db")
|
||||
settings = SettingsStore(tmp_path)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
def runner(job_id, params, manager):
|
||||
for unit in ["one", "two"]:
|
||||
manager.checkpoint(job_id)
|
||||
if manager.unit_done(job_id, unit):
|
||||
continue
|
||||
if unit == "one":
|
||||
entered.set()
|
||||
release.wait(3)
|
||||
manager.complete_unit(job_id, unit)
|
||||
jobs = JobManager(db, settings, runner=runner)
|
||||
job = jobs.start({"mode": "catalog", "year": 2027})
|
||||
assert entered.wait(3)
|
||||
jobs.pause(job["id"])
|
||||
release.set()
|
||||
time.sleep(.1)
|
||||
assert jobs.get(job["id"])["status"] == "paused"
|
||||
jobs.resume(job["id"])
|
||||
deadline = time.monotonic() + 3
|
||||
while jobs.get(job["id"])["status"] != "completed" and time.monotonic() < deadline:
|
||||
time.sleep(.02)
|
||||
assert jobs.get(job["id"])["status"] == "completed"
|
||||
assert jobs.get(job["id"])["completed_units"] == 2
|
||||
jobs.close()
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import requests
|
||||
import pytest
|
||||
|
||||
from studio.network import ProxyRouter, Transport, ProxyUnavailable
|
||||
from studio.settings import SettingsStore
|
||||
from studio.auth import CookieVault
|
||||
|
||||
|
||||
class Response:
|
||||
def __init__(self, data, status=200, text=""):
|
||||
self.data = data
|
||||
self.status_code = status
|
||||
self.text = text
|
||||
self.headers = {}
|
||||
self.url = "https://yz.chsi.com.cn/zsml/"
|
||||
|
||||
def json(self):
|
||||
return self.data
|
||||
|
||||
def raise_for_status(self):
|
||||
if self.status_code >= 400:
|
||||
raise requests.HTTPError(response=self)
|
||||
|
||||
|
||||
class Pool:
|
||||
def __init__(self, proxies=None):
|
||||
self.proxies = proxies if proxies is not None else [{"proxy": "1.2.3.4:8080", "https": True}, {"proxy": "5.6.7.8:9090", "https": True}]
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
if url.endswith("/all/"):
|
||||
return Response(self.proxies)
|
||||
if url.endswith("/count/"):
|
||||
return Response({"count": len(self.proxies), "http_type": {"https": len(self.proxies)}})
|
||||
return Response(self.proxies[0] if self.proxies else {"code": 0, "src": "no proxy"})
|
||||
|
||||
|
||||
class Remote:
|
||||
def __init__(self, fail_once=False):
|
||||
self.fail_once = fail_once
|
||||
self.cookies = requests.cookies.RequestsCookieJar()
|
||||
|
||||
def request(self, method, url, **kwargs):
|
||||
if self.fail_once:
|
||||
self.fail_once = False
|
||||
raise requests.exceptions.ProxyError("failed")
|
||||
return Response({"routed_through": (kwargs.get("proxies") or {}).get("https"), "url": url})
|
||||
|
||||
|
||||
def setup_network(tmp_path, mode="proxy_only", pool=None, remote=None):
|
||||
settings = SettingsStore(tmp_path)
|
||||
settings.update({"network_mode": mode, "request_interval": 0.3})
|
||||
router = ProxyRouter(settings, session=pool or Pool())
|
||||
transport = Transport(settings, router, session=remote or Remote(), wait=lambda _: None)
|
||||
return settings, router, transport
|
||||
|
||||
|
||||
def test_catalog_and_score_requests_share_the_selected_proxy(tmp_path):
|
||||
_, router, transport = setup_network(tmp_path)
|
||||
first = transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/dws.do").json()
|
||||
second = transport.request("POST", "https://api.kaoyan.cn/pc/school/schoolScore").json()
|
||||
assert first["routed_through"] == second["routed_through"]
|
||||
assert first["routed_through"] in ["http://1.2.3.4:8080", "http://5.6.7.8:9090"]
|
||||
|
||||
|
||||
def test_strict_proxy_mode_never_falls_back_to_direct_connection(tmp_path):
|
||||
_, _, transport = setup_network(tmp_path, pool=Pool([]))
|
||||
with pytest.raises(ProxyUnavailable):
|
||||
transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/dws.do")
|
||||
|
||||
|
||||
def test_prefer_proxy_mode_exposes_direct_fallback(tmp_path):
|
||||
_, router, transport = setup_network(tmp_path, mode="prefer_proxy", pool=Pool([]))
|
||||
result = transport.request("GET", "https://api.kaoyan.cn/pc/school/schoolList").json()
|
||||
assert result["routed_through"] is None
|
||||
assert router.status()["route"] == "direct"
|
||||
assert router.status()["fallback"] is True
|
||||
|
||||
|
||||
def test_network_failure_rotates_to_another_proxy(tmp_path):
|
||||
_, router, transport = setup_network(tmp_path, remote=Remote(fail_once=True))
|
||||
original = router.resolve()
|
||||
result = transport.request("GET", "https://yz.chsi.com.cn/zsml/").json()
|
||||
assert result["routed_through"] != original
|
||||
assert len(router.failed) == 1
|
||||
|
||||
|
||||
def test_cookie_is_encrypted_and_not_returned_in_settings(tmp_path):
|
||||
vault = CookieVault(tmp_path)
|
||||
vault.save({"JSESSIONID": "secret-test-cookie", "CASTGC": "ticket"})
|
||||
assert vault.load()["JSESSIONID"] == "secret-test-cookie"
|
||||
assert b"secret-test-cookie" not in vault.path.read_bytes()
|
||||
assert "cookie" not in SettingsStore(tmp_path).read()
|
||||
|
||||
Reference in New Issue
Block a user