feat: implement unified data sources, proxy routing and local API

This commit is contained in:
Codex
2026-10-11 13:49:07 +08:00
parent 25cccd4372
commit 389b2aa3a5
20 changed files with 21601 additions and 4 deletions
+2
View File
@@ -7,4 +7,6 @@ frontend/node_modules/
frontend/dist/ frontend/dist/
frontend/*.tsbuildinfo frontend/*.tsbuildinfo
.superpowers/ .superpowers/
*.egg-info/
*-install.log
+15787
View File
File diff suppressed because it is too large Load Diff
+2762
View File
File diff suppressed because it is too large Load Diff
+6 -4
View File
@@ -3,10 +3,12 @@
用户已明确要求创建项目,并选择本地网页;依照会话授权直接推进开发。 用户已明确要求创建项目,并选择本地网页;依照会话授权直接推进开发。
原目录检查:YAM 无本地修改;yzw 有本地修改;代理池运行中。新独立仓库隔离开发,不修改以上目录。 原目录检查:YAM 无本地修改;yzw 有本地修改;代理池运行中。新独立仓库隔离开发,不修改以上目录。
Pre-flight:数据库/任务/网络/API 的数据接口按设计共享。图形界面通过 HTTP 使用这些接口,不依赖原项目环境。 Pre-flight:数据库/任务/网络/API 的数据接口按设计共享。图形界面通过 HTTP 使用这些接口,不依赖原项目环境。
Task 1: pending Task 1: complete — 5 tests green; cross-year records, alternatives, conservative quota, score granularity.
Task 2: pending Task 2: complete — 5 tests green; shared proxy, strict route, fallback, rotation, DPAPI vault.
Task 3: pending Task 3: complete — 5 tests green; pagination, business login status, year mismatch, exact campus, pause/resume.
Task 4: pending Task 4: complete — 5 tests green; file import/export, YAM SQLite, favorites, origin guard, formula-safe exports.
Task 5: pending Task 5: pending
Task 6: pending Task 6: pending
Public index: 2,255 major entries and 460 unique schools extracted from YAM public data files. Seed index contains no admissions snapshots, cookies or signed request tokens.
+1337
View File
File diff suppressed because it is too large Load Diff
+2
View File
@@ -0,0 +1,2 @@
"""考研数据工作台。"""
+14
View File
@@ -0,0 +1,14 @@
import argparse
from pathlib import Path
import uvicorn
from studio.api import create_app
def main():
parser = argparse.ArgumentParser(description="考研数据工作台")
parser.add_argument("--port", type=int, default=8765)
parser.add_argument("--data-dir", type=Path)
args = parser.parse_args()
uvicorn.run(create_app(args.data_dir), host="127.0.0.1", port=args.port, log_level="info")
if __name__ == "__main__":
main()
+230
View File
@@ -0,0 +1,230 @@
"""Local-only HTTP workbench and production SPA host."""
import json
import subprocess
import uuid
from contextlib import asynccontextmanager
from pathlib import Path
from urllib.parse import urlparse
from fastapi import FastAPI, HTTPException, Request, UploadFile, File, Form
from fastapi.responses import FileResponse, JSONResponse, Response
from fastapi.staticfiles import StaticFiles
from studio.auth import AuthService, CookieVault
from studio.database import Database
from studio.domain import PROVINCES
from studio.imports import import_file, export_sheet
from studio.jobs import JobManager
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable
from studio.settings import SettingsStore
ROOT = Path(__file__).resolve().parent.parent
def create_app(data_dir=None):
directory = Path(data_dir or ROOT / ".runtime")
directory.mkdir(parents=True, exist_ok=True)
db = Database(directory / "studio.db")
settings = SettingsStore(directory)
router = ProxyRouter(settings)
transport = Transport(settings, router)
auth = AuthService(transport, CookieVault(directory))
jobs = JobManager(db, settings, transport)
if (ROOT / "data/schools.json").exists():
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
@asynccontextmanager
async def lifespan(app):
yield
jobs.close()
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
app.state.db, app.state.jobs = db, jobs
app.state.settings, app.state.router = settings, router
@app.middleware("http")
async def local_access(request: Request, call_next):
host = request.url.hostname
if host not in ("127.0.0.1", "localhost", "::1"):
return JSONResponse({"detail": "仅允许本机访问"}, status_code=403)
if request.method not in ("GET", "HEAD", "OPTIONS"):
origin = request.headers.get("origin")
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
return await call_next(request)
@app.exception_handler(ValueError)
async def invalid(request, exc):
return JSONResponse({"detail": str(exc)}, status_code=400)
@app.exception_handler(ProxyUnavailable)
@app.exception_handler(RemoteUnavailable)
async def remote_error(request, exc):
return JSONResponse({"detail": str(exc)}, status_code=503)
def filters(request):
values = dict(request.query_params)
for key in ("year", "min_quota", "max_score"):
values[key] = int(values[key]) if values.get(key) else None
values["favorites"] = values.get("favorites") in ("true", "1")
return values
@app.get("/api/health")
def health():
return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve())}
@app.get("/api/overview")
def overview():
return {**db.overview(), "provinces_list": [{"code": k, "name": v} for k, v in PROVINCES.items()], "sources": [{"name": "研招网", "role": "全国院校 · 专业 · 方向 · 科目", "url": "https://yz.chsi.com.cn/zsml/"}, {"name": "掌上考研", "role": "历年分数线 · 招生计划", "url": "https://www.kaoyan.cn/"}]}
@app.get("/api/majors")
def majors():
path = ROOT / "data/majors.json"
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else []
@app.get("/api/schools")
def schools(keyword: str = "", province: str = ""):
return db.schools(keyword, province)
@app.get("/api/catalog")
def catalog(request: Request, page: int = 1, page_size: int = 30):
return db.query_catalog(filters(request), page, page_size)
@app.get("/api/catalog/{rid}")
def detail(rid: int):
result = db.history(rid)
if result is None:
raise HTTPException(404, "记录不存在")
return result
@app.put("/api/catalog/{rid}/favorite")
def favorite(rid: int, body: dict):
if not db.favorite(rid, bool(body.get("enabled"))):
raise HTTPException(404, "记录不存在")
return {"ok": True}
@app.get("/api/jobs")
def list_jobs():
return jobs.get()
@app.post("/api/jobs")
def start_job(body: dict):
return jobs.start(body)
@app.get("/api/jobs/{jid}/logs")
def job_logs(jid: str, after: int = 0):
return db.logs(jid, after)
@app.post("/api/jobs/{jid}/{action}")
def job_action(jid: str, action: str):
if action not in ("pause", "resume", "cancel"):
raise HTTPException(404, "操作不存在")
return getattr(jobs, action)(jid)
@app.get("/api/settings")
def get_settings():
return settings.read()
@app.put("/api/settings")
def put_settings(body: dict):
result = settings.update(body)
router.reset()
return result
@app.get("/api/auth")
def auth_status():
return auth.status()
@app.post("/api/auth/{action}")
def auth_action(action: str, body: dict = {}):
if action == "cookie":
return auth.cookie(body.get("cookie", ""))
if action == "login":
return auth.login(body.get("username", ""), body.get("password", ""))
if action in ("check", "browser", "clear"):
return getattr(auth, action)()
raise HTTPException(404, "操作不存在")
@app.get("/api/proxy")
def proxy_status(refresh: bool = False):
if refresh:
router.refresh()
try:
router.resolve()
except ProxyUnavailable:
pass
return router.status()
@app.post("/api/proxy/rotate")
def rotate():
router.rotate()
return router.status()
@app.post("/api/proxy/manage")
def manage(body: dict):
action = body.get("action")
if action not in ("start", "stop", "status"):
raise ValueError("代理池操作无效")
script = Path(settings.read()["pool_project"]) / "manage.ps1"
if not script.is_file():
raise ValueError("代理池目录没有 manage.ps1,请检查配置;也可手动启动原代理池")
result = subprocess.run(["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", str(script), action], capture_output=True, timeout=55, creationflags=subprocess.CREATE_NO_WINDOW)
output = (result.stdout + result.stderr).decode("utf-8", errors="replace")[-5000:]
router.reset()
return {"ok": result.returncode == 0, "output": output, "action": action}
@app.post("/api/import")
def upload(file: UploadFile = File(...), year: int | None = Form(None)):
suffix = Path(file.filename or "").suffix.lower()
folder = directory / "imports"
folder.mkdir(exist_ok=True)
temporary = folder / (uuid.uuid4().hex + suffix)
try:
total = 0
with temporary.open("wb") as stream:
while chunk := file.file.read(1024 * 1024):
total += len(chunk)
if total > 100 * 1024 * 1024:
raise ValueError("一次最多上传 100 MB,请拆分文件")
stream.write(chunk)
records, scores = import_file(temporary, year)
db.upsert_catalog(records)
db.upsert_scores(scores)
return {"records": len(records), "scores": len(scores), "message": "导入完成,相同记录按年份更新"}
except (UnicodeError, json.JSONDecodeError, KeyError) as exc:
raise ValueError(f"文件编码或结构无法识别({type(exc).__name__})") from None
finally:
temporary.unlink(missing_ok=True)
file.file.close()
@app.get("/api/export")
def export(request: Request, format: str = "csv"):
if format not in ("csv", "xlsx", "json"):
raise ValueError("导出格式无效")
rows = db.all_catalog(filters(request))
if format == "json":
pairs = {(r["school_code"], r["major_code"]) for r in rows}
with db.connection() as conn:
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs]
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode()
mime = "application/json"
else:
payload = export_sheet(rows, format)
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'})
dist = ROOT / "frontend/dist"
if (dist / "assets").exists():
app.mount("/assets", StaticFiles(directory=dist / "assets"), name="assets")
@app.get("/{path:path}")
def frontend(path: str):
if path.startswith("api/"):
raise HTTPException(404, "API 不存在")
index = dist / "index.html"
if index.exists():
return FileResponse(index)
return Response("请先运行 frontend 的 npm run build,再启动工作台。", media_type="text/plain", status_code=503)
return app
+163
View File
@@ -0,0 +1,163 @@
"""Windows DPAPI protects local CHSI cookies; passwords are never stored."""
import ctypes
import json
import os
import threading
import time
from pathlib import Path
from bs4 import BeautifulSoup
LOGIN_URL = "https://account.chsi.com.cn/passport/login?entrytype=yzgr&service=https%3A%2F%2Fyz.chsi.com.cn%2Fj_spring_cas_security_check"
class Blob(ctypes.Structure):
_fields_ = [("size", ctypes.c_ulong), ("data", ctypes.POINTER(ctypes.c_ubyte))]
def protect(payload, decrypt=False):
if os.name != "nt":
raise RuntimeError("凭据加密存储需要 Windows DPAPI")
memory = ctypes.create_string_buffer(payload)
source = Blob(len(payload), ctypes.cast(memory, ctypes.POINTER(ctypes.c_ubyte)))
result = Blob()
crypt = ctypes.windll.crypt32
fn = crypt.CryptUnprotectData if decrypt else crypt.CryptProtectData
# UI_FORBIDDEN prevents a modal system prompt in a background job.
if not fn(ctypes.byref(source), None, None, None, None, 1, ctypes.byref(result)):
raise RuntimeError("Windows 凭据加密失败,请使用当前 Windows 用户重新登录")
try:
return ctypes.string_at(result.data, result.size)
finally:
ctypes.windll.kernel32.LocalFree(result.data)
class CookieVault:
def __init__(self, directory):
self.path = Path(directory) / "chsi-cookie.enc"
self.path.parent.mkdir(parents=True, exist_ok=True)
def save(self, cookies):
temporary = self.path.with_suffix(".tmp")
temporary.write_bytes(protect(json.dumps(cookies, ensure_ascii=False).encode()))
temporary.replace(self.path)
def load(self):
if not self.path.exists():
return {}
return json.loads(protect(self.path.read_bytes(), decrypt=True))
def clear(self):
self.path.unlink(missing_ok=True)
class AuthService:
def __init__(self, transport, vault):
self.transport, self.vault = transport, vault
self.message = "尚未验证登录状态"
self.valid = False
self.browser_state = "idle"
self.browser_message = ""
self.browser_thread = None
try:
self._load(self.vault.load())
except RuntimeError:
self.message = "保存的 Cookie 无法解密,请重新登录"
def _load(self, cookies):
with self.transport.lock:
self.transport.session.cookies.clear()
for name, value in cookies.items():
self.transport.session.cookies.set(name, value, domain=".chsi.com.cn", path="/")
def status(self):
return {"saved": self.vault.path.exists(), "valid": self.valid, "message": self.message, "browser_state": self.browser_state, "browser_message": self.browser_message}
def cookie(self, text):
if not text.strip() or "\n" in text or "\r" in text:
raise ValueError("请粘贴单行 Cookie 请求头")
text = text.removeprefix("Cookie:").strip()
cookies = {}
for token in text.split(";"):
if "=" in token:
name, value = token.strip().split("=", 1)
if name and value:
cookies[name] = value
if not cookies:
raise ValueError("未识别到 Cookie 键值")
self._load(cookies)
self.vault.save(cookies)
self.valid = False
self.message = "Cookie 已加密保存,点击验证确认目录访问权限"
return self.status()
def check(self):
response = self.transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/dws.do", data={"ssdm": "11", "curPage": 1, "start": 0, "pageSize": 1, "dwlxs[0]": "all"})
try:
body = response.json()
except ValueError:
self.valid = False
self.message = "目录接口未返回有效数据,可能需要登录或验证码"
return self.status()
self.valid = body.get("flag") is True and isinstance(body.get("msg"), dict)
self.message = "研招目录接口验证成功" if self.valid else "登录未生效或数据源限流,请在浏览器完成登录后重试"
if self.valid:
self.vault.save(self.transport.session.cookies.get_dict())
return self.status()
def login(self, username, password):
if not username or not password:
raise ValueError("请输入学信网账号和密码")
response = self.transport.request("GET", LOGIN_URL)
soup = BeautifulSoup(response.text, "html.parser")
form = {node.get("name"): node.get("value", "") for node in soup.select("input[type=hidden][name]")}
if "lt" not in form or "execution" not in form:
raise ValueError("登录页面需要人工验证,请使用浏览器登录")
form.update(username=username, password=password, _eventId="submit")
self.transport.request("POST", LOGIN_URL, data=form)
return self.check()
def clear(self):
self._load({})
self.vault.clear()
self.valid = False
self.message = "已清除本项目保存的登录凭据"
return self.status()
def browser(self):
if self.browser_thread and self.browser_thread.is_alive():
return self.status()
proxy = self.transport.router.resolve()
self.browser_state = "opening"
self.browser_message = "正在打开独立 Edge 登录窗口,完成登录后可关闭窗口"
def run():
from playwright.sync_api import sync_playwright
try:
with sync_playwright() as playwright:
browser = playwright.chromium.launch(channel="msedge", headless=False, proxy={"server": proxy} if proxy else None)
context = browser.new_context()
page = context.new_page()
page.goto(LOGIN_URL, wait_until="domcontentloaded", timeout=60000)
self.browser_state = "waiting"
self.browser_message = "请在 Edge 完成学信网登录和验证码;窗口保持打开,工作台自动检测"
deadline = time.monotonic() + 300
while time.monotonic() < deadline:
page.wait_for_timeout(2000)
if "yz.chsi.com.cn" in page.url:
cookies = {c["name"]: c["value"] for c in context.cookies() if "chsi.com.cn" in c["domain"]}
if cookies:
self._load(cookies)
if self.check()["valid"]:
self.browser_state = "completed"
self.browser_message = "已获取登录 Cookie,目录接口验证成功"
browser.close()
return
browser.close()
self.browser_state = "expired"
self.browser_message = "登录等待超时,可再次打开浏览器"
except Exception:
self.browser_state = "failed"
self.browser_message = "登录窗口已关闭或连接失败,可重试或粘贴 Cookie"
self.browser_thread = threading.Thread(target=run, daemon=True, name="browser-login")
self.browser_thread.start()
return self.status()
+133
View File
@@ -0,0 +1,133 @@
"""Independent adapters for the two projects' public data sources."""
import json
import re
from studio.domain import finish_record, integer, normalized_name, select_score_lines
from studio.network import RemoteUnavailable
class LoginRequired(RuntimeError):
pass
class SourceYearMismatch(RuntimeError):
pass
class CatalogClient:
BASE = "https://yz.chsi.com.cn/zsml/"
def __init__(self, transport):
self.transport = transport
def verify_year(self, year):
response = self.transport.request("GET", self.BASE)
text = re.sub("<[^>]*>", " ", response.text)
matches = re.findall(r"(20\d{2})\s*年.{0,30}(?:硕士|招生|专业目录)", text)
years = set(map(int, matches))
if not years:
raise SourceYearMismatch("未能核实研招网当前目录年份,已停止写入;请检查登录及目录开放状态")
current = max(years)
if int(year) != current:
raise SourceYearMismatch(f"研招网当前公开目录为 {current} 年,任务选择了 {year} 年;历史目录请导入已有快照")
return current
def pages(self, endpoint, payload, page_size=50):
seen = set()
for page in range(1, 10001):
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
try:
body = response.json()
except ValueError:
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
if body.get("flag") is not True:
message = str(body.get("msg", ""))
if any(token in message for token in ("登录", "登陆", "认证")):
raise LoginRequired("研招网要求登录,请更新 Cookie 后恢复任务")
if any(token in message for token in ("频繁", "稍后", "验证码")):
raise RemoteUnavailable("研招网访问受限,请增加请求间隔并完成人工验证后恢复")
raise RemoteUnavailable("研招网目录接口未成功返回数据")
data = body.get("msg") or {}
if not isinstance(data, dict) or not isinstance(data.get("list"), list):
raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据")
rows = data["list"]
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen:
raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页")
seen.add(signature)
yield from rows
more = data.get("nextPageAvailable")
total = integer(data.get("totalCount"))
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and page * page_size >= total) or len(rows) < page_size):
return
raise RemoteUnavailable("目录分页超过安全上限")
def schools(self, province, keyword=""):
return self.pages("dws", {"ssdm": province, "dwmc": keyword, "dwdm": "", "xxfs": "", "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"})
def majors(self, school, major_code="", study_mode=""):
return self.pages("dwzys", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major_code, "zymc": "", "mldm": "", "yjxkdm": "", "xwlx": "", "xxfs": study_mode})
def directions(self, school, major, study_mode=""):
return self.pages("yjfxs", {"dwdm": school["dwdm"], "dwmc": school.get("dwmc", ""), "zydm": major["zydm"], "zymc": major.get("zymc", ""), "xxfs": study_mode, "tydxs": "", "jsggjh": "", "dwlxs[0]": "all"})
ALIASES = {"湖州师范学院": "湖州师范大学", "闽江学院": "闽江大学", "皖南医学院": "皖南医科大学", "中国航空研究院(613所)": "中国航空研究院", "中国电子科技集团有限公司研究生院": "中国电子科技集团", "湖南理工学院": "湖南理工大学"}
class ScoreClient:
BASE = "https://api.kaoyan.cn/pc"
def __init__(self, transport):
self.transport = transport
self.cache = {}
def post(self, endpoint, payload):
response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"})
try:
body = response.json()
except ValueError:
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
if str(body.get("code")) != "0000":
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
return body.get("data") or {}
def pages(self, endpoint, payload, limit=100):
seen = set()
for page in range(1, 10001):
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data
if not isinstance(rows, list):
raise RemoteUnavailable("掌上考研响应结构已变化")
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen:
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
seen.add(signature)
yield from rows
total = integer(data.get("total")) if isinstance(data, dict) else None
last = integer(data.get("last_page")) if isinstance(data, dict) else None
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total:
return
raise RemoteUnavailable("历史数据分页超过安全上限")
def school_id(self, name):
key = normalized_name(name)
if key in self.cache:
return self.cache[key]
for query in dict.fromkeys([name, ALIASES.get(key, name)]):
matches = [row for row in self.pages("/school/schoolList", {"school_name": query}, limit=10) if normalized_name(row.get("school_name", "")) == normalized_name(query)]
if len(matches) == 1 and matches[0].get("school_id"):
self.cache[key] = matches[0]["school_id"]
return self.cache[key]
return None
def score_lines(self, school, major_code, year, school_id):
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
def plans(self, school, major_code, year, school_id):
result = []
for row in self.pages("/school/planListV2", {"school_id": school_id, "year": year, "keyword": major_code}):
if str(row.get("special_code", "")) != major_code:
continue
result.append(finish_record({**school, "year": year, "major_code": major_code, "major_name": row.get("special_name", ""), "department_code": str(row.get("depart_id", "") or ""), "department_name": row.get("depart_name", ""), "direction_code": str(row.get("spe_id", "") or row.get("plan_id", "")), "direction": row.get("research_direction", ""), "quota": integer(row.get("recruit_number")), "quota_text": str(row.get("recruit_number", "") or ""), "quota_scope": "major", "subjects": [], "remarks": row.get("remark", ""), "reference_books": [], "source": "掌上考研", "source_url": "https://www.kaoyan.cn/", "year_verified": True, "raw_evidence": row}))
return result
+196
View File
@@ -0,0 +1,196 @@
"""按年份保存招生目录、来源证据、分数线和任务断点。"""
import json
import sqlite3
from contextlib import contextmanager
from pathlib import Path
from studio.domain import now, normalized_name
SCHEMA = """
CREATE TABLE IF NOT EXISTS catalog(
id INTEGER PRIMARY KEY, record_key TEXT NOT NULL UNIQUE, year INTEGER NOT NULL,
school_code TEXT, school_name TEXT, province TEXT, major_code TEXT, major_name TEXT,
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
CREATE TABLE IF NOT EXISTS scores(
id INTEGER PRIMARY KEY, school_code TEXT, major_code TEXT, year INTEGER,
department_id TEXT, department_name TEXT, scope TEXT, total INTEGER, source TEXT, data TEXT NOT NULL,
UNIQUE(school_code,major_code,year,department_id,department_name,scope,source));
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
"""
def encode(value):
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
class Database:
def __init__(self, path: Path):
self.path = Path(path)
self.path.parent.mkdir(parents=True, exist_ok=True)
with self.connection() as conn:
conn.executescript(SCHEMA)
conn.execute("UPDATE jobs SET status='paused',message='服务重启,任务可从断点恢复' WHERE status IN ('running','queued')")
@contextmanager
def connection(self):
conn = sqlite3.connect(self.path, timeout=30)
conn.row_factory = sqlite3.Row
conn.create_function("norm_name", 1, normalized_name, deterministic=True)
conn.execute("PRAGMA journal_mode=WAL")
try:
yield conn
conn.commit()
except BaseException:
conn.rollback()
raise
finally:
conn.close()
def upsert_catalog(self, records: list[dict]):
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
values = []
for record in records:
row = {**record, "subjects":" | ".join(record.get("subjects", [])), "data":encode(record)}
values.append([row.get(field) for field in fields])
if not values:
return
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
with self.connection() as conn:
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
for record in records:
self._school(conn, record)
def _school(self, conn, record):
conn.execute("INSERT INTO schools VALUES(?,?,?,?,?) ON CONFLICT(school_code) DO UPDATE SET school_name=excluded.school_name,province=CASE WHEN excluded.province<>'' THEN excluded.province ELSE schools.province END,level=CASE WHEN excluded.level<>'' THEN excluded.level ELSE schools.level END", (record["school_code"],record["school_name"],record.get("province", ""),record.get("level", ""),encode(record)))
def index_schools(self, rows):
with self.connection() as conn:
for row in rows:
self._school(conn, row)
def upsert_scores(self, records):
with self.connection() as conn:
for row in records:
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", ""),row.get("department_name", ""),row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row)))
def _query(self, filters: dict):
conditions, params = ["1=1"], []
for field in ["year","province","major_code","study_mode","degree_type","school_code"]:
if filters.get(field) not in (None, "", 0):
conditions.append(f"c.{field}=?")
params.append(filters[field])
for field in ["level","subjects","department_name"]:
value = filters.get(field) or (filters.get("subject") if field == "subjects" else "")
if value:
conditions.append(f"c.{field} LIKE ?")
params.append(f"%{value}%")
if filters.get("direction"):
conditions.append("json_extract(c.data,'$.direction') LIKE ?")
params.append(f"%{filters['direction']}%")
if filters.get("keyword"):
conditions.append("(c.school_name LIKE ? OR c.major_name LIKE ? OR c.department_name LIKE ? OR c.major_code LIKE ?)")
params.extend([f"%{filters['keyword']}%"] * 4)
if filters.get("favorites"):
conditions.append("c.favorite=1")
if filters.get("min_quota") is not None:
conditions.append("c.quota>=?")
params.append(filters["min_quota"])
for field, index in [("english", 1), ("math", 2)]:
if filters.get(field):
conditions.append(f"json_extract(c.data,'$.subjects[{index}]') LIKE ?")
params.append(str(filters[field])+"%")
select = """SELECT c.*,s.data AS score_data,s.total AS score_total,s.year AS score_year,s.scope AS score_scope FROM catalog c LEFT JOIN scores s ON s.id=(SELECT x.id FROM scores x WHERE x.school_code=c.school_code AND x.major_code=c.major_code AND x.year<=c.year AND (x.department_name='' AND x.department_id='' OR x.department_id<>'' AND x.department_id=c.department_code OR x.department_name<>'' AND norm_name(x.department_name)=norm_name(c.department_name)) AND coalesce(json_extract(x.data,'$.note'),'') NOT LIKE '%专项%' AND coalesce(json_extract(x.data,'$.note'),'') NOT LIKE '%少干%' ORDER BY x.year DESC,CASE x.scope WHEN 'major' THEN 0 WHEN 'discipline' THEN 1 ELSE 2 END,x.id DESC LIMIT 1) WHERE """ + " AND ".join(conditions)
if filters.get("max_score") is not None:
select = "SELECT * FROM ("+select+") q WHERE q.score_total<=?"
params.append(filters["max_score"])
return select, params
def query_catalog(self, filters: dict, page=1, page_size=50):
select, params = self._query(filters)
page, page_size = max(1,int(page)), max(1,min(int(page_size),200))
with self.connection() as conn:
total = conn.execute("SELECT count(*) FROM ("+select+")", params).fetchone()[0]
order = {"school":"school_name,major_code,department_name,id", "quota":"quota IS NULL,quota DESC,school_name,id", "score":"score_total IS NULL,score_total,school_name,id", "year":"year DESC,school_name,major_code,department_name,id"}.get(filters.get("sort"), "year DESC,school_name,major_code,department_name,id")
rows = conn.execute(select+" ORDER BY "+order+" LIMIT ? OFFSET ?",[*params,page_size,(page-1)*page_size]).fetchall()
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
def all_catalog(self, filters):
select, params = self._query(filters)
with self.connection() as conn:
return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)]
def _catalog_row(self, row):
data = json.loads(row["data"])
return {**data,"id":row["id"],"favorite":bool(row["favorite"]),"score_total":row["score_total"],"score_year":row["score_year"],"score_scope":row["score_scope"],"score":json.loads(row["score_data"]) if row["score_data"] else None}
def record(self, rid):
with self.connection() as conn:
row = conn.execute("SELECT data,favorite FROM catalog WHERE id=?",(rid,)).fetchone()
return {**json.loads(row["data"]),"id":rid,"favorite":bool(row["favorite"])} if row else None
def favorite(self, rid, enabled):
with self.connection() as conn:
return conn.execute("UPDATE catalog SET favorite=? WHERE id=?",(int(enabled),rid)).rowcount > 0
def history(self, rid):
record = self.record(rid)
if record is None:
return None
with self.connection() as conn:
scores = conn.execute("SELECT data FROM scores WHERE school_code=? AND major_code=? ORDER BY year,scope,department_name",(record["school_code"],record["major_code"])).fetchall()
plans = conn.execute("SELECT data FROM catalog WHERE school_code=? AND major_code=? AND norm_name(department_name)=norm_name(?) ORDER BY year,id",(record["school_code"],record["major_code"],record["department_name"])).fetchall()
def matches(row):
return not row.get("department_name") and not row.get("department_id") or row.get("department_id") and row["department_id"] == record["department_code"] or row.get("department_name") and normalized_name(row["department_name"]) == normalized_name(record["department_name"])
return {"record":record,"scores":[row for entry in scores if matches(row:=json.loads(entry["data"]))],"plans":[json.loads(row["data"]) for row in plans]}
def overview(self):
with self.connection() as conn:
totals = dict(conn.execute("SELECT count(*) AS records,count(DISTINCT school_code) AS schools,count(DISTINCT major_code) AS majors,sum(favorite) AS favorites FROM catalog").fetchone())
totals["indexed_schools"] = conn.execute("SELECT count(*) FROM schools").fetchone()[0]
totals["scores"] = conn.execute("SELECT count(*) FROM scores").fetchone()[0]
totals["years"] = [row[0] for row in conn.execute("SELECT DISTINCT year FROM catalog ORDER BY year DESC")]
totals["provinces"] = [dict(row) for row in conn.execute("SELECT province,count(DISTINCT school_code) AS count FROM catalog WHERE province<>'' GROUP BY province ORDER BY count DESC LIMIT 10")]
return totals
def schools(self, keyword="", province=""):
with self.connection() as conn:
return [dict(row) for row in conn.execute("SELECT school_code,school_name,province,level FROM schools WHERE school_name LIKE ? AND (?='' OR province=?) ORDER BY school_name LIMIT 2000",(f"%{keyword}%",province,province))]
def job_create(self, jid, params):
with self.connection() as conn:
conn.execute("INSERT INTO jobs(id,params,status,created_at,updated_at,message) VALUES(?,?,'queued',?,?,?)",(jid,encode(params),now(),now(),"等待开始"))
def job_update(self, jid, **changes):
allowed = {"status","message","errors","records"}
changes = {key:value for key,value in changes.items() if key in allowed}
changes["updated_at"] = now()
with self.connection() as conn:
conn.execute("UPDATE jobs SET "+",".join(f"{key}=?" for key in changes)+" WHERE id=?",[*changes.values(),jid])
def job_get(self, jid=None):
with self.connection() as conn:
rows = conn.execute("SELECT j.*,(SELECT count(*) FROM job_units u WHERE u.job_id=j.id) AS completed_units FROM jobs j "+("WHERE j.id=?" if jid else "ORDER BY j.created_at DESC LIMIT 100"), (jid,) if jid else ()).fetchall()
result = [{**dict(row),"params":json.loads(row["params"])} for row in rows]
return (result[0] if result else None) if jid else result
def unit_done(self, jid, unit):
with self.connection() as conn:
return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None
def complete_unit(self, jid, unit):
with self.connection() as conn:
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
def log(self, jid, level, message):
with self.connection() as conn:
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
def logs(self, jid, after=0):
with self.connection() as conn:
return [dict(row) for row in conn.execute("SELECT * FROM job_logs WHERE job_id=? AND id>? ORDER BY id DESC LIMIT 300",(jid,after))][::-1]
+111
View File
@@ -0,0 +1,111 @@
"""招生目录与分数线规范化;来源口径和未知值始终保留。"""
import hashlib
import json
import re
from datetime import datetime, timezone, timedelta
from typing import Any
from urllib.parse import urlencode
PROVINCES = {"11":"北京","12":"天津","13":"河北","14":"山西","15":"内蒙古","21":"辽宁","22":"吉林","23":"黑龙江","31":"上海","32":"江苏","33":"浙江","34":"安徽","35":"福建","36":"江西","37":"山东","41":"河南","42":"湖北","43":"湖南","44":"广东","45":"广西","46":"海南","50":"重庆","51":"四川","52":"贵州","53":"云南","54":"西藏","61":"陕西","62":"甘肃","63":"青海","64":"宁夏","65":"新疆"}
def now() -> str:
return datetime.now(timezone(timedelta(hours=8))).isoformat(timespec="seconds")
def normalized_name(value: str) -> str:
return str(value or "").replace("(", "(").replace(")", ")").replace(" ", "").strip()
def integer(value: Any) -> int | None:
if value is None or isinstance(value, bool):
return None
text = str(value).strip()
return int(text) if re.fullmatch(r"\d+", text) else None
def parse_quota(text: str) -> tuple[int | None, str]:
text = str(text or "").strip()
match = re.search(r"专业\s*[::]\s*(\d+)", text)
if match and not re.match(r"\s*[-—~至或]", text[match.end():]):
return int(match.group(1)), "major"
if "学院" in text or "院系" in text:
return None, "college"
if "方向" in text:
return None, "direction"
if re.fullmatch(r"\d+", text):
return int(text), "unspecified"
return None, "unknown"
def record_key(record: dict) -> str:
fields = ["year","school_code","school_name","major_code","department_code","department_name","direction_code","direction","study_mode","exam_type","subjects","special_plans","source"]
serialized = json.dumps([record.get(key, "") for key in fields], ensure_ascii=False, sort_keys=True)
return hashlib.sha256(serialized.encode()).hexdigest()
def finish_record(record: dict) -> dict:
defaults = {"school_code":"", "school_name":"", "province":"", "major_code":"", "major_name":"", "department_code":"", "department_name":"", "direction_code":"", "direction":"", "study_mode":"", "degree_type":"", "exam_type":"", "advisor":"", "quota_text":"", "quota":None, "quota_scope":"unknown", "subjects":[], "special_plans":[], "level":"", "remarks":"", "source":"", "source_url":"", "fetched_at":now(), "year_verified":False}
row = {**defaults, **record}
if not row["school_code"]:
row["school_code"] = "name:" + normalized_name(row["school_name"])
row["record_key"] = record_key(row)
return row
def normalize_catalog(item: dict, year: int, school: dict | None = None, major: dict | None = None) -> list[dict]:
item = {**(school or {}), **(major or {}), **item}
quota, scope = parse_quota(item.get("nzsrsstr", ""))
if not item.get("nzsrsstr") and integer(item.get("nzsrs")) not in (None, 0):
quota, scope = integer(item["nzsrs"]), "unspecified"
labels = []
if str(item.get("syl", "")) == "1":
labels.append("双一流")
base = {
"year": year, "school_code": str(item.get("dwdm", "")), "school_name": item.get("dwmc", ""),
"province": item.get("szss") or PROVINCES.get(str(item.get("ssdm", item.get("szssm", ""))), ""),
"major_code": str(item.get("zydm", "")), "major_name": item.get("zymc", ""),
"department_code": str(item.get("yxsdm", "")), "department_name": item.get("yxsmc", ""),
"direction_code": str(item.get("yjfxdm", "")), "direction": item.get("yjfxmc", ""),
"study_mode": {"1":"全日制", "2":"非全日制"}.get(str(item.get("xxfs", "")), ""),
"degree_type": item.get("xwlxmc", ""), "exam_type": item.get("ksfsmc", ""),
"advisor": item.get("zdjs", ""), "quota_text": item.get("nzsrsstr", ""), "quota": quota, "quota_scope": scope,
"special_plans": [label for key,label in [("tydxs","退役大学生士兵"),("jsggjh","少数民族骨干")] if str(item.get(key, "")) == "1"],
"level": "/".join(labels), "remarks": item.get("bz", item.get("zybz", "")),
"source":"研招网", "source_url":"https://yz.chsi.com.cn/zsml/zydetail.do?" + urlencode({"zydm": item.get("zydm", ""), "zymc": item.get("zymc", "")}),
}
combinations = item.get("kskmz") or [{}]
result = []
for combination in combinations:
subjects = []
for number in range(1, 5):
subject = combination.get(f"km{number}Vo") or {}
name = str(subject.get("kskmmc", "") or "")
code = str(subject.get("kskmdm", "") or "")
subjects.append((code if code and not name.startswith(code) else "") + name)
result.append(finish_record({**base, "subjects":subjects}))
return result
def select_score_lines(items: list[dict], major_code: str, school_code: str, school_name: str, year: int) -> list[dict]:
levels = {6:[], 4:[], 2:[]}
for item in items:
code = str(item.get("code", "")).strip()
if len(code) in levels and (code == major_code or len(code) < 6 and major_code.startswith(code)):
levels[len(code)].append(item)
result = []
for size in (6, 4, 2):
if not levels[size]:
continue
for item in levels[size]:
result.append({
"school_code":school_code, "school_name":school_name, "major_code":major_code, "year":year,
"department_id":str(item.get("depart_id", "") or ""), "department_name":item.get("depart_name", "") or "",
"scope":{6:"major",4:"discipline",2:"category"}[size], "raw_code":str(item.get("code", "")),
"total":integer(item.get("total")), "politics":integer(item.get("politics")), "english":integer(item.get("english")),
"special_one":integer(item.get("special_one")), "special_two":integer(item.get("special_two")),
"note":item.get("note", "") or "", "source":"掌上考研", "source_url":"https://www.kaoyan.cn/", "fetched_at":now(), "raw_evidence":item,
})
break
return result
+178
View File
@@ -0,0 +1,178 @@
"""Exchange formats for yzw CSV, YAM SQLite and studio snapshots."""
import csv
import io
import json
import sqlite3
from pathlib import Path
from openpyxl import load_workbook, Workbook
from studio.domain import finish_record, integer, parse_quota
LABELS = {"year": "年份", "school_name": "院校", "school_code": "院校代码", "province": "省份", "level": "院校层次", "major_code": "专业代码", "major_name": "专业", "degree_type": "学位类型", "department_name": "院系", "direction": "研究方向", "study_mode": "学习方式", "exam_type": "考试方式", "advisor": "导师", "quota_text": "招生人数原文", "quota": "明确人数", "quota_scope": "人数口径", "subjects": "考试科目", "special_plans": "专项计划", "score_year": "参考线年份", "score_total": "参考线总分", "score_scope": "分数线口径", "remarks": "备注", "source": "来源", "source_url": "来源链接", "year_verified": "来源年份已核实", "fetched_at": "采集时间"}
ALIASES = {"院校名称": "school_name", "学校名称": "school_name", "专业名称": "major_name", "院系所": "department_name", "拟招生人数": "quota_text", "planned_enrollment": "quota_text", "nzsrsstr": "quota_text", "school_level": "level", "department": "department_name", "research_direction": "direction", "enrollment_text": "quota_text", "enrollment_count": "quota", "exam_subjects": "subjects"}
ALIASES.update({v: k for k, v in LABELS.items()})
def list_field(value):
if isinstance(value, list):
return value
if not value:
return []
if str(value).startswith("["):
try:
decoded = json.loads(value)
if isinstance(decoded, list):
return decoded
except ValueError:
pass
return str(value).split(" | ")
def row_record(raw, year=None, source="文件导入"):
row = {ALIASES.get(str(k).strip(), str(k).strip()): (v if v is not None else "") for k, v in raw.items() if k is not None}
explicit_year = integer(row.get("year")) or integer(year)
if explicit_year is None:
raise ValueError("文件未包含年份,请明确指定这份数据的招生年份")
if not 2000 <= explicit_year <= 2100:
raise ValueError("文件年份无效")
if not row.get("school_name") or not row.get("major_code"):
raise ValueError("文件需包含院校名称和专业代码")
row["major_code"] = str(row["major_code"]).strip().zfill(6)
if not row["major_code"].isdigit() or len(row["major_code"]) != 6:
raise ValueError("文件专业代码应为 6 位数字")
row["school_code"] = str(row.get("school_code", "")).strip()
subjects = list_field(row.get("subjects"))
if not subjects:
subjects = [str(row.get(f"exam_subject{i}") or row.get(f"subject{i}") or row.get(f"考试科目{i}") or "") for i in range(1, 5)]
quota, scope = parse_quota(row.get("quota_text", ""))
if quota is None and row.get("quota") != "":
quota = integer(row.get("quota"))
scope = row.get("quota_scope") or "unspecified"
plans = list_field(row.get("special_plans"))
if row.get("veteran_program") in ("是", "1", 1):
plans.append("退役大学生士兵")
if row.get("shaogu_program") in ("是", "1", 1):
plans.append("少数民族骨干")
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
def import_file(path, year=None):
path = Path(path)
suffix = path.suffix.lower()
scores = []
if suffix in (".db", ".sqlite", ".sqlite3"):
return import_yam(path, year)
if suffix == ".json":
content = json.loads(path.read_text(encoding="utf-8-sig"))
if isinstance(content, dict):
rows = content.get("catalog", content.get("records", []))
scores = content.get("scores", [])
else:
rows = content
if not isinstance(rows, list) or not isinstance(scores, list):
raise ValueError("JSON 应是记录数组或包含 catalog/scores 的工作台快照")
elif suffix == ".csv":
payload = path.read_bytes()
try:
text = payload.decode("utf-8-sig")
except UnicodeDecodeError:
text = payload.decode("gb18030")
rows = list(csv.DictReader(io.StringIO(text)))
elif suffix == ".xlsx":
book = load_workbook(path, read_only=True, data_only=True)
try:
sheet = book.active
if sheet.max_row > 300001 or sheet.max_column > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
values = sheet.iter_rows(values_only=True)
headers = next(values, [])
rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)]
finally:
book.close()
else:
raise ValueError("支持 CSV、XLSX、JSON 和 YAM SQLite 文件")
if len(rows) > 300000:
raise ValueError("一次最多导入 30 万条,请拆分文件")
records = [row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入") for row in rows]
for score in scores:
if not all(k in score for k in ("school_code", "major_code", "year", "scope")):
raise ValueError("快照分数线缺少院校/专业/年份/口径")
score["year"] = int(score["year"])
score["total"] = integer(score.get("total"))
score.setdefault("note", "")
return records, scores
def import_yam(path, year):
conn = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)
conn.row_factory = sqlite3.Row
try:
conn.execute("PRAGMA query_only=ON")
tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
if not {"schools", "departments"}.issubset(tables):
raise ValueError("此 SQLite 不包含 YAM schools/departments 表;请从原软件导出 CSV/JSON")
schools = [dict(row) for row in conn.execute("SELECT * FROM schools")]
lookup = {(str(r["school_id"]), str(r.get("major_code", ""))): r for r in schools}
records, departments = [], {}
for entry in conn.execute("SELECT * FROM departments"):
row = dict(entry)
school = lookup.get((str(row["school_id"]), str(row.get("major_code", "")))) or next((r for r in schools if str(r["school_id"]) == str(row["school_id"])), {})
department = str(row.get("department_id", ""))
departments[(str(row["school_id"]), department)] = row.get("name", "")
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "level": school.get("level", ""), "department_code": department, "department_name": row.get("name", ""), "source": row.get("source") or "YAM 导入"}, year))
scores = []
if "score_lines" in tables:
for entry in conn.execute("SELECT * FROM score_lines"):
row = dict(entry)
if not integer(row.get("year")):
continue
raw = str(row.get("raw_code", ""))
scope = row.get("match_scope") or ({2: "category", 4: "discipline", 6: "major"}.get(len(raw)) if raw else "unknown")
scores.append({**row, "school_code": str(row["school_id"]), "major_code": str(row.get("major_code", "")), "department_id": str(row.get("department_id", "") or ""), "department_name": row.get("raw_department_name") or departments.get((str(row["school_id"]), str(row.get("department_id", ""))), ""), "scope": scope, "note": row.get("note") or "", "source": row.get("source") or "YAM 导入"})
if "admission_plans" in tables:
for entry in conn.execute("SELECT * FROM admission_plans"):
row = dict(entry)
school = lookup.get((str(row["school_id"]), str(row.get("major_code", ""))), {})
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "direction": row.get("research_direction", ""), "direction_code": str(row.get("spe_id") or row.get("plan_id") or ""), "department_code": str(row.get("department_id", "") or ""), "source": row.get("source") or "YAM 导入", "remarks": row.get("note", "")}, year))
return records, scores
except sqlite3.DatabaseError:
raise ValueError("SQLite 文件无法读取,请确认是完整导出的数据库") from None
finally:
conn.close()
def spreadsheet_value(value):
if isinstance(value, (list, dict)):
value = " | ".join(str(v) for v in value) if isinstance(value, list) else json.dumps(value, ensure_ascii=False)
if value is None:
return ""
if isinstance(value, str) and value.lstrip().startswith(("=", "+", "-", "@", "\t", "\r")):
return "'" + value
return value
def export_sheet(records, kind):
rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records]
if kind == "csv":
stream = io.StringIO(newline="")
writer = csv.writer(stream)
writer.writerow(list(LABELS.values()))
writer.writerows(rows)
return stream.getvalue().encode("utf-8-sig")
book = Workbook()
sheet = book.active
sheet.title = "院校专业"
sheet.append(list(LABELS.values()))
for row in rows:
sheet.append(row)
sheet.freeze_panes = "A2"
sheet.auto_filter.ref = sheet.dimensions
from openpyxl.styles import Font, PatternFill, Alignment
for cell in sheet[1]:
cell.font = Font(color="FFFFFF", bold=True)
cell.fill = PatternFill("solid", fgColor="183447")
cell.alignment = Alignment(vertical="center")
for col in sheet.columns:
sheet.column_dimensions[col[0].column_letter].width = 24
output = io.BytesIO()
book.save(output)
return output.getvalue()
+233
View File
@@ -0,0 +1,233 @@
"""Persistent units and cooperative pause/cancel for one local worker."""
import threading
import uuid
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
from studio.domain import PROVINCES, normalize_catalog
from studio.network import ProxyUnavailable, RemoteUnavailable
class Cancelled(Exception):
pass
class JobManager:
def __init__(self, database, settings, transport=None, runner=None):
self.db, self.settings, self.transport = database, settings, transport
self.runner = runner or self._collect
self.controls = {}
self.lock = threading.RLock()
self.closed = False
def get(self, jid=None):
return self.db.job_get(jid)
def start(self, params):
params = {**params}
if params.get("mode") not in ("catalog", "scores", "combined"):
raise ValueError("请选择有效的采集模式")
if not 2000 <= int(params.get("year", 0)) <= 2100:
raise ValueError("请选择目录年份")
if params.get("major_code") and (not str(params["major_code"]).isdigit() or len(str(params["major_code"])) != 6):
raise ValueError("专业代码应为 6 位数字")
provinces = params.get("provinces") or list(PROVINCES)
if any(str(p) not in PROVINCES for p in provinces):
raise ValueError("省份代码无效")
params["provinces"] = list(map(str, provinces))
params["score_years"] = sorted(set(int(y) for y in params.get("score_years", [])))
if any(not 2000 <= y <= 2100 for y in params["score_years"]):
raise ValueError("历史年份无效")
if params["mode"] in ("combined", "scores") and not params["score_years"]:
raise ValueError("请至少选择一个分数线年份")
with self.lock:
self._ensure_available()
jid = uuid.uuid4().hex
self.db.job_create(jid, params)
self._launch(jid)
return self.get(jid)
def _ensure_available(self, same=None):
if self.closed:
raise ValueError("任务服务正在关闭")
for jid, control in self.controls.items():
if jid != same and control["thread"].is_alive():
raise ValueError("已有正在运行或暂停的任务,请完成或取消后再新建")
def _launch(self, jid):
condition = threading.Condition()
control = {"condition": condition, "paused": False, "cancelled": False}
thread = threading.Thread(target=self._run, args=(jid,), daemon=True, name="collector-" + jid[:8])
control["thread"] = thread
self.controls[jid] = control
thread.start()
def _run(self, jid):
job = self.get(jid)
self.db.job_update(jid, status="running", errors=0, message="正在采集")
self.db.log(jid, "info", "任务开始/恢复,将跳过已完成单元")
if self.transport:
self.transport.local.guard = lambda: self.checkpoint(jid)
try:
self.runner(jid, job["params"], self)
self.checkpoint(jid)
errors = self.get(jid)["errors"]
self.db.job_update(jid, status="partial" if errors else "completed", message="部分单元失败,可恢复重试" if errors else "采集完成")
except Cancelled:
self.db.job_update(jid, status="cancelled", message="已取消,已保存数据保留")
except LoginRequired as exc:
self.db.job_update(jid, status="waiting_login", message=str(exc))
self.db.log(jid, "warning", str(exc))
except ProxyUnavailable as exc:
self.db.job_update(jid, status="waiting_proxy", message=str(exc))
self.db.log(jid, "warning", str(exc))
except (RemoteUnavailable, SourceYearMismatch, ValueError) as exc:
self.db.job_update(jid, status="failed", message=str(exc))
self.db.log(jid, "error", str(exc))
except Exception as exc:
self.db.job_update(jid, status="failed", message=f"采集异常({type(exc).__name__}),已保存断点")
self.db.log(jid, "error", f"采集异常类型:{type(exc).__name__}")
finally:
if self.transport:
self.transport.local.guard = None
def checkpoint(self, jid):
control = self.controls.get(jid)
if not control:
return
with control["condition"]:
while control["paused"] and not control["cancelled"]:
control["condition"].wait(.25)
if control["cancelled"]:
raise Cancelled()
def unit_done(self, jid, unit):
return self.db.unit_done(jid, unit)
def complete_unit(self, jid, unit):
self.db.complete_unit(jid, unit)
def pause(self, jid):
control = self.controls.get(jid)
if not control or not control["thread"].is_alive():
raise ValueError("任务当前不在运行")
with control["condition"]:
control["paused"] = True
self.db.job_update(jid, status="paused", message="已暂停,当前请求结束后停止")
return self.get(jid)
def resume(self, jid):
with self.lock:
job = self.get(jid)
if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy"):
raise ValueError("此任务当前不能恢复")
self._ensure_available(jid)
control = self.controls.get(jid)
if control and control["thread"].is_alive():
with control["condition"]:
self.db.job_update(jid, status="running", message="正在继续采集")
control["paused"] = False
control["condition"].notify_all()
else:
self._launch(jid)
return self.get(jid)
def cancel(self, jid):
if not self.get(jid):
raise ValueError("任务不存在")
control = self.controls.get(jid)
if control and control["thread"].is_alive():
with control["condition"]:
control["cancelled"] = True
control["condition"].notify_all()
self.db.job_update(jid, message="正在取消,等待当前请求结束")
else:
self.db.job_update(jid, status="cancelled", message="已取消")
return self.get(jid)
def close(self):
self.closed = True
for jid, control in list(self.controls.items()):
if control["thread"].is_alive():
self.cancel(jid)
for control in list(self.controls.values()):
control["thread"].join(timeout=1)
def _save(self, jid, records):
self.db.upsert_catalog(records)
self.db.job_update(jid, records=self.get(jid)["records"] + len(records))
def _error(self, jid, message):
self.db.job_update(jid, errors=self.get(jid)["errors"] + 1)
self.db.log(jid, "error", message)
def _collect(self, jid, params, manager):
if params["mode"] in ("catalog", "combined"):
self._catalog(jid, params)
if params["mode"] in ("scores", "combined"):
self._scores(jid, params)
def _catalog(self, jid, params):
client = CatalogClient(self.transport)
client.verify_year(params["year"])
for province in params["provinces"]:
self.checkpoint(jid)
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
for school in client.schools(province, params.get("school_keyword", "")):
self.checkpoint(jid)
school = {"ssdm": province, **school}
for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")):
if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]):
continue
unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}"
if self.unit_done(jid, unit):
continue
self.checkpoint(jid)
label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}"
self.db.job_update(jid, message=label)
try:
records = []
for row in client.directions(school, major, params.get("study_mode", "")):
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
self._save(jid, records)
self.complete_unit(jid, unit)
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}")
def _scores(self, jid, params):
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "keyword": params.get("school_keyword", "")}
records = self.db.all_catalog(filters)
targets = {}
for row in records:
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]:
continue
targets[(row["school_code"], row["major_code"])] = row
if not targets and params.get("major_code"):
for row in self.db.schools(params.get("school_keyword", "")):
if row["province"] in [PROVINCES[p] for p in params["provinces"]]:
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
if not targets:
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
client = ScoreClient(self.transport)
for (_, code), record in targets.items():
self.checkpoint(jid)
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")]
if not years:
continue
school_id = client.school_id(school["school_name"])
if school_id is None:
self._error(jid, school["school_name"] + ":掌上考研未找到精确匹配,已跳过")
continue
for year in years:
self.checkpoint(jid)
label = f"{school['school_name']} · {code} · {year} 年"
self.db.job_update(jid, message=label)
try:
scores = client.score_lines(school, code, year, school_id)
self.db.upsert_scores(scores)
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
self._save(jid, plans)
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}")
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}")
+149
View File
@@ -0,0 +1,149 @@
"""One explicit route for CHSI login, catalog and historical data."""
import threading
import time
import requests
from studio.settings import proxy_url
class ProxyUnavailable(RuntimeError):
pass
class RemoteUnavailable(RuntimeError):
pass
class ProxyRouter:
def __init__(self, settings, session=None):
self.settings = settings
self.session = session or requests.Session()
self.session.trust_env = False
self.active = None
self.failed = set()
self.candidates = []
self.last_error = ""
self.counts = {}
self.fallback = False
self.lock = threading.RLock()
def refresh(self):
with self.lock:
config = self.settings.read()
try:
response = self.session.get(config["pool_url"] + "/all/", params={"type": "https"}, timeout=5)
response.raise_for_status()
payload = response.json()
if not isinstance(payload, list):
raise ValueError("代理池返回格式不正确")
proxies = []
for entry in payload:
# The pool already filters HTTPS CONNECT-capable entries.
value = entry.get("proxy") if isinstance(entry, dict) else entry
if value:
try:
proxies.append(proxy_url(value))
except ValueError:
continue
self.last_error = ""
try:
self.counts = self.session.get(config["pool_url"] + "/count/", timeout=5).json()
except (requests.RequestException, ValueError):
self.counts = {"https_available": len(proxies)}
except (requests.RequestException, ValueError):
proxies = []
self.last_error = "无法连接代理池 API,请检查地址和服务状态"
self.counts = {}
self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"]))
return self.status()
def resolve(self):
with self.lock:
config = self.settings.read()
if config["network_mode"] == "direct":
self.fallback = False
return None
if self.active and self.active not in self.failed:
self.fallback = False
return self.active
available = [p for p in self.candidates if p not in self.failed]
if not available:
self.refresh()
available = [p for p in self.candidates if p not in self.failed]
if available:
self.active = available[0]
self.fallback = False
return self.active
self.active = None
if config["network_mode"] == "proxy_only":
self.fallback = False
raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务")
self.fallback = True
return None
def rotate(self, failed=None):
with self.lock:
old = self.active
if failed:
self.failed.add(failed)
self.active = None
if old in self.candidates:
self.candidates.remove(old)
self.candidates.append(old)
return self.resolve()
def reset(self):
with self.lock:
self.active = None
self.failed.clear()
self.candidates = []
self.fallback = False
def status(self):
config = self.settings.read()
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
class Transport:
def __init__(self, settings, router, session=None, wait=None):
self.settings, self.router = settings, router
self.session = session or requests.Session()
self.session.trust_env = False
self.session.headers = getattr(self.session, "headers", {})
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
self.wait = wait or time.sleep
self.local = threading.local()
self.lock = threading.RLock()
def sleep(self, seconds):
remaining = seconds
while remaining > 0:
if guard := getattr(self.local, "guard", None):
guard()
step = min(.25, remaining)
self.wait(step)
remaining -= step
def request(self, method, url, **kwargs):
config = self.settings.read()
for attempt in range(config["retries"]):
self.sleep(config["request_interval"] if attempt == 0 else min(20, 2 ** attempt))
proxy = self.router.resolve()
kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {}
kwargs.setdefault("timeout", config["timeout"])
try:
with self.lock:
response = self.session.request(method, url, **kwargs)
if response.status_code in (429, 503):
# Rate limiting does not prove an IP is broken.
self.sleep(min(30, 5 * (attempt + 1)))
continue
response.raise_for_status()
return response
except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout):
if proxy:
self.router.rotate(failed=proxy)
if attempt + 1 == config["retries"]:
raise RemoteUnavailable("网络连接失败,已尝试切换代理;请检查网络或恢复任务") from None
except requests.HTTPError as exc:
raise RemoteUnavailable(f"数据源返回 HTTP {exc.response.status_code}") from None
raise RemoteUnavailable("数据源限流,请降低请求频率后重试")
+62
View File
@@ -0,0 +1,62 @@
"""Project-local, non-secret configuration."""
import json
import threading
from pathlib import Path
from urllib.parse import urlparse
DEFAULTS = {
"network_mode": "prefer_proxy", "pool_url": "http://127.0.0.1:5010",
"pool_project": r"C:\Users\Jayson\Desktop\proxy_pool",
"backup_proxies": [], "request_interval": 1.2, "timeout": 30, "retries": 3,
}
def proxy_url(value):
value = str(value).strip()
if not value:
return ""
value = value if "://" in value else "http://" + value
parsed = urlparse(value)
try:
port = parsed.port
except ValueError:
port = None
if parsed.scheme not in ("http", "https", "socks5") or not parsed.hostname or not port:
raise ValueError("代理格式应为 IP:端口或 http://IP:端口")
if parsed.username or parsed.password:
raise ValueError("此版本请使用不带账号密码的代理地址")
return value
class SettingsStore:
def __init__(self, directory):
self.path = Path(directory) / "settings.json"
self.path.parent.mkdir(parents=True, exist_ok=True)
self.lock = threading.RLock()
def read(self):
with self.lock:
values = json.loads(self.path.read_text(encoding="utf-8")) if self.path.exists() else {}
return {**DEFAULTS, **{k: v for k, v in values.items() if k in DEFAULTS}}
def update(self, values):
with self.lock:
values = {**self.read(), **{k: v for k, v in values.items() if k in DEFAULTS}}
if values["network_mode"] not in ("direct", "prefer_proxy", "proxy_only"):
raise ValueError("未知网络策略")
parsed = urlparse(values["pool_url"])
if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:
raise ValueError("代理池 API 地址无效")
values["pool_url"] = values["pool_url"].rstrip("/")
for key, low, high in [("request_interval", .3, 60), ("timeout", 5, 120), ("retries", 1, 6)]:
values[key] = float(values[key]) if key == "request_interval" else int(values[key])
if not low <= values[key] <= high:
raise ValueError(f"{key} 应在 {low} 到 {high} 之间")
proxies = values["backup_proxies"]
if isinstance(proxies, str):
proxies = proxies.splitlines()
values["backup_proxies"] = list(dict.fromkeys(proxy_url(p) for p in proxies if str(p).strip()))
temporary = self.path.with_suffix(".tmp")
temporary.write_text(json.dumps(values, ensure_ascii=False, indent=2), encoding="utf-8")
temporary.replace(self.path)
return values
+1
View File
@@ -0,0 +1 @@
+68
View File
@@ -0,0 +1,68 @@
import io
import json
import sqlite3
import pytest
from fastapi.testclient import TestClient
from studio.api import create_app
@pytest.fixture
def client(tmp_path):
with TestClient(create_app(tmp_path), base_url="http://127.0.0.1:8765") as client:
yield client
def csv_upload(client, year=2026):
text = 'school_name,school_code,major_code,major_name,department_name,nzsrsstr,subject1,subject2,subject3,subject4\n北京大学,10001,081200,计算机科学与技术,信息学院,专业:6(不含推免),101思想政治理论,201英语一,301数学一,408计算机学科专业基础\n'
return client.post("/api/import", files={"file": ("yzw.csv", text.encode("utf-8-sig"))}, data={"year": year})
def test_csv_upload_filter_favorite_and_export(client):
assert csv_upload(client).json()["records"] == 1
rows = client.get("/api/catalog", params={"year": 2026, "subject": "408"}).json()
assert rows["total"] == 1
record = rows["items"][0]
assert record["quota"] == 6
assert client.put(f'/api/catalog/{record["id"]}/favorite', json={"enabled": True}).status_code == 200
assert client.get("/api/catalog", params={"favorites": True}).json()["total"] == 1
exported = client.get("/api/export", params={"format": "json", "year": 2026}).json()
assert exported["catalog"][0]["year"] == 2026
assert client.get("/api/export", params={"format": "xlsx"}).content.startswith(b"PK")
def test_import_does_not_invent_year_when_csv_has_no_year(client):
content = b"school_name,major_code\nTest,081200\n"
response = client.post("/api/import", files={"file": ("data.csv", content)})
assert response.status_code == 400
assert "年份" in response.json()["detail"]
def test_cross_origin_write_is_blocked_and_secrets_are_not_in_settings(client):
response = client.put("/api/settings", json={"cookie": "secret", "network_mode": "direct"}, headers={"Origin": "https://untrusted.example"})
assert response.status_code == 403
assert "cookie" not in client.get("/api/settings").json()
def test_export_prevents_spreadsheet_formula_execution(client):
data = {"catalog": [{"year": 2026, "school_name": "=1+1", "major_code": "081200", "major_name": "test"}]}
response = client.post("/api/import", files={"file": ("studio.json", json.dumps(data).encode())})
assert response.status_code == 200
response = client.get("/api/export", params={"format": "csv"})
assert "'=1+1" in response.content.decode("utf-8-sig")
def test_import_yam_sqlite_keeps_year_and_score_scope(client, tmp_path):
path = tmp_path / "yam.db"
conn = sqlite3.connect(path)
conn.executescript('CREATE TABLE schools(school_id TEXT,major_code TEXT,name TEXT,province TEXT); CREATE TABLE departments(department_id TEXT,school_id TEXT,major_code TEXT,name TEXT,research_direction TEXT,exam_subjects TEXT,enrollment_text TEXT); CREATE TABLE score_lines(school_id TEXT,department_id TEXT,major_code TEXT,year INTEGER,total INTEGER,match_scope TEXT,raw_code TEXT);')
conn.execute("INSERT INTO schools VALUES('10001','081200','北京大学','北京')")
conn.execute("INSERT INTO departments VALUES('001','10001','081200','信息学院','人工智能','[\"201英语一\"]','专业:6')")
conn.execute("INSERT INTO score_lines VALUES('10001','001','081200',2025,310,'discipline','0812')")
conn.commit()
conn.close()
response = client.post("/api/import", files={"file": ("yam.db", path.read_bytes())}, data={"year": 2026})
assert response.status_code == 200, response.text
rows = client.get("/api/catalog").json()["items"]
assert rows[0]["year"] == 2026
assert rows[0]["score_year"] == 2025
assert rows[0]["score_scope"] == "discipline"
+74
View File
@@ -0,0 +1,74 @@
import threading
import time
import pytest
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
from tests.test_network import Response
class CatalogTransport:
def request(self, method, url, **kwargs):
if method == "GET":
return Response({}, text="2027年硕士专业目录查询")
page = int(kwargs["data"].get("curPage", 1))
rows = [{"dwdm": "10001", "dwmc": "北京大学"}] if page == 1 else [{"dwdm": "10002", "dwmc": "中国人民大学"}]
return Response({"flag": True, "msg": {"list": rows, "nextPageAvailable": page == 1, "totalCount": 2}})
def test_catalog_pagination_consumes_all_pages():
client = CatalogClient(CatalogTransport())
assert [row["dwdm"] for row in client.schools("11")] == ["10001", "10002"]
def test_false_login_business_status_cannot_be_success():
class Unauthenticated(CatalogTransport):
def request(self, *args, **kwargs):
return Response({"flag": False, "msg": "请登录"})
with pytest.raises(LoginRequired):
list(CatalogClient(Unauthenticated()).schools("11"))
def test_source_year_mismatch_prevents_wrongly_dated_catalog():
with pytest.raises(SourceYearMismatch):
CatalogClient(CatalogTransport()).verify_year(2026)
def test_school_lookup_rejects_ambiguous_or_wrong_campus():
class SearchTransport:
def request(self, *args, **kwargs):
return Response({"code": "0000", "data": {"data": [{"school_name": "中国地质大学(武汉)", "school_id": 10}]}})
assert ScoreClient(SearchTransport()).school_id("中国地质大学(北京)") is None
def test_task_pause_resume_reuses_persisted_work_without_duplicate_entries(tmp_path):
db = Database(tmp_path / "studio.db")
settings = SettingsStore(tmp_path)
entered, release = threading.Event(), threading.Event()
def runner(job_id, params, manager):
for unit in ["one", "two"]:
manager.checkpoint(job_id)
if manager.unit_done(job_id, unit):
continue
if unit == "one":
entered.set()
release.wait(3)
manager.complete_unit(job_id, unit)
jobs = JobManager(db, settings, runner=runner)
job = jobs.start({"mode": "catalog", "year": 2027})
assert entered.wait(3)
jobs.pause(job["id"])
release.set()
time.sleep(.1)
assert jobs.get(job["id"])["status"] == "paused"
jobs.resume(job["id"])
deadline = time.monotonic() + 3
while jobs.get(job["id"])["status"] != "completed" and time.monotonic() < deadline:
time.sleep(.02)
assert jobs.get(job["id"])["status"] == "completed"
assert jobs.get(job["id"])["completed_units"] == 2
jobs.close()
+93
View File
@@ -0,0 +1,93 @@
import requests
import pytest
from studio.network import ProxyRouter, Transport, ProxyUnavailable
from studio.settings import SettingsStore
from studio.auth import CookieVault
class Response:
def __init__(self, data, status=200, text=""):
self.data = data
self.status_code = status
self.text = text
self.headers = {}
self.url = "https://yz.chsi.com.cn/zsml/"
def json(self):
return self.data
def raise_for_status(self):
if self.status_code >= 400:
raise requests.HTTPError(response=self)
class Pool:
def __init__(self, proxies=None):
self.proxies = proxies if proxies is not None else [{"proxy": "1.2.3.4:8080", "https": True}, {"proxy": "5.6.7.8:9090", "https": True}]
def get(self, url, **kwargs):
if url.endswith("/all/"):
return Response(self.proxies)
if url.endswith("/count/"):
return Response({"count": len(self.proxies), "http_type": {"https": len(self.proxies)}})
return Response(self.proxies[0] if self.proxies else {"code": 0, "src": "no proxy"})
class Remote:
def __init__(self, fail_once=False):
self.fail_once = fail_once
self.cookies = requests.cookies.RequestsCookieJar()
def request(self, method, url, **kwargs):
if self.fail_once:
self.fail_once = False
raise requests.exceptions.ProxyError("failed")
return Response({"routed_through": (kwargs.get("proxies") or {}).get("https"), "url": url})
def setup_network(tmp_path, mode="proxy_only", pool=None, remote=None):
settings = SettingsStore(tmp_path)
settings.update({"network_mode": mode, "request_interval": 0.3})
router = ProxyRouter(settings, session=pool or Pool())
transport = Transport(settings, router, session=remote or Remote(), wait=lambda _: None)
return settings, router, transport
def test_catalog_and_score_requests_share_the_selected_proxy(tmp_path):
_, router, transport = setup_network(tmp_path)
first = transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/dws.do").json()
second = transport.request("POST", "https://api.kaoyan.cn/pc/school/schoolScore").json()
assert first["routed_through"] == second["routed_through"]
assert first["routed_through"] in ["http://1.2.3.4:8080", "http://5.6.7.8:9090"]
def test_strict_proxy_mode_never_falls_back_to_direct_connection(tmp_path):
_, _, transport = setup_network(tmp_path, pool=Pool([]))
with pytest.raises(ProxyUnavailable):
transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/dws.do")
def test_prefer_proxy_mode_exposes_direct_fallback(tmp_path):
_, router, transport = setup_network(tmp_path, mode="prefer_proxy", pool=Pool([]))
result = transport.request("GET", "https://api.kaoyan.cn/pc/school/schoolList").json()
assert result["routed_through"] is None
assert router.status()["route"] == "direct"
assert router.status()["fallback"] is True
def test_network_failure_rotates_to_another_proxy(tmp_path):
_, router, transport = setup_network(tmp_path, remote=Remote(fail_once=True))
original = router.resolve()
result = transport.request("GET", "https://yz.chsi.com.cn/zsml/").json()
assert result["routed_through"] != original
assert len(router.failed) == 1
def test_cookie_is_encrypted_and_not_returned_in_settings(tmp_path):
vault = CookieVault(tmp_path)
vault.save({"JSESSIONID": "secret-test-cookie", "CASTGC": "ticket"})
assert vault.load()["JSESSIONID"] == "secret-test-cookie"
assert b"secret-test-cookie" not in vault.path.read_bytes()
assert "cookie" not in SettingsStore(tmp_path).read()