feat: implement unified data sources, proxy routing and local API

This commit is contained in:
Codex
2026-10-11 13:49:07 +08:00
parent 25cccd4372
commit 389b2aa3a5
20 changed files with 21601 additions and 4 deletions
+230
View File
@@ -0,0 +1,230 @@
"""Local-only HTTP workbench and production SPA host."""
import json
import subprocess
import uuid
from contextlib import asynccontextmanager
from pathlib import Path
from urllib.parse import urlparse
from fastapi import FastAPI, HTTPException, Request, UploadFile, File, Form
from fastapi.responses import FileResponse, JSONResponse, Response
from fastapi.staticfiles import StaticFiles
from studio.auth import AuthService, CookieVault
from studio.database import Database
from studio.domain import PROVINCES
from studio.imports import import_file, export_sheet
from studio.jobs import JobManager
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable
from studio.settings import SettingsStore
ROOT = Path(__file__).resolve().parent.parent
def create_app(data_dir=None):
directory = Path(data_dir or ROOT / ".runtime")
directory.mkdir(parents=True, exist_ok=True)
db = Database(directory / "studio.db")
settings = SettingsStore(directory)
router = ProxyRouter(settings)
transport = Transport(settings, router)
auth = AuthService(transport, CookieVault(directory))
jobs = JobManager(db, settings, transport)
if (ROOT / "data/schools.json").exists():
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
@asynccontextmanager
async def lifespan(app):
yield
jobs.close()
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
app.state.db, app.state.jobs = db, jobs
app.state.settings, app.state.router = settings, router
@app.middleware("http")
async def local_access(request: Request, call_next):
host = request.url.hostname
if host not in ("127.0.0.1", "localhost", "::1"):
return JSONResponse({"detail": "仅允许本机访问"}, status_code=403)
if request.method not in ("GET", "HEAD", "OPTIONS"):
origin = request.headers.get("origin")
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
return await call_next(request)
@app.exception_handler(ValueError)
async def invalid(request, exc):
return JSONResponse({"detail": str(exc)}, status_code=400)
@app.exception_handler(ProxyUnavailable)
@app.exception_handler(RemoteUnavailable)
async def remote_error(request, exc):
return JSONResponse({"detail": str(exc)}, status_code=503)
def filters(request):
values = dict(request.query_params)
for key in ("year", "min_quota", "max_score"):
values[key] = int(values[key]) if values.get(key) else None
values["favorites"] = values.get("favorites") in ("true", "1")
return values
@app.get("/api/health")
def health():
return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve())}
@app.get("/api/overview")
def overview():
return {**db.overview(), "provinces_list": [{"code": k, "name": v} for k, v in PROVINCES.items()], "sources": [{"name": "研招网", "role": "全国院校 · 专业 · 方向 · 科目", "url": "https://yz.chsi.com.cn/zsml/"}, {"name": "掌上考研", "role": "历年分数线 · 招生计划", "url": "https://www.kaoyan.cn/"}]}
@app.get("/api/majors")
def majors():
path = ROOT / "data/majors.json"
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else []
@app.get("/api/schools")
def schools(keyword: str = "", province: str = ""):
return db.schools(keyword, province)
@app.get("/api/catalog")
def catalog(request: Request, page: int = 1, page_size: int = 30):
return db.query_catalog(filters(request), page, page_size)
@app.get("/api/catalog/{rid}")
def detail(rid: int):
result = db.history(rid)
if result is None:
raise HTTPException(404, "记录不存在")
return result
@app.put("/api/catalog/{rid}/favorite")
def favorite(rid: int, body: dict):
if not db.favorite(rid, bool(body.get("enabled"))):
raise HTTPException(404, "记录不存在")
return {"ok": True}
@app.get("/api/jobs")
def list_jobs():
return jobs.get()
@app.post("/api/jobs")
def start_job(body: dict):
return jobs.start(body)
@app.get("/api/jobs/{jid}/logs")
def job_logs(jid: str, after: int = 0):
return db.logs(jid, after)
@app.post("/api/jobs/{jid}/{action}")
def job_action(jid: str, action: str):
if action not in ("pause", "resume", "cancel"):
raise HTTPException(404, "操作不存在")
return getattr(jobs, action)(jid)
@app.get("/api/settings")
def get_settings():
return settings.read()
@app.put("/api/settings")
def put_settings(body: dict):
result = settings.update(body)
router.reset()
return result
@app.get("/api/auth")
def auth_status():
return auth.status()
@app.post("/api/auth/{action}")
def auth_action(action: str, body: dict = {}):
if action == "cookie":
return auth.cookie(body.get("cookie", ""))
if action == "login":
return auth.login(body.get("username", ""), body.get("password", ""))
if action in ("check", "browser", "clear"):
return getattr(auth, action)()
raise HTTPException(404, "操作不存在")
@app.get("/api/proxy")
def proxy_status(refresh: bool = False):
if refresh:
router.refresh()
try:
router.resolve()
except ProxyUnavailable:
pass
return router.status()
@app.post("/api/proxy/rotate")
def rotate():
router.rotate()
return router.status()
@app.post("/api/proxy/manage")
def manage(body: dict):
action = body.get("action")
if action not in ("start", "stop", "status"):
raise ValueError("代理池操作无效")
script = Path(settings.read()["pool_project"]) / "manage.ps1"
if not script.is_file():
raise ValueError("代理池目录没有 manage.ps1,请检查配置;也可手动启动原代理池")
result = subprocess.run(["powershell.exe", "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", str(script), action], capture_output=True, timeout=55, creationflags=subprocess.CREATE_NO_WINDOW)
output = (result.stdout + result.stderr).decode("utf-8", errors="replace")[-5000:]
router.reset()
return {"ok": result.returncode == 0, "output": output, "action": action}
@app.post("/api/import")
def upload(file: UploadFile = File(...), year: int | None = Form(None)):
suffix = Path(file.filename or "").suffix.lower()
folder = directory / "imports"
folder.mkdir(exist_ok=True)
temporary = folder / (uuid.uuid4().hex + suffix)
try:
total = 0
with temporary.open("wb") as stream:
while chunk := file.file.read(1024 * 1024):
total += len(chunk)
if total > 100 * 1024 * 1024:
raise ValueError("一次最多上传 100 MB,请拆分文件")
stream.write(chunk)
records, scores = import_file(temporary, year)
db.upsert_catalog(records)
db.upsert_scores(scores)
return {"records": len(records), "scores": len(scores), "message": "导入完成,相同记录按年份更新"}
except (UnicodeError, json.JSONDecodeError, KeyError) as exc:
raise ValueError(f"文件编码或结构无法识别({type(exc).__name__})") from None
finally:
temporary.unlink(missing_ok=True)
file.file.close()
@app.get("/api/export")
def export(request: Request, format: str = "csv"):
if format not in ("csv", "xlsx", "json"):
raise ValueError("导出格式无效")
rows = db.all_catalog(filters(request))
if format == "json":
pairs = {(r["school_code"], r["major_code"]) for r in rows}
with db.connection() as conn:
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs]
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode()
mime = "application/json"
else:
payload = export_sheet(rows, format)
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'})
dist = ROOT / "frontend/dist"
if (dist / "assets").exists():
app.mount("/assets", StaticFiles(directory=dist / "assets"), name="assets")
@app.get("/{path:path}")
def frontend(path: str):
if path.startswith("api/"):
raise HTTPException(404, "API 不存在")
index = dist / "index.html"
if index.exists():
return FileResponse(index)
return Response("请先运行 frontend 的 npm run build,再启动工作台。", media_type="text/plain", status_code=503)
return app