285 lines
11 KiB
Python
285 lines
11 KiB
Python
"""Local-only HTTP workbench and production SPA host."""
|
||
import json
|
||
import uuid
|
||
import os
|
||
import tempfile
|
||
import threading
|
||
from contextlib import asynccontextmanager
|
||
from pathlib import Path
|
||
from urllib.parse import urlparse
|
||
from fastapi import FastAPI, HTTPException, Request, UploadFile, File, Form
|
||
from fastapi.responses import FileResponse, JSONResponse, Response
|
||
from fastapi.staticfiles import StaticFiles
|
||
from studio.auth import AuthService, CookieVault
|
||
from studio.database import Database
|
||
from studio.domain import PROVINCES
|
||
from studio.imports import import_file, write_xlsx
|
||
from studio.exporting import ExportStream, csv_chunks, json_chunks, file_chunks
|
||
from studio.memory import release_idle_memory
|
||
from studio.jobs import JobManager
|
||
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
|
||
from studio.settings import SettingsStore
|
||
from studio.pool_service import PoolService
|
||
|
||
ROOT = Path(__file__).resolve().parent.parent
|
||
|
||
|
||
class LocalAccessMiddleware:
|
||
"""Check hosts and origins without buffering or wrapping response streams."""
|
||
|
||
def __init__(self, app, allowed_hosts):
|
||
self.app = app
|
||
self.allowed_hosts = allowed_hosts
|
||
|
||
async def __call__(self, scope, receive, send):
|
||
if scope['type'] != 'http':
|
||
return await self.app(scope, receive, send)
|
||
request = Request(scope)
|
||
if request.url.hostname not in self.allowed_hosts:
|
||
response = JSONResponse({'detail': '访问主机不在允许列表中'}, status_code=403)
|
||
return await response(scope, receive, send)
|
||
if request.method not in ('GET', 'HEAD', 'OPTIONS'):
|
||
origin = request.headers.get('origin')
|
||
if origin:
|
||
parsed = urlparse(origin)
|
||
allowed = {str(request.base_url).rstrip('/'), 'http://127.0.0.1:5173', 'http://localhost:5173'}
|
||
if parsed.scheme not in ('http', 'https') or origin.rstrip('/') not in allowed:
|
||
response = JSONResponse({'detail': '已拒绝跨来源修改请求'}, status_code=403)
|
||
return await response(scope, receive, send)
|
||
return await self.app(scope, receive, send)
|
||
|
||
|
||
def create_app(data_dir=None):
|
||
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
|
||
directory.mkdir(parents=True, exist_ok=True)
|
||
database_config = os.environ.get("STUDIO_DATABASE_CONFIG")
|
||
if database_config:
|
||
from studio.mysql_database import MySQLDatabase
|
||
db = MySQLDatabase(json.loads(Path(database_config).read_text(encoding="utf-8")))
|
||
else:
|
||
db = Database(directory / "studio.db")
|
||
allowed_hosts = {"127.0.0.1", "localhost", "::1"} | {host.strip().lower() for host in os.environ.get("STUDIO_ALLOWED_HOSTS", "").split(",") if host.strip()}
|
||
settings = SettingsStore(directory)
|
||
pool = PoolService(settings)
|
||
router = ProxyRouter(settings)
|
||
transport = Transport(settings, router)
|
||
auth = AuthService(transport, CookieVault(directory))
|
||
jobs = JobManager(db, settings, transport, auth=auth)
|
||
export_lock = threading.BoundedSemaphore(1)
|
||
export_directory = directory / 'exports'
|
||
export_directory.mkdir(exist_ok=True)
|
||
if (ROOT / "data/schools.json").exists():
|
||
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(app):
|
||
if os.environ.get("STUDIO_AUTO_RESUME") == "1":
|
||
for jid in db.recovered_job_ids:
|
||
jobs.resume(jid)
|
||
yield
|
||
jobs.close()
|
||
transport.close()
|
||
router.session.close()
|
||
|
||
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
|
||
app.state.db, app.state.jobs = db, jobs
|
||
app.state.settings, app.state.router = settings, router
|
||
app.state.pool = pool
|
||
app.state.auth = auth
|
||
|
||
app.add_middleware(LocalAccessMiddleware, allowed_hosts=allowed_hosts)
|
||
|
||
@app.exception_handler(ValueError)
|
||
async def invalid(request, exc):
|
||
return JSONResponse({"detail": str(exc)}, status_code=400)
|
||
|
||
@app.exception_handler(ProxyUnavailable)
|
||
@app.exception_handler(RemoteUnavailable)
|
||
@app.exception_handler(RateLimited)
|
||
async def remote_error(request, exc):
|
||
return JSONResponse({"detail": str(exc)}, status_code=503)
|
||
|
||
def filters(request):
|
||
values = dict(request.query_params)
|
||
for key in ("year", "min_quota", "max_score"):
|
||
values[key] = int(values[key]) if values.get(key) else None
|
||
values["favorites"] = values.get("favorites") in ("true", "1")
|
||
return values
|
||
|
||
@app.get("/api/health")
|
||
def health():
|
||
return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve()), "database": "mysql" if database_config else "sqlite", "platform": "windows" if os.name == "nt" else "linux"}
|
||
|
||
@app.get("/api/overview")
|
||
def overview():
|
||
return {**db.overview(), "provinces_list": [{"code": k, "name": v} for k, v in PROVINCES.items()], "sources": [{"name": "研招网", "role": "全国院校 · 专业 · 方向 · 科目", "url": "https://yz.chsi.com.cn/zsml/"}, {"name": "掌上考研", "role": "历年分数线 · 招生计划", "url": "https://www.kaoyan.cn/"}]}
|
||
|
||
@app.get("/api/majors")
|
||
def majors():
|
||
path = ROOT / "data/majors.json"
|
||
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else []
|
||
|
||
@app.get("/api/schools")
|
||
def schools(keyword: str = "", province: str = ""):
|
||
return db.schools(keyword, province)
|
||
|
||
@app.get("/api/catalog")
|
||
def catalog(request: Request, page: int = 1, page_size: int = 30):
|
||
return db.query_catalog(filters(request), page, page_size)
|
||
|
||
@app.get("/api/catalog/{rid}")
|
||
def detail(rid: int):
|
||
result = db.history(rid)
|
||
if result is None:
|
||
raise HTTPException(404, "记录不存在")
|
||
return result
|
||
|
||
@app.put("/api/catalog/{rid}/favorite")
|
||
def favorite(rid: int, body: dict):
|
||
if not db.favorite(rid, bool(body.get("enabled"))):
|
||
raise HTTPException(404, "记录不存在")
|
||
return {"ok": True}
|
||
|
||
@app.get("/api/jobs")
|
||
def list_jobs():
|
||
return jobs.get()
|
||
|
||
@app.post("/api/jobs")
|
||
def start_job(body: dict):
|
||
return jobs.start(body)
|
||
|
||
@app.get("/api/jobs/{jid}/logs")
|
||
def job_logs(jid: str, after: int = 0):
|
||
return db.logs(jid, after)
|
||
|
||
@app.post("/api/jobs/{jid}/{action}")
|
||
def job_action(jid: str, action: str):
|
||
if action not in ("pause", "resume", "cancel"):
|
||
raise HTTPException(404, "操作不存在")
|
||
return getattr(jobs, action)(jid)
|
||
|
||
@app.get("/api/settings")
|
||
def get_settings():
|
||
return settings.read()
|
||
|
||
@app.put("/api/settings")
|
||
def put_settings(body: dict):
|
||
result = settings.update(body)
|
||
router.reset()
|
||
return result
|
||
|
||
@app.get("/api/auth")
|
||
def auth_status():
|
||
return auth.status()
|
||
|
||
@app.post("/api/auth/{action}")
|
||
def auth_action(action: str, body: dict = {}):
|
||
if action == "cookie":
|
||
return auth.cookie(body.get("cookie", ""))
|
||
if action == "login":
|
||
return auth.login(body.get("username", ""), body.get("password", ""))
|
||
if action in ("check", "browser", "clear"):
|
||
return getattr(auth, action)()
|
||
raise HTTPException(404, "操作不存在")
|
||
|
||
@app.get("/api/proxy")
|
||
def proxy_status(refresh: bool = False):
|
||
if refresh:
|
||
router.refresh()
|
||
try:
|
||
router.resolve()
|
||
except ProxyUnavailable:
|
||
pass
|
||
return router.status()
|
||
|
||
@app.post("/api/proxy/rotate")
|
||
def rotate():
|
||
router.rotate()
|
||
return router.status()
|
||
|
||
@app.get("/api/proxy/services")
|
||
def pool_status():
|
||
return pool.status()
|
||
|
||
@app.post("/api/proxy/manage")
|
||
def manage(body: dict):
|
||
result = pool.manage(body.get("action"))
|
||
if result["changed"]:
|
||
router.reset()
|
||
return result
|
||
|
||
@app.post("/api/import")
|
||
def upload(file: UploadFile = File(...), year: int | None = Form(None)):
|
||
suffix = Path(file.filename or "").suffix.lower()
|
||
folder = directory / "imports"
|
||
folder.mkdir(exist_ok=True)
|
||
temporary = folder / (uuid.uuid4().hex + suffix)
|
||
try:
|
||
total = 0
|
||
with temporary.open("wb") as stream:
|
||
while chunk := file.file.read(1024 * 1024):
|
||
total += len(chunk)
|
||
if total > 100 * 1024 * 1024:
|
||
raise ValueError("一次最多上传 100 MB,请拆分文件")
|
||
stream.write(chunk)
|
||
result = import_file(temporary, year)
|
||
records, scores = result
|
||
db.upsert_catalog(records)
|
||
db.upsert_scores(scores)
|
||
db.index_schools(result.schools)
|
||
return {"records": len(records), "scores": len(scores), "indexed_schools": len(result.schools), "message": "导入完成,相同记录按年份更新;无详情的 YAM 种子只作为院校索引"}
|
||
except (UnicodeError, json.JSONDecodeError, KeyError) as exc:
|
||
raise ValueError(f"文件编码或结构无法识别({type(exc).__name__})") from None
|
||
finally:
|
||
temporary.unlink(missing_ok=True)
|
||
file.file.close()
|
||
|
||
@app.get("/api/export")
|
||
def export(request: Request, format: str = "csv"):
|
||
if format not in ("csv", "xlsx", "json"):
|
||
raise ValueError("导出格式无效")
|
||
selected = filters(request)
|
||
if not export_lock.acquire(False):
|
||
raise HTTPException(429, '已有导出正在进行,请等待完成后重试')
|
||
temporary = None
|
||
def cleanup():
|
||
try:
|
||
if temporary is not None:
|
||
temporary.unlink(missing_ok=True)
|
||
finally:
|
||
export_lock.release()
|
||
try:
|
||
if format == 'csv':
|
||
source, mime = csv_chunks(db.iter_catalog(selected)), 'text/csv; charset=utf-8'
|
||
elif format == 'json':
|
||
source, mime = json_chunks(db, selected), 'application/json'
|
||
else:
|
||
descriptor, path = tempfile.mkstemp(prefix='kaoyan-', suffix='.xlsx', dir=export_directory)
|
||
os.close(descriptor)
|
||
temporary = Path(path)
|
||
write_xlsx(db.iter_catalog(selected), temporary)
|
||
source = file_chunks(temporary)
|
||
mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
||
return ExportStream(source, cleanup, media_type=mime,
|
||
headers={'Content-Disposition': f'attachment; filename="kaoyan-data.{format}"',
|
||
'X-Accel-Buffering': 'no'})
|
||
except BaseException:
|
||
cleanup()
|
||
release_idle_memory()
|
||
raise
|
||
|
||
dist = ROOT / "frontend/dist"
|
||
if (dist / "assets").exists():
|
||
app.mount("/assets", StaticFiles(directory=dist / "assets"), name="assets")
|
||
|
||
@app.get("/{path:path}")
|
||
def frontend(path: str):
|
||
if path.startswith("api/"):
|
||
raise HTTPException(404, "API 不存在")
|
||
index = dist / "index.html"
|
||
if index.exists():
|
||
return FileResponse(index)
|
||
return Response("请先运行 frontend 的 npm run build,再启动工作台。", media_type="text/plain", status_code=503)
|
||
|
||
return app
|