Fix historical pagination and bound export and proxy memory

This commit is contained in:
2026-10-13 00:31:34 +08:00
parent 4c510759fb
commit ec2c897752
20 changed files with 1050 additions and 87 deletions
+65 -25
View File
@@ -2,6 +2,8 @@
import json
import uuid
import os
import tempfile
import threading
from contextlib import asynccontextmanager
from pathlib import Path
from urllib.parse import urlparse
@@ -11,7 +13,9 @@ from fastapi.staticfiles import StaticFiles
from studio.auth import AuthService, CookieVault
from studio.database import Database
from studio.domain import PROVINCES
from studio.imports import import_file, export_sheet
from studio.imports import import_file, write_xlsx
from studio.exporting import ExportStream, csv_chunks, json_chunks, file_chunks
from studio.memory import release_idle_memory
from studio.jobs import JobManager
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
from studio.settings import SettingsStore
@@ -20,6 +24,31 @@ from studio.pool_service import PoolService
ROOT = Path(__file__).resolve().parent.parent
class LocalAccessMiddleware:
"""Check hosts and origins without buffering or wrapping response streams."""
def __init__(self, app, allowed_hosts):
self.app = app
self.allowed_hosts = allowed_hosts
async def __call__(self, scope, receive, send):
if scope['type'] != 'http':
return await self.app(scope, receive, send)
request = Request(scope)
if request.url.hostname not in self.allowed_hosts:
response = JSONResponse({'detail': '访问主机不在允许列表中'}, status_code=403)
return await response(scope, receive, send)
if request.method not in ('GET', 'HEAD', 'OPTIONS'):
origin = request.headers.get('origin')
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip('/'), 'http://127.0.0.1:5173', 'http://localhost:5173'}
if parsed.scheme not in ('http', 'https') or origin.rstrip('/') not in allowed:
response = JSONResponse({'detail': '已拒绝跨来源修改请求'}, status_code=403)
return await response(scope, receive, send)
return await self.app(scope, receive, send)
def create_app(data_dir=None):
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
directory.mkdir(parents=True, exist_ok=True)
@@ -36,6 +65,9 @@ def create_app(data_dir=None):
transport = Transport(settings, router)
auth = AuthService(transport, CookieVault(directory))
jobs = JobManager(db, settings, transport, auth=auth)
export_lock = threading.BoundedSemaphore(1)
export_directory = directory / 'exports'
export_directory.mkdir(exist_ok=True)
if (ROOT / "data/schools.json").exists():
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
@@ -46,6 +78,8 @@ def create_app(data_dir=None):
jobs.resume(jid)
yield
jobs.close()
transport.close()
router.session.close()
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
app.state.db, app.state.jobs = db, jobs
@@ -53,19 +87,7 @@ def create_app(data_dir=None):
app.state.pool = pool
app.state.auth = auth
@app.middleware("http")
async def local_access(request: Request, call_next):
host = request.url.hostname
if host not in allowed_hosts:
return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
if request.method not in ("GET", "HEAD", "OPTIONS"):
origin = request.headers.get("origin")
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
return await call_next(request)
app.add_middleware(LocalAccessMiddleware, allowed_hosts=allowed_hosts)
@app.exception_handler(ValueError)
async def invalid(request, exc):
@@ -216,17 +238,35 @@ def create_app(data_dir=None):
def export(request: Request, format: str = "csv"):
if format not in ("csv", "xlsx", "json"):
raise ValueError("导出格式无效")
rows = db.all_catalog(filters(request))
if format == "json":
pairs = {(r["school_code"], r["major_code"]) for r in rows}
with db.connection() as conn:
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs]
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode()
mime = "application/json"
else:
payload = export_sheet(rows, format)
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'})
selected = filters(request)
if not export_lock.acquire(False):
raise HTTPException(429, '已有导出正在进行,请等待完成后重试')
temporary = None
def cleanup():
try:
if temporary is not None:
temporary.unlink(missing_ok=True)
finally:
export_lock.release()
try:
if format == 'csv':
source, mime = csv_chunks(db.iter_catalog(selected)), 'text/csv; charset=utf-8'
elif format == 'json':
source, mime = json_chunks(db, selected), 'application/json'
else:
descriptor, path = tempfile.mkstemp(prefix='kaoyan-', suffix='.xlsx', dir=export_directory)
os.close(descriptor)
temporary = Path(path)
write_xlsx(db.iter_catalog(selected), temporary)
source = file_chunks(temporary)
mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
return ExportStream(source, cleanup, media_type=mime,
headers={'Content-Disposition': f'attachment; filename="kaoyan-data.{format}"',
'X-Accel-Buffering': 'no'})
except BaseException:
cleanup()
release_idle_memory()
raise
dist = ROOT / "frontend/dist"
if (dist / "assets").exists():