Fix historical pagination and bound export and proxy memory
This commit is contained in:
+65
-25
@@ -2,6 +2,8 @@
|
||||
import json
|
||||
import uuid
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
@@ -11,7 +13,9 @@ from fastapi.staticfiles import StaticFiles
|
||||
from studio.auth import AuthService, CookieVault
|
||||
from studio.database import Database
|
||||
from studio.domain import PROVINCES
|
||||
from studio.imports import import_file, export_sheet
|
||||
from studio.imports import import_file, write_xlsx
|
||||
from studio.exporting import ExportStream, csv_chunks, json_chunks, file_chunks
|
||||
from studio.memory import release_idle_memory
|
||||
from studio.jobs import JobManager
|
||||
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||
from studio.settings import SettingsStore
|
||||
@@ -20,6 +24,31 @@ from studio.pool_service import PoolService
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
class LocalAccessMiddleware:
|
||||
"""Check hosts and origins without buffering or wrapping response streams."""
|
||||
|
||||
def __init__(self, app, allowed_hosts):
|
||||
self.app = app
|
||||
self.allowed_hosts = allowed_hosts
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope['type'] != 'http':
|
||||
return await self.app(scope, receive, send)
|
||||
request = Request(scope)
|
||||
if request.url.hostname not in self.allowed_hosts:
|
||||
response = JSONResponse({'detail': '访问主机不在允许列表中'}, status_code=403)
|
||||
return await response(scope, receive, send)
|
||||
if request.method not in ('GET', 'HEAD', 'OPTIONS'):
|
||||
origin = request.headers.get('origin')
|
||||
if origin:
|
||||
parsed = urlparse(origin)
|
||||
allowed = {str(request.base_url).rstrip('/'), 'http://127.0.0.1:5173', 'http://localhost:5173'}
|
||||
if parsed.scheme not in ('http', 'https') or origin.rstrip('/') not in allowed:
|
||||
response = JSONResponse({'detail': '已拒绝跨来源修改请求'}, status_code=403)
|
||||
return await response(scope, receive, send)
|
||||
return await self.app(scope, receive, send)
|
||||
|
||||
|
||||
def create_app(data_dir=None):
|
||||
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
@@ -36,6 +65,9 @@ def create_app(data_dir=None):
|
||||
transport = Transport(settings, router)
|
||||
auth = AuthService(transport, CookieVault(directory))
|
||||
jobs = JobManager(db, settings, transport, auth=auth)
|
||||
export_lock = threading.BoundedSemaphore(1)
|
||||
export_directory = directory / 'exports'
|
||||
export_directory.mkdir(exist_ok=True)
|
||||
if (ROOT / "data/schools.json").exists():
|
||||
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
|
||||
|
||||
@@ -46,6 +78,8 @@ def create_app(data_dir=None):
|
||||
jobs.resume(jid)
|
||||
yield
|
||||
jobs.close()
|
||||
transport.close()
|
||||
router.session.close()
|
||||
|
||||
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
|
||||
app.state.db, app.state.jobs = db, jobs
|
||||
@@ -53,19 +87,7 @@ def create_app(data_dir=None):
|
||||
app.state.pool = pool
|
||||
app.state.auth = auth
|
||||
|
||||
@app.middleware("http")
|
||||
async def local_access(request: Request, call_next):
|
||||
host = request.url.hostname
|
||||
if host not in allowed_hosts:
|
||||
return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
|
||||
if request.method not in ("GET", "HEAD", "OPTIONS"):
|
||||
origin = request.headers.get("origin")
|
||||
if origin:
|
||||
parsed = urlparse(origin)
|
||||
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
|
||||
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
|
||||
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
|
||||
return await call_next(request)
|
||||
app.add_middleware(LocalAccessMiddleware, allowed_hosts=allowed_hosts)
|
||||
|
||||
@app.exception_handler(ValueError)
|
||||
async def invalid(request, exc):
|
||||
@@ -216,17 +238,35 @@ def create_app(data_dir=None):
|
||||
def export(request: Request, format: str = "csv"):
|
||||
if format not in ("csv", "xlsx", "json"):
|
||||
raise ValueError("导出格式无效")
|
||||
rows = db.all_catalog(filters(request))
|
||||
if format == "json":
|
||||
pairs = {(r["school_code"], r["major_code"]) for r in rows}
|
||||
with db.connection() as conn:
|
||||
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs]
|
||||
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode()
|
||||
mime = "application/json"
|
||||
else:
|
||||
payload = export_sheet(rows, format)
|
||||
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'})
|
||||
selected = filters(request)
|
||||
if not export_lock.acquire(False):
|
||||
raise HTTPException(429, '已有导出正在进行,请等待完成后重试')
|
||||
temporary = None
|
||||
def cleanup():
|
||||
try:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
finally:
|
||||
export_lock.release()
|
||||
try:
|
||||
if format == 'csv':
|
||||
source, mime = csv_chunks(db.iter_catalog(selected)), 'text/csv; charset=utf-8'
|
||||
elif format == 'json':
|
||||
source, mime = json_chunks(db, selected), 'application/json'
|
||||
else:
|
||||
descriptor, path = tempfile.mkstemp(prefix='kaoyan-', suffix='.xlsx', dir=export_directory)
|
||||
os.close(descriptor)
|
||||
temporary = Path(path)
|
||||
write_xlsx(db.iter_catalog(selected), temporary)
|
||||
source = file_chunks(temporary)
|
||||
mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
||||
return ExportStream(source, cleanup, media_type=mime,
|
||||
headers={'Content-Disposition': f'attachment; filename="kaoyan-data.{format}"',
|
||||
'X-Accel-Buffering': 'no'})
|
||||
except BaseException:
|
||||
cleanup()
|
||||
release_idle_memory()
|
||||
raise
|
||||
|
||||
dist = ROOT / "frontend/dist"
|
||||
if (dist / "assets").exists():
|
||||
|
||||
Reference in New Issue
Block a user