Fix historical pagination and bound export and proxy memory
This commit is contained in:
+65
-25
@@ -2,6 +2,8 @@
|
||||
import json
|
||||
import uuid
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
@@ -11,7 +13,9 @@ from fastapi.staticfiles import StaticFiles
|
||||
from studio.auth import AuthService, CookieVault
|
||||
from studio.database import Database
|
||||
from studio.domain import PROVINCES
|
||||
from studio.imports import import_file, export_sheet
|
||||
from studio.imports import import_file, write_xlsx
|
||||
from studio.exporting import ExportStream, csv_chunks, json_chunks, file_chunks
|
||||
from studio.memory import release_idle_memory
|
||||
from studio.jobs import JobManager
|
||||
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||
from studio.settings import SettingsStore
|
||||
@@ -20,6 +24,31 @@ from studio.pool_service import PoolService
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
class LocalAccessMiddleware:
|
||||
"""Check hosts and origins without buffering or wrapping response streams."""
|
||||
|
||||
def __init__(self, app, allowed_hosts):
|
||||
self.app = app
|
||||
self.allowed_hosts = allowed_hosts
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope['type'] != 'http':
|
||||
return await self.app(scope, receive, send)
|
||||
request = Request(scope)
|
||||
if request.url.hostname not in self.allowed_hosts:
|
||||
response = JSONResponse({'detail': '访问主机不在允许列表中'}, status_code=403)
|
||||
return await response(scope, receive, send)
|
||||
if request.method not in ('GET', 'HEAD', 'OPTIONS'):
|
||||
origin = request.headers.get('origin')
|
||||
if origin:
|
||||
parsed = urlparse(origin)
|
||||
allowed = {str(request.base_url).rstrip('/'), 'http://127.0.0.1:5173', 'http://localhost:5173'}
|
||||
if parsed.scheme not in ('http', 'https') or origin.rstrip('/') not in allowed:
|
||||
response = JSONResponse({'detail': '已拒绝跨来源修改请求'}, status_code=403)
|
||||
return await response(scope, receive, send)
|
||||
return await self.app(scope, receive, send)
|
||||
|
||||
|
||||
def create_app(data_dir=None):
|
||||
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
@@ -36,6 +65,9 @@ def create_app(data_dir=None):
|
||||
transport = Transport(settings, router)
|
||||
auth = AuthService(transport, CookieVault(directory))
|
||||
jobs = JobManager(db, settings, transport, auth=auth)
|
||||
export_lock = threading.BoundedSemaphore(1)
|
||||
export_directory = directory / 'exports'
|
||||
export_directory.mkdir(exist_ok=True)
|
||||
if (ROOT / "data/schools.json").exists():
|
||||
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
|
||||
|
||||
@@ -46,6 +78,8 @@ def create_app(data_dir=None):
|
||||
jobs.resume(jid)
|
||||
yield
|
||||
jobs.close()
|
||||
transport.close()
|
||||
router.session.close()
|
||||
|
||||
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
|
||||
app.state.db, app.state.jobs = db, jobs
|
||||
@@ -53,19 +87,7 @@ def create_app(data_dir=None):
|
||||
app.state.pool = pool
|
||||
app.state.auth = auth
|
||||
|
||||
@app.middleware("http")
|
||||
async def local_access(request: Request, call_next):
|
||||
host = request.url.hostname
|
||||
if host not in allowed_hosts:
|
||||
return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
|
||||
if request.method not in ("GET", "HEAD", "OPTIONS"):
|
||||
origin = request.headers.get("origin")
|
||||
if origin:
|
||||
parsed = urlparse(origin)
|
||||
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
|
||||
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
|
||||
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
|
||||
return await call_next(request)
|
||||
app.add_middleware(LocalAccessMiddleware, allowed_hosts=allowed_hosts)
|
||||
|
||||
@app.exception_handler(ValueError)
|
||||
async def invalid(request, exc):
|
||||
@@ -216,17 +238,35 @@ def create_app(data_dir=None):
|
||||
def export(request: Request, format: str = "csv"):
|
||||
if format not in ("csv", "xlsx", "json"):
|
||||
raise ValueError("导出格式无效")
|
||||
rows = db.all_catalog(filters(request))
|
||||
if format == "json":
|
||||
pairs = {(r["school_code"], r["major_code"]) for r in rows}
|
||||
with db.connection() as conn:
|
||||
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs]
|
||||
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode()
|
||||
mime = "application/json"
|
||||
else:
|
||||
payload = export_sheet(rows, format)
|
||||
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'})
|
||||
selected = filters(request)
|
||||
if not export_lock.acquire(False):
|
||||
raise HTTPException(429, '已有导出正在进行,请等待完成后重试')
|
||||
temporary = None
|
||||
def cleanup():
|
||||
try:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
finally:
|
||||
export_lock.release()
|
||||
try:
|
||||
if format == 'csv':
|
||||
source, mime = csv_chunks(db.iter_catalog(selected)), 'text/csv; charset=utf-8'
|
||||
elif format == 'json':
|
||||
source, mime = json_chunks(db, selected), 'application/json'
|
||||
else:
|
||||
descriptor, path = tempfile.mkstemp(prefix='kaoyan-', suffix='.xlsx', dir=export_directory)
|
||||
os.close(descriptor)
|
||||
temporary = Path(path)
|
||||
write_xlsx(db.iter_catalog(selected), temporary)
|
||||
source = file_chunks(temporary)
|
||||
mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
||||
return ExportStream(source, cleanup, media_type=mime,
|
||||
headers={'Content-Disposition': f'attachment; filename="kaoyan-data.{format}"',
|
||||
'X-Accel-Buffering': 'no'})
|
||||
except BaseException:
|
||||
cleanup()
|
||||
release_idle_memory()
|
||||
raise
|
||||
|
||||
dist = ROOT / "frontend/dist"
|
||||
if (dist / "assets").exists():
|
||||
|
||||
+43
-8
@@ -2,6 +2,7 @@
|
||||
import json
|
||||
import re
|
||||
import hashlib
|
||||
from collections import OrderedDict
|
||||
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
||||
from studio.network import RemoteUnavailable, RateLimited
|
||||
|
||||
@@ -153,6 +154,7 @@ class ScoreClient:
|
||||
def __init__(self, transport):
|
||||
self.transport = transport
|
||||
self.cache = {}
|
||||
self.score_cache = OrderedDict()
|
||||
|
||||
def post(self, endpoint, payload):
|
||||
for attempt in range(3):
|
||||
@@ -161,8 +163,13 @@ class ScoreClient:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
|
||||
if not isinstance(body, dict):
|
||||
raise RemoteUnavailable("掌上考研响应结构已变化")
|
||||
if str(body.get("code")) == "0000":
|
||||
return body.get("data") or {}
|
||||
data = body.get("data")
|
||||
if not isinstance(data, (dict, list)):
|
||||
raise RemoteUnavailable("掌上考研缺少有效数据,未写入不完整结果")
|
||||
return data
|
||||
message = str(body.get('message', body.get('msg', '')))
|
||||
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
|
||||
if attempt == 2:
|
||||
@@ -172,20 +179,40 @@ class ScoreClient:
|
||||
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
|
||||
|
||||
def pages(self, endpoint, payload, limit=100):
|
||||
seen = set()
|
||||
seen, consumed = set(), 0
|
||||
for page in range(1, 10001):
|
||||
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
|
||||
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data
|
||||
if not isinstance(rows, list):
|
||||
full_list = isinstance(data, list)
|
||||
rows = data.get("data", data.get("list")) if isinstance(data, dict) else data
|
||||
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
|
||||
raise RemoteUnavailable("掌上考研响应结构已变化")
|
||||
total = integer(data.get("total")) if isinstance(data, dict) else None
|
||||
last = integer(data.get("last_page")) if isinstance(data, dict) else None
|
||||
if isinstance(data, dict):
|
||||
for field, value in (('total', total), ('last_page', last)):
|
||||
if data.get(field) is not None and (value is None or value < 0 or isinstance(data[field], bool)):
|
||||
raise RemoteUnavailable("掌上考研分页信息无效,未写入不完整结果")
|
||||
if not rows and (total is not None and consumed < total or last is not None and page < last):
|
||||
raise RemoteUnavailable("掌上考研提前返回空页,未写入不完整结果")
|
||||
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
||||
if rows and signature in seen:
|
||||
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
|
||||
seen.add(signature)
|
||||
consumed += len(rows)
|
||||
yield from rows
|
||||
total = integer(data.get("total")) if isinstance(data, dict) else None
|
||||
last = integer(data.get("last_page")) if isinstance(data, dict) else None
|
||||
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total:
|
||||
# Bare arrays are complete responses; wrapped pages use their actual
|
||||
# metadata even when the server ignores the requested page size.
|
||||
terminal = full_list or not rows
|
||||
if not terminal:
|
||||
if last is not None:
|
||||
terminal = page >= last
|
||||
if terminal and total is not None and consumed < total:
|
||||
raise RemoteUnavailable("掌上考研分页未返回全部记录")
|
||||
elif total is not None:
|
||||
terminal = consumed >= total
|
||||
else:
|
||||
terminal = len(rows) < limit
|
||||
if terminal:
|
||||
return
|
||||
raise RemoteUnavailable("历史数据分页超过安全上限")
|
||||
|
||||
@@ -198,10 +225,18 @@ class ScoreClient:
|
||||
if len(matches) == 1 and matches[0].get("school_id"):
|
||||
self.cache[key] = matches[0]["school_id"]
|
||||
return self.cache[key]
|
||||
self.cache[key] = None
|
||||
return None
|
||||
|
||||
def score_lines(self, school, major_code, year, school_id):
|
||||
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
|
||||
key = (str(school_id), int(year))
|
||||
if key not in self.score_cache:
|
||||
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
|
||||
self.score_cache[key] = rows
|
||||
if len(self.score_cache) > 8:
|
||||
self.score_cache.popitem(last=False)
|
||||
self.score_cache.move_to_end(key)
|
||||
rows = self.score_cache[key]
|
||||
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
|
||||
|
||||
def plans(self, school, major_code, year, school_id):
|
||||
|
||||
+90
-7
@@ -19,6 +19,8 @@ CREATE TABLE IF NOT EXISTS catalog(
|
||||
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
|
||||
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
|
||||
CREATE INDEX IF NOT EXISTS catalog_export ON catalog(year DESC,school_name,id);
|
||||
CREATE INDEX IF NOT EXISTS catalog_targets ON catalog(year,school_code,major_code);
|
||||
""" + SCORE_SCHEMA + """
|
||||
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
|
||||
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
||||
@@ -69,6 +71,10 @@ class Database:
|
||||
conn.close()
|
||||
|
||||
def upsert_catalog(self, records: list[dict]):
|
||||
with self.connection() as conn:
|
||||
self._upsert_catalog(conn, records)
|
||||
|
||||
def _upsert_catalog(self, conn, records):
|
||||
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
|
||||
values = []
|
||||
for record in records:
|
||||
@@ -77,10 +83,9 @@ class Database:
|
||||
if not values:
|
||||
return
|
||||
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
|
||||
def _school(self, conn, record):
|
||||
record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))}
|
||||
@@ -93,8 +98,11 @@ class Database:
|
||||
|
||||
def upsert_scores(self, records):
|
||||
with self.connection() as conn:
|
||||
for row in records:
|
||||
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
|
||||
self._upsert_scores(conn, records)
|
||||
|
||||
def _upsert_scores(self, conn, records):
|
||||
for row in records:
|
||||
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
|
||||
|
||||
def _query(self, filters: dict):
|
||||
conditions, params = ["1=1"], []
|
||||
@@ -141,9 +149,73 @@ class Database:
|
||||
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
|
||||
|
||||
def all_catalog(self, filters):
|
||||
return list(self.iter_catalog(filters))
|
||||
|
||||
def iter_catalog(self, filters, batch_size=200):
|
||||
select, params = self._query(filters)
|
||||
with self.connection() as conn:
|
||||
return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)]
|
||||
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM catalog').fetchone()[0]
|
||||
after = None
|
||||
while True:
|
||||
query = 'SELECT * FROM (' + select + ') export_rows WHERE id<=?'
|
||||
values = [*params, cutoff]
|
||||
if after:
|
||||
year, school, rid = after
|
||||
if school is None:
|
||||
query += ' AND (year<? OR (year=? AND (school_name IS NOT NULL OR (school_name IS NULL AND id>?))))'
|
||||
values.extend([year, year, rid])
|
||||
else:
|
||||
query += ' AND (year<? OR (year=? AND (school_name>? OR (school_name=? AND id>?))))'
|
||||
values.extend([year, year, school, school, rid])
|
||||
query += ' ORDER BY year DESC,school_name,id LIMIT ?'
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(query, [*values, batch_size]).fetchall()
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
yield self._catalog_row(row)
|
||||
last = rows[-1]
|
||||
after = (last['year'], last['school_name'], last['id'])
|
||||
|
||||
def iter_scores(self, pairs, batch_size=200):
|
||||
after = 0
|
||||
with self.connection() as conn:
|
||||
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM scores').fetchone()[0]
|
||||
while True:
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute('SELECT id,school_code,major_code,data FROM scores WHERE id>? AND id<=? ORDER BY id LIMIT ?', (after, cutoff, batch_size)).fetchall()
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
if (row['school_code'], row['major_code']) in pairs:
|
||||
yield json.loads(row['data'])
|
||||
after = rows[-1]['id']
|
||||
|
||||
def score_targets(self, filters, batch_size=200):
|
||||
conditions, params = ['year=?'], [filters['year']]
|
||||
if filters.get('major_code'):
|
||||
conditions.append('major_code=?')
|
||||
params.append(filters['major_code'])
|
||||
if filters.get('school_keyword'):
|
||||
conditions.append('school_name LIKE ?')
|
||||
params.append('%' + filters['school_keyword'] + '%')
|
||||
if filters.get('provinces'):
|
||||
conditions.append("(province='' OR province IN (" + ','.join('?' for _ in filters['provinces']) + '))')
|
||||
params.extend(filters['provinces'])
|
||||
after = None
|
||||
while True:
|
||||
where, values = ' AND '.join(conditions), list(params)
|
||||
if after:
|
||||
where += ' AND (school_code>? OR (school_code=? AND major_code>?))'
|
||||
values.extend([after[0], after[0], after[1]])
|
||||
query = 'SELECT school_code,major_code,MAX(school_name) AS school_name,MAX(province) AS province,MAX(level) AS level FROM catalog WHERE ' + where + ' GROUP BY school_code,major_code ORDER BY school_code,major_code LIMIT ?'
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(query, [*values, batch_size]).fetchall()
|
||||
if not rows:
|
||||
return
|
||||
yield from (dict(row) for row in rows)
|
||||
last = rows[-1]
|
||||
after = (last['school_code'], last['major_code'])
|
||||
|
||||
def _catalog_row(self, row):
|
||||
data = json.loads(row["data"])
|
||||
@@ -219,6 +291,17 @@ class Database:
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
|
||||
|
||||
def save_unit(self, jid, unit, records, scores=()):
|
||||
with self.connection() as conn:
|
||||
if not conn.execute('SELECT 1 FROM jobs WHERE id=?', (jid,)).fetchone():
|
||||
raise ValueError('任务不存在')
|
||||
if not conn.execute('INSERT OR IGNORE INTO job_units VALUES(?,?)', (jid, unit)).rowcount:
|
||||
return False
|
||||
self._upsert_scores(conn, scores)
|
||||
self._upsert_catalog(conn, records)
|
||||
conn.execute('UPDATE jobs SET records=records+?,updated_at=? WHERE id=?', (len(records), now(), jid))
|
||||
return True
|
||||
|
||||
def log(self, jid, level, message):
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Bounded exports, one download at a time, with disconnect cleanup."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
|
||||
from fastapi.responses import StreamingResponse
|
||||
from studio.imports import LABELS, spreadsheet_value
|
||||
from studio.memory import release_idle_memory
|
||||
|
||||
|
||||
class ExportStream(StreamingResponse):
|
||||
def __init__(self, source, cleanup, **kwargs):
|
||||
super().__init__(source, **kwargs)
|
||||
self.source, self.cleanup = source, cleanup
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
try:
|
||||
await super().__call__(scope, receive, send)
|
||||
finally:
|
||||
try:
|
||||
self.source.close()
|
||||
finally:
|
||||
try:
|
||||
self.cleanup()
|
||||
finally:
|
||||
release_idle_memory()
|
||||
|
||||
|
||||
def csv_chunks(records):
|
||||
yield b'\xef\xbb\xbf'
|
||||
stream = io.StringIO(newline='')
|
||||
writer = csv.writer(stream)
|
||||
writer.writerow(list(LABELS.values()))
|
||||
yield stream.getvalue().encode('utf-8')
|
||||
stream.seek(0)
|
||||
stream.truncate(0)
|
||||
for record in records:
|
||||
writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS])
|
||||
if stream.tell() >= 65536:
|
||||
yield stream.getvalue().encode('utf-8')
|
||||
stream.seek(0)
|
||||
stream.truncate(0)
|
||||
if stream.tell():
|
||||
yield stream.getvalue().encode('utf-8')
|
||||
|
||||
|
||||
def json_chunks(database, filters):
|
||||
pairs = set()
|
||||
yield b'{"version":1,"catalog":['
|
||||
first = True
|
||||
for row in database.iter_catalog(filters):
|
||||
pairs.add((row['school_code'], row['major_code']))
|
||||
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
|
||||
first = False
|
||||
yield b'],"scores":['
|
||||
first = True
|
||||
for row in database.iter_scores(pairs):
|
||||
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
|
||||
first = False
|
||||
yield b']}'
|
||||
|
||||
|
||||
def file_chunks(path):
|
||||
with path.open('rb') as stream:
|
||||
while chunk := stream.read(65536):
|
||||
yield chunk
|
||||
+45
-18
@@ -2,6 +2,7 @@
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
from contextlib import suppress
|
||||
import sqlite3
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
@@ -94,11 +95,18 @@ def import_file(path, year=None):
|
||||
book = load_workbook(path, read_only=True, data_only=True)
|
||||
try:
|
||||
sheet = book.active
|
||||
if sheet.max_row > 300001 or sheet.max_column > 100:
|
||||
if (sheet.max_row or 0) > 300001 or (sheet.max_column or 0) > 100:
|
||||
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
|
||||
values = sheet.iter_rows(values_only=True)
|
||||
headers = next(values, [])
|
||||
rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)]
|
||||
if len(headers) > 100:
|
||||
raise ValueError("Excel 超过 100 列,请拆分后导入")
|
||||
rows = []
|
||||
for index, row in enumerate(values, 2):
|
||||
if index > 300001 or len(row) > 100:
|
||||
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
|
||||
if any(v is not None for v in row):
|
||||
rows.append(dict(zip(headers, row)))
|
||||
finally:
|
||||
book.close()
|
||||
else:
|
||||
@@ -179,28 +187,47 @@ def spreadsheet_value(value):
|
||||
|
||||
|
||||
def export_sheet(records, kind):
|
||||
rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records]
|
||||
if kind == "csv":
|
||||
stream = io.StringIO(newline="")
|
||||
writer = csv.writer(stream)
|
||||
writer.writerow(list(LABELS.values()))
|
||||
writer.writerows(rows)
|
||||
writer.writerows([spreadsheet_value(record.get(field)) for field in LABELS] for record in records)
|
||||
return stream.getvalue().encode("utf-8-sig")
|
||||
book = Workbook()
|
||||
sheet = book.active
|
||||
sheet.title = "院校专业"
|
||||
sheet.append(list(LABELS.values()))
|
||||
for row in rows:
|
||||
sheet.append(row)
|
||||
sheet.freeze_panes = "A2"
|
||||
sheet.auto_filter.ref = sheet.dimensions
|
||||
output = io.BytesIO()
|
||||
write_xlsx(records, output)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def write_xlsx(records, output):
|
||||
from openpyxl.cell import WriteOnlyCell
|
||||
from openpyxl.utils import get_column_letter
|
||||
from openpyxl.styles import Font, PatternFill, Alignment
|
||||
for cell in sheet[1]:
|
||||
book = Workbook(write_only=True)
|
||||
sheet = book.create_sheet("院校专业")
|
||||
sheet.freeze_panes = "A2"
|
||||
header = []
|
||||
for index, label in enumerate(LABELS.values(), 1):
|
||||
cell = WriteOnlyCell(sheet, value=label)
|
||||
cell.font = Font(color="FFFFFF", bold=True)
|
||||
cell.fill = PatternFill("solid", fgColor="183447")
|
||||
cell.alignment = Alignment(vertical="center")
|
||||
for col in sheet.columns:
|
||||
sheet.column_dimensions[col[0].column_letter].width = 24
|
||||
output = io.BytesIO()
|
||||
book.save(output)
|
||||
return output.getvalue()
|
||||
header.append(cell)
|
||||
sheet.column_dimensions[get_column_letter(index)].width = 24
|
||||
count = 1
|
||||
try:
|
||||
sheet.append(header)
|
||||
for record in records:
|
||||
sheet.append([spreadsheet_value(record.get(field)) for field in LABELS])
|
||||
count += 1
|
||||
sheet.auto_filter.ref = f'A1:{get_column_letter(len(LABELS))}{count}'
|
||||
book.save(output)
|
||||
except BaseException:
|
||||
with suppress(Exception):
|
||||
if not sheet.closed:
|
||||
sheet.close()
|
||||
with suppress(Exception):
|
||||
if sheet._writer:
|
||||
sheet._writer.cleanup()
|
||||
raise
|
||||
finally:
|
||||
book.close()
|
||||
|
||||
+20
-19
@@ -1,6 +1,7 @@
|
||||
"""Persistent units and cooperative pause/cancel for one local worker."""
|
||||
import threading
|
||||
import uuid
|
||||
from itertools import chain
|
||||
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
|
||||
from studio.domain import PROVINCES, normalize_catalog
|
||||
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||
@@ -230,8 +231,7 @@ class JobManager:
|
||||
records = []
|
||||
for row in client.directions(school, major, params.get("study_mode", "")):
|
||||
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
|
||||
self._save(jid, records)
|
||||
self.complete_unit(jid, unit)
|
||||
self.db.save_unit(jid, unit, records)
|
||||
completed.add(unit)
|
||||
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
|
||||
except RemoteUnavailable as exc:
|
||||
@@ -240,24 +240,25 @@ class JobManager:
|
||||
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
|
||||
|
||||
def _scores(self, jid, params):
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
|
||||
records = self.db.all_catalog(filters)
|
||||
targets = {}
|
||||
for row in records:
|
||||
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]:
|
||||
continue
|
||||
targets[(row["school_code"], row["major_code"])] = row
|
||||
if not targets and params.get("major_code"):
|
||||
for row in self.db.schools(params.get("school_keyword", "")):
|
||||
if row["province"] in [PROVINCES[p] for p in params["provinces"]]:
|
||||
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
|
||||
if not targets:
|
||||
provinces = [PROVINCES[p] for p in params['provinces']]
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""),
|
||||
"school_keyword": params.get("school_keyword", ""), 'provinces': provinces}
|
||||
targets = iter(self.db.score_targets(filters))
|
||||
first = next(targets, None)
|
||||
if first is None and params.get('major_code'):
|
||||
targets = ({**row, 'major_code': params['major_code']}
|
||||
for row in self.db.schools(params.get('school_keyword', '')) if row['province'] in provinces)
|
||||
first = next(targets, None)
|
||||
if first is None:
|
||||
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
|
||||
completed = self.db.completed_units(jid)
|
||||
self.db.log(jid, 'info', '开始补充分数线和历史计划,分批读取专业目标并复用院校年份分数线')
|
||||
client = ScoreClient(self.transport)
|
||||
for (_, code), record in targets.items():
|
||||
for record in chain((first,), targets):
|
||||
self.checkpoint(jid)
|
||||
code = record['major_code']
|
||||
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
|
||||
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")]
|
||||
years = [y for y in params["score_years"] if f"scores:{school['school_code']}:{code}:{y}" not in completed]
|
||||
if not years:
|
||||
continue
|
||||
school_id = client.school_id(school["school_name"])
|
||||
@@ -270,10 +271,10 @@ class JobManager:
|
||||
self.db.job_update(jid, message=label)
|
||||
try:
|
||||
scores = client.score_lines(school, code, year, school_id)
|
||||
self.db.upsert_scores(scores)
|
||||
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
|
||||
self._save(jid, plans)
|
||||
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}")
|
||||
unit = f"scores:{school['school_code']}:{code}:{year}"
|
||||
self.db.save_unit(jid, unit, plans, scores=scores)
|
||||
completed.add(unit)
|
||||
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
|
||||
except RemoteUnavailable as exc:
|
||||
self._error(jid, f"{label}:{exc}")
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Best-effort return of idle allocator memory after large operations."""
|
||||
import ctypes
|
||||
import gc
|
||||
import sys
|
||||
|
||||
|
||||
def release_idle_memory():
|
||||
gc.collect()
|
||||
if sys.platform == 'linux':
|
||||
try:
|
||||
trim = ctypes.CDLL(None).malloc_trim
|
||||
trim.argtypes = [ctypes.c_size_t]
|
||||
trim.restype = ctypes.c_int
|
||||
return bool(trim(0))
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
return False
|
||||
@@ -107,6 +107,10 @@ class MySQLDatabase(Database):
|
||||
with self.connection() as conn:
|
||||
for statement in SCHEMA:
|
||||
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
|
||||
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_export'").fetchone():
|
||||
conn.execute('CREATE INDEX catalog_export ON catalog(year DESC,school_name,id)')
|
||||
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_targets'").fetchone():
|
||||
conn.execute('CREATE INDEX catalog_targets ON catalog(year,school_code,major_code)')
|
||||
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
|
||||
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
|
||||
if recover_jobs:
|
||||
@@ -130,7 +134,7 @@ class MySQLDatabase(Database):
|
||||
finally:
|
||||
raw.close()
|
||||
|
||||
def upsert_catalog(self, records):
|
||||
def _upsert_catalog(self, conn, records):
|
||||
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
|
||||
values = []
|
||||
for record in records:
|
||||
@@ -139,12 +143,11 @@ class MySQLDatabase(Database):
|
||||
if not values:
|
||||
return
|
||||
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
|
||||
def upsert_scores(self, records):
|
||||
def _upsert_scores(self, conn, records):
|
||||
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
|
||||
values = []
|
||||
for record in records:
|
||||
@@ -152,8 +155,7 @@ class MySQLDatabase(Database):
|
||||
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
|
||||
values.append([row.get(field) for field in fields])
|
||||
if values:
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
|
||||
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
|
||||
|
||||
def favorite(self, rid, enabled):
|
||||
# MariaDB reports changed rows; an already-selected favorite is still success.
|
||||
|
||||
+31
-1
@@ -3,6 +3,7 @@ import threading
|
||||
import time
|
||||
import random
|
||||
from contextlib import contextmanager
|
||||
from collections import OrderedDict
|
||||
import requests
|
||||
from studio.settings import proxy_url
|
||||
|
||||
@@ -19,6 +20,31 @@ class RateLimited(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class BoundedProxyAdapter(requests.adapters.HTTPAdapter):
|
||||
"""Close obsolete proxy pools rather than retain every historical exit."""
|
||||
|
||||
def __init__(self, max_proxy_pools=4, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.max_proxy_pools = max(1, int(max_proxy_pools))
|
||||
self.proxy_manager = OrderedDict()
|
||||
|
||||
def proxy_manager_for(self, proxy, **proxy_kwargs):
|
||||
manager = super().proxy_manager_for(proxy, **proxy_kwargs)
|
||||
self.proxy_manager.move_to_end(proxy)
|
||||
while len(self.proxy_manager) > self.max_proxy_pools:
|
||||
_, obsolete = self.proxy_manager.popitem(last=False)
|
||||
obsolete.clear()
|
||||
return manager
|
||||
|
||||
|
||||
def bounded_session():
|
||||
session = requests.Session()
|
||||
session.trust_env = False
|
||||
for scheme in ('http://', 'https://'):
|
||||
session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4))
|
||||
return session
|
||||
|
||||
|
||||
class ProxyRouter:
|
||||
def __init__(self, settings, session=None):
|
||||
self.settings = settings
|
||||
@@ -141,7 +167,7 @@ class ProxyRouter:
|
||||
class Transport:
|
||||
def __init__(self, settings, router, session=None, wait=None):
|
||||
self.settings, self.router = settings, router
|
||||
self.session = session or requests.Session()
|
||||
self.session = session or bounded_session()
|
||||
self.session.trust_env = False
|
||||
self.session.headers = getattr(self.session, "headers", {})
|
||||
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
|
||||
@@ -149,6 +175,10 @@ class Transport:
|
||||
self.local = threading.local()
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def close(self):
|
||||
with self.lock:
|
||||
self.session.close()
|
||||
|
||||
def sleep(self, seconds):
|
||||
remaining = seconds
|
||||
while remaining > 0:
|
||||
|
||||
Reference in New Issue
Block a user