Fix historical pagination and bound export and proxy memory

This commit is contained in:
2026-10-13 00:31:34 +08:00
parent 4c510759fb
commit ec2c897752
20 changed files with 1050 additions and 87 deletions
+65 -25
View File
@@ -2,6 +2,8 @@
import json
import uuid
import os
import tempfile
import threading
from contextlib import asynccontextmanager
from pathlib import Path
from urllib.parse import urlparse
@@ -11,7 +13,9 @@ from fastapi.staticfiles import StaticFiles
from studio.auth import AuthService, CookieVault
from studio.database import Database
from studio.domain import PROVINCES
from studio.imports import import_file, export_sheet
from studio.imports import import_file, write_xlsx
from studio.exporting import ExportStream, csv_chunks, json_chunks, file_chunks
from studio.memory import release_idle_memory
from studio.jobs import JobManager
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
from studio.settings import SettingsStore
@@ -20,6 +24,31 @@ from studio.pool_service import PoolService
ROOT = Path(__file__).resolve().parent.parent
class LocalAccessMiddleware:
"""Check hosts and origins without buffering or wrapping response streams."""
def __init__(self, app, allowed_hosts):
self.app = app
self.allowed_hosts = allowed_hosts
async def __call__(self, scope, receive, send):
if scope['type'] != 'http':
return await self.app(scope, receive, send)
request = Request(scope)
if request.url.hostname not in self.allowed_hosts:
response = JSONResponse({'detail': '访问主机不在允许列表中'}, status_code=403)
return await response(scope, receive, send)
if request.method not in ('GET', 'HEAD', 'OPTIONS'):
origin = request.headers.get('origin')
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip('/'), 'http://127.0.0.1:5173', 'http://localhost:5173'}
if parsed.scheme not in ('http', 'https') or origin.rstrip('/') not in allowed:
response = JSONResponse({'detail': '已拒绝跨来源修改请求'}, status_code=403)
return await response(scope, receive, send)
return await self.app(scope, receive, send)
def create_app(data_dir=None):
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
directory.mkdir(parents=True, exist_ok=True)
@@ -36,6 +65,9 @@ def create_app(data_dir=None):
transport = Transport(settings, router)
auth = AuthService(transport, CookieVault(directory))
jobs = JobManager(db, settings, transport, auth=auth)
export_lock = threading.BoundedSemaphore(1)
export_directory = directory / 'exports'
export_directory.mkdir(exist_ok=True)
if (ROOT / "data/schools.json").exists():
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
@@ -46,6 +78,8 @@ def create_app(data_dir=None):
jobs.resume(jid)
yield
jobs.close()
transport.close()
router.session.close()
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
app.state.db, app.state.jobs = db, jobs
@@ -53,19 +87,7 @@ def create_app(data_dir=None):
app.state.pool = pool
app.state.auth = auth
@app.middleware("http")
async def local_access(request: Request, call_next):
host = request.url.hostname
if host not in allowed_hosts:
return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
if request.method not in ("GET", "HEAD", "OPTIONS"):
origin = request.headers.get("origin")
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
return await call_next(request)
app.add_middleware(LocalAccessMiddleware, allowed_hosts=allowed_hosts)
@app.exception_handler(ValueError)
async def invalid(request, exc):
@@ -216,17 +238,35 @@ def create_app(data_dir=None):
def export(request: Request, format: str = "csv"):
if format not in ("csv", "xlsx", "json"):
raise ValueError("导出格式无效")
rows = db.all_catalog(filters(request))
if format == "json":
pairs = {(r["school_code"], r["major_code"]) for r in rows}
with db.connection() as conn:
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs]
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode()
mime = "application/json"
else:
payload = export_sheet(rows, format)
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'})
selected = filters(request)
if not export_lock.acquire(False):
raise HTTPException(429, '已有导出正在进行,请等待完成后重试')
temporary = None
def cleanup():
try:
if temporary is not None:
temporary.unlink(missing_ok=True)
finally:
export_lock.release()
try:
if format == 'csv':
source, mime = csv_chunks(db.iter_catalog(selected)), 'text/csv; charset=utf-8'
elif format == 'json':
source, mime = json_chunks(db, selected), 'application/json'
else:
descriptor, path = tempfile.mkstemp(prefix='kaoyan-', suffix='.xlsx', dir=export_directory)
os.close(descriptor)
temporary = Path(path)
write_xlsx(db.iter_catalog(selected), temporary)
source = file_chunks(temporary)
mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
return ExportStream(source, cleanup, media_type=mime,
headers={'Content-Disposition': f'attachment; filename="kaoyan-data.{format}"',
'X-Accel-Buffering': 'no'})
except BaseException:
cleanup()
release_idle_memory()
raise
dist = ROOT / "frontend/dist"
if (dist / "assets").exists():
+43 -8
View File
@@ -2,6 +2,7 @@
import json
import re
import hashlib
from collections import OrderedDict
from studio.domain import finish_record, integer, normalized_name, select_score_lines
from studio.network import RemoteUnavailable, RateLimited
@@ -153,6 +154,7 @@ class ScoreClient:
def __init__(self, transport):
self.transport = transport
self.cache = {}
self.score_cache = OrderedDict()
def post(self, endpoint, payload):
for attempt in range(3):
@@ -161,8 +163,13 @@ class ScoreClient:
body = response.json()
except ValueError:
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
if not isinstance(body, dict):
raise RemoteUnavailable("掌上考研响应结构已变化")
if str(body.get("code")) == "0000":
return body.get("data") or {}
data = body.get("data")
if not isinstance(data, (dict, list)):
raise RemoteUnavailable("掌上考研缺少有效数据,未写入不完整结果")
return data
message = str(body.get('message', body.get('msg', '')))
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
if attempt == 2:
@@ -172,20 +179,40 @@ class ScoreClient:
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
def pages(self, endpoint, payload, limit=100):
seen = set()
seen, consumed = set(), 0
for page in range(1, 10001):
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data
if not isinstance(rows, list):
full_list = isinstance(data, list)
rows = data.get("data", data.get("list")) if isinstance(data, dict) else data
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
raise RemoteUnavailable("掌上考研响应结构已变化")
total = integer(data.get("total")) if isinstance(data, dict) else None
last = integer(data.get("last_page")) if isinstance(data, dict) else None
if isinstance(data, dict):
for field, value in (('total', total), ('last_page', last)):
if data.get(field) is not None and (value is None or value < 0 or isinstance(data[field], bool)):
raise RemoteUnavailable("掌上考研分页信息无效,未写入不完整结果")
if not rows and (total is not None and consumed < total or last is not None and page < last):
raise RemoteUnavailable("掌上考研提前返回空页,未写入不完整结果")
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen:
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
seen.add(signature)
consumed += len(rows)
yield from rows
total = integer(data.get("total")) if isinstance(data, dict) else None
last = integer(data.get("last_page")) if isinstance(data, dict) else None
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total:
# Bare arrays are complete responses; wrapped pages use their actual
# metadata even when the server ignores the requested page size.
terminal = full_list or not rows
if not terminal:
if last is not None:
terminal = page >= last
if terminal and total is not None and consumed < total:
raise RemoteUnavailable("掌上考研分页未返回全部记录")
elif total is not None:
terminal = consumed >= total
else:
terminal = len(rows) < limit
if terminal:
return
raise RemoteUnavailable("历史数据分页超过安全上限")
@@ -198,10 +225,18 @@ class ScoreClient:
if len(matches) == 1 and matches[0].get("school_id"):
self.cache[key] = matches[0]["school_id"]
return self.cache[key]
self.cache[key] = None
return None
def score_lines(self, school, major_code, year, school_id):
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
key = (str(school_id), int(year))
if key not in self.score_cache:
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
self.score_cache[key] = rows
if len(self.score_cache) > 8:
self.score_cache.popitem(last=False)
self.score_cache.move_to_end(key)
rows = self.score_cache[key]
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
def plans(self, school, major_code, year, school_id):
+90 -7
View File
@@ -19,6 +19,8 @@ CREATE TABLE IF NOT EXISTS catalog(
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
CREATE INDEX IF NOT EXISTS catalog_export ON catalog(year DESC,school_name,id);
CREATE INDEX IF NOT EXISTS catalog_targets ON catalog(year,school_code,major_code);
""" + SCORE_SCHEMA + """
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
@@ -69,6 +71,10 @@ class Database:
conn.close()
def upsert_catalog(self, records: list[dict]):
with self.connection() as conn:
self._upsert_catalog(conn, records)
def _upsert_catalog(self, conn, records):
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
values = []
for record in records:
@@ -77,10 +83,9 @@ class Database:
if not values:
return
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
with self.connection() as conn:
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
for record in records:
self._school(conn, record)
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
for record in records:
self._school(conn, record)
def _school(self, conn, record):
record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))}
@@ -93,8 +98,11 @@ class Database:
def upsert_scores(self, records):
with self.connection() as conn:
for row in records:
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
self._upsert_scores(conn, records)
def _upsert_scores(self, conn, records):
for row in records:
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
def _query(self, filters: dict):
conditions, params = ["1=1"], []
@@ -141,9 +149,73 @@ class Database:
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
def all_catalog(self, filters):
return list(self.iter_catalog(filters))
def iter_catalog(self, filters, batch_size=200):
select, params = self._query(filters)
with self.connection() as conn:
return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)]
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM catalog').fetchone()[0]
after = None
while True:
query = 'SELECT * FROM (' + select + ') export_rows WHERE id<=?'
values = [*params, cutoff]
if after:
year, school, rid = after
if school is None:
query += ' AND (year<? OR (year=? AND (school_name IS NOT NULL OR (school_name IS NULL AND id>?))))'
values.extend([year, year, rid])
else:
query += ' AND (year<? OR (year=? AND (school_name>? OR (school_name=? AND id>?))))'
values.extend([year, year, school, school, rid])
query += ' ORDER BY year DESC,school_name,id LIMIT ?'
with self.connection() as conn:
rows = conn.execute(query, [*values, batch_size]).fetchall()
if not rows:
return
for row in rows:
yield self._catalog_row(row)
last = rows[-1]
after = (last['year'], last['school_name'], last['id'])
def iter_scores(self, pairs, batch_size=200):
after = 0
with self.connection() as conn:
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM scores').fetchone()[0]
while True:
with self.connection() as conn:
rows = conn.execute('SELECT id,school_code,major_code,data FROM scores WHERE id>? AND id<=? ORDER BY id LIMIT ?', (after, cutoff, batch_size)).fetchall()
if not rows:
return
for row in rows:
if (row['school_code'], row['major_code']) in pairs:
yield json.loads(row['data'])
after = rows[-1]['id']
def score_targets(self, filters, batch_size=200):
conditions, params = ['year=?'], [filters['year']]
if filters.get('major_code'):
conditions.append('major_code=?')
params.append(filters['major_code'])
if filters.get('school_keyword'):
conditions.append('school_name LIKE ?')
params.append('%' + filters['school_keyword'] + '%')
if filters.get('provinces'):
conditions.append("(province='' OR province IN (" + ','.join('?' for _ in filters['provinces']) + '))')
params.extend(filters['provinces'])
after = None
while True:
where, values = ' AND '.join(conditions), list(params)
if after:
where += ' AND (school_code>? OR (school_code=? AND major_code>?))'
values.extend([after[0], after[0], after[1]])
query = 'SELECT school_code,major_code,MAX(school_name) AS school_name,MAX(province) AS province,MAX(level) AS level FROM catalog WHERE ' + where + ' GROUP BY school_code,major_code ORDER BY school_code,major_code LIMIT ?'
with self.connection() as conn:
rows = conn.execute(query, [*values, batch_size]).fetchall()
if not rows:
return
yield from (dict(row) for row in rows)
last = rows[-1]
after = (last['school_code'], last['major_code'])
def _catalog_row(self, row):
data = json.loads(row["data"])
@@ -219,6 +291,17 @@ class Database:
with self.connection() as conn:
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
def save_unit(self, jid, unit, records, scores=()):
with self.connection() as conn:
if not conn.execute('SELECT 1 FROM jobs WHERE id=?', (jid,)).fetchone():
raise ValueError('任务不存在')
if not conn.execute('INSERT OR IGNORE INTO job_units VALUES(?,?)', (jid, unit)).rowcount:
return False
self._upsert_scores(conn, scores)
self._upsert_catalog(conn, records)
conn.execute('UPDATE jobs SET records=records+?,updated_at=? WHERE id=?', (len(records), now(), jid))
return True
def log(self, jid, level, message):
with self.connection() as conn:
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
+66
View File
@@ -0,0 +1,66 @@
"""Bounded exports, one download at a time, with disconnect cleanup."""
import csv
import io
import json
from fastapi.responses import StreamingResponse
from studio.imports import LABELS, spreadsheet_value
from studio.memory import release_idle_memory
class ExportStream(StreamingResponse):
def __init__(self, source, cleanup, **kwargs):
super().__init__(source, **kwargs)
self.source, self.cleanup = source, cleanup
async def __call__(self, scope, receive, send):
try:
await super().__call__(scope, receive, send)
finally:
try:
self.source.close()
finally:
try:
self.cleanup()
finally:
release_idle_memory()
def csv_chunks(records):
yield b'\xef\xbb\xbf'
stream = io.StringIO(newline='')
writer = csv.writer(stream)
writer.writerow(list(LABELS.values()))
yield stream.getvalue().encode('utf-8')
stream.seek(0)
stream.truncate(0)
for record in records:
writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS])
if stream.tell() >= 65536:
yield stream.getvalue().encode('utf-8')
stream.seek(0)
stream.truncate(0)
if stream.tell():
yield stream.getvalue().encode('utf-8')
def json_chunks(database, filters):
pairs = set()
yield b'{"version":1,"catalog":['
first = True
for row in database.iter_catalog(filters):
pairs.add((row['school_code'], row['major_code']))
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
first = False
yield b'],"scores":['
first = True
for row in database.iter_scores(pairs):
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
first = False
yield b']}'
def file_chunks(path):
with path.open('rb') as stream:
while chunk := stream.read(65536):
yield chunk
+45 -18
View File
@@ -2,6 +2,7 @@
import csv
import io
import json
from contextlib import suppress
import sqlite3
from dataclasses import dataclass, field
from pathlib import Path
@@ -94,11 +95,18 @@ def import_file(path, year=None):
book = load_workbook(path, read_only=True, data_only=True)
try:
sheet = book.active
if sheet.max_row > 300001 or sheet.max_column > 100:
if (sheet.max_row or 0) > 300001 or (sheet.max_column or 0) > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
values = sheet.iter_rows(values_only=True)
headers = next(values, [])
rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)]
if len(headers) > 100:
raise ValueError("Excel 超过 100 列,请拆分后导入")
rows = []
for index, row in enumerate(values, 2):
if index > 300001 or len(row) > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
if any(v is not None for v in row):
rows.append(dict(zip(headers, row)))
finally:
book.close()
else:
@@ -179,28 +187,47 @@ def spreadsheet_value(value):
def export_sheet(records, kind):
rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records]
if kind == "csv":
stream = io.StringIO(newline="")
writer = csv.writer(stream)
writer.writerow(list(LABELS.values()))
writer.writerows(rows)
writer.writerows([spreadsheet_value(record.get(field)) for field in LABELS] for record in records)
return stream.getvalue().encode("utf-8-sig")
book = Workbook()
sheet = book.active
sheet.title = "院校专业"
sheet.append(list(LABELS.values()))
for row in rows:
sheet.append(row)
sheet.freeze_panes = "A2"
sheet.auto_filter.ref = sheet.dimensions
output = io.BytesIO()
write_xlsx(records, output)
return output.getvalue()
def write_xlsx(records, output):
from openpyxl.cell import WriteOnlyCell
from openpyxl.utils import get_column_letter
from openpyxl.styles import Font, PatternFill, Alignment
for cell in sheet[1]:
book = Workbook(write_only=True)
sheet = book.create_sheet("院校专业")
sheet.freeze_panes = "A2"
header = []
for index, label in enumerate(LABELS.values(), 1):
cell = WriteOnlyCell(sheet, value=label)
cell.font = Font(color="FFFFFF", bold=True)
cell.fill = PatternFill("solid", fgColor="183447")
cell.alignment = Alignment(vertical="center")
for col in sheet.columns:
sheet.column_dimensions[col[0].column_letter].width = 24
output = io.BytesIO()
book.save(output)
return output.getvalue()
header.append(cell)
sheet.column_dimensions[get_column_letter(index)].width = 24
count = 1
try:
sheet.append(header)
for record in records:
sheet.append([spreadsheet_value(record.get(field)) for field in LABELS])
count += 1
sheet.auto_filter.ref = f'A1:{get_column_letter(len(LABELS))}{count}'
book.save(output)
except BaseException:
with suppress(Exception):
if not sheet.closed:
sheet.close()
with suppress(Exception):
if sheet._writer:
sheet._writer.cleanup()
raise
finally:
book.close()
+20 -19
View File
@@ -1,6 +1,7 @@
"""Persistent units and cooperative pause/cancel for one local worker."""
import threading
import uuid
from itertools import chain
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
from studio.domain import PROVINCES, normalize_catalog
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
@@ -230,8 +231,7 @@ class JobManager:
records = []
for row in client.directions(school, major, params.get("study_mode", "")):
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
self._save(jid, records)
self.complete_unit(jid, unit)
self.db.save_unit(jid, unit, records)
completed.add(unit)
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
except RemoteUnavailable as exc:
@@ -240,24 +240,25 @@ class JobManager:
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
def _scores(self, jid, params):
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
records = self.db.all_catalog(filters)
targets = {}
for row in records:
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]:
continue
targets[(row["school_code"], row["major_code"])] = row
if not targets and params.get("major_code"):
for row in self.db.schools(params.get("school_keyword", "")):
if row["province"] in [PROVINCES[p] for p in params["provinces"]]:
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
if not targets:
provinces = [PROVINCES[p] for p in params['provinces']]
filters = {"year": params["year"], "major_code": params.get("major_code", ""),
"school_keyword": params.get("school_keyword", ""), 'provinces': provinces}
targets = iter(self.db.score_targets(filters))
first = next(targets, None)
if first is None and params.get('major_code'):
targets = ({**row, 'major_code': params['major_code']}
for row in self.db.schools(params.get('school_keyword', '')) if row['province'] in provinces)
first = next(targets, None)
if first is None:
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
completed = self.db.completed_units(jid)
self.db.log(jid, 'info', '开始补充分数线和历史计划,分批读取专业目标并复用院校年份分数线')
client = ScoreClient(self.transport)
for (_, code), record in targets.items():
for record in chain((first,), targets):
self.checkpoint(jid)
code = record['major_code']
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")]
years = [y for y in params["score_years"] if f"scores:{school['school_code']}:{code}:{y}" not in completed]
if not years:
continue
school_id = client.school_id(school["school_name"])
@@ -270,10 +271,10 @@ class JobManager:
self.db.job_update(jid, message=label)
try:
scores = client.score_lines(school, code, year, school_id)
self.db.upsert_scores(scores)
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
self._save(jid, plans)
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}")
unit = f"scores:{school['school_code']}:{code}:{year}"
self.db.save_unit(jid, unit, plans, scores=scores)
completed.add(unit)
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}")
+17
View File
@@ -0,0 +1,17 @@
"""Best-effort return of idle allocator memory after large operations."""
import ctypes
import gc
import sys
def release_idle_memory():
gc.collect()
if sys.platform == 'linux':
try:
trim = ctypes.CDLL(None).malloc_trim
trim.argtypes = [ctypes.c_size_t]
trim.restype = ctypes.c_int
return bool(trim(0))
except (AttributeError, OSError):
pass
return False
+10 -8
View File
@@ -107,6 +107,10 @@ class MySQLDatabase(Database):
with self.connection() as conn:
for statement in SCHEMA:
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_export'").fetchone():
conn.execute('CREATE INDEX catalog_export ON catalog(year DESC,school_name,id)')
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_targets'").fetchone():
conn.execute('CREATE INDEX catalog_targets ON catalog(year,school_code,major_code)')
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
if recover_jobs:
@@ -130,7 +134,7 @@ class MySQLDatabase(Database):
finally:
raw.close()
def upsert_catalog(self, records):
def _upsert_catalog(self, conn, records):
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
values = []
for record in records:
@@ -139,12 +143,11 @@ class MySQLDatabase(Database):
if not values:
return
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
with self.connection() as conn:
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
for record in records:
self._school(conn, record)
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
for record in records:
self._school(conn, record)
def upsert_scores(self, records):
def _upsert_scores(self, conn, records):
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
values = []
for record in records:
@@ -152,8 +155,7 @@ class MySQLDatabase(Database):
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
values.append([row.get(field) for field in fields])
if values:
with self.connection() as conn:
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
def favorite(self, rid, enabled):
# MariaDB reports changed rows; an already-selected favorite is still success.
+31 -1
View File
@@ -3,6 +3,7 @@ import threading
import time
import random
from contextlib import contextmanager
from collections import OrderedDict
import requests
from studio.settings import proxy_url
@@ -19,6 +20,31 @@ class RateLimited(RuntimeError):
pass
class BoundedProxyAdapter(requests.adapters.HTTPAdapter):
"""Close obsolete proxy pools rather than retain every historical exit."""
def __init__(self, max_proxy_pools=4, **kwargs):
super().__init__(**kwargs)
self.max_proxy_pools = max(1, int(max_proxy_pools))
self.proxy_manager = OrderedDict()
def proxy_manager_for(self, proxy, **proxy_kwargs):
manager = super().proxy_manager_for(proxy, **proxy_kwargs)
self.proxy_manager.move_to_end(proxy)
while len(self.proxy_manager) > self.max_proxy_pools:
_, obsolete = self.proxy_manager.popitem(last=False)
obsolete.clear()
return manager
def bounded_session():
session = requests.Session()
session.trust_env = False
for scheme in ('http://', 'https://'):
session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4))
return session
class ProxyRouter:
def __init__(self, settings, session=None):
self.settings = settings
@@ -141,7 +167,7 @@ class ProxyRouter:
class Transport:
def __init__(self, settings, router, session=None, wait=None):
self.settings, self.router = settings, router
self.session = session or requests.Session()
self.session = session or bounded_session()
self.session.trust_env = False
self.session.headers = getattr(self.session, "headers", {})
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
@@ -149,6 +175,10 @@ class Transport:
self.local = threading.local()
self.lock = threading.RLock()
def close(self):
with self.lock:
self.session.close()
def sleep(self, seconds):
remaining = seconds
while remaining > 0: