Fix historical pagination and bound export and proxy memory

This commit is contained in:
2026-10-13 00:31:34 +08:00
parent 4c510759fb
commit ec2c897752
20 changed files with 1050 additions and 87 deletions
+4
View File
@@ -62,6 +62,10 @@ Debian 12 / MariaDB 服务器模式、HTTPS 访问与代理池后台运行见 [
可将当前筛选结果导出 CSV、Excel 或完整 JSON。表格导出保留院系/方向代码和字段口径,回导不会凭空增加同一记录。JSON 同时保留相关学校专业的历史分数线;凭据不进入导出。CSV/Excel 对可能被表格软件当作公式的文本加保护前缀。 可将当前筛选结果导出 CSV、Excel 或完整 JSON。表格导出保留院系/方向代码和字段口径,回导不会凭空增加同一记录。JSON 同时保留相关学校专业的历史分数线;凭据不进入导出。CSV/Excel 对可能被表格软件当作公式的文本加保护前缀。
全量导出按 200 条分批读取,CSV/JSON 流式下载,Excel 使用写入模式和临时文件;同一实例一次只执行一个导出,重复请求返回 429。下载结束或中断后清理临时资源。导出期间新增的记录留待下次导出;导出不是事务一致性备份。大于 100 MB 的文件仍受导入上限限制,可按地区等条件分别导出;完整数据库备份按服务器部署说明操作。
历史采集在招生目录阶段完成后开始,按院校和年份复用全校分数线,每次最多缓存 8 份,历史计划继续逐专业获取。接口直接返回的数组按完整结果处理,带分页信息的结果按实际数量继续;不完整的响应不会被写成成功断点。数据、保存计数和完成断点在同一数据库事务中提交。
## 本地数据与开发 ## 本地数据与开发
所有运行数据位于项目 `.runtime`:`studio.db` 保存目录、分数、收藏、任务和断点;`settings.json` 保存非机密配置;`chsi-cookie.enc` 由当前 Windows 用户的 DPAPI 加密。密码不持久化,接口不返回 Cookie。备份数据库时先停止工作台,再复制 `.runtime`;恢复加密 Cookie 需要原 Windows 用户环境,否则重新登录即可。 所有运行数据位于项目 `.runtime`:`studio.db` 保存目录、分数、收藏、任务和断点;`settings.json` 保存非机密配置;`chsi-cookie.enc` 由当前 Windows 用户的 DPAPI 加密。密码不持久化,接口不返回 Cookie。备份数据库时先停止工作台,再复制 `.runtime`;恢复加密 Cookie 需要原 Windows 用户环境,否则重新登录即可。
+4 -1
View File
@@ -8,9 +8,12 @@ Requires=docker.service
User=kaoyan User=kaoyan
Group=kaoyan Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py schedule Environment=MALLOC_ARENA_MAX=2
Environment=PROXY_CHECK_WORKERS=8
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python /opt/kaoyan-studio-deploy/app/deploy/proxy_scheduler.py
Restart=on-failure Restart=on-failure
RestartSec=10 RestartSec=10
TimeoutStopSec=120
UMask=0077 UMask=0077
PrivateTmp=true PrivateTmp=true
ProtectSystem=full ProtectSystem=full
+1
View File
@@ -9,6 +9,7 @@ User=kaoyan
Group=kaoyan Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/app WorkingDirectory=/opt/kaoyan-studio-deploy/app
EnvironmentFile=/etc/kaoyan-studio.env EnvironmentFile=/etc/kaoyan-studio.env
Environment=MALLOC_ARENA_MAX=2
ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio
Restart=on-failure Restart=on-failure
RestartSec=10 RestartSec=10
+122
View File
@@ -0,0 +1,122 @@
"""Run upstream proxy_pool checks with reusable workers and serialized jobs."""
import os
import queue
import signal
import sys
import threading
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
APP_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(APP_ROOT))
from studio.memory import release_idle_memory
class StopQueue(queue.Queue):
def __init__(self, stopping):
super().__init__()
self.stopping = stopping
def get(self, *args, **kwargs):
if self.stopping.is_set():
raise queue.Empty
return super().get(*args, **kwargs)
class ProxyMaintainer:
def __init__(self, workers=8, checker_class=None, fetcher_factory=None, handler_factory=None, pool_min=20):
self.worker_count = max(1, min(20, int(workers)))
self.checker_class = checker_class
self.fetcher_factory, self.handler_factory = fetcher_factory, handler_factory
self.pool_min = pool_min
self.stopping = threading.Event()
self.pipeline = threading.RLock()
self.workers = ThreadPoolExecutor(max_workers=self.worker_count, thread_name_prefix='proxy-check')
def validate(self, work_type, proxies):
with self.pipeline:
if self.stopping.is_set():
return
target = StopQueue(self.stopping)
for proxy in proxies:
if self.stopping.is_set():
return
target.put(proxy)
if self.checker_class is None:
from helper.check import _ThreadChecker
self.checker_class = _ThreadChecker
count = min(self.worker_count, target.qsize())
futures = [self.workers.submit(self.checker_class(work_type, target, f'worker_{i:02}').run)
for i in range(count)]
try:
failure = None
for future in futures:
try:
future.result()
except Exception as exc:
if failure is None:
failure = exc
if failure is not None:
raise failure
finally:
release_idle_memory()
def fetch(self):
with self.pipeline:
if not self.stopping.is_set():
self.validate('raw', self.fetcher_factory().run())
def check(self):
with self.pipeline:
if self.stopping.is_set():
return
handler = self.handler_factory()
if handler.db.getCount().get('total', 0) < self.pool_min:
self.fetch()
self.validate('use', handler.getAll())
def stop(self):
self.stopping.set()
def close(self):
self.stop()
with self.pipeline:
self.workers.shutdown(wait=True, cancel_futures=True)
def main():
proxy_root = Path(os.environ.get('STUDIO_PROXY_POOL_ROOT', APP_ROOT.parent / 'proxy_pool'))
sys.path.insert(0, str(proxy_root))
from apscheduler.schedulers.blocking import BlockingScheduler
from helper.check import _ThreadChecker
from helper.fetch import Fetcher
from handler.configHandler import ConfigHandler
from handler.logHandler import LogHandler
from handler.proxyHandler import ProxyHandler
config = ConfigHandler()
maintainer = ProxyMaintainer(workers=int(os.environ.get('PROXY_CHECK_WORKERS', '8')),
checker_class=_ThreadChecker, fetcher_factory=Fetcher,
handler_factory=ProxyHandler, pool_min=config.poolSizeMin)
scheduler = BlockingScheduler(timezone=config.timezone, logger=LogHandler('scheduler'),
executors={'default': {'type': 'threadpool', 'max_workers': 1}},
job_defaults={'coalesce': True, 'max_instances': 1, 'misfire_grace_time': 120})
scheduler.add_job(maintainer.fetch, 'interval', minutes=5, id='proxy_fetch')
scheduler.add_job(maintainer.check, 'interval', minutes=2, id='proxy_check',
next_run_time=datetime.now(timezone.utc))
def stop(signum, frame):
maintainer.stop()
if scheduler.running:
scheduler.shutdown(wait=False)
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
if not maintainer.stopping.is_set():
scheduler.start()
finally:
maintainer.close()
if __name__ == '__main__':
main()
+4
View File
@@ -16,6 +16,8 @@
安装模板在 `deploy/`。Python 依赖按 `requirements.lock` 安装;前端 `npm ci && npm run build`。不要启动多个工作台进程连接同一数据库,否则任务恢复状态会互相影响。原有站点、数据库和 fail2ban 配置不作修改。 安装模板在 `deploy/`。Python 依赖按 `requirements.lock` 安装;前端 `npm ci && npm run build`。不要启动多个工作台进程连接同一数据库,否则任务恢复状态会互相影响。原有站点、数据库和 fail2ban 配置不作修改。
代理调度器运行本项目 `deploy/proxy_scheduler.py` 包装器,继续使用原版代理池的获取源、检测逻辑和 Redis 数据,不修改上游源码。默认复用 8 个检测线程,抓取与检查轮次串行运行,避免定时任务重叠。工作台和调度器设定 `MALLOC_ARENA_MAX=2`;每次导出/检查结束尝试释放分配器空闲内存。该操作不释放仍在使用的数据,实际内存以运行监测为准。
## 操作 ## 操作
```bash ```bash
@@ -28,6 +30,8 @@ systemctl restart kaoyan-studio
默认每次请求随机等待 1.2–2.4 秒,每 100 次成功请求轮换代理。设置页可修改上下限和轮换次数;0 表示关闭自动轮换。代理页显示实际成功请求路由、计数和切换次数。登录表单、会话同步、权限验证及紧随其后的失败页重试保持同一路由。免费代理池初次启动可能没有 HTTPS 候选,部署时保留了原电脑代理池的候选作为备用;这些候选仍会经过实际请求与失败隔离。 默认每次请求随机等待 1.2–2.4 秒,每 100 次成功请求轮换代理。设置页可修改上下限和轮换次数;0 表示关闭自动轮换。代理页显示实际成功请求路由、计数和切换次数。登录表单、会话同步、权限验证及紧随其后的失败页重试保持同一路由。免费代理池初次启动可能没有 HTTPS 候选,部署时保留了原电脑代理池的候选作为备用;这些候选仍会经过实际请求与失败隔离。
当前全国任务采用 0 次自动轮换,维持通过登录验证的出口;连接失败仍会切换。Cookie 验证状态与出口关联,出口变化后需要重新验证,不宜为了让 IP 显示变化而频繁轮换。HTTP 会话最多保留 4 个代理连接池,淘汰旧池时主动关闭连接。
Linux 无桌面环境,界面提供研招网登录入口与粘贴 Cookie。请在自己的电脑浏览器打开研招目录并登录,从开发者工具中复制目录接口请求的单行 Cookie,粘贴到工作台后点验证。服务器重新同步会话并检查受保护的第二页。迁移 Windows Cookie 时只经 SSH 传输加密封装,在服务器重新加密;不继承原验证标记。 Linux 无桌面环境,界面提供研招网登录入口与粘贴 Cookie。请在自己的电脑浏览器打开研招目录并登录,从开发者工具中复制目录接口请求的单行 Cookie,粘贴到工作台后点验证。服务器重新同步会话并检查受保护的第二页。迁移 Windows Cookie 时只经 SSH 传输加密封装,在服务器重新加密;不继承原验证标记。
## 迁移与验证 ## 迁移与验证
+65 -25
View File
@@ -2,6 +2,8 @@
import json import json
import uuid import uuid
import os import os
import tempfile
import threading
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse
@@ -11,7 +13,9 @@ from fastapi.staticfiles import StaticFiles
from studio.auth import AuthService, CookieVault from studio.auth import AuthService, CookieVault
from studio.database import Database from studio.database import Database
from studio.domain import PROVINCES from studio.domain import PROVINCES
from studio.imports import import_file, export_sheet from studio.imports import import_file, write_xlsx
from studio.exporting import ExportStream, csv_chunks, json_chunks, file_chunks
from studio.memory import release_idle_memory
from studio.jobs import JobManager from studio.jobs import JobManager
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
from studio.settings import SettingsStore from studio.settings import SettingsStore
@@ -20,6 +24,31 @@ from studio.pool_service import PoolService
ROOT = Path(__file__).resolve().parent.parent ROOT = Path(__file__).resolve().parent.parent
class LocalAccessMiddleware:
"""Check hosts and origins without buffering or wrapping response streams."""
def __init__(self, app, allowed_hosts):
self.app = app
self.allowed_hosts = allowed_hosts
async def __call__(self, scope, receive, send):
if scope['type'] != 'http':
return await self.app(scope, receive, send)
request = Request(scope)
if request.url.hostname not in self.allowed_hosts:
response = JSONResponse({'detail': '访问主机不在允许列表中'}, status_code=403)
return await response(scope, receive, send)
if request.method not in ('GET', 'HEAD', 'OPTIONS'):
origin = request.headers.get('origin')
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip('/'), 'http://127.0.0.1:5173', 'http://localhost:5173'}
if parsed.scheme not in ('http', 'https') or origin.rstrip('/') not in allowed:
response = JSONResponse({'detail': '已拒绝跨来源修改请求'}, status_code=403)
return await response(scope, receive, send)
return await self.app(scope, receive, send)
def create_app(data_dir=None): def create_app(data_dir=None):
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime") directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
directory.mkdir(parents=True, exist_ok=True) directory.mkdir(parents=True, exist_ok=True)
@@ -36,6 +65,9 @@ def create_app(data_dir=None):
transport = Transport(settings, router) transport = Transport(settings, router)
auth = AuthService(transport, CookieVault(directory)) auth = AuthService(transport, CookieVault(directory))
jobs = JobManager(db, settings, transport, auth=auth) jobs = JobManager(db, settings, transport, auth=auth)
export_lock = threading.BoundedSemaphore(1)
export_directory = directory / 'exports'
export_directory.mkdir(exist_ok=True)
if (ROOT / "data/schools.json").exists(): if (ROOT / "data/schools.json").exists():
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8"))) db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
@@ -46,6 +78,8 @@ def create_app(data_dir=None):
jobs.resume(jid) jobs.resume(jid)
yield yield
jobs.close() jobs.close()
transport.close()
router.session.close()
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None) app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
app.state.db, app.state.jobs = db, jobs app.state.db, app.state.jobs = db, jobs
@@ -53,19 +87,7 @@ def create_app(data_dir=None):
app.state.pool = pool app.state.pool = pool
app.state.auth = auth app.state.auth = auth
@app.middleware("http") app.add_middleware(LocalAccessMiddleware, allowed_hosts=allowed_hosts)
async def local_access(request: Request, call_next):
host = request.url.hostname
if host not in allowed_hosts:
return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
if request.method not in ("GET", "HEAD", "OPTIONS"):
origin = request.headers.get("origin")
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
return await call_next(request)
@app.exception_handler(ValueError) @app.exception_handler(ValueError)
async def invalid(request, exc): async def invalid(request, exc):
@@ -216,17 +238,35 @@ def create_app(data_dir=None):
def export(request: Request, format: str = "csv"): def export(request: Request, format: str = "csv"):
if format not in ("csv", "xlsx", "json"): if format not in ("csv", "xlsx", "json"):
raise ValueError("导出格式无效") raise ValueError("导出格式无效")
rows = db.all_catalog(filters(request)) selected = filters(request)
if format == "json": if not export_lock.acquire(False):
pairs = {(r["school_code"], r["major_code"]) for r in rows} raise HTTPException(429, '已有导出正在进行,请等待完成后重试')
with db.connection() as conn: temporary = None
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs] def cleanup():
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode() try:
mime = "application/json" if temporary is not None:
else: temporary.unlink(missing_ok=True)
payload = export_sheet(rows, format) finally:
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" export_lock.release()
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'}) try:
if format == 'csv':
source, mime = csv_chunks(db.iter_catalog(selected)), 'text/csv; charset=utf-8'
elif format == 'json':
source, mime = json_chunks(db, selected), 'application/json'
else:
descriptor, path = tempfile.mkstemp(prefix='kaoyan-', suffix='.xlsx', dir=export_directory)
os.close(descriptor)
temporary = Path(path)
write_xlsx(db.iter_catalog(selected), temporary)
source = file_chunks(temporary)
mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
return ExportStream(source, cleanup, media_type=mime,
headers={'Content-Disposition': f'attachment; filename="kaoyan-data.{format}"',
'X-Accel-Buffering': 'no'})
except BaseException:
cleanup()
release_idle_memory()
raise
dist = ROOT / "frontend/dist" dist = ROOT / "frontend/dist"
if (dist / "assets").exists(): if (dist / "assets").exists():
+43 -8
View File
@@ -2,6 +2,7 @@
import json import json
import re import re
import hashlib import hashlib
from collections import OrderedDict
from studio.domain import finish_record, integer, normalized_name, select_score_lines from studio.domain import finish_record, integer, normalized_name, select_score_lines
from studio.network import RemoteUnavailable, RateLimited from studio.network import RemoteUnavailable, RateLimited
@@ -153,6 +154,7 @@ class ScoreClient:
def __init__(self, transport): def __init__(self, transport):
self.transport = transport self.transport = transport
self.cache = {} self.cache = {}
self.score_cache = OrderedDict()
def post(self, endpoint, payload): def post(self, endpoint, payload):
for attempt in range(3): for attempt in range(3):
@@ -161,8 +163,13 @@ class ScoreClient:
body = response.json() body = response.json()
except ValueError: except ValueError:
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
if not isinstance(body, dict):
raise RemoteUnavailable("掌上考研响应结构已变化")
if str(body.get("code")) == "0000": if str(body.get("code")) == "0000":
return body.get("data") or {} data = body.get("data")
if not isinstance(data, (dict, list)):
raise RemoteUnavailable("掌上考研缺少有效数据,未写入不完整结果")
return data
message = str(body.get('message', body.get('msg', ''))) message = str(body.get('message', body.get('msg', '')))
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')): if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
if attempt == 2: if attempt == 2:
@@ -172,20 +179,40 @@ class ScoreClient:
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试") raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
def pages(self, endpoint, payload, limit=100): def pages(self, endpoint, payload, limit=100):
seen = set() seen, consumed = set(), 0
for page in range(1, 10001): for page in range(1, 10001):
data = self.post(endpoint, {**payload, "page": page, "limit": limit}) data = self.post(endpoint, {**payload, "page": page, "limit": limit})
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data full_list = isinstance(data, list)
if not isinstance(rows, list): rows = data.get("data", data.get("list")) if isinstance(data, dict) else data
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
raise RemoteUnavailable("掌上考研响应结构已变化") raise RemoteUnavailable("掌上考研响应结构已变化")
total = integer(data.get("total")) if isinstance(data, dict) else None
last = integer(data.get("last_page")) if isinstance(data, dict) else None
if isinstance(data, dict):
for field, value in (('total', total), ('last_page', last)):
if data.get(field) is not None and (value is None or value < 0 or isinstance(data[field], bool)):
raise RemoteUnavailable("掌上考研分页信息无效,未写入不完整结果")
if not rows and (total is not None and consumed < total or last is not None and page < last):
raise RemoteUnavailable("掌上考研提前返回空页,未写入不完整结果")
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True) signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen: if rows and signature in seen:
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页") raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
seen.add(signature) seen.add(signature)
consumed += len(rows)
yield from rows yield from rows
total = integer(data.get("total")) if isinstance(data, dict) else None # Bare arrays are complete responses; wrapped pages use their actual
last = integer(data.get("last_page")) if isinstance(data, dict) else None # metadata even when the server ignores the requested page size.
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total: terminal = full_list or not rows
if not terminal:
if last is not None:
terminal = page >= last
if terminal and total is not None and consumed < total:
raise RemoteUnavailable("掌上考研分页未返回全部记录")
elif total is not None:
terminal = consumed >= total
else:
terminal = len(rows) < limit
if terminal:
return return
raise RemoteUnavailable("历史数据分页超过安全上限") raise RemoteUnavailable("历史数据分页超过安全上限")
@@ -198,10 +225,18 @@ class ScoreClient:
if len(matches) == 1 and matches[0].get("school_id"): if len(matches) == 1 and matches[0].get("school_id"):
self.cache[key] = matches[0]["school_id"] self.cache[key] = matches[0]["school_id"]
return self.cache[key] return self.cache[key]
self.cache[key] = None
return None return None
def score_lines(self, school, major_code, year, school_id): def score_lines(self, school, major_code, year, school_id):
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year})) key = (str(school_id), int(year))
if key not in self.score_cache:
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
self.score_cache[key] = rows
if len(self.score_cache) > 8:
self.score_cache.popitem(last=False)
self.score_cache.move_to_end(key)
rows = self.score_cache[key]
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year) return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
def plans(self, school, major_code, year, school_id): def plans(self, school, major_code, year, school_id):
+90 -7
View File
@@ -19,6 +19,8 @@ CREATE TABLE IF NOT EXISTS catalog(
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT, department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL); subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code); CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
CREATE INDEX IF NOT EXISTS catalog_export ON catalog(year DESC,school_name,id);
CREATE INDEX IF NOT EXISTS catalog_targets ON catalog(year,school_code,major_code);
""" + SCORE_SCHEMA + """ """ + SCORE_SCHEMA + """
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC); CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT); CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
@@ -69,6 +71,10 @@ class Database:
conn.close() conn.close()
def upsert_catalog(self, records: list[dict]): def upsert_catalog(self, records: list[dict]):
with self.connection() as conn:
self._upsert_catalog(conn, records)
def _upsert_catalog(self, conn, records):
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"] fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
values = [] values = []
for record in records: for record in records:
@@ -77,10 +83,9 @@ class Database:
if not values: if not values:
return return
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:]) update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
with self.connection() as conn: conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values) for record in records:
for record in records: self._school(conn, record)
self._school(conn, record)
def _school(self, conn, record): def _school(self, conn, record):
record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))} record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))}
@@ -93,8 +98,11 @@ class Database:
def upsert_scores(self, records): def upsert_scores(self, records):
with self.connection() as conn: with self.connection() as conn:
for row in records: self._upsert_scores(conn, records)
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
def _upsert_scores(self, conn, records):
for row in records:
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
def _query(self, filters: dict): def _query(self, filters: dict):
conditions, params = ["1=1"], [] conditions, params = ["1=1"], []
@@ -141,9 +149,73 @@ class Database:
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]} return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
def all_catalog(self, filters): def all_catalog(self, filters):
return list(self.iter_catalog(filters))
def iter_catalog(self, filters, batch_size=200):
select, params = self._query(filters) select, params = self._query(filters)
with self.connection() as conn: with self.connection() as conn:
return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)] cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM catalog').fetchone()[0]
after = None
while True:
query = 'SELECT * FROM (' + select + ') export_rows WHERE id<=?'
values = [*params, cutoff]
if after:
year, school, rid = after
if school is None:
query += ' AND (year<? OR (year=? AND (school_name IS NOT NULL OR (school_name IS NULL AND id>?))))'
values.extend([year, year, rid])
else:
query += ' AND (year<? OR (year=? AND (school_name>? OR (school_name=? AND id>?))))'
values.extend([year, year, school, school, rid])
query += ' ORDER BY year DESC,school_name,id LIMIT ?'
with self.connection() as conn:
rows = conn.execute(query, [*values, batch_size]).fetchall()
if not rows:
return
for row in rows:
yield self._catalog_row(row)
last = rows[-1]
after = (last['year'], last['school_name'], last['id'])
def iter_scores(self, pairs, batch_size=200):
after = 0
with self.connection() as conn:
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM scores').fetchone()[0]
while True:
with self.connection() as conn:
rows = conn.execute('SELECT id,school_code,major_code,data FROM scores WHERE id>? AND id<=? ORDER BY id LIMIT ?', (after, cutoff, batch_size)).fetchall()
if not rows:
return
for row in rows:
if (row['school_code'], row['major_code']) in pairs:
yield json.loads(row['data'])
after = rows[-1]['id']
def score_targets(self, filters, batch_size=200):
conditions, params = ['year=?'], [filters['year']]
if filters.get('major_code'):
conditions.append('major_code=?')
params.append(filters['major_code'])
if filters.get('school_keyword'):
conditions.append('school_name LIKE ?')
params.append('%' + filters['school_keyword'] + '%')
if filters.get('provinces'):
conditions.append("(province='' OR province IN (" + ','.join('?' for _ in filters['provinces']) + '))')
params.extend(filters['provinces'])
after = None
while True:
where, values = ' AND '.join(conditions), list(params)
if after:
where += ' AND (school_code>? OR (school_code=? AND major_code>?))'
values.extend([after[0], after[0], after[1]])
query = 'SELECT school_code,major_code,MAX(school_name) AS school_name,MAX(province) AS province,MAX(level) AS level FROM catalog WHERE ' + where + ' GROUP BY school_code,major_code ORDER BY school_code,major_code LIMIT ?'
with self.connection() as conn:
rows = conn.execute(query, [*values, batch_size]).fetchall()
if not rows:
return
yield from (dict(row) for row in rows)
last = rows[-1]
after = (last['school_code'], last['major_code'])
def _catalog_row(self, row): def _catalog_row(self, row):
data = json.loads(row["data"]) data = json.loads(row["data"])
@@ -219,6 +291,17 @@ class Database:
with self.connection() as conn: with self.connection() as conn:
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit)) conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
def save_unit(self, jid, unit, records, scores=()):
with self.connection() as conn:
if not conn.execute('SELECT 1 FROM jobs WHERE id=?', (jid,)).fetchone():
raise ValueError('任务不存在')
if not conn.execute('INSERT OR IGNORE INTO job_units VALUES(?,?)', (jid, unit)).rowcount:
return False
self._upsert_scores(conn, scores)
self._upsert_catalog(conn, records)
conn.execute('UPDATE jobs SET records=records+?,updated_at=? WHERE id=?', (len(records), now(), jid))
return True
def log(self, jid, level, message): def log(self, jid, level, message):
with self.connection() as conn: with self.connection() as conn:
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000])) conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
+66
View File
@@ -0,0 +1,66 @@
"""Bounded exports, one download at a time, with disconnect cleanup."""
import csv
import io
import json
from fastapi.responses import StreamingResponse
from studio.imports import LABELS, spreadsheet_value
from studio.memory import release_idle_memory
class ExportStream(StreamingResponse):
def __init__(self, source, cleanup, **kwargs):
super().__init__(source, **kwargs)
self.source, self.cleanup = source, cleanup
async def __call__(self, scope, receive, send):
try:
await super().__call__(scope, receive, send)
finally:
try:
self.source.close()
finally:
try:
self.cleanup()
finally:
release_idle_memory()
def csv_chunks(records):
yield b'\xef\xbb\xbf'
stream = io.StringIO(newline='')
writer = csv.writer(stream)
writer.writerow(list(LABELS.values()))
yield stream.getvalue().encode('utf-8')
stream.seek(0)
stream.truncate(0)
for record in records:
writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS])
if stream.tell() >= 65536:
yield stream.getvalue().encode('utf-8')
stream.seek(0)
stream.truncate(0)
if stream.tell():
yield stream.getvalue().encode('utf-8')
def json_chunks(database, filters):
pairs = set()
yield b'{"version":1,"catalog":['
first = True
for row in database.iter_catalog(filters):
pairs.add((row['school_code'], row['major_code']))
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
first = False
yield b'],"scores":['
first = True
for row in database.iter_scores(pairs):
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
first = False
yield b']}'
def file_chunks(path):
with path.open('rb') as stream:
while chunk := stream.read(65536):
yield chunk
+45 -18
View File
@@ -2,6 +2,7 @@
import csv import csv
import io import io
import json import json
from contextlib import suppress
import sqlite3 import sqlite3
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
@@ -94,11 +95,18 @@ def import_file(path, year=None):
book = load_workbook(path, read_only=True, data_only=True) book = load_workbook(path, read_only=True, data_only=True)
try: try:
sheet = book.active sheet = book.active
if sheet.max_row > 300001 or sheet.max_column > 100: if (sheet.max_row or 0) > 300001 or (sheet.max_column or 0) > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入") raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
values = sheet.iter_rows(values_only=True) values = sheet.iter_rows(values_only=True)
headers = next(values, []) headers = next(values, [])
rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)] if len(headers) > 100:
raise ValueError("Excel 超过 100 列,请拆分后导入")
rows = []
for index, row in enumerate(values, 2):
if index > 300001 or len(row) > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
if any(v is not None for v in row):
rows.append(dict(zip(headers, row)))
finally: finally:
book.close() book.close()
else: else:
@@ -179,28 +187,47 @@ def spreadsheet_value(value):
def export_sheet(records, kind): def export_sheet(records, kind):
rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records]
if kind == "csv": if kind == "csv":
stream = io.StringIO(newline="") stream = io.StringIO(newline="")
writer = csv.writer(stream) writer = csv.writer(stream)
writer.writerow(list(LABELS.values())) writer.writerow(list(LABELS.values()))
writer.writerows(rows) writer.writerows([spreadsheet_value(record.get(field)) for field in LABELS] for record in records)
return stream.getvalue().encode("utf-8-sig") return stream.getvalue().encode("utf-8-sig")
book = Workbook() output = io.BytesIO()
sheet = book.active write_xlsx(records, output)
sheet.title = "院校专业" return output.getvalue()
sheet.append(list(LABELS.values()))
for row in rows:
sheet.append(row) def write_xlsx(records, output):
sheet.freeze_panes = "A2" from openpyxl.cell import WriteOnlyCell
sheet.auto_filter.ref = sheet.dimensions from openpyxl.utils import get_column_letter
from openpyxl.styles import Font, PatternFill, Alignment from openpyxl.styles import Font, PatternFill, Alignment
for cell in sheet[1]: book = Workbook(write_only=True)
sheet = book.create_sheet("院校专业")
sheet.freeze_panes = "A2"
header = []
for index, label in enumerate(LABELS.values(), 1):
cell = WriteOnlyCell(sheet, value=label)
cell.font = Font(color="FFFFFF", bold=True) cell.font = Font(color="FFFFFF", bold=True)
cell.fill = PatternFill("solid", fgColor="183447") cell.fill = PatternFill("solid", fgColor="183447")
cell.alignment = Alignment(vertical="center") cell.alignment = Alignment(vertical="center")
for col in sheet.columns: header.append(cell)
sheet.column_dimensions[col[0].column_letter].width = 24 sheet.column_dimensions[get_column_letter(index)].width = 24
output = io.BytesIO() count = 1
book.save(output) try:
return output.getvalue() sheet.append(header)
for record in records:
sheet.append([spreadsheet_value(record.get(field)) for field in LABELS])
count += 1
sheet.auto_filter.ref = f'A1:{get_column_letter(len(LABELS))}{count}'
book.save(output)
except BaseException:
with suppress(Exception):
if not sheet.closed:
sheet.close()
with suppress(Exception):
if sheet._writer:
sheet._writer.cleanup()
raise
finally:
book.close()
+20 -19
View File
@@ -1,6 +1,7 @@
"""Persistent units and cooperative pause/cancel for one local worker.""" """Persistent units and cooperative pause/cancel for one local worker."""
import threading import threading
import uuid import uuid
from itertools import chain
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
from studio.domain import PROVINCES, normalize_catalog from studio.domain import PROVINCES, normalize_catalog
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
@@ -230,8 +231,7 @@ class JobManager:
records = [] records = []
for row in client.directions(school, major, params.get("study_mode", "")): for row in client.directions(school, major, params.get("study_mode", "")):
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major)) records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
self._save(jid, records) self.db.save_unit(jid, unit, records)
self.complete_unit(jid, unit)
completed.add(unit) completed.add(unit)
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合") self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
except RemoteUnavailable as exc: except RemoteUnavailable as exc:
@@ -240,24 +240,25 @@ class JobManager:
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业") self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
def _scores(self, jid, params): def _scores(self, jid, params):
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")} provinces = [PROVINCES[p] for p in params['provinces']]
records = self.db.all_catalog(filters) filters = {"year": params["year"], "major_code": params.get("major_code", ""),
targets = {} "school_keyword": params.get("school_keyword", ""), 'provinces': provinces}
for row in records: targets = iter(self.db.score_targets(filters))
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]: first = next(targets, None)
continue if first is None and params.get('major_code'):
targets[(row["school_code"], row["major_code"])] = row targets = ({**row, 'major_code': params['major_code']}
if not targets and params.get("major_code"): for row in self.db.schools(params.get('school_keyword', '')) if row['province'] in provinces)
for row in self.db.schools(params.get("school_keyword", "")): first = next(targets, None)
if row["province"] in [PROVINCES[p] for p in params["provinces"]]: if first is None:
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
if not targets:
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引") raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
completed = self.db.completed_units(jid)
self.db.log(jid, 'info', '开始补充分数线和历史计划,分批读取专业目标并复用院校年份分数线')
client = ScoreClient(self.transport) client = ScoreClient(self.transport)
for (_, code), record in targets.items(): for record in chain((first,), targets):
self.checkpoint(jid) self.checkpoint(jid)
code = record['major_code']
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")} school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")] years = [y for y in params["score_years"] if f"scores:{school['school_code']}:{code}:{y}" not in completed]
if not years: if not years:
continue continue
school_id = client.school_id(school["school_name"]) school_id = client.school_id(school["school_name"])
@@ -270,10 +271,10 @@ class JobManager:
self.db.job_update(jid, message=label) self.db.job_update(jid, message=label)
try: try:
scores = client.score_lines(school, code, year, school_id) scores = client.score_lines(school, code, year, school_id)
self.db.upsert_scores(scores)
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else [] plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
self._save(jid, plans) unit = f"scores:{school['school_code']}:{code}:{year}"
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}") self.db.save_unit(jid, unit, plans, scores=scores)
completed.add(unit)
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条") self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
except RemoteUnavailable as exc: except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}") self._error(jid, f"{label}:{exc}")
+17
View File
@@ -0,0 +1,17 @@
"""Best-effort return of idle allocator memory after large operations."""
import ctypes
import gc
import sys
def release_idle_memory():
gc.collect()
if sys.platform == 'linux':
try:
trim = ctypes.CDLL(None).malloc_trim
trim.argtypes = [ctypes.c_size_t]
trim.restype = ctypes.c_int
return bool(trim(0))
except (AttributeError, OSError):
pass
return False
+10 -8
View File
@@ -107,6 +107,10 @@ class MySQLDatabase(Database):
with self.connection() as conn: with self.connection() as conn:
for statement in SCHEMA: for statement in SCHEMA:
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin') conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_export'").fetchone():
conn.execute('CREATE INDEX catalog_export ON catalog(year DESC,school_name,id)')
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_targets'").fetchone():
conn.execute('CREATE INDEX catalog_targets ON catalog(year,school_code,major_code)')
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone(): if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0') conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
if recover_jobs: if recover_jobs:
@@ -130,7 +134,7 @@ class MySQLDatabase(Database):
finally: finally:
raw.close() raw.close()
def upsert_catalog(self, records): def _upsert_catalog(self, conn, records):
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm'] fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
values = [] values = []
for record in records: for record in records:
@@ -139,12 +143,11 @@ class MySQLDatabase(Database):
if not values: if not values:
return return
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:]) update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
with self.connection() as conn: conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values) for record in records:
for record in records: self._school(conn, record)
self._school(conn, record)
def upsert_scores(self, records): def _upsert_scores(self, conn, records):
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line'] fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
values = [] values = []
for record in records: for record in records:
@@ -152,8 +155,7 @@ class MySQLDatabase(Database):
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note']))) row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
values.append([row.get(field) for field in fields]) values.append([row.get(field) for field in fields])
if values: if values:
with self.connection() as conn: conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
def favorite(self, rid, enabled): def favorite(self, rid, enabled):
# MariaDB reports changed rows; an already-selected favorite is still success. # MariaDB reports changed rows; an already-selected favorite is still success.
+31 -1
View File
@@ -3,6 +3,7 @@ import threading
import time import time
import random import random
from contextlib import contextmanager from contextlib import contextmanager
from collections import OrderedDict
import requests import requests
from studio.settings import proxy_url from studio.settings import proxy_url
@@ -19,6 +20,31 @@ class RateLimited(RuntimeError):
pass pass
class BoundedProxyAdapter(requests.adapters.HTTPAdapter):
"""Close obsolete proxy pools rather than retain every historical exit."""
def __init__(self, max_proxy_pools=4, **kwargs):
super().__init__(**kwargs)
self.max_proxy_pools = max(1, int(max_proxy_pools))
self.proxy_manager = OrderedDict()
def proxy_manager_for(self, proxy, **proxy_kwargs):
manager = super().proxy_manager_for(proxy, **proxy_kwargs)
self.proxy_manager.move_to_end(proxy)
while len(self.proxy_manager) > self.max_proxy_pools:
_, obsolete = self.proxy_manager.popitem(last=False)
obsolete.clear()
return manager
def bounded_session():
session = requests.Session()
session.trust_env = False
for scheme in ('http://', 'https://'):
session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4))
return session
class ProxyRouter: class ProxyRouter:
def __init__(self, settings, session=None): def __init__(self, settings, session=None):
self.settings = settings self.settings = settings
@@ -141,7 +167,7 @@ class ProxyRouter:
class Transport: class Transport:
def __init__(self, settings, router, session=None, wait=None): def __init__(self, settings, router, session=None, wait=None):
self.settings, self.router = settings, router self.settings, self.router = settings, router
self.session = session or requests.Session() self.session = session or bounded_session()
self.session.trust_env = False self.session.trust_env = False
self.session.headers = getattr(self.session, "headers", {}) self.session.headers = getattr(self.session, "headers", {})
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"}) self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
@@ -149,6 +175,10 @@ class Transport:
self.local = threading.local() self.local = threading.local()
self.lock = threading.RLock() self.lock = threading.RLock()
def close(self):
with self.lock:
self.session.close()
def sleep(self, seconds): def sleep(self, seconds):
remaining = seconds remaining = seconds
while remaining > 0: while remaining > 0:
+75
View File
@@ -0,0 +1,75 @@
import pytest
from studio.database import Database
from studio.domain import finish_record
def record(code='10001', name='甲大学', major='081200', province='北京', year=2027, direction='方向一'):
return finish_record({'school_code': code, 'school_name': name, 'major_code': major,
'major_name': major, 'province': province, 'year': year,
'direction': direction, 'subjects': ['101政治'], 'source': '测试'})
def test_history_targets_deduplicate_directions_and_preserve_filters(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_catalog([
record(direction='一'), record(direction='二'), record(major='081201'),
record(code='10002', name='乙大学', province='天津'),
record(code='10003', name='丙大学', province=''), record(year=2026),
])
rows = list(db.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
assert [(r['school_code'], r['major_code']) for r in rows] == [
('10001', '081200'), ('10001', '081201'), ('10003', '081200')]
assert all('data' not in row and 'subjects' not in row for row in rows)
rows = list(db.score_targets({'year': 2027, 'school_keyword': '甲', 'major_code': '081201'}))
assert [(r['school_code'], r['major_code']) for r in rows] == [('10001', '081201')]
def test_catalog_iterator_keeps_export_order_and_excludes_new_inserts(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_catalog([record(name='B大学'), record(name='A大学', direction='二'),
record(name='A大学', direction='三'), record(year=2026),
record(code='10002', name='C大学')])
iterator = db.iter_catalog({}, batch_size=2)
first = next(iterator)
db.upsert_catalog([record(name='Z大学', year=2028)])
rows = [first, *iterator]
assert [(r['year'], r['school_name']) for r in rows] == [
(2027, 'A大学'), (2027, 'A大学'), (2027, 'B大学'), (2027, 'C大学'), (2026, '甲大学')]
assert [r['school_name'] for r in db.iter_catalog({'school_keyword': 'A'}, batch_size=1)] == ['A大学', 'A大学']
def test_score_iterator_exports_only_selected_school_major_pairs(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_scores([
{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310},
{'school_code': '10002', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 320},
])
rows = list(db.iter_scores({('10001', '081200')}, batch_size=1))
assert len(rows) == 1
assert rows[0]['total'] == 310
def test_unit_commit_is_idempotent_for_records_count_and_checkpoint(tmp_path):
db = Database(tmp_path / 'data.db')
db.job_create('job', {})
rows = [record(direction='一'), record(direction='二')]
assert db.save_unit('job', 'catalog:test', rows) is True
assert db.save_unit('job', 'catalog:test', rows) is False
assert db.overview()['records'] == 2
assert db.job_get('job')['records'] == 2
assert db.unit_done('job', 'catalog:test')
def test_failed_unit_transaction_rolls_back_scores_records_and_checkpoint(tmp_path, monkeypatch):
db = Database(tmp_path / 'data.db')
db.job_create('job', {})
def fail(*args):
raise RuntimeError('interrupted persistence')
monkeypatch.setattr(db, '_school', fail)
score = {'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}
with pytest.raises(RuntimeError):
db.save_unit('job', 'scores:test', [record()], scores=[score])
assert db.overview()['records'] == db.overview()['scores'] == 0
assert db.job_get('job')['records'] == 0
assert not db.unit_done('job', 'scores:test')
+42
View File
@@ -0,0 +1,42 @@
from studio import network
def test_proxy_managers_are_bounded_and_evicted_pools_are_closed():
adapter = network.BoundedProxyAdapter(max_proxy_pools=3)
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
closed = []
original = first.clear
def observe():
original()
closed.append(True)
first.clear = observe
for port in range(10002, 10006):
adapter.proxy_manager_for(f'http://127.0.0.1:{port}')
assert len(adapter.proxy_manager) <= 3
assert closed == [True]
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is not first
adapter.close()
def test_active_proxy_pool_is_reused_while_old_pools_are_evicted():
adapter = network.BoundedProxyAdapter(max_proxy_pools=2)
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
adapter.proxy_manager_for('http://127.0.0.1:10002')
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
adapter.proxy_manager_for('http://127.0.0.1:10003')
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
assert 'http://127.0.0.1:10002' not in adapter.proxy_manager
adapter.close()
def test_default_transport_limits_pools_without_changing_route_or_tls(tmp_path):
from studio.settings import SettingsStore
from tests.test_network import Pool
settings = SettingsStore(tmp_path)
router = network.ProxyRouter(settings, session=Pool())
transport = network.Transport(settings, router)
adapter = transport.session.get_adapter('https://example.test')
assert isinstance(adapter, network.BoundedProxyAdapter)
assert transport.session.verify is True
assert router.resolve() == 'http://1.2.3.4:8080'
transport.close()
+125
View File
@@ -0,0 +1,125 @@
import asyncio
import csv
import io
import threading
import pytest
from fastapi.testclient import TestClient
from openpyxl import load_workbook
from studio.api import create_app
from tests.test_bounded_storage import record
@pytest.mark.parametrize('kind', ['csv', 'json', 'xlsx'])
def test_export_uses_bounded_rows_and_preserves_content(tmp_path, monkeypatch, kind):
app = create_app(tmp_path)
app.state.db.upsert_catalog([record(name='=1+1'), record(code='10002', name='含逗号,大学')])
app.state.db.upsert_scores([{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}])
def forbidden(*args, **kwargs):
raise AssertionError('export materializes all catalog rows')
monkeypatch.setattr(app.state.db, 'all_catalog', forbidden)
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
response = client.get('/api/export', params={'format': kind})
assert response.status_code == 200
if kind == 'json':
data = response.json()
assert {r['school_name'] for r in data['catalog']} == {'=1+1', '含逗号,大学'}
assert data['scores'][0]['total'] == 310
elif kind == 'csv':
rows = list(csv.reader(io.StringIO(response.content.decode('utf-8-sig'))))
assert len(rows) == 3
assert any("'=1+1" in row for row in rows)
assert any('含逗号,大学' in row for row in rows)
else:
book = load_workbook(io.BytesIO(response.content), read_only=True)
rows = list(book.active.values)
assert len(rows) == 3
assert any("'=1+1" in row for row in rows)
book.close()
assert not list((tmp_path / 'exports').glob('*'))
def test_concurrent_export_is_rejected_and_lock_is_released(tmp_path, monkeypatch):
app = create_app(tmp_path)
entered, release = threading.Event(), threading.Event()
original = app.state.db.iter_catalog
def slow_rows(*args, **kwargs):
if not entered.is_set():
entered.set()
assert release.wait(5)
yield from original(*args, **kwargs)
monkeypatch.setattr(app.state.db, 'iter_catalog', slow_rows)
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
responses = []
thread = threading.Thread(target=lambda: responses.append(client.get('/api/export')))
thread.start()
assert entered.wait(3)
blocked = client.get('/api/export', params={'format': 'json'})
release.set()
thread.join(5)
assert blocked.status_code == 429
assert responses[0].status_code == 200
assert client.get('/api/export').status_code == 200
def test_disconnected_stream_closes_source_and_releases_export_lock():
from studio.exporting import ExportStream
from starlette.requests import ClientDisconnect
lock = threading.BoundedSemaphore(1)
assert lock.acquire(False)
closed = []
def source():
try:
yield b'data'
finally:
closed.append(True)
async def send(message):
if message['type'] == 'http.response.body':
raise OSError('client disconnected')
async def receive():
return {'type': 'http.disconnect'}
response = ExportStream(source(), lock.release, media_type='text/csv')
with pytest.raises(ClientDisconnect):
asyncio.run(response({'type': 'http', 'asgi': {'spec_version': '2.4'}}, receive, send))
assert closed == [True]
assert lock.acquire(False)
def test_xlsx_packaging_error_removes_xml_tempfile_without_masking_error(tmp_path, monkeypatch):
from pathlib import Path
from studio.imports import write_xlsx
from openpyxl import Workbook
paths = []
def packaging_failure(book, filename):
sheet = book.worksheets[0]
sheet.close()
paths.append(Path(sheet._writer.out))
raise OSError('archive write failed')
monkeypatch.setattr(Workbook, 'save', packaging_failure)
with pytest.raises(OSError, match='archive write failed'):
write_xlsx([record()], tmp_path / 'failed.xlsx')
assert paths and not paths[0].exists()
def test_failed_export_batch_does_not_finish_a_successful_truncated_download(tmp_path, monkeypatch):
app = create_app(tmp_path)
def broken_rows(*args, **kwargs):
yield record()
raise RuntimeError('database batch failed')
monkeypatch.setattr(app.state.db, 'iter_catalog', broken_rows)
sent = []
async def send(message):
sent.append(message)
async def receive():
await asyncio.Event().wait()
scope = {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'},
'http_version': '1.1', 'method': 'GET', 'scheme': 'http',
'path': '/api/export', 'raw_path': b'/api/export', 'root_path': '',
'query_string': b'format=csv', 'headers': [(b'host', b'127.0.0.1:8765')],
'server': ('127.0.0.1', 8765), 'client': ('127.0.0.1', 12345)}
with pytest.raises(RuntimeError, match='database batch failed'):
asyncio.run(app(scope, receive, send))
assert not any(message['type'] == 'http.response.body' and not message.get('more_body', False) for message in sent)
+147
View File
@@ -0,0 +1,147 @@
import pytest
from studio.connectors import ScoreClient
from studio.network import RemoteUnavailable
from tests.test_network import Response
class HistorySource:
def __init__(self, pages):
self.pages = pages
self.calls = []
def request(self, method, url, **kwargs):
payload = kwargs['data']
self.calls.append((url.rsplit('/', 1)[-1], payload.copy()))
data = self.pages(payload.get('page', 1))
return Response({'code': '0000', 'data': data})
def test_full_score_array_over_page_limit_is_consumed_once():
rows = [{'code': '081200', 'total': 310, 'depart_name': '信息学院'}]
rows += [{'code': '999999', 'total': 200} for _ in range(197)]
source = HistorySource(lambda page: rows)
lines = ScoreClient(source).score_lines({'school_code': '10001', 'school_name': '北京大学'}, '081200', 2025, 1)
assert len(lines) == 1
assert lines[0]['total'] == 310
assert [call[1]['page'] for call in source.calls] == [1]
def test_short_wrapped_pages_follow_explicit_total():
source = HistorySource(lambda page: {'data': [{'value': page}], 'total': 2})
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
def test_short_wrapped_pages_follow_explicit_last_page():
source = HistorySource(lambda page: {'list': [{'value': page}], 'last_page': 2})
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
@pytest.mark.parametrize('data', [None, {}, {'data': None}, {'data': [None]}, {'list': [42]}])
def test_invalid_history_data_cannot_become_a_successful_empty_unit(data):
with pytest.raises(RemoteUnavailable):
list(ScoreClient(HistorySource(lambda page: data)).pages('/example', {}))
def test_empty_full_array_is_a_valid_empty_result():
assert list(ScoreClient(HistorySource(lambda page: [])).pages('/example', {})) == []
def test_repeated_real_page_still_raises_instead_of_returning_partial_data():
source = HistorySource(lambda page: {'data': [{'value': 1}], 'total': 2})
with pytest.raises(RemoteUnavailable):
list(ScoreClient(source).pages('/example', {}))
def test_empty_page_before_declared_total_is_an_error():
source = HistorySource(lambda page: {'data': [{'value': 1}] if page == 1 else [], 'total': 2})
with pytest.raises(RemoteUnavailable):
list(ScoreClient(source).pages('/example', {}))
def test_two_majors_reuse_one_complete_school_year_score_table():
rows = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
source = HistorySource(lambda page: rows)
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
assert client.score_lines(school, '081201', 2025, 1)[0]['total'] == 320
assert len(source.calls) == 1
def test_score_cache_separates_schools_and_years_and_has_a_bound():
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}])
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
for sid in range(20):
client.score_lines(school, '081200', 2025, sid)
client.score_lines(school, '081200', 2026, 19)
before = len(source.calls)
client.score_lines(school, '081200', 2026, 19)
assert len(source.calls) == before
client.score_lines(school, '081200', 2025, 0)
assert len(source.calls) == before + 1
def test_failed_lookup_is_cached_for_the_current_client_only():
source = HistorySource(lambda page: {'data': [], 'total': 0})
client = ScoreClient(source)
assert client.school_id('暂未收录大学') is None
assert client.school_id('暂未收录大学') is None
assert len(source.calls) == 1
assert ScoreClient(source).school_id('暂未收录大学') is None
assert len(source.calls) == 2
def test_invalid_score_response_is_not_cached():
valid = False
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}] if valid else {})
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
with pytest.raises(RemoteUnavailable):
client.score_lines(school, '081200', 2025, 1)
valid = True
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
def test_history_runner_uses_thin_targets_and_bulk_checkpoints(tmp_path, monkeypatch):
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
from tests.test_bounded_storage import record
db = Database(tmp_path / 'data.db')
db.upsert_catalog([record(), record(major='081201'), record(direction='二')])
params = {'year': 2027, 'provinces': ['11'], 'score_years': [2025, 2026], 'include_plans': True}
db.job_create('history', params)
class Source:
def __init__(self):
self.calls = []
def request(self, method, url, **kwargs):
payload = kwargs['data']
self.calls.append(url.rsplit('/', 1)[-1])
if url.endswith('/schoolList'):
data = {'data': [{'school_name': '甲大学', 'school_id': 1}], 'total': 1}
elif url.endswith('/schoolScore'):
data = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
else:
data = {'data': [{'special_code': payload['keyword'], 'special_name': payload['keyword'],
'plan_id': str(payload['year']) + payload['keyword'], 'recruit_number': 1}], 'total': 1}
return Response({'code': '0000', 'data': data})
def forbidden(*args, **kwargs):
raise AssertionError('full catalog decoding or per-unit checkpoint query')
monkeypatch.setattr(db, 'all_catalog', forbidden)
monkeypatch.setattr(db, 'unit_done', forbidden)
source = Source()
manager = JobManager(db, SettingsStore(tmp_path), transport=source)
manager._scores('history', params)
assert db.job_get('history')['completed_units'] == 4
assert db.overview()['scores'] == 4
assert db.job_get('history')['records'] == 4
assert source.calls.count('schoolScore') == 2
assert source.calls.count('planListV2') == 4
before = len(source.calls)
manager._scores('history', params)
assert len(source.calls) == before
+41
View File
@@ -98,6 +98,47 @@ def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypa
assert creations == [] assert creations == []
def test_mysql_bounded_catalog_and_history_targets_preserve_filters(mysql):
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
scores = select_score_lines([{'code': '081200', 'total': 310, 'depart_name': '计算机学院'}], '081200', '10001', '北京大学', 2026)
mysql.upsert_scores(scores)
targets = list(mysql.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
assert len(targets) == 1
assert targets[0]['major_code'] == '081200'
assert 'data' not in targets[0]
rows = list(mysql.iter_catalog({'year': 2027}, batch_size=1))
assert len(rows) == 2
assert len(list(mysql.iter_catalog({'year': 2026, 'max_score': 310}, batch_size=1))) == 2
assert len(list(mysql.iter_scores({('10001', '081200')}, batch_size=1))) == 1
def test_mysql_unit_commit_is_atomic_and_idempotent(mysql):
mysql.job_create('atomic-job', {})
rows = normalize_catalog(raw_catalog(), 2027)
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
assert mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert not mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert mysql.job_get('atomic-job')['records'] == 2
assert mysql.job_get('atomic-job')['completed_units'] == 1
assert mysql.overview()['records'] == 2
assert mysql.overview()['scores'] == 1
def test_mysql_failed_unit_rolls_back_every_write(mysql, monkeypatch):
mysql.job_create('atomic-job', {})
def fail(*args):
raise RuntimeError('interrupted unit')
monkeypatch.setattr(mysql, '_school', fail)
rows = normalize_catalog(raw_catalog(), 2027)
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
with pytest.raises(RuntimeError):
mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert mysql.job_get('atomic-job')['records'] == 0
assert mysql.job_get('atomic-job')['completed_units'] == 0
assert mysql.overview()['records'] == mysql.overview()['scores'] == 0
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path): def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
from studio.migrate import migrate_sqlite from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'snapshot.db') source = Database(tmp_path / 'snapshot.db')
+98
View File
@@ -0,0 +1,98 @@
import importlib.util
import queue
import threading
import time
from pathlib import Path
def maintainer_class():
path = Path(__file__).resolve().parents[1] / 'deploy/proxy_scheduler.py'
spec = importlib.util.spec_from_file_location('bounded_proxy_scheduler', path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.ProxyMaintainer
def test_detection_workers_are_bounded_reused_and_rounds_do_not_overlap():
state = {'active': 0, 'peak': 0, 'types': set(), 'mixed': False, 'done': [], 'thread_ids': set()}
lock = threading.Lock()
class Checker:
def __init__(self, work_type, target_queue, name):
self.work_type, self.queue = work_type, target_queue
def run(self):
while True:
try:
item = self.queue.get(block=False)
except queue.Empty:
return
with lock:
state['active'] += 1
state['peak'] = max(state['peak'], state['active'])
state['types'].add(self.work_type)
state['mixed'] |= len(state['types']) > 1
state['thread_ids'].add(threading.get_ident())
time.sleep(.01)
with lock:
state['done'].append(item)
state['active'] -= 1
if not state['active']:
state['types'].clear()
self.queue.task_done()
maintainer = maintainer_class()(workers=2, checker_class=Checker)
first = threading.Thread(target=maintainer.validate, args=('raw', range(10)))
second = threading.Thread(target=maintainer.validate, args=('use', range(10, 20)))
first.start(); second.start()
first.join(4); second.join(4)
assert not first.is_alive() and not second.is_alive()
before = set(state['thread_ids'])
maintainer.validate('raw', range(20, 25))
maintainer.close()
assert sorted(state['done']) == list(range(25))
assert state['peak'] <= 2
assert state['mixed'] is False
assert state['thread_ids'] == before
def test_stopped_detector_does_not_begin_new_rounds():
class Forbidden:
def __init__(self, *args):
raise AssertionError('detection began after shutdown')
maintainer = maintainer_class()(workers=2, checker_class=Forbidden)
maintainer.close()
maintainer.validate('raw', [1, 2, 3])
def test_failed_worker_waits_for_other_workers_before_next_round():
slow_started, release, next_round = threading.Event(), threading.Event(), threading.Event()
class Checker:
def __init__(self, work_type, target_queue, name):
self.kind, self.name = work_type, name
def run(self):
if self.kind == 'raw' and self.name == 'worker_00':
assert slow_started.wait(2)
raise RuntimeError('validator failed')
if self.kind == 'raw':
slow_started.set()
assert release.wait(3)
maintainer = maintainer_class()(workers=2, checker_class=Checker)
failures = []
def first_round():
try:
maintainer.validate('raw', [1, 2])
except RuntimeError:
failures.append(True)
first = threading.Thread(target=first_round)
first.start()
assert slow_started.wait(2)
def second_proxies():
next_round.set()
yield 3
second = threading.Thread(target=maintainer.validate, args=('use', second_proxies()))
second.start()
began_early = next_round.wait(.15)
release.set()
first.join(3); second.join(3)
maintainer.close()
assert began_early is False
assert next_round.is_set()
assert failures == [True]