From ec2c897752fea88bdd1ddc36ad75e572624f32e8 Mon Sep 17 00:00:00 2001 From: Jayson Date: Tue, 13 Oct 2026 00:26:47 +0800 Subject: [PATCH] Fix historical pagination and bound export and proxy memory --- README.md | 4 + deploy/kaoyan-proxy-scheduler.service | 5 +- deploy/kaoyan-studio.service | 1 + deploy/proxy_scheduler.py | 122 +++++++++++++++++++++ docs/debian-deployment.md | 4 + studio/api.py | 90 +++++++++++----- studio/connectors.py | 51 +++++++-- studio/database.py | 97 +++++++++++++++-- studio/exporting.py | 66 ++++++++++++ studio/imports.py | 63 +++++++---- studio/jobs.py | 39 +++---- studio/memory.py | 17 +++ studio/mysql_database.py | 18 ++-- studio/network.py | 32 +++++- tests/test_bounded_storage.py | 75 +++++++++++++ tests/test_connection_bounds.py | 42 ++++++++ tests/test_export_streams.py | 125 ++++++++++++++++++++++ tests/test_history_collection.py | 147 ++++++++++++++++++++++++++ tests/test_mysql.py | 41 +++++++ tests/test_proxy_scheduler_bounds.py | 98 +++++++++++++++++ 20 files changed, 1050 insertions(+), 87 deletions(-) create mode 100644 deploy/proxy_scheduler.py create mode 100644 studio/exporting.py create mode 100644 studio/memory.py create mode 100644 tests/test_bounded_storage.py create mode 100644 tests/test_connection_bounds.py create mode 100644 tests/test_export_streams.py create mode 100644 tests/test_history_collection.py create mode 100644 tests/test_proxy_scheduler_bounds.py diff --git a/README.md b/README.md index bfd0af0..51465e1 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,10 @@ Debian 12 / MariaDB 服务器模式、HTTPS 访问与代理池后台运行见 [ 可将当前筛选结果导出 CSV、Excel 或完整 JSON。表格导出保留院系/方向代码和字段口径,回导不会凭空增加同一记录。JSON 同时保留相关学校专业的历史分数线;凭据不进入导出。CSV/Excel 对可能被表格软件当作公式的文本加保护前缀。 +全量导出按 200 条分批读取,CSV/JSON 流式下载,Excel 使用写入模式和临时文件;同一实例一次只执行一个导出,重复请求返回 429。下载结束或中断后清理临时资源。导出期间新增的记录留待下次导出;导出不是事务一致性备份。大于 100 MB 的文件仍受导入上限限制,可按地区等条件分别导出;完整数据库备份按服务器部署说明操作。 + +历史采集在招生目录阶段完成后开始,按院校和年份复用全校分数线,每次最多缓存 8 份,历史计划继续逐专业获取。接口直接返回的数组按完整结果处理,带分页信息的结果按实际数量继续;不完整的响应不会被写成成功断点。数据、保存计数和完成断点在同一数据库事务中提交。 + ## 本地数据与开发 所有运行数据位于项目 `.runtime`:`studio.db` 保存目录、分数、收藏、任务和断点;`settings.json` 保存非机密配置;`chsi-cookie.enc` 由当前 Windows 用户的 DPAPI 加密。密码不持久化,接口不返回 Cookie。备份数据库时先停止工作台,再复制 `.runtime`;恢复加密 Cookie 需要原 Windows 用户环境,否则重新登录即可。 diff --git a/deploy/kaoyan-proxy-scheduler.service b/deploy/kaoyan-proxy-scheduler.service index d47ca80..23fef21 100644 --- a/deploy/kaoyan-proxy-scheduler.service +++ b/deploy/kaoyan-proxy-scheduler.service @@ -8,9 +8,12 @@ Requires=docker.service User=kaoyan Group=kaoyan WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool -ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py schedule +Environment=MALLOC_ARENA_MAX=2 +Environment=PROXY_CHECK_WORKERS=8 +ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python /opt/kaoyan-studio-deploy/app/deploy/proxy_scheduler.py Restart=on-failure RestartSec=10 +TimeoutStopSec=120 UMask=0077 PrivateTmp=true ProtectSystem=full diff --git a/deploy/kaoyan-studio.service b/deploy/kaoyan-studio.service index 9dbe2a4..44f4a84 100644 --- a/deploy/kaoyan-studio.service +++ b/deploy/kaoyan-studio.service @@ -9,6 +9,7 @@ User=kaoyan Group=kaoyan WorkingDirectory=/opt/kaoyan-studio-deploy/app EnvironmentFile=/etc/kaoyan-studio.env +Environment=MALLOC_ARENA_MAX=2 ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio Restart=on-failure RestartSec=10 diff --git a/deploy/proxy_scheduler.py b/deploy/proxy_scheduler.py new file mode 100644 index 0000000..0809773 --- /dev/null +++ b/deploy/proxy_scheduler.py @@ -0,0 +1,122 @@ +"""Run upstream proxy_pool checks with reusable workers and serialized jobs.""" +import os +import queue +import signal +import sys +import threading +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path + +APP_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(APP_ROOT)) +from studio.memory import release_idle_memory + + +class StopQueue(queue.Queue): + def __init__(self, stopping): + super().__init__() + self.stopping = stopping + + def get(self, *args, **kwargs): + if self.stopping.is_set(): + raise queue.Empty + return super().get(*args, **kwargs) + + +class ProxyMaintainer: + def __init__(self, workers=8, checker_class=None, fetcher_factory=None, handler_factory=None, pool_min=20): + self.worker_count = max(1, min(20, int(workers))) + self.checker_class = checker_class + self.fetcher_factory, self.handler_factory = fetcher_factory, handler_factory + self.pool_min = pool_min + self.stopping = threading.Event() + self.pipeline = threading.RLock() + self.workers = ThreadPoolExecutor(max_workers=self.worker_count, thread_name_prefix='proxy-check') + + def validate(self, work_type, proxies): + with self.pipeline: + if self.stopping.is_set(): + return + target = StopQueue(self.stopping) + for proxy in proxies: + if self.stopping.is_set(): + return + target.put(proxy) + if self.checker_class is None: + from helper.check import _ThreadChecker + self.checker_class = _ThreadChecker + count = min(self.worker_count, target.qsize()) + futures = [self.workers.submit(self.checker_class(work_type, target, f'worker_{i:02}').run) + for i in range(count)] + try: + failure = None + for future in futures: + try: + future.result() + except Exception as exc: + if failure is None: + failure = exc + if failure is not None: + raise failure + finally: + release_idle_memory() + + def fetch(self): + with self.pipeline: + if not self.stopping.is_set(): + self.validate('raw', self.fetcher_factory().run()) + + def check(self): + with self.pipeline: + if self.stopping.is_set(): + return + handler = self.handler_factory() + if handler.db.getCount().get('total', 0) < self.pool_min: + self.fetch() + self.validate('use', handler.getAll()) + + def stop(self): + self.stopping.set() + + def close(self): + self.stop() + with self.pipeline: + self.workers.shutdown(wait=True, cancel_futures=True) + + +def main(): + proxy_root = Path(os.environ.get('STUDIO_PROXY_POOL_ROOT', APP_ROOT.parent / 'proxy_pool')) + sys.path.insert(0, str(proxy_root)) + from apscheduler.schedulers.blocking import BlockingScheduler + from helper.check import _ThreadChecker + from helper.fetch import Fetcher + from handler.configHandler import ConfigHandler + from handler.logHandler import LogHandler + from handler.proxyHandler import ProxyHandler + + config = ConfigHandler() + maintainer = ProxyMaintainer(workers=int(os.environ.get('PROXY_CHECK_WORKERS', '8')), + checker_class=_ThreadChecker, fetcher_factory=Fetcher, + handler_factory=ProxyHandler, pool_min=config.poolSizeMin) + scheduler = BlockingScheduler(timezone=config.timezone, logger=LogHandler('scheduler'), + executors={'default': {'type': 'threadpool', 'max_workers': 1}}, + job_defaults={'coalesce': True, 'max_instances': 1, 'misfire_grace_time': 120}) + scheduler.add_job(maintainer.fetch, 'interval', minutes=5, id='proxy_fetch') + scheduler.add_job(maintainer.check, 'interval', minutes=2, id='proxy_check', + next_run_time=datetime.now(timezone.utc)) + def stop(signum, frame): + maintainer.stop() + if scheduler.running: + scheduler.shutdown(wait=False) + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + try: + if not maintainer.stopping.is_set(): + scheduler.start() + finally: + maintainer.close() + + +if __name__ == '__main__': + main() diff --git a/docs/debian-deployment.md b/docs/debian-deployment.md index aef78d5..50dd196 100644 --- a/docs/debian-deployment.md +++ b/docs/debian-deployment.md @@ -16,6 +16,8 @@ 安装模板在 `deploy/`。Python 依赖按 `requirements.lock` 安装;前端 `npm ci && npm run build`。不要启动多个工作台进程连接同一数据库,否则任务恢复状态会互相影响。原有站点、数据库和 fail2ban 配置不作修改。 +代理调度器运行本项目 `deploy/proxy_scheduler.py` 包装器,继续使用原版代理池的获取源、检测逻辑和 Redis 数据,不修改上游源码。默认复用 8 个检测线程,抓取与检查轮次串行运行,避免定时任务重叠。工作台和调度器设定 `MALLOC_ARENA_MAX=2`;每次导出/检查结束尝试释放分配器空闲内存。该操作不释放仍在使用的数据,实际内存以运行监测为准。 + ## 操作 ```bash @@ -28,6 +30,8 @@ systemctl restart kaoyan-studio 默认每次请求随机等待 1.2–2.4 秒,每 100 次成功请求轮换代理。设置页可修改上下限和轮换次数;0 表示关闭自动轮换。代理页显示实际成功请求路由、计数和切换次数。登录表单、会话同步、权限验证及紧随其后的失败页重试保持同一路由。免费代理池初次启动可能没有 HTTPS 候选,部署时保留了原电脑代理池的候选作为备用;这些候选仍会经过实际请求与失败隔离。 +当前全国任务采用 0 次自动轮换,维持通过登录验证的出口;连接失败仍会切换。Cookie 验证状态与出口关联,出口变化后需要重新验证,不宜为了让 IP 显示变化而频繁轮换。HTTP 会话最多保留 4 个代理连接池,淘汰旧池时主动关闭连接。 + Linux 无桌面环境,界面提供研招网登录入口与粘贴 Cookie。请在自己的电脑浏览器打开研招目录并登录,从开发者工具中复制目录接口请求的单行 Cookie,粘贴到工作台后点验证。服务器重新同步会话并检查受保护的第二页。迁移 Windows Cookie 时只经 SSH 传输加密封装,在服务器重新加密;不继承原验证标记。 ## 迁移与验证 diff --git a/studio/api.py b/studio/api.py index ae87bf1..581669c 100644 --- a/studio/api.py +++ b/studio/api.py @@ -2,6 +2,8 @@ import json import uuid import os +import tempfile +import threading from contextlib import asynccontextmanager from pathlib import Path from urllib.parse import urlparse @@ -11,7 +13,9 @@ from fastapi.staticfiles import StaticFiles from studio.auth import AuthService, CookieVault from studio.database import Database from studio.domain import PROVINCES -from studio.imports import import_file, export_sheet +from studio.imports import import_file, write_xlsx +from studio.exporting import ExportStream, csv_chunks, json_chunks, file_chunks +from studio.memory import release_idle_memory from studio.jobs import JobManager from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited from studio.settings import SettingsStore @@ -20,6 +24,31 @@ from studio.pool_service import PoolService ROOT = Path(__file__).resolve().parent.parent +class LocalAccessMiddleware: + """Check hosts and origins without buffering or wrapping response streams.""" + + def __init__(self, app, allowed_hosts): + self.app = app + self.allowed_hosts = allowed_hosts + + async def __call__(self, scope, receive, send): + if scope['type'] != 'http': + return await self.app(scope, receive, send) + request = Request(scope) + if request.url.hostname not in self.allowed_hosts: + response = JSONResponse({'detail': '访问主机不在允许列表中'}, status_code=403) + return await response(scope, receive, send) + if request.method not in ('GET', 'HEAD', 'OPTIONS'): + origin = request.headers.get('origin') + if origin: + parsed = urlparse(origin) + allowed = {str(request.base_url).rstrip('/'), 'http://127.0.0.1:5173', 'http://localhost:5173'} + if parsed.scheme not in ('http', 'https') or origin.rstrip('/') not in allowed: + response = JSONResponse({'detail': '已拒绝跨来源修改请求'}, status_code=403) + return await response(scope, receive, send) + return await self.app(scope, receive, send) + + def create_app(data_dir=None): directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime") directory.mkdir(parents=True, exist_ok=True) @@ -36,6 +65,9 @@ def create_app(data_dir=None): transport = Transport(settings, router) auth = AuthService(transport, CookieVault(directory)) jobs = JobManager(db, settings, transport, auth=auth) + export_lock = threading.BoundedSemaphore(1) + export_directory = directory / 'exports' + export_directory.mkdir(exist_ok=True) if (ROOT / "data/schools.json").exists(): db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8"))) @@ -46,6 +78,8 @@ def create_app(data_dir=None): jobs.resume(jid) yield jobs.close() + transport.close() + router.session.close() app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None) app.state.db, app.state.jobs = db, jobs @@ -53,19 +87,7 @@ def create_app(data_dir=None): app.state.pool = pool app.state.auth = auth - @app.middleware("http") - async def local_access(request: Request, call_next): - host = request.url.hostname - if host not in allowed_hosts: - return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403) - if request.method not in ("GET", "HEAD", "OPTIONS"): - origin = request.headers.get("origin") - if origin: - parsed = urlparse(origin) - allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"} - if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed: - return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403) - return await call_next(request) + app.add_middleware(LocalAccessMiddleware, allowed_hosts=allowed_hosts) @app.exception_handler(ValueError) async def invalid(request, exc): @@ -216,17 +238,35 @@ def create_app(data_dir=None): def export(request: Request, format: str = "csv"): if format not in ("csv", "xlsx", "json"): raise ValueError("导出格式无效") - rows = db.all_catalog(filters(request)) - if format == "json": - pairs = {(r["school_code"], r["major_code"]) for r in rows} - with db.connection() as conn: - scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs] - payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode() - mime = "application/json" - else: - payload = export_sheet(rows, format) - mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'}) + selected = filters(request) + if not export_lock.acquire(False): + raise HTTPException(429, '已有导出正在进行,请等待完成后重试') + temporary = None + def cleanup(): + try: + if temporary is not None: + temporary.unlink(missing_ok=True) + finally: + export_lock.release() + try: + if format == 'csv': + source, mime = csv_chunks(db.iter_catalog(selected)), 'text/csv; charset=utf-8' + elif format == 'json': + source, mime = json_chunks(db, selected), 'application/json' + else: + descriptor, path = tempfile.mkstemp(prefix='kaoyan-', suffix='.xlsx', dir=export_directory) + os.close(descriptor) + temporary = Path(path) + write_xlsx(db.iter_catalog(selected), temporary) + source = file_chunks(temporary) + mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + return ExportStream(source, cleanup, media_type=mime, + headers={'Content-Disposition': f'attachment; filename="kaoyan-data.{format}"', + 'X-Accel-Buffering': 'no'}) + except BaseException: + cleanup() + release_idle_memory() + raise dist = ROOT / "frontend/dist" if (dist / "assets").exists(): diff --git a/studio/connectors.py b/studio/connectors.py index 0400fd8..b28170a 100644 --- a/studio/connectors.py +++ b/studio/connectors.py @@ -2,6 +2,7 @@ import json import re import hashlib +from collections import OrderedDict from studio.domain import finish_record, integer, normalized_name, select_score_lines from studio.network import RemoteUnavailable, RateLimited @@ -153,6 +154,7 @@ class ScoreClient: def __init__(self, transport): self.transport = transport self.cache = {} + self.score_cache = OrderedDict() def post(self, endpoint, payload): for attempt in range(3): @@ -161,8 +163,13 @@ class ScoreClient: body = response.json() except ValueError: raise RemoteUnavailable("掌上考研未返回有效 JSON") from None + if not isinstance(body, dict): + raise RemoteUnavailable("掌上考研响应结构已变化") if str(body.get("code")) == "0000": - return body.get("data") or {} + data = body.get("data") + if not isinstance(data, (dict, list)): + raise RemoteUnavailable("掌上考研缺少有效数据,未写入不完整结果") + return data message = str(body.get('message', body.get('msg', ''))) if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')): if attempt == 2: @@ -172,20 +179,40 @@ class ScoreClient: raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试") def pages(self, endpoint, payload, limit=100): - seen = set() + seen, consumed = set(), 0 for page in range(1, 10001): data = self.post(endpoint, {**payload, "page": page, "limit": limit}) - rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data - if not isinstance(rows, list): + full_list = isinstance(data, list) + rows = data.get("data", data.get("list")) if isinstance(data, dict) else data + if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows): raise RemoteUnavailable("掌上考研响应结构已变化") + total = integer(data.get("total")) if isinstance(data, dict) else None + last = integer(data.get("last_page")) if isinstance(data, dict) else None + if isinstance(data, dict): + for field, value in (('total', total), ('last_page', last)): + if data.get(field) is not None and (value is None or value < 0 or isinstance(data[field], bool)): + raise RemoteUnavailable("掌上考研分页信息无效,未写入不完整结果") + if not rows and (total is not None and consumed < total or last is not None and page < last): + raise RemoteUnavailable("掌上考研提前返回空页,未写入不完整结果") signature = json.dumps(rows, ensure_ascii=False, sort_keys=True) if rows and signature in seen: raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页") seen.add(signature) + consumed += len(rows) yield from rows - total = integer(data.get("total")) if isinstance(data, dict) else None - last = integer(data.get("last_page")) if isinstance(data, dict) else None - if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total: + # Bare arrays are complete responses; wrapped pages use their actual + # metadata even when the server ignores the requested page size. + terminal = full_list or not rows + if not terminal: + if last is not None: + terminal = page >= last + if terminal and total is not None and consumed < total: + raise RemoteUnavailable("掌上考研分页未返回全部记录") + elif total is not None: + terminal = consumed >= total + else: + terminal = len(rows) < limit + if terminal: return raise RemoteUnavailable("历史数据分页超过安全上限") @@ -198,10 +225,18 @@ class ScoreClient: if len(matches) == 1 and matches[0].get("school_id"): self.cache[key] = matches[0]["school_id"] return self.cache[key] + self.cache[key] = None return None def score_lines(self, school, major_code, year, school_id): - rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year})) + key = (str(school_id), int(year)) + if key not in self.score_cache: + rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year})) + self.score_cache[key] = rows + if len(self.score_cache) > 8: + self.score_cache.popitem(last=False) + self.score_cache.move_to_end(key) + rows = self.score_cache[key] return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year) def plans(self, school, major_code, year, school_id): diff --git a/studio/database.py b/studio/database.py index d3353f6..b36f7ad 100644 --- a/studio/database.py +++ b/studio/database.py @@ -19,6 +19,8 @@ CREATE TABLE IF NOT EXISTS catalog( department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT, subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL); CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code); +CREATE INDEX IF NOT EXISTS catalog_export ON catalog(year DESC,school_name,id); +CREATE INDEX IF NOT EXISTS catalog_targets ON catalog(year,school_code,major_code); """ + SCORE_SCHEMA + """ CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC); CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT); @@ -69,6 +71,10 @@ class Database: conn.close() def upsert_catalog(self, records: list[dict]): + with self.connection() as conn: + self._upsert_catalog(conn, records) + + def _upsert_catalog(self, conn, records): fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"] values = [] for record in records: @@ -77,10 +83,9 @@ class Database: if not values: return update = ",".join(f"{field}=excluded.{field}" for field in fields[1:]) - with self.connection() as conn: - conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values) - for record in records: - self._school(conn, record) + conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values) + for record in records: + self._school(conn, record) def _school(self, conn, record): record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))} @@ -93,8 +98,11 @@ class Database: def upsert_scores(self, records): with self.connection() as conn: - for row in records: - conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or '')) + self._upsert_scores(conn, records) + + def _upsert_scores(self, conn, records): + for row in records: + conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or '')) def _query(self, filters: dict): conditions, params = ["1=1"], [] @@ -141,9 +149,73 @@ class Database: return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]} def all_catalog(self, filters): + return list(self.iter_catalog(filters)) + + def iter_catalog(self, filters, batch_size=200): select, params = self._query(filters) with self.connection() as conn: - return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)] + cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM catalog').fetchone()[0] + after = None + while True: + query = 'SELECT * FROM (' + select + ') export_rows WHERE id<=?' + values = [*params, cutoff] + if after: + year, school, rid = after + if school is None: + query += ' AND (year?))))' + values.extend([year, year, rid]) + else: + query += ' AND (year? OR (school_name=? AND id>?))))' + values.extend([year, year, school, school, rid]) + query += ' ORDER BY year DESC,school_name,id LIMIT ?' + with self.connection() as conn: + rows = conn.execute(query, [*values, batch_size]).fetchall() + if not rows: + return + for row in rows: + yield self._catalog_row(row) + last = rows[-1] + after = (last['year'], last['school_name'], last['id']) + + def iter_scores(self, pairs, batch_size=200): + after = 0 + with self.connection() as conn: + cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM scores').fetchone()[0] + while True: + with self.connection() as conn: + rows = conn.execute('SELECT id,school_code,major_code,data FROM scores WHERE id>? AND id<=? ORDER BY id LIMIT ?', (after, cutoff, batch_size)).fetchall() + if not rows: + return + for row in rows: + if (row['school_code'], row['major_code']) in pairs: + yield json.loads(row['data']) + after = rows[-1]['id'] + + def score_targets(self, filters, batch_size=200): + conditions, params = ['year=?'], [filters['year']] + if filters.get('major_code'): + conditions.append('major_code=?') + params.append(filters['major_code']) + if filters.get('school_keyword'): + conditions.append('school_name LIKE ?') + params.append('%' + filters['school_keyword'] + '%') + if filters.get('provinces'): + conditions.append("(province='' OR province IN (" + ','.join('?' for _ in filters['provinces']) + '))') + params.extend(filters['provinces']) + after = None + while True: + where, values = ' AND '.join(conditions), list(params) + if after: + where += ' AND (school_code>? OR (school_code=? AND major_code>?))' + values.extend([after[0], after[0], after[1]]) + query = 'SELECT school_code,major_code,MAX(school_name) AS school_name,MAX(province) AS province,MAX(level) AS level FROM catalog WHERE ' + where + ' GROUP BY school_code,major_code ORDER BY school_code,major_code LIMIT ?' + with self.connection() as conn: + rows = conn.execute(query, [*values, batch_size]).fetchall() + if not rows: + return + yield from (dict(row) for row in rows) + last = rows[-1] + after = (last['school_code'], last['major_code']) def _catalog_row(self, row): data = json.loads(row["data"]) @@ -219,6 +291,17 @@ class Database: with self.connection() as conn: conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit)) + def save_unit(self, jid, unit, records, scores=()): + with self.connection() as conn: + if not conn.execute('SELECT 1 FROM jobs WHERE id=?', (jid,)).fetchone(): + raise ValueError('任务不存在') + if not conn.execute('INSERT OR IGNORE INTO job_units VALUES(?,?)', (jid, unit)).rowcount: + return False + self._upsert_scores(conn, scores) + self._upsert_catalog(conn, records) + conn.execute('UPDATE jobs SET records=records+?,updated_at=? WHERE id=?', (len(records), now(), jid)) + return True + def log(self, jid, level, message): with self.connection() as conn: conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000])) diff --git a/studio/exporting.py b/studio/exporting.py new file mode 100644 index 0000000..bbaac85 --- /dev/null +++ b/studio/exporting.py @@ -0,0 +1,66 @@ +"""Bounded exports, one download at a time, with disconnect cleanup.""" +import csv +import io +import json + +from fastapi.responses import StreamingResponse +from studio.imports import LABELS, spreadsheet_value +from studio.memory import release_idle_memory + + +class ExportStream(StreamingResponse): + def __init__(self, source, cleanup, **kwargs): + super().__init__(source, **kwargs) + self.source, self.cleanup = source, cleanup + + async def __call__(self, scope, receive, send): + try: + await super().__call__(scope, receive, send) + finally: + try: + self.source.close() + finally: + try: + self.cleanup() + finally: + release_idle_memory() + + +def csv_chunks(records): + yield b'\xef\xbb\xbf' + stream = io.StringIO(newline='') + writer = csv.writer(stream) + writer.writerow(list(LABELS.values())) + yield stream.getvalue().encode('utf-8') + stream.seek(0) + stream.truncate(0) + for record in records: + writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS]) + if stream.tell() >= 65536: + yield stream.getvalue().encode('utf-8') + stream.seek(0) + stream.truncate(0) + if stream.tell(): + yield stream.getvalue().encode('utf-8') + + +def json_chunks(database, filters): + pairs = set() + yield b'{"version":1,"catalog":[' + first = True + for row in database.iter_catalog(filters): + pairs.add((row['school_code'], row['major_code'])) + yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8') + first = False + yield b'],"scores":[' + first = True + for row in database.iter_scores(pairs): + yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8') + first = False + yield b']}' + + +def file_chunks(path): + with path.open('rb') as stream: + while chunk := stream.read(65536): + yield chunk diff --git a/studio/imports.py b/studio/imports.py index afe3d47..86e31a8 100644 --- a/studio/imports.py +++ b/studio/imports.py @@ -2,6 +2,7 @@ import csv import io import json +from contextlib import suppress import sqlite3 from dataclasses import dataclass, field from pathlib import Path @@ -94,11 +95,18 @@ def import_file(path, year=None): book = load_workbook(path, read_only=True, data_only=True) try: sheet = book.active - if sheet.max_row > 300001 or sheet.max_column > 100: + if (sheet.max_row or 0) > 300001 or (sheet.max_column or 0) > 100: raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入") values = sheet.iter_rows(values_only=True) headers = next(values, []) - rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)] + if len(headers) > 100: + raise ValueError("Excel 超过 100 列,请拆分后导入") + rows = [] + for index, row in enumerate(values, 2): + if index > 300001 or len(row) > 100: + raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入") + if any(v is not None for v in row): + rows.append(dict(zip(headers, row))) finally: book.close() else: @@ -179,28 +187,47 @@ def spreadsheet_value(value): def export_sheet(records, kind): - rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records] if kind == "csv": stream = io.StringIO(newline="") writer = csv.writer(stream) writer.writerow(list(LABELS.values())) - writer.writerows(rows) + writer.writerows([spreadsheet_value(record.get(field)) for field in LABELS] for record in records) return stream.getvalue().encode("utf-8-sig") - book = Workbook() - sheet = book.active - sheet.title = "院校专业" - sheet.append(list(LABELS.values())) - for row in rows: - sheet.append(row) - sheet.freeze_panes = "A2" - sheet.auto_filter.ref = sheet.dimensions + output = io.BytesIO() + write_xlsx(records, output) + return output.getvalue() + + +def write_xlsx(records, output): + from openpyxl.cell import WriteOnlyCell + from openpyxl.utils import get_column_letter from openpyxl.styles import Font, PatternFill, Alignment - for cell in sheet[1]: + book = Workbook(write_only=True) + sheet = book.create_sheet("院校专业") + sheet.freeze_panes = "A2" + header = [] + for index, label in enumerate(LABELS.values(), 1): + cell = WriteOnlyCell(sheet, value=label) cell.font = Font(color="FFFFFF", bold=True) cell.fill = PatternFill("solid", fgColor="183447") cell.alignment = Alignment(vertical="center") - for col in sheet.columns: - sheet.column_dimensions[col[0].column_letter].width = 24 - output = io.BytesIO() - book.save(output) - return output.getvalue() + header.append(cell) + sheet.column_dimensions[get_column_letter(index)].width = 24 + count = 1 + try: + sheet.append(header) + for record in records: + sheet.append([spreadsheet_value(record.get(field)) for field in LABELS]) + count += 1 + sheet.auto_filter.ref = f'A1:{get_column_letter(len(LABELS))}{count}' + book.save(output) + except BaseException: + with suppress(Exception): + if not sheet.closed: + sheet.close() + with suppress(Exception): + if sheet._writer: + sheet._writer.cleanup() + raise + finally: + book.close() diff --git a/studio/jobs.py b/studio/jobs.py index 8efb4d7..1869f7f 100644 --- a/studio/jobs.py +++ b/studio/jobs.py @@ -1,6 +1,7 @@ """Persistent units and cooperative pause/cancel for one local worker.""" import threading import uuid +from itertools import chain from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch from studio.domain import PROVINCES, normalize_catalog from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited @@ -230,8 +231,7 @@ class JobManager: records = [] for row in client.directions(school, major, params.get("study_mode", "")): records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major)) - self._save(jid, records) - self.complete_unit(jid, unit) + self.db.save_unit(jid, unit, records) completed.add(unit) self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合") except RemoteUnavailable as exc: @@ -240,24 +240,25 @@ class JobManager: self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业") def _scores(self, jid, params): - filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")} - records = self.db.all_catalog(filters) - targets = {} - for row in records: - if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]: - continue - targets[(row["school_code"], row["major_code"])] = row - if not targets and params.get("major_code"): - for row in self.db.schools(params.get("school_keyword", "")): - if row["province"] in [PROVINCES[p] for p in params["provinces"]]: - targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]} - if not targets: + provinces = [PROVINCES[p] for p in params['provinces']] + filters = {"year": params["year"], "major_code": params.get("major_code", ""), + "school_keyword": params.get("school_keyword", ""), 'provinces': provinces} + targets = iter(self.db.score_targets(filters)) + first = next(targets, None) + if first is None and params.get('major_code'): + targets = ({**row, 'major_code': params['major_code']} + for row in self.db.schools(params.get('school_keyword', '')) if row['province'] in provinces) + first = next(targets, None) + if first is None: raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引") + completed = self.db.completed_units(jid) + self.db.log(jid, 'info', '开始补充分数线和历史计划,分批读取专业目标并复用院校年份分数线') client = ScoreClient(self.transport) - for (_, code), record in targets.items(): + for record in chain((first,), targets): self.checkpoint(jid) + code = record['major_code'] school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")} - years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")] + years = [y for y in params["score_years"] if f"scores:{school['school_code']}:{code}:{y}" not in completed] if not years: continue school_id = client.school_id(school["school_name"]) @@ -270,10 +271,10 @@ class JobManager: self.db.job_update(jid, message=label) try: scores = client.score_lines(school, code, year, school_id) - self.db.upsert_scores(scores) plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else [] - self._save(jid, plans) - self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}") + unit = f"scores:{school['school_code']}:{code}:{year}" + self.db.save_unit(jid, unit, plans, scores=scores) + completed.add(unit) self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条") except RemoteUnavailable as exc: self._error(jid, f"{label}:{exc}") diff --git a/studio/memory.py b/studio/memory.py new file mode 100644 index 0000000..ab3fff1 --- /dev/null +++ b/studio/memory.py @@ -0,0 +1,17 @@ +"""Best-effort return of idle allocator memory after large operations.""" +import ctypes +import gc +import sys + + +def release_idle_memory(): + gc.collect() + if sys.platform == 'linux': + try: + trim = ctypes.CDLL(None).malloc_trim + trim.argtypes = [ctypes.c_size_t] + trim.restype = ctypes.c_int + return bool(trim(0)) + except (AttributeError, OSError): + pass + return False diff --git a/studio/mysql_database.py b/studio/mysql_database.py index 2886245..ed69999 100644 --- a/studio/mysql_database.py +++ b/studio/mysql_database.py @@ -107,6 +107,10 @@ class MySQLDatabase(Database): with self.connection() as conn: for statement in SCHEMA: conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin') + if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_export'").fetchone(): + conn.execute('CREATE INDEX catalog_export ON catalog(year DESC,school_name,id)') + if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_targets'").fetchone(): + conn.execute('CREATE INDEX catalog_targets ON catalog(year,school_code,major_code)') if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone(): conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0') if recover_jobs: @@ -130,7 +134,7 @@ class MySQLDatabase(Database): finally: raw.close() - def upsert_catalog(self, records): + def _upsert_catalog(self, conn, records): fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm'] values = [] for record in records: @@ -139,12 +143,11 @@ class MySQLDatabase(Database): if not values: return update = ','.join(f'{field}=VALUES({field})' for field in fields[1:]) - with self.connection() as conn: - conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values) - for record in records: - self._school(conn, record) + conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values) + for record in records: + self._school(conn, record) - def upsert_scores(self, records): + def _upsert_scores(self, conn, records): fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line'] values = [] for record in records: @@ -152,8 +155,7 @@ class MySQLDatabase(Database): row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note']))) values.append([row.get(field) for field in fields]) if values: - with self.connection() as conn: - conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values) + conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values) def favorite(self, rid, enabled): # MariaDB reports changed rows; an already-selected favorite is still success. diff --git a/studio/network.py b/studio/network.py index 9e9a87e..52107d7 100644 --- a/studio/network.py +++ b/studio/network.py @@ -3,6 +3,7 @@ import threading import time import random from contextlib import contextmanager +from collections import OrderedDict import requests from studio.settings import proxy_url @@ -19,6 +20,31 @@ class RateLimited(RuntimeError): pass +class BoundedProxyAdapter(requests.adapters.HTTPAdapter): + """Close obsolete proxy pools rather than retain every historical exit.""" + + def __init__(self, max_proxy_pools=4, **kwargs): + super().__init__(**kwargs) + self.max_proxy_pools = max(1, int(max_proxy_pools)) + self.proxy_manager = OrderedDict() + + def proxy_manager_for(self, proxy, **proxy_kwargs): + manager = super().proxy_manager_for(proxy, **proxy_kwargs) + self.proxy_manager.move_to_end(proxy) + while len(self.proxy_manager) > self.max_proxy_pools: + _, obsolete = self.proxy_manager.popitem(last=False) + obsolete.clear() + return manager + + +def bounded_session(): + session = requests.Session() + session.trust_env = False + for scheme in ('http://', 'https://'): + session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4)) + return session + + class ProxyRouter: def __init__(self, settings, session=None): self.settings = settings @@ -141,7 +167,7 @@ class ProxyRouter: class Transport: def __init__(self, settings, router, session=None, wait=None): self.settings, self.router = settings, router - self.session = session or requests.Session() + self.session = session or bounded_session() self.session.trust_env = False self.session.headers = getattr(self.session, "headers", {}) self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"}) @@ -149,6 +175,10 @@ class Transport: self.local = threading.local() self.lock = threading.RLock() + def close(self): + with self.lock: + self.session.close() + def sleep(self, seconds): remaining = seconds while remaining > 0: diff --git a/tests/test_bounded_storage.py b/tests/test_bounded_storage.py new file mode 100644 index 0000000..56fc128 --- /dev/null +++ b/tests/test_bounded_storage.py @@ -0,0 +1,75 @@ +import pytest + +from studio.database import Database +from studio.domain import finish_record + + +def record(code='10001', name='甲大学', major='081200', province='北京', year=2027, direction='方向一'): + return finish_record({'school_code': code, 'school_name': name, 'major_code': major, + 'major_name': major, 'province': province, 'year': year, + 'direction': direction, 'subjects': ['101政治'], 'source': '测试'}) + + +def test_history_targets_deduplicate_directions_and_preserve_filters(tmp_path): + db = Database(tmp_path / 'data.db') + db.upsert_catalog([ + record(direction='一'), record(direction='二'), record(major='081201'), + record(code='10002', name='乙大学', province='天津'), + record(code='10003', name='丙大学', province=''), record(year=2026), + ]) + rows = list(db.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1)) + assert [(r['school_code'], r['major_code']) for r in rows] == [ + ('10001', '081200'), ('10001', '081201'), ('10003', '081200')] + assert all('data' not in row and 'subjects' not in row for row in rows) + rows = list(db.score_targets({'year': 2027, 'school_keyword': '甲', 'major_code': '081201'})) + assert [(r['school_code'], r['major_code']) for r in rows] == [('10001', '081201')] + + +def test_catalog_iterator_keeps_export_order_and_excludes_new_inserts(tmp_path): + db = Database(tmp_path / 'data.db') + db.upsert_catalog([record(name='B大学'), record(name='A大学', direction='二'), + record(name='A大学', direction='三'), record(year=2026), + record(code='10002', name='C大学')]) + iterator = db.iter_catalog({}, batch_size=2) + first = next(iterator) + db.upsert_catalog([record(name='Z大学', year=2028)]) + rows = [first, *iterator] + assert [(r['year'], r['school_name']) for r in rows] == [ + (2027, 'A大学'), (2027, 'A大学'), (2027, 'B大学'), (2027, 'C大学'), (2026, '甲大学')] + assert [r['school_name'] for r in db.iter_catalog({'school_keyword': 'A'}, batch_size=1)] == ['A大学', 'A大学'] + + +def test_score_iterator_exports_only_selected_school_major_pairs(tmp_path): + db = Database(tmp_path / 'data.db') + db.upsert_scores([ + {'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}, + {'school_code': '10002', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 320}, + ]) + rows = list(db.iter_scores({('10001', '081200')}, batch_size=1)) + assert len(rows) == 1 + assert rows[0]['total'] == 310 + + +def test_unit_commit_is_idempotent_for_records_count_and_checkpoint(tmp_path): + db = Database(tmp_path / 'data.db') + db.job_create('job', {}) + rows = [record(direction='一'), record(direction='二')] + assert db.save_unit('job', 'catalog:test', rows) is True + assert db.save_unit('job', 'catalog:test', rows) is False + assert db.overview()['records'] == 2 + assert db.job_get('job')['records'] == 2 + assert db.unit_done('job', 'catalog:test') + + +def test_failed_unit_transaction_rolls_back_scores_records_and_checkpoint(tmp_path, monkeypatch): + db = Database(tmp_path / 'data.db') + db.job_create('job', {}) + def fail(*args): + raise RuntimeError('interrupted persistence') + monkeypatch.setattr(db, '_school', fail) + score = {'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310} + with pytest.raises(RuntimeError): + db.save_unit('job', 'scores:test', [record()], scores=[score]) + assert db.overview()['records'] == db.overview()['scores'] == 0 + assert db.job_get('job')['records'] == 0 + assert not db.unit_done('job', 'scores:test') diff --git a/tests/test_connection_bounds.py b/tests/test_connection_bounds.py new file mode 100644 index 0000000..4a11f36 --- /dev/null +++ b/tests/test_connection_bounds.py @@ -0,0 +1,42 @@ +from studio import network + + +def test_proxy_managers_are_bounded_and_evicted_pools_are_closed(): + adapter = network.BoundedProxyAdapter(max_proxy_pools=3) + first = adapter.proxy_manager_for('http://127.0.0.1:10001') + closed = [] + original = first.clear + def observe(): + original() + closed.append(True) + first.clear = observe + for port in range(10002, 10006): + adapter.proxy_manager_for(f'http://127.0.0.1:{port}') + assert len(adapter.proxy_manager) <= 3 + assert closed == [True] + assert adapter.proxy_manager_for('http://127.0.0.1:10001') is not first + adapter.close() + + +def test_active_proxy_pool_is_reused_while_old_pools_are_evicted(): + adapter = network.BoundedProxyAdapter(max_proxy_pools=2) + first = adapter.proxy_manager_for('http://127.0.0.1:10001') + adapter.proxy_manager_for('http://127.0.0.1:10002') + assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first + adapter.proxy_manager_for('http://127.0.0.1:10003') + assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first + assert 'http://127.0.0.1:10002' not in adapter.proxy_manager + adapter.close() + + +def test_default_transport_limits_pools_without_changing_route_or_tls(tmp_path): + from studio.settings import SettingsStore + from tests.test_network import Pool + settings = SettingsStore(tmp_path) + router = network.ProxyRouter(settings, session=Pool()) + transport = network.Transport(settings, router) + adapter = transport.session.get_adapter('https://example.test') + assert isinstance(adapter, network.BoundedProxyAdapter) + assert transport.session.verify is True + assert router.resolve() == 'http://1.2.3.4:8080' + transport.close() diff --git a/tests/test_export_streams.py b/tests/test_export_streams.py new file mode 100644 index 0000000..bb0ee9f --- /dev/null +++ b/tests/test_export_streams.py @@ -0,0 +1,125 @@ +import asyncio +import csv +import io +import threading + +import pytest +from fastapi.testclient import TestClient +from openpyxl import load_workbook + +from studio.api import create_app +from tests.test_bounded_storage import record + + +@pytest.mark.parametrize('kind', ['csv', 'json', 'xlsx']) +def test_export_uses_bounded_rows_and_preserves_content(tmp_path, monkeypatch, kind): + app = create_app(tmp_path) + app.state.db.upsert_catalog([record(name='=1+1'), record(code='10002', name='含逗号,大学')]) + app.state.db.upsert_scores([{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}]) + def forbidden(*args, **kwargs): + raise AssertionError('export materializes all catalog rows') + monkeypatch.setattr(app.state.db, 'all_catalog', forbidden) + with TestClient(app, base_url='http://127.0.0.1:8765') as client: + response = client.get('/api/export', params={'format': kind}) + assert response.status_code == 200 + if kind == 'json': + data = response.json() + assert {r['school_name'] for r in data['catalog']} == {'=1+1', '含逗号,大学'} + assert data['scores'][0]['total'] == 310 + elif kind == 'csv': + rows = list(csv.reader(io.StringIO(response.content.decode('utf-8-sig')))) + assert len(rows) == 3 + assert any("'=1+1" in row for row in rows) + assert any('含逗号,大学' in row for row in rows) + else: + book = load_workbook(io.BytesIO(response.content), read_only=True) + rows = list(book.active.values) + assert len(rows) == 3 + assert any("'=1+1" in row for row in rows) + book.close() + assert not list((tmp_path / 'exports').glob('*')) + + +def test_concurrent_export_is_rejected_and_lock_is_released(tmp_path, monkeypatch): + app = create_app(tmp_path) + entered, release = threading.Event(), threading.Event() + original = app.state.db.iter_catalog + def slow_rows(*args, **kwargs): + if not entered.is_set(): + entered.set() + assert release.wait(5) + yield from original(*args, **kwargs) + monkeypatch.setattr(app.state.db, 'iter_catalog', slow_rows) + with TestClient(app, base_url='http://127.0.0.1:8765') as client: + responses = [] + thread = threading.Thread(target=lambda: responses.append(client.get('/api/export'))) + thread.start() + assert entered.wait(3) + blocked = client.get('/api/export', params={'format': 'json'}) + release.set() + thread.join(5) + assert blocked.status_code == 429 + assert responses[0].status_code == 200 + assert client.get('/api/export').status_code == 200 + + +def test_disconnected_stream_closes_source_and_releases_export_lock(): + from studio.exporting import ExportStream + from starlette.requests import ClientDisconnect + lock = threading.BoundedSemaphore(1) + assert lock.acquire(False) + + + closed = [] + def source(): + try: + yield b'data' + finally: + closed.append(True) + async def send(message): + if message['type'] == 'http.response.body': + raise OSError('client disconnected') + async def receive(): + return {'type': 'http.disconnect'} + response = ExportStream(source(), lock.release, media_type='text/csv') + with pytest.raises(ClientDisconnect): + asyncio.run(response({'type': 'http', 'asgi': {'spec_version': '2.4'}}, receive, send)) + assert closed == [True] + assert lock.acquire(False) + + +def test_xlsx_packaging_error_removes_xml_tempfile_without_masking_error(tmp_path, monkeypatch): + from pathlib import Path + from studio.imports import write_xlsx + from openpyxl import Workbook + paths = [] + def packaging_failure(book, filename): + sheet = book.worksheets[0] + sheet.close() + paths.append(Path(sheet._writer.out)) + raise OSError('archive write failed') + monkeypatch.setattr(Workbook, 'save', packaging_failure) + with pytest.raises(OSError, match='archive write failed'): + write_xlsx([record()], tmp_path / 'failed.xlsx') + assert paths and not paths[0].exists() + + +def test_failed_export_batch_does_not_finish_a_successful_truncated_download(tmp_path, monkeypatch): + app = create_app(tmp_path) + def broken_rows(*args, **kwargs): + yield record() + raise RuntimeError('database batch failed') + monkeypatch.setattr(app.state.db, 'iter_catalog', broken_rows) + sent = [] + async def send(message): + sent.append(message) + async def receive(): + await asyncio.Event().wait() + scope = {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'}, + 'http_version': '1.1', 'method': 'GET', 'scheme': 'http', + 'path': '/api/export', 'raw_path': b'/api/export', 'root_path': '', + 'query_string': b'format=csv', 'headers': [(b'host', b'127.0.0.1:8765')], + 'server': ('127.0.0.1', 8765), 'client': ('127.0.0.1', 12345)} + with pytest.raises(RuntimeError, match='database batch failed'): + asyncio.run(app(scope, receive, send)) + assert not any(message['type'] == 'http.response.body' and not message.get('more_body', False) for message in sent) diff --git a/tests/test_history_collection.py b/tests/test_history_collection.py new file mode 100644 index 0000000..4ec0850 --- /dev/null +++ b/tests/test_history_collection.py @@ -0,0 +1,147 @@ +import pytest + +from studio.connectors import ScoreClient +from studio.network import RemoteUnavailable +from tests.test_network import Response + + +class HistorySource: + def __init__(self, pages): + self.pages = pages + self.calls = [] + + def request(self, method, url, **kwargs): + payload = kwargs['data'] + self.calls.append((url.rsplit('/', 1)[-1], payload.copy())) + data = self.pages(payload.get('page', 1)) + return Response({'code': '0000', 'data': data}) + + +def test_full_score_array_over_page_limit_is_consumed_once(): + rows = [{'code': '081200', 'total': 310, 'depart_name': '信息学院'}] + rows += [{'code': '999999', 'total': 200} for _ in range(197)] + source = HistorySource(lambda page: rows) + lines = ScoreClient(source).score_lines({'school_code': '10001', 'school_name': '北京大学'}, '081200', 2025, 1) + assert len(lines) == 1 + assert lines[0]['total'] == 310 + assert [call[1]['page'] for call in source.calls] == [1] + + +def test_short_wrapped_pages_follow_explicit_total(): + source = HistorySource(lambda page: {'data': [{'value': page}], 'total': 2}) + assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}] + + +def test_short_wrapped_pages_follow_explicit_last_page(): + source = HistorySource(lambda page: {'list': [{'value': page}], 'last_page': 2}) + assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}] + + +@pytest.mark.parametrize('data', [None, {}, {'data': None}, {'data': [None]}, {'list': [42]}]) +def test_invalid_history_data_cannot_become_a_successful_empty_unit(data): + with pytest.raises(RemoteUnavailable): + list(ScoreClient(HistorySource(lambda page: data)).pages('/example', {})) + + +def test_empty_full_array_is_a_valid_empty_result(): + assert list(ScoreClient(HistorySource(lambda page: [])).pages('/example', {})) == [] + + +def test_repeated_real_page_still_raises_instead_of_returning_partial_data(): + source = HistorySource(lambda page: {'data': [{'value': 1}], 'total': 2}) + with pytest.raises(RemoteUnavailable): + list(ScoreClient(source).pages('/example', {})) + + +def test_empty_page_before_declared_total_is_an_error(): + source = HistorySource(lambda page: {'data': [{'value': 1}] if page == 1 else [], 'total': 2}) + with pytest.raises(RemoteUnavailable): + list(ScoreClient(source).pages('/example', {})) + + +def test_two_majors_reuse_one_complete_school_year_score_table(): + rows = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}] + source = HistorySource(lambda page: rows) + client = ScoreClient(source) + school = {'school_code': '10001', 'school_name': '北京大学'} + assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310 + assert client.score_lines(school, '081201', 2025, 1)[0]['total'] == 320 + assert len(source.calls) == 1 + + +def test_score_cache_separates_schools_and_years_and_has_a_bound(): + source = HistorySource(lambda page: [{'code': '081200', 'total': 310}]) + client = ScoreClient(source) + school = {'school_code': '10001', 'school_name': '北京大学'} + for sid in range(20): + client.score_lines(school, '081200', 2025, sid) + client.score_lines(school, '081200', 2026, 19) + before = len(source.calls) + client.score_lines(school, '081200', 2026, 19) + assert len(source.calls) == before + client.score_lines(school, '081200', 2025, 0) + assert len(source.calls) == before + 1 + + +def test_failed_lookup_is_cached_for_the_current_client_only(): + source = HistorySource(lambda page: {'data': [], 'total': 0}) + client = ScoreClient(source) + assert client.school_id('暂未收录大学') is None + assert client.school_id('暂未收录大学') is None + assert len(source.calls) == 1 + assert ScoreClient(source).school_id('暂未收录大学') is None + assert len(source.calls) == 2 + + +def test_invalid_score_response_is_not_cached(): + valid = False + source = HistorySource(lambda page: [{'code': '081200', 'total': 310}] if valid else {}) + client = ScoreClient(source) + school = {'school_code': '10001', 'school_name': '北京大学'} + with pytest.raises(RemoteUnavailable): + client.score_lines(school, '081200', 2025, 1) + valid = True + assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310 + + +def test_history_runner_uses_thin_targets_and_bulk_checkpoints(tmp_path, monkeypatch): + from studio.database import Database + from studio.jobs import JobManager + from studio.settings import SettingsStore + from tests.test_bounded_storage import record + + db = Database(tmp_path / 'data.db') + db.upsert_catalog([record(), record(major='081201'), record(direction='二')]) + params = {'year': 2027, 'provinces': ['11'], 'score_years': [2025, 2026], 'include_plans': True} + db.job_create('history', params) + + class Source: + def __init__(self): + self.calls = [] + def request(self, method, url, **kwargs): + payload = kwargs['data'] + self.calls.append(url.rsplit('/', 1)[-1]) + if url.endswith('/schoolList'): + data = {'data': [{'school_name': '甲大学', 'school_id': 1}], 'total': 1} + elif url.endswith('/schoolScore'): + data = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}] + else: + data = {'data': [{'special_code': payload['keyword'], 'special_name': payload['keyword'], + 'plan_id': str(payload['year']) + payload['keyword'], 'recruit_number': 1}], 'total': 1} + return Response({'code': '0000', 'data': data}) + + def forbidden(*args, **kwargs): + raise AssertionError('full catalog decoding or per-unit checkpoint query') + monkeypatch.setattr(db, 'all_catalog', forbidden) + monkeypatch.setattr(db, 'unit_done', forbidden) + source = Source() + manager = JobManager(db, SettingsStore(tmp_path), transport=source) + manager._scores('history', params) + assert db.job_get('history')['completed_units'] == 4 + assert db.overview()['scores'] == 4 + assert db.job_get('history')['records'] == 4 + assert source.calls.count('schoolScore') == 2 + assert source.calls.count('planListV2') == 4 + before = len(source.calls) + manager._scores('history', params) + assert len(source.calls) == before diff --git a/tests/test_mysql.py b/tests/test_mysql.py index 18296d2..4e5ff33 100644 --- a/tests/test_mysql.py +++ b/tests/test_mysql.py @@ -98,6 +98,47 @@ def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypa assert creations == [] +def test_mysql_bounded_catalog_and_history_targets_preserve_filters(mysql): + mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027)) + mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026)) + scores = select_score_lines([{'code': '081200', 'total': 310, 'depart_name': '计算机学院'}], '081200', '10001', '北京大学', 2026) + mysql.upsert_scores(scores) + targets = list(mysql.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1)) + assert len(targets) == 1 + assert targets[0]['major_code'] == '081200' + assert 'data' not in targets[0] + rows = list(mysql.iter_catalog({'year': 2027}, batch_size=1)) + assert len(rows) == 2 + assert len(list(mysql.iter_catalog({'year': 2026, 'max_score': 310}, batch_size=1))) == 2 + assert len(list(mysql.iter_scores({('10001', '081200')}, batch_size=1))) == 1 + + +def test_mysql_unit_commit_is_atomic_and_idempotent(mysql): + mysql.job_create('atomic-job', {}) + rows = normalize_catalog(raw_catalog(), 2027) + scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026) + assert mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores) + assert not mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores) + assert mysql.job_get('atomic-job')['records'] == 2 + assert mysql.job_get('atomic-job')['completed_units'] == 1 + assert mysql.overview()['records'] == 2 + assert mysql.overview()['scores'] == 1 + + +def test_mysql_failed_unit_rolls_back_every_write(mysql, monkeypatch): + mysql.job_create('atomic-job', {}) + def fail(*args): + raise RuntimeError('interrupted unit') + monkeypatch.setattr(mysql, '_school', fail) + rows = normalize_catalog(raw_catalog(), 2027) + scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026) + with pytest.raises(RuntimeError): + mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores) + assert mysql.job_get('atomic-job')['records'] == 0 + assert mysql.job_get('atomic-job')['completed_units'] == 0 + assert mysql.overview()['records'] == mysql.overview()['scores'] == 0 + + def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path): from studio.migrate import migrate_sqlite source = Database(tmp_path / 'snapshot.db') diff --git a/tests/test_proxy_scheduler_bounds.py b/tests/test_proxy_scheduler_bounds.py new file mode 100644 index 0000000..71d0645 --- /dev/null +++ b/tests/test_proxy_scheduler_bounds.py @@ -0,0 +1,98 @@ +import importlib.util +import queue +import threading +import time +from pathlib import Path + + +def maintainer_class(): + path = Path(__file__).resolve().parents[1] / 'deploy/proxy_scheduler.py' + spec = importlib.util.spec_from_file_location('bounded_proxy_scheduler', path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.ProxyMaintainer + + +def test_detection_workers_are_bounded_reused_and_rounds_do_not_overlap(): + state = {'active': 0, 'peak': 0, 'types': set(), 'mixed': False, 'done': [], 'thread_ids': set()} + lock = threading.Lock() + class Checker: + def __init__(self, work_type, target_queue, name): + self.work_type, self.queue = work_type, target_queue + def run(self): + while True: + try: + item = self.queue.get(block=False) + except queue.Empty: + return + with lock: + state['active'] += 1 + state['peak'] = max(state['peak'], state['active']) + state['types'].add(self.work_type) + state['mixed'] |= len(state['types']) > 1 + state['thread_ids'].add(threading.get_ident()) + time.sleep(.01) + with lock: + state['done'].append(item) + state['active'] -= 1 + if not state['active']: + state['types'].clear() + self.queue.task_done() + maintainer = maintainer_class()(workers=2, checker_class=Checker) + first = threading.Thread(target=maintainer.validate, args=('raw', range(10))) + second = threading.Thread(target=maintainer.validate, args=('use', range(10, 20))) + first.start(); second.start() + first.join(4); second.join(4) + assert not first.is_alive() and not second.is_alive() + before = set(state['thread_ids']) + maintainer.validate('raw', range(20, 25)) + maintainer.close() + assert sorted(state['done']) == list(range(25)) + assert state['peak'] <= 2 + assert state['mixed'] is False + assert state['thread_ids'] == before + + +def test_stopped_detector_does_not_begin_new_rounds(): + class Forbidden: + def __init__(self, *args): + raise AssertionError('detection began after shutdown') + maintainer = maintainer_class()(workers=2, checker_class=Forbidden) + maintainer.close() + maintainer.validate('raw', [1, 2, 3]) + + +def test_failed_worker_waits_for_other_workers_before_next_round(): + slow_started, release, next_round = threading.Event(), threading.Event(), threading.Event() + class Checker: + def __init__(self, work_type, target_queue, name): + self.kind, self.name = work_type, name + def run(self): + if self.kind == 'raw' and self.name == 'worker_00': + assert slow_started.wait(2) + raise RuntimeError('validator failed') + if self.kind == 'raw': + slow_started.set() + assert release.wait(3) + maintainer = maintainer_class()(workers=2, checker_class=Checker) + failures = [] + def first_round(): + try: + maintainer.validate('raw', [1, 2]) + except RuntimeError: + failures.append(True) + first = threading.Thread(target=first_round) + first.start() + assert slow_started.wait(2) + def second_proxies(): + next_round.set() + yield 3 + second = threading.Thread(target=maintainer.validate, args=('use', second_proxies())) + second.start() + began_early = next_round.wait(.15) + release.set() + first.join(3); second.join(3) + maintainer.close() + assert began_early is False + assert next_round.is_set() + assert failures == [True]