Fix historical pagination and bound export and proxy memory
This commit is contained in:
@@ -62,6 +62,10 @@ Debian 12 / MariaDB 服务器模式、HTTPS 访问与代理池后台运行见 [
|
||||
|
||||
可将当前筛选结果导出 CSV、Excel 或完整 JSON。表格导出保留院系/方向代码和字段口径,回导不会凭空增加同一记录。JSON 同时保留相关学校专业的历史分数线;凭据不进入导出。CSV/Excel 对可能被表格软件当作公式的文本加保护前缀。
|
||||
|
||||
全量导出按 200 条分批读取,CSV/JSON 流式下载,Excel 使用写入模式和临时文件;同一实例一次只执行一个导出,重复请求返回 429。下载结束或中断后清理临时资源。导出期间新增的记录留待下次导出;导出不是事务一致性备份。大于 100 MB 的文件仍受导入上限限制,可按地区等条件分别导出;完整数据库备份按服务器部署说明操作。
|
||||
|
||||
历史采集在招生目录阶段完成后开始,按院校和年份复用全校分数线,每次最多缓存 8 份,历史计划继续逐专业获取。接口直接返回的数组按完整结果处理,带分页信息的结果按实际数量继续;不完整的响应不会被写成成功断点。数据、保存计数和完成断点在同一数据库事务中提交。
|
||||
|
||||
## 本地数据与开发
|
||||
|
||||
所有运行数据位于项目 `.runtime`:`studio.db` 保存目录、分数、收藏、任务和断点;`settings.json` 保存非机密配置;`chsi-cookie.enc` 由当前 Windows 用户的 DPAPI 加密。密码不持久化,接口不返回 Cookie。备份数据库时先停止工作台,再复制 `.runtime`;恢复加密 Cookie 需要原 Windows 用户环境,否则重新登录即可。
|
||||
|
||||
@@ -8,9 +8,12 @@ Requires=docker.service
|
||||
User=kaoyan
|
||||
Group=kaoyan
|
||||
WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
|
||||
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py schedule
|
||||
Environment=MALLOC_ARENA_MAX=2
|
||||
Environment=PROXY_CHECK_WORKERS=8
|
||||
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python /opt/kaoyan-studio-deploy/app/deploy/proxy_scheduler.py
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
TimeoutStopSec=120
|
||||
UMask=0077
|
||||
PrivateTmp=true
|
||||
ProtectSystem=full
|
||||
|
||||
@@ -9,6 +9,7 @@ User=kaoyan
|
||||
Group=kaoyan
|
||||
WorkingDirectory=/opt/kaoyan-studio-deploy/app
|
||||
EnvironmentFile=/etc/kaoyan-studio.env
|
||||
Environment=MALLOC_ARENA_MAX=2
|
||||
ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Run upstream proxy_pool checks with reusable workers and serialized jobs."""
|
||||
import os
|
||||
import queue
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
APP_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(APP_ROOT))
|
||||
from studio.memory import release_idle_memory
|
||||
|
||||
|
||||
class StopQueue(queue.Queue):
|
||||
def __init__(self, stopping):
|
||||
super().__init__()
|
||||
self.stopping = stopping
|
||||
|
||||
def get(self, *args, **kwargs):
|
||||
if self.stopping.is_set():
|
||||
raise queue.Empty
|
||||
return super().get(*args, **kwargs)
|
||||
|
||||
|
||||
class ProxyMaintainer:
|
||||
def __init__(self, workers=8, checker_class=None, fetcher_factory=None, handler_factory=None, pool_min=20):
|
||||
self.worker_count = max(1, min(20, int(workers)))
|
||||
self.checker_class = checker_class
|
||||
self.fetcher_factory, self.handler_factory = fetcher_factory, handler_factory
|
||||
self.pool_min = pool_min
|
||||
self.stopping = threading.Event()
|
||||
self.pipeline = threading.RLock()
|
||||
self.workers = ThreadPoolExecutor(max_workers=self.worker_count, thread_name_prefix='proxy-check')
|
||||
|
||||
def validate(self, work_type, proxies):
|
||||
with self.pipeline:
|
||||
if self.stopping.is_set():
|
||||
return
|
||||
target = StopQueue(self.stopping)
|
||||
for proxy in proxies:
|
||||
if self.stopping.is_set():
|
||||
return
|
||||
target.put(proxy)
|
||||
if self.checker_class is None:
|
||||
from helper.check import _ThreadChecker
|
||||
self.checker_class = _ThreadChecker
|
||||
count = min(self.worker_count, target.qsize())
|
||||
futures = [self.workers.submit(self.checker_class(work_type, target, f'worker_{i:02}').run)
|
||||
for i in range(count)]
|
||||
try:
|
||||
failure = None
|
||||
for future in futures:
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc:
|
||||
if failure is None:
|
||||
failure = exc
|
||||
if failure is not None:
|
||||
raise failure
|
||||
finally:
|
||||
release_idle_memory()
|
||||
|
||||
def fetch(self):
|
||||
with self.pipeline:
|
||||
if not self.stopping.is_set():
|
||||
self.validate('raw', self.fetcher_factory().run())
|
||||
|
||||
def check(self):
|
||||
with self.pipeline:
|
||||
if self.stopping.is_set():
|
||||
return
|
||||
handler = self.handler_factory()
|
||||
if handler.db.getCount().get('total', 0) < self.pool_min:
|
||||
self.fetch()
|
||||
self.validate('use', handler.getAll())
|
||||
|
||||
def stop(self):
|
||||
self.stopping.set()
|
||||
|
||||
def close(self):
|
||||
self.stop()
|
||||
with self.pipeline:
|
||||
self.workers.shutdown(wait=True, cancel_futures=True)
|
||||
|
||||
|
||||
def main():
|
||||
proxy_root = Path(os.environ.get('STUDIO_PROXY_POOL_ROOT', APP_ROOT.parent / 'proxy_pool'))
|
||||
sys.path.insert(0, str(proxy_root))
|
||||
from apscheduler.schedulers.blocking import BlockingScheduler
|
||||
from helper.check import _ThreadChecker
|
||||
from helper.fetch import Fetcher
|
||||
from handler.configHandler import ConfigHandler
|
||||
from handler.logHandler import LogHandler
|
||||
from handler.proxyHandler import ProxyHandler
|
||||
|
||||
config = ConfigHandler()
|
||||
maintainer = ProxyMaintainer(workers=int(os.environ.get('PROXY_CHECK_WORKERS', '8')),
|
||||
checker_class=_ThreadChecker, fetcher_factory=Fetcher,
|
||||
handler_factory=ProxyHandler, pool_min=config.poolSizeMin)
|
||||
scheduler = BlockingScheduler(timezone=config.timezone, logger=LogHandler('scheduler'),
|
||||
executors={'default': {'type': 'threadpool', 'max_workers': 1}},
|
||||
job_defaults={'coalesce': True, 'max_instances': 1, 'misfire_grace_time': 120})
|
||||
scheduler.add_job(maintainer.fetch, 'interval', minutes=5, id='proxy_fetch')
|
||||
scheduler.add_job(maintainer.check, 'interval', minutes=2, id='proxy_check',
|
||||
next_run_time=datetime.now(timezone.utc))
|
||||
def stop(signum, frame):
|
||||
maintainer.stop()
|
||||
if scheduler.running:
|
||||
scheduler.shutdown(wait=False)
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
if not maintainer.stopping.is_set():
|
||||
scheduler.start()
|
||||
finally:
|
||||
maintainer.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -16,6 +16,8 @@
|
||||
|
||||
安装模板在 `deploy/`。Python 依赖按 `requirements.lock` 安装;前端 `npm ci && npm run build`。不要启动多个工作台进程连接同一数据库,否则任务恢复状态会互相影响。原有站点、数据库和 fail2ban 配置不作修改。
|
||||
|
||||
代理调度器运行本项目 `deploy/proxy_scheduler.py` 包装器,继续使用原版代理池的获取源、检测逻辑和 Redis 数据,不修改上游源码。默认复用 8 个检测线程,抓取与检查轮次串行运行,避免定时任务重叠。工作台和调度器设定 `MALLOC_ARENA_MAX=2`;每次导出/检查结束尝试释放分配器空闲内存。该操作不释放仍在使用的数据,实际内存以运行监测为准。
|
||||
|
||||
## 操作
|
||||
|
||||
```bash
|
||||
@@ -28,6 +30,8 @@ systemctl restart kaoyan-studio
|
||||
|
||||
默认每次请求随机等待 1.2–2.4 秒,每 100 次成功请求轮换代理。设置页可修改上下限和轮换次数;0 表示关闭自动轮换。代理页显示实际成功请求路由、计数和切换次数。登录表单、会话同步、权限验证及紧随其后的失败页重试保持同一路由。免费代理池初次启动可能没有 HTTPS 候选,部署时保留了原电脑代理池的候选作为备用;这些候选仍会经过实际请求与失败隔离。
|
||||
|
||||
当前全国任务采用 0 次自动轮换,维持通过登录验证的出口;连接失败仍会切换。Cookie 验证状态与出口关联,出口变化后需要重新验证,不宜为了让 IP 显示变化而频繁轮换。HTTP 会话最多保留 4 个代理连接池,淘汰旧池时主动关闭连接。
|
||||
|
||||
Linux 无桌面环境,界面提供研招网登录入口与粘贴 Cookie。请在自己的电脑浏览器打开研招目录并登录,从开发者工具中复制目录接口请求的单行 Cookie,粘贴到工作台后点验证。服务器重新同步会话并检查受保护的第二页。迁移 Windows Cookie 时只经 SSH 传输加密封装,在服务器重新加密;不继承原验证标记。
|
||||
|
||||
## 迁移与验证
|
||||
|
||||
+64
-24
@@ -2,6 +2,8 @@
|
||||
import json
|
||||
import uuid
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
@@ -11,7 +13,9 @@ from fastapi.staticfiles import StaticFiles
|
||||
from studio.auth import AuthService, CookieVault
|
||||
from studio.database import Database
|
||||
from studio.domain import PROVINCES
|
||||
from studio.imports import import_file, export_sheet
|
||||
from studio.imports import import_file, write_xlsx
|
||||
from studio.exporting import ExportStream, csv_chunks, json_chunks, file_chunks
|
||||
from studio.memory import release_idle_memory
|
||||
from studio.jobs import JobManager
|
||||
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||
from studio.settings import SettingsStore
|
||||
@@ -20,6 +24,31 @@ from studio.pool_service import PoolService
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
class LocalAccessMiddleware:
|
||||
"""Check hosts and origins without buffering or wrapping response streams."""
|
||||
|
||||
def __init__(self, app, allowed_hosts):
|
||||
self.app = app
|
||||
self.allowed_hosts = allowed_hosts
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope['type'] != 'http':
|
||||
return await self.app(scope, receive, send)
|
||||
request = Request(scope)
|
||||
if request.url.hostname not in self.allowed_hosts:
|
||||
response = JSONResponse({'detail': '访问主机不在允许列表中'}, status_code=403)
|
||||
return await response(scope, receive, send)
|
||||
if request.method not in ('GET', 'HEAD', 'OPTIONS'):
|
||||
origin = request.headers.get('origin')
|
||||
if origin:
|
||||
parsed = urlparse(origin)
|
||||
allowed = {str(request.base_url).rstrip('/'), 'http://127.0.0.1:5173', 'http://localhost:5173'}
|
||||
if parsed.scheme not in ('http', 'https') or origin.rstrip('/') not in allowed:
|
||||
response = JSONResponse({'detail': '已拒绝跨来源修改请求'}, status_code=403)
|
||||
return await response(scope, receive, send)
|
||||
return await self.app(scope, receive, send)
|
||||
|
||||
|
||||
def create_app(data_dir=None):
|
||||
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
@@ -36,6 +65,9 @@ def create_app(data_dir=None):
|
||||
transport = Transport(settings, router)
|
||||
auth = AuthService(transport, CookieVault(directory))
|
||||
jobs = JobManager(db, settings, transport, auth=auth)
|
||||
export_lock = threading.BoundedSemaphore(1)
|
||||
export_directory = directory / 'exports'
|
||||
export_directory.mkdir(exist_ok=True)
|
||||
if (ROOT / "data/schools.json").exists():
|
||||
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
|
||||
|
||||
@@ -46,6 +78,8 @@ def create_app(data_dir=None):
|
||||
jobs.resume(jid)
|
||||
yield
|
||||
jobs.close()
|
||||
transport.close()
|
||||
router.session.close()
|
||||
|
||||
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
|
||||
app.state.db, app.state.jobs = db, jobs
|
||||
@@ -53,19 +87,7 @@ def create_app(data_dir=None):
|
||||
app.state.pool = pool
|
||||
app.state.auth = auth
|
||||
|
||||
@app.middleware("http")
|
||||
async def local_access(request: Request, call_next):
|
||||
host = request.url.hostname
|
||||
if host not in allowed_hosts:
|
||||
return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
|
||||
if request.method not in ("GET", "HEAD", "OPTIONS"):
|
||||
origin = request.headers.get("origin")
|
||||
if origin:
|
||||
parsed = urlparse(origin)
|
||||
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
|
||||
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
|
||||
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
|
||||
return await call_next(request)
|
||||
app.add_middleware(LocalAccessMiddleware, allowed_hosts=allowed_hosts)
|
||||
|
||||
@app.exception_handler(ValueError)
|
||||
async def invalid(request, exc):
|
||||
@@ -216,17 +238,35 @@ def create_app(data_dir=None):
|
||||
def export(request: Request, format: str = "csv"):
|
||||
if format not in ("csv", "xlsx", "json"):
|
||||
raise ValueError("导出格式无效")
|
||||
rows = db.all_catalog(filters(request))
|
||||
if format == "json":
|
||||
pairs = {(r["school_code"], r["major_code"]) for r in rows}
|
||||
with db.connection() as conn:
|
||||
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs]
|
||||
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode()
|
||||
mime = "application/json"
|
||||
selected = filters(request)
|
||||
if not export_lock.acquire(False):
|
||||
raise HTTPException(429, '已有导出正在进行,请等待完成后重试')
|
||||
temporary = None
|
||||
def cleanup():
|
||||
try:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
finally:
|
||||
export_lock.release()
|
||||
try:
|
||||
if format == 'csv':
|
||||
source, mime = csv_chunks(db.iter_catalog(selected)), 'text/csv; charset=utf-8'
|
||||
elif format == 'json':
|
||||
source, mime = json_chunks(db, selected), 'application/json'
|
||||
else:
|
||||
payload = export_sheet(rows, format)
|
||||
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'})
|
||||
descriptor, path = tempfile.mkstemp(prefix='kaoyan-', suffix='.xlsx', dir=export_directory)
|
||||
os.close(descriptor)
|
||||
temporary = Path(path)
|
||||
write_xlsx(db.iter_catalog(selected), temporary)
|
||||
source = file_chunks(temporary)
|
||||
mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
||||
return ExportStream(source, cleanup, media_type=mime,
|
||||
headers={'Content-Disposition': f'attachment; filename="kaoyan-data.{format}"',
|
||||
'X-Accel-Buffering': 'no'})
|
||||
except BaseException:
|
||||
cleanup()
|
||||
release_idle_memory()
|
||||
raise
|
||||
|
||||
dist = ROOT / "frontend/dist"
|
||||
if (dist / "assets").exists():
|
||||
|
||||
+42
-7
@@ -2,6 +2,7 @@
|
||||
import json
|
||||
import re
|
||||
import hashlib
|
||||
from collections import OrderedDict
|
||||
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
||||
from studio.network import RemoteUnavailable, RateLimited
|
||||
|
||||
@@ -153,6 +154,7 @@ class ScoreClient:
|
||||
def __init__(self, transport):
|
||||
self.transport = transport
|
||||
self.cache = {}
|
||||
self.score_cache = OrderedDict()
|
||||
|
||||
def post(self, endpoint, payload):
|
||||
for attempt in range(3):
|
||||
@@ -161,8 +163,13 @@ class ScoreClient:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
|
||||
if not isinstance(body, dict):
|
||||
raise RemoteUnavailable("掌上考研响应结构已变化")
|
||||
if str(body.get("code")) == "0000":
|
||||
return body.get("data") or {}
|
||||
data = body.get("data")
|
||||
if not isinstance(data, (dict, list)):
|
||||
raise RemoteUnavailable("掌上考研缺少有效数据,未写入不完整结果")
|
||||
return data
|
||||
message = str(body.get('message', body.get('msg', '')))
|
||||
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
|
||||
if attempt == 2:
|
||||
@@ -172,20 +179,40 @@ class ScoreClient:
|
||||
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
|
||||
|
||||
def pages(self, endpoint, payload, limit=100):
|
||||
seen = set()
|
||||
seen, consumed = set(), 0
|
||||
for page in range(1, 10001):
|
||||
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
|
||||
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data
|
||||
if not isinstance(rows, list):
|
||||
full_list = isinstance(data, list)
|
||||
rows = data.get("data", data.get("list")) if isinstance(data, dict) else data
|
||||
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
|
||||
raise RemoteUnavailable("掌上考研响应结构已变化")
|
||||
total = integer(data.get("total")) if isinstance(data, dict) else None
|
||||
last = integer(data.get("last_page")) if isinstance(data, dict) else None
|
||||
if isinstance(data, dict):
|
||||
for field, value in (('total', total), ('last_page', last)):
|
||||
if data.get(field) is not None and (value is None or value < 0 or isinstance(data[field], bool)):
|
||||
raise RemoteUnavailable("掌上考研分页信息无效,未写入不完整结果")
|
||||
if not rows and (total is not None and consumed < total or last is not None and page < last):
|
||||
raise RemoteUnavailable("掌上考研提前返回空页,未写入不完整结果")
|
||||
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
||||
if rows and signature in seen:
|
||||
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
|
||||
seen.add(signature)
|
||||
consumed += len(rows)
|
||||
yield from rows
|
||||
total = integer(data.get("total")) if isinstance(data, dict) else None
|
||||
last = integer(data.get("last_page")) if isinstance(data, dict) else None
|
||||
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total:
|
||||
# Bare arrays are complete responses; wrapped pages use their actual
|
||||
# metadata even when the server ignores the requested page size.
|
||||
terminal = full_list or not rows
|
||||
if not terminal:
|
||||
if last is not None:
|
||||
terminal = page >= last
|
||||
if terminal and total is not None and consumed < total:
|
||||
raise RemoteUnavailable("掌上考研分页未返回全部记录")
|
||||
elif total is not None:
|
||||
terminal = consumed >= total
|
||||
else:
|
||||
terminal = len(rows) < limit
|
||||
if terminal:
|
||||
return
|
||||
raise RemoteUnavailable("历史数据分页超过安全上限")
|
||||
|
||||
@@ -198,10 +225,18 @@ class ScoreClient:
|
||||
if len(matches) == 1 and matches[0].get("school_id"):
|
||||
self.cache[key] = matches[0]["school_id"]
|
||||
return self.cache[key]
|
||||
self.cache[key] = None
|
||||
return None
|
||||
|
||||
def score_lines(self, school, major_code, year, school_id):
|
||||
key = (str(school_id), int(year))
|
||||
if key not in self.score_cache:
|
||||
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
|
||||
self.score_cache[key] = rows
|
||||
if len(self.score_cache) > 8:
|
||||
self.score_cache.popitem(last=False)
|
||||
self.score_cache.move_to_end(key)
|
||||
rows = self.score_cache[key]
|
||||
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
|
||||
|
||||
def plans(self, school, major_code, year, school_id):
|
||||
|
||||
+85
-2
@@ -19,6 +19,8 @@ CREATE TABLE IF NOT EXISTS catalog(
|
||||
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
|
||||
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
|
||||
CREATE INDEX IF NOT EXISTS catalog_export ON catalog(year DESC,school_name,id);
|
||||
CREATE INDEX IF NOT EXISTS catalog_targets ON catalog(year,school_code,major_code);
|
||||
""" + SCORE_SCHEMA + """
|
||||
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
|
||||
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
||||
@@ -69,6 +71,10 @@ class Database:
|
||||
conn.close()
|
||||
|
||||
def upsert_catalog(self, records: list[dict]):
|
||||
with self.connection() as conn:
|
||||
self._upsert_catalog(conn, records)
|
||||
|
||||
def _upsert_catalog(self, conn, records):
|
||||
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
|
||||
values = []
|
||||
for record in records:
|
||||
@@ -77,7 +83,6 @@ class Database:
|
||||
if not values:
|
||||
return
|
||||
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
@@ -93,6 +98,9 @@ class Database:
|
||||
|
||||
def upsert_scores(self, records):
|
||||
with self.connection() as conn:
|
||||
self._upsert_scores(conn, records)
|
||||
|
||||
def _upsert_scores(self, conn, records):
|
||||
for row in records:
|
||||
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
|
||||
|
||||
@@ -141,9 +149,73 @@ class Database:
|
||||
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
|
||||
|
||||
def all_catalog(self, filters):
|
||||
return list(self.iter_catalog(filters))
|
||||
|
||||
def iter_catalog(self, filters, batch_size=200):
|
||||
select, params = self._query(filters)
|
||||
with self.connection() as conn:
|
||||
return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)]
|
||||
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM catalog').fetchone()[0]
|
||||
after = None
|
||||
while True:
|
||||
query = 'SELECT * FROM (' + select + ') export_rows WHERE id<=?'
|
||||
values = [*params, cutoff]
|
||||
if after:
|
||||
year, school, rid = after
|
||||
if school is None:
|
||||
query += ' AND (year<? OR (year=? AND (school_name IS NOT NULL OR (school_name IS NULL AND id>?))))'
|
||||
values.extend([year, year, rid])
|
||||
else:
|
||||
query += ' AND (year<? OR (year=? AND (school_name>? OR (school_name=? AND id>?))))'
|
||||
values.extend([year, year, school, school, rid])
|
||||
query += ' ORDER BY year DESC,school_name,id LIMIT ?'
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(query, [*values, batch_size]).fetchall()
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
yield self._catalog_row(row)
|
||||
last = rows[-1]
|
||||
after = (last['year'], last['school_name'], last['id'])
|
||||
|
||||
def iter_scores(self, pairs, batch_size=200):
|
||||
after = 0
|
||||
with self.connection() as conn:
|
||||
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM scores').fetchone()[0]
|
||||
while True:
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute('SELECT id,school_code,major_code,data FROM scores WHERE id>? AND id<=? ORDER BY id LIMIT ?', (after, cutoff, batch_size)).fetchall()
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
if (row['school_code'], row['major_code']) in pairs:
|
||||
yield json.loads(row['data'])
|
||||
after = rows[-1]['id']
|
||||
|
||||
def score_targets(self, filters, batch_size=200):
|
||||
conditions, params = ['year=?'], [filters['year']]
|
||||
if filters.get('major_code'):
|
||||
conditions.append('major_code=?')
|
||||
params.append(filters['major_code'])
|
||||
if filters.get('school_keyword'):
|
||||
conditions.append('school_name LIKE ?')
|
||||
params.append('%' + filters['school_keyword'] + '%')
|
||||
if filters.get('provinces'):
|
||||
conditions.append("(province='' OR province IN (" + ','.join('?' for _ in filters['provinces']) + '))')
|
||||
params.extend(filters['provinces'])
|
||||
after = None
|
||||
while True:
|
||||
where, values = ' AND '.join(conditions), list(params)
|
||||
if after:
|
||||
where += ' AND (school_code>? OR (school_code=? AND major_code>?))'
|
||||
values.extend([after[0], after[0], after[1]])
|
||||
query = 'SELECT school_code,major_code,MAX(school_name) AS school_name,MAX(province) AS province,MAX(level) AS level FROM catalog WHERE ' + where + ' GROUP BY school_code,major_code ORDER BY school_code,major_code LIMIT ?'
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(query, [*values, batch_size]).fetchall()
|
||||
if not rows:
|
||||
return
|
||||
yield from (dict(row) for row in rows)
|
||||
last = rows[-1]
|
||||
after = (last['school_code'], last['major_code'])
|
||||
|
||||
def _catalog_row(self, row):
|
||||
data = json.loads(row["data"])
|
||||
@@ -219,6 +291,17 @@ class Database:
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
|
||||
|
||||
def save_unit(self, jid, unit, records, scores=()):
|
||||
with self.connection() as conn:
|
||||
if not conn.execute('SELECT 1 FROM jobs WHERE id=?', (jid,)).fetchone():
|
||||
raise ValueError('任务不存在')
|
||||
if not conn.execute('INSERT OR IGNORE INTO job_units VALUES(?,?)', (jid, unit)).rowcount:
|
||||
return False
|
||||
self._upsert_scores(conn, scores)
|
||||
self._upsert_catalog(conn, records)
|
||||
conn.execute('UPDATE jobs SET records=records+?,updated_at=? WHERE id=?', (len(records), now(), jid))
|
||||
return True
|
||||
|
||||
def log(self, jid, level, message):
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Bounded exports, one download at a time, with disconnect cleanup."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
|
||||
from fastapi.responses import StreamingResponse
|
||||
from studio.imports import LABELS, spreadsheet_value
|
||||
from studio.memory import release_idle_memory
|
||||
|
||||
|
||||
class ExportStream(StreamingResponse):
|
||||
def __init__(self, source, cleanup, **kwargs):
|
||||
super().__init__(source, **kwargs)
|
||||
self.source, self.cleanup = source, cleanup
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
try:
|
||||
await super().__call__(scope, receive, send)
|
||||
finally:
|
||||
try:
|
||||
self.source.close()
|
||||
finally:
|
||||
try:
|
||||
self.cleanup()
|
||||
finally:
|
||||
release_idle_memory()
|
||||
|
||||
|
||||
def csv_chunks(records):
|
||||
yield b'\xef\xbb\xbf'
|
||||
stream = io.StringIO(newline='')
|
||||
writer = csv.writer(stream)
|
||||
writer.writerow(list(LABELS.values()))
|
||||
yield stream.getvalue().encode('utf-8')
|
||||
stream.seek(0)
|
||||
stream.truncate(0)
|
||||
for record in records:
|
||||
writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS])
|
||||
if stream.tell() >= 65536:
|
||||
yield stream.getvalue().encode('utf-8')
|
||||
stream.seek(0)
|
||||
stream.truncate(0)
|
||||
if stream.tell():
|
||||
yield stream.getvalue().encode('utf-8')
|
||||
|
||||
|
||||
def json_chunks(database, filters):
|
||||
pairs = set()
|
||||
yield b'{"version":1,"catalog":['
|
||||
first = True
|
||||
for row in database.iter_catalog(filters):
|
||||
pairs.add((row['school_code'], row['major_code']))
|
||||
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
|
||||
first = False
|
||||
yield b'],"scores":['
|
||||
first = True
|
||||
for row in database.iter_scores(pairs):
|
||||
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
|
||||
first = False
|
||||
yield b']}'
|
||||
|
||||
|
||||
def file_chunks(path):
|
||||
with path.open('rb') as stream:
|
||||
while chunk := stream.read(65536):
|
||||
yield chunk
|
||||
+44
-17
@@ -2,6 +2,7 @@
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
from contextlib import suppress
|
||||
import sqlite3
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
@@ -94,11 +95,18 @@ def import_file(path, year=None):
|
||||
book = load_workbook(path, read_only=True, data_only=True)
|
||||
try:
|
||||
sheet = book.active
|
||||
if sheet.max_row > 300001 or sheet.max_column > 100:
|
||||
if (sheet.max_row or 0) > 300001 or (sheet.max_column or 0) > 100:
|
||||
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
|
||||
values = sheet.iter_rows(values_only=True)
|
||||
headers = next(values, [])
|
||||
rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)]
|
||||
if len(headers) > 100:
|
||||
raise ValueError("Excel 超过 100 列,请拆分后导入")
|
||||
rows = []
|
||||
for index, row in enumerate(values, 2):
|
||||
if index > 300001 or len(row) > 100:
|
||||
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
|
||||
if any(v is not None for v in row):
|
||||
rows.append(dict(zip(headers, row)))
|
||||
finally:
|
||||
book.close()
|
||||
else:
|
||||
@@ -179,28 +187,47 @@ def spreadsheet_value(value):
|
||||
|
||||
|
||||
def export_sheet(records, kind):
|
||||
rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records]
|
||||
if kind == "csv":
|
||||
stream = io.StringIO(newline="")
|
||||
writer = csv.writer(stream)
|
||||
writer.writerow(list(LABELS.values()))
|
||||
writer.writerows(rows)
|
||||
writer.writerows([spreadsheet_value(record.get(field)) for field in LABELS] for record in records)
|
||||
return stream.getvalue().encode("utf-8-sig")
|
||||
book = Workbook()
|
||||
sheet = book.active
|
||||
sheet.title = "院校专业"
|
||||
sheet.append(list(LABELS.values()))
|
||||
for row in rows:
|
||||
sheet.append(row)
|
||||
sheet.freeze_panes = "A2"
|
||||
sheet.auto_filter.ref = sheet.dimensions
|
||||
output = io.BytesIO()
|
||||
write_xlsx(records, output)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def write_xlsx(records, output):
|
||||
from openpyxl.cell import WriteOnlyCell
|
||||
from openpyxl.utils import get_column_letter
|
||||
from openpyxl.styles import Font, PatternFill, Alignment
|
||||
for cell in sheet[1]:
|
||||
book = Workbook(write_only=True)
|
||||
sheet = book.create_sheet("院校专业")
|
||||
sheet.freeze_panes = "A2"
|
||||
header = []
|
||||
for index, label in enumerate(LABELS.values(), 1):
|
||||
cell = WriteOnlyCell(sheet, value=label)
|
||||
cell.font = Font(color="FFFFFF", bold=True)
|
||||
cell.fill = PatternFill("solid", fgColor="183447")
|
||||
cell.alignment = Alignment(vertical="center")
|
||||
for col in sheet.columns:
|
||||
sheet.column_dimensions[col[0].column_letter].width = 24
|
||||
output = io.BytesIO()
|
||||
header.append(cell)
|
||||
sheet.column_dimensions[get_column_letter(index)].width = 24
|
||||
count = 1
|
||||
try:
|
||||
sheet.append(header)
|
||||
for record in records:
|
||||
sheet.append([spreadsheet_value(record.get(field)) for field in LABELS])
|
||||
count += 1
|
||||
sheet.auto_filter.ref = f'A1:{get_column_letter(len(LABELS))}{count}'
|
||||
book.save(output)
|
||||
return output.getvalue()
|
||||
except BaseException:
|
||||
with suppress(Exception):
|
||||
if not sheet.closed:
|
||||
sheet.close()
|
||||
with suppress(Exception):
|
||||
if sheet._writer:
|
||||
sheet._writer.cleanup()
|
||||
raise
|
||||
finally:
|
||||
book.close()
|
||||
|
||||
+20
-19
@@ -1,6 +1,7 @@
|
||||
"""Persistent units and cooperative pause/cancel for one local worker."""
|
||||
import threading
|
||||
import uuid
|
||||
from itertools import chain
|
||||
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
|
||||
from studio.domain import PROVINCES, normalize_catalog
|
||||
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||
@@ -230,8 +231,7 @@ class JobManager:
|
||||
records = []
|
||||
for row in client.directions(school, major, params.get("study_mode", "")):
|
||||
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
|
||||
self._save(jid, records)
|
||||
self.complete_unit(jid, unit)
|
||||
self.db.save_unit(jid, unit, records)
|
||||
completed.add(unit)
|
||||
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
|
||||
except RemoteUnavailable as exc:
|
||||
@@ -240,24 +240,25 @@ class JobManager:
|
||||
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
|
||||
|
||||
def _scores(self, jid, params):
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
|
||||
records = self.db.all_catalog(filters)
|
||||
targets = {}
|
||||
for row in records:
|
||||
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]:
|
||||
continue
|
||||
targets[(row["school_code"], row["major_code"])] = row
|
||||
if not targets and params.get("major_code"):
|
||||
for row in self.db.schools(params.get("school_keyword", "")):
|
||||
if row["province"] in [PROVINCES[p] for p in params["provinces"]]:
|
||||
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
|
||||
if not targets:
|
||||
provinces = [PROVINCES[p] for p in params['provinces']]
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""),
|
||||
"school_keyword": params.get("school_keyword", ""), 'provinces': provinces}
|
||||
targets = iter(self.db.score_targets(filters))
|
||||
first = next(targets, None)
|
||||
if first is None and params.get('major_code'):
|
||||
targets = ({**row, 'major_code': params['major_code']}
|
||||
for row in self.db.schools(params.get('school_keyword', '')) if row['province'] in provinces)
|
||||
first = next(targets, None)
|
||||
if first is None:
|
||||
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
|
||||
completed = self.db.completed_units(jid)
|
||||
self.db.log(jid, 'info', '开始补充分数线和历史计划,分批读取专业目标并复用院校年份分数线')
|
||||
client = ScoreClient(self.transport)
|
||||
for (_, code), record in targets.items():
|
||||
for record in chain((first,), targets):
|
||||
self.checkpoint(jid)
|
||||
code = record['major_code']
|
||||
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
|
||||
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")]
|
||||
years = [y for y in params["score_years"] if f"scores:{school['school_code']}:{code}:{y}" not in completed]
|
||||
if not years:
|
||||
continue
|
||||
school_id = client.school_id(school["school_name"])
|
||||
@@ -270,10 +271,10 @@ class JobManager:
|
||||
self.db.job_update(jid, message=label)
|
||||
try:
|
||||
scores = client.score_lines(school, code, year, school_id)
|
||||
self.db.upsert_scores(scores)
|
||||
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
|
||||
self._save(jid, plans)
|
||||
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}")
|
||||
unit = f"scores:{school['school_code']}:{code}:{year}"
|
||||
self.db.save_unit(jid, unit, plans, scores=scores)
|
||||
completed.add(unit)
|
||||
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
|
||||
except RemoteUnavailable as exc:
|
||||
self._error(jid, f"{label}:{exc}")
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Best-effort return of idle allocator memory after large operations."""
|
||||
import ctypes
|
||||
import gc
|
||||
import sys
|
||||
|
||||
|
||||
def release_idle_memory():
|
||||
gc.collect()
|
||||
if sys.platform == 'linux':
|
||||
try:
|
||||
trim = ctypes.CDLL(None).malloc_trim
|
||||
trim.argtypes = [ctypes.c_size_t]
|
||||
trim.restype = ctypes.c_int
|
||||
return bool(trim(0))
|
||||
except (AttributeError, OSError):
|
||||
pass
|
||||
return False
|
||||
@@ -107,6 +107,10 @@ class MySQLDatabase(Database):
|
||||
with self.connection() as conn:
|
||||
for statement in SCHEMA:
|
||||
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
|
||||
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_export'").fetchone():
|
||||
conn.execute('CREATE INDEX catalog_export ON catalog(year DESC,school_name,id)')
|
||||
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_targets'").fetchone():
|
||||
conn.execute('CREATE INDEX catalog_targets ON catalog(year,school_code,major_code)')
|
||||
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
|
||||
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
|
||||
if recover_jobs:
|
||||
@@ -130,7 +134,7 @@ class MySQLDatabase(Database):
|
||||
finally:
|
||||
raw.close()
|
||||
|
||||
def upsert_catalog(self, records):
|
||||
def _upsert_catalog(self, conn, records):
|
||||
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
|
||||
values = []
|
||||
for record in records:
|
||||
@@ -139,12 +143,11 @@ class MySQLDatabase(Database):
|
||||
if not values:
|
||||
return
|
||||
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
|
||||
def upsert_scores(self, records):
|
||||
def _upsert_scores(self, conn, records):
|
||||
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
|
||||
values = []
|
||||
for record in records:
|
||||
@@ -152,7 +155,6 @@ class MySQLDatabase(Database):
|
||||
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
|
||||
values.append([row.get(field) for field in fields])
|
||||
if values:
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
|
||||
|
||||
def favorite(self, rid, enabled):
|
||||
|
||||
+31
-1
@@ -3,6 +3,7 @@ import threading
|
||||
import time
|
||||
import random
|
||||
from contextlib import contextmanager
|
||||
from collections import OrderedDict
|
||||
import requests
|
||||
from studio.settings import proxy_url
|
||||
|
||||
@@ -19,6 +20,31 @@ class RateLimited(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class BoundedProxyAdapter(requests.adapters.HTTPAdapter):
|
||||
"""Close obsolete proxy pools rather than retain every historical exit."""
|
||||
|
||||
def __init__(self, max_proxy_pools=4, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.max_proxy_pools = max(1, int(max_proxy_pools))
|
||||
self.proxy_manager = OrderedDict()
|
||||
|
||||
def proxy_manager_for(self, proxy, **proxy_kwargs):
|
||||
manager = super().proxy_manager_for(proxy, **proxy_kwargs)
|
||||
self.proxy_manager.move_to_end(proxy)
|
||||
while len(self.proxy_manager) > self.max_proxy_pools:
|
||||
_, obsolete = self.proxy_manager.popitem(last=False)
|
||||
obsolete.clear()
|
||||
return manager
|
||||
|
||||
|
||||
def bounded_session():
|
||||
session = requests.Session()
|
||||
session.trust_env = False
|
||||
for scheme in ('http://', 'https://'):
|
||||
session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4))
|
||||
return session
|
||||
|
||||
|
||||
class ProxyRouter:
|
||||
def __init__(self, settings, session=None):
|
||||
self.settings = settings
|
||||
@@ -141,7 +167,7 @@ class ProxyRouter:
|
||||
class Transport:
|
||||
def __init__(self, settings, router, session=None, wait=None):
|
||||
self.settings, self.router = settings, router
|
||||
self.session = session or requests.Session()
|
||||
self.session = session or bounded_session()
|
||||
self.session.trust_env = False
|
||||
self.session.headers = getattr(self.session, "headers", {})
|
||||
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
|
||||
@@ -149,6 +175,10 @@ class Transport:
|
||||
self.local = threading.local()
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def close(self):
|
||||
with self.lock:
|
||||
self.session.close()
|
||||
|
||||
def sleep(self, seconds):
|
||||
remaining = seconds
|
||||
while remaining > 0:
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import pytest
|
||||
|
||||
from studio.database import Database
|
||||
from studio.domain import finish_record
|
||||
|
||||
|
||||
def record(code='10001', name='甲大学', major='081200', province='北京', year=2027, direction='方向一'):
|
||||
return finish_record({'school_code': code, 'school_name': name, 'major_code': major,
|
||||
'major_name': major, 'province': province, 'year': year,
|
||||
'direction': direction, 'subjects': ['101政治'], 'source': '测试'})
|
||||
|
||||
|
||||
def test_history_targets_deduplicate_directions_and_preserve_filters(tmp_path):
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.upsert_catalog([
|
||||
record(direction='一'), record(direction='二'), record(major='081201'),
|
||||
record(code='10002', name='乙大学', province='天津'),
|
||||
record(code='10003', name='丙大学', province=''), record(year=2026),
|
||||
])
|
||||
rows = list(db.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
|
||||
assert [(r['school_code'], r['major_code']) for r in rows] == [
|
||||
('10001', '081200'), ('10001', '081201'), ('10003', '081200')]
|
||||
assert all('data' not in row and 'subjects' not in row for row in rows)
|
||||
rows = list(db.score_targets({'year': 2027, 'school_keyword': '甲', 'major_code': '081201'}))
|
||||
assert [(r['school_code'], r['major_code']) for r in rows] == [('10001', '081201')]
|
||||
|
||||
|
||||
def test_catalog_iterator_keeps_export_order_and_excludes_new_inserts(tmp_path):
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.upsert_catalog([record(name='B大学'), record(name='A大学', direction='二'),
|
||||
record(name='A大学', direction='三'), record(year=2026),
|
||||
record(code='10002', name='C大学')])
|
||||
iterator = db.iter_catalog({}, batch_size=2)
|
||||
first = next(iterator)
|
||||
db.upsert_catalog([record(name='Z大学', year=2028)])
|
||||
rows = [first, *iterator]
|
||||
assert [(r['year'], r['school_name']) for r in rows] == [
|
||||
(2027, 'A大学'), (2027, 'A大学'), (2027, 'B大学'), (2027, 'C大学'), (2026, '甲大学')]
|
||||
assert [r['school_name'] for r in db.iter_catalog({'school_keyword': 'A'}, batch_size=1)] == ['A大学', 'A大学']
|
||||
|
||||
|
||||
def test_score_iterator_exports_only_selected_school_major_pairs(tmp_path):
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.upsert_scores([
|
||||
{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310},
|
||||
{'school_code': '10002', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 320},
|
||||
])
|
||||
rows = list(db.iter_scores({('10001', '081200')}, batch_size=1))
|
||||
assert len(rows) == 1
|
||||
assert rows[0]['total'] == 310
|
||||
|
||||
|
||||
def test_unit_commit_is_idempotent_for_records_count_and_checkpoint(tmp_path):
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.job_create('job', {})
|
||||
rows = [record(direction='一'), record(direction='二')]
|
||||
assert db.save_unit('job', 'catalog:test', rows) is True
|
||||
assert db.save_unit('job', 'catalog:test', rows) is False
|
||||
assert db.overview()['records'] == 2
|
||||
assert db.job_get('job')['records'] == 2
|
||||
assert db.unit_done('job', 'catalog:test')
|
||||
|
||||
|
||||
def test_failed_unit_transaction_rolls_back_scores_records_and_checkpoint(tmp_path, monkeypatch):
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.job_create('job', {})
|
||||
def fail(*args):
|
||||
raise RuntimeError('interrupted persistence')
|
||||
monkeypatch.setattr(db, '_school', fail)
|
||||
score = {'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}
|
||||
with pytest.raises(RuntimeError):
|
||||
db.save_unit('job', 'scores:test', [record()], scores=[score])
|
||||
assert db.overview()['records'] == db.overview()['scores'] == 0
|
||||
assert db.job_get('job')['records'] == 0
|
||||
assert not db.unit_done('job', 'scores:test')
|
||||
@@ -0,0 +1,42 @@
|
||||
from studio import network
|
||||
|
||||
|
||||
def test_proxy_managers_are_bounded_and_evicted_pools_are_closed():
|
||||
adapter = network.BoundedProxyAdapter(max_proxy_pools=3)
|
||||
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
|
||||
closed = []
|
||||
original = first.clear
|
||||
def observe():
|
||||
original()
|
||||
closed.append(True)
|
||||
first.clear = observe
|
||||
for port in range(10002, 10006):
|
||||
adapter.proxy_manager_for(f'http://127.0.0.1:{port}')
|
||||
assert len(adapter.proxy_manager) <= 3
|
||||
assert closed == [True]
|
||||
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is not first
|
||||
adapter.close()
|
||||
|
||||
|
||||
def test_active_proxy_pool_is_reused_while_old_pools_are_evicted():
|
||||
adapter = network.BoundedProxyAdapter(max_proxy_pools=2)
|
||||
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
|
||||
adapter.proxy_manager_for('http://127.0.0.1:10002')
|
||||
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
|
||||
adapter.proxy_manager_for('http://127.0.0.1:10003')
|
||||
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
|
||||
assert 'http://127.0.0.1:10002' not in adapter.proxy_manager
|
||||
adapter.close()
|
||||
|
||||
|
||||
def test_default_transport_limits_pools_without_changing_route_or_tls(tmp_path):
|
||||
from studio.settings import SettingsStore
|
||||
from tests.test_network import Pool
|
||||
settings = SettingsStore(tmp_path)
|
||||
router = network.ProxyRouter(settings, session=Pool())
|
||||
transport = network.Transport(settings, router)
|
||||
adapter = transport.session.get_adapter('https://example.test')
|
||||
assert isinstance(adapter, network.BoundedProxyAdapter)
|
||||
assert transport.session.verify is True
|
||||
assert router.resolve() == 'http://1.2.3.4:8080'
|
||||
transport.close()
|
||||
@@ -0,0 +1,125 @@
|
||||
import asyncio
|
||||
import csv
|
||||
import io
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from studio.api import create_app
|
||||
from tests.test_bounded_storage import record
|
||||
|
||||
|
||||
@pytest.mark.parametrize('kind', ['csv', 'json', 'xlsx'])
|
||||
def test_export_uses_bounded_rows_and_preserves_content(tmp_path, monkeypatch, kind):
|
||||
app = create_app(tmp_path)
|
||||
app.state.db.upsert_catalog([record(name='=1+1'), record(code='10002', name='含逗号,大学')])
|
||||
app.state.db.upsert_scores([{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}])
|
||||
def forbidden(*args, **kwargs):
|
||||
raise AssertionError('export materializes all catalog rows')
|
||||
monkeypatch.setattr(app.state.db, 'all_catalog', forbidden)
|
||||
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
|
||||
response = client.get('/api/export', params={'format': kind})
|
||||
assert response.status_code == 200
|
||||
if kind == 'json':
|
||||
data = response.json()
|
||||
assert {r['school_name'] for r in data['catalog']} == {'=1+1', '含逗号,大学'}
|
||||
assert data['scores'][0]['total'] == 310
|
||||
elif kind == 'csv':
|
||||
rows = list(csv.reader(io.StringIO(response.content.decode('utf-8-sig'))))
|
||||
assert len(rows) == 3
|
||||
assert any("'=1+1" in row for row in rows)
|
||||
assert any('含逗号,大学' in row for row in rows)
|
||||
else:
|
||||
book = load_workbook(io.BytesIO(response.content), read_only=True)
|
||||
rows = list(book.active.values)
|
||||
assert len(rows) == 3
|
||||
assert any("'=1+1" in row for row in rows)
|
||||
book.close()
|
||||
assert not list((tmp_path / 'exports').glob('*'))
|
||||
|
||||
|
||||
def test_concurrent_export_is_rejected_and_lock_is_released(tmp_path, monkeypatch):
|
||||
app = create_app(tmp_path)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original = app.state.db.iter_catalog
|
||||
def slow_rows(*args, **kwargs):
|
||||
if not entered.is_set():
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
yield from original(*args, **kwargs)
|
||||
monkeypatch.setattr(app.state.db, 'iter_catalog', slow_rows)
|
||||
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
|
||||
responses = []
|
||||
thread = threading.Thread(target=lambda: responses.append(client.get('/api/export')))
|
||||
thread.start()
|
||||
assert entered.wait(3)
|
||||
blocked = client.get('/api/export', params={'format': 'json'})
|
||||
release.set()
|
||||
thread.join(5)
|
||||
assert blocked.status_code == 429
|
||||
assert responses[0].status_code == 200
|
||||
assert client.get('/api/export').status_code == 200
|
||||
|
||||
|
||||
def test_disconnected_stream_closes_source_and_releases_export_lock():
|
||||
from studio.exporting import ExportStream
|
||||
from starlette.requests import ClientDisconnect
|
||||
lock = threading.BoundedSemaphore(1)
|
||||
assert lock.acquire(False)
|
||||
|
||||
|
||||
closed = []
|
||||
def source():
|
||||
try:
|
||||
yield b'data'
|
||||
finally:
|
||||
closed.append(True)
|
||||
async def send(message):
|
||||
if message['type'] == 'http.response.body':
|
||||
raise OSError('client disconnected')
|
||||
async def receive():
|
||||
return {'type': 'http.disconnect'}
|
||||
response = ExportStream(source(), lock.release, media_type='text/csv')
|
||||
with pytest.raises(ClientDisconnect):
|
||||
asyncio.run(response({'type': 'http', 'asgi': {'spec_version': '2.4'}}, receive, send))
|
||||
assert closed == [True]
|
||||
assert lock.acquire(False)
|
||||
|
||||
|
||||
def test_xlsx_packaging_error_removes_xml_tempfile_without_masking_error(tmp_path, monkeypatch):
|
||||
from pathlib import Path
|
||||
from studio.imports import write_xlsx
|
||||
from openpyxl import Workbook
|
||||
paths = []
|
||||
def packaging_failure(book, filename):
|
||||
sheet = book.worksheets[0]
|
||||
sheet.close()
|
||||
paths.append(Path(sheet._writer.out))
|
||||
raise OSError('archive write failed')
|
||||
monkeypatch.setattr(Workbook, 'save', packaging_failure)
|
||||
with pytest.raises(OSError, match='archive write failed'):
|
||||
write_xlsx([record()], tmp_path / 'failed.xlsx')
|
||||
assert paths and not paths[0].exists()
|
||||
|
||||
|
||||
def test_failed_export_batch_does_not_finish_a_successful_truncated_download(tmp_path, monkeypatch):
|
||||
app = create_app(tmp_path)
|
||||
def broken_rows(*args, **kwargs):
|
||||
yield record()
|
||||
raise RuntimeError('database batch failed')
|
||||
monkeypatch.setattr(app.state.db, 'iter_catalog', broken_rows)
|
||||
sent = []
|
||||
async def send(message):
|
||||
sent.append(message)
|
||||
async def receive():
|
||||
await asyncio.Event().wait()
|
||||
scope = {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'},
|
||||
'http_version': '1.1', 'method': 'GET', 'scheme': 'http',
|
||||
'path': '/api/export', 'raw_path': b'/api/export', 'root_path': '',
|
||||
'query_string': b'format=csv', 'headers': [(b'host', b'127.0.0.1:8765')],
|
||||
'server': ('127.0.0.1', 8765), 'client': ('127.0.0.1', 12345)}
|
||||
with pytest.raises(RuntimeError, match='database batch failed'):
|
||||
asyncio.run(app(scope, receive, send))
|
||||
assert not any(message['type'] == 'http.response.body' and not message.get('more_body', False) for message in sent)
|
||||
@@ -0,0 +1,147 @@
|
||||
import pytest
|
||||
|
||||
from studio.connectors import ScoreClient
|
||||
from studio.network import RemoteUnavailable
|
||||
from tests.test_network import Response
|
||||
|
||||
|
||||
class HistorySource:
|
||||
def __init__(self, pages):
|
||||
self.pages = pages
|
||||
self.calls = []
|
||||
|
||||
def request(self, method, url, **kwargs):
|
||||
payload = kwargs['data']
|
||||
self.calls.append((url.rsplit('/', 1)[-1], payload.copy()))
|
||||
data = self.pages(payload.get('page', 1))
|
||||
return Response({'code': '0000', 'data': data})
|
||||
|
||||
|
||||
def test_full_score_array_over_page_limit_is_consumed_once():
|
||||
rows = [{'code': '081200', 'total': 310, 'depart_name': '信息学院'}]
|
||||
rows += [{'code': '999999', 'total': 200} for _ in range(197)]
|
||||
source = HistorySource(lambda page: rows)
|
||||
lines = ScoreClient(source).score_lines({'school_code': '10001', 'school_name': '北京大学'}, '081200', 2025, 1)
|
||||
assert len(lines) == 1
|
||||
assert lines[0]['total'] == 310
|
||||
assert [call[1]['page'] for call in source.calls] == [1]
|
||||
|
||||
|
||||
def test_short_wrapped_pages_follow_explicit_total():
|
||||
source = HistorySource(lambda page: {'data': [{'value': page}], 'total': 2})
|
||||
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
|
||||
|
||||
|
||||
def test_short_wrapped_pages_follow_explicit_last_page():
|
||||
source = HistorySource(lambda page: {'list': [{'value': page}], 'last_page': 2})
|
||||
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('data', [None, {}, {'data': None}, {'data': [None]}, {'list': [42]}])
|
||||
def test_invalid_history_data_cannot_become_a_successful_empty_unit(data):
|
||||
with pytest.raises(RemoteUnavailable):
|
||||
list(ScoreClient(HistorySource(lambda page: data)).pages('/example', {}))
|
||||
|
||||
|
||||
def test_empty_full_array_is_a_valid_empty_result():
|
||||
assert list(ScoreClient(HistorySource(lambda page: [])).pages('/example', {})) == []
|
||||
|
||||
|
||||
def test_repeated_real_page_still_raises_instead_of_returning_partial_data():
|
||||
source = HistorySource(lambda page: {'data': [{'value': 1}], 'total': 2})
|
||||
with pytest.raises(RemoteUnavailable):
|
||||
list(ScoreClient(source).pages('/example', {}))
|
||||
|
||||
|
||||
def test_empty_page_before_declared_total_is_an_error():
|
||||
source = HistorySource(lambda page: {'data': [{'value': 1}] if page == 1 else [], 'total': 2})
|
||||
with pytest.raises(RemoteUnavailable):
|
||||
list(ScoreClient(source).pages('/example', {}))
|
||||
|
||||
|
||||
def test_two_majors_reuse_one_complete_school_year_score_table():
|
||||
rows = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
|
||||
source = HistorySource(lambda page: rows)
|
||||
client = ScoreClient(source)
|
||||
school = {'school_code': '10001', 'school_name': '北京大学'}
|
||||
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
|
||||
assert client.score_lines(school, '081201', 2025, 1)[0]['total'] == 320
|
||||
assert len(source.calls) == 1
|
||||
|
||||
|
||||
def test_score_cache_separates_schools_and_years_and_has_a_bound():
|
||||
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}])
|
||||
client = ScoreClient(source)
|
||||
school = {'school_code': '10001', 'school_name': '北京大学'}
|
||||
for sid in range(20):
|
||||
client.score_lines(school, '081200', 2025, sid)
|
||||
client.score_lines(school, '081200', 2026, 19)
|
||||
before = len(source.calls)
|
||||
client.score_lines(school, '081200', 2026, 19)
|
||||
assert len(source.calls) == before
|
||||
client.score_lines(school, '081200', 2025, 0)
|
||||
assert len(source.calls) == before + 1
|
||||
|
||||
|
||||
def test_failed_lookup_is_cached_for_the_current_client_only():
|
||||
source = HistorySource(lambda page: {'data': [], 'total': 0})
|
||||
client = ScoreClient(source)
|
||||
assert client.school_id('暂未收录大学') is None
|
||||
assert client.school_id('暂未收录大学') is None
|
||||
assert len(source.calls) == 1
|
||||
assert ScoreClient(source).school_id('暂未收录大学') is None
|
||||
assert len(source.calls) == 2
|
||||
|
||||
|
||||
def test_invalid_score_response_is_not_cached():
|
||||
valid = False
|
||||
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}] if valid else {})
|
||||
client = ScoreClient(source)
|
||||
school = {'school_code': '10001', 'school_name': '北京大学'}
|
||||
with pytest.raises(RemoteUnavailable):
|
||||
client.score_lines(school, '081200', 2025, 1)
|
||||
valid = True
|
||||
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
|
||||
|
||||
|
||||
def test_history_runner_uses_thin_targets_and_bulk_checkpoints(tmp_path, monkeypatch):
|
||||
from studio.database import Database
|
||||
from studio.jobs import JobManager
|
||||
from studio.settings import SettingsStore
|
||||
from tests.test_bounded_storage import record
|
||||
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.upsert_catalog([record(), record(major='081201'), record(direction='二')])
|
||||
params = {'year': 2027, 'provinces': ['11'], 'score_years': [2025, 2026], 'include_plans': True}
|
||||
db.job_create('history', params)
|
||||
|
||||
class Source:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
def request(self, method, url, **kwargs):
|
||||
payload = kwargs['data']
|
||||
self.calls.append(url.rsplit('/', 1)[-1])
|
||||
if url.endswith('/schoolList'):
|
||||
data = {'data': [{'school_name': '甲大学', 'school_id': 1}], 'total': 1}
|
||||
elif url.endswith('/schoolScore'):
|
||||
data = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
|
||||
else:
|
||||
data = {'data': [{'special_code': payload['keyword'], 'special_name': payload['keyword'],
|
||||
'plan_id': str(payload['year']) + payload['keyword'], 'recruit_number': 1}], 'total': 1}
|
||||
return Response({'code': '0000', 'data': data})
|
||||
|
||||
def forbidden(*args, **kwargs):
|
||||
raise AssertionError('full catalog decoding or per-unit checkpoint query')
|
||||
monkeypatch.setattr(db, 'all_catalog', forbidden)
|
||||
monkeypatch.setattr(db, 'unit_done', forbidden)
|
||||
source = Source()
|
||||
manager = JobManager(db, SettingsStore(tmp_path), transport=source)
|
||||
manager._scores('history', params)
|
||||
assert db.job_get('history')['completed_units'] == 4
|
||||
assert db.overview()['scores'] == 4
|
||||
assert db.job_get('history')['records'] == 4
|
||||
assert source.calls.count('schoolScore') == 2
|
||||
assert source.calls.count('planListV2') == 4
|
||||
before = len(source.calls)
|
||||
manager._scores('history', params)
|
||||
assert len(source.calls) == before
|
||||
@@ -98,6 +98,47 @@ def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypa
|
||||
assert creations == []
|
||||
|
||||
|
||||
def test_mysql_bounded_catalog_and_history_targets_preserve_filters(mysql):
|
||||
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
|
||||
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
|
||||
scores = select_score_lines([{'code': '081200', 'total': 310, 'depart_name': '计算机学院'}], '081200', '10001', '北京大学', 2026)
|
||||
mysql.upsert_scores(scores)
|
||||
targets = list(mysql.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
|
||||
assert len(targets) == 1
|
||||
assert targets[0]['major_code'] == '081200'
|
||||
assert 'data' not in targets[0]
|
||||
rows = list(mysql.iter_catalog({'year': 2027}, batch_size=1))
|
||||
assert len(rows) == 2
|
||||
assert len(list(mysql.iter_catalog({'year': 2026, 'max_score': 310}, batch_size=1))) == 2
|
||||
assert len(list(mysql.iter_scores({('10001', '081200')}, batch_size=1))) == 1
|
||||
|
||||
|
||||
def test_mysql_unit_commit_is_atomic_and_idempotent(mysql):
|
||||
mysql.job_create('atomic-job', {})
|
||||
rows = normalize_catalog(raw_catalog(), 2027)
|
||||
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
|
||||
assert mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
|
||||
assert not mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
|
||||
assert mysql.job_get('atomic-job')['records'] == 2
|
||||
assert mysql.job_get('atomic-job')['completed_units'] == 1
|
||||
assert mysql.overview()['records'] == 2
|
||||
assert mysql.overview()['scores'] == 1
|
||||
|
||||
|
||||
def test_mysql_failed_unit_rolls_back_every_write(mysql, monkeypatch):
|
||||
mysql.job_create('atomic-job', {})
|
||||
def fail(*args):
|
||||
raise RuntimeError('interrupted unit')
|
||||
monkeypatch.setattr(mysql, '_school', fail)
|
||||
rows = normalize_catalog(raw_catalog(), 2027)
|
||||
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
|
||||
with pytest.raises(RuntimeError):
|
||||
mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
|
||||
assert mysql.job_get('atomic-job')['records'] == 0
|
||||
assert mysql.job_get('atomic-job')['completed_units'] == 0
|
||||
assert mysql.overview()['records'] == mysql.overview()['scores'] == 0
|
||||
|
||||
|
||||
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
|
||||
from studio.migrate import migrate_sqlite
|
||||
source = Database(tmp_path / 'snapshot.db')
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import importlib.util
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def maintainer_class():
|
||||
path = Path(__file__).resolve().parents[1] / 'deploy/proxy_scheduler.py'
|
||||
spec = importlib.util.spec_from_file_location('bounded_proxy_scheduler', path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.ProxyMaintainer
|
||||
|
||||
|
||||
def test_detection_workers_are_bounded_reused_and_rounds_do_not_overlap():
|
||||
state = {'active': 0, 'peak': 0, 'types': set(), 'mixed': False, 'done': [], 'thread_ids': set()}
|
||||
lock = threading.Lock()
|
||||
class Checker:
|
||||
def __init__(self, work_type, target_queue, name):
|
||||
self.work_type, self.queue = work_type, target_queue
|
||||
def run(self):
|
||||
while True:
|
||||
try:
|
||||
item = self.queue.get(block=False)
|
||||
except queue.Empty:
|
||||
return
|
||||
with lock:
|
||||
state['active'] += 1
|
||||
state['peak'] = max(state['peak'], state['active'])
|
||||
state['types'].add(self.work_type)
|
||||
state['mixed'] |= len(state['types']) > 1
|
||||
state['thread_ids'].add(threading.get_ident())
|
||||
time.sleep(.01)
|
||||
with lock:
|
||||
state['done'].append(item)
|
||||
state['active'] -= 1
|
||||
if not state['active']:
|
||||
state['types'].clear()
|
||||
self.queue.task_done()
|
||||
maintainer = maintainer_class()(workers=2, checker_class=Checker)
|
||||
first = threading.Thread(target=maintainer.validate, args=('raw', range(10)))
|
||||
second = threading.Thread(target=maintainer.validate, args=('use', range(10, 20)))
|
||||
first.start(); second.start()
|
||||
first.join(4); second.join(4)
|
||||
assert not first.is_alive() and not second.is_alive()
|
||||
before = set(state['thread_ids'])
|
||||
maintainer.validate('raw', range(20, 25))
|
||||
maintainer.close()
|
||||
assert sorted(state['done']) == list(range(25))
|
||||
assert state['peak'] <= 2
|
||||
assert state['mixed'] is False
|
||||
assert state['thread_ids'] == before
|
||||
|
||||
|
||||
def test_stopped_detector_does_not_begin_new_rounds():
|
||||
class Forbidden:
|
||||
def __init__(self, *args):
|
||||
raise AssertionError('detection began after shutdown')
|
||||
maintainer = maintainer_class()(workers=2, checker_class=Forbidden)
|
||||
maintainer.close()
|
||||
maintainer.validate('raw', [1, 2, 3])
|
||||
|
||||
|
||||
def test_failed_worker_waits_for_other_workers_before_next_round():
|
||||
slow_started, release, next_round = threading.Event(), threading.Event(), threading.Event()
|
||||
class Checker:
|
||||
def __init__(self, work_type, target_queue, name):
|
||||
self.kind, self.name = work_type, name
|
||||
def run(self):
|
||||
if self.kind == 'raw' and self.name == 'worker_00':
|
||||
assert slow_started.wait(2)
|
||||
raise RuntimeError('validator failed')
|
||||
if self.kind == 'raw':
|
||||
slow_started.set()
|
||||
assert release.wait(3)
|
||||
maintainer = maintainer_class()(workers=2, checker_class=Checker)
|
||||
failures = []
|
||||
def first_round():
|
||||
try:
|
||||
maintainer.validate('raw', [1, 2])
|
||||
except RuntimeError:
|
||||
failures.append(True)
|
||||
first = threading.Thread(target=first_round)
|
||||
first.start()
|
||||
assert slow_started.wait(2)
|
||||
def second_proxies():
|
||||
next_round.set()
|
||||
yield 3
|
||||
second = threading.Thread(target=maintainer.validate, args=('use', second_proxies()))
|
||||
second.start()
|
||||
began_early = next_round.wait(.15)
|
||||
release.set()
|
||||
first.join(3); second.join(3)
|
||||
maintainer.close()
|
||||
assert began_early is False
|
||||
assert next_round.is_set()
|
||||
assert failures == [True]
|
||||
Reference in New Issue
Block a user