Fix historical pagination and bound export and proxy memory
This commit is contained in:
@@ -62,6 +62,10 @@ Debian 12 / MariaDB 服务器模式、HTTPS 访问与代理池后台运行见 [
|
|||||||
|
|
||||||
可将当前筛选结果导出 CSV、Excel 或完整 JSON。表格导出保留院系/方向代码和字段口径,回导不会凭空增加同一记录。JSON 同时保留相关学校专业的历史分数线;凭据不进入导出。CSV/Excel 对可能被表格软件当作公式的文本加保护前缀。
|
可将当前筛选结果导出 CSV、Excel 或完整 JSON。表格导出保留院系/方向代码和字段口径,回导不会凭空增加同一记录。JSON 同时保留相关学校专业的历史分数线;凭据不进入导出。CSV/Excel 对可能被表格软件当作公式的文本加保护前缀。
|
||||||
|
|
||||||
|
全量导出按 200 条分批读取,CSV/JSON 流式下载,Excel 使用写入模式和临时文件;同一实例一次只执行一个导出,重复请求返回 429。下载结束或中断后清理临时资源。导出期间新增的记录留待下次导出;导出不是事务一致性备份。大于 100 MB 的文件仍受导入上限限制,可按地区等条件分别导出;完整数据库备份按服务器部署说明操作。
|
||||||
|
|
||||||
|
历史采集在招生目录阶段完成后开始,按院校和年份复用全校分数线,每次最多缓存 8 份,历史计划继续逐专业获取。接口直接返回的数组按完整结果处理,带分页信息的结果按实际数量继续;不完整的响应不会被写成成功断点。数据、保存计数和完成断点在同一数据库事务中提交。
|
||||||
|
|
||||||
## 本地数据与开发
|
## 本地数据与开发
|
||||||
|
|
||||||
所有运行数据位于项目 `.runtime`:`studio.db` 保存目录、分数、收藏、任务和断点;`settings.json` 保存非机密配置;`chsi-cookie.enc` 由当前 Windows 用户的 DPAPI 加密。密码不持久化,接口不返回 Cookie。备份数据库时先停止工作台,再复制 `.runtime`;恢复加密 Cookie 需要原 Windows 用户环境,否则重新登录即可。
|
所有运行数据位于项目 `.runtime`:`studio.db` 保存目录、分数、收藏、任务和断点;`settings.json` 保存非机密配置;`chsi-cookie.enc` 由当前 Windows 用户的 DPAPI 加密。密码不持久化,接口不返回 Cookie。备份数据库时先停止工作台,再复制 `.runtime`;恢复加密 Cookie 需要原 Windows 用户环境,否则重新登录即可。
|
||||||
|
|||||||
@@ -8,9 +8,12 @@ Requires=docker.service
|
|||||||
User=kaoyan
|
User=kaoyan
|
||||||
Group=kaoyan
|
Group=kaoyan
|
||||||
WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
|
WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
|
||||||
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py schedule
|
Environment=MALLOC_ARENA_MAX=2
|
||||||
|
Environment=PROXY_CHECK_WORKERS=8
|
||||||
|
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python /opt/kaoyan-studio-deploy/app/deploy/proxy_scheduler.py
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
RestartSec=10
|
RestartSec=10
|
||||||
|
TimeoutStopSec=120
|
||||||
UMask=0077
|
UMask=0077
|
||||||
PrivateTmp=true
|
PrivateTmp=true
|
||||||
ProtectSystem=full
|
ProtectSystem=full
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ User=kaoyan
|
|||||||
Group=kaoyan
|
Group=kaoyan
|
||||||
WorkingDirectory=/opt/kaoyan-studio-deploy/app
|
WorkingDirectory=/opt/kaoyan-studio-deploy/app
|
||||||
EnvironmentFile=/etc/kaoyan-studio.env
|
EnvironmentFile=/etc/kaoyan-studio.env
|
||||||
|
Environment=MALLOC_ARENA_MAX=2
|
||||||
ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio
|
ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio
|
||||||
Restart=on-failure
|
Restart=on-failure
|
||||||
RestartSec=10
|
RestartSec=10
|
||||||
|
|||||||
@@ -0,0 +1,122 @@
|
|||||||
|
"""Run upstream proxy_pool checks with reusable workers and serialized jobs."""
|
||||||
|
import os
|
||||||
|
import queue
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
APP_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(APP_ROOT))
|
||||||
|
from studio.memory import release_idle_memory
|
||||||
|
|
||||||
|
|
||||||
|
class StopQueue(queue.Queue):
|
||||||
|
def __init__(self, stopping):
|
||||||
|
super().__init__()
|
||||||
|
self.stopping = stopping
|
||||||
|
|
||||||
|
def get(self, *args, **kwargs):
|
||||||
|
if self.stopping.is_set():
|
||||||
|
raise queue.Empty
|
||||||
|
return super().get(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class ProxyMaintainer:
|
||||||
|
def __init__(self, workers=8, checker_class=None, fetcher_factory=None, handler_factory=None, pool_min=20):
|
||||||
|
self.worker_count = max(1, min(20, int(workers)))
|
||||||
|
self.checker_class = checker_class
|
||||||
|
self.fetcher_factory, self.handler_factory = fetcher_factory, handler_factory
|
||||||
|
self.pool_min = pool_min
|
||||||
|
self.stopping = threading.Event()
|
||||||
|
self.pipeline = threading.RLock()
|
||||||
|
self.workers = ThreadPoolExecutor(max_workers=self.worker_count, thread_name_prefix='proxy-check')
|
||||||
|
|
||||||
|
def validate(self, work_type, proxies):
|
||||||
|
with self.pipeline:
|
||||||
|
if self.stopping.is_set():
|
||||||
|
return
|
||||||
|
target = StopQueue(self.stopping)
|
||||||
|
for proxy in proxies:
|
||||||
|
if self.stopping.is_set():
|
||||||
|
return
|
||||||
|
target.put(proxy)
|
||||||
|
if self.checker_class is None:
|
||||||
|
from helper.check import _ThreadChecker
|
||||||
|
self.checker_class = _ThreadChecker
|
||||||
|
count = min(self.worker_count, target.qsize())
|
||||||
|
futures = [self.workers.submit(self.checker_class(work_type, target, f'worker_{i:02}').run)
|
||||||
|
for i in range(count)]
|
||||||
|
try:
|
||||||
|
failure = None
|
||||||
|
for future in futures:
|
||||||
|
try:
|
||||||
|
future.result()
|
||||||
|
except Exception as exc:
|
||||||
|
if failure is None:
|
||||||
|
failure = exc
|
||||||
|
if failure is not None:
|
||||||
|
raise failure
|
||||||
|
finally:
|
||||||
|
release_idle_memory()
|
||||||
|
|
||||||
|
def fetch(self):
|
||||||
|
with self.pipeline:
|
||||||
|
if not self.stopping.is_set():
|
||||||
|
self.validate('raw', self.fetcher_factory().run())
|
||||||
|
|
||||||
|
def check(self):
|
||||||
|
with self.pipeline:
|
||||||
|
if self.stopping.is_set():
|
||||||
|
return
|
||||||
|
handler = self.handler_factory()
|
||||||
|
if handler.db.getCount().get('total', 0) < self.pool_min:
|
||||||
|
self.fetch()
|
||||||
|
self.validate('use', handler.getAll())
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.stopping.set()
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.stop()
|
||||||
|
with self.pipeline:
|
||||||
|
self.workers.shutdown(wait=True, cancel_futures=True)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
proxy_root = Path(os.environ.get('STUDIO_PROXY_POOL_ROOT', APP_ROOT.parent / 'proxy_pool'))
|
||||||
|
sys.path.insert(0, str(proxy_root))
|
||||||
|
from apscheduler.schedulers.blocking import BlockingScheduler
|
||||||
|
from helper.check import _ThreadChecker
|
||||||
|
from helper.fetch import Fetcher
|
||||||
|
from handler.configHandler import ConfigHandler
|
||||||
|
from handler.logHandler import LogHandler
|
||||||
|
from handler.proxyHandler import ProxyHandler
|
||||||
|
|
||||||
|
config = ConfigHandler()
|
||||||
|
maintainer = ProxyMaintainer(workers=int(os.environ.get('PROXY_CHECK_WORKERS', '8')),
|
||||||
|
checker_class=_ThreadChecker, fetcher_factory=Fetcher,
|
||||||
|
handler_factory=ProxyHandler, pool_min=config.poolSizeMin)
|
||||||
|
scheduler = BlockingScheduler(timezone=config.timezone, logger=LogHandler('scheduler'),
|
||||||
|
executors={'default': {'type': 'threadpool', 'max_workers': 1}},
|
||||||
|
job_defaults={'coalesce': True, 'max_instances': 1, 'misfire_grace_time': 120})
|
||||||
|
scheduler.add_job(maintainer.fetch, 'interval', minutes=5, id='proxy_fetch')
|
||||||
|
scheduler.add_job(maintainer.check, 'interval', minutes=2, id='proxy_check',
|
||||||
|
next_run_time=datetime.now(timezone.utc))
|
||||||
|
def stop(signum, frame):
|
||||||
|
maintainer.stop()
|
||||||
|
if scheduler.running:
|
||||||
|
scheduler.shutdown(wait=False)
|
||||||
|
signal.signal(signal.SIGTERM, stop)
|
||||||
|
signal.signal(signal.SIGINT, stop)
|
||||||
|
try:
|
||||||
|
if not maintainer.stopping.is_set():
|
||||||
|
scheduler.start()
|
||||||
|
finally:
|
||||||
|
maintainer.close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@@ -16,6 +16,8 @@
|
|||||||
|
|
||||||
安装模板在 `deploy/`。Python 依赖按 `requirements.lock` 安装;前端 `npm ci && npm run build`。不要启动多个工作台进程连接同一数据库,否则任务恢复状态会互相影响。原有站点、数据库和 fail2ban 配置不作修改。
|
安装模板在 `deploy/`。Python 依赖按 `requirements.lock` 安装;前端 `npm ci && npm run build`。不要启动多个工作台进程连接同一数据库,否则任务恢复状态会互相影响。原有站点、数据库和 fail2ban 配置不作修改。
|
||||||
|
|
||||||
|
代理调度器运行本项目 `deploy/proxy_scheduler.py` 包装器,继续使用原版代理池的获取源、检测逻辑和 Redis 数据,不修改上游源码。默认复用 8 个检测线程,抓取与检查轮次串行运行,避免定时任务重叠。工作台和调度器设定 `MALLOC_ARENA_MAX=2`;每次导出/检查结束尝试释放分配器空闲内存。该操作不释放仍在使用的数据,实际内存以运行监测为准。
|
||||||
|
|
||||||
## 操作
|
## 操作
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -28,6 +30,8 @@ systemctl restart kaoyan-studio
|
|||||||
|
|
||||||
默认每次请求随机等待 1.2–2.4 秒,每 100 次成功请求轮换代理。设置页可修改上下限和轮换次数;0 表示关闭自动轮换。代理页显示实际成功请求路由、计数和切换次数。登录表单、会话同步、权限验证及紧随其后的失败页重试保持同一路由。免费代理池初次启动可能没有 HTTPS 候选,部署时保留了原电脑代理池的候选作为备用;这些候选仍会经过实际请求与失败隔离。
|
默认每次请求随机等待 1.2–2.4 秒,每 100 次成功请求轮换代理。设置页可修改上下限和轮换次数;0 表示关闭自动轮换。代理页显示实际成功请求路由、计数和切换次数。登录表单、会话同步、权限验证及紧随其后的失败页重试保持同一路由。免费代理池初次启动可能没有 HTTPS 候选,部署时保留了原电脑代理池的候选作为备用;这些候选仍会经过实际请求与失败隔离。
|
||||||
|
|
||||||
|
当前全国任务采用 0 次自动轮换,维持通过登录验证的出口;连接失败仍会切换。Cookie 验证状态与出口关联,出口变化后需要重新验证,不宜为了让 IP 显示变化而频繁轮换。HTTP 会话最多保留 4 个代理连接池,淘汰旧池时主动关闭连接。
|
||||||
|
|
||||||
Linux 无桌面环境,界面提供研招网登录入口与粘贴 Cookie。请在自己的电脑浏览器打开研招目录并登录,从开发者工具中复制目录接口请求的单行 Cookie,粘贴到工作台后点验证。服务器重新同步会话并检查受保护的第二页。迁移 Windows Cookie 时只经 SSH 传输加密封装,在服务器重新加密;不继承原验证标记。
|
Linux 无桌面环境,界面提供研招网登录入口与粘贴 Cookie。请在自己的电脑浏览器打开研招目录并登录,从开发者工具中复制目录接口请求的单行 Cookie,粘贴到工作台后点验证。服务器重新同步会话并检查受保护的第二页。迁移 Windows Cookie 时只经 SSH 传输加密封装,在服务器重新加密;不继承原验证标记。
|
||||||
|
|
||||||
## 迁移与验证
|
## 迁移与验证
|
||||||
|
|||||||
+65
-25
@@ -2,6 +2,8 @@
|
|||||||
import json
|
import json
|
||||||
import uuid
|
import uuid
|
||||||
import os
|
import os
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
@@ -11,7 +13,9 @@ from fastapi.staticfiles import StaticFiles
|
|||||||
from studio.auth import AuthService, CookieVault
|
from studio.auth import AuthService, CookieVault
|
||||||
from studio.database import Database
|
from studio.database import Database
|
||||||
from studio.domain import PROVINCES
|
from studio.domain import PROVINCES
|
||||||
from studio.imports import import_file, export_sheet
|
from studio.imports import import_file, write_xlsx
|
||||||
|
from studio.exporting import ExportStream, csv_chunks, json_chunks, file_chunks
|
||||||
|
from studio.memory import release_idle_memory
|
||||||
from studio.jobs import JobManager
|
from studio.jobs import JobManager
|
||||||
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
|
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||||
from studio.settings import SettingsStore
|
from studio.settings import SettingsStore
|
||||||
@@ -20,6 +24,31 @@ from studio.pool_service import PoolService
|
|||||||
ROOT = Path(__file__).resolve().parent.parent
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
|
||||||
|
|
||||||
|
class LocalAccessMiddleware:
|
||||||
|
"""Check hosts and origins without buffering or wrapping response streams."""
|
||||||
|
|
||||||
|
def __init__(self, app, allowed_hosts):
|
||||||
|
self.app = app
|
||||||
|
self.allowed_hosts = allowed_hosts
|
||||||
|
|
||||||
|
async def __call__(self, scope, receive, send):
|
||||||
|
if scope['type'] != 'http':
|
||||||
|
return await self.app(scope, receive, send)
|
||||||
|
request = Request(scope)
|
||||||
|
if request.url.hostname not in self.allowed_hosts:
|
||||||
|
response = JSONResponse({'detail': '访问主机不在允许列表中'}, status_code=403)
|
||||||
|
return await response(scope, receive, send)
|
||||||
|
if request.method not in ('GET', 'HEAD', 'OPTIONS'):
|
||||||
|
origin = request.headers.get('origin')
|
||||||
|
if origin:
|
||||||
|
parsed = urlparse(origin)
|
||||||
|
allowed = {str(request.base_url).rstrip('/'), 'http://127.0.0.1:5173', 'http://localhost:5173'}
|
||||||
|
if parsed.scheme not in ('http', 'https') or origin.rstrip('/') not in allowed:
|
||||||
|
response = JSONResponse({'detail': '已拒绝跨来源修改请求'}, status_code=403)
|
||||||
|
return await response(scope, receive, send)
|
||||||
|
return await self.app(scope, receive, send)
|
||||||
|
|
||||||
|
|
||||||
def create_app(data_dir=None):
|
def create_app(data_dir=None):
|
||||||
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
|
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
|
||||||
directory.mkdir(parents=True, exist_ok=True)
|
directory.mkdir(parents=True, exist_ok=True)
|
||||||
@@ -36,6 +65,9 @@ def create_app(data_dir=None):
|
|||||||
transport = Transport(settings, router)
|
transport = Transport(settings, router)
|
||||||
auth = AuthService(transport, CookieVault(directory))
|
auth = AuthService(transport, CookieVault(directory))
|
||||||
jobs = JobManager(db, settings, transport, auth=auth)
|
jobs = JobManager(db, settings, transport, auth=auth)
|
||||||
|
export_lock = threading.BoundedSemaphore(1)
|
||||||
|
export_directory = directory / 'exports'
|
||||||
|
export_directory.mkdir(exist_ok=True)
|
||||||
if (ROOT / "data/schools.json").exists():
|
if (ROOT / "data/schools.json").exists():
|
||||||
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
|
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
|
||||||
|
|
||||||
@@ -46,6 +78,8 @@ def create_app(data_dir=None):
|
|||||||
jobs.resume(jid)
|
jobs.resume(jid)
|
||||||
yield
|
yield
|
||||||
jobs.close()
|
jobs.close()
|
||||||
|
transport.close()
|
||||||
|
router.session.close()
|
||||||
|
|
||||||
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
|
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
|
||||||
app.state.db, app.state.jobs = db, jobs
|
app.state.db, app.state.jobs = db, jobs
|
||||||
@@ -53,19 +87,7 @@ def create_app(data_dir=None):
|
|||||||
app.state.pool = pool
|
app.state.pool = pool
|
||||||
app.state.auth = auth
|
app.state.auth = auth
|
||||||
|
|
||||||
@app.middleware("http")
|
app.add_middleware(LocalAccessMiddleware, allowed_hosts=allowed_hosts)
|
||||||
async def local_access(request: Request, call_next):
|
|
||||||
host = request.url.hostname
|
|
||||||
if host not in allowed_hosts:
|
|
||||||
return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
|
|
||||||
if request.method not in ("GET", "HEAD", "OPTIONS"):
|
|
||||||
origin = request.headers.get("origin")
|
|
||||||
if origin:
|
|
||||||
parsed = urlparse(origin)
|
|
||||||
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
|
|
||||||
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
|
|
||||||
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
|
|
||||||
return await call_next(request)
|
|
||||||
|
|
||||||
@app.exception_handler(ValueError)
|
@app.exception_handler(ValueError)
|
||||||
async def invalid(request, exc):
|
async def invalid(request, exc):
|
||||||
@@ -216,17 +238,35 @@ def create_app(data_dir=None):
|
|||||||
def export(request: Request, format: str = "csv"):
|
def export(request: Request, format: str = "csv"):
|
||||||
if format not in ("csv", "xlsx", "json"):
|
if format not in ("csv", "xlsx", "json"):
|
||||||
raise ValueError("导出格式无效")
|
raise ValueError("导出格式无效")
|
||||||
rows = db.all_catalog(filters(request))
|
selected = filters(request)
|
||||||
if format == "json":
|
if not export_lock.acquire(False):
|
||||||
pairs = {(r["school_code"], r["major_code"]) for r in rows}
|
raise HTTPException(429, '已有导出正在进行,请等待完成后重试')
|
||||||
with db.connection() as conn:
|
temporary = None
|
||||||
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs]
|
def cleanup():
|
||||||
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode()
|
try:
|
||||||
mime = "application/json"
|
if temporary is not None:
|
||||||
else:
|
temporary.unlink(missing_ok=True)
|
||||||
payload = export_sheet(rows, format)
|
finally:
|
||||||
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
export_lock.release()
|
||||||
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'})
|
try:
|
||||||
|
if format == 'csv':
|
||||||
|
source, mime = csv_chunks(db.iter_catalog(selected)), 'text/csv; charset=utf-8'
|
||||||
|
elif format == 'json':
|
||||||
|
source, mime = json_chunks(db, selected), 'application/json'
|
||||||
|
else:
|
||||||
|
descriptor, path = tempfile.mkstemp(prefix='kaoyan-', suffix='.xlsx', dir=export_directory)
|
||||||
|
os.close(descriptor)
|
||||||
|
temporary = Path(path)
|
||||||
|
write_xlsx(db.iter_catalog(selected), temporary)
|
||||||
|
source = file_chunks(temporary)
|
||||||
|
mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
||||||
|
return ExportStream(source, cleanup, media_type=mime,
|
||||||
|
headers={'Content-Disposition': f'attachment; filename="kaoyan-data.{format}"',
|
||||||
|
'X-Accel-Buffering': 'no'})
|
||||||
|
except BaseException:
|
||||||
|
cleanup()
|
||||||
|
release_idle_memory()
|
||||||
|
raise
|
||||||
|
|
||||||
dist = ROOT / "frontend/dist"
|
dist = ROOT / "frontend/dist"
|
||||||
if (dist / "assets").exists():
|
if (dist / "assets").exists():
|
||||||
|
|||||||
+43
-8
@@ -2,6 +2,7 @@
|
|||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import hashlib
|
import hashlib
|
||||||
|
from collections import OrderedDict
|
||||||
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
||||||
from studio.network import RemoteUnavailable, RateLimited
|
from studio.network import RemoteUnavailable, RateLimited
|
||||||
|
|
||||||
@@ -153,6 +154,7 @@ class ScoreClient:
|
|||||||
def __init__(self, transport):
|
def __init__(self, transport):
|
||||||
self.transport = transport
|
self.transport = transport
|
||||||
self.cache = {}
|
self.cache = {}
|
||||||
|
self.score_cache = OrderedDict()
|
||||||
|
|
||||||
def post(self, endpoint, payload):
|
def post(self, endpoint, payload):
|
||||||
for attempt in range(3):
|
for attempt in range(3):
|
||||||
@@ -161,8 +163,13 @@ class ScoreClient:
|
|||||||
body = response.json()
|
body = response.json()
|
||||||
except ValueError:
|
except ValueError:
|
||||||
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
|
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
|
||||||
|
if not isinstance(body, dict):
|
||||||
|
raise RemoteUnavailable("掌上考研响应结构已变化")
|
||||||
if str(body.get("code")) == "0000":
|
if str(body.get("code")) == "0000":
|
||||||
return body.get("data") or {}
|
data = body.get("data")
|
||||||
|
if not isinstance(data, (dict, list)):
|
||||||
|
raise RemoteUnavailable("掌上考研缺少有效数据,未写入不完整结果")
|
||||||
|
return data
|
||||||
message = str(body.get('message', body.get('msg', '')))
|
message = str(body.get('message', body.get('msg', '')))
|
||||||
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
|
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
|
||||||
if attempt == 2:
|
if attempt == 2:
|
||||||
@@ -172,20 +179,40 @@ class ScoreClient:
|
|||||||
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
|
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
|
||||||
|
|
||||||
def pages(self, endpoint, payload, limit=100):
|
def pages(self, endpoint, payload, limit=100):
|
||||||
seen = set()
|
seen, consumed = set(), 0
|
||||||
for page in range(1, 10001):
|
for page in range(1, 10001):
|
||||||
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
|
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
|
||||||
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data
|
full_list = isinstance(data, list)
|
||||||
if not isinstance(rows, list):
|
rows = data.get("data", data.get("list")) if isinstance(data, dict) else data
|
||||||
|
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
|
||||||
raise RemoteUnavailable("掌上考研响应结构已变化")
|
raise RemoteUnavailable("掌上考研响应结构已变化")
|
||||||
|
total = integer(data.get("total")) if isinstance(data, dict) else None
|
||||||
|
last = integer(data.get("last_page")) if isinstance(data, dict) else None
|
||||||
|
if isinstance(data, dict):
|
||||||
|
for field, value in (('total', total), ('last_page', last)):
|
||||||
|
if data.get(field) is not None and (value is None or value < 0 or isinstance(data[field], bool)):
|
||||||
|
raise RemoteUnavailable("掌上考研分页信息无效,未写入不完整结果")
|
||||||
|
if not rows and (total is not None and consumed < total or last is not None and page < last):
|
||||||
|
raise RemoteUnavailable("掌上考研提前返回空页,未写入不完整结果")
|
||||||
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
||||||
if rows and signature in seen:
|
if rows and signature in seen:
|
||||||
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
|
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
|
||||||
seen.add(signature)
|
seen.add(signature)
|
||||||
|
consumed += len(rows)
|
||||||
yield from rows
|
yield from rows
|
||||||
total = integer(data.get("total")) if isinstance(data, dict) else None
|
# Bare arrays are complete responses; wrapped pages use their actual
|
||||||
last = integer(data.get("last_page")) if isinstance(data, dict) else None
|
# metadata even when the server ignores the requested page size.
|
||||||
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total:
|
terminal = full_list or not rows
|
||||||
|
if not terminal:
|
||||||
|
if last is not None:
|
||||||
|
terminal = page >= last
|
||||||
|
if terminal and total is not None and consumed < total:
|
||||||
|
raise RemoteUnavailable("掌上考研分页未返回全部记录")
|
||||||
|
elif total is not None:
|
||||||
|
terminal = consumed >= total
|
||||||
|
else:
|
||||||
|
terminal = len(rows) < limit
|
||||||
|
if terminal:
|
||||||
return
|
return
|
||||||
raise RemoteUnavailable("历史数据分页超过安全上限")
|
raise RemoteUnavailable("历史数据分页超过安全上限")
|
||||||
|
|
||||||
@@ -198,10 +225,18 @@ class ScoreClient:
|
|||||||
if len(matches) == 1 and matches[0].get("school_id"):
|
if len(matches) == 1 and matches[0].get("school_id"):
|
||||||
self.cache[key] = matches[0]["school_id"]
|
self.cache[key] = matches[0]["school_id"]
|
||||||
return self.cache[key]
|
return self.cache[key]
|
||||||
|
self.cache[key] = None
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def score_lines(self, school, major_code, year, school_id):
|
def score_lines(self, school, major_code, year, school_id):
|
||||||
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
|
key = (str(school_id), int(year))
|
||||||
|
if key not in self.score_cache:
|
||||||
|
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
|
||||||
|
self.score_cache[key] = rows
|
||||||
|
if len(self.score_cache) > 8:
|
||||||
|
self.score_cache.popitem(last=False)
|
||||||
|
self.score_cache.move_to_end(key)
|
||||||
|
rows = self.score_cache[key]
|
||||||
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
|
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
|
||||||
|
|
||||||
def plans(self, school, major_code, year, school_id):
|
def plans(self, school, major_code, year, school_id):
|
||||||
|
|||||||
+90
-7
@@ -19,6 +19,8 @@ CREATE TABLE IF NOT EXISTS catalog(
|
|||||||
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
|
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
|
||||||
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
|
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
|
||||||
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
|
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
|
||||||
|
CREATE INDEX IF NOT EXISTS catalog_export ON catalog(year DESC,school_name,id);
|
||||||
|
CREATE INDEX IF NOT EXISTS catalog_targets ON catalog(year,school_code,major_code);
|
||||||
""" + SCORE_SCHEMA + """
|
""" + SCORE_SCHEMA + """
|
||||||
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
|
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
|
||||||
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
||||||
@@ -69,6 +71,10 @@ class Database:
|
|||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
def upsert_catalog(self, records: list[dict]):
|
def upsert_catalog(self, records: list[dict]):
|
||||||
|
with self.connection() as conn:
|
||||||
|
self._upsert_catalog(conn, records)
|
||||||
|
|
||||||
|
def _upsert_catalog(self, conn, records):
|
||||||
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
|
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
|
||||||
values = []
|
values = []
|
||||||
for record in records:
|
for record in records:
|
||||||
@@ -77,10 +83,9 @@ class Database:
|
|||||||
if not values:
|
if not values:
|
||||||
return
|
return
|
||||||
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
|
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
|
||||||
with self.connection() as conn:
|
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
|
||||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
|
for record in records:
|
||||||
for record in records:
|
self._school(conn, record)
|
||||||
self._school(conn, record)
|
|
||||||
|
|
||||||
def _school(self, conn, record):
|
def _school(self, conn, record):
|
||||||
record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))}
|
record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))}
|
||||||
@@ -93,8 +98,11 @@ class Database:
|
|||||||
|
|
||||||
def upsert_scores(self, records):
|
def upsert_scores(self, records):
|
||||||
with self.connection() as conn:
|
with self.connection() as conn:
|
||||||
for row in records:
|
self._upsert_scores(conn, records)
|
||||||
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
|
|
||||||
|
def _upsert_scores(self, conn, records):
|
||||||
|
for row in records:
|
||||||
|
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
|
||||||
|
|
||||||
def _query(self, filters: dict):
|
def _query(self, filters: dict):
|
||||||
conditions, params = ["1=1"], []
|
conditions, params = ["1=1"], []
|
||||||
@@ -141,9 +149,73 @@ class Database:
|
|||||||
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
|
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
|
||||||
|
|
||||||
def all_catalog(self, filters):
|
def all_catalog(self, filters):
|
||||||
|
return list(self.iter_catalog(filters))
|
||||||
|
|
||||||
|
def iter_catalog(self, filters, batch_size=200):
|
||||||
select, params = self._query(filters)
|
select, params = self._query(filters)
|
||||||
with self.connection() as conn:
|
with self.connection() as conn:
|
||||||
return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)]
|
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM catalog').fetchone()[0]
|
||||||
|
after = None
|
||||||
|
while True:
|
||||||
|
query = 'SELECT * FROM (' + select + ') export_rows WHERE id<=?'
|
||||||
|
values = [*params, cutoff]
|
||||||
|
if after:
|
||||||
|
year, school, rid = after
|
||||||
|
if school is None:
|
||||||
|
query += ' AND (year<? OR (year=? AND (school_name IS NOT NULL OR (school_name IS NULL AND id>?))))'
|
||||||
|
values.extend([year, year, rid])
|
||||||
|
else:
|
||||||
|
query += ' AND (year<? OR (year=? AND (school_name>? OR (school_name=? AND id>?))))'
|
||||||
|
values.extend([year, year, school, school, rid])
|
||||||
|
query += ' ORDER BY year DESC,school_name,id LIMIT ?'
|
||||||
|
with self.connection() as conn:
|
||||||
|
rows = conn.execute(query, [*values, batch_size]).fetchall()
|
||||||
|
if not rows:
|
||||||
|
return
|
||||||
|
for row in rows:
|
||||||
|
yield self._catalog_row(row)
|
||||||
|
last = rows[-1]
|
||||||
|
after = (last['year'], last['school_name'], last['id'])
|
||||||
|
|
||||||
|
def iter_scores(self, pairs, batch_size=200):
|
||||||
|
after = 0
|
||||||
|
with self.connection() as conn:
|
||||||
|
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM scores').fetchone()[0]
|
||||||
|
while True:
|
||||||
|
with self.connection() as conn:
|
||||||
|
rows = conn.execute('SELECT id,school_code,major_code,data FROM scores WHERE id>? AND id<=? ORDER BY id LIMIT ?', (after, cutoff, batch_size)).fetchall()
|
||||||
|
if not rows:
|
||||||
|
return
|
||||||
|
for row in rows:
|
||||||
|
if (row['school_code'], row['major_code']) in pairs:
|
||||||
|
yield json.loads(row['data'])
|
||||||
|
after = rows[-1]['id']
|
||||||
|
|
||||||
|
def score_targets(self, filters, batch_size=200):
|
||||||
|
conditions, params = ['year=?'], [filters['year']]
|
||||||
|
if filters.get('major_code'):
|
||||||
|
conditions.append('major_code=?')
|
||||||
|
params.append(filters['major_code'])
|
||||||
|
if filters.get('school_keyword'):
|
||||||
|
conditions.append('school_name LIKE ?')
|
||||||
|
params.append('%' + filters['school_keyword'] + '%')
|
||||||
|
if filters.get('provinces'):
|
||||||
|
conditions.append("(province='' OR province IN (" + ','.join('?' for _ in filters['provinces']) + '))')
|
||||||
|
params.extend(filters['provinces'])
|
||||||
|
after = None
|
||||||
|
while True:
|
||||||
|
where, values = ' AND '.join(conditions), list(params)
|
||||||
|
if after:
|
||||||
|
where += ' AND (school_code>? OR (school_code=? AND major_code>?))'
|
||||||
|
values.extend([after[0], after[0], after[1]])
|
||||||
|
query = 'SELECT school_code,major_code,MAX(school_name) AS school_name,MAX(province) AS province,MAX(level) AS level FROM catalog WHERE ' + where + ' GROUP BY school_code,major_code ORDER BY school_code,major_code LIMIT ?'
|
||||||
|
with self.connection() as conn:
|
||||||
|
rows = conn.execute(query, [*values, batch_size]).fetchall()
|
||||||
|
if not rows:
|
||||||
|
return
|
||||||
|
yield from (dict(row) for row in rows)
|
||||||
|
last = rows[-1]
|
||||||
|
after = (last['school_code'], last['major_code'])
|
||||||
|
|
||||||
def _catalog_row(self, row):
|
def _catalog_row(self, row):
|
||||||
data = json.loads(row["data"])
|
data = json.loads(row["data"])
|
||||||
@@ -219,6 +291,17 @@ class Database:
|
|||||||
with self.connection() as conn:
|
with self.connection() as conn:
|
||||||
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
|
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
|
||||||
|
|
||||||
|
def save_unit(self, jid, unit, records, scores=()):
|
||||||
|
with self.connection() as conn:
|
||||||
|
if not conn.execute('SELECT 1 FROM jobs WHERE id=?', (jid,)).fetchone():
|
||||||
|
raise ValueError('任务不存在')
|
||||||
|
if not conn.execute('INSERT OR IGNORE INTO job_units VALUES(?,?)', (jid, unit)).rowcount:
|
||||||
|
return False
|
||||||
|
self._upsert_scores(conn, scores)
|
||||||
|
self._upsert_catalog(conn, records)
|
||||||
|
conn.execute('UPDATE jobs SET records=records+?,updated_at=? WHERE id=?', (len(records), now(), jid))
|
||||||
|
return True
|
||||||
|
|
||||||
def log(self, jid, level, message):
|
def log(self, jid, level, message):
|
||||||
with self.connection() as conn:
|
with self.connection() as conn:
|
||||||
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
|
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
|
||||||
|
|||||||
@@ -0,0 +1,66 @@
|
|||||||
|
"""Bounded exports, one download at a time, with disconnect cleanup."""
|
||||||
|
import csv
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
|
||||||
|
from fastapi.responses import StreamingResponse
|
||||||
|
from studio.imports import LABELS, spreadsheet_value
|
||||||
|
from studio.memory import release_idle_memory
|
||||||
|
|
||||||
|
|
||||||
|
class ExportStream(StreamingResponse):
|
||||||
|
def __init__(self, source, cleanup, **kwargs):
|
||||||
|
super().__init__(source, **kwargs)
|
||||||
|
self.source, self.cleanup = source, cleanup
|
||||||
|
|
||||||
|
async def __call__(self, scope, receive, send):
|
||||||
|
try:
|
||||||
|
await super().__call__(scope, receive, send)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
self.source.close()
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
self.cleanup()
|
||||||
|
finally:
|
||||||
|
release_idle_memory()
|
||||||
|
|
||||||
|
|
||||||
|
def csv_chunks(records):
|
||||||
|
yield b'\xef\xbb\xbf'
|
||||||
|
stream = io.StringIO(newline='')
|
||||||
|
writer = csv.writer(stream)
|
||||||
|
writer.writerow(list(LABELS.values()))
|
||||||
|
yield stream.getvalue().encode('utf-8')
|
||||||
|
stream.seek(0)
|
||||||
|
stream.truncate(0)
|
||||||
|
for record in records:
|
||||||
|
writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS])
|
||||||
|
if stream.tell() >= 65536:
|
||||||
|
yield stream.getvalue().encode('utf-8')
|
||||||
|
stream.seek(0)
|
||||||
|
stream.truncate(0)
|
||||||
|
if stream.tell():
|
||||||
|
yield stream.getvalue().encode('utf-8')
|
||||||
|
|
||||||
|
|
||||||
|
def json_chunks(database, filters):
|
||||||
|
pairs = set()
|
||||||
|
yield b'{"version":1,"catalog":['
|
||||||
|
first = True
|
||||||
|
for row in database.iter_catalog(filters):
|
||||||
|
pairs.add((row['school_code'], row['major_code']))
|
||||||
|
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
|
||||||
|
first = False
|
||||||
|
yield b'],"scores":['
|
||||||
|
first = True
|
||||||
|
for row in database.iter_scores(pairs):
|
||||||
|
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
|
||||||
|
first = False
|
||||||
|
yield b']}'
|
||||||
|
|
||||||
|
|
||||||
|
def file_chunks(path):
|
||||||
|
with path.open('rb') as stream:
|
||||||
|
while chunk := stream.read(65536):
|
||||||
|
yield chunk
|
||||||
+45
-18
@@ -2,6 +2,7 @@
|
|||||||
import csv
|
import csv
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
|
from contextlib import suppress
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -94,11 +95,18 @@ def import_file(path, year=None):
|
|||||||
book = load_workbook(path, read_only=True, data_only=True)
|
book = load_workbook(path, read_only=True, data_only=True)
|
||||||
try:
|
try:
|
||||||
sheet = book.active
|
sheet = book.active
|
||||||
if sheet.max_row > 300001 or sheet.max_column > 100:
|
if (sheet.max_row or 0) > 300001 or (sheet.max_column or 0) > 100:
|
||||||
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
|
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
|
||||||
values = sheet.iter_rows(values_only=True)
|
values = sheet.iter_rows(values_only=True)
|
||||||
headers = next(values, [])
|
headers = next(values, [])
|
||||||
rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)]
|
if len(headers) > 100:
|
||||||
|
raise ValueError("Excel 超过 100 列,请拆分后导入")
|
||||||
|
rows = []
|
||||||
|
for index, row in enumerate(values, 2):
|
||||||
|
if index > 300001 or len(row) > 100:
|
||||||
|
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
|
||||||
|
if any(v is not None for v in row):
|
||||||
|
rows.append(dict(zip(headers, row)))
|
||||||
finally:
|
finally:
|
||||||
book.close()
|
book.close()
|
||||||
else:
|
else:
|
||||||
@@ -179,28 +187,47 @@ def spreadsheet_value(value):
|
|||||||
|
|
||||||
|
|
||||||
def export_sheet(records, kind):
|
def export_sheet(records, kind):
|
||||||
rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records]
|
|
||||||
if kind == "csv":
|
if kind == "csv":
|
||||||
stream = io.StringIO(newline="")
|
stream = io.StringIO(newline="")
|
||||||
writer = csv.writer(stream)
|
writer = csv.writer(stream)
|
||||||
writer.writerow(list(LABELS.values()))
|
writer.writerow(list(LABELS.values()))
|
||||||
writer.writerows(rows)
|
writer.writerows([spreadsheet_value(record.get(field)) for field in LABELS] for record in records)
|
||||||
return stream.getvalue().encode("utf-8-sig")
|
return stream.getvalue().encode("utf-8-sig")
|
||||||
book = Workbook()
|
output = io.BytesIO()
|
||||||
sheet = book.active
|
write_xlsx(records, output)
|
||||||
sheet.title = "院校专业"
|
return output.getvalue()
|
||||||
sheet.append(list(LABELS.values()))
|
|
||||||
for row in rows:
|
|
||||||
sheet.append(row)
|
def write_xlsx(records, output):
|
||||||
sheet.freeze_panes = "A2"
|
from openpyxl.cell import WriteOnlyCell
|
||||||
sheet.auto_filter.ref = sheet.dimensions
|
from openpyxl.utils import get_column_letter
|
||||||
from openpyxl.styles import Font, PatternFill, Alignment
|
from openpyxl.styles import Font, PatternFill, Alignment
|
||||||
for cell in sheet[1]:
|
book = Workbook(write_only=True)
|
||||||
|
sheet = book.create_sheet("院校专业")
|
||||||
|
sheet.freeze_panes = "A2"
|
||||||
|
header = []
|
||||||
|
for index, label in enumerate(LABELS.values(), 1):
|
||||||
|
cell = WriteOnlyCell(sheet, value=label)
|
||||||
cell.font = Font(color="FFFFFF", bold=True)
|
cell.font = Font(color="FFFFFF", bold=True)
|
||||||
cell.fill = PatternFill("solid", fgColor="183447")
|
cell.fill = PatternFill("solid", fgColor="183447")
|
||||||
cell.alignment = Alignment(vertical="center")
|
cell.alignment = Alignment(vertical="center")
|
||||||
for col in sheet.columns:
|
header.append(cell)
|
||||||
sheet.column_dimensions[col[0].column_letter].width = 24
|
sheet.column_dimensions[get_column_letter(index)].width = 24
|
||||||
output = io.BytesIO()
|
count = 1
|
||||||
book.save(output)
|
try:
|
||||||
return output.getvalue()
|
sheet.append(header)
|
||||||
|
for record in records:
|
||||||
|
sheet.append([spreadsheet_value(record.get(field)) for field in LABELS])
|
||||||
|
count += 1
|
||||||
|
sheet.auto_filter.ref = f'A1:{get_column_letter(len(LABELS))}{count}'
|
||||||
|
book.save(output)
|
||||||
|
except BaseException:
|
||||||
|
with suppress(Exception):
|
||||||
|
if not sheet.closed:
|
||||||
|
sheet.close()
|
||||||
|
with suppress(Exception):
|
||||||
|
if sheet._writer:
|
||||||
|
sheet._writer.cleanup()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
book.close()
|
||||||
|
|||||||
+20
-19
@@ -1,6 +1,7 @@
|
|||||||
"""Persistent units and cooperative pause/cancel for one local worker."""
|
"""Persistent units and cooperative pause/cancel for one local worker."""
|
||||||
import threading
|
import threading
|
||||||
import uuid
|
import uuid
|
||||||
|
from itertools import chain
|
||||||
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
|
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
|
||||||
from studio.domain import PROVINCES, normalize_catalog
|
from studio.domain import PROVINCES, normalize_catalog
|
||||||
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
|
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||||
@@ -230,8 +231,7 @@ class JobManager:
|
|||||||
records = []
|
records = []
|
||||||
for row in client.directions(school, major, params.get("study_mode", "")):
|
for row in client.directions(school, major, params.get("study_mode", "")):
|
||||||
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
|
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
|
||||||
self._save(jid, records)
|
self.db.save_unit(jid, unit, records)
|
||||||
self.complete_unit(jid, unit)
|
|
||||||
completed.add(unit)
|
completed.add(unit)
|
||||||
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
|
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
|
||||||
except RemoteUnavailable as exc:
|
except RemoteUnavailable as exc:
|
||||||
@@ -240,24 +240,25 @@ class JobManager:
|
|||||||
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
|
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
|
||||||
|
|
||||||
def _scores(self, jid, params):
|
def _scores(self, jid, params):
|
||||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
|
provinces = [PROVINCES[p] for p in params['provinces']]
|
||||||
records = self.db.all_catalog(filters)
|
filters = {"year": params["year"], "major_code": params.get("major_code", ""),
|
||||||
targets = {}
|
"school_keyword": params.get("school_keyword", ""), 'provinces': provinces}
|
||||||
for row in records:
|
targets = iter(self.db.score_targets(filters))
|
||||||
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]:
|
first = next(targets, None)
|
||||||
continue
|
if first is None and params.get('major_code'):
|
||||||
targets[(row["school_code"], row["major_code"])] = row
|
targets = ({**row, 'major_code': params['major_code']}
|
||||||
if not targets and params.get("major_code"):
|
for row in self.db.schools(params.get('school_keyword', '')) if row['province'] in provinces)
|
||||||
for row in self.db.schools(params.get("school_keyword", "")):
|
first = next(targets, None)
|
||||||
if row["province"] in [PROVINCES[p] for p in params["provinces"]]:
|
if first is None:
|
||||||
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
|
|
||||||
if not targets:
|
|
||||||
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
|
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
|
||||||
|
completed = self.db.completed_units(jid)
|
||||||
|
self.db.log(jid, 'info', '开始补充分数线和历史计划,分批读取专业目标并复用院校年份分数线')
|
||||||
client = ScoreClient(self.transport)
|
client = ScoreClient(self.transport)
|
||||||
for (_, code), record in targets.items():
|
for record in chain((first,), targets):
|
||||||
self.checkpoint(jid)
|
self.checkpoint(jid)
|
||||||
|
code = record['major_code']
|
||||||
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
|
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
|
||||||
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")]
|
years = [y for y in params["score_years"] if f"scores:{school['school_code']}:{code}:{y}" not in completed]
|
||||||
if not years:
|
if not years:
|
||||||
continue
|
continue
|
||||||
school_id = client.school_id(school["school_name"])
|
school_id = client.school_id(school["school_name"])
|
||||||
@@ -270,10 +271,10 @@ class JobManager:
|
|||||||
self.db.job_update(jid, message=label)
|
self.db.job_update(jid, message=label)
|
||||||
try:
|
try:
|
||||||
scores = client.score_lines(school, code, year, school_id)
|
scores = client.score_lines(school, code, year, school_id)
|
||||||
self.db.upsert_scores(scores)
|
|
||||||
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
|
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
|
||||||
self._save(jid, plans)
|
unit = f"scores:{school['school_code']}:{code}:{year}"
|
||||||
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}")
|
self.db.save_unit(jid, unit, plans, scores=scores)
|
||||||
|
completed.add(unit)
|
||||||
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
|
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
|
||||||
except RemoteUnavailable as exc:
|
except RemoteUnavailable as exc:
|
||||||
self._error(jid, f"{label}:{exc}")
|
self._error(jid, f"{label}:{exc}")
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
"""Best-effort return of idle allocator memory after large operations."""
|
||||||
|
import ctypes
|
||||||
|
import gc
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
def release_idle_memory():
|
||||||
|
gc.collect()
|
||||||
|
if sys.platform == 'linux':
|
||||||
|
try:
|
||||||
|
trim = ctypes.CDLL(None).malloc_trim
|
||||||
|
trim.argtypes = [ctypes.c_size_t]
|
||||||
|
trim.restype = ctypes.c_int
|
||||||
|
return bool(trim(0))
|
||||||
|
except (AttributeError, OSError):
|
||||||
|
pass
|
||||||
|
return False
|
||||||
@@ -107,6 +107,10 @@ class MySQLDatabase(Database):
|
|||||||
with self.connection() as conn:
|
with self.connection() as conn:
|
||||||
for statement in SCHEMA:
|
for statement in SCHEMA:
|
||||||
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
|
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
|
||||||
|
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_export'").fetchone():
|
||||||
|
conn.execute('CREATE INDEX catalog_export ON catalog(year DESC,school_name,id)')
|
||||||
|
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_targets'").fetchone():
|
||||||
|
conn.execute('CREATE INDEX catalog_targets ON catalog(year,school_code,major_code)')
|
||||||
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
|
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
|
||||||
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
|
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
|
||||||
if recover_jobs:
|
if recover_jobs:
|
||||||
@@ -130,7 +134,7 @@ class MySQLDatabase(Database):
|
|||||||
finally:
|
finally:
|
||||||
raw.close()
|
raw.close()
|
||||||
|
|
||||||
def upsert_catalog(self, records):
|
def _upsert_catalog(self, conn, records):
|
||||||
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
|
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
|
||||||
values = []
|
values = []
|
||||||
for record in records:
|
for record in records:
|
||||||
@@ -139,12 +143,11 @@ class MySQLDatabase(Database):
|
|||||||
if not values:
|
if not values:
|
||||||
return
|
return
|
||||||
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
|
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
|
||||||
with self.connection() as conn:
|
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
|
||||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
|
for record in records:
|
||||||
for record in records:
|
self._school(conn, record)
|
||||||
self._school(conn, record)
|
|
||||||
|
|
||||||
def upsert_scores(self, records):
|
def _upsert_scores(self, conn, records):
|
||||||
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
|
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
|
||||||
values = []
|
values = []
|
||||||
for record in records:
|
for record in records:
|
||||||
@@ -152,8 +155,7 @@ class MySQLDatabase(Database):
|
|||||||
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
|
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
|
||||||
values.append([row.get(field) for field in fields])
|
values.append([row.get(field) for field in fields])
|
||||||
if values:
|
if values:
|
||||||
with self.connection() as conn:
|
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
|
||||||
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
|
|
||||||
|
|
||||||
def favorite(self, rid, enabled):
|
def favorite(self, rid, enabled):
|
||||||
# MariaDB reports changed rows; an already-selected favorite is still success.
|
# MariaDB reports changed rows; an already-selected favorite is still success.
|
||||||
|
|||||||
+31
-1
@@ -3,6 +3,7 @@ import threading
|
|||||||
import time
|
import time
|
||||||
import random
|
import random
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
|
from collections import OrderedDict
|
||||||
import requests
|
import requests
|
||||||
from studio.settings import proxy_url
|
from studio.settings import proxy_url
|
||||||
|
|
||||||
@@ -19,6 +20,31 @@ class RateLimited(RuntimeError):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class BoundedProxyAdapter(requests.adapters.HTTPAdapter):
|
||||||
|
"""Close obsolete proxy pools rather than retain every historical exit."""
|
||||||
|
|
||||||
|
def __init__(self, max_proxy_pools=4, **kwargs):
|
||||||
|
super().__init__(**kwargs)
|
||||||
|
self.max_proxy_pools = max(1, int(max_proxy_pools))
|
||||||
|
self.proxy_manager = OrderedDict()
|
||||||
|
|
||||||
|
def proxy_manager_for(self, proxy, **proxy_kwargs):
|
||||||
|
manager = super().proxy_manager_for(proxy, **proxy_kwargs)
|
||||||
|
self.proxy_manager.move_to_end(proxy)
|
||||||
|
while len(self.proxy_manager) > self.max_proxy_pools:
|
||||||
|
_, obsolete = self.proxy_manager.popitem(last=False)
|
||||||
|
obsolete.clear()
|
||||||
|
return manager
|
||||||
|
|
||||||
|
|
||||||
|
def bounded_session():
|
||||||
|
session = requests.Session()
|
||||||
|
session.trust_env = False
|
||||||
|
for scheme in ('http://', 'https://'):
|
||||||
|
session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4))
|
||||||
|
return session
|
||||||
|
|
||||||
|
|
||||||
class ProxyRouter:
|
class ProxyRouter:
|
||||||
def __init__(self, settings, session=None):
|
def __init__(self, settings, session=None):
|
||||||
self.settings = settings
|
self.settings = settings
|
||||||
@@ -141,7 +167,7 @@ class ProxyRouter:
|
|||||||
class Transport:
|
class Transport:
|
||||||
def __init__(self, settings, router, session=None, wait=None):
|
def __init__(self, settings, router, session=None, wait=None):
|
||||||
self.settings, self.router = settings, router
|
self.settings, self.router = settings, router
|
||||||
self.session = session or requests.Session()
|
self.session = session or bounded_session()
|
||||||
self.session.trust_env = False
|
self.session.trust_env = False
|
||||||
self.session.headers = getattr(self.session, "headers", {})
|
self.session.headers = getattr(self.session, "headers", {})
|
||||||
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
|
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
|
||||||
@@ -149,6 +175,10 @@ class Transport:
|
|||||||
self.local = threading.local()
|
self.local = threading.local()
|
||||||
self.lock = threading.RLock()
|
self.lock = threading.RLock()
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
with self.lock:
|
||||||
|
self.session.close()
|
||||||
|
|
||||||
def sleep(self, seconds):
|
def sleep(self, seconds):
|
||||||
remaining = seconds
|
remaining = seconds
|
||||||
while remaining > 0:
|
while remaining > 0:
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import pytest
|
||||||
|
|
||||||
|
from studio.database import Database
|
||||||
|
from studio.domain import finish_record
|
||||||
|
|
||||||
|
|
||||||
|
def record(code='10001', name='甲大学', major='081200', province='北京', year=2027, direction='方向一'):
|
||||||
|
return finish_record({'school_code': code, 'school_name': name, 'major_code': major,
|
||||||
|
'major_name': major, 'province': province, 'year': year,
|
||||||
|
'direction': direction, 'subjects': ['101政治'], 'source': '测试'})
|
||||||
|
|
||||||
|
|
||||||
|
def test_history_targets_deduplicate_directions_and_preserve_filters(tmp_path):
|
||||||
|
db = Database(tmp_path / 'data.db')
|
||||||
|
db.upsert_catalog([
|
||||||
|
record(direction='一'), record(direction='二'), record(major='081201'),
|
||||||
|
record(code='10002', name='乙大学', province='天津'),
|
||||||
|
record(code='10003', name='丙大学', province=''), record(year=2026),
|
||||||
|
])
|
||||||
|
rows = list(db.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
|
||||||
|
assert [(r['school_code'], r['major_code']) for r in rows] == [
|
||||||
|
('10001', '081200'), ('10001', '081201'), ('10003', '081200')]
|
||||||
|
assert all('data' not in row and 'subjects' not in row for row in rows)
|
||||||
|
rows = list(db.score_targets({'year': 2027, 'school_keyword': '甲', 'major_code': '081201'}))
|
||||||
|
assert [(r['school_code'], r['major_code']) for r in rows] == [('10001', '081201')]
|
||||||
|
|
||||||
|
|
||||||
|
def test_catalog_iterator_keeps_export_order_and_excludes_new_inserts(tmp_path):
|
||||||
|
db = Database(tmp_path / 'data.db')
|
||||||
|
db.upsert_catalog([record(name='B大学'), record(name='A大学', direction='二'),
|
||||||
|
record(name='A大学', direction='三'), record(year=2026),
|
||||||
|
record(code='10002', name='C大学')])
|
||||||
|
iterator = db.iter_catalog({}, batch_size=2)
|
||||||
|
first = next(iterator)
|
||||||
|
db.upsert_catalog([record(name='Z大学', year=2028)])
|
||||||
|
rows = [first, *iterator]
|
||||||
|
assert [(r['year'], r['school_name']) for r in rows] == [
|
||||||
|
(2027, 'A大学'), (2027, 'A大学'), (2027, 'B大学'), (2027, 'C大学'), (2026, '甲大学')]
|
||||||
|
assert [r['school_name'] for r in db.iter_catalog({'school_keyword': 'A'}, batch_size=1)] == ['A大学', 'A大学']
|
||||||
|
|
||||||
|
|
||||||
|
def test_score_iterator_exports_only_selected_school_major_pairs(tmp_path):
|
||||||
|
db = Database(tmp_path / 'data.db')
|
||||||
|
db.upsert_scores([
|
||||||
|
{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310},
|
||||||
|
{'school_code': '10002', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 320},
|
||||||
|
])
|
||||||
|
rows = list(db.iter_scores({('10001', '081200')}, batch_size=1))
|
||||||
|
assert len(rows) == 1
|
||||||
|
assert rows[0]['total'] == 310
|
||||||
|
|
||||||
|
|
||||||
|
def test_unit_commit_is_idempotent_for_records_count_and_checkpoint(tmp_path):
|
||||||
|
db = Database(tmp_path / 'data.db')
|
||||||
|
db.job_create('job', {})
|
||||||
|
rows = [record(direction='一'), record(direction='二')]
|
||||||
|
assert db.save_unit('job', 'catalog:test', rows) is True
|
||||||
|
assert db.save_unit('job', 'catalog:test', rows) is False
|
||||||
|
assert db.overview()['records'] == 2
|
||||||
|
assert db.job_get('job')['records'] == 2
|
||||||
|
assert db.unit_done('job', 'catalog:test')
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_unit_transaction_rolls_back_scores_records_and_checkpoint(tmp_path, monkeypatch):
|
||||||
|
db = Database(tmp_path / 'data.db')
|
||||||
|
db.job_create('job', {})
|
||||||
|
def fail(*args):
|
||||||
|
raise RuntimeError('interrupted persistence')
|
||||||
|
monkeypatch.setattr(db, '_school', fail)
|
||||||
|
score = {'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
db.save_unit('job', 'scores:test', [record()], scores=[score])
|
||||||
|
assert db.overview()['records'] == db.overview()['scores'] == 0
|
||||||
|
assert db.job_get('job')['records'] == 0
|
||||||
|
assert not db.unit_done('job', 'scores:test')
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
from studio import network
|
||||||
|
|
||||||
|
|
||||||
|
def test_proxy_managers_are_bounded_and_evicted_pools_are_closed():
|
||||||
|
adapter = network.BoundedProxyAdapter(max_proxy_pools=3)
|
||||||
|
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
|
||||||
|
closed = []
|
||||||
|
original = first.clear
|
||||||
|
def observe():
|
||||||
|
original()
|
||||||
|
closed.append(True)
|
||||||
|
first.clear = observe
|
||||||
|
for port in range(10002, 10006):
|
||||||
|
adapter.proxy_manager_for(f'http://127.0.0.1:{port}')
|
||||||
|
assert len(adapter.proxy_manager) <= 3
|
||||||
|
assert closed == [True]
|
||||||
|
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is not first
|
||||||
|
adapter.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_active_proxy_pool_is_reused_while_old_pools_are_evicted():
|
||||||
|
adapter = network.BoundedProxyAdapter(max_proxy_pools=2)
|
||||||
|
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
|
||||||
|
adapter.proxy_manager_for('http://127.0.0.1:10002')
|
||||||
|
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
|
||||||
|
adapter.proxy_manager_for('http://127.0.0.1:10003')
|
||||||
|
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
|
||||||
|
assert 'http://127.0.0.1:10002' not in adapter.proxy_manager
|
||||||
|
adapter.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_transport_limits_pools_without_changing_route_or_tls(tmp_path):
|
||||||
|
from studio.settings import SettingsStore
|
||||||
|
from tests.test_network import Pool
|
||||||
|
settings = SettingsStore(tmp_path)
|
||||||
|
router = network.ProxyRouter(settings, session=Pool())
|
||||||
|
transport = network.Transport(settings, router)
|
||||||
|
adapter = transport.session.get_adapter('https://example.test')
|
||||||
|
assert isinstance(adapter, network.BoundedProxyAdapter)
|
||||||
|
assert transport.session.verify is True
|
||||||
|
assert router.resolve() == 'http://1.2.3.4:8080'
|
||||||
|
transport.close()
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
import asyncio
|
||||||
|
import csv
|
||||||
|
import io
|
||||||
|
import threading
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from openpyxl import load_workbook
|
||||||
|
|
||||||
|
from studio.api import create_app
|
||||||
|
from tests.test_bounded_storage import record
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('kind', ['csv', 'json', 'xlsx'])
|
||||||
|
def test_export_uses_bounded_rows_and_preserves_content(tmp_path, monkeypatch, kind):
|
||||||
|
app = create_app(tmp_path)
|
||||||
|
app.state.db.upsert_catalog([record(name='=1+1'), record(code='10002', name='含逗号,大学')])
|
||||||
|
app.state.db.upsert_scores([{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}])
|
||||||
|
def forbidden(*args, **kwargs):
|
||||||
|
raise AssertionError('export materializes all catalog rows')
|
||||||
|
monkeypatch.setattr(app.state.db, 'all_catalog', forbidden)
|
||||||
|
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
|
||||||
|
response = client.get('/api/export', params={'format': kind})
|
||||||
|
assert response.status_code == 200
|
||||||
|
if kind == 'json':
|
||||||
|
data = response.json()
|
||||||
|
assert {r['school_name'] for r in data['catalog']} == {'=1+1', '含逗号,大学'}
|
||||||
|
assert data['scores'][0]['total'] == 310
|
||||||
|
elif kind == 'csv':
|
||||||
|
rows = list(csv.reader(io.StringIO(response.content.decode('utf-8-sig'))))
|
||||||
|
assert len(rows) == 3
|
||||||
|
assert any("'=1+1" in row for row in rows)
|
||||||
|
assert any('含逗号,大学' in row for row in rows)
|
||||||
|
else:
|
||||||
|
book = load_workbook(io.BytesIO(response.content), read_only=True)
|
||||||
|
rows = list(book.active.values)
|
||||||
|
assert len(rows) == 3
|
||||||
|
assert any("'=1+1" in row for row in rows)
|
||||||
|
book.close()
|
||||||
|
assert not list((tmp_path / 'exports').glob('*'))
|
||||||
|
|
||||||
|
|
||||||
|
def test_concurrent_export_is_rejected_and_lock_is_released(tmp_path, monkeypatch):
|
||||||
|
app = create_app(tmp_path)
|
||||||
|
entered, release = threading.Event(), threading.Event()
|
||||||
|
original = app.state.db.iter_catalog
|
||||||
|
def slow_rows(*args, **kwargs):
|
||||||
|
if not entered.is_set():
|
||||||
|
entered.set()
|
||||||
|
assert release.wait(5)
|
||||||
|
yield from original(*args, **kwargs)
|
||||||
|
monkeypatch.setattr(app.state.db, 'iter_catalog', slow_rows)
|
||||||
|
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
|
||||||
|
responses = []
|
||||||
|
thread = threading.Thread(target=lambda: responses.append(client.get('/api/export')))
|
||||||
|
thread.start()
|
||||||
|
assert entered.wait(3)
|
||||||
|
blocked = client.get('/api/export', params={'format': 'json'})
|
||||||
|
release.set()
|
||||||
|
thread.join(5)
|
||||||
|
assert blocked.status_code == 429
|
||||||
|
assert responses[0].status_code == 200
|
||||||
|
assert client.get('/api/export').status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
def test_disconnected_stream_closes_source_and_releases_export_lock():
|
||||||
|
from studio.exporting import ExportStream
|
||||||
|
from starlette.requests import ClientDisconnect
|
||||||
|
lock = threading.BoundedSemaphore(1)
|
||||||
|
assert lock.acquire(False)
|
||||||
|
|
||||||
|
|
||||||
|
closed = []
|
||||||
|
def source():
|
||||||
|
try:
|
||||||
|
yield b'data'
|
||||||
|
finally:
|
||||||
|
closed.append(True)
|
||||||
|
async def send(message):
|
||||||
|
if message['type'] == 'http.response.body':
|
||||||
|
raise OSError('client disconnected')
|
||||||
|
async def receive():
|
||||||
|
return {'type': 'http.disconnect'}
|
||||||
|
response = ExportStream(source(), lock.release, media_type='text/csv')
|
||||||
|
with pytest.raises(ClientDisconnect):
|
||||||
|
asyncio.run(response({'type': 'http', 'asgi': {'spec_version': '2.4'}}, receive, send))
|
||||||
|
assert closed == [True]
|
||||||
|
assert lock.acquire(False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_xlsx_packaging_error_removes_xml_tempfile_without_masking_error(tmp_path, monkeypatch):
|
||||||
|
from pathlib import Path
|
||||||
|
from studio.imports import write_xlsx
|
||||||
|
from openpyxl import Workbook
|
||||||
|
paths = []
|
||||||
|
def packaging_failure(book, filename):
|
||||||
|
sheet = book.worksheets[0]
|
||||||
|
sheet.close()
|
||||||
|
paths.append(Path(sheet._writer.out))
|
||||||
|
raise OSError('archive write failed')
|
||||||
|
monkeypatch.setattr(Workbook, 'save', packaging_failure)
|
||||||
|
with pytest.raises(OSError, match='archive write failed'):
|
||||||
|
write_xlsx([record()], tmp_path / 'failed.xlsx')
|
||||||
|
assert paths and not paths[0].exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_export_batch_does_not_finish_a_successful_truncated_download(tmp_path, monkeypatch):
|
||||||
|
app = create_app(tmp_path)
|
||||||
|
def broken_rows(*args, **kwargs):
|
||||||
|
yield record()
|
||||||
|
raise RuntimeError('database batch failed')
|
||||||
|
monkeypatch.setattr(app.state.db, 'iter_catalog', broken_rows)
|
||||||
|
sent = []
|
||||||
|
async def send(message):
|
||||||
|
sent.append(message)
|
||||||
|
async def receive():
|
||||||
|
await asyncio.Event().wait()
|
||||||
|
scope = {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'},
|
||||||
|
'http_version': '1.1', 'method': 'GET', 'scheme': 'http',
|
||||||
|
'path': '/api/export', 'raw_path': b'/api/export', 'root_path': '',
|
||||||
|
'query_string': b'format=csv', 'headers': [(b'host', b'127.0.0.1:8765')],
|
||||||
|
'server': ('127.0.0.1', 8765), 'client': ('127.0.0.1', 12345)}
|
||||||
|
with pytest.raises(RuntimeError, match='database batch failed'):
|
||||||
|
asyncio.run(app(scope, receive, send))
|
||||||
|
assert not any(message['type'] == 'http.response.body' and not message.get('more_body', False) for message in sent)
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
import pytest
|
||||||
|
|
||||||
|
from studio.connectors import ScoreClient
|
||||||
|
from studio.network import RemoteUnavailable
|
||||||
|
from tests.test_network import Response
|
||||||
|
|
||||||
|
|
||||||
|
class HistorySource:
|
||||||
|
def __init__(self, pages):
|
||||||
|
self.pages = pages
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def request(self, method, url, **kwargs):
|
||||||
|
payload = kwargs['data']
|
||||||
|
self.calls.append((url.rsplit('/', 1)[-1], payload.copy()))
|
||||||
|
data = self.pages(payload.get('page', 1))
|
||||||
|
return Response({'code': '0000', 'data': data})
|
||||||
|
|
||||||
|
|
||||||
|
def test_full_score_array_over_page_limit_is_consumed_once():
|
||||||
|
rows = [{'code': '081200', 'total': 310, 'depart_name': '信息学院'}]
|
||||||
|
rows += [{'code': '999999', 'total': 200} for _ in range(197)]
|
||||||
|
source = HistorySource(lambda page: rows)
|
||||||
|
lines = ScoreClient(source).score_lines({'school_code': '10001', 'school_name': '北京大学'}, '081200', 2025, 1)
|
||||||
|
assert len(lines) == 1
|
||||||
|
assert lines[0]['total'] == 310
|
||||||
|
assert [call[1]['page'] for call in source.calls] == [1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_short_wrapped_pages_follow_explicit_total():
|
||||||
|
source = HistorySource(lambda page: {'data': [{'value': page}], 'total': 2})
|
||||||
|
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_short_wrapped_pages_follow_explicit_last_page():
|
||||||
|
source = HistorySource(lambda page: {'list': [{'value': page}], 'last_page': 2})
|
||||||
|
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('data', [None, {}, {'data': None}, {'data': [None]}, {'list': [42]}])
|
||||||
|
def test_invalid_history_data_cannot_become_a_successful_empty_unit(data):
|
||||||
|
with pytest.raises(RemoteUnavailable):
|
||||||
|
list(ScoreClient(HistorySource(lambda page: data)).pages('/example', {}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_full_array_is_a_valid_empty_result():
|
||||||
|
assert list(ScoreClient(HistorySource(lambda page: [])).pages('/example', {})) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_repeated_real_page_still_raises_instead_of_returning_partial_data():
|
||||||
|
source = HistorySource(lambda page: {'data': [{'value': 1}], 'total': 2})
|
||||||
|
with pytest.raises(RemoteUnavailable):
|
||||||
|
list(ScoreClient(source).pages('/example', {}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_page_before_declared_total_is_an_error():
|
||||||
|
source = HistorySource(lambda page: {'data': [{'value': 1}] if page == 1 else [], 'total': 2})
|
||||||
|
with pytest.raises(RemoteUnavailable):
|
||||||
|
list(ScoreClient(source).pages('/example', {}))
|
||||||
|
|
||||||
|
|
||||||
|
def test_two_majors_reuse_one_complete_school_year_score_table():
|
||||||
|
rows = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
|
||||||
|
source = HistorySource(lambda page: rows)
|
||||||
|
client = ScoreClient(source)
|
||||||
|
school = {'school_code': '10001', 'school_name': '北京大学'}
|
||||||
|
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
|
||||||
|
assert client.score_lines(school, '081201', 2025, 1)[0]['total'] == 320
|
||||||
|
assert len(source.calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_score_cache_separates_schools_and_years_and_has_a_bound():
|
||||||
|
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}])
|
||||||
|
client = ScoreClient(source)
|
||||||
|
school = {'school_code': '10001', 'school_name': '北京大学'}
|
||||||
|
for sid in range(20):
|
||||||
|
client.score_lines(school, '081200', 2025, sid)
|
||||||
|
client.score_lines(school, '081200', 2026, 19)
|
||||||
|
before = len(source.calls)
|
||||||
|
client.score_lines(school, '081200', 2026, 19)
|
||||||
|
assert len(source.calls) == before
|
||||||
|
client.score_lines(school, '081200', 2025, 0)
|
||||||
|
assert len(source.calls) == before + 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_lookup_is_cached_for_the_current_client_only():
|
||||||
|
source = HistorySource(lambda page: {'data': [], 'total': 0})
|
||||||
|
client = ScoreClient(source)
|
||||||
|
assert client.school_id('暂未收录大学') is None
|
||||||
|
assert client.school_id('暂未收录大学') is None
|
||||||
|
assert len(source.calls) == 1
|
||||||
|
assert ScoreClient(source).school_id('暂未收录大学') is None
|
||||||
|
assert len(source.calls) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_score_response_is_not_cached():
|
||||||
|
valid = False
|
||||||
|
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}] if valid else {})
|
||||||
|
client = ScoreClient(source)
|
||||||
|
school = {'school_code': '10001', 'school_name': '北京大学'}
|
||||||
|
with pytest.raises(RemoteUnavailable):
|
||||||
|
client.score_lines(school, '081200', 2025, 1)
|
||||||
|
valid = True
|
||||||
|
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
|
||||||
|
|
||||||
|
|
||||||
|
def test_history_runner_uses_thin_targets_and_bulk_checkpoints(tmp_path, monkeypatch):
|
||||||
|
from studio.database import Database
|
||||||
|
from studio.jobs import JobManager
|
||||||
|
from studio.settings import SettingsStore
|
||||||
|
from tests.test_bounded_storage import record
|
||||||
|
|
||||||
|
db = Database(tmp_path / 'data.db')
|
||||||
|
db.upsert_catalog([record(), record(major='081201'), record(direction='二')])
|
||||||
|
params = {'year': 2027, 'provinces': ['11'], 'score_years': [2025, 2026], 'include_plans': True}
|
||||||
|
db.job_create('history', params)
|
||||||
|
|
||||||
|
class Source:
|
||||||
|
def __init__(self):
|
||||||
|
self.calls = []
|
||||||
|
def request(self, method, url, **kwargs):
|
||||||
|
payload = kwargs['data']
|
||||||
|
self.calls.append(url.rsplit('/', 1)[-1])
|
||||||
|
if url.endswith('/schoolList'):
|
||||||
|
data = {'data': [{'school_name': '甲大学', 'school_id': 1}], 'total': 1}
|
||||||
|
elif url.endswith('/schoolScore'):
|
||||||
|
data = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
|
||||||
|
else:
|
||||||
|
data = {'data': [{'special_code': payload['keyword'], 'special_name': payload['keyword'],
|
||||||
|
'plan_id': str(payload['year']) + payload['keyword'], 'recruit_number': 1}], 'total': 1}
|
||||||
|
return Response({'code': '0000', 'data': data})
|
||||||
|
|
||||||
|
def forbidden(*args, **kwargs):
|
||||||
|
raise AssertionError('full catalog decoding or per-unit checkpoint query')
|
||||||
|
monkeypatch.setattr(db, 'all_catalog', forbidden)
|
||||||
|
monkeypatch.setattr(db, 'unit_done', forbidden)
|
||||||
|
source = Source()
|
||||||
|
manager = JobManager(db, SettingsStore(tmp_path), transport=source)
|
||||||
|
manager._scores('history', params)
|
||||||
|
assert db.job_get('history')['completed_units'] == 4
|
||||||
|
assert db.overview()['scores'] == 4
|
||||||
|
assert db.job_get('history')['records'] == 4
|
||||||
|
assert source.calls.count('schoolScore') == 2
|
||||||
|
assert source.calls.count('planListV2') == 4
|
||||||
|
before = len(source.calls)
|
||||||
|
manager._scores('history', params)
|
||||||
|
assert len(source.calls) == before
|
||||||
@@ -98,6 +98,47 @@ def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypa
|
|||||||
assert creations == []
|
assert creations == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_mysql_bounded_catalog_and_history_targets_preserve_filters(mysql):
|
||||||
|
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
|
||||||
|
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
|
||||||
|
scores = select_score_lines([{'code': '081200', 'total': 310, 'depart_name': '计算机学院'}], '081200', '10001', '北京大学', 2026)
|
||||||
|
mysql.upsert_scores(scores)
|
||||||
|
targets = list(mysql.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
|
||||||
|
assert len(targets) == 1
|
||||||
|
assert targets[0]['major_code'] == '081200'
|
||||||
|
assert 'data' not in targets[0]
|
||||||
|
rows = list(mysql.iter_catalog({'year': 2027}, batch_size=1))
|
||||||
|
assert len(rows) == 2
|
||||||
|
assert len(list(mysql.iter_catalog({'year': 2026, 'max_score': 310}, batch_size=1))) == 2
|
||||||
|
assert len(list(mysql.iter_scores({('10001', '081200')}, batch_size=1))) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_mysql_unit_commit_is_atomic_and_idempotent(mysql):
|
||||||
|
mysql.job_create('atomic-job', {})
|
||||||
|
rows = normalize_catalog(raw_catalog(), 2027)
|
||||||
|
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
|
||||||
|
assert mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
|
||||||
|
assert not mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
|
||||||
|
assert mysql.job_get('atomic-job')['records'] == 2
|
||||||
|
assert mysql.job_get('atomic-job')['completed_units'] == 1
|
||||||
|
assert mysql.overview()['records'] == 2
|
||||||
|
assert mysql.overview()['scores'] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_mysql_failed_unit_rolls_back_every_write(mysql, monkeypatch):
|
||||||
|
mysql.job_create('atomic-job', {})
|
||||||
|
def fail(*args):
|
||||||
|
raise RuntimeError('interrupted unit')
|
||||||
|
monkeypatch.setattr(mysql, '_school', fail)
|
||||||
|
rows = normalize_catalog(raw_catalog(), 2027)
|
||||||
|
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
|
||||||
|
assert mysql.job_get('atomic-job')['records'] == 0
|
||||||
|
assert mysql.job_get('atomic-job')['completed_units'] == 0
|
||||||
|
assert mysql.overview()['records'] == mysql.overview()['scores'] == 0
|
||||||
|
|
||||||
|
|
||||||
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
|
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
|
||||||
from studio.migrate import migrate_sqlite
|
from studio.migrate import migrate_sqlite
|
||||||
source = Database(tmp_path / 'snapshot.db')
|
source = Database(tmp_path / 'snapshot.db')
|
||||||
|
|||||||
@@ -0,0 +1,98 @@
|
|||||||
|
import importlib.util
|
||||||
|
import queue
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def maintainer_class():
|
||||||
|
path = Path(__file__).resolve().parents[1] / 'deploy/proxy_scheduler.py'
|
||||||
|
spec = importlib.util.spec_from_file_location('bounded_proxy_scheduler', path)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module.ProxyMaintainer
|
||||||
|
|
||||||
|
|
||||||
|
def test_detection_workers_are_bounded_reused_and_rounds_do_not_overlap():
|
||||||
|
state = {'active': 0, 'peak': 0, 'types': set(), 'mixed': False, 'done': [], 'thread_ids': set()}
|
||||||
|
lock = threading.Lock()
|
||||||
|
class Checker:
|
||||||
|
def __init__(self, work_type, target_queue, name):
|
||||||
|
self.work_type, self.queue = work_type, target_queue
|
||||||
|
def run(self):
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
item = self.queue.get(block=False)
|
||||||
|
except queue.Empty:
|
||||||
|
return
|
||||||
|
with lock:
|
||||||
|
state['active'] += 1
|
||||||
|
state['peak'] = max(state['peak'], state['active'])
|
||||||
|
state['types'].add(self.work_type)
|
||||||
|
state['mixed'] |= len(state['types']) > 1
|
||||||
|
state['thread_ids'].add(threading.get_ident())
|
||||||
|
time.sleep(.01)
|
||||||
|
with lock:
|
||||||
|
state['done'].append(item)
|
||||||
|
state['active'] -= 1
|
||||||
|
if not state['active']:
|
||||||
|
state['types'].clear()
|
||||||
|
self.queue.task_done()
|
||||||
|
maintainer = maintainer_class()(workers=2, checker_class=Checker)
|
||||||
|
first = threading.Thread(target=maintainer.validate, args=('raw', range(10)))
|
||||||
|
second = threading.Thread(target=maintainer.validate, args=('use', range(10, 20)))
|
||||||
|
first.start(); second.start()
|
||||||
|
first.join(4); second.join(4)
|
||||||
|
assert not first.is_alive() and not second.is_alive()
|
||||||
|
before = set(state['thread_ids'])
|
||||||
|
maintainer.validate('raw', range(20, 25))
|
||||||
|
maintainer.close()
|
||||||
|
assert sorted(state['done']) == list(range(25))
|
||||||
|
assert state['peak'] <= 2
|
||||||
|
assert state['mixed'] is False
|
||||||
|
assert state['thread_ids'] == before
|
||||||
|
|
||||||
|
|
||||||
|
def test_stopped_detector_does_not_begin_new_rounds():
|
||||||
|
class Forbidden:
|
||||||
|
def __init__(self, *args):
|
||||||
|
raise AssertionError('detection began after shutdown')
|
||||||
|
maintainer = maintainer_class()(workers=2, checker_class=Forbidden)
|
||||||
|
maintainer.close()
|
||||||
|
maintainer.validate('raw', [1, 2, 3])
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_worker_waits_for_other_workers_before_next_round():
|
||||||
|
slow_started, release, next_round = threading.Event(), threading.Event(), threading.Event()
|
||||||
|
class Checker:
|
||||||
|
def __init__(self, work_type, target_queue, name):
|
||||||
|
self.kind, self.name = work_type, name
|
||||||
|
def run(self):
|
||||||
|
if self.kind == 'raw' and self.name == 'worker_00':
|
||||||
|
assert slow_started.wait(2)
|
||||||
|
raise RuntimeError('validator failed')
|
||||||
|
if self.kind == 'raw':
|
||||||
|
slow_started.set()
|
||||||
|
assert release.wait(3)
|
||||||
|
maintainer = maintainer_class()(workers=2, checker_class=Checker)
|
||||||
|
failures = []
|
||||||
|
def first_round():
|
||||||
|
try:
|
||||||
|
maintainer.validate('raw', [1, 2])
|
||||||
|
except RuntimeError:
|
||||||
|
failures.append(True)
|
||||||
|
first = threading.Thread(target=first_round)
|
||||||
|
first.start()
|
||||||
|
assert slow_started.wait(2)
|
||||||
|
def second_proxies():
|
||||||
|
next_round.set()
|
||||||
|
yield 3
|
||||||
|
second = threading.Thread(target=maintainer.validate, args=('use', second_proxies()))
|
||||||
|
second.start()
|
||||||
|
began_early = next_round.wait(.15)
|
||||||
|
release.set()
|
||||||
|
first.join(3); second.join(3)
|
||||||
|
maintainer.close()
|
||||||
|
assert began_early is False
|
||||||
|
assert next_round.is_set()
|
||||||
|
assert failures == [True]
|
||||||
Reference in New Issue
Block a user