Compare commits

...
32 changed files with 1628 additions and 137 deletions
+8
View File
@@ -4,6 +4,8 @@
当前安装位置:`F:\@Other\kaoyan-studio`。原 YAM、yzw 和 proxy_pool 源码不作修改。
Debian 12 / MariaDB 服务器模式、HTTPS 访问与代理池后台运行见 [服务器部署说明](docs/debian-deployment.md)。服务器适配在 `feat/debian-server` 分支,Windows 继续默认使用 SQLite。设置页支持随机请求间隔上下限及按成功请求次数自动轮换代理。
## 启动
双击 **启动工作台.cmd**,浏览器打开 **http://127.0.0.1:8765**。窗口关闭后后台服务继续运行;双击 **停止工作台.cmd** 结束本项目服务。
@@ -26,6 +28,8 @@
4. 在「采集任务」选择内容、目录年份、地区、专业和院校关键词。建议先按一个地区、一个专业或一所院校验证,再创建全国任务。专业代码可以从内置目录选择,也可直接输入 6 位代码。
5. 查看任务日志。暂停可继续;需要登录、代理或限流解除时,处理后点击恢复;取消保留已有数据。关闭或重启服务会保留可恢复断点。
任务会保存本次采集的院校和专业列表分页,登录恢复或服务重启后复用,并一次加载已完成单元。界面显示当前核对的院校和正在请求的页数。修复前创建的任务首次恢复仍需建立列表缓存;新建任务会获取新列表。研究方向和登录验证保持实时请求,异常分页不会写入缓存。
研招网目录第一页可匿名返回,包括部分研究方向;不能用第一页判断登录成功。工作台访问研招目录的登录入口,将已有学信网 Cookie 同步到目录会话,再验证需要登录的第二页。采集开始/恢复时自动完成此检查;途中遇到登录提示,只同步一次并重试原页,失败后保留断点并提示重新登录。验证码仍需在浏览器人工完成,限流和网络失败会单独提示。登录卡片会持续更新状态;切换网络出口后需要重新验证,原 Cookie 保留。
分页使用接口返回的下一页位置和实际页大小。研招网可能将每页条数限制为 10,程序按实际位置继续,避免请求较大页时遗漏记录。目录采集先核对研招页面配置中的年份,避免将当前目录标为历史年份。
@@ -58,6 +62,10 @@
可将当前筛选结果导出 CSV、Excel 或完整 JSON。表格导出保留院系/方向代码和字段口径,回导不会凭空增加同一记录。JSON 同时保留相关学校专业的历史分数线;凭据不进入导出。CSV/Excel 对可能被表格软件当作公式的文本加保护前缀。
全量导出按 200 条分批读取,CSV/JSON 流式下载,Excel 使用写入模式和临时文件;同一实例一次只执行一个导出,重复请求返回 429。下载结束或中断后清理临时资源。导出期间新增的记录留待下次导出;导出不是事务一致性备份。大于 100 MB 的文件仍受导入上限限制,可按地区等条件分别导出;完整数据库备份按服务器部署说明操作。
历史采集在招生目录阶段完成后开始,按院校和年份复用全校分数线,每次最多缓存 8 份,历史计划继续逐专业获取。接口直接返回的数组按完整结果处理,带分页信息的结果按实际数量继续;不完整的响应不会被写成成功断点。数据、保存计数和完成断点在同一数据库事务中提交。
## 本地数据与开发
所有运行数据位于项目 `.runtime`:`studio.db` 保存目录、分数、收藏、任务和断点;`settings.json` 保存非机密配置;`chsi-cookie.enc` 由当前 Windows 用户的 DPAPI 加密。密码不持久化,接口不返回 Cookie。备份数据库时先停止工作台,再复制 `.runtime`;恢复加密 Cookie 需要原 Windows 用户环境,否则重新登录即可。
+23
View File
@@ -0,0 +1,23 @@
#!/bin/bash
# Preserve existing sites. Prefer a domain certificate; CDN IP origins can use
# the server's existing valid IP certificate when HTTP-01 is unavailable.
set -euo pipefail
root=/opt/kaoyan-studio-deploy/app
test -f /etc/nginx/kaoyan-studio.htpasswd
python3 - <<'PY'
from pathlib import Path
root = Path('/opt/kaoyan-studio-deploy/app/deploy')
text = (root / 'nginx-http-bootstrap.conf').read_text() + '\n' + (root / 'nginx-https.conf').read_text()
cert = Path('/etc/letsencrypt/live/kaoyan.fjayson.com/fullchain.pem')
if not cert.exists():
replacement = '/etc/letsencrypt/live/186.241.68.230/'
if not Path(replacement, 'fullchain.pem').exists():
raise SystemExit('Install a valid origin certificate before configuring HTTPS')
text = text.replace('/etc/letsencrypt/live/kaoyan.fjayson.com/', replacement)
if Path('/etc/letsencrypt/live/186.241.68.230/fullchain.pem').exists():
text += '\n' + (root / 'nginx-ip-origin.conf').read_text()
Path('/etc/nginx/sites-available/kaoyan-studio').write_text(text)
PY
ln -sfn /etc/nginx/sites-available/kaoyan-studio /etc/nginx/sites-enabled/kaoyan-studio
nginx -t
systemctl reload nginx
+4 -1
View File
@@ -8,9 +8,12 @@ Requires=docker.service
User=kaoyan
Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py schedule
Environment=MALLOC_ARENA_MAX=2
Environment=PROXY_CHECK_WORKERS=8
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python /opt/kaoyan-studio-deploy/app/deploy/proxy_scheduler.py
Restart=on-failure
RestartSec=10
TimeoutStopSec=120
UMask=0077
PrivateTmp=true
ProtectSystem=full
+1
View File
@@ -9,6 +9,7 @@ User=kaoyan
Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/app
EnvironmentFile=/etc/kaoyan-studio.env
Environment=MALLOC_ARENA_MAX=2
ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio
Restart=on-failure
RestartSec=10
+5
View File
@@ -8,6 +8,11 @@ server {
client_max_body_size 100m;
auth_basic "Kaoyan Workbench";
auth_basic_user_file /etc/nginx/kaoyan-studio.htpasswd;
location ^~ /.well-known/acme-challenge/ {
root /var/www/kaoyan-acme;
auth_basic off;
add_header Cache-Control no-store;
}
location / {
proxy_pass http://127.0.0.1:8765;
proxy_set_header Host $http_host;
+25
View File
@@ -0,0 +1,25 @@
# An authenticated HTTPS entry while the CDN domain is being configured.
# Uses the server's separately renewed IP certificate and an unused port.
server {
listen 9443 ssl;
listen [::]:9443 ssl;
server_name 186.241.68.230;
ssl_certificate /etc/letsencrypt/live/186.241.68.230/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/186.241.68.230/privkey.pem;
ssl_protocols TLSv1.2 TLSv1.3;
client_max_body_size 100m;
auth_basic "Kaoyan Workbench";
auth_basic_user_file /etc/nginx/kaoyan-studio.htpasswd;
location / {
proxy_pass http://127.0.0.1:8765;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_read_timeout 120s;
proxy_hide_header Cache-Control;
add_header Cache-Control "private, no-store" always;
add_header X-Content-Type-Options nosniff always;
add_header X-Frame-Options DENY always;
}
}
+122
View File
@@ -0,0 +1,122 @@
"""Run upstream proxy_pool checks with reusable workers and serialized jobs."""
import os
import queue
import signal
import sys
import threading
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
APP_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(APP_ROOT))
from studio.memory import release_idle_memory
class StopQueue(queue.Queue):
def __init__(self, stopping):
super().__init__()
self.stopping = stopping
def get(self, *args, **kwargs):
if self.stopping.is_set():
raise queue.Empty
return super().get(*args, **kwargs)
class ProxyMaintainer:
def __init__(self, workers=8, checker_class=None, fetcher_factory=None, handler_factory=None, pool_min=20):
self.worker_count = max(1, min(20, int(workers)))
self.checker_class = checker_class
self.fetcher_factory, self.handler_factory = fetcher_factory, handler_factory
self.pool_min = pool_min
self.stopping = threading.Event()
self.pipeline = threading.RLock()
self.workers = ThreadPoolExecutor(max_workers=self.worker_count, thread_name_prefix='proxy-check')
def validate(self, work_type, proxies):
with self.pipeline:
if self.stopping.is_set():
return
target = StopQueue(self.stopping)
for proxy in proxies:
if self.stopping.is_set():
return
target.put(proxy)
if self.checker_class is None:
from helper.check import _ThreadChecker
self.checker_class = _ThreadChecker
count = min(self.worker_count, target.qsize())
futures = [self.workers.submit(self.checker_class(work_type, target, f'worker_{i:02}').run)
for i in range(count)]
try:
failure = None
for future in futures:
try:
future.result()
except Exception as exc:
if failure is None:
failure = exc
if failure is not None:
raise failure
finally:
release_idle_memory()
def fetch(self):
with self.pipeline:
if not self.stopping.is_set():
self.validate('raw', self.fetcher_factory().run())
def check(self):
with self.pipeline:
if self.stopping.is_set():
return
handler = self.handler_factory()
if handler.db.getCount().get('total', 0) < self.pool_min:
self.fetch()
self.validate('use', handler.getAll())
def stop(self):
self.stopping.set()
def close(self):
self.stop()
with self.pipeline:
self.workers.shutdown(wait=True, cancel_futures=True)
def main():
proxy_root = Path(os.environ.get('STUDIO_PROXY_POOL_ROOT', APP_ROOT.parent / 'proxy_pool'))
sys.path.insert(0, str(proxy_root))
from apscheduler.schedulers.blocking import BlockingScheduler
from helper.check import _ThreadChecker
from helper.fetch import Fetcher
from handler.configHandler import ConfigHandler
from handler.logHandler import LogHandler
from handler.proxyHandler import ProxyHandler
config = ConfigHandler()
maintainer = ProxyMaintainer(workers=int(os.environ.get('PROXY_CHECK_WORKERS', '8')),
checker_class=_ThreadChecker, fetcher_factory=Fetcher,
handler_factory=ProxyHandler, pool_min=config.poolSizeMin)
scheduler = BlockingScheduler(timezone=config.timezone, logger=LogHandler('scheduler'),
executors={'default': {'type': 'threadpool', 'max_workers': 1}},
job_defaults={'coalesce': True, 'max_instances': 1, 'misfire_grace_time': 120})
scheduler.add_job(maintainer.fetch, 'interval', minutes=5, id='proxy_fetch')
scheduler.add_job(maintainer.check, 'interval', minutes=2, id='proxy_check',
next_run_time=datetime.now(timezone.utc))
def stop(signum, frame):
maintainer.stop()
if scheduler.running:
scheduler.shutdown(wait=False)
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
if not maintainer.stopping.is_set():
scheduler.start()
finally:
maintainer.close()
if __name__ == '__main__':
main()
+22
View File
@@ -0,0 +1,22 @@
# 目录任务恢复与进度修复
2026-10-11,`feat/debian-server`。
恢复全国目录任务时,已完成单元虽然不会重复保存,但院校、专业列表会从第一地区重新请求。每个已完成专业还会创建 MariaDB 连接查询断点。用户因此看到记录数和日志长时间不变。
服务器实测原短连接加断点查询平均 0.135 秒;复用已协商的 TLS 上下文后约 0.017 秒。一次读取 4,284 个断点约 0.034 秒。
修复包括:
- 一次加载该任务已完成单元,用集合核对专业断点。
- 在 `job_catalog_pages` 中按任务和请求参数保存院校、专业列表的有效分页;恢复时复用。研究方向、目录年份及登录权限验证保持实时请求。
- 先验证必需代码、分页位置和重复页面,再缓存、返回记录。失败响应不缓存,源站恢复后可以重新请求。
- 更新当前院校、列表/方向页码和断点扫描日志;遍历缓存时仍检查暂停和取消。
- MariaDB 已协商 TLS 时复用相同的 TLS 上下文。数据库事务和短连接的关闭方式保持原有行为。
- SQLite 和 MariaDB 自动创建缓存表;迁移同时兼容没有该表的旧快照。
旧任务首次使用修复仍需获取并保存列表分页。缓存限定于该任务;需要获取更新后的院校/专业列表时新建任务。
验证:Windows 后端 87 passed、9 skipped;Debian 后端 82 passed、14 skipped,包含独立 MariaDB 测试库和 TLS 测试。跳过项为平台或未配置独立数据库的测试。两平台均有既有 Starlette TestClient 的 httpx 弃用提示,不影响生产采集。
11 项新增目录恢复回归覆盖跨服务重启复用分页、批量断点、扫描进度、任务隔离、无效分页、重复页面,以及缺失/空/错误类型的院校与专业代码。独立只读审查未发现剩余 Critical 或 Important 问题。
+75
View File
@@ -0,0 +1,75 @@
# Debian 12 服务器部署
服务器:186.241.68.230;域名:https://kaoyan.fjayson.com;备用入口:https://186.241.68.230:9443。部署分支 `feat/debian-server`,应用监听回环 8765,Nginx 提供访问认证;域名通过用户的 CDN 回源。备用入口使用有效 IP 证书,与域名共用访问账号。账号密码单独交付,不放入 Git。浏览器第一次访问会提示输入工作台访问账号,这与学信网账号不同。
## 目录与服务
- `/opt/kaoyan-studio-deploy/app`:代码与构建网页。
- `/opt/kaoyan-studio-deploy/venv`:Python 3.11 虚拟环境。
- `/var/lib/kaoyan-studio`:设置、加密 Cookie 与权限 0600 的加密密钥。
- `/opt/kaoyan-studio-deploy/mysql.json`:MariaDB 私密连接配置,root:kaoyan 权限 0640。
- `/opt/kaoyan-studio-deploy/proxy_pool` 与 `proxy-venv`:独立原版代理池。
- MariaDB 数据库 `kaoyan_studio`;数据、收藏、历史分数、任务、日志与断点均写入此库。
- `kaoyan-studio.service`:工作台,单进程,异常退出自动重启,已设置开机启动。
- `kaoyan-proxy-api.service`、`kaoyan-proxy-scheduler.service`:代理 API 与定时抓取检测,开机启动。
- `kaoyan-proxy-redis`:独立 Redis 容器,只映射 `127.0.0.1:6381`,不操作原服务器的 Redis。
安装模板在 `deploy/`。Python 依赖按 `requirements.lock` 安装;前端 `npm ci && npm run build`。不要启动多个工作台进程连接同一数据库,否则任务恢复状态会互相影响。原有站点、数据库和 fail2ban 配置不作修改。
代理调度器运行本项目 `deploy/proxy_scheduler.py` 包装器,继续使用原版代理池的获取源、检测逻辑和 Redis 数据,不修改上游源码。默认复用 8 个检测线程,抓取与检查轮次串行运行,避免定时任务重叠。工作台和调度器设定 `MALLOC_ARENA_MAX=2`;每次导出/检查结束尝试释放分配器空闲内存。该操作不释放仍在使用的数据,实际内存以运行监测为准。
## 操作
```bash
systemctl status kaoyan-studio kaoyan-proxy-api kaoyan-proxy-scheduler
journalctl -u kaoyan-studio -f
systemctl restart kaoyan-studio
```
后台采集与浏览器连接无关,关掉电脑或网页后服务器继续运行。重启仅恢复被服务停止打断的运行任务;手动暂停、取消和等待登录的任务保持原状态。仍可能遇到研招验证码、登录到期、源站限流或免费代理不可用,需要到页面处理后恢复。
默认每次请求随机等待 1.2–2.4 秒,每 100 次成功请求轮换代理。设置页可修改上下限和轮换次数;0 表示关闭自动轮换。代理页显示实际成功请求路由、计数和切换次数。登录表单、会话同步、权限验证及紧随其后的失败页重试保持同一路由。免费代理池初次启动可能没有 HTTPS 候选,部署时保留了原电脑代理池的候选作为备用;这些候选仍会经过实际请求与失败隔离。
当前全国任务采用 0 次自动轮换,维持通过登录验证的出口;连接失败仍会切换。Cookie 验证状态与出口关联,出口变化后需要重新验证,不宜为了让 IP 显示变化而频繁轮换。HTTP 会话最多保留 4 个代理连接池,淘汰旧池时主动关闭连接。
Linux 无桌面环境,界面提供研招网登录入口与粘贴 Cookie。请在自己的电脑浏览器打开研招目录并登录,从开发者工具中复制目录接口请求的单行 Cookie,粘贴到工作台后点验证。服务器重新同步会话并检查受保护的第二页。迁移 Windows Cookie 时只经 SSH 传输加密封装,在服务器重新加密;不继承原验证标记。
## 迁移与验证
对原 SQLite 使用在线 `backup()` 取得一致性快照,源库只读,原任务保持原状态。使用以下命令导入一个空 MySQL 库:
```bash
python -m studio.migrate /path/to/studio-snapshot.db --config /path/to/private-mysql.json
```
目标非空时拒绝覆盖,并保留已有任务状态;导入失败回滚整批写入。CLI 不执行应用启动的任务恢复。记录 ID、收藏、任务参数、日志 ID 与完成单元均保留,迁移来的任务暂停,验证登录后由用户恢复。
正式快照:27,901 条目录、0 条历史线、559 条院校索引、4 个任务、4,302 个断点、4,319 条日志;当前全国任务含 4,282 个完成单元。数据仍处于 2027 招生目录阶段,历史数据不会因部署自动出现。逐表内容校验结果位于服务器 `/opt/kaoyan-studio-deploy/migration-verification.json`。
目录、历史线、任务、断点和日志逐表内容一致。应用启动载入内置院校索引时更新了两个索引名称:82605 更新为“冶金自动化研究设计院”,83401 更新为“中国石油勘探开发研究院”;目录记录中的原始名称与内容均保留,因此运行后的院校索引全表哈希不同。迁移 Cookie 的实际第二页验证返回要求登录,必须在页面更新 Cookie 后再恢复原暂停任务。
## HTTPS 与 CDN
CDN 回源地址为该服务器,应用保持原始 Host `kaoyan.fjayson.com`,使用 HTTPS 443 回源,并转发 Authorization、请求方法与正文。禁止缓存工作台 HTML/API,源站已返回 `Cache-Control: private, no-store`。域名的浏览器证书由 CDN 提供;源站优先使用域名证书,也可使用当前有效的 IP 证书用于 CDN IP 回源。
部署时 HTTP-01 的本地回源探测成功,但证书机构访问另一 CDN 节点收到 404,因此没有替换服务器现有证书。`configure-nginx.sh` 优先选择域名证书,否则保留现有源站 IP 证书,并提供独立 9443 入口。可通过 DNS-01 或修正全部 CDN 节点的挑战路径回源后申请域名证书,再重新运行配置脚本。
源站 IP 证书使用服务器原有 `/opt/certbot-venv/bin/certbot` 5.8.0 维护,`certbot.timer` 已启用,续期后自动 reload Nginx。2026-10-11 对该 IP 证书执行 `renew --cert-name 186.241.68.230 --dry-run --no-random-sleep-on-renew` 成功。不要误用 `/usr/bin/certbot` 2.1.0 维护此 IP 证书。[Let's Encrypt 的 IP 证书说明](https://letsencrypt.org/2026/03/11/shorter-certs-certbot/)要求受支持的新版客户端。
HTTPS 域名若重定向到完全相同的网址,通常是 CDN 正在用 HTTP 回源源站的强制 HTTPS 入口。请改为 HTTPS 443 回源,保持请求 Host 与 TLS SNI 为 `kaoyan.fjayson.com`。源站当前证书的名称是 IP,供应商须支持按源站 IP 校验;若仅支持域名名称严格校验,则先取得域名证书。域名配置完成前可直接使用 9443 备用入口。
## 备份和升级
```bash
# root 使用 MariaDB 原有本机 socket 认证,备份不在命令行放密码
umask 077
mkdir -p /var/backups/kaoyan-studio
mariadb-dump --single-transaction --default-character-set=utf8mb4 kaoyan_studio > /var/backups/kaoyan-studio/database.sql
cp -a /var/lib/kaoyan-studio /var/backups/kaoyan-studio/runtime
```
同时保存数据库、运行目录(含 `cookie.key`)、私密 MySQL 配置和访问认证文件。不要把这些文件上传 Git 或公开网盘。升级前备份,停止工作台,拉取该部署分支并构建前端、安装锁定依赖,再启动工作台。原 Windows 项目继续保留 SQLite 模式,可作为原始备份。
代理池使用 [jhao104/proxy_pool](https://github.com/jhao104/proxy_pool),数据库适配参考 [PyMySQL 官方文档](https://pymysql.readthedocs.io/en/latest/user/examples.html) 和 [MariaDB 更新冲突记录的语法](https://mariadb.com/docs/server/reference/sql-statements/data-manipulation/inserting-loading-data/insert-on-duplicate-key-update)。
2026-10-11 CDN 改为 HTTPS 回源后,正式域名已通过实际认证 API 和浏览器验收,不再循环跳转。目录查询 27,901 条、同来源 POST 状态检查成功,跨来源修改拒绝,网页和 API 返回 no-store。备用 9443 入口保留。学信网 Cookie 仍需更新,原全国任务仍保持手动暂停。
+41
View File
@@ -0,0 +1,41 @@
# 服务器验收记录
日期:2026-10-11。代码:`feat/debian-server`;运行版本包含 `fcdf6f2`。这份记录只列公开运行状态,不包含访问密码、数据库密码、Cookie 或密钥。
## 自动检查
| 检查 | 结果 |
| --- | --- |
| Windows 后端完整测试 | 76 通过,6 个需要实际 MariaDB 的测试跳过 |
| Debian Python 3.11 后端完整测试 | 68 通过,14 个 Windows 平台测试跳过;包含 6 个实际 MariaDB 集成测试 |
| 前端测试 | Windows、Debian 均 3 通过 |
| TypeScript 与 Vite 生产构建 | 两端均通过,服务器实际使用构建后的网页 |
| 独立全分支审查 | 登录验证后的重试固定出口、账号登录固定出口、迁移拒绝不修改旧任务三项问题均已修复并通过回归验证 |
| `nginx -t` | 全服务器配置通过 |
后端有一条既有 Starlette 弃用提示。真实 MariaDB 测试使用独立 `kaoyan_studio_test` 数据库,未把测试记录写入正式库。
## 实际服务与数据
- 工作台、代理 API、抓取调度器均为 `active`、`enabled`;独立 Redis 容器运行且设置 `unless-stopped`。
- MariaDB 10.11.18,正式数据库 `kaoyan_studio`;HTTP 健康接口报告 `database=mysql`、`platform=linux`。
- 从本机在线 SQLite 快照迁移目录 27,901 条、历史线 0 条、院校索引 559 条、任务 4 个、完成单元 4,302 条、日志 4,319 条。
- 目录、历史线、任务、完成单元、日志与原 SQLite 内容相同;启动时两个院校索引名称被内置种子刷新,详见部署说明。目录的原始名称不变。
- 原全国任务 `19f9b363a1dd4f2b8a384d84351b1b61` 保持手动暂停,记录 27,901、完成单元 4,282;服务器重启后仍保持暂停。本机原任务也保持暂停。
- 登录 Cookie 在服务器以权限 0600 的密钥重新加密保存。实际第二页验证返回 `login_required`,界面据实显示需重新登录;不会继承 Windows 已验证标记。
- 请求间隔上下限 1.2、2.4 秒;自动代理轮换阈值 100 次成功请求。代理池启动时已有 56 个 HTTP 候选、没有 HTTPS 候选;工作台保留原电脑的 24 个 HTTPS 备用出口,实际验证请求通过其中的代理完成。
## HTTPS 与浏览器
- `https://186.241.68.230:9443` 从用户电脑可访问;使用系统信任的有效 IP 证书,API 验收和浏览器验收均未忽略证书错误。
- 未认证请求返回 401;提供工作台账号后目录查询、详情、任务、设置、代理服务接口均返回正常结果。
- 真实跨来源修改返回 403;同来源保存原设置返回 200。
- 通过页面使用的管理 API 停止和启动代理服务成功,只操作本项目 API 与调度器,独立 Redis 保持运行。
- Edge 无界面验收:数据概览、实际目录表格、Linux 登录提示、代理池状态、手机宽度正常,没有浏览器脚本异常。
- Cookie 说明、顶部运行位置与数据保存位置显示为服务器;Linux 页面不显示打开服务器 Edge 窗口的按钮。
- IP 证书的续期模拟成功,现有 `certbot.timer` 调用 Certbot 5.8.0,续期后 reload Nginx。
- 用户改为 HTTPS 回源后,`https://kaoyan.fjayson.com` 不再重定向循环;未认证返回 401,认证后健康接口、目录、任务和代理服务接口均返回 200,页面验收通过。同来源 POST 状态检查成功,跨来源修改返回 403,响应包含 `Cache-Control: private, no-store`。DNS 指向 CDN 节点本身不影响部署。
## 私密备份
服务器 `/var/backups/kaoyan-studio/` 权限 0700,数据库备份权限 0600;保存数据库、含 Cookie 加密密钥的运行目录、私密数据库配置和 Nginx 认证文件。原 SQLite 快照保留在部署目录的 `incoming/`。访问账号单独交付到用户电脑。
@@ -30,34 +30,38 @@
Files: studio/settings.py, studio/network.py, studio/auth.py, tests/test_server_features.py.
- [ ] Add failing tests for varied bounded delays, invalid bounds, automatic route changes, and pinned verification.
- [ ] Implement interval maximum and proxy rotation settings, actual request route/count reporting, and route pinning during auth verification.
- [ ] Run network and auth tests, then commit.
- [x] Add failing tests for varied bounded delays, invalid bounds, automatic route changes, and pinned verification.
- [x] Implement interval maximum and proxy rotation settings, actual request route/count reporting, and route pinning during auth verification.
- [x] Run network and auth tests, then commit.
### Task 2: MariaDB backend and migration
Files: studio/mysql_database.py, studio/migrate.py, studio/api.py, tests/test_mysql.py, pyproject.toml.
- [ ] Write tests against a real isolated MariaDB database for upserts, historical scopes, favorites, jobs, and atomic migration.
- [ ] Implement MySQLDatabase.connection(), preserving the existing Database API; load private database configuration through STUDIO_DATABASE_CONFIG.
- [ ] Implement migrate_sqlite(source, target) preserving IDs, logs and unit keys and rejecting nonempty targets.
- [ ] Verify all existing tests and real MariaDB integration tests, then commit.
- [x] Write tests against a real isolated MariaDB database for upserts, historical scopes, favorites, jobs, and atomic migration.
- [x] Implement MySQLDatabase.connection(), preserving the existing Database API; load private database configuration through STUDIO_DATABASE_CONFIG.
- [x] Implement migrate_sqlite(source, target) preserving IDs, logs and unit keys and rejecting nonempty targets.
- [x] Verify all existing tests and real MariaDB integration tests, then commit.
### Task 3: Linux operation and user interface
Files: studio/auth.py, studio/linux_pool.py, studio/pool_service.py, studio/__main__.py, studio/api.py, frontend/src/App.tsx, frontend/src/types.ts.
- [ ] Add failing tests for Linux vault roundtrip/permissions, server host filtering, and interrupted-task recovery selection.
- [ ] Implement encrypted Linux storage, explicit manual-cookie login instructions, Linux systemd pool controls, configured hosts/data directory, and opt-in restart resumption.
- [ ] Add interval range, rotation settings and counters to the UI; use capability metadata for platform instructions.
- [ ] Run full backend tests and frontend tests/build, then commit.
- [x] Add failing tests for Linux vault roundtrip/permissions, server host filtering, and interrupted-task recovery selection.
- [x] Implement encrypted Linux storage, explicit manual-cookie login instructions, Linux systemd pool controls, configured hosts/data directory, and opt-in restart resumption.
- [x] Add interval range, rotation settings and counters to the UI; use capability metadata for platform instructions.
- [x] Run full backend tests and frontend tests/build, then commit.
### Task 4: Provision, migrate, and deploy
Files: deploy/*, docs/debian-deployment.md, README.md.
- [ ] Prepare reproducible systemd, Nginx and private configuration templates with rollback and backup instructions.
- [ ] Provision independent MariaDB account/database, dedicated loopback Redis, proxy pool and workbench services.
- [ ] Transfer online SQLite backup and credentials through SSH without exposing secrets; verify migrated data and server login.
- [ ] Validate authenticated HTTPS, browser workflows, proxy API/scheduler and restart persistence; resume migrated task only after valid authentication.
- [ ] Fresh reviewer checks the whole branch; fix material findings and verify again. Publish the deployment branch to the user's repository.
- [x] Prepare reproducible systemd, Nginx and private configuration templates with rollback and backup instructions.
- [x] Provision independent MariaDB account/database, dedicated loopback Redis, proxy pool and workbench services.
- [x] Transfer online SQLite backup and credentials through SSH without exposing secrets; verify migrated data and server login.
- [x] Validate authenticated IP HTTPS, browser workflows, proxy API/scheduler and restart persistence; preserve the manually paused migrated task and report its actual login state.
- [x] Fresh reviewer checks the whole branch; fix material findings and verify again. Publish the deployment branch to the user's repository.
### External follow-up
- [x] User changed CDN origin to HTTPS; authenticated public domain access, browser pages, same-origin POST and cross-origin rejection all verified. Authenticated IP HTTPS access remains available.
+13 -9
View File
@@ -195,6 +195,7 @@ export default function App() {
const [tab, setTab] = useState<Tab>("overview");
const [overview, setOverview] = useState<Overview | null>(null);
const [majors, setMajors] = useState<Major[]>([]);
const [serverMode, setServerMode] = useState(false);
const [version, setVersion] = useState(0);
const [notice, setNotice] = useState<{
message: string;
@@ -210,10 +211,12 @@ export default function App() {
Promise.all([
api<Overview>("/overview", { signal: controller.signal }),
api<Major[]>("/majors", { signal: controller.signal }),
api<{ platform: string }>("/health", { signal: controller.signal }),
])
.then(([o, m]) => {
.then(([o, m, health]) => {
setOverview(o);
setMajors(m);
setServerMode(health.platform === "linux");
})
.catch((e) => {
if (e.name !== "AbortError") notify(e.message, true);
@@ -247,7 +250,7 @@ export default function App() {
<small>考研数据工作台</small>
</span>
</a>
<div className="nav-label">WORKSPACE / 本地工作区</div>
<div className="nav-label">WORKSPACE / {serverMode ? "服务器工作区" : "本地工作区"}</div>
<nav aria-label="主导航">
{NAV.map((n) => (
<button
@@ -263,7 +266,7 @@ export default function App() {
</nav>
<div className="sidebar-bottom">
<div className="local-indicator">
<span /> 本机数据 · 独立保存
<span /> {serverMode ? "服务器数据 · 持续保存" : "本机数据 · 独立保存"}
</div>
<p>
把全国目录与历年考情
@@ -283,7 +286,7 @@ export default function App() {
</div>
<div className="topbar-right">
<span className="local-tag">
<ShieldCheck size={14} /> 本地运行
<ShieldCheck size={14} /> {serverMode ? "服务器运行" : "本地运行"}
</span>
<button
className="icon-button"
@@ -303,7 +306,7 @@ export default function App() {
</main>
<footer>
研途 · 数据以院校当年招生简章为准
<span>全国招生目录 / 历年考情 / 本地工作区</span>
<span>全国招生目录 / 历年考情 / {serverMode ? "服务器工作区" : "本地工作区"}</span>
</footer>
</div>
{notice && (
@@ -1754,7 +1757,7 @@ function ProxyPanel({ notify }: { notify: Notify }) {
</section>
<section className="card pool-controls">
<div className="card-title">
<h2>本机代理池服务</h2>
<h2>{services?.manager === "systemd" ? "服务器代理池服务" : "本机代理池服务"}</h2>
<Badge tone={services?.api_ready ? "green" : ""}>
{services
? services.api_ready
@@ -1763,7 +1766,7 @@ function ProxyPanel({ notify }: { notify: Notify }) {
: "正在查询"}
</Badge>
</div>
<p>查看 API、代理抓取与检测、Redis 的运行状态,管理本机服务。</p>
<p>查看 API、代理抓取与检测、Redis 的运行状态,管理{services?.manager === "systemd" ? "服务器" : "本机"}服务。</p>
{services && (
<ul className="pool-service-list">
{services.items.map((service) => (
@@ -2154,8 +2157,9 @@ function SettingsPanel({ notify }: { notify: Notify }) {
</button>
</div>
<div className="info-box">
Cookie 由当前 Windows 用户的 DPAPI
加密,保存在项目的本地数据目录中,导出文件不包含凭据。
{auth?.browser_supported === false
? "Cookie 加密保存在服务器的数据目录中,导出文件不包含凭据。"
: "Cookie 由当前 Windows 用户的 DPAPI 加密,保存在项目的本地数据目录中,导出文件不包含凭据。"}
</div>
</section>
</div>
+1
View File
@@ -113,6 +113,7 @@ export type Proxy = {
proxies: { url: string; failed: boolean; active: boolean }[];
};
export type PoolServiceStatus = {
manager?: string;
api_ready: boolean;
items: {
name: string;
+64 -24
View File
@@ -2,6 +2,8 @@
import json
import uuid
import os
import tempfile
import threading
from contextlib import asynccontextmanager
from pathlib import Path
from urllib.parse import urlparse
@@ -11,7 +13,9 @@ from fastapi.staticfiles import StaticFiles
from studio.auth import AuthService, CookieVault
from studio.database import Database
from studio.domain import PROVINCES
from studio.imports import import_file, export_sheet
from studio.imports import import_file, write_xlsx
from studio.exporting import ExportStream, csv_chunks, json_chunks, file_chunks
from studio.memory import release_idle_memory
from studio.jobs import JobManager
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
from studio.settings import SettingsStore
@@ -20,6 +24,31 @@ from studio.pool_service import PoolService
ROOT = Path(__file__).resolve().parent.parent
class LocalAccessMiddleware:
"""Check hosts and origins without buffering or wrapping response streams."""
def __init__(self, app, allowed_hosts):
self.app = app
self.allowed_hosts = allowed_hosts
async def __call__(self, scope, receive, send):
if scope['type'] != 'http':
return await self.app(scope, receive, send)
request = Request(scope)
if request.url.hostname not in self.allowed_hosts:
response = JSONResponse({'detail': '访问主机不在允许列表中'}, status_code=403)
return await response(scope, receive, send)
if request.method not in ('GET', 'HEAD', 'OPTIONS'):
origin = request.headers.get('origin')
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip('/'), 'http://127.0.0.1:5173', 'http://localhost:5173'}
if parsed.scheme not in ('http', 'https') or origin.rstrip('/') not in allowed:
response = JSONResponse({'detail': '已拒绝跨来源修改请求'}, status_code=403)
return await response(scope, receive, send)
return await self.app(scope, receive, send)
def create_app(data_dir=None):
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
directory.mkdir(parents=True, exist_ok=True)
@@ -36,6 +65,9 @@ def create_app(data_dir=None):
transport = Transport(settings, router)
auth = AuthService(transport, CookieVault(directory))
jobs = JobManager(db, settings, transport, auth=auth)
export_lock = threading.BoundedSemaphore(1)
export_directory = directory / 'exports'
export_directory.mkdir(exist_ok=True)
if (ROOT / "data/schools.json").exists():
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
@@ -46,6 +78,8 @@ def create_app(data_dir=None):
jobs.resume(jid)
yield
jobs.close()
transport.close()
router.session.close()
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
app.state.db, app.state.jobs = db, jobs
@@ -53,19 +87,7 @@ def create_app(data_dir=None):
app.state.pool = pool
app.state.auth = auth
@app.middleware("http")
async def local_access(request: Request, call_next):
host = request.url.hostname
if host not in allowed_hosts:
return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
if request.method not in ("GET", "HEAD", "OPTIONS"):
origin = request.headers.get("origin")
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
return await call_next(request)
app.add_middleware(LocalAccessMiddleware, allowed_hosts=allowed_hosts)
@app.exception_handler(ValueError)
async def invalid(request, exc):
@@ -216,17 +238,35 @@ def create_app(data_dir=None):
def export(request: Request, format: str = "csv"):
if format not in ("csv", "xlsx", "json"):
raise ValueError("导出格式无效")
rows = db.all_catalog(filters(request))
if format == "json":
pairs = {(r["school_code"], r["major_code"]) for r in rows}
with db.connection() as conn:
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs]
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode()
mime = "application/json"
selected = filters(request)
if not export_lock.acquire(False):
raise HTTPException(429, '已有导出正在进行,请等待完成后重试')
temporary = None
def cleanup():
try:
if temporary is not None:
temporary.unlink(missing_ok=True)
finally:
export_lock.release()
try:
if format == 'csv':
source, mime = csv_chunks(db.iter_catalog(selected)), 'text/csv; charset=utf-8'
elif format == 'json':
source, mime = json_chunks(db, selected), 'application/json'
else:
payload = export_sheet(rows, format)
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'})
descriptor, path = tempfile.mkstemp(prefix='kaoyan-', suffix='.xlsx', dir=export_directory)
os.close(descriptor)
temporary = Path(path)
write_xlsx(db.iter_catalog(selected), temporary)
source = file_chunks(temporary)
mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
return ExportStream(source, cleanup, media_type=mime,
headers={'Content-Disposition': f'attachment; filename="kaoyan-data.{format}"',
'X-Accel-Buffering': 'no'})
except BaseException:
cleanup()
release_idle_memory()
raise
dist = ROOT / "frontend/dist"
if (dist / "assets").exists():
+2 -2
View File
@@ -187,10 +187,10 @@ class AuthService:
raise ValueError("请输入学信网账号和密码")
if self.browser_cancel:
self.browser_cancel.set()
with self.check_lock:
with self.check_lock, self.transport.router.hold_route():
response = self.transport.request("GET", LOGIN_URL)
soup = BeautifulSoup(response.text, "html.parser")
form = {node.get("name"): node.get("value", "") for node in soup.select("input[type=hidden][name]")}
form = {node.get("name"): node.get("value", "") for node in soup.find_all("input", attrs={"type": "hidden", "name": True})}
if "lt" not in form or "execution" not in form:
raise ValueError("登录页面需要人工验证,请使用浏览器登录")
form.update(username=username, password=password, _eventId="submit")
+85 -15
View File
@@ -1,6 +1,8 @@
"""Independent adapters for the two projects' public data sources."""
import json
import re
import hashlib
from collections import OrderedDict
from studio.domain import finish_record, integer, normalized_name, select_score_lines
from studio.network import RemoteUnavailable, RateLimited
@@ -18,8 +20,9 @@ class SourceYearMismatch(RuntimeError):
class CatalogClient:
BASE = "https://yz.chsi.com.cn/zsml/"
def __init__(self, transport, auth=None):
def __init__(self, transport, auth=None, page_cache=None, save_page=None, on_request=None):
self.transport, self.auth = transport, auth
self.page_cache, self.save_page, self.on_request = page_cache, save_page, on_request
def verify_year(self, year):
response = self.transport.request("GET", self.BASE)
@@ -65,37 +68,70 @@ class CatalogClient:
return data
def fetch_page(self, endpoint, payload):
# Lists are a snapshot for this task. Direction records and login proofs
# remain live requests; failed/partial responses never enter the cache.
cacheable = self.page_cache is not None and endpoint in ('dws', 'dwzys')
key = self._page_key(endpoint, payload) if cacheable else None
if cacheable and key in self.page_cache:
return self.page_cache[key]
if self.on_request:
self.on_request(endpoint, payload)
try:
return self._fetch_page(endpoint, payload)
data = self._fetch_page(endpoint, payload)
except LoginRequired as exc:
if not self.auth or exc.reason == "captcha":
raise
# A CAS cookie may still be valid while the catalogue session has expired.
# Refresh once, then retry exactly the same page; never skip its records.
with self.transport.router.hold_route():
status = self.auth.check()
if not status["valid"]:
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None
return self._fetch_page(endpoint, payload)
data = self._fetch_page(endpoint, payload)
return data
@staticmethod
def _page_key(endpoint, payload):
return hashlib.sha256(json.dumps([endpoint, payload], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
def _cache_page(self, endpoint, payload, data):
if self.page_cache is None or endpoint not in ('dws', 'dwzys'):
return
key = self._page_key(endpoint, payload)
if key not in self.page_cache:
if self.save_page:
self.save_page(key, data)
self.page_cache[key] = data
def pages(self, endpoint, payload, page_size=10):
seen, start = set(), 0
for page in range(1, 10001):
data = self.fetch_page(endpoint, {**payload, "curPage": page, "start": start, "pageSize": page_size})
page_payload = {**payload, "curPage": page, "start": start, "pageSize": page_size}
data = self.fetch_page(endpoint, page_payload)
rows = data["list"]
identifier = {'dws': 'dwdm', 'dwzys': 'zydm'}.get(endpoint)
if identifier:
for row in rows:
code = row.get(identifier) if isinstance(row, dict) else None
if isinstance(code, bool) or not isinstance(code, (str, int)) or not str(code).strip():
label = '院校' if endpoint == 'dws' else '专业'
raise RemoteUnavailable(f'研招目录{label}代码缺失或结构异常,已停止以避免缓存不完整数据')
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen:
raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页")
seen.add(signature)
yield from rows
more = data.get("nextPageAvailable")
total = integer(data.get("totalCount"))
size = integer(data.get("size")) or page_size
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size):
return
terminal = not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size)
next_start = integer(data.get("startOfNextPage"))
next_start = start + len(rows) if next_start is None else next_start
if next_start <= start:
if not terminal and next_start <= start:
raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页")
self._cache_page(endpoint, page_payload, data)
yield from rows
if terminal:
return
start, page_size = next_start, size
raise RemoteUnavailable("目录分页超过安全上限")
@@ -118,6 +154,7 @@ class ScoreClient:
def __init__(self, transport):
self.transport = transport
self.cache = {}
self.score_cache = OrderedDict()
def post(self, endpoint, payload):
for attempt in range(3):
@@ -126,8 +163,13 @@ class ScoreClient:
body = response.json()
except ValueError:
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
if not isinstance(body, dict):
raise RemoteUnavailable("掌上考研响应结构已变化")
if str(body.get("code")) == "0000":
return body.get("data") or {}
data = body.get("data")
if not isinstance(data, (dict, list)):
raise RemoteUnavailable("掌上考研缺少有效数据,未写入不完整结果")
return data
message = str(body.get('message', body.get('msg', '')))
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
if attempt == 2:
@@ -137,20 +179,40 @@ class ScoreClient:
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
def pages(self, endpoint, payload, limit=100):
seen = set()
seen, consumed = set(), 0
for page in range(1, 10001):
data = self.post(endpoint, {**payload, "page": page, "limit": limit})
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data
if not isinstance(rows, list):
full_list = isinstance(data, list)
rows = data.get("data", data.get("list")) if isinstance(data, dict) else data
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
raise RemoteUnavailable("掌上考研响应结构已变化")
total = integer(data.get("total")) if isinstance(data, dict) else None
last = integer(data.get("last_page")) if isinstance(data, dict) else None
if isinstance(data, dict):
for field, value in (('total', total), ('last_page', last)):
if data.get(field) is not None and (value is None or value < 0 or isinstance(data[field], bool)):
raise RemoteUnavailable("掌上考研分页信息无效,未写入不完整结果")
if not rows and (total is not None and consumed < total or last is not None and page < last):
raise RemoteUnavailable("掌上考研提前返回空页,未写入不完整结果")
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen:
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
seen.add(signature)
consumed += len(rows)
yield from rows
total = integer(data.get("total")) if isinstance(data, dict) else None
last = integer(data.get("last_page")) if isinstance(data, dict) else None
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total:
# Bare arrays are complete responses; wrapped pages use their actual
# metadata even when the server ignores the requested page size.
terminal = full_list or not rows
if not terminal:
if last is not None:
terminal = page >= last
if terminal and total is not None and consumed < total:
raise RemoteUnavailable("掌上考研分页未返回全部记录")
elif total is not None:
terminal = consumed >= total
else:
terminal = len(rows) < limit
if terminal:
return
raise RemoteUnavailable("历史数据分页超过安全上限")
@@ -163,10 +225,18 @@ class ScoreClient:
if len(matches) == 1 and matches[0].get("school_id"):
self.cache[key] = matches[0]["school_id"]
return self.cache[key]
self.cache[key] = None
return None
def score_lines(self, school, major_code, year, school_id):
key = (str(school_id), int(year))
if key not in self.score_cache:
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
self.score_cache[key] = rows
if len(self.score_cache) > 8:
self.score_cache.popitem(last=False)
self.score_cache.move_to_end(key)
rows = self.score_cache[key]
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
def plans(self, school, major_code, year, school_id):
+98 -2
View File
@@ -19,11 +19,14 @@ CREATE TABLE IF NOT EXISTS catalog(
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
CREATE INDEX IF NOT EXISTS catalog_export ON catalog(year DESC,school_name,id);
CREATE INDEX IF NOT EXISTS catalog_targets ON catalog(year,school_code,major_code);
""" + SCORE_SCHEMA + """
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id TEXT,page_key TEXT,data TEXT NOT NULL,PRIMARY KEY(job_id,page_key));
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
"""
@@ -68,6 +71,10 @@ class Database:
conn.close()
def upsert_catalog(self, records: list[dict]):
with self.connection() as conn:
self._upsert_catalog(conn, records)
def _upsert_catalog(self, conn, records):
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
values = []
for record in records:
@@ -76,7 +83,6 @@ class Database:
if not values:
return
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
with self.connection() as conn:
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
for record in records:
self._school(conn, record)
@@ -92,6 +98,9 @@ class Database:
def upsert_scores(self, records):
with self.connection() as conn:
self._upsert_scores(conn, records)
def _upsert_scores(self, conn, records):
for row in records:
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
@@ -140,9 +149,73 @@ class Database:
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
def all_catalog(self, filters):
return list(self.iter_catalog(filters))
def iter_catalog(self, filters, batch_size=200):
select, params = self._query(filters)
with self.connection() as conn:
return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)]
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM catalog').fetchone()[0]
after = None
while True:
query = 'SELECT * FROM (' + select + ') export_rows WHERE id<=?'
values = [*params, cutoff]
if after:
year, school, rid = after
if school is None:
query += ' AND (year<? OR (year=? AND (school_name IS NOT NULL OR (school_name IS NULL AND id>?))))'
values.extend([year, year, rid])
else:
query += ' AND (year<? OR (year=? AND (school_name>? OR (school_name=? AND id>?))))'
values.extend([year, year, school, school, rid])
query += ' ORDER BY year DESC,school_name,id LIMIT ?'
with self.connection() as conn:
rows = conn.execute(query, [*values, batch_size]).fetchall()
if not rows:
return
for row in rows:
yield self._catalog_row(row)
last = rows[-1]
after = (last['year'], last['school_name'], last['id'])
def iter_scores(self, pairs, batch_size=200):
after = 0
with self.connection() as conn:
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM scores').fetchone()[0]
while True:
with self.connection() as conn:
rows = conn.execute('SELECT id,school_code,major_code,data FROM scores WHERE id>? AND id<=? ORDER BY id LIMIT ?', (after, cutoff, batch_size)).fetchall()
if not rows:
return
for row in rows:
if (row['school_code'], row['major_code']) in pairs:
yield json.loads(row['data'])
after = rows[-1]['id']
def score_targets(self, filters, batch_size=200):
conditions, params = ['year=?'], [filters['year']]
if filters.get('major_code'):
conditions.append('major_code=?')
params.append(filters['major_code'])
if filters.get('school_keyword'):
conditions.append('school_name LIKE ?')
params.append('%' + filters['school_keyword'] + '%')
if filters.get('provinces'):
conditions.append("(province='' OR province IN (" + ','.join('?' for _ in filters['provinces']) + '))')
params.extend(filters['provinces'])
after = None
while True:
where, values = ' AND '.join(conditions), list(params)
if after:
where += ' AND (school_code>? OR (school_code=? AND major_code>?))'
values.extend([after[0], after[0], after[1]])
query = 'SELECT school_code,major_code,MAX(school_name) AS school_name,MAX(province) AS province,MAX(level) AS level FROM catalog WHERE ' + where + ' GROUP BY school_code,major_code ORDER BY school_code,major_code LIMIT ?'
with self.connection() as conn:
rows = conn.execute(query, [*values, batch_size]).fetchall()
if not rows:
return
yield from (dict(row) for row in rows)
last = rows[-1]
after = (last['school_code'], last['major_code'])
def _catalog_row(self, row):
data = json.loads(row["data"])
@@ -202,10 +275,33 @@ class Database:
with self.connection() as conn:
return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None
def completed_units(self, jid):
with self.connection() as conn:
return {row['unit'] for row in conn.execute('SELECT unit FROM job_units WHERE job_id=?', (jid,))}
def catalog_pages(self, jid):
with self.connection() as conn:
return {row['page_key']: json.loads(row['data']) for row in conn.execute('SELECT page_key,data FROM job_catalog_pages WHERE job_id=?', (jid,))}
def cache_catalog_page(self, jid, page_key, data):
with self.connection() as conn:
conn.execute('INSERT INTO job_catalog_pages(job_id,page_key,data) VALUES(?,?,?) ON CONFLICT(job_id,page_key) DO UPDATE SET data=excluded.data', (jid, page_key, encode(data)))
def complete_unit(self, jid, unit):
with self.connection() as conn:
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
def save_unit(self, jid, unit, records, scores=()):
with self.connection() as conn:
if not conn.execute('SELECT 1 FROM jobs WHERE id=?', (jid,)).fetchone():
raise ValueError('任务不存在')
if not conn.execute('INSERT OR IGNORE INTO job_units VALUES(?,?)', (jid, unit)).rowcount:
return False
self._upsert_scores(conn, scores)
self._upsert_catalog(conn, records)
conn.execute('UPDATE jobs SET records=records+?,updated_at=? WHERE id=?', (len(records), now(), jid))
return True
def log(self, jid, level, message):
with self.connection() as conn:
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
+66
View File
@@ -0,0 +1,66 @@
"""Bounded exports, one download at a time, with disconnect cleanup."""
import csv
import io
import json
from fastapi.responses import StreamingResponse
from studio.imports import LABELS, spreadsheet_value
from studio.memory import release_idle_memory
class ExportStream(StreamingResponse):
def __init__(self, source, cleanup, **kwargs):
super().__init__(source, **kwargs)
self.source, self.cleanup = source, cleanup
async def __call__(self, scope, receive, send):
try:
await super().__call__(scope, receive, send)
finally:
try:
self.source.close()
finally:
try:
self.cleanup()
finally:
release_idle_memory()
def csv_chunks(records):
yield b'\xef\xbb\xbf'
stream = io.StringIO(newline='')
writer = csv.writer(stream)
writer.writerow(list(LABELS.values()))
yield stream.getvalue().encode('utf-8')
stream.seek(0)
stream.truncate(0)
for record in records:
writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS])
if stream.tell() >= 65536:
yield stream.getvalue().encode('utf-8')
stream.seek(0)
stream.truncate(0)
if stream.tell():
yield stream.getvalue().encode('utf-8')
def json_chunks(database, filters):
pairs = set()
yield b'{"version":1,"catalog":['
first = True
for row in database.iter_catalog(filters):
pairs.add((row['school_code'], row['major_code']))
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
first = False
yield b'],"scores":['
first = True
for row in database.iter_scores(pairs):
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
first = False
yield b']}'
def file_chunks(path):
with path.open('rb') as stream:
while chunk := stream.read(65536):
yield chunk
+44 -17
View File
@@ -2,6 +2,7 @@
import csv
import io
import json
from contextlib import suppress
import sqlite3
from dataclasses import dataclass, field
from pathlib import Path
@@ -94,11 +95,18 @@ def import_file(path, year=None):
book = load_workbook(path, read_only=True, data_only=True)
try:
sheet = book.active
if sheet.max_row > 300001 or sheet.max_column > 100:
if (sheet.max_row or 0) > 300001 or (sheet.max_column or 0) > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
values = sheet.iter_rows(values_only=True)
headers = next(values, [])
rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)]
if len(headers) > 100:
raise ValueError("Excel 超过 100 列,请拆分后导入")
rows = []
for index, row in enumerate(values, 2):
if index > 300001 or len(row) > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
if any(v is not None for v in row):
rows.append(dict(zip(headers, row)))
finally:
book.close()
else:
@@ -179,28 +187,47 @@ def spreadsheet_value(value):
def export_sheet(records, kind):
rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records]
if kind == "csv":
stream = io.StringIO(newline="")
writer = csv.writer(stream)
writer.writerow(list(LABELS.values()))
writer.writerows(rows)
writer.writerows([spreadsheet_value(record.get(field)) for field in LABELS] for record in records)
return stream.getvalue().encode("utf-8-sig")
book = Workbook()
sheet = book.active
sheet.title = "院校专业"
sheet.append(list(LABELS.values()))
for row in rows:
sheet.append(row)
sheet.freeze_panes = "A2"
sheet.auto_filter.ref = sheet.dimensions
output = io.BytesIO()
write_xlsx(records, output)
return output.getvalue()
def write_xlsx(records, output):
from openpyxl.cell import WriteOnlyCell
from openpyxl.utils import get_column_letter
from openpyxl.styles import Font, PatternFill, Alignment
for cell in sheet[1]:
book = Workbook(write_only=True)
sheet = book.create_sheet("院校专业")
sheet.freeze_panes = "A2"
header = []
for index, label in enumerate(LABELS.values(), 1):
cell = WriteOnlyCell(sheet, value=label)
cell.font = Font(color="FFFFFF", bold=True)
cell.fill = PatternFill("solid", fgColor="183447")
cell.alignment = Alignment(vertical="center")
for col in sheet.columns:
sheet.column_dimensions[col[0].column_letter].width = 24
output = io.BytesIO()
header.append(cell)
sheet.column_dimensions[get_column_letter(index)].width = 24
count = 1
try:
sheet.append(header)
for record in records:
sheet.append([spreadsheet_value(record.get(field)) for field in LABELS])
count += 1
sheet.auto_filter.ref = f'A1:{get_column_letter(len(LABELS))}{count}'
book.save(output)
return output.getvalue()
except BaseException:
with suppress(Exception):
if not sheet.closed:
sheet.close()
with suppress(Exception):
if sheet._writer:
sheet._writer.cleanup()
raise
finally:
book.close()
+40 -21
View File
@@ -1,6 +1,7 @@
"""Persistent units and cooperative pause/cancel for one local worker."""
import threading
import uuid
from itertools import chain
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
from studio.domain import PROVINCES, normalize_catalog
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
@@ -189,7 +190,13 @@ class JobManager:
self._scores(jid, params)
def _catalog(self, jid, params):
client = CatalogClient(self.transport, auth=self.auth)
completed = self.db.completed_units(jid)
location = '研招目录'
def on_request(endpoint, payload):
phase = {'dws': '院校列表', 'dwzys': '专业列表', 'yjfxs': '研究方向'}[endpoint]
self.db.job_update(jid, message=f"{location}:正在请求{phase}第 {payload.get('curPage', 1)} 页")
client = CatalogClient(self.transport, auth=self.auth, page_cache=self.db.catalog_pages(jid),
save_page=lambda key, data: self.db.cache_catalog_page(jid, key, data), on_request=on_request)
client.verify_year(params["year"])
if self.auth:
status = self.auth.check()
@@ -198,48 +205,60 @@ class JobManager:
self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功")
for province in params["provinces"]:
self.checkpoint(jid)
location = PROVINCES[province]
self.db.job_update(jid, message=location + ':正在核对院校列表')
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
for school in client.schools(province, params.get("school_keyword", "")):
self.checkpoint(jid)
school = {"ssdm": province, **school}
location = f"{PROVINCES[province]} · {school.get('dwmc', school['dwdm'])}"
self.db.job_update(jid, message=location + ':正在核对专业列表与已完成断点')
self.db.log(jid, 'info', location + ':核对专业列表与断点')
skipped = 0
for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")):
self.checkpoint(jid)
if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]):
continue
unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}"
if self.unit_done(jid, unit):
if unit in completed:
skipped += 1
continue
self.checkpoint(jid)
label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}"
location = label
self.db.job_update(jid, message=label)
try:
records = []
for row in client.directions(school, major, params.get("study_mode", "")):
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
self._save(jid, records)
self.complete_unit(jid, unit)
self.db.save_unit(jid, unit, records)
completed.add(unit)
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}")
if skipped:
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
def _scores(self, jid, params):
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
records = self.db.all_catalog(filters)
targets = {}
for row in records:
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]:
continue
targets[(row["school_code"], row["major_code"])] = row
if not targets and params.get("major_code"):
for row in self.db.schools(params.get("school_keyword", "")):
if row["province"] in [PROVINCES[p] for p in params["provinces"]]:
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
if not targets:
provinces = [PROVINCES[p] for p in params['provinces']]
filters = {"year": params["year"], "major_code": params.get("major_code", ""),
"school_keyword": params.get("school_keyword", ""), 'provinces': provinces}
targets = iter(self.db.score_targets(filters))
first = next(targets, None)
if first is None and params.get('major_code'):
targets = ({**row, 'major_code': params['major_code']}
for row in self.db.schools(params.get('school_keyword', '')) if row['province'] in provinces)
first = next(targets, None)
if first is None:
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
completed = self.db.completed_units(jid)
self.db.log(jid, 'info', '开始补充分数线和历史计划,分批读取专业目标并复用院校年份分数线')
client = ScoreClient(self.transport)
for (_, code), record in targets.items():
for record in chain((first,), targets):
self.checkpoint(jid)
code = record['major_code']
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")]
years = [y for y in params["score_years"] if f"scores:{school['school_code']}:{code}:{y}" not in completed]
if not years:
continue
school_id = client.school_id(school["school_name"])
@@ -252,10 +271,10 @@ class JobManager:
self.db.job_update(jid, message=label)
try:
scores = client.score_lines(school, code, year, school_id)
self.db.upsert_scores(scores)
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
self._save(jid, plans)
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}")
unit = f"scores:{school['school_code']}:{code}:{year}"
self.db.save_unit(jid, unit, plans, scores=scores)
completed.add(unit)
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}")
+17
View File
@@ -0,0 +1,17 @@
"""Best-effort return of idle allocator memory after large operations."""
import ctypes
import gc
import sys
def release_idle_memory():
gc.collect()
if sys.platform == 'linux':
try:
trim = ctypes.CDLL(None).malloc_trim
trim.argtypes = [ctypes.c_size_t]
trim.restype = ctypes.c_int
return bool(trim(0))
except (AttributeError, OSError):
pass
return False
+5 -2
View File
@@ -6,7 +6,7 @@ from pathlib import Path
from studio.domain import normalized_name, is_special_score
from studio.mysql_database import MySQLDatabase, score_key
TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs')
TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs', 'job_catalog_pages')
def migrate_sqlite(source, target):
@@ -16,6 +16,7 @@ def migrate_sqlite(source, target):
counts = {}
try:
conn.execute('BEGIN')
source_tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
with target.connection() as output:
# Serialize attempted imports to the same database.
acquired = output.execute("SELECT GET_LOCK('kaoyan_studio_migration',10)").fetchone()[0]
@@ -27,6 +28,8 @@ def migrate_sqlite(source, target):
raise ValueError(f'目标数据库非空({table}),拒绝覆盖;请使用空数据库')
for table in TABLES:
counts[table] = 0
if table == 'job_catalog_pages' and table not in source_tables:
continue # Existing snapshots predate the optional page cache.
reader = conn.execute(f'SELECT * FROM {table}')
while batch := reader.fetchmany(500):
rows = []
@@ -67,7 +70,7 @@ def main():
parser.add_argument('--config', type=Path, required=True)
args = parser.parse_args()
config = json.loads(args.config.read_text())
print(json.dumps(migrate_sqlite(args.snapshot, MySQLDatabase(config)), ensure_ascii=False))
print(json.dumps(migrate_sqlite(args.snapshot, MySQLDatabase(config, recover_jobs=False)), ensure_ascii=False))
if __name__ == '__main__':
+16 -6
View File
@@ -2,6 +2,7 @@
import hashlib
import json
import re
import ssl
from contextlib import contextmanager
import pymysql
from studio.database import Database, encode
@@ -23,6 +24,7 @@ SCHEMA = [
"CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)",
"CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)",
"CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))",
"CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id VARCHAR(64),page_key VARCHAR(64),data LONGTEXT NOT NULL,PRIMARY KEY(job_id,page_key))",
"CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))",
]
@@ -96,24 +98,34 @@ class Connection:
class MySQLDatabase(Database):
def __init__(self, config):
def __init__(self, config, recover_jobs=True):
self.config = {key: value for key, value in config.items() if key in ('host', 'port', 'user', 'password', 'database', 'unix_socket')}
if not self.config.get('database'):
raise ValueError('数据库配置缺少 database')
self.recovered_job_ids = []
self._ssl_context = None
with self.connection() as conn:
for statement in SCHEMA:
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_export'").fetchone():
conn.execute('CREATE INDEX catalog_export ON catalog(year DESC,school_name,id)')
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_targets'").fetchone():
conn.execute('CREATE INDEX catalog_targets ON catalog(year,school_code,major_code)')
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
if recover_jobs:
recovery = "status IN ('running','queued') OR (status='paused' AND resume_pending=1)"
self.recovered_job_ids = [row['id'] for row in conn.execute('SELECT id FROM jobs WHERE ' + recovery)]
conn.execute("UPDATE jobs SET status='paused',resume_pending=0,message='服务重启,任务可从断点恢复' WHERE " + recovery)
@contextmanager
def connection(self):
raw = pymysql.connect(**self.config, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
raw = pymysql.connect(**self.config, ssl=self._ssl_context, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
try:
# Retain the driver's negotiated context rather than reload its CA
# store for every short-lived query. Plain servers retain default behavior.
if self._ssl_context is None and isinstance(getattr(raw, '_sock', None), ssl.SSLSocket):
self._ssl_context = raw.ctx
yield Connection(raw)
raw.commit()
except BaseException:
@@ -122,7 +134,7 @@ class MySQLDatabase(Database):
finally:
raw.close()
def upsert_catalog(self, records):
def _upsert_catalog(self, conn, records):
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
values = []
for record in records:
@@ -131,12 +143,11 @@ class MySQLDatabase(Database):
if not values:
return
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
with self.connection() as conn:
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
for record in records:
self._school(conn, record)
def upsert_scores(self, records):
def _upsert_scores(self, conn, records):
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
values = []
for record in records:
@@ -144,7 +155,6 @@ class MySQLDatabase(Database):
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
values.append([row.get(field) for field in fields])
if values:
with self.connection() as conn:
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
def favorite(self, rid, enabled):
+31 -1
View File
@@ -3,6 +3,7 @@ import threading
import time
import random
from contextlib import contextmanager
from collections import OrderedDict
import requests
from studio.settings import proxy_url
@@ -19,6 +20,31 @@ class RateLimited(RuntimeError):
pass
class BoundedProxyAdapter(requests.adapters.HTTPAdapter):
"""Close obsolete proxy pools rather than retain every historical exit."""
def __init__(self, max_proxy_pools=4, **kwargs):
super().__init__(**kwargs)
self.max_proxy_pools = max(1, int(max_proxy_pools))
self.proxy_manager = OrderedDict()
def proxy_manager_for(self, proxy, **proxy_kwargs):
manager = super().proxy_manager_for(proxy, **proxy_kwargs)
self.proxy_manager.move_to_end(proxy)
while len(self.proxy_manager) > self.max_proxy_pools:
_, obsolete = self.proxy_manager.popitem(last=False)
obsolete.clear()
return manager
def bounded_session():
session = requests.Session()
session.trust_env = False
for scheme in ('http://', 'https://'):
session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4))
return session
class ProxyRouter:
def __init__(self, settings, session=None):
self.settings = settings
@@ -141,7 +167,7 @@ class ProxyRouter:
class Transport:
def __init__(self, settings, router, session=None, wait=None):
self.settings, self.router = settings, router
self.session = session or requests.Session()
self.session = session or bounded_session()
self.session.trust_env = False
self.session.headers = getattr(self.session, "headers", {})
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
@@ -149,6 +175,10 @@ class Transport:
self.local = threading.local()
self.lock = threading.RLock()
def close(self):
with self.lock:
self.session.close()
def sleep(self, seconds):
remaining = seconds
while remaining > 0:
+75
View File
@@ -0,0 +1,75 @@
import pytest
from studio.database import Database
from studio.domain import finish_record
def record(code='10001', name='甲大学', major='081200', province='北京', year=2027, direction='方向一'):
return finish_record({'school_code': code, 'school_name': name, 'major_code': major,
'major_name': major, 'province': province, 'year': year,
'direction': direction, 'subjects': ['101政治'], 'source': '测试'})
def test_history_targets_deduplicate_directions_and_preserve_filters(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_catalog([
record(direction='一'), record(direction='二'), record(major='081201'),
record(code='10002', name='乙大学', province='天津'),
record(code='10003', name='丙大学', province=''), record(year=2026),
])
rows = list(db.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
assert [(r['school_code'], r['major_code']) for r in rows] == [
('10001', '081200'), ('10001', '081201'), ('10003', '081200')]
assert all('data' not in row and 'subjects' not in row for row in rows)
rows = list(db.score_targets({'year': 2027, 'school_keyword': '甲', 'major_code': '081201'}))
assert [(r['school_code'], r['major_code']) for r in rows] == [('10001', '081201')]
def test_catalog_iterator_keeps_export_order_and_excludes_new_inserts(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_catalog([record(name='B大学'), record(name='A大学', direction='二'),
record(name='A大学', direction='三'), record(year=2026),
record(code='10002', name='C大学')])
iterator = db.iter_catalog({}, batch_size=2)
first = next(iterator)
db.upsert_catalog([record(name='Z大学', year=2028)])
rows = [first, *iterator]
assert [(r['year'], r['school_name']) for r in rows] == [
(2027, 'A大学'), (2027, 'A大学'), (2027, 'B大学'), (2027, 'C大学'), (2026, '甲大学')]
assert [r['school_name'] for r in db.iter_catalog({'school_keyword': 'A'}, batch_size=1)] == ['A大学', 'A大学']
def test_score_iterator_exports_only_selected_school_major_pairs(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_scores([
{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310},
{'school_code': '10002', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 320},
])
rows = list(db.iter_scores({('10001', '081200')}, batch_size=1))
assert len(rows) == 1
assert rows[0]['total'] == 310
def test_unit_commit_is_idempotent_for_records_count_and_checkpoint(tmp_path):
db = Database(tmp_path / 'data.db')
db.job_create('job', {})
rows = [record(direction='一'), record(direction='二')]
assert db.save_unit('job', 'catalog:test', rows) is True
assert db.save_unit('job', 'catalog:test', rows) is False
assert db.overview()['records'] == 2
assert db.job_get('job')['records'] == 2
assert db.unit_done('job', 'catalog:test')
def test_failed_unit_transaction_rolls_back_scores_records_and_checkpoint(tmp_path, monkeypatch):
db = Database(tmp_path / 'data.db')
db.job_create('job', {})
def fail(*args):
raise RuntimeError('interrupted persistence')
monkeypatch.setattr(db, '_school', fail)
score = {'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}
with pytest.raises(RuntimeError):
db.save_unit('job', 'scores:test', [record()], scores=[score])
assert db.overview()['records'] == db.overview()['scores'] == 0
assert db.job_get('job')['records'] == 0
assert not db.unit_done('job', 'scores:test')
+153
View File
@@ -0,0 +1,153 @@
from collections import Counter
import pytest
from studio.connectors import CatalogClient, LoginRequired
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
from studio.network import RemoteUnavailable
from tests.test_data import raw_catalog
from tests.test_network import Response
PARAMS = {'mode': 'catalog', 'year': 2027, 'provinces': ['11']}
class InterruptedCatalog:
def __init__(self, observe=None):
self.blocked = True
self.calls = Counter()
self.observe = observe
def request(self, method, url, **kwargs):
if method == 'GET':
return Response({}, text='2027年硕士专业目录查询')
endpoint = url.rsplit('/', 1)[-1]
school = kwargs['data'].get('dwdm', '')
self.calls[endpoint, school] += 1
if endpoint == 'dws.do':
rows = [{'dwdm': '10001', 'dwmc': '北京大学'}, {'dwdm': '10002', 'dwmc': '中国人民大学'}]
elif endpoint == 'dwzys.do':
if self.observe:
self.observe(school)
rows = [{'zydm': '081200', 'zymc': '计算机科学与技术', 'xxfs': '1'}]
else:
if school == '10002' and self.blocked:
return Response({'flag': False, 'msg': '请登录'})
rows = [{**raw_catalog(), 'dwdm': school}]
return Response({'flag': True, 'msg': {'list': rows, 'nextPageAvailable': False}})
def prepare(tmp_path, transport):
db = Database(tmp_path / 'studio.db')
db.job_create('resume', PARAMS)
return db, JobManager(db, SettingsStore(tmp_path), transport=transport)
def test_login_recovery_reuses_successful_catalog_pages_across_service_restart(tmp_path):
source = InterruptedCatalog()
db, manager = prepare(tmp_path, source)
with pytest.raises(LoginRequired):
manager._catalog('resume', PARAMS)
assert db.job_get('resume')['completed_units'] == 1
source.blocked = False
reopened = Database(tmp_path / 'studio.db')
JobManager(reopened, SettingsStore(tmp_path), transport=source)._catalog('resume', PARAMS)
assert reopened.job_get('resume')['completed_units'] == 2
assert source.calls['dws.do', ''] == 1
assert source.calls['dwzys.do', '10001'] == 1
assert source.calls['dwzys.do', '10002'] == 1
assert source.calls['yjfxs.do', '10002'] == 2 # Failed responses must never become cached successes.
def test_resume_checks_completed_units_in_memory_instead_of_one_query_per_major(tmp_path):
source = InterruptedCatalog()
source.blocked = False
db, manager = prepare(tmp_path, source)
for school in ('10001', '10002'):
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
queries = []
original = db.unit_done
def observe(jid, unit):
queries.append(unit)
return original(jid, unit)
db.unit_done = observe
manager._catalog('resume', PARAMS)
assert queries == []
assert not any(endpoint == 'yjfxs.do' for endpoint, school in source.calls)
def test_resume_displays_current_school_while_checking_its_major_list(tmp_path):
messages = {}
source = InterruptedCatalog(lambda school: messages.setdefault(school, db.job_get('resume')['message']))
source.blocked = False
db, manager = prepare(tmp_path, source)
for school in ('10001', '10002'):
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
manager._catalog('resume', PARAMS)
assert '北京大学' in messages['10001']
assert '中国人民大学' in messages['10002']
def test_cached_pages_do_not_cross_job_boundaries(tmp_path):
source = InterruptedCatalog()
source.blocked = False
db, manager = prepare(tmp_path, source)
manager._catalog('resume', PARAMS)
db.job_create('fresh', PARAMS)
manager._catalog('fresh', PARAMS)
assert source.calls['dws.do', ''] == 2
assert db.job_get('fresh')['completed_units'] == 2
def test_invalid_pagination_is_not_persisted_as_a_reusable_page():
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
body = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': self.blocked, 'startOfNextPage': 0}
self.blocked = False
return Response({'flag': True, 'msg': body})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='下一页位置无效'):
list(client.schools('11'))
assert cache == {}
assert list(client.schools('11')) == [{'dwdm': '10001'}]
def test_repeated_source_page_is_retried_live_after_recovery():
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
page = kwargs['data']['curPage']
school = '10001' if page == 1 or self.blocked else '10002'
return Response({'flag': True, 'msg': {'list': [{'dwdm': school}], 'nextPageAvailable': page == 1}})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='重复返回同一页'):
list(client.schools('11'))
assert len(cache) == 1
source.blocked = False
assert [row['dwdm'] for row in client.schools('11')] == ['10001', '10002']
@pytest.mark.parametrize('endpoint,broken_row', [
('dws', {'dwmc': '缺少代码的院校'}),
('dwzys', {'zymc': '缺少代码的专业'}),
('dws', 'unexpected row'),
('dwzys', {'zydm': ''}),
('dwzys', {'zydm': None}),
])
def test_malformed_list_page_is_retried_after_source_recovers(endpoint, broken_row):
identifier = 'dwdm' if endpoint == 'dws' else 'zydm'
good_row = {identifier: '10001' if endpoint == 'dws' else '081200'}
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
row = broken_row if self.blocked else good_row
return Response({'flag': True, 'msg': {'list': [row], 'nextPageAvailable': False}})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='代码'):
list(client.pages(endpoint, {}))
assert cache == {}
source.blocked = False
assert list(client.pages(endpoint, {})) == [good_row]
+42
View File
@@ -0,0 +1,42 @@
from studio import network
def test_proxy_managers_are_bounded_and_evicted_pools_are_closed():
adapter = network.BoundedProxyAdapter(max_proxy_pools=3)
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
closed = []
original = first.clear
def observe():
original()
closed.append(True)
first.clear = observe
for port in range(10002, 10006):
adapter.proxy_manager_for(f'http://127.0.0.1:{port}')
assert len(adapter.proxy_manager) <= 3
assert closed == [True]
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is not first
adapter.close()
def test_active_proxy_pool_is_reused_while_old_pools_are_evicted():
adapter = network.BoundedProxyAdapter(max_proxy_pools=2)
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
adapter.proxy_manager_for('http://127.0.0.1:10002')
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
adapter.proxy_manager_for('http://127.0.0.1:10003')
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
assert 'http://127.0.0.1:10002' not in adapter.proxy_manager
adapter.close()
def test_default_transport_limits_pools_without_changing_route_or_tls(tmp_path):
from studio.settings import SettingsStore
from tests.test_network import Pool
settings = SettingsStore(tmp_path)
router = network.ProxyRouter(settings, session=Pool())
transport = network.Transport(settings, router)
adapter = transport.session.get_adapter('https://example.test')
assert isinstance(adapter, network.BoundedProxyAdapter)
assert transport.session.verify is True
assert router.resolve() == 'http://1.2.3.4:8080'
transport.close()
+125
View File
@@ -0,0 +1,125 @@
import asyncio
import csv
import io
import threading
import pytest
from fastapi.testclient import TestClient
from openpyxl import load_workbook
from studio.api import create_app
from tests.test_bounded_storage import record
@pytest.mark.parametrize('kind', ['csv', 'json', 'xlsx'])
def test_export_uses_bounded_rows_and_preserves_content(tmp_path, monkeypatch, kind):
app = create_app(tmp_path)
app.state.db.upsert_catalog([record(name='=1+1'), record(code='10002', name='含逗号,大学')])
app.state.db.upsert_scores([{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}])
def forbidden(*args, **kwargs):
raise AssertionError('export materializes all catalog rows')
monkeypatch.setattr(app.state.db, 'all_catalog', forbidden)
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
response = client.get('/api/export', params={'format': kind})
assert response.status_code == 200
if kind == 'json':
data = response.json()
assert {r['school_name'] for r in data['catalog']} == {'=1+1', '含逗号,大学'}
assert data['scores'][0]['total'] == 310
elif kind == 'csv':
rows = list(csv.reader(io.StringIO(response.content.decode('utf-8-sig'))))
assert len(rows) == 3
assert any("'=1+1" in row for row in rows)
assert any('含逗号,大学' in row for row in rows)
else:
book = load_workbook(io.BytesIO(response.content), read_only=True)
rows = list(book.active.values)
assert len(rows) == 3
assert any("'=1+1" in row for row in rows)
book.close()
assert not list((tmp_path / 'exports').glob('*'))
def test_concurrent_export_is_rejected_and_lock_is_released(tmp_path, monkeypatch):
app = create_app(tmp_path)
entered, release = threading.Event(), threading.Event()
original = app.state.db.iter_catalog
def slow_rows(*args, **kwargs):
if not entered.is_set():
entered.set()
assert release.wait(5)
yield from original(*args, **kwargs)
monkeypatch.setattr(app.state.db, 'iter_catalog', slow_rows)
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
responses = []
thread = threading.Thread(target=lambda: responses.append(client.get('/api/export')))
thread.start()
assert entered.wait(3)
blocked = client.get('/api/export', params={'format': 'json'})
release.set()
thread.join(5)
assert blocked.status_code == 429
assert responses[0].status_code == 200
assert client.get('/api/export').status_code == 200
def test_disconnected_stream_closes_source_and_releases_export_lock():
from studio.exporting import ExportStream
from starlette.requests import ClientDisconnect
lock = threading.BoundedSemaphore(1)
assert lock.acquire(False)
closed = []
def source():
try:
yield b'data'
finally:
closed.append(True)
async def send(message):
if message['type'] == 'http.response.body':
raise OSError('client disconnected')
async def receive():
return {'type': 'http.disconnect'}
response = ExportStream(source(), lock.release, media_type='text/csv')
with pytest.raises(ClientDisconnect):
asyncio.run(response({'type': 'http', 'asgi': {'spec_version': '2.4'}}, receive, send))
assert closed == [True]
assert lock.acquire(False)
def test_xlsx_packaging_error_removes_xml_tempfile_without_masking_error(tmp_path, monkeypatch):
from pathlib import Path
from studio.imports import write_xlsx
from openpyxl import Workbook
paths = []
def packaging_failure(book, filename):
sheet = book.worksheets[0]
sheet.close()
paths.append(Path(sheet._writer.out))
raise OSError('archive write failed')
monkeypatch.setattr(Workbook, 'save', packaging_failure)
with pytest.raises(OSError, match='archive write failed'):
write_xlsx([record()], tmp_path / 'failed.xlsx')
assert paths and not paths[0].exists()
def test_failed_export_batch_does_not_finish_a_successful_truncated_download(tmp_path, monkeypatch):
app = create_app(tmp_path)
def broken_rows(*args, **kwargs):
yield record()
raise RuntimeError('database batch failed')
monkeypatch.setattr(app.state.db, 'iter_catalog', broken_rows)
sent = []
async def send(message):
sent.append(message)
async def receive():
await asyncio.Event().wait()
scope = {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'},
'http_version': '1.1', 'method': 'GET', 'scheme': 'http',
'path': '/api/export', 'raw_path': b'/api/export', 'root_path': '',
'query_string': b'format=csv', 'headers': [(b'host', b'127.0.0.1:8765')],
'server': ('127.0.0.1', 8765), 'client': ('127.0.0.1', 12345)}
with pytest.raises(RuntimeError, match='database batch failed'):
asyncio.run(app(scope, receive, send))
assert not any(message['type'] == 'http.response.body' and not message.get('more_body', False) for message in sent)
+147
View File
@@ -0,0 +1,147 @@
import pytest
from studio.connectors import ScoreClient
from studio.network import RemoteUnavailable
from tests.test_network import Response
class HistorySource:
def __init__(self, pages):
self.pages = pages
self.calls = []
def request(self, method, url, **kwargs):
payload = kwargs['data']
self.calls.append((url.rsplit('/', 1)[-1], payload.copy()))
data = self.pages(payload.get('page', 1))
return Response({'code': '0000', 'data': data})
def test_full_score_array_over_page_limit_is_consumed_once():
rows = [{'code': '081200', 'total': 310, 'depart_name': '信息学院'}]
rows += [{'code': '999999', 'total': 200} for _ in range(197)]
source = HistorySource(lambda page: rows)
lines = ScoreClient(source).score_lines({'school_code': '10001', 'school_name': '北京大学'}, '081200', 2025, 1)
assert len(lines) == 1
assert lines[0]['total'] == 310
assert [call[1]['page'] for call in source.calls] == [1]
def test_short_wrapped_pages_follow_explicit_total():
source = HistorySource(lambda page: {'data': [{'value': page}], 'total': 2})
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
def test_short_wrapped_pages_follow_explicit_last_page():
source = HistorySource(lambda page: {'list': [{'value': page}], 'last_page': 2})
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
@pytest.mark.parametrize('data', [None, {}, {'data': None}, {'data': [None]}, {'list': [42]}])
def test_invalid_history_data_cannot_become_a_successful_empty_unit(data):
with pytest.raises(RemoteUnavailable):
list(ScoreClient(HistorySource(lambda page: data)).pages('/example', {}))
def test_empty_full_array_is_a_valid_empty_result():
assert list(ScoreClient(HistorySource(lambda page: [])).pages('/example', {})) == []
def test_repeated_real_page_still_raises_instead_of_returning_partial_data():
source = HistorySource(lambda page: {'data': [{'value': 1}], 'total': 2})
with pytest.raises(RemoteUnavailable):
list(ScoreClient(source).pages('/example', {}))
def test_empty_page_before_declared_total_is_an_error():
source = HistorySource(lambda page: {'data': [{'value': 1}] if page == 1 else [], 'total': 2})
with pytest.raises(RemoteUnavailable):
list(ScoreClient(source).pages('/example', {}))
def test_two_majors_reuse_one_complete_school_year_score_table():
rows = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
source = HistorySource(lambda page: rows)
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
assert client.score_lines(school, '081201', 2025, 1)[0]['total'] == 320
assert len(source.calls) == 1
def test_score_cache_separates_schools_and_years_and_has_a_bound():
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}])
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
for sid in range(20):
client.score_lines(school, '081200', 2025, sid)
client.score_lines(school, '081200', 2026, 19)
before = len(source.calls)
client.score_lines(school, '081200', 2026, 19)
assert len(source.calls) == before
client.score_lines(school, '081200', 2025, 0)
assert len(source.calls) == before + 1
def test_failed_lookup_is_cached_for_the_current_client_only():
source = HistorySource(lambda page: {'data': [], 'total': 0})
client = ScoreClient(source)
assert client.school_id('暂未收录大学') is None
assert client.school_id('暂未收录大学') is None
assert len(source.calls) == 1
assert ScoreClient(source).school_id('暂未收录大学') is None
assert len(source.calls) == 2
def test_invalid_score_response_is_not_cached():
valid = False
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}] if valid else {})
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
with pytest.raises(RemoteUnavailable):
client.score_lines(school, '081200', 2025, 1)
valid = True
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
def test_history_runner_uses_thin_targets_and_bulk_checkpoints(tmp_path, monkeypatch):
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
from tests.test_bounded_storage import record
db = Database(tmp_path / 'data.db')
db.upsert_catalog([record(), record(major='081201'), record(direction='二')])
params = {'year': 2027, 'provinces': ['11'], 'score_years': [2025, 2026], 'include_plans': True}
db.job_create('history', params)
class Source:
def __init__(self):
self.calls = []
def request(self, method, url, **kwargs):
payload = kwargs['data']
self.calls.append(url.rsplit('/', 1)[-1])
if url.endswith('/schoolList'):
data = {'data': [{'school_name': '甲大学', 'school_id': 1}], 'total': 1}
elif url.endswith('/schoolScore'):
data = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
else:
data = {'data': [{'special_code': payload['keyword'], 'special_name': payload['keyword'],
'plan_id': str(payload['year']) + payload['keyword'], 'recruit_number': 1}], 'total': 1}
return Response({'code': '0000', 'data': data})
def forbidden(*args, **kwargs):
raise AssertionError('full catalog decoding or per-unit checkpoint query')
monkeypatch.setattr(db, 'all_catalog', forbidden)
monkeypatch.setattr(db, 'unit_done', forbidden)
source = Source()
manager = JobManager(db, SettingsStore(tmp_path), transport=source)
manager._scores('history', params)
assert db.job_get('history')['completed_units'] == 4
assert db.overview()['scores'] == 4
assert db.job_get('history')['records'] == 4
assert source.calls.count('schoolScore') == 2
assert source.calls.count('planListV2') == 4
before = len(source.calls)
manager._scores('history', params)
assert len(source.calls) == before
+97 -1
View File
@@ -17,7 +17,7 @@ def mysql():
assert config['database'].endswith('_test'), 'refusing to clear a production database'
db = MySQLDatabase(config)
with db.connection() as conn:
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs'):
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs', 'job_catalog_pages'):
conn.execute(f'DELETE FROM {table}')
return db
@@ -68,6 +68,77 @@ def test_mysql_tasks_logs_and_checkpoints(mysql):
assert mysql.logs('contract-job')[0]['message'] == '数据库测试'
def test_mysql_catalog_page_cache_and_bulk_checkpoints_are_durable(mysql):
mysql.job_create('cached-job', {'year': 2027})
mysql.complete_unit('cached-job', 'catalog:11:10001:081200:1')
page = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': False}
mysql.cache_catalog_page('cached-job', 'page-key', page)
from studio.mysql_database import MySQLDatabase
reopened = MySQLDatabase(mysql.config, recover_jobs=False)
assert reopened.completed_units('cached-job') == {'catalog:11:10001:081200:1'}
assert reopened.catalog_pages('cached-job') == {'page-key': page}
assert reopened.catalog_pages('another-job') == {}
def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypatch):
import ssl
with mysql.connection() as conn:
cipher = conn.execute("SHOW SESSION STATUS LIKE 'Ssl_cipher'").fetchone()[1]
if not cipher:
pytest.skip('requires a TLS-enabled MariaDB test server')
creations = []
original = ssl.create_default_context
def observe(*args, **kwargs):
creations.append(True)
return original(*args, **kwargs)
monkeypatch.setattr(ssl, 'create_default_context', observe)
for _ in range(2):
with mysql.connection() as conn:
assert conn.execute('SELECT 1').fetchone()[0] == 1
assert creations == []
def test_mysql_bounded_catalog_and_history_targets_preserve_filters(mysql):
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
scores = select_score_lines([{'code': '081200', 'total': 310, 'depart_name': '计算机学院'}], '081200', '10001', '北京大学', 2026)
mysql.upsert_scores(scores)
targets = list(mysql.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
assert len(targets) == 1
assert targets[0]['major_code'] == '081200'
assert 'data' not in targets[0]
rows = list(mysql.iter_catalog({'year': 2027}, batch_size=1))
assert len(rows) == 2
assert len(list(mysql.iter_catalog({'year': 2026, 'max_score': 310}, batch_size=1))) == 2
assert len(list(mysql.iter_scores({('10001', '081200')}, batch_size=1))) == 1
def test_mysql_unit_commit_is_atomic_and_idempotent(mysql):
mysql.job_create('atomic-job', {})
rows = normalize_catalog(raw_catalog(), 2027)
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
assert mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert not mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert mysql.job_get('atomic-job')['records'] == 2
assert mysql.job_get('atomic-job')['completed_units'] == 1
assert mysql.overview()['records'] == 2
assert mysql.overview()['scores'] == 1
def test_mysql_failed_unit_rolls_back_every_write(mysql, monkeypatch):
mysql.job_create('atomic-job', {})
def fail(*args):
raise RuntimeError('interrupted unit')
monkeypatch.setattr(mysql, '_school', fail)
rows = normalize_catalog(raw_catalog(), 2027)
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
with pytest.raises(RuntimeError):
mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert mysql.job_get('atomic-job')['records'] == 0
assert mysql.job_get('atomic-job')['completed_units'] == 0
assert mysql.overview()['records'] == mysql.overview()['scores'] == 0
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'snapshot.db')
@@ -77,16 +148,30 @@ def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_p
source.job_create('migrated-job', {'year': 2027})
source.complete_unit('migrated-job', 'catalog:11:10001:081200:1')
source.log('migrated-job', 'info', 'preserve log')
source.cache_catalog_page('migrated-job', 'migration-page', {'list': [], 'nextPageAvailable': False})
counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql)
assert counts['catalog'] == 2 and counts['scores'] == 1
assert mysql.record(1)['favorite']
assert mysql.query_catalog({})['items'][0]['score_total'] == 320
assert mysql.job_get('migrated-job')['completed_units'] == 1
assert mysql.logs('migrated-job')[0]['id'] == 1
assert counts['job_catalog_pages'] == 1
assert mysql.catalog_pages('migrated-job')['migration-page']['list'] == []
with pytest.raises(ValueError, match='非空'):
migrate_sqlite(tmp_path / 'snapshot.db', mysql)
def test_legacy_snapshot_without_catalog_page_table_can_still_be_migrated(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'legacy.db')
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
with source.connection() as conn:
conn.execute('DROP TABLE job_catalog_pages')
counts = migrate_sqlite(source.path, mysql)
assert counts['catalog'] == 2
assert counts['job_catalog_pages'] == 0
def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'bad.db')
@@ -97,3 +182,14 @@ def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
migrate_sqlite(tmp_path / 'bad.db', mysql)
assert mysql.overview()['records'] == 0
assert mysql.overview()['scores'] == 0
def test_rejected_migration_cli_does_not_pause_destination_jobs(mysql, tmp_path):
import subprocess
import sys
source = Database(tmp_path / 'source.db')
mysql.job_create('existing-running-job', {'year': 2027})
mysql.job_update('existing-running-job', status='running')
result = subprocess.run([sys.executable, '-m', 'studio.migrate', str(source.path), '--config', os.environ['STUDIO_MYSQL_TEST_CONFIG']], capture_output=True, text=True)
assert result.returncode != 0
assert mysql.job_get('existing-running-job')['status'] == 'running'
+98
View File
@@ -0,0 +1,98 @@
import importlib.util
import queue
import threading
import time
from pathlib import Path
def maintainer_class():
path = Path(__file__).resolve().parents[1] / 'deploy/proxy_scheduler.py'
spec = importlib.util.spec_from_file_location('bounded_proxy_scheduler', path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.ProxyMaintainer
def test_detection_workers_are_bounded_reused_and_rounds_do_not_overlap():
state = {'active': 0, 'peak': 0, 'types': set(), 'mixed': False, 'done': [], 'thread_ids': set()}
lock = threading.Lock()
class Checker:
def __init__(self, work_type, target_queue, name):
self.work_type, self.queue = work_type, target_queue
def run(self):
while True:
try:
item = self.queue.get(block=False)
except queue.Empty:
return
with lock:
state['active'] += 1
state['peak'] = max(state['peak'], state['active'])
state['types'].add(self.work_type)
state['mixed'] |= len(state['types']) > 1
state['thread_ids'].add(threading.get_ident())
time.sleep(.01)
with lock:
state['done'].append(item)
state['active'] -= 1
if not state['active']:
state['types'].clear()
self.queue.task_done()
maintainer = maintainer_class()(workers=2, checker_class=Checker)
first = threading.Thread(target=maintainer.validate, args=('raw', range(10)))
second = threading.Thread(target=maintainer.validate, args=('use', range(10, 20)))
first.start(); second.start()
first.join(4); second.join(4)
assert not first.is_alive() and not second.is_alive()
before = set(state['thread_ids'])
maintainer.validate('raw', range(20, 25))
maintainer.close()
assert sorted(state['done']) == list(range(25))
assert state['peak'] <= 2
assert state['mixed'] is False
assert state['thread_ids'] == before
def test_stopped_detector_does_not_begin_new_rounds():
class Forbidden:
def __init__(self, *args):
raise AssertionError('detection began after shutdown')
maintainer = maintainer_class()(workers=2, checker_class=Forbidden)
maintainer.close()
maintainer.validate('raw', [1, 2, 3])
def test_failed_worker_waits_for_other_workers_before_next_round():
slow_started, release, next_round = threading.Event(), threading.Event(), threading.Event()
class Checker:
def __init__(self, work_type, target_queue, name):
self.kind, self.name = work_type, name
def run(self):
if self.kind == 'raw' and self.name == 'worker_00':
assert slow_started.wait(2)
raise RuntimeError('validator failed')
if self.kind == 'raw':
slow_started.set()
assert release.wait(3)
maintainer = maintainer_class()(workers=2, checker_class=Checker)
failures = []
def first_round():
try:
maintainer.validate('raw', [1, 2])
except RuntimeError:
failures.append(True)
first = threading.Thread(target=first_round)
first.start()
assert slow_started.wait(2)
def second_proxies():
next_round.set()
yield 3
second = threading.Thread(target=maintainer.validate, args=('use', second_proxies()))
second.start()
began_early = next_round.wait(.15)
release.set()
first.join(3); second.join(3)
maintainer.close()
assert began_early is False
assert next_round.is_set()
assert failures == [True]
+45 -2
View File
@@ -3,8 +3,10 @@ import random
import os
import pytest
from studio.auth import CookieVault
from tests.test_network import setup_network
from studio.auth import CookieVault, AuthService, LOGIN_URL, CATALOG_ENTRY
from studio.connectors import CatalogClient
from studio.network import RemoteUnavailable
from tests.test_network import setup_network, Remote, Response
def test_requests_sample_a_fresh_delay_inside_configured_bounds(tmp_path, monkeypatch):
@@ -45,6 +47,47 @@ def test_route_pin_defers_rotation_until_login_proof_finishes(tmp_path):
assert transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] == 'http://5.6.7.8:9090'
class RouteBoundSession(Remote):
def __init__(self):
super().__init__()
self.bound = set()
self.form_route = None
def request(self, method, url, **kwargs):
route = kwargs.get('proxies', {}).get('https')
if url == LOGIN_URL and method == 'GET':
self.form_route = route
return Response({}, text='<input type="hidden" name="lt" value="test"><input type="hidden" name="execution" value="test">')
if url == LOGIN_URL and method == 'POST':
if route != self.form_route:
raise RemoteUnavailable('login form and credentials used different routes')
return Response({})
if url == CATALOG_ENTRY:
self.bound = {route}
return Response({})
if url.endswith('/yjfxs.do'):
return Response({'flag': True, 'msg': {'list': [{'direction': 'confirmed'}]}}) if route in self.bound else Response({'flag': False, 'msg': '请登录'})
return super().request(method, url, **kwargs)
@pytest.mark.parametrize('threshold', [1, 2])
def test_auth_recovery_preserves_proven_route_through_catalog_retry(tmp_path, threshold):
settings, _, transport = setup_network(tmp_path, remote=RouteBoundSession())
settings.update({'proxy_rotate_every': threshold})
auth = AuthService(transport, CookieVault(tmp_path))
assert auth.check()['valid']
payload = CatalogClient(transport, auth).fetch_page('yjfxs', {'curPage': 2, 'start': 10, 'pageSize': 10})
assert payload['list'][0]['direction'] == 'confirmed'
def test_credential_login_pins_form_submission_and_verification(tmp_path):
settings, _, transport = setup_network(tmp_path, remote=RouteBoundSession())
settings.update({'proxy_rotate_every': 2})
transport.request('GET', 'https://yz.chsi.com.cn/zsml/')
auth = AuthService(transport, CookieVault(tmp_path))
assert auth.login('test-user', 'test-password')['valid']
def test_linux_cookie_vault_encrypts_and_preserves_domain_cookies(tmp_path, monkeypatch):
monkeypatch.setattr(platform, 'system', lambda: 'Linux')
cookies = [{'name': 'JSESSIONID', 'value': 'private-account-cookie', 'domain': 'account.chsi.com.cn'},