Compare commits

..
4 Commits
25 changed files with 1422 additions and 116 deletions
+6
View File
@@ -28,6 +28,8 @@ Debian 12 / MariaDB 服务器模式、HTTPS 访问与代理池后台运行见 [
4. 在「采集任务」选择内容、目录年份、地区、专业和院校关键词。建议先按一个地区、一个专业或一所院校验证,再创建全国任务。专业代码可以从内置目录选择,也可直接输入 6 位代码。 4. 在「采集任务」选择内容、目录年份、地区、专业和院校关键词。建议先按一个地区、一个专业或一所院校验证,再创建全国任务。专业代码可以从内置目录选择,也可直接输入 6 位代码。
5. 查看任务日志。暂停可继续;需要登录、代理或限流解除时,处理后点击恢复;取消保留已有数据。关闭或重启服务会保留可恢复断点。 5. 查看任务日志。暂停可继续;需要登录、代理或限流解除时,处理后点击恢复;取消保留已有数据。关闭或重启服务会保留可恢复断点。
任务会保存本次采集的院校和专业列表分页,登录恢复或服务重启后复用,并一次加载已完成单元。界面显示当前核对的院校和正在请求的页数。修复前创建的任务首次恢复仍需建立列表缓存;新建任务会获取新列表。研究方向和登录验证保持实时请求,异常分页不会写入缓存。
研招网目录第一页可匿名返回,包括部分研究方向;不能用第一页判断登录成功。工作台访问研招目录的登录入口,将已有学信网 Cookie 同步到目录会话,再验证需要登录的第二页。采集开始/恢复时自动完成此检查;途中遇到登录提示,只同步一次并重试原页,失败后保留断点并提示重新登录。验证码仍需在浏览器人工完成,限流和网络失败会单独提示。登录卡片会持续更新状态;切换网络出口后需要重新验证,原 Cookie 保留。 研招网目录第一页可匿名返回,包括部分研究方向;不能用第一页判断登录成功。工作台访问研招目录的登录入口,将已有学信网 Cookie 同步到目录会话,再验证需要登录的第二页。采集开始/恢复时自动完成此检查;途中遇到登录提示,只同步一次并重试原页,失败后保留断点并提示重新登录。验证码仍需在浏览器人工完成,限流和网络失败会单独提示。登录卡片会持续更新状态;切换网络出口后需要重新验证,原 Cookie 保留。
分页使用接口返回的下一页位置和实际页大小。研招网可能将每页条数限制为 10,程序按实际位置继续,避免请求较大页时遗漏记录。目录采集先核对研招页面配置中的年份,避免将当前目录标为历史年份。 分页使用接口返回的下一页位置和实际页大小。研招网可能将每页条数限制为 10,程序按实际位置继续,避免请求较大页时遗漏记录。目录采集先核对研招页面配置中的年份,避免将当前目录标为历史年份。
@@ -60,6 +62,10 @@ Debian 12 / MariaDB 服务器模式、HTTPS 访问与代理池后台运行见 [
可将当前筛选结果导出 CSV、Excel 或完整 JSON。表格导出保留院系/方向代码和字段口径,回导不会凭空增加同一记录。JSON 同时保留相关学校专业的历史分数线;凭据不进入导出。CSV/Excel 对可能被表格软件当作公式的文本加保护前缀。 可将当前筛选结果导出 CSV、Excel 或完整 JSON。表格导出保留院系/方向代码和字段口径,回导不会凭空增加同一记录。JSON 同时保留相关学校专业的历史分数线;凭据不进入导出。CSV/Excel 对可能被表格软件当作公式的文本加保护前缀。
全量导出按 200 条分批读取,CSV/JSON 流式下载,Excel 使用写入模式和临时文件;同一实例一次只执行一个导出,重复请求返回 429。下载结束或中断后清理临时资源。导出期间新增的记录留待下次导出;导出不是事务一致性备份。大于 100 MB 的文件仍受导入上限限制,可按地区等条件分别导出;完整数据库备份按服务器部署说明操作。
历史采集在招生目录阶段完成后开始,按院校和年份复用全校分数线,每次最多缓存 8 份,历史计划继续逐专业获取。接口直接返回的数组按完整结果处理,带分页信息的结果按实际数量继续;不完整的响应不会被写成成功断点。数据、保存计数和完成断点在同一数据库事务中提交。
## 本地数据与开发 ## 本地数据与开发
所有运行数据位于项目 `.runtime`:`studio.db` 保存目录、分数、收藏、任务和断点;`settings.json` 保存非机密配置;`chsi-cookie.enc` 由当前 Windows 用户的 DPAPI 加密。密码不持久化,接口不返回 Cookie。备份数据库时先停止工作台,再复制 `.runtime`;恢复加密 Cookie 需要原 Windows 用户环境,否则重新登录即可。 所有运行数据位于项目 `.runtime`:`studio.db` 保存目录、分数、收藏、任务和断点;`settings.json` 保存非机密配置;`chsi-cookie.enc` 由当前 Windows 用户的 DPAPI 加密。密码不持久化,接口不返回 Cookie。备份数据库时先停止工作台,再复制 `.runtime`;恢复加密 Cookie 需要原 Windows 用户环境,否则重新登录即可。
+4 -1
View File
@@ -8,9 +8,12 @@ Requires=docker.service
User=kaoyan User=kaoyan
Group=kaoyan Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py schedule Environment=MALLOC_ARENA_MAX=2
Environment=PROXY_CHECK_WORKERS=8
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python /opt/kaoyan-studio-deploy/app/deploy/proxy_scheduler.py
Restart=on-failure Restart=on-failure
RestartSec=10 RestartSec=10
TimeoutStopSec=120
UMask=0077 UMask=0077
PrivateTmp=true PrivateTmp=true
ProtectSystem=full ProtectSystem=full
+1
View File
@@ -9,6 +9,7 @@ User=kaoyan
Group=kaoyan Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/app WorkingDirectory=/opt/kaoyan-studio-deploy/app
EnvironmentFile=/etc/kaoyan-studio.env EnvironmentFile=/etc/kaoyan-studio.env
Environment=MALLOC_ARENA_MAX=2
ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio
Restart=on-failure Restart=on-failure
RestartSec=10 RestartSec=10
+122
View File
@@ -0,0 +1,122 @@
"""Run upstream proxy_pool checks with reusable workers and serialized jobs."""
import os
import queue
import signal
import sys
import threading
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
APP_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(APP_ROOT))
from studio.memory import release_idle_memory
class StopQueue(queue.Queue):
def __init__(self, stopping):
super().__init__()
self.stopping = stopping
def get(self, *args, **kwargs):
if self.stopping.is_set():
raise queue.Empty
return super().get(*args, **kwargs)
class ProxyMaintainer:
def __init__(self, workers=8, checker_class=None, fetcher_factory=None, handler_factory=None, pool_min=20):
self.worker_count = max(1, min(20, int(workers)))
self.checker_class = checker_class
self.fetcher_factory, self.handler_factory = fetcher_factory, handler_factory
self.pool_min = pool_min
self.stopping = threading.Event()
self.pipeline = threading.RLock()
self.workers = ThreadPoolExecutor(max_workers=self.worker_count, thread_name_prefix='proxy-check')
def validate(self, work_type, proxies):
with self.pipeline:
if self.stopping.is_set():
return
target = StopQueue(self.stopping)
for proxy in proxies:
if self.stopping.is_set():
return
target.put(proxy)
if self.checker_class is None:
from helper.check import _ThreadChecker
self.checker_class = _ThreadChecker
count = min(self.worker_count, target.qsize())
futures = [self.workers.submit(self.checker_class(work_type, target, f'worker_{i:02}').run)
for i in range(count)]
try:
failure = None
for future in futures:
try:
future.result()
except Exception as exc:
if failure is None:
failure = exc
if failure is not None:
raise failure
finally:
release_idle_memory()
def fetch(self):
with self.pipeline:
if not self.stopping.is_set():
self.validate('raw', self.fetcher_factory().run())
def check(self):
with self.pipeline:
if self.stopping.is_set():
return
handler = self.handler_factory()
if handler.db.getCount().get('total', 0) < self.pool_min:
self.fetch()
self.validate('use', handler.getAll())
def stop(self):
self.stopping.set()
def close(self):
self.stop()
with self.pipeline:
self.workers.shutdown(wait=True, cancel_futures=True)
def main():
proxy_root = Path(os.environ.get('STUDIO_PROXY_POOL_ROOT', APP_ROOT.parent / 'proxy_pool'))
sys.path.insert(0, str(proxy_root))
from apscheduler.schedulers.blocking import BlockingScheduler
from helper.check import _ThreadChecker
from helper.fetch import Fetcher
from handler.configHandler import ConfigHandler
from handler.logHandler import LogHandler
from handler.proxyHandler import ProxyHandler
config = ConfigHandler()
maintainer = ProxyMaintainer(workers=int(os.environ.get('PROXY_CHECK_WORKERS', '8')),
checker_class=_ThreadChecker, fetcher_factory=Fetcher,
handler_factory=ProxyHandler, pool_min=config.poolSizeMin)
scheduler = BlockingScheduler(timezone=config.timezone, logger=LogHandler('scheduler'),
executors={'default': {'type': 'threadpool', 'max_workers': 1}},
job_defaults={'coalesce': True, 'max_instances': 1, 'misfire_grace_time': 120})
scheduler.add_job(maintainer.fetch, 'interval', minutes=5, id='proxy_fetch')
scheduler.add_job(maintainer.check, 'interval', minutes=2, id='proxy_check',
next_run_time=datetime.now(timezone.utc))
def stop(signum, frame):
maintainer.stop()
if scheduler.running:
scheduler.shutdown(wait=False)
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
if not maintainer.stopping.is_set():
scheduler.start()
finally:
maintainer.close()
if __name__ == '__main__':
main()
+22
View File
@@ -0,0 +1,22 @@
# 目录任务恢复与进度修复
2026-10-11,`feat/debian-server`。
恢复全国目录任务时,已完成单元虽然不会重复保存,但院校、专业列表会从第一地区重新请求。每个已完成专业还会创建 MariaDB 连接查询断点。用户因此看到记录数和日志长时间不变。
服务器实测原短连接加断点查询平均 0.135 秒;复用已协商的 TLS 上下文后约 0.017 秒。一次读取 4,284 个断点约 0.034 秒。
修复包括:
- 一次加载该任务已完成单元,用集合核对专业断点。
- 在 `job_catalog_pages` 中按任务和请求参数保存院校、专业列表的有效分页;恢复时复用。研究方向、目录年份及登录权限验证保持实时请求。
- 先验证必需代码、分页位置和重复页面,再缓存、返回记录。失败响应不缓存,源站恢复后可以重新请求。
- 更新当前院校、列表/方向页码和断点扫描日志;遍历缓存时仍检查暂停和取消。
- MariaDB 已协商 TLS 时复用相同的 TLS 上下文。数据库事务和短连接的关闭方式保持原有行为。
- SQLite 和 MariaDB 自动创建缓存表;迁移同时兼容没有该表的旧快照。
旧任务首次使用修复仍需获取并保存列表分页。缓存限定于该任务;需要获取更新后的院校/专业列表时新建任务。
验证:Windows 后端 87 passed、9 skipped;Debian 后端 82 passed、14 skipped,包含独立 MariaDB 测试库和 TLS 测试。跳过项为平台或未配置独立数据库的测试。两平台均有既有 Starlette TestClient 的 httpx 弃用提示,不影响生产采集。
11 项新增目录恢复回归覆盖跨服务重启复用分页、批量断点、扫描进度、任务隔离、无效分页、重复页面,以及缺失/空/错误类型的院校与专业代码。独立只读审查未发现剩余 Critical 或 Important 问题。
+6
View File
@@ -16,6 +16,8 @@
安装模板在 `deploy/`。Python 依赖按 `requirements.lock` 安装;前端 `npm ci && npm run build`。不要启动多个工作台进程连接同一数据库,否则任务恢复状态会互相影响。原有站点、数据库和 fail2ban 配置不作修改。 安装模板在 `deploy/`。Python 依赖按 `requirements.lock` 安装;前端 `npm ci && npm run build`。不要启动多个工作台进程连接同一数据库,否则任务恢复状态会互相影响。原有站点、数据库和 fail2ban 配置不作修改。
代理调度器运行本项目 `deploy/proxy_scheduler.py` 包装器,继续使用原版代理池的获取源、检测逻辑和 Redis 数据,不修改上游源码。默认复用 8 个检测线程,抓取与检查轮次串行运行,避免定时任务重叠。工作台和调度器设定 `MALLOC_ARENA_MAX=2`;每次导出/检查结束尝试释放分配器空闲内存。该操作不释放仍在使用的数据,实际内存以运行监测为准。
## 操作 ## 操作
```bash ```bash
@@ -28,6 +30,8 @@ systemctl restart kaoyan-studio
默认每次请求随机等待 1.2–2.4 秒,每 100 次成功请求轮换代理。设置页可修改上下限和轮换次数;0 表示关闭自动轮换。代理页显示实际成功请求路由、计数和切换次数。登录表单、会话同步、权限验证及紧随其后的失败页重试保持同一路由。免费代理池初次启动可能没有 HTTPS 候选,部署时保留了原电脑代理池的候选作为备用;这些候选仍会经过实际请求与失败隔离。 默认每次请求随机等待 1.2–2.4 秒,每 100 次成功请求轮换代理。设置页可修改上下限和轮换次数;0 表示关闭自动轮换。代理页显示实际成功请求路由、计数和切换次数。登录表单、会话同步、权限验证及紧随其后的失败页重试保持同一路由。免费代理池初次启动可能没有 HTTPS 候选,部署时保留了原电脑代理池的候选作为备用;这些候选仍会经过实际请求与失败隔离。
当前全国任务采用 0 次自动轮换,维持通过登录验证的出口;连接失败仍会切换。Cookie 验证状态与出口关联,出口变化后需要重新验证,不宜为了让 IP 显示变化而频繁轮换。HTTP 会话最多保留 4 个代理连接池,淘汰旧池时主动关闭连接。
Linux 无桌面环境,界面提供研招网登录入口与粘贴 Cookie。请在自己的电脑浏览器打开研招目录并登录,从开发者工具中复制目录接口请求的单行 Cookie,粘贴到工作台后点验证。服务器重新同步会话并检查受保护的第二页。迁移 Windows Cookie 时只经 SSH 传输加密封装,在服务器重新加密;不继承原验证标记。 Linux 无桌面环境,界面提供研招网登录入口与粘贴 Cookie。请在自己的电脑浏览器打开研招目录并登录,从开发者工具中复制目录接口请求的单行 Cookie,粘贴到工作台后点验证。服务器重新同步会话并检查受保护的第二页。迁移 Windows Cookie 时只经 SSH 传输加密封装,在服务器重新加密;不继承原验证标记。
## 迁移与验证 ## 迁移与验证
@@ -67,3 +71,5 @@ cp -a /var/lib/kaoyan-studio /var/backups/kaoyan-studio/runtime
同时保存数据库、运行目录(含 `cookie.key`)、私密 MySQL 配置和访问认证文件。不要把这些文件上传 Git 或公开网盘。升级前备份,停止工作台,拉取该部署分支并构建前端、安装锁定依赖,再启动工作台。原 Windows 项目继续保留 SQLite 模式,可作为原始备份。 同时保存数据库、运行目录(含 `cookie.key`)、私密 MySQL 配置和访问认证文件。不要把这些文件上传 Git 或公开网盘。升级前备份,停止工作台,拉取该部署分支并构建前端、安装锁定依赖,再启动工作台。原 Windows 项目继续保留 SQLite 模式,可作为原始备份。
代理池使用 [jhao104/proxy_pool](https://github.com/jhao104/proxy_pool),数据库适配参考 [PyMySQL 官方文档](https://pymysql.readthedocs.io/en/latest/user/examples.html) 和 [MariaDB 更新冲突记录的语法](https://mariadb.com/docs/server/reference/sql-statements/data-manipulation/inserting-loading-data/insert-on-duplicate-key-update)。 代理池使用 [jhao104/proxy_pool](https://github.com/jhao104/proxy_pool),数据库适配参考 [PyMySQL 官方文档](https://pymysql.readthedocs.io/en/latest/user/examples.html) 和 [MariaDB 更新冲突记录的语法](https://mariadb.com/docs/server/reference/sql-statements/data-manipulation/inserting-loading-data/insert-on-duplicate-key-update)。
2026-10-11 CDN 改为 HTTPS 回源后,正式域名已通过实际认证 API 和浏览器验收,不再循环跳转。目录查询 27,901 条、同来源 POST 状态检查成功,跨来源修改拒绝,网页和 API 返回 no-store。备用 9443 入口保留。学信网 Cookie 仍需更新,原全国任务仍保持手动暂停。
+41
View File
@@ -0,0 +1,41 @@
# 服务器验收记录
日期:2026-10-11。代码:`feat/debian-server`;运行版本包含 `fcdf6f2`。这份记录只列公开运行状态,不包含访问密码、数据库密码、Cookie 或密钥。
## 自动检查
| 检查 | 结果 |
| --- | --- |
| Windows 后端完整测试 | 76 通过,6 个需要实际 MariaDB 的测试跳过 |
| Debian Python 3.11 后端完整测试 | 68 通过,14 个 Windows 平台测试跳过;包含 6 个实际 MariaDB 集成测试 |
| 前端测试 | Windows、Debian 均 3 通过 |
| TypeScript 与 Vite 生产构建 | 两端均通过,服务器实际使用构建后的网页 |
| 独立全分支审查 | 登录验证后的重试固定出口、账号登录固定出口、迁移拒绝不修改旧任务三项问题均已修复并通过回归验证 |
| `nginx -t` | 全服务器配置通过 |
后端有一条既有 Starlette 弃用提示。真实 MariaDB 测试使用独立 `kaoyan_studio_test` 数据库,未把测试记录写入正式库。
## 实际服务与数据
- 工作台、代理 API、抓取调度器均为 `active`、`enabled`;独立 Redis 容器运行且设置 `unless-stopped`。
- MariaDB 10.11.18,正式数据库 `kaoyan_studio`;HTTP 健康接口报告 `database=mysql`、`platform=linux`。
- 从本机在线 SQLite 快照迁移目录 27,901 条、历史线 0 条、院校索引 559 条、任务 4 个、完成单元 4,302 条、日志 4,319 条。
- 目录、历史线、任务、完成单元、日志与原 SQLite 内容相同;启动时两个院校索引名称被内置种子刷新,详见部署说明。目录的原始名称不变。
- 原全国任务 `19f9b363a1dd4f2b8a384d84351b1b61` 保持手动暂停,记录 27,901、完成单元 4,282;服务器重启后仍保持暂停。本机原任务也保持暂停。
- 登录 Cookie 在服务器以权限 0600 的密钥重新加密保存。实际第二页验证返回 `login_required`,界面据实显示需重新登录;不会继承 Windows 已验证标记。
- 请求间隔上下限 1.2、2.4 秒;自动代理轮换阈值 100 次成功请求。代理池启动时已有 56 个 HTTP 候选、没有 HTTPS 候选;工作台保留原电脑的 24 个 HTTPS 备用出口,实际验证请求通过其中的代理完成。
## HTTPS 与浏览器
- `https://186.241.68.230:9443` 从用户电脑可访问;使用系统信任的有效 IP 证书,API 验收和浏览器验收均未忽略证书错误。
- 未认证请求返回 401;提供工作台账号后目录查询、详情、任务、设置、代理服务接口均返回正常结果。
- 真实跨来源修改返回 403;同来源保存原设置返回 200。
- 通过页面使用的管理 API 停止和启动代理服务成功,只操作本项目 API 与调度器,独立 Redis 保持运行。
- Edge 无界面验收:数据概览、实际目录表格、Linux 登录提示、代理池状态、手机宽度正常,没有浏览器脚本异常。
- Cookie 说明、顶部运行位置与数据保存位置显示为服务器;Linux 页面不显示打开服务器 Edge 窗口的按钮。
- IP 证书的续期模拟成功,现有 `certbot.timer` 调用 Certbot 5.8.0,续期后 reload Nginx。
- 用户改为 HTTPS 回源后,`https://kaoyan.fjayson.com` 不再重定向循环;未认证返回 401,认证后健康接口、目录、任务和代理服务接口均返回 200,页面验收通过。同来源 POST 状态检查成功,跨来源修改返回 403,响应包含 `Cache-Control: private, no-store`。DNS 指向 CDN 节点本身不影响部署。
## 私密备份
服务器 `/var/backups/kaoyan-studio/` 权限 0700,数据库备份权限 0600;保存数据库、含 Cookie 加密密钥的运行目录、私密数据库配置和 Nginx 认证文件。原 SQLite 快照保留在部署目录的 `incoming/`。访问账号单独交付到用户电脑。
@@ -30,34 +30,38 @@
Files: studio/settings.py, studio/network.py, studio/auth.py, tests/test_server_features.py. Files: studio/settings.py, studio/network.py, studio/auth.py, tests/test_server_features.py.
- [ ] Add failing tests for varied bounded delays, invalid bounds, automatic route changes, and pinned verification. - [x] Add failing tests for varied bounded delays, invalid bounds, automatic route changes, and pinned verification.
- [ ] Implement interval maximum and proxy rotation settings, actual request route/count reporting, and route pinning during auth verification. - [x] Implement interval maximum and proxy rotation settings, actual request route/count reporting, and route pinning during auth verification.
- [ ] Run network and auth tests, then commit. - [x] Run network and auth tests, then commit.
### Task 2: MariaDB backend and migration ### Task 2: MariaDB backend and migration
Files: studio/mysql_database.py, studio/migrate.py, studio/api.py, tests/test_mysql.py, pyproject.toml. Files: studio/mysql_database.py, studio/migrate.py, studio/api.py, tests/test_mysql.py, pyproject.toml.
- [ ] Write tests against a real isolated MariaDB database for upserts, historical scopes, favorites, jobs, and atomic migration. - [x] Write tests against a real isolated MariaDB database for upserts, historical scopes, favorites, jobs, and atomic migration.
- [ ] Implement MySQLDatabase.connection(), preserving the existing Database API; load private database configuration through STUDIO_DATABASE_CONFIG. - [x] Implement MySQLDatabase.connection(), preserving the existing Database API; load private database configuration through STUDIO_DATABASE_CONFIG.
- [ ] Implement migrate_sqlite(source, target) preserving IDs, logs and unit keys and rejecting nonempty targets. - [x] Implement migrate_sqlite(source, target) preserving IDs, logs and unit keys and rejecting nonempty targets.
- [ ] Verify all existing tests and real MariaDB integration tests, then commit. - [x] Verify all existing tests and real MariaDB integration tests, then commit.
### Task 3: Linux operation and user interface ### Task 3: Linux operation and user interface
Files: studio/auth.py, studio/linux_pool.py, studio/pool_service.py, studio/__main__.py, studio/api.py, frontend/src/App.tsx, frontend/src/types.ts. Files: studio/auth.py, studio/linux_pool.py, studio/pool_service.py, studio/__main__.py, studio/api.py, frontend/src/App.tsx, frontend/src/types.ts.
- [ ] Add failing tests for Linux vault roundtrip/permissions, server host filtering, and interrupted-task recovery selection. - [x] Add failing tests for Linux vault roundtrip/permissions, server host filtering, and interrupted-task recovery selection.
- [ ] Implement encrypted Linux storage, explicit manual-cookie login instructions, Linux systemd pool controls, configured hosts/data directory, and opt-in restart resumption. - [x] Implement encrypted Linux storage, explicit manual-cookie login instructions, Linux systemd pool controls, configured hosts/data directory, and opt-in restart resumption.
- [ ] Add interval range, rotation settings and counters to the UI; use capability metadata for platform instructions. - [x] Add interval range, rotation settings and counters to the UI; use capability metadata for platform instructions.
- [ ] Run full backend tests and frontend tests/build, then commit. - [x] Run full backend tests and frontend tests/build, then commit.
### Task 4: Provision, migrate, and deploy ### Task 4: Provision, migrate, and deploy
Files: deploy/*, docs/debian-deployment.md, README.md. Files: deploy/*, docs/debian-deployment.md, README.md.
- [ ] Prepare reproducible systemd, Nginx and private configuration templates with rollback and backup instructions. - [x] Prepare reproducible systemd, Nginx and private configuration templates with rollback and backup instructions.
- [ ] Provision independent MariaDB account/database, dedicated loopback Redis, proxy pool and workbench services. - [x] Provision independent MariaDB account/database, dedicated loopback Redis, proxy pool and workbench services.
- [ ] Transfer online SQLite backup and credentials through SSH without exposing secrets; verify migrated data and server login. - [x] Transfer online SQLite backup and credentials through SSH without exposing secrets; verify migrated data and server login.
- [ ] Validate authenticated HTTPS, browser workflows, proxy API/scheduler and restart persistence; resume migrated task only after valid authentication. - [x] Validate authenticated IP HTTPS, browser workflows, proxy API/scheduler and restart persistence; preserve the manually paused migrated task and report its actual login state.
- [ ] Fresh reviewer checks the whole branch; fix material findings and verify again. Publish the deployment branch to the user's repository. - [x] Fresh reviewer checks the whole branch; fix material findings and verify again. Publish the deployment branch to the user's repository.
### External follow-up
- [x] User changed CDN origin to HTTPS; authenticated public domain access, browser pages, same-origin POST and cross-origin rejection all verified. Authenticated IP HTTPS access remains available.
+65 -25
View File
@@ -2,6 +2,8 @@
import json import json
import uuid import uuid
import os import os
import tempfile
import threading
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse
@@ -11,7 +13,9 @@ from fastapi.staticfiles import StaticFiles
from studio.auth import AuthService, CookieVault from studio.auth import AuthService, CookieVault
from studio.database import Database from studio.database import Database
from studio.domain import PROVINCES from studio.domain import PROVINCES
from studio.imports import import_file, export_sheet from studio.imports import import_file, write_xlsx
from studio.exporting import ExportStream, csv_chunks, json_chunks, file_chunks
from studio.memory import release_idle_memory
from studio.jobs import JobManager from studio.jobs import JobManager
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
from studio.settings import SettingsStore from studio.settings import SettingsStore
@@ -20,6 +24,31 @@ from studio.pool_service import PoolService
ROOT = Path(__file__).resolve().parent.parent ROOT = Path(__file__).resolve().parent.parent
class LocalAccessMiddleware:
"""Check hosts and origins without buffering or wrapping response streams."""
def __init__(self, app, allowed_hosts):
self.app = app
self.allowed_hosts = allowed_hosts
async def __call__(self, scope, receive, send):
if scope['type'] != 'http':
return await self.app(scope, receive, send)
request = Request(scope)
if request.url.hostname not in self.allowed_hosts:
response = JSONResponse({'detail': '访问主机不在允许列表中'}, status_code=403)
return await response(scope, receive, send)
if request.method not in ('GET', 'HEAD', 'OPTIONS'):
origin = request.headers.get('origin')
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip('/'), 'http://127.0.0.1:5173', 'http://localhost:5173'}
if parsed.scheme not in ('http', 'https') or origin.rstrip('/') not in allowed:
response = JSONResponse({'detail': '已拒绝跨来源修改请求'}, status_code=403)
return await response(scope, receive, send)
return await self.app(scope, receive, send)
def create_app(data_dir=None): def create_app(data_dir=None):
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime") directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
directory.mkdir(parents=True, exist_ok=True) directory.mkdir(parents=True, exist_ok=True)
@@ -36,6 +65,9 @@ def create_app(data_dir=None):
transport = Transport(settings, router) transport = Transport(settings, router)
auth = AuthService(transport, CookieVault(directory)) auth = AuthService(transport, CookieVault(directory))
jobs = JobManager(db, settings, transport, auth=auth) jobs = JobManager(db, settings, transport, auth=auth)
export_lock = threading.BoundedSemaphore(1)
export_directory = directory / 'exports'
export_directory.mkdir(exist_ok=True)
if (ROOT / "data/schools.json").exists(): if (ROOT / "data/schools.json").exists():
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8"))) db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
@@ -46,6 +78,8 @@ def create_app(data_dir=None):
jobs.resume(jid) jobs.resume(jid)
yield yield
jobs.close() jobs.close()
transport.close()
router.session.close()
app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None) app = FastAPI(title="考研数据工作台", lifespan=lifespan, docs_url=None, redoc_url=None)
app.state.db, app.state.jobs = db, jobs app.state.db, app.state.jobs = db, jobs
@@ -53,19 +87,7 @@ def create_app(data_dir=None):
app.state.pool = pool app.state.pool = pool
app.state.auth = auth app.state.auth = auth
@app.middleware("http") app.add_middleware(LocalAccessMiddleware, allowed_hosts=allowed_hosts)
async def local_access(request: Request, call_next):
host = request.url.hostname
if host not in allowed_hosts:
return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
if request.method not in ("GET", "HEAD", "OPTIONS"):
origin = request.headers.get("origin")
if origin:
parsed = urlparse(origin)
allowed = {str(request.base_url).rstrip("/"), "http://127.0.0.1:5173", "http://localhost:5173"}
if parsed.scheme not in ("http", "https") or origin.rstrip("/") not in allowed:
return JSONResponse({"detail": "已拒绝跨来源修改请求"}, status_code=403)
return await call_next(request)
@app.exception_handler(ValueError) @app.exception_handler(ValueError)
async def invalid(request, exc): async def invalid(request, exc):
@@ -216,17 +238,35 @@ def create_app(data_dir=None):
def export(request: Request, format: str = "csv"): def export(request: Request, format: str = "csv"):
if format not in ("csv", "xlsx", "json"): if format not in ("csv", "xlsx", "json"):
raise ValueError("导出格式无效") raise ValueError("导出格式无效")
rows = db.all_catalog(filters(request)) selected = filters(request)
if format == "json": if not export_lock.acquire(False):
pairs = {(r["school_code"], r["major_code"]) for r in rows} raise HTTPException(429, '已有导出正在进行,请等待完成后重试')
with db.connection() as conn: temporary = None
scores = [json.loads(r["data"]) for r in conn.execute("SELECT data FROM scores") if (json.loads(r["data"])["school_code"], json.loads(r["data"])["major_code"]) in pairs] def cleanup():
payload = json.dumps({"version": 1, "catalog": rows, "scores": scores}, ensure_ascii=False, indent=2).encode() try:
mime = "application/json" if temporary is not None:
else: temporary.unlink(missing_ok=True)
payload = export_sheet(rows, format) finally:
mime = "text/csv; charset=utf-8" if format == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" export_lock.release()
return Response(payload, media_type=mime, headers={"Content-Disposition": f'attachment; filename="kaoyan-data.{format}"'}) try:
if format == 'csv':
source, mime = csv_chunks(db.iter_catalog(selected)), 'text/csv; charset=utf-8'
elif format == 'json':
source, mime = json_chunks(db, selected), 'application/json'
else:
descriptor, path = tempfile.mkstemp(prefix='kaoyan-', suffix='.xlsx', dir=export_directory)
os.close(descriptor)
temporary = Path(path)
write_xlsx(db.iter_catalog(selected), temporary)
source = file_chunks(temporary)
mime = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
return ExportStream(source, cleanup, media_type=mime,
headers={'Content-Disposition': f'attachment; filename="kaoyan-data.{format}"',
'X-Accel-Buffering': 'no'})
except BaseException:
cleanup()
release_idle_memory()
raise
dist = ROOT / "frontend/dist" dist = ROOT / "frontend/dist"
if (dist / "assets").exists(): if (dist / "assets").exists():
+85 -16
View File
@@ -1,6 +1,8 @@
"""Independent adapters for the two projects' public data sources.""" """Independent adapters for the two projects' public data sources."""
import json import json
import re import re
import hashlib
from collections import OrderedDict
from studio.domain import finish_record, integer, normalized_name, select_score_lines from studio.domain import finish_record, integer, normalized_name, select_score_lines
from studio.network import RemoteUnavailable, RateLimited from studio.network import RemoteUnavailable, RateLimited
@@ -18,8 +20,9 @@ class SourceYearMismatch(RuntimeError):
class CatalogClient: class CatalogClient:
BASE = "https://yz.chsi.com.cn/zsml/" BASE = "https://yz.chsi.com.cn/zsml/"
def __init__(self, transport, auth=None): def __init__(self, transport, auth=None, page_cache=None, save_page=None, on_request=None):
self.transport, self.auth = transport, auth self.transport, self.auth = transport, auth
self.page_cache, self.save_page, self.on_request = page_cache, save_page, on_request
def verify_year(self, year): def verify_year(self, year):
response = self.transport.request("GET", self.BASE) response = self.transport.request("GET", self.BASE)
@@ -65,8 +68,16 @@ class CatalogClient:
return data return data
def fetch_page(self, endpoint, payload): def fetch_page(self, endpoint, payload):
# Lists are a snapshot for this task. Direction records and login proofs
# remain live requests; failed/partial responses never enter the cache.
cacheable = self.page_cache is not None and endpoint in ('dws', 'dwzys')
key = self._page_key(endpoint, payload) if cacheable else None
if cacheable and key in self.page_cache:
return self.page_cache[key]
if self.on_request:
self.on_request(endpoint, payload)
try: try:
return self._fetch_page(endpoint, payload) data = self._fetch_page(endpoint, payload)
except LoginRequired as exc: except LoginRequired as exc:
if not self.auth or exc.reason == "captcha": if not self.auth or exc.reason == "captcha":
raise raise
@@ -76,27 +87,51 @@ class CatalogClient:
status = self.auth.check() status = self.auth.check()
if not status["valid"]: if not status["valid"]:
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None
return self._fetch_page(endpoint, payload) data = self._fetch_page(endpoint, payload)
return data
@staticmethod
def _page_key(endpoint, payload):
return hashlib.sha256(json.dumps([endpoint, payload], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
def _cache_page(self, endpoint, payload, data):
if self.page_cache is None or endpoint not in ('dws', 'dwzys'):
return
key = self._page_key(endpoint, payload)
if key not in self.page_cache:
if self.save_page:
self.save_page(key, data)
self.page_cache[key] = data
def pages(self, endpoint, payload, page_size=10): def pages(self, endpoint, payload, page_size=10):
seen, start = set(), 0 seen, start = set(), 0
for page in range(1, 10001): for page in range(1, 10001):
data = self.fetch_page(endpoint, {**payload, "curPage": page, "start": start, "pageSize": page_size}) page_payload = {**payload, "curPage": page, "start": start, "pageSize": page_size}
data = self.fetch_page(endpoint, page_payload)
rows = data["list"] rows = data["list"]
identifier = {'dws': 'dwdm', 'dwzys': 'zydm'}.get(endpoint)
if identifier:
for row in rows:
code = row.get(identifier) if isinstance(row, dict) else None
if isinstance(code, bool) or not isinstance(code, (str, int)) or not str(code).strip():
label = '院校' if endpoint == 'dws' else '专业'
raise RemoteUnavailable(f'研招目录{label}代码缺失或结构异常,已停止以避免缓存不完整数据')
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True) signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen: if rows and signature in seen:
raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页") raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页")
seen.add(signature) seen.add(signature)
yield from rows
more = data.get("nextPageAvailable") more = data.get("nextPageAvailable")
total = integer(data.get("totalCount")) total = integer(data.get("totalCount"))
size = integer(data.get("size")) or page_size size = integer(data.get("size")) or page_size
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size): terminal = not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size)
return
next_start = integer(data.get("startOfNextPage")) next_start = integer(data.get("startOfNextPage"))
next_start = start + len(rows) if next_start is None else next_start next_start = start + len(rows) if next_start is None else next_start
if next_start <= start: if not terminal and next_start <= start:
raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页") raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页")
self._cache_page(endpoint, page_payload, data)
yield from rows
if terminal:
return
start, page_size = next_start, size start, page_size = next_start, size
raise RemoteUnavailable("目录分页超过安全上限") raise RemoteUnavailable("目录分页超过安全上限")
@@ -119,6 +154,7 @@ class ScoreClient:
def __init__(self, transport): def __init__(self, transport):
self.transport = transport self.transport = transport
self.cache = {} self.cache = {}
self.score_cache = OrderedDict()
def post(self, endpoint, payload): def post(self, endpoint, payload):
for attempt in range(3): for attempt in range(3):
@@ -127,8 +163,13 @@ class ScoreClient:
body = response.json() body = response.json()
except ValueError: except ValueError:
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
if not isinstance(body, dict):
raise RemoteUnavailable("掌上考研响应结构已变化")
if str(body.get("code")) == "0000": if str(body.get("code")) == "0000":
return body.get("data") or {} data = body.get("data")
if not isinstance(data, (dict, list)):
raise RemoteUnavailable("掌上考研缺少有效数据,未写入不完整结果")
return data
message = str(body.get('message', body.get('msg', ''))) message = str(body.get('message', body.get('msg', '')))
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')): if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
if attempt == 2: if attempt == 2:
@@ -138,20 +179,40 @@ class ScoreClient:
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试") raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
def pages(self, endpoint, payload, limit=100): def pages(self, endpoint, payload, limit=100):
seen = set() seen, consumed = set(), 0
for page in range(1, 10001): for page in range(1, 10001):
data = self.post(endpoint, {**payload, "page": page, "limit": limit}) data = self.post(endpoint, {**payload, "page": page, "limit": limit})
rows = data.get("data", data.get("list", [])) if isinstance(data, dict) else data full_list = isinstance(data, list)
if not isinstance(rows, list): rows = data.get("data", data.get("list")) if isinstance(data, dict) else data
if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
raise RemoteUnavailable("掌上考研响应结构已变化") raise RemoteUnavailable("掌上考研响应结构已变化")
total = integer(data.get("total")) if isinstance(data, dict) else None
last = integer(data.get("last_page")) if isinstance(data, dict) else None
if isinstance(data, dict):
for field, value in (('total', total), ('last_page', last)):
if data.get(field) is not None and (value is None or value < 0 or isinstance(data[field], bool)):
raise RemoteUnavailable("掌上考研分页信息无效,未写入不完整结果")
if not rows and (total is not None and consumed < total or last is not None and page < last):
raise RemoteUnavailable("掌上考研提前返回空页,未写入不完整结果")
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True) signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen: if rows and signature in seen:
raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页") raise RemoteUnavailable("掌上考研重复返回同一页,请检查接口分页")
seen.add(signature) seen.add(signature)
consumed += len(rows)
yield from rows yield from rows
total = integer(data.get("total")) if isinstance(data, dict) else None # Bare arrays are complete responses; wrapped pages use their actual
last = integer(data.get("last_page")) if isinstance(data, dict) else None # metadata even when the server ignores the requested page size.
if len(rows) < limit or last is not None and page >= last or total is not None and page * limit >= total: terminal = full_list or not rows
if not terminal:
if last is not None:
terminal = page >= last
if terminal and total is not None and consumed < total:
raise RemoteUnavailable("掌上考研分页未返回全部记录")
elif total is not None:
terminal = consumed >= total
else:
terminal = len(rows) < limit
if terminal:
return return
raise RemoteUnavailable("历史数据分页超过安全上限") raise RemoteUnavailable("历史数据分页超过安全上限")
@@ -164,10 +225,18 @@ class ScoreClient:
if len(matches) == 1 and matches[0].get("school_id"): if len(matches) == 1 and matches[0].get("school_id"):
self.cache[key] = matches[0]["school_id"] self.cache[key] = matches[0]["school_id"]
return self.cache[key] return self.cache[key]
self.cache[key] = None
return None return None
def score_lines(self, school, major_code, year, school_id): def score_lines(self, school, major_code, year, school_id):
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year})) key = (str(school_id), int(year))
if key not in self.score_cache:
rows = list(self.pages("/school/schoolScore", {"school_id": school_id, "year": year}))
self.score_cache[key] = rows
if len(self.score_cache) > 8:
self.score_cache.popitem(last=False)
self.score_cache.move_to_end(key)
rows = self.score_cache[key]
return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year) return select_score_lines(rows, major_code, school["school_code"], school["school_name"], year)
def plans(self, school, major_code, year, school_id): def plans(self, school, major_code, year, school_id):
+103 -7
View File
@@ -19,11 +19,14 @@ CREATE TABLE IF NOT EXISTS catalog(
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT, department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL); subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code); CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
CREATE INDEX IF NOT EXISTS catalog_export ON catalog(year DESC,school_name,id);
CREATE INDEX IF NOT EXISTS catalog_targets ON catalog(year,school_code,major_code);
""" + SCORE_SCHEMA + """ """ + SCORE_SCHEMA + """
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC); CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT); CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0); CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit)); CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id TEXT,page_key TEXT,data TEXT NOT NULL,PRIMARY KEY(job_id,page_key));
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT); CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
""" """
@@ -68,6 +71,10 @@ class Database:
conn.close() conn.close()
def upsert_catalog(self, records: list[dict]): def upsert_catalog(self, records: list[dict]):
with self.connection() as conn:
self._upsert_catalog(conn, records)
def _upsert_catalog(self, conn, records):
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"] fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
values = [] values = []
for record in records: for record in records:
@@ -76,10 +83,9 @@ class Database:
if not values: if not values:
return return
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:]) update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
with self.connection() as conn: conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values) for record in records:
for record in records: self._school(conn, record)
self._school(conn, record)
def _school(self, conn, record): def _school(self, conn, record):
record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))} record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))}
@@ -92,8 +98,11 @@ class Database:
def upsert_scores(self, records): def upsert_scores(self, records):
with self.connection() as conn: with self.connection() as conn:
for row in records: self._upsert_scores(conn, records)
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
def _upsert_scores(self, conn, records):
for row in records:
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
def _query(self, filters: dict): def _query(self, filters: dict):
conditions, params = ["1=1"], [] conditions, params = ["1=1"], []
@@ -140,9 +149,73 @@ class Database:
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]} return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
def all_catalog(self, filters): def all_catalog(self, filters):
return list(self.iter_catalog(filters))
def iter_catalog(self, filters, batch_size=200):
select, params = self._query(filters) select, params = self._query(filters)
with self.connection() as conn: with self.connection() as conn:
return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)] cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM catalog').fetchone()[0]
after = None
while True:
query = 'SELECT * FROM (' + select + ') export_rows WHERE id<=?'
values = [*params, cutoff]
if after:
year, school, rid = after
if school is None:
query += ' AND (year<? OR (year=? AND (school_name IS NOT NULL OR (school_name IS NULL AND id>?))))'
values.extend([year, year, rid])
else:
query += ' AND (year<? OR (year=? AND (school_name>? OR (school_name=? AND id>?))))'
values.extend([year, year, school, school, rid])
query += ' ORDER BY year DESC,school_name,id LIMIT ?'
with self.connection() as conn:
rows = conn.execute(query, [*values, batch_size]).fetchall()
if not rows:
return
for row in rows:
yield self._catalog_row(row)
last = rows[-1]
after = (last['year'], last['school_name'], last['id'])
def iter_scores(self, pairs, batch_size=200):
after = 0
with self.connection() as conn:
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM scores').fetchone()[0]
while True:
with self.connection() as conn:
rows = conn.execute('SELECT id,school_code,major_code,data FROM scores WHERE id>? AND id<=? ORDER BY id LIMIT ?', (after, cutoff, batch_size)).fetchall()
if not rows:
return
for row in rows:
if (row['school_code'], row['major_code']) in pairs:
yield json.loads(row['data'])
after = rows[-1]['id']
def score_targets(self, filters, batch_size=200):
conditions, params = ['year=?'], [filters['year']]
if filters.get('major_code'):
conditions.append('major_code=?')
params.append(filters['major_code'])
if filters.get('school_keyword'):
conditions.append('school_name LIKE ?')
params.append('%' + filters['school_keyword'] + '%')
if filters.get('provinces'):
conditions.append("(province='' OR province IN (" + ','.join('?' for _ in filters['provinces']) + '))')
params.extend(filters['provinces'])
after = None
while True:
where, values = ' AND '.join(conditions), list(params)
if after:
where += ' AND (school_code>? OR (school_code=? AND major_code>?))'
values.extend([after[0], after[0], after[1]])
query = 'SELECT school_code,major_code,MAX(school_name) AS school_name,MAX(province) AS province,MAX(level) AS level FROM catalog WHERE ' + where + ' GROUP BY school_code,major_code ORDER BY school_code,major_code LIMIT ?'
with self.connection() as conn:
rows = conn.execute(query, [*values, batch_size]).fetchall()
if not rows:
return
yield from (dict(row) for row in rows)
last = rows[-1]
after = (last['school_code'], last['major_code'])
def _catalog_row(self, row): def _catalog_row(self, row):
data = json.loads(row["data"]) data = json.loads(row["data"])
@@ -202,10 +275,33 @@ class Database:
with self.connection() as conn: with self.connection() as conn:
return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None
def completed_units(self, jid):
with self.connection() as conn:
return {row['unit'] for row in conn.execute('SELECT unit FROM job_units WHERE job_id=?', (jid,))}
def catalog_pages(self, jid):
with self.connection() as conn:
return {row['page_key']: json.loads(row['data']) for row in conn.execute('SELECT page_key,data FROM job_catalog_pages WHERE job_id=?', (jid,))}
def cache_catalog_page(self, jid, page_key, data):
with self.connection() as conn:
conn.execute('INSERT INTO job_catalog_pages(job_id,page_key,data) VALUES(?,?,?) ON CONFLICT(job_id,page_key) DO UPDATE SET data=excluded.data', (jid, page_key, encode(data)))
def complete_unit(self, jid, unit): def complete_unit(self, jid, unit):
with self.connection() as conn: with self.connection() as conn:
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit)) conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
def save_unit(self, jid, unit, records, scores=()):
with self.connection() as conn:
if not conn.execute('SELECT 1 FROM jobs WHERE id=?', (jid,)).fetchone():
raise ValueError('任务不存在')
if not conn.execute('INSERT OR IGNORE INTO job_units VALUES(?,?)', (jid, unit)).rowcount:
return False
self._upsert_scores(conn, scores)
self._upsert_catalog(conn, records)
conn.execute('UPDATE jobs SET records=records+?,updated_at=? WHERE id=?', (len(records), now(), jid))
return True
def log(self, jid, level, message): def log(self, jid, level, message):
with self.connection() as conn: with self.connection() as conn:
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000])) conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
+66
View File
@@ -0,0 +1,66 @@
"""Bounded exports, one download at a time, with disconnect cleanup."""
import csv
import io
import json
from fastapi.responses import StreamingResponse
from studio.imports import LABELS, spreadsheet_value
from studio.memory import release_idle_memory
class ExportStream(StreamingResponse):
def __init__(self, source, cleanup, **kwargs):
super().__init__(source, **kwargs)
self.source, self.cleanup = source, cleanup
async def __call__(self, scope, receive, send):
try:
await super().__call__(scope, receive, send)
finally:
try:
self.source.close()
finally:
try:
self.cleanup()
finally:
release_idle_memory()
def csv_chunks(records):
yield b'\xef\xbb\xbf'
stream = io.StringIO(newline='')
writer = csv.writer(stream)
writer.writerow(list(LABELS.values()))
yield stream.getvalue().encode('utf-8')
stream.seek(0)
stream.truncate(0)
for record in records:
writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS])
if stream.tell() >= 65536:
yield stream.getvalue().encode('utf-8')
stream.seek(0)
stream.truncate(0)
if stream.tell():
yield stream.getvalue().encode('utf-8')
def json_chunks(database, filters):
pairs = set()
yield b'{"version":1,"catalog":['
first = True
for row in database.iter_catalog(filters):
pairs.add((row['school_code'], row['major_code']))
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
first = False
yield b'],"scores":['
first = True
for row in database.iter_scores(pairs):
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
first = False
yield b']}'
def file_chunks(path):
with path.open('rb') as stream:
while chunk := stream.read(65536):
yield chunk
+45 -18
View File
@@ -2,6 +2,7 @@
import csv import csv
import io import io
import json import json
from contextlib import suppress
import sqlite3 import sqlite3
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
@@ -94,11 +95,18 @@ def import_file(path, year=None):
book = load_workbook(path, read_only=True, data_only=True) book = load_workbook(path, read_only=True, data_only=True)
try: try:
sheet = book.active sheet = book.active
if sheet.max_row > 300001 or sheet.max_column > 100: if (sheet.max_row or 0) > 300001 or (sheet.max_column or 0) > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入") raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
values = sheet.iter_rows(values_only=True) values = sheet.iter_rows(values_only=True)
headers = next(values, []) headers = next(values, [])
rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)] if len(headers) > 100:
raise ValueError("Excel 超过 100 列,请拆分后导入")
rows = []
for index, row in enumerate(values, 2):
if index > 300001 or len(row) > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
if any(v is not None for v in row):
rows.append(dict(zip(headers, row)))
finally: finally:
book.close() book.close()
else: else:
@@ -179,28 +187,47 @@ def spreadsheet_value(value):
def export_sheet(records, kind): def export_sheet(records, kind):
rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records]
if kind == "csv": if kind == "csv":
stream = io.StringIO(newline="") stream = io.StringIO(newline="")
writer = csv.writer(stream) writer = csv.writer(stream)
writer.writerow(list(LABELS.values())) writer.writerow(list(LABELS.values()))
writer.writerows(rows) writer.writerows([spreadsheet_value(record.get(field)) for field in LABELS] for record in records)
return stream.getvalue().encode("utf-8-sig") return stream.getvalue().encode("utf-8-sig")
book = Workbook() output = io.BytesIO()
sheet = book.active write_xlsx(records, output)
sheet.title = "院校专业" return output.getvalue()
sheet.append(list(LABELS.values()))
for row in rows:
sheet.append(row) def write_xlsx(records, output):
sheet.freeze_panes = "A2" from openpyxl.cell import WriteOnlyCell
sheet.auto_filter.ref = sheet.dimensions from openpyxl.utils import get_column_letter
from openpyxl.styles import Font, PatternFill, Alignment from openpyxl.styles import Font, PatternFill, Alignment
for cell in sheet[1]: book = Workbook(write_only=True)
sheet = book.create_sheet("院校专业")
sheet.freeze_panes = "A2"
header = []
for index, label in enumerate(LABELS.values(), 1):
cell = WriteOnlyCell(sheet, value=label)
cell.font = Font(color="FFFFFF", bold=True) cell.font = Font(color="FFFFFF", bold=True)
cell.fill = PatternFill("solid", fgColor="183447") cell.fill = PatternFill("solid", fgColor="183447")
cell.alignment = Alignment(vertical="center") cell.alignment = Alignment(vertical="center")
for col in sheet.columns: header.append(cell)
sheet.column_dimensions[col[0].column_letter].width = 24 sheet.column_dimensions[get_column_letter(index)].width = 24
output = io.BytesIO() count = 1
book.save(output) try:
return output.getvalue() sheet.append(header)
for record in records:
sheet.append([spreadsheet_value(record.get(field)) for field in LABELS])
count += 1
sheet.auto_filter.ref = f'A1:{get_column_letter(len(LABELS))}{count}'
book.save(output)
except BaseException:
with suppress(Exception):
if not sheet.closed:
sheet.close()
with suppress(Exception):
if sheet._writer:
sheet._writer.cleanup()
raise
finally:
book.close()
+40 -21
View File
@@ -1,6 +1,7 @@
"""Persistent units and cooperative pause/cancel for one local worker.""" """Persistent units and cooperative pause/cancel for one local worker."""
import threading import threading
import uuid import uuid
from itertools import chain
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
from studio.domain import PROVINCES, normalize_catalog from studio.domain import PROVINCES, normalize_catalog
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
@@ -189,7 +190,13 @@ class JobManager:
self._scores(jid, params) self._scores(jid, params)
def _catalog(self, jid, params): def _catalog(self, jid, params):
client = CatalogClient(self.transport, auth=self.auth) completed = self.db.completed_units(jid)
location = '研招目录'
def on_request(endpoint, payload):
phase = {'dws': '院校列表', 'dwzys': '专业列表', 'yjfxs': '研究方向'}[endpoint]
self.db.job_update(jid, message=f"{location}:正在请求{phase}第 {payload.get('curPage', 1)} 页")
client = CatalogClient(self.transport, auth=self.auth, page_cache=self.db.catalog_pages(jid),
save_page=lambda key, data: self.db.cache_catalog_page(jid, key, data), on_request=on_request)
client.verify_year(params["year"]) client.verify_year(params["year"])
if self.auth: if self.auth:
status = self.auth.check() status = self.auth.check()
@@ -198,48 +205,60 @@ class JobManager:
self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功") self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功")
for province in params["provinces"]: for province in params["provinces"]:
self.checkpoint(jid) self.checkpoint(jid)
location = PROVINCES[province]
self.db.job_update(jid, message=location + ':正在核对院校列表')
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校") self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
for school in client.schools(province, params.get("school_keyword", "")): for school in client.schools(province, params.get("school_keyword", "")):
self.checkpoint(jid) self.checkpoint(jid)
school = {"ssdm": province, **school} school = {"ssdm": province, **school}
location = f"{PROVINCES[province]} · {school.get('dwmc', school['dwdm'])}"
self.db.job_update(jid, message=location + ':正在核对专业列表与已完成断点')
self.db.log(jid, 'info', location + ':核对专业列表与断点')
skipped = 0
for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")): for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")):
self.checkpoint(jid)
if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]): if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]):
continue continue
unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}" unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}"
if self.unit_done(jid, unit): if unit in completed:
skipped += 1
continue continue
self.checkpoint(jid) self.checkpoint(jid)
label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}" label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}"
location = label
self.db.job_update(jid, message=label) self.db.job_update(jid, message=label)
try: try:
records = [] records = []
for row in client.directions(school, major, params.get("study_mode", "")): for row in client.directions(school, major, params.get("study_mode", "")):
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major)) records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
self._save(jid, records) self.db.save_unit(jid, unit, records)
self.complete_unit(jid, unit) completed.add(unit)
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合") self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
except RemoteUnavailable as exc: except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}") self._error(jid, f"{label}:{exc}")
if skipped:
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
def _scores(self, jid, params): def _scores(self, jid, params):
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")} provinces = [PROVINCES[p] for p in params['provinces']]
records = self.db.all_catalog(filters) filters = {"year": params["year"], "major_code": params.get("major_code", ""),
targets = {} "school_keyword": params.get("school_keyword", ""), 'provinces': provinces}
for row in records: targets = iter(self.db.score_targets(filters))
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]: first = next(targets, None)
continue if first is None and params.get('major_code'):
targets[(row["school_code"], row["major_code"])] = row targets = ({**row, 'major_code': params['major_code']}
if not targets and params.get("major_code"): for row in self.db.schools(params.get('school_keyword', '')) if row['province'] in provinces)
for row in self.db.schools(params.get("school_keyword", "")): first = next(targets, None)
if row["province"] in [PROVINCES[p] for p in params["provinces"]]: if first is None:
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
if not targets:
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引") raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
completed = self.db.completed_units(jid)
self.db.log(jid, 'info', '开始补充分数线和历史计划,分批读取专业目标并复用院校年份分数线')
client = ScoreClient(self.transport) client = ScoreClient(self.transport)
for (_, code), record in targets.items(): for record in chain((first,), targets):
self.checkpoint(jid) self.checkpoint(jid)
code = record['major_code']
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")} school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")] years = [y for y in params["score_years"] if f"scores:{school['school_code']}:{code}:{y}" not in completed]
if not years: if not years:
continue continue
school_id = client.school_id(school["school_name"]) school_id = client.school_id(school["school_name"])
@@ -252,10 +271,10 @@ class JobManager:
self.db.job_update(jid, message=label) self.db.job_update(jid, message=label)
try: try:
scores = client.score_lines(school, code, year, school_id) scores = client.score_lines(school, code, year, school_id)
self.db.upsert_scores(scores)
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else [] plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
self._save(jid, plans) unit = f"scores:{school['school_code']}:{code}:{year}"
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}") self.db.save_unit(jid, unit, plans, scores=scores)
completed.add(unit)
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条") self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
except RemoteUnavailable as exc: except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}") self._error(jid, f"{label}:{exc}")
+17
View File
@@ -0,0 +1,17 @@
"""Best-effort return of idle allocator memory after large operations."""
import ctypes
import gc
import sys
def release_idle_memory():
gc.collect()
if sys.platform == 'linux':
try:
trim = ctypes.CDLL(None).malloc_trim
trim.argtypes = [ctypes.c_size_t]
trim.restype = ctypes.c_int
return bool(trim(0))
except (AttributeError, OSError):
pass
return False
+4 -1
View File
@@ -6,7 +6,7 @@ from pathlib import Path
from studio.domain import normalized_name, is_special_score from studio.domain import normalized_name, is_special_score
from studio.mysql_database import MySQLDatabase, score_key from studio.mysql_database import MySQLDatabase, score_key
TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs') TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs', 'job_catalog_pages')
def migrate_sqlite(source, target): def migrate_sqlite(source, target):
@@ -16,6 +16,7 @@ def migrate_sqlite(source, target):
counts = {} counts = {}
try: try:
conn.execute('BEGIN') conn.execute('BEGIN')
source_tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
with target.connection() as output: with target.connection() as output:
# Serialize attempted imports to the same database. # Serialize attempted imports to the same database.
acquired = output.execute("SELECT GET_LOCK('kaoyan_studio_migration',10)").fetchone()[0] acquired = output.execute("SELECT GET_LOCK('kaoyan_studio_migration',10)").fetchone()[0]
@@ -27,6 +28,8 @@ def migrate_sqlite(source, target):
raise ValueError(f'目标数据库非空({table}),拒绝覆盖;请使用空数据库') raise ValueError(f'目标数据库非空({table}),拒绝覆盖;请使用空数据库')
for table in TABLES: for table in TABLES:
counts[table] = 0 counts[table] = 0
if table == 'job_catalog_pages' and table not in source_tables:
continue # Existing snapshots predate the optional page cache.
reader = conn.execute(f'SELECT * FROM {table}') reader = conn.execute(f'SELECT * FROM {table}')
while batch := reader.fetchmany(500): while batch := reader.fetchmany(500):
rows = [] rows = []
+18 -9
View File
@@ -2,6 +2,7 @@
import hashlib import hashlib
import json import json
import re import re
import ssl
from contextlib import contextmanager from contextlib import contextmanager
import pymysql import pymysql
from studio.database import Database, encode from studio.database import Database, encode
@@ -23,6 +24,7 @@ SCHEMA = [
"CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)", "CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)",
"CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)", "CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)",
"CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))", "CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))",
"CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id VARCHAR(64),page_key VARCHAR(64),data LONGTEXT NOT NULL,PRIMARY KEY(job_id,page_key))",
"CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))", "CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))",
] ]
@@ -101,9 +103,14 @@ class MySQLDatabase(Database):
if not self.config.get('database'): if not self.config.get('database'):
raise ValueError('数据库配置缺少 database') raise ValueError('数据库配置缺少 database')
self.recovered_job_ids = [] self.recovered_job_ids = []
self._ssl_context = None
with self.connection() as conn: with self.connection() as conn:
for statement in SCHEMA: for statement in SCHEMA:
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin') conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_export'").fetchone():
conn.execute('CREATE INDEX catalog_export ON catalog(year DESC,school_name,id)')
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_targets'").fetchone():
conn.execute('CREATE INDEX catalog_targets ON catalog(year,school_code,major_code)')
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone(): if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0') conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
if recover_jobs: if recover_jobs:
@@ -113,8 +120,12 @@ class MySQLDatabase(Database):
@contextmanager @contextmanager
def connection(self): def connection(self):
raw = pymysql.connect(**self.config, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120) raw = pymysql.connect(**self.config, ssl=self._ssl_context, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
try: try:
# Retain the driver's negotiated context rather than reload its CA
# store for every short-lived query. Plain servers retain default behavior.
if self._ssl_context is None and isinstance(getattr(raw, '_sock', None), ssl.SSLSocket):
self._ssl_context = raw.ctx
yield Connection(raw) yield Connection(raw)
raw.commit() raw.commit()
except BaseException: except BaseException:
@@ -123,7 +134,7 @@ class MySQLDatabase(Database):
finally: finally:
raw.close() raw.close()
def upsert_catalog(self, records): def _upsert_catalog(self, conn, records):
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm'] fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
values = [] values = []
for record in records: for record in records:
@@ -132,12 +143,11 @@ class MySQLDatabase(Database):
if not values: if not values:
return return
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:]) update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
with self.connection() as conn: conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values) for record in records:
for record in records: self._school(conn, record)
self._school(conn, record)
def upsert_scores(self, records): def _upsert_scores(self, conn, records):
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line'] fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
values = [] values = []
for record in records: for record in records:
@@ -145,8 +155,7 @@ class MySQLDatabase(Database):
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note']))) row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
values.append([row.get(field) for field in fields]) values.append([row.get(field) for field in fields])
if values: if values:
with self.connection() as conn: conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
def favorite(self, rid, enabled): def favorite(self, rid, enabled):
# MariaDB reports changed rows; an already-selected favorite is still success. # MariaDB reports changed rows; an already-selected favorite is still success.
+31 -1
View File
@@ -3,6 +3,7 @@ import threading
import time import time
import random import random
from contextlib import contextmanager from contextlib import contextmanager
from collections import OrderedDict
import requests import requests
from studio.settings import proxy_url from studio.settings import proxy_url
@@ -19,6 +20,31 @@ class RateLimited(RuntimeError):
pass pass
class BoundedProxyAdapter(requests.adapters.HTTPAdapter):
"""Close obsolete proxy pools rather than retain every historical exit."""
def __init__(self, max_proxy_pools=4, **kwargs):
super().__init__(**kwargs)
self.max_proxy_pools = max(1, int(max_proxy_pools))
self.proxy_manager = OrderedDict()
def proxy_manager_for(self, proxy, **proxy_kwargs):
manager = super().proxy_manager_for(proxy, **proxy_kwargs)
self.proxy_manager.move_to_end(proxy)
while len(self.proxy_manager) > self.max_proxy_pools:
_, obsolete = self.proxy_manager.popitem(last=False)
obsolete.clear()
return manager
def bounded_session():
session = requests.Session()
session.trust_env = False
for scheme in ('http://', 'https://'):
session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4))
return session
class ProxyRouter: class ProxyRouter:
def __init__(self, settings, session=None): def __init__(self, settings, session=None):
self.settings = settings self.settings = settings
@@ -141,7 +167,7 @@ class ProxyRouter:
class Transport: class Transport:
def __init__(self, settings, router, session=None, wait=None): def __init__(self, settings, router, session=None, wait=None):
self.settings, self.router = settings, router self.settings, self.router = settings, router
self.session = session or requests.Session() self.session = session or bounded_session()
self.session.trust_env = False self.session.trust_env = False
self.session.headers = getattr(self.session, "headers", {}) self.session.headers = getattr(self.session, "headers", {})
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"}) self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
@@ -149,6 +175,10 @@ class Transport:
self.local = threading.local() self.local = threading.local()
self.lock = threading.RLock() self.lock = threading.RLock()
def close(self):
with self.lock:
self.session.close()
def sleep(self, seconds): def sleep(self, seconds):
remaining = seconds remaining = seconds
while remaining > 0: while remaining > 0:
+75
View File
@@ -0,0 +1,75 @@
import pytest
from studio.database import Database
from studio.domain import finish_record
def record(code='10001', name='甲大学', major='081200', province='北京', year=2027, direction='方向一'):
return finish_record({'school_code': code, 'school_name': name, 'major_code': major,
'major_name': major, 'province': province, 'year': year,
'direction': direction, 'subjects': ['101政治'], 'source': '测试'})
def test_history_targets_deduplicate_directions_and_preserve_filters(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_catalog([
record(direction='一'), record(direction='二'), record(major='081201'),
record(code='10002', name='乙大学', province='天津'),
record(code='10003', name='丙大学', province=''), record(year=2026),
])
rows = list(db.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
assert [(r['school_code'], r['major_code']) for r in rows] == [
('10001', '081200'), ('10001', '081201'), ('10003', '081200')]
assert all('data' not in row and 'subjects' not in row for row in rows)
rows = list(db.score_targets({'year': 2027, 'school_keyword': '甲', 'major_code': '081201'}))
assert [(r['school_code'], r['major_code']) for r in rows] == [('10001', '081201')]
def test_catalog_iterator_keeps_export_order_and_excludes_new_inserts(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_catalog([record(name='B大学'), record(name='A大学', direction='二'),
record(name='A大学', direction='三'), record(year=2026),
record(code='10002', name='C大学')])
iterator = db.iter_catalog({}, batch_size=2)
first = next(iterator)
db.upsert_catalog([record(name='Z大学', year=2028)])
rows = [first, *iterator]
assert [(r['year'], r['school_name']) for r in rows] == [
(2027, 'A大学'), (2027, 'A大学'), (2027, 'B大学'), (2027, 'C大学'), (2026, '甲大学')]
assert [r['school_name'] for r in db.iter_catalog({'school_keyword': 'A'}, batch_size=1)] == ['A大学', 'A大学']
def test_score_iterator_exports_only_selected_school_major_pairs(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_scores([
{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310},
{'school_code': '10002', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 320},
])
rows = list(db.iter_scores({('10001', '081200')}, batch_size=1))
assert len(rows) == 1
assert rows[0]['total'] == 310
def test_unit_commit_is_idempotent_for_records_count_and_checkpoint(tmp_path):
db = Database(tmp_path / 'data.db')
db.job_create('job', {})
rows = [record(direction='一'), record(direction='二')]
assert db.save_unit('job', 'catalog:test', rows) is True
assert db.save_unit('job', 'catalog:test', rows) is False
assert db.overview()['records'] == 2
assert db.job_get('job')['records'] == 2
assert db.unit_done('job', 'catalog:test')
def test_failed_unit_transaction_rolls_back_scores_records_and_checkpoint(tmp_path, monkeypatch):
db = Database(tmp_path / 'data.db')
db.job_create('job', {})
def fail(*args):
raise RuntimeError('interrupted persistence')
monkeypatch.setattr(db, '_school', fail)
score = {'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}
with pytest.raises(RuntimeError):
db.save_unit('job', 'scores:test', [record()], scores=[score])
assert db.overview()['records'] == db.overview()['scores'] == 0
assert db.job_get('job')['records'] == 0
assert not db.unit_done('job', 'scores:test')
+153
View File
@@ -0,0 +1,153 @@
from collections import Counter
import pytest
from studio.connectors import CatalogClient, LoginRequired
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
from studio.network import RemoteUnavailable
from tests.test_data import raw_catalog
from tests.test_network import Response
PARAMS = {'mode': 'catalog', 'year': 2027, 'provinces': ['11']}
class InterruptedCatalog:
def __init__(self, observe=None):
self.blocked = True
self.calls = Counter()
self.observe = observe
def request(self, method, url, **kwargs):
if method == 'GET':
return Response({}, text='2027年硕士专业目录查询')
endpoint = url.rsplit('/', 1)[-1]
school = kwargs['data'].get('dwdm', '')
self.calls[endpoint, school] += 1
if endpoint == 'dws.do':
rows = [{'dwdm': '10001', 'dwmc': '北京大学'}, {'dwdm': '10002', 'dwmc': '中国人民大学'}]
elif endpoint == 'dwzys.do':
if self.observe:
self.observe(school)
rows = [{'zydm': '081200', 'zymc': '计算机科学与技术', 'xxfs': '1'}]
else:
if school == '10002' and self.blocked:
return Response({'flag': False, 'msg': '请登录'})
rows = [{**raw_catalog(), 'dwdm': school}]
return Response({'flag': True, 'msg': {'list': rows, 'nextPageAvailable': False}})
def prepare(tmp_path, transport):
db = Database(tmp_path / 'studio.db')
db.job_create('resume', PARAMS)
return db, JobManager(db, SettingsStore(tmp_path), transport=transport)
def test_login_recovery_reuses_successful_catalog_pages_across_service_restart(tmp_path):
source = InterruptedCatalog()
db, manager = prepare(tmp_path, source)
with pytest.raises(LoginRequired):
manager._catalog('resume', PARAMS)
assert db.job_get('resume')['completed_units'] == 1
source.blocked = False
reopened = Database(tmp_path / 'studio.db')
JobManager(reopened, SettingsStore(tmp_path), transport=source)._catalog('resume', PARAMS)
assert reopened.job_get('resume')['completed_units'] == 2
assert source.calls['dws.do', ''] == 1
assert source.calls['dwzys.do', '10001'] == 1
assert source.calls['dwzys.do', '10002'] == 1
assert source.calls['yjfxs.do', '10002'] == 2 # Failed responses must never become cached successes.
def test_resume_checks_completed_units_in_memory_instead_of_one_query_per_major(tmp_path):
source = InterruptedCatalog()
source.blocked = False
db, manager = prepare(tmp_path, source)
for school in ('10001', '10002'):
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
queries = []
original = db.unit_done
def observe(jid, unit):
queries.append(unit)
return original(jid, unit)
db.unit_done = observe
manager._catalog('resume', PARAMS)
assert queries == []
assert not any(endpoint == 'yjfxs.do' for endpoint, school in source.calls)
def test_resume_displays_current_school_while_checking_its_major_list(tmp_path):
messages = {}
source = InterruptedCatalog(lambda school: messages.setdefault(school, db.job_get('resume')['message']))
source.blocked = False
db, manager = prepare(tmp_path, source)
for school in ('10001', '10002'):
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
manager._catalog('resume', PARAMS)
assert '北京大学' in messages['10001']
assert '中国人民大学' in messages['10002']
def test_cached_pages_do_not_cross_job_boundaries(tmp_path):
source = InterruptedCatalog()
source.blocked = False
db, manager = prepare(tmp_path, source)
manager._catalog('resume', PARAMS)
db.job_create('fresh', PARAMS)
manager._catalog('fresh', PARAMS)
assert source.calls['dws.do', ''] == 2
assert db.job_get('fresh')['completed_units'] == 2
def test_invalid_pagination_is_not_persisted_as_a_reusable_page():
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
body = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': self.blocked, 'startOfNextPage': 0}
self.blocked = False
return Response({'flag': True, 'msg': body})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='下一页位置无效'):
list(client.schools('11'))
assert cache == {}
assert list(client.schools('11')) == [{'dwdm': '10001'}]
def test_repeated_source_page_is_retried_live_after_recovery():
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
page = kwargs['data']['curPage']
school = '10001' if page == 1 or self.blocked else '10002'
return Response({'flag': True, 'msg': {'list': [{'dwdm': school}], 'nextPageAvailable': page == 1}})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='重复返回同一页'):
list(client.schools('11'))
assert len(cache) == 1
source.blocked = False
assert [row['dwdm'] for row in client.schools('11')] == ['10001', '10002']
@pytest.mark.parametrize('endpoint,broken_row', [
('dws', {'dwmc': '缺少代码的院校'}),
('dwzys', {'zymc': '缺少代码的专业'}),
('dws', 'unexpected row'),
('dwzys', {'zydm': ''}),
('dwzys', {'zydm': None}),
])
def test_malformed_list_page_is_retried_after_source_recovers(endpoint, broken_row):
identifier = 'dwdm' if endpoint == 'dws' else 'zydm'
good_row = {identifier: '10001' if endpoint == 'dws' else '081200'}
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
row = broken_row if self.blocked else good_row
return Response({'flag': True, 'msg': {'list': [row], 'nextPageAvailable': False}})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='代码'):
list(client.pages(endpoint, {}))
assert cache == {}
source.blocked = False
assert list(client.pages(endpoint, {})) == [good_row]
+42
View File
@@ -0,0 +1,42 @@
from studio import network
def test_proxy_managers_are_bounded_and_evicted_pools_are_closed():
adapter = network.BoundedProxyAdapter(max_proxy_pools=3)
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
closed = []
original = first.clear
def observe():
original()
closed.append(True)
first.clear = observe
for port in range(10002, 10006):
adapter.proxy_manager_for(f'http://127.0.0.1:{port}')
assert len(adapter.proxy_manager) <= 3
assert closed == [True]
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is not first
adapter.close()
def test_active_proxy_pool_is_reused_while_old_pools_are_evicted():
adapter = network.BoundedProxyAdapter(max_proxy_pools=2)
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
adapter.proxy_manager_for('http://127.0.0.1:10002')
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
adapter.proxy_manager_for('http://127.0.0.1:10003')
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
assert 'http://127.0.0.1:10002' not in adapter.proxy_manager
adapter.close()
def test_default_transport_limits_pools_without_changing_route_or_tls(tmp_path):
from studio.settings import SettingsStore
from tests.test_network import Pool
settings = SettingsStore(tmp_path)
router = network.ProxyRouter(settings, session=Pool())
transport = network.Transport(settings, router)
adapter = transport.session.get_adapter('https://example.test')
assert isinstance(adapter, network.BoundedProxyAdapter)
assert transport.session.verify is True
assert router.resolve() == 'http://1.2.3.4:8080'
transport.close()
+125
View File
@@ -0,0 +1,125 @@
import asyncio
import csv
import io
import threading
import pytest
from fastapi.testclient import TestClient
from openpyxl import load_workbook
from studio.api import create_app
from tests.test_bounded_storage import record
@pytest.mark.parametrize('kind', ['csv', 'json', 'xlsx'])
def test_export_uses_bounded_rows_and_preserves_content(tmp_path, monkeypatch, kind):
app = create_app(tmp_path)
app.state.db.upsert_catalog([record(name='=1+1'), record(code='10002', name='含逗号,大学')])
app.state.db.upsert_scores([{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}])
def forbidden(*args, **kwargs):
raise AssertionError('export materializes all catalog rows')
monkeypatch.setattr(app.state.db, 'all_catalog', forbidden)
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
response = client.get('/api/export', params={'format': kind})
assert response.status_code == 200
if kind == 'json':
data = response.json()
assert {r['school_name'] for r in data['catalog']} == {'=1+1', '含逗号,大学'}
assert data['scores'][0]['total'] == 310
elif kind == 'csv':
rows = list(csv.reader(io.StringIO(response.content.decode('utf-8-sig'))))
assert len(rows) == 3
assert any("'=1+1" in row for row in rows)
assert any('含逗号,大学' in row for row in rows)
else:
book = load_workbook(io.BytesIO(response.content), read_only=True)
rows = list(book.active.values)
assert len(rows) == 3
assert any("'=1+1" in row for row in rows)
book.close()
assert not list((tmp_path / 'exports').glob('*'))
def test_concurrent_export_is_rejected_and_lock_is_released(tmp_path, monkeypatch):
app = create_app(tmp_path)
entered, release = threading.Event(), threading.Event()
original = app.state.db.iter_catalog
def slow_rows(*args, **kwargs):
if not entered.is_set():
entered.set()
assert release.wait(5)
yield from original(*args, **kwargs)
monkeypatch.setattr(app.state.db, 'iter_catalog', slow_rows)
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
responses = []
thread = threading.Thread(target=lambda: responses.append(client.get('/api/export')))
thread.start()
assert entered.wait(3)
blocked = client.get('/api/export', params={'format': 'json'})
release.set()
thread.join(5)
assert blocked.status_code == 429
assert responses[0].status_code == 200
assert client.get('/api/export').status_code == 200
def test_disconnected_stream_closes_source_and_releases_export_lock():
from studio.exporting import ExportStream
from starlette.requests import ClientDisconnect
lock = threading.BoundedSemaphore(1)
assert lock.acquire(False)
closed = []
def source():
try:
yield b'data'
finally:
closed.append(True)
async def send(message):
if message['type'] == 'http.response.body':
raise OSError('client disconnected')
async def receive():
return {'type': 'http.disconnect'}
response = ExportStream(source(), lock.release, media_type='text/csv')
with pytest.raises(ClientDisconnect):
asyncio.run(response({'type': 'http', 'asgi': {'spec_version': '2.4'}}, receive, send))
assert closed == [True]
assert lock.acquire(False)
def test_xlsx_packaging_error_removes_xml_tempfile_without_masking_error(tmp_path, monkeypatch):
from pathlib import Path
from studio.imports import write_xlsx
from openpyxl import Workbook
paths = []
def packaging_failure(book, filename):
sheet = book.worksheets[0]
sheet.close()
paths.append(Path(sheet._writer.out))
raise OSError('archive write failed')
monkeypatch.setattr(Workbook, 'save', packaging_failure)
with pytest.raises(OSError, match='archive write failed'):
write_xlsx([record()], tmp_path / 'failed.xlsx')
assert paths and not paths[0].exists()
def test_failed_export_batch_does_not_finish_a_successful_truncated_download(tmp_path, monkeypatch):
app = create_app(tmp_path)
def broken_rows(*args, **kwargs):
yield record()
raise RuntimeError('database batch failed')
monkeypatch.setattr(app.state.db, 'iter_catalog', broken_rows)
sent = []
async def send(message):
sent.append(message)
async def receive():
await asyncio.Event().wait()
scope = {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'},
'http_version': '1.1', 'method': 'GET', 'scheme': 'http',
'path': '/api/export', 'raw_path': b'/api/export', 'root_path': '',
'query_string': b'format=csv', 'headers': [(b'host', b'127.0.0.1:8765')],
'server': ('127.0.0.1', 8765), 'client': ('127.0.0.1', 12345)}
with pytest.raises(RuntimeError, match='database batch failed'):
asyncio.run(app(scope, receive, send))
assert not any(message['type'] == 'http.response.body' and not message.get('more_body', False) for message in sent)
+147
View File
@@ -0,0 +1,147 @@
import pytest
from studio.connectors import ScoreClient
from studio.network import RemoteUnavailable
from tests.test_network import Response
class HistorySource:
def __init__(self, pages):
self.pages = pages
self.calls = []
def request(self, method, url, **kwargs):
payload = kwargs['data']
self.calls.append((url.rsplit('/', 1)[-1], payload.copy()))
data = self.pages(payload.get('page', 1))
return Response({'code': '0000', 'data': data})
def test_full_score_array_over_page_limit_is_consumed_once():
rows = [{'code': '081200', 'total': 310, 'depart_name': '信息学院'}]
rows += [{'code': '999999', 'total': 200} for _ in range(197)]
source = HistorySource(lambda page: rows)
lines = ScoreClient(source).score_lines({'school_code': '10001', 'school_name': '北京大学'}, '081200', 2025, 1)
assert len(lines) == 1
assert lines[0]['total'] == 310
assert [call[1]['page'] for call in source.calls] == [1]
def test_short_wrapped_pages_follow_explicit_total():
source = HistorySource(lambda page: {'data': [{'value': page}], 'total': 2})
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
def test_short_wrapped_pages_follow_explicit_last_page():
source = HistorySource(lambda page: {'list': [{'value': page}], 'last_page': 2})
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
@pytest.mark.parametrize('data', [None, {}, {'data': None}, {'data': [None]}, {'list': [42]}])
def test_invalid_history_data_cannot_become_a_successful_empty_unit(data):
with pytest.raises(RemoteUnavailable):
list(ScoreClient(HistorySource(lambda page: data)).pages('/example', {}))
def test_empty_full_array_is_a_valid_empty_result():
assert list(ScoreClient(HistorySource(lambda page: [])).pages('/example', {})) == []
def test_repeated_real_page_still_raises_instead_of_returning_partial_data():
source = HistorySource(lambda page: {'data': [{'value': 1}], 'total': 2})
with pytest.raises(RemoteUnavailable):
list(ScoreClient(source).pages('/example', {}))
def test_empty_page_before_declared_total_is_an_error():
source = HistorySource(lambda page: {'data': [{'value': 1}] if page == 1 else [], 'total': 2})
with pytest.raises(RemoteUnavailable):
list(ScoreClient(source).pages('/example', {}))
def test_two_majors_reuse_one_complete_school_year_score_table():
rows = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
source = HistorySource(lambda page: rows)
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
assert client.score_lines(school, '081201', 2025, 1)[0]['total'] == 320
assert len(source.calls) == 1
def test_score_cache_separates_schools_and_years_and_has_a_bound():
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}])
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
for sid in range(20):
client.score_lines(school, '081200', 2025, sid)
client.score_lines(school, '081200', 2026, 19)
before = len(source.calls)
client.score_lines(school, '081200', 2026, 19)
assert len(source.calls) == before
client.score_lines(school, '081200', 2025, 0)
assert len(source.calls) == before + 1
def test_failed_lookup_is_cached_for_the_current_client_only():
source = HistorySource(lambda page: {'data': [], 'total': 0})
client = ScoreClient(source)
assert client.school_id('暂未收录大学') is None
assert client.school_id('暂未收录大学') is None
assert len(source.calls) == 1
assert ScoreClient(source).school_id('暂未收录大学') is None
assert len(source.calls) == 2
def test_invalid_score_response_is_not_cached():
valid = False
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}] if valid else {})
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
with pytest.raises(RemoteUnavailable):
client.score_lines(school, '081200', 2025, 1)
valid = True
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
def test_history_runner_uses_thin_targets_and_bulk_checkpoints(tmp_path, monkeypatch):
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
from tests.test_bounded_storage import record
db = Database(tmp_path / 'data.db')
db.upsert_catalog([record(), record(major='081201'), record(direction='二')])
params = {'year': 2027, 'provinces': ['11'], 'score_years': [2025, 2026], 'include_plans': True}
db.job_create('history', params)
class Source:
def __init__(self):
self.calls = []
def request(self, method, url, **kwargs):
payload = kwargs['data']
self.calls.append(url.rsplit('/', 1)[-1])
if url.endswith('/schoolList'):
data = {'data': [{'school_name': '甲大学', 'school_id': 1}], 'total': 1}
elif url.endswith('/schoolScore'):
data = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
else:
data = {'data': [{'special_code': payload['keyword'], 'special_name': payload['keyword'],
'plan_id': str(payload['year']) + payload['keyword'], 'recruit_number': 1}], 'total': 1}
return Response({'code': '0000', 'data': data})
def forbidden(*args, **kwargs):
raise AssertionError('full catalog decoding or per-unit checkpoint query')
monkeypatch.setattr(db, 'all_catalog', forbidden)
monkeypatch.setattr(db, 'unit_done', forbidden)
source = Source()
manager = JobManager(db, SettingsStore(tmp_path), transport=source)
manager._scores('history', params)
assert db.job_get('history')['completed_units'] == 4
assert db.overview()['scores'] == 4
assert db.job_get('history')['records'] == 4
assert source.calls.count('schoolScore') == 2
assert source.calls.count('planListV2') == 4
before = len(source.calls)
manager._scores('history', params)
assert len(source.calls) == before
+86 -1
View File
@@ -17,7 +17,7 @@ def mysql():
assert config['database'].endswith('_test'), 'refusing to clear a production database' assert config['database'].endswith('_test'), 'refusing to clear a production database'
db = MySQLDatabase(config) db = MySQLDatabase(config)
with db.connection() as conn: with db.connection() as conn:
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs'): for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs', 'job_catalog_pages'):
conn.execute(f'DELETE FROM {table}') conn.execute(f'DELETE FROM {table}')
return db return db
@@ -68,6 +68,77 @@ def test_mysql_tasks_logs_and_checkpoints(mysql):
assert mysql.logs('contract-job')[0]['message'] == '数据库测试' assert mysql.logs('contract-job')[0]['message'] == '数据库测试'
def test_mysql_catalog_page_cache_and_bulk_checkpoints_are_durable(mysql):
mysql.job_create('cached-job', {'year': 2027})
mysql.complete_unit('cached-job', 'catalog:11:10001:081200:1')
page = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': False}
mysql.cache_catalog_page('cached-job', 'page-key', page)
from studio.mysql_database import MySQLDatabase
reopened = MySQLDatabase(mysql.config, recover_jobs=False)
assert reopened.completed_units('cached-job') == {'catalog:11:10001:081200:1'}
assert reopened.catalog_pages('cached-job') == {'page-key': page}
assert reopened.catalog_pages('another-job') == {}
def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypatch):
import ssl
with mysql.connection() as conn:
cipher = conn.execute("SHOW SESSION STATUS LIKE 'Ssl_cipher'").fetchone()[1]
if not cipher:
pytest.skip('requires a TLS-enabled MariaDB test server')
creations = []
original = ssl.create_default_context
def observe(*args, **kwargs):
creations.append(True)
return original(*args, **kwargs)
monkeypatch.setattr(ssl, 'create_default_context', observe)
for _ in range(2):
with mysql.connection() as conn:
assert conn.execute('SELECT 1').fetchone()[0] == 1
assert creations == []
def test_mysql_bounded_catalog_and_history_targets_preserve_filters(mysql):
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
scores = select_score_lines([{'code': '081200', 'total': 310, 'depart_name': '计算机学院'}], '081200', '10001', '北京大学', 2026)
mysql.upsert_scores(scores)
targets = list(mysql.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
assert len(targets) == 1
assert targets[0]['major_code'] == '081200'
assert 'data' not in targets[0]
rows = list(mysql.iter_catalog({'year': 2027}, batch_size=1))
assert len(rows) == 2
assert len(list(mysql.iter_catalog({'year': 2026, 'max_score': 310}, batch_size=1))) == 2
assert len(list(mysql.iter_scores({('10001', '081200')}, batch_size=1))) == 1
def test_mysql_unit_commit_is_atomic_and_idempotent(mysql):
mysql.job_create('atomic-job', {})
rows = normalize_catalog(raw_catalog(), 2027)
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
assert mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert not mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert mysql.job_get('atomic-job')['records'] == 2
assert mysql.job_get('atomic-job')['completed_units'] == 1
assert mysql.overview()['records'] == 2
assert mysql.overview()['scores'] == 1
def test_mysql_failed_unit_rolls_back_every_write(mysql, monkeypatch):
mysql.job_create('atomic-job', {})
def fail(*args):
raise RuntimeError('interrupted unit')
monkeypatch.setattr(mysql, '_school', fail)
rows = normalize_catalog(raw_catalog(), 2027)
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
with pytest.raises(RuntimeError):
mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert mysql.job_get('atomic-job')['records'] == 0
assert mysql.job_get('atomic-job')['completed_units'] == 0
assert mysql.overview()['records'] == mysql.overview()['scores'] == 0
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path): def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
from studio.migrate import migrate_sqlite from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'snapshot.db') source = Database(tmp_path / 'snapshot.db')
@@ -77,16 +148,30 @@ def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_p
source.job_create('migrated-job', {'year': 2027}) source.job_create('migrated-job', {'year': 2027})
source.complete_unit('migrated-job', 'catalog:11:10001:081200:1') source.complete_unit('migrated-job', 'catalog:11:10001:081200:1')
source.log('migrated-job', 'info', 'preserve log') source.log('migrated-job', 'info', 'preserve log')
source.cache_catalog_page('migrated-job', 'migration-page', {'list': [], 'nextPageAvailable': False})
counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql) counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql)
assert counts['catalog'] == 2 and counts['scores'] == 1 assert counts['catalog'] == 2 and counts['scores'] == 1
assert mysql.record(1)['favorite'] assert mysql.record(1)['favorite']
assert mysql.query_catalog({})['items'][0]['score_total'] == 320 assert mysql.query_catalog({})['items'][0]['score_total'] == 320
assert mysql.job_get('migrated-job')['completed_units'] == 1 assert mysql.job_get('migrated-job')['completed_units'] == 1
assert mysql.logs('migrated-job')[0]['id'] == 1 assert mysql.logs('migrated-job')[0]['id'] == 1
assert counts['job_catalog_pages'] == 1
assert mysql.catalog_pages('migrated-job')['migration-page']['list'] == []
with pytest.raises(ValueError, match='非空'): with pytest.raises(ValueError, match='非空'):
migrate_sqlite(tmp_path / 'snapshot.db', mysql) migrate_sqlite(tmp_path / 'snapshot.db', mysql)
def test_legacy_snapshot_without_catalog_page_table_can_still_be_migrated(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'legacy.db')
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
with source.connection() as conn:
conn.execute('DROP TABLE job_catalog_pages')
counts = migrate_sqlite(source.path, mysql)
assert counts['catalog'] == 2
assert counts['job_catalog_pages'] == 0
def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path): def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
from studio.migrate import migrate_sqlite from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'bad.db') source = Database(tmp_path / 'bad.db')
+98
View File
@@ -0,0 +1,98 @@
import importlib.util
import queue
import threading
import time
from pathlib import Path
def maintainer_class():
path = Path(__file__).resolve().parents[1] / 'deploy/proxy_scheduler.py'
spec = importlib.util.spec_from_file_location('bounded_proxy_scheduler', path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.ProxyMaintainer
def test_detection_workers_are_bounded_reused_and_rounds_do_not_overlap():
state = {'active': 0, 'peak': 0, 'types': set(), 'mixed': False, 'done': [], 'thread_ids': set()}
lock = threading.Lock()
class Checker:
def __init__(self, work_type, target_queue, name):
self.work_type, self.queue = work_type, target_queue
def run(self):
while True:
try:
item = self.queue.get(block=False)
except queue.Empty:
return
with lock:
state['active'] += 1
state['peak'] = max(state['peak'], state['active'])
state['types'].add(self.work_type)
state['mixed'] |= len(state['types']) > 1
state['thread_ids'].add(threading.get_ident())
time.sleep(.01)
with lock:
state['done'].append(item)
state['active'] -= 1
if not state['active']:
state['types'].clear()
self.queue.task_done()
maintainer = maintainer_class()(workers=2, checker_class=Checker)
first = threading.Thread(target=maintainer.validate, args=('raw', range(10)))
second = threading.Thread(target=maintainer.validate, args=('use', range(10, 20)))
first.start(); second.start()
first.join(4); second.join(4)
assert not first.is_alive() and not second.is_alive()
before = set(state['thread_ids'])
maintainer.validate('raw', range(20, 25))
maintainer.close()
assert sorted(state['done']) == list(range(25))
assert state['peak'] <= 2
assert state['mixed'] is False
assert state['thread_ids'] == before
def test_stopped_detector_does_not_begin_new_rounds():
class Forbidden:
def __init__(self, *args):
raise AssertionError('detection began after shutdown')
maintainer = maintainer_class()(workers=2, checker_class=Forbidden)
maintainer.close()
maintainer.validate('raw', [1, 2, 3])
def test_failed_worker_waits_for_other_workers_before_next_round():
slow_started, release, next_round = threading.Event(), threading.Event(), threading.Event()
class Checker:
def __init__(self, work_type, target_queue, name):
self.kind, self.name = work_type, name
def run(self):
if self.kind == 'raw' and self.name == 'worker_00':
assert slow_started.wait(2)
raise RuntimeError('validator failed')
if self.kind == 'raw':
slow_started.set()
assert release.wait(3)
maintainer = maintainer_class()(workers=2, checker_class=Checker)
failures = []
def first_round():
try:
maintainer.validate('raw', [1, 2])
except RuntimeError:
failures.append(True)
first = threading.Thread(target=first_round)
first.start()
assert slow_started.wait(2)
def second_proxies():
next_round.set()
yield 3
second = threading.Thread(target=maintainer.validate, args=('use', second_proxies()))
second.start()
began_early = next_round.wait(.15)
release.set()
first.join(3); second.join(3)
maintainer.close()
assert began_early is False
assert next_round.is_set()
assert failures == [True]