Add Debian deployment, MariaDB storage and randomized proxy routing #1

Merged
Jayson merged 1 commits from feat/debian-server into main 2026-10-11 19:09:15 +08:00
27 changed files with 896 additions and 32 deletions
Showing only changes of commit 915241a2fd - Show all commits
+1
View File
@@ -0,0 +1 @@
kaoyan ALL=(root) NOPASSWD: /usr/bin/systemctl start kaoyan-proxy-api.service kaoyan-proxy-scheduler.service, /usr/bin/systemctl stop kaoyan-proxy-api.service kaoyan-proxy-scheduler.service
+20
View File
@@ -0,0 +1,20 @@
[Unit]
Description=Kaoyan proxy-pool API
After=network-online.target docker.service
Wants=network-online.target
Requires=docker.service
[Service]
User=kaoyan
Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py server
Restart=on-failure
RestartSec=10
UMask=0077
PrivateTmp=true
ProtectSystem=full
ProtectHome=true
[Install]
WantedBy=multi-user.target
+20
View File
@@ -0,0 +1,20 @@
[Unit]
Description=Kaoyan proxy-pool fetching and validation
After=network-online.target docker.service
Wants=network-online.target
Requires=docker.service
[Service]
User=kaoyan
Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py schedule
Restart=on-failure
RestartSec=10
UMask=0077
PrivateTmp=true
ProtectSystem=full
ProtectHome=true
[Install]
WantedBy=multi-user.target
+5
View File
@@ -0,0 +1,5 @@
STUDIO_DATABASE_CONFIG=/opt/kaoyan-studio-deploy/mysql.json
STUDIO_DATA_DIR=/var/lib/kaoyan-studio
STUDIO_ALLOWED_HOSTS=kaoyan.fjayson.com,186.241.68.230
STUDIO_AUTO_RESUME=1
PYTHONUNBUFFERED=1
+22
View File
@@ -0,0 +1,22 @@
[Unit]
Description=Kaoyan data workbench
After=network-online.target mariadb.service kaoyan-proxy-api.service
Wants=network-online.target kaoyan-proxy-api.service kaoyan-proxy-scheduler.service
[Service]
Type=simple
User=kaoyan
Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/app
EnvironmentFile=/etc/kaoyan-studio.env
ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio
Restart=on-failure
RestartSec=10
TimeoutStopSec=120
UMask=0077
PrivateTmp=true
ProtectSystem=full
ProtectHome=true
[Install]
WantedBy=multi-user.target
+10
View File
@@ -0,0 +1,10 @@
server {
listen 80;
listen [::]:80;
server_name kaoyan.fjayson.com;
location ^~ /.well-known/acme-challenge/ {
root /var/www/kaoyan-acme;
auth_basic off;
}
location / { return 301 https://$host$request_uri; }
}
+23
View File
@@ -0,0 +1,23 @@
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name kaoyan.fjayson.com;
ssl_certificate /etc/letsencrypt/live/kaoyan.fjayson.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/kaoyan.fjayson.com/privkey.pem;
ssl_protocols TLSv1.2 TLSv1.3;
client_max_body_size 100m;
auth_basic "Kaoyan Workbench";
auth_basic_user_file /etc/nginx/kaoyan-studio.htpasswd;
location / {
proxy_pass http://127.0.0.1:8765;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_read_timeout 120s;
proxy_hide_header Cache-Control;
add_header Cache-Control "private, no-store" always;
add_header X-Content-Type-Options nosniff always;
add_header X-Frame-Options DENY always;
}
}
+35
View File
@@ -0,0 +1,35 @@
#!/bin/bash
# Run as root after installing the two virtual environments and private MySQL config.
set -euo pipefail
root=/opt/kaoyan-studio-deploy
test -f "$root/mysql.json"
test -x "$root/venv/bin/python"
test -x "$root/proxy-venv/bin/python"
id kaoyan >/dev/null 2>&1 || useradd --system --user-group --home-dir /var/lib/kaoyan-studio --shell /usr/sbin/nologin kaoyan
install -d -m 700 -o kaoyan -g kaoyan /var/lib/kaoyan-studio
install -d -m 750 -o kaoyan -g kaoyan "$root/proxy_pool/log"
chown root:kaoyan "$root/mysql.json"
chmod 640 "$root/mysql.json"
install -m 640 -o root -g kaoyan "$root/app/deploy/kaoyan-studio.env.example" /etc/kaoyan-studio.env
for unit in kaoyan-studio kaoyan-proxy-api kaoyan-proxy-scheduler; do
install -m 644 "$root/app/deploy/$unit.service" "/etc/systemd/system/$unit.service"
done
visudo -cf "$root/app/deploy/kaoyan-pool-sudoers"
install -m 440 "$root/app/deploy/kaoyan-pool-sudoers" /etc/sudoers.d/kaoyan-pool
if ! docker container inspect kaoyan-proxy-redis >/dev/null 2>&1; then
docker run -d --name kaoyan-proxy-redis --restart unless-stopped \
-p 127.0.0.1:6381:6379 -v kaoyan-proxy-redis:/data \
redis:7.2 redis-server --appendonly yes
fi
python3 - <<'PY'
from pathlib import Path
path = Path('/opt/kaoyan-studio-deploy/proxy_pool/setting.py')
text = path.read_text()
text = text.replace('HOST = "0.0.0.0"', 'HOST = "127.0.0.1"')
text = text.replace("DB_CONN = 'redis://:pwdstring@127.0.0.1:6379/0'", "DB_CONN = 'redis://127.0.0.1:6381/0'")
path.write_text(text)
PY
systemctl daemon-reload
systemctl enable kaoyan-studio kaoyan-proxy-api kaoyan-proxy-scheduler
systemctl start kaoyan-proxy-api kaoyan-proxy-scheduler
# The workbench starts only after its database/credential migration is validated.
@@ -0,0 +1,63 @@
# Debian server implementation plan
> **For agentic workers:** Use superpowers:executing-plans to implement this plan inline, with a fresh whole-branch reviewer.
**Goal:** Run the existing workbench continuously on the user's Debian 12 server with MariaDB, a managed proxy pool, randomized pacing, and preserved collected data.
**Architecture:** Keep SQLite as the local default. Add a MySQL database adapter behind the existing Database interface, Linux credential encryption and fixed systemd pool management. Serve the single-worker application on loopback behind authenticated HTTPS.
**Tech Stack:** Python 3.11+, FastAPI, PyMySQL, cryptography, React, MariaDB 10.11, systemd, Nginx, proxy_pool, dedicated Redis.
**Spec:** ../specs/2026-10-11-debian-server-design.md
## Global Constraints
- Preserve the currently running Windows collection.
- Never print or commit cookies, encryption keys, database passwords, or access passwords.
- Reuse SSH connections; do not alter existing databases or unrelated services.
- Random request bounds: 0.3–60 seconds; server proxy rotation: 100 successful requests.
- No automated captcha solving; no inherited login verification after migration.
## Review Focus
- Same-name cookies across CHSI domains must survive encrypted migration.
- Normalized department names and special-plan score rules must match SQLite results.
- Failed migrations must not leave a partly imported database; source remains untouched.
- Automatic rotation must not split the SSO binding and access proof across proxies.
- Manual pause and login-wait states must survive service restart without automatic resumption.
### Task 1: Random pacing and proxy rotation
Files: studio/settings.py, studio/network.py, studio/auth.py, tests/test_server_features.py.
- [ ] Add failing tests for varied bounded delays, invalid bounds, automatic route changes, and pinned verification.
- [ ] Implement interval maximum and proxy rotation settings, actual request route/count reporting, and route pinning during auth verification.
- [ ] Run network and auth tests, then commit.
### Task 2: MariaDB backend and migration
Files: studio/mysql_database.py, studio/migrate.py, studio/api.py, tests/test_mysql.py, pyproject.toml.
- [ ] Write tests against a real isolated MariaDB database for upserts, historical scopes, favorites, jobs, and atomic migration.
- [ ] Implement MySQLDatabase.connection(), preserving the existing Database API; load private database configuration through STUDIO_DATABASE_CONFIG.
- [ ] Implement migrate_sqlite(source, target) preserving IDs, logs and unit keys and rejecting nonempty targets.
- [ ] Verify all existing tests and real MariaDB integration tests, then commit.
### Task 3: Linux operation and user interface
Files: studio/auth.py, studio/linux_pool.py, studio/pool_service.py, studio/__main__.py, studio/api.py, frontend/src/App.tsx, frontend/src/types.ts.
- [ ] Add failing tests for Linux vault roundtrip/permissions, server host filtering, and interrupted-task recovery selection.
- [ ] Implement encrypted Linux storage, explicit manual-cookie login instructions, Linux systemd pool controls, configured hosts/data directory, and opt-in restart resumption.
- [ ] Add interval range, rotation settings and counters to the UI; use capability metadata for platform instructions.
- [ ] Run full backend tests and frontend tests/build, then commit.
### Task 4: Provision, migrate, and deploy
Files: deploy/*, docs/debian-deployment.md, README.md.
- [ ] Prepare reproducible systemd, Nginx and private configuration templates with rollback and backup instructions.
- [ ] Provision independent MariaDB account/database, dedicated loopback Redis, proxy pool and workbench services.
- [ ] Transfer online SQLite backup and credentials through SSH without exposing secrets; verify migrated data and server login.
- [ ] Validate authenticated HTTPS, browser workflows, proxy API/scheduler and restart persistence; resume migrated task only after valid authentication.
- [ ] Fresh reviewer checks the whole branch; fix material findings and verify again. Publish the deployment branch to the user's repository.
@@ -0,0 +1,29 @@
# Debian 12 持续采集部署
用户要求将工作台部署到 186.241.68.230,通过已有 SSH 密钥操作,避免频繁连接触发 fail2ban;使用服务器实际 MariaDB,部署 jhao104/proxy_pool,核实代理复用,并加入随机请求间隔。保留现有本机运行任务。
## 运行与访问
使用 Debian 的 Python 3.11、独立虚拟环境和 systemd 服务,普通专用用户运行工作台。现有 MariaDB 10.11 创建独立数据库与最小权限账号,不改其他项目数据库。Nginx 提供 HTTPS 和访问认证;优先指定域名,DNS 未就绪时使用已有有效 IP 证书及独立 9443 端口。后端、代理池与 Redis 仅监听回环地址。操作 SSH 尽量复用持续连接。
## 数据
增加 MariaDB/MySQL 存储实现,复用现有数据查询与任务接口;Windows 默认继续使用 SQLite。MariaDB 保存目录、分数线、院校索引、收藏、任务、日志和完成断点。使用 utf8mb4 与稳定记录键,保留原文、年份和特殊计划的匹配规则。
SQLite 在线 backup 生成一致性快照;显式迁移到空的新库,保留记录 ID 和全部断点。迁移完成核对每张表的行数及记录内容。旧电脑任务保持原状态,服务器副本初次迁移暂停,登录与网络验证成功后才恢复。重启时仅自动恢复原先运行的任务,用户暂停、取消、等待登录任务不自动启动。
## 凭据与无桌面登录
Windows 继续 DPAPI;Linux 使用本地权限 0600 的密钥和加密 Cookie。密钥与数据库配置存放运行目录,排除 Git 和公开下载。服务器无需桌面:界面说明在用户电脑完成学信网登录并粘贴 Cookie;账号登录仍可用,验证码保留人工流程。迁移原有 Cookie 通过 SSH 安全传输,并在服务器重新验证,不能继承原“已验证”标记。
## 请求与代理
每次请求从配置的最小与最大间隔随机采样,最大不得小于最小,两者均在 0.3–60 秒;相等时允许固定间隔。保留限流退避与人工验证码处理。
增加“每 N 次成功请求轮换代理”,0 表示保留原策略;服务器默认 100。显示当前代理的请求计数、累计切换次数及实际成功请求路由。会话同步和第二页验证期间延迟自动轮换,轮换后的目录会话重新验证。健康轮换保留旧 IP,不将限流误认成失效。
Linux 代理 API 与调度器使用专门 systemd 单元,独立 Redis 容器只绑定 127.0.0.1:6381。界面管理固定单元名称,只允许启动和停止本项目 API/调度器,不接受任意 shell 命令,不停止其他项目 Redis。
## 验证
现有测试继续通过。新增随机间隔边界、轮换与会话固定、Linux 凭据往返、主机与跨来源保护测试。在实际 MariaDB 临时测试库验证重复更新、收藏、历史线匹配、特殊线隔离、任务断点和 SQLite 迁移。构建网页,在服务器验证 HTTPS 认证、页面、数据查询、代理池、登录状态、systemd 重启恢复和现有站点未受影响。
+25 -5
View File
@@ -1713,6 +1713,11 @@ function ProxyPanel({ notify }: { notify: Notify }) {
? "代理池暂无可用 IP,优先代理模式已回退直连。"
: "登录、招生目录和历年考情共用该路由。"}
</p>
<p className="form-hint">
当前代理已成功请求 {data?.route_requests || 0} 次 · 已切换 {data?.rotations || 0} 次
{data?.rotate_every ? ` · 每 ${data.rotate_every} 次自动轮换` : " · 连接失败或手动切换时轮换"}
{data?.last_request_route && ` · 最近成功请求:${data.last_request_route === "direct" ? "直连" : data.last_request_route}`}
</p>
</div>
</div>
<Badge tone={data?.fallback ? "amber" : "green"}>
@@ -1972,7 +1977,7 @@ function SettingsPanel({ notify }: { notify: Notify }) {
</label>
<div className="three-fields">
<label>
请求间隔(秒)
请求间隔下限(秒)
<input
type="number"
min="0.3"
@@ -1984,6 +1989,18 @@ function SettingsPanel({ notify }: { notify: Notify }) {
}
/>
</label>
<label>
请求间隔上限(秒)
<input type="number" min={config.request_interval} max="60" step="0.1"
value={config.request_interval_max}
onChange={(e) => update("request_interval_max", Number(e.target.value))} />
</label>
<label>
自动轮换代理(请求次数)
<input type="number" min="0" max="100000" step="1"
value={config.proxy_rotate_every}
onChange={(e) => update("proxy_rotate_every", Number(e.target.value))} />
</label>
<label>
超时(秒)
<input
@@ -2005,6 +2022,7 @@ function SettingsPanel({ notify }: { notify: Notify }) {
/>
</label>
</div>
<p className="form-hint">每次请求随机取上下限之间的间隔;两者相等时固定。轮换次数填 0 可关闭自动轮换;登录会话验证期间保留当前代理。</p>
<button className="primary" disabled={!!busy}>
{busy === "save" ? (
<Loader2 className="spin" size={16} />
@@ -2048,17 +2066,19 @@ function SettingsPanel({ notify }: { notify: Notify }) {
<div>
<Globe2 size={22} />
<div>
<b>在 Edge 中完成登录</b>
<small>支持验证码,自动获取并加密保存 Cookie</small>
<b>{auth?.browser_supported === false ? "在自己电脑上完成登录" : "在 Edge 中完成登录"}</b>
<small>{auth?.browser_supported === false ? "服务器无桌面,请登录研招网后在下方粘贴 Cookie" : "支持验证码,自动获取并加密保存 Cookie"}</small>
</div>
</div>
<button
{auth?.browser_supported === false ? (
<a className="primary" href="https://yz.chsi.com.cn/zsml/a/dw.do" target="_blank" rel="noopener noreferrer">打开研招网登录页面 <ArrowUpRight size={16} /></a>
) : <button
className="primary"
disabled={!!busy || !!browserWaiting}
onClick={() => authenticate("browser")}
>
打开登录窗口 <ArrowUpRight size={16} />
</button>
</button>}
{auth?.browser_message && (
<p className="form-hint">{auth.browser_message}</p>
)}
+7
View File
@@ -92,10 +92,16 @@ export type Config = {
pool_project: string;
backup_proxies: string[];
request_interval: number;
request_interval_max: number;
proxy_rotate_every: number;
timeout: number;
retries: number;
};
export type Proxy = {
route_requests: number;
rotations: number;
last_request_route: string | null;
rotate_every: number;
mode: string;
route: string;
active_proxy: string | null;
@@ -136,6 +142,7 @@ export type PoolManageResult = {
status: PoolServiceStatus;
};
export type Auth = {
browser_supported: boolean;
saved: boolean;
valid: boolean;
state: "unverified" | "verified" | "login_required" | "captcha_required";
+3 -1
View File
@@ -2,7 +2,7 @@
name = "kaoyan-studio"
version = "1.0.0"
description = "本地考研招生目录、历年分数线与代理池工作台"
requires-python = ">=3.12"
requires-python = ">=3.11"
dependencies = [
"fastapi>=0.115,<1",
"uvicorn>=0.34,<1",
@@ -11,6 +11,8 @@ dependencies = [
"openpyxl>=3.1,<4",
"python-multipart>=0.0.20,<1",
"playwright>=1.50,<2",
"PyMySQL>=1.1,<2",
"cryptography>=44,<47",
]
[project.optional-dependencies]
+4
View File
@@ -3,9 +3,11 @@ annotated-types==0.8.0
anyio==4.15.1
beautifulsoup4==4.15.0
certifi==2026.7.22
cffi==2.1.1
charset-normalizer==3.5.2
click==8.5.0
colorama==0.4.6
cryptography==50.0.2
et_xmlfile==2.0.0
fastapi==0.143.0
greenlet==3.5.6
@@ -19,10 +21,12 @@ opentelemetry-api==1.45.1
packaging==26.3
playwright==1.63.0
pluggy==1.6.0
pycparser==3.11
pydantic==2.14.0
pydantic_core==2.50.0
pyee==13.0.1
Pygments==2.21.0
PyMySQL==1.2.3
pytest==9.1.1
python-multipart==0.0.32
requests==2.34.2
+14 -4
View File
@@ -1,6 +1,7 @@
"""Local-only HTTP workbench and production SPA host."""
import json
import uuid
import os
from contextlib import asynccontextmanager
from pathlib import Path
from urllib.parse import urlparse
@@ -20,9 +21,15 @@ ROOT = Path(__file__).resolve().parent.parent
def create_app(data_dir=None):
directory = Path(data_dir or ROOT / ".runtime")
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
directory.mkdir(parents=True, exist_ok=True)
database_config = os.environ.get("STUDIO_DATABASE_CONFIG")
if database_config:
from studio.mysql_database import MySQLDatabase
db = MySQLDatabase(json.loads(Path(database_config).read_text(encoding="utf-8")))
else:
db = Database(directory / "studio.db")
allowed_hosts = {"127.0.0.1", "localhost", "::1"} | {host.strip().lower() for host in os.environ.get("STUDIO_ALLOWED_HOSTS", "").split(",") if host.strip()}
settings = SettingsStore(directory)
pool = PoolService(settings)
router = ProxyRouter(settings)
@@ -34,6 +41,9 @@ def create_app(data_dir=None):
@asynccontextmanager
async def lifespan(app):
if os.environ.get("STUDIO_AUTO_RESUME") == "1":
for jid in db.recovered_job_ids:
jobs.resume(jid)
yield
jobs.close()
@@ -46,8 +56,8 @@ def create_app(data_dir=None):
@app.middleware("http")
async def local_access(request: Request, call_next):
host = request.url.hostname
if host not in ("127.0.0.1", "localhost", "::1"):
return JSONResponse({"detail": "仅允许本机访问"}, status_code=403)
if host not in allowed_hosts:
return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
if request.method not in ("GET", "HEAD", "OPTIONS"):
origin = request.headers.get("origin")
if origin:
@@ -76,7 +86,7 @@ def create_app(data_dir=None):
@app.get("/api/health")
def health():
return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve())}
return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve()), "database": "mysql" if database_config else "sqlite", "platform": "windows" if os.name == "nt" else "linux"}
@app.get("/api/overview")
def overview():
+35 -4
View File
@@ -2,6 +2,7 @@
import ctypes
import json
import os
import platform
import threading
import time
from pathlib import Path
@@ -40,15 +41,41 @@ class CookieVault:
self.path = Path(directory) / "chsi-cookie.enc"
self.path.parent.mkdir(parents=True, exist_ok=True)
def _cipher(self):
from cryptography.fernet import Fernet
key_path = self.path.parent / "cookie.key"
if not key_path.exists():
try:
descriptor = os.open(key_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
except FileExistsError:
pass
else:
with os.fdopen(descriptor, "wb") as stream:
stream.write(Fernet.generate_key())
return Fernet(key_path.read_bytes())
def save(self, cookies):
temporary = self.path.with_suffix(".tmp")
temporary.write_bytes(protect(json.dumps(cookies, ensure_ascii=False).encode()))
payload = json.dumps(cookies, ensure_ascii=False).encode()
encrypted = protect(payload) if platform.system() == "Windows" else self._cipher().encrypt(payload)
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(descriptor, "wb") as stream:
stream.write(encrypted)
temporary.replace(self.path)
def load(self):
if not self.path.exists():
return {}
return json.loads(protect(self.path.read_bytes(), decrypt=True))
payload = self.path.read_bytes()
if platform.system() == "Windows":
payload = protect(payload, decrypt=True)
else:
from cryptography.fernet import InvalidToken
try:
payload = self._cipher().decrypt(payload)
except InvalidToken:
raise ValueError("保存的 Cookie 无法解密,请重新登录") from None
return json.loads(payload)
def clear(self):
self.path.unlink(missing_ok=True)
@@ -94,7 +121,7 @@ class AuthService:
with self.state_lock:
if self.valid and self.verified_route != self.transport.router.status()["route"]:
self.invalidate("网络出口已变化,请验证目录访问后继续采集", state="unverified")
return {"saved": self.vault.path.exists(), "valid": self.valid, "state": self.state, "message": self.message, "checked_at": self.checked_at, "browser_state": self.browser_state, "browser_message": self.browser_message}
return {"saved": self.vault.path.exists(), "valid": self.valid, "state": self.state, "message": self.message, "checked_at": self.checked_at, "browser_state": self.browser_state, "browser_message": self.browser_message, "browser_supported": platform.system() == "Windows"}
def invalidate(self, message, state="login_required"):
with self.state_lock:
@@ -128,7 +155,7 @@ class AuthService:
return self.status()
def check(self):
with self.check_lock:
with self.check_lock, self.transport.router.hold_route():
with self.state_lock:
self.invalidate("正在同步研招目录会话并验证访问权限", state="unverified")
revision = self.revision
@@ -182,6 +209,10 @@ class AuthService:
return self.status()
def browser(self):
if platform.system() != "Windows":
self.browser_state = "unsupported"
self.browser_message = "服务器无桌面环境:请在自己电脑的浏览器完成学信网登录,再粘贴研招网 Cookie 并验证目录访问"
return self.status()
if self.browser_thread and self.browser_thread.is_alive() and not self.browser_cancel.is_set():
return self.status()
proxy = self.transport.router.resolve()
+7 -3
View File
@@ -22,7 +22,7 @@ CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,sc
""" + SCORE_SCHEMA + """
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
"""
@@ -45,7 +45,11 @@ class Database:
conn.execute('DROP TABLE scores')
conn.execute('ALTER TABLE scores_new RENAME TO scores')
conn.execute('CREATE INDEX scores_lookup ON scores(school_code,major_code,year DESC)')
conn.execute("UPDATE jobs SET status='paused',message='服务重启,任务可从断点恢复' WHERE status IN ('running','queued')")
if 'resume_pending' not in {row['name'] for row in conn.execute('PRAGMA table_info(jobs)')}:
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending INTEGER DEFAULT 0')
recovery = "status IN ('running','queued') OR (status='paused' AND resume_pending=1)"
self.recovered_job_ids = [row['id'] for row in conn.execute('SELECT id FROM jobs WHERE ' + recovery)]
conn.execute("UPDATE jobs SET status='paused',resume_pending=0,message='服务重启,任务可从断点恢复' WHERE " + recovery)
@contextmanager
def connection(self):
@@ -182,7 +186,7 @@ class Database:
conn.execute("INSERT INTO jobs(id,params,status,created_at,updated_at,message) VALUES(?,?,'queued',?,?,?)",(jid,encode(params),now(),now(),"等待开始"))
def job_update(self, jid, **changes):
allowed = {"status","message","errors","records"}
allowed = {"status","message","errors","records","resume_pending"}
changes = {key:value for key,value in changes.items() if key in allowed}
changes["updated_at"] = now()
with self.connection() as conn:
+3 -3
View File
@@ -67,7 +67,7 @@ class JobManager:
def _run(self, jid):
job = self.get(jid)
self.db.job_update(jid, status="running", errors=0, message="正在采集")
self.db.job_update(jid, status="running", errors=0, resume_pending=0, message="正在采集")
self.db.log(jid, "info", "任务开始/恢复,将跳过已完成单元")
if self.transport:
self.transport.local.guard = lambda: self.checkpoint(jid)
@@ -127,7 +127,7 @@ class JobManager:
raise ValueError("任务当前不在运行")
with control["condition"]:
control["paused"] = True
self.db.job_update(jid, status="paused", message="已暂停,当前请求结束后停止")
self.db.job_update(jid, status="paused", resume_pending=0, message="已暂停,当前请求结束后停止")
return self.get(jid)
def resume(self, jid):
@@ -169,7 +169,7 @@ class JobManager:
if control["thread"].is_alive():
with control['condition']:
control['shutdown'] = True
self.db.job_update(jid, status='paused', message='服务停止,任务可从断点恢复')
self.db.job_update(jid, status='paused', resume_pending=int(not control['paused']), message='服务停止,任务可从断点恢复')
control['condition'].notify_all()
for control in list(self.controls.values()):
control["thread"].join(timeout=1)
+61
View File
@@ -0,0 +1,61 @@
"""Control only the two fixed, dedicated proxy-pool systemd units."""
import socket
import subprocess
from urllib.parse import urlparse
import requests
UNITS = ('kaoyan-proxy-api.service', 'kaoyan-proxy-scheduler.service')
class LinuxPoolService:
def __init__(self, settings):
self.settings = settings
self.session = requests.Session()
self.session.trust_env = False
def status(self, config=None):
config = config or self.settings.read()
api_ready, counts = False, {}
try:
response = self.session.get(config['pool_url'] + '/count/', timeout=2)
response.raise_for_status()
counts = response.json()
api_ready = isinstance(counts, dict) and type(counts.get('count')) is int
except (requests.RequestException, ValueError):
pass
items = []
for name, label, unit in zip(('server', 'schedule'), ('代理 API', '抓取与检测'), UNITS):
try:
result = subprocess.run(['/usr/bin/systemctl', 'show', unit, '--property=LoadState,ActiveState,MainPID'], capture_output=True, text=True, timeout=5)
facts = dict(line.split('=', 1) for line in result.stdout.splitlines() if '=' in line)
installed = facts.get('LoadState') == 'loaded'
active = installed and facts.get('ActiveState') == 'active'
items.append({'name': name, 'label': label, 'state': 'running' if active else 'stopped', 'pid': int(facts.get('MainPID', '0')) or None, 'verified': active, 'elevated': False, 'detail': 'systemd 运行中' if active else ('已停止' if installed else '未安装本项目 systemd 单元')})
except (OSError, subprocess.TimeoutExpired, ValueError):
items.append({'name': name, 'label': label, 'state': 'unknown', 'pid': None, 'verified': False, 'elevated': False, 'detail': '无法查询 systemd 状态'})
try:
with socket.create_connection(('127.0.0.1', 6381), timeout=1):
redis_ready = True
except OSError:
redis_ready = False
items.append({'name': 'redis', 'label': 'Redis', 'state': 'external' if redis_ready else 'stopped', 'pid': None, 'verified': False, 'elevated': False, 'detail': '独立 Redis 服务(127.0.0.1:6381)' if redis_ready else '未连接到本项目 Redis'})
local = urlparse(config['pool_url']).hostname in ('127.0.0.1', 'localhost', '::1')
return {'api_ready': api_ready, 'counts': counts if api_ready else {}, 'items': items, 'can_stop': local and any(item['verified'] for item in items[:2]), 'warning': '' if local else '当前使用外部代理池,请在其所在服务器管理服务', 'admin_command': '', 'manager': 'systemd'}
def manage(self, action):
if action not in ('start', 'stop', 'status'):
raise ValueError('代理池操作无效')
status = self.status()
result = {'ok': True, 'action': action, 'changed': False, 'status': status, 'message': '服务状态已更新', 'output': '\n'.join(item['label'] + ':' + item['detail'] for item in status['items'])}
if action == 'status':
return result
if status['warning']:
return {**result, 'ok': False, 'message': status['warning']}
if action == 'start' and all(item['state'] == 'running' for item in status['items'][:2]) and status['api_ready']:
return {**result, 'message': '代理池已在运行,已复用现有服务'}
try:
completed = subprocess.run(['/usr/bin/sudo', '-n', '/usr/bin/systemctl', action, *UNITS], capture_output=True, text=True, timeout=55)
message = '代理池启动完成' if action == 'start' else '代理池已停止'
return {**result, 'ok': completed.returncode == 0, 'changed': True, 'status': self.status(), 'message': message if completed.returncode == 0 else 'systemd 操作失败,请检查本项目 sudoers 与服务日志', 'output': (completed.stdout + completed.stderr)[-5000:] or message}
except (OSError, subprocess.TimeoutExpired):
return {**result, 'ok': False, 'message': '代理池管理超时或命令不可用,请检查 systemd 服务'}
+74
View File
@@ -0,0 +1,74 @@
"""Explicit, atomic migration from a consistent SQLite snapshot into an empty MySQL database."""
import argparse
import json
import sqlite3
from pathlib import Path
from studio.domain import normalized_name, is_special_score
from studio.mysql_database import MySQLDatabase, score_key
TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs')
def migrate_sqlite(source, target):
source = Path(source).resolve()
conn = sqlite3.connect(source.as_uri() + '?mode=ro', uri=True)
conn.row_factory = sqlite3.Row
counts = {}
try:
conn.execute('BEGIN')
with target.connection() as output:
# Serialize attempted imports to the same database.
acquired = output.execute("SELECT GET_LOCK('kaoyan_studio_migration',10)").fetchone()[0]
if acquired != 1:
raise ValueError('已有数据迁移正在进行')
try:
for table in TABLES:
if output.execute(f'SELECT count(*) FROM {table}').fetchone()[0]:
raise ValueError(f'目标数据库非空({table}),拒绝覆盖;请使用空数据库')
for table in TABLES:
counts[table] = 0
reader = conn.execute(f'SELECT * FROM {table}')
while batch := reader.fetchmany(500):
rows = []
for original in batch:
row = dict(original)
if table in ('catalog', 'scores'):
payload = json.loads(row['data'])
row['department_norm'] = normalized_name(row.get('department_name'))
if table == 'scores':
row['note'] = row.get('note') or payload.get('note') or ''
row['raw_code'] = row.get('raw_code') or payload.get('raw_code') or ''
row['score_key'] = score_key(row)
row['special_line'] = int(is_special_score(row['note']))
if table == 'jobs' and row['status'] in ('running', 'queued'):
row.update(status='paused', message='已迁移到服务器,验证登录后可从断点恢复')
if table == 'jobs':
row['resume_pending'] = 0
rows.append(row)
fields = list(rows[0])
output.executemany(f"INSERT INTO {table}({','.join(fields)}) VALUES({','.join('?' for _ in fields)})", [[row[field] for field in fields] for row in rows])
counts[table] += len(rows)
for table, count in counts.items():
if output.execute(f'SELECT count(*) FROM {table}').fetchone()[0] != count:
raise ValueError(f'迁移行数校验失败:{table}')
finally:
# The private connection closes after commit/rollback, releasing its
# advisory lock only after the entire import becomes visible.
pass
return counts
finally:
conn.rollback()
conn.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('snapshot', type=Path)
parser.add_argument('--config', type=Path, required=True)
args = parser.parse_args()
config = json.loads(args.config.read_text())
print(json.dumps(migrate_sqlite(args.snapshot, MySQLDatabase(config)), ensure_ascii=False))
if __name__ == '__main__':
main()
+165
View File
@@ -0,0 +1,165 @@
"""MariaDB/MySQL storage retaining the local Database's public interface."""
import hashlib
import json
import re
from contextlib import contextmanager
import pymysql
from studio.database import Database, encode
from studio.domain import normalized_name, is_special_score
SCHEMA = [
"""CREATE TABLE IF NOT EXISTS catalog(
id BIGINT PRIMARY KEY AUTO_INCREMENT,record_key CHAR(64) NOT NULL UNIQUE,year INT NOT NULL,
school_code VARCHAR(32),school_name VARCHAR(255),province VARCHAR(64),major_code VARCHAR(32),major_name VARCHAR(255),
department_code VARCHAR(64),department_name VARCHAR(255),study_mode VARCHAR(32),degree_type VARCHAR(64),level VARCHAR(128),
subjects TEXT,quota INT,source VARCHAR(128),favorite TINYINT NOT NULL DEFAULT 0,data LONGTEXT NOT NULL,
department_norm VARCHAR(255) NOT NULL DEFAULT '',INDEX catalog_filter(year,province,major_code,school_code))""",
"""CREATE TABLE IF NOT EXISTS scores(
id BIGINT PRIMARY KEY AUTO_INCREMENT,school_code VARCHAR(32),major_code VARCHAR(32),year INT,
department_id VARCHAR(64),department_name VARCHAR(255),scope VARCHAR(32),total INT,source VARCHAR(128),data LONGTEXT NOT NULL,
note TEXT NOT NULL,raw_code VARCHAR(64) NOT NULL,score_key CHAR(64) NOT NULL UNIQUE,
department_norm VARCHAR(255) NOT NULL DEFAULT '',special_line TINYINT NOT NULL DEFAULT 0,
INDEX scores_lookup(school_code,major_code,year))""",
"CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)",
"CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)",
"CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))",
"CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))",
]
def score_key(row):
identity = [row.get(field) or '' for field in ('school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'source', 'note', 'raw_code')]
return hashlib.sha256(encode(identity).encode()).hexdigest()
class Row(dict):
def __getitem__(self, key):
return tuple(self.values())[key] if isinstance(key, int) else super().__getitem__(key)
class Result:
def __init__(self, cursor):
self.rowcount = cursor.rowcount
self.columns = [item[0] for item in cursor.description] if cursor.description else []
self.rows = cursor.fetchall() if cursor.description else ()
self.offset = 0
cursor.close()
def fetchone(self):
if self.offset >= len(self.rows):
return None
result = Row(zip(self.columns, self.rows[self.offset]))
self.offset += 1
return result
def fetchall(self):
return list(self)
def __iter__(self):
while self.offset < len(self.rows):
yield self.fetchone()
def mysql_sql(sql):
if sql.startswith('SELECT count(*) FROM (') and sql.endswith(')'):
sql += ' AS studio_count'
sql = sql.replace('INSERT OR IGNORE', 'INSERT IGNORE')
sql = re.sub(r'ON CONFLICT\([^)]*\) DO UPDATE SET', 'ON DUPLICATE KEY UPDATE', sql)
sql = re.sub(r'excluded\.(\w+)', r'VALUES(\1)', sql)
sql = sql.replace("is_special_score(json_extract(x.data,'$.note'))=0", 'x.special_line=0')
sql = sql.replace('norm_name(x.department_name)', 'x.department_norm').replace('norm_name(c.department_name)', 'c.department_norm')
sql = re.sub(r"json_extract\(([^,]+),('[^']+')\)", r'JSON_UNQUOTE(JSON_EXTRACT(\1,\2))', sql)
return sql.replace('?', '%s')
class Connection:
def __init__(self, raw):
self.raw = raw
def execute(self, sql, params=None):
cursor = self.raw.cursor()
try:
cursor.execute(mysql_sql(sql), params)
return Result(cursor)
except BaseException:
cursor.close()
raise
def executemany(self, sql, values):
cursor = self.raw.cursor()
try:
cursor.executemany(mysql_sql(sql), values)
return Result(cursor)
except BaseException:
cursor.close()
raise
class MySQLDatabase(Database):
def __init__(self, config):
self.config = {key: value for key, value in config.items() if key in ('host', 'port', 'user', 'password', 'database', 'unix_socket')}
if not self.config.get('database'):
raise ValueError('数据库配置缺少 database')
self.recovered_job_ids = []
with self.connection() as conn:
for statement in SCHEMA:
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
recovery = "status IN ('running','queued') OR (status='paused' AND resume_pending=1)"
self.recovered_job_ids = [row['id'] for row in conn.execute('SELECT id FROM jobs WHERE ' + recovery)]
conn.execute("UPDATE jobs SET status='paused',resume_pending=0,message='服务重启,任务可从断点恢复' WHERE " + recovery)
@contextmanager
def connection(self):
raw = pymysql.connect(**self.config, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
try:
yield Connection(raw)
raw.commit()
except BaseException:
raw.rollback()
raise
finally:
raw.close()
def upsert_catalog(self, records):
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
values = []
for record in records:
row = {**record, 'subjects': ' | '.join(record.get('subjects', [])), 'data': encode(record), 'department_norm': normalized_name(record.get('department_name'))}
values.append([row.get(field) for field in fields])
if not values:
return
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
with self.connection() as conn:
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
for record in records:
self._school(conn, record)
def upsert_scores(self, records):
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
values = []
for record in records:
row = {**record, 'department_id': record.get('department_id') or '', 'department_name': record.get('department_name') or '', 'source': record.get('source', '掌上考研'), 'note': record.get('note') or '', 'raw_code': record.get('raw_code') or ''}
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
values.append([row.get(field) for field in fields])
if values:
with self.connection() as conn:
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
def favorite(self, rid, enabled):
# MariaDB reports changed rows; an already-selected favorite is still success.
with self.connection() as conn:
conn.execute('UPDATE catalog SET favorite=? WHERE id=?', (int(enabled), rid))
return conn.execute('SELECT 1 FROM catalog WHERE id=?', (rid,)).fetchone() is not None
def history(self, rid):
record = self.record(rid)
if record is None:
return None
with self.connection() as conn:
scores = conn.execute('SELECT data FROM scores WHERE school_code=? AND major_code=? ORDER BY year,scope,department_name', (record['school_code'], record['major_code'])).fetchall()
plans = conn.execute('SELECT data FROM catalog WHERE school_code=? AND major_code=? AND department_norm=? ORDER BY year,id', (record['school_code'], record['major_code'], normalized_name(record['department_name']))).fetchall()
def matches(row):
return not row.get('department_name') and not row.get('department_id') or row.get('department_id') and row['department_id'] == record['department_code'] or row.get('department_name') and normalized_name(row['department_name']) == normalized_name(record['department_name'])
return {'record': record, 'scores': [row for entry in scores if matches(row := json.loads(entry['data']))], 'plans': [json.loads(row['data']) for row in plans]}
+37 -5
View File
@@ -1,6 +1,8 @@
"""One explicit route for CHSI login, catalog and historical data."""
import threading
import time
import random
from contextlib import contextmanager
import requests
from studio.settings import proxy_url
@@ -29,6 +31,10 @@ class ProxyRouter:
self.counts = {}
self.fallback = False
self.lock = threading.RLock()
self.route_requests = 0
self.rotations = 0
self.last_request_route = None
self.holds = 0
def refresh(self):
with self.lock:
@@ -60,12 +66,31 @@ class ProxyRouter:
self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"]))
return self.status()
def resolve(self):
@contextmanager
def hold_route(self):
with self.lock:
self.holds += 1
try:
yield
finally:
with self.lock:
self.holds -= 1
def note_success(self, proxy):
with self.lock:
self.last_request_route = proxy or "direct"
if proxy == self.active:
self.route_requests += 1
def resolve(self, for_request=False):
with self.lock:
config = self.settings.read()
if config["network_mode"] == "direct":
self.fallback = False
return None
threshold = config["proxy_rotate_every"]
if for_request and not self.holds and threshold and self.active and self.route_requests >= threshold:
self.rotate()
if self.active and self.active not in self.failed:
self.fallback = False
return self.active
@@ -75,9 +100,11 @@ class ProxyRouter:
available = [p for p in self.candidates if p not in self.failed]
if available:
self.active = available[0]
self.route_requests = 0
self.fallback = False
return self.active
self.active = None
self.route_requests = 0
if config["network_mode"] == "proxy_only":
self.fallback = False
raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务")
@@ -93,7 +120,10 @@ class ProxyRouter:
if old in self.candidates:
self.candidates.remove(old)
self.candidates.append(old)
return self.resolve()
selected = self.resolve()
if old and selected != old:
self.rotations += 1
return selected
def reset(self):
with self.lock:
@@ -101,10 +131,11 @@ class ProxyRouter:
self.failed.clear()
self.candidates = []
self.fallback = False
self.route_requests = 0
def status(self):
config = self.settings.read()
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "route_requests": self.route_requests, "rotations": self.rotations, "last_request_route": self.last_request_route, "rotate_every": config["proxy_rotate_every"], "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
class Transport:
@@ -130,8 +161,8 @@ class Transport:
def request(self, method, url, **kwargs):
config = self.settings.read()
for attempt in range(config["retries"]):
self.sleep(config["request_interval"] if attempt == 0 else min(20, 2 ** attempt))
proxy = self.router.resolve()
self.sleep(random.uniform(config["request_interval"], config["request_interval_max"]) if attempt == 0 else min(20, 2 ** attempt))
proxy = self.router.resolve(for_request=True)
kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {}
kwargs.setdefault("timeout", config["timeout"])
try:
@@ -143,6 +174,7 @@ class Transport:
continue
response.raise_for_status()
self.local.response_route = proxy or "direct"
self.router.note_success(proxy)
return response
except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout):
if proxy:
+8
View File
@@ -137,6 +137,10 @@ def _pool_entry(arguments, project, service):
class PoolService:
def __init__(self, settings):
self.settings = settings
self.linux = None
if os.name != "nt":
from studio.linux_pool import LinuxPoolService
self.linux = LinuxPoolService(settings)
self.session = requests.Session()
self.session.trust_env = False
self.lock = threading.Lock()
@@ -154,6 +158,8 @@ class PoolService:
return json.loads(result.stdout.decode("utf-8-sig"))
def status(self, config=None):
if self.linux:
return self.linux.status(config)
if config is None:
config = self.settings.read()
project = Path(config["pool_project"])
@@ -233,6 +239,8 @@ class PoolService:
return "\n".join(lines)
def manage(self, action):
if self.linux:
return self.linux.manage(action)
if action not in ("start", "stop", "status"):
raise ValueError("代理池操作无效")
with self.lock:
+14 -6
View File
@@ -1,13 +1,16 @@
"""Project-local, non-secret configuration."""
import json
import threading
import math
import os
from pathlib import Path
from urllib.parse import urlparse
DEFAULTS = {
"network_mode": "prefer_proxy", "pool_url": "http://127.0.0.1:5010",
"pool_project": r"C:\Users\Jayson\Desktop\proxy_pool",
"backup_proxies": [], "request_interval": 1.2, "timeout": 30, "retries": 3,
"pool_project": r"C:\Users\Jayson\Desktop\proxy_pool" if os.name == "nt" else "/opt/kaoyan-studio-deploy/proxy_pool",
"backup_proxies": [], "request_interval": 1.2, "request_interval_max": 2.4,
"proxy_rotate_every": 0, "timeout": 30, "retries": 3,
}
@@ -37,7 +40,10 @@ class SettingsStore:
def read(self):
with self.lock:
values = json.loads(self.path.read_text(encoding="utf-8")) if self.path.exists() else {}
return {**DEFAULTS, **{k: v for k, v in values.items() if k in DEFAULTS}}
result = {**DEFAULTS, **{k: v for k, v in values.items() if k in DEFAULTS}}
if "request_interval_max" not in values:
result["request_interval_max"] = max(result["request_interval"], DEFAULTS["request_interval_max"])
return result
def update(self, values):
with self.lock:
@@ -48,10 +54,12 @@ class SettingsStore:
if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:
raise ValueError("代理池 API 地址无效")
values["pool_url"] = values["pool_url"].rstrip("/")
for key, low, high in [("request_interval", .3, 60), ("timeout", 5, 120), ("retries", 1, 6)]:
values[key] = float(values[key]) if key == "request_interval" else int(values[key])
if not low <= values[key] <= high:
for key, low, high in [("request_interval", .3, 60), ("request_interval_max", .3, 60), ("timeout", 5, 120), ("retries", 1, 6), ("proxy_rotate_every", 0, 100000)]:
values[key] = float(values[key]) if key.startswith("request_interval") else int(values[key])
if not math.isfinite(values[key]) or not low <= values[key] <= high:
raise ValueError(f"{key} 应在 {low} 到 {high} 之间")
if values["request_interval_max"] < values["request_interval"]:
raise ValueError("请求间隔上限不能小于下限")
proxies = values["backup_proxies"]
if isinstance(proxies, str):
proxies = proxies.splitlines()
+99
View File
@@ -0,0 +1,99 @@
"""Real MariaDB contract tests; run only against an explicitly supplied test database."""
import json
import os
from pathlib import Path
import pytest
from studio.domain import normalize_catalog, select_score_lines
from studio.database import Database
from tests.test_data import raw_catalog
pytestmark = pytest.mark.skipif(not os.environ.get('STUDIO_MYSQL_TEST_CONFIG'), reason='requires an isolated MariaDB test database')
@pytest.fixture
def mysql():
from studio.mysql_database import MySQLDatabase
config = json.loads(Path(os.environ['STUDIO_MYSQL_TEST_CONFIG']).read_text())
assert config['database'].endswith('_test'), 'refusing to clear a production database'
db = MySQLDatabase(config)
with db.connection() as conn:
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs'):
conn.execute(f'DELETE FROM {table}')
return db
def test_mysql_preserves_upserts_favorites_and_history(mysql):
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
first = mysql.query_catalog({})['items'][0]
mysql.favorite(first['id'], True)
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
mysql.upsert_scores(select_score_lines([
{'code': '081200', 'total': 320, 'depart_name': '计算机学院'},
{'code': '081200', 'total': 260, 'depart_name': '计算机学院', 'note': '少干专项计划'},
{'code': '081200', 'total': 390, 'depart_name': '其他学院'},
], '081200', '10001', '北京大学', 2026))
result = mysql.query_catalog({'year': 2027, 'direction': '人工', 'english': '203'})
assert result['total'] == 1
assert result['items'][0]['score_total'] == 320
assert mysql.query_catalog({'favorites': True})['total'] == 1
assert mysql.overview()['records'] == 4
assert len(mysql.history(first['id'])['plans']) == 4
assert len(mysql.history(first['id'])['scores']) == 2
def test_mysql_score_unique_key_preserves_long_notes_and_normalized_department(mysql):
raw = raw_catalog()
raw['yxsmc'] = '计算机 ( 学院 )'
mysql.upsert_catalog(normalize_catalog(raw, 2027))
scores = select_score_lines([
{'code': '081200', 'total': 320, 'depart_name': '计算机(学院)', 'note': '普通计划' * 300},
{'code': '081200', 'total': 321, 'depart_name': '计算机(学院)', 'note': '普通计划' * 300 + 'B'},
], '081200', '10001', '北京大学', 2026)
mysql.upsert_scores(scores)
mysql.upsert_scores(scores)
assert mysql.overview()['scores'] == 2
assert mysql.query_catalog({})['items'][0]['score_total'] == 321
def test_mysql_tasks_logs_and_checkpoints(mysql):
mysql.job_create('contract-job', {'year': 2027})
mysql.job_update('contract-job', status='running', records=7)
mysql.complete_unit('contract-job', 'catalog:11:10001:081200:1')
mysql.complete_unit('contract-job', 'catalog:11:10001:081200:1')
mysql.log('contract-job', 'info', '数据库测试')
assert mysql.unit_done('contract-job', 'catalog:11:10001:081200:1')
assert mysql.job_get('contract-job')['completed_units'] == 1
assert mysql.job_get('contract-job')['records'] == 7
assert mysql.logs('contract-job')[0]['message'] == '数据库测试'
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'snapshot.db')
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
source.upsert_scores(select_score_lines([{'code': '081200', 'total': 320}], '081200', '10001', '北京大学', 2026))
source.favorite(1, True)
source.job_create('migrated-job', {'year': 2027})
source.complete_unit('migrated-job', 'catalog:11:10001:081200:1')
source.log('migrated-job', 'info', 'preserve log')
counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql)
assert counts['catalog'] == 2 and counts['scores'] == 1
assert mysql.record(1)['favorite']
assert mysql.query_catalog({})['items'][0]['score_total'] == 320
assert mysql.job_get('migrated-job')['completed_units'] == 1
assert mysql.logs('migrated-job')[0]['id'] == 1
with pytest.raises(ValueError, match='非空'):
migrate_sqlite(tmp_path / 'snapshot.db', mysql)
def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'bad.db')
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
with source.connection() as conn:
conn.execute("INSERT INTO scores(school_code,major_code,year,scope,data) VALUES('10001','081200',2026,'major','not json')")
with pytest.raises(ValueError):
migrate_sqlite(tmp_path / 'bad.db', mysql)
assert mysql.overview()['records'] == 0
assert mysql.overview()['scores'] == 0
+2
View File
@@ -11,6 +11,8 @@ from fastapi.testclient import TestClient
from studio.api import create_app
pytestmark = pytest.mark.skipif(os.name != 'nt', reason='Windows pool process and PowerShell behavior')
@pytest.fixture
def pool_client(tmp_path):
+109
View File
@@ -0,0 +1,109 @@
import platform
import random
import os
import pytest
from studio.auth import CookieVault
from tests.test_network import setup_network
def test_requests_sample_a_fresh_delay_inside_configured_bounds(tmp_path, monkeypatch):
settings, _, transport = setup_network(tmp_path)
settings.update({'request_interval': .3, 'request_interval_max': .5})
samples = iter([.31, .47, .39])
monkeypatch.setattr(random, 'uniform', lambda low, high: next(samples))
delays = []
transport.sleep = delays.append
for _ in range(3):
transport.request('GET', 'https://yz.chsi.com.cn/zsml/')
assert delays == [.31, .47, .39]
@pytest.mark.parametrize('maximum', [.2, 61, float('nan')])
def test_invalid_interval_range_is_rejected(tmp_path, maximum):
settings, _, _ = setup_network(tmp_path)
with pytest.raises(ValueError):
settings.update({'request_interval': .3, 'request_interval_max': maximum})
def test_successful_requests_rotate_without_quarantining_old_proxy(tmp_path):
settings, router, transport = setup_network(tmp_path)
settings.update({'proxy_rotate_every': 2})
routes = [transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] for _ in range(3)]
assert routes == ['http://1.2.3.4:8080', 'http://1.2.3.4:8080', 'http://5.6.7.8:9090']
assert router.failed == set()
assert router.status()['last_request_route'] == 'http://5.6.7.8:9090'
assert router.status()['route_requests'] == 1
def test_route_pin_defers_rotation_until_login_proof_finishes(tmp_path):
settings, router, transport = setup_network(tmp_path)
settings.update({'proxy_rotate_every': 1})
with router.hold_route():
routes = [transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] for _ in range(2)]
assert routes == ['http://1.2.3.4:8080'] * 2
assert transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] == 'http://5.6.7.8:9090'
def test_linux_cookie_vault_encrypts_and_preserves_domain_cookies(tmp_path, monkeypatch):
monkeypatch.setattr(platform, 'system', lambda: 'Linux')
cookies = [{'name': 'JSESSIONID', 'value': 'private-account-cookie', 'domain': 'account.chsi.com.cn'},
{'name': 'JSESSIONID', 'value': 'private-catalog-cookie', 'domain': 'yz.chsi.com.cn'}]
vault = CookieVault(tmp_path)
vault.save(cookies)
assert (tmp_path / 'cookie.key').exists()
assert b'private-account-cookie' not in vault.path.read_bytes()
assert CookieVault(tmp_path).load() == cookies
if os.name != 'nt':
assert (tmp_path / 'cookie.key').stat().st_mode & 0o777 == 0o600
assert vault.path.stat().st_mode & 0o777 == 0o600
def test_configured_server_host_is_allowed_but_other_hosts_and_origins_are_blocked(tmp_path, monkeypatch):
from fastapi.testclient import TestClient
from studio.api import create_app
monkeypatch.setenv('STUDIO_ALLOWED_HOSTS', 'kaoyan.fjayson.com')
with TestClient(create_app(tmp_path), base_url='https://kaoyan.fjayson.com') as client:
assert client.get('/api/health').status_code == 200
assert client.get('/api/health', headers={'Host': 'evil.example'}).status_code == 403
assert client.put('/api/settings', json={'network_mode': 'direct'}, headers={'Origin': 'https://evil.example'}).status_code == 403
assert client.put('/api/settings', json={'network_mode': 'direct'}, headers={'Origin': 'https://kaoyan.fjayson.com'}).status_code == 200
def test_only_interrupted_running_tasks_are_selected_for_restart(tmp_path):
from studio.database import Database
db = Database(tmp_path / 'resume.db')
for status in ('running', 'queued', 'paused', 'waiting_login', 'cancelled'):
db.job_create(status, {'year': 2027})
db.job_update(status, status=status)
restarted = Database(tmp_path / 'resume.db')
assert set(restarted.recovered_job_ids) == {'running', 'queued'}
assert restarted.job_get('paused')['status'] == 'paused'
assert restarted.job_get('waiting_login')['status'] == 'waiting_login'
@pytest.mark.parametrize('manually_paused', [False, True])
def test_graceful_restart_remembers_running_but_not_manual_pause(tmp_path, manually_paused):
import threading
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
ready = threading.Event()
stop = threading.Event()
def runner(jid, params, manager):
ready.set()
while not stop.wait(.01):
manager.checkpoint(jid)
db = Database(tmp_path / 'shutdown.db')
manager = JobManager(db, SettingsStore(tmp_path), runner=runner)
job = manager.start({'mode': 'catalog', 'year': 2027, 'provinces': ['11']})
assert ready.wait(3)
try:
if manually_paused:
manager.pause(job['id'])
manager.close()
restarted = Database(tmp_path / 'shutdown.db')
assert (job['id'] in restarted.recovered_job_ids) is (not manually_paused)
finally:
stop.set()
manager.close()