Add Debian deployment, MariaDB storage and randomized proxy routing

This commit is contained in:
Codex
2026-10-11 19:07:27 +08:00
parent 2e3439dbb9
commit 915241a2fd
27 changed files with 896 additions and 32 deletions
+1
View File
@@ -0,0 +1 @@
kaoyan ALL=(root) NOPASSWD: /usr/bin/systemctl start kaoyan-proxy-api.service kaoyan-proxy-scheduler.service, /usr/bin/systemctl stop kaoyan-proxy-api.service kaoyan-proxy-scheduler.service
+20
View File
@@ -0,0 +1,20 @@
[Unit]
Description=Kaoyan proxy-pool API
After=network-online.target docker.service
Wants=network-online.target
Requires=docker.service
[Service]
User=kaoyan
Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py server
Restart=on-failure
RestartSec=10
UMask=0077
PrivateTmp=true
ProtectSystem=full
ProtectHome=true
[Install]
WantedBy=multi-user.target
+20
View File
@@ -0,0 +1,20 @@
[Unit]
Description=Kaoyan proxy-pool fetching and validation
After=network-online.target docker.service
Wants=network-online.target
Requires=docker.service
[Service]
User=kaoyan
Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py schedule
Restart=on-failure
RestartSec=10
UMask=0077
PrivateTmp=true
ProtectSystem=full
ProtectHome=true
[Install]
WantedBy=multi-user.target
+5
View File
@@ -0,0 +1,5 @@
STUDIO_DATABASE_CONFIG=/opt/kaoyan-studio-deploy/mysql.json
STUDIO_DATA_DIR=/var/lib/kaoyan-studio
STUDIO_ALLOWED_HOSTS=kaoyan.fjayson.com,186.241.68.230
STUDIO_AUTO_RESUME=1
PYTHONUNBUFFERED=1
+22
View File
@@ -0,0 +1,22 @@
[Unit]
Description=Kaoyan data workbench
After=network-online.target mariadb.service kaoyan-proxy-api.service
Wants=network-online.target kaoyan-proxy-api.service kaoyan-proxy-scheduler.service
[Service]
Type=simple
User=kaoyan
Group=kaoyan
WorkingDirectory=/opt/kaoyan-studio-deploy/app
EnvironmentFile=/etc/kaoyan-studio.env
ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio
Restart=on-failure
RestartSec=10
TimeoutStopSec=120
UMask=0077
PrivateTmp=true
ProtectSystem=full
ProtectHome=true
[Install]
WantedBy=multi-user.target
+10
View File
@@ -0,0 +1,10 @@
server {
listen 80;
listen [::]:80;
server_name kaoyan.fjayson.com;
location ^~ /.well-known/acme-challenge/ {
root /var/www/kaoyan-acme;
auth_basic off;
}
location / { return 301 https://$host$request_uri; }
}
+23
View File
@@ -0,0 +1,23 @@
server {
listen 443 ssl;
listen [::]:443 ssl;
server_name kaoyan.fjayson.com;
ssl_certificate /etc/letsencrypt/live/kaoyan.fjayson.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/kaoyan.fjayson.com/privkey.pem;
ssl_protocols TLSv1.2 TLSv1.3;
client_max_body_size 100m;
auth_basic "Kaoyan Workbench";
auth_basic_user_file /etc/nginx/kaoyan-studio.htpasswd;
location / {
proxy_pass http://127.0.0.1:8765;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_read_timeout 120s;
proxy_hide_header Cache-Control;
add_header Cache-Control "private, no-store" always;
add_header X-Content-Type-Options nosniff always;
add_header X-Frame-Options DENY always;
}
}
+35
View File
@@ -0,0 +1,35 @@
#!/bin/bash
# Run as root after installing the two virtual environments and private MySQL config.
set -euo pipefail
root=/opt/kaoyan-studio-deploy
test -f "$root/mysql.json"
test -x "$root/venv/bin/python"
test -x "$root/proxy-venv/bin/python"
id kaoyan >/dev/null 2>&1 || useradd --system --user-group --home-dir /var/lib/kaoyan-studio --shell /usr/sbin/nologin kaoyan
install -d -m 700 -o kaoyan -g kaoyan /var/lib/kaoyan-studio
install -d -m 750 -o kaoyan -g kaoyan "$root/proxy_pool/log"
chown root:kaoyan "$root/mysql.json"
chmod 640 "$root/mysql.json"
install -m 640 -o root -g kaoyan "$root/app/deploy/kaoyan-studio.env.example" /etc/kaoyan-studio.env
for unit in kaoyan-studio kaoyan-proxy-api kaoyan-proxy-scheduler; do
install -m 644 "$root/app/deploy/$unit.service" "/etc/systemd/system/$unit.service"
done
visudo -cf "$root/app/deploy/kaoyan-pool-sudoers"
install -m 440 "$root/app/deploy/kaoyan-pool-sudoers" /etc/sudoers.d/kaoyan-pool
if ! docker container inspect kaoyan-proxy-redis >/dev/null 2>&1; then
docker run -d --name kaoyan-proxy-redis --restart unless-stopped \
-p 127.0.0.1:6381:6379 -v kaoyan-proxy-redis:/data \
redis:7.2 redis-server --appendonly yes
fi
python3 - <<'PY'
from pathlib import Path
path = Path('/opt/kaoyan-studio-deploy/proxy_pool/setting.py')
text = path.read_text()
text = text.replace('HOST = "0.0.0.0"', 'HOST = "127.0.0.1"')
text = text.replace("DB_CONN = 'redis://:pwdstring@127.0.0.1:6379/0'", "DB_CONN = 'redis://127.0.0.1:6381/0'")
path.write_text(text)
PY
systemctl daemon-reload
systemctl enable kaoyan-studio kaoyan-proxy-api kaoyan-proxy-scheduler
systemctl start kaoyan-proxy-api kaoyan-proxy-scheduler
# The workbench starts only after its database/credential migration is validated.
@@ -0,0 +1,63 @@
# Debian server implementation plan
> **For agentic workers:** Use superpowers:executing-plans to implement this plan inline, with a fresh whole-branch reviewer.
**Goal:** Run the existing workbench continuously on the user's Debian 12 server with MariaDB, a managed proxy pool, randomized pacing, and preserved collected data.
**Architecture:** Keep SQLite as the local default. Add a MySQL database adapter behind the existing Database interface, Linux credential encryption and fixed systemd pool management. Serve the single-worker application on loopback behind authenticated HTTPS.
**Tech Stack:** Python 3.11+, FastAPI, PyMySQL, cryptography, React, MariaDB 10.11, systemd, Nginx, proxy_pool, dedicated Redis.
**Spec:** ../specs/2026-10-11-debian-server-design.md
## Global Constraints
- Preserve the currently running Windows collection.
- Never print or commit cookies, encryption keys, database passwords, or access passwords.
- Reuse SSH connections; do not alter existing databases or unrelated services.
- Random request bounds: 0.3–60 seconds; server proxy rotation: 100 successful requests.
- No automated captcha solving; no inherited login verification after migration.
## Review Focus
- Same-name cookies across CHSI domains must survive encrypted migration.
- Normalized department names and special-plan score rules must match SQLite results.
- Failed migrations must not leave a partly imported database; source remains untouched.
- Automatic rotation must not split the SSO binding and access proof across proxies.
- Manual pause and login-wait states must survive service restart without automatic resumption.
### Task 1: Random pacing and proxy rotation
Files: studio/settings.py, studio/network.py, studio/auth.py, tests/test_server_features.py.
- [ ] Add failing tests for varied bounded delays, invalid bounds, automatic route changes, and pinned verification.
- [ ] Implement interval maximum and proxy rotation settings, actual request route/count reporting, and route pinning during auth verification.
- [ ] Run network and auth tests, then commit.
### Task 2: MariaDB backend and migration
Files: studio/mysql_database.py, studio/migrate.py, studio/api.py, tests/test_mysql.py, pyproject.toml.
- [ ] Write tests against a real isolated MariaDB database for upserts, historical scopes, favorites, jobs, and atomic migration.
- [ ] Implement MySQLDatabase.connection(), preserving the existing Database API; load private database configuration through STUDIO_DATABASE_CONFIG.
- [ ] Implement migrate_sqlite(source, target) preserving IDs, logs and unit keys and rejecting nonempty targets.
- [ ] Verify all existing tests and real MariaDB integration tests, then commit.
### Task 3: Linux operation and user interface
Files: studio/auth.py, studio/linux_pool.py, studio/pool_service.py, studio/__main__.py, studio/api.py, frontend/src/App.tsx, frontend/src/types.ts.
- [ ] Add failing tests for Linux vault roundtrip/permissions, server host filtering, and interrupted-task recovery selection.
- [ ] Implement encrypted Linux storage, explicit manual-cookie login instructions, Linux systemd pool controls, configured hosts/data directory, and opt-in restart resumption.
- [ ] Add interval range, rotation settings and counters to the UI; use capability metadata for platform instructions.
- [ ] Run full backend tests and frontend tests/build, then commit.
### Task 4: Provision, migrate, and deploy
Files: deploy/*, docs/debian-deployment.md, README.md.
- [ ] Prepare reproducible systemd, Nginx and private configuration templates with rollback and backup instructions.
- [ ] Provision independent MariaDB account/database, dedicated loopback Redis, proxy pool and workbench services.
- [ ] Transfer online SQLite backup and credentials through SSH without exposing secrets; verify migrated data and server login.
- [ ] Validate authenticated HTTPS, browser workflows, proxy API/scheduler and restart persistence; resume migrated task only after valid authentication.
- [ ] Fresh reviewer checks the whole branch; fix material findings and verify again. Publish the deployment branch to the user's repository.
@@ -0,0 +1,29 @@
# Debian 12 持续采集部署
用户要求将工作台部署到 186.241.68.230,通过已有 SSH 密钥操作,避免频繁连接触发 fail2ban;使用服务器实际 MariaDB,部署 jhao104/proxy_pool,核实代理复用,并加入随机请求间隔。保留现有本机运行任务。
## 运行与访问
使用 Debian 的 Python 3.11、独立虚拟环境和 systemd 服务,普通专用用户运行工作台。现有 MariaDB 10.11 创建独立数据库与最小权限账号,不改其他项目数据库。Nginx 提供 HTTPS 和访问认证;优先指定域名,DNS 未就绪时使用已有有效 IP 证书及独立 9443 端口。后端、代理池与 Redis 仅监听回环地址。操作 SSH 尽量复用持续连接。
## 数据
增加 MariaDB/MySQL 存储实现,复用现有数据查询与任务接口;Windows 默认继续使用 SQLite。MariaDB 保存目录、分数线、院校索引、收藏、任务、日志和完成断点。使用 utf8mb4 与稳定记录键,保留原文、年份和特殊计划的匹配规则。
SQLite 在线 backup 生成一致性快照;显式迁移到空的新库,保留记录 ID 和全部断点。迁移完成核对每张表的行数及记录内容。旧电脑任务保持原状态,服务器副本初次迁移暂停,登录与网络验证成功后才恢复。重启时仅自动恢复原先运行的任务,用户暂停、取消、等待登录任务不自动启动。
## 凭据与无桌面登录
Windows 继续 DPAPI;Linux 使用本地权限 0600 的密钥和加密 Cookie。密钥与数据库配置存放运行目录,排除 Git 和公开下载。服务器无需桌面:界面说明在用户电脑完成学信网登录并粘贴 Cookie;账号登录仍可用,验证码保留人工流程。迁移原有 Cookie 通过 SSH 安全传输,并在服务器重新验证,不能继承原“已验证”标记。
## 请求与代理
每次请求从配置的最小与最大间隔随机采样,最大不得小于最小,两者均在 0.3–60 秒;相等时允许固定间隔。保留限流退避与人工验证码处理。
增加“每 N 次成功请求轮换代理”,0 表示保留原策略;服务器默认 100。显示当前代理的请求计数、累计切换次数及实际成功请求路由。会话同步和第二页验证期间延迟自动轮换,轮换后的目录会话重新验证。健康轮换保留旧 IP,不将限流误认成失效。
Linux 代理 API 与调度器使用专门 systemd 单元,独立 Redis 容器只绑定 127.0.0.1:6381。界面管理固定单元名称,只允许启动和停止本项目 API/调度器,不接受任意 shell 命令,不停止其他项目 Redis。
## 验证
现有测试继续通过。新增随机间隔边界、轮换与会话固定、Linux 凭据往返、主机与跨来源保护测试。在实际 MariaDB 临时测试库验证重复更新、收藏、历史线匹配、特殊线隔离、任务断点和 SQLite 迁移。构建网页,在服务器验证 HTTPS 认证、页面、数据查询、代理池、登录状态、systemd 重启恢复和现有站点未受影响。
+25 -5
View File
@@ -1713,6 +1713,11 @@ function ProxyPanel({ notify }: { notify: Notify }) {
? "代理池暂无可用 IP,优先代理模式已回退直连。" ? "代理池暂无可用 IP,优先代理模式已回退直连。"
: "登录、招生目录和历年考情共用该路由。"} : "登录、招生目录和历年考情共用该路由。"}
</p> </p>
<p className="form-hint">
当前代理已成功请求 {data?.route_requests || 0} 次 · 已切换 {data?.rotations || 0} 次
{data?.rotate_every ? ` · 每 ${data.rotate_every} 次自动轮换` : " · 连接失败或手动切换时轮换"}
{data?.last_request_route && ` · 最近成功请求:${data.last_request_route === "direct" ? "直连" : data.last_request_route}`}
</p>
</div> </div>
</div> </div>
<Badge tone={data?.fallback ? "amber" : "green"}> <Badge tone={data?.fallback ? "amber" : "green"}>
@@ -1972,7 +1977,7 @@ function SettingsPanel({ notify }: { notify: Notify }) {
</label> </label>
<div className="three-fields"> <div className="three-fields">
<label> <label>
请求间隔(秒) 请求间隔下限(秒)
<input <input
type="number" type="number"
min="0.3" min="0.3"
@@ -1984,6 +1989,18 @@ function SettingsPanel({ notify }: { notify: Notify }) {
} }
/> />
</label> </label>
<label>
请求间隔上限(秒)
<input type="number" min={config.request_interval} max="60" step="0.1"
value={config.request_interval_max}
onChange={(e) => update("request_interval_max", Number(e.target.value))} />
</label>
<label>
自动轮换代理(请求次数)
<input type="number" min="0" max="100000" step="1"
value={config.proxy_rotate_every}
onChange={(e) => update("proxy_rotate_every", Number(e.target.value))} />
</label>
<label> <label>
超时(秒) 超时(秒)
<input <input
@@ -2005,6 +2022,7 @@ function SettingsPanel({ notify }: { notify: Notify }) {
/> />
</label> </label>
</div> </div>
<p className="form-hint">每次请求随机取上下限之间的间隔;两者相等时固定。轮换次数填 0 可关闭自动轮换;登录会话验证期间保留当前代理。</p>
<button className="primary" disabled={!!busy}> <button className="primary" disabled={!!busy}>
{busy === "save" ? ( {busy === "save" ? (
<Loader2 className="spin" size={16} /> <Loader2 className="spin" size={16} />
@@ -2048,17 +2066,19 @@ function SettingsPanel({ notify }: { notify: Notify }) {
<div> <div>
<Globe2 size={22} /> <Globe2 size={22} />
<div> <div>
<b>在 Edge 中完成登录</b> <b>{auth?.browser_supported === false ? "在自己电脑上完成登录" : "在 Edge 中完成登录"}</b>
<small>支持验证码,自动获取并加密保存 Cookie</small> <small>{auth?.browser_supported === false ? "服务器无桌面,请登录研招网后在下方粘贴 Cookie" : "支持验证码,自动获取并加密保存 Cookie"}</small>
</div> </div>
</div> </div>
<button {auth?.browser_supported === false ? (
<a className="primary" href="https://yz.chsi.com.cn/zsml/a/dw.do" target="_blank" rel="noopener noreferrer">打开研招网登录页面 <ArrowUpRight size={16} /></a>
) : <button
className="primary" className="primary"
disabled={!!busy || !!browserWaiting} disabled={!!busy || !!browserWaiting}
onClick={() => authenticate("browser")} onClick={() => authenticate("browser")}
> >
打开登录窗口 <ArrowUpRight size={16} /> 打开登录窗口 <ArrowUpRight size={16} />
</button> </button>}
{auth?.browser_message && ( {auth?.browser_message && (
<p className="form-hint">{auth.browser_message}</p> <p className="form-hint">{auth.browser_message}</p>
)} )}
+7
View File
@@ -92,10 +92,16 @@ export type Config = {
pool_project: string; pool_project: string;
backup_proxies: string[]; backup_proxies: string[];
request_interval: number; request_interval: number;
request_interval_max: number;
proxy_rotate_every: number;
timeout: number; timeout: number;
retries: number; retries: number;
}; };
export type Proxy = { export type Proxy = {
route_requests: number;
rotations: number;
last_request_route: string | null;
rotate_every: number;
mode: string; mode: string;
route: string; route: string;
active_proxy: string | null; active_proxy: string | null;
@@ -136,6 +142,7 @@ export type PoolManageResult = {
status: PoolServiceStatus; status: PoolServiceStatus;
}; };
export type Auth = { export type Auth = {
browser_supported: boolean;
saved: boolean; saved: boolean;
valid: boolean; valid: boolean;
state: "unverified" | "verified" | "login_required" | "captcha_required"; state: "unverified" | "verified" | "login_required" | "captcha_required";
+3 -1
View File
@@ -2,7 +2,7 @@
name = "kaoyan-studio" name = "kaoyan-studio"
version = "1.0.0" version = "1.0.0"
description = "本地考研招生目录、历年分数线与代理池工作台" description = "本地考研招生目录、历年分数线与代理池工作台"
requires-python = ">=3.12" requires-python = ">=3.11"
dependencies = [ dependencies = [
"fastapi>=0.115,<1", "fastapi>=0.115,<1",
"uvicorn>=0.34,<1", "uvicorn>=0.34,<1",
@@ -11,6 +11,8 @@ dependencies = [
"openpyxl>=3.1,<4", "openpyxl>=3.1,<4",
"python-multipart>=0.0.20,<1", "python-multipart>=0.0.20,<1",
"playwright>=1.50,<2", "playwright>=1.50,<2",
"PyMySQL>=1.1,<2",
"cryptography>=44,<47",
] ]
[project.optional-dependencies] [project.optional-dependencies]
+4
View File
@@ -3,9 +3,11 @@ annotated-types==0.8.0
anyio==4.15.1 anyio==4.15.1
beautifulsoup4==4.15.0 beautifulsoup4==4.15.0
certifi==2026.7.22 certifi==2026.7.22
cffi==2.1.1
charset-normalizer==3.5.2 charset-normalizer==3.5.2
click==8.5.0 click==8.5.0
colorama==0.4.6 colorama==0.4.6
cryptography==50.0.2
et_xmlfile==2.0.0 et_xmlfile==2.0.0
fastapi==0.143.0 fastapi==0.143.0
greenlet==3.5.6 greenlet==3.5.6
@@ -19,10 +21,12 @@ opentelemetry-api==1.45.1
packaging==26.3 packaging==26.3
playwright==1.63.0 playwright==1.63.0
pluggy==1.6.0 pluggy==1.6.0
pycparser==3.11
pydantic==2.14.0 pydantic==2.14.0
pydantic_core==2.50.0 pydantic_core==2.50.0
pyee==13.0.1 pyee==13.0.1
Pygments==2.21.0 Pygments==2.21.0
PyMySQL==1.2.3
pytest==9.1.1 pytest==9.1.1
python-multipart==0.0.32 python-multipart==0.0.32
requests==2.34.2 requests==2.34.2
+15 -5
View File
@@ -1,6 +1,7 @@
"""Local-only HTTP workbench and production SPA host.""" """Local-only HTTP workbench and production SPA host."""
import json import json
import uuid import uuid
import os
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse
@@ -20,9 +21,15 @@ ROOT = Path(__file__).resolve().parent.parent
def create_app(data_dir=None): def create_app(data_dir=None):
directory = Path(data_dir or ROOT / ".runtime") directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
directory.mkdir(parents=True, exist_ok=True) directory.mkdir(parents=True, exist_ok=True)
db = Database(directory / "studio.db") database_config = os.environ.get("STUDIO_DATABASE_CONFIG")
if database_config:
from studio.mysql_database import MySQLDatabase
db = MySQLDatabase(json.loads(Path(database_config).read_text(encoding="utf-8")))
else:
db = Database(directory / "studio.db")
allowed_hosts = {"127.0.0.1", "localhost", "::1"} | {host.strip().lower() for host in os.environ.get("STUDIO_ALLOWED_HOSTS", "").split(",") if host.strip()}
settings = SettingsStore(directory) settings = SettingsStore(directory)
pool = PoolService(settings) pool = PoolService(settings)
router = ProxyRouter(settings) router = ProxyRouter(settings)
@@ -34,6 +41,9 @@ def create_app(data_dir=None):
@asynccontextmanager @asynccontextmanager
async def lifespan(app): async def lifespan(app):
if os.environ.get("STUDIO_AUTO_RESUME") == "1":
for jid in db.recovered_job_ids:
jobs.resume(jid)
yield yield
jobs.close() jobs.close()
@@ -46,8 +56,8 @@ def create_app(data_dir=None):
@app.middleware("http") @app.middleware("http")
async def local_access(request: Request, call_next): async def local_access(request: Request, call_next):
host = request.url.hostname host = request.url.hostname
if host not in ("127.0.0.1", "localhost", "::1"): if host not in allowed_hosts:
return JSONResponse({"detail": "仅允许本机访问"}, status_code=403) return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
if request.method not in ("GET", "HEAD", "OPTIONS"): if request.method not in ("GET", "HEAD", "OPTIONS"):
origin = request.headers.get("origin") origin = request.headers.get("origin")
if origin: if origin:
@@ -76,7 +86,7 @@ def create_app(data_dir=None):
@app.get("/api/health") @app.get("/api/health")
def health(): def health():
return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve())} return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve()), "database": "mysql" if database_config else "sqlite", "platform": "windows" if os.name == "nt" else "linux"}
@app.get("/api/overview") @app.get("/api/overview")
def overview(): def overview():
+35 -4
View File
@@ -2,6 +2,7 @@
import ctypes import ctypes
import json import json
import os import os
import platform
import threading import threading
import time import time
from pathlib import Path from pathlib import Path
@@ -40,15 +41,41 @@ class CookieVault:
self.path = Path(directory) / "chsi-cookie.enc" self.path = Path(directory) / "chsi-cookie.enc"
self.path.parent.mkdir(parents=True, exist_ok=True) self.path.parent.mkdir(parents=True, exist_ok=True)
def _cipher(self):
from cryptography.fernet import Fernet
key_path = self.path.parent / "cookie.key"
if not key_path.exists():
try:
descriptor = os.open(key_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
except FileExistsError:
pass
else:
with os.fdopen(descriptor, "wb") as stream:
stream.write(Fernet.generate_key())
return Fernet(key_path.read_bytes())
def save(self, cookies): def save(self, cookies):
temporary = self.path.with_suffix(".tmp") temporary = self.path.with_suffix(".tmp")
temporary.write_bytes(protect(json.dumps(cookies, ensure_ascii=False).encode())) payload = json.dumps(cookies, ensure_ascii=False).encode()
encrypted = protect(payload) if platform.system() == "Windows" else self._cipher().encrypt(payload)
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(descriptor, "wb") as stream:
stream.write(encrypted)
temporary.replace(self.path) temporary.replace(self.path)
def load(self): def load(self):
if not self.path.exists(): if not self.path.exists():
return {} return {}
return json.loads(protect(self.path.read_bytes(), decrypt=True)) payload = self.path.read_bytes()
if platform.system() == "Windows":
payload = protect(payload, decrypt=True)
else:
from cryptography.fernet import InvalidToken
try:
payload = self._cipher().decrypt(payload)
except InvalidToken:
raise ValueError("保存的 Cookie 无法解密,请重新登录") from None
return json.loads(payload)
def clear(self): def clear(self):
self.path.unlink(missing_ok=True) self.path.unlink(missing_ok=True)
@@ -94,7 +121,7 @@ class AuthService:
with self.state_lock: with self.state_lock:
if self.valid and self.verified_route != self.transport.router.status()["route"]: if self.valid and self.verified_route != self.transport.router.status()["route"]:
self.invalidate("网络出口已变化,请验证目录访问后继续采集", state="unverified") self.invalidate("网络出口已变化,请验证目录访问后继续采集", state="unverified")
return {"saved": self.vault.path.exists(), "valid": self.valid, "state": self.state, "message": self.message, "checked_at": self.checked_at, "browser_state": self.browser_state, "browser_message": self.browser_message} return {"saved": self.vault.path.exists(), "valid": self.valid, "state": self.state, "message": self.message, "checked_at": self.checked_at, "browser_state": self.browser_state, "browser_message": self.browser_message, "browser_supported": platform.system() == "Windows"}
def invalidate(self, message, state="login_required"): def invalidate(self, message, state="login_required"):
with self.state_lock: with self.state_lock:
@@ -128,7 +155,7 @@ class AuthService:
return self.status() return self.status()
def check(self): def check(self):
with self.check_lock: with self.check_lock, self.transport.router.hold_route():
with self.state_lock: with self.state_lock:
self.invalidate("正在同步研招目录会话并验证访问权限", state="unverified") self.invalidate("正在同步研招目录会话并验证访问权限", state="unverified")
revision = self.revision revision = self.revision
@@ -182,6 +209,10 @@ class AuthService:
return self.status() return self.status()
def browser(self): def browser(self):
if platform.system() != "Windows":
self.browser_state = "unsupported"
self.browser_message = "服务器无桌面环境:请在自己电脑的浏览器完成学信网登录,再粘贴研招网 Cookie 并验证目录访问"
return self.status()
if self.browser_thread and self.browser_thread.is_alive() and not self.browser_cancel.is_set(): if self.browser_thread and self.browser_thread.is_alive() and not self.browser_cancel.is_set():
return self.status() return self.status()
proxy = self.transport.router.resolve() proxy = self.transport.router.resolve()
+7 -3
View File
@@ -22,7 +22,7 @@ CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,sc
""" + SCORE_SCHEMA + """ """ + SCORE_SCHEMA + """
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC); CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT); CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0); CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit)); CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT); CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
""" """
@@ -45,7 +45,11 @@ class Database:
conn.execute('DROP TABLE scores') conn.execute('DROP TABLE scores')
conn.execute('ALTER TABLE scores_new RENAME TO scores') conn.execute('ALTER TABLE scores_new RENAME TO scores')
conn.execute('CREATE INDEX scores_lookup ON scores(school_code,major_code,year DESC)') conn.execute('CREATE INDEX scores_lookup ON scores(school_code,major_code,year DESC)')
conn.execute("UPDATE jobs SET status='paused',message='服务重启,任务可从断点恢复' WHERE status IN ('running','queued')") if 'resume_pending' not in {row['name'] for row in conn.execute('PRAGMA table_info(jobs)')}:
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending INTEGER DEFAULT 0')
recovery = "status IN ('running','queued') OR (status='paused' AND resume_pending=1)"
self.recovered_job_ids = [row['id'] for row in conn.execute('SELECT id FROM jobs WHERE ' + recovery)]
conn.execute("UPDATE jobs SET status='paused',resume_pending=0,message='服务重启,任务可从断点恢复' WHERE " + recovery)
@contextmanager @contextmanager
def connection(self): def connection(self):
@@ -182,7 +186,7 @@ class Database:
conn.execute("INSERT INTO jobs(id,params,status,created_at,updated_at,message) VALUES(?,?,'queued',?,?,?)",(jid,encode(params),now(),now(),"等待开始")) conn.execute("INSERT INTO jobs(id,params,status,created_at,updated_at,message) VALUES(?,?,'queued',?,?,?)",(jid,encode(params),now(),now(),"等待开始"))
def job_update(self, jid, **changes): def job_update(self, jid, **changes):
allowed = {"status","message","errors","records"} allowed = {"status","message","errors","records","resume_pending"}
changes = {key:value for key,value in changes.items() if key in allowed} changes = {key:value for key,value in changes.items() if key in allowed}
changes["updated_at"] = now() changes["updated_at"] = now()
with self.connection() as conn: with self.connection() as conn:
+3 -3
View File
@@ -67,7 +67,7 @@ class JobManager:
def _run(self, jid): def _run(self, jid):
job = self.get(jid) job = self.get(jid)
self.db.job_update(jid, status="running", errors=0, message="正在采集") self.db.job_update(jid, status="running", errors=0, resume_pending=0, message="正在采集")
self.db.log(jid, "info", "任务开始/恢复,将跳过已完成单元") self.db.log(jid, "info", "任务开始/恢复,将跳过已完成单元")
if self.transport: if self.transport:
self.transport.local.guard = lambda: self.checkpoint(jid) self.transport.local.guard = lambda: self.checkpoint(jid)
@@ -127,7 +127,7 @@ class JobManager:
raise ValueError("任务当前不在运行") raise ValueError("任务当前不在运行")
with control["condition"]: with control["condition"]:
control["paused"] = True control["paused"] = True
self.db.job_update(jid, status="paused", message="已暂停,当前请求结束后停止") self.db.job_update(jid, status="paused", resume_pending=0, message="已暂停,当前请求结束后停止")
return self.get(jid) return self.get(jid)
def resume(self, jid): def resume(self, jid):
@@ -169,7 +169,7 @@ class JobManager:
if control["thread"].is_alive(): if control["thread"].is_alive():
with control['condition']: with control['condition']:
control['shutdown'] = True control['shutdown'] = True
self.db.job_update(jid, status='paused', message='服务停止,任务可从断点恢复') self.db.job_update(jid, status='paused', resume_pending=int(not control['paused']), message='服务停止,任务可从断点恢复')
control['condition'].notify_all() control['condition'].notify_all()
for control in list(self.controls.values()): for control in list(self.controls.values()):
control["thread"].join(timeout=1) control["thread"].join(timeout=1)
+61
View File
@@ -0,0 +1,61 @@
"""Control only the two fixed, dedicated proxy-pool systemd units."""
import socket
import subprocess
from urllib.parse import urlparse
import requests
UNITS = ('kaoyan-proxy-api.service', 'kaoyan-proxy-scheduler.service')
class LinuxPoolService:
def __init__(self, settings):
self.settings = settings
self.session = requests.Session()
self.session.trust_env = False
def status(self, config=None):
config = config or self.settings.read()
api_ready, counts = False, {}
try:
response = self.session.get(config['pool_url'] + '/count/', timeout=2)
response.raise_for_status()
counts = response.json()
api_ready = isinstance(counts, dict) and type(counts.get('count')) is int
except (requests.RequestException, ValueError):
pass
items = []
for name, label, unit in zip(('server', 'schedule'), ('代理 API', '抓取与检测'), UNITS):
try:
result = subprocess.run(['/usr/bin/systemctl', 'show', unit, '--property=LoadState,ActiveState,MainPID'], capture_output=True, text=True, timeout=5)
facts = dict(line.split('=', 1) for line in result.stdout.splitlines() if '=' in line)
installed = facts.get('LoadState') == 'loaded'
active = installed and facts.get('ActiveState') == 'active'
items.append({'name': name, 'label': label, 'state': 'running' if active else 'stopped', 'pid': int(facts.get('MainPID', '0')) or None, 'verified': active, 'elevated': False, 'detail': 'systemd 运行中' if active else ('已停止' if installed else '未安装本项目 systemd 单元')})
except (OSError, subprocess.TimeoutExpired, ValueError):
items.append({'name': name, 'label': label, 'state': 'unknown', 'pid': None, 'verified': False, 'elevated': False, 'detail': '无法查询 systemd 状态'})
try:
with socket.create_connection(('127.0.0.1', 6381), timeout=1):
redis_ready = True
except OSError:
redis_ready = False
items.append({'name': 'redis', 'label': 'Redis', 'state': 'external' if redis_ready else 'stopped', 'pid': None, 'verified': False, 'elevated': False, 'detail': '独立 Redis 服务(127.0.0.1:6381)' if redis_ready else '未连接到本项目 Redis'})
local = urlparse(config['pool_url']).hostname in ('127.0.0.1', 'localhost', '::1')
return {'api_ready': api_ready, 'counts': counts if api_ready else {}, 'items': items, 'can_stop': local and any(item['verified'] for item in items[:2]), 'warning': '' if local else '当前使用外部代理池,请在其所在服务器管理服务', 'admin_command': '', 'manager': 'systemd'}
def manage(self, action):
if action not in ('start', 'stop', 'status'):
raise ValueError('代理池操作无效')
status = self.status()
result = {'ok': True, 'action': action, 'changed': False, 'status': status, 'message': '服务状态已更新', 'output': '\n'.join(item['label'] + ':' + item['detail'] for item in status['items'])}
if action == 'status':
return result
if status['warning']:
return {**result, 'ok': False, 'message': status['warning']}
if action == 'start' and all(item['state'] == 'running' for item in status['items'][:2]) and status['api_ready']:
return {**result, 'message': '代理池已在运行,已复用现有服务'}
try:
completed = subprocess.run(['/usr/bin/sudo', '-n', '/usr/bin/systemctl', action, *UNITS], capture_output=True, text=True, timeout=55)
message = '代理池启动完成' if action == 'start' else '代理池已停止'
return {**result, 'ok': completed.returncode == 0, 'changed': True, 'status': self.status(), 'message': message if completed.returncode == 0 else 'systemd 操作失败,请检查本项目 sudoers 与服务日志', 'output': (completed.stdout + completed.stderr)[-5000:] or message}
except (OSError, subprocess.TimeoutExpired):
return {**result, 'ok': False, 'message': '代理池管理超时或命令不可用,请检查 systemd 服务'}
+74
View File
@@ -0,0 +1,74 @@
"""Explicit, atomic migration from a consistent SQLite snapshot into an empty MySQL database."""
import argparse
import json
import sqlite3
from pathlib import Path
from studio.domain import normalized_name, is_special_score
from studio.mysql_database import MySQLDatabase, score_key
TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs')
def migrate_sqlite(source, target):
source = Path(source).resolve()
conn = sqlite3.connect(source.as_uri() + '?mode=ro', uri=True)
conn.row_factory = sqlite3.Row
counts = {}
try:
conn.execute('BEGIN')
with target.connection() as output:
# Serialize attempted imports to the same database.
acquired = output.execute("SELECT GET_LOCK('kaoyan_studio_migration',10)").fetchone()[0]
if acquired != 1:
raise ValueError('已有数据迁移正在进行')
try:
for table in TABLES:
if output.execute(f'SELECT count(*) FROM {table}').fetchone()[0]:
raise ValueError(f'目标数据库非空({table}),拒绝覆盖;请使用空数据库')
for table in TABLES:
counts[table] = 0
reader = conn.execute(f'SELECT * FROM {table}')
while batch := reader.fetchmany(500):
rows = []
for original in batch:
row = dict(original)
if table in ('catalog', 'scores'):
payload = json.loads(row['data'])
row['department_norm'] = normalized_name(row.get('department_name'))
if table == 'scores':
row['note'] = row.get('note') or payload.get('note') or ''
row['raw_code'] = row.get('raw_code') or payload.get('raw_code') or ''
row['score_key'] = score_key(row)
row['special_line'] = int(is_special_score(row['note']))
if table == 'jobs' and row['status'] in ('running', 'queued'):
row.update(status='paused', message='已迁移到服务器,验证登录后可从断点恢复')
if table == 'jobs':
row['resume_pending'] = 0
rows.append(row)
fields = list(rows[0])
output.executemany(f"INSERT INTO {table}({','.join(fields)}) VALUES({','.join('?' for _ in fields)})", [[row[field] for field in fields] for row in rows])
counts[table] += len(rows)
for table, count in counts.items():
if output.execute(f'SELECT count(*) FROM {table}').fetchone()[0] != count:
raise ValueError(f'迁移行数校验失败:{table}')
finally:
# The private connection closes after commit/rollback, releasing its
# advisory lock only after the entire import becomes visible.
pass
return counts
finally:
conn.rollback()
conn.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('snapshot', type=Path)
parser.add_argument('--config', type=Path, required=True)
args = parser.parse_args()
config = json.loads(args.config.read_text())
print(json.dumps(migrate_sqlite(args.snapshot, MySQLDatabase(config)), ensure_ascii=False))
if __name__ == '__main__':
main()
+165
View File
@@ -0,0 +1,165 @@
"""MariaDB/MySQL storage retaining the local Database's public interface."""
import hashlib
import json
import re
from contextlib import contextmanager
import pymysql
from studio.database import Database, encode
from studio.domain import normalized_name, is_special_score
SCHEMA = [
"""CREATE TABLE IF NOT EXISTS catalog(
id BIGINT PRIMARY KEY AUTO_INCREMENT,record_key CHAR(64) NOT NULL UNIQUE,year INT NOT NULL,
school_code VARCHAR(32),school_name VARCHAR(255),province VARCHAR(64),major_code VARCHAR(32),major_name VARCHAR(255),
department_code VARCHAR(64),department_name VARCHAR(255),study_mode VARCHAR(32),degree_type VARCHAR(64),level VARCHAR(128),
subjects TEXT,quota INT,source VARCHAR(128),favorite TINYINT NOT NULL DEFAULT 0,data LONGTEXT NOT NULL,
department_norm VARCHAR(255) NOT NULL DEFAULT '',INDEX catalog_filter(year,province,major_code,school_code))""",
"""CREATE TABLE IF NOT EXISTS scores(
id BIGINT PRIMARY KEY AUTO_INCREMENT,school_code VARCHAR(32),major_code VARCHAR(32),year INT,
department_id VARCHAR(64),department_name VARCHAR(255),scope VARCHAR(32),total INT,source VARCHAR(128),data LONGTEXT NOT NULL,
note TEXT NOT NULL,raw_code VARCHAR(64) NOT NULL,score_key CHAR(64) NOT NULL UNIQUE,
department_norm VARCHAR(255) NOT NULL DEFAULT '',special_line TINYINT NOT NULL DEFAULT 0,
INDEX scores_lookup(school_code,major_code,year))""",
"CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)",
"CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)",
"CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))",
"CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))",
]
def score_key(row):
identity = [row.get(field) or '' for field in ('school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'source', 'note', 'raw_code')]
return hashlib.sha256(encode(identity).encode()).hexdigest()
class Row(dict):
def __getitem__(self, key):
return tuple(self.values())[key] if isinstance(key, int) else super().__getitem__(key)
class Result:
def __init__(self, cursor):
self.rowcount = cursor.rowcount
self.columns = [item[0] for item in cursor.description] if cursor.description else []
self.rows = cursor.fetchall() if cursor.description else ()
self.offset = 0
cursor.close()
def fetchone(self):
if self.offset >= len(self.rows):
return None
result = Row(zip(self.columns, self.rows[self.offset]))
self.offset += 1
return result
def fetchall(self):
return list(self)
def __iter__(self):
while self.offset < len(self.rows):
yield self.fetchone()
def mysql_sql(sql):
if sql.startswith('SELECT count(*) FROM (') and sql.endswith(')'):
sql += ' AS studio_count'
sql = sql.replace('INSERT OR IGNORE', 'INSERT IGNORE')
sql = re.sub(r'ON CONFLICT\([^)]*\) DO UPDATE SET', 'ON DUPLICATE KEY UPDATE', sql)
sql = re.sub(r'excluded\.(\w+)', r'VALUES(\1)', sql)
sql = sql.replace("is_special_score(json_extract(x.data,'$.note'))=0", 'x.special_line=0')
sql = sql.replace('norm_name(x.department_name)', 'x.department_norm').replace('norm_name(c.department_name)', 'c.department_norm')
sql = re.sub(r"json_extract\(([^,]+),('[^']+')\)", r'JSON_UNQUOTE(JSON_EXTRACT(\1,\2))', sql)
return sql.replace('?', '%s')
class Connection:
def __init__(self, raw):
self.raw = raw
def execute(self, sql, params=None):
cursor = self.raw.cursor()
try:
cursor.execute(mysql_sql(sql), params)
return Result(cursor)
except BaseException:
cursor.close()
raise
def executemany(self, sql, values):
cursor = self.raw.cursor()
try:
cursor.executemany(mysql_sql(sql), values)
return Result(cursor)
except BaseException:
cursor.close()
raise
class MySQLDatabase(Database):
def __init__(self, config):
self.config = {key: value for key, value in config.items() if key in ('host', 'port', 'user', 'password', 'database', 'unix_socket')}
if not self.config.get('database'):
raise ValueError('数据库配置缺少 database')
self.recovered_job_ids = []
with self.connection() as conn:
for statement in SCHEMA:
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
recovery = "status IN ('running','queued') OR (status='paused' AND resume_pending=1)"
self.recovered_job_ids = [row['id'] for row in conn.execute('SELECT id FROM jobs WHERE ' + recovery)]
conn.execute("UPDATE jobs SET status='paused',resume_pending=0,message='服务重启,任务可从断点恢复' WHERE " + recovery)
@contextmanager
def connection(self):
raw = pymysql.connect(**self.config, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
try:
yield Connection(raw)
raw.commit()
except BaseException:
raw.rollback()
raise
finally:
raw.close()
def upsert_catalog(self, records):
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
values = []
for record in records:
row = {**record, 'subjects': ' | '.join(record.get('subjects', [])), 'data': encode(record), 'department_norm': normalized_name(record.get('department_name'))}
values.append([row.get(field) for field in fields])
if not values:
return
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
with self.connection() as conn:
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
for record in records:
self._school(conn, record)
def upsert_scores(self, records):
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
values = []
for record in records:
row = {**record, 'department_id': record.get('department_id') or '', 'department_name': record.get('department_name') or '', 'source': record.get('source', '掌上考研'), 'note': record.get('note') or '', 'raw_code': record.get('raw_code') or ''}
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
values.append([row.get(field) for field in fields])
if values:
with self.connection() as conn:
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
def favorite(self, rid, enabled):
# MariaDB reports changed rows; an already-selected favorite is still success.
with self.connection() as conn:
conn.execute('UPDATE catalog SET favorite=? WHERE id=?', (int(enabled), rid))
return conn.execute('SELECT 1 FROM catalog WHERE id=?', (rid,)).fetchone() is not None
def history(self, rid):
record = self.record(rid)
if record is None:
return None
with self.connection() as conn:
scores = conn.execute('SELECT data FROM scores WHERE school_code=? AND major_code=? ORDER BY year,scope,department_name', (record['school_code'], record['major_code'])).fetchall()
plans = conn.execute('SELECT data FROM catalog WHERE school_code=? AND major_code=? AND department_norm=? ORDER BY year,id', (record['school_code'], record['major_code'], normalized_name(record['department_name']))).fetchall()
def matches(row):
return not row.get('department_name') and not row.get('department_id') or row.get('department_id') and row['department_id'] == record['department_code'] or row.get('department_name') and normalized_name(row['department_name']) == normalized_name(record['department_name'])
return {'record': record, 'scores': [row for entry in scores if matches(row := json.loads(entry['data']))], 'plans': [json.loads(row['data']) for row in plans]}
+37 -5
View File
@@ -1,6 +1,8 @@
"""One explicit route for CHSI login, catalog and historical data.""" """One explicit route for CHSI login, catalog and historical data."""
import threading import threading
import time import time
import random
from contextlib import contextmanager
import requests import requests
from studio.settings import proxy_url from studio.settings import proxy_url
@@ -29,6 +31,10 @@ class ProxyRouter:
self.counts = {} self.counts = {}
self.fallback = False self.fallback = False
self.lock = threading.RLock() self.lock = threading.RLock()
self.route_requests = 0
self.rotations = 0
self.last_request_route = None
self.holds = 0
def refresh(self): def refresh(self):
with self.lock: with self.lock:
@@ -60,12 +66,31 @@ class ProxyRouter:
self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"])) self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"]))
return self.status() return self.status()
def resolve(self): @contextmanager
def hold_route(self):
with self.lock:
self.holds += 1
try:
yield
finally:
with self.lock:
self.holds -= 1
def note_success(self, proxy):
with self.lock:
self.last_request_route = proxy or "direct"
if proxy == self.active:
self.route_requests += 1
def resolve(self, for_request=False):
with self.lock: with self.lock:
config = self.settings.read() config = self.settings.read()
if config["network_mode"] == "direct": if config["network_mode"] == "direct":
self.fallback = False self.fallback = False
return None return None
threshold = config["proxy_rotate_every"]
if for_request and not self.holds and threshold and self.active and self.route_requests >= threshold:
self.rotate()
if self.active and self.active not in self.failed: if self.active and self.active not in self.failed:
self.fallback = False self.fallback = False
return self.active return self.active
@@ -75,9 +100,11 @@ class ProxyRouter:
available = [p for p in self.candidates if p not in self.failed] available = [p for p in self.candidates if p not in self.failed]
if available: if available:
self.active = available[0] self.active = available[0]
self.route_requests = 0
self.fallback = False self.fallback = False
return self.active return self.active
self.active = None self.active = None
self.route_requests = 0
if config["network_mode"] == "proxy_only": if config["network_mode"] == "proxy_only":
self.fallback = False self.fallback = False
raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务") raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务")
@@ -93,7 +120,10 @@ class ProxyRouter:
if old in self.candidates: if old in self.candidates:
self.candidates.remove(old) self.candidates.remove(old)
self.candidates.append(old) self.candidates.append(old)
return self.resolve() selected = self.resolve()
if old and selected != old:
self.rotations += 1
return selected
def reset(self): def reset(self):
with self.lock: with self.lock:
@@ -101,10 +131,11 @@ class ProxyRouter:
self.failed.clear() self.failed.clear()
self.candidates = [] self.candidates = []
self.fallback = False self.fallback = False
self.route_requests = 0
def status(self): def status(self):
config = self.settings.read() config = self.settings.read()
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]} return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "route_requests": self.route_requests, "rotations": self.rotations, "last_request_route": self.last_request_route, "rotate_every": config["proxy_rotate_every"], "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
class Transport: class Transport:
@@ -130,8 +161,8 @@ class Transport:
def request(self, method, url, **kwargs): def request(self, method, url, **kwargs):
config = self.settings.read() config = self.settings.read()
for attempt in range(config["retries"]): for attempt in range(config["retries"]):
self.sleep(config["request_interval"] if attempt == 0 else min(20, 2 ** attempt)) self.sleep(random.uniform(config["request_interval"], config["request_interval_max"]) if attempt == 0 else min(20, 2 ** attempt))
proxy = self.router.resolve() proxy = self.router.resolve(for_request=True)
kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {} kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {}
kwargs.setdefault("timeout", config["timeout"]) kwargs.setdefault("timeout", config["timeout"])
try: try:
@@ -143,6 +174,7 @@ class Transport:
continue continue
response.raise_for_status() response.raise_for_status()
self.local.response_route = proxy or "direct" self.local.response_route = proxy or "direct"
self.router.note_success(proxy)
return response return response
except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout): except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout):
if proxy: if proxy:
+8
View File
@@ -137,6 +137,10 @@ def _pool_entry(arguments, project, service):
class PoolService: class PoolService:
def __init__(self, settings): def __init__(self, settings):
self.settings = settings self.settings = settings
self.linux = None
if os.name != "nt":
from studio.linux_pool import LinuxPoolService
self.linux = LinuxPoolService(settings)
self.session = requests.Session() self.session = requests.Session()
self.session.trust_env = False self.session.trust_env = False
self.lock = threading.Lock() self.lock = threading.Lock()
@@ -154,6 +158,8 @@ class PoolService:
return json.loads(result.stdout.decode("utf-8-sig")) return json.loads(result.stdout.decode("utf-8-sig"))
def status(self, config=None): def status(self, config=None):
if self.linux:
return self.linux.status(config)
if config is None: if config is None:
config = self.settings.read() config = self.settings.read()
project = Path(config["pool_project"]) project = Path(config["pool_project"])
@@ -233,6 +239,8 @@ class PoolService:
return "\n".join(lines) return "\n".join(lines)
def manage(self, action): def manage(self, action):
if self.linux:
return self.linux.manage(action)
if action not in ("start", "stop", "status"): if action not in ("start", "stop", "status"):
raise ValueError("代理池操作无效") raise ValueError("代理池操作无效")
with self.lock: with self.lock:
+14 -6
View File
@@ -1,13 +1,16 @@
"""Project-local, non-secret configuration.""" """Project-local, non-secret configuration."""
import json import json
import threading import threading
import math
import os
from pathlib import Path from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse
DEFAULTS = { DEFAULTS = {
"network_mode": "prefer_proxy", "pool_url": "http://127.0.0.1:5010", "network_mode": "prefer_proxy", "pool_url": "http://127.0.0.1:5010",
"pool_project": r"C:\Users\Jayson\Desktop\proxy_pool", "pool_project": r"C:\Users\Jayson\Desktop\proxy_pool" if os.name == "nt" else "/opt/kaoyan-studio-deploy/proxy_pool",
"backup_proxies": [], "request_interval": 1.2, "timeout": 30, "retries": 3, "backup_proxies": [], "request_interval": 1.2, "request_interval_max": 2.4,
"proxy_rotate_every": 0, "timeout": 30, "retries": 3,
} }
@@ -37,7 +40,10 @@ class SettingsStore:
def read(self): def read(self):
with self.lock: with self.lock:
values = json.loads(self.path.read_text(encoding="utf-8")) if self.path.exists() else {} values = json.loads(self.path.read_text(encoding="utf-8")) if self.path.exists() else {}
return {**DEFAULTS, **{k: v for k, v in values.items() if k in DEFAULTS}} result = {**DEFAULTS, **{k: v for k, v in values.items() if k in DEFAULTS}}
if "request_interval_max" not in values:
result["request_interval_max"] = max(result["request_interval"], DEFAULTS["request_interval_max"])
return result
def update(self, values): def update(self, values):
with self.lock: with self.lock:
@@ -48,10 +54,12 @@ class SettingsStore:
if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password: if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:
raise ValueError("代理池 API 地址无效") raise ValueError("代理池 API 地址无效")
values["pool_url"] = values["pool_url"].rstrip("/") values["pool_url"] = values["pool_url"].rstrip("/")
for key, low, high in [("request_interval", .3, 60), ("timeout", 5, 120), ("retries", 1, 6)]: for key, low, high in [("request_interval", .3, 60), ("request_interval_max", .3, 60), ("timeout", 5, 120), ("retries", 1, 6), ("proxy_rotate_every", 0, 100000)]:
values[key] = float(values[key]) if key == "request_interval" else int(values[key]) values[key] = float(values[key]) if key.startswith("request_interval") else int(values[key])
if not low <= values[key] <= high: if not math.isfinite(values[key]) or not low <= values[key] <= high:
raise ValueError(f"{key} 应在 {low} 到 {high} 之间") raise ValueError(f"{key} 应在 {low} 到 {high} 之间")
if values["request_interval_max"] < values["request_interval"]:
raise ValueError("请求间隔上限不能小于下限")
proxies = values["backup_proxies"] proxies = values["backup_proxies"]
if isinstance(proxies, str): if isinstance(proxies, str):
proxies = proxies.splitlines() proxies = proxies.splitlines()
+99
View File
@@ -0,0 +1,99 @@
"""Real MariaDB contract tests; run only against an explicitly supplied test database."""
import json
import os
from pathlib import Path
import pytest
from studio.domain import normalize_catalog, select_score_lines
from studio.database import Database
from tests.test_data import raw_catalog
pytestmark = pytest.mark.skipif(not os.environ.get('STUDIO_MYSQL_TEST_CONFIG'), reason='requires an isolated MariaDB test database')
@pytest.fixture
def mysql():
from studio.mysql_database import MySQLDatabase
config = json.loads(Path(os.environ['STUDIO_MYSQL_TEST_CONFIG']).read_text())
assert config['database'].endswith('_test'), 'refusing to clear a production database'
db = MySQLDatabase(config)
with db.connection() as conn:
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs'):
conn.execute(f'DELETE FROM {table}')
return db
def test_mysql_preserves_upserts_favorites_and_history(mysql):
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
first = mysql.query_catalog({})['items'][0]
mysql.favorite(first['id'], True)
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
mysql.upsert_scores(select_score_lines([
{'code': '081200', 'total': 320, 'depart_name': '计算机学院'},
{'code': '081200', 'total': 260, 'depart_name': '计算机学院', 'note': '少干专项计划'},
{'code': '081200', 'total': 390, 'depart_name': '其他学院'},
], '081200', '10001', '北京大学', 2026))
result = mysql.query_catalog({'year': 2027, 'direction': '人工', 'english': '203'})
assert result['total'] == 1
assert result['items'][0]['score_total'] == 320
assert mysql.query_catalog({'favorites': True})['total'] == 1
assert mysql.overview()['records'] == 4
assert len(mysql.history(first['id'])['plans']) == 4
assert len(mysql.history(first['id'])['scores']) == 2
def test_mysql_score_unique_key_preserves_long_notes_and_normalized_department(mysql):
raw = raw_catalog()
raw['yxsmc'] = '计算机 ( 学院 )'
mysql.upsert_catalog(normalize_catalog(raw, 2027))
scores = select_score_lines([
{'code': '081200', 'total': 320, 'depart_name': '计算机(学院)', 'note': '普通计划' * 300},
{'code': '081200', 'total': 321, 'depart_name': '计算机(学院)', 'note': '普通计划' * 300 + 'B'},
], '081200', '10001', '北京大学', 2026)
mysql.upsert_scores(scores)
mysql.upsert_scores(scores)
assert mysql.overview()['scores'] == 2
assert mysql.query_catalog({})['items'][0]['score_total'] == 321
def test_mysql_tasks_logs_and_checkpoints(mysql):
mysql.job_create('contract-job', {'year': 2027})
mysql.job_update('contract-job', status='running', records=7)
mysql.complete_unit('contract-job', 'catalog:11:10001:081200:1')
mysql.complete_unit('contract-job', 'catalog:11:10001:081200:1')
mysql.log('contract-job', 'info', '数据库测试')
assert mysql.unit_done('contract-job', 'catalog:11:10001:081200:1')
assert mysql.job_get('contract-job')['completed_units'] == 1
assert mysql.job_get('contract-job')['records'] == 7
assert mysql.logs('contract-job')[0]['message'] == '数据库测试'
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'snapshot.db')
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
source.upsert_scores(select_score_lines([{'code': '081200', 'total': 320}], '081200', '10001', '北京大学', 2026))
source.favorite(1, True)
source.job_create('migrated-job', {'year': 2027})
source.complete_unit('migrated-job', 'catalog:11:10001:081200:1')
source.log('migrated-job', 'info', 'preserve log')
counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql)
assert counts['catalog'] == 2 and counts['scores'] == 1
assert mysql.record(1)['favorite']
assert mysql.query_catalog({})['items'][0]['score_total'] == 320
assert mysql.job_get('migrated-job')['completed_units'] == 1
assert mysql.logs('migrated-job')[0]['id'] == 1
with pytest.raises(ValueError, match='非空'):
migrate_sqlite(tmp_path / 'snapshot.db', mysql)
def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'bad.db')
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
with source.connection() as conn:
conn.execute("INSERT INTO scores(school_code,major_code,year,scope,data) VALUES('10001','081200',2026,'major','not json')")
with pytest.raises(ValueError):
migrate_sqlite(tmp_path / 'bad.db', mysql)
assert mysql.overview()['records'] == 0
assert mysql.overview()['scores'] == 0
+2
View File
@@ -11,6 +11,8 @@ from fastapi.testclient import TestClient
from studio.api import create_app from studio.api import create_app
pytestmark = pytest.mark.skipif(os.name != 'nt', reason='Windows pool process and PowerShell behavior')
@pytest.fixture @pytest.fixture
def pool_client(tmp_path): def pool_client(tmp_path):
+109
View File
@@ -0,0 +1,109 @@
import platform
import random
import os
import pytest
from studio.auth import CookieVault
from tests.test_network import setup_network
def test_requests_sample_a_fresh_delay_inside_configured_bounds(tmp_path, monkeypatch):
settings, _, transport = setup_network(tmp_path)
settings.update({'request_interval': .3, 'request_interval_max': .5})
samples = iter([.31, .47, .39])
monkeypatch.setattr(random, 'uniform', lambda low, high: next(samples))
delays = []
transport.sleep = delays.append
for _ in range(3):
transport.request('GET', 'https://yz.chsi.com.cn/zsml/')
assert delays == [.31, .47, .39]
@pytest.mark.parametrize('maximum', [.2, 61, float('nan')])
def test_invalid_interval_range_is_rejected(tmp_path, maximum):
settings, _, _ = setup_network(tmp_path)
with pytest.raises(ValueError):
settings.update({'request_interval': .3, 'request_interval_max': maximum})
def test_successful_requests_rotate_without_quarantining_old_proxy(tmp_path):
settings, router, transport = setup_network(tmp_path)
settings.update({'proxy_rotate_every': 2})
routes = [transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] for _ in range(3)]
assert routes == ['http://1.2.3.4:8080', 'http://1.2.3.4:8080', 'http://5.6.7.8:9090']
assert router.failed == set()
assert router.status()['last_request_route'] == 'http://5.6.7.8:9090'
assert router.status()['route_requests'] == 1
def test_route_pin_defers_rotation_until_login_proof_finishes(tmp_path):
settings, router, transport = setup_network(tmp_path)
settings.update({'proxy_rotate_every': 1})
with router.hold_route():
routes = [transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] for _ in range(2)]
assert routes == ['http://1.2.3.4:8080'] * 2
assert transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] == 'http://5.6.7.8:9090'
def test_linux_cookie_vault_encrypts_and_preserves_domain_cookies(tmp_path, monkeypatch):
monkeypatch.setattr(platform, 'system', lambda: 'Linux')
cookies = [{'name': 'JSESSIONID', 'value': 'private-account-cookie', 'domain': 'account.chsi.com.cn'},
{'name': 'JSESSIONID', 'value': 'private-catalog-cookie', 'domain': 'yz.chsi.com.cn'}]
vault = CookieVault(tmp_path)
vault.save(cookies)
assert (tmp_path / 'cookie.key').exists()
assert b'private-account-cookie' not in vault.path.read_bytes()
assert CookieVault(tmp_path).load() == cookies
if os.name != 'nt':
assert (tmp_path / 'cookie.key').stat().st_mode & 0o777 == 0o600
assert vault.path.stat().st_mode & 0o777 == 0o600
def test_configured_server_host_is_allowed_but_other_hosts_and_origins_are_blocked(tmp_path, monkeypatch):
from fastapi.testclient import TestClient
from studio.api import create_app
monkeypatch.setenv('STUDIO_ALLOWED_HOSTS', 'kaoyan.fjayson.com')
with TestClient(create_app(tmp_path), base_url='https://kaoyan.fjayson.com') as client:
assert client.get('/api/health').status_code == 200
assert client.get('/api/health', headers={'Host': 'evil.example'}).status_code == 403
assert client.put('/api/settings', json={'network_mode': 'direct'}, headers={'Origin': 'https://evil.example'}).status_code == 403
assert client.put('/api/settings', json={'network_mode': 'direct'}, headers={'Origin': 'https://kaoyan.fjayson.com'}).status_code == 200
def test_only_interrupted_running_tasks_are_selected_for_restart(tmp_path):
from studio.database import Database
db = Database(tmp_path / 'resume.db')
for status in ('running', 'queued', 'paused', 'waiting_login', 'cancelled'):
db.job_create(status, {'year': 2027})
db.job_update(status, status=status)
restarted = Database(tmp_path / 'resume.db')
assert set(restarted.recovered_job_ids) == {'running', 'queued'}
assert restarted.job_get('paused')['status'] == 'paused'
assert restarted.job_get('waiting_login')['status'] == 'waiting_login'
@pytest.mark.parametrize('manually_paused', [False, True])
def test_graceful_restart_remembers_running_but_not_manual_pause(tmp_path, manually_paused):
import threading
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
ready = threading.Event()
stop = threading.Event()
def runner(jid, params, manager):
ready.set()
while not stop.wait(.01):
manager.checkpoint(jid)
db = Database(tmp_path / 'shutdown.db')
manager = JobManager(db, SettingsStore(tmp_path), runner=runner)
job = manager.start({'mode': 'catalog', 'year': 2027, 'provinces': ['11']})
assert ready.wait(3)
try:
if manually_paused:
manager.pause(job['id'])
manager.close()
restarted = Database(tmp_path / 'shutdown.db')
assert (job['id'] in restarted.recovered_job_ids) is (not manually_paused)
finally:
stop.set()
manager.close()