From 915241a2fdd3f0a30119427a53359b56ea0f2eb5 Mon Sep 17 00:00:00 2001
From: Codex
Date: Sun, 11 Oct 2026 19:07:27 +0800
Subject: [PATCH] Add Debian deployment, MariaDB storage and randomized proxy
routing
---
deploy/kaoyan-pool-sudoers | 1 +
deploy/kaoyan-proxy-api.service | 20 +++
deploy/kaoyan-proxy-scheduler.service | 20 +++
deploy/kaoyan-studio.env.example | 5 +
deploy/kaoyan-studio.service | 22 +++
deploy/nginx-http-bootstrap.conf | 10 ++
deploy/nginx-https.conf | 23 +++
deploy/provision-services.sh | 35 ++++
.../plans/2026-10-11-debian-server.md | 63 +++++++
.../specs/2026-10-11-debian-server-design.md | 29 +++
frontend/src/App.tsx | 30 +++-
frontend/src/types.ts | 7 +
pyproject.toml | 4 +-
requirements.lock | 4 +
studio/api.py | 20 ++-
studio/auth.py | 39 ++++-
studio/database.py | 10 +-
studio/jobs.py | 6 +-
studio/linux_pool.py | 61 +++++++
studio/migrate.py | 74 ++++++++
studio/mysql_database.py | 165 ++++++++++++++++++
studio/network.py | 42 ++++-
studio/pool_service.py | 8 +
studio/settings.py | 20 ++-
tests/test_mysql.py | 99 +++++++++++
tests/test_pool_service.py | 2 +
tests/test_server_features.py | 109 ++++++++++++
27 files changed, 896 insertions(+), 32 deletions(-)
create mode 100644 deploy/kaoyan-pool-sudoers
create mode 100644 deploy/kaoyan-proxy-api.service
create mode 100644 deploy/kaoyan-proxy-scheduler.service
create mode 100644 deploy/kaoyan-studio.env.example
create mode 100644 deploy/kaoyan-studio.service
create mode 100644 deploy/nginx-http-bootstrap.conf
create mode 100644 deploy/nginx-https.conf
create mode 100644 deploy/provision-services.sh
create mode 100644 docs/superpowers/plans/2026-10-11-debian-server.md
create mode 100644 docs/superpowers/specs/2026-10-11-debian-server-design.md
create mode 100644 studio/linux_pool.py
create mode 100644 studio/migrate.py
create mode 100644 studio/mysql_database.py
create mode 100644 tests/test_mysql.py
create mode 100644 tests/test_server_features.py
diff --git a/deploy/kaoyan-pool-sudoers b/deploy/kaoyan-pool-sudoers
new file mode 100644
index 0000000..3b00bf5
--- /dev/null
+++ b/deploy/kaoyan-pool-sudoers
@@ -0,0 +1 @@
+kaoyan ALL=(root) NOPASSWD: /usr/bin/systemctl start kaoyan-proxy-api.service kaoyan-proxy-scheduler.service, /usr/bin/systemctl stop kaoyan-proxy-api.service kaoyan-proxy-scheduler.service
diff --git a/deploy/kaoyan-proxy-api.service b/deploy/kaoyan-proxy-api.service
new file mode 100644
index 0000000..cbe6aa1
--- /dev/null
+++ b/deploy/kaoyan-proxy-api.service
@@ -0,0 +1,20 @@
+[Unit]
+Description=Kaoyan proxy-pool API
+After=network-online.target docker.service
+Wants=network-online.target
+Requires=docker.service
+
+[Service]
+User=kaoyan
+Group=kaoyan
+WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
+ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py server
+Restart=on-failure
+RestartSec=10
+UMask=0077
+PrivateTmp=true
+ProtectSystem=full
+ProtectHome=true
+
+[Install]
+WantedBy=multi-user.target
diff --git a/deploy/kaoyan-proxy-scheduler.service b/deploy/kaoyan-proxy-scheduler.service
new file mode 100644
index 0000000..d47ca80
--- /dev/null
+++ b/deploy/kaoyan-proxy-scheduler.service
@@ -0,0 +1,20 @@
+[Unit]
+Description=Kaoyan proxy-pool fetching and validation
+After=network-online.target docker.service
+Wants=network-online.target
+Requires=docker.service
+
+[Service]
+User=kaoyan
+Group=kaoyan
+WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
+ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py schedule
+Restart=on-failure
+RestartSec=10
+UMask=0077
+PrivateTmp=true
+ProtectSystem=full
+ProtectHome=true
+
+[Install]
+WantedBy=multi-user.target
diff --git a/deploy/kaoyan-studio.env.example b/deploy/kaoyan-studio.env.example
new file mode 100644
index 0000000..ea14f16
--- /dev/null
+++ b/deploy/kaoyan-studio.env.example
@@ -0,0 +1,5 @@
+STUDIO_DATABASE_CONFIG=/opt/kaoyan-studio-deploy/mysql.json
+STUDIO_DATA_DIR=/var/lib/kaoyan-studio
+STUDIO_ALLOWED_HOSTS=kaoyan.fjayson.com,186.241.68.230
+STUDIO_AUTO_RESUME=1
+PYTHONUNBUFFERED=1
diff --git a/deploy/kaoyan-studio.service b/deploy/kaoyan-studio.service
new file mode 100644
index 0000000..9dbe2a4
--- /dev/null
+++ b/deploy/kaoyan-studio.service
@@ -0,0 +1,22 @@
+[Unit]
+Description=Kaoyan data workbench
+After=network-online.target mariadb.service kaoyan-proxy-api.service
+Wants=network-online.target kaoyan-proxy-api.service kaoyan-proxy-scheduler.service
+
+[Service]
+Type=simple
+User=kaoyan
+Group=kaoyan
+WorkingDirectory=/opt/kaoyan-studio-deploy/app
+EnvironmentFile=/etc/kaoyan-studio.env
+ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio
+Restart=on-failure
+RestartSec=10
+TimeoutStopSec=120
+UMask=0077
+PrivateTmp=true
+ProtectSystem=full
+ProtectHome=true
+
+[Install]
+WantedBy=multi-user.target
diff --git a/deploy/nginx-http-bootstrap.conf b/deploy/nginx-http-bootstrap.conf
new file mode 100644
index 0000000..68353d2
--- /dev/null
+++ b/deploy/nginx-http-bootstrap.conf
@@ -0,0 +1,10 @@
+server {
+ listen 80;
+ listen [::]:80;
+ server_name kaoyan.fjayson.com;
+ location ^~ /.well-known/acme-challenge/ {
+ root /var/www/kaoyan-acme;
+ auth_basic off;
+ }
+ location / { return 301 https://$host$request_uri; }
+}
diff --git a/deploy/nginx-https.conf b/deploy/nginx-https.conf
new file mode 100644
index 0000000..97197fd
--- /dev/null
+++ b/deploy/nginx-https.conf
@@ -0,0 +1,23 @@
+server {
+ listen 443 ssl;
+ listen [::]:443 ssl;
+ server_name kaoyan.fjayson.com;
+ ssl_certificate /etc/letsencrypt/live/kaoyan.fjayson.com/fullchain.pem;
+ ssl_certificate_key /etc/letsencrypt/live/kaoyan.fjayson.com/privkey.pem;
+ ssl_protocols TLSv1.2 TLSv1.3;
+ client_max_body_size 100m;
+ auth_basic "Kaoyan Workbench";
+ auth_basic_user_file /etc/nginx/kaoyan-studio.htpasswd;
+ location / {
+ proxy_pass http://127.0.0.1:8765;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_read_timeout 120s;
+ proxy_hide_header Cache-Control;
+ add_header Cache-Control "private, no-store" always;
+ add_header X-Content-Type-Options nosniff always;
+ add_header X-Frame-Options DENY always;
+ }
+}
diff --git a/deploy/provision-services.sh b/deploy/provision-services.sh
new file mode 100644
index 0000000..53785da
--- /dev/null
+++ b/deploy/provision-services.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Run as root after installing the two virtual environments and private MySQL config.
+set -euo pipefail
+root=/opt/kaoyan-studio-deploy
+test -f "$root/mysql.json"
+test -x "$root/venv/bin/python"
+test -x "$root/proxy-venv/bin/python"
+id kaoyan >/dev/null 2>&1 || useradd --system --user-group --home-dir /var/lib/kaoyan-studio --shell /usr/sbin/nologin kaoyan
+install -d -m 700 -o kaoyan -g kaoyan /var/lib/kaoyan-studio
+install -d -m 750 -o kaoyan -g kaoyan "$root/proxy_pool/log"
+chown root:kaoyan "$root/mysql.json"
+chmod 640 "$root/mysql.json"
+install -m 640 -o root -g kaoyan "$root/app/deploy/kaoyan-studio.env.example" /etc/kaoyan-studio.env
+for unit in kaoyan-studio kaoyan-proxy-api kaoyan-proxy-scheduler; do
+ install -m 644 "$root/app/deploy/$unit.service" "/etc/systemd/system/$unit.service"
+done
+visudo -cf "$root/app/deploy/kaoyan-pool-sudoers"
+install -m 440 "$root/app/deploy/kaoyan-pool-sudoers" /etc/sudoers.d/kaoyan-pool
+if ! docker container inspect kaoyan-proxy-redis >/dev/null 2>&1; then
+ docker run -d --name kaoyan-proxy-redis --restart unless-stopped \
+ -p 127.0.0.1:6381:6379 -v kaoyan-proxy-redis:/data \
+ redis:7.2 redis-server --appendonly yes
+fi
+python3 - <<'PY'
+from pathlib import Path
+path = Path('/opt/kaoyan-studio-deploy/proxy_pool/setting.py')
+text = path.read_text()
+text = text.replace('HOST = "0.0.0.0"', 'HOST = "127.0.0.1"')
+text = text.replace("DB_CONN = 'redis://:pwdstring@127.0.0.1:6379/0'", "DB_CONN = 'redis://127.0.0.1:6381/0'")
+path.write_text(text)
+PY
+systemctl daemon-reload
+systemctl enable kaoyan-studio kaoyan-proxy-api kaoyan-proxy-scheduler
+systemctl start kaoyan-proxy-api kaoyan-proxy-scheduler
+# The workbench starts only after its database/credential migration is validated.
diff --git a/docs/superpowers/plans/2026-10-11-debian-server.md b/docs/superpowers/plans/2026-10-11-debian-server.md
new file mode 100644
index 0000000..1411b94
--- /dev/null
+++ b/docs/superpowers/plans/2026-10-11-debian-server.md
@@ -0,0 +1,63 @@
+# Debian server implementation plan
+
+> **For agentic workers:** Use superpowers:executing-plans to implement this plan inline, with a fresh whole-branch reviewer.
+
+**Goal:** Run the existing workbench continuously on the user's Debian 12 server with MariaDB, a managed proxy pool, randomized pacing, and preserved collected data.
+
+**Architecture:** Keep SQLite as the local default. Add a MySQL database adapter behind the existing Database interface, Linux credential encryption and fixed systemd pool management. Serve the single-worker application on loopback behind authenticated HTTPS.
+
+**Tech Stack:** Python 3.11+, FastAPI, PyMySQL, cryptography, React, MariaDB 10.11, systemd, Nginx, proxy_pool, dedicated Redis.
+
+**Spec:** ../specs/2026-10-11-debian-server-design.md
+
+## Global Constraints
+
+- Preserve the currently running Windows collection.
+- Never print or commit cookies, encryption keys, database passwords, or access passwords.
+- Reuse SSH connections; do not alter existing databases or unrelated services.
+- Random request bounds: 0.3–60 seconds; server proxy rotation: 100 successful requests.
+- No automated captcha solving; no inherited login verification after migration.
+
+## Review Focus
+
+- Same-name cookies across CHSI domains must survive encrypted migration.
+- Normalized department names and special-plan score rules must match SQLite results.
+- Failed migrations must not leave a partly imported database; source remains untouched.
+- Automatic rotation must not split the SSO binding and access proof across proxies.
+- Manual pause and login-wait states must survive service restart without automatic resumption.
+
+### Task 1: Random pacing and proxy rotation
+
+Files: studio/settings.py, studio/network.py, studio/auth.py, tests/test_server_features.py.
+
+- [ ] Add failing tests for varied bounded delays, invalid bounds, automatic route changes, and pinned verification.
+- [ ] Implement interval maximum and proxy rotation settings, actual request route/count reporting, and route pinning during auth verification.
+- [ ] Run network and auth tests, then commit.
+
+### Task 2: MariaDB backend and migration
+
+Files: studio/mysql_database.py, studio/migrate.py, studio/api.py, tests/test_mysql.py, pyproject.toml.
+
+- [ ] Write tests against a real isolated MariaDB database for upserts, historical scopes, favorites, jobs, and atomic migration.
+- [ ] Implement MySQLDatabase.connection(), preserving the existing Database API; load private database configuration through STUDIO_DATABASE_CONFIG.
+- [ ] Implement migrate_sqlite(source, target) preserving IDs, logs and unit keys and rejecting nonempty targets.
+- [ ] Verify all existing tests and real MariaDB integration tests, then commit.
+
+### Task 3: Linux operation and user interface
+
+Files: studio/auth.py, studio/linux_pool.py, studio/pool_service.py, studio/__main__.py, studio/api.py, frontend/src/App.tsx, frontend/src/types.ts.
+
+- [ ] Add failing tests for Linux vault roundtrip/permissions, server host filtering, and interrupted-task recovery selection.
+- [ ] Implement encrypted Linux storage, explicit manual-cookie login instructions, Linux systemd pool controls, configured hosts/data directory, and opt-in restart resumption.
+- [ ] Add interval range, rotation settings and counters to the UI; use capability metadata for platform instructions.
+- [ ] Run full backend tests and frontend tests/build, then commit.
+
+### Task 4: Provision, migrate, and deploy
+
+Files: deploy/*, docs/debian-deployment.md, README.md.
+
+- [ ] Prepare reproducible systemd, Nginx and private configuration templates with rollback and backup instructions.
+- [ ] Provision independent MariaDB account/database, dedicated loopback Redis, proxy pool and workbench services.
+- [ ] Transfer online SQLite backup and credentials through SSH without exposing secrets; verify migrated data and server login.
+- [ ] Validate authenticated HTTPS, browser workflows, proxy API/scheduler and restart persistence; resume migrated task only after valid authentication.
+- [ ] Fresh reviewer checks the whole branch; fix material findings and verify again. Publish the deployment branch to the user's repository.
diff --git a/docs/superpowers/specs/2026-10-11-debian-server-design.md b/docs/superpowers/specs/2026-10-11-debian-server-design.md
new file mode 100644
index 0000000..e2617f5
--- /dev/null
+++ b/docs/superpowers/specs/2026-10-11-debian-server-design.md
@@ -0,0 +1,29 @@
+# Debian 12 持续采集部署
+
+用户要求将工作台部署到 186.241.68.230,通过已有 SSH 密钥操作,避免频繁连接触发 fail2ban;使用服务器实际 MariaDB,部署 jhao104/proxy_pool,核实代理复用,并加入随机请求间隔。保留现有本机运行任务。
+
+## 运行与访问
+
+使用 Debian 的 Python 3.11、独立虚拟环境和 systemd 服务,普通专用用户运行工作台。现有 MariaDB 10.11 创建独立数据库与最小权限账号,不改其他项目数据库。Nginx 提供 HTTPS 和访问认证;优先指定域名,DNS 未就绪时使用已有有效 IP 证书及独立 9443 端口。后端、代理池与 Redis 仅监听回环地址。操作 SSH 尽量复用持续连接。
+
+## 数据
+
+增加 MariaDB/MySQL 存储实现,复用现有数据查询与任务接口;Windows 默认继续使用 SQLite。MariaDB 保存目录、分数线、院校索引、收藏、任务、日志和完成断点。使用 utf8mb4 与稳定记录键,保留原文、年份和特殊计划的匹配规则。
+
+SQLite 在线 backup 生成一致性快照;显式迁移到空的新库,保留记录 ID 和全部断点。迁移完成核对每张表的行数及记录内容。旧电脑任务保持原状态,服务器副本初次迁移暂停,登录与网络验证成功后才恢复。重启时仅自动恢复原先运行的任务,用户暂停、取消、等待登录任务不自动启动。
+
+## 凭据与无桌面登录
+
+Windows 继续 DPAPI;Linux 使用本地权限 0600 的密钥和加密 Cookie。密钥与数据库配置存放运行目录,排除 Git 和公开下载。服务器无需桌面:界面说明在用户电脑完成学信网登录并粘贴 Cookie;账号登录仍可用,验证码保留人工流程。迁移原有 Cookie 通过 SSH 安全传输,并在服务器重新验证,不能继承原“已验证”标记。
+
+## 请求与代理
+
+每次请求从配置的最小与最大间隔随机采样,最大不得小于最小,两者均在 0.3–60 秒;相等时允许固定间隔。保留限流退避与人工验证码处理。
+
+增加“每 N 次成功请求轮换代理”,0 表示保留原策略;服务器默认 100。显示当前代理的请求计数、累计切换次数及实际成功请求路由。会话同步和第二页验证期间延迟自动轮换,轮换后的目录会话重新验证。健康轮换保留旧 IP,不将限流误认成失效。
+
+Linux 代理 API 与调度器使用专门 systemd 单元,独立 Redis 容器只绑定 127.0.0.1:6381。界面管理固定单元名称,只允许启动和停止本项目 API/调度器,不接受任意 shell 命令,不停止其他项目 Redis。
+
+## 验证
+
+现有测试继续通过。新增随机间隔边界、轮换与会话固定、Linux 凭据往返、主机与跨来源保护测试。在实际 MariaDB 临时测试库验证重复更新、收藏、历史线匹配、特殊线隔离、任务断点和 SQLite 迁移。构建网页,在服务器验证 HTTPS 认证、页面、数据查询、代理池、登录状态、systemd 重启恢复和现有站点未受影响。
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 9af0386..a530ea1 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -1713,6 +1713,11 @@ function ProxyPanel({ notify }: { notify: Notify }) {
? "代理池暂无可用 IP,优先代理模式已回退直连。"
: "登录、招生目录和历年考情共用该路由。"}
+
+ 当前代理已成功请求 {data?.route_requests || 0} 次 · 已切换 {data?.rotations || 0} 次
+ {data?.rotate_every ? ` · 每 ${data.rotate_every} 次自动轮换` : " · 连接失败或手动切换时轮换"}
+ {data?.last_request_route && ` · 最近成功请求:${data.last_request_route === "direct" ? "直连" : data.last_request_route}`}
+
@@ -1972,7 +1977,7 @@ function SettingsPanel({ notify }: { notify: Notify }) {
+
+
+ 每次请求随机取上下限之间的间隔;两者相等时固定。轮换次数填 0 可关闭自动轮换;登录会话验证期间保留当前代理。