diff --git a/deploy/kaoyan-pool-sudoers b/deploy/kaoyan-pool-sudoers new file mode 100644 index 0000000..3b00bf5 --- /dev/null +++ b/deploy/kaoyan-pool-sudoers @@ -0,0 +1 @@ +kaoyan ALL=(root) NOPASSWD: /usr/bin/systemctl start kaoyan-proxy-api.service kaoyan-proxy-scheduler.service, /usr/bin/systemctl stop kaoyan-proxy-api.service kaoyan-proxy-scheduler.service diff --git a/deploy/kaoyan-proxy-api.service b/deploy/kaoyan-proxy-api.service new file mode 100644 index 0000000..cbe6aa1 --- /dev/null +++ b/deploy/kaoyan-proxy-api.service @@ -0,0 +1,20 @@ +[Unit] +Description=Kaoyan proxy-pool API +After=network-online.target docker.service +Wants=network-online.target +Requires=docker.service + +[Service] +User=kaoyan +Group=kaoyan +WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool +ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py server +Restart=on-failure +RestartSec=10 +UMask=0077 +PrivateTmp=true +ProtectSystem=full +ProtectHome=true + +[Install] +WantedBy=multi-user.target diff --git a/deploy/kaoyan-proxy-scheduler.service b/deploy/kaoyan-proxy-scheduler.service new file mode 100644 index 0000000..d47ca80 --- /dev/null +++ b/deploy/kaoyan-proxy-scheduler.service @@ -0,0 +1,20 @@ +[Unit] +Description=Kaoyan proxy-pool fetching and validation +After=network-online.target docker.service +Wants=network-online.target +Requires=docker.service + +[Service] +User=kaoyan +Group=kaoyan +WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool +ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py schedule +Restart=on-failure +RestartSec=10 +UMask=0077 +PrivateTmp=true +ProtectSystem=full +ProtectHome=true + +[Install] +WantedBy=multi-user.target diff --git a/deploy/kaoyan-studio.env.example b/deploy/kaoyan-studio.env.example new file mode 100644 index 0000000..ea14f16 --- /dev/null +++ b/deploy/kaoyan-studio.env.example @@ -0,0 +1,5 @@ +STUDIO_DATABASE_CONFIG=/opt/kaoyan-studio-deploy/mysql.json +STUDIO_DATA_DIR=/var/lib/kaoyan-studio +STUDIO_ALLOWED_HOSTS=kaoyan.fjayson.com,186.241.68.230 +STUDIO_AUTO_RESUME=1 +PYTHONUNBUFFERED=1 diff --git a/deploy/kaoyan-studio.service b/deploy/kaoyan-studio.service new file mode 100644 index 0000000..9dbe2a4 --- /dev/null +++ b/deploy/kaoyan-studio.service @@ -0,0 +1,22 @@ +[Unit] +Description=Kaoyan data workbench +After=network-online.target mariadb.service kaoyan-proxy-api.service +Wants=network-online.target kaoyan-proxy-api.service kaoyan-proxy-scheduler.service + +[Service] +Type=simple +User=kaoyan +Group=kaoyan +WorkingDirectory=/opt/kaoyan-studio-deploy/app +EnvironmentFile=/etc/kaoyan-studio.env +ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio +Restart=on-failure +RestartSec=10 +TimeoutStopSec=120 +UMask=0077 +PrivateTmp=true +ProtectSystem=full +ProtectHome=true + +[Install] +WantedBy=multi-user.target diff --git a/deploy/nginx-http-bootstrap.conf b/deploy/nginx-http-bootstrap.conf new file mode 100644 index 0000000..68353d2 --- /dev/null +++ b/deploy/nginx-http-bootstrap.conf @@ -0,0 +1,10 @@ +server { + listen 80; + listen [::]:80; + server_name kaoyan.fjayson.com; + location ^~ /.well-known/acme-challenge/ { + root /var/www/kaoyan-acme; + auth_basic off; + } + location / { return 301 https://$host$request_uri; } +} diff --git a/deploy/nginx-https.conf b/deploy/nginx-https.conf new file mode 100644 index 0000000..97197fd --- /dev/null +++ b/deploy/nginx-https.conf @@ -0,0 +1,23 @@ +server { + listen 443 ssl; + listen [::]:443 ssl; + server_name kaoyan.fjayson.com; + ssl_certificate /etc/letsencrypt/live/kaoyan.fjayson.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/kaoyan.fjayson.com/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + client_max_body_size 100m; + auth_basic "Kaoyan Workbench"; + auth_basic_user_file /etc/nginx/kaoyan-studio.htpasswd; + location / { + proxy_pass http://127.0.0.1:8765; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_read_timeout 120s; + proxy_hide_header Cache-Control; + add_header Cache-Control "private, no-store" always; + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + } +} diff --git a/deploy/provision-services.sh b/deploy/provision-services.sh new file mode 100644 index 0000000..53785da --- /dev/null +++ b/deploy/provision-services.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Run as root after installing the two virtual environments and private MySQL config. +set -euo pipefail +root=/opt/kaoyan-studio-deploy +test -f "$root/mysql.json" +test -x "$root/venv/bin/python" +test -x "$root/proxy-venv/bin/python" +id kaoyan >/dev/null 2>&1 || useradd --system --user-group --home-dir /var/lib/kaoyan-studio --shell /usr/sbin/nologin kaoyan +install -d -m 700 -o kaoyan -g kaoyan /var/lib/kaoyan-studio +install -d -m 750 -o kaoyan -g kaoyan "$root/proxy_pool/log" +chown root:kaoyan "$root/mysql.json" +chmod 640 "$root/mysql.json" +install -m 640 -o root -g kaoyan "$root/app/deploy/kaoyan-studio.env.example" /etc/kaoyan-studio.env +for unit in kaoyan-studio kaoyan-proxy-api kaoyan-proxy-scheduler; do + install -m 644 "$root/app/deploy/$unit.service" "/etc/systemd/system/$unit.service" +done +visudo -cf "$root/app/deploy/kaoyan-pool-sudoers" +install -m 440 "$root/app/deploy/kaoyan-pool-sudoers" /etc/sudoers.d/kaoyan-pool +if ! docker container inspect kaoyan-proxy-redis >/dev/null 2>&1; then + docker run -d --name kaoyan-proxy-redis --restart unless-stopped \ + -p 127.0.0.1:6381:6379 -v kaoyan-proxy-redis:/data \ + redis:7.2 redis-server --appendonly yes +fi +python3 - <<'PY' +from pathlib import Path +path = Path('/opt/kaoyan-studio-deploy/proxy_pool/setting.py') +text = path.read_text() +text = text.replace('HOST = "0.0.0.0"', 'HOST = "127.0.0.1"') +text = text.replace("DB_CONN = 'redis://:pwdstring@127.0.0.1:6379/0'", "DB_CONN = 'redis://127.0.0.1:6381/0'") +path.write_text(text) +PY +systemctl daemon-reload +systemctl enable kaoyan-studio kaoyan-proxy-api kaoyan-proxy-scheduler +systemctl start kaoyan-proxy-api kaoyan-proxy-scheduler +# The workbench starts only after its database/credential migration is validated. diff --git a/docs/superpowers/plans/2026-10-11-debian-server.md b/docs/superpowers/plans/2026-10-11-debian-server.md new file mode 100644 index 0000000..1411b94 --- /dev/null +++ b/docs/superpowers/plans/2026-10-11-debian-server.md @@ -0,0 +1,63 @@ +# Debian server implementation plan + +> **For agentic workers:** Use superpowers:executing-plans to implement this plan inline, with a fresh whole-branch reviewer. + +**Goal:** Run the existing workbench continuously on the user's Debian 12 server with MariaDB, a managed proxy pool, randomized pacing, and preserved collected data. + +**Architecture:** Keep SQLite as the local default. Add a MySQL database adapter behind the existing Database interface, Linux credential encryption and fixed systemd pool management. Serve the single-worker application on loopback behind authenticated HTTPS. + +**Tech Stack:** Python 3.11+, FastAPI, PyMySQL, cryptography, React, MariaDB 10.11, systemd, Nginx, proxy_pool, dedicated Redis. + +**Spec:** ../specs/2026-10-11-debian-server-design.md + +## Global Constraints + +- Preserve the currently running Windows collection. +- Never print or commit cookies, encryption keys, database passwords, or access passwords. +- Reuse SSH connections; do not alter existing databases or unrelated services. +- Random request bounds: 0.3–60 seconds; server proxy rotation: 100 successful requests. +- No automated captcha solving; no inherited login verification after migration. + +## Review Focus + +- Same-name cookies across CHSI domains must survive encrypted migration. +- Normalized department names and special-plan score rules must match SQLite results. +- Failed migrations must not leave a partly imported database; source remains untouched. +- Automatic rotation must not split the SSO binding and access proof across proxies. +- Manual pause and login-wait states must survive service restart without automatic resumption. + +### Task 1: Random pacing and proxy rotation + +Files: studio/settings.py, studio/network.py, studio/auth.py, tests/test_server_features.py. + +- [ ] Add failing tests for varied bounded delays, invalid bounds, automatic route changes, and pinned verification. +- [ ] Implement interval maximum and proxy rotation settings, actual request route/count reporting, and route pinning during auth verification. +- [ ] Run network and auth tests, then commit. + +### Task 2: MariaDB backend and migration + +Files: studio/mysql_database.py, studio/migrate.py, studio/api.py, tests/test_mysql.py, pyproject.toml. + +- [ ] Write tests against a real isolated MariaDB database for upserts, historical scopes, favorites, jobs, and atomic migration. +- [ ] Implement MySQLDatabase.connection(), preserving the existing Database API; load private database configuration through STUDIO_DATABASE_CONFIG. +- [ ] Implement migrate_sqlite(source, target) preserving IDs, logs and unit keys and rejecting nonempty targets. +- [ ] Verify all existing tests and real MariaDB integration tests, then commit. + +### Task 3: Linux operation and user interface + +Files: studio/auth.py, studio/linux_pool.py, studio/pool_service.py, studio/__main__.py, studio/api.py, frontend/src/App.tsx, frontend/src/types.ts. + +- [ ] Add failing tests for Linux vault roundtrip/permissions, server host filtering, and interrupted-task recovery selection. +- [ ] Implement encrypted Linux storage, explicit manual-cookie login instructions, Linux systemd pool controls, configured hosts/data directory, and opt-in restart resumption. +- [ ] Add interval range, rotation settings and counters to the UI; use capability metadata for platform instructions. +- [ ] Run full backend tests and frontend tests/build, then commit. + +### Task 4: Provision, migrate, and deploy + +Files: deploy/*, docs/debian-deployment.md, README.md. + +- [ ] Prepare reproducible systemd, Nginx and private configuration templates with rollback and backup instructions. +- [ ] Provision independent MariaDB account/database, dedicated loopback Redis, proxy pool and workbench services. +- [ ] Transfer online SQLite backup and credentials through SSH without exposing secrets; verify migrated data and server login. +- [ ] Validate authenticated HTTPS, browser workflows, proxy API/scheduler and restart persistence; resume migrated task only after valid authentication. +- [ ] Fresh reviewer checks the whole branch; fix material findings and verify again. Publish the deployment branch to the user's repository. diff --git a/docs/superpowers/specs/2026-10-11-debian-server-design.md b/docs/superpowers/specs/2026-10-11-debian-server-design.md new file mode 100644 index 0000000..e2617f5 --- /dev/null +++ b/docs/superpowers/specs/2026-10-11-debian-server-design.md @@ -0,0 +1,29 @@ +# Debian 12 持续采集部署 + +用户要求将工作台部署到 186.241.68.230,通过已有 SSH 密钥操作,避免频繁连接触发 fail2ban;使用服务器实际 MariaDB,部署 jhao104/proxy_pool,核实代理复用,并加入随机请求间隔。保留现有本机运行任务。 + +## 运行与访问 + +使用 Debian 的 Python 3.11、独立虚拟环境和 systemd 服务,普通专用用户运行工作台。现有 MariaDB 10.11 创建独立数据库与最小权限账号,不改其他项目数据库。Nginx 提供 HTTPS 和访问认证;优先指定域名,DNS 未就绪时使用已有有效 IP 证书及独立 9443 端口。后端、代理池与 Redis 仅监听回环地址。操作 SSH 尽量复用持续连接。 + +## 数据 + +增加 MariaDB/MySQL 存储实现,复用现有数据查询与任务接口;Windows 默认继续使用 SQLite。MariaDB 保存目录、分数线、院校索引、收藏、任务、日志和完成断点。使用 utf8mb4 与稳定记录键,保留原文、年份和特殊计划的匹配规则。 + +SQLite 在线 backup 生成一致性快照;显式迁移到空的新库,保留记录 ID 和全部断点。迁移完成核对每张表的行数及记录内容。旧电脑任务保持原状态,服务器副本初次迁移暂停,登录与网络验证成功后才恢复。重启时仅自动恢复原先运行的任务,用户暂停、取消、等待登录任务不自动启动。 + +## 凭据与无桌面登录 + +Windows 继续 DPAPI;Linux 使用本地权限 0600 的密钥和加密 Cookie。密钥与数据库配置存放运行目录,排除 Git 和公开下载。服务器无需桌面:界面说明在用户电脑完成学信网登录并粘贴 Cookie;账号登录仍可用,验证码保留人工流程。迁移原有 Cookie 通过 SSH 安全传输,并在服务器重新验证,不能继承原“已验证”标记。 + +## 请求与代理 + +每次请求从配置的最小与最大间隔随机采样,最大不得小于最小,两者均在 0.3–60 秒;相等时允许固定间隔。保留限流退避与人工验证码处理。 + +增加“每 N 次成功请求轮换代理”,0 表示保留原策略;服务器默认 100。显示当前代理的请求计数、累计切换次数及实际成功请求路由。会话同步和第二页验证期间延迟自动轮换,轮换后的目录会话重新验证。健康轮换保留旧 IP,不将限流误认成失效。 + +Linux 代理 API 与调度器使用专门 systemd 单元,独立 Redis 容器只绑定 127.0.0.1:6381。界面管理固定单元名称,只允许启动和停止本项目 API/调度器,不接受任意 shell 命令,不停止其他项目 Redis。 + +## 验证 + +现有测试继续通过。新增随机间隔边界、轮换与会话固定、Linux 凭据往返、主机与跨来源保护测试。在实际 MariaDB 临时测试库验证重复更新、收藏、历史线匹配、特殊线隔离、任务断点和 SQLite 迁移。构建网页,在服务器验证 HTTPS 认证、页面、数据查询、代理池、登录状态、systemd 重启恢复和现有站点未受影响。 diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 9af0386..a530ea1 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1713,6 +1713,11 @@ function ProxyPanel({ notify }: { notify: Notify }) { ? "代理池暂无可用 IP,优先代理模式已回退直连。" : "登录、招生目录和历年考情共用该路由。"}

+

+ 当前代理已成功请求 {data?.route_requests || 0} 次 · 已切换 {data?.rotations || 0} 次 + {data?.rotate_every ? ` · 每 ${data.rotate_every} 次自动轮换` : " · 连接失败或手动切换时轮换"} + {data?.last_request_route && ` · 最近成功请求:${data.last_request_route === "direct" ? "直连" : data.last_request_route}`} +

@@ -1972,7 +1977,7 @@ function SettingsPanel({ notify }: { notify: Notify }) {
+ +
+

每次请求随机取上下限之间的间隔;两者相等时固定。轮换次数填 0 可关闭自动轮换;登录会话验证期间保留当前代理。

+ } {auth?.browser_message && (

{auth.browser_message}

)} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 6a07679..d82c0c2 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -92,10 +92,16 @@ export type Config = { pool_project: string; backup_proxies: string[]; request_interval: number; + request_interval_max: number; + proxy_rotate_every: number; timeout: number; retries: number; }; export type Proxy = { + route_requests: number; + rotations: number; + last_request_route: string | null; + rotate_every: number; mode: string; route: string; active_proxy: string | null; @@ -136,6 +142,7 @@ export type PoolManageResult = { status: PoolServiceStatus; }; export type Auth = { + browser_supported: boolean; saved: boolean; valid: boolean; state: "unverified" | "verified" | "login_required" | "captcha_required"; diff --git a/pyproject.toml b/pyproject.toml index 20183f4..300b06c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "kaoyan-studio" version = "1.0.0" description = "本地考研招生目录、历年分数线与代理池工作台" -requires-python = ">=3.12" +requires-python = ">=3.11" dependencies = [ "fastapi>=0.115,<1", "uvicorn>=0.34,<1", @@ -11,6 +11,8 @@ dependencies = [ "openpyxl>=3.1,<4", "python-multipart>=0.0.20,<1", "playwright>=1.50,<2", + "PyMySQL>=1.1,<2", + "cryptography>=44,<47", ] [project.optional-dependencies] diff --git a/requirements.lock b/requirements.lock index 1d88e3a..b8c39f9 100644 --- a/requirements.lock +++ b/requirements.lock @@ -3,9 +3,11 @@ annotated-types==0.8.0 anyio==4.15.1 beautifulsoup4==4.15.0 certifi==2026.7.22 +cffi==2.1.1 charset-normalizer==3.5.2 click==8.5.0 colorama==0.4.6 +cryptography==50.0.2 et_xmlfile==2.0.0 fastapi==0.143.0 greenlet==3.5.6 @@ -19,10 +21,12 @@ opentelemetry-api==1.45.1 packaging==26.3 playwright==1.63.0 pluggy==1.6.0 +pycparser==3.11 pydantic==2.14.0 pydantic_core==2.50.0 pyee==13.0.1 Pygments==2.21.0 +PyMySQL==1.2.3 pytest==9.1.1 python-multipart==0.0.32 requests==2.34.2 diff --git a/studio/api.py b/studio/api.py index 9ba8b31..ae87bf1 100644 --- a/studio/api.py +++ b/studio/api.py @@ -1,6 +1,7 @@ """Local-only HTTP workbench and production SPA host.""" import json import uuid +import os from contextlib import asynccontextmanager from pathlib import Path from urllib.parse import urlparse @@ -20,9 +21,15 @@ ROOT = Path(__file__).resolve().parent.parent def create_app(data_dir=None): - directory = Path(data_dir or ROOT / ".runtime") + directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime") directory.mkdir(parents=True, exist_ok=True) - db = Database(directory / "studio.db") + database_config = os.environ.get("STUDIO_DATABASE_CONFIG") + if database_config: + from studio.mysql_database import MySQLDatabase + db = MySQLDatabase(json.loads(Path(database_config).read_text(encoding="utf-8"))) + else: + db = Database(directory / "studio.db") + allowed_hosts = {"127.0.0.1", "localhost", "::1"} | {host.strip().lower() for host in os.environ.get("STUDIO_ALLOWED_HOSTS", "").split(",") if host.strip()} settings = SettingsStore(directory) pool = PoolService(settings) router = ProxyRouter(settings) @@ -34,6 +41,9 @@ def create_app(data_dir=None): @asynccontextmanager async def lifespan(app): + if os.environ.get("STUDIO_AUTO_RESUME") == "1": + for jid in db.recovered_job_ids: + jobs.resume(jid) yield jobs.close() @@ -46,8 +56,8 @@ def create_app(data_dir=None): @app.middleware("http") async def local_access(request: Request, call_next): host = request.url.hostname - if host not in ("127.0.0.1", "localhost", "::1"): - return JSONResponse({"detail": "仅允许本机访问"}, status_code=403) + if host not in allowed_hosts: + return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403) if request.method not in ("GET", "HEAD", "OPTIONS"): origin = request.headers.get("origin") if origin: @@ -76,7 +86,7 @@ def create_app(data_dir=None): @app.get("/api/health") def health(): - return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve())} + return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve()), "database": "mysql" if database_config else "sqlite", "platform": "windows" if os.name == "nt" else "linux"} @app.get("/api/overview") def overview(): diff --git a/studio/auth.py b/studio/auth.py index 3f79be2..7803ddf 100644 --- a/studio/auth.py +++ b/studio/auth.py @@ -2,6 +2,7 @@ import ctypes import json import os +import platform import threading import time from pathlib import Path @@ -40,15 +41,41 @@ class CookieVault: self.path = Path(directory) / "chsi-cookie.enc" self.path.parent.mkdir(parents=True, exist_ok=True) + def _cipher(self): + from cryptography.fernet import Fernet + key_path = self.path.parent / "cookie.key" + if not key_path.exists(): + try: + descriptor = os.open(key_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + pass + else: + with os.fdopen(descriptor, "wb") as stream: + stream.write(Fernet.generate_key()) + return Fernet(key_path.read_bytes()) + def save(self, cookies): temporary = self.path.with_suffix(".tmp") - temporary.write_bytes(protect(json.dumps(cookies, ensure_ascii=False).encode())) + payload = json.dumps(cookies, ensure_ascii=False).encode() + encrypted = protect(payload) if platform.system() == "Windows" else self._cipher().encrypt(payload) + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(encrypted) temporary.replace(self.path) def load(self): if not self.path.exists(): return {} - return json.loads(protect(self.path.read_bytes(), decrypt=True)) + payload = self.path.read_bytes() + if platform.system() == "Windows": + payload = protect(payload, decrypt=True) + else: + from cryptography.fernet import InvalidToken + try: + payload = self._cipher().decrypt(payload) + except InvalidToken: + raise ValueError("保存的 Cookie 无法解密,请重新登录") from None + return json.loads(payload) def clear(self): self.path.unlink(missing_ok=True) @@ -94,7 +121,7 @@ class AuthService: with self.state_lock: if self.valid and self.verified_route != self.transport.router.status()["route"]: self.invalidate("网络出口已变化,请验证目录访问后继续采集", state="unverified") - return {"saved": self.vault.path.exists(), "valid": self.valid, "state": self.state, "message": self.message, "checked_at": self.checked_at, "browser_state": self.browser_state, "browser_message": self.browser_message} + return {"saved": self.vault.path.exists(), "valid": self.valid, "state": self.state, "message": self.message, "checked_at": self.checked_at, "browser_state": self.browser_state, "browser_message": self.browser_message, "browser_supported": platform.system() == "Windows"} def invalidate(self, message, state="login_required"): with self.state_lock: @@ -128,7 +155,7 @@ class AuthService: return self.status() def check(self): - with self.check_lock: + with self.check_lock, self.transport.router.hold_route(): with self.state_lock: self.invalidate("正在同步研招目录会话并验证访问权限", state="unverified") revision = self.revision @@ -182,6 +209,10 @@ class AuthService: return self.status() def browser(self): + if platform.system() != "Windows": + self.browser_state = "unsupported" + self.browser_message = "服务器无桌面环境:请在自己电脑的浏览器完成学信网登录,再粘贴研招网 Cookie 并验证目录访问" + return self.status() if self.browser_thread and self.browser_thread.is_alive() and not self.browser_cancel.is_set(): return self.status() proxy = self.transport.router.resolve() diff --git a/studio/database.py b/studio/database.py index 872bd1a..7cf2f41 100644 --- a/studio/database.py +++ b/studio/database.py @@ -22,7 +22,7 @@ CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,sc """ + SCORE_SCHEMA + """ CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC); CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT); -CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0); +CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0); CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit)); CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT); """ @@ -45,7 +45,11 @@ class Database: conn.execute('DROP TABLE scores') conn.execute('ALTER TABLE scores_new RENAME TO scores') conn.execute('CREATE INDEX scores_lookup ON scores(school_code,major_code,year DESC)') - conn.execute("UPDATE jobs SET status='paused',message='服务重启,任务可从断点恢复' WHERE status IN ('running','queued')") + if 'resume_pending' not in {row['name'] for row in conn.execute('PRAGMA table_info(jobs)')}: + conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending INTEGER DEFAULT 0') + recovery = "status IN ('running','queued') OR (status='paused' AND resume_pending=1)" + self.recovered_job_ids = [row['id'] for row in conn.execute('SELECT id FROM jobs WHERE ' + recovery)] + conn.execute("UPDATE jobs SET status='paused',resume_pending=0,message='服务重启,任务可从断点恢复' WHERE " + recovery) @contextmanager def connection(self): @@ -182,7 +186,7 @@ class Database: conn.execute("INSERT INTO jobs(id,params,status,created_at,updated_at,message) VALUES(?,?,'queued',?,?,?)",(jid,encode(params),now(),now(),"等待开始")) def job_update(self, jid, **changes): - allowed = {"status","message","errors","records"} + allowed = {"status","message","errors","records","resume_pending"} changes = {key:value for key,value in changes.items() if key in allowed} changes["updated_at"] = now() with self.connection() as conn: diff --git a/studio/jobs.py b/studio/jobs.py index 226d43e..9ef3f26 100644 --- a/studio/jobs.py +++ b/studio/jobs.py @@ -67,7 +67,7 @@ class JobManager: def _run(self, jid): job = self.get(jid) - self.db.job_update(jid, status="running", errors=0, message="正在采集") + self.db.job_update(jid, status="running", errors=0, resume_pending=0, message="正在采集") self.db.log(jid, "info", "任务开始/恢复,将跳过已完成单元") if self.transport: self.transport.local.guard = lambda: self.checkpoint(jid) @@ -127,7 +127,7 @@ class JobManager: raise ValueError("任务当前不在运行") with control["condition"]: control["paused"] = True - self.db.job_update(jid, status="paused", message="已暂停,当前请求结束后停止") + self.db.job_update(jid, status="paused", resume_pending=0, message="已暂停,当前请求结束后停止") return self.get(jid) def resume(self, jid): @@ -169,7 +169,7 @@ class JobManager: if control["thread"].is_alive(): with control['condition']: control['shutdown'] = True - self.db.job_update(jid, status='paused', message='服务停止,任务可从断点恢复') + self.db.job_update(jid, status='paused', resume_pending=int(not control['paused']), message='服务停止,任务可从断点恢复') control['condition'].notify_all() for control in list(self.controls.values()): control["thread"].join(timeout=1) diff --git a/studio/linux_pool.py b/studio/linux_pool.py new file mode 100644 index 0000000..83e15fa --- /dev/null +++ b/studio/linux_pool.py @@ -0,0 +1,61 @@ +"""Control only the two fixed, dedicated proxy-pool systemd units.""" +import socket +import subprocess +from urllib.parse import urlparse +import requests + +UNITS = ('kaoyan-proxy-api.service', 'kaoyan-proxy-scheduler.service') + + +class LinuxPoolService: + def __init__(self, settings): + self.settings = settings + self.session = requests.Session() + self.session.trust_env = False + + def status(self, config=None): + config = config or self.settings.read() + api_ready, counts = False, {} + try: + response = self.session.get(config['pool_url'] + '/count/', timeout=2) + response.raise_for_status() + counts = response.json() + api_ready = isinstance(counts, dict) and type(counts.get('count')) is int + except (requests.RequestException, ValueError): + pass + items = [] + for name, label, unit in zip(('server', 'schedule'), ('代理 API', '抓取与检测'), UNITS): + try: + result = subprocess.run(['/usr/bin/systemctl', 'show', unit, '--property=LoadState,ActiveState,MainPID'], capture_output=True, text=True, timeout=5) + facts = dict(line.split('=', 1) for line in result.stdout.splitlines() if '=' in line) + installed = facts.get('LoadState') == 'loaded' + active = installed and facts.get('ActiveState') == 'active' + items.append({'name': name, 'label': label, 'state': 'running' if active else 'stopped', 'pid': int(facts.get('MainPID', '0')) or None, 'verified': active, 'elevated': False, 'detail': 'systemd 运行中' if active else ('已停止' if installed else '未安装本项目 systemd 单元')}) + except (OSError, subprocess.TimeoutExpired, ValueError): + items.append({'name': name, 'label': label, 'state': 'unknown', 'pid': None, 'verified': False, 'elevated': False, 'detail': '无法查询 systemd 状态'}) + try: + with socket.create_connection(('127.0.0.1', 6381), timeout=1): + redis_ready = True + except OSError: + redis_ready = False + items.append({'name': 'redis', 'label': 'Redis', 'state': 'external' if redis_ready else 'stopped', 'pid': None, 'verified': False, 'elevated': False, 'detail': '独立 Redis 服务(127.0.0.1:6381)' if redis_ready else '未连接到本项目 Redis'}) + local = urlparse(config['pool_url']).hostname in ('127.0.0.1', 'localhost', '::1') + return {'api_ready': api_ready, 'counts': counts if api_ready else {}, 'items': items, 'can_stop': local and any(item['verified'] for item in items[:2]), 'warning': '' if local else '当前使用外部代理池,请在其所在服务器管理服务', 'admin_command': '', 'manager': 'systemd'} + + def manage(self, action): + if action not in ('start', 'stop', 'status'): + raise ValueError('代理池操作无效') + status = self.status() + result = {'ok': True, 'action': action, 'changed': False, 'status': status, 'message': '服务状态已更新', 'output': '\n'.join(item['label'] + ':' + item['detail'] for item in status['items'])} + if action == 'status': + return result + if status['warning']: + return {**result, 'ok': False, 'message': status['warning']} + if action == 'start' and all(item['state'] == 'running' for item in status['items'][:2]) and status['api_ready']: + return {**result, 'message': '代理池已在运行,已复用现有服务'} + try: + completed = subprocess.run(['/usr/bin/sudo', '-n', '/usr/bin/systemctl', action, *UNITS], capture_output=True, text=True, timeout=55) + message = '代理池启动完成' if action == 'start' else '代理池已停止' + return {**result, 'ok': completed.returncode == 0, 'changed': True, 'status': self.status(), 'message': message if completed.returncode == 0 else 'systemd 操作失败,请检查本项目 sudoers 与服务日志', 'output': (completed.stdout + completed.stderr)[-5000:] or message} + except (OSError, subprocess.TimeoutExpired): + return {**result, 'ok': False, 'message': '代理池管理超时或命令不可用,请检查 systemd 服务'} diff --git a/studio/migrate.py b/studio/migrate.py new file mode 100644 index 0000000..9cc7b3e --- /dev/null +++ b/studio/migrate.py @@ -0,0 +1,74 @@ +"""Explicit, atomic migration from a consistent SQLite snapshot into an empty MySQL database.""" +import argparse +import json +import sqlite3 +from pathlib import Path +from studio.domain import normalized_name, is_special_score +from studio.mysql_database import MySQLDatabase, score_key + +TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs') + + +def migrate_sqlite(source, target): + source = Path(source).resolve() + conn = sqlite3.connect(source.as_uri() + '?mode=ro', uri=True) + conn.row_factory = sqlite3.Row + counts = {} + try: + conn.execute('BEGIN') + with target.connection() as output: + # Serialize attempted imports to the same database. + acquired = output.execute("SELECT GET_LOCK('kaoyan_studio_migration',10)").fetchone()[0] + if acquired != 1: + raise ValueError('已有数据迁移正在进行') + try: + for table in TABLES: + if output.execute(f'SELECT count(*) FROM {table}').fetchone()[0]: + raise ValueError(f'目标数据库非空({table}),拒绝覆盖;请使用空数据库') + for table in TABLES: + counts[table] = 0 + reader = conn.execute(f'SELECT * FROM {table}') + while batch := reader.fetchmany(500): + rows = [] + for original in batch: + row = dict(original) + if table in ('catalog', 'scores'): + payload = json.loads(row['data']) + row['department_norm'] = normalized_name(row.get('department_name')) + if table == 'scores': + row['note'] = row.get('note') or payload.get('note') or '' + row['raw_code'] = row.get('raw_code') or payload.get('raw_code') or '' + row['score_key'] = score_key(row) + row['special_line'] = int(is_special_score(row['note'])) + if table == 'jobs' and row['status'] in ('running', 'queued'): + row.update(status='paused', message='已迁移到服务器,验证登录后可从断点恢复') + if table == 'jobs': + row['resume_pending'] = 0 + rows.append(row) + fields = list(rows[0]) + output.executemany(f"INSERT INTO {table}({','.join(fields)}) VALUES({','.join('?' for _ in fields)})", [[row[field] for field in fields] for row in rows]) + counts[table] += len(rows) + for table, count in counts.items(): + if output.execute(f'SELECT count(*) FROM {table}').fetchone()[0] != count: + raise ValueError(f'迁移行数校验失败:{table}') + finally: + # The private connection closes after commit/rollback, releasing its + # advisory lock only after the entire import becomes visible. + pass + return counts + finally: + conn.rollback() + conn.close() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('snapshot', type=Path) + parser.add_argument('--config', type=Path, required=True) + args = parser.parse_args() + config = json.loads(args.config.read_text()) + print(json.dumps(migrate_sqlite(args.snapshot, MySQLDatabase(config)), ensure_ascii=False)) + + +if __name__ == '__main__': + main() diff --git a/studio/mysql_database.py b/studio/mysql_database.py new file mode 100644 index 0000000..0201444 --- /dev/null +++ b/studio/mysql_database.py @@ -0,0 +1,165 @@ +"""MariaDB/MySQL storage retaining the local Database's public interface.""" +import hashlib +import json +import re +from contextlib import contextmanager +import pymysql +from studio.database import Database, encode +from studio.domain import normalized_name, is_special_score + +SCHEMA = [ + """CREATE TABLE IF NOT EXISTS catalog( + id BIGINT PRIMARY KEY AUTO_INCREMENT,record_key CHAR(64) NOT NULL UNIQUE,year INT NOT NULL, + school_code VARCHAR(32),school_name VARCHAR(255),province VARCHAR(64),major_code VARCHAR(32),major_name VARCHAR(255), + department_code VARCHAR(64),department_name VARCHAR(255),study_mode VARCHAR(32),degree_type VARCHAR(64),level VARCHAR(128), + subjects TEXT,quota INT,source VARCHAR(128),favorite TINYINT NOT NULL DEFAULT 0,data LONGTEXT NOT NULL, + department_norm VARCHAR(255) NOT NULL DEFAULT '',INDEX catalog_filter(year,province,major_code,school_code))""", + """CREATE TABLE IF NOT EXISTS scores( + id BIGINT PRIMARY KEY AUTO_INCREMENT,school_code VARCHAR(32),major_code VARCHAR(32),year INT, + department_id VARCHAR(64),department_name VARCHAR(255),scope VARCHAR(32),total INT,source VARCHAR(128),data LONGTEXT NOT NULL, + note TEXT NOT NULL,raw_code VARCHAR(64) NOT NULL,score_key CHAR(64) NOT NULL UNIQUE, + department_norm VARCHAR(255) NOT NULL DEFAULT '',special_line TINYINT NOT NULL DEFAULT 0, + INDEX scores_lookup(school_code,major_code,year))""", + "CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)", + "CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)", + "CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))", + "CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))", +] + + +def score_key(row): + identity = [row.get(field) or '' for field in ('school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'source', 'note', 'raw_code')] + return hashlib.sha256(encode(identity).encode()).hexdigest() + + +class Row(dict): + def __getitem__(self, key): + return tuple(self.values())[key] if isinstance(key, int) else super().__getitem__(key) + + +class Result: + def __init__(self, cursor): + self.rowcount = cursor.rowcount + self.columns = [item[0] for item in cursor.description] if cursor.description else [] + self.rows = cursor.fetchall() if cursor.description else () + self.offset = 0 + cursor.close() + + def fetchone(self): + if self.offset >= len(self.rows): + return None + result = Row(zip(self.columns, self.rows[self.offset])) + self.offset += 1 + return result + + def fetchall(self): + return list(self) + + def __iter__(self): + while self.offset < len(self.rows): + yield self.fetchone() + + +def mysql_sql(sql): + if sql.startswith('SELECT count(*) FROM (') and sql.endswith(')'): + sql += ' AS studio_count' + sql = sql.replace('INSERT OR IGNORE', 'INSERT IGNORE') + sql = re.sub(r'ON CONFLICT\([^)]*\) DO UPDATE SET', 'ON DUPLICATE KEY UPDATE', sql) + sql = re.sub(r'excluded\.(\w+)', r'VALUES(\1)', sql) + sql = sql.replace("is_special_score(json_extract(x.data,'$.note'))=0", 'x.special_line=0') + sql = sql.replace('norm_name(x.department_name)', 'x.department_norm').replace('norm_name(c.department_name)', 'c.department_norm') + sql = re.sub(r"json_extract\(([^,]+),('[^']+')\)", r'JSON_UNQUOTE(JSON_EXTRACT(\1,\2))', sql) + return sql.replace('?', '%s') + + +class Connection: + def __init__(self, raw): + self.raw = raw + + def execute(self, sql, params=None): + cursor = self.raw.cursor() + try: + cursor.execute(mysql_sql(sql), params) + return Result(cursor) + except BaseException: + cursor.close() + raise + + def executemany(self, sql, values): + cursor = self.raw.cursor() + try: + cursor.executemany(mysql_sql(sql), values) + return Result(cursor) + except BaseException: + cursor.close() + raise + + +class MySQLDatabase(Database): + def __init__(self, config): + self.config = {key: value for key, value in config.items() if key in ('host', 'port', 'user', 'password', 'database', 'unix_socket')} + if not self.config.get('database'): + raise ValueError('数据库配置缺少 database') + self.recovered_job_ids = [] + with self.connection() as conn: + for statement in SCHEMA: + conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin') + if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone(): + conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0') + recovery = "status IN ('running','queued') OR (status='paused' AND resume_pending=1)" + self.recovered_job_ids = [row['id'] for row in conn.execute('SELECT id FROM jobs WHERE ' + recovery)] + conn.execute("UPDATE jobs SET status='paused',resume_pending=0,message='服务重启,任务可从断点恢复' WHERE " + recovery) + + @contextmanager + def connection(self): + raw = pymysql.connect(**self.config, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120) + try: + yield Connection(raw) + raw.commit() + except BaseException: + raw.rollback() + raise + finally: + raw.close() + + def upsert_catalog(self, records): + fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm'] + values = [] + for record in records: + row = {**record, 'subjects': ' | '.join(record.get('subjects', [])), 'data': encode(record), 'department_norm': normalized_name(record.get('department_name'))} + values.append([row.get(field) for field in fields]) + if not values: + return + update = ','.join(f'{field}=VALUES({field})' for field in fields[1:]) + with self.connection() as conn: + conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values) + for record in records: + self._school(conn, record) + + def upsert_scores(self, records): + fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line'] + values = [] + for record in records: + row = {**record, 'department_id': record.get('department_id') or '', 'department_name': record.get('department_name') or '', 'source': record.get('source', '掌上考研'), 'note': record.get('note') or '', 'raw_code': record.get('raw_code') or ''} + row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note']))) + values.append([row.get(field) for field in fields]) + if values: + with self.connection() as conn: + conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values) + + def favorite(self, rid, enabled): + # MariaDB reports changed rows; an already-selected favorite is still success. + with self.connection() as conn: + conn.execute('UPDATE catalog SET favorite=? WHERE id=?', (int(enabled), rid)) + return conn.execute('SELECT 1 FROM catalog WHERE id=?', (rid,)).fetchone() is not None + + def history(self, rid): + record = self.record(rid) + if record is None: + return None + with self.connection() as conn: + scores = conn.execute('SELECT data FROM scores WHERE school_code=? AND major_code=? ORDER BY year,scope,department_name', (record['school_code'], record['major_code'])).fetchall() + plans = conn.execute('SELECT data FROM catalog WHERE school_code=? AND major_code=? AND department_norm=? ORDER BY year,id', (record['school_code'], record['major_code'], normalized_name(record['department_name']))).fetchall() + def matches(row): + return not row.get('department_name') and not row.get('department_id') or row.get('department_id') and row['department_id'] == record['department_code'] or row.get('department_name') and normalized_name(row['department_name']) == normalized_name(record['department_name']) + return {'record': record, 'scores': [row for entry in scores if matches(row := json.loads(entry['data']))], 'plans': [json.loads(row['data']) for row in plans]} diff --git a/studio/network.py b/studio/network.py index 464a27d..9e9a87e 100644 --- a/studio/network.py +++ b/studio/network.py @@ -1,6 +1,8 @@ """One explicit route for CHSI login, catalog and historical data.""" import threading import time +import random +from contextlib import contextmanager import requests from studio.settings import proxy_url @@ -29,6 +31,10 @@ class ProxyRouter: self.counts = {} self.fallback = False self.lock = threading.RLock() + self.route_requests = 0 + self.rotations = 0 + self.last_request_route = None + self.holds = 0 def refresh(self): with self.lock: @@ -60,12 +66,31 @@ class ProxyRouter: self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"])) return self.status() - def resolve(self): + @contextmanager + def hold_route(self): + with self.lock: + self.holds += 1 + try: + yield + finally: + with self.lock: + self.holds -= 1 + + def note_success(self, proxy): + with self.lock: + self.last_request_route = proxy or "direct" + if proxy == self.active: + self.route_requests += 1 + + def resolve(self, for_request=False): with self.lock: config = self.settings.read() if config["network_mode"] == "direct": self.fallback = False return None + threshold = config["proxy_rotate_every"] + if for_request and not self.holds and threshold and self.active and self.route_requests >= threshold: + self.rotate() if self.active and self.active not in self.failed: self.fallback = False return self.active @@ -75,9 +100,11 @@ class ProxyRouter: available = [p for p in self.candidates if p not in self.failed] if available: self.active = available[0] + self.route_requests = 0 self.fallback = False return self.active self.active = None + self.route_requests = 0 if config["network_mode"] == "proxy_only": self.fallback = False raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务") @@ -93,7 +120,10 @@ class ProxyRouter: if old in self.candidates: self.candidates.remove(old) self.candidates.append(old) - return self.resolve() + selected = self.resolve() + if old and selected != old: + self.rotations += 1 + return selected def reset(self): with self.lock: @@ -101,10 +131,11 @@ class ProxyRouter: self.failed.clear() self.candidates = [] self.fallback = False + self.route_requests = 0 def status(self): config = self.settings.read() - return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]} + return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "route_requests": self.route_requests, "rotations": self.rotations, "last_request_route": self.last_request_route, "rotate_every": config["proxy_rotate_every"], "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]} class Transport: @@ -130,8 +161,8 @@ class Transport: def request(self, method, url, **kwargs): config = self.settings.read() for attempt in range(config["retries"]): - self.sleep(config["request_interval"] if attempt == 0 else min(20, 2 ** attempt)) - proxy = self.router.resolve() + self.sleep(random.uniform(config["request_interval"], config["request_interval_max"]) if attempt == 0 else min(20, 2 ** attempt)) + proxy = self.router.resolve(for_request=True) kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {} kwargs.setdefault("timeout", config["timeout"]) try: @@ -143,6 +174,7 @@ class Transport: continue response.raise_for_status() self.local.response_route = proxy or "direct" + self.router.note_success(proxy) return response except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout): if proxy: diff --git a/studio/pool_service.py b/studio/pool_service.py index 23d5b8f..d2a0901 100644 --- a/studio/pool_service.py +++ b/studio/pool_service.py @@ -137,6 +137,10 @@ def _pool_entry(arguments, project, service): class PoolService: def __init__(self, settings): self.settings = settings + self.linux = None + if os.name != "nt": + from studio.linux_pool import LinuxPoolService + self.linux = LinuxPoolService(settings) self.session = requests.Session() self.session.trust_env = False self.lock = threading.Lock() @@ -154,6 +158,8 @@ class PoolService: return json.loads(result.stdout.decode("utf-8-sig")) def status(self, config=None): + if self.linux: + return self.linux.status(config) if config is None: config = self.settings.read() project = Path(config["pool_project"]) @@ -233,6 +239,8 @@ class PoolService: return "\n".join(lines) def manage(self, action): + if self.linux: + return self.linux.manage(action) if action not in ("start", "stop", "status"): raise ValueError("代理池操作无效") with self.lock: diff --git a/studio/settings.py b/studio/settings.py index 7e0e14a..29521a7 100644 --- a/studio/settings.py +++ b/studio/settings.py @@ -1,13 +1,16 @@ """Project-local, non-secret configuration.""" import json import threading +import math +import os from pathlib import Path from urllib.parse import urlparse DEFAULTS = { "network_mode": "prefer_proxy", "pool_url": "http://127.0.0.1:5010", - "pool_project": r"C:\Users\Jayson\Desktop\proxy_pool", - "backup_proxies": [], "request_interval": 1.2, "timeout": 30, "retries": 3, + "pool_project": r"C:\Users\Jayson\Desktop\proxy_pool" if os.name == "nt" else "/opt/kaoyan-studio-deploy/proxy_pool", + "backup_proxies": [], "request_interval": 1.2, "request_interval_max": 2.4, + "proxy_rotate_every": 0, "timeout": 30, "retries": 3, } @@ -37,7 +40,10 @@ class SettingsStore: def read(self): with self.lock: values = json.loads(self.path.read_text(encoding="utf-8")) if self.path.exists() else {} - return {**DEFAULTS, **{k: v for k, v in values.items() if k in DEFAULTS}} + result = {**DEFAULTS, **{k: v for k, v in values.items() if k in DEFAULTS}} + if "request_interval_max" not in values: + result["request_interval_max"] = max(result["request_interval"], DEFAULTS["request_interval_max"]) + return result def update(self, values): with self.lock: @@ -48,10 +54,12 @@ class SettingsStore: if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password: raise ValueError("代理池 API 地址无效") values["pool_url"] = values["pool_url"].rstrip("/") - for key, low, high in [("request_interval", .3, 60), ("timeout", 5, 120), ("retries", 1, 6)]: - values[key] = float(values[key]) if key == "request_interval" else int(values[key]) - if not low <= values[key] <= high: + for key, low, high in [("request_interval", .3, 60), ("request_interval_max", .3, 60), ("timeout", 5, 120), ("retries", 1, 6), ("proxy_rotate_every", 0, 100000)]: + values[key] = float(values[key]) if key.startswith("request_interval") else int(values[key]) + if not math.isfinite(values[key]) or not low <= values[key] <= high: raise ValueError(f"{key} 应在 {low} 到 {high} 之间") + if values["request_interval_max"] < values["request_interval"]: + raise ValueError("请求间隔上限不能小于下限") proxies = values["backup_proxies"] if isinstance(proxies, str): proxies = proxies.splitlines() diff --git a/tests/test_mysql.py b/tests/test_mysql.py new file mode 100644 index 0000000..8865338 --- /dev/null +++ b/tests/test_mysql.py @@ -0,0 +1,99 @@ +"""Real MariaDB contract tests; run only against an explicitly supplied test database.""" +import json +import os +from pathlib import Path +import pytest +from studio.domain import normalize_catalog, select_score_lines +from studio.database import Database +from tests.test_data import raw_catalog + +pytestmark = pytest.mark.skipif(not os.environ.get('STUDIO_MYSQL_TEST_CONFIG'), reason='requires an isolated MariaDB test database') + + +@pytest.fixture +def mysql(): + from studio.mysql_database import MySQLDatabase + config = json.loads(Path(os.environ['STUDIO_MYSQL_TEST_CONFIG']).read_text()) + assert config['database'].endswith('_test'), 'refusing to clear a production database' + db = MySQLDatabase(config) + with db.connection() as conn: + for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs'): + conn.execute(f'DELETE FROM {table}') + return db + + +def test_mysql_preserves_upserts_favorites_and_history(mysql): + mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027)) + first = mysql.query_catalog({})['items'][0] + mysql.favorite(first['id'], True) + mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027)) + mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026)) + mysql.upsert_scores(select_score_lines([ + {'code': '081200', 'total': 320, 'depart_name': '计算机学院'}, + {'code': '081200', 'total': 260, 'depart_name': '计算机学院', 'note': '少干专项计划'}, + {'code': '081200', 'total': 390, 'depart_name': '其他学院'}, + ], '081200', '10001', '北京大学', 2026)) + result = mysql.query_catalog({'year': 2027, 'direction': '人工', 'english': '203'}) + assert result['total'] == 1 + assert result['items'][0]['score_total'] == 320 + assert mysql.query_catalog({'favorites': True})['total'] == 1 + assert mysql.overview()['records'] == 4 + assert len(mysql.history(first['id'])['plans']) == 4 + assert len(mysql.history(first['id'])['scores']) == 2 + + +def test_mysql_score_unique_key_preserves_long_notes_and_normalized_department(mysql): + raw = raw_catalog() + raw['yxsmc'] = '计算机 ( 学院 )' + mysql.upsert_catalog(normalize_catalog(raw, 2027)) + scores = select_score_lines([ + {'code': '081200', 'total': 320, 'depart_name': '计算机(学院)', 'note': '普通计划' * 300}, + {'code': '081200', 'total': 321, 'depart_name': '计算机(学院)', 'note': '普通计划' * 300 + 'B'}, + ], '081200', '10001', '北京大学', 2026) + mysql.upsert_scores(scores) + mysql.upsert_scores(scores) + assert mysql.overview()['scores'] == 2 + assert mysql.query_catalog({})['items'][0]['score_total'] == 321 + + +def test_mysql_tasks_logs_and_checkpoints(mysql): + mysql.job_create('contract-job', {'year': 2027}) + mysql.job_update('contract-job', status='running', records=7) + mysql.complete_unit('contract-job', 'catalog:11:10001:081200:1') + mysql.complete_unit('contract-job', 'catalog:11:10001:081200:1') + mysql.log('contract-job', 'info', '数据库测试') + assert mysql.unit_done('contract-job', 'catalog:11:10001:081200:1') + assert mysql.job_get('contract-job')['completed_units'] == 1 + assert mysql.job_get('contract-job')['records'] == 7 + assert mysql.logs('contract-job')[0]['message'] == '数据库测试' + + +def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path): + from studio.migrate import migrate_sqlite + source = Database(tmp_path / 'snapshot.db') + source.upsert_catalog(normalize_catalog(raw_catalog(), 2027)) + source.upsert_scores(select_score_lines([{'code': '081200', 'total': 320}], '081200', '10001', '北京大学', 2026)) + source.favorite(1, True) + source.job_create('migrated-job', {'year': 2027}) + source.complete_unit('migrated-job', 'catalog:11:10001:081200:1') + source.log('migrated-job', 'info', 'preserve log') + counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql) + assert counts['catalog'] == 2 and counts['scores'] == 1 + assert mysql.record(1)['favorite'] + assert mysql.query_catalog({})['items'][0]['score_total'] == 320 + assert mysql.job_get('migrated-job')['completed_units'] == 1 + assert mysql.logs('migrated-job')[0]['id'] == 1 + with pytest.raises(ValueError, match='非空'): + migrate_sqlite(tmp_path / 'snapshot.db', mysql) + + +def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path): + from studio.migrate import migrate_sqlite + source = Database(tmp_path / 'bad.db') + source.upsert_catalog(normalize_catalog(raw_catalog(), 2027)) + with source.connection() as conn: + conn.execute("INSERT INTO scores(school_code,major_code,year,scope,data) VALUES('10001','081200',2026,'major','not json')") + with pytest.raises(ValueError): + migrate_sqlite(tmp_path / 'bad.db', mysql) + assert mysql.overview()['records'] == 0 + assert mysql.overview()['scores'] == 0 diff --git a/tests/test_pool_service.py b/tests/test_pool_service.py index fca32a5..c3ad4a1 100644 --- a/tests/test_pool_service.py +++ b/tests/test_pool_service.py @@ -11,6 +11,8 @@ from fastapi.testclient import TestClient from studio.api import create_app +pytestmark = pytest.mark.skipif(os.name != 'nt', reason='Windows pool process and PowerShell behavior') + @pytest.fixture def pool_client(tmp_path): diff --git a/tests/test_server_features.py b/tests/test_server_features.py new file mode 100644 index 0000000..faa481f --- /dev/null +++ b/tests/test_server_features.py @@ -0,0 +1,109 @@ +import platform +import random +import os +import pytest + +from studio.auth import CookieVault +from tests.test_network import setup_network + + +def test_requests_sample_a_fresh_delay_inside_configured_bounds(tmp_path, monkeypatch): + settings, _, transport = setup_network(tmp_path) + settings.update({'request_interval': .3, 'request_interval_max': .5}) + samples = iter([.31, .47, .39]) + monkeypatch.setattr(random, 'uniform', lambda low, high: next(samples)) + delays = [] + transport.sleep = delays.append + for _ in range(3): + transport.request('GET', 'https://yz.chsi.com.cn/zsml/') + assert delays == [.31, .47, .39] + + +@pytest.mark.parametrize('maximum', [.2, 61, float('nan')]) +def test_invalid_interval_range_is_rejected(tmp_path, maximum): + settings, _, _ = setup_network(tmp_path) + with pytest.raises(ValueError): + settings.update({'request_interval': .3, 'request_interval_max': maximum}) + + +def test_successful_requests_rotate_without_quarantining_old_proxy(tmp_path): + settings, router, transport = setup_network(tmp_path) + settings.update({'proxy_rotate_every': 2}) + routes = [transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] for _ in range(3)] + assert routes == ['http://1.2.3.4:8080', 'http://1.2.3.4:8080', 'http://5.6.7.8:9090'] + assert router.failed == set() + assert router.status()['last_request_route'] == 'http://5.6.7.8:9090' + assert router.status()['route_requests'] == 1 + + +def test_route_pin_defers_rotation_until_login_proof_finishes(tmp_path): + settings, router, transport = setup_network(tmp_path) + settings.update({'proxy_rotate_every': 1}) + with router.hold_route(): + routes = [transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] for _ in range(2)] + assert routes == ['http://1.2.3.4:8080'] * 2 + assert transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] == 'http://5.6.7.8:9090' + + +def test_linux_cookie_vault_encrypts_and_preserves_domain_cookies(tmp_path, monkeypatch): + monkeypatch.setattr(platform, 'system', lambda: 'Linux') + cookies = [{'name': 'JSESSIONID', 'value': 'private-account-cookie', 'domain': 'account.chsi.com.cn'}, + {'name': 'JSESSIONID', 'value': 'private-catalog-cookie', 'domain': 'yz.chsi.com.cn'}] + vault = CookieVault(tmp_path) + vault.save(cookies) + assert (tmp_path / 'cookie.key').exists() + assert b'private-account-cookie' not in vault.path.read_bytes() + assert CookieVault(tmp_path).load() == cookies + if os.name != 'nt': + assert (tmp_path / 'cookie.key').stat().st_mode & 0o777 == 0o600 + assert vault.path.stat().st_mode & 0o777 == 0o600 + + +def test_configured_server_host_is_allowed_but_other_hosts_and_origins_are_blocked(tmp_path, monkeypatch): + from fastapi.testclient import TestClient + from studio.api import create_app + monkeypatch.setenv('STUDIO_ALLOWED_HOSTS', 'kaoyan.fjayson.com') + with TestClient(create_app(tmp_path), base_url='https://kaoyan.fjayson.com') as client: + assert client.get('/api/health').status_code == 200 + assert client.get('/api/health', headers={'Host': 'evil.example'}).status_code == 403 + assert client.put('/api/settings', json={'network_mode': 'direct'}, headers={'Origin': 'https://evil.example'}).status_code == 403 + assert client.put('/api/settings', json={'network_mode': 'direct'}, headers={'Origin': 'https://kaoyan.fjayson.com'}).status_code == 200 + + +def test_only_interrupted_running_tasks_are_selected_for_restart(tmp_path): + from studio.database import Database + db = Database(tmp_path / 'resume.db') + for status in ('running', 'queued', 'paused', 'waiting_login', 'cancelled'): + db.job_create(status, {'year': 2027}) + db.job_update(status, status=status) + restarted = Database(tmp_path / 'resume.db') + assert set(restarted.recovered_job_ids) == {'running', 'queued'} + assert restarted.job_get('paused')['status'] == 'paused' + assert restarted.job_get('waiting_login')['status'] == 'waiting_login' + + +@pytest.mark.parametrize('manually_paused', [False, True]) +def test_graceful_restart_remembers_running_but_not_manual_pause(tmp_path, manually_paused): + import threading + from studio.database import Database + from studio.jobs import JobManager + from studio.settings import SettingsStore + ready = threading.Event() + stop = threading.Event() + def runner(jid, params, manager): + ready.set() + while not stop.wait(.01): + manager.checkpoint(jid) + db = Database(tmp_path / 'shutdown.db') + manager = JobManager(db, SettingsStore(tmp_path), runner=runner) + job = manager.start({'mode': 'catalog', 'year': 2027, 'provinces': ['11']}) + assert ready.wait(3) + try: + if manually_paused: + manager.pause(job['id']) + manager.close() + restarted = Database(tmp_path / 'shutdown.db') + assert (job['id'] in restarted.recovered_job_ids) is (not manually_paused) + finally: + stop.set() + manager.close()