Add Debian deployment, MariaDB storage and randomized proxy routing
This commit is contained in:
+15
-5
@@ -1,6 +1,7 @@
|
||||
"""Local-only HTTP workbench and production SPA host."""
|
||||
import json
|
||||
import uuid
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
@@ -20,9 +21,15 @@ ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def create_app(data_dir=None):
|
||||
directory = Path(data_dir or ROOT / ".runtime")
|
||||
directory = Path(data_dir or os.environ.get("STUDIO_DATA_DIR") or ROOT / ".runtime")
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
db = Database(directory / "studio.db")
|
||||
database_config = os.environ.get("STUDIO_DATABASE_CONFIG")
|
||||
if database_config:
|
||||
from studio.mysql_database import MySQLDatabase
|
||||
db = MySQLDatabase(json.loads(Path(database_config).read_text(encoding="utf-8")))
|
||||
else:
|
||||
db = Database(directory / "studio.db")
|
||||
allowed_hosts = {"127.0.0.1", "localhost", "::1"} | {host.strip().lower() for host in os.environ.get("STUDIO_ALLOWED_HOSTS", "").split(",") if host.strip()}
|
||||
settings = SettingsStore(directory)
|
||||
pool = PoolService(settings)
|
||||
router = ProxyRouter(settings)
|
||||
@@ -34,6 +41,9 @@ def create_app(data_dir=None):
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
if os.environ.get("STUDIO_AUTO_RESUME") == "1":
|
||||
for jid in db.recovered_job_ids:
|
||||
jobs.resume(jid)
|
||||
yield
|
||||
jobs.close()
|
||||
|
||||
@@ -46,8 +56,8 @@ def create_app(data_dir=None):
|
||||
@app.middleware("http")
|
||||
async def local_access(request: Request, call_next):
|
||||
host = request.url.hostname
|
||||
if host not in ("127.0.0.1", "localhost", "::1"):
|
||||
return JSONResponse({"detail": "仅允许本机访问"}, status_code=403)
|
||||
if host not in allowed_hosts:
|
||||
return JSONResponse({"detail": "访问主机不在允许列表中"}, status_code=403)
|
||||
if request.method not in ("GET", "HEAD", "OPTIONS"):
|
||||
origin = request.headers.get("origin")
|
||||
if origin:
|
||||
@@ -76,7 +86,7 @@ def create_app(data_dir=None):
|
||||
|
||||
@app.get("/api/health")
|
||||
def health():
|
||||
return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve())}
|
||||
return {"app": "kaoyan-studio", "version": "1.0.0", "root": str(ROOT), "data_dir": str(directory.resolve()), "database": "mysql" if database_config else "sqlite", "platform": "windows" if os.name == "nt" else "linux"}
|
||||
|
||||
@app.get("/api/overview")
|
||||
def overview():
|
||||
|
||||
+35
-4
@@ -2,6 +2,7 @@
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -40,15 +41,41 @@ class CookieVault:
|
||||
self.path = Path(directory) / "chsi-cookie.enc"
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _cipher(self):
|
||||
from cryptography.fernet import Fernet
|
||||
key_path = self.path.parent / "cookie.key"
|
||||
if not key_path.exists():
|
||||
try:
|
||||
descriptor = os.open(key_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
except FileExistsError:
|
||||
pass
|
||||
else:
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
stream.write(Fernet.generate_key())
|
||||
return Fernet(key_path.read_bytes())
|
||||
|
||||
def save(self, cookies):
|
||||
temporary = self.path.with_suffix(".tmp")
|
||||
temporary.write_bytes(protect(json.dumps(cookies, ensure_ascii=False).encode()))
|
||||
payload = json.dumps(cookies, ensure_ascii=False).encode()
|
||||
encrypted = protect(payload) if platform.system() == "Windows" else self._cipher().encrypt(payload)
|
||||
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
stream.write(encrypted)
|
||||
temporary.replace(self.path)
|
||||
|
||||
def load(self):
|
||||
if not self.path.exists():
|
||||
return {}
|
||||
return json.loads(protect(self.path.read_bytes(), decrypt=True))
|
||||
payload = self.path.read_bytes()
|
||||
if platform.system() == "Windows":
|
||||
payload = protect(payload, decrypt=True)
|
||||
else:
|
||||
from cryptography.fernet import InvalidToken
|
||||
try:
|
||||
payload = self._cipher().decrypt(payload)
|
||||
except InvalidToken:
|
||||
raise ValueError("保存的 Cookie 无法解密,请重新登录") from None
|
||||
return json.loads(payload)
|
||||
|
||||
def clear(self):
|
||||
self.path.unlink(missing_ok=True)
|
||||
@@ -94,7 +121,7 @@ class AuthService:
|
||||
with self.state_lock:
|
||||
if self.valid and self.verified_route != self.transport.router.status()["route"]:
|
||||
self.invalidate("网络出口已变化,请验证目录访问后继续采集", state="unverified")
|
||||
return {"saved": self.vault.path.exists(), "valid": self.valid, "state": self.state, "message": self.message, "checked_at": self.checked_at, "browser_state": self.browser_state, "browser_message": self.browser_message}
|
||||
return {"saved": self.vault.path.exists(), "valid": self.valid, "state": self.state, "message": self.message, "checked_at": self.checked_at, "browser_state": self.browser_state, "browser_message": self.browser_message, "browser_supported": platform.system() == "Windows"}
|
||||
|
||||
def invalidate(self, message, state="login_required"):
|
||||
with self.state_lock:
|
||||
@@ -128,7 +155,7 @@ class AuthService:
|
||||
return self.status()
|
||||
|
||||
def check(self):
|
||||
with self.check_lock:
|
||||
with self.check_lock, self.transport.router.hold_route():
|
||||
with self.state_lock:
|
||||
self.invalidate("正在同步研招目录会话并验证访问权限", state="unverified")
|
||||
revision = self.revision
|
||||
@@ -182,6 +209,10 @@ class AuthService:
|
||||
return self.status()
|
||||
|
||||
def browser(self):
|
||||
if platform.system() != "Windows":
|
||||
self.browser_state = "unsupported"
|
||||
self.browser_message = "服务器无桌面环境:请在自己电脑的浏览器完成学信网登录,再粘贴研招网 Cookie 并验证目录访问"
|
||||
return self.status()
|
||||
if self.browser_thread and self.browser_thread.is_alive() and not self.browser_cancel.is_set():
|
||||
return self.status()
|
||||
proxy = self.transport.router.resolve()
|
||||
|
||||
+7
-3
@@ -22,7 +22,7 @@ CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,sc
|
||||
""" + SCORE_SCHEMA + """
|
||||
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
|
||||
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
||||
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
|
||||
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
|
||||
"""
|
||||
@@ -45,7 +45,11 @@ class Database:
|
||||
conn.execute('DROP TABLE scores')
|
||||
conn.execute('ALTER TABLE scores_new RENAME TO scores')
|
||||
conn.execute('CREATE INDEX scores_lookup ON scores(school_code,major_code,year DESC)')
|
||||
conn.execute("UPDATE jobs SET status='paused',message='服务重启,任务可从断点恢复' WHERE status IN ('running','queued')")
|
||||
if 'resume_pending' not in {row['name'] for row in conn.execute('PRAGMA table_info(jobs)')}:
|
||||
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending INTEGER DEFAULT 0')
|
||||
recovery = "status IN ('running','queued') OR (status='paused' AND resume_pending=1)"
|
||||
self.recovered_job_ids = [row['id'] for row in conn.execute('SELECT id FROM jobs WHERE ' + recovery)]
|
||||
conn.execute("UPDATE jobs SET status='paused',resume_pending=0,message='服务重启,任务可从断点恢复' WHERE " + recovery)
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
@@ -182,7 +186,7 @@ class Database:
|
||||
conn.execute("INSERT INTO jobs(id,params,status,created_at,updated_at,message) VALUES(?,?,'queued',?,?,?)",(jid,encode(params),now(),now(),"等待开始"))
|
||||
|
||||
def job_update(self, jid, **changes):
|
||||
allowed = {"status","message","errors","records"}
|
||||
allowed = {"status","message","errors","records","resume_pending"}
|
||||
changes = {key:value for key,value in changes.items() if key in allowed}
|
||||
changes["updated_at"] = now()
|
||||
with self.connection() as conn:
|
||||
|
||||
+3
-3
@@ -67,7 +67,7 @@ class JobManager:
|
||||
|
||||
def _run(self, jid):
|
||||
job = self.get(jid)
|
||||
self.db.job_update(jid, status="running", errors=0, message="正在采集")
|
||||
self.db.job_update(jid, status="running", errors=0, resume_pending=0, message="正在采集")
|
||||
self.db.log(jid, "info", "任务开始/恢复,将跳过已完成单元")
|
||||
if self.transport:
|
||||
self.transport.local.guard = lambda: self.checkpoint(jid)
|
||||
@@ -127,7 +127,7 @@ class JobManager:
|
||||
raise ValueError("任务当前不在运行")
|
||||
with control["condition"]:
|
||||
control["paused"] = True
|
||||
self.db.job_update(jid, status="paused", message="已暂停,当前请求结束后停止")
|
||||
self.db.job_update(jid, status="paused", resume_pending=0, message="已暂停,当前请求结束后停止")
|
||||
return self.get(jid)
|
||||
|
||||
def resume(self, jid):
|
||||
@@ -169,7 +169,7 @@ class JobManager:
|
||||
if control["thread"].is_alive():
|
||||
with control['condition']:
|
||||
control['shutdown'] = True
|
||||
self.db.job_update(jid, status='paused', message='服务停止,任务可从断点恢复')
|
||||
self.db.job_update(jid, status='paused', resume_pending=int(not control['paused']), message='服务停止,任务可从断点恢复')
|
||||
control['condition'].notify_all()
|
||||
for control in list(self.controls.values()):
|
||||
control["thread"].join(timeout=1)
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Control only the two fixed, dedicated proxy-pool systemd units."""
|
||||
import socket
|
||||
import subprocess
|
||||
from urllib.parse import urlparse
|
||||
import requests
|
||||
|
||||
UNITS = ('kaoyan-proxy-api.service', 'kaoyan-proxy-scheduler.service')
|
||||
|
||||
|
||||
class LinuxPoolService:
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
self.session = requests.Session()
|
||||
self.session.trust_env = False
|
||||
|
||||
def status(self, config=None):
|
||||
config = config or self.settings.read()
|
||||
api_ready, counts = False, {}
|
||||
try:
|
||||
response = self.session.get(config['pool_url'] + '/count/', timeout=2)
|
||||
response.raise_for_status()
|
||||
counts = response.json()
|
||||
api_ready = isinstance(counts, dict) and type(counts.get('count')) is int
|
||||
except (requests.RequestException, ValueError):
|
||||
pass
|
||||
items = []
|
||||
for name, label, unit in zip(('server', 'schedule'), ('代理 API', '抓取与检测'), UNITS):
|
||||
try:
|
||||
result = subprocess.run(['/usr/bin/systemctl', 'show', unit, '--property=LoadState,ActiveState,MainPID'], capture_output=True, text=True, timeout=5)
|
||||
facts = dict(line.split('=', 1) for line in result.stdout.splitlines() if '=' in line)
|
||||
installed = facts.get('LoadState') == 'loaded'
|
||||
active = installed and facts.get('ActiveState') == 'active'
|
||||
items.append({'name': name, 'label': label, 'state': 'running' if active else 'stopped', 'pid': int(facts.get('MainPID', '0')) or None, 'verified': active, 'elevated': False, 'detail': 'systemd 运行中' if active else ('已停止' if installed else '未安装本项目 systemd 单元')})
|
||||
except (OSError, subprocess.TimeoutExpired, ValueError):
|
||||
items.append({'name': name, 'label': label, 'state': 'unknown', 'pid': None, 'verified': False, 'elevated': False, 'detail': '无法查询 systemd 状态'})
|
||||
try:
|
||||
with socket.create_connection(('127.0.0.1', 6381), timeout=1):
|
||||
redis_ready = True
|
||||
except OSError:
|
||||
redis_ready = False
|
||||
items.append({'name': 'redis', 'label': 'Redis', 'state': 'external' if redis_ready else 'stopped', 'pid': None, 'verified': False, 'elevated': False, 'detail': '独立 Redis 服务(127.0.0.1:6381)' if redis_ready else '未连接到本项目 Redis'})
|
||||
local = urlparse(config['pool_url']).hostname in ('127.0.0.1', 'localhost', '::1')
|
||||
return {'api_ready': api_ready, 'counts': counts if api_ready else {}, 'items': items, 'can_stop': local and any(item['verified'] for item in items[:2]), 'warning': '' if local else '当前使用外部代理池,请在其所在服务器管理服务', 'admin_command': '', 'manager': 'systemd'}
|
||||
|
||||
def manage(self, action):
|
||||
if action not in ('start', 'stop', 'status'):
|
||||
raise ValueError('代理池操作无效')
|
||||
status = self.status()
|
||||
result = {'ok': True, 'action': action, 'changed': False, 'status': status, 'message': '服务状态已更新', 'output': '\n'.join(item['label'] + ':' + item['detail'] for item in status['items'])}
|
||||
if action == 'status':
|
||||
return result
|
||||
if status['warning']:
|
||||
return {**result, 'ok': False, 'message': status['warning']}
|
||||
if action == 'start' and all(item['state'] == 'running' for item in status['items'][:2]) and status['api_ready']:
|
||||
return {**result, 'message': '代理池已在运行,已复用现有服务'}
|
||||
try:
|
||||
completed = subprocess.run(['/usr/bin/sudo', '-n', '/usr/bin/systemctl', action, *UNITS], capture_output=True, text=True, timeout=55)
|
||||
message = '代理池启动完成' if action == 'start' else '代理池已停止'
|
||||
return {**result, 'ok': completed.returncode == 0, 'changed': True, 'status': self.status(), 'message': message if completed.returncode == 0 else 'systemd 操作失败,请检查本项目 sudoers 与服务日志', 'output': (completed.stdout + completed.stderr)[-5000:] or message}
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return {**result, 'ok': False, 'message': '代理池管理超时或命令不可用,请检查 systemd 服务'}
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Explicit, atomic migration from a consistent SQLite snapshot into an empty MySQL database."""
|
||||
import argparse
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from studio.domain import normalized_name, is_special_score
|
||||
from studio.mysql_database import MySQLDatabase, score_key
|
||||
|
||||
TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs')
|
||||
|
||||
|
||||
def migrate_sqlite(source, target):
|
||||
source = Path(source).resolve()
|
||||
conn = sqlite3.connect(source.as_uri() + '?mode=ro', uri=True)
|
||||
conn.row_factory = sqlite3.Row
|
||||
counts = {}
|
||||
try:
|
||||
conn.execute('BEGIN')
|
||||
with target.connection() as output:
|
||||
# Serialize attempted imports to the same database.
|
||||
acquired = output.execute("SELECT GET_LOCK('kaoyan_studio_migration',10)").fetchone()[0]
|
||||
if acquired != 1:
|
||||
raise ValueError('已有数据迁移正在进行')
|
||||
try:
|
||||
for table in TABLES:
|
||||
if output.execute(f'SELECT count(*) FROM {table}').fetchone()[0]:
|
||||
raise ValueError(f'目标数据库非空({table}),拒绝覆盖;请使用空数据库')
|
||||
for table in TABLES:
|
||||
counts[table] = 0
|
||||
reader = conn.execute(f'SELECT * FROM {table}')
|
||||
while batch := reader.fetchmany(500):
|
||||
rows = []
|
||||
for original in batch:
|
||||
row = dict(original)
|
||||
if table in ('catalog', 'scores'):
|
||||
payload = json.loads(row['data'])
|
||||
row['department_norm'] = normalized_name(row.get('department_name'))
|
||||
if table == 'scores':
|
||||
row['note'] = row.get('note') or payload.get('note') or ''
|
||||
row['raw_code'] = row.get('raw_code') or payload.get('raw_code') or ''
|
||||
row['score_key'] = score_key(row)
|
||||
row['special_line'] = int(is_special_score(row['note']))
|
||||
if table == 'jobs' and row['status'] in ('running', 'queued'):
|
||||
row.update(status='paused', message='已迁移到服务器,验证登录后可从断点恢复')
|
||||
if table == 'jobs':
|
||||
row['resume_pending'] = 0
|
||||
rows.append(row)
|
||||
fields = list(rows[0])
|
||||
output.executemany(f"INSERT INTO {table}({','.join(fields)}) VALUES({','.join('?' for _ in fields)})", [[row[field] for field in fields] for row in rows])
|
||||
counts[table] += len(rows)
|
||||
for table, count in counts.items():
|
||||
if output.execute(f'SELECT count(*) FROM {table}').fetchone()[0] != count:
|
||||
raise ValueError(f'迁移行数校验失败:{table}')
|
||||
finally:
|
||||
# The private connection closes after commit/rollback, releasing its
|
||||
# advisory lock only after the entire import becomes visible.
|
||||
pass
|
||||
return counts
|
||||
finally:
|
||||
conn.rollback()
|
||||
conn.close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('snapshot', type=Path)
|
||||
parser.add_argument('--config', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
config = json.loads(args.config.read_text())
|
||||
print(json.dumps(migrate_sqlite(args.snapshot, MySQLDatabase(config)), ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,165 @@
|
||||
"""MariaDB/MySQL storage retaining the local Database's public interface."""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from contextlib import contextmanager
|
||||
import pymysql
|
||||
from studio.database import Database, encode
|
||||
from studio.domain import normalized_name, is_special_score
|
||||
|
||||
SCHEMA = [
|
||||
"""CREATE TABLE IF NOT EXISTS catalog(
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,record_key CHAR(64) NOT NULL UNIQUE,year INT NOT NULL,
|
||||
school_code VARCHAR(32),school_name VARCHAR(255),province VARCHAR(64),major_code VARCHAR(32),major_name VARCHAR(255),
|
||||
department_code VARCHAR(64),department_name VARCHAR(255),study_mode VARCHAR(32),degree_type VARCHAR(64),level VARCHAR(128),
|
||||
subjects TEXT,quota INT,source VARCHAR(128),favorite TINYINT NOT NULL DEFAULT 0,data LONGTEXT NOT NULL,
|
||||
department_norm VARCHAR(255) NOT NULL DEFAULT '',INDEX catalog_filter(year,province,major_code,school_code))""",
|
||||
"""CREATE TABLE IF NOT EXISTS scores(
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,school_code VARCHAR(32),major_code VARCHAR(32),year INT,
|
||||
department_id VARCHAR(64),department_name VARCHAR(255),scope VARCHAR(32),total INT,source VARCHAR(128),data LONGTEXT NOT NULL,
|
||||
note TEXT NOT NULL,raw_code VARCHAR(64) NOT NULL,score_key CHAR(64) NOT NULL UNIQUE,
|
||||
department_norm VARCHAR(255) NOT NULL DEFAULT '',special_line TINYINT NOT NULL DEFAULT 0,
|
||||
INDEX scores_lookup(school_code,major_code,year))""",
|
||||
"CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)",
|
||||
"CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)",
|
||||
"CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))",
|
||||
"CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))",
|
||||
]
|
||||
|
||||
|
||||
def score_key(row):
|
||||
identity = [row.get(field) or '' for field in ('school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'source', 'note', 'raw_code')]
|
||||
return hashlib.sha256(encode(identity).encode()).hexdigest()
|
||||
|
||||
|
||||
class Row(dict):
|
||||
def __getitem__(self, key):
|
||||
return tuple(self.values())[key] if isinstance(key, int) else super().__getitem__(key)
|
||||
|
||||
|
||||
class Result:
|
||||
def __init__(self, cursor):
|
||||
self.rowcount = cursor.rowcount
|
||||
self.columns = [item[0] for item in cursor.description] if cursor.description else []
|
||||
self.rows = cursor.fetchall() if cursor.description else ()
|
||||
self.offset = 0
|
||||
cursor.close()
|
||||
|
||||
def fetchone(self):
|
||||
if self.offset >= len(self.rows):
|
||||
return None
|
||||
result = Row(zip(self.columns, self.rows[self.offset]))
|
||||
self.offset += 1
|
||||
return result
|
||||
|
||||
def fetchall(self):
|
||||
return list(self)
|
||||
|
||||
def __iter__(self):
|
||||
while self.offset < len(self.rows):
|
||||
yield self.fetchone()
|
||||
|
||||
|
||||
def mysql_sql(sql):
|
||||
if sql.startswith('SELECT count(*) FROM (') and sql.endswith(')'):
|
||||
sql += ' AS studio_count'
|
||||
sql = sql.replace('INSERT OR IGNORE', 'INSERT IGNORE')
|
||||
sql = re.sub(r'ON CONFLICT\([^)]*\) DO UPDATE SET', 'ON DUPLICATE KEY UPDATE', sql)
|
||||
sql = re.sub(r'excluded\.(\w+)', r'VALUES(\1)', sql)
|
||||
sql = sql.replace("is_special_score(json_extract(x.data,'$.note'))=0", 'x.special_line=0')
|
||||
sql = sql.replace('norm_name(x.department_name)', 'x.department_norm').replace('norm_name(c.department_name)', 'c.department_norm')
|
||||
sql = re.sub(r"json_extract\(([^,]+),('[^']+')\)", r'JSON_UNQUOTE(JSON_EXTRACT(\1,\2))', sql)
|
||||
return sql.replace('?', '%s')
|
||||
|
||||
|
||||
class Connection:
|
||||
def __init__(self, raw):
|
||||
self.raw = raw
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
cursor = self.raw.cursor()
|
||||
try:
|
||||
cursor.execute(mysql_sql(sql), params)
|
||||
return Result(cursor)
|
||||
except BaseException:
|
||||
cursor.close()
|
||||
raise
|
||||
|
||||
def executemany(self, sql, values):
|
||||
cursor = self.raw.cursor()
|
||||
try:
|
||||
cursor.executemany(mysql_sql(sql), values)
|
||||
return Result(cursor)
|
||||
except BaseException:
|
||||
cursor.close()
|
||||
raise
|
||||
|
||||
|
||||
class MySQLDatabase(Database):
|
||||
def __init__(self, config):
|
||||
self.config = {key: value for key, value in config.items() if key in ('host', 'port', 'user', 'password', 'database', 'unix_socket')}
|
||||
if not self.config.get('database'):
|
||||
raise ValueError('数据库配置缺少 database')
|
||||
self.recovered_job_ids = []
|
||||
with self.connection() as conn:
|
||||
for statement in SCHEMA:
|
||||
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
|
||||
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
|
||||
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
|
||||
recovery = "status IN ('running','queued') OR (status='paused' AND resume_pending=1)"
|
||||
self.recovered_job_ids = [row['id'] for row in conn.execute('SELECT id FROM jobs WHERE ' + recovery)]
|
||||
conn.execute("UPDATE jobs SET status='paused',resume_pending=0,message='服务重启,任务可从断点恢复' WHERE " + recovery)
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
raw = pymysql.connect(**self.config, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
|
||||
try:
|
||||
yield Connection(raw)
|
||||
raw.commit()
|
||||
except BaseException:
|
||||
raw.rollback()
|
||||
raise
|
||||
finally:
|
||||
raw.close()
|
||||
|
||||
def upsert_catalog(self, records):
|
||||
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
|
||||
values = []
|
||||
for record in records:
|
||||
row = {**record, 'subjects': ' | '.join(record.get('subjects', [])), 'data': encode(record), 'department_norm': normalized_name(record.get('department_name'))}
|
||||
values.append([row.get(field) for field in fields])
|
||||
if not values:
|
||||
return
|
||||
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
|
||||
def upsert_scores(self, records):
|
||||
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
|
||||
values = []
|
||||
for record in records:
|
||||
row = {**record, 'department_id': record.get('department_id') or '', 'department_name': record.get('department_name') or '', 'source': record.get('source', '掌上考研'), 'note': record.get('note') or '', 'raw_code': record.get('raw_code') or ''}
|
||||
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
|
||||
values.append([row.get(field) for field in fields])
|
||||
if values:
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
|
||||
|
||||
def favorite(self, rid, enabled):
|
||||
# MariaDB reports changed rows; an already-selected favorite is still success.
|
||||
with self.connection() as conn:
|
||||
conn.execute('UPDATE catalog SET favorite=? WHERE id=?', (int(enabled), rid))
|
||||
return conn.execute('SELECT 1 FROM catalog WHERE id=?', (rid,)).fetchone() is not None
|
||||
|
||||
def history(self, rid):
|
||||
record = self.record(rid)
|
||||
if record is None:
|
||||
return None
|
||||
with self.connection() as conn:
|
||||
scores = conn.execute('SELECT data FROM scores WHERE school_code=? AND major_code=? ORDER BY year,scope,department_name', (record['school_code'], record['major_code'])).fetchall()
|
||||
plans = conn.execute('SELECT data FROM catalog WHERE school_code=? AND major_code=? AND department_norm=? ORDER BY year,id', (record['school_code'], record['major_code'], normalized_name(record['department_name']))).fetchall()
|
||||
def matches(row):
|
||||
return not row.get('department_name') and not row.get('department_id') or row.get('department_id') and row['department_id'] == record['department_code'] or row.get('department_name') and normalized_name(row['department_name']) == normalized_name(record['department_name'])
|
||||
return {'record': record, 'scores': [row for entry in scores if matches(row := json.loads(entry['data']))], 'plans': [json.loads(row['data']) for row in plans]}
|
||||
+37
-5
@@ -1,6 +1,8 @@
|
||||
"""One explicit route for CHSI login, catalog and historical data."""
|
||||
import threading
|
||||
import time
|
||||
import random
|
||||
from contextlib import contextmanager
|
||||
import requests
|
||||
from studio.settings import proxy_url
|
||||
|
||||
@@ -29,6 +31,10 @@ class ProxyRouter:
|
||||
self.counts = {}
|
||||
self.fallback = False
|
||||
self.lock = threading.RLock()
|
||||
self.route_requests = 0
|
||||
self.rotations = 0
|
||||
self.last_request_route = None
|
||||
self.holds = 0
|
||||
|
||||
def refresh(self):
|
||||
with self.lock:
|
||||
@@ -60,12 +66,31 @@ class ProxyRouter:
|
||||
self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"]))
|
||||
return self.status()
|
||||
|
||||
def resolve(self):
|
||||
@contextmanager
|
||||
def hold_route(self):
|
||||
with self.lock:
|
||||
self.holds += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with self.lock:
|
||||
self.holds -= 1
|
||||
|
||||
def note_success(self, proxy):
|
||||
with self.lock:
|
||||
self.last_request_route = proxy or "direct"
|
||||
if proxy == self.active:
|
||||
self.route_requests += 1
|
||||
|
||||
def resolve(self, for_request=False):
|
||||
with self.lock:
|
||||
config = self.settings.read()
|
||||
if config["network_mode"] == "direct":
|
||||
self.fallback = False
|
||||
return None
|
||||
threshold = config["proxy_rotate_every"]
|
||||
if for_request and not self.holds and threshold and self.active and self.route_requests >= threshold:
|
||||
self.rotate()
|
||||
if self.active and self.active not in self.failed:
|
||||
self.fallback = False
|
||||
return self.active
|
||||
@@ -75,9 +100,11 @@ class ProxyRouter:
|
||||
available = [p for p in self.candidates if p not in self.failed]
|
||||
if available:
|
||||
self.active = available[0]
|
||||
self.route_requests = 0
|
||||
self.fallback = False
|
||||
return self.active
|
||||
self.active = None
|
||||
self.route_requests = 0
|
||||
if config["network_mode"] == "proxy_only":
|
||||
self.fallback = False
|
||||
raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务")
|
||||
@@ -93,7 +120,10 @@ class ProxyRouter:
|
||||
if old in self.candidates:
|
||||
self.candidates.remove(old)
|
||||
self.candidates.append(old)
|
||||
return self.resolve()
|
||||
selected = self.resolve()
|
||||
if old and selected != old:
|
||||
self.rotations += 1
|
||||
return selected
|
||||
|
||||
def reset(self):
|
||||
with self.lock:
|
||||
@@ -101,10 +131,11 @@ class ProxyRouter:
|
||||
self.failed.clear()
|
||||
self.candidates = []
|
||||
self.fallback = False
|
||||
self.route_requests = 0
|
||||
|
||||
def status(self):
|
||||
config = self.settings.read()
|
||||
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
|
||||
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "route_requests": self.route_requests, "rotations": self.rotations, "last_request_route": self.last_request_route, "rotate_every": config["proxy_rotate_every"], "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
|
||||
|
||||
|
||||
class Transport:
|
||||
@@ -130,8 +161,8 @@ class Transport:
|
||||
def request(self, method, url, **kwargs):
|
||||
config = self.settings.read()
|
||||
for attempt in range(config["retries"]):
|
||||
self.sleep(config["request_interval"] if attempt == 0 else min(20, 2 ** attempt))
|
||||
proxy = self.router.resolve()
|
||||
self.sleep(random.uniform(config["request_interval"], config["request_interval_max"]) if attempt == 0 else min(20, 2 ** attempt))
|
||||
proxy = self.router.resolve(for_request=True)
|
||||
kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {}
|
||||
kwargs.setdefault("timeout", config["timeout"])
|
||||
try:
|
||||
@@ -143,6 +174,7 @@ class Transport:
|
||||
continue
|
||||
response.raise_for_status()
|
||||
self.local.response_route = proxy or "direct"
|
||||
self.router.note_success(proxy)
|
||||
return response
|
||||
except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
if proxy:
|
||||
|
||||
@@ -137,6 +137,10 @@ def _pool_entry(arguments, project, service):
|
||||
class PoolService:
|
||||
def __init__(self, settings):
|
||||
self.settings = settings
|
||||
self.linux = None
|
||||
if os.name != "nt":
|
||||
from studio.linux_pool import LinuxPoolService
|
||||
self.linux = LinuxPoolService(settings)
|
||||
self.session = requests.Session()
|
||||
self.session.trust_env = False
|
||||
self.lock = threading.Lock()
|
||||
@@ -154,6 +158,8 @@ class PoolService:
|
||||
return json.loads(result.stdout.decode("utf-8-sig"))
|
||||
|
||||
def status(self, config=None):
|
||||
if self.linux:
|
||||
return self.linux.status(config)
|
||||
if config is None:
|
||||
config = self.settings.read()
|
||||
project = Path(config["pool_project"])
|
||||
@@ -233,6 +239,8 @@ class PoolService:
|
||||
return "\n".join(lines)
|
||||
|
||||
def manage(self, action):
|
||||
if self.linux:
|
||||
return self.linux.manage(action)
|
||||
if action not in ("start", "stop", "status"):
|
||||
raise ValueError("代理池操作无效")
|
||||
with self.lock:
|
||||
|
||||
+14
-6
@@ -1,13 +1,16 @@
|
||||
"""Project-local, non-secret configuration."""
|
||||
import json
|
||||
import threading
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
DEFAULTS = {
|
||||
"network_mode": "prefer_proxy", "pool_url": "http://127.0.0.1:5010",
|
||||
"pool_project": r"C:\Users\Jayson\Desktop\proxy_pool",
|
||||
"backup_proxies": [], "request_interval": 1.2, "timeout": 30, "retries": 3,
|
||||
"pool_project": r"C:\Users\Jayson\Desktop\proxy_pool" if os.name == "nt" else "/opt/kaoyan-studio-deploy/proxy_pool",
|
||||
"backup_proxies": [], "request_interval": 1.2, "request_interval_max": 2.4,
|
||||
"proxy_rotate_every": 0, "timeout": 30, "retries": 3,
|
||||
}
|
||||
|
||||
|
||||
@@ -37,7 +40,10 @@ class SettingsStore:
|
||||
def read(self):
|
||||
with self.lock:
|
||||
values = json.loads(self.path.read_text(encoding="utf-8")) if self.path.exists() else {}
|
||||
return {**DEFAULTS, **{k: v for k, v in values.items() if k in DEFAULTS}}
|
||||
result = {**DEFAULTS, **{k: v for k, v in values.items() if k in DEFAULTS}}
|
||||
if "request_interval_max" not in values:
|
||||
result["request_interval_max"] = max(result["request_interval"], DEFAULTS["request_interval_max"])
|
||||
return result
|
||||
|
||||
def update(self, values):
|
||||
with self.lock:
|
||||
@@ -48,10 +54,12 @@ class SettingsStore:
|
||||
if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password:
|
||||
raise ValueError("代理池 API 地址无效")
|
||||
values["pool_url"] = values["pool_url"].rstrip("/")
|
||||
for key, low, high in [("request_interval", .3, 60), ("timeout", 5, 120), ("retries", 1, 6)]:
|
||||
values[key] = float(values[key]) if key == "request_interval" else int(values[key])
|
||||
if not low <= values[key] <= high:
|
||||
for key, low, high in [("request_interval", .3, 60), ("request_interval_max", .3, 60), ("timeout", 5, 120), ("retries", 1, 6), ("proxy_rotate_every", 0, 100000)]:
|
||||
values[key] = float(values[key]) if key.startswith("request_interval") else int(values[key])
|
||||
if not math.isfinite(values[key]) or not low <= values[key] <= high:
|
||||
raise ValueError(f"{key} 应在 {low} 到 {high} 之间")
|
||||
if values["request_interval_max"] < values["request_interval"]:
|
||||
raise ValueError("请求间隔上限不能小于下限")
|
||||
proxies = values["backup_proxies"]
|
||||
if isinstance(proxies, str):
|
||||
proxies = proxies.splitlines()
|
||||
|
||||
Reference in New Issue
Block a user