Add Debian deployment, MariaDB storage and randomized proxy routing

This commit is contained in:
Codex
2026-10-11 19:07:27 +08:00
parent 2e3439dbb9
commit 915241a2fd
27 changed files with 896 additions and 32 deletions
+99
View File
@@ -0,0 +1,99 @@
"""Real MariaDB contract tests; run only against an explicitly supplied test database."""
import json
import os
from pathlib import Path
import pytest
from studio.domain import normalize_catalog, select_score_lines
from studio.database import Database
from tests.test_data import raw_catalog
pytestmark = pytest.mark.skipif(not os.environ.get('STUDIO_MYSQL_TEST_CONFIG'), reason='requires an isolated MariaDB test database')
@pytest.fixture
def mysql():
from studio.mysql_database import MySQLDatabase
config = json.loads(Path(os.environ['STUDIO_MYSQL_TEST_CONFIG']).read_text())
assert config['database'].endswith('_test'), 'refusing to clear a production database'
db = MySQLDatabase(config)
with db.connection() as conn:
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs'):
conn.execute(f'DELETE FROM {table}')
return db
def test_mysql_preserves_upserts_favorites_and_history(mysql):
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
first = mysql.query_catalog({})['items'][0]
mysql.favorite(first['id'], True)
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
mysql.upsert_scores(select_score_lines([
{'code': '081200', 'total': 320, 'depart_name': '计算机学院'},
{'code': '081200', 'total': 260, 'depart_name': '计算机学院', 'note': '少干专项计划'},
{'code': '081200', 'total': 390, 'depart_name': '其他学院'},
], '081200', '10001', '北京大学', 2026))
result = mysql.query_catalog({'year': 2027, 'direction': '人工', 'english': '203'})
assert result['total'] == 1
assert result['items'][0]['score_total'] == 320
assert mysql.query_catalog({'favorites': True})['total'] == 1
assert mysql.overview()['records'] == 4
assert len(mysql.history(first['id'])['plans']) == 4
assert len(mysql.history(first['id'])['scores']) == 2
def test_mysql_score_unique_key_preserves_long_notes_and_normalized_department(mysql):
raw = raw_catalog()
raw['yxsmc'] = '计算机 ( 学院 )'
mysql.upsert_catalog(normalize_catalog(raw, 2027))
scores = select_score_lines([
{'code': '081200', 'total': 320, 'depart_name': '计算机(学院)', 'note': '普通计划' * 300},
{'code': '081200', 'total': 321, 'depart_name': '计算机(学院)', 'note': '普通计划' * 300 + 'B'},
], '081200', '10001', '北京大学', 2026)
mysql.upsert_scores(scores)
mysql.upsert_scores(scores)
assert mysql.overview()['scores'] == 2
assert mysql.query_catalog({})['items'][0]['score_total'] == 321
def test_mysql_tasks_logs_and_checkpoints(mysql):
mysql.job_create('contract-job', {'year': 2027})
mysql.job_update('contract-job', status='running', records=7)
mysql.complete_unit('contract-job', 'catalog:11:10001:081200:1')
mysql.complete_unit('contract-job', 'catalog:11:10001:081200:1')
mysql.log('contract-job', 'info', '数据库测试')
assert mysql.unit_done('contract-job', 'catalog:11:10001:081200:1')
assert mysql.job_get('contract-job')['completed_units'] == 1
assert mysql.job_get('contract-job')['records'] == 7
assert mysql.logs('contract-job')[0]['message'] == '数据库测试'
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'snapshot.db')
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
source.upsert_scores(select_score_lines([{'code': '081200', 'total': 320}], '081200', '10001', '北京大学', 2026))
source.favorite(1, True)
source.job_create('migrated-job', {'year': 2027})
source.complete_unit('migrated-job', 'catalog:11:10001:081200:1')
source.log('migrated-job', 'info', 'preserve log')
counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql)
assert counts['catalog'] == 2 and counts['scores'] == 1
assert mysql.record(1)['favorite']
assert mysql.query_catalog({})['items'][0]['score_total'] == 320
assert mysql.job_get('migrated-job')['completed_units'] == 1
assert mysql.logs('migrated-job')[0]['id'] == 1
with pytest.raises(ValueError, match='非空'):
migrate_sqlite(tmp_path / 'snapshot.db', mysql)
def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'bad.db')
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
with source.connection() as conn:
conn.execute("INSERT INTO scores(school_code,major_code,year,scope,data) VALUES('10001','081200',2026,'major','not json')")
with pytest.raises(ValueError):
migrate_sqlite(tmp_path / 'bad.db', mysql)
assert mysql.overview()['records'] == 0
assert mysql.overview()['scores'] == 0
+2
View File
@@ -11,6 +11,8 @@ from fastapi.testclient import TestClient
from studio.api import create_app
pytestmark = pytest.mark.skipif(os.name != 'nt', reason='Windows pool process and PowerShell behavior')
@pytest.fixture
def pool_client(tmp_path):
+109
View File
@@ -0,0 +1,109 @@
import platform
import random
import os
import pytest
from studio.auth import CookieVault
from tests.test_network import setup_network
def test_requests_sample_a_fresh_delay_inside_configured_bounds(tmp_path, monkeypatch):
settings, _, transport = setup_network(tmp_path)
settings.update({'request_interval': .3, 'request_interval_max': .5})
samples = iter([.31, .47, .39])
monkeypatch.setattr(random, 'uniform', lambda low, high: next(samples))
delays = []
transport.sleep = delays.append
for _ in range(3):
transport.request('GET', 'https://yz.chsi.com.cn/zsml/')
assert delays == [.31, .47, .39]
@pytest.mark.parametrize('maximum', [.2, 61, float('nan')])
def test_invalid_interval_range_is_rejected(tmp_path, maximum):
settings, _, _ = setup_network(tmp_path)
with pytest.raises(ValueError):
settings.update({'request_interval': .3, 'request_interval_max': maximum})
def test_successful_requests_rotate_without_quarantining_old_proxy(tmp_path):
settings, router, transport = setup_network(tmp_path)
settings.update({'proxy_rotate_every': 2})
routes = [transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] for _ in range(3)]
assert routes == ['http://1.2.3.4:8080', 'http://1.2.3.4:8080', 'http://5.6.7.8:9090']
assert router.failed == set()
assert router.status()['last_request_route'] == 'http://5.6.7.8:9090'
assert router.status()['route_requests'] == 1
def test_route_pin_defers_rotation_until_login_proof_finishes(tmp_path):
settings, router, transport = setup_network(tmp_path)
settings.update({'proxy_rotate_every': 1})
with router.hold_route():
routes = [transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] for _ in range(2)]
assert routes == ['http://1.2.3.4:8080'] * 2
assert transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] == 'http://5.6.7.8:9090'
def test_linux_cookie_vault_encrypts_and_preserves_domain_cookies(tmp_path, monkeypatch):
monkeypatch.setattr(platform, 'system', lambda: 'Linux')
cookies = [{'name': 'JSESSIONID', 'value': 'private-account-cookie', 'domain': 'account.chsi.com.cn'},
{'name': 'JSESSIONID', 'value': 'private-catalog-cookie', 'domain': 'yz.chsi.com.cn'}]
vault = CookieVault(tmp_path)
vault.save(cookies)
assert (tmp_path / 'cookie.key').exists()
assert b'private-account-cookie' not in vault.path.read_bytes()
assert CookieVault(tmp_path).load() == cookies
if os.name != 'nt':
assert (tmp_path / 'cookie.key').stat().st_mode & 0o777 == 0o600
assert vault.path.stat().st_mode & 0o777 == 0o600
def test_configured_server_host_is_allowed_but_other_hosts_and_origins_are_blocked(tmp_path, monkeypatch):
from fastapi.testclient import TestClient
from studio.api import create_app
monkeypatch.setenv('STUDIO_ALLOWED_HOSTS', 'kaoyan.fjayson.com')
with TestClient(create_app(tmp_path), base_url='https://kaoyan.fjayson.com') as client:
assert client.get('/api/health').status_code == 200
assert client.get('/api/health', headers={'Host': 'evil.example'}).status_code == 403
assert client.put('/api/settings', json={'network_mode': 'direct'}, headers={'Origin': 'https://evil.example'}).status_code == 403
assert client.put('/api/settings', json={'network_mode': 'direct'}, headers={'Origin': 'https://kaoyan.fjayson.com'}).status_code == 200
def test_only_interrupted_running_tasks_are_selected_for_restart(tmp_path):
from studio.database import Database
db = Database(tmp_path / 'resume.db')
for status in ('running', 'queued', 'paused', 'waiting_login', 'cancelled'):
db.job_create(status, {'year': 2027})
db.job_update(status, status=status)
restarted = Database(tmp_path / 'resume.db')
assert set(restarted.recovered_job_ids) == {'running', 'queued'}
assert restarted.job_get('paused')['status'] == 'paused'
assert restarted.job_get('waiting_login')['status'] == 'waiting_login'
@pytest.mark.parametrize('manually_paused', [False, True])
def test_graceful_restart_remembers_running_but_not_manual_pause(tmp_path, manually_paused):
import threading
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
ready = threading.Event()
stop = threading.Event()
def runner(jid, params, manager):
ready.set()
while not stop.wait(.01):
manager.checkpoint(jid)
db = Database(tmp_path / 'shutdown.db')
manager = JobManager(db, SettingsStore(tmp_path), runner=runner)
job = manager.start({'mode': 'catalog', 'year': 2027, 'provinces': ['11']})
assert ready.wait(3)
try:
if manually_paused:
manager.pause(job['id'])
manager.close()
restarted = Database(tmp_path / 'shutdown.db')
assert (job['id'] in restarted.recovered_job_ids) is (not manually_paused)
finally:
stop.set()
manager.close()