Add Debian deployment, MariaDB storage and randomized proxy routing
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
"""Real MariaDB contract tests; run only against an explicitly supplied test database."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
from studio.domain import normalize_catalog, select_score_lines
|
||||
from studio.database import Database
|
||||
from tests.test_data import raw_catalog
|
||||
|
||||
pytestmark = pytest.mark.skipif(not os.environ.get('STUDIO_MYSQL_TEST_CONFIG'), reason='requires an isolated MariaDB test database')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mysql():
|
||||
from studio.mysql_database import MySQLDatabase
|
||||
config = json.loads(Path(os.environ['STUDIO_MYSQL_TEST_CONFIG']).read_text())
|
||||
assert config['database'].endswith('_test'), 'refusing to clear a production database'
|
||||
db = MySQLDatabase(config)
|
||||
with db.connection() as conn:
|
||||
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs'):
|
||||
conn.execute(f'DELETE FROM {table}')
|
||||
return db
|
||||
|
||||
|
||||
def test_mysql_preserves_upserts_favorites_and_history(mysql):
|
||||
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
|
||||
first = mysql.query_catalog({})['items'][0]
|
||||
mysql.favorite(first['id'], True)
|
||||
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
|
||||
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
|
||||
mysql.upsert_scores(select_score_lines([
|
||||
{'code': '081200', 'total': 320, 'depart_name': '计算机学院'},
|
||||
{'code': '081200', 'total': 260, 'depart_name': '计算机学院', 'note': '少干专项计划'},
|
||||
{'code': '081200', 'total': 390, 'depart_name': '其他学院'},
|
||||
], '081200', '10001', '北京大学', 2026))
|
||||
result = mysql.query_catalog({'year': 2027, 'direction': '人工', 'english': '203'})
|
||||
assert result['total'] == 1
|
||||
assert result['items'][0]['score_total'] == 320
|
||||
assert mysql.query_catalog({'favorites': True})['total'] == 1
|
||||
assert mysql.overview()['records'] == 4
|
||||
assert len(mysql.history(first['id'])['plans']) == 4
|
||||
assert len(mysql.history(first['id'])['scores']) == 2
|
||||
|
||||
|
||||
def test_mysql_score_unique_key_preserves_long_notes_and_normalized_department(mysql):
|
||||
raw = raw_catalog()
|
||||
raw['yxsmc'] = '计算机 ( 学院 )'
|
||||
mysql.upsert_catalog(normalize_catalog(raw, 2027))
|
||||
scores = select_score_lines([
|
||||
{'code': '081200', 'total': 320, 'depart_name': '计算机(学院)', 'note': '普通计划' * 300},
|
||||
{'code': '081200', 'total': 321, 'depart_name': '计算机(学院)', 'note': '普通计划' * 300 + 'B'},
|
||||
], '081200', '10001', '北京大学', 2026)
|
||||
mysql.upsert_scores(scores)
|
||||
mysql.upsert_scores(scores)
|
||||
assert mysql.overview()['scores'] == 2
|
||||
assert mysql.query_catalog({})['items'][0]['score_total'] == 321
|
||||
|
||||
|
||||
def test_mysql_tasks_logs_and_checkpoints(mysql):
|
||||
mysql.job_create('contract-job', {'year': 2027})
|
||||
mysql.job_update('contract-job', status='running', records=7)
|
||||
mysql.complete_unit('contract-job', 'catalog:11:10001:081200:1')
|
||||
mysql.complete_unit('contract-job', 'catalog:11:10001:081200:1')
|
||||
mysql.log('contract-job', 'info', '数据库测试')
|
||||
assert mysql.unit_done('contract-job', 'catalog:11:10001:081200:1')
|
||||
assert mysql.job_get('contract-job')['completed_units'] == 1
|
||||
assert mysql.job_get('contract-job')['records'] == 7
|
||||
assert mysql.logs('contract-job')[0]['message'] == '数据库测试'
|
||||
|
||||
|
||||
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
|
||||
from studio.migrate import migrate_sqlite
|
||||
source = Database(tmp_path / 'snapshot.db')
|
||||
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
|
||||
source.upsert_scores(select_score_lines([{'code': '081200', 'total': 320}], '081200', '10001', '北京大学', 2026))
|
||||
source.favorite(1, True)
|
||||
source.job_create('migrated-job', {'year': 2027})
|
||||
source.complete_unit('migrated-job', 'catalog:11:10001:081200:1')
|
||||
source.log('migrated-job', 'info', 'preserve log')
|
||||
counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql)
|
||||
assert counts['catalog'] == 2 and counts['scores'] == 1
|
||||
assert mysql.record(1)['favorite']
|
||||
assert mysql.query_catalog({})['items'][0]['score_total'] == 320
|
||||
assert mysql.job_get('migrated-job')['completed_units'] == 1
|
||||
assert mysql.logs('migrated-job')[0]['id'] == 1
|
||||
with pytest.raises(ValueError, match='非空'):
|
||||
migrate_sqlite(tmp_path / 'snapshot.db', mysql)
|
||||
|
||||
|
||||
def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
|
||||
from studio.migrate import migrate_sqlite
|
||||
source = Database(tmp_path / 'bad.db')
|
||||
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
|
||||
with source.connection() as conn:
|
||||
conn.execute("INSERT INTO scores(school_code,major_code,year,scope,data) VALUES('10001','081200',2026,'major','not json')")
|
||||
with pytest.raises(ValueError):
|
||||
migrate_sqlite(tmp_path / 'bad.db', mysql)
|
||||
assert mysql.overview()['records'] == 0
|
||||
assert mysql.overview()['scores'] == 0
|
||||
@@ -11,6 +11,8 @@ from fastapi.testclient import TestClient
|
||||
|
||||
from studio.api import create_app
|
||||
|
||||
pytestmark = pytest.mark.skipif(os.name != 'nt', reason='Windows pool process and PowerShell behavior')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pool_client(tmp_path):
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import platform
|
||||
import random
|
||||
import os
|
||||
import pytest
|
||||
|
||||
from studio.auth import CookieVault
|
||||
from tests.test_network import setup_network
|
||||
|
||||
|
||||
def test_requests_sample_a_fresh_delay_inside_configured_bounds(tmp_path, monkeypatch):
|
||||
settings, _, transport = setup_network(tmp_path)
|
||||
settings.update({'request_interval': .3, 'request_interval_max': .5})
|
||||
samples = iter([.31, .47, .39])
|
||||
monkeypatch.setattr(random, 'uniform', lambda low, high: next(samples))
|
||||
delays = []
|
||||
transport.sleep = delays.append
|
||||
for _ in range(3):
|
||||
transport.request('GET', 'https://yz.chsi.com.cn/zsml/')
|
||||
assert delays == [.31, .47, .39]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('maximum', [.2, 61, float('nan')])
|
||||
def test_invalid_interval_range_is_rejected(tmp_path, maximum):
|
||||
settings, _, _ = setup_network(tmp_path)
|
||||
with pytest.raises(ValueError):
|
||||
settings.update({'request_interval': .3, 'request_interval_max': maximum})
|
||||
|
||||
|
||||
def test_successful_requests_rotate_without_quarantining_old_proxy(tmp_path):
|
||||
settings, router, transport = setup_network(tmp_path)
|
||||
settings.update({'proxy_rotate_every': 2})
|
||||
routes = [transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] for _ in range(3)]
|
||||
assert routes == ['http://1.2.3.4:8080', 'http://1.2.3.4:8080', 'http://5.6.7.8:9090']
|
||||
assert router.failed == set()
|
||||
assert router.status()['last_request_route'] == 'http://5.6.7.8:9090'
|
||||
assert router.status()['route_requests'] == 1
|
||||
|
||||
|
||||
def test_route_pin_defers_rotation_until_login_proof_finishes(tmp_path):
|
||||
settings, router, transport = setup_network(tmp_path)
|
||||
settings.update({'proxy_rotate_every': 1})
|
||||
with router.hold_route():
|
||||
routes = [transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] for _ in range(2)]
|
||||
assert routes == ['http://1.2.3.4:8080'] * 2
|
||||
assert transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] == 'http://5.6.7.8:9090'
|
||||
|
||||
|
||||
def test_linux_cookie_vault_encrypts_and_preserves_domain_cookies(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(platform, 'system', lambda: 'Linux')
|
||||
cookies = [{'name': 'JSESSIONID', 'value': 'private-account-cookie', 'domain': 'account.chsi.com.cn'},
|
||||
{'name': 'JSESSIONID', 'value': 'private-catalog-cookie', 'domain': 'yz.chsi.com.cn'}]
|
||||
vault = CookieVault(tmp_path)
|
||||
vault.save(cookies)
|
||||
assert (tmp_path / 'cookie.key').exists()
|
||||
assert b'private-account-cookie' not in vault.path.read_bytes()
|
||||
assert CookieVault(tmp_path).load() == cookies
|
||||
if os.name != 'nt':
|
||||
assert (tmp_path / 'cookie.key').stat().st_mode & 0o777 == 0o600
|
||||
assert vault.path.stat().st_mode & 0o777 == 0o600
|
||||
|
||||
|
||||
def test_configured_server_host_is_allowed_but_other_hosts_and_origins_are_blocked(tmp_path, monkeypatch):
|
||||
from fastapi.testclient import TestClient
|
||||
from studio.api import create_app
|
||||
monkeypatch.setenv('STUDIO_ALLOWED_HOSTS', 'kaoyan.fjayson.com')
|
||||
with TestClient(create_app(tmp_path), base_url='https://kaoyan.fjayson.com') as client:
|
||||
assert client.get('/api/health').status_code == 200
|
||||
assert client.get('/api/health', headers={'Host': 'evil.example'}).status_code == 403
|
||||
assert client.put('/api/settings', json={'network_mode': 'direct'}, headers={'Origin': 'https://evil.example'}).status_code == 403
|
||||
assert client.put('/api/settings', json={'network_mode': 'direct'}, headers={'Origin': 'https://kaoyan.fjayson.com'}).status_code == 200
|
||||
|
||||
|
||||
def test_only_interrupted_running_tasks_are_selected_for_restart(tmp_path):
|
||||
from studio.database import Database
|
||||
db = Database(tmp_path / 'resume.db')
|
||||
for status in ('running', 'queued', 'paused', 'waiting_login', 'cancelled'):
|
||||
db.job_create(status, {'year': 2027})
|
||||
db.job_update(status, status=status)
|
||||
restarted = Database(tmp_path / 'resume.db')
|
||||
assert set(restarted.recovered_job_ids) == {'running', 'queued'}
|
||||
assert restarted.job_get('paused')['status'] == 'paused'
|
||||
assert restarted.job_get('waiting_login')['status'] == 'waiting_login'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('manually_paused', [False, True])
|
||||
def test_graceful_restart_remembers_running_but_not_manual_pause(tmp_path, manually_paused):
|
||||
import threading
|
||||
from studio.database import Database
|
||||
from studio.jobs import JobManager
|
||||
from studio.settings import SettingsStore
|
||||
ready = threading.Event()
|
||||
stop = threading.Event()
|
||||
def runner(jid, params, manager):
|
||||
ready.set()
|
||||
while not stop.wait(.01):
|
||||
manager.checkpoint(jid)
|
||||
db = Database(tmp_path / 'shutdown.db')
|
||||
manager = JobManager(db, SettingsStore(tmp_path), runner=runner)
|
||||
job = manager.start({'mode': 'catalog', 'year': 2027, 'provinces': ['11']})
|
||||
assert ready.wait(3)
|
||||
try:
|
||||
if manually_paused:
|
||||
manager.pause(job['id'])
|
||||
manager.close()
|
||||
restarted = Database(tmp_path / 'shutdown.db')
|
||||
assert (job['id'] in restarted.recovered_job_ids) is (not manually_paused)
|
||||
finally:
|
||||
stop.set()
|
||||
manager.close()
|
||||
Reference in New Issue
Block a user