Preserve login routes across rotation and keep rejected migrations read-only

This commit is contained in:
Codex
2026-10-11 19:15:05 +08:00
parent 915241a2fd
commit ab4e4a4f79
6 changed files with 68 additions and 12 deletions
+1 -1
View File
@@ -187,7 +187,7 @@ class AuthService:
raise ValueError("请输入学信网账号和密码") raise ValueError("请输入学信网账号和密码")
if self.browser_cancel: if self.browser_cancel:
self.browser_cancel.set() self.browser_cancel.set()
with self.check_lock: with self.check_lock, self.transport.router.hold_route():
response = self.transport.request("GET", LOGIN_URL) response = self.transport.request("GET", LOGIN_URL)
soup = BeautifulSoup(response.text, "html.parser") soup = BeautifulSoup(response.text, "html.parser")
form = {node.get("name"): node.get("value", "") for node in soup.select("input[type=hidden][name]")} form = {node.get("name"): node.get("value", "") for node in soup.select("input[type=hidden][name]")}
+5 -4
View File
@@ -72,10 +72,11 @@ class CatalogClient:
raise raise
# A CAS cookie may still be valid while the catalogue session has expired. # A CAS cookie may still be valid while the catalogue session has expired.
# Refresh once, then retry exactly the same page; never skip its records. # Refresh once, then retry exactly the same page; never skip its records.
status = self.auth.check() with self.transport.router.hold_route():
if not status["valid"]: status = self.auth.check()
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None if not status["valid"]:
return self._fetch_page(endpoint, payload) raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None
return self._fetch_page(endpoint, payload)
def pages(self, endpoint, payload, page_size=10): def pages(self, endpoint, payload, page_size=10):
seen, start = set(), 0 seen, start = set(), 0
+1 -1
View File
@@ -67,7 +67,7 @@ def main():
parser.add_argument('--config', type=Path, required=True) parser.add_argument('--config', type=Path, required=True)
args = parser.parse_args() args = parser.parse_args()
config = json.loads(args.config.read_text()) config = json.loads(args.config.read_text())
print(json.dumps(migrate_sqlite(args.snapshot, MySQLDatabase(config)), ensure_ascii=False)) print(json.dumps(migrate_sqlite(args.snapshot, MySQLDatabase(config, recover_jobs=False)), ensure_ascii=False))
if __name__ == '__main__': if __name__ == '__main__':
+5 -4
View File
@@ -96,7 +96,7 @@ class Connection:
class MySQLDatabase(Database): class MySQLDatabase(Database):
def __init__(self, config): def __init__(self, config, recover_jobs=True):
self.config = {key: value for key, value in config.items() if key in ('host', 'port', 'user', 'password', 'database', 'unix_socket')} self.config = {key: value for key, value in config.items() if key in ('host', 'port', 'user', 'password', 'database', 'unix_socket')}
if not self.config.get('database'): if not self.config.get('database'):
raise ValueError('数据库配置缺少 database') raise ValueError('数据库配置缺少 database')
@@ -106,9 +106,10 @@ class MySQLDatabase(Database):
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin') conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone(): if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0') conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
recovery = "status IN ('running','queued') OR (status='paused' AND resume_pending=1)" if recover_jobs:
self.recovered_job_ids = [row['id'] for row in conn.execute('SELECT id FROM jobs WHERE ' + recovery)] recovery = "status IN ('running','queued') OR (status='paused' AND resume_pending=1)"
conn.execute("UPDATE jobs SET status='paused',resume_pending=0,message='服务重启,任务可从断点恢复' WHERE " + recovery) self.recovered_job_ids = [row['id'] for row in conn.execute('SELECT id FROM jobs WHERE ' + recovery)]
conn.execute("UPDATE jobs SET status='paused',resume_pending=0,message='服务重启,任务可从断点恢复' WHERE " + recovery)
@contextmanager @contextmanager
def connection(self): def connection(self):
+11
View File
@@ -97,3 +97,14 @@ def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
migrate_sqlite(tmp_path / 'bad.db', mysql) migrate_sqlite(tmp_path / 'bad.db', mysql)
assert mysql.overview()['records'] == 0 assert mysql.overview()['records'] == 0
assert mysql.overview()['scores'] == 0 assert mysql.overview()['scores'] == 0
def test_rejected_migration_cli_does_not_pause_destination_jobs(mysql, tmp_path):
import subprocess
import sys
source = Database(tmp_path / 'source.db')
mysql.job_create('existing-running-job', {'year': 2027})
mysql.job_update('existing-running-job', status='running')
result = subprocess.run([sys.executable, '-m', 'studio.migrate', str(source.path), '--config', os.environ['STUDIO_MYSQL_TEST_CONFIG']], capture_output=True, text=True)
assert result.returncode != 0
assert mysql.job_get('existing-running-job')['status'] == 'running'
+45 -2
View File
@@ -3,8 +3,10 @@ import random
import os import os
import pytest import pytest
from studio.auth import CookieVault from studio.auth import CookieVault, AuthService, LOGIN_URL, CATALOG_ENTRY
from tests.test_network import setup_network from studio.connectors import CatalogClient
from studio.network import RemoteUnavailable
from tests.test_network import setup_network, Remote, Response
def test_requests_sample_a_fresh_delay_inside_configured_bounds(tmp_path, monkeypatch): def test_requests_sample_a_fresh_delay_inside_configured_bounds(tmp_path, monkeypatch):
@@ -45,6 +47,47 @@ def test_route_pin_defers_rotation_until_login_proof_finishes(tmp_path):
assert transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] == 'http://5.6.7.8:9090' assert transport.request('GET', 'https://yz.chsi.com.cn/zsml/').json()['routed_through'] == 'http://5.6.7.8:9090'
class RouteBoundSession(Remote):
def __init__(self):
super().__init__()
self.bound = set()
self.form_route = None
def request(self, method, url, **kwargs):
route = kwargs.get('proxies', {}).get('https')
if url == LOGIN_URL and method == 'GET':
self.form_route = route
return Response({}, text='<input type="hidden" name="lt" value="test"><input type="hidden" name="execution" value="test">')
if url == LOGIN_URL and method == 'POST':
if route != self.form_route:
raise RemoteUnavailable('login form and credentials used different routes')
return Response({})
if url == CATALOG_ENTRY:
self.bound = {route}
return Response({})
if url.endswith('/yjfxs.do'):
return Response({'flag': True, 'msg': {'list': [{'direction': 'confirmed'}]}}) if route in self.bound else Response({'flag': False, 'msg': '请登录'})
return super().request(method, url, **kwargs)
@pytest.mark.parametrize('threshold', [1, 2])
def test_auth_recovery_preserves_proven_route_through_catalog_retry(tmp_path, threshold):
settings, _, transport = setup_network(tmp_path, remote=RouteBoundSession())
settings.update({'proxy_rotate_every': threshold})
auth = AuthService(transport, CookieVault(tmp_path))
assert auth.check()['valid']
payload = CatalogClient(transport, auth).fetch_page('yjfxs', {'curPage': 2, 'start': 10, 'pageSize': 10})
assert payload['list'][0]['direction'] == 'confirmed'
def test_credential_login_pins_form_submission_and_verification(tmp_path):
settings, _, transport = setup_network(tmp_path, remote=RouteBoundSession())
settings.update({'proxy_rotate_every': 2})
transport.request('GET', 'https://yz.chsi.com.cn/zsml/')
auth = AuthService(transport, CookieVault(tmp_path))
assert auth.login('test-user', 'test-password')['valid']
def test_linux_cookie_vault_encrypts_and_preserves_domain_cookies(tmp_path, monkeypatch): def test_linux_cookie_vault_encrypts_and_preserves_domain_cookies(tmp_path, monkeypatch):
monkeypatch.setattr(platform, 'system', lambda: 'Linux') monkeypatch.setattr(platform, 'system', lambda: 'Linux')
cookies = [{'name': 'JSESSIONID', 'value': 'private-account-cookie', 'domain': 'account.chsi.com.cn'}, cookies = [{'name': 'JSESSIONID', 'value': 'private-account-cookie', 'domain': 'account.chsi.com.cn'},