fix: resume catalog jobs without repeated checkpoint queries

This commit is contained in:
2026-10-11 21:18:49 +08:00
parent c4a71e88a1
commit 4c510759fb
9 changed files with 309 additions and 13 deletions
+45 -1
View File
@@ -17,7 +17,7 @@ def mysql():
assert config['database'].endswith('_test'), 'refusing to clear a production database'
db = MySQLDatabase(config)
with db.connection() as conn:
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs'):
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs', 'job_catalog_pages'):
conn.execute(f'DELETE FROM {table}')
return db
@@ -68,6 +68,36 @@ def test_mysql_tasks_logs_and_checkpoints(mysql):
assert mysql.logs('contract-job')[0]['message'] == '数据库测试'
def test_mysql_catalog_page_cache_and_bulk_checkpoints_are_durable(mysql):
mysql.job_create('cached-job', {'year': 2027})
mysql.complete_unit('cached-job', 'catalog:11:10001:081200:1')
page = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': False}
mysql.cache_catalog_page('cached-job', 'page-key', page)
from studio.mysql_database import MySQLDatabase
reopened = MySQLDatabase(mysql.config, recover_jobs=False)
assert reopened.completed_units('cached-job') == {'catalog:11:10001:081200:1'}
assert reopened.catalog_pages('cached-job') == {'page-key': page}
assert reopened.catalog_pages('another-job') == {}
def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypatch):
import ssl
with mysql.connection() as conn:
cipher = conn.execute("SHOW SESSION STATUS LIKE 'Ssl_cipher'").fetchone()[1]
if not cipher:
pytest.skip('requires a TLS-enabled MariaDB test server')
creations = []
original = ssl.create_default_context
def observe(*args, **kwargs):
creations.append(True)
return original(*args, **kwargs)
monkeypatch.setattr(ssl, 'create_default_context', observe)
for _ in range(2):
with mysql.connection() as conn:
assert conn.execute('SELECT 1').fetchone()[0] == 1
assert creations == []
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'snapshot.db')
@@ -77,16 +107,30 @@ def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_p
source.job_create('migrated-job', {'year': 2027})
source.complete_unit('migrated-job', 'catalog:11:10001:081200:1')
source.log('migrated-job', 'info', 'preserve log')
source.cache_catalog_page('migrated-job', 'migration-page', {'list': [], 'nextPageAvailable': False})
counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql)
assert counts['catalog'] == 2 and counts['scores'] == 1
assert mysql.record(1)['favorite']
assert mysql.query_catalog({})['items'][0]['score_total'] == 320
assert mysql.job_get('migrated-job')['completed_units'] == 1
assert mysql.logs('migrated-job')[0]['id'] == 1
assert counts['job_catalog_pages'] == 1
assert mysql.catalog_pages('migrated-job')['migration-page']['list'] == []
with pytest.raises(ValueError, match='非空'):
migrate_sqlite(tmp_path / 'snapshot.db', mysql)
def test_legacy_snapshot_without_catalog_page_table_can_still_be_migrated(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'legacy.db')
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
with source.connection() as conn:
conn.execute('DROP TABLE job_catalog_pages')
counts = migrate_sqlite(source.path, mysql)
assert counts['catalog'] == 2
assert counts['job_catalog_pages'] == 0
def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'bad.db')