fix: resume catalog jobs without repeated checkpoint queries

This commit is contained in:
2026-10-11 21:18:49 +08:00
parent c4a71e88a1
commit 4c510759fb
9 changed files with 309 additions and 13 deletions
+153
View File
@@ -0,0 +1,153 @@
from collections import Counter
import pytest
from studio.connectors import CatalogClient, LoginRequired
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
from studio.network import RemoteUnavailable
from tests.test_data import raw_catalog
from tests.test_network import Response
PARAMS = {'mode': 'catalog', 'year': 2027, 'provinces': ['11']}
class InterruptedCatalog:
def __init__(self, observe=None):
self.blocked = True
self.calls = Counter()
self.observe = observe
def request(self, method, url, **kwargs):
if method == 'GET':
return Response({}, text='2027年硕士专业目录查询')
endpoint = url.rsplit('/', 1)[-1]
school = kwargs['data'].get('dwdm', '')
self.calls[endpoint, school] += 1
if endpoint == 'dws.do':
rows = [{'dwdm': '10001', 'dwmc': '北京大学'}, {'dwdm': '10002', 'dwmc': '中国人民大学'}]
elif endpoint == 'dwzys.do':
if self.observe:
self.observe(school)
rows = [{'zydm': '081200', 'zymc': '计算机科学与技术', 'xxfs': '1'}]
else:
if school == '10002' and self.blocked:
return Response({'flag': False, 'msg': '请登录'})
rows = [{**raw_catalog(), 'dwdm': school}]
return Response({'flag': True, 'msg': {'list': rows, 'nextPageAvailable': False}})
def prepare(tmp_path, transport):
db = Database(tmp_path / 'studio.db')
db.job_create('resume', PARAMS)
return db, JobManager(db, SettingsStore(tmp_path), transport=transport)
def test_login_recovery_reuses_successful_catalog_pages_across_service_restart(tmp_path):
source = InterruptedCatalog()
db, manager = prepare(tmp_path, source)
with pytest.raises(LoginRequired):
manager._catalog('resume', PARAMS)
assert db.job_get('resume')['completed_units'] == 1
source.blocked = False
reopened = Database(tmp_path / 'studio.db')
JobManager(reopened, SettingsStore(tmp_path), transport=source)._catalog('resume', PARAMS)
assert reopened.job_get('resume')['completed_units'] == 2
assert source.calls['dws.do', ''] == 1
assert source.calls['dwzys.do', '10001'] == 1
assert source.calls['dwzys.do', '10002'] == 1
assert source.calls['yjfxs.do', '10002'] == 2 # Failed responses must never become cached successes.
def test_resume_checks_completed_units_in_memory_instead_of_one_query_per_major(tmp_path):
source = InterruptedCatalog()
source.blocked = False
db, manager = prepare(tmp_path, source)
for school in ('10001', '10002'):
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
queries = []
original = db.unit_done
def observe(jid, unit):
queries.append(unit)
return original(jid, unit)
db.unit_done = observe
manager._catalog('resume', PARAMS)
assert queries == []
assert not any(endpoint == 'yjfxs.do' for endpoint, school in source.calls)
def test_resume_displays_current_school_while_checking_its_major_list(tmp_path):
messages = {}
source = InterruptedCatalog(lambda school: messages.setdefault(school, db.job_get('resume')['message']))
source.blocked = False
db, manager = prepare(tmp_path, source)
for school in ('10001', '10002'):
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
manager._catalog('resume', PARAMS)
assert '北京大学' in messages['10001']
assert '中国人民大学' in messages['10002']
def test_cached_pages_do_not_cross_job_boundaries(tmp_path):
source = InterruptedCatalog()
source.blocked = False
db, manager = prepare(tmp_path, source)
manager._catalog('resume', PARAMS)
db.job_create('fresh', PARAMS)
manager._catalog('fresh', PARAMS)
assert source.calls['dws.do', ''] == 2
assert db.job_get('fresh')['completed_units'] == 2
def test_invalid_pagination_is_not_persisted_as_a_reusable_page():
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
body = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': self.blocked, 'startOfNextPage': 0}
self.blocked = False
return Response({'flag': True, 'msg': body})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='下一页位置无效'):
list(client.schools('11'))
assert cache == {}
assert list(client.schools('11')) == [{'dwdm': '10001'}]
def test_repeated_source_page_is_retried_live_after_recovery():
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
page = kwargs['data']['curPage']
school = '10001' if page == 1 or self.blocked else '10002'
return Response({'flag': True, 'msg': {'list': [{'dwdm': school}], 'nextPageAvailable': page == 1}})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='重复返回同一页'):
list(client.schools('11'))
assert len(cache) == 1
source.blocked = False
assert [row['dwdm'] for row in client.schools('11')] == ['10001', '10002']
@pytest.mark.parametrize('endpoint,broken_row', [
('dws', {'dwmc': '缺少代码的院校'}),
('dwzys', {'zymc': '缺少代码的专业'}),
('dws', 'unexpected row'),
('dwzys', {'zydm': ''}),
('dwzys', {'zydm': None}),
])
def test_malformed_list_page_is_retried_after_source_recovers(endpoint, broken_row):
identifier = 'dwdm' if endpoint == 'dws' else 'zydm'
good_row = {identifier: '10001' if endpoint == 'dws' else '081200'}
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
row = broken_row if self.blocked else good_row
return Response({'flag': True, 'msg': {'list': [row], 'nextPageAvailable': False}})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='代码'):
list(client.pages(endpoint, {}))
assert cache == {}
source.blocked = False
assert list(client.pages(endpoint, {})) == [good_row]
+45 -1
View File
@@ -17,7 +17,7 @@ def mysql():
assert config['database'].endswith('_test'), 'refusing to clear a production database'
db = MySQLDatabase(config)
with db.connection() as conn:
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs'):
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs', 'job_catalog_pages'):
conn.execute(f'DELETE FROM {table}')
return db
@@ -68,6 +68,36 @@ def test_mysql_tasks_logs_and_checkpoints(mysql):
assert mysql.logs('contract-job')[0]['message'] == '数据库测试'
def test_mysql_catalog_page_cache_and_bulk_checkpoints_are_durable(mysql):
mysql.job_create('cached-job', {'year': 2027})
mysql.complete_unit('cached-job', 'catalog:11:10001:081200:1')
page = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': False}
mysql.cache_catalog_page('cached-job', 'page-key', page)
from studio.mysql_database import MySQLDatabase
reopened = MySQLDatabase(mysql.config, recover_jobs=False)
assert reopened.completed_units('cached-job') == {'catalog:11:10001:081200:1'}
assert reopened.catalog_pages('cached-job') == {'page-key': page}
assert reopened.catalog_pages('another-job') == {}
def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypatch):
import ssl
with mysql.connection() as conn:
cipher = conn.execute("SHOW SESSION STATUS LIKE 'Ssl_cipher'").fetchone()[1]
if not cipher:
pytest.skip('requires a TLS-enabled MariaDB test server')
creations = []
original = ssl.create_default_context
def observe(*args, **kwargs):
creations.append(True)
return original(*args, **kwargs)
monkeypatch.setattr(ssl, 'create_default_context', observe)
for _ in range(2):
with mysql.connection() as conn:
assert conn.execute('SELECT 1').fetchone()[0] == 1
assert creations == []
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'snapshot.db')
@@ -77,16 +107,30 @@ def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_p
source.job_create('migrated-job', {'year': 2027})
source.complete_unit('migrated-job', 'catalog:11:10001:081200:1')
source.log('migrated-job', 'info', 'preserve log')
source.cache_catalog_page('migrated-job', 'migration-page', {'list': [], 'nextPageAvailable': False})
counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql)
assert counts['catalog'] == 2 and counts['scores'] == 1
assert mysql.record(1)['favorite']
assert mysql.query_catalog({})['items'][0]['score_total'] == 320
assert mysql.job_get('migrated-job')['completed_units'] == 1
assert mysql.logs('migrated-job')[0]['id'] == 1
assert counts['job_catalog_pages'] == 1
assert mysql.catalog_pages('migrated-job')['migration-page']['list'] == []
with pytest.raises(ValueError, match='非空'):
migrate_sqlite(tmp_path / 'snapshot.db', mysql)
def test_legacy_snapshot_without_catalog_page_table_can_still_be_migrated(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'legacy.db')
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
with source.connection() as conn:
conn.execute('DROP TABLE job_catalog_pages')
counts = migrate_sqlite(source.path, mysql)
assert counts['catalog'] == 2
assert counts['job_catalog_pages'] == 0
def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'bad.db')