154 lines
6.1 KiB
Python
154 lines
6.1 KiB
Python
from collections import Counter
|
|
|
|
import pytest
|
|
|
|
from studio.connectors import CatalogClient, LoginRequired
|
|
from studio.database import Database
|
|
from studio.jobs import JobManager
|
|
from studio.settings import SettingsStore
|
|
from studio.network import RemoteUnavailable
|
|
from tests.test_data import raw_catalog
|
|
from tests.test_network import Response
|
|
|
|
|
|
PARAMS = {'mode': 'catalog', 'year': 2027, 'provinces': ['11']}
|
|
|
|
|
|
class InterruptedCatalog:
|
|
def __init__(self, observe=None):
|
|
self.blocked = True
|
|
self.calls = Counter()
|
|
self.observe = observe
|
|
|
|
def request(self, method, url, **kwargs):
|
|
if method == 'GET':
|
|
return Response({}, text='2027年硕士专业目录查询')
|
|
endpoint = url.rsplit('/', 1)[-1]
|
|
school = kwargs['data'].get('dwdm', '')
|
|
self.calls[endpoint, school] += 1
|
|
if endpoint == 'dws.do':
|
|
rows = [{'dwdm': '10001', 'dwmc': '北京大学'}, {'dwdm': '10002', 'dwmc': '中国人民大学'}]
|
|
elif endpoint == 'dwzys.do':
|
|
if self.observe:
|
|
self.observe(school)
|
|
rows = [{'zydm': '081200', 'zymc': '计算机科学与技术', 'xxfs': '1'}]
|
|
else:
|
|
if school == '10002' and self.blocked:
|
|
return Response({'flag': False, 'msg': '请登录'})
|
|
rows = [{**raw_catalog(), 'dwdm': school}]
|
|
return Response({'flag': True, 'msg': {'list': rows, 'nextPageAvailable': False}})
|
|
|
|
|
|
def prepare(tmp_path, transport):
|
|
db = Database(tmp_path / 'studio.db')
|
|
db.job_create('resume', PARAMS)
|
|
return db, JobManager(db, SettingsStore(tmp_path), transport=transport)
|
|
|
|
|
|
def test_login_recovery_reuses_successful_catalog_pages_across_service_restart(tmp_path):
|
|
source = InterruptedCatalog()
|
|
db, manager = prepare(tmp_path, source)
|
|
with pytest.raises(LoginRequired):
|
|
manager._catalog('resume', PARAMS)
|
|
assert db.job_get('resume')['completed_units'] == 1
|
|
source.blocked = False
|
|
reopened = Database(tmp_path / 'studio.db')
|
|
JobManager(reopened, SettingsStore(tmp_path), transport=source)._catalog('resume', PARAMS)
|
|
assert reopened.job_get('resume')['completed_units'] == 2
|
|
assert source.calls['dws.do', ''] == 1
|
|
assert source.calls['dwzys.do', '10001'] == 1
|
|
assert source.calls['dwzys.do', '10002'] == 1
|
|
assert source.calls['yjfxs.do', '10002'] == 2 # Failed responses must never become cached successes.
|
|
|
|
|
|
def test_resume_checks_completed_units_in_memory_instead_of_one_query_per_major(tmp_path):
|
|
source = InterruptedCatalog()
|
|
source.blocked = False
|
|
db, manager = prepare(tmp_path, source)
|
|
for school in ('10001', '10002'):
|
|
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
|
|
queries = []
|
|
original = db.unit_done
|
|
def observe(jid, unit):
|
|
queries.append(unit)
|
|
return original(jid, unit)
|
|
db.unit_done = observe
|
|
manager._catalog('resume', PARAMS)
|
|
assert queries == []
|
|
assert not any(endpoint == 'yjfxs.do' for endpoint, school in source.calls)
|
|
|
|
|
|
def test_resume_displays_current_school_while_checking_its_major_list(tmp_path):
|
|
messages = {}
|
|
source = InterruptedCatalog(lambda school: messages.setdefault(school, db.job_get('resume')['message']))
|
|
source.blocked = False
|
|
db, manager = prepare(tmp_path, source)
|
|
for school in ('10001', '10002'):
|
|
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
|
|
manager._catalog('resume', PARAMS)
|
|
assert '北京大学' in messages['10001']
|
|
assert '中国人民大学' in messages['10002']
|
|
|
|
|
|
def test_cached_pages_do_not_cross_job_boundaries(tmp_path):
|
|
source = InterruptedCatalog()
|
|
source.blocked = False
|
|
db, manager = prepare(tmp_path, source)
|
|
manager._catalog('resume', PARAMS)
|
|
db.job_create('fresh', PARAMS)
|
|
manager._catalog('fresh', PARAMS)
|
|
assert source.calls['dws.do', ''] == 2
|
|
assert db.job_get('fresh')['completed_units'] == 2
|
|
|
|
|
|
def test_invalid_pagination_is_not_persisted_as_a_reusable_page():
|
|
class Source(InterruptedCatalog):
|
|
def request(self, method, url, **kwargs):
|
|
body = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': self.blocked, 'startOfNextPage': 0}
|
|
self.blocked = False
|
|
return Response({'flag': True, 'msg': body})
|
|
source, cache = Source(), {}
|
|
client = CatalogClient(source, page_cache=cache)
|
|
with pytest.raises(RemoteUnavailable, match='下一页位置无效'):
|
|
list(client.schools('11'))
|
|
assert cache == {}
|
|
assert list(client.schools('11')) == [{'dwdm': '10001'}]
|
|
|
|
|
|
def test_repeated_source_page_is_retried_live_after_recovery():
|
|
class Source(InterruptedCatalog):
|
|
def request(self, method, url, **kwargs):
|
|
page = kwargs['data']['curPage']
|
|
school = '10001' if page == 1 or self.blocked else '10002'
|
|
return Response({'flag': True, 'msg': {'list': [{'dwdm': school}], 'nextPageAvailable': page == 1}})
|
|
source, cache = Source(), {}
|
|
client = CatalogClient(source, page_cache=cache)
|
|
with pytest.raises(RemoteUnavailable, match='重复返回同一页'):
|
|
list(client.schools('11'))
|
|
assert len(cache) == 1
|
|
source.blocked = False
|
|
assert [row['dwdm'] for row in client.schools('11')] == ['10001', '10002']
|
|
|
|
|
|
@pytest.mark.parametrize('endpoint,broken_row', [
|
|
('dws', {'dwmc': '缺少代码的院校'}),
|
|
('dwzys', {'zymc': '缺少代码的专业'}),
|
|
('dws', 'unexpected row'),
|
|
('dwzys', {'zydm': ''}),
|
|
('dwzys', {'zydm': None}),
|
|
])
|
|
def test_malformed_list_page_is_retried_after_source_recovers(endpoint, broken_row):
|
|
identifier = 'dwdm' if endpoint == 'dws' else 'zydm'
|
|
good_row = {identifier: '10001' if endpoint == 'dws' else '081200'}
|
|
class Source(InterruptedCatalog):
|
|
def request(self, method, url, **kwargs):
|
|
row = broken_row if self.blocked else good_row
|
|
return Response({'flag': True, 'msg': {'list': [row], 'nextPageAvailable': False}})
|
|
source, cache = Source(), {}
|
|
client = CatalogClient(source, page_cache=cache)
|
|
with pytest.raises(RemoteUnavailable, match='代码'):
|
|
list(client.pages(endpoint, {}))
|
|
assert cache == {}
|
|
source.blocked = False
|
|
assert list(client.pages(endpoint, {})) == [good_row]
|