Fix historical pagination and bound export and proxy memory

This commit is contained in:
2026-10-13 00:31:34 +08:00
parent 4c510759fb
commit ec2c897752
20 changed files with 1050 additions and 87 deletions
+147
View File
@@ -0,0 +1,147 @@
import pytest
from studio.connectors import ScoreClient
from studio.network import RemoteUnavailable
from tests.test_network import Response
class HistorySource:
def __init__(self, pages):
self.pages = pages
self.calls = []
def request(self, method, url, **kwargs):
payload = kwargs['data']
self.calls.append((url.rsplit('/', 1)[-1], payload.copy()))
data = self.pages(payload.get('page', 1))
return Response({'code': '0000', 'data': data})
def test_full_score_array_over_page_limit_is_consumed_once():
rows = [{'code': '081200', 'total': 310, 'depart_name': '信息学院'}]
rows += [{'code': '999999', 'total': 200} for _ in range(197)]
source = HistorySource(lambda page: rows)
lines = ScoreClient(source).score_lines({'school_code': '10001', 'school_name': '北京大学'}, '081200', 2025, 1)
assert len(lines) == 1
assert lines[0]['total'] == 310
assert [call[1]['page'] for call in source.calls] == [1]
def test_short_wrapped_pages_follow_explicit_total():
source = HistorySource(lambda page: {'data': [{'value': page}], 'total': 2})
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
def test_short_wrapped_pages_follow_explicit_last_page():
source = HistorySource(lambda page: {'list': [{'value': page}], 'last_page': 2})
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
@pytest.mark.parametrize('data', [None, {}, {'data': None}, {'data': [None]}, {'list': [42]}])
def test_invalid_history_data_cannot_become_a_successful_empty_unit(data):
with pytest.raises(RemoteUnavailable):
list(ScoreClient(HistorySource(lambda page: data)).pages('/example', {}))
def test_empty_full_array_is_a_valid_empty_result():
assert list(ScoreClient(HistorySource(lambda page: [])).pages('/example', {})) == []
def test_repeated_real_page_still_raises_instead_of_returning_partial_data():
source = HistorySource(lambda page: {'data': [{'value': 1}], 'total': 2})
with pytest.raises(RemoteUnavailable):
list(ScoreClient(source).pages('/example', {}))
def test_empty_page_before_declared_total_is_an_error():
source = HistorySource(lambda page: {'data': [{'value': 1}] if page == 1 else [], 'total': 2})
with pytest.raises(RemoteUnavailable):
list(ScoreClient(source).pages('/example', {}))
def test_two_majors_reuse_one_complete_school_year_score_table():
rows = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
source = HistorySource(lambda page: rows)
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
assert client.score_lines(school, '081201', 2025, 1)[0]['total'] == 320
assert len(source.calls) == 1
def test_score_cache_separates_schools_and_years_and_has_a_bound():
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}])
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
for sid in range(20):
client.score_lines(school, '081200', 2025, sid)
client.score_lines(school, '081200', 2026, 19)
before = len(source.calls)
client.score_lines(school, '081200', 2026, 19)
assert len(source.calls) == before
client.score_lines(school, '081200', 2025, 0)
assert len(source.calls) == before + 1
def test_failed_lookup_is_cached_for_the_current_client_only():
source = HistorySource(lambda page: {'data': [], 'total': 0})
client = ScoreClient(source)
assert client.school_id('暂未收录大学') is None
assert client.school_id('暂未收录大学') is None
assert len(source.calls) == 1
assert ScoreClient(source).school_id('暂未收录大学') is None
assert len(source.calls) == 2
def test_invalid_score_response_is_not_cached():
valid = False
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}] if valid else {})
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
with pytest.raises(RemoteUnavailable):
client.score_lines(school, '081200', 2025, 1)
valid = True
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
def test_history_runner_uses_thin_targets_and_bulk_checkpoints(tmp_path, monkeypatch):
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
from tests.test_bounded_storage import record
db = Database(tmp_path / 'data.db')
db.upsert_catalog([record(), record(major='081201'), record(direction='二')])
params = {'year': 2027, 'provinces': ['11'], 'score_years': [2025, 2026], 'include_plans': True}
db.job_create('history', params)
class Source:
def __init__(self):
self.calls = []
def request(self, method, url, **kwargs):
payload = kwargs['data']
self.calls.append(url.rsplit('/', 1)[-1])
if url.endswith('/schoolList'):
data = {'data': [{'school_name': '甲大学', 'school_id': 1}], 'total': 1}
elif url.endswith('/schoolScore'):
data = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
else:
data = {'data': [{'special_code': payload['keyword'], 'special_name': payload['keyword'],
'plan_id': str(payload['year']) + payload['keyword'], 'recruit_number': 1}], 'total': 1}
return Response({'code': '0000', 'data': data})
def forbidden(*args, **kwargs):
raise AssertionError('full catalog decoding or per-unit checkpoint query')
monkeypatch.setattr(db, 'all_catalog', forbidden)
monkeypatch.setattr(db, 'unit_done', forbidden)
source = Source()
manager = JobManager(db, SettingsStore(tmp_path), transport=source)
manager._scores('history', params)
assert db.job_get('history')['completed_units'] == 4
assert db.overview()['scores'] == 4
assert db.job_get('history')['records'] == 4
assert source.calls.count('schoolScore') == 2
assert source.calls.count('planListV2') == 4
before = len(source.calls)
manager._scores('history', params)
assert len(source.calls) == before