Fix historical pagination and bound export and proxy memory

This commit is contained in:
2026-10-13 00:31:34 +08:00
parent 4c510759fb
commit ec2c897752
20 changed files with 1050 additions and 87 deletions
+75
View File
@@ -0,0 +1,75 @@
import pytest
from studio.database import Database
from studio.domain import finish_record
def record(code='10001', name='甲大学', major='081200', province='北京', year=2027, direction='方向一'):
return finish_record({'school_code': code, 'school_name': name, 'major_code': major,
'major_name': major, 'province': province, 'year': year,
'direction': direction, 'subjects': ['101政治'], 'source': '测试'})
def test_history_targets_deduplicate_directions_and_preserve_filters(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_catalog([
record(direction='一'), record(direction='二'), record(major='081201'),
record(code='10002', name='乙大学', province='天津'),
record(code='10003', name='丙大学', province=''), record(year=2026),
])
rows = list(db.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
assert [(r['school_code'], r['major_code']) for r in rows] == [
('10001', '081200'), ('10001', '081201'), ('10003', '081200')]
assert all('data' not in row and 'subjects' not in row for row in rows)
rows = list(db.score_targets({'year': 2027, 'school_keyword': '甲', 'major_code': '081201'}))
assert [(r['school_code'], r['major_code']) for r in rows] == [('10001', '081201')]
def test_catalog_iterator_keeps_export_order_and_excludes_new_inserts(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_catalog([record(name='B大学'), record(name='A大学', direction='二'),
record(name='A大学', direction='三'), record(year=2026),
record(code='10002', name='C大学')])
iterator = db.iter_catalog({}, batch_size=2)
first = next(iterator)
db.upsert_catalog([record(name='Z大学', year=2028)])
rows = [first, *iterator]
assert [(r['year'], r['school_name']) for r in rows] == [
(2027, 'A大学'), (2027, 'A大学'), (2027, 'B大学'), (2027, 'C大学'), (2026, '甲大学')]
assert [r['school_name'] for r in db.iter_catalog({'school_keyword': 'A'}, batch_size=1)] == ['A大学', 'A大学']
def test_score_iterator_exports_only_selected_school_major_pairs(tmp_path):
db = Database(tmp_path / 'data.db')
db.upsert_scores([
{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310},
{'school_code': '10002', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 320},
])
rows = list(db.iter_scores({('10001', '081200')}, batch_size=1))
assert len(rows) == 1
assert rows[0]['total'] == 310
def test_unit_commit_is_idempotent_for_records_count_and_checkpoint(tmp_path):
db = Database(tmp_path / 'data.db')
db.job_create('job', {})
rows = [record(direction='一'), record(direction='二')]
assert db.save_unit('job', 'catalog:test', rows) is True
assert db.save_unit('job', 'catalog:test', rows) is False
assert db.overview()['records'] == 2
assert db.job_get('job')['records'] == 2
assert db.unit_done('job', 'catalog:test')
def test_failed_unit_transaction_rolls_back_scores_records_and_checkpoint(tmp_path, monkeypatch):
db = Database(tmp_path / 'data.db')
db.job_create('job', {})
def fail(*args):
raise RuntimeError('interrupted persistence')
monkeypatch.setattr(db, '_school', fail)
score = {'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}
with pytest.raises(RuntimeError):
db.save_unit('job', 'scores:test', [record()], scores=[score])
assert db.overview()['records'] == db.overview()['scores'] == 0
assert db.job_get('job')['records'] == 0
assert not db.unit_done('job', 'scores:test')
+42
View File
@@ -0,0 +1,42 @@
from studio import network
def test_proxy_managers_are_bounded_and_evicted_pools_are_closed():
adapter = network.BoundedProxyAdapter(max_proxy_pools=3)
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
closed = []
original = first.clear
def observe():
original()
closed.append(True)
first.clear = observe
for port in range(10002, 10006):
adapter.proxy_manager_for(f'http://127.0.0.1:{port}')
assert len(adapter.proxy_manager) <= 3
assert closed == [True]
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is not first
adapter.close()
def test_active_proxy_pool_is_reused_while_old_pools_are_evicted():
adapter = network.BoundedProxyAdapter(max_proxy_pools=2)
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
adapter.proxy_manager_for('http://127.0.0.1:10002')
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
adapter.proxy_manager_for('http://127.0.0.1:10003')
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
assert 'http://127.0.0.1:10002' not in adapter.proxy_manager
adapter.close()
def test_default_transport_limits_pools_without_changing_route_or_tls(tmp_path):
from studio.settings import SettingsStore
from tests.test_network import Pool
settings = SettingsStore(tmp_path)
router = network.ProxyRouter(settings, session=Pool())
transport = network.Transport(settings, router)
adapter = transport.session.get_adapter('https://example.test')
assert isinstance(adapter, network.BoundedProxyAdapter)
assert transport.session.verify is True
assert router.resolve() == 'http://1.2.3.4:8080'
transport.close()
+125
View File
@@ -0,0 +1,125 @@
import asyncio
import csv
import io
import threading
import pytest
from fastapi.testclient import TestClient
from openpyxl import load_workbook
from studio.api import create_app
from tests.test_bounded_storage import record
@pytest.mark.parametrize('kind', ['csv', 'json', 'xlsx'])
def test_export_uses_bounded_rows_and_preserves_content(tmp_path, monkeypatch, kind):
app = create_app(tmp_path)
app.state.db.upsert_catalog([record(name='=1+1'), record(code='10002', name='含逗号,大学')])
app.state.db.upsert_scores([{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}])
def forbidden(*args, **kwargs):
raise AssertionError('export materializes all catalog rows')
monkeypatch.setattr(app.state.db, 'all_catalog', forbidden)
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
response = client.get('/api/export', params={'format': kind})
assert response.status_code == 200
if kind == 'json':
data = response.json()
assert {r['school_name'] for r in data['catalog']} == {'=1+1', '含逗号,大学'}
assert data['scores'][0]['total'] == 310
elif kind == 'csv':
rows = list(csv.reader(io.StringIO(response.content.decode('utf-8-sig'))))
assert len(rows) == 3
assert any("'=1+1" in row for row in rows)
assert any('含逗号,大学' in row for row in rows)
else:
book = load_workbook(io.BytesIO(response.content), read_only=True)
rows = list(book.active.values)
assert len(rows) == 3
assert any("'=1+1" in row for row in rows)
book.close()
assert not list((tmp_path / 'exports').glob('*'))
def test_concurrent_export_is_rejected_and_lock_is_released(tmp_path, monkeypatch):
app = create_app(tmp_path)
entered, release = threading.Event(), threading.Event()
original = app.state.db.iter_catalog
def slow_rows(*args, **kwargs):
if not entered.is_set():
entered.set()
assert release.wait(5)
yield from original(*args, **kwargs)
monkeypatch.setattr(app.state.db, 'iter_catalog', slow_rows)
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
responses = []
thread = threading.Thread(target=lambda: responses.append(client.get('/api/export')))
thread.start()
assert entered.wait(3)
blocked = client.get('/api/export', params={'format': 'json'})
release.set()
thread.join(5)
assert blocked.status_code == 429
assert responses[0].status_code == 200
assert client.get('/api/export').status_code == 200
def test_disconnected_stream_closes_source_and_releases_export_lock():
from studio.exporting import ExportStream
from starlette.requests import ClientDisconnect
lock = threading.BoundedSemaphore(1)
assert lock.acquire(False)
closed = []
def source():
try:
yield b'data'
finally:
closed.append(True)
async def send(message):
if message['type'] == 'http.response.body':
raise OSError('client disconnected')
async def receive():
return {'type': 'http.disconnect'}
response = ExportStream(source(), lock.release, media_type='text/csv')
with pytest.raises(ClientDisconnect):
asyncio.run(response({'type': 'http', 'asgi': {'spec_version': '2.4'}}, receive, send))
assert closed == [True]
assert lock.acquire(False)
def test_xlsx_packaging_error_removes_xml_tempfile_without_masking_error(tmp_path, monkeypatch):
from pathlib import Path
from studio.imports import write_xlsx
from openpyxl import Workbook
paths = []
def packaging_failure(book, filename):
sheet = book.worksheets[0]
sheet.close()
paths.append(Path(sheet._writer.out))
raise OSError('archive write failed')
monkeypatch.setattr(Workbook, 'save', packaging_failure)
with pytest.raises(OSError, match='archive write failed'):
write_xlsx([record()], tmp_path / 'failed.xlsx')
assert paths and not paths[0].exists()
def test_failed_export_batch_does_not_finish_a_successful_truncated_download(tmp_path, monkeypatch):
app = create_app(tmp_path)
def broken_rows(*args, **kwargs):
yield record()
raise RuntimeError('database batch failed')
monkeypatch.setattr(app.state.db, 'iter_catalog', broken_rows)
sent = []
async def send(message):
sent.append(message)
async def receive():
await asyncio.Event().wait()
scope = {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'},
'http_version': '1.1', 'method': 'GET', 'scheme': 'http',
'path': '/api/export', 'raw_path': b'/api/export', 'root_path': '',
'query_string': b'format=csv', 'headers': [(b'host', b'127.0.0.1:8765')],
'server': ('127.0.0.1', 8765), 'client': ('127.0.0.1', 12345)}
with pytest.raises(RuntimeError, match='database batch failed'):
asyncio.run(app(scope, receive, send))
assert not any(message['type'] == 'http.response.body' and not message.get('more_body', False) for message in sent)
+147
View File
@@ -0,0 +1,147 @@
import pytest
from studio.connectors import ScoreClient
from studio.network import RemoteUnavailable
from tests.test_network import Response
class HistorySource:
def __init__(self, pages):
self.pages = pages
self.calls = []
def request(self, method, url, **kwargs):
payload = kwargs['data']
self.calls.append((url.rsplit('/', 1)[-1], payload.copy()))
data = self.pages(payload.get('page', 1))
return Response({'code': '0000', 'data': data})
def test_full_score_array_over_page_limit_is_consumed_once():
rows = [{'code': '081200', 'total': 310, 'depart_name': '信息学院'}]
rows += [{'code': '999999', 'total': 200} for _ in range(197)]
source = HistorySource(lambda page: rows)
lines = ScoreClient(source).score_lines({'school_code': '10001', 'school_name': '北京大学'}, '081200', 2025, 1)
assert len(lines) == 1
assert lines[0]['total'] == 310
assert [call[1]['page'] for call in source.calls] == [1]
def test_short_wrapped_pages_follow_explicit_total():
source = HistorySource(lambda page: {'data': [{'value': page}], 'total': 2})
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
def test_short_wrapped_pages_follow_explicit_last_page():
source = HistorySource(lambda page: {'list': [{'value': page}], 'last_page': 2})
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
@pytest.mark.parametrize('data', [None, {}, {'data': None}, {'data': [None]}, {'list': [42]}])
def test_invalid_history_data_cannot_become_a_successful_empty_unit(data):
with pytest.raises(RemoteUnavailable):
list(ScoreClient(HistorySource(lambda page: data)).pages('/example', {}))
def test_empty_full_array_is_a_valid_empty_result():
assert list(ScoreClient(HistorySource(lambda page: [])).pages('/example', {})) == []
def test_repeated_real_page_still_raises_instead_of_returning_partial_data():
source = HistorySource(lambda page: {'data': [{'value': 1}], 'total': 2})
with pytest.raises(RemoteUnavailable):
list(ScoreClient(source).pages('/example', {}))
def test_empty_page_before_declared_total_is_an_error():
source = HistorySource(lambda page: {'data': [{'value': 1}] if page == 1 else [], 'total': 2})
with pytest.raises(RemoteUnavailable):
list(ScoreClient(source).pages('/example', {}))
def test_two_majors_reuse_one_complete_school_year_score_table():
rows = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
source = HistorySource(lambda page: rows)
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
assert client.score_lines(school, '081201', 2025, 1)[0]['total'] == 320
assert len(source.calls) == 1
def test_score_cache_separates_schools_and_years_and_has_a_bound():
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}])
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
for sid in range(20):
client.score_lines(school, '081200', 2025, sid)
client.score_lines(school, '081200', 2026, 19)
before = len(source.calls)
client.score_lines(school, '081200', 2026, 19)
assert len(source.calls) == before
client.score_lines(school, '081200', 2025, 0)
assert len(source.calls) == before + 1
def test_failed_lookup_is_cached_for_the_current_client_only():
source = HistorySource(lambda page: {'data': [], 'total': 0})
client = ScoreClient(source)
assert client.school_id('暂未收录大学') is None
assert client.school_id('暂未收录大学') is None
assert len(source.calls) == 1
assert ScoreClient(source).school_id('暂未收录大学') is None
assert len(source.calls) == 2
def test_invalid_score_response_is_not_cached():
valid = False
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}] if valid else {})
client = ScoreClient(source)
school = {'school_code': '10001', 'school_name': '北京大学'}
with pytest.raises(RemoteUnavailable):
client.score_lines(school, '081200', 2025, 1)
valid = True
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
def test_history_runner_uses_thin_targets_and_bulk_checkpoints(tmp_path, monkeypatch):
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
from tests.test_bounded_storage import record
db = Database(tmp_path / 'data.db')
db.upsert_catalog([record(), record(major='081201'), record(direction='二')])
params = {'year': 2027, 'provinces': ['11'], 'score_years': [2025, 2026], 'include_plans': True}
db.job_create('history', params)
class Source:
def __init__(self):
self.calls = []
def request(self, method, url, **kwargs):
payload = kwargs['data']
self.calls.append(url.rsplit('/', 1)[-1])
if url.endswith('/schoolList'):
data = {'data': [{'school_name': '甲大学', 'school_id': 1}], 'total': 1}
elif url.endswith('/schoolScore'):
data = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
else:
data = {'data': [{'special_code': payload['keyword'], 'special_name': payload['keyword'],
'plan_id': str(payload['year']) + payload['keyword'], 'recruit_number': 1}], 'total': 1}
return Response({'code': '0000', 'data': data})
def forbidden(*args, **kwargs):
raise AssertionError('full catalog decoding or per-unit checkpoint query')
monkeypatch.setattr(db, 'all_catalog', forbidden)
monkeypatch.setattr(db, 'unit_done', forbidden)
source = Source()
manager = JobManager(db, SettingsStore(tmp_path), transport=source)
manager._scores('history', params)
assert db.job_get('history')['completed_units'] == 4
assert db.overview()['scores'] == 4
assert db.job_get('history')['records'] == 4
assert source.calls.count('schoolScore') == 2
assert source.calls.count('planListV2') == 4
before = len(source.calls)
manager._scores('history', params)
assert len(source.calls) == before
+41
View File
@@ -98,6 +98,47 @@ def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypa
assert creations == []
def test_mysql_bounded_catalog_and_history_targets_preserve_filters(mysql):
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
scores = select_score_lines([{'code': '081200', 'total': 310, 'depart_name': '计算机学院'}], '081200', '10001', '北京大学', 2026)
mysql.upsert_scores(scores)
targets = list(mysql.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
assert len(targets) == 1
assert targets[0]['major_code'] == '081200'
assert 'data' not in targets[0]
rows = list(mysql.iter_catalog({'year': 2027}, batch_size=1))
assert len(rows) == 2
assert len(list(mysql.iter_catalog({'year': 2026, 'max_score': 310}, batch_size=1))) == 2
assert len(list(mysql.iter_scores({('10001', '081200')}, batch_size=1))) == 1
def test_mysql_unit_commit_is_atomic_and_idempotent(mysql):
mysql.job_create('atomic-job', {})
rows = normalize_catalog(raw_catalog(), 2027)
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
assert mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert not mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert mysql.job_get('atomic-job')['records'] == 2
assert mysql.job_get('atomic-job')['completed_units'] == 1
assert mysql.overview()['records'] == 2
assert mysql.overview()['scores'] == 1
def test_mysql_failed_unit_rolls_back_every_write(mysql, monkeypatch):
mysql.job_create('atomic-job', {})
def fail(*args):
raise RuntimeError('interrupted unit')
monkeypatch.setattr(mysql, '_school', fail)
rows = normalize_catalog(raw_catalog(), 2027)
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
with pytest.raises(RuntimeError):
mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
assert mysql.job_get('atomic-job')['records'] == 0
assert mysql.job_get('atomic-job')['completed_units'] == 0
assert mysql.overview()['records'] == mysql.overview()['scores'] == 0
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'snapshot.db')
+98
View File
@@ -0,0 +1,98 @@
import importlib.util
import queue
import threading
import time
from pathlib import Path
def maintainer_class():
path = Path(__file__).resolve().parents[1] / 'deploy/proxy_scheduler.py'
spec = importlib.util.spec_from_file_location('bounded_proxy_scheduler', path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module.ProxyMaintainer
def test_detection_workers_are_bounded_reused_and_rounds_do_not_overlap():
state = {'active': 0, 'peak': 0, 'types': set(), 'mixed': False, 'done': [], 'thread_ids': set()}
lock = threading.Lock()
class Checker:
def __init__(self, work_type, target_queue, name):
self.work_type, self.queue = work_type, target_queue
def run(self):
while True:
try:
item = self.queue.get(block=False)
except queue.Empty:
return
with lock:
state['active'] += 1
state['peak'] = max(state['peak'], state['active'])
state['types'].add(self.work_type)
state['mixed'] |= len(state['types']) > 1
state['thread_ids'].add(threading.get_ident())
time.sleep(.01)
with lock:
state['done'].append(item)
state['active'] -= 1
if not state['active']:
state['types'].clear()
self.queue.task_done()
maintainer = maintainer_class()(workers=2, checker_class=Checker)
first = threading.Thread(target=maintainer.validate, args=('raw', range(10)))
second = threading.Thread(target=maintainer.validate, args=('use', range(10, 20)))
first.start(); second.start()
first.join(4); second.join(4)
assert not first.is_alive() and not second.is_alive()
before = set(state['thread_ids'])
maintainer.validate('raw', range(20, 25))
maintainer.close()
assert sorted(state['done']) == list(range(25))
assert state['peak'] <= 2
assert state['mixed'] is False
assert state['thread_ids'] == before
def test_stopped_detector_does_not_begin_new_rounds():
class Forbidden:
def __init__(self, *args):
raise AssertionError('detection began after shutdown')
maintainer = maintainer_class()(workers=2, checker_class=Forbidden)
maintainer.close()
maintainer.validate('raw', [1, 2, 3])
def test_failed_worker_waits_for_other_workers_before_next_round():
slow_started, release, next_round = threading.Event(), threading.Event(), threading.Event()
class Checker:
def __init__(self, work_type, target_queue, name):
self.kind, self.name = work_type, name
def run(self):
if self.kind == 'raw' and self.name == 'worker_00':
assert slow_started.wait(2)
raise RuntimeError('validator failed')
if self.kind == 'raw':
slow_started.set()
assert release.wait(3)
maintainer = maintainer_class()(workers=2, checker_class=Checker)
failures = []
def first_round():
try:
maintainer.validate('raw', [1, 2])
except RuntimeError:
failures.append(True)
first = threading.Thread(target=first_round)
first.start()
assert slow_started.wait(2)
def second_proxies():
next_round.set()
yield 3
second = threading.Thread(target=maintainer.validate, args=('use', second_proxies()))
second.start()
began_early = next_round.wait(.15)
release.set()
first.join(3); second.join(3)
maintainer.close()
assert began_early is False
assert next_round.is_set()
assert failures == [True]