Fix historical pagination and bound export and proxy memory
This commit is contained in:
@@ -0,0 +1,75 @@
|
||||
import pytest
|
||||
|
||||
from studio.database import Database
|
||||
from studio.domain import finish_record
|
||||
|
||||
|
||||
def record(code='10001', name='甲大学', major='081200', province='北京', year=2027, direction='方向一'):
|
||||
return finish_record({'school_code': code, 'school_name': name, 'major_code': major,
|
||||
'major_name': major, 'province': province, 'year': year,
|
||||
'direction': direction, 'subjects': ['101政治'], 'source': '测试'})
|
||||
|
||||
|
||||
def test_history_targets_deduplicate_directions_and_preserve_filters(tmp_path):
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.upsert_catalog([
|
||||
record(direction='一'), record(direction='二'), record(major='081201'),
|
||||
record(code='10002', name='乙大学', province='天津'),
|
||||
record(code='10003', name='丙大学', province=''), record(year=2026),
|
||||
])
|
||||
rows = list(db.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
|
||||
assert [(r['school_code'], r['major_code']) for r in rows] == [
|
||||
('10001', '081200'), ('10001', '081201'), ('10003', '081200')]
|
||||
assert all('data' not in row and 'subjects' not in row for row in rows)
|
||||
rows = list(db.score_targets({'year': 2027, 'school_keyword': '甲', 'major_code': '081201'}))
|
||||
assert [(r['school_code'], r['major_code']) for r in rows] == [('10001', '081201')]
|
||||
|
||||
|
||||
def test_catalog_iterator_keeps_export_order_and_excludes_new_inserts(tmp_path):
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.upsert_catalog([record(name='B大学'), record(name='A大学', direction='二'),
|
||||
record(name='A大学', direction='三'), record(year=2026),
|
||||
record(code='10002', name='C大学')])
|
||||
iterator = db.iter_catalog({}, batch_size=2)
|
||||
first = next(iterator)
|
||||
db.upsert_catalog([record(name='Z大学', year=2028)])
|
||||
rows = [first, *iterator]
|
||||
assert [(r['year'], r['school_name']) for r in rows] == [
|
||||
(2027, 'A大学'), (2027, 'A大学'), (2027, 'B大学'), (2027, 'C大学'), (2026, '甲大学')]
|
||||
assert [r['school_name'] for r in db.iter_catalog({'school_keyword': 'A'}, batch_size=1)] == ['A大学', 'A大学']
|
||||
|
||||
|
||||
def test_score_iterator_exports_only_selected_school_major_pairs(tmp_path):
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.upsert_scores([
|
||||
{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310},
|
||||
{'school_code': '10002', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 320},
|
||||
])
|
||||
rows = list(db.iter_scores({('10001', '081200')}, batch_size=1))
|
||||
assert len(rows) == 1
|
||||
assert rows[0]['total'] == 310
|
||||
|
||||
|
||||
def test_unit_commit_is_idempotent_for_records_count_and_checkpoint(tmp_path):
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.job_create('job', {})
|
||||
rows = [record(direction='一'), record(direction='二')]
|
||||
assert db.save_unit('job', 'catalog:test', rows) is True
|
||||
assert db.save_unit('job', 'catalog:test', rows) is False
|
||||
assert db.overview()['records'] == 2
|
||||
assert db.job_get('job')['records'] == 2
|
||||
assert db.unit_done('job', 'catalog:test')
|
||||
|
||||
|
||||
def test_failed_unit_transaction_rolls_back_scores_records_and_checkpoint(tmp_path, monkeypatch):
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.job_create('job', {})
|
||||
def fail(*args):
|
||||
raise RuntimeError('interrupted persistence')
|
||||
monkeypatch.setattr(db, '_school', fail)
|
||||
score = {'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}
|
||||
with pytest.raises(RuntimeError):
|
||||
db.save_unit('job', 'scores:test', [record()], scores=[score])
|
||||
assert db.overview()['records'] == db.overview()['scores'] == 0
|
||||
assert db.job_get('job')['records'] == 0
|
||||
assert not db.unit_done('job', 'scores:test')
|
||||
@@ -0,0 +1,42 @@
|
||||
from studio import network
|
||||
|
||||
|
||||
def test_proxy_managers_are_bounded_and_evicted_pools_are_closed():
|
||||
adapter = network.BoundedProxyAdapter(max_proxy_pools=3)
|
||||
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
|
||||
closed = []
|
||||
original = first.clear
|
||||
def observe():
|
||||
original()
|
||||
closed.append(True)
|
||||
first.clear = observe
|
||||
for port in range(10002, 10006):
|
||||
adapter.proxy_manager_for(f'http://127.0.0.1:{port}')
|
||||
assert len(adapter.proxy_manager) <= 3
|
||||
assert closed == [True]
|
||||
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is not first
|
||||
adapter.close()
|
||||
|
||||
|
||||
def test_active_proxy_pool_is_reused_while_old_pools_are_evicted():
|
||||
adapter = network.BoundedProxyAdapter(max_proxy_pools=2)
|
||||
first = adapter.proxy_manager_for('http://127.0.0.1:10001')
|
||||
adapter.proxy_manager_for('http://127.0.0.1:10002')
|
||||
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
|
||||
adapter.proxy_manager_for('http://127.0.0.1:10003')
|
||||
assert adapter.proxy_manager_for('http://127.0.0.1:10001') is first
|
||||
assert 'http://127.0.0.1:10002' not in adapter.proxy_manager
|
||||
adapter.close()
|
||||
|
||||
|
||||
def test_default_transport_limits_pools_without_changing_route_or_tls(tmp_path):
|
||||
from studio.settings import SettingsStore
|
||||
from tests.test_network import Pool
|
||||
settings = SettingsStore(tmp_path)
|
||||
router = network.ProxyRouter(settings, session=Pool())
|
||||
transport = network.Transport(settings, router)
|
||||
adapter = transport.session.get_adapter('https://example.test')
|
||||
assert isinstance(adapter, network.BoundedProxyAdapter)
|
||||
assert transport.session.verify is True
|
||||
assert router.resolve() == 'http://1.2.3.4:8080'
|
||||
transport.close()
|
||||
@@ -0,0 +1,125 @@
|
||||
import asyncio
|
||||
import csv
|
||||
import io
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from studio.api import create_app
|
||||
from tests.test_bounded_storage import record
|
||||
|
||||
|
||||
@pytest.mark.parametrize('kind', ['csv', 'json', 'xlsx'])
|
||||
def test_export_uses_bounded_rows_and_preserves_content(tmp_path, monkeypatch, kind):
|
||||
app = create_app(tmp_path)
|
||||
app.state.db.upsert_catalog([record(name='=1+1'), record(code='10002', name='含逗号,大学')])
|
||||
app.state.db.upsert_scores([{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}])
|
||||
def forbidden(*args, **kwargs):
|
||||
raise AssertionError('export materializes all catalog rows')
|
||||
monkeypatch.setattr(app.state.db, 'all_catalog', forbidden)
|
||||
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
|
||||
response = client.get('/api/export', params={'format': kind})
|
||||
assert response.status_code == 200
|
||||
if kind == 'json':
|
||||
data = response.json()
|
||||
assert {r['school_name'] for r in data['catalog']} == {'=1+1', '含逗号,大学'}
|
||||
assert data['scores'][0]['total'] == 310
|
||||
elif kind == 'csv':
|
||||
rows = list(csv.reader(io.StringIO(response.content.decode('utf-8-sig'))))
|
||||
assert len(rows) == 3
|
||||
assert any("'=1+1" in row for row in rows)
|
||||
assert any('含逗号,大学' in row for row in rows)
|
||||
else:
|
||||
book = load_workbook(io.BytesIO(response.content), read_only=True)
|
||||
rows = list(book.active.values)
|
||||
assert len(rows) == 3
|
||||
assert any("'=1+1" in row for row in rows)
|
||||
book.close()
|
||||
assert not list((tmp_path / 'exports').glob('*'))
|
||||
|
||||
|
||||
def test_concurrent_export_is_rejected_and_lock_is_released(tmp_path, monkeypatch):
|
||||
app = create_app(tmp_path)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original = app.state.db.iter_catalog
|
||||
def slow_rows(*args, **kwargs):
|
||||
if not entered.is_set():
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
yield from original(*args, **kwargs)
|
||||
monkeypatch.setattr(app.state.db, 'iter_catalog', slow_rows)
|
||||
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
|
||||
responses = []
|
||||
thread = threading.Thread(target=lambda: responses.append(client.get('/api/export')))
|
||||
thread.start()
|
||||
assert entered.wait(3)
|
||||
blocked = client.get('/api/export', params={'format': 'json'})
|
||||
release.set()
|
||||
thread.join(5)
|
||||
assert blocked.status_code == 429
|
||||
assert responses[0].status_code == 200
|
||||
assert client.get('/api/export').status_code == 200
|
||||
|
||||
|
||||
def test_disconnected_stream_closes_source_and_releases_export_lock():
|
||||
from studio.exporting import ExportStream
|
||||
from starlette.requests import ClientDisconnect
|
||||
lock = threading.BoundedSemaphore(1)
|
||||
assert lock.acquire(False)
|
||||
|
||||
|
||||
closed = []
|
||||
def source():
|
||||
try:
|
||||
yield b'data'
|
||||
finally:
|
||||
closed.append(True)
|
||||
async def send(message):
|
||||
if message['type'] == 'http.response.body':
|
||||
raise OSError('client disconnected')
|
||||
async def receive():
|
||||
return {'type': 'http.disconnect'}
|
||||
response = ExportStream(source(), lock.release, media_type='text/csv')
|
||||
with pytest.raises(ClientDisconnect):
|
||||
asyncio.run(response({'type': 'http', 'asgi': {'spec_version': '2.4'}}, receive, send))
|
||||
assert closed == [True]
|
||||
assert lock.acquire(False)
|
||||
|
||||
|
||||
def test_xlsx_packaging_error_removes_xml_tempfile_without_masking_error(tmp_path, monkeypatch):
|
||||
from pathlib import Path
|
||||
from studio.imports import write_xlsx
|
||||
from openpyxl import Workbook
|
||||
paths = []
|
||||
def packaging_failure(book, filename):
|
||||
sheet = book.worksheets[0]
|
||||
sheet.close()
|
||||
paths.append(Path(sheet._writer.out))
|
||||
raise OSError('archive write failed')
|
||||
monkeypatch.setattr(Workbook, 'save', packaging_failure)
|
||||
with pytest.raises(OSError, match='archive write failed'):
|
||||
write_xlsx([record()], tmp_path / 'failed.xlsx')
|
||||
assert paths and not paths[0].exists()
|
||||
|
||||
|
||||
def test_failed_export_batch_does_not_finish_a_successful_truncated_download(tmp_path, monkeypatch):
|
||||
app = create_app(tmp_path)
|
||||
def broken_rows(*args, **kwargs):
|
||||
yield record()
|
||||
raise RuntimeError('database batch failed')
|
||||
monkeypatch.setattr(app.state.db, 'iter_catalog', broken_rows)
|
||||
sent = []
|
||||
async def send(message):
|
||||
sent.append(message)
|
||||
async def receive():
|
||||
await asyncio.Event().wait()
|
||||
scope = {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'},
|
||||
'http_version': '1.1', 'method': 'GET', 'scheme': 'http',
|
||||
'path': '/api/export', 'raw_path': b'/api/export', 'root_path': '',
|
||||
'query_string': b'format=csv', 'headers': [(b'host', b'127.0.0.1:8765')],
|
||||
'server': ('127.0.0.1', 8765), 'client': ('127.0.0.1', 12345)}
|
||||
with pytest.raises(RuntimeError, match='database batch failed'):
|
||||
asyncio.run(app(scope, receive, send))
|
||||
assert not any(message['type'] == 'http.response.body' and not message.get('more_body', False) for message in sent)
|
||||
@@ -0,0 +1,147 @@
|
||||
import pytest
|
||||
|
||||
from studio.connectors import ScoreClient
|
||||
from studio.network import RemoteUnavailable
|
||||
from tests.test_network import Response
|
||||
|
||||
|
||||
class HistorySource:
|
||||
def __init__(self, pages):
|
||||
self.pages = pages
|
||||
self.calls = []
|
||||
|
||||
def request(self, method, url, **kwargs):
|
||||
payload = kwargs['data']
|
||||
self.calls.append((url.rsplit('/', 1)[-1], payload.copy()))
|
||||
data = self.pages(payload.get('page', 1))
|
||||
return Response({'code': '0000', 'data': data})
|
||||
|
||||
|
||||
def test_full_score_array_over_page_limit_is_consumed_once():
|
||||
rows = [{'code': '081200', 'total': 310, 'depart_name': '信息学院'}]
|
||||
rows += [{'code': '999999', 'total': 200} for _ in range(197)]
|
||||
source = HistorySource(lambda page: rows)
|
||||
lines = ScoreClient(source).score_lines({'school_code': '10001', 'school_name': '北京大学'}, '081200', 2025, 1)
|
||||
assert len(lines) == 1
|
||||
assert lines[0]['total'] == 310
|
||||
assert [call[1]['page'] for call in source.calls] == [1]
|
||||
|
||||
|
||||
def test_short_wrapped_pages_follow_explicit_total():
|
||||
source = HistorySource(lambda page: {'data': [{'value': page}], 'total': 2})
|
||||
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
|
||||
|
||||
|
||||
def test_short_wrapped_pages_follow_explicit_last_page():
|
||||
source = HistorySource(lambda page: {'list': [{'value': page}], 'last_page': 2})
|
||||
assert list(ScoreClient(source).pages('/example', {})) == [{'value': 1}, {'value': 2}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('data', [None, {}, {'data': None}, {'data': [None]}, {'list': [42]}])
|
||||
def test_invalid_history_data_cannot_become_a_successful_empty_unit(data):
|
||||
with pytest.raises(RemoteUnavailable):
|
||||
list(ScoreClient(HistorySource(lambda page: data)).pages('/example', {}))
|
||||
|
||||
|
||||
def test_empty_full_array_is_a_valid_empty_result():
|
||||
assert list(ScoreClient(HistorySource(lambda page: [])).pages('/example', {})) == []
|
||||
|
||||
|
||||
def test_repeated_real_page_still_raises_instead_of_returning_partial_data():
|
||||
source = HistorySource(lambda page: {'data': [{'value': 1}], 'total': 2})
|
||||
with pytest.raises(RemoteUnavailable):
|
||||
list(ScoreClient(source).pages('/example', {}))
|
||||
|
||||
|
||||
def test_empty_page_before_declared_total_is_an_error():
|
||||
source = HistorySource(lambda page: {'data': [{'value': 1}] if page == 1 else [], 'total': 2})
|
||||
with pytest.raises(RemoteUnavailable):
|
||||
list(ScoreClient(source).pages('/example', {}))
|
||||
|
||||
|
||||
def test_two_majors_reuse_one_complete_school_year_score_table():
|
||||
rows = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
|
||||
source = HistorySource(lambda page: rows)
|
||||
client = ScoreClient(source)
|
||||
school = {'school_code': '10001', 'school_name': '北京大学'}
|
||||
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
|
||||
assert client.score_lines(school, '081201', 2025, 1)[0]['total'] == 320
|
||||
assert len(source.calls) == 1
|
||||
|
||||
|
||||
def test_score_cache_separates_schools_and_years_and_has_a_bound():
|
||||
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}])
|
||||
client = ScoreClient(source)
|
||||
school = {'school_code': '10001', 'school_name': '北京大学'}
|
||||
for sid in range(20):
|
||||
client.score_lines(school, '081200', 2025, sid)
|
||||
client.score_lines(school, '081200', 2026, 19)
|
||||
before = len(source.calls)
|
||||
client.score_lines(school, '081200', 2026, 19)
|
||||
assert len(source.calls) == before
|
||||
client.score_lines(school, '081200', 2025, 0)
|
||||
assert len(source.calls) == before + 1
|
||||
|
||||
|
||||
def test_failed_lookup_is_cached_for_the_current_client_only():
|
||||
source = HistorySource(lambda page: {'data': [], 'total': 0})
|
||||
client = ScoreClient(source)
|
||||
assert client.school_id('暂未收录大学') is None
|
||||
assert client.school_id('暂未收录大学') is None
|
||||
assert len(source.calls) == 1
|
||||
assert ScoreClient(source).school_id('暂未收录大学') is None
|
||||
assert len(source.calls) == 2
|
||||
|
||||
|
||||
def test_invalid_score_response_is_not_cached():
|
||||
valid = False
|
||||
source = HistorySource(lambda page: [{'code': '081200', 'total': 310}] if valid else {})
|
||||
client = ScoreClient(source)
|
||||
school = {'school_code': '10001', 'school_name': '北京大学'}
|
||||
with pytest.raises(RemoteUnavailable):
|
||||
client.score_lines(school, '081200', 2025, 1)
|
||||
valid = True
|
||||
assert client.score_lines(school, '081200', 2025, 1)[0]['total'] == 310
|
||||
|
||||
|
||||
def test_history_runner_uses_thin_targets_and_bulk_checkpoints(tmp_path, monkeypatch):
|
||||
from studio.database import Database
|
||||
from studio.jobs import JobManager
|
||||
from studio.settings import SettingsStore
|
||||
from tests.test_bounded_storage import record
|
||||
|
||||
db = Database(tmp_path / 'data.db')
|
||||
db.upsert_catalog([record(), record(major='081201'), record(direction='二')])
|
||||
params = {'year': 2027, 'provinces': ['11'], 'score_years': [2025, 2026], 'include_plans': True}
|
||||
db.job_create('history', params)
|
||||
|
||||
class Source:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
def request(self, method, url, **kwargs):
|
||||
payload = kwargs['data']
|
||||
self.calls.append(url.rsplit('/', 1)[-1])
|
||||
if url.endswith('/schoolList'):
|
||||
data = {'data': [{'school_name': '甲大学', 'school_id': 1}], 'total': 1}
|
||||
elif url.endswith('/schoolScore'):
|
||||
data = [{'code': '081200', 'total': 310}, {'code': '081201', 'total': 320}]
|
||||
else:
|
||||
data = {'data': [{'special_code': payload['keyword'], 'special_name': payload['keyword'],
|
||||
'plan_id': str(payload['year']) + payload['keyword'], 'recruit_number': 1}], 'total': 1}
|
||||
return Response({'code': '0000', 'data': data})
|
||||
|
||||
def forbidden(*args, **kwargs):
|
||||
raise AssertionError('full catalog decoding or per-unit checkpoint query')
|
||||
monkeypatch.setattr(db, 'all_catalog', forbidden)
|
||||
monkeypatch.setattr(db, 'unit_done', forbidden)
|
||||
source = Source()
|
||||
manager = JobManager(db, SettingsStore(tmp_path), transport=source)
|
||||
manager._scores('history', params)
|
||||
assert db.job_get('history')['completed_units'] == 4
|
||||
assert db.overview()['scores'] == 4
|
||||
assert db.job_get('history')['records'] == 4
|
||||
assert source.calls.count('schoolScore') == 2
|
||||
assert source.calls.count('planListV2') == 4
|
||||
before = len(source.calls)
|
||||
manager._scores('history', params)
|
||||
assert len(source.calls) == before
|
||||
@@ -98,6 +98,47 @@ def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypa
|
||||
assert creations == []
|
||||
|
||||
|
||||
def test_mysql_bounded_catalog_and_history_targets_preserve_filters(mysql):
|
||||
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
|
||||
mysql.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
|
||||
scores = select_score_lines([{'code': '081200', 'total': 310, 'depart_name': '计算机学院'}], '081200', '10001', '北京大学', 2026)
|
||||
mysql.upsert_scores(scores)
|
||||
targets = list(mysql.score_targets({'year': 2027, 'provinces': ['北京']}, batch_size=1))
|
||||
assert len(targets) == 1
|
||||
assert targets[0]['major_code'] == '081200'
|
||||
assert 'data' not in targets[0]
|
||||
rows = list(mysql.iter_catalog({'year': 2027}, batch_size=1))
|
||||
assert len(rows) == 2
|
||||
assert len(list(mysql.iter_catalog({'year': 2026, 'max_score': 310}, batch_size=1))) == 2
|
||||
assert len(list(mysql.iter_scores({('10001', '081200')}, batch_size=1))) == 1
|
||||
|
||||
|
||||
def test_mysql_unit_commit_is_atomic_and_idempotent(mysql):
|
||||
mysql.job_create('atomic-job', {})
|
||||
rows = normalize_catalog(raw_catalog(), 2027)
|
||||
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
|
||||
assert mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
|
||||
assert not mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
|
||||
assert mysql.job_get('atomic-job')['records'] == 2
|
||||
assert mysql.job_get('atomic-job')['completed_units'] == 1
|
||||
assert mysql.overview()['records'] == 2
|
||||
assert mysql.overview()['scores'] == 1
|
||||
|
||||
|
||||
def test_mysql_failed_unit_rolls_back_every_write(mysql, monkeypatch):
|
||||
mysql.job_create('atomic-job', {})
|
||||
def fail(*args):
|
||||
raise RuntimeError('interrupted unit')
|
||||
monkeypatch.setattr(mysql, '_school', fail)
|
||||
rows = normalize_catalog(raw_catalog(), 2027)
|
||||
scores = select_score_lines([{'code': '081200', 'total': 310}], '081200', '10001', '北京大学', 2026)
|
||||
with pytest.raises(RuntimeError):
|
||||
mysql.save_unit('atomic-job', 'catalog:test', rows, scores=scores)
|
||||
assert mysql.job_get('atomic-job')['records'] == 0
|
||||
assert mysql.job_get('atomic-job')['completed_units'] == 0
|
||||
assert mysql.overview()['records'] == mysql.overview()['scores'] == 0
|
||||
|
||||
|
||||
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
|
||||
from studio.migrate import migrate_sqlite
|
||||
source = Database(tmp_path / 'snapshot.db')
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import importlib.util
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def maintainer_class():
|
||||
path = Path(__file__).resolve().parents[1] / 'deploy/proxy_scheduler.py'
|
||||
spec = importlib.util.spec_from_file_location('bounded_proxy_scheduler', path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.ProxyMaintainer
|
||||
|
||||
|
||||
def test_detection_workers_are_bounded_reused_and_rounds_do_not_overlap():
|
||||
state = {'active': 0, 'peak': 0, 'types': set(), 'mixed': False, 'done': [], 'thread_ids': set()}
|
||||
lock = threading.Lock()
|
||||
class Checker:
|
||||
def __init__(self, work_type, target_queue, name):
|
||||
self.work_type, self.queue = work_type, target_queue
|
||||
def run(self):
|
||||
while True:
|
||||
try:
|
||||
item = self.queue.get(block=False)
|
||||
except queue.Empty:
|
||||
return
|
||||
with lock:
|
||||
state['active'] += 1
|
||||
state['peak'] = max(state['peak'], state['active'])
|
||||
state['types'].add(self.work_type)
|
||||
state['mixed'] |= len(state['types']) > 1
|
||||
state['thread_ids'].add(threading.get_ident())
|
||||
time.sleep(.01)
|
||||
with lock:
|
||||
state['done'].append(item)
|
||||
state['active'] -= 1
|
||||
if not state['active']:
|
||||
state['types'].clear()
|
||||
self.queue.task_done()
|
||||
maintainer = maintainer_class()(workers=2, checker_class=Checker)
|
||||
first = threading.Thread(target=maintainer.validate, args=('raw', range(10)))
|
||||
second = threading.Thread(target=maintainer.validate, args=('use', range(10, 20)))
|
||||
first.start(); second.start()
|
||||
first.join(4); second.join(4)
|
||||
assert not first.is_alive() and not second.is_alive()
|
||||
before = set(state['thread_ids'])
|
||||
maintainer.validate('raw', range(20, 25))
|
||||
maintainer.close()
|
||||
assert sorted(state['done']) == list(range(25))
|
||||
assert state['peak'] <= 2
|
||||
assert state['mixed'] is False
|
||||
assert state['thread_ids'] == before
|
||||
|
||||
|
||||
def test_stopped_detector_does_not_begin_new_rounds():
|
||||
class Forbidden:
|
||||
def __init__(self, *args):
|
||||
raise AssertionError('detection began after shutdown')
|
||||
maintainer = maintainer_class()(workers=2, checker_class=Forbidden)
|
||||
maintainer.close()
|
||||
maintainer.validate('raw', [1, 2, 3])
|
||||
|
||||
|
||||
def test_failed_worker_waits_for_other_workers_before_next_round():
|
||||
slow_started, release, next_round = threading.Event(), threading.Event(), threading.Event()
|
||||
class Checker:
|
||||
def __init__(self, work_type, target_queue, name):
|
||||
self.kind, self.name = work_type, name
|
||||
def run(self):
|
||||
if self.kind == 'raw' and self.name == 'worker_00':
|
||||
assert slow_started.wait(2)
|
||||
raise RuntimeError('validator failed')
|
||||
if self.kind == 'raw':
|
||||
slow_started.set()
|
||||
assert release.wait(3)
|
||||
maintainer = maintainer_class()(workers=2, checker_class=Checker)
|
||||
failures = []
|
||||
def first_round():
|
||||
try:
|
||||
maintainer.validate('raw', [1, 2])
|
||||
except RuntimeError:
|
||||
failures.append(True)
|
||||
first = threading.Thread(target=first_round)
|
||||
first.start()
|
||||
assert slow_started.wait(2)
|
||||
def second_proxies():
|
||||
next_round.set()
|
||||
yield 3
|
||||
second = threading.Thread(target=maintainer.validate, args=('use', second_proxies()))
|
||||
second.start()
|
||||
began_early = next_round.wait(.15)
|
||||
release.set()
|
||||
first.join(3); second.join(3)
|
||||
maintainer.close()
|
||||
assert began_early is False
|
||||
assert next_round.is_set()
|
||||
assert failures == [True]
|
||||
Reference in New Issue
Block a user