Fix historical pagination and bound export and proxy memory
This commit is contained in:
@@ -0,0 +1,98 @@
|
||||
import importlib.util
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def maintainer_class():
|
||||
path = Path(__file__).resolve().parents[1] / 'deploy/proxy_scheduler.py'
|
||||
spec = importlib.util.spec_from_file_location('bounded_proxy_scheduler', path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.ProxyMaintainer
|
||||
|
||||
|
||||
def test_detection_workers_are_bounded_reused_and_rounds_do_not_overlap():
|
||||
state = {'active': 0, 'peak': 0, 'types': set(), 'mixed': False, 'done': [], 'thread_ids': set()}
|
||||
lock = threading.Lock()
|
||||
class Checker:
|
||||
def __init__(self, work_type, target_queue, name):
|
||||
self.work_type, self.queue = work_type, target_queue
|
||||
def run(self):
|
||||
while True:
|
||||
try:
|
||||
item = self.queue.get(block=False)
|
||||
except queue.Empty:
|
||||
return
|
||||
with lock:
|
||||
state['active'] += 1
|
||||
state['peak'] = max(state['peak'], state['active'])
|
||||
state['types'].add(self.work_type)
|
||||
state['mixed'] |= len(state['types']) > 1
|
||||
state['thread_ids'].add(threading.get_ident())
|
||||
time.sleep(.01)
|
||||
with lock:
|
||||
state['done'].append(item)
|
||||
state['active'] -= 1
|
||||
if not state['active']:
|
||||
state['types'].clear()
|
||||
self.queue.task_done()
|
||||
maintainer = maintainer_class()(workers=2, checker_class=Checker)
|
||||
first = threading.Thread(target=maintainer.validate, args=('raw', range(10)))
|
||||
second = threading.Thread(target=maintainer.validate, args=('use', range(10, 20)))
|
||||
first.start(); second.start()
|
||||
first.join(4); second.join(4)
|
||||
assert not first.is_alive() and not second.is_alive()
|
||||
before = set(state['thread_ids'])
|
||||
maintainer.validate('raw', range(20, 25))
|
||||
maintainer.close()
|
||||
assert sorted(state['done']) == list(range(25))
|
||||
assert state['peak'] <= 2
|
||||
assert state['mixed'] is False
|
||||
assert state['thread_ids'] == before
|
||||
|
||||
|
||||
def test_stopped_detector_does_not_begin_new_rounds():
|
||||
class Forbidden:
|
||||
def __init__(self, *args):
|
||||
raise AssertionError('detection began after shutdown')
|
||||
maintainer = maintainer_class()(workers=2, checker_class=Forbidden)
|
||||
maintainer.close()
|
||||
maintainer.validate('raw', [1, 2, 3])
|
||||
|
||||
|
||||
def test_failed_worker_waits_for_other_workers_before_next_round():
|
||||
slow_started, release, next_round = threading.Event(), threading.Event(), threading.Event()
|
||||
class Checker:
|
||||
def __init__(self, work_type, target_queue, name):
|
||||
self.kind, self.name = work_type, name
|
||||
def run(self):
|
||||
if self.kind == 'raw' and self.name == 'worker_00':
|
||||
assert slow_started.wait(2)
|
||||
raise RuntimeError('validator failed')
|
||||
if self.kind == 'raw':
|
||||
slow_started.set()
|
||||
assert release.wait(3)
|
||||
maintainer = maintainer_class()(workers=2, checker_class=Checker)
|
||||
failures = []
|
||||
def first_round():
|
||||
try:
|
||||
maintainer.validate('raw', [1, 2])
|
||||
except RuntimeError:
|
||||
failures.append(True)
|
||||
first = threading.Thread(target=first_round)
|
||||
first.start()
|
||||
assert slow_started.wait(2)
|
||||
def second_proxies():
|
||||
next_round.set()
|
||||
yield 3
|
||||
second = threading.Thread(target=maintainer.validate, args=('use', second_proxies()))
|
||||
second.start()
|
||||
began_early = next_round.wait(.15)
|
||||
release.set()
|
||||
first.join(3); second.join(3)
|
||||
maintainer.close()
|
||||
assert began_early is False
|
||||
assert next_round.is_set()
|
||||
assert failures == [True]
|
||||
Reference in New Issue
Block a user