99 lines
3.7 KiB
Python
99 lines
3.7 KiB
Python
import importlib.util
|
|
import queue
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
|
|
def maintainer_class():
|
|
path = Path(__file__).resolve().parents[1] / 'deploy/proxy_scheduler.py'
|
|
spec = importlib.util.spec_from_file_location('bounded_proxy_scheduler', path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module.ProxyMaintainer
|
|
|
|
|
|
def test_detection_workers_are_bounded_reused_and_rounds_do_not_overlap():
|
|
state = {'active': 0, 'peak': 0, 'types': set(), 'mixed': False, 'done': [], 'thread_ids': set()}
|
|
lock = threading.Lock()
|
|
class Checker:
|
|
def __init__(self, work_type, target_queue, name):
|
|
self.work_type, self.queue = work_type, target_queue
|
|
def run(self):
|
|
while True:
|
|
try:
|
|
item = self.queue.get(block=False)
|
|
except queue.Empty:
|
|
return
|
|
with lock:
|
|
state['active'] += 1
|
|
state['peak'] = max(state['peak'], state['active'])
|
|
state['types'].add(self.work_type)
|
|
state['mixed'] |= len(state['types']) > 1
|
|
state['thread_ids'].add(threading.get_ident())
|
|
time.sleep(.01)
|
|
with lock:
|
|
state['done'].append(item)
|
|
state['active'] -= 1
|
|
if not state['active']:
|
|
state['types'].clear()
|
|
self.queue.task_done()
|
|
maintainer = maintainer_class()(workers=2, checker_class=Checker)
|
|
first = threading.Thread(target=maintainer.validate, args=('raw', range(10)))
|
|
second = threading.Thread(target=maintainer.validate, args=('use', range(10, 20)))
|
|
first.start(); second.start()
|
|
first.join(4); second.join(4)
|
|
assert not first.is_alive() and not second.is_alive()
|
|
before = set(state['thread_ids'])
|
|
maintainer.validate('raw', range(20, 25))
|
|
maintainer.close()
|
|
assert sorted(state['done']) == list(range(25))
|
|
assert state['peak'] <= 2
|
|
assert state['mixed'] is False
|
|
assert state['thread_ids'] == before
|
|
|
|
|
|
def test_stopped_detector_does_not_begin_new_rounds():
|
|
class Forbidden:
|
|
def __init__(self, *args):
|
|
raise AssertionError('detection began after shutdown')
|
|
maintainer = maintainer_class()(workers=2, checker_class=Forbidden)
|
|
maintainer.close()
|
|
maintainer.validate('raw', [1, 2, 3])
|
|
|
|
|
|
def test_failed_worker_waits_for_other_workers_before_next_round():
|
|
slow_started, release, next_round = threading.Event(), threading.Event(), threading.Event()
|
|
class Checker:
|
|
def __init__(self, work_type, target_queue, name):
|
|
self.kind, self.name = work_type, name
|
|
def run(self):
|
|
if self.kind == 'raw' and self.name == 'worker_00':
|
|
assert slow_started.wait(2)
|
|
raise RuntimeError('validator failed')
|
|
if self.kind == 'raw':
|
|
slow_started.set()
|
|
assert release.wait(3)
|
|
maintainer = maintainer_class()(workers=2, checker_class=Checker)
|
|
failures = []
|
|
def first_round():
|
|
try:
|
|
maintainer.validate('raw', [1, 2])
|
|
except RuntimeError:
|
|
failures.append(True)
|
|
first = threading.Thread(target=first_round)
|
|
first.start()
|
|
assert slow_started.wait(2)
|
|
def second_proxies():
|
|
next_round.set()
|
|
yield 3
|
|
second = threading.Thread(target=maintainer.validate, args=('use', second_proxies()))
|
|
second.start()
|
|
began_early = next_round.wait(.15)
|
|
release.set()
|
|
first.join(3); second.join(3)
|
|
maintainer.close()
|
|
assert began_early is False
|
|
assert next_round.is_set()
|
|
assert failures == [True]
|