"""Run upstream proxy_pool checks with reusable workers and serialized jobs.""" import os import queue import signal import sys import threading from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from pathlib import Path APP_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(APP_ROOT)) from studio.memory import release_idle_memory class StopQueue(queue.Queue): def __init__(self, stopping): super().__init__() self.stopping = stopping def get(self, *args, **kwargs): if self.stopping.is_set(): raise queue.Empty return super().get(*args, **kwargs) class ProxyMaintainer: def __init__(self, workers=8, checker_class=None, fetcher_factory=None, handler_factory=None, pool_min=20): self.worker_count = max(1, min(20, int(workers))) self.checker_class = checker_class self.fetcher_factory, self.handler_factory = fetcher_factory, handler_factory self.pool_min = pool_min self.stopping = threading.Event() self.pipeline = threading.RLock() self.workers = ThreadPoolExecutor(max_workers=self.worker_count, thread_name_prefix='proxy-check') def validate(self, work_type, proxies): with self.pipeline: if self.stopping.is_set(): return target = StopQueue(self.stopping) for proxy in proxies: if self.stopping.is_set(): return target.put(proxy) if self.checker_class is None: from helper.check import _ThreadChecker self.checker_class = _ThreadChecker count = min(self.worker_count, target.qsize()) futures = [self.workers.submit(self.checker_class(work_type, target, f'worker_{i:02}').run) for i in range(count)] try: failure = None for future in futures: try: future.result() except Exception as exc: if failure is None: failure = exc if failure is not None: raise failure finally: release_idle_memory() def fetch(self): with self.pipeline: if not self.stopping.is_set(): self.validate('raw', self.fetcher_factory().run()) def check(self): with self.pipeline: if self.stopping.is_set(): return handler = self.handler_factory() if handler.db.getCount().get('total', 0) < self.pool_min: self.fetch() self.validate('use', handler.getAll()) def stop(self): self.stopping.set() def close(self): self.stop() with self.pipeline: self.workers.shutdown(wait=True, cancel_futures=True) def main(): proxy_root = Path(os.environ.get('STUDIO_PROXY_POOL_ROOT', APP_ROOT.parent / 'proxy_pool')) sys.path.insert(0, str(proxy_root)) from apscheduler.schedulers.blocking import BlockingScheduler from helper.check import _ThreadChecker from helper.fetch import Fetcher from handler.configHandler import ConfigHandler from handler.logHandler import LogHandler from handler.proxyHandler import ProxyHandler config = ConfigHandler() maintainer = ProxyMaintainer(workers=int(os.environ.get('PROXY_CHECK_WORKERS', '8')), checker_class=_ThreadChecker, fetcher_factory=Fetcher, handler_factory=ProxyHandler, pool_min=config.poolSizeMin) scheduler = BlockingScheduler(timezone=config.timezone, logger=LogHandler('scheduler'), executors={'default': {'type': 'threadpool', 'max_workers': 1}}, job_defaults={'coalesce': True, 'max_instances': 1, 'misfire_grace_time': 120}) scheduler.add_job(maintainer.fetch, 'interval', minutes=5, id='proxy_fetch') scheduler.add_job(maintainer.check, 'interval', minutes=2, id='proxy_check', next_run_time=datetime.now(timezone.utc)) def stop(signum, frame): maintainer.stop() if scheduler.running: scheduler.shutdown(wait=False) signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGINT, stop) try: if not maintainer.stopping.is_set(): scheduler.start() finally: maintainer.close() if __name__ == '__main__': main()