Files
kaoyan/deploy/proxy_scheduler.py

123 lines
4.5 KiB
Python

"""Run upstream proxy_pool checks with reusable workers and serialized jobs."""
import os
import queue
import signal
import sys
import threading
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
APP_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(APP_ROOT))
from studio.memory import release_idle_memory
class StopQueue(queue.Queue):
def __init__(self, stopping):
super().__init__()
self.stopping = stopping
def get(self, *args, **kwargs):
if self.stopping.is_set():
raise queue.Empty
return super().get(*args, **kwargs)
class ProxyMaintainer:
def __init__(self, workers=8, checker_class=None, fetcher_factory=None, handler_factory=None, pool_min=20):
self.worker_count = max(1, min(20, int(workers)))
self.checker_class = checker_class
self.fetcher_factory, self.handler_factory = fetcher_factory, handler_factory
self.pool_min = pool_min
self.stopping = threading.Event()
self.pipeline = threading.RLock()
self.workers = ThreadPoolExecutor(max_workers=self.worker_count, thread_name_prefix='proxy-check')
def validate(self, work_type, proxies):
with self.pipeline:
if self.stopping.is_set():
return
target = StopQueue(self.stopping)
for proxy in proxies:
if self.stopping.is_set():
return
target.put(proxy)
if self.checker_class is None:
from helper.check import _ThreadChecker
self.checker_class = _ThreadChecker
count = min(self.worker_count, target.qsize())
futures = [self.workers.submit(self.checker_class(work_type, target, f'worker_{i:02}').run)
for i in range(count)]
try:
failure = None
for future in futures:
try:
future.result()
except Exception as exc:
if failure is None:
failure = exc
if failure is not None:
raise failure
finally:
release_idle_memory()
def fetch(self):
with self.pipeline:
if not self.stopping.is_set():
self.validate('raw', self.fetcher_factory().run())
def check(self):
with self.pipeline:
if self.stopping.is_set():
return
handler = self.handler_factory()
if handler.db.getCount().get('total', 0) < self.pool_min:
self.fetch()
self.validate('use', handler.getAll())
def stop(self):
self.stopping.set()
def close(self):
self.stop()
with self.pipeline:
self.workers.shutdown(wait=True, cancel_futures=True)
def main():
proxy_root = Path(os.environ.get('STUDIO_PROXY_POOL_ROOT', APP_ROOT.parent / 'proxy_pool'))
sys.path.insert(0, str(proxy_root))
from apscheduler.schedulers.blocking import BlockingScheduler
from helper.check import _ThreadChecker
from helper.fetch import Fetcher
from handler.configHandler import ConfigHandler
from handler.logHandler import LogHandler
from handler.proxyHandler import ProxyHandler
config = ConfigHandler()
maintainer = ProxyMaintainer(workers=int(os.environ.get('PROXY_CHECK_WORKERS', '8')),
checker_class=_ThreadChecker, fetcher_factory=Fetcher,
handler_factory=ProxyHandler, pool_min=config.poolSizeMin)
scheduler = BlockingScheduler(timezone=config.timezone, logger=LogHandler('scheduler'),
executors={'default': {'type': 'threadpool', 'max_workers': 1}},
job_defaults={'coalesce': True, 'max_instances': 1, 'misfire_grace_time': 120})
scheduler.add_job(maintainer.fetch, 'interval', minutes=5, id='proxy_fetch')
scheduler.add_job(maintainer.check, 'interval', minutes=2, id='proxy_check',
next_run_time=datetime.now(timezone.utc))
def stop(signum, frame):
maintainer.stop()
if scheduler.running:
scheduler.shutdown(wait=False)
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
try:
if not maintainer.stopping.is_set():
scheduler.start()
finally:
maintainer.close()
if __name__ == '__main__':
main()