123 lines
4.5 KiB
Python
123 lines
4.5 KiB
Python
"""Run upstream proxy_pool checks with reusable workers and serialized jobs."""
|
|
import os
|
|
import queue
|
|
import signal
|
|
import sys
|
|
import threading
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
APP_ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(APP_ROOT))
|
|
from studio.memory import release_idle_memory
|
|
|
|
|
|
class StopQueue(queue.Queue):
|
|
def __init__(self, stopping):
|
|
super().__init__()
|
|
self.stopping = stopping
|
|
|
|
def get(self, *args, **kwargs):
|
|
if self.stopping.is_set():
|
|
raise queue.Empty
|
|
return super().get(*args, **kwargs)
|
|
|
|
|
|
class ProxyMaintainer:
|
|
def __init__(self, workers=8, checker_class=None, fetcher_factory=None, handler_factory=None, pool_min=20):
|
|
self.worker_count = max(1, min(20, int(workers)))
|
|
self.checker_class = checker_class
|
|
self.fetcher_factory, self.handler_factory = fetcher_factory, handler_factory
|
|
self.pool_min = pool_min
|
|
self.stopping = threading.Event()
|
|
self.pipeline = threading.RLock()
|
|
self.workers = ThreadPoolExecutor(max_workers=self.worker_count, thread_name_prefix='proxy-check')
|
|
|
|
def validate(self, work_type, proxies):
|
|
with self.pipeline:
|
|
if self.stopping.is_set():
|
|
return
|
|
target = StopQueue(self.stopping)
|
|
for proxy in proxies:
|
|
if self.stopping.is_set():
|
|
return
|
|
target.put(proxy)
|
|
if self.checker_class is None:
|
|
from helper.check import _ThreadChecker
|
|
self.checker_class = _ThreadChecker
|
|
count = min(self.worker_count, target.qsize())
|
|
futures = [self.workers.submit(self.checker_class(work_type, target, f'worker_{i:02}').run)
|
|
for i in range(count)]
|
|
try:
|
|
failure = None
|
|
for future in futures:
|
|
try:
|
|
future.result()
|
|
except Exception as exc:
|
|
if failure is None:
|
|
failure = exc
|
|
if failure is not None:
|
|
raise failure
|
|
finally:
|
|
release_idle_memory()
|
|
|
|
def fetch(self):
|
|
with self.pipeline:
|
|
if not self.stopping.is_set():
|
|
self.validate('raw', self.fetcher_factory().run())
|
|
|
|
def check(self):
|
|
with self.pipeline:
|
|
if self.stopping.is_set():
|
|
return
|
|
handler = self.handler_factory()
|
|
if handler.db.getCount().get('total', 0) < self.pool_min:
|
|
self.fetch()
|
|
self.validate('use', handler.getAll())
|
|
|
|
def stop(self):
|
|
self.stopping.set()
|
|
|
|
def close(self):
|
|
self.stop()
|
|
with self.pipeline:
|
|
self.workers.shutdown(wait=True, cancel_futures=True)
|
|
|
|
|
|
def main():
|
|
proxy_root = Path(os.environ.get('STUDIO_PROXY_POOL_ROOT', APP_ROOT.parent / 'proxy_pool'))
|
|
sys.path.insert(0, str(proxy_root))
|
|
from apscheduler.schedulers.blocking import BlockingScheduler
|
|
from helper.check import _ThreadChecker
|
|
from helper.fetch import Fetcher
|
|
from handler.configHandler import ConfigHandler
|
|
from handler.logHandler import LogHandler
|
|
from handler.proxyHandler import ProxyHandler
|
|
|
|
config = ConfigHandler()
|
|
maintainer = ProxyMaintainer(workers=int(os.environ.get('PROXY_CHECK_WORKERS', '8')),
|
|
checker_class=_ThreadChecker, fetcher_factory=Fetcher,
|
|
handler_factory=ProxyHandler, pool_min=config.poolSizeMin)
|
|
scheduler = BlockingScheduler(timezone=config.timezone, logger=LogHandler('scheduler'),
|
|
executors={'default': {'type': 'threadpool', 'max_workers': 1}},
|
|
job_defaults={'coalesce': True, 'max_instances': 1, 'misfire_grace_time': 120})
|
|
scheduler.add_job(maintainer.fetch, 'interval', minutes=5, id='proxy_fetch')
|
|
scheduler.add_job(maintainer.check, 'interval', minutes=2, id='proxy_check',
|
|
next_run_time=datetime.now(timezone.utc))
|
|
def stop(signum, frame):
|
|
maintainer.stop()
|
|
if scheduler.running:
|
|
scheduler.shutdown(wait=False)
|
|
signal.signal(signal.SIGTERM, stop)
|
|
signal.signal(signal.SIGINT, stop)
|
|
try:
|
|
if not maintainer.stopping.is_set():
|
|
scheduler.start()
|
|
finally:
|
|
maintainer.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|