Fix historical pagination and bound export and proxy memory
This commit is contained in:
@@ -8,9 +8,12 @@ Requires=docker.service
|
||||
User=kaoyan
|
||||
Group=kaoyan
|
||||
WorkingDirectory=/opt/kaoyan-studio-deploy/proxy_pool
|
||||
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python proxyPool.py schedule
|
||||
Environment=MALLOC_ARENA_MAX=2
|
||||
Environment=PROXY_CHECK_WORKERS=8
|
||||
ExecStart=/opt/kaoyan-studio-deploy/proxy-venv/bin/python /opt/kaoyan-studio-deploy/app/deploy/proxy_scheduler.py
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
TimeoutStopSec=120
|
||||
UMask=0077
|
||||
PrivateTmp=true
|
||||
ProtectSystem=full
|
||||
|
||||
@@ -9,6 +9,7 @@ User=kaoyan
|
||||
Group=kaoyan
|
||||
WorkingDirectory=/opt/kaoyan-studio-deploy/app
|
||||
EnvironmentFile=/etc/kaoyan-studio.env
|
||||
Environment=MALLOC_ARENA_MAX=2
|
||||
ExecStart=/opt/kaoyan-studio-deploy/venv/bin/python -m studio --data-dir /var/lib/kaoyan-studio
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Run upstream proxy_pool checks with reusable workers and serialized jobs."""
|
||||
import os
|
||||
import queue
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
APP_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(APP_ROOT))
|
||||
from studio.memory import release_idle_memory
|
||||
|
||||
|
||||
class StopQueue(queue.Queue):
|
||||
def __init__(self, stopping):
|
||||
super().__init__()
|
||||
self.stopping = stopping
|
||||
|
||||
def get(self, *args, **kwargs):
|
||||
if self.stopping.is_set():
|
||||
raise queue.Empty
|
||||
return super().get(*args, **kwargs)
|
||||
|
||||
|
||||
class ProxyMaintainer:
|
||||
def __init__(self, workers=8, checker_class=None, fetcher_factory=None, handler_factory=None, pool_min=20):
|
||||
self.worker_count = max(1, min(20, int(workers)))
|
||||
self.checker_class = checker_class
|
||||
self.fetcher_factory, self.handler_factory = fetcher_factory, handler_factory
|
||||
self.pool_min = pool_min
|
||||
self.stopping = threading.Event()
|
||||
self.pipeline = threading.RLock()
|
||||
self.workers = ThreadPoolExecutor(max_workers=self.worker_count, thread_name_prefix='proxy-check')
|
||||
|
||||
def validate(self, work_type, proxies):
|
||||
with self.pipeline:
|
||||
if self.stopping.is_set():
|
||||
return
|
||||
target = StopQueue(self.stopping)
|
||||
for proxy in proxies:
|
||||
if self.stopping.is_set():
|
||||
return
|
||||
target.put(proxy)
|
||||
if self.checker_class is None:
|
||||
from helper.check import _ThreadChecker
|
||||
self.checker_class = _ThreadChecker
|
||||
count = min(self.worker_count, target.qsize())
|
||||
futures = [self.workers.submit(self.checker_class(work_type, target, f'worker_{i:02}').run)
|
||||
for i in range(count)]
|
||||
try:
|
||||
failure = None
|
||||
for future in futures:
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc:
|
||||
if failure is None:
|
||||
failure = exc
|
||||
if failure is not None:
|
||||
raise failure
|
||||
finally:
|
||||
release_idle_memory()
|
||||
|
||||
def fetch(self):
|
||||
with self.pipeline:
|
||||
if not self.stopping.is_set():
|
||||
self.validate('raw', self.fetcher_factory().run())
|
||||
|
||||
def check(self):
|
||||
with self.pipeline:
|
||||
if self.stopping.is_set():
|
||||
return
|
||||
handler = self.handler_factory()
|
||||
if handler.db.getCount().get('total', 0) < self.pool_min:
|
||||
self.fetch()
|
||||
self.validate('use', handler.getAll())
|
||||
|
||||
def stop(self):
|
||||
self.stopping.set()
|
||||
|
||||
def close(self):
|
||||
self.stop()
|
||||
with self.pipeline:
|
||||
self.workers.shutdown(wait=True, cancel_futures=True)
|
||||
|
||||
|
||||
def main():
|
||||
proxy_root = Path(os.environ.get('STUDIO_PROXY_POOL_ROOT', APP_ROOT.parent / 'proxy_pool'))
|
||||
sys.path.insert(0, str(proxy_root))
|
||||
from apscheduler.schedulers.blocking import BlockingScheduler
|
||||
from helper.check import _ThreadChecker
|
||||
from helper.fetch import Fetcher
|
||||
from handler.configHandler import ConfigHandler
|
||||
from handler.logHandler import LogHandler
|
||||
from handler.proxyHandler import ProxyHandler
|
||||
|
||||
config = ConfigHandler()
|
||||
maintainer = ProxyMaintainer(workers=int(os.environ.get('PROXY_CHECK_WORKERS', '8')),
|
||||
checker_class=_ThreadChecker, fetcher_factory=Fetcher,
|
||||
handler_factory=ProxyHandler, pool_min=config.poolSizeMin)
|
||||
scheduler = BlockingScheduler(timezone=config.timezone, logger=LogHandler('scheduler'),
|
||||
executors={'default': {'type': 'threadpool', 'max_workers': 1}},
|
||||
job_defaults={'coalesce': True, 'max_instances': 1, 'misfire_grace_time': 120})
|
||||
scheduler.add_job(maintainer.fetch, 'interval', minutes=5, id='proxy_fetch')
|
||||
scheduler.add_job(maintainer.check, 'interval', minutes=2, id='proxy_check',
|
||||
next_run_time=datetime.now(timezone.utc))
|
||||
def stop(signum, frame):
|
||||
maintainer.stop()
|
||||
if scheduler.running:
|
||||
scheduler.shutdown(wait=False)
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
try:
|
||||
if not maintainer.stopping.is_set():
|
||||
scheduler.start()
|
||||
finally:
|
||||
maintainer.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user