Fix historical pagination and bound export and proxy memory
This commit is contained in:
+31
-1
@@ -3,6 +3,7 @@ import threading
|
||||
import time
|
||||
import random
|
||||
from contextlib import contextmanager
|
||||
from collections import OrderedDict
|
||||
import requests
|
||||
from studio.settings import proxy_url
|
||||
|
||||
@@ -19,6 +20,31 @@ class RateLimited(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class BoundedProxyAdapter(requests.adapters.HTTPAdapter):
|
||||
"""Close obsolete proxy pools rather than retain every historical exit."""
|
||||
|
||||
def __init__(self, max_proxy_pools=4, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.max_proxy_pools = max(1, int(max_proxy_pools))
|
||||
self.proxy_manager = OrderedDict()
|
||||
|
||||
def proxy_manager_for(self, proxy, **proxy_kwargs):
|
||||
manager = super().proxy_manager_for(proxy, **proxy_kwargs)
|
||||
self.proxy_manager.move_to_end(proxy)
|
||||
while len(self.proxy_manager) > self.max_proxy_pools:
|
||||
_, obsolete = self.proxy_manager.popitem(last=False)
|
||||
obsolete.clear()
|
||||
return manager
|
||||
|
||||
|
||||
def bounded_session():
|
||||
session = requests.Session()
|
||||
session.trust_env = False
|
||||
for scheme in ('http://', 'https://'):
|
||||
session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4))
|
||||
return session
|
||||
|
||||
|
||||
class ProxyRouter:
|
||||
def __init__(self, settings, session=None):
|
||||
self.settings = settings
|
||||
@@ -141,7 +167,7 @@ class ProxyRouter:
|
||||
class Transport:
|
||||
def __init__(self, settings, router, session=None, wait=None):
|
||||
self.settings, self.router = settings, router
|
||||
self.session = session or requests.Session()
|
||||
self.session = session or bounded_session()
|
||||
self.session.trust_env = False
|
||||
self.session.headers = getattr(self.session, "headers", {})
|
||||
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
|
||||
@@ -149,6 +175,10 @@ class Transport:
|
||||
self.local = threading.local()
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def close(self):
|
||||
with self.lock:
|
||||
self.session.close()
|
||||
|
||||
def sleep(self, seconds):
|
||||
remaining = seconds
|
||||
while remaining > 0:
|
||||
|
||||
Reference in New Issue
Block a user