Fix historical pagination and bound export and proxy memory

This commit is contained in:
2026-10-13 00:31:34 +08:00
parent 4c510759fb
commit ec2c897752
20 changed files with 1050 additions and 87 deletions
+31 -1
View File
@@ -3,6 +3,7 @@ import threading
import time
import random
from contextlib import contextmanager
from collections import OrderedDict
import requests
from studio.settings import proxy_url
@@ -19,6 +20,31 @@ class RateLimited(RuntimeError):
pass
class BoundedProxyAdapter(requests.adapters.HTTPAdapter):
"""Close obsolete proxy pools rather than retain every historical exit."""
def __init__(self, max_proxy_pools=4, **kwargs):
super().__init__(**kwargs)
self.max_proxy_pools = max(1, int(max_proxy_pools))
self.proxy_manager = OrderedDict()
def proxy_manager_for(self, proxy, **proxy_kwargs):
manager = super().proxy_manager_for(proxy, **proxy_kwargs)
self.proxy_manager.move_to_end(proxy)
while len(self.proxy_manager) > self.max_proxy_pools:
_, obsolete = self.proxy_manager.popitem(last=False)
obsolete.clear()
return manager
def bounded_session():
session = requests.Session()
session.trust_env = False
for scheme in ('http://', 'https://'):
session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4))
return session
class ProxyRouter:
def __init__(self, settings, session=None):
self.settings = settings
@@ -141,7 +167,7 @@ class ProxyRouter:
class Transport:
def __init__(self, settings, router, session=None, wait=None):
self.settings, self.router = settings, router
self.session = session or requests.Session()
self.session = session or bounded_session()
self.session.trust_env = False
self.session.headers = getattr(self.session, "headers", {})
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
@@ -149,6 +175,10 @@ class Transport:
self.local = threading.local()
self.lock = threading.RLock()
def close(self):
with self.lock:
self.session.close()
def sleep(self, seconds):
remaining = seconds
while remaining > 0: