Files
kaoyan/studio/network.py

217 lines
8.7 KiB
Python

"""One explicit route for CHSI login, catalog and historical data."""
import threading
import time
import random
from contextlib import contextmanager
from collections import OrderedDict
import requests
from studio.settings import proxy_url
class ProxyUnavailable(RuntimeError):
pass
class RemoteUnavailable(RuntimeError):
pass
class RateLimited(RuntimeError):
pass
class BoundedProxyAdapter(requests.adapters.HTTPAdapter):
"""Close obsolete proxy pools rather than retain every historical exit."""
def __init__(self, max_proxy_pools=4, **kwargs):
super().__init__(**kwargs)
self.max_proxy_pools = max(1, int(max_proxy_pools))
self.proxy_manager = OrderedDict()
def proxy_manager_for(self, proxy, **proxy_kwargs):
manager = super().proxy_manager_for(proxy, **proxy_kwargs)
self.proxy_manager.move_to_end(proxy)
while len(self.proxy_manager) > self.max_proxy_pools:
_, obsolete = self.proxy_manager.popitem(last=False)
obsolete.clear()
return manager
def bounded_session():
session = requests.Session()
session.trust_env = False
for scheme in ('http://', 'https://'):
session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4))
return session
class ProxyRouter:
def __init__(self, settings, session=None):
self.settings = settings
self.session = session or requests.Session()
self.session.trust_env = False
self.active = None
self.failed = set()
self.candidates = []
self.last_error = ""
self.counts = {}
self.fallback = False
self.lock = threading.RLock()
self.route_requests = 0
self.rotations = 0
self.last_request_route = None
self.holds = 0
def refresh(self):
with self.lock:
config = self.settings.read()
try:
response = self.session.get(config["pool_url"] + "/all/", params={"type": "https"}, timeout=5)
response.raise_for_status()
payload = response.json()
if not isinstance(payload, list):
raise ValueError("代理池返回格式不正确")
proxies = []
for entry in payload:
# The pool already filters HTTPS CONNECT-capable entries.
value = entry.get("proxy") if isinstance(entry, dict) else entry
if value:
try:
proxies.append(proxy_url(value))
except ValueError:
continue
self.last_error = ""
try:
self.counts = self.session.get(config["pool_url"] + "/count/", timeout=5).json()
except (requests.RequestException, ValueError):
self.counts = {"https_available": len(proxies)}
except (requests.RequestException, ValueError):
proxies = []
self.last_error = "无法连接代理池 API,请检查地址和服务状态"
self.counts = {}
self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"]))
return self.status()
@contextmanager
def hold_route(self):
with self.lock:
self.holds += 1
try:
yield
finally:
with self.lock:
self.holds -= 1
def note_success(self, proxy):
with self.lock:
self.last_request_route = proxy or "direct"
if proxy == self.active:
self.route_requests += 1
def resolve(self, for_request=False):
with self.lock:
config = self.settings.read()
if config["network_mode"] == "direct":
self.fallback = False
return None
threshold = config["proxy_rotate_every"]
if for_request and not self.holds and threshold and self.active and self.route_requests >= threshold:
self.rotate()
if self.active and self.active not in self.failed:
self.fallback = False
return self.active
available = [p for p in self.candidates if p not in self.failed]
if not available:
self.refresh()
available = [p for p in self.candidates if p not in self.failed]
if available:
self.active = available[0]
self.route_requests = 0
self.fallback = False
return self.active
self.active = None
self.route_requests = 0
if config["network_mode"] == "proxy_only":
self.fallback = False
raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务")
self.fallback = True
return None
def rotate(self, failed=None):
with self.lock:
old = self.active
if failed:
self.failed.add(failed)
self.active = None
if old in self.candidates:
self.candidates.remove(old)
self.candidates.append(old)
selected = self.resolve()
if old and selected != old:
self.rotations += 1
return selected
def reset(self):
with self.lock:
self.active = None
self.failed.clear()
self.candidates = []
self.fallback = False
self.route_requests = 0
def status(self):
config = self.settings.read()
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "route_requests": self.route_requests, "rotations": self.rotations, "last_request_route": self.last_request_route, "rotate_every": config["proxy_rotate_every"], "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
class Transport:
def __init__(self, settings, router, session=None, wait=None):
self.settings, self.router = settings, router
self.session = session or bounded_session()
self.session.trust_env = False
self.session.headers = getattr(self.session, "headers", {})
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
self.wait = wait or time.sleep
self.local = threading.local()
self.lock = threading.RLock()
def close(self):
with self.lock:
self.session.close()
def sleep(self, seconds):
remaining = seconds
while remaining > 0:
if guard := getattr(self.local, "guard", None):
guard()
step = min(.25, remaining)
self.wait(step)
remaining -= step
def request(self, method, url, **kwargs):
config = self.settings.read()
for attempt in range(config["retries"]):
self.sleep(random.uniform(config["request_interval"], config["request_interval_max"]) if attempt == 0 else min(20, 2 ** attempt))
proxy = self.router.resolve(for_request=True)
kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {}
kwargs.setdefault("timeout", config["timeout"])
try:
with self.lock:
response = self.session.request(method, url, **kwargs)
if response.status_code in (429, 503):
# Rate limiting does not prove an IP is broken.
self.sleep(min(30, 5 * (attempt + 1)))
continue
response.raise_for_status()
self.local.response_route = proxy or "direct"
self.router.note_success(proxy)
return response
except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout):
if proxy:
self.router.rotate(failed=proxy)
if attempt + 1 == config["retries"]:
raise RemoteUnavailable("网络连接失败,已尝试切换代理;请检查网络或恢复任务") from None
except requests.HTTPError as exc:
raise RemoteUnavailable(f"数据源返回 HTTP {exc.response.status_code}") from None
raise RateLimited("数据源持续限流,任务已等待;请降低请求频率后恢复")