"""One explicit route for CHSI login, catalog and historical data.""" import threading import time import random from contextlib import contextmanager from collections import OrderedDict import requests from studio.settings import proxy_url class ProxyUnavailable(RuntimeError): pass class RemoteUnavailable(RuntimeError): pass class RateLimited(RuntimeError): pass class BoundedProxyAdapter(requests.adapters.HTTPAdapter): """Close obsolete proxy pools rather than retain every historical exit.""" def __init__(self, max_proxy_pools=4, **kwargs): super().__init__(**kwargs) self.max_proxy_pools = max(1, int(max_proxy_pools)) self.proxy_manager = OrderedDict() def proxy_manager_for(self, proxy, **proxy_kwargs): manager = super().proxy_manager_for(proxy, **proxy_kwargs) self.proxy_manager.move_to_end(proxy) while len(self.proxy_manager) > self.max_proxy_pools: _, obsolete = self.proxy_manager.popitem(last=False) obsolete.clear() return manager def bounded_session(): session = requests.Session() session.trust_env = False for scheme in ('http://', 'https://'): session.mount(scheme, BoundedProxyAdapter(pool_connections=8, pool_maxsize=4)) return session class ProxyRouter: def __init__(self, settings, session=None): self.settings = settings self.session = session or requests.Session() self.session.trust_env = False self.active = None self.failed = set() self.candidates = [] self.last_error = "" self.counts = {} self.fallback = False self.lock = threading.RLock() self.route_requests = 0 self.rotations = 0 self.last_request_route = None self.holds = 0 def refresh(self): with self.lock: config = self.settings.read() try: response = self.session.get(config["pool_url"] + "/all/", params={"type": "https"}, timeout=5) response.raise_for_status() payload = response.json() if not isinstance(payload, list): raise ValueError("代理池返回格式不正确") proxies = [] for entry in payload: # The pool already filters HTTPS CONNECT-capable entries. value = entry.get("proxy") if isinstance(entry, dict) else entry if value: try: proxies.append(proxy_url(value)) except ValueError: continue self.last_error = "" try: self.counts = self.session.get(config["pool_url"] + "/count/", timeout=5).json() except (requests.RequestException, ValueError): self.counts = {"https_available": len(proxies)} except (requests.RequestException, ValueError): proxies = [] self.last_error = "无法连接代理池 API,请检查地址和服务状态" self.counts = {} self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"])) return self.status() @contextmanager def hold_route(self): with self.lock: self.holds += 1 try: yield finally: with self.lock: self.holds -= 1 def note_success(self, proxy): with self.lock: self.last_request_route = proxy or "direct" if proxy == self.active: self.route_requests += 1 def resolve(self, for_request=False): with self.lock: config = self.settings.read() if config["network_mode"] == "direct": self.fallback = False return None threshold = config["proxy_rotate_every"] if for_request and not self.holds and threshold and self.active and self.route_requests >= threshold: self.rotate() if self.active and self.active not in self.failed: self.fallback = False return self.active available = [p for p in self.candidates if p not in self.failed] if not available: self.refresh() available = [p for p in self.candidates if p not in self.failed] if available: self.active = available[0] self.route_requests = 0 self.fallback = False return self.active self.active = None self.route_requests = 0 if config["network_mode"] == "proxy_only": self.fallback = False raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务") self.fallback = True return None def rotate(self, failed=None): with self.lock: old = self.active if failed: self.failed.add(failed) self.active = None if old in self.candidates: self.candidates.remove(old) self.candidates.append(old) selected = self.resolve() if old and selected != old: self.rotations += 1 return selected def reset(self): with self.lock: self.active = None self.failed.clear() self.candidates = [] self.fallback = False self.route_requests = 0 def status(self): config = self.settings.read() return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "route_requests": self.route_requests, "rotations": self.rotations, "last_request_route": self.last_request_route, "rotate_every": config["proxy_rotate_every"], "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]} class Transport: def __init__(self, settings, router, session=None, wait=None): self.settings, self.router = settings, router self.session = session or bounded_session() self.session.trust_env = False self.session.headers = getattr(self.session, "headers", {}) self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"}) self.wait = wait or time.sleep self.local = threading.local() self.lock = threading.RLock() def close(self): with self.lock: self.session.close() def sleep(self, seconds): remaining = seconds while remaining > 0: if guard := getattr(self.local, "guard", None): guard() step = min(.25, remaining) self.wait(step) remaining -= step def request(self, method, url, **kwargs): config = self.settings.read() for attempt in range(config["retries"]): self.sleep(random.uniform(config["request_interval"], config["request_interval_max"]) if attempt == 0 else min(20, 2 ** attempt)) proxy = self.router.resolve(for_request=True) kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {} kwargs.setdefault("timeout", config["timeout"]) try: with self.lock: response = self.session.request(method, url, **kwargs) if response.status_code in (429, 503): # Rate limiting does not prove an IP is broken. self.sleep(min(30, 5 * (attempt + 1))) continue response.raise_for_status() self.local.response_route = proxy or "direct" self.router.note_success(proxy) return response except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout): if proxy: self.router.rotate(failed=proxy) if attempt + 1 == config["retries"]: raise RemoteUnavailable("网络连接失败,已尝试切换代理;请检查网络或恢复任务") from None except requests.HTTPError as exc: raise RemoteUnavailable(f"数据源返回 HTTP {exc.response.status_code}") from None raise RateLimited("数据源持续限流,任务已等待;请降低请求频率后恢复")