Files
kaoyan/studio/network.py
T

150 lines
6.3 KiB
Python

"""One explicit route for CHSI login, catalog and historical data."""
import threading
import time
import requests
from studio.settings import proxy_url
class ProxyUnavailable(RuntimeError):
pass
class RemoteUnavailable(RuntimeError):
pass
class ProxyRouter:
def __init__(self, settings, session=None):
self.settings = settings
self.session = session or requests.Session()
self.session.trust_env = False
self.active = None
self.failed = set()
self.candidates = []
self.last_error = ""
self.counts = {}
self.fallback = False
self.lock = threading.RLock()
def refresh(self):
with self.lock:
config = self.settings.read()
try:
response = self.session.get(config["pool_url"] + "/all/", params={"type": "https"}, timeout=5)
response.raise_for_status()
payload = response.json()
if not isinstance(payload, list):
raise ValueError("代理池返回格式不正确")
proxies = []
for entry in payload:
# The pool already filters HTTPS CONNECT-capable entries.
value = entry.get("proxy") if isinstance(entry, dict) else entry
if value:
try:
proxies.append(proxy_url(value))
except ValueError:
continue
self.last_error = ""
try:
self.counts = self.session.get(config["pool_url"] + "/count/", timeout=5).json()
except (requests.RequestException, ValueError):
self.counts = {"https_available": len(proxies)}
except (requests.RequestException, ValueError):
proxies = []
self.last_error = "无法连接代理池 API,请检查地址和服务状态"
self.counts = {}
self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"]))
return self.status()
def resolve(self):
with self.lock:
config = self.settings.read()
if config["network_mode"] == "direct":
self.fallback = False
return None
if self.active and self.active not in self.failed:
self.fallback = False
return self.active
available = [p for p in self.candidates if p not in self.failed]
if not available:
self.refresh()
available = [p for p in self.candidates if p not in self.failed]
if available:
self.active = available[0]
self.fallback = False
return self.active
self.active = None
if config["network_mode"] == "proxy_only":
self.fallback = False
raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务")
self.fallback = True
return None
def rotate(self, failed=None):
with self.lock:
old = self.active
if failed:
self.failed.add(failed)
self.active = None
if old in self.candidates:
self.candidates.remove(old)
self.candidates.append(old)
return self.resolve()
def reset(self):
with self.lock:
self.active = None
self.failed.clear()
self.candidates = []
self.fallback = False
def status(self):
config = self.settings.read()
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
class Transport:
def __init__(self, settings, router, session=None, wait=None):
self.settings, self.router = settings, router
self.session = session or requests.Session()
self.session.trust_env = False
self.session.headers = getattr(self.session, "headers", {})
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
self.wait = wait or time.sleep
self.local = threading.local()
self.lock = threading.RLock()
def sleep(self, seconds):
remaining = seconds
while remaining > 0:
if guard := getattr(self.local, "guard", None):
guard()
step = min(.25, remaining)
self.wait(step)
remaining -= step
def request(self, method, url, **kwargs):
config = self.settings.read()
for attempt in range(config["retries"]):
self.sleep(config["request_interval"] if attempt == 0 else min(20, 2 ** attempt))
proxy = self.router.resolve()
kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {}
kwargs.setdefault("timeout", config["timeout"])
try:
with self.lock:
response = self.session.request(method, url, **kwargs)
if response.status_code in (429, 503):
# Rate limiting does not prove an IP is broken.
self.sleep(min(30, 5 * (attempt + 1)))
continue
response.raise_for_status()
return response
except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout):
if proxy:
self.router.rotate(failed=proxy)
if attempt + 1 == config["retries"]:
raise RemoteUnavailable("网络连接失败,已尝试切换代理;请检查网络或恢复任务") from None
except requests.HTTPError as exc:
raise RemoteUnavailable(f"数据源返回 HTTP {exc.response.status_code}") from None
raise RemoteUnavailable("数据源限流,请降低请求频率后重试")