150 lines
6.3 KiB
Python
150 lines
6.3 KiB
Python
"""One explicit route for CHSI login, catalog and historical data."""
|
|
import threading
|
|
import time
|
|
import requests
|
|
from studio.settings import proxy_url
|
|
|
|
|
|
class ProxyUnavailable(RuntimeError):
|
|
pass
|
|
|
|
|
|
class RemoteUnavailable(RuntimeError):
|
|
pass
|
|
|
|
|
|
class ProxyRouter:
|
|
def __init__(self, settings, session=None):
|
|
self.settings = settings
|
|
self.session = session or requests.Session()
|
|
self.session.trust_env = False
|
|
self.active = None
|
|
self.failed = set()
|
|
self.candidates = []
|
|
self.last_error = ""
|
|
self.counts = {}
|
|
self.fallback = False
|
|
self.lock = threading.RLock()
|
|
|
|
def refresh(self):
|
|
with self.lock:
|
|
config = self.settings.read()
|
|
try:
|
|
response = self.session.get(config["pool_url"] + "/all/", params={"type": "https"}, timeout=5)
|
|
response.raise_for_status()
|
|
payload = response.json()
|
|
if not isinstance(payload, list):
|
|
raise ValueError("代理池返回格式不正确")
|
|
proxies = []
|
|
for entry in payload:
|
|
# The pool already filters HTTPS CONNECT-capable entries.
|
|
value = entry.get("proxy") if isinstance(entry, dict) else entry
|
|
if value:
|
|
try:
|
|
proxies.append(proxy_url(value))
|
|
except ValueError:
|
|
continue
|
|
self.last_error = ""
|
|
try:
|
|
self.counts = self.session.get(config["pool_url"] + "/count/", timeout=5).json()
|
|
except (requests.RequestException, ValueError):
|
|
self.counts = {"https_available": len(proxies)}
|
|
except (requests.RequestException, ValueError):
|
|
proxies = []
|
|
self.last_error = "无法连接代理池 API,请检查地址和服务状态"
|
|
self.counts = {}
|
|
self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"]))
|
|
return self.status()
|
|
|
|
def resolve(self):
|
|
with self.lock:
|
|
config = self.settings.read()
|
|
if config["network_mode"] == "direct":
|
|
self.fallback = False
|
|
return None
|
|
if self.active and self.active not in self.failed:
|
|
self.fallback = False
|
|
return self.active
|
|
available = [p for p in self.candidates if p not in self.failed]
|
|
if not available:
|
|
self.refresh()
|
|
available = [p for p in self.candidates if p not in self.failed]
|
|
if available:
|
|
self.active = available[0]
|
|
self.fallback = False
|
|
return self.active
|
|
self.active = None
|
|
if config["network_mode"] == "proxy_only":
|
|
self.fallback = False
|
|
raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务")
|
|
self.fallback = True
|
|
return None
|
|
|
|
def rotate(self, failed=None):
|
|
with self.lock:
|
|
old = self.active
|
|
if failed:
|
|
self.failed.add(failed)
|
|
self.active = None
|
|
if old in self.candidates:
|
|
self.candidates.remove(old)
|
|
self.candidates.append(old)
|
|
return self.resolve()
|
|
|
|
def reset(self):
|
|
with self.lock:
|
|
self.active = None
|
|
self.failed.clear()
|
|
self.candidates = []
|
|
self.fallback = False
|
|
|
|
def status(self):
|
|
config = self.settings.read()
|
|
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
|
|
|
|
|
|
class Transport:
|
|
def __init__(self, settings, router, session=None, wait=None):
|
|
self.settings, self.router = settings, router
|
|
self.session = session or requests.Session()
|
|
self.session.trust_env = False
|
|
self.session.headers = getattr(self.session, "headers", {})
|
|
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
|
|
self.wait = wait or time.sleep
|
|
self.local = threading.local()
|
|
self.lock = threading.RLock()
|
|
|
|
def sleep(self, seconds):
|
|
remaining = seconds
|
|
while remaining > 0:
|
|
if guard := getattr(self.local, "guard", None):
|
|
guard()
|
|
step = min(.25, remaining)
|
|
self.wait(step)
|
|
remaining -= step
|
|
|
|
def request(self, method, url, **kwargs):
|
|
config = self.settings.read()
|
|
for attempt in range(config["retries"]):
|
|
self.sleep(config["request_interval"] if attempt == 0 else min(20, 2 ** attempt))
|
|
proxy = self.router.resolve()
|
|
kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {}
|
|
kwargs.setdefault("timeout", config["timeout"])
|
|
try:
|
|
with self.lock:
|
|
response = self.session.request(method, url, **kwargs)
|
|
if response.status_code in (429, 503):
|
|
# Rate limiting does not prove an IP is broken.
|
|
self.sleep(min(30, 5 * (attempt + 1)))
|
|
continue
|
|
response.raise_for_status()
|
|
return response
|
|
except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
|
if proxy:
|
|
self.router.rotate(failed=proxy)
|
|
if attempt + 1 == config["retries"]:
|
|
raise RemoteUnavailable("网络连接失败,已尝试切换代理;请检查网络或恢复任务") from None
|
|
except requests.HTTPError as exc:
|
|
raise RemoteUnavailable(f"数据源返回 HTTP {exc.response.status_code}") from None
|
|
raise RemoteUnavailable("数据源限流,请降低请求频率后重试")
|