feat: implement unified data sources, proxy routing and local API
This commit is contained in:
@@ -0,0 +1,149 @@
|
||||
"""One explicit route for CHSI login, catalog and historical data."""
|
||||
import threading
|
||||
import time
|
||||
import requests
|
||||
from studio.settings import proxy_url
|
||||
|
||||
|
||||
class ProxyUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class RemoteUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ProxyRouter:
|
||||
def __init__(self, settings, session=None):
|
||||
self.settings = settings
|
||||
self.session = session or requests.Session()
|
||||
self.session.trust_env = False
|
||||
self.active = None
|
||||
self.failed = set()
|
||||
self.candidates = []
|
||||
self.last_error = ""
|
||||
self.counts = {}
|
||||
self.fallback = False
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def refresh(self):
|
||||
with self.lock:
|
||||
config = self.settings.read()
|
||||
try:
|
||||
response = self.session.get(config["pool_url"] + "/all/", params={"type": "https"}, timeout=5)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if not isinstance(payload, list):
|
||||
raise ValueError("代理池返回格式不正确")
|
||||
proxies = []
|
||||
for entry in payload:
|
||||
# The pool already filters HTTPS CONNECT-capable entries.
|
||||
value = entry.get("proxy") if isinstance(entry, dict) else entry
|
||||
if value:
|
||||
try:
|
||||
proxies.append(proxy_url(value))
|
||||
except ValueError:
|
||||
continue
|
||||
self.last_error = ""
|
||||
try:
|
||||
self.counts = self.session.get(config["pool_url"] + "/count/", timeout=5).json()
|
||||
except (requests.RequestException, ValueError):
|
||||
self.counts = {"https_available": len(proxies)}
|
||||
except (requests.RequestException, ValueError):
|
||||
proxies = []
|
||||
self.last_error = "无法连接代理池 API,请检查地址和服务状态"
|
||||
self.counts = {}
|
||||
self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"]))
|
||||
return self.status()
|
||||
|
||||
def resolve(self):
|
||||
with self.lock:
|
||||
config = self.settings.read()
|
||||
if config["network_mode"] == "direct":
|
||||
self.fallback = False
|
||||
return None
|
||||
if self.active and self.active not in self.failed:
|
||||
self.fallback = False
|
||||
return self.active
|
||||
available = [p for p in self.candidates if p not in self.failed]
|
||||
if not available:
|
||||
self.refresh()
|
||||
available = [p for p in self.candidates if p not in self.failed]
|
||||
if available:
|
||||
self.active = available[0]
|
||||
self.fallback = False
|
||||
return self.active
|
||||
self.active = None
|
||||
if config["network_mode"] == "proxy_only":
|
||||
self.fallback = False
|
||||
raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务")
|
||||
self.fallback = True
|
||||
return None
|
||||
|
||||
def rotate(self, failed=None):
|
||||
with self.lock:
|
||||
old = self.active
|
||||
if failed:
|
||||
self.failed.add(failed)
|
||||
self.active = None
|
||||
if old in self.candidates:
|
||||
self.candidates.remove(old)
|
||||
self.candidates.append(old)
|
||||
return self.resolve()
|
||||
|
||||
def reset(self):
|
||||
with self.lock:
|
||||
self.active = None
|
||||
self.failed.clear()
|
||||
self.candidates = []
|
||||
self.fallback = False
|
||||
|
||||
def status(self):
|
||||
config = self.settings.read()
|
||||
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
|
||||
|
||||
|
||||
class Transport:
|
||||
def __init__(self, settings, router, session=None, wait=None):
|
||||
self.settings, self.router = settings, router
|
||||
self.session = session or requests.Session()
|
||||
self.session.trust_env = False
|
||||
self.session.headers = getattr(self.session, "headers", {})
|
||||
self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"})
|
||||
self.wait = wait or time.sleep
|
||||
self.local = threading.local()
|
||||
self.lock = threading.RLock()
|
||||
|
||||
def sleep(self, seconds):
|
||||
remaining = seconds
|
||||
while remaining > 0:
|
||||
if guard := getattr(self.local, "guard", None):
|
||||
guard()
|
||||
step = min(.25, remaining)
|
||||
self.wait(step)
|
||||
remaining -= step
|
||||
|
||||
def request(self, method, url, **kwargs):
|
||||
config = self.settings.read()
|
||||
for attempt in range(config["retries"]):
|
||||
self.sleep(config["request_interval"] if attempt == 0 else min(20, 2 ** attempt))
|
||||
proxy = self.router.resolve()
|
||||
kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {}
|
||||
kwargs.setdefault("timeout", config["timeout"])
|
||||
try:
|
||||
with self.lock:
|
||||
response = self.session.request(method, url, **kwargs)
|
||||
if response.status_code in (429, 503):
|
||||
# Rate limiting does not prove an IP is broken.
|
||||
self.sleep(min(30, 5 * (attempt + 1)))
|
||||
continue
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
if proxy:
|
||||
self.router.rotate(failed=proxy)
|
||||
if attempt + 1 == config["retries"]:
|
||||
raise RemoteUnavailable("网络连接失败,已尝试切换代理;请检查网络或恢复任务") from None
|
||||
except requests.HTTPError as exc:
|
||||
raise RemoteUnavailable(f"数据源返回 HTTP {exc.response.status_code}") from None
|
||||
raise RemoteUnavailable("数据源限流,请降低请求频率后重试")
|
||||
Reference in New Issue
Block a user