Add Debian deployment, MariaDB storage and randomized proxy routing
This commit is contained in:
+37
-5
@@ -1,6 +1,8 @@
|
||||
"""One explicit route for CHSI login, catalog and historical data."""
|
||||
import threading
|
||||
import time
|
||||
import random
|
||||
from contextlib import contextmanager
|
||||
import requests
|
||||
from studio.settings import proxy_url
|
||||
|
||||
@@ -29,6 +31,10 @@ class ProxyRouter:
|
||||
self.counts = {}
|
||||
self.fallback = False
|
||||
self.lock = threading.RLock()
|
||||
self.route_requests = 0
|
||||
self.rotations = 0
|
||||
self.last_request_route = None
|
||||
self.holds = 0
|
||||
|
||||
def refresh(self):
|
||||
with self.lock:
|
||||
@@ -60,12 +66,31 @@ class ProxyRouter:
|
||||
self.candidates = list(dict.fromkeys(proxies + config["backup_proxies"]))
|
||||
return self.status()
|
||||
|
||||
def resolve(self):
|
||||
@contextmanager
|
||||
def hold_route(self):
|
||||
with self.lock:
|
||||
self.holds += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with self.lock:
|
||||
self.holds -= 1
|
||||
|
||||
def note_success(self, proxy):
|
||||
with self.lock:
|
||||
self.last_request_route = proxy or "direct"
|
||||
if proxy == self.active:
|
||||
self.route_requests += 1
|
||||
|
||||
def resolve(self, for_request=False):
|
||||
with self.lock:
|
||||
config = self.settings.read()
|
||||
if config["network_mode"] == "direct":
|
||||
self.fallback = False
|
||||
return None
|
||||
threshold = config["proxy_rotate_every"]
|
||||
if for_request and not self.holds and threshold and self.active and self.route_requests >= threshold:
|
||||
self.rotate()
|
||||
if self.active and self.active not in self.failed:
|
||||
self.fallback = False
|
||||
return self.active
|
||||
@@ -75,9 +100,11 @@ class ProxyRouter:
|
||||
available = [p for p in self.candidates if p not in self.failed]
|
||||
if available:
|
||||
self.active = available[0]
|
||||
self.route_requests = 0
|
||||
self.fallback = False
|
||||
return self.active
|
||||
self.active = None
|
||||
self.route_requests = 0
|
||||
if config["network_mode"] == "proxy_only":
|
||||
self.fallback = False
|
||||
raise ProxyUnavailable("严格代理模式下没有可用 HTTPS 代理;请补充代理池后恢复任务")
|
||||
@@ -93,7 +120,10 @@ class ProxyRouter:
|
||||
if old in self.candidates:
|
||||
self.candidates.remove(old)
|
||||
self.candidates.append(old)
|
||||
return self.resolve()
|
||||
selected = self.resolve()
|
||||
if old and selected != old:
|
||||
self.rotations += 1
|
||||
return selected
|
||||
|
||||
def reset(self):
|
||||
with self.lock:
|
||||
@@ -101,10 +131,11 @@ class ProxyRouter:
|
||||
self.failed.clear()
|
||||
self.candidates = []
|
||||
self.fallback = False
|
||||
self.route_requests = 0
|
||||
|
||||
def status(self):
|
||||
config = self.settings.read()
|
||||
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
|
||||
return {"mode": config["network_mode"], "route": "direct" if config["network_mode"] == "direct" or self.fallback else (self.active or "pending"), "active_proxy": self.active, "fallback": self.fallback, "counts": self.counts, "available": len([p for p in self.candidates if p not in self.failed]), "quarantined": len(self.failed), "error": self.last_error, "route_requests": self.route_requests, "rotations": self.rotations, "last_request_route": self.last_request_route, "rotate_every": config["proxy_rotate_every"], "proxies": [{"url": p, "failed": p in self.failed, "active": p == self.active} for p in self.candidates]}
|
||||
|
||||
|
||||
class Transport:
|
||||
@@ -130,8 +161,8 @@ class Transport:
|
||||
def request(self, method, url, **kwargs):
|
||||
config = self.settings.read()
|
||||
for attempt in range(config["retries"]):
|
||||
self.sleep(config["request_interval"] if attempt == 0 else min(20, 2 ** attempt))
|
||||
proxy = self.router.resolve()
|
||||
self.sleep(random.uniform(config["request_interval"], config["request_interval_max"]) if attempt == 0 else min(20, 2 ** attempt))
|
||||
proxy = self.router.resolve(for_request=True)
|
||||
kwargs["proxies"] = {"http": proxy, "https": proxy} if proxy else {}
|
||||
kwargs.setdefault("timeout", config["timeout"])
|
||||
try:
|
||||
@@ -143,6 +174,7 @@ class Transport:
|
||||
continue
|
||||
response.raise_for_status()
|
||||
self.local.response_route = proxy or "direct"
|
||||
self.router.note_success(proxy)
|
||||
return response
|
||||
except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
if proxy:
|
||||
|
||||
Reference in New Issue
Block a user