Fix CHSI catalogue session verification and pagination
This commit is contained in:
+2
-1
@@ -28,7 +28,7 @@ def create_app(data_dir=None):
|
||||
router = ProxyRouter(settings)
|
||||
transport = Transport(settings, router)
|
||||
auth = AuthService(transport, CookieVault(directory))
|
||||
jobs = JobManager(db, settings, transport)
|
||||
jobs = JobManager(db, settings, transport, auth=auth)
|
||||
if (ROOT / "data/schools.json").exists():
|
||||
db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8")))
|
||||
|
||||
@@ -41,6 +41,7 @@ def create_app(data_dir=None):
|
||||
app.state.db, app.state.jobs = db, jobs
|
||||
app.state.settings, app.state.router = settings, router
|
||||
app.state.pool = pool
|
||||
app.state.auth = auth
|
||||
|
||||
@app.middleware("http")
|
||||
async def local_access(request: Request, call_next):
|
||||
|
||||
+98
-39
@@ -6,8 +6,12 @@ import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from bs4 import BeautifulSoup
|
||||
from studio.connectors import CatalogClient, LoginRequired
|
||||
from studio.domain import now
|
||||
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||
|
||||
LOGIN_URL = "https://account.chsi.com.cn/passport/login?entrytype=yzgr&service=https%3A%2F%2Fyz.chsi.com.cn%2Fj_spring_cas_security_check"
|
||||
CATALOG_ENTRY = "https://yz.chsi.com.cn/zsml/a/dw.do"
|
||||
|
||||
|
||||
class Blob(ctypes.Structure):
|
||||
@@ -55,9 +59,16 @@ class AuthService:
|
||||
self.transport, self.vault = transport, vault
|
||||
self.message = "尚未验证登录状态"
|
||||
self.valid = False
|
||||
self.state = "unverified"
|
||||
self.checked_at = None
|
||||
self.verified_route = None
|
||||
self.revision = 0
|
||||
self.state_lock = threading.RLock()
|
||||
self.check_lock = threading.RLock()
|
||||
self.browser_state = "idle"
|
||||
self.browser_message = ""
|
||||
self.browser_thread = None
|
||||
self.browser_cancel = None
|
||||
try:
|
||||
self._load(self.vault.load())
|
||||
except (RuntimeError, OSError, ValueError):
|
||||
@@ -80,7 +91,19 @@ class AuthService:
|
||||
return [{"name": c.name, "value": c.value, "domain": c.domain, "path": c.path, "secure": c.secure, "expires": c.expires} for c in self.transport.session.cookies]
|
||||
|
||||
def status(self):
|
||||
return {"saved": self.vault.path.exists(), "valid": self.valid, "message": self.message, "browser_state": self.browser_state, "browser_message": self.browser_message}
|
||||
with self.state_lock:
|
||||
if self.valid and self.verified_route != self.transport.router.status()["route"]:
|
||||
self.invalidate("网络出口已变化,请验证目录访问后继续采集", state="unverified")
|
||||
return {"saved": self.vault.path.exists(), "valid": self.valid, "state": self.state, "message": self.message, "checked_at": self.checked_at, "browser_state": self.browser_state, "browser_message": self.browser_message}
|
||||
|
||||
def invalidate(self, message, state="login_required"):
|
||||
with self.state_lock:
|
||||
self.revision += 1
|
||||
self.valid = False
|
||||
self.state, self.message = state, message
|
||||
self.verified_route = None
|
||||
if self.browser_state == "completed":
|
||||
self.browser_state, self.browser_message = "idle", ""
|
||||
|
||||
def cookie(self, text):
|
||||
if not text.strip() or "\n" in text or "\r" in text:
|
||||
@@ -94,50 +117,76 @@ class AuthService:
|
||||
cookies[name] = value
|
||||
if not cookies:
|
||||
raise ValueError("未识别到 Cookie 键值")
|
||||
self._load(cookies)
|
||||
self.vault.save(cookies)
|
||||
self.valid = False
|
||||
self.message = "Cookie 已加密保存,点击验证确认目录访问权限"
|
||||
return self.status()
|
||||
if self.browser_cancel:
|
||||
self.browser_cancel.set()
|
||||
with self.check_lock:
|
||||
self._load(cookies)
|
||||
self.vault.save(cookies)
|
||||
self.invalidate("Cookie 已加密保存,点击验证确认目录访问权限", state="unverified")
|
||||
self.checked_at = None
|
||||
self.browser_state, self.browser_message = "idle", ""
|
||||
return self.status()
|
||||
|
||||
def check(self):
|
||||
# The school/major indexes can be public while direction details require login.
|
||||
response = self.transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/yjfxs.do", data={"dwdm": "10001", "zydm": "081200", "zymc": "计算机科学与技术", "curPage": 1, "start": 0, "pageSize": 1, "dwlxs[0]": "all"})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
self.valid = False
|
||||
self.message = "目录接口未返回有效数据,可能需要登录或验证码"
|
||||
with self.check_lock:
|
||||
with self.state_lock:
|
||||
self.invalidate("正在同步研招目录会话并验证访问权限", state="unverified")
|
||||
revision = self.revision
|
||||
try:
|
||||
# The first ten rows are available anonymously, even for directions.
|
||||
# Enter the CAS-protected page to bind cookies, then prove page-two access.
|
||||
self.transport.request("GET", CATALOG_ENTRY)
|
||||
CatalogClient(self.transport).fetch_page("yjfxs", {"dwdm": "10001", "zydm": "081200", "zymc": "计算机科学与技术", "curPage": 2, "start": 10, "pageSize": 10, "dwlxs[0]": "all"})
|
||||
proof_route = self.transport.local.response_route
|
||||
self.vault.save(self._snapshot())
|
||||
except LoginRequired as exc:
|
||||
self.invalidate(str(exc), state="captcha_required" if exc.reason == "captcha" else "login_required")
|
||||
except (ProxyUnavailable, RemoteUnavailable, RateLimited):
|
||||
self.invalidate("暂时无法验证研招目录,请检查网络或稍后重试;已保存的 Cookie 保留", state="unverified")
|
||||
raise
|
||||
else:
|
||||
with self.state_lock:
|
||||
# A later crawler login/captcha failure supersedes this older proof.
|
||||
if self.revision == revision:
|
||||
self.valid, self.state = True, "verified"
|
||||
self.message = "研招目录登录会话已同步,第二页访问验证成功"
|
||||
self.verified_route = proof_route
|
||||
finally:
|
||||
self.checked_at = now()
|
||||
return self.status()
|
||||
self.valid = body.get("flag") is True and isinstance(body.get("msg"), dict)
|
||||
self.message = "研招目录接口验证成功" if self.valid else "登录未生效或数据源限流,请在浏览器完成登录后重试"
|
||||
if self.valid:
|
||||
self.vault.save(self._snapshot())
|
||||
return self.status()
|
||||
|
||||
def login(self, username, password):
|
||||
if not username or not password:
|
||||
raise ValueError("请输入学信网账号和密码")
|
||||
response = self.transport.request("GET", LOGIN_URL)
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
form = {node.get("name"): node.get("value", "") for node in soup.select("input[type=hidden][name]")}
|
||||
if "lt" not in form or "execution" not in form:
|
||||
raise ValueError("登录页面需要人工验证,请使用浏览器登录")
|
||||
form.update(username=username, password=password, _eventId="submit")
|
||||
self.transport.request("POST", LOGIN_URL, data=form)
|
||||
return self.check()
|
||||
if self.browser_cancel:
|
||||
self.browser_cancel.set()
|
||||
with self.check_lock:
|
||||
response = self.transport.request("GET", LOGIN_URL)
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
form = {node.get("name"): node.get("value", "") for node in soup.select("input[type=hidden][name]")}
|
||||
if "lt" not in form or "execution" not in form:
|
||||
raise ValueError("登录页面需要人工验证,请使用浏览器登录")
|
||||
form.update(username=username, password=password, _eventId="submit")
|
||||
self.transport.request("POST", LOGIN_URL, data=form)
|
||||
return self.check()
|
||||
|
||||
def clear(self):
|
||||
self._load({})
|
||||
self.vault.clear()
|
||||
self.valid = False
|
||||
self.message = "已清除本项目保存的登录凭据"
|
||||
return self.status()
|
||||
if self.browser_cancel:
|
||||
self.browser_cancel.set()
|
||||
with self.check_lock:
|
||||
self._load({})
|
||||
self.vault.clear()
|
||||
self.invalidate("已清除本项目保存的登录凭据", state="unverified")
|
||||
self.checked_at = None
|
||||
self.browser_state, self.browser_message = "idle", ""
|
||||
return self.status()
|
||||
|
||||
def browser(self):
|
||||
if self.browser_thread and self.browser_thread.is_alive():
|
||||
if self.browser_thread and self.browser_thread.is_alive() and not self.browser_cancel.is_set():
|
||||
return self.status()
|
||||
proxy = self.transport.router.resolve()
|
||||
cancelled = threading.Event()
|
||||
self.browser_cancel = cancelled
|
||||
self.browser_state = "opening"
|
||||
self.browser_message = "正在打开独立 Edge 登录窗口,完成登录后可关闭窗口"
|
||||
|
||||
@@ -149,24 +198,34 @@ class AuthService:
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
page.goto(LOGIN_URL, wait_until="domcontentloaded", timeout=60000)
|
||||
if cancelled.is_set():
|
||||
browser.close()
|
||||
return
|
||||
self.browser_state = "waiting"
|
||||
self.browser_message = "请在 Edge 完成学信网登录和验证码;窗口保持打开,工作台自动检测"
|
||||
deadline = time.monotonic() + 300
|
||||
while time.monotonic() < deadline:
|
||||
while time.monotonic() < deadline and not cancelled.is_set():
|
||||
page.wait_for_timeout(2000)
|
||||
if "yz.chsi.com.cn" in page.url:
|
||||
cookies = [c for c in context.cookies() if c["domain"].lstrip(".") == "chsi.com.cn" or c["domain"].endswith(".chsi.com.cn")]
|
||||
if cookies:
|
||||
self._load(cookies)
|
||||
if self.check()["valid"]:
|
||||
self.browser_state = "completed"
|
||||
self.browser_message = "已获取登录 Cookie,目录接口验证成功"
|
||||
browser.close()
|
||||
return
|
||||
with self.check_lock:
|
||||
if cancelled.is_set():
|
||||
break
|
||||
self._load(cookies)
|
||||
if self.check()["valid"] and not cancelled.is_set():
|
||||
self.browser_state = "completed"
|
||||
self.browser_message = "已获取登录 Cookie,目录接口验证成功"
|
||||
browser.close()
|
||||
return
|
||||
browser.close()
|
||||
if cancelled.is_set():
|
||||
return
|
||||
self.browser_state = "expired"
|
||||
self.browser_message = "登录等待超时,可再次打开浏览器"
|
||||
except Exception:
|
||||
if cancelled.is_set():
|
||||
return
|
||||
self.browser_state = "failed"
|
||||
self.browser_message = "登录窗口已关闭或连接失败,可重试或粘贴 Cookie"
|
||||
self.browser_thread = threading.Thread(target=run, daemon=True, name="browser-login")
|
||||
|
||||
+54
-25
@@ -6,7 +6,9 @@ from studio.network import RemoteUnavailable, RateLimited
|
||||
|
||||
|
||||
class LoginRequired(RuntimeError):
|
||||
pass
|
||||
def __init__(self, message, reason="login"):
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
class SourceYearMismatch(RuntimeError):
|
||||
@@ -16,8 +18,8 @@ class SourceYearMismatch(RuntimeError):
|
||||
class CatalogClient:
|
||||
BASE = "https://yz.chsi.com.cn/zsml/"
|
||||
|
||||
def __init__(self, transport):
|
||||
self.transport = transport
|
||||
def __init__(self, transport, auth=None):
|
||||
self.transport, self.auth = transport, auth
|
||||
|
||||
def verify_year(self, year):
|
||||
response = self.transport.request("GET", self.BASE)
|
||||
@@ -32,32 +34,53 @@ class CatalogClient:
|
||||
raise SourceYearMismatch(f"研招网当前公开目录为 {current} 年,任务选择了 {year} 年;历史目录请导入已有快照")
|
||||
return current
|
||||
|
||||
def pages(self, endpoint, payload, page_size=50):
|
||||
seen = set()
|
||||
for page in range(1, 10001):
|
||||
for attempt in range(3):
|
||||
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
|
||||
message = str(body.get('msg', ''))
|
||||
if body.get('flag') is not True and '验证码' in message:
|
||||
raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复')
|
||||
if body.get('flag') is not True and any(token in message for token in ('频繁', '稍后')):
|
||||
if attempt == 2:
|
||||
raise RateLimited('研招网业务限流持续出现,已暂停;请增加请求间隔后恢复')
|
||||
self.transport.sleep(5 * 2 ** attempt)
|
||||
continue
|
||||
break
|
||||
def _fetch_page(self, endpoint, payload):
|
||||
for attempt in range(3):
|
||||
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data=payload)
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
if '验证码' in response.text:
|
||||
raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复', reason="captcha") from None
|
||||
if re.search(r"passport/login|j_spring_cas_security_check|name=[\"']password[\"']", response.text) or '/passport/login' in response.url:
|
||||
raise LoginRequired("研招网返回登录页面,请在浏览器完成登录后恢复") from None
|
||||
raise RemoteUnavailable("研招目录接口未返回有效 JSON,请稍后重试") from None
|
||||
if not isinstance(body, dict):
|
||||
raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据")
|
||||
message = str(body.get('msg', ''))
|
||||
if body.get('flag') is not True and '验证码' in message:
|
||||
raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复', reason="captcha")
|
||||
if body.get('flag') is not True and any(token in message for token in ('频繁', '稍后')):
|
||||
if attempt == 2:
|
||||
raise RateLimited('研招网业务限流持续出现,已暂停;请增加请求间隔后恢复')
|
||||
self.transport.sleep(5 * 2 ** attempt)
|
||||
continue
|
||||
if body.get("flag") is not True:
|
||||
message = str(body.get("msg", ""))
|
||||
if any(token in message for token in ("登录", "登陆", "认证")):
|
||||
raise LoginRequired("研招网要求登录,请更新 Cookie 后恢复任务")
|
||||
raise LoginRequired("研招网要求登录,请重新验证或在浏览器登录后恢复任务")
|
||||
raise RemoteUnavailable("研招网目录接口未成功返回数据")
|
||||
data = body.get("msg") or {}
|
||||
data = body.get("msg")
|
||||
if not isinstance(data, dict) or not isinstance(data.get("list"), list):
|
||||
raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据")
|
||||
return data
|
||||
|
||||
def fetch_page(self, endpoint, payload):
|
||||
try:
|
||||
return self._fetch_page(endpoint, payload)
|
||||
except LoginRequired as exc:
|
||||
if not self.auth or exc.reason == "captcha":
|
||||
raise
|
||||
# A CAS cookie may still be valid while the catalogue session has expired.
|
||||
# Refresh once, then retry exactly the same page; never skip its records.
|
||||
status = self.auth.check()
|
||||
if not status["valid"]:
|
||||
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None
|
||||
return self._fetch_page(endpoint, payload)
|
||||
|
||||
def pages(self, endpoint, payload, page_size=10):
|
||||
seen, start = set(), 0
|
||||
for page in range(1, 10001):
|
||||
data = self.fetch_page(endpoint, {**payload, "curPage": page, "start": start, "pageSize": page_size})
|
||||
rows = data["list"]
|
||||
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
||||
if rows and signature in seen:
|
||||
@@ -66,8 +89,14 @@ class CatalogClient:
|
||||
yield from rows
|
||||
more = data.get("nextPageAvailable")
|
||||
total = integer(data.get("totalCount"))
|
||||
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and page * page_size >= total) or len(rows) < page_size):
|
||||
size = integer(data.get("size")) or page_size
|
||||
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size):
|
||||
return
|
||||
next_start = integer(data.get("startOfNextPage"))
|
||||
next_start = start + len(rows) if next_start is None else next_start
|
||||
if next_start <= start:
|
||||
raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页")
|
||||
start, page_size = next_start, size
|
||||
raise RemoteUnavailable("目录分页超过安全上限")
|
||||
|
||||
def schools(self, province, keyword=""):
|
||||
|
||||
+10
-2
@@ -15,8 +15,9 @@ class Shutdown(Exception):
|
||||
|
||||
|
||||
class JobManager:
|
||||
def __init__(self, database, settings, transport=None, runner=None):
|
||||
def __init__(self, database, settings, transport=None, runner=None, auth=None):
|
||||
self.db, self.settings, self.transport = database, settings, transport
|
||||
self.auth = auth
|
||||
self.runner = runner or self._collect
|
||||
self.controls = {}
|
||||
self.lock = threading.RLock()
|
||||
@@ -80,6 +81,8 @@ class JobManager:
|
||||
except Cancelled:
|
||||
self.db.job_update(jid, status="cancelled", message="已取消,已保存数据保留")
|
||||
except LoginRequired as exc:
|
||||
if self.auth:
|
||||
self.auth.invalidate(str(exc), state="captcha_required" if exc.reason == "captcha" else "login_required")
|
||||
self.db.job_update(jid, status="waiting_login", message=str(exc))
|
||||
self.db.log(jid, "warning", str(exc))
|
||||
except ProxyUnavailable as exc:
|
||||
@@ -186,8 +189,13 @@ class JobManager:
|
||||
self._scores(jid, params)
|
||||
|
||||
def _catalog(self, jid, params):
|
||||
client = CatalogClient(self.transport)
|
||||
client = CatalogClient(self.transport, auth=self.auth)
|
||||
client.verify_year(params["year"])
|
||||
if self.auth:
|
||||
status = self.auth.check()
|
||||
if not status["valid"]:
|
||||
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login")
|
||||
self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功")
|
||||
for province in params["provinces"]:
|
||||
self.checkpoint(jid)
|
||||
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
|
||||
|
||||
@@ -142,6 +142,7 @@ class Transport:
|
||||
self.sleep(min(30, 5 * (attempt + 1)))
|
||||
continue
|
||||
response.raise_for_status()
|
||||
self.local.response_route = proxy or "direct"
|
||||
return response
|
||||
except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout):
|
||||
if proxy:
|
||||
|
||||
Reference in New Issue
Block a user