From 2e3439dbb9a48c022254a1b539aef4b9b371c536 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 11 Oct 2026 15:28:14 +0800 Subject: [PATCH] Fix CHSI catalogue session verification and pagination --- README.md | 6 +- docs/verification.md | 16 ++- frontend/src/App.tsx | 44 +++++-- frontend/src/types.ts | 2 + studio/api.py | 3 +- studio/auth.py | 137 +++++++++++++++------ studio/connectors.py | 79 ++++++++---- studio/jobs.py | 12 +- studio/network.py | 1 + tests/test_auth_session.py | 242 +++++++++++++++++++++++++++++++++++++ tools/auth_browser_qa.py | 39 ++++++ 11 files changed, 496 insertions(+), 85 deletions(-) create mode 100644 tests/test_auth_session.py create mode 100644 tools/auth_browser_qa.py diff --git a/README.md b/README.md index e9b823a..4472747 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,13 @@ 1. 在「代理 IP 池」刷新,查看池内 HTTPS 候选和当前路由。默认连接 `http://127.0.0.1:5010`;服务目录为 `C:\Users\Jayson\Desktop\proxy_pool`。 2. 在「设置与登录」选择网络策略。优先代理在池为空时允许直连;严格代理没有可用 IP 时等待处理。备用代理每行一个 `IP:端口`。代理池存在候选并不保证该 IP 对所有目标网站可用,连接失败会切换并隔离。 -3. 点击「打开登录窗口」,在独立 Edge 窗口完成学信网登录和验证码。工作台会检测登录并保存 Cookie。也可粘贴已登录研招网的 Cookie 请求头,或使用账号密码登录。点击「验证目录访问」确认详情接口权限。 +3. 点击「打开登录窗口」,在独立 Edge 窗口完成学信网登录和验证码。工作台会同步研招目录登录会话、验证第二页访问权限,再加密保存 Cookie。也可粘贴已登录研招网的 Cookie 请求头,或使用账号密码登录。点击「验证目录访问」会重新同步并检查权限。 4. 在「采集任务」选择内容、目录年份、地区、专业和院校关键词。建议先按一个地区、一个专业或一所院校验证,再创建全国任务。专业代码可以从内置目录选择,也可直接输入 6 位代码。 5. 查看任务日志。暂停可继续;需要登录、代理或限流解除时,处理后点击恢复;取消保留已有数据。关闭或重启服务会保留可恢复断点。 -研招网学校、专业索引有时可以匿名读取,但方向与科目详情需要登录。程序验证实际业务返回状态,不将 HTTP 200 当作登录成功。目录采集先核对研招页面配置中的年份,避免将当前目录标为历史年份。 +研招网目录第一页可匿名返回,包括部分研究方向;不能用第一页判断登录成功。工作台访问研招目录的登录入口,将已有学信网 Cookie 同步到目录会话,再验证需要登录的第二页。采集开始/恢复时自动完成此检查;途中遇到登录提示,只同步一次并重试原页,失败后保留断点并提示重新登录。验证码仍需在浏览器人工完成,限流和网络失败会单独提示。登录卡片会持续更新状态;切换网络出口后需要重新验证,原 Cookie 保留。 + +分页使用接口返回的下一页位置和实际页大小。研招网可能将每页条数限制为 10,程序按实际位置继续,避免请求较大页时遗漏记录。目录采集先核对研招页面配置中的年份,避免将当前目录标为历史年份。 ## 数据查询 diff --git a/docs/verification.md b/docs/verification.md index 5aa914d..2da89a0 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -2,14 +2,14 @@ 2026-10-11,Windows、本地独立虚拟环境,Python 3.12.13 / Node 24.16。 -- 后端:`python -m pytest -q`,49 passed。Starlette TestClient 提示一条 httpx 迁移 warning,测试退出 0,生产服务不使用 TestClient。 +- 后端:`python -m pytest -q`,62 passed。Starlette TestClient 提示一条 httpx 迁移 warning,测试退出 0,生产服务不使用 TestClient。 - 前端:`npm test`,3 passed;`npm run build`,TypeScript 检查与 Vite 生产构建退出 0。 - Edge 实际浏览器:独立端口 8766、独立 `.runtime/qa/db`,验证导入→科目筛选→收藏→对比→历史趋势→Excel 下载;任务/代理/设置页面;设置保存;390px 手机宽度无页面溢出;无 pageerror。测试数据均标明「自动验收样例」,没有写入正式数据库。 - 启动脚本:Windows PowerShell 首启、重复启动、停止、重新启动均退出 0;正式服务在 127.0.0.1:8765,健康检查应用身份和项目路径匹配。 -- 真实数据源:proxy_pool `/count/` 正常响应;研招页面 `curYear=2027` 可核实;匿名北京大学学校及 081200 专业索引可返回,方向接口要求登录,按 LoginRequired 处理;掌上考研精确匹配北京大学,2025 年 081200 查询返回 1 条门类参考线和 2 条历史计划。真实探测只读取少量公开数据,不写入正式目录。 +- 真实数据源:proxy_pool `/count/` 正常响应;研招页面 `curYear=2027` 可核实;研招目录第一页(包括 081200 方向)可以匿名返回,第二页需要登录,详见下方登录会话修复;掌上考研精确匹配北京大学,2025 年 081200 查询返回 1 条门类参考线和 2 条历史计划。真实探测不写入正式目录。 - 独立只读审查:复现并修复专项线覆盖、同名跨域 Cookie、任务失败恢复竞态、服务退出被误取消、文件往返丢失身份字段、学院人数误升格、YAM 种子误识别、院校关键词范围、参考书 JSON 字符串、业务限流等问题,补充相应回归。 -自动验收未使用真人学信网账号,未对全部院校进行实网抓取。登录后的全国采集能力取决于账号状态、代理可用性、开放目录年份和数据源限流。 +初次自动验收未使用真人学信网账号。后续登录修复使用用户已保存的 Cookie 进行了有限的只读分页验证,未对全部院校进行实网抓取。登录后的全国采集能力取决于账号状态、代理可用性、开放目录年份和数据源限流。 复跑浏览器验收(只使用测试库): @@ -28,3 +28,13 @@ - `tools/proxy_service_qa.py`:复制管理脚本到临时测试目录,启动无抓取行为的假 API / 调度器;实际验证普通权限 start → 重复 start → status → stop → API 关闭,PASS。仅只读检查现有 Redis,正式代理池未停止。 - 独立代码审查确认真实脚本入口校验和一次操作使用同一配置快照;最终无 Critical / Important。 - 正式界面 Edge 验收:显示 API 正常、两个进程的管理员权限、外部 Redis;点击检查状态成功,点击启动复用现有进程,无乱码或原错误堆栈,状态检查与复用保留当前路由,停止按钮给出权限限制说明。390px 代理页无溢出,无浏览器错误;截图保存为「代理池修复预览.png」。完整独立测试库浏览器流程亦再次通过。 + +登录会话与分页修复(2026-10-11): + +- 复现:两次任务都在北京大学的前 10 个专业、107 条记录后收到「请登录」。匿名方向第一页也返回 flag=true;原来的第一页验证不能证明登录成功,且任务的登录失败没有更新登录卡片。 +- 使用相同代理和已保存 Cookie,只读请求专业第二页仍提示登录;访问 `/zsml/a/dw.do` 同步 CAS 会话后,专业第二页和方向第二页均正常。没有要求用户重新提供凭据,没有输出 Cookie 或密码。 +- 请求 pageSize=50 时,真实接口返回 size=10、startOfNextPage=10。修复按实际下一页位置继续,防止用 50 作为偏移跳过记录。 +- 13 项新增行为回归覆盖匿名第一页误判、已有 SSO Cookie 初始化、服务端页大小限制、原页自动恢复、恢复次数限制、任务与登录状态同步及断点保留、限流分类、人工验证码、网络出口变化,以及验证中切换出口、清除/替换凭据、新验证码拦截覆盖旧证明的并发情形。 +- 修正后实网只读验证:15 个专业跨两页、081200 的全部 15 条原始研究方向跨两页,当前代理保持不变,正式数据库写入 0 条。 +- `tools/auth_browser_qa.py`:模拟登录已完成后收到失效状态,原版因不再轮询而失败;修正后登录卡片自动显示「需重新登录」并清除旧成功提示。独立端口 8766 和正式端口的 Edge 验收通过;390px 无溢出,无浏览器错误。原完整工作区浏览器验收再次通过。 +- 正式服务重启后,使用已有加密 Cookie 再次验证第二页成功,登录卡片显示最新检查结果。原任务保留 107 条记录和 10 个已完成单元,仍可从断点恢复,未自动启动全国采集。截图为「登录修复预览.png」。最终 62 项后端测试、3 项前端测试和生产构建通过;独立只读审查未发现剩余 Critical / Important。 diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 8ed71b0..9af0386 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1857,16 +1857,22 @@ function SettingsPanel({ notify }: { notify: Notify }) { const browserWaiting = auth && ["opening", "waiting"].includes(auth.browser_state); useEffect(() => { - if (!browserWaiting) return; - const timer = setInterval( - () => - api("/auth") - .then(setAuth) - .catch(() => {}), - 2000, - ); - return () => clearInterval(timer); - }, [browserWaiting]); + if (busy) return; + let active = true; + const refreshAuth = () => + api("/auth") + .then((result) => { + if (active) setAuth(result); + }) + .catch(() => {}); + const timer = setInterval(refreshAuth, 2000); + window.addEventListener("focus", refreshAuth); + return () => { + active = false; + clearInterval(timer); + window.removeEventListener("focus", refreshAuth); + }; + }, [busy]); async function save(e: React.FormEvent) { e.preventDefault(); setBusy("save"); @@ -2020,12 +2026,24 @@ function SettingsPanel({ notify }: { notify: Notify }) { {auth?.valid ? "已验证" - : auth?.saved - ? "已保存,待验证" - : "尚未登录"} + : auth?.state === "captcha_required" + ? "需人工验证" + : auth?.state === "login_required" + ? "需重新登录" + : auth?.saved + ? "已保存,待验证" + : "尚未登录"}

{auth?.message}

+ {auth?.checked_at && ( +

+ 最近检查: + {new Date(auth.checked_at).toLocaleString("zh-CN", { + hour12: false, + })} +

+ )}
diff --git a/frontend/src/types.ts b/frontend/src/types.ts index c725a71..6a07679 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -138,6 +138,8 @@ export type PoolManageResult = { export type Auth = { saved: boolean; valid: boolean; + state: "unverified" | "verified" | "login_required" | "captcha_required"; + checked_at: string | null; message: string; browser_state: string; browser_message: string; diff --git a/studio/api.py b/studio/api.py index 5549ec8..9ba8b31 100644 --- a/studio/api.py +++ b/studio/api.py @@ -28,7 +28,7 @@ def create_app(data_dir=None): router = ProxyRouter(settings) transport = Transport(settings, router) auth = AuthService(transport, CookieVault(directory)) - jobs = JobManager(db, settings, transport) + jobs = JobManager(db, settings, transport, auth=auth) if (ROOT / "data/schools.json").exists(): db.index_schools(json.loads((ROOT / "data/schools.json").read_text(encoding="utf-8"))) @@ -41,6 +41,7 @@ def create_app(data_dir=None): app.state.db, app.state.jobs = db, jobs app.state.settings, app.state.router = settings, router app.state.pool = pool + app.state.auth = auth @app.middleware("http") async def local_access(request: Request, call_next): diff --git a/studio/auth.py b/studio/auth.py index 7684210..3f79be2 100644 --- a/studio/auth.py +++ b/studio/auth.py @@ -6,8 +6,12 @@ import threading import time from pathlib import Path from bs4 import BeautifulSoup +from studio.connectors import CatalogClient, LoginRequired +from studio.domain import now +from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited LOGIN_URL = "https://account.chsi.com.cn/passport/login?entrytype=yzgr&service=https%3A%2F%2Fyz.chsi.com.cn%2Fj_spring_cas_security_check" +CATALOG_ENTRY = "https://yz.chsi.com.cn/zsml/a/dw.do" class Blob(ctypes.Structure): @@ -55,9 +59,16 @@ class AuthService: self.transport, self.vault = transport, vault self.message = "尚未验证登录状态" self.valid = False + self.state = "unverified" + self.checked_at = None + self.verified_route = None + self.revision = 0 + self.state_lock = threading.RLock() + self.check_lock = threading.RLock() self.browser_state = "idle" self.browser_message = "" self.browser_thread = None + self.browser_cancel = None try: self._load(self.vault.load()) except (RuntimeError, OSError, ValueError): @@ -80,7 +91,19 @@ class AuthService: return [{"name": c.name, "value": c.value, "domain": c.domain, "path": c.path, "secure": c.secure, "expires": c.expires} for c in self.transport.session.cookies] def status(self): - return {"saved": self.vault.path.exists(), "valid": self.valid, "message": self.message, "browser_state": self.browser_state, "browser_message": self.browser_message} + with self.state_lock: + if self.valid and self.verified_route != self.transport.router.status()["route"]: + self.invalidate("网络出口已变化,请验证目录访问后继续采集", state="unverified") + return {"saved": self.vault.path.exists(), "valid": self.valid, "state": self.state, "message": self.message, "checked_at": self.checked_at, "browser_state": self.browser_state, "browser_message": self.browser_message} + + def invalidate(self, message, state="login_required"): + with self.state_lock: + self.revision += 1 + self.valid = False + self.state, self.message = state, message + self.verified_route = None + if self.browser_state == "completed": + self.browser_state, self.browser_message = "idle", "" def cookie(self, text): if not text.strip() or "\n" in text or "\r" in text: @@ -94,50 +117,76 @@ class AuthService: cookies[name] = value if not cookies: raise ValueError("未识别到 Cookie 键值") - self._load(cookies) - self.vault.save(cookies) - self.valid = False - self.message = "Cookie 已加密保存,点击验证确认目录访问权限" - return self.status() + if self.browser_cancel: + self.browser_cancel.set() + with self.check_lock: + self._load(cookies) + self.vault.save(cookies) + self.invalidate("Cookie 已加密保存,点击验证确认目录访问权限", state="unverified") + self.checked_at = None + self.browser_state, self.browser_message = "idle", "" + return self.status() def check(self): - # The school/major indexes can be public while direction details require login. - response = self.transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/yjfxs.do", data={"dwdm": "10001", "zydm": "081200", "zymc": "计算机科学与技术", "curPage": 1, "start": 0, "pageSize": 1, "dwlxs[0]": "all"}) - try: - body = response.json() - except ValueError: - self.valid = False - self.message = "目录接口未返回有效数据,可能需要登录或验证码" + with self.check_lock: + with self.state_lock: + self.invalidate("正在同步研招目录会话并验证访问权限", state="unverified") + revision = self.revision + try: + # The first ten rows are available anonymously, even for directions. + # Enter the CAS-protected page to bind cookies, then prove page-two access. + self.transport.request("GET", CATALOG_ENTRY) + CatalogClient(self.transport).fetch_page("yjfxs", {"dwdm": "10001", "zydm": "081200", "zymc": "计算机科学与技术", "curPage": 2, "start": 10, "pageSize": 10, "dwlxs[0]": "all"}) + proof_route = self.transport.local.response_route + self.vault.save(self._snapshot()) + except LoginRequired as exc: + self.invalidate(str(exc), state="captcha_required" if exc.reason == "captcha" else "login_required") + except (ProxyUnavailable, RemoteUnavailable, RateLimited): + self.invalidate("暂时无法验证研招目录,请检查网络或稍后重试;已保存的 Cookie 保留", state="unverified") + raise + else: + with self.state_lock: + # A later crawler login/captcha failure supersedes this older proof. + if self.revision == revision: + self.valid, self.state = True, "verified" + self.message = "研招目录登录会话已同步,第二页访问验证成功" + self.verified_route = proof_route + finally: + self.checked_at = now() return self.status() - self.valid = body.get("flag") is True and isinstance(body.get("msg"), dict) - self.message = "研招目录接口验证成功" if self.valid else "登录未生效或数据源限流,请在浏览器完成登录后重试" - if self.valid: - self.vault.save(self._snapshot()) - return self.status() def login(self, username, password): if not username or not password: raise ValueError("请输入学信网账号和密码") - response = self.transport.request("GET", LOGIN_URL) - soup = BeautifulSoup(response.text, "html.parser") - form = {node.get("name"): node.get("value", "") for node in soup.select("input[type=hidden][name]")} - if "lt" not in form or "execution" not in form: - raise ValueError("登录页面需要人工验证,请使用浏览器登录") - form.update(username=username, password=password, _eventId="submit") - self.transport.request("POST", LOGIN_URL, data=form) - return self.check() + if self.browser_cancel: + self.browser_cancel.set() + with self.check_lock: + response = self.transport.request("GET", LOGIN_URL) + soup = BeautifulSoup(response.text, "html.parser") + form = {node.get("name"): node.get("value", "") for node in soup.select("input[type=hidden][name]")} + if "lt" not in form or "execution" not in form: + raise ValueError("登录页面需要人工验证,请使用浏览器登录") + form.update(username=username, password=password, _eventId="submit") + self.transport.request("POST", LOGIN_URL, data=form) + return self.check() def clear(self): - self._load({}) - self.vault.clear() - self.valid = False - self.message = "已清除本项目保存的登录凭据" - return self.status() + if self.browser_cancel: + self.browser_cancel.set() + with self.check_lock: + self._load({}) + self.vault.clear() + self.invalidate("已清除本项目保存的登录凭据", state="unverified") + self.checked_at = None + self.browser_state, self.browser_message = "idle", "" + return self.status() def browser(self): - if self.browser_thread and self.browser_thread.is_alive(): + if self.browser_thread and self.browser_thread.is_alive() and not self.browser_cancel.is_set(): return self.status() proxy = self.transport.router.resolve() + cancelled = threading.Event() + self.browser_cancel = cancelled self.browser_state = "opening" self.browser_message = "正在打开独立 Edge 登录窗口,完成登录后可关闭窗口" @@ -149,24 +198,34 @@ class AuthService: context = browser.new_context() page = context.new_page() page.goto(LOGIN_URL, wait_until="domcontentloaded", timeout=60000) + if cancelled.is_set(): + browser.close() + return self.browser_state = "waiting" self.browser_message = "请在 Edge 完成学信网登录和验证码;窗口保持打开,工作台自动检测" deadline = time.monotonic() + 300 - while time.monotonic() < deadline: + while time.monotonic() < deadline and not cancelled.is_set(): page.wait_for_timeout(2000) if "yz.chsi.com.cn" in page.url: cookies = [c for c in context.cookies() if c["domain"].lstrip(".") == "chsi.com.cn" or c["domain"].endswith(".chsi.com.cn")] if cookies: - self._load(cookies) - if self.check()["valid"]: - self.browser_state = "completed" - self.browser_message = "已获取登录 Cookie,目录接口验证成功" - browser.close() - return + with self.check_lock: + if cancelled.is_set(): + break + self._load(cookies) + if self.check()["valid"] and not cancelled.is_set(): + self.browser_state = "completed" + self.browser_message = "已获取登录 Cookie,目录接口验证成功" + browser.close() + return browser.close() + if cancelled.is_set(): + return self.browser_state = "expired" self.browser_message = "登录等待超时,可再次打开浏览器" except Exception: + if cancelled.is_set(): + return self.browser_state = "failed" self.browser_message = "登录窗口已关闭或连接失败,可重试或粘贴 Cookie" self.browser_thread = threading.Thread(target=run, daemon=True, name="browser-login") diff --git a/studio/connectors.py b/studio/connectors.py index 07a283b..76a2377 100644 --- a/studio/connectors.py +++ b/studio/connectors.py @@ -6,7 +6,9 @@ from studio.network import RemoteUnavailable, RateLimited class LoginRequired(RuntimeError): - pass + def __init__(self, message, reason="login"): + super().__init__(message) + self.reason = reason class SourceYearMismatch(RuntimeError): @@ -16,8 +18,8 @@ class SourceYearMismatch(RuntimeError): class CatalogClient: BASE = "https://yz.chsi.com.cn/zsml/" - def __init__(self, transport): - self.transport = transport + def __init__(self, transport, auth=None): + self.transport, self.auth = transport, auth def verify_year(self, year): response = self.transport.request("GET", self.BASE) @@ -32,32 +34,53 @@ class CatalogClient: raise SourceYearMismatch(f"研招网当前公开目录为 {current} 年,任务选择了 {year} 年;历史目录请导入已有快照") return current - def pages(self, endpoint, payload, page_size=50): - seen = set() - for page in range(1, 10001): - for attempt in range(3): - response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size}) - try: - body = response.json() - except ValueError: - raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None - message = str(body.get('msg', '')) - if body.get('flag') is not True and '验证码' in message: - raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复') - if body.get('flag') is not True and any(token in message for token in ('频繁', '稍后')): - if attempt == 2: - raise RateLimited('研招网业务限流持续出现,已暂停;请增加请求间隔后恢复') - self.transport.sleep(5 * 2 ** attempt) - continue - break + def _fetch_page(self, endpoint, payload): + for attempt in range(3): + response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data=payload) + try: + body = response.json() + except ValueError: + if '验证码' in response.text: + raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复', reason="captcha") from None + if re.search(r"passport/login|j_spring_cas_security_check|name=[\"']password[\"']", response.text) or '/passport/login' in response.url: + raise LoginRequired("研招网返回登录页面,请在浏览器完成登录后恢复") from None + raise RemoteUnavailable("研招目录接口未返回有效 JSON,请稍后重试") from None + if not isinstance(body, dict): + raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据") + message = str(body.get('msg', '')) + if body.get('flag') is not True and '验证码' in message: + raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复', reason="captcha") + if body.get('flag') is not True and any(token in message for token in ('频繁', '稍后')): + if attempt == 2: + raise RateLimited('研招网业务限流持续出现,已暂停;请增加请求间隔后恢复') + self.transport.sleep(5 * 2 ** attempt) + continue if body.get("flag") is not True: - message = str(body.get("msg", "")) if any(token in message for token in ("登录", "登陆", "认证")): - raise LoginRequired("研招网要求登录,请更新 Cookie 后恢复任务") + raise LoginRequired("研招网要求登录,请重新验证或在浏览器登录后恢复任务") raise RemoteUnavailable("研招网目录接口未成功返回数据") - data = body.get("msg") or {} + data = body.get("msg") if not isinstance(data, dict) or not isinstance(data.get("list"), list): raise RemoteUnavailable("研招网目录响应结构已变化,未写入不完整数据") + return data + + def fetch_page(self, endpoint, payload): + try: + return self._fetch_page(endpoint, payload) + except LoginRequired as exc: + if not self.auth or exc.reason == "captcha": + raise + # A CAS cookie may still be valid while the catalogue session has expired. + # Refresh once, then retry exactly the same page; never skip its records. + status = self.auth.check() + if not status["valid"]: + raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None + return self._fetch_page(endpoint, payload) + + def pages(self, endpoint, payload, page_size=10): + seen, start = set(), 0 + for page in range(1, 10001): + data = self.fetch_page(endpoint, {**payload, "curPage": page, "start": start, "pageSize": page_size}) rows = data["list"] signature = json.dumps(rows, ensure_ascii=False, sort_keys=True) if rows and signature in seen: @@ -66,8 +89,14 @@ class CatalogClient: yield from rows more = data.get("nextPageAvailable") total = integer(data.get("totalCount")) - if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and page * page_size >= total) or len(rows) < page_size): + size = integer(data.get("size")) or page_size + if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size): return + next_start = integer(data.get("startOfNextPage")) + next_start = start + len(rows) if next_start is None else next_start + if next_start <= start: + raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页") + start, page_size = next_start, size raise RemoteUnavailable("目录分页超过安全上限") def schools(self, province, keyword=""): diff --git a/studio/jobs.py b/studio/jobs.py index b87d620..226d43e 100644 --- a/studio/jobs.py +++ b/studio/jobs.py @@ -15,8 +15,9 @@ class Shutdown(Exception): class JobManager: - def __init__(self, database, settings, transport=None, runner=None): + def __init__(self, database, settings, transport=None, runner=None, auth=None): self.db, self.settings, self.transport = database, settings, transport + self.auth = auth self.runner = runner or self._collect self.controls = {} self.lock = threading.RLock() @@ -80,6 +81,8 @@ class JobManager: except Cancelled: self.db.job_update(jid, status="cancelled", message="已取消,已保存数据保留") except LoginRequired as exc: + if self.auth: + self.auth.invalidate(str(exc), state="captcha_required" if exc.reason == "captcha" else "login_required") self.db.job_update(jid, status="waiting_login", message=str(exc)) self.db.log(jid, "warning", str(exc)) except ProxyUnavailable as exc: @@ -186,8 +189,13 @@ class JobManager: self._scores(jid, params) def _catalog(self, jid, params): - client = CatalogClient(self.transport) + client = CatalogClient(self.transport, auth=self.auth) client.verify_year(params["year"]) + if self.auth: + status = self.auth.check() + if not status["valid"]: + raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") + self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功") for province in params["provinces"]: self.checkpoint(jid) self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校") diff --git a/studio/network.py b/studio/network.py index 29f3fd7..464a27d 100644 --- a/studio/network.py +++ b/studio/network.py @@ -142,6 +142,7 @@ class Transport: self.sleep(min(30, 5 * (attempt + 1))) continue response.raise_for_status() + self.local.response_route = proxy or "direct" return response except (requests.exceptions.ProxyError, requests.exceptions.ConnectionError, requests.exceptions.Timeout): if proxy: diff --git a/tests/test_auth_session.py b/tests/test_auth_session.py new file mode 100644 index 0000000..9d16682 --- /dev/null +++ b/tests/test_auth_session.py @@ -0,0 +1,242 @@ +"""CHSI exposes page one publicly; page two proves the catalogue session.""" +import threading +import pytest + +from studio.auth import AuthService, CookieVault +from studio.connectors import CatalogClient, LoginRequired +from studio.database import Database +from studio.jobs import JobManager +from studio.network import RateLimited +from tests.test_network import Remote, Response, setup_network + + +class CatalogSession(Remote): + def __init__(self, accepts_sso=True): + super().__init__() + self.accepts_sso = accepts_sso + self.authorized = False + self.calls = [] + self.blocked = None + + def request(self, method, url, **kwargs): + data = kwargs.get('data', {}) + self.calls.append((method, url, data.copy())) + if method == 'GET': + if url.endswith('/a/dw.do'): + self.authorized = self.accepts_sso + return Response({}, text='2027年硕士专业目录查询') + if self.blocked: + return Response({'flag': False, 'msg': self.blocked}) + start = int(data.get('start', 0)) + if start > 0 and not self.authorized: + return Response({'flag': False, 'msg': '请登录'}) + # The server caps the requested 50 rows at ten and supplies its next offset. + rows = [{'dwdm': str(10000 + i)} for i in range(start, min(start + 10, 23))] + return Response({'flag': True, 'msg': {'list': rows, 'size': 10, + 'startOfNextPage': start + 10, 'totalCount': 23, + 'nextPageAvailable': start + 10 < 23}}) + + +def service(tmp_path, remote): + settings, router, transport = setup_network(tmp_path, remote=remote) + auth = AuthService(transport, CookieVault(tmp_path)) + auth.cookie('CASTGC=fixture-sso-cookie') + return auth, settings, router, transport + + +def test_public_first_page_is_not_valid_login_evidence(tmp_path): + remote = CatalogSession(accepts_sso=False) + auth, _, _, _ = service(tmp_path, remote) + result = auth.check() + assert result['valid'] is False + assert result['saved'] is True + assert any(int(data.get('start', 0)) > 0 for method, _, data in remote.calls if method == 'POST') + + +def test_existing_sso_cookie_is_bound_before_testing_page_two(tmp_path): + remote = CatalogSession() + auth, _, _, _ = service(tmp_path, remote) + assert auth.check()['valid'] is True + assert remote.calls[0][0] == 'GET' + assert remote.calls[0][1].endswith('/a/dw.do') + assert int(remote.calls[-1][2]['start']) == 10 + + +def test_server_pagination_offsets_keep_every_row_when_page_size_is_capped(tmp_path): + remote = CatalogSession() + remote.authorized = True + _, _, transport = setup_network(tmp_path, remote=remote) + rows = list(CatalogClient(transport).pages('dws', {}, page_size=50)) + assert [r['dwdm'] for r in rows] == [str(10000 + i) for i in range(23)] + assert [data['start'] for _, _, data in remote.calls] == [0, 10, 20] + + +def test_sso_recovery_retries_the_same_failed_page_without_skipping_rows(tmp_path): + remote = CatalogSession() + auth, _, _, transport = service(tmp_path, remote) + rows = list(CatalogClient(transport, auth=auth).pages('dws', {})) + assert [r['dwdm'] for r in rows] == [str(10000 + i) for i in range(23)] + assert auth.status()['valid'] is True + assert sum(url.endswith('/a/dw.do') for _, url, _ in remote.calls) == 1 + + +def test_unrecoverable_login_stops_after_one_sso_attempt(tmp_path): + remote = CatalogSession(accepts_sso=False) + auth, _, _, transport = service(tmp_path, remote) + with pytest.raises(LoginRequired): + list(CatalogClient(transport, auth=auth).pages('dws', {})) + assert sum(url.endswith('/a/dw.do') for _, url, _ in remote.calls) == 1 + assert auth.status()['valid'] is False + + +def test_job_login_failure_invalidates_verified_badge_and_preserves_checkpoint(tmp_path): + auth, settings, _, transport = service(tmp_path, CatalogSession()) + assert auth.check()['valid'] is True + auth.browser_state = 'completed' + auth.browser_message = '已获取登录 Cookie,目录接口验证成功' + db = Database(tmp_path / 'studio.db') + + def runner(jid, params, manager): + manager.complete_unit(jid, 'already-saved') + db.job_update(jid, records=107) + raise LoginRequired('研招网要求登录,请重新验证后恢复任务') + + jobs = JobManager(db, settings, transport, runner=runner, auth=auth) + try: + job = jobs.start({'mode': 'catalog', 'year': 2027}) + jobs.controls[job['id']]['thread'].join(timeout=3) + result = jobs.get(job['id']) + assert result['status'] == 'waiting_login' + assert result['records'] == 107 and result['completed_units'] == 1 + status = auth.status() + assert status['valid'] is False + assert status['state'] == 'login_required' + assert status['saved'] is True + assert status['browser_message'] == '' + finally: + jobs.close() + + +def test_rate_limiting_during_validation_is_not_reported_as_bad_cookie(tmp_path): + remote = CatalogSession() + auth, _, _, _ = service(tmp_path, remote) + assert auth.check()['valid'] is True + remote.blocked = '访问太频繁,请稍后重试' + with pytest.raises(RateLimited): + auth.check() + assert auth.status()['valid'] is False + assert auth.status()['state'] == 'unverified' + assert auth.status()['saved'] is True + + +def test_captcha_requires_human_verification_without_sso_retry(tmp_path): + remote = CatalogSession() + auth, _, _, transport = service(tmp_path, remote) + remote.blocked = '需要验证码' + with pytest.raises(LoginRequired): + list(CatalogClient(transport, auth=auth).pages('dws', {})) + assert not any(url.endswith('/a/dw.do') for _, url, _ in remote.calls) + + +def test_changing_route_invalidates_previous_verification_without_deleting_cookie(tmp_path): + auth, _, router, _ = service(tmp_path, CatalogSession()) + assert auth.check()['valid'] is True + router.rotate() + status = auth.status() + assert status['valid'] is False + assert status['state'] == 'unverified' + assert status['saved'] is True + + +def test_route_changed_after_proof_is_not_marked_as_verified(tmp_path): + auth, _, router, _ = service(tmp_path, CatalogSession()) + original_save = auth.vault.save + + def switch_before_saving(cookies): + router.rotate() + original_save(cookies) + + auth.vault.save = switch_before_saving + status = auth.check() + assert status['valid'] is False + assert status['state'] == 'unverified' + assert status['saved'] is True + + +def test_later_captcha_invalidation_is_not_overwritten_by_pending_verification(tmp_path): + auth, _, _, _ = service(tmp_path, CatalogSession()) + before_save, finish_save = threading.Event(), threading.Event() + original_save = auth.vault.save + + def delayed_save(cookies): + before_save.set() + assert finish_save.wait(3) + original_save(cookies) + + auth.vault.save = delayed_save + result = [] + checking = threading.Thread(target=lambda: result.append(auth.check())) + checking.start() + try: + assert before_save.wait(3) + auth.invalidate('采集中发现验证码,请在浏览器完成人工验证', state='captcha_required') + finally: + finish_save.set() + checking.join(timeout=3) + assert not checking.is_alive() + assert result[0]['valid'] is False + assert result[0]['state'] == 'captcha_required' + assert '验证码' in result[0]['message'] + assert result[0]['saved'] is True + + +@pytest.mark.parametrize('action', ['clear', 'replace']) +def test_pending_check_cannot_override_a_later_credential_change(tmp_path, action): + auth, _, _, _ = service(tmp_path, CatalogSession()) + before_save, finish_save, change_started = threading.Event(), threading.Event(), threading.Event() + original_save = auth.vault.save + + def delayed_save(cookies): + # Only delay the verifier, not the user's later replacement. + if threading.current_thread().name == 'verification': + before_save.set() + assert finish_save.wait(3) + original_save(cookies) + + auth.vault.save = delayed_save + errors = [] + + def check(): + try: + auth.check() + except Exception as error: + errors.append(error) + + def change(): + change_started.set() + if action == 'clear': + auth.clear() + else: + auth.cookie('CASTGC=new-fixture-cookie') + + checking = threading.Thread(target=check, name='verification') + changing = threading.Thread(target=change, name='credential-change') + checking.start() + try: + assert before_save.wait(3) + changing.start() + assert change_started.wait(3) + changing.join(timeout=.1) + finally: + finish_save.set() + checking.join(timeout=3) + if changing.ident: + changing.join(timeout=3) + assert not checking.is_alive() and not changing.is_alive() + assert not errors, errors + status = auth.status() + assert status['valid'] is False + assert status['state'] == 'unverified' + assert status['saved'] is (action == 'replace') + if action == 'replace': + assert auth.vault.load()['CASTGC'] == 'new-fixture-cookie' diff --git a/tools/auth_browser_qa.py b/tools/auth_browser_qa.py new file mode 100644 index 0000000..ab62d39 --- /dev/null +++ b/tools/auth_browser_qa.py @@ -0,0 +1,39 @@ +"""Read-only UI regression: a completed login must keep tracking auth state.""" +import argparse +import json +from pathlib import Path +from playwright.sync_api import sync_playwright, expect + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--url', default='http://127.0.0.1:8765') + args = parser.parse_args() + status = {'saved': True, 'valid': True, 'state': 'verified', + 'message': '研招目录第二页接口验证成功', 'checked_at': '2026-10-11T15:00:00+08:00', + 'browser_state': 'completed', 'browser_message': '已获取登录 Cookie,目录接口验证成功'} + with sync_playwright() as p: + browser = p.chromium.launch(channel='msedge', headless=True) + page = browser.new_page(viewport={'width': 1480, 'height': 1050}) + errors = [] + page.on('pageerror', lambda error: errors.append(str(error))) + page.route('**/api/auth', lambda route: route.fulfill(content_type='application/json', body=json.dumps(status))) + page.goto(args.url, wait_until='networkidle') + page.get_by_role('navigation').get_by_role('button', name='设置与登录').click() + card = page.locator('.auth-card') + expect(card).to_contain_text('已验证') + status.update(valid=False, state='login_required', message='研招网要求登录,请重新验证后恢复任务', browser_state='idle', browser_message='') + expect(card).to_contain_text('需重新登录', timeout=7000) + expect(card).not_to_contain_text('目录接口验证成功') + page.set_viewport_size({'width': 390, 'height': 844}) + assert page.evaluate('document.documentElement.scrollWidth <= window.innerWidth'), 'mobile overflow' + assert not errors, errors + out = Path(__file__).resolve().parents[1] / '.runtime/qa' + out.mkdir(parents=True, exist_ok=True) + page.screenshot(path=str(out / 'auth-state.png'), full_page=True) + browser.close() + print('PASS: completed login polls invalidation, badge/message stay consistent, mobile layout, no browser errors') + + +if __name__ == '__main__': + main()