fix: resume catalog jobs without repeated checkpoint queries

This commit is contained in:
2026-10-11 21:18:49 +08:00
parent c4a71e88a1
commit 4c510759fb
9 changed files with 309 additions and 13 deletions
+2
View File
@@ -28,6 +28,8 @@ Debian 12 / MariaDB 服务器模式、HTTPS 访问与代理池后台运行见 [
4. 在「采集任务」选择内容、目录年份、地区、专业和院校关键词。建议先按一个地区、一个专业或一所院校验证,再创建全国任务。专业代码可以从内置目录选择,也可直接输入 6 位代码。 4. 在「采集任务」选择内容、目录年份、地区、专业和院校关键词。建议先按一个地区、一个专业或一所院校验证,再创建全国任务。专业代码可以从内置目录选择,也可直接输入 6 位代码。
5. 查看任务日志。暂停可继续;需要登录、代理或限流解除时,处理后点击恢复;取消保留已有数据。关闭或重启服务会保留可恢复断点。 5. 查看任务日志。暂停可继续;需要登录、代理或限流解除时,处理后点击恢复;取消保留已有数据。关闭或重启服务会保留可恢复断点。
任务会保存本次采集的院校和专业列表分页,登录恢复或服务重启后复用,并一次加载已完成单元。界面显示当前核对的院校和正在请求的页数。修复前创建的任务首次恢复仍需建立列表缓存;新建任务会获取新列表。研究方向和登录验证保持实时请求,异常分页不会写入缓存。
研招网目录第一页可匿名返回,包括部分研究方向;不能用第一页判断登录成功。工作台访问研招目录的登录入口,将已有学信网 Cookie 同步到目录会话,再验证需要登录的第二页。采集开始/恢复时自动完成此检查;途中遇到登录提示,只同步一次并重试原页,失败后保留断点并提示重新登录。验证码仍需在浏览器人工完成,限流和网络失败会单独提示。登录卡片会持续更新状态;切换网络出口后需要重新验证,原 Cookie 保留。 研招网目录第一页可匿名返回,包括部分研究方向;不能用第一页判断登录成功。工作台访问研招目录的登录入口,将已有学信网 Cookie 同步到目录会话,再验证需要登录的第二页。采集开始/恢复时自动完成此检查;途中遇到登录提示,只同步一次并重试原页,失败后保留断点并提示重新登录。验证码仍需在浏览器人工完成,限流和网络失败会单独提示。登录卡片会持续更新状态;切换网络出口后需要重新验证,原 Cookie 保留。
分页使用接口返回的下一页位置和实际页大小。研招网可能将每页条数限制为 10,程序按实际位置继续,避免请求较大页时遗漏记录。目录采集先核对研招页面配置中的年份,避免将当前目录标为历史年份。 分页使用接口返回的下一页位置和实际页大小。研招网可能将每页条数限制为 10,程序按实际位置继续,避免请求较大页时遗漏记录。目录采集先核对研招页面配置中的年份,避免将当前目录标为历史年份。
+22
View File
@@ -0,0 +1,22 @@
# 目录任务恢复与进度修复
2026-10-11,`feat/debian-server`。
恢复全国目录任务时,已完成单元虽然不会重复保存,但院校、专业列表会从第一地区重新请求。每个已完成专业还会创建 MariaDB 连接查询断点。用户因此看到记录数和日志长时间不变。
服务器实测原短连接加断点查询平均 0.135 秒;复用已协商的 TLS 上下文后约 0.017 秒。一次读取 4,284 个断点约 0.034 秒。
修复包括:
- 一次加载该任务已完成单元,用集合核对专业断点。
- 在 `job_catalog_pages` 中按任务和请求参数保存院校、专业列表的有效分页;恢复时复用。研究方向、目录年份及登录权限验证保持实时请求。
- 先验证必需代码、分页位置和重复页面,再缓存、返回记录。失败响应不缓存,源站恢复后可以重新请求。
- 更新当前院校、列表/方向页码和断点扫描日志;遍历缓存时仍检查暂停和取消。
- MariaDB 已协商 TLS 时复用相同的 TLS 上下文。数据库事务和短连接的关闭方式保持原有行为。
- SQLite 和 MariaDB 自动创建缓存表;迁移同时兼容没有该表的旧快照。
旧任务首次使用修复仍需获取并保存列表分页。缓存限定于该任务;需要获取更新后的院校/专业列表时新建任务。
验证:Windows 后端 87 passed、9 skipped;Debian 后端 82 passed、14 skipped,包含独立 MariaDB 测试库和 TLS 测试。跳过项为平台或未配置独立数据库的测试。两平台均有既有 Starlette TestClient 的 httpx 弃用提示,不影响生产采集。
11 项新增目录恢复回归覆盖跨服务重启复用分页、批量断点、扫描进度、任务隔离、无效分页、重复页面,以及缺失/空/错误类型的院校与专业代码。独立只读审查未发现剩余 Critical 或 Important 问题。
+42 -8
View File
@@ -1,6 +1,7 @@
"""Independent adapters for the two projects' public data sources.""" """Independent adapters for the two projects' public data sources."""
import json import json
import re import re
import hashlib
from studio.domain import finish_record, integer, normalized_name, select_score_lines from studio.domain import finish_record, integer, normalized_name, select_score_lines
from studio.network import RemoteUnavailable, RateLimited from studio.network import RemoteUnavailable, RateLimited
@@ -18,8 +19,9 @@ class SourceYearMismatch(RuntimeError):
class CatalogClient: class CatalogClient:
BASE = "https://yz.chsi.com.cn/zsml/" BASE = "https://yz.chsi.com.cn/zsml/"
def __init__(self, transport, auth=None): def __init__(self, transport, auth=None, page_cache=None, save_page=None, on_request=None):
self.transport, self.auth = transport, auth self.transport, self.auth = transport, auth
self.page_cache, self.save_page, self.on_request = page_cache, save_page, on_request
def verify_year(self, year): def verify_year(self, year):
response = self.transport.request("GET", self.BASE) response = self.transport.request("GET", self.BASE)
@@ -65,8 +67,16 @@ class CatalogClient:
return data return data
def fetch_page(self, endpoint, payload): def fetch_page(self, endpoint, payload):
# Lists are a snapshot for this task. Direction records and login proofs
# remain live requests; failed/partial responses never enter the cache.
cacheable = self.page_cache is not None and endpoint in ('dws', 'dwzys')
key = self._page_key(endpoint, payload) if cacheable else None
if cacheable and key in self.page_cache:
return self.page_cache[key]
if self.on_request:
self.on_request(endpoint, payload)
try: try:
return self._fetch_page(endpoint, payload) data = self._fetch_page(endpoint, payload)
except LoginRequired as exc: except LoginRequired as exc:
if not self.auth or exc.reason == "captcha": if not self.auth or exc.reason == "captcha":
raise raise
@@ -76,27 +86,51 @@ class CatalogClient:
status = self.auth.check() status = self.auth.check()
if not status["valid"]: if not status["valid"]:
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None
return self._fetch_page(endpoint, payload) data = self._fetch_page(endpoint, payload)
return data
@staticmethod
def _page_key(endpoint, payload):
return hashlib.sha256(json.dumps([endpoint, payload], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
def _cache_page(self, endpoint, payload, data):
if self.page_cache is None or endpoint not in ('dws', 'dwzys'):
return
key = self._page_key(endpoint, payload)
if key not in self.page_cache:
if self.save_page:
self.save_page(key, data)
self.page_cache[key] = data
def pages(self, endpoint, payload, page_size=10): def pages(self, endpoint, payload, page_size=10):
seen, start = set(), 0 seen, start = set(), 0
for page in range(1, 10001): for page in range(1, 10001):
data = self.fetch_page(endpoint, {**payload, "curPage": page, "start": start, "pageSize": page_size}) page_payload = {**payload, "curPage": page, "start": start, "pageSize": page_size}
data = self.fetch_page(endpoint, page_payload)
rows = data["list"] rows = data["list"]
identifier = {'dws': 'dwdm', 'dwzys': 'zydm'}.get(endpoint)
if identifier:
for row in rows:
code = row.get(identifier) if isinstance(row, dict) else None
if isinstance(code, bool) or not isinstance(code, (str, int)) or not str(code).strip():
label = '院校' if endpoint == 'dws' else '专业'
raise RemoteUnavailable(f'研招目录{label}代码缺失或结构异常,已停止以避免缓存不完整数据')
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True) signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
if rows and signature in seen: if rows and signature in seen:
raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页") raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页")
seen.add(signature) seen.add(signature)
yield from rows
more = data.get("nextPageAvailable") more = data.get("nextPageAvailable")
total = integer(data.get("totalCount")) total = integer(data.get("totalCount"))
size = integer(data.get("size")) or page_size size = integer(data.get("size")) or page_size
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size): terminal = not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size)
return
next_start = integer(data.get("startOfNextPage")) next_start = integer(data.get("startOfNextPage"))
next_start = start + len(rows) if next_start is None else next_start next_start = start + len(rows) if next_start is None else next_start
if next_start <= start: if not terminal and next_start <= start:
raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页") raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页")
self._cache_page(endpoint, page_payload, data)
yield from rows
if terminal:
return
start, page_size = next_start, size start, page_size = next_start, size
raise RemoteUnavailable("目录分页超过安全上限") raise RemoteUnavailable("目录分页超过安全上限")
+13
View File
@@ -24,6 +24,7 @@ CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year D
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT); CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0); CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit)); CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id TEXT,page_key TEXT,data TEXT NOT NULL,PRIMARY KEY(job_id,page_key));
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT); CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
""" """
@@ -202,6 +203,18 @@ class Database:
with self.connection() as conn: with self.connection() as conn:
return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None
def completed_units(self, jid):
with self.connection() as conn:
return {row['unit'] for row in conn.execute('SELECT unit FROM job_units WHERE job_id=?', (jid,))}
def catalog_pages(self, jid):
with self.connection() as conn:
return {row['page_key']: json.loads(row['data']) for row in conn.execute('SELECT page_key,data FROM job_catalog_pages WHERE job_id=?', (jid,))}
def cache_catalog_page(self, jid, page_key, data):
with self.connection() as conn:
conn.execute('INSERT INTO job_catalog_pages(job_id,page_key,data) VALUES(?,?,?) ON CONFLICT(job_id,page_key) DO UPDATE SET data=excluded.data', (jid, page_key, encode(data)))
def complete_unit(self, jid, unit): def complete_unit(self, jid, unit):
with self.connection() as conn: with self.connection() as conn:
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit)) conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
+20 -2
View File
@@ -189,7 +189,13 @@ class JobManager:
self._scores(jid, params) self._scores(jid, params)
def _catalog(self, jid, params): def _catalog(self, jid, params):
client = CatalogClient(self.transport, auth=self.auth) completed = self.db.completed_units(jid)
location = '研招目录'
def on_request(endpoint, payload):
phase = {'dws': '院校列表', 'dwzys': '专业列表', 'yjfxs': '研究方向'}[endpoint]
self.db.job_update(jid, message=f"{location}:正在请求{phase}第 {payload.get('curPage', 1)} 页")
client = CatalogClient(self.transport, auth=self.auth, page_cache=self.db.catalog_pages(jid),
save_page=lambda key, data: self.db.cache_catalog_page(jid, key, data), on_request=on_request)
client.verify_year(params["year"]) client.verify_year(params["year"])
if self.auth: if self.auth:
status = self.auth.check() status = self.auth.check()
@@ -198,18 +204,27 @@ class JobManager:
self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功") self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功")
for province in params["provinces"]: for province in params["provinces"]:
self.checkpoint(jid) self.checkpoint(jid)
location = PROVINCES[province]
self.db.job_update(jid, message=location + ':正在核对院校列表')
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校") self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
for school in client.schools(province, params.get("school_keyword", "")): for school in client.schools(province, params.get("school_keyword", "")):
self.checkpoint(jid) self.checkpoint(jid)
school = {"ssdm": province, **school} school = {"ssdm": province, **school}
location = f"{PROVINCES[province]} · {school.get('dwmc', school['dwdm'])}"
self.db.job_update(jid, message=location + ':正在核对专业列表与已完成断点')
self.db.log(jid, 'info', location + ':核对专业列表与断点')
skipped = 0
for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")): for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")):
self.checkpoint(jid)
if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]): if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]):
continue continue
unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}" unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}"
if self.unit_done(jid, unit): if unit in completed:
skipped += 1
continue continue
self.checkpoint(jid) self.checkpoint(jid)
label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}" label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}"
location = label
self.db.job_update(jid, message=label) self.db.job_update(jid, message=label)
try: try:
records = [] records = []
@@ -217,9 +232,12 @@ class JobManager:
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major)) records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
self._save(jid, records) self._save(jid, records)
self.complete_unit(jid, unit) self.complete_unit(jid, unit)
completed.add(unit)
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合") self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
except RemoteUnavailable as exc: except RemoteUnavailable as exc:
self._error(jid, f"{label}:{exc}") self._error(jid, f"{label}:{exc}")
if skipped:
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
def _scores(self, jid, params): def _scores(self, jid, params):
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")} filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
+4 -1
View File
@@ -6,7 +6,7 @@ from pathlib import Path
from studio.domain import normalized_name, is_special_score from studio.domain import normalized_name, is_special_score
from studio.mysql_database import MySQLDatabase, score_key from studio.mysql_database import MySQLDatabase, score_key
TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs') TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs', 'job_catalog_pages')
def migrate_sqlite(source, target): def migrate_sqlite(source, target):
@@ -16,6 +16,7 @@ def migrate_sqlite(source, target):
counts = {} counts = {}
try: try:
conn.execute('BEGIN') conn.execute('BEGIN')
source_tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
with target.connection() as output: with target.connection() as output:
# Serialize attempted imports to the same database. # Serialize attempted imports to the same database.
acquired = output.execute("SELECT GET_LOCK('kaoyan_studio_migration',10)").fetchone()[0] acquired = output.execute("SELECT GET_LOCK('kaoyan_studio_migration',10)").fetchone()[0]
@@ -27,6 +28,8 @@ def migrate_sqlite(source, target):
raise ValueError(f'目标数据库非空({table}),拒绝覆盖;请使用空数据库') raise ValueError(f'目标数据库非空({table}),拒绝覆盖;请使用空数据库')
for table in TABLES: for table in TABLES:
counts[table] = 0 counts[table] = 0
if table == 'job_catalog_pages' and table not in source_tables:
continue # Existing snapshots predate the optional page cache.
reader = conn.execute(f'SELECT * FROM {table}') reader = conn.execute(f'SELECT * FROM {table}')
while batch := reader.fetchmany(500): while batch := reader.fetchmany(500):
rows = [] rows = []
+8 -1
View File
@@ -2,6 +2,7 @@
import hashlib import hashlib
import json import json
import re import re
import ssl
from contextlib import contextmanager from contextlib import contextmanager
import pymysql import pymysql
from studio.database import Database, encode from studio.database import Database, encode
@@ -23,6 +24,7 @@ SCHEMA = [
"CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)", "CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)",
"CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)", "CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)",
"CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))", "CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))",
"CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id VARCHAR(64),page_key VARCHAR(64),data LONGTEXT NOT NULL,PRIMARY KEY(job_id,page_key))",
"CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))", "CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))",
] ]
@@ -101,6 +103,7 @@ class MySQLDatabase(Database):
if not self.config.get('database'): if not self.config.get('database'):
raise ValueError('数据库配置缺少 database') raise ValueError('数据库配置缺少 database')
self.recovered_job_ids = [] self.recovered_job_ids = []
self._ssl_context = None
with self.connection() as conn: with self.connection() as conn:
for statement in SCHEMA: for statement in SCHEMA:
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin') conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
@@ -113,8 +116,12 @@ class MySQLDatabase(Database):
@contextmanager @contextmanager
def connection(self): def connection(self):
raw = pymysql.connect(**self.config, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120) raw = pymysql.connect(**self.config, ssl=self._ssl_context, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
try: try:
# Retain the driver's negotiated context rather than reload its CA
# store for every short-lived query. Plain servers retain default behavior.
if self._ssl_context is None and isinstance(getattr(raw, '_sock', None), ssl.SSLSocket):
self._ssl_context = raw.ctx
yield Connection(raw) yield Connection(raw)
raw.commit() raw.commit()
except BaseException: except BaseException:
+153
View File
@@ -0,0 +1,153 @@
from collections import Counter
import pytest
from studio.connectors import CatalogClient, LoginRequired
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
from studio.network import RemoteUnavailable
from tests.test_data import raw_catalog
from tests.test_network import Response
PARAMS = {'mode': 'catalog', 'year': 2027, 'provinces': ['11']}
class InterruptedCatalog:
def __init__(self, observe=None):
self.blocked = True
self.calls = Counter()
self.observe = observe
def request(self, method, url, **kwargs):
if method == 'GET':
return Response({}, text='2027年硕士专业目录查询')
endpoint = url.rsplit('/', 1)[-1]
school = kwargs['data'].get('dwdm', '')
self.calls[endpoint, school] += 1
if endpoint == 'dws.do':
rows = [{'dwdm': '10001', 'dwmc': '北京大学'}, {'dwdm': '10002', 'dwmc': '中国人民大学'}]
elif endpoint == 'dwzys.do':
if self.observe:
self.observe(school)
rows = [{'zydm': '081200', 'zymc': '计算机科学与技术', 'xxfs': '1'}]
else:
if school == '10002' and self.blocked:
return Response({'flag': False, 'msg': '请登录'})
rows = [{**raw_catalog(), 'dwdm': school}]
return Response({'flag': True, 'msg': {'list': rows, 'nextPageAvailable': False}})
def prepare(tmp_path, transport):
db = Database(tmp_path / 'studio.db')
db.job_create('resume', PARAMS)
return db, JobManager(db, SettingsStore(tmp_path), transport=transport)
def test_login_recovery_reuses_successful_catalog_pages_across_service_restart(tmp_path):
source = InterruptedCatalog()
db, manager = prepare(tmp_path, source)
with pytest.raises(LoginRequired):
manager._catalog('resume', PARAMS)
assert db.job_get('resume')['completed_units'] == 1
source.blocked = False
reopened = Database(tmp_path / 'studio.db')
JobManager(reopened, SettingsStore(tmp_path), transport=source)._catalog('resume', PARAMS)
assert reopened.job_get('resume')['completed_units'] == 2
assert source.calls['dws.do', ''] == 1
assert source.calls['dwzys.do', '10001'] == 1
assert source.calls['dwzys.do', '10002'] == 1
assert source.calls['yjfxs.do', '10002'] == 2 # Failed responses must never become cached successes.
def test_resume_checks_completed_units_in_memory_instead_of_one_query_per_major(tmp_path):
source = InterruptedCatalog()
source.blocked = False
db, manager = prepare(tmp_path, source)
for school in ('10001', '10002'):
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
queries = []
original = db.unit_done
def observe(jid, unit):
queries.append(unit)
return original(jid, unit)
db.unit_done = observe
manager._catalog('resume', PARAMS)
assert queries == []
assert not any(endpoint == 'yjfxs.do' for endpoint, school in source.calls)
def test_resume_displays_current_school_while_checking_its_major_list(tmp_path):
messages = {}
source = InterruptedCatalog(lambda school: messages.setdefault(school, db.job_get('resume')['message']))
source.blocked = False
db, manager = prepare(tmp_path, source)
for school in ('10001', '10002'):
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
manager._catalog('resume', PARAMS)
assert '北京大学' in messages['10001']
assert '中国人民大学' in messages['10002']
def test_cached_pages_do_not_cross_job_boundaries(tmp_path):
source = InterruptedCatalog()
source.blocked = False
db, manager = prepare(tmp_path, source)
manager._catalog('resume', PARAMS)
db.job_create('fresh', PARAMS)
manager._catalog('fresh', PARAMS)
assert source.calls['dws.do', ''] == 2
assert db.job_get('fresh')['completed_units'] == 2
def test_invalid_pagination_is_not_persisted_as_a_reusable_page():
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
body = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': self.blocked, 'startOfNextPage': 0}
self.blocked = False
return Response({'flag': True, 'msg': body})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='下一页位置无效'):
list(client.schools('11'))
assert cache == {}
assert list(client.schools('11')) == [{'dwdm': '10001'}]
def test_repeated_source_page_is_retried_live_after_recovery():
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
page = kwargs['data']['curPage']
school = '10001' if page == 1 or self.blocked else '10002'
return Response({'flag': True, 'msg': {'list': [{'dwdm': school}], 'nextPageAvailable': page == 1}})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='重复返回同一页'):
list(client.schools('11'))
assert len(cache) == 1
source.blocked = False
assert [row['dwdm'] for row in client.schools('11')] == ['10001', '10002']
@pytest.mark.parametrize('endpoint,broken_row', [
('dws', {'dwmc': '缺少代码的院校'}),
('dwzys', {'zymc': '缺少代码的专业'}),
('dws', 'unexpected row'),
('dwzys', {'zydm': ''}),
('dwzys', {'zydm': None}),
])
def test_malformed_list_page_is_retried_after_source_recovers(endpoint, broken_row):
identifier = 'dwdm' if endpoint == 'dws' else 'zydm'
good_row = {identifier: '10001' if endpoint == 'dws' else '081200'}
class Source(InterruptedCatalog):
def request(self, method, url, **kwargs):
row = broken_row if self.blocked else good_row
return Response({'flag': True, 'msg': {'list': [row], 'nextPageAvailable': False}})
source, cache = Source(), {}
client = CatalogClient(source, page_cache=cache)
with pytest.raises(RemoteUnavailable, match='代码'):
list(client.pages(endpoint, {}))
assert cache == {}
source.blocked = False
assert list(client.pages(endpoint, {})) == [good_row]
+45 -1
View File
@@ -17,7 +17,7 @@ def mysql():
assert config['database'].endswith('_test'), 'refusing to clear a production database' assert config['database'].endswith('_test'), 'refusing to clear a production database'
db = MySQLDatabase(config) db = MySQLDatabase(config)
with db.connection() as conn: with db.connection() as conn:
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs'): for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs', 'job_catalog_pages'):
conn.execute(f'DELETE FROM {table}') conn.execute(f'DELETE FROM {table}')
return db return db
@@ -68,6 +68,36 @@ def test_mysql_tasks_logs_and_checkpoints(mysql):
assert mysql.logs('contract-job')[0]['message'] == '数据库测试' assert mysql.logs('contract-job')[0]['message'] == '数据库测试'
def test_mysql_catalog_page_cache_and_bulk_checkpoints_are_durable(mysql):
mysql.job_create('cached-job', {'year': 2027})
mysql.complete_unit('cached-job', 'catalog:11:10001:081200:1')
page = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': False}
mysql.cache_catalog_page('cached-job', 'page-key', page)
from studio.mysql_database import MySQLDatabase
reopened = MySQLDatabase(mysql.config, recover_jobs=False)
assert reopened.completed_units('cached-job') == {'catalog:11:10001:081200:1'}
assert reopened.catalog_pages('cached-job') == {'page-key': page}
assert reopened.catalog_pages('another-job') == {}
def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypatch):
import ssl
with mysql.connection() as conn:
cipher = conn.execute("SHOW SESSION STATUS LIKE 'Ssl_cipher'").fetchone()[1]
if not cipher:
pytest.skip('requires a TLS-enabled MariaDB test server')
creations = []
original = ssl.create_default_context
def observe(*args, **kwargs):
creations.append(True)
return original(*args, **kwargs)
monkeypatch.setattr(ssl, 'create_default_context', observe)
for _ in range(2):
with mysql.connection() as conn:
assert conn.execute('SELECT 1').fetchone()[0] == 1
assert creations == []
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path): def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
from studio.migrate import migrate_sqlite from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'snapshot.db') source = Database(tmp_path / 'snapshot.db')
@@ -77,16 +107,30 @@ def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_p
source.job_create('migrated-job', {'year': 2027}) source.job_create('migrated-job', {'year': 2027})
source.complete_unit('migrated-job', 'catalog:11:10001:081200:1') source.complete_unit('migrated-job', 'catalog:11:10001:081200:1')
source.log('migrated-job', 'info', 'preserve log') source.log('migrated-job', 'info', 'preserve log')
source.cache_catalog_page('migrated-job', 'migration-page', {'list': [], 'nextPageAvailable': False})
counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql) counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql)
assert counts['catalog'] == 2 and counts['scores'] == 1 assert counts['catalog'] == 2 and counts['scores'] == 1
assert mysql.record(1)['favorite'] assert mysql.record(1)['favorite']
assert mysql.query_catalog({})['items'][0]['score_total'] == 320 assert mysql.query_catalog({})['items'][0]['score_total'] == 320
assert mysql.job_get('migrated-job')['completed_units'] == 1 assert mysql.job_get('migrated-job')['completed_units'] == 1
assert mysql.logs('migrated-job')[0]['id'] == 1 assert mysql.logs('migrated-job')[0]['id'] == 1
assert counts['job_catalog_pages'] == 1
assert mysql.catalog_pages('migrated-job')['migration-page']['list'] == []
with pytest.raises(ValueError, match='非空'): with pytest.raises(ValueError, match='非空'):
migrate_sqlite(tmp_path / 'snapshot.db', mysql) migrate_sqlite(tmp_path / 'snapshot.db', mysql)
def test_legacy_snapshot_without_catalog_page_table_can_still_be_migrated(mysql, tmp_path):
from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'legacy.db')
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
with source.connection() as conn:
conn.execute('DROP TABLE job_catalog_pages')
counts = migrate_sqlite(source.path, mysql)
assert counts['catalog'] == 2
assert counts['job_catalog_pages'] == 0
def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path): def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
from studio.migrate import migrate_sqlite from studio.migrate import migrate_sqlite
source = Database(tmp_path / 'bad.db') source = Database(tmp_path / 'bad.db')