fix: resume catalog jobs without repeated checkpoint queries
This commit is contained in:
@@ -28,6 +28,8 @@ Debian 12 / MariaDB 服务器模式、HTTPS 访问与代理池后台运行见 [
|
||||
4. 在「采集任务」选择内容、目录年份、地区、专业和院校关键词。建议先按一个地区、一个专业或一所院校验证,再创建全国任务。专业代码可以从内置目录选择,也可直接输入 6 位代码。
|
||||
5. 查看任务日志。暂停可继续;需要登录、代理或限流解除时,处理后点击恢复;取消保留已有数据。关闭或重启服务会保留可恢复断点。
|
||||
|
||||
任务会保存本次采集的院校和专业列表分页,登录恢复或服务重启后复用,并一次加载已完成单元。界面显示当前核对的院校和正在请求的页数。修复前创建的任务首次恢复仍需建立列表缓存;新建任务会获取新列表。研究方向和登录验证保持实时请求,异常分页不会写入缓存。
|
||||
|
||||
研招网目录第一页可匿名返回,包括部分研究方向;不能用第一页判断登录成功。工作台访问研招目录的登录入口,将已有学信网 Cookie 同步到目录会话,再验证需要登录的第二页。采集开始/恢复时自动完成此检查;途中遇到登录提示,只同步一次并重试原页,失败后保留断点并提示重新登录。验证码仍需在浏览器人工完成,限流和网络失败会单独提示。登录卡片会持续更新状态;切换网络出口后需要重新验证,原 Cookie 保留。
|
||||
|
||||
分页使用接口返回的下一页位置和实际页大小。研招网可能将每页条数限制为 10,程序按实际位置继续,避免请求较大页时遗漏记录。目录采集先核对研招页面配置中的年份,避免将当前目录标为历史年份。
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# 目录任务恢复与进度修复
|
||||
|
||||
2026-10-11,`feat/debian-server`。
|
||||
|
||||
恢复全国目录任务时,已完成单元虽然不会重复保存,但院校、专业列表会从第一地区重新请求。每个已完成专业还会创建 MariaDB 连接查询断点。用户因此看到记录数和日志长时间不变。
|
||||
|
||||
服务器实测原短连接加断点查询平均 0.135 秒;复用已协商的 TLS 上下文后约 0.017 秒。一次读取 4,284 个断点约 0.034 秒。
|
||||
|
||||
修复包括:
|
||||
|
||||
- 一次加载该任务已完成单元,用集合核对专业断点。
|
||||
- 在 `job_catalog_pages` 中按任务和请求参数保存院校、专业列表的有效分页;恢复时复用。研究方向、目录年份及登录权限验证保持实时请求。
|
||||
- 先验证必需代码、分页位置和重复页面,再缓存、返回记录。失败响应不缓存,源站恢复后可以重新请求。
|
||||
- 更新当前院校、列表/方向页码和断点扫描日志;遍历缓存时仍检查暂停和取消。
|
||||
- MariaDB 已协商 TLS 时复用相同的 TLS 上下文。数据库事务和短连接的关闭方式保持原有行为。
|
||||
- SQLite 和 MariaDB 自动创建缓存表;迁移同时兼容没有该表的旧快照。
|
||||
|
||||
旧任务首次使用修复仍需获取并保存列表分页。缓存限定于该任务;需要获取更新后的院校/专业列表时新建任务。
|
||||
|
||||
验证:Windows 后端 87 passed、9 skipped;Debian 后端 82 passed、14 skipped,包含独立 MariaDB 测试库和 TLS 测试。跳过项为平台或未配置独立数据库的测试。两平台均有既有 Starlette TestClient 的 httpx 弃用提示,不影响生产采集。
|
||||
|
||||
11 项新增目录恢复回归覆盖跨服务重启复用分页、批量断点、扫描进度、任务隔离、无效分页、重复页面,以及缺失/空/错误类型的院校与专业代码。独立只读审查未发现剩余 Critical 或 Important 问题。
|
||||
+42
-8
@@ -1,6 +1,7 @@
|
||||
"""Independent adapters for the two projects' public data sources."""
|
||||
import json
|
||||
import re
|
||||
import hashlib
|
||||
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
||||
from studio.network import RemoteUnavailable, RateLimited
|
||||
|
||||
@@ -18,8 +19,9 @@ class SourceYearMismatch(RuntimeError):
|
||||
class CatalogClient:
|
||||
BASE = "https://yz.chsi.com.cn/zsml/"
|
||||
|
||||
def __init__(self, transport, auth=None):
|
||||
def __init__(self, transport, auth=None, page_cache=None, save_page=None, on_request=None):
|
||||
self.transport, self.auth = transport, auth
|
||||
self.page_cache, self.save_page, self.on_request = page_cache, save_page, on_request
|
||||
|
||||
def verify_year(self, year):
|
||||
response = self.transport.request("GET", self.BASE)
|
||||
@@ -65,8 +67,16 @@ class CatalogClient:
|
||||
return data
|
||||
|
||||
def fetch_page(self, endpoint, payload):
|
||||
# Lists are a snapshot for this task. Direction records and login proofs
|
||||
# remain live requests; failed/partial responses never enter the cache.
|
||||
cacheable = self.page_cache is not None and endpoint in ('dws', 'dwzys')
|
||||
key = self._page_key(endpoint, payload) if cacheable else None
|
||||
if cacheable and key in self.page_cache:
|
||||
return self.page_cache[key]
|
||||
if self.on_request:
|
||||
self.on_request(endpoint, payload)
|
||||
try:
|
||||
return self._fetch_page(endpoint, payload)
|
||||
data = self._fetch_page(endpoint, payload)
|
||||
except LoginRequired as exc:
|
||||
if not self.auth or exc.reason == "captcha":
|
||||
raise
|
||||
@@ -76,27 +86,51 @@ class CatalogClient:
|
||||
status = self.auth.check()
|
||||
if not status["valid"]:
|
||||
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None
|
||||
return self._fetch_page(endpoint, payload)
|
||||
data = self._fetch_page(endpoint, payload)
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _page_key(endpoint, payload):
|
||||
return hashlib.sha256(json.dumps([endpoint, payload], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
|
||||
def _cache_page(self, endpoint, payload, data):
|
||||
if self.page_cache is None or endpoint not in ('dws', 'dwzys'):
|
||||
return
|
||||
key = self._page_key(endpoint, payload)
|
||||
if key not in self.page_cache:
|
||||
if self.save_page:
|
||||
self.save_page(key, data)
|
||||
self.page_cache[key] = data
|
||||
|
||||
def pages(self, endpoint, payload, page_size=10):
|
||||
seen, start = set(), 0
|
||||
for page in range(1, 10001):
|
||||
data = self.fetch_page(endpoint, {**payload, "curPage": page, "start": start, "pageSize": page_size})
|
||||
page_payload = {**payload, "curPage": page, "start": start, "pageSize": page_size}
|
||||
data = self.fetch_page(endpoint, page_payload)
|
||||
rows = data["list"]
|
||||
identifier = {'dws': 'dwdm', 'dwzys': 'zydm'}.get(endpoint)
|
||||
if identifier:
|
||||
for row in rows:
|
||||
code = row.get(identifier) if isinstance(row, dict) else None
|
||||
if isinstance(code, bool) or not isinstance(code, (str, int)) or not str(code).strip():
|
||||
label = '院校' if endpoint == 'dws' else '专业'
|
||||
raise RemoteUnavailable(f'研招目录{label}代码缺失或结构异常,已停止以避免缓存不完整数据')
|
||||
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
||||
if rows and signature in seen:
|
||||
raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页")
|
||||
seen.add(signature)
|
||||
yield from rows
|
||||
more = data.get("nextPageAvailable")
|
||||
total = integer(data.get("totalCount"))
|
||||
size = integer(data.get("size")) or page_size
|
||||
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size):
|
||||
return
|
||||
terminal = not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size)
|
||||
next_start = integer(data.get("startOfNextPage"))
|
||||
next_start = start + len(rows) if next_start is None else next_start
|
||||
if next_start <= start:
|
||||
if not terminal and next_start <= start:
|
||||
raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页")
|
||||
self._cache_page(endpoint, page_payload, data)
|
||||
yield from rows
|
||||
if terminal:
|
||||
return
|
||||
start, page_size = next_start, size
|
||||
raise RemoteUnavailable("目录分页超过安全上限")
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year D
|
||||
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
||||
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
|
||||
CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id TEXT,page_key TEXT,data TEXT NOT NULL,PRIMARY KEY(job_id,page_key));
|
||||
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
|
||||
"""
|
||||
|
||||
@@ -202,6 +203,18 @@ class Database:
|
||||
with self.connection() as conn:
|
||||
return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None
|
||||
|
||||
def completed_units(self, jid):
|
||||
with self.connection() as conn:
|
||||
return {row['unit'] for row in conn.execute('SELECT unit FROM job_units WHERE job_id=?', (jid,))}
|
||||
|
||||
def catalog_pages(self, jid):
|
||||
with self.connection() as conn:
|
||||
return {row['page_key']: json.loads(row['data']) for row in conn.execute('SELECT page_key,data FROM job_catalog_pages WHERE job_id=?', (jid,))}
|
||||
|
||||
def cache_catalog_page(self, jid, page_key, data):
|
||||
with self.connection() as conn:
|
||||
conn.execute('INSERT INTO job_catalog_pages(job_id,page_key,data) VALUES(?,?,?) ON CONFLICT(job_id,page_key) DO UPDATE SET data=excluded.data', (jid, page_key, encode(data)))
|
||||
|
||||
def complete_unit(self, jid, unit):
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
|
||||
|
||||
+20
-2
@@ -189,7 +189,13 @@ class JobManager:
|
||||
self._scores(jid, params)
|
||||
|
||||
def _catalog(self, jid, params):
|
||||
client = CatalogClient(self.transport, auth=self.auth)
|
||||
completed = self.db.completed_units(jid)
|
||||
location = '研招目录'
|
||||
def on_request(endpoint, payload):
|
||||
phase = {'dws': '院校列表', 'dwzys': '专业列表', 'yjfxs': '研究方向'}[endpoint]
|
||||
self.db.job_update(jid, message=f"{location}:正在请求{phase}第 {payload.get('curPage', 1)} 页")
|
||||
client = CatalogClient(self.transport, auth=self.auth, page_cache=self.db.catalog_pages(jid),
|
||||
save_page=lambda key, data: self.db.cache_catalog_page(jid, key, data), on_request=on_request)
|
||||
client.verify_year(params["year"])
|
||||
if self.auth:
|
||||
status = self.auth.check()
|
||||
@@ -198,18 +204,27 @@ class JobManager:
|
||||
self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功")
|
||||
for province in params["provinces"]:
|
||||
self.checkpoint(jid)
|
||||
location = PROVINCES[province]
|
||||
self.db.job_update(jid, message=location + ':正在核对院校列表')
|
||||
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
|
||||
for school in client.schools(province, params.get("school_keyword", "")):
|
||||
self.checkpoint(jid)
|
||||
school = {"ssdm": province, **school}
|
||||
location = f"{PROVINCES[province]} · {school.get('dwmc', school['dwdm'])}"
|
||||
self.db.job_update(jid, message=location + ':正在核对专业列表与已完成断点')
|
||||
self.db.log(jid, 'info', location + ':核对专业列表与断点')
|
||||
skipped = 0
|
||||
for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")):
|
||||
self.checkpoint(jid)
|
||||
if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]):
|
||||
continue
|
||||
unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}"
|
||||
if self.unit_done(jid, unit):
|
||||
if unit in completed:
|
||||
skipped += 1
|
||||
continue
|
||||
self.checkpoint(jid)
|
||||
label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}"
|
||||
location = label
|
||||
self.db.job_update(jid, message=label)
|
||||
try:
|
||||
records = []
|
||||
@@ -217,9 +232,12 @@ class JobManager:
|
||||
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
|
||||
self._save(jid, records)
|
||||
self.complete_unit(jid, unit)
|
||||
completed.add(unit)
|
||||
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
|
||||
except RemoteUnavailable as exc:
|
||||
self._error(jid, f"{label}:{exc}")
|
||||
if skipped:
|
||||
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
|
||||
|
||||
def _scores(self, jid, params):
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
|
||||
|
||||
+4
-1
@@ -6,7 +6,7 @@ from pathlib import Path
|
||||
from studio.domain import normalized_name, is_special_score
|
||||
from studio.mysql_database import MySQLDatabase, score_key
|
||||
|
||||
TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs')
|
||||
TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs', 'job_catalog_pages')
|
||||
|
||||
|
||||
def migrate_sqlite(source, target):
|
||||
@@ -16,6 +16,7 @@ def migrate_sqlite(source, target):
|
||||
counts = {}
|
||||
try:
|
||||
conn.execute('BEGIN')
|
||||
source_tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
with target.connection() as output:
|
||||
# Serialize attempted imports to the same database.
|
||||
acquired = output.execute("SELECT GET_LOCK('kaoyan_studio_migration',10)").fetchone()[0]
|
||||
@@ -27,6 +28,8 @@ def migrate_sqlite(source, target):
|
||||
raise ValueError(f'目标数据库非空({table}),拒绝覆盖;请使用空数据库')
|
||||
for table in TABLES:
|
||||
counts[table] = 0
|
||||
if table == 'job_catalog_pages' and table not in source_tables:
|
||||
continue # Existing snapshots predate the optional page cache.
|
||||
reader = conn.execute(f'SELECT * FROM {table}')
|
||||
while batch := reader.fetchmany(500):
|
||||
rows = []
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
from contextlib import contextmanager
|
||||
import pymysql
|
||||
from studio.database import Database, encode
|
||||
@@ -23,6 +24,7 @@ SCHEMA = [
|
||||
"CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)",
|
||||
"CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)",
|
||||
"CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))",
|
||||
"CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id VARCHAR(64),page_key VARCHAR(64),data LONGTEXT NOT NULL,PRIMARY KEY(job_id,page_key))",
|
||||
"CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))",
|
||||
]
|
||||
|
||||
@@ -101,6 +103,7 @@ class MySQLDatabase(Database):
|
||||
if not self.config.get('database'):
|
||||
raise ValueError('数据库配置缺少 database')
|
||||
self.recovered_job_ids = []
|
||||
self._ssl_context = None
|
||||
with self.connection() as conn:
|
||||
for statement in SCHEMA:
|
||||
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
|
||||
@@ -113,8 +116,12 @@ class MySQLDatabase(Database):
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
raw = pymysql.connect(**self.config, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
|
||||
raw = pymysql.connect(**self.config, ssl=self._ssl_context, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
|
||||
try:
|
||||
# Retain the driver's negotiated context rather than reload its CA
|
||||
# store for every short-lived query. Plain servers retain default behavior.
|
||||
if self._ssl_context is None and isinstance(getattr(raw, '_sock', None), ssl.SSLSocket):
|
||||
self._ssl_context = raw.ctx
|
||||
yield Connection(raw)
|
||||
raw.commit()
|
||||
except BaseException:
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
from collections import Counter
|
||||
|
||||
import pytest
|
||||
|
||||
from studio.connectors import CatalogClient, LoginRequired
|
||||
from studio.database import Database
|
||||
from studio.jobs import JobManager
|
||||
from studio.settings import SettingsStore
|
||||
from studio.network import RemoteUnavailable
|
||||
from tests.test_data import raw_catalog
|
||||
from tests.test_network import Response
|
||||
|
||||
|
||||
PARAMS = {'mode': 'catalog', 'year': 2027, 'provinces': ['11']}
|
||||
|
||||
|
||||
class InterruptedCatalog:
|
||||
def __init__(self, observe=None):
|
||||
self.blocked = True
|
||||
self.calls = Counter()
|
||||
self.observe = observe
|
||||
|
||||
def request(self, method, url, **kwargs):
|
||||
if method == 'GET':
|
||||
return Response({}, text='2027年硕士专业目录查询')
|
||||
endpoint = url.rsplit('/', 1)[-1]
|
||||
school = kwargs['data'].get('dwdm', '')
|
||||
self.calls[endpoint, school] += 1
|
||||
if endpoint == 'dws.do':
|
||||
rows = [{'dwdm': '10001', 'dwmc': '北京大学'}, {'dwdm': '10002', 'dwmc': '中国人民大学'}]
|
||||
elif endpoint == 'dwzys.do':
|
||||
if self.observe:
|
||||
self.observe(school)
|
||||
rows = [{'zydm': '081200', 'zymc': '计算机科学与技术', 'xxfs': '1'}]
|
||||
else:
|
||||
if school == '10002' and self.blocked:
|
||||
return Response({'flag': False, 'msg': '请登录'})
|
||||
rows = [{**raw_catalog(), 'dwdm': school}]
|
||||
return Response({'flag': True, 'msg': {'list': rows, 'nextPageAvailable': False}})
|
||||
|
||||
|
||||
def prepare(tmp_path, transport):
|
||||
db = Database(tmp_path / 'studio.db')
|
||||
db.job_create('resume', PARAMS)
|
||||
return db, JobManager(db, SettingsStore(tmp_path), transport=transport)
|
||||
|
||||
|
||||
def test_login_recovery_reuses_successful_catalog_pages_across_service_restart(tmp_path):
|
||||
source = InterruptedCatalog()
|
||||
db, manager = prepare(tmp_path, source)
|
||||
with pytest.raises(LoginRequired):
|
||||
manager._catalog('resume', PARAMS)
|
||||
assert db.job_get('resume')['completed_units'] == 1
|
||||
source.blocked = False
|
||||
reopened = Database(tmp_path / 'studio.db')
|
||||
JobManager(reopened, SettingsStore(tmp_path), transport=source)._catalog('resume', PARAMS)
|
||||
assert reopened.job_get('resume')['completed_units'] == 2
|
||||
assert source.calls['dws.do', ''] == 1
|
||||
assert source.calls['dwzys.do', '10001'] == 1
|
||||
assert source.calls['dwzys.do', '10002'] == 1
|
||||
assert source.calls['yjfxs.do', '10002'] == 2 # Failed responses must never become cached successes.
|
||||
|
||||
|
||||
def test_resume_checks_completed_units_in_memory_instead_of_one_query_per_major(tmp_path):
|
||||
source = InterruptedCatalog()
|
||||
source.blocked = False
|
||||
db, manager = prepare(tmp_path, source)
|
||||
for school in ('10001', '10002'):
|
||||
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
|
||||
queries = []
|
||||
original = db.unit_done
|
||||
def observe(jid, unit):
|
||||
queries.append(unit)
|
||||
return original(jid, unit)
|
||||
db.unit_done = observe
|
||||
manager._catalog('resume', PARAMS)
|
||||
assert queries == []
|
||||
assert not any(endpoint == 'yjfxs.do' for endpoint, school in source.calls)
|
||||
|
||||
|
||||
def test_resume_displays_current_school_while_checking_its_major_list(tmp_path):
|
||||
messages = {}
|
||||
source = InterruptedCatalog(lambda school: messages.setdefault(school, db.job_get('resume')['message']))
|
||||
source.blocked = False
|
||||
db, manager = prepare(tmp_path, source)
|
||||
for school in ('10001', '10002'):
|
||||
db.complete_unit('resume', f'catalog:11:{school}:081200:1')
|
||||
manager._catalog('resume', PARAMS)
|
||||
assert '北京大学' in messages['10001']
|
||||
assert '中国人民大学' in messages['10002']
|
||||
|
||||
|
||||
def test_cached_pages_do_not_cross_job_boundaries(tmp_path):
|
||||
source = InterruptedCatalog()
|
||||
source.blocked = False
|
||||
db, manager = prepare(tmp_path, source)
|
||||
manager._catalog('resume', PARAMS)
|
||||
db.job_create('fresh', PARAMS)
|
||||
manager._catalog('fresh', PARAMS)
|
||||
assert source.calls['dws.do', ''] == 2
|
||||
assert db.job_get('fresh')['completed_units'] == 2
|
||||
|
||||
|
||||
def test_invalid_pagination_is_not_persisted_as_a_reusable_page():
|
||||
class Source(InterruptedCatalog):
|
||||
def request(self, method, url, **kwargs):
|
||||
body = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': self.blocked, 'startOfNextPage': 0}
|
||||
self.blocked = False
|
||||
return Response({'flag': True, 'msg': body})
|
||||
source, cache = Source(), {}
|
||||
client = CatalogClient(source, page_cache=cache)
|
||||
with pytest.raises(RemoteUnavailable, match='下一页位置无效'):
|
||||
list(client.schools('11'))
|
||||
assert cache == {}
|
||||
assert list(client.schools('11')) == [{'dwdm': '10001'}]
|
||||
|
||||
|
||||
def test_repeated_source_page_is_retried_live_after_recovery():
|
||||
class Source(InterruptedCatalog):
|
||||
def request(self, method, url, **kwargs):
|
||||
page = kwargs['data']['curPage']
|
||||
school = '10001' if page == 1 or self.blocked else '10002'
|
||||
return Response({'flag': True, 'msg': {'list': [{'dwdm': school}], 'nextPageAvailable': page == 1}})
|
||||
source, cache = Source(), {}
|
||||
client = CatalogClient(source, page_cache=cache)
|
||||
with pytest.raises(RemoteUnavailable, match='重复返回同一页'):
|
||||
list(client.schools('11'))
|
||||
assert len(cache) == 1
|
||||
source.blocked = False
|
||||
assert [row['dwdm'] for row in client.schools('11')] == ['10001', '10002']
|
||||
|
||||
|
||||
@pytest.mark.parametrize('endpoint,broken_row', [
|
||||
('dws', {'dwmc': '缺少代码的院校'}),
|
||||
('dwzys', {'zymc': '缺少代码的专业'}),
|
||||
('dws', 'unexpected row'),
|
||||
('dwzys', {'zydm': ''}),
|
||||
('dwzys', {'zydm': None}),
|
||||
])
|
||||
def test_malformed_list_page_is_retried_after_source_recovers(endpoint, broken_row):
|
||||
identifier = 'dwdm' if endpoint == 'dws' else 'zydm'
|
||||
good_row = {identifier: '10001' if endpoint == 'dws' else '081200'}
|
||||
class Source(InterruptedCatalog):
|
||||
def request(self, method, url, **kwargs):
|
||||
row = broken_row if self.blocked else good_row
|
||||
return Response({'flag': True, 'msg': {'list': [row], 'nextPageAvailable': False}})
|
||||
source, cache = Source(), {}
|
||||
client = CatalogClient(source, page_cache=cache)
|
||||
with pytest.raises(RemoteUnavailable, match='代码'):
|
||||
list(client.pages(endpoint, {}))
|
||||
assert cache == {}
|
||||
source.blocked = False
|
||||
assert list(client.pages(endpoint, {})) == [good_row]
|
||||
+45
-1
@@ -17,7 +17,7 @@ def mysql():
|
||||
assert config['database'].endswith('_test'), 'refusing to clear a production database'
|
||||
db = MySQLDatabase(config)
|
||||
with db.connection() as conn:
|
||||
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs'):
|
||||
for table in ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs', 'job_catalog_pages'):
|
||||
conn.execute(f'DELETE FROM {table}')
|
||||
return db
|
||||
|
||||
@@ -68,6 +68,36 @@ def test_mysql_tasks_logs_and_checkpoints(mysql):
|
||||
assert mysql.logs('contract-job')[0]['message'] == '数据库测试'
|
||||
|
||||
|
||||
def test_mysql_catalog_page_cache_and_bulk_checkpoints_are_durable(mysql):
|
||||
mysql.job_create('cached-job', {'year': 2027})
|
||||
mysql.complete_unit('cached-job', 'catalog:11:10001:081200:1')
|
||||
page = {'list': [{'dwdm': '10001'}], 'nextPageAvailable': False}
|
||||
mysql.cache_catalog_page('cached-job', 'page-key', page)
|
||||
from studio.mysql_database import MySQLDatabase
|
||||
reopened = MySQLDatabase(mysql.config, recover_jobs=False)
|
||||
assert reopened.completed_units('cached-job') == {'catalog:11:10001:081200:1'}
|
||||
assert reopened.catalog_pages('cached-job') == {'page-key': page}
|
||||
assert reopened.catalog_pages('another-job') == {}
|
||||
|
||||
|
||||
def test_mysql_reuses_negotiated_tls_context_between_connections(mysql, monkeypatch):
|
||||
import ssl
|
||||
with mysql.connection() as conn:
|
||||
cipher = conn.execute("SHOW SESSION STATUS LIKE 'Ssl_cipher'").fetchone()[1]
|
||||
if not cipher:
|
||||
pytest.skip('requires a TLS-enabled MariaDB test server')
|
||||
creations = []
|
||||
original = ssl.create_default_context
|
||||
def observe(*args, **kwargs):
|
||||
creations.append(True)
|
||||
return original(*args, **kwargs)
|
||||
monkeypatch.setattr(ssl, 'create_default_context', observe)
|
||||
for _ in range(2):
|
||||
with mysql.connection() as conn:
|
||||
assert conn.execute('SELECT 1').fetchone()[0] == 1
|
||||
assert creations == []
|
||||
|
||||
|
||||
def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_path):
|
||||
from studio.migrate import migrate_sqlite
|
||||
source = Database(tmp_path / 'snapshot.db')
|
||||
@@ -77,16 +107,30 @@ def test_sqlite_migration_preserves_ids_and_refuses_nonempty_target(mysql, tmp_p
|
||||
source.job_create('migrated-job', {'year': 2027})
|
||||
source.complete_unit('migrated-job', 'catalog:11:10001:081200:1')
|
||||
source.log('migrated-job', 'info', 'preserve log')
|
||||
source.cache_catalog_page('migrated-job', 'migration-page', {'list': [], 'nextPageAvailable': False})
|
||||
counts = migrate_sqlite(tmp_path / 'snapshot.db', mysql)
|
||||
assert counts['catalog'] == 2 and counts['scores'] == 1
|
||||
assert mysql.record(1)['favorite']
|
||||
assert mysql.query_catalog({})['items'][0]['score_total'] == 320
|
||||
assert mysql.job_get('migrated-job')['completed_units'] == 1
|
||||
assert mysql.logs('migrated-job')[0]['id'] == 1
|
||||
assert counts['job_catalog_pages'] == 1
|
||||
assert mysql.catalog_pages('migrated-job')['migration-page']['list'] == []
|
||||
with pytest.raises(ValueError, match='非空'):
|
||||
migrate_sqlite(tmp_path / 'snapshot.db', mysql)
|
||||
|
||||
|
||||
def test_legacy_snapshot_without_catalog_page_table_can_still_be_migrated(mysql, tmp_path):
|
||||
from studio.migrate import migrate_sqlite
|
||||
source = Database(tmp_path / 'legacy.db')
|
||||
source.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
|
||||
with source.connection() as conn:
|
||||
conn.execute('DROP TABLE job_catalog_pages')
|
||||
counts = migrate_sqlite(source.path, mysql)
|
||||
assert counts['catalog'] == 2
|
||||
assert counts['job_catalog_pages'] == 0
|
||||
|
||||
|
||||
def test_migration_rolls_back_every_table_on_bad_source(mysql, tmp_path):
|
||||
from studio.migrate import migrate_sqlite
|
||||
source = Database(tmp_path / 'bad.db')
|
||||
|
||||
Reference in New Issue
Block a user