fix: resume catalog jobs without repeated checkpoint queries
This commit is contained in:
+42
-8
@@ -1,6 +1,7 @@
|
||||
"""Independent adapters for the two projects' public data sources."""
|
||||
import json
|
||||
import re
|
||||
import hashlib
|
||||
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
||||
from studio.network import RemoteUnavailable, RateLimited
|
||||
|
||||
@@ -18,8 +19,9 @@ class SourceYearMismatch(RuntimeError):
|
||||
class CatalogClient:
|
||||
BASE = "https://yz.chsi.com.cn/zsml/"
|
||||
|
||||
def __init__(self, transport, auth=None):
|
||||
def __init__(self, transport, auth=None, page_cache=None, save_page=None, on_request=None):
|
||||
self.transport, self.auth = transport, auth
|
||||
self.page_cache, self.save_page, self.on_request = page_cache, save_page, on_request
|
||||
|
||||
def verify_year(self, year):
|
||||
response = self.transport.request("GET", self.BASE)
|
||||
@@ -65,8 +67,16 @@ class CatalogClient:
|
||||
return data
|
||||
|
||||
def fetch_page(self, endpoint, payload):
|
||||
# Lists are a snapshot for this task. Direction records and login proofs
|
||||
# remain live requests; failed/partial responses never enter the cache.
|
||||
cacheable = self.page_cache is not None and endpoint in ('dws', 'dwzys')
|
||||
key = self._page_key(endpoint, payload) if cacheable else None
|
||||
if cacheable and key in self.page_cache:
|
||||
return self.page_cache[key]
|
||||
if self.on_request:
|
||||
self.on_request(endpoint, payload)
|
||||
try:
|
||||
return self._fetch_page(endpoint, payload)
|
||||
data = self._fetch_page(endpoint, payload)
|
||||
except LoginRequired as exc:
|
||||
if not self.auth or exc.reason == "captcha":
|
||||
raise
|
||||
@@ -76,27 +86,51 @@ class CatalogClient:
|
||||
status = self.auth.check()
|
||||
if not status["valid"]:
|
||||
raise LoginRequired(status["message"], reason="captcha" if status["state"] == "captcha_required" else "login") from None
|
||||
return self._fetch_page(endpoint, payload)
|
||||
data = self._fetch_page(endpoint, payload)
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _page_key(endpoint, payload):
|
||||
return hashlib.sha256(json.dumps([endpoint, payload], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
|
||||
def _cache_page(self, endpoint, payload, data):
|
||||
if self.page_cache is None or endpoint not in ('dws', 'dwzys'):
|
||||
return
|
||||
key = self._page_key(endpoint, payload)
|
||||
if key not in self.page_cache:
|
||||
if self.save_page:
|
||||
self.save_page(key, data)
|
||||
self.page_cache[key] = data
|
||||
|
||||
def pages(self, endpoint, payload, page_size=10):
|
||||
seen, start = set(), 0
|
||||
for page in range(1, 10001):
|
||||
data = self.fetch_page(endpoint, {**payload, "curPage": page, "start": start, "pageSize": page_size})
|
||||
page_payload = {**payload, "curPage": page, "start": start, "pageSize": page_size}
|
||||
data = self.fetch_page(endpoint, page_payload)
|
||||
rows = data["list"]
|
||||
identifier = {'dws': 'dwdm', 'dwzys': 'zydm'}.get(endpoint)
|
||||
if identifier:
|
||||
for row in rows:
|
||||
code = row.get(identifier) if isinstance(row, dict) else None
|
||||
if isinstance(code, bool) or not isinstance(code, (str, int)) or not str(code).strip():
|
||||
label = '院校' if endpoint == 'dws' else '专业'
|
||||
raise RemoteUnavailable(f'研招目录{label}代码缺失或结构异常,已停止以避免缓存不完整数据')
|
||||
signature = json.dumps(rows, ensure_ascii=False, sort_keys=True)
|
||||
if rows and signature in seen:
|
||||
raise RemoteUnavailable("数据源重复返回同一页,已停止以避免遗漏分页")
|
||||
seen.add(signature)
|
||||
yield from rows
|
||||
more = data.get("nextPageAvailable")
|
||||
total = integer(data.get("totalCount"))
|
||||
size = integer(data.get("size")) or page_size
|
||||
if not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size):
|
||||
return
|
||||
terminal = not rows or more is False or more in ("false", "0", 0) or more is None and ((total is not None and start + len(rows) >= total) or len(rows) < size)
|
||||
next_start = integer(data.get("startOfNextPage"))
|
||||
next_start = start + len(rows) if next_start is None else next_start
|
||||
if next_start <= start:
|
||||
if not terminal and next_start <= start:
|
||||
raise RemoteUnavailable("研招目录下一页位置无效,已停止以避免遗漏分页")
|
||||
self._cache_page(endpoint, page_payload, data)
|
||||
yield from rows
|
||||
if terminal:
|
||||
return
|
||||
start, page_size = next_start, size
|
||||
raise RemoteUnavailable("目录分页超过安全上限")
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year D
|
||||
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
||||
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
|
||||
CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id TEXT,page_key TEXT,data TEXT NOT NULL,PRIMARY KEY(job_id,page_key));
|
||||
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
|
||||
"""
|
||||
|
||||
@@ -202,6 +203,18 @@ class Database:
|
||||
with self.connection() as conn:
|
||||
return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None
|
||||
|
||||
def completed_units(self, jid):
|
||||
with self.connection() as conn:
|
||||
return {row['unit'] for row in conn.execute('SELECT unit FROM job_units WHERE job_id=?', (jid,))}
|
||||
|
||||
def catalog_pages(self, jid):
|
||||
with self.connection() as conn:
|
||||
return {row['page_key']: json.loads(row['data']) for row in conn.execute('SELECT page_key,data FROM job_catalog_pages WHERE job_id=?', (jid,))}
|
||||
|
||||
def cache_catalog_page(self, jid, page_key, data):
|
||||
with self.connection() as conn:
|
||||
conn.execute('INSERT INTO job_catalog_pages(job_id,page_key,data) VALUES(?,?,?) ON CONFLICT(job_id,page_key) DO UPDATE SET data=excluded.data', (jid, page_key, encode(data)))
|
||||
|
||||
def complete_unit(self, jid, unit):
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
|
||||
|
||||
+20
-2
@@ -189,7 +189,13 @@ class JobManager:
|
||||
self._scores(jid, params)
|
||||
|
||||
def _catalog(self, jid, params):
|
||||
client = CatalogClient(self.transport, auth=self.auth)
|
||||
completed = self.db.completed_units(jid)
|
||||
location = '研招目录'
|
||||
def on_request(endpoint, payload):
|
||||
phase = {'dws': '院校列表', 'dwzys': '专业列表', 'yjfxs': '研究方向'}[endpoint]
|
||||
self.db.job_update(jid, message=f"{location}:正在请求{phase}第 {payload.get('curPage', 1)} 页")
|
||||
client = CatalogClient(self.transport, auth=self.auth, page_cache=self.db.catalog_pages(jid),
|
||||
save_page=lambda key, data: self.db.cache_catalog_page(jid, key, data), on_request=on_request)
|
||||
client.verify_year(params["year"])
|
||||
if self.auth:
|
||||
status = self.auth.check()
|
||||
@@ -198,18 +204,27 @@ class JobManager:
|
||||
self.db.log(jid, "info", "研招目录会话已同步,第二页访问验证成功")
|
||||
for province in params["provinces"]:
|
||||
self.checkpoint(jid)
|
||||
location = PROVINCES[province]
|
||||
self.db.job_update(jid, message=location + ':正在核对院校列表')
|
||||
self.db.log(jid, "info", "查询 " + PROVINCES[province] + " 院校")
|
||||
for school in client.schools(province, params.get("school_keyword", "")):
|
||||
self.checkpoint(jid)
|
||||
school = {"ssdm": province, **school}
|
||||
location = f"{PROVINCES[province]} · {school.get('dwmc', school['dwdm'])}"
|
||||
self.db.job_update(jid, message=location + ':正在核对专业列表与已完成断点')
|
||||
self.db.log(jid, 'info', location + ':核对专业列表与断点')
|
||||
skipped = 0
|
||||
for major in client.majors(school, params.get("major_code", ""), params.get("study_mode", "")):
|
||||
self.checkpoint(jid)
|
||||
if params.get("major_code") and str(major.get("zydm")) != str(params["major_code"]):
|
||||
continue
|
||||
unit = f"catalog:{province}:{school['dwdm']}:{major['zydm']}:{major.get('xxfs','')}"
|
||||
if self.unit_done(jid, unit):
|
||||
if unit in completed:
|
||||
skipped += 1
|
||||
continue
|
||||
self.checkpoint(jid)
|
||||
label = f"{school.get('dwmc', '')} · {major.get('zymc', '')}"
|
||||
location = label
|
||||
self.db.job_update(jid, message=label)
|
||||
try:
|
||||
records = []
|
||||
@@ -217,9 +232,12 @@ class JobManager:
|
||||
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
|
||||
self._save(jid, records)
|
||||
self.complete_unit(jid, unit)
|
||||
completed.add(unit)
|
||||
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
|
||||
except RemoteUnavailable as exc:
|
||||
self._error(jid, f"{label}:{exc}")
|
||||
if skipped:
|
||||
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
|
||||
|
||||
def _scores(self, jid, params):
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
|
||||
|
||||
+4
-1
@@ -6,7 +6,7 @@ from pathlib import Path
|
||||
from studio.domain import normalized_name, is_special_score
|
||||
from studio.mysql_database import MySQLDatabase, score_key
|
||||
|
||||
TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs')
|
||||
TABLES = ('catalog', 'scores', 'schools', 'jobs', 'job_units', 'job_logs', 'job_catalog_pages')
|
||||
|
||||
|
||||
def migrate_sqlite(source, target):
|
||||
@@ -16,6 +16,7 @@ def migrate_sqlite(source, target):
|
||||
counts = {}
|
||||
try:
|
||||
conn.execute('BEGIN')
|
||||
source_tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
||||
with target.connection() as output:
|
||||
# Serialize attempted imports to the same database.
|
||||
acquired = output.execute("SELECT GET_LOCK('kaoyan_studio_migration',10)").fetchone()[0]
|
||||
@@ -27,6 +28,8 @@ def migrate_sqlite(source, target):
|
||||
raise ValueError(f'目标数据库非空({table}),拒绝覆盖;请使用空数据库')
|
||||
for table in TABLES:
|
||||
counts[table] = 0
|
||||
if table == 'job_catalog_pages' and table not in source_tables:
|
||||
continue # Existing snapshots predate the optional page cache.
|
||||
reader = conn.execute(f'SELECT * FROM {table}')
|
||||
while batch := reader.fetchmany(500):
|
||||
rows = []
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
from contextlib import contextmanager
|
||||
import pymysql
|
||||
from studio.database import Database, encode
|
||||
@@ -23,6 +24,7 @@ SCHEMA = [
|
||||
"CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)",
|
||||
"CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)",
|
||||
"CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))",
|
||||
"CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id VARCHAR(64),page_key VARCHAR(64),data LONGTEXT NOT NULL,PRIMARY KEY(job_id,page_key))",
|
||||
"CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))",
|
||||
]
|
||||
|
||||
@@ -101,6 +103,7 @@ class MySQLDatabase(Database):
|
||||
if not self.config.get('database'):
|
||||
raise ValueError('数据库配置缺少 database')
|
||||
self.recovered_job_ids = []
|
||||
self._ssl_context = None
|
||||
with self.connection() as conn:
|
||||
for statement in SCHEMA:
|
||||
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
|
||||
@@ -113,8 +116,12 @@ class MySQLDatabase(Database):
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
raw = pymysql.connect(**self.config, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
|
||||
raw = pymysql.connect(**self.config, ssl=self._ssl_context, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
|
||||
try:
|
||||
# Retain the driver's negotiated context rather than reload its CA
|
||||
# store for every short-lived query. Plain servers retain default behavior.
|
||||
if self._ssl_context is None and isinstance(getattr(raw, '_sock', None), ssl.SSLSocket):
|
||||
self._ssl_context = raw.ctx
|
||||
yield Connection(raw)
|
||||
raw.commit()
|
||||
except BaseException:
|
||||
|
||||
Reference in New Issue
Block a user