fix: resume catalog jobs without repeated checkpoint queries

This commit is contained in:
2026-10-11 21:18:49 +08:00
parent c4a71e88a1
commit 4c510759fb
9 changed files with 309 additions and 13 deletions
+8 -1
View File
@@ -2,6 +2,7 @@
import hashlib
import json
import re
import ssl
from contextlib import contextmanager
import pymysql
from studio.database import Database, encode
@@ -23,6 +24,7 @@ SCHEMA = [
"CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)",
"CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)",
"CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))",
"CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id VARCHAR(64),page_key VARCHAR(64),data LONGTEXT NOT NULL,PRIMARY KEY(job_id,page_key))",
"CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))",
]
@@ -101,6 +103,7 @@ class MySQLDatabase(Database):
if not self.config.get('database'):
raise ValueError('数据库配置缺少 database')
self.recovered_job_ids = []
self._ssl_context = None
with self.connection() as conn:
for statement in SCHEMA:
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
@@ -113,8 +116,12 @@ class MySQLDatabase(Database):
@contextmanager
def connection(self):
raw = pymysql.connect(**self.config, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
raw = pymysql.connect(**self.config, ssl=self._ssl_context, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
try:
# Retain the driver's negotiated context rather than reload its CA
# store for every short-lived query. Plain servers retain default behavior.
if self._ssl_context is None and isinstance(getattr(raw, '_sock', None), ssl.SSLSocket):
self._ssl_context = raw.ctx
yield Connection(raw)
raw.commit()
except BaseException: