fix: resume catalog jobs without repeated checkpoint queries
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
from contextlib import contextmanager
|
||||
import pymysql
|
||||
from studio.database import Database, encode
|
||||
@@ -23,6 +24,7 @@ SCHEMA = [
|
||||
"CREATE TABLE IF NOT EXISTS schools(school_code VARCHAR(32) PRIMARY KEY,school_name VARCHAR(255),province VARCHAR(64),level VARCHAR(128),data LONGTEXT)",
|
||||
"CREATE TABLE IF NOT EXISTS jobs(id VARCHAR(64) PRIMARY KEY,params LONGTEXT,status VARCHAR(32),created_at VARCHAR(40),updated_at VARCHAR(40),message TEXT,errors INT DEFAULT 0,records INT DEFAULT 0,resume_pending TINYINT DEFAULT 0)",
|
||||
"CREATE TABLE IF NOT EXISTS job_units(job_id VARCHAR(64),unit VARCHAR(255),PRIMARY KEY(job_id,unit))",
|
||||
"CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id VARCHAR(64),page_key VARCHAR(64),data LONGTEXT NOT NULL,PRIMARY KEY(job_id,page_key))",
|
||||
"CREATE TABLE IF NOT EXISTS job_logs(id BIGINT PRIMARY KEY AUTO_INCREMENT,job_id VARCHAR(64),created_at VARCHAR(40),level VARCHAR(32),message TEXT,INDEX job_logs_lookup(job_id,id))",
|
||||
]
|
||||
|
||||
@@ -101,6 +103,7 @@ class MySQLDatabase(Database):
|
||||
if not self.config.get('database'):
|
||||
raise ValueError('数据库配置缺少 database')
|
||||
self.recovered_job_ids = []
|
||||
self._ssl_context = None
|
||||
with self.connection() as conn:
|
||||
for statement in SCHEMA:
|
||||
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
|
||||
@@ -113,8 +116,12 @@ class MySQLDatabase(Database):
|
||||
|
||||
@contextmanager
|
||||
def connection(self):
|
||||
raw = pymysql.connect(**self.config, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
|
||||
raw = pymysql.connect(**self.config, ssl=self._ssl_context, charset='utf8mb4', autocommit=False, connect_timeout=10, read_timeout=120, write_timeout=120)
|
||||
try:
|
||||
# Retain the driver's negotiated context rather than reload its CA
|
||||
# store for every short-lived query. Plain servers retain default behavior.
|
||||
if self._ssl_context is None and isinstance(getattr(raw, '_sock', None), ssl.SSLSocket):
|
||||
self._ssl_context = raw.ctx
|
||||
yield Connection(raw)
|
||||
raw.commit()
|
||||
except BaseException:
|
||||
|
||||
Reference in New Issue
Block a user