fix: resume catalog jobs without repeated checkpoint queries
This commit is contained in:
@@ -24,6 +24,7 @@ CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year D
|
||||
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
||||
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0,resume_pending INTEGER DEFAULT 0);
|
||||
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
|
||||
CREATE TABLE IF NOT EXISTS job_catalog_pages(job_id TEXT,page_key TEXT,data TEXT NOT NULL,PRIMARY KEY(job_id,page_key));
|
||||
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
|
||||
"""
|
||||
|
||||
@@ -202,6 +203,18 @@ class Database:
|
||||
with self.connection() as conn:
|
||||
return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None
|
||||
|
||||
def completed_units(self, jid):
|
||||
with self.connection() as conn:
|
||||
return {row['unit'] for row in conn.execute('SELECT unit FROM job_units WHERE job_id=?', (jid,))}
|
||||
|
||||
def catalog_pages(self, jid):
|
||||
with self.connection() as conn:
|
||||
return {row['page_key']: json.loads(row['data']) for row in conn.execute('SELECT page_key,data FROM job_catalog_pages WHERE job_id=?', (jid,))}
|
||||
|
||||
def cache_catalog_page(self, jid, page_key, data):
|
||||
with self.connection() as conn:
|
||||
conn.execute('INSERT INTO job_catalog_pages(job_id,page_key,data) VALUES(?,?,?) ON CONFLICT(job_id,page_key) DO UPDATE SET data=excluded.data', (jid, page_key, encode(data)))
|
||||
|
||||
def complete_unit(self, jid, unit):
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
|
||||
|
||||
Reference in New Issue
Block a user