Fix historical pagination and bound export and proxy memory
This commit is contained in:
@@ -107,6 +107,10 @@ class MySQLDatabase(Database):
|
||||
with self.connection() as conn:
|
||||
for statement in SCHEMA:
|
||||
conn.execute(statement + ' ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin')
|
||||
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_export'").fetchone():
|
||||
conn.execute('CREATE INDEX catalog_export ON catalog(year DESC,school_name,id)')
|
||||
if not conn.execute("SHOW INDEX FROM catalog WHERE Key_name='catalog_targets'").fetchone():
|
||||
conn.execute('CREATE INDEX catalog_targets ON catalog(year,school_code,major_code)')
|
||||
if not conn.execute("SHOW COLUMNS FROM jobs LIKE 'resume_pending'").fetchone():
|
||||
conn.execute('ALTER TABLE jobs ADD COLUMN resume_pending TINYINT DEFAULT 0')
|
||||
if recover_jobs:
|
||||
@@ -130,7 +134,7 @@ class MySQLDatabase(Database):
|
||||
finally:
|
||||
raw.close()
|
||||
|
||||
def upsert_catalog(self, records):
|
||||
def _upsert_catalog(self, conn, records):
|
||||
fields = ['record_key', 'year', 'school_code', 'school_name', 'province', 'major_code', 'major_name', 'department_code', 'department_name', 'study_mode', 'degree_type', 'level', 'subjects', 'quota', 'source', 'data', 'department_norm']
|
||||
values = []
|
||||
for record in records:
|
||||
@@ -139,12 +143,11 @@ class MySQLDatabase(Database):
|
||||
if not values:
|
||||
return
|
||||
update = ','.join(f'{field}=VALUES({field})' for field in fields[1:])
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
|
||||
def upsert_scores(self, records):
|
||||
def _upsert_scores(self, conn, records):
|
||||
fields = ['school_code', 'major_code', 'year', 'department_id', 'department_name', 'scope', 'total', 'source', 'data', 'note', 'raw_code', 'score_key', 'department_norm', 'special_line']
|
||||
values = []
|
||||
for record in records:
|
||||
@@ -152,8 +155,7 @@ class MySQLDatabase(Database):
|
||||
row.update(data=encode(record), score_key=score_key(row), department_norm=normalized_name(row['department_name']), special_line=int(is_special_score(row['note'])))
|
||||
values.append([row.get(field) for field in fields])
|
||||
if values:
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
|
||||
conn.executemany(f"INSERT INTO scores({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON DUPLICATE KEY UPDATE total=VALUES(total),data=VALUES(data),department_norm=VALUES(department_norm),special_line=VALUES(special_line)", values)
|
||||
|
||||
def favorite(self, rid, enabled):
|
||||
# MariaDB reports changed rows; an already-selected favorite is still success.
|
||||
|
||||
Reference in New Issue
Block a user