Fix historical pagination and bound export and proxy memory
This commit is contained in:
+90
-7
@@ -19,6 +19,8 @@ CREATE TABLE IF NOT EXISTS catalog(
|
||||
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
|
||||
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
|
||||
CREATE INDEX IF NOT EXISTS catalog_export ON catalog(year DESC,school_name,id);
|
||||
CREATE INDEX IF NOT EXISTS catalog_targets ON catalog(year,school_code,major_code);
|
||||
""" + SCORE_SCHEMA + """
|
||||
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
|
||||
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
||||
@@ -69,6 +71,10 @@ class Database:
|
||||
conn.close()
|
||||
|
||||
def upsert_catalog(self, records: list[dict]):
|
||||
with self.connection() as conn:
|
||||
self._upsert_catalog(conn, records)
|
||||
|
||||
def _upsert_catalog(self, conn, records):
|
||||
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
|
||||
values = []
|
||||
for record in records:
|
||||
@@ -77,10 +83,9 @@ class Database:
|
||||
if not values:
|
||||
return
|
||||
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
|
||||
with self.connection() as conn:
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
|
||||
for record in records:
|
||||
self._school(conn, record)
|
||||
|
||||
def _school(self, conn, record):
|
||||
record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))}
|
||||
@@ -93,8 +98,11 @@ class Database:
|
||||
|
||||
def upsert_scores(self, records):
|
||||
with self.connection() as conn:
|
||||
for row in records:
|
||||
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
|
||||
self._upsert_scores(conn, records)
|
||||
|
||||
def _upsert_scores(self, conn, records):
|
||||
for row in records:
|
||||
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
|
||||
|
||||
def _query(self, filters: dict):
|
||||
conditions, params = ["1=1"], []
|
||||
@@ -141,9 +149,73 @@ class Database:
|
||||
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
|
||||
|
||||
def all_catalog(self, filters):
|
||||
return list(self.iter_catalog(filters))
|
||||
|
||||
def iter_catalog(self, filters, batch_size=200):
|
||||
select, params = self._query(filters)
|
||||
with self.connection() as conn:
|
||||
return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)]
|
||||
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM catalog').fetchone()[0]
|
||||
after = None
|
||||
while True:
|
||||
query = 'SELECT * FROM (' + select + ') export_rows WHERE id<=?'
|
||||
values = [*params, cutoff]
|
||||
if after:
|
||||
year, school, rid = after
|
||||
if school is None:
|
||||
query += ' AND (year<? OR (year=? AND (school_name IS NOT NULL OR (school_name IS NULL AND id>?))))'
|
||||
values.extend([year, year, rid])
|
||||
else:
|
||||
query += ' AND (year<? OR (year=? AND (school_name>? OR (school_name=? AND id>?))))'
|
||||
values.extend([year, year, school, school, rid])
|
||||
query += ' ORDER BY year DESC,school_name,id LIMIT ?'
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(query, [*values, batch_size]).fetchall()
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
yield self._catalog_row(row)
|
||||
last = rows[-1]
|
||||
after = (last['year'], last['school_name'], last['id'])
|
||||
|
||||
def iter_scores(self, pairs, batch_size=200):
|
||||
after = 0
|
||||
with self.connection() as conn:
|
||||
cutoff = conn.execute('SELECT COALESCE(MAX(id),0) FROM scores').fetchone()[0]
|
||||
while True:
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute('SELECT id,school_code,major_code,data FROM scores WHERE id>? AND id<=? ORDER BY id LIMIT ?', (after, cutoff, batch_size)).fetchall()
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
if (row['school_code'], row['major_code']) in pairs:
|
||||
yield json.loads(row['data'])
|
||||
after = rows[-1]['id']
|
||||
|
||||
def score_targets(self, filters, batch_size=200):
|
||||
conditions, params = ['year=?'], [filters['year']]
|
||||
if filters.get('major_code'):
|
||||
conditions.append('major_code=?')
|
||||
params.append(filters['major_code'])
|
||||
if filters.get('school_keyword'):
|
||||
conditions.append('school_name LIKE ?')
|
||||
params.append('%' + filters['school_keyword'] + '%')
|
||||
if filters.get('provinces'):
|
||||
conditions.append("(province='' OR province IN (" + ','.join('?' for _ in filters['provinces']) + '))')
|
||||
params.extend(filters['provinces'])
|
||||
after = None
|
||||
while True:
|
||||
where, values = ' AND '.join(conditions), list(params)
|
||||
if after:
|
||||
where += ' AND (school_code>? OR (school_code=? AND major_code>?))'
|
||||
values.extend([after[0], after[0], after[1]])
|
||||
query = 'SELECT school_code,major_code,MAX(school_name) AS school_name,MAX(province) AS province,MAX(level) AS level FROM catalog WHERE ' + where + ' GROUP BY school_code,major_code ORDER BY school_code,major_code LIMIT ?'
|
||||
with self.connection() as conn:
|
||||
rows = conn.execute(query, [*values, batch_size]).fetchall()
|
||||
if not rows:
|
||||
return
|
||||
yield from (dict(row) for row in rows)
|
||||
last = rows[-1]
|
||||
after = (last['school_code'], last['major_code'])
|
||||
|
||||
def _catalog_row(self, row):
|
||||
data = json.loads(row["data"])
|
||||
@@ -219,6 +291,17 @@ class Database:
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
|
||||
|
||||
def save_unit(self, jid, unit, records, scores=()):
|
||||
with self.connection() as conn:
|
||||
if not conn.execute('SELECT 1 FROM jobs WHERE id=?', (jid,)).fetchone():
|
||||
raise ValueError('任务不存在')
|
||||
if not conn.execute('INSERT OR IGNORE INTO job_units VALUES(?,?)', (jid, unit)).rowcount:
|
||||
return False
|
||||
self._upsert_scores(conn, scores)
|
||||
self._upsert_catalog(conn, records)
|
||||
conn.execute('UPDATE jobs SET records=records+?,updated_at=? WHERE id=?', (len(records), now(), jid))
|
||||
return True
|
||||
|
||||
def log(self, jid, level, message):
|
||||
with self.connection() as conn:
|
||||
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
|
||||
|
||||
Reference in New Issue
Block a user