fix: preserve data granularity and recover tasks safely
This commit is contained in:
+23
-7
@@ -3,8 +3,15 @@ import json
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from studio.domain import now, normalized_name
|
||||
from studio.domain import now, normalized_name, is_special_score, school_level
|
||||
|
||||
SCORE_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS scores(
|
||||
id INTEGER PRIMARY KEY, school_code TEXT, major_code TEXT, year INTEGER,
|
||||
department_id TEXT, department_name TEXT, scope TEXT, total INTEGER, source TEXT, data TEXT NOT NULL,
|
||||
note TEXT NOT NULL DEFAULT '',raw_code TEXT NOT NULL DEFAULT '',
|
||||
UNIQUE(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code));
|
||||
"""
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS catalog(
|
||||
id INTEGER PRIMARY KEY, record_key TEXT NOT NULL UNIQUE, year INTEGER NOT NULL,
|
||||
@@ -12,10 +19,7 @@ CREATE TABLE IF NOT EXISTS catalog(
|
||||
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
|
||||
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
|
||||
CREATE TABLE IF NOT EXISTS scores(
|
||||
id INTEGER PRIMARY KEY, school_code TEXT, major_code TEXT, year INTEGER,
|
||||
department_id TEXT, department_name TEXT, scope TEXT, total INTEGER, source TEXT, data TEXT NOT NULL,
|
||||
UNIQUE(school_code,major_code,year,department_id,department_name,scope,source));
|
||||
""" + SCORE_SCHEMA + """
|
||||
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
|
||||
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
||||
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0);
|
||||
@@ -34,6 +38,13 @@ class Database:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.connection() as conn:
|
||||
conn.executescript(SCHEMA)
|
||||
if 'note' not in {row['name'] for row in conn.execute('PRAGMA table_info(scores)')}:
|
||||
# Rebuild atomically to replace the early schema's narrower unique key.
|
||||
conn.execute(SCORE_SCHEMA.replace('IF NOT EXISTS scores', 'scores_new'))
|
||||
conn.execute("""INSERT INTO scores_new(id,school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) SELECT id,school_code,major_code,year,department_id,department_name,scope,total,source,data,coalesce(json_extract(data,'$.note'),''),coalesce(json_extract(data,'$.raw_code'),'') FROM scores""")
|
||||
conn.execute('DROP TABLE scores')
|
||||
conn.execute('ALTER TABLE scores_new RENAME TO scores')
|
||||
conn.execute('CREATE INDEX scores_lookup ON scores(school_code,major_code,year DESC)')
|
||||
conn.execute("UPDATE jobs SET status='paused',message='服务重启,任务可从断点恢复' WHERE status IN ('running','queued')")
|
||||
|
||||
@contextmanager
|
||||
@@ -41,6 +52,7 @@ class Database:
|
||||
conn = sqlite3.connect(self.path, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.create_function("norm_name", 1, normalized_name, deterministic=True)
|
||||
conn.create_function("is_special_score", 1, is_special_score, deterministic=True)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
try:
|
||||
yield conn
|
||||
@@ -66,6 +78,7 @@ class Database:
|
||||
self._school(conn, record)
|
||||
|
||||
def _school(self, conn, record):
|
||||
record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))}
|
||||
conn.execute("INSERT INTO schools VALUES(?,?,?,?,?) ON CONFLICT(school_code) DO UPDATE SET school_name=excluded.school_name,province=CASE WHEN excluded.province<>'' THEN excluded.province ELSE schools.province END,level=CASE WHEN excluded.level<>'' THEN excluded.level ELSE schools.level END", (record["school_code"],record["school_name"],record.get("province", ""),record.get("level", ""),encode(record)))
|
||||
|
||||
def index_schools(self, rows):
|
||||
@@ -76,7 +89,7 @@ class Database:
|
||||
def upsert_scores(self, records):
|
||||
with self.connection() as conn:
|
||||
for row in records:
|
||||
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", ""),row.get("department_name", ""),row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row)))
|
||||
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
|
||||
|
||||
def _query(self, filters: dict):
|
||||
conditions, params = ["1=1"], []
|
||||
@@ -95,6 +108,9 @@ class Database:
|
||||
if filters.get("keyword"):
|
||||
conditions.append("(c.school_name LIKE ? OR c.major_name LIKE ? OR c.department_name LIKE ? OR c.major_code LIKE ?)")
|
||||
params.extend([f"%{filters['keyword']}%"] * 4)
|
||||
if filters.get('school_keyword'):
|
||||
conditions.append('c.school_name LIKE ?')
|
||||
params.append(f"%{filters['school_keyword']}%")
|
||||
if filters.get("favorites"):
|
||||
conditions.append("c.favorite=1")
|
||||
if filters.get("min_quota") is not None:
|
||||
@@ -104,7 +120,7 @@ class Database:
|
||||
if filters.get(field):
|
||||
conditions.append(f"json_extract(c.data,'$.subjects[{index}]') LIKE ?")
|
||||
params.append(str(filters[field])+"%")
|
||||
select = """SELECT c.*,s.data AS score_data,s.total AS score_total,s.year AS score_year,s.scope AS score_scope FROM catalog c LEFT JOIN scores s ON s.id=(SELECT x.id FROM scores x WHERE x.school_code=c.school_code AND x.major_code=c.major_code AND x.year<=c.year AND (x.department_name='' AND x.department_id='' OR x.department_id<>'' AND x.department_id=c.department_code OR x.department_name<>'' AND norm_name(x.department_name)=norm_name(c.department_name)) AND coalesce(json_extract(x.data,'$.note'),'') NOT LIKE '%专项%' AND coalesce(json_extract(x.data,'$.note'),'') NOT LIKE '%少干%' ORDER BY x.year DESC,CASE x.scope WHEN 'major' THEN 0 WHEN 'discipline' THEN 1 ELSE 2 END,x.id DESC LIMIT 1) WHERE """ + " AND ".join(conditions)
|
||||
select = """SELECT c.*,s.data AS score_data,s.total AS score_total,s.year AS score_year,s.scope AS score_scope FROM catalog c LEFT JOIN scores s ON s.id=(SELECT x.id FROM scores x WHERE x.school_code=c.school_code AND x.major_code=c.major_code AND x.year<=c.year AND (x.department_name='' AND x.department_id='' OR x.department_id<>'' AND x.department_id=c.department_code OR x.department_name<>'' AND norm_name(x.department_name)=norm_name(c.department_name)) AND is_special_score(json_extract(x.data,'$.note'))=0 ORDER BY x.year DESC,CASE x.scope WHEN 'major' THEN 0 WHEN 'discipline' THEN 1 ELSE 2 END,x.id DESC LIMIT 1) WHERE """ + " AND ".join(conditions)
|
||||
if filters.get("max_score") is not None:
|
||||
select = "SELECT * FROM ("+select+") q WHERE q.score_total<=?"
|
||||
params.append(filters["max_score"])
|
||||
|
||||
Reference in New Issue
Block a user