213 lines
14 KiB
Python
213 lines
14 KiB
Python
"""按年份保存招生目录、来源证据、分数线和任务断点。"""
|
|
import json
|
|
import sqlite3
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from studio.domain import now, normalized_name, is_special_score, school_level
|
|
|
|
SCORE_SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS scores(
|
|
id INTEGER PRIMARY KEY, school_code TEXT, major_code TEXT, year INTEGER,
|
|
department_id TEXT, department_name TEXT, scope TEXT, total INTEGER, source TEXT, data TEXT NOT NULL,
|
|
note TEXT NOT NULL DEFAULT '',raw_code TEXT NOT NULL DEFAULT '',
|
|
UNIQUE(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code));
|
|
"""
|
|
SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS catalog(
|
|
id INTEGER PRIMARY KEY, record_key TEXT NOT NULL UNIQUE, year INTEGER NOT NULL,
|
|
school_code TEXT, school_name TEXT, province TEXT, major_code TEXT, major_name TEXT,
|
|
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
|
|
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
|
|
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
|
|
""" + SCORE_SCHEMA + """
|
|
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
|
|
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
|
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0);
|
|
CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit));
|
|
CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT);
|
|
"""
|
|
|
|
|
|
def encode(value):
|
|
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
|
|
|
|
class Database:
|
|
def __init__(self, path: Path):
|
|
self.path = Path(path)
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
with self.connection() as conn:
|
|
conn.executescript(SCHEMA)
|
|
if 'note' not in {row['name'] for row in conn.execute('PRAGMA table_info(scores)')}:
|
|
# Rebuild atomically to replace the early schema's narrower unique key.
|
|
conn.execute(SCORE_SCHEMA.replace('IF NOT EXISTS scores', 'scores_new'))
|
|
conn.execute("""INSERT INTO scores_new(id,school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) SELECT id,school_code,major_code,year,department_id,department_name,scope,total,source,data,coalesce(json_extract(data,'$.note'),''),coalesce(json_extract(data,'$.raw_code'),'') FROM scores""")
|
|
conn.execute('DROP TABLE scores')
|
|
conn.execute('ALTER TABLE scores_new RENAME TO scores')
|
|
conn.execute('CREATE INDEX scores_lookup ON scores(school_code,major_code,year DESC)')
|
|
conn.execute("UPDATE jobs SET status='paused',message='服务重启,任务可从断点恢复' WHERE status IN ('running','queued')")
|
|
|
|
@contextmanager
|
|
def connection(self):
|
|
conn = sqlite3.connect(self.path, timeout=30)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.create_function("norm_name", 1, normalized_name, deterministic=True)
|
|
conn.create_function("is_special_score", 1, is_special_score, deterministic=True)
|
|
conn.execute("PRAGMA journal_mode=WAL")
|
|
try:
|
|
yield conn
|
|
conn.commit()
|
|
except BaseException:
|
|
conn.rollback()
|
|
raise
|
|
finally:
|
|
conn.close()
|
|
|
|
def upsert_catalog(self, records: list[dict]):
|
|
fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"]
|
|
values = []
|
|
for record in records:
|
|
row = {**record, "subjects":" | ".join(record.get("subjects", [])), "data":encode(record)}
|
|
values.append([row.get(field) for field in fields])
|
|
if not values:
|
|
return
|
|
update = ",".join(f"{field}=excluded.{field}" for field in fields[1:])
|
|
with self.connection() as conn:
|
|
conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values)
|
|
for record in records:
|
|
self._school(conn, record)
|
|
|
|
def _school(self, conn, record):
|
|
record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))}
|
|
conn.execute("INSERT INTO schools VALUES(?,?,?,?,?) ON CONFLICT(school_code) DO UPDATE SET school_name=excluded.school_name,province=CASE WHEN excluded.province<>'' THEN excluded.province ELSE schools.province END,level=CASE WHEN excluded.level<>'' THEN excluded.level ELSE schools.level END", (record["school_code"],record["school_name"],record.get("province", ""),record.get("level", ""),encode(record)))
|
|
|
|
def index_schools(self, rows):
|
|
with self.connection() as conn:
|
|
for row in rows:
|
|
self._school(conn, row)
|
|
|
|
def upsert_scores(self, records):
|
|
with self.connection() as conn:
|
|
for row in records:
|
|
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
|
|
|
|
def _query(self, filters: dict):
|
|
conditions, params = ["1=1"], []
|
|
for field in ["year","province","major_code","study_mode","degree_type","school_code"]:
|
|
if filters.get(field) not in (None, "", 0):
|
|
conditions.append(f"c.{field}=?")
|
|
params.append(filters[field])
|
|
for field in ["level","subjects","department_name"]:
|
|
value = filters.get(field) or (filters.get("subject") if field == "subjects" else "")
|
|
if value:
|
|
conditions.append(f"c.{field} LIKE ?")
|
|
params.append(f"%{value}%")
|
|
if filters.get("direction"):
|
|
conditions.append("json_extract(c.data,'$.direction') LIKE ?")
|
|
params.append(f"%{filters['direction']}%")
|
|
if filters.get("keyword"):
|
|
conditions.append("(c.school_name LIKE ? OR c.major_name LIKE ? OR c.department_name LIKE ? OR c.major_code LIKE ?)")
|
|
params.extend([f"%{filters['keyword']}%"] * 4)
|
|
if filters.get('school_keyword'):
|
|
conditions.append('c.school_name LIKE ?')
|
|
params.append(f"%{filters['school_keyword']}%")
|
|
if filters.get("favorites"):
|
|
conditions.append("c.favorite=1")
|
|
if filters.get("min_quota") is not None:
|
|
conditions.append("c.quota>=?")
|
|
params.append(filters["min_quota"])
|
|
for field, index in [("english", 1), ("math", 2)]:
|
|
if filters.get(field):
|
|
conditions.append(f"json_extract(c.data,'$.subjects[{index}]') LIKE ?")
|
|
params.append(str(filters[field])+"%")
|
|
select = """SELECT c.*,s.data AS score_data,s.total AS score_total,s.year AS score_year,s.scope AS score_scope FROM catalog c LEFT JOIN scores s ON s.id=(SELECT x.id FROM scores x WHERE x.school_code=c.school_code AND x.major_code=c.major_code AND x.year<=c.year AND (x.department_name='' AND x.department_id='' OR x.department_id<>'' AND x.department_id=c.department_code OR x.department_name<>'' AND norm_name(x.department_name)=norm_name(c.department_name)) AND is_special_score(json_extract(x.data,'$.note'))=0 ORDER BY x.year DESC,CASE x.scope WHEN 'major' THEN 0 WHEN 'discipline' THEN 1 ELSE 2 END,x.id DESC LIMIT 1) WHERE """ + " AND ".join(conditions)
|
|
if filters.get("max_score") is not None:
|
|
select = "SELECT * FROM ("+select+") q WHERE q.score_total<=?"
|
|
params.append(filters["max_score"])
|
|
return select, params
|
|
|
|
def query_catalog(self, filters: dict, page=1, page_size=50):
|
|
select, params = self._query(filters)
|
|
page, page_size = max(1,int(page)), max(1,min(int(page_size),200))
|
|
with self.connection() as conn:
|
|
total = conn.execute("SELECT count(*) FROM ("+select+")", params).fetchone()[0]
|
|
order = {"school":"school_name,major_code,department_name,id", "quota":"quota IS NULL,quota DESC,school_name,id", "score":"score_total IS NULL,score_total,school_name,id", "year":"year DESC,school_name,major_code,department_name,id"}.get(filters.get("sort"), "year DESC,school_name,major_code,department_name,id")
|
|
rows = conn.execute(select+" ORDER BY "+order+" LIMIT ? OFFSET ?",[*params,page_size,(page-1)*page_size]).fetchall()
|
|
return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]}
|
|
|
|
def all_catalog(self, filters):
|
|
select, params = self._query(filters)
|
|
with self.connection() as conn:
|
|
return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)]
|
|
|
|
def _catalog_row(self, row):
|
|
data = json.loads(row["data"])
|
|
return {**data,"id":row["id"],"favorite":bool(row["favorite"]),"score_total":row["score_total"],"score_year":row["score_year"],"score_scope":row["score_scope"],"score":json.loads(row["score_data"]) if row["score_data"] else None}
|
|
|
|
def record(self, rid):
|
|
with self.connection() as conn:
|
|
row = conn.execute("SELECT data,favorite FROM catalog WHERE id=?",(rid,)).fetchone()
|
|
return {**json.loads(row["data"]),"id":rid,"favorite":bool(row["favorite"])} if row else None
|
|
|
|
def favorite(self, rid, enabled):
|
|
with self.connection() as conn:
|
|
return conn.execute("UPDATE catalog SET favorite=? WHERE id=?",(int(enabled),rid)).rowcount > 0
|
|
|
|
def history(self, rid):
|
|
record = self.record(rid)
|
|
if record is None:
|
|
return None
|
|
with self.connection() as conn:
|
|
scores = conn.execute("SELECT data FROM scores WHERE school_code=? AND major_code=? ORDER BY year,scope,department_name",(record["school_code"],record["major_code"])).fetchall()
|
|
plans = conn.execute("SELECT data FROM catalog WHERE school_code=? AND major_code=? AND norm_name(department_name)=norm_name(?) ORDER BY year,id",(record["school_code"],record["major_code"],record["department_name"])).fetchall()
|
|
def matches(row):
|
|
return not row.get("department_name") and not row.get("department_id") or row.get("department_id") and row["department_id"] == record["department_code"] or row.get("department_name") and normalized_name(row["department_name"]) == normalized_name(record["department_name"])
|
|
return {"record":record,"scores":[row for entry in scores if matches(row:=json.loads(entry["data"]))],"plans":[json.loads(row["data"]) for row in plans]}
|
|
|
|
def overview(self):
|
|
with self.connection() as conn:
|
|
totals = dict(conn.execute("SELECT count(*) AS records,count(DISTINCT school_code) AS schools,count(DISTINCT major_code) AS majors,sum(favorite) AS favorites FROM catalog").fetchone())
|
|
totals["indexed_schools"] = conn.execute("SELECT count(*) FROM schools").fetchone()[0]
|
|
totals["scores"] = conn.execute("SELECT count(*) FROM scores").fetchone()[0]
|
|
totals["years"] = [row[0] for row in conn.execute("SELECT DISTINCT year FROM catalog ORDER BY year DESC")]
|
|
totals["provinces"] = [dict(row) for row in conn.execute("SELECT province,count(DISTINCT school_code) AS count FROM catalog WHERE province<>'' GROUP BY province ORDER BY count DESC LIMIT 10")]
|
|
return totals
|
|
|
|
def schools(self, keyword="", province=""):
|
|
with self.connection() as conn:
|
|
return [dict(row) for row in conn.execute("SELECT school_code,school_name,province,level FROM schools WHERE school_name LIKE ? AND (?='' OR province=?) ORDER BY school_name LIMIT 2000",(f"%{keyword}%",province,province))]
|
|
|
|
def job_create(self, jid, params):
|
|
with self.connection() as conn:
|
|
conn.execute("INSERT INTO jobs(id,params,status,created_at,updated_at,message) VALUES(?,?,'queued',?,?,?)",(jid,encode(params),now(),now(),"等待开始"))
|
|
|
|
def job_update(self, jid, **changes):
|
|
allowed = {"status","message","errors","records"}
|
|
changes = {key:value for key,value in changes.items() if key in allowed}
|
|
changes["updated_at"] = now()
|
|
with self.connection() as conn:
|
|
conn.execute("UPDATE jobs SET "+",".join(f"{key}=?" for key in changes)+" WHERE id=?",[*changes.values(),jid])
|
|
|
|
def job_get(self, jid=None):
|
|
with self.connection() as conn:
|
|
rows = conn.execute("SELECT j.*,(SELECT count(*) FROM job_units u WHERE u.job_id=j.id) AS completed_units FROM jobs j "+("WHERE j.id=?" if jid else "ORDER BY j.created_at DESC LIMIT 100"), (jid,) if jid else ()).fetchall()
|
|
result = [{**dict(row),"params":json.loads(row["params"])} for row in rows]
|
|
return (result[0] if result else None) if jid else result
|
|
|
|
def unit_done(self, jid, unit):
|
|
with self.connection() as conn:
|
|
return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None
|
|
|
|
def complete_unit(self, jid, unit):
|
|
with self.connection() as conn:
|
|
conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit))
|
|
|
|
def log(self, jid, level, message):
|
|
with self.connection() as conn:
|
|
conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000]))
|
|
|
|
def logs(self, jid, after=0):
|
|
with self.connection() as conn:
|
|
return [dict(row) for row in conn.execute("SELECT * FROM job_logs WHERE job_id=? AND id>? ORDER BY id DESC LIMIT 300",(jid,after))][::-1]
|
|
|