"""按年份保存招生目录、来源证据、分数线和任务断点。""" import json import sqlite3 from contextlib import contextmanager from pathlib import Path from studio.domain import now, normalized_name SCHEMA = """ CREATE TABLE IF NOT EXISTS catalog( id INTEGER PRIMARY KEY, record_key TEXT NOT NULL UNIQUE, year INTEGER NOT NULL, school_code TEXT, school_name TEXT, province TEXT, major_code TEXT, major_name TEXT, department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT, subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL); CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code); CREATE TABLE IF NOT EXISTS scores( id INTEGER PRIMARY KEY, school_code TEXT, major_code TEXT, year INTEGER, department_id TEXT, department_name TEXT, scope TEXT, total INTEGER, source TEXT, data TEXT NOT NULL, UNIQUE(school_code,major_code,year,department_id,department_name,scope,source)); CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC); CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT); CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0); CREATE TABLE IF NOT EXISTS job_units(job_id TEXT, unit TEXT, PRIMARY KEY(job_id,unit)); CREATE TABLE IF NOT EXISTS job_logs(id INTEGER PRIMARY KEY,job_id TEXT,created_at TEXT,level TEXT,message TEXT); """ def encode(value): return json.dumps(value, ensure_ascii=False, separators=(",", ":")) class Database: def __init__(self, path: Path): self.path = Path(path) self.path.parent.mkdir(parents=True, exist_ok=True) with self.connection() as conn: conn.executescript(SCHEMA) conn.execute("UPDATE jobs SET status='paused',message='服务重启,任务可从断点恢复' WHERE status IN ('running','queued')") @contextmanager def connection(self): conn = sqlite3.connect(self.path, timeout=30) conn.row_factory = sqlite3.Row conn.create_function("norm_name", 1, normalized_name, deterministic=True) conn.execute("PRAGMA journal_mode=WAL") try: yield conn conn.commit() except BaseException: conn.rollback() raise finally: conn.close() def upsert_catalog(self, records: list[dict]): fields = ["record_key","year","school_code","school_name","province","major_code","major_name","department_code","department_name","study_mode","degree_type","level","subjects","quota","source","data"] values = [] for record in records: row = {**record, "subjects":" | ".join(record.get("subjects", [])), "data":encode(record)} values.append([row.get(field) for field in fields]) if not values: return update = ",".join(f"{field}=excluded.{field}" for field in fields[1:]) with self.connection() as conn: conn.executemany(f"INSERT INTO catalog({','.join(fields)}) VALUES({','.join('?' for _ in fields)}) ON CONFLICT(record_key) DO UPDATE SET {update}", values) for record in records: self._school(conn, record) def _school(self, conn, record): conn.execute("INSERT INTO schools VALUES(?,?,?,?,?) ON CONFLICT(school_code) DO UPDATE SET school_name=excluded.school_name,province=CASE WHEN excluded.province<>'' THEN excluded.province ELSE schools.province END,level=CASE WHEN excluded.level<>'' THEN excluded.level ELSE schools.level END", (record["school_code"],record["school_name"],record.get("province", ""),record.get("level", ""),encode(record))) def index_schools(self, rows): with self.connection() as conn: for row in rows: self._school(conn, row) def upsert_scores(self, records): with self.connection() as conn: for row in records: conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", ""),row.get("department_name", ""),row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row))) def _query(self, filters: dict): conditions, params = ["1=1"], [] for field in ["year","province","major_code","study_mode","degree_type","school_code"]: if filters.get(field) not in (None, "", 0): conditions.append(f"c.{field}=?") params.append(filters[field]) for field in ["level","subjects","department_name"]: value = filters.get(field) or (filters.get("subject") if field == "subjects" else "") if value: conditions.append(f"c.{field} LIKE ?") params.append(f"%{value}%") if filters.get("direction"): conditions.append("json_extract(c.data,'$.direction') LIKE ?") params.append(f"%{filters['direction']}%") if filters.get("keyword"): conditions.append("(c.school_name LIKE ? OR c.major_name LIKE ? OR c.department_name LIKE ? OR c.major_code LIKE ?)") params.extend([f"%{filters['keyword']}%"] * 4) if filters.get("favorites"): conditions.append("c.favorite=1") if filters.get("min_quota") is not None: conditions.append("c.quota>=?") params.append(filters["min_quota"]) for field, index in [("english", 1), ("math", 2)]: if filters.get(field): conditions.append(f"json_extract(c.data,'$.subjects[{index}]') LIKE ?") params.append(str(filters[field])+"%") select = """SELECT c.*,s.data AS score_data,s.total AS score_total,s.year AS score_year,s.scope AS score_scope FROM catalog c LEFT JOIN scores s ON s.id=(SELECT x.id FROM scores x WHERE x.school_code=c.school_code AND x.major_code=c.major_code AND x.year<=c.year AND (x.department_name='' AND x.department_id='' OR x.department_id<>'' AND x.department_id=c.department_code OR x.department_name<>'' AND norm_name(x.department_name)=norm_name(c.department_name)) AND coalesce(json_extract(x.data,'$.note'),'') NOT LIKE '%专项%' AND coalesce(json_extract(x.data,'$.note'),'') NOT LIKE '%少干%' ORDER BY x.year DESC,CASE x.scope WHEN 'major' THEN 0 WHEN 'discipline' THEN 1 ELSE 2 END,x.id DESC LIMIT 1) WHERE """ + " AND ".join(conditions) if filters.get("max_score") is not None: select = "SELECT * FROM ("+select+") q WHERE q.score_total<=?" params.append(filters["max_score"]) return select, params def query_catalog(self, filters: dict, page=1, page_size=50): select, params = self._query(filters) page, page_size = max(1,int(page)), max(1,min(int(page_size),200)) with self.connection() as conn: total = conn.execute("SELECT count(*) FROM ("+select+")", params).fetchone()[0] order = {"school":"school_name,major_code,department_name,id", "quota":"quota IS NULL,quota DESC,school_name,id", "score":"score_total IS NULL,score_total,school_name,id", "year":"year DESC,school_name,major_code,department_name,id"}.get(filters.get("sort"), "year DESC,school_name,major_code,department_name,id") rows = conn.execute(select+" ORDER BY "+order+" LIMIT ? OFFSET ?",[*params,page_size,(page-1)*page_size]).fetchall() return {"total":total,"page":page,"page_size":page_size,"items":[self._catalog_row(row) for row in rows]} def all_catalog(self, filters): select, params = self._query(filters) with self.connection() as conn: return [self._catalog_row(row) for row in conn.execute(select+" ORDER BY year DESC,school_name,id",params)] def _catalog_row(self, row): data = json.loads(row["data"]) return {**data,"id":row["id"],"favorite":bool(row["favorite"]),"score_total":row["score_total"],"score_year":row["score_year"],"score_scope":row["score_scope"],"score":json.loads(row["score_data"]) if row["score_data"] else None} def record(self, rid): with self.connection() as conn: row = conn.execute("SELECT data,favorite FROM catalog WHERE id=?",(rid,)).fetchone() return {**json.loads(row["data"]),"id":rid,"favorite":bool(row["favorite"])} if row else None def favorite(self, rid, enabled): with self.connection() as conn: return conn.execute("UPDATE catalog SET favorite=? WHERE id=?",(int(enabled),rid)).rowcount > 0 def history(self, rid): record = self.record(rid) if record is None: return None with self.connection() as conn: scores = conn.execute("SELECT data FROM scores WHERE school_code=? AND major_code=? ORDER BY year,scope,department_name",(record["school_code"],record["major_code"])).fetchall() plans = conn.execute("SELECT data FROM catalog WHERE school_code=? AND major_code=? AND norm_name(department_name)=norm_name(?) ORDER BY year,id",(record["school_code"],record["major_code"],record["department_name"])).fetchall() def matches(row): return not row.get("department_name") and not row.get("department_id") or row.get("department_id") and row["department_id"] == record["department_code"] or row.get("department_name") and normalized_name(row["department_name"]) == normalized_name(record["department_name"]) return {"record":record,"scores":[row for entry in scores if matches(row:=json.loads(entry["data"]))],"plans":[json.loads(row["data"]) for row in plans]} def overview(self): with self.connection() as conn: totals = dict(conn.execute("SELECT count(*) AS records,count(DISTINCT school_code) AS schools,count(DISTINCT major_code) AS majors,sum(favorite) AS favorites FROM catalog").fetchone()) totals["indexed_schools"] = conn.execute("SELECT count(*) FROM schools").fetchone()[0] totals["scores"] = conn.execute("SELECT count(*) FROM scores").fetchone()[0] totals["years"] = [row[0] for row in conn.execute("SELECT DISTINCT year FROM catalog ORDER BY year DESC")] totals["provinces"] = [dict(row) for row in conn.execute("SELECT province,count(DISTINCT school_code) AS count FROM catalog WHERE province<>'' GROUP BY province ORDER BY count DESC LIMIT 10")] return totals def schools(self, keyword="", province=""): with self.connection() as conn: return [dict(row) for row in conn.execute("SELECT school_code,school_name,province,level FROM schools WHERE school_name LIKE ? AND (?='' OR province=?) ORDER BY school_name LIMIT 2000",(f"%{keyword}%",province,province))] def job_create(self, jid, params): with self.connection() as conn: conn.execute("INSERT INTO jobs(id,params,status,created_at,updated_at,message) VALUES(?,?,'queued',?,?,?)",(jid,encode(params),now(),now(),"等待开始")) def job_update(self, jid, **changes): allowed = {"status","message","errors","records"} changes = {key:value for key,value in changes.items() if key in allowed} changes["updated_at"] = now() with self.connection() as conn: conn.execute("UPDATE jobs SET "+",".join(f"{key}=?" for key in changes)+" WHERE id=?",[*changes.values(),jid]) def job_get(self, jid=None): with self.connection() as conn: rows = conn.execute("SELECT j.*,(SELECT count(*) FROM job_units u WHERE u.job_id=j.id) AS completed_units FROM jobs j "+("WHERE j.id=?" if jid else "ORDER BY j.created_at DESC LIMIT 100"), (jid,) if jid else ()).fetchall() result = [{**dict(row),"params":json.loads(row["params"])} for row in rows] return (result[0] if result else None) if jid else result def unit_done(self, jid, unit): with self.connection() as conn: return conn.execute("SELECT 1 FROM job_units WHERE job_id=? AND unit=?",(jid,unit)).fetchone() is not None def complete_unit(self, jid, unit): with self.connection() as conn: conn.execute("INSERT OR IGNORE INTO job_units VALUES(?,?)",(jid,unit)) def log(self, jid, level, message): with self.connection() as conn: conn.execute("INSERT INTO job_logs(job_id,created_at,level,message) VALUES(?,?,?,?)",(jid,now(),level,str(message)[:2000])) def logs(self, jid, after=0): with self.connection() as conn: return [dict(row) for row in conn.execute("SELECT * FROM job_logs WHERE job_id=? AND id>? ORDER BY id DESC LIMIT 300",(jid,after))][::-1]