feat: implement unified data sources, proxy routing and local API

This commit is contained in:
Codex
2026-10-11 13:49:07 +08:00
parent 25cccd4372
commit 389b2aa3a5
20 changed files with 21601 additions and 4 deletions
+178
View File
@@ -0,0 +1,178 @@
"""Exchange formats for yzw CSV, YAM SQLite and studio snapshots."""
import csv
import io
import json
import sqlite3
from pathlib import Path
from openpyxl import load_workbook, Workbook
from studio.domain import finish_record, integer, parse_quota
LABELS = {"year": "年份", "school_name": "院校", "school_code": "院校代码", "province": "省份", "level": "院校层次", "major_code": "专业代码", "major_name": "专业", "degree_type": "学位类型", "department_name": "院系", "direction": "研究方向", "study_mode": "学习方式", "exam_type": "考试方式", "advisor": "导师", "quota_text": "招生人数原文", "quota": "明确人数", "quota_scope": "人数口径", "subjects": "考试科目", "special_plans": "专项计划", "score_year": "参考线年份", "score_total": "参考线总分", "score_scope": "分数线口径", "remarks": "备注", "source": "来源", "source_url": "来源链接", "year_verified": "来源年份已核实", "fetched_at": "采集时间"}
ALIASES = {"院校名称": "school_name", "学校名称": "school_name", "专业名称": "major_name", "院系所": "department_name", "拟招生人数": "quota_text", "planned_enrollment": "quota_text", "nzsrsstr": "quota_text", "school_level": "level", "department": "department_name", "research_direction": "direction", "enrollment_text": "quota_text", "enrollment_count": "quota", "exam_subjects": "subjects"}
ALIASES.update({v: k for k, v in LABELS.items()})
def list_field(value):
if isinstance(value, list):
return value
if not value:
return []
if str(value).startswith("["):
try:
decoded = json.loads(value)
if isinstance(decoded, list):
return decoded
except ValueError:
pass
return str(value).split(" | ")
def row_record(raw, year=None, source="文件导入"):
row = {ALIASES.get(str(k).strip(), str(k).strip()): (v if v is not None else "") for k, v in raw.items() if k is not None}
explicit_year = integer(row.get("year")) or integer(year)
if explicit_year is None:
raise ValueError("文件未包含年份,请明确指定这份数据的招生年份")
if not 2000 <= explicit_year <= 2100:
raise ValueError("文件年份无效")
if not row.get("school_name") or not row.get("major_code"):
raise ValueError("文件需包含院校名称和专业代码")
row["major_code"] = str(row["major_code"]).strip().zfill(6)
if not row["major_code"].isdigit() or len(row["major_code"]) != 6:
raise ValueError("文件专业代码应为 6 位数字")
row["school_code"] = str(row.get("school_code", "")).strip()
subjects = list_field(row.get("subjects"))
if not subjects:
subjects = [str(row.get(f"exam_subject{i}") or row.get(f"subject{i}") or row.get(f"考试科目{i}") or "") for i in range(1, 5)]
quota, scope = parse_quota(row.get("quota_text", ""))
if quota is None and row.get("quota") != "":
quota = integer(row.get("quota"))
scope = row.get("quota_scope") or "unspecified"
plans = list_field(row.get("special_plans"))
if row.get("veteran_program") in ("是", "1", 1):
plans.append("退役大学生士兵")
if row.get("shaogu_program") in ("是", "1", 1):
plans.append("少数民族骨干")
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
def import_file(path, year=None):
path = Path(path)
suffix = path.suffix.lower()
scores = []
if suffix in (".db", ".sqlite", ".sqlite3"):
return import_yam(path, year)
if suffix == ".json":
content = json.loads(path.read_text(encoding="utf-8-sig"))
if isinstance(content, dict):
rows = content.get("catalog", content.get("records", []))
scores = content.get("scores", [])
else:
rows = content
if not isinstance(rows, list) or not isinstance(scores, list):
raise ValueError("JSON 应是记录数组或包含 catalog/scores 的工作台快照")
elif suffix == ".csv":
payload = path.read_bytes()
try:
text = payload.decode("utf-8-sig")
except UnicodeDecodeError:
text = payload.decode("gb18030")
rows = list(csv.DictReader(io.StringIO(text)))
elif suffix == ".xlsx":
book = load_workbook(path, read_only=True, data_only=True)
try:
sheet = book.active
if sheet.max_row > 300001 or sheet.max_column > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
values = sheet.iter_rows(values_only=True)
headers = next(values, [])
rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)]
finally:
book.close()
else:
raise ValueError("支持 CSV、XLSX、JSON 和 YAM SQLite 文件")
if len(rows) > 300000:
raise ValueError("一次最多导入 30 万条,请拆分文件")
records = [row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入") for row in rows]
for score in scores:
if not all(k in score for k in ("school_code", "major_code", "year", "scope")):
raise ValueError("快照分数线缺少院校/专业/年份/口径")
score["year"] = int(score["year"])
score["total"] = integer(score.get("total"))
score.setdefault("note", "")
return records, scores
def import_yam(path, year):
conn = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)
conn.row_factory = sqlite3.Row
try:
conn.execute("PRAGMA query_only=ON")
tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
if not {"schools", "departments"}.issubset(tables):
raise ValueError("此 SQLite 不包含 YAM schools/departments 表;请从原软件导出 CSV/JSON")
schools = [dict(row) for row in conn.execute("SELECT * FROM schools")]
lookup = {(str(r["school_id"]), str(r.get("major_code", ""))): r for r in schools}
records, departments = [], {}
for entry in conn.execute("SELECT * FROM departments"):
row = dict(entry)
school = lookup.get((str(row["school_id"]), str(row.get("major_code", "")))) or next((r for r in schools if str(r["school_id"]) == str(row["school_id"])), {})
department = str(row.get("department_id", ""))
departments[(str(row["school_id"]), department)] = row.get("name", "")
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "level": school.get("level", ""), "department_code": department, "department_name": row.get("name", ""), "source": row.get("source") or "YAM 导入"}, year))
scores = []
if "score_lines" in tables:
for entry in conn.execute("SELECT * FROM score_lines"):
row = dict(entry)
if not integer(row.get("year")):
continue
raw = str(row.get("raw_code", ""))
scope = row.get("match_scope") or ({2: "category", 4: "discipline", 6: "major"}.get(len(raw)) if raw else "unknown")
scores.append({**row, "school_code": str(row["school_id"]), "major_code": str(row.get("major_code", "")), "department_id": str(row.get("department_id", "") or ""), "department_name": row.get("raw_department_name") or departments.get((str(row["school_id"]), str(row.get("department_id", ""))), ""), "scope": scope, "note": row.get("note") or "", "source": row.get("source") or "YAM 导入"})
if "admission_plans" in tables:
for entry in conn.execute("SELECT * FROM admission_plans"):
row = dict(entry)
school = lookup.get((str(row["school_id"]), str(row.get("major_code", ""))), {})
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "direction": row.get("research_direction", ""), "direction_code": str(row.get("spe_id") or row.get("plan_id") or ""), "department_code": str(row.get("department_id", "") or ""), "source": row.get("source") or "YAM 导入", "remarks": row.get("note", "")}, year))
return records, scores
except sqlite3.DatabaseError:
raise ValueError("SQLite 文件无法读取,请确认是完整导出的数据库") from None
finally:
conn.close()
def spreadsheet_value(value):
if isinstance(value, (list, dict)):
value = " | ".join(str(v) for v in value) if isinstance(value, list) else json.dumps(value, ensure_ascii=False)
if value is None:
return ""
if isinstance(value, str) and value.lstrip().startswith(("=", "+", "-", "@", "\t", "\r")):
return "'" + value
return value
def export_sheet(records, kind):
rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records]
if kind == "csv":
stream = io.StringIO(newline="")
writer = csv.writer(stream)
writer.writerow(list(LABELS.values()))
writer.writerows(rows)
return stream.getvalue().encode("utf-8-sig")
book = Workbook()
sheet = book.active
sheet.title = "院校专业"
sheet.append(list(LABELS.values()))
for row in rows:
sheet.append(row)
sheet.freeze_panes = "A2"
sheet.auto_filter.ref = sheet.dimensions
from openpyxl.styles import Font, PatternFill, Alignment
for cell in sheet[1]:
cell.font = Font(color="FFFFFF", bold=True)
cell.fill = PatternFill("solid", fgColor="183447")
cell.alignment = Alignment(vertical="center")
for col in sheet.columns:
sheet.column_dimensions[col[0].column_letter].width = 24
output = io.BytesIO()
book.save(output)
return output.getvalue()