Files
kaoyan/studio/imports.py
T

179 lines
9.9 KiB
Python

"""Exchange formats for yzw CSV, YAM SQLite and studio snapshots."""
import csv
import io
import json
import sqlite3
from pathlib import Path
from openpyxl import load_workbook, Workbook
from studio.domain import finish_record, integer, parse_quota
LABELS = {"year": "年份", "school_name": "院校", "school_code": "院校代码", "province": "省份", "level": "院校层次", "major_code": "专业代码", "major_name": "专业", "degree_type": "学位类型", "department_name": "院系", "direction": "研究方向", "study_mode": "学习方式", "exam_type": "考试方式", "advisor": "导师", "quota_text": "招生人数原文", "quota": "明确人数", "quota_scope": "人数口径", "subjects": "考试科目", "special_plans": "专项计划", "score_year": "参考线年份", "score_total": "参考线总分", "score_scope": "分数线口径", "remarks": "备注", "source": "来源", "source_url": "来源链接", "year_verified": "来源年份已核实", "fetched_at": "采集时间"}
ALIASES = {"院校名称": "school_name", "学校名称": "school_name", "专业名称": "major_name", "院系所": "department_name", "拟招生人数": "quota_text", "planned_enrollment": "quota_text", "nzsrsstr": "quota_text", "school_level": "level", "department": "department_name", "research_direction": "direction", "enrollment_text": "quota_text", "enrollment_count": "quota", "exam_subjects": "subjects"}
ALIASES.update({v: k for k, v in LABELS.items()})
def list_field(value):
if isinstance(value, list):
return value
if not value:
return []
if str(value).startswith("["):
try:
decoded = json.loads(value)
if isinstance(decoded, list):
return decoded
except ValueError:
pass
return str(value).split(" | ")
def row_record(raw, year=None, source="文件导入"):
row = {ALIASES.get(str(k).strip(), str(k).strip()): (v if v is not None else "") for k, v in raw.items() if k is not None}
explicit_year = integer(row.get("year")) or integer(year)
if explicit_year is None:
raise ValueError("文件未包含年份,请明确指定这份数据的招生年份")
if not 2000 <= explicit_year <= 2100:
raise ValueError("文件年份无效")
if not row.get("school_name") or not row.get("major_code"):
raise ValueError("文件需包含院校名称和专业代码")
row["major_code"] = str(row["major_code"]).strip().zfill(6)
if not row["major_code"].isdigit() or len(row["major_code"]) != 6:
raise ValueError("文件专业代码应为 6 位数字")
row["school_code"] = str(row.get("school_code", "")).strip()
subjects = list_field(row.get("subjects"))
if not subjects:
subjects = [str(row.get(f"exam_subject{i}") or row.get(f"subject{i}") or row.get(f"考试科目{i}") or "") for i in range(1, 5)]
quota, scope = parse_quota(row.get("quota_text", ""))
if quota is None and row.get("quota") != "":
quota = integer(row.get("quota"))
scope = row.get("quota_scope") or "unspecified"
plans = list_field(row.get("special_plans"))
if row.get("veteran_program") in ("是", "1", 1):
plans.append("退役大学生士兵")
if row.get("shaogu_program") in ("是", "1", 1):
plans.append("少数民族骨干")
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
def import_file(path, year=None):
path = Path(path)
suffix = path.suffix.lower()
scores = []
if suffix in (".db", ".sqlite", ".sqlite3"):
return import_yam(path, year)
if suffix == ".json":
content = json.loads(path.read_text(encoding="utf-8-sig"))
if isinstance(content, dict):
rows = content.get("catalog", content.get("records", []))
scores = content.get("scores", [])
else:
rows = content
if not isinstance(rows, list) or not isinstance(scores, list):
raise ValueError("JSON 应是记录数组或包含 catalog/scores 的工作台快照")
elif suffix == ".csv":
payload = path.read_bytes()
try:
text = payload.decode("utf-8-sig")
except UnicodeDecodeError:
text = payload.decode("gb18030")
rows = list(csv.DictReader(io.StringIO(text)))
elif suffix == ".xlsx":
book = load_workbook(path, read_only=True, data_only=True)
try:
sheet = book.active
if sheet.max_row > 300001 or sheet.max_column > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
values = sheet.iter_rows(values_only=True)
headers = next(values, [])
rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)]
finally:
book.close()
else:
raise ValueError("支持 CSV、XLSX、JSON 和 YAM SQLite 文件")
if len(rows) > 300000:
raise ValueError("一次最多导入 30 万条,请拆分文件")
records = [row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入") for row in rows]
for score in scores:
if not all(k in score for k in ("school_code", "major_code", "year", "scope")):
raise ValueError("快照分数线缺少院校/专业/年份/口径")
score["year"] = int(score["year"])
score["total"] = integer(score.get("total"))
score.setdefault("note", "")
return records, scores
def import_yam(path, year):
conn = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)
conn.row_factory = sqlite3.Row
try:
conn.execute("PRAGMA query_only=ON")
tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
if not {"schools", "departments"}.issubset(tables):
raise ValueError("此 SQLite 不包含 YAM schools/departments 表;请从原软件导出 CSV/JSON")
schools = [dict(row) for row in conn.execute("SELECT * FROM schools")]
lookup = {(str(r["school_id"]), str(r.get("major_code", ""))): r for r in schools}
records, departments = [], {}
for entry in conn.execute("SELECT * FROM departments"):
row = dict(entry)
school = lookup.get((str(row["school_id"]), str(row.get("major_code", "")))) or next((r for r in schools if str(r["school_id"]) == str(row["school_id"])), {})
department = str(row.get("department_id", ""))
departments[(str(row["school_id"]), department)] = row.get("name", "")
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "level": school.get("level", ""), "department_code": department, "department_name": row.get("name", ""), "source": row.get("source") or "YAM 导入"}, year))
scores = []
if "score_lines" in tables:
for entry in conn.execute("SELECT * FROM score_lines"):
row = dict(entry)
if not integer(row.get("year")):
continue
raw = str(row.get("raw_code", ""))
scope = row.get("match_scope") or ({2: "category", 4: "discipline", 6: "major"}.get(len(raw)) if raw else "unknown")
scores.append({**row, "school_code": str(row["school_id"]), "major_code": str(row.get("major_code", "")), "department_id": str(row.get("department_id", "") or ""), "department_name": row.get("raw_department_name") or departments.get((str(row["school_id"]), str(row.get("department_id", ""))), ""), "scope": scope, "note": row.get("note") or "", "source": row.get("source") or "YAM 导入"})
if "admission_plans" in tables:
for entry in conn.execute("SELECT * FROM admission_plans"):
row = dict(entry)
school = lookup.get((str(row["school_id"]), str(row.get("major_code", ""))), {})
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "direction": row.get("research_direction", ""), "direction_code": str(row.get("spe_id") or row.get("plan_id") or ""), "department_code": str(row.get("department_id", "") or ""), "source": row.get("source") or "YAM 导入", "remarks": row.get("note", "")}, year))
return records, scores
except sqlite3.DatabaseError:
raise ValueError("SQLite 文件无法读取,请确认是完整导出的数据库") from None
finally:
conn.close()
def spreadsheet_value(value):
if isinstance(value, (list, dict)):
value = " | ".join(str(v) for v in value) if isinstance(value, list) else json.dumps(value, ensure_ascii=False)
if value is None:
return ""
if isinstance(value, str) and value.lstrip().startswith(("=", "+", "-", "@", "\t", "\r")):
return "'" + value
return value
def export_sheet(records, kind):
rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records]
if kind == "csv":
stream = io.StringIO(newline="")
writer = csv.writer(stream)
writer.writerow(list(LABELS.values()))
writer.writerows(rows)
return stream.getvalue().encode("utf-8-sig")
book = Workbook()
sheet = book.active
sheet.title = "院校专业"
sheet.append(list(LABELS.values()))
for row in rows:
sheet.append(row)
sheet.freeze_panes = "A2"
sheet.auto_filter.ref = sheet.dimensions
from openpyxl.styles import Font, PatternFill, Alignment
for cell in sheet[1]:
cell.font = Font(color="FFFFFF", bold=True)
cell.fill = PatternFill("solid", fgColor="183447")
cell.alignment = Alignment(vertical="center")
for col in sheet.columns:
sheet.column_dimensions[col[0].column_letter].width = 24
output = io.BytesIO()
book.save(output)
return output.getvalue()