Files
kaoyan/studio/imports.py
T

234 lines
12 KiB
Python

"""Exchange formats for yzw CSV, YAM SQLite and studio snapshots."""
import csv
import io
import json
from contextlib import suppress
import sqlite3
from dataclasses import dataclass, field
from pathlib import Path
from openpyxl import load_workbook, Workbook
from studio.domain import finish_record, integer, parse_quota, normalize_catalog, PROVINCES
LABELS = {"year": "年份", "school_name": "院校", "school_code": "院校代码", "province": "省份", "level": "院校层次", "major_code": "专业代码", "major_name": "专业", "degree_type": "学位类型", "department_name": "院系", "direction": "研究方向", "study_mode": "学习方式", "exam_type": "考试方式", "advisor": "导师", "quota_text": "招生人数原文", "quota": "明确人数", "quota_scope": "人数口径", "subjects": "考试科目", "special_plans": "专项计划", "score_year": "参考线年份", "score_total": "参考线总分", "score_scope": "分数线口径", "remarks": "备注", "source": "来源", "source_url": "来源链接", "year_verified": "来源年份已核实", "fetched_at": "采集时间"}
LABELS.update({"department_code": "院系代码", "direction_code": "方向代码"})
ALIASES = {"院校名称": "school_name", "学校名称": "school_name", "专业名称": "major_name", "院系所": "department_name", "拟招生人数": "quota_text", "planned_enrollment": "quota_text", "nzsrsstr": "quota_text", "school_level": "level", "department": "department_name", "research_direction": "direction", "enrollment_text": "quota_text", "enrollment_count": "quota", "exam_subjects": "subjects"}
ALIASES.update({v: k for k, v in LABELS.items()})
@dataclass
class ImportResult:
records: list
scores: list
schools: list = field(default_factory=list)
def __iter__(self):
yield self.records
yield self.scores
def list_field(value):
if isinstance(value, list):
return value
if not value:
return []
if str(value).startswith("["):
try:
decoded = json.loads(value)
if isinstance(decoded, list):
return decoded
except ValueError:
pass
return str(value).split(" | ")
def row_record(raw, year=None, source="文件导入"):
row = {ALIASES.get(str(k).strip(), str(k).strip()): (v if v is not None else "") for k, v in raw.items() if k is not None}
explicit_year = integer(row.get("year")) or integer(year)
if explicit_year is None:
raise ValueError("文件未包含年份,请明确指定这份数据的招生年份")
if not 2000 <= explicit_year <= 2100:
raise ValueError("文件年份无效")
if not row.get("school_name") or not row.get("major_code"):
raise ValueError("文件需包含院校名称和专业代码")
row["major_code"] = str(row["major_code"]).strip().zfill(6)
if not row["major_code"].isdigit() or len(row["major_code"]) != 6:
raise ValueError("文件专业代码应为 6 位数字")
row["school_code"] = str(row.get("school_code", "")).strip()
subjects = list_field(row.get("subjects"))
if not subjects:
subjects = [str(row.get(f"exam_subject{i}") or row.get(f"subject{i}") or row.get(f"考试科目{i}") or "") for i in range(1, 5)]
quota, scope = parse_quota(row.get("quota_text", ""))
if quota is None and not str(row.get('quota_text') or '').strip():
quota = integer(row.get("quota"))
scope = row.get("quota_scope") or "unspecified"
plans = list_field(row.get("special_plans"))
if row.get("veteran_program") in ("是", "1", 1):
plans.append("退役大学生士兵")
if row.get("shaogu_program") in ("是", "1", 1):
plans.append("少数民族骨干")
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "reference_books": list_field(row.get('reference_books')), "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
def import_file(path, year=None):
path = Path(path)
suffix = path.suffix.lower()
scores = []
if suffix in (".db", ".sqlite", ".sqlite3"):
return import_yam(path, year)
if suffix == ".json":
content = json.loads(path.read_text(encoding="utf-8-sig"))
if isinstance(content, dict):
rows = content.get("catalog", content.get("records", []))
scores = content.get("scores", [])
else:
rows = content
if not isinstance(rows, list) or not isinstance(scores, list):
raise ValueError("JSON 应是记录数组或包含 catalog/scores 的工作台快照")
elif suffix == ".csv":
payload = path.read_bytes()
try:
text = payload.decode("utf-8-sig")
except UnicodeDecodeError:
text = payload.decode("gb18030")
rows = list(csv.DictReader(io.StringIO(text)))
elif suffix == ".xlsx":
book = load_workbook(path, read_only=True, data_only=True)
try:
sheet = book.active
if (sheet.max_row or 0) > 300001 or (sheet.max_column or 0) > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
values = sheet.iter_rows(values_only=True)
headers = next(values, [])
if len(headers) > 100:
raise ValueError("Excel 超过 100 列,请拆分后导入")
rows = []
for index, row in enumerate(values, 2):
if index > 300001 or len(row) > 100:
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
if any(v is not None for v in row):
rows.append(dict(zip(headers, row)))
finally:
book.close()
else:
raise ValueError("支持 CSV、XLSX、JSON 和 YAM SQLite 文件")
if len(rows) > 300000:
raise ValueError("一次最多导入 30 万条,请拆分文件")
records, schools = [], []
for row in rows:
if isinstance(row, dict) and row.get('dwdm') and row.get('dwmc'):
schools.append({'school_code': str(row['dwdm']), 'school_name': row['dwmc'], 'province': row.get('szss') or PROVINCES.get(str(row.get('szssm', '')), ''), 'level': '双一流' if str(row.get('syl')) == '1' else ''})
directions = row.get('data') or ([row] if row.get('kskmz') else [])
if not isinstance(directions, list):
raise ValueError('YAM 种子 data 应为方向数组')
for direction in directions:
if not isinstance(direction, dict) or not any(k in direction for k in ('yjfxmc', 'kskmz', 'yxsmc')):
raise ValueError('YAM 方向数据结构无法识别')
explicit = integer(row.get('year')) or integer(year)
if not explicit:
raise ValueError('种子含有目录详情,请指定其招生年份')
records.extend(normalize_catalog(direction, explicit, row))
else:
records.append(row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入"))
for score in scores:
if not all(k in score for k in ("school_code", "major_code", "year", "scope")):
raise ValueError("快照分数线缺少院校/专业/年份/口径")
score["year"] = int(score["year"])
score["total"] = integer(score.get("total"))
score.setdefault("note", "")
return ImportResult(records, scores, schools)
def import_yam(path, year):
conn = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)
conn.row_factory = sqlite3.Row
try:
conn.execute("PRAGMA query_only=ON")
tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
if not {"schools", "departments"}.issubset(tables):
raise ValueError("此 SQLite 不包含 YAM schools/departments 表;请从原软件导出 CSV/JSON")
schools = [dict(row) for row in conn.execute("SELECT * FROM schools")]
lookup = {(str(r["school_id"]), str(r.get("major_code", ""))): r for r in schools}
records, departments = [], {}
for entry in conn.execute("SELECT * FROM departments"):
row = dict(entry)
school = lookup.get((str(row["school_id"]), str(row.get("major_code", "")))) or next((r for r in schools if str(r["school_id"]) == str(row["school_id"])), {})
department = str(row.get("department_id", ""))
departments[(str(row["school_id"]), department)] = row.get("name", "")
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "level": school.get("level", ""), "department_code": department, "department_name": row.get("name", ""), "source": row.get("source") or "YAM 导入"}, year))
scores = []
if "score_lines" in tables:
for entry in conn.execute("SELECT * FROM score_lines"):
row = dict(entry)
if not integer(row.get("year")):
continue
raw = str(row.get("raw_code", ""))
scope = row.get("match_scope") or ({2: "category", 4: "discipline", 6: "major"}.get(len(raw)) if raw else "unknown")
scores.append({**row, "school_code": str(row["school_id"]), "major_code": str(row.get("major_code", "")), "department_id": str(row.get("department_id", "") or ""), "department_name": row.get("raw_department_name") or departments.get((str(row["school_id"]), str(row.get("department_id", ""))), ""), "scope": scope, "note": row.get("note") or "", "source": row.get("source") or "YAM 导入"})
if "admission_plans" in tables:
for entry in conn.execute("SELECT * FROM admission_plans"):
row = dict(entry)
school = lookup.get((str(row["school_id"]), str(row.get("major_code", ""))), {})
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "direction": row.get("research_direction", ""), "direction_code": str(row.get("spe_id") or row.get("plan_id") or ""), "department_code": str(row.get("department_id", "") or ""), "source": row.get("source") or "YAM 导入", "remarks": row.get("note", "")}, year))
return ImportResult(records, scores)
except sqlite3.DatabaseError:
raise ValueError("SQLite 文件无法读取,请确认是完整导出的数据库") from None
finally:
conn.close()
def spreadsheet_value(value):
if isinstance(value, (list, dict)):
value = " | ".join(str(v) for v in value) if isinstance(value, list) else json.dumps(value, ensure_ascii=False)
if value is None:
return ""
if isinstance(value, str) and value.lstrip().startswith(("=", "+", "-", "@", "\t", "\r")):
return "'" + value
return value
def export_sheet(records, kind):
if kind == "csv":
stream = io.StringIO(newline="")
writer = csv.writer(stream)
writer.writerow(list(LABELS.values()))
writer.writerows([spreadsheet_value(record.get(field)) for field in LABELS] for record in records)
return stream.getvalue().encode("utf-8-sig")
output = io.BytesIO()
write_xlsx(records, output)
return output.getvalue()
def write_xlsx(records, output):
from openpyxl.cell import WriteOnlyCell
from openpyxl.utils import get_column_letter
from openpyxl.styles import Font, PatternFill, Alignment
book = Workbook(write_only=True)
sheet = book.create_sheet("院校专业")
sheet.freeze_panes = "A2"
header = []
for index, label in enumerate(LABELS.values(), 1):
cell = WriteOnlyCell(sheet, value=label)
cell.font = Font(color="FFFFFF", bold=True)
cell.fill = PatternFill("solid", fgColor="183447")
cell.alignment = Alignment(vertical="center")
header.append(cell)
sheet.column_dimensions[get_column_letter(index)].width = 24
count = 1
try:
sheet.append(header)
for record in records:
sheet.append([spreadsheet_value(record.get(field)) for field in LABELS])
count += 1
sheet.auto_filter.ref = f'A1:{get_column_letter(len(LABELS))}{count}'
book.save(output)
except BaseException:
with suppress(Exception):
if not sheet.closed:
sheet.close()
with suppress(Exception):
if sheet._writer:
sheet._writer.cleanup()
raise
finally:
book.close()