234 lines
12 KiB
Python
234 lines
12 KiB
Python
"""Exchange formats for yzw CSV, YAM SQLite and studio snapshots."""
|
|
import csv
|
|
import io
|
|
import json
|
|
from contextlib import suppress
|
|
import sqlite3
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from openpyxl import load_workbook, Workbook
|
|
from studio.domain import finish_record, integer, parse_quota, normalize_catalog, PROVINCES
|
|
|
|
LABELS = {"year": "年份", "school_name": "院校", "school_code": "院校代码", "province": "省份", "level": "院校层次", "major_code": "专业代码", "major_name": "专业", "degree_type": "学位类型", "department_name": "院系", "direction": "研究方向", "study_mode": "学习方式", "exam_type": "考试方式", "advisor": "导师", "quota_text": "招生人数原文", "quota": "明确人数", "quota_scope": "人数口径", "subjects": "考试科目", "special_plans": "专项计划", "score_year": "参考线年份", "score_total": "参考线总分", "score_scope": "分数线口径", "remarks": "备注", "source": "来源", "source_url": "来源链接", "year_verified": "来源年份已核实", "fetched_at": "采集时间"}
|
|
LABELS.update({"department_code": "院系代码", "direction_code": "方向代码"})
|
|
ALIASES = {"院校名称": "school_name", "学校名称": "school_name", "专业名称": "major_name", "院系所": "department_name", "拟招生人数": "quota_text", "planned_enrollment": "quota_text", "nzsrsstr": "quota_text", "school_level": "level", "department": "department_name", "research_direction": "direction", "enrollment_text": "quota_text", "enrollment_count": "quota", "exam_subjects": "subjects"}
|
|
ALIASES.update({v: k for k, v in LABELS.items()})
|
|
|
|
|
|
@dataclass
|
|
class ImportResult:
|
|
records: list
|
|
scores: list
|
|
schools: list = field(default_factory=list)
|
|
|
|
def __iter__(self):
|
|
yield self.records
|
|
yield self.scores
|
|
|
|
|
|
def list_field(value):
|
|
if isinstance(value, list):
|
|
return value
|
|
if not value:
|
|
return []
|
|
if str(value).startswith("["):
|
|
try:
|
|
decoded = json.loads(value)
|
|
if isinstance(decoded, list):
|
|
return decoded
|
|
except ValueError:
|
|
pass
|
|
return str(value).split(" | ")
|
|
|
|
|
|
def row_record(raw, year=None, source="文件导入"):
|
|
row = {ALIASES.get(str(k).strip(), str(k).strip()): (v if v is not None else "") for k, v in raw.items() if k is not None}
|
|
explicit_year = integer(row.get("year")) or integer(year)
|
|
if explicit_year is None:
|
|
raise ValueError("文件未包含年份,请明确指定这份数据的招生年份")
|
|
if not 2000 <= explicit_year <= 2100:
|
|
raise ValueError("文件年份无效")
|
|
if not row.get("school_name") or not row.get("major_code"):
|
|
raise ValueError("文件需包含院校名称和专业代码")
|
|
row["major_code"] = str(row["major_code"]).strip().zfill(6)
|
|
if not row["major_code"].isdigit() or len(row["major_code"]) != 6:
|
|
raise ValueError("文件专业代码应为 6 位数字")
|
|
row["school_code"] = str(row.get("school_code", "")).strip()
|
|
subjects = list_field(row.get("subjects"))
|
|
if not subjects:
|
|
subjects = [str(row.get(f"exam_subject{i}") or row.get(f"subject{i}") or row.get(f"考试科目{i}") or "") for i in range(1, 5)]
|
|
quota, scope = parse_quota(row.get("quota_text", ""))
|
|
if quota is None and not str(row.get('quota_text') or '').strip():
|
|
quota = integer(row.get("quota"))
|
|
scope = row.get("quota_scope") or "unspecified"
|
|
plans = list_field(row.get("special_plans"))
|
|
if row.get("veteran_program") in ("是", "1", 1):
|
|
plans.append("退役大学生士兵")
|
|
if row.get("shaogu_program") in ("是", "1", 1):
|
|
plans.append("少数民族骨干")
|
|
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "reference_books": list_field(row.get('reference_books')), "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
|
|
|
|
|
|
def import_file(path, year=None):
|
|
path = Path(path)
|
|
suffix = path.suffix.lower()
|
|
scores = []
|
|
if suffix in (".db", ".sqlite", ".sqlite3"):
|
|
return import_yam(path, year)
|
|
if suffix == ".json":
|
|
content = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
if isinstance(content, dict):
|
|
rows = content.get("catalog", content.get("records", []))
|
|
scores = content.get("scores", [])
|
|
else:
|
|
rows = content
|
|
if not isinstance(rows, list) or not isinstance(scores, list):
|
|
raise ValueError("JSON 应是记录数组或包含 catalog/scores 的工作台快照")
|
|
elif suffix == ".csv":
|
|
payload = path.read_bytes()
|
|
try:
|
|
text = payload.decode("utf-8-sig")
|
|
except UnicodeDecodeError:
|
|
text = payload.decode("gb18030")
|
|
rows = list(csv.DictReader(io.StringIO(text)))
|
|
elif suffix == ".xlsx":
|
|
book = load_workbook(path, read_only=True, data_only=True)
|
|
try:
|
|
sheet = book.active
|
|
if (sheet.max_row or 0) > 300001 or (sheet.max_column or 0) > 100:
|
|
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
|
|
values = sheet.iter_rows(values_only=True)
|
|
headers = next(values, [])
|
|
if len(headers) > 100:
|
|
raise ValueError("Excel 超过 100 列,请拆分后导入")
|
|
rows = []
|
|
for index, row in enumerate(values, 2):
|
|
if index > 300001 or len(row) > 100:
|
|
raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入")
|
|
if any(v is not None for v in row):
|
|
rows.append(dict(zip(headers, row)))
|
|
finally:
|
|
book.close()
|
|
else:
|
|
raise ValueError("支持 CSV、XLSX、JSON 和 YAM SQLite 文件")
|
|
if len(rows) > 300000:
|
|
raise ValueError("一次最多导入 30 万条,请拆分文件")
|
|
records, schools = [], []
|
|
for row in rows:
|
|
if isinstance(row, dict) and row.get('dwdm') and row.get('dwmc'):
|
|
schools.append({'school_code': str(row['dwdm']), 'school_name': row['dwmc'], 'province': row.get('szss') or PROVINCES.get(str(row.get('szssm', '')), ''), 'level': '双一流' if str(row.get('syl')) == '1' else ''})
|
|
directions = row.get('data') or ([row] if row.get('kskmz') else [])
|
|
if not isinstance(directions, list):
|
|
raise ValueError('YAM 种子 data 应为方向数组')
|
|
for direction in directions:
|
|
if not isinstance(direction, dict) or not any(k in direction for k in ('yjfxmc', 'kskmz', 'yxsmc')):
|
|
raise ValueError('YAM 方向数据结构无法识别')
|
|
explicit = integer(row.get('year')) or integer(year)
|
|
if not explicit:
|
|
raise ValueError('种子含有目录详情,请指定其招生年份')
|
|
records.extend(normalize_catalog(direction, explicit, row))
|
|
else:
|
|
records.append(row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入"))
|
|
for score in scores:
|
|
if not all(k in score for k in ("school_code", "major_code", "year", "scope")):
|
|
raise ValueError("快照分数线缺少院校/专业/年份/口径")
|
|
score["year"] = int(score["year"])
|
|
score["total"] = integer(score.get("total"))
|
|
score.setdefault("note", "")
|
|
return ImportResult(records, scores, schools)
|
|
|
|
|
|
def import_yam(path, year):
|
|
conn = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True)
|
|
conn.row_factory = sqlite3.Row
|
|
try:
|
|
conn.execute("PRAGMA query_only=ON")
|
|
tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
|
|
if not {"schools", "departments"}.issubset(tables):
|
|
raise ValueError("此 SQLite 不包含 YAM schools/departments 表;请从原软件导出 CSV/JSON")
|
|
schools = [dict(row) for row in conn.execute("SELECT * FROM schools")]
|
|
lookup = {(str(r["school_id"]), str(r.get("major_code", ""))): r for r in schools}
|
|
records, departments = [], {}
|
|
for entry in conn.execute("SELECT * FROM departments"):
|
|
row = dict(entry)
|
|
school = lookup.get((str(row["school_id"]), str(row.get("major_code", "")))) or next((r for r in schools if str(r["school_id"]) == str(row["school_id"])), {})
|
|
department = str(row.get("department_id", ""))
|
|
departments[(str(row["school_id"]), department)] = row.get("name", "")
|
|
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "level": school.get("level", ""), "department_code": department, "department_name": row.get("name", ""), "source": row.get("source") or "YAM 导入"}, year))
|
|
scores = []
|
|
if "score_lines" in tables:
|
|
for entry in conn.execute("SELECT * FROM score_lines"):
|
|
row = dict(entry)
|
|
if not integer(row.get("year")):
|
|
continue
|
|
raw = str(row.get("raw_code", ""))
|
|
scope = row.get("match_scope") or ({2: "category", 4: "discipline", 6: "major"}.get(len(raw)) if raw else "unknown")
|
|
scores.append({**row, "school_code": str(row["school_id"]), "major_code": str(row.get("major_code", "")), "department_id": str(row.get("department_id", "") or ""), "department_name": row.get("raw_department_name") or departments.get((str(row["school_id"]), str(row.get("department_id", ""))), ""), "scope": scope, "note": row.get("note") or "", "source": row.get("source") or "YAM 导入"})
|
|
if "admission_plans" in tables:
|
|
for entry in conn.execute("SELECT * FROM admission_plans"):
|
|
row = dict(entry)
|
|
school = lookup.get((str(row["school_id"]), str(row.get("major_code", ""))), {})
|
|
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "direction": row.get("research_direction", ""), "direction_code": str(row.get("spe_id") or row.get("plan_id") or ""), "department_code": str(row.get("department_id", "") or ""), "source": row.get("source") or "YAM 导入", "remarks": row.get("note", "")}, year))
|
|
return ImportResult(records, scores)
|
|
except sqlite3.DatabaseError:
|
|
raise ValueError("SQLite 文件无法读取,请确认是完整导出的数据库") from None
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def spreadsheet_value(value):
|
|
if isinstance(value, (list, dict)):
|
|
value = " | ".join(str(v) for v in value) if isinstance(value, list) else json.dumps(value, ensure_ascii=False)
|
|
if value is None:
|
|
return ""
|
|
if isinstance(value, str) and value.lstrip().startswith(("=", "+", "-", "@", "\t", "\r")):
|
|
return "'" + value
|
|
return value
|
|
|
|
|
|
def export_sheet(records, kind):
|
|
if kind == "csv":
|
|
stream = io.StringIO(newline="")
|
|
writer = csv.writer(stream)
|
|
writer.writerow(list(LABELS.values()))
|
|
writer.writerows([spreadsheet_value(record.get(field)) for field in LABELS] for record in records)
|
|
return stream.getvalue().encode("utf-8-sig")
|
|
output = io.BytesIO()
|
|
write_xlsx(records, output)
|
|
return output.getvalue()
|
|
|
|
|
|
def write_xlsx(records, output):
|
|
from openpyxl.cell import WriteOnlyCell
|
|
from openpyxl.utils import get_column_letter
|
|
from openpyxl.styles import Font, PatternFill, Alignment
|
|
book = Workbook(write_only=True)
|
|
sheet = book.create_sheet("院校专业")
|
|
sheet.freeze_panes = "A2"
|
|
header = []
|
|
for index, label in enumerate(LABELS.values(), 1):
|
|
cell = WriteOnlyCell(sheet, value=label)
|
|
cell.font = Font(color="FFFFFF", bold=True)
|
|
cell.fill = PatternFill("solid", fgColor="183447")
|
|
cell.alignment = Alignment(vertical="center")
|
|
header.append(cell)
|
|
sheet.column_dimensions[get_column_letter(index)].width = 24
|
|
count = 1
|
|
try:
|
|
sheet.append(header)
|
|
for record in records:
|
|
sheet.append([spreadsheet_value(record.get(field)) for field in LABELS])
|
|
count += 1
|
|
sheet.auto_filter.ref = f'A1:{get_column_letter(len(LABELS))}{count}'
|
|
book.save(output)
|
|
except BaseException:
|
|
with suppress(Exception):
|
|
if not sheet.closed:
|
|
sheet.close()
|
|
with suppress(Exception):
|
|
if sheet._writer:
|
|
sheet._writer.cleanup()
|
|
raise
|
|
finally:
|
|
book.close()
|