"""Exchange formats for yzw CSV, YAM SQLite and studio snapshots.""" import csv import io import json import sqlite3 from pathlib import Path from openpyxl import load_workbook, Workbook from studio.domain import finish_record, integer, parse_quota LABELS = {"year": "年份", "school_name": "院校", "school_code": "院校代码", "province": "省份", "level": "院校层次", "major_code": "专业代码", "major_name": "专业", "degree_type": "学位类型", "department_name": "院系", "direction": "研究方向", "study_mode": "学习方式", "exam_type": "考试方式", "advisor": "导师", "quota_text": "招生人数原文", "quota": "明确人数", "quota_scope": "人数口径", "subjects": "考试科目", "special_plans": "专项计划", "score_year": "参考线年份", "score_total": "参考线总分", "score_scope": "分数线口径", "remarks": "备注", "source": "来源", "source_url": "来源链接", "year_verified": "来源年份已核实", "fetched_at": "采集时间"} ALIASES = {"院校名称": "school_name", "学校名称": "school_name", "专业名称": "major_name", "院系所": "department_name", "拟招生人数": "quota_text", "planned_enrollment": "quota_text", "nzsrsstr": "quota_text", "school_level": "level", "department": "department_name", "research_direction": "direction", "enrollment_text": "quota_text", "enrollment_count": "quota", "exam_subjects": "subjects"} ALIASES.update({v: k for k, v in LABELS.items()}) def list_field(value): if isinstance(value, list): return value if not value: return [] if str(value).startswith("["): try: decoded = json.loads(value) if isinstance(decoded, list): return decoded except ValueError: pass return str(value).split(" | ") def row_record(raw, year=None, source="文件导入"): row = {ALIASES.get(str(k).strip(), str(k).strip()): (v if v is not None else "") for k, v in raw.items() if k is not None} explicit_year = integer(row.get("year")) or integer(year) if explicit_year is None: raise ValueError("文件未包含年份,请明确指定这份数据的招生年份") if not 2000 <= explicit_year <= 2100: raise ValueError("文件年份无效") if not row.get("school_name") or not row.get("major_code"): raise ValueError("文件需包含院校名称和专业代码") row["major_code"] = str(row["major_code"]).strip().zfill(6) if not row["major_code"].isdigit() or len(row["major_code"]) != 6: raise ValueError("文件专业代码应为 6 位数字") row["school_code"] = str(row.get("school_code", "")).strip() subjects = list_field(row.get("subjects")) if not subjects: subjects = [str(row.get(f"exam_subject{i}") or row.get(f"subject{i}") or row.get(f"考试科目{i}") or "") for i in range(1, 5)] quota, scope = parse_quota(row.get("quota_text", "")) if quota is None and row.get("quota") != "": quota = integer(row.get("quota")) scope = row.get("quota_scope") or "unspecified" plans = list_field(row.get("special_plans")) if row.get("veteran_program") in ("是", "1", 1): plans.append("退役大学生士兵") if row.get("shaogu_program") in ("是", "1", 1): plans.append("少数民族骨干") return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"}) def import_file(path, year=None): path = Path(path) suffix = path.suffix.lower() scores = [] if suffix in (".db", ".sqlite", ".sqlite3"): return import_yam(path, year) if suffix == ".json": content = json.loads(path.read_text(encoding="utf-8-sig")) if isinstance(content, dict): rows = content.get("catalog", content.get("records", [])) scores = content.get("scores", []) else: rows = content if not isinstance(rows, list) or not isinstance(scores, list): raise ValueError("JSON 应是记录数组或包含 catalog/scores 的工作台快照") elif suffix == ".csv": payload = path.read_bytes() try: text = payload.decode("utf-8-sig") except UnicodeDecodeError: text = payload.decode("gb18030") rows = list(csv.DictReader(io.StringIO(text))) elif suffix == ".xlsx": book = load_workbook(path, read_only=True, data_only=True) try: sheet = book.active if sheet.max_row > 300001 or sheet.max_column > 100: raise ValueError("Excel 超过 30 万行或 100 列,请拆分后导入") values = sheet.iter_rows(values_only=True) headers = next(values, []) rows = [dict(zip(headers, row)) for row in values if any(v is not None for v in row)] finally: book.close() else: raise ValueError("支持 CSV、XLSX、JSON 和 YAM SQLite 文件") if len(rows) > 300000: raise ValueError("一次最多导入 30 万条,请拆分文件") records = [row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入") for row in rows] for score in scores: if not all(k in score for k in ("school_code", "major_code", "year", "scope")): raise ValueError("快照分数线缺少院校/专业/年份/口径") score["year"] = int(score["year"]) score["total"] = integer(score.get("total")) score.setdefault("note", "") return records, scores def import_yam(path, year): conn = sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) conn.row_factory = sqlite3.Row try: conn.execute("PRAGMA query_only=ON") tables = {row[0] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")} if not {"schools", "departments"}.issubset(tables): raise ValueError("此 SQLite 不包含 YAM schools/departments 表;请从原软件导出 CSV/JSON") schools = [dict(row) for row in conn.execute("SELECT * FROM schools")] lookup = {(str(r["school_id"]), str(r.get("major_code", ""))): r for r in schools} records, departments = [], {} for entry in conn.execute("SELECT * FROM departments"): row = dict(entry) school = lookup.get((str(row["school_id"]), str(row.get("major_code", "")))) or next((r for r in schools if str(r["school_id"]) == str(row["school_id"])), {}) department = str(row.get("department_id", "")) departments[(str(row["school_id"]), department)] = row.get("name", "") records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "level": school.get("level", ""), "department_code": department, "department_name": row.get("name", ""), "source": row.get("source") or "YAM 导入"}, year)) scores = [] if "score_lines" in tables: for entry in conn.execute("SELECT * FROM score_lines"): row = dict(entry) if not integer(row.get("year")): continue raw = str(row.get("raw_code", "")) scope = row.get("match_scope") or ({2: "category", 4: "discipline", 6: "major"}.get(len(raw)) if raw else "unknown") scores.append({**row, "school_code": str(row["school_id"]), "major_code": str(row.get("major_code", "")), "department_id": str(row.get("department_id", "") or ""), "department_name": row.get("raw_department_name") or departments.get((str(row["school_id"]), str(row.get("department_id", ""))), ""), "scope": scope, "note": row.get("note") or "", "source": row.get("source") or "YAM 导入"}) if "admission_plans" in tables: for entry in conn.execute("SELECT * FROM admission_plans"): row = dict(entry) school = lookup.get((str(row["school_id"]), str(row.get("major_code", ""))), {}) records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "direction": row.get("research_direction", ""), "direction_code": str(row.get("spe_id") or row.get("plan_id") or ""), "department_code": str(row.get("department_id", "") or ""), "source": row.get("source") or "YAM 导入", "remarks": row.get("note", "")}, year)) return records, scores except sqlite3.DatabaseError: raise ValueError("SQLite 文件无法读取,请确认是完整导出的数据库") from None finally: conn.close() def spreadsheet_value(value): if isinstance(value, (list, dict)): value = " | ".join(str(v) for v in value) if isinstance(value, list) else json.dumps(value, ensure_ascii=False) if value is None: return "" if isinstance(value, str) and value.lstrip().startswith(("=", "+", "-", "@", "\t", "\r")): return "'" + value return value def export_sheet(records, kind): rows = [[spreadsheet_value(record.get(field)) for field in LABELS] for record in records] if kind == "csv": stream = io.StringIO(newline="") writer = csv.writer(stream) writer.writerow(list(LABELS.values())) writer.writerows(rows) return stream.getvalue().encode("utf-8-sig") book = Workbook() sheet = book.active sheet.title = "院校专业" sheet.append(list(LABELS.values())) for row in rows: sheet.append(row) sheet.freeze_panes = "A2" sheet.auto_filter.ref = sheet.dimensions from openpyxl.styles import Font, PatternFill, Alignment for cell in sheet[1]: cell.font = Font(color="FFFFFF", bold=True) cell.fill = PatternFill("solid", fgColor="183447") cell.alignment = Alignment(vertical="center") for col in sheet.columns: sheet.column_dimensions[col[0].column_letter].width = 24 output = io.BytesIO() book.save(output) return output.getvalue()