fix: preserve data granularity and recover tasks safely
This commit is contained in:
+34
-6
@@ -3,15 +3,28 @@ import csv
|
||||
import io
|
||||
import json
|
||||
import sqlite3
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from openpyxl import load_workbook, Workbook
|
||||
from studio.domain import finish_record, integer, parse_quota
|
||||
from studio.domain import finish_record, integer, parse_quota, normalize_catalog, PROVINCES
|
||||
|
||||
LABELS = {"year": "年份", "school_name": "院校", "school_code": "院校代码", "province": "省份", "level": "院校层次", "major_code": "专业代码", "major_name": "专业", "degree_type": "学位类型", "department_name": "院系", "direction": "研究方向", "study_mode": "学习方式", "exam_type": "考试方式", "advisor": "导师", "quota_text": "招生人数原文", "quota": "明确人数", "quota_scope": "人数口径", "subjects": "考试科目", "special_plans": "专项计划", "score_year": "参考线年份", "score_total": "参考线总分", "score_scope": "分数线口径", "remarks": "备注", "source": "来源", "source_url": "来源链接", "year_verified": "来源年份已核实", "fetched_at": "采集时间"}
|
||||
LABELS.update({"department_code": "院系代码", "direction_code": "方向代码"})
|
||||
ALIASES = {"院校名称": "school_name", "学校名称": "school_name", "专业名称": "major_name", "院系所": "department_name", "拟招生人数": "quota_text", "planned_enrollment": "quota_text", "nzsrsstr": "quota_text", "school_level": "level", "department": "department_name", "research_direction": "direction", "enrollment_text": "quota_text", "enrollment_count": "quota", "exam_subjects": "subjects"}
|
||||
ALIASES.update({v: k for k, v in LABELS.items()})
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportResult:
|
||||
records: list
|
||||
scores: list
|
||||
schools: list = field(default_factory=list)
|
||||
|
||||
def __iter__(self):
|
||||
yield self.records
|
||||
yield self.scores
|
||||
|
||||
|
||||
def list_field(value):
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
@@ -44,7 +57,7 @@ def row_record(raw, year=None, source="文件导入"):
|
||||
if not subjects:
|
||||
subjects = [str(row.get(f"exam_subject{i}") or row.get(f"subject{i}") or row.get(f"考试科目{i}") or "") for i in range(1, 5)]
|
||||
quota, scope = parse_quota(row.get("quota_text", ""))
|
||||
if quota is None and row.get("quota") != "":
|
||||
if quota is None and not str(row.get('quota_text') or '').strip():
|
||||
quota = integer(row.get("quota"))
|
||||
scope = row.get("quota_scope") or "unspecified"
|
||||
plans = list_field(row.get("special_plans"))
|
||||
@@ -52,7 +65,7 @@ def row_record(raw, year=None, source="文件导入"):
|
||||
plans.append("退役大学生士兵")
|
||||
if row.get("shaogu_program") in ("是", "1", 1):
|
||||
plans.append("少数民族骨干")
|
||||
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
|
||||
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "reference_books": list_field(row.get('reference_books')), "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
|
||||
|
||||
|
||||
def import_file(path, year=None):
|
||||
@@ -92,14 +105,29 @@ def import_file(path, year=None):
|
||||
raise ValueError("支持 CSV、XLSX、JSON 和 YAM SQLite 文件")
|
||||
if len(rows) > 300000:
|
||||
raise ValueError("一次最多导入 30 万条,请拆分文件")
|
||||
records = [row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入") for row in rows]
|
||||
records, schools = [], []
|
||||
for row in rows:
|
||||
if isinstance(row, dict) and row.get('dwdm') and row.get('dwmc'):
|
||||
schools.append({'school_code': str(row['dwdm']), 'school_name': row['dwmc'], 'province': row.get('szss') or PROVINCES.get(str(row.get('szssm', '')), ''), 'level': '双一流' if str(row.get('syl')) == '1' else ''})
|
||||
directions = row.get('data') or ([row] if row.get('kskmz') else [])
|
||||
if not isinstance(directions, list):
|
||||
raise ValueError('YAM 种子 data 应为方向数组')
|
||||
for direction in directions:
|
||||
if not isinstance(direction, dict) or not any(k in direction for k in ('yjfxmc', 'kskmz', 'yxsmc')):
|
||||
raise ValueError('YAM 方向数据结构无法识别')
|
||||
explicit = integer(row.get('year')) or integer(year)
|
||||
if not explicit:
|
||||
raise ValueError('种子含有目录详情,请指定其招生年份')
|
||||
records.extend(normalize_catalog(direction, explicit, row))
|
||||
else:
|
||||
records.append(row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入"))
|
||||
for score in scores:
|
||||
if not all(k in score for k in ("school_code", "major_code", "year", "scope")):
|
||||
raise ValueError("快照分数线缺少院校/专业/年份/口径")
|
||||
score["year"] = int(score["year"])
|
||||
score["total"] = integer(score.get("total"))
|
||||
score.setdefault("note", "")
|
||||
return records, scores
|
||||
return ImportResult(records, scores, schools)
|
||||
|
||||
|
||||
def import_yam(path, year):
|
||||
@@ -133,7 +161,7 @@ def import_yam(path, year):
|
||||
row = dict(entry)
|
||||
school = lookup.get((str(row["school_id"]), str(row.get("major_code", ""))), {})
|
||||
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "direction": row.get("research_direction", ""), "direction_code": str(row.get("spe_id") or row.get("plan_id") or ""), "department_code": str(row.get("department_id", "") or ""), "source": row.get("source") or "YAM 导入", "remarks": row.get("note", "")}, year))
|
||||
return records, scores
|
||||
return ImportResult(records, scores)
|
||||
except sqlite3.DatabaseError:
|
||||
raise ValueError("SQLite 文件无法读取,请确认是完整导出的数据库") from None
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user