fix: preserve data granularity and recover tasks safely

This commit is contained in:
Codex
2026-10-11 14:18:56 +08:00
parent 23f3a15a15
commit 603184549b
29 changed files with 4557 additions and 418 deletions
+34 -6
View File
@@ -3,15 +3,28 @@ import csv
import io
import json
import sqlite3
from dataclasses import dataclass, field
from pathlib import Path
from openpyxl import load_workbook, Workbook
from studio.domain import finish_record, integer, parse_quota
from studio.domain import finish_record, integer, parse_quota, normalize_catalog, PROVINCES
LABELS = {"year": "年份", "school_name": "院校", "school_code": "院校代码", "province": "省份", "level": "院校层次", "major_code": "专业代码", "major_name": "专业", "degree_type": "学位类型", "department_name": "院系", "direction": "研究方向", "study_mode": "学习方式", "exam_type": "考试方式", "advisor": "导师", "quota_text": "招生人数原文", "quota": "明确人数", "quota_scope": "人数口径", "subjects": "考试科目", "special_plans": "专项计划", "score_year": "参考线年份", "score_total": "参考线总分", "score_scope": "分数线口径", "remarks": "备注", "source": "来源", "source_url": "来源链接", "year_verified": "来源年份已核实", "fetched_at": "采集时间"}
LABELS.update({"department_code": "院系代码", "direction_code": "方向代码"})
ALIASES = {"院校名称": "school_name", "学校名称": "school_name", "专业名称": "major_name", "院系所": "department_name", "拟招生人数": "quota_text", "planned_enrollment": "quota_text", "nzsrsstr": "quota_text", "school_level": "level", "department": "department_name", "research_direction": "direction", "enrollment_text": "quota_text", "enrollment_count": "quota", "exam_subjects": "subjects"}
ALIASES.update({v: k for k, v in LABELS.items()})
@dataclass
class ImportResult:
records: list
scores: list
schools: list = field(default_factory=list)
def __iter__(self):
yield self.records
yield self.scores
def list_field(value):
if isinstance(value, list):
return value
@@ -44,7 +57,7 @@ def row_record(raw, year=None, source="文件导入"):
if not subjects:
subjects = [str(row.get(f"exam_subject{i}") or row.get(f"subject{i}") or row.get(f"考试科目{i}") or "") for i in range(1, 5)]
quota, scope = parse_quota(row.get("quota_text", ""))
if quota is None and row.get("quota") != "":
if quota is None and not str(row.get('quota_text') or '').strip():
quota = integer(row.get("quota"))
scope = row.get("quota_scope") or "unspecified"
plans = list_field(row.get("special_plans"))
@@ -52,7 +65,7 @@ def row_record(raw, year=None, source="文件导入"):
plans.append("退役大学生士兵")
if row.get("shaogu_program") in ("是", "1", 1):
plans.append("少数民族骨干")
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "reference_books": list_field(row.get('reference_books')), "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
def import_file(path, year=None):
@@ -92,14 +105,29 @@ def import_file(path, year=None):
raise ValueError("支持 CSV、XLSX、JSON 和 YAM SQLite 文件")
if len(rows) > 300000:
raise ValueError("一次最多导入 30 万条,请拆分文件")
records = [row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入") for row in rows]
records, schools = [], []
for row in rows:
if isinstance(row, dict) and row.get('dwdm') and row.get('dwmc'):
schools.append({'school_code': str(row['dwdm']), 'school_name': row['dwmc'], 'province': row.get('szss') or PROVINCES.get(str(row.get('szssm', '')), ''), 'level': '双一流' if str(row.get('syl')) == '1' else ''})
directions = row.get('data') or ([row] if row.get('kskmz') else [])
if not isinstance(directions, list):
raise ValueError('YAM 种子 data 应为方向数组')
for direction in directions:
if not isinstance(direction, dict) or not any(k in direction for k in ('yjfxmc', 'kskmz', 'yxsmc')):
raise ValueError('YAM 方向数据结构无法识别')
explicit = integer(row.get('year')) or integer(year)
if not explicit:
raise ValueError('种子含有目录详情,请指定其招生年份')
records.extend(normalize_catalog(direction, explicit, row))
else:
records.append(row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入"))
for score in scores:
if not all(k in score for k in ("school_code", "major_code", "year", "scope")):
raise ValueError("快照分数线缺少院校/专业/年份/口径")
score["year"] = int(score["year"])
score["total"] = integer(score.get("total"))
score.setdefault("note", "")
return records, scores
return ImportResult(records, scores, schools)
def import_yam(path, year):
@@ -133,7 +161,7 @@ def import_yam(path, year):
row = dict(entry)
school = lookup.get((str(row["school_id"]), str(row.get("major_code", ""))), {})
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "direction": row.get("research_direction", ""), "direction_code": str(row.get("spe_id") or row.get("plan_id") or ""), "department_code": str(row.get("department_id", "") or ""), "source": row.get("source") or "YAM 导入", "remarks": row.get("note", "")}, year))
return records, scores
return ImportResult(records, scores)
except sqlite3.DatabaseError:
raise ValueError("SQLite 文件无法读取,请确认是完整导出的数据库") from None
finally: