fix: preserve data granularity and recover tasks safely
This commit is contained in:
+6
-3
@@ -13,7 +13,7 @@ from studio.database import Database
|
||||
from studio.domain import PROVINCES
|
||||
from studio.imports import import_file, export_sheet
|
||||
from studio.jobs import JobManager
|
||||
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable
|
||||
from studio.network import ProxyRouter, Transport, ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||
from studio.settings import SettingsStore
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
@@ -60,6 +60,7 @@ def create_app(data_dir=None):
|
||||
|
||||
@app.exception_handler(ProxyUnavailable)
|
||||
@app.exception_handler(RemoteUnavailable)
|
||||
@app.exception_handler(RateLimited)
|
||||
async def remote_error(request, exc):
|
||||
return JSONResponse({"detail": str(exc)}, status_code=503)
|
||||
|
||||
@@ -188,10 +189,12 @@ def create_app(data_dir=None):
|
||||
if total > 100 * 1024 * 1024:
|
||||
raise ValueError("一次最多上传 100 MB,请拆分文件")
|
||||
stream.write(chunk)
|
||||
records, scores = import_file(temporary, year)
|
||||
result = import_file(temporary, year)
|
||||
records, scores = result
|
||||
db.upsert_catalog(records)
|
||||
db.upsert_scores(scores)
|
||||
return {"records": len(records), "scores": len(scores), "message": "导入完成,相同记录按年份更新"}
|
||||
db.index_schools(result.schools)
|
||||
return {"records": len(records), "scores": len(scores), "indexed_schools": len(result.schools), "message": "导入完成,相同记录按年份更新;无详情的 YAM 种子只作为院校索引"}
|
||||
except (UnicodeError, json.JSONDecodeError, KeyError) as exc:
|
||||
raise ValueError(f"文件编码或结构无法识别({type(exc).__name__})") from None
|
||||
finally:
|
||||
|
||||
+17
-6
@@ -60,14 +60,24 @@ class AuthService:
|
||||
self.browser_thread = None
|
||||
try:
|
||||
self._load(self.vault.load())
|
||||
except RuntimeError:
|
||||
except (RuntimeError, OSError, ValueError):
|
||||
self.message = "保存的 Cookie 无法解密,请重新登录"
|
||||
|
||||
def _load(self, cookies):
|
||||
if isinstance(cookies, dict):
|
||||
cookies = [{"name": name, "value": value, "domain": ".chsi.com.cn", "path": "/"} for name, value in cookies.items()]
|
||||
with self.transport.lock:
|
||||
self.transport.session.cookies.clear()
|
||||
for name, value in cookies.items():
|
||||
self.transport.session.cookies.set(name, value, domain=".chsi.com.cn", path="/")
|
||||
for cookie in cookies:
|
||||
domain = str(cookie.get("domain", ".chsi.com.cn"))
|
||||
if domain.lstrip(".") != "chsi.com.cn" and not domain.endswith(".chsi.com.cn"):
|
||||
continue
|
||||
expires = cookie.get("expires")
|
||||
self.transport.session.cookies.set(cookie["name"], cookie["value"], domain=domain, path=cookie.get("path", "/"), secure=bool(cookie.get("secure")), expires=int(expires) if expires and expires > 0 else None)
|
||||
|
||||
def _snapshot(self):
|
||||
with self.transport.lock:
|
||||
return [{"name": c.name, "value": c.value, "domain": c.domain, "path": c.path, "secure": c.secure, "expires": c.expires} for c in self.transport.session.cookies]
|
||||
|
||||
def status(self):
|
||||
return {"saved": self.vault.path.exists(), "valid": self.valid, "message": self.message, "browser_state": self.browser_state, "browser_message": self.browser_message}
|
||||
@@ -91,7 +101,8 @@ class AuthService:
|
||||
return self.status()
|
||||
|
||||
def check(self):
|
||||
response = self.transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/dws.do", data={"ssdm": "11", "curPage": 1, "start": 0, "pageSize": 1, "dwlxs[0]": "all"})
|
||||
# The school/major indexes can be public while direction details require login.
|
||||
response = self.transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/yjfxs.do", data={"dwdm": "10001", "zydm": "081200", "zymc": "计算机科学与技术", "curPage": 1, "start": 0, "pageSize": 1, "dwlxs[0]": "all"})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
@@ -101,7 +112,7 @@ class AuthService:
|
||||
self.valid = body.get("flag") is True and isinstance(body.get("msg"), dict)
|
||||
self.message = "研招目录接口验证成功" if self.valid else "登录未生效或数据源限流,请在浏览器完成登录后重试"
|
||||
if self.valid:
|
||||
self.vault.save(self.transport.session.cookies.get_dict())
|
||||
self.vault.save(self._snapshot())
|
||||
return self.status()
|
||||
|
||||
def login(self, username, password):
|
||||
@@ -144,7 +155,7 @@ class AuthService:
|
||||
while time.monotonic() < deadline:
|
||||
page.wait_for_timeout(2000)
|
||||
if "yz.chsi.com.cn" in page.url:
|
||||
cookies = {c["name"]: c["value"] for c in context.cookies() if "chsi.com.cn" in c["domain"]}
|
||||
cookies = [c for c in context.cookies() if c["domain"].lstrip(".") == "chsi.com.cn" or c["domain"].endswith(".chsi.com.cn")]
|
||||
if cookies:
|
||||
self._load(cookies)
|
||||
if self.check()["valid"]:
|
||||
|
||||
+31
-16
@@ -2,7 +2,7 @@
|
||||
import json
|
||||
import re
|
||||
from studio.domain import finish_record, integer, normalized_name, select_score_lines
|
||||
from studio.network import RemoteUnavailable
|
||||
from studio.network import RemoteUnavailable, RateLimited
|
||||
|
||||
|
||||
class LoginRequired(RuntimeError):
|
||||
@@ -35,17 +35,25 @@ class CatalogClient:
|
||||
def pages(self, endpoint, payload, page_size=50):
|
||||
seen = set()
|
||||
for page in range(1, 10001):
|
||||
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
|
||||
for attempt in range(3):
|
||||
response = self.transport.request("POST", self.BASE + "rs/" + endpoint + ".do", data={**payload, "curPage": page, "start": (page - 1) * page_size, "pageSize": page_size})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise LoginRequired("研招网返回登录/验证页面,请先完成登录") from None
|
||||
message = str(body.get('msg', ''))
|
||||
if body.get('flag') is not True and '验证码' in message:
|
||||
raise LoginRequired('研招网需要人工验证码,请在浏览器完成验证后恢复')
|
||||
if body.get('flag') is not True and any(token in message for token in ('频繁', '稍后')):
|
||||
if attempt == 2:
|
||||
raise RateLimited('研招网业务限流持续出现,已暂停;请增加请求间隔后恢复')
|
||||
self.transport.sleep(5 * 2 ** attempt)
|
||||
continue
|
||||
break
|
||||
if body.get("flag") is not True:
|
||||
message = str(body.get("msg", ""))
|
||||
if any(token in message for token in ("登录", "登陆", "认证")):
|
||||
raise LoginRequired("研招网要求登录,请更新 Cookie 后恢复任务")
|
||||
if any(token in message for token in ("频繁", "稍后", "验证码")):
|
||||
raise RemoteUnavailable("研招网访问受限,请增加请求间隔并完成人工验证后恢复")
|
||||
raise RemoteUnavailable("研招网目录接口未成功返回数据")
|
||||
data = body.get("msg") or {}
|
||||
if not isinstance(data, dict) or not isinstance(data.get("list"), list):
|
||||
@@ -83,14 +91,21 @@ class ScoreClient:
|
||||
self.cache = {}
|
||||
|
||||
def post(self, endpoint, payload):
|
||||
response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
|
||||
if str(body.get("code")) != "0000":
|
||||
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
|
||||
return body.get("data") or {}
|
||||
for attempt in range(3):
|
||||
response = self.transport.request("POST", self.BASE + endpoint, data=payload, headers={"Referer": "https://www.kaoyan.cn/"})
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
raise RemoteUnavailable("掌上考研未返回有效 JSON") from None
|
||||
if str(body.get("code")) == "0000":
|
||||
return body.get("data") or {}
|
||||
message = str(body.get('message', body.get('msg', '')))
|
||||
if any(token in message for token in ('频繁', '稍后', '限流', '繁忙')):
|
||||
if attempt == 2:
|
||||
raise RateLimited('掌上考研业务限流持续出现,请增加请求间隔后恢复')
|
||||
self.transport.sleep(5 * 2 ** attempt)
|
||||
else:
|
||||
raise RemoteUnavailable("掌上考研接口未成功返回数据,请稍后重试")
|
||||
|
||||
def pages(self, endpoint, payload, limit=100):
|
||||
seen = set()
|
||||
|
||||
+23
-7
@@ -3,8 +3,15 @@ import json
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from studio.domain import now, normalized_name
|
||||
from studio.domain import now, normalized_name, is_special_score, school_level
|
||||
|
||||
SCORE_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS scores(
|
||||
id INTEGER PRIMARY KEY, school_code TEXT, major_code TEXT, year INTEGER,
|
||||
department_id TEXT, department_name TEXT, scope TEXT, total INTEGER, source TEXT, data TEXT NOT NULL,
|
||||
note TEXT NOT NULL DEFAULT '',raw_code TEXT NOT NULL DEFAULT '',
|
||||
UNIQUE(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code));
|
||||
"""
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS catalog(
|
||||
id INTEGER PRIMARY KEY, record_key TEXT NOT NULL UNIQUE, year INTEGER NOT NULL,
|
||||
@@ -12,10 +19,7 @@ CREATE TABLE IF NOT EXISTS catalog(
|
||||
department_code TEXT, department_name TEXT, study_mode TEXT, degree_type TEXT, level TEXT,
|
||||
subjects TEXT, quota INTEGER, source TEXT, favorite INTEGER NOT NULL DEFAULT 0, data TEXT NOT NULL);
|
||||
CREATE INDEX IF NOT EXISTS catalog_filter ON catalog(year,province,major_code,school_code);
|
||||
CREATE TABLE IF NOT EXISTS scores(
|
||||
id INTEGER PRIMARY KEY, school_code TEXT, major_code TEXT, year INTEGER,
|
||||
department_id TEXT, department_name TEXT, scope TEXT, total INTEGER, source TEXT, data TEXT NOT NULL,
|
||||
UNIQUE(school_code,major_code,year,department_id,department_name,scope,source));
|
||||
""" + SCORE_SCHEMA + """
|
||||
CREATE INDEX IF NOT EXISTS scores_lookup ON scores(school_code,major_code,year DESC);
|
||||
CREATE TABLE IF NOT EXISTS schools(school_code TEXT PRIMARY KEY, school_name TEXT, province TEXT, level TEXT, data TEXT);
|
||||
CREATE TABLE IF NOT EXISTS jobs(id TEXT PRIMARY KEY, params TEXT, status TEXT, created_at TEXT, updated_at TEXT, message TEXT, errors INTEGER DEFAULT 0, records INTEGER DEFAULT 0);
|
||||
@@ -34,6 +38,13 @@ class Database:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.connection() as conn:
|
||||
conn.executescript(SCHEMA)
|
||||
if 'note' not in {row['name'] for row in conn.execute('PRAGMA table_info(scores)')}:
|
||||
# Rebuild atomically to replace the early schema's narrower unique key.
|
||||
conn.execute(SCORE_SCHEMA.replace('IF NOT EXISTS scores', 'scores_new'))
|
||||
conn.execute("""INSERT INTO scores_new(id,school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) SELECT id,school_code,major_code,year,department_id,department_name,scope,total,source,data,coalesce(json_extract(data,'$.note'),''),coalesce(json_extract(data,'$.raw_code'),'') FROM scores""")
|
||||
conn.execute('DROP TABLE scores')
|
||||
conn.execute('ALTER TABLE scores_new RENAME TO scores')
|
||||
conn.execute('CREATE INDEX scores_lookup ON scores(school_code,major_code,year DESC)')
|
||||
conn.execute("UPDATE jobs SET status='paused',message='服务重启,任务可从断点恢复' WHERE status IN ('running','queued')")
|
||||
|
||||
@contextmanager
|
||||
@@ -41,6 +52,7 @@ class Database:
|
||||
conn = sqlite3.connect(self.path, timeout=30)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.create_function("norm_name", 1, normalized_name, deterministic=True)
|
||||
conn.create_function("is_special_score", 1, is_special_score, deterministic=True)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
try:
|
||||
yield conn
|
||||
@@ -66,6 +78,7 @@ class Database:
|
||||
self._school(conn, record)
|
||||
|
||||
def _school(self, conn, record):
|
||||
record = {**record, 'level': school_level(record['school_name'], record.get('level', ''))}
|
||||
conn.execute("INSERT INTO schools VALUES(?,?,?,?,?) ON CONFLICT(school_code) DO UPDATE SET school_name=excluded.school_name,province=CASE WHEN excluded.province<>'' THEN excluded.province ELSE schools.province END,level=CASE WHEN excluded.level<>'' THEN excluded.level ELSE schools.level END", (record["school_code"],record["school_name"],record.get("province", ""),record.get("level", ""),encode(record)))
|
||||
|
||||
def index_schools(self, rows):
|
||||
@@ -76,7 +89,7 @@ class Database:
|
||||
def upsert_scores(self, records):
|
||||
with self.connection() as conn:
|
||||
for row in records:
|
||||
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data) VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", ""),row.get("department_name", ""),row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row)))
|
||||
conn.execute("""INSERT INTO scores(school_code,major_code,year,department_id,department_name,scope,total,source,data,note,raw_code) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(school_code,major_code,year,department_id,department_name,scope,source,note,raw_code) DO UPDATE SET total=excluded.total,data=excluded.data""",(row["school_code"],row["major_code"],row["year"],row.get("department_id", "") or "",row.get("department_name", "") or "",row["scope"],row.get("total"),row.get("source", "掌上考研"),encode(row),row.get('note') or '',row.get('raw_code') or ''))
|
||||
|
||||
def _query(self, filters: dict):
|
||||
conditions, params = ["1=1"], []
|
||||
@@ -95,6 +108,9 @@ class Database:
|
||||
if filters.get("keyword"):
|
||||
conditions.append("(c.school_name LIKE ? OR c.major_name LIKE ? OR c.department_name LIKE ? OR c.major_code LIKE ?)")
|
||||
params.extend([f"%{filters['keyword']}%"] * 4)
|
||||
if filters.get('school_keyword'):
|
||||
conditions.append('c.school_name LIKE ?')
|
||||
params.append(f"%{filters['school_keyword']}%")
|
||||
if filters.get("favorites"):
|
||||
conditions.append("c.favorite=1")
|
||||
if filters.get("min_quota") is not None:
|
||||
@@ -104,7 +120,7 @@ class Database:
|
||||
if filters.get(field):
|
||||
conditions.append(f"json_extract(c.data,'$.subjects[{index}]') LIKE ?")
|
||||
params.append(str(filters[field])+"%")
|
||||
select = """SELECT c.*,s.data AS score_data,s.total AS score_total,s.year AS score_year,s.scope AS score_scope FROM catalog c LEFT JOIN scores s ON s.id=(SELECT x.id FROM scores x WHERE x.school_code=c.school_code AND x.major_code=c.major_code AND x.year<=c.year AND (x.department_name='' AND x.department_id='' OR x.department_id<>'' AND x.department_id=c.department_code OR x.department_name<>'' AND norm_name(x.department_name)=norm_name(c.department_name)) AND coalesce(json_extract(x.data,'$.note'),'') NOT LIKE '%专项%' AND coalesce(json_extract(x.data,'$.note'),'') NOT LIKE '%少干%' ORDER BY x.year DESC,CASE x.scope WHEN 'major' THEN 0 WHEN 'discipline' THEN 1 ELSE 2 END,x.id DESC LIMIT 1) WHERE """ + " AND ".join(conditions)
|
||||
select = """SELECT c.*,s.data AS score_data,s.total AS score_total,s.year AS score_year,s.scope AS score_scope FROM catalog c LEFT JOIN scores s ON s.id=(SELECT x.id FROM scores x WHERE x.school_code=c.school_code AND x.major_code=c.major_code AND x.year<=c.year AND (x.department_name='' AND x.department_id='' OR x.department_id<>'' AND x.department_id=c.department_code OR x.department_name<>'' AND norm_name(x.department_name)=norm_name(c.department_name)) AND is_special_score(json_extract(x.data,'$.note'))=0 ORDER BY x.year DESC,CASE x.scope WHEN 'major' THEN 0 WHEN 'discipline' THEN 1 ELSE 2 END,x.id DESC LIMIT 1) WHERE """ + " AND ".join(conditions)
|
||||
if filters.get("max_score") is not None:
|
||||
select = "SELECT * FROM ("+select+") q WHERE q.score_total<=?"
|
||||
params.append(filters["max_score"])
|
||||
|
||||
@@ -4,11 +4,27 @@ import json
|
||||
import re
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Any
|
||||
from pathlib import Path
|
||||
from functools import lru_cache
|
||||
from urllib.parse import urlencode
|
||||
|
||||
PROVINCES = {"11":"北京","12":"天津","13":"河北","14":"山西","15":"内蒙古","21":"辽宁","22":"吉林","23":"黑龙江","31":"上海","32":"江苏","33":"浙江","34":"安徽","35":"福建","36":"江西","37":"山东","41":"河南","42":"湖北","43":"湖南","44":"广东","45":"广西","46":"海南","50":"重庆","51":"四川","52":"贵州","53":"云南","54":"西藏","61":"陕西","62":"甘肃","63":"青海","64":"宁夏","65":"新疆"}
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def school_metadata():
|
||||
path = Path(__file__).resolve().parent.parent / 'data/school_levels.json'
|
||||
data = json.loads(path.read_text(encoding='utf-8')) if path.exists() else {}
|
||||
return {k: {normalized_name(n) for n in data.get(k, [])} for k in ('985','211','double_first_class_2022_known')}
|
||||
|
||||
|
||||
def school_level(name, existing=''):
|
||||
metadata = school_metadata()
|
||||
name = normalized_name(name)
|
||||
levels = [label for key, label in [('985','985'),('211','211'),('double_first_class_2022_known','双一流')] if name in metadata[key] or label in str(existing).split('/')]
|
||||
return '/'.join(levels) or existing
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return datetime.now(timezone(timedelta(hours=8))).isoformat(timespec="seconds")
|
||||
|
||||
@@ -17,6 +33,16 @@ def normalized_name(value: str) -> str:
|
||||
return str(value or "").replace("(", "(").replace(")", ")").replace(" ", "").strip()
|
||||
|
||||
|
||||
def is_special_score(note):
|
||||
text = str(note or '')
|
||||
tokens = ('专项', '少干', '少骨', '少数民族骨干', '退役大学生士兵', '退役士兵', '强军', '援藏', '单独考试')
|
||||
for token in tokens:
|
||||
cleaned = re.sub(r'(?:不含|不包括|除外)\s*' + re.escape(token), '', text)
|
||||
if token in cleaned:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def integer(value: Any) -> int | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
@@ -47,6 +73,7 @@ def record_key(record: dict) -> str:
|
||||
def finish_record(record: dict) -> dict:
|
||||
defaults = {"school_code":"", "school_name":"", "province":"", "major_code":"", "major_name":"", "department_code":"", "department_name":"", "direction_code":"", "direction":"", "study_mode":"", "degree_type":"", "exam_type":"", "advisor":"", "quota_text":"", "quota":None, "quota_scope":"unknown", "subjects":[], "special_plans":[], "level":"", "remarks":"", "source":"", "source_url":"", "fetched_at":now(), "year_verified":False}
|
||||
row = {**defaults, **record}
|
||||
row['level'] = school_level(row['school_name'], row['level'])
|
||||
if not row["school_code"]:
|
||||
row["school_code"] = "name:" + normalized_name(row["school_name"])
|
||||
row["record_key"] = record_key(row)
|
||||
|
||||
+34
-6
@@ -3,15 +3,28 @@ import csv
|
||||
import io
|
||||
import json
|
||||
import sqlite3
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from openpyxl import load_workbook, Workbook
|
||||
from studio.domain import finish_record, integer, parse_quota
|
||||
from studio.domain import finish_record, integer, parse_quota, normalize_catalog, PROVINCES
|
||||
|
||||
LABELS = {"year": "年份", "school_name": "院校", "school_code": "院校代码", "province": "省份", "level": "院校层次", "major_code": "专业代码", "major_name": "专业", "degree_type": "学位类型", "department_name": "院系", "direction": "研究方向", "study_mode": "学习方式", "exam_type": "考试方式", "advisor": "导师", "quota_text": "招生人数原文", "quota": "明确人数", "quota_scope": "人数口径", "subjects": "考试科目", "special_plans": "专项计划", "score_year": "参考线年份", "score_total": "参考线总分", "score_scope": "分数线口径", "remarks": "备注", "source": "来源", "source_url": "来源链接", "year_verified": "来源年份已核实", "fetched_at": "采集时间"}
|
||||
LABELS.update({"department_code": "院系代码", "direction_code": "方向代码"})
|
||||
ALIASES = {"院校名称": "school_name", "学校名称": "school_name", "专业名称": "major_name", "院系所": "department_name", "拟招生人数": "quota_text", "planned_enrollment": "quota_text", "nzsrsstr": "quota_text", "school_level": "level", "department": "department_name", "research_direction": "direction", "enrollment_text": "quota_text", "enrollment_count": "quota", "exam_subjects": "subjects"}
|
||||
ALIASES.update({v: k for k, v in LABELS.items()})
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportResult:
|
||||
records: list
|
||||
scores: list
|
||||
schools: list = field(default_factory=list)
|
||||
|
||||
def __iter__(self):
|
||||
yield self.records
|
||||
yield self.scores
|
||||
|
||||
|
||||
def list_field(value):
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
@@ -44,7 +57,7 @@ def row_record(raw, year=None, source="文件导入"):
|
||||
if not subjects:
|
||||
subjects = [str(row.get(f"exam_subject{i}") or row.get(f"subject{i}") or row.get(f"考试科目{i}") or "") for i in range(1, 5)]
|
||||
quota, scope = parse_quota(row.get("quota_text", ""))
|
||||
if quota is None and row.get("quota") != "":
|
||||
if quota is None and not str(row.get('quota_text') or '').strip():
|
||||
quota = integer(row.get("quota"))
|
||||
scope = row.get("quota_scope") or "unspecified"
|
||||
plans = list_field(row.get("special_plans"))
|
||||
@@ -52,7 +65,7 @@ def row_record(raw, year=None, source="文件导入"):
|
||||
plans.append("退役大学生士兵")
|
||||
if row.get("shaogu_program") in ("是", "1", 1):
|
||||
plans.append("少数民族骨干")
|
||||
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
|
||||
return finish_record({**row, "year": explicit_year, "source": row.get("source") or source, "subjects": subjects, "reference_books": list_field(row.get('reference_books')), "special_plans": plans, "quota": quota, "quota_scope": scope, "year_verified": row.get("year_verified") is True or row.get("year_verified") == "True"})
|
||||
|
||||
|
||||
def import_file(path, year=None):
|
||||
@@ -92,14 +105,29 @@ def import_file(path, year=None):
|
||||
raise ValueError("支持 CSV、XLSX、JSON 和 YAM SQLite 文件")
|
||||
if len(rows) > 300000:
|
||||
raise ValueError("一次最多导入 30 万条,请拆分文件")
|
||||
records = [row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入") for row in rows]
|
||||
records, schools = [], []
|
||||
for row in rows:
|
||||
if isinstance(row, dict) and row.get('dwdm') and row.get('dwmc'):
|
||||
schools.append({'school_code': str(row['dwdm']), 'school_name': row['dwmc'], 'province': row.get('szss') or PROVINCES.get(str(row.get('szssm', '')), ''), 'level': '双一流' if str(row.get('syl')) == '1' else ''})
|
||||
directions = row.get('data') or ([row] if row.get('kskmz') else [])
|
||||
if not isinstance(directions, list):
|
||||
raise ValueError('YAM 种子 data 应为方向数组')
|
||||
for direction in directions:
|
||||
if not isinstance(direction, dict) or not any(k in direction for k in ('yjfxmc', 'kskmz', 'yxsmc')):
|
||||
raise ValueError('YAM 方向数据结构无法识别')
|
||||
explicit = integer(row.get('year')) or integer(year)
|
||||
if not explicit:
|
||||
raise ValueError('种子含有目录详情,请指定其招生年份')
|
||||
records.extend(normalize_catalog(direction, explicit, row))
|
||||
else:
|
||||
records.append(row_record(row, year, "yzw 导入" if suffix == ".csv" else "文件导入"))
|
||||
for score in scores:
|
||||
if not all(k in score for k in ("school_code", "major_code", "year", "scope")):
|
||||
raise ValueError("快照分数线缺少院校/专业/年份/口径")
|
||||
score["year"] = int(score["year"])
|
||||
score["total"] = integer(score.get("total"))
|
||||
score.setdefault("note", "")
|
||||
return records, scores
|
||||
return ImportResult(records, scores, schools)
|
||||
|
||||
|
||||
def import_yam(path, year):
|
||||
@@ -133,7 +161,7 @@ def import_yam(path, year):
|
||||
row = dict(entry)
|
||||
school = lookup.get((str(row["school_id"]), str(row.get("major_code", ""))), {})
|
||||
records.append(row_record({**row, "school_code": str(row["school_id"]), "school_name": school.get("name", ""), "province": school.get("province", ""), "direction": row.get("research_direction", ""), "direction_code": str(row.get("spe_id") or row.get("plan_id") or ""), "department_code": str(row.get("department_id", "") or ""), "source": row.get("source") or "YAM 导入", "remarks": row.get("note", "")}, year))
|
||||
return records, scores
|
||||
return ImportResult(records, scores)
|
||||
except sqlite3.DatabaseError:
|
||||
raise ValueError("SQLite 文件无法读取,请确认是完整导出的数据库") from None
|
||||
finally:
|
||||
|
||||
+27
-7
@@ -3,13 +3,17 @@ import threading
|
||||
import uuid
|
||||
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
|
||||
from studio.domain import PROVINCES, normalize_catalog
|
||||
from studio.network import ProxyUnavailable, RemoteUnavailable
|
||||
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||
|
||||
|
||||
class Cancelled(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Shutdown(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class JobManager:
|
||||
def __init__(self, database, settings, transport=None, runner=None):
|
||||
self.db, self.settings, self.transport = database, settings, transport
|
||||
@@ -54,7 +58,7 @@ class JobManager:
|
||||
|
||||
def _launch(self, jid):
|
||||
condition = threading.Condition()
|
||||
control = {"condition": condition, "paused": False, "cancelled": False}
|
||||
control = {"condition": condition, "paused": False, "cancelled": False, "shutdown": False}
|
||||
thread = threading.Thread(target=self._run, args=(jid,), daemon=True, name="collector-" + jid[:8])
|
||||
control["thread"] = thread
|
||||
self.controls[jid] = control
|
||||
@@ -71,6 +75,8 @@ class JobManager:
|
||||
self.checkpoint(jid)
|
||||
errors = self.get(jid)["errors"]
|
||||
self.db.job_update(jid, status="partial" if errors else "completed", message="部分单元失败,可恢复重试" if errors else "采集完成")
|
||||
except Shutdown:
|
||||
self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复")
|
||||
except Cancelled:
|
||||
self.db.job_update(jid, status="cancelled", message="已取消,已保存数据保留")
|
||||
except LoginRequired as exc:
|
||||
@@ -79,6 +85,9 @@ class JobManager:
|
||||
except ProxyUnavailable as exc:
|
||||
self.db.job_update(jid, status="waiting_proxy", message=str(exc))
|
||||
self.db.log(jid, "warning", str(exc))
|
||||
except RateLimited as exc:
|
||||
self.db.job_update(jid, status="waiting_source", message=str(exc))
|
||||
self.db.log(jid, "warning", str(exc))
|
||||
except (RemoteUnavailable, SourceYearMismatch, ValueError) as exc:
|
||||
self.db.job_update(jid, status="failed", message=str(exc))
|
||||
self.db.log(jid, "error", str(exc))
|
||||
@@ -88,16 +97,20 @@ class JobManager:
|
||||
finally:
|
||||
if self.transport:
|
||||
self.transport.local.guard = None
|
||||
if self.controls[jid]["shutdown"] and self.get(jid)["status"] != 'cancelled':
|
||||
self.db.job_update(jid, status="paused", message="服务停止,任务可从断点恢复")
|
||||
|
||||
def checkpoint(self, jid):
|
||||
control = self.controls.get(jid)
|
||||
if not control:
|
||||
return
|
||||
with control["condition"]:
|
||||
while control["paused"] and not control["cancelled"]:
|
||||
while control["paused"] and not control["cancelled"] and not control["shutdown"]:
|
||||
control["condition"].wait(.25)
|
||||
if control["cancelled"]:
|
||||
raise Cancelled()
|
||||
if control["shutdown"]:
|
||||
raise Shutdown()
|
||||
|
||||
def unit_done(self, jid, unit):
|
||||
return self.db.unit_done(jid, unit)
|
||||
@@ -117,16 +130,20 @@ class JobManager:
|
||||
def resume(self, jid):
|
||||
with self.lock:
|
||||
job = self.get(jid)
|
||||
if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy"):
|
||||
if not job or job["status"] not in ("paused", "partial", "failed", "waiting_login", "waiting_proxy", "waiting_source"):
|
||||
raise ValueError("此任务当前不能恢复")
|
||||
self._ensure_available(jid)
|
||||
control = self.controls.get(jid)
|
||||
if control and control["thread"].is_alive():
|
||||
if control and control["thread"].is_alive() and control["paused"] and job['status'] == 'paused':
|
||||
with control["condition"]:
|
||||
self.db.job_update(jid, status="running", message="正在继续采集")
|
||||
control["paused"] = False
|
||||
control["condition"].notify_all()
|
||||
else:
|
||||
if control and control['thread'].is_alive():
|
||||
control['thread'].join(timeout=1)
|
||||
if control['thread'].is_alive():
|
||||
raise ValueError("旧任务正在结束,请稍后点击恢复")
|
||||
self._launch(jid)
|
||||
return self.get(jid)
|
||||
|
||||
@@ -147,7 +164,10 @@ class JobManager:
|
||||
self.closed = True
|
||||
for jid, control in list(self.controls.items()):
|
||||
if control["thread"].is_alive():
|
||||
self.cancel(jid)
|
||||
with control['condition']:
|
||||
control['shutdown'] = True
|
||||
self.db.job_update(jid, status='paused', message='服务停止,任务可从断点恢复')
|
||||
control['condition'].notify_all()
|
||||
for control in list(self.controls.values()):
|
||||
control["thread"].join(timeout=1)
|
||||
|
||||
@@ -194,7 +214,7 @@ class JobManager:
|
||||
self._error(jid, f"{label}:{exc}")
|
||||
|
||||
def _scores(self, jid, params):
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "keyword": params.get("school_keyword", "")}
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
|
||||
records = self.db.all_catalog(filters)
|
||||
targets = {}
|
||||
for row in records:
|
||||
|
||||
+5
-1
@@ -13,6 +13,10 @@ class RemoteUnavailable(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class RateLimited(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ProxyRouter:
|
||||
def __init__(self, settings, session=None):
|
||||
self.settings = settings
|
||||
@@ -146,4 +150,4 @@ class Transport:
|
||||
raise RemoteUnavailable("网络连接失败,已尝试切换代理;请检查网络或恢复任务") from None
|
||||
except requests.HTTPError as exc:
|
||||
raise RemoteUnavailable(f"数据源返回 HTTP {exc.response.status_code}") from None
|
||||
raise RemoteUnavailable("数据源限流,请降低请求频率后重试")
|
||||
raise RateLimited("数据源持续限流,任务已等待;请降低请求频率后恢复")
|
||||
|
||||
Reference in New Issue
Block a user