Fix historical pagination and bound export and proxy memory
This commit is contained in:
+20
-19
@@ -1,6 +1,7 @@
|
||||
"""Persistent units and cooperative pause/cancel for one local worker."""
|
||||
import threading
|
||||
import uuid
|
||||
from itertools import chain
|
||||
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
|
||||
from studio.domain import PROVINCES, normalize_catalog
|
||||
from studio.network import ProxyUnavailable, RemoteUnavailable, RateLimited
|
||||
@@ -230,8 +231,7 @@ class JobManager:
|
||||
records = []
|
||||
for row in client.directions(school, major, params.get("study_mode", "")):
|
||||
records.extend({**r, "year_verified": True} for r in normalize_catalog(row, params["year"], school, major))
|
||||
self._save(jid, records)
|
||||
self.complete_unit(jid, unit)
|
||||
self.db.save_unit(jid, unit, records)
|
||||
completed.add(unit)
|
||||
self.db.log(jid, "info", f"{label}:保存 {len(records)} 条方向/科目组合")
|
||||
except RemoteUnavailable as exc:
|
||||
@@ -240,24 +240,25 @@ class JobManager:
|
||||
self.db.log(jid, 'info', f"{school.get('dwmc', school['dwdm'])}:已跳过 {skipped} 个完成专业")
|
||||
|
||||
def _scores(self, jid, params):
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""), "school_keyword": params.get("school_keyword", "")}
|
||||
records = self.db.all_catalog(filters)
|
||||
targets = {}
|
||||
for row in records:
|
||||
if row["province"] and row["province"] not in [PROVINCES[p] for p in params["provinces"]]:
|
||||
continue
|
||||
targets[(row["school_code"], row["major_code"])] = row
|
||||
if not targets and params.get("major_code"):
|
||||
for row in self.db.schools(params.get("school_keyword", "")):
|
||||
if row["province"] in [PROVINCES[p] for p in params["provinces"]]:
|
||||
targets[(row["school_code"], params["major_code"])] = {**row, "major_code": params["major_code"]}
|
||||
if not targets:
|
||||
provinces = [PROVINCES[p] for p in params['provinces']]
|
||||
filters = {"year": params["year"], "major_code": params.get("major_code", ""),
|
||||
"school_keyword": params.get("school_keyword", ""), 'provinces': provinces}
|
||||
targets = iter(self.db.score_targets(filters))
|
||||
first = next(targets, None)
|
||||
if first is None and params.get('major_code'):
|
||||
targets = ({**row, 'major_code': params['major_code']}
|
||||
for row in self.db.schools(params.get('school_keyword', '')) if row['province'] in provinces)
|
||||
first = next(targets, None)
|
||||
if first is None:
|
||||
raise ValueError("没有符合条件的院校专业,请先采集/导入目录,或输入专业代码使用院校索引")
|
||||
completed = self.db.completed_units(jid)
|
||||
self.db.log(jid, 'info', '开始补充分数线和历史计划,分批读取专业目标并复用院校年份分数线')
|
||||
client = ScoreClient(self.transport)
|
||||
for (_, code), record in targets.items():
|
||||
for record in chain((first,), targets):
|
||||
self.checkpoint(jid)
|
||||
code = record['major_code']
|
||||
school = {key: record.get(key, "") for key in ("school_code", "school_name", "province", "level")}
|
||||
years = [y for y in params["score_years"] if not self.unit_done(jid, f"scores:{school['school_code']}:{code}:{y}")]
|
||||
years = [y for y in params["score_years"] if f"scores:{school['school_code']}:{code}:{y}" not in completed]
|
||||
if not years:
|
||||
continue
|
||||
school_id = client.school_id(school["school_name"])
|
||||
@@ -270,10 +271,10 @@ class JobManager:
|
||||
self.db.job_update(jid, message=label)
|
||||
try:
|
||||
scores = client.score_lines(school, code, year, school_id)
|
||||
self.db.upsert_scores(scores)
|
||||
plans = client.plans(school, code, year, school_id) if params.get("include_plans", True) else []
|
||||
self._save(jid, plans)
|
||||
self.complete_unit(jid, f"scores:{school['school_code']}:{code}:{year}")
|
||||
unit = f"scores:{school['school_code']}:{code}:{year}"
|
||||
self.db.save_unit(jid, unit, plans, scores=scores)
|
||||
completed.add(unit)
|
||||
self.db.log(jid, "info", f"{label}:分数线 {len(scores)} 条,历史计划 {len(plans)} 条")
|
||||
except RemoteUnavailable as exc:
|
||||
self._error(jid, f"{label}:{exc}")
|
||||
|
||||
Reference in New Issue
Block a user