fix: preserve data granularity and recover tasks safely

This commit is contained in:
Codex
2026-10-11 14:18:56 +08:00
parent 23f3a15a15
commit 603184549b
29 changed files with 4557 additions and 418 deletions
+10
View File
@@ -66,3 +66,13 @@ def test_import_yam_sqlite_keeps_year_and_score_scope(client, tmp_path):
assert rows[0]["year"] == 2026
assert rows[0]["score_year"] == 2025
assert rows[0]["score_scope"] == "discipline"
def test_yam_seed_json_imports_school_index_without_inventing_admissions(client):
payload = [{'dwdm': '10001', 'dwmc': '北京大学', 'zydm': '081200', 'zymc': '计算机科学与技术', 'szss': '北京', 'data': [], 'sign': 'token-must-not-copy'}]
response = client.post('/api/import', files={'file': ('yam-seed.json', json.dumps(payload).encode())})
assert response.status_code == 200, response.text
assert response.json()['records'] == 0
assert response.json()['indexed_schools'] == 1
assert client.get('/api/catalog').json()['total'] == 0
assert 'token-must-not-copy' not in client.get('/api/schools').text
+61
View File
@@ -79,3 +79,64 @@ def test_task_pause_resume_reuses_persisted_work_without_duplicate_entries(tmp_p
assert jobs.get(job["id"])["completed_units"] == 2
jobs.close()
def test_service_shutdown_pauses_instead_of_cancelling_persisted_job(tmp_path):
db, settings = Database(tmp_path / 'studio.db'), SettingsStore(tmp_path)
entered = threading.Event()
def runner(jid, params, manager):
entered.set()
while True:
manager.checkpoint(jid)
time.sleep(.01)
manager = JobManager(db, settings, runner=runner)
job = manager.start({'mode': 'catalog', 'year': 2027})
assert entered.wait(2)
manager.close()
assert Database(tmp_path / 'studio.db').job_get(job['id'])['status'] == 'paused'
def test_business_rate_limit_uses_bounded_backoff_without_skipping_a_page():
from studio.network import RateLimited
class Limited(CatalogTransport):
def __init__(self):
self.calls, self.waits = 0, []
def request(self, *args, **kwargs):
self.calls += 1
return Response({'flag': False, 'msg': '访问太频繁,请稍后重试'})
def sleep(self, seconds):
self.waits.append(seconds)
transport = Limited()
with pytest.raises(RateLimited):
list(CatalogClient(transport).schools('11'))
assert transport.calls == 3
assert transport.waits == [5, 10]
def test_recovery_waits_for_failed_worker_to_exit_before_launching_another(tmp_path):
db, settings = Database(tmp_path / 'studio.db'), SettingsStore(tmp_path)
entered, release = threading.Event(), threading.Event()
original_log = db.log
def guarded_log(jid, level, message):
if level == 'error':
entered.set()
release.wait(2)
original_log(jid, level, message)
db.log = guarded_log
attempts = []
def runner(jid, params, manager):
attempts.append(jid)
if len(attempts) == 1:
raise ValueError('failure fixture')
manager.complete_unit(jid, 'recovered')
manager = JobManager(db, settings, runner=runner)
job = manager.start({'mode': 'catalog', 'year': 2027})
assert entered.wait(2)
threading.Timer(.05, release.set).start()
manager.resume(job['id'])
deadline = time.monotonic() + 2
while manager.get(job['id'])['status'] != 'completed' and time.monotonic() < deadline:
time.sleep(.02)
assert manager.get(job['id'])['status'] == 'completed'
assert len(attempts) == 2
manager.close()
+58
View File
@@ -64,3 +64,61 @@ def test_same_year_different_college_lines_are_not_attached_arbitrarily(tmp_path
db.upsert_scores(select_score_lines(rows, "081200", "10001", "北京大学", 2026))
assert db.query_catalog({"year": 2026})["items"][0]["score_total"] is None
def test_special_plan_score_does_not_overwrite_regular_admission_line(tmp_path):
db = Database(tmp_path / "studio.db")
db.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
rows = [{"code": "081200", "total": "320", "depart_id": "001", "note": ""}, {"code": "081200", "total": "260", "depart_id": "001", "note": "少干专项计划"}]
db.upsert_scores(select_score_lines(rows, "081200", "10001", "北京大学", 2026))
assert db.overview()["scores"] == 2
assert db.query_catalog({})["items"][0]["score_total"] == 320
def test_special_plan_full_names_are_not_used_as_regular_reference_line(tmp_path):
db = Database(tmp_path / 'studio.db')
db.upsert_catalog(normalize_catalog(raw_catalog(), 2026))
rows = [{'code': '081200', 'total': 320, 'depart_id': '001', 'note': ''}, {'code': '081200', 'total': 260, 'depart_id': '001', 'note': '少数民族骨干计划'}, {'code': '081200', 'total': 250, 'depart_id': '001', 'note': '退役大学生士兵计划'}]
db.upsert_scores(select_score_lines(rows, '081200', '10001', '北京大学', 2026))
assert db.query_catalog({})['items'][0]['score_total'] == 320
def test_spreadsheet_roundtrip_preserves_record_identity(tmp_path):
from studio.imports import export_sheet, import_file
db = Database(tmp_path / 'studio.db')
records = normalize_catalog(raw_catalog(), 2027)
db.upsert_catalog(records)
for kind in ('csv', 'xlsx'):
path = tmp_path / ('data.' + kind)
path.write_bytes(export_sheet(records, kind))
imported, _ = import_file(path)
db.upsert_catalog(imported)
assert db.query_catalog({})['total'] == 2
def test_known_historical_school_levels_are_available_in_filters(tmp_path):
db = Database(tmp_path / 'studio.db')
db.upsert_catalog(normalize_catalog(raw_catalog(), 2027))
assert db.query_catalog({'level': '985'})['total'] == 2
assert db.query_catalog({'level': '211'})['total'] == 2
def test_imported_college_quota_cannot_be_promoted_by_numeric_column():
from studio.imports import row_record
row = row_record({'school_name':'Test','major_code':'081200','enrollment_text':'学院:100','enrollment_count':100},2026)
assert row['quota'] is None
assert row['quota_scope'] == 'college'
def test_job_school_keyword_does_not_match_department_names(tmp_path):
db = Database(tmp_path / 'studio.db')
item = raw_catalog()
item['yxsmc'] = '信息学院'
db.upsert_catalog(normalize_catalog(item, 2027))
assert db.query_catalog({'school_keyword': '信息'})['total'] == 0
def test_yam_reference_books_are_arrays_for_the_detail_view():
from studio.imports import row_record
row = row_record({'school_name':'Test','major_code':'081200','reference_books':'[]'},2026)
assert row['reference_books'] == []
+38 -1
View File
@@ -3,7 +3,7 @@ import pytest
from studio.network import ProxyRouter, Transport, ProxyUnavailable
from studio.settings import SettingsStore
from studio.auth import CookieVault
from studio.auth import CookieVault, AuthService
class Response:
@@ -91,3 +91,40 @@ def test_cookie_is_encrypted_and_not_returned_in_settings(tmp_path):
assert b"secret-test-cookie" not in vault.path.read_bytes()
assert "cookie" not in SettingsStore(tmp_path).read()
def test_rate_limiting_does_not_quarantine_a_healthy_proxy(tmp_path):
class Limited(Remote):
def __init__(self):
super().__init__()
self.calls = 0
def request(self, *args, **kwargs):
self.calls += 1
if self.calls == 1:
return Response({}, status=429)
return super().request(*args, **kwargs)
_, router, transport = setup_network(tmp_path, remote=Limited())
original = router.resolve()
assert transport.request("GET", "https://yz.chsi.com.cn/zsml/").json()["routed_through"] == original
assert router.failed == set()
def test_browser_cookie_domains_preserve_same_name_sessions(tmp_path):
_, _, transport = setup_network(tmp_path)
vault = CookieVault(tmp_path)
vault.save([{"name": "JSESSIONID", "value": "account-cookie", "domain": "account.chsi.com.cn", "path": "/"}, {"name": "JSESSIONID", "value": "catalog-cookie", "domain": "yz.chsi.com.cn", "path": "/"}])
AuthService(transport, vault)
cookies = transport.session.cookies
assert cookies.get("JSESSIONID", domain="account.chsi.com.cn") == "account-cookie"
assert cookies.get("JSESSIONID", domain="yz.chsi.com.cn") == "catalog-cookie"
def test_auth_checks_protected_direction_api_rather_than_public_school_index(tmp_path):
class PublicIndex(Remote):
def request(self, method, url, **kwargs):
if url.endswith('/dws.do'):
return Response({'flag': True, 'msg': {'list': []}})
return Response({'flag': False, 'msg': '请登录'})
_, _, transport = setup_network(tmp_path, remote=PublicIndex())
auth = AuthService(transport, CookieVault(tmp_path))
assert auth.check()['valid'] is False