Files
kaoyan/tests/test_collectors.py
T

143 lines
5.5 KiB
Python

import threading
import time
import pytest
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
from studio.database import Database
from studio.jobs import JobManager
from studio.settings import SettingsStore
from tests.test_network import Response
class CatalogTransport:
def request(self, method, url, **kwargs):
if method == "GET":
return Response({}, text="2027年硕士专业目录查询")
page = int(kwargs["data"].get("curPage", 1))
rows = [{"dwdm": "10001", "dwmc": "北京大学"}] if page == 1 else [{"dwdm": "10002", "dwmc": "中国人民大学"}]
return Response({"flag": True, "msg": {"list": rows, "nextPageAvailable": page == 1, "totalCount": 2}})
def test_catalog_pagination_consumes_all_pages():
client = CatalogClient(CatalogTransport())
assert [row["dwdm"] for row in client.schools("11")] == ["10001", "10002"]
def test_false_login_business_status_cannot_be_success():
class Unauthenticated(CatalogTransport):
def request(self, *args, **kwargs):
return Response({"flag": False, "msg": "请登录"})
with pytest.raises(LoginRequired):
list(CatalogClient(Unauthenticated()).schools("11"))
def test_source_year_mismatch_prevents_wrongly_dated_catalog():
with pytest.raises(SourceYearMismatch):
CatalogClient(CatalogTransport()).verify_year(2026)
def test_current_catalog_year_is_read_from_vue_page_configuration():
class ConfiguredYear(CatalogTransport):
def request(self, *args, **kwargs):
return Response({}, text='<h2>{{curYear}}年硕士专业目录</h2><script>curYear: "2027",</script>Copyright 2026')
assert CatalogClient(ConfiguredYear()).verify_year(2027) == 2027
def test_school_lookup_rejects_ambiguous_or_wrong_campus():
class SearchTransport:
def request(self, *args, **kwargs):
return Response({"code": "0000", "data": {"data": [{"school_name": "中国地质大学(武汉)", "school_id": 10}]}})
assert ScoreClient(SearchTransport()).school_id("中国地质大学(北京)") is None
def test_task_pause_resume_reuses_persisted_work_without_duplicate_entries(tmp_path):
db = Database(tmp_path / "studio.db")
settings = SettingsStore(tmp_path)
entered, release = threading.Event(), threading.Event()
def runner(job_id, params, manager):
for unit in ["one", "two"]:
manager.checkpoint(job_id)
if manager.unit_done(job_id, unit):
continue
if unit == "one":
entered.set()
release.wait(3)
manager.complete_unit(job_id, unit)
jobs = JobManager(db, settings, runner=runner)
job = jobs.start({"mode": "catalog", "year": 2027})
assert entered.wait(3)
jobs.pause(job["id"])
release.set()
time.sleep(.1)
assert jobs.get(job["id"])["status"] == "paused"
jobs.resume(job["id"])
deadline = time.monotonic() + 3
while jobs.get(job["id"])["status"] != "completed" and time.monotonic() < deadline:
time.sleep(.02)
assert jobs.get(job["id"])["status"] == "completed"
assert jobs.get(job["id"])["completed_units"] == 2
jobs.close()
def test_service_shutdown_pauses_instead_of_cancelling_persisted_job(tmp_path):
db, settings = Database(tmp_path / 'studio.db'), SettingsStore(tmp_path)
entered = threading.Event()
def runner(jid, params, manager):
entered.set()
while True:
manager.checkpoint(jid)
time.sleep(.01)
manager = JobManager(db, settings, runner=runner)
job = manager.start({'mode': 'catalog', 'year': 2027})
assert entered.wait(2)
manager.close()
assert Database(tmp_path / 'studio.db').job_get(job['id'])['status'] == 'paused'
def test_business_rate_limit_uses_bounded_backoff_without_skipping_a_page():
from studio.network import RateLimited
class Limited(CatalogTransport):
def __init__(self):
self.calls, self.waits = 0, []
def request(self, *args, **kwargs):
self.calls += 1
return Response({'flag': False, 'msg': '访问太频繁,请稍后重试'})
def sleep(self, seconds):
self.waits.append(seconds)
transport = Limited()
with pytest.raises(RateLimited):
list(CatalogClient(transport).schools('11'))
assert transport.calls == 3
assert transport.waits == [5, 10]
def test_recovery_waits_for_failed_worker_to_exit_before_launching_another(tmp_path):
db, settings = Database(tmp_path / 'studio.db'), SettingsStore(tmp_path)
entered, release = threading.Event(), threading.Event()
original_log = db.log
def guarded_log(jid, level, message):
if level == 'error':
entered.set()
release.wait(2)
original_log(jid, level, message)
db.log = guarded_log
attempts = []
def runner(jid, params, manager):
attempts.append(jid)
if len(attempts) == 1:
raise ValueError('failure fixture')
manager.complete_unit(jid, 'recovered')
manager = JobManager(db, settings, runner=runner)
job = manager.start({'mode': 'catalog', 'year': 2027})
assert entered.wait(2)
threading.Timer(.05, release.set).start()
manager.resume(job['id'])
deadline = time.monotonic() + 2
while manager.get(job['id'])['status'] != 'completed' and time.monotonic() < deadline:
time.sleep(.02)
assert manager.get(job['id'])['status'] == 'completed'
assert len(attempts) == 2
manager.close()