feat: implement unified data sources, proxy routing and local API
This commit is contained in:
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import io
|
||||
import json
|
||||
import sqlite3
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from studio.api import create_app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(tmp_path):
|
||||
with TestClient(create_app(tmp_path), base_url="http://127.0.0.1:8765") as client:
|
||||
yield client
|
||||
|
||||
|
||||
def csv_upload(client, year=2026):
|
||||
text = 'school_name,school_code,major_code,major_name,department_name,nzsrsstr,subject1,subject2,subject3,subject4\n北京大学,10001,081200,计算机科学与技术,信息学院,专业:6(不含推免),101思想政治理论,201英语一,301数学一,408计算机学科专业基础\n'
|
||||
return client.post("/api/import", files={"file": ("yzw.csv", text.encode("utf-8-sig"))}, data={"year": year})
|
||||
|
||||
|
||||
def test_csv_upload_filter_favorite_and_export(client):
|
||||
assert csv_upload(client).json()["records"] == 1
|
||||
rows = client.get("/api/catalog", params={"year": 2026, "subject": "408"}).json()
|
||||
assert rows["total"] == 1
|
||||
record = rows["items"][0]
|
||||
assert record["quota"] == 6
|
||||
assert client.put(f'/api/catalog/{record["id"]}/favorite', json={"enabled": True}).status_code == 200
|
||||
assert client.get("/api/catalog", params={"favorites": True}).json()["total"] == 1
|
||||
exported = client.get("/api/export", params={"format": "json", "year": 2026}).json()
|
||||
assert exported["catalog"][0]["year"] == 2026
|
||||
assert client.get("/api/export", params={"format": "xlsx"}).content.startswith(b"PK")
|
||||
|
||||
|
||||
def test_import_does_not_invent_year_when_csv_has_no_year(client):
|
||||
content = b"school_name,major_code\nTest,081200\n"
|
||||
response = client.post("/api/import", files={"file": ("data.csv", content)})
|
||||
assert response.status_code == 400
|
||||
assert "年份" in response.json()["detail"]
|
||||
|
||||
|
||||
def test_cross_origin_write_is_blocked_and_secrets_are_not_in_settings(client):
|
||||
response = client.put("/api/settings", json={"cookie": "secret", "network_mode": "direct"}, headers={"Origin": "https://untrusted.example"})
|
||||
assert response.status_code == 403
|
||||
assert "cookie" not in client.get("/api/settings").json()
|
||||
|
||||
|
||||
def test_export_prevents_spreadsheet_formula_execution(client):
|
||||
data = {"catalog": [{"year": 2026, "school_name": "=1+1", "major_code": "081200", "major_name": "test"}]}
|
||||
response = client.post("/api/import", files={"file": ("studio.json", json.dumps(data).encode())})
|
||||
assert response.status_code == 200
|
||||
response = client.get("/api/export", params={"format": "csv"})
|
||||
assert "'=1+1" in response.content.decode("utf-8-sig")
|
||||
|
||||
|
||||
def test_import_yam_sqlite_keeps_year_and_score_scope(client, tmp_path):
|
||||
path = tmp_path / "yam.db"
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript('CREATE TABLE schools(school_id TEXT,major_code TEXT,name TEXT,province TEXT); CREATE TABLE departments(department_id TEXT,school_id TEXT,major_code TEXT,name TEXT,research_direction TEXT,exam_subjects TEXT,enrollment_text TEXT); CREATE TABLE score_lines(school_id TEXT,department_id TEXT,major_code TEXT,year INTEGER,total INTEGER,match_scope TEXT,raw_code TEXT);')
|
||||
conn.execute("INSERT INTO schools VALUES('10001','081200','北京大学','北京')")
|
||||
conn.execute("INSERT INTO departments VALUES('001','10001','081200','信息学院','人工智能','[\"201英语一\"]','专业:6')")
|
||||
conn.execute("INSERT INTO score_lines VALUES('10001','001','081200',2025,310,'discipline','0812')")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
response = client.post("/api/import", files={"file": ("yam.db", path.read_bytes())}, data={"year": 2026})
|
||||
assert response.status_code == 200, response.text
|
||||
rows = client.get("/api/catalog").json()["items"]
|
||||
assert rows[0]["year"] == 2026
|
||||
assert rows[0]["score_year"] == 2025
|
||||
assert rows[0]["score_scope"] == "discipline"
|
||||
@@ -0,0 +1,74 @@
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from studio.connectors import CatalogClient, ScoreClient, LoginRequired, SourceYearMismatch
|
||||
from studio.database import Database
|
||||
from studio.jobs import JobManager
|
||||
from studio.settings import SettingsStore
|
||||
from tests.test_network import Response
|
||||
|
||||
|
||||
class CatalogTransport:
|
||||
def request(self, method, url, **kwargs):
|
||||
if method == "GET":
|
||||
return Response({}, text="2027年硕士专业目录查询")
|
||||
page = int(kwargs["data"].get("curPage", 1))
|
||||
rows = [{"dwdm": "10001", "dwmc": "北京大学"}] if page == 1 else [{"dwdm": "10002", "dwmc": "中国人民大学"}]
|
||||
return Response({"flag": True, "msg": {"list": rows, "nextPageAvailable": page == 1, "totalCount": 2}})
|
||||
|
||||
|
||||
def test_catalog_pagination_consumes_all_pages():
|
||||
client = CatalogClient(CatalogTransport())
|
||||
assert [row["dwdm"] for row in client.schools("11")] == ["10001", "10002"]
|
||||
|
||||
|
||||
def test_false_login_business_status_cannot_be_success():
|
||||
class Unauthenticated(CatalogTransport):
|
||||
def request(self, *args, **kwargs):
|
||||
return Response({"flag": False, "msg": "请登录"})
|
||||
with pytest.raises(LoginRequired):
|
||||
list(CatalogClient(Unauthenticated()).schools("11"))
|
||||
|
||||
|
||||
def test_source_year_mismatch_prevents_wrongly_dated_catalog():
|
||||
with pytest.raises(SourceYearMismatch):
|
||||
CatalogClient(CatalogTransport()).verify_year(2026)
|
||||
|
||||
|
||||
def test_school_lookup_rejects_ambiguous_or_wrong_campus():
|
||||
class SearchTransport:
|
||||
def request(self, *args, **kwargs):
|
||||
return Response({"code": "0000", "data": {"data": [{"school_name": "中国地质大学(武汉)", "school_id": 10}]}})
|
||||
assert ScoreClient(SearchTransport()).school_id("中国地质大学(北京)") is None
|
||||
|
||||
|
||||
def test_task_pause_resume_reuses_persisted_work_without_duplicate_entries(tmp_path):
|
||||
db = Database(tmp_path / "studio.db")
|
||||
settings = SettingsStore(tmp_path)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
def runner(job_id, params, manager):
|
||||
for unit in ["one", "two"]:
|
||||
manager.checkpoint(job_id)
|
||||
if manager.unit_done(job_id, unit):
|
||||
continue
|
||||
if unit == "one":
|
||||
entered.set()
|
||||
release.wait(3)
|
||||
manager.complete_unit(job_id, unit)
|
||||
jobs = JobManager(db, settings, runner=runner)
|
||||
job = jobs.start({"mode": "catalog", "year": 2027})
|
||||
assert entered.wait(3)
|
||||
jobs.pause(job["id"])
|
||||
release.set()
|
||||
time.sleep(.1)
|
||||
assert jobs.get(job["id"])["status"] == "paused"
|
||||
jobs.resume(job["id"])
|
||||
deadline = time.monotonic() + 3
|
||||
while jobs.get(job["id"])["status"] != "completed" and time.monotonic() < deadline:
|
||||
time.sleep(.02)
|
||||
assert jobs.get(job["id"])["status"] == "completed"
|
||||
assert jobs.get(job["id"])["completed_units"] == 2
|
||||
jobs.close()
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import requests
|
||||
import pytest
|
||||
|
||||
from studio.network import ProxyRouter, Transport, ProxyUnavailable
|
||||
from studio.settings import SettingsStore
|
||||
from studio.auth import CookieVault
|
||||
|
||||
|
||||
class Response:
|
||||
def __init__(self, data, status=200, text=""):
|
||||
self.data = data
|
||||
self.status_code = status
|
||||
self.text = text
|
||||
self.headers = {}
|
||||
self.url = "https://yz.chsi.com.cn/zsml/"
|
||||
|
||||
def json(self):
|
||||
return self.data
|
||||
|
||||
def raise_for_status(self):
|
||||
if self.status_code >= 400:
|
||||
raise requests.HTTPError(response=self)
|
||||
|
||||
|
||||
class Pool:
|
||||
def __init__(self, proxies=None):
|
||||
self.proxies = proxies if proxies is not None else [{"proxy": "1.2.3.4:8080", "https": True}, {"proxy": "5.6.7.8:9090", "https": True}]
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
if url.endswith("/all/"):
|
||||
return Response(self.proxies)
|
||||
if url.endswith("/count/"):
|
||||
return Response({"count": len(self.proxies), "http_type": {"https": len(self.proxies)}})
|
||||
return Response(self.proxies[0] if self.proxies else {"code": 0, "src": "no proxy"})
|
||||
|
||||
|
||||
class Remote:
|
||||
def __init__(self, fail_once=False):
|
||||
self.fail_once = fail_once
|
||||
self.cookies = requests.cookies.RequestsCookieJar()
|
||||
|
||||
def request(self, method, url, **kwargs):
|
||||
if self.fail_once:
|
||||
self.fail_once = False
|
||||
raise requests.exceptions.ProxyError("failed")
|
||||
return Response({"routed_through": (kwargs.get("proxies") or {}).get("https"), "url": url})
|
||||
|
||||
|
||||
def setup_network(tmp_path, mode="proxy_only", pool=None, remote=None):
|
||||
settings = SettingsStore(tmp_path)
|
||||
settings.update({"network_mode": mode, "request_interval": 0.3})
|
||||
router = ProxyRouter(settings, session=pool or Pool())
|
||||
transport = Transport(settings, router, session=remote or Remote(), wait=lambda _: None)
|
||||
return settings, router, transport
|
||||
|
||||
|
||||
def test_catalog_and_score_requests_share_the_selected_proxy(tmp_path):
|
||||
_, router, transport = setup_network(tmp_path)
|
||||
first = transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/dws.do").json()
|
||||
second = transport.request("POST", "https://api.kaoyan.cn/pc/school/schoolScore").json()
|
||||
assert first["routed_through"] == second["routed_through"]
|
||||
assert first["routed_through"] in ["http://1.2.3.4:8080", "http://5.6.7.8:9090"]
|
||||
|
||||
|
||||
def test_strict_proxy_mode_never_falls_back_to_direct_connection(tmp_path):
|
||||
_, _, transport = setup_network(tmp_path, pool=Pool([]))
|
||||
with pytest.raises(ProxyUnavailable):
|
||||
transport.request("POST", "https://yz.chsi.com.cn/zsml/rs/dws.do")
|
||||
|
||||
|
||||
def test_prefer_proxy_mode_exposes_direct_fallback(tmp_path):
|
||||
_, router, transport = setup_network(tmp_path, mode="prefer_proxy", pool=Pool([]))
|
||||
result = transport.request("GET", "https://api.kaoyan.cn/pc/school/schoolList").json()
|
||||
assert result["routed_through"] is None
|
||||
assert router.status()["route"] == "direct"
|
||||
assert router.status()["fallback"] is True
|
||||
|
||||
|
||||
def test_network_failure_rotates_to_another_proxy(tmp_path):
|
||||
_, router, transport = setup_network(tmp_path, remote=Remote(fail_once=True))
|
||||
original = router.resolve()
|
||||
result = transport.request("GET", "https://yz.chsi.com.cn/zsml/").json()
|
||||
assert result["routed_through"] != original
|
||||
assert len(router.failed) == 1
|
||||
|
||||
|
||||
def test_cookie_is_encrypted_and_not_returned_in_settings(tmp_path):
|
||||
vault = CookieVault(tmp_path)
|
||||
vault.save({"JSESSIONID": "secret-test-cookie", "CASTGC": "ticket"})
|
||||
assert vault.load()["JSESSIONID"] == "secret-test-cookie"
|
||||
assert b"secret-test-cookie" not in vault.path.read_bytes()
|
||||
assert "cookie" not in SettingsStore(tmp_path).read()
|
||||
|
||||
Reference in New Issue
Block a user