Fix historical pagination and bound export and proxy memory
This commit is contained in:
@@ -0,0 +1,125 @@
|
||||
import asyncio
|
||||
import csv
|
||||
import io
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from studio.api import create_app
|
||||
from tests.test_bounded_storage import record
|
||||
|
||||
|
||||
@pytest.mark.parametrize('kind', ['csv', 'json', 'xlsx'])
|
||||
def test_export_uses_bounded_rows_and_preserves_content(tmp_path, monkeypatch, kind):
|
||||
app = create_app(tmp_path)
|
||||
app.state.db.upsert_catalog([record(name='=1+1'), record(code='10002', name='含逗号,大学')])
|
||||
app.state.db.upsert_scores([{'school_code': '10001', 'major_code': '081200', 'year': 2025, 'scope': 'major', 'total': 310}])
|
||||
def forbidden(*args, **kwargs):
|
||||
raise AssertionError('export materializes all catalog rows')
|
||||
monkeypatch.setattr(app.state.db, 'all_catalog', forbidden)
|
||||
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
|
||||
response = client.get('/api/export', params={'format': kind})
|
||||
assert response.status_code == 200
|
||||
if kind == 'json':
|
||||
data = response.json()
|
||||
assert {r['school_name'] for r in data['catalog']} == {'=1+1', '含逗号,大学'}
|
||||
assert data['scores'][0]['total'] == 310
|
||||
elif kind == 'csv':
|
||||
rows = list(csv.reader(io.StringIO(response.content.decode('utf-8-sig'))))
|
||||
assert len(rows) == 3
|
||||
assert any("'=1+1" in row for row in rows)
|
||||
assert any('含逗号,大学' in row for row in rows)
|
||||
else:
|
||||
book = load_workbook(io.BytesIO(response.content), read_only=True)
|
||||
rows = list(book.active.values)
|
||||
assert len(rows) == 3
|
||||
assert any("'=1+1" in row for row in rows)
|
||||
book.close()
|
||||
assert not list((tmp_path / 'exports').glob('*'))
|
||||
|
||||
|
||||
def test_concurrent_export_is_rejected_and_lock_is_released(tmp_path, monkeypatch):
|
||||
app = create_app(tmp_path)
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
original = app.state.db.iter_catalog
|
||||
def slow_rows(*args, **kwargs):
|
||||
if not entered.is_set():
|
||||
entered.set()
|
||||
assert release.wait(5)
|
||||
yield from original(*args, **kwargs)
|
||||
monkeypatch.setattr(app.state.db, 'iter_catalog', slow_rows)
|
||||
with TestClient(app, base_url='http://127.0.0.1:8765') as client:
|
||||
responses = []
|
||||
thread = threading.Thread(target=lambda: responses.append(client.get('/api/export')))
|
||||
thread.start()
|
||||
assert entered.wait(3)
|
||||
blocked = client.get('/api/export', params={'format': 'json'})
|
||||
release.set()
|
||||
thread.join(5)
|
||||
assert blocked.status_code == 429
|
||||
assert responses[0].status_code == 200
|
||||
assert client.get('/api/export').status_code == 200
|
||||
|
||||
|
||||
def test_disconnected_stream_closes_source_and_releases_export_lock():
|
||||
from studio.exporting import ExportStream
|
||||
from starlette.requests import ClientDisconnect
|
||||
lock = threading.BoundedSemaphore(1)
|
||||
assert lock.acquire(False)
|
||||
|
||||
|
||||
closed = []
|
||||
def source():
|
||||
try:
|
||||
yield b'data'
|
||||
finally:
|
||||
closed.append(True)
|
||||
async def send(message):
|
||||
if message['type'] == 'http.response.body':
|
||||
raise OSError('client disconnected')
|
||||
async def receive():
|
||||
return {'type': 'http.disconnect'}
|
||||
response = ExportStream(source(), lock.release, media_type='text/csv')
|
||||
with pytest.raises(ClientDisconnect):
|
||||
asyncio.run(response({'type': 'http', 'asgi': {'spec_version': '2.4'}}, receive, send))
|
||||
assert closed == [True]
|
||||
assert lock.acquire(False)
|
||||
|
||||
|
||||
def test_xlsx_packaging_error_removes_xml_tempfile_without_masking_error(tmp_path, monkeypatch):
|
||||
from pathlib import Path
|
||||
from studio.imports import write_xlsx
|
||||
from openpyxl import Workbook
|
||||
paths = []
|
||||
def packaging_failure(book, filename):
|
||||
sheet = book.worksheets[0]
|
||||
sheet.close()
|
||||
paths.append(Path(sheet._writer.out))
|
||||
raise OSError('archive write failed')
|
||||
monkeypatch.setattr(Workbook, 'save', packaging_failure)
|
||||
with pytest.raises(OSError, match='archive write failed'):
|
||||
write_xlsx([record()], tmp_path / 'failed.xlsx')
|
||||
assert paths and not paths[0].exists()
|
||||
|
||||
|
||||
def test_failed_export_batch_does_not_finish_a_successful_truncated_download(tmp_path, monkeypatch):
|
||||
app = create_app(tmp_path)
|
||||
def broken_rows(*args, **kwargs):
|
||||
yield record()
|
||||
raise RuntimeError('database batch failed')
|
||||
monkeypatch.setattr(app.state.db, 'iter_catalog', broken_rows)
|
||||
sent = []
|
||||
async def send(message):
|
||||
sent.append(message)
|
||||
async def receive():
|
||||
await asyncio.Event().wait()
|
||||
scope = {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'},
|
||||
'http_version': '1.1', 'method': 'GET', 'scheme': 'http',
|
||||
'path': '/api/export', 'raw_path': b'/api/export', 'root_path': '',
|
||||
'query_string': b'format=csv', 'headers': [(b'host', b'127.0.0.1:8765')],
|
||||
'server': ('127.0.0.1', 8765), 'client': ('127.0.0.1', 12345)}
|
||||
with pytest.raises(RuntimeError, match='database batch failed'):
|
||||
asyncio.run(app(scope, receive, send))
|
||||
assert not any(message['type'] == 'http.response.body' and not message.get('more_body', False) for message in sent)
|
||||
Reference in New Issue
Block a user