Fix historical pagination and bound export and proxy memory

This commit is contained in:
2026-10-13 00:31:34 +08:00
parent 4c510759fb
commit ec2c897752
20 changed files with 1050 additions and 87 deletions
+66
View File
@@ -0,0 +1,66 @@
"""Bounded exports, one download at a time, with disconnect cleanup."""
import csv
import io
import json
from fastapi.responses import StreamingResponse
from studio.imports import LABELS, spreadsheet_value
from studio.memory import release_idle_memory
class ExportStream(StreamingResponse):
def __init__(self, source, cleanup, **kwargs):
super().__init__(source, **kwargs)
self.source, self.cleanup = source, cleanup
async def __call__(self, scope, receive, send):
try:
await super().__call__(scope, receive, send)
finally:
try:
self.source.close()
finally:
try:
self.cleanup()
finally:
release_idle_memory()
def csv_chunks(records):
yield b'\xef\xbb\xbf'
stream = io.StringIO(newline='')
writer = csv.writer(stream)
writer.writerow(list(LABELS.values()))
yield stream.getvalue().encode('utf-8')
stream.seek(0)
stream.truncate(0)
for record in records:
writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS])
if stream.tell() >= 65536:
yield stream.getvalue().encode('utf-8')
stream.seek(0)
stream.truncate(0)
if stream.tell():
yield stream.getvalue().encode('utf-8')
def json_chunks(database, filters):
pairs = set()
yield b'{"version":1,"catalog":['
first = True
for row in database.iter_catalog(filters):
pairs.add((row['school_code'], row['major_code']))
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
first = False
yield b'],"scores":['
first = True
for row in database.iter_scores(pairs):
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
first = False
yield b']}'
def file_chunks(path):
with path.open('rb') as stream:
while chunk := stream.read(65536):
yield chunk