Fix historical pagination and bound export and proxy memory
This commit is contained in:
@@ -0,0 +1,66 @@
|
||||
"""Bounded exports, one download at a time, with disconnect cleanup."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
|
||||
from fastapi.responses import StreamingResponse
|
||||
from studio.imports import LABELS, spreadsheet_value
|
||||
from studio.memory import release_idle_memory
|
||||
|
||||
|
||||
class ExportStream(StreamingResponse):
|
||||
def __init__(self, source, cleanup, **kwargs):
|
||||
super().__init__(source, **kwargs)
|
||||
self.source, self.cleanup = source, cleanup
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
try:
|
||||
await super().__call__(scope, receive, send)
|
||||
finally:
|
||||
try:
|
||||
self.source.close()
|
||||
finally:
|
||||
try:
|
||||
self.cleanup()
|
||||
finally:
|
||||
release_idle_memory()
|
||||
|
||||
|
||||
def csv_chunks(records):
|
||||
yield b'\xef\xbb\xbf'
|
||||
stream = io.StringIO(newline='')
|
||||
writer = csv.writer(stream)
|
||||
writer.writerow(list(LABELS.values()))
|
||||
yield stream.getvalue().encode('utf-8')
|
||||
stream.seek(0)
|
||||
stream.truncate(0)
|
||||
for record in records:
|
||||
writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS])
|
||||
if stream.tell() >= 65536:
|
||||
yield stream.getvalue().encode('utf-8')
|
||||
stream.seek(0)
|
||||
stream.truncate(0)
|
||||
if stream.tell():
|
||||
yield stream.getvalue().encode('utf-8')
|
||||
|
||||
|
||||
def json_chunks(database, filters):
|
||||
pairs = set()
|
||||
yield b'{"version":1,"catalog":['
|
||||
first = True
|
||||
for row in database.iter_catalog(filters):
|
||||
pairs.add((row['school_code'], row['major_code']))
|
||||
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
|
||||
first = False
|
||||
yield b'],"scores":['
|
||||
first = True
|
||||
for row in database.iter_scores(pairs):
|
||||
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
|
||||
first = False
|
||||
yield b']}'
|
||||
|
||||
|
||||
def file_chunks(path):
|
||||
with path.open('rb') as stream:
|
||||
while chunk := stream.read(65536):
|
||||
yield chunk
|
||||
Reference in New Issue
Block a user