67 lines
2.0 KiB
Python
67 lines
2.0 KiB
Python
"""Bounded exports, one download at a time, with disconnect cleanup."""
|
|
import csv
|
|
import io
|
|
import json
|
|
|
|
from fastapi.responses import StreamingResponse
|
|
from studio.imports import LABELS, spreadsheet_value
|
|
from studio.memory import release_idle_memory
|
|
|
|
|
|
class ExportStream(StreamingResponse):
|
|
def __init__(self, source, cleanup, **kwargs):
|
|
super().__init__(source, **kwargs)
|
|
self.source, self.cleanup = source, cleanup
|
|
|
|
async def __call__(self, scope, receive, send):
|
|
try:
|
|
await super().__call__(scope, receive, send)
|
|
finally:
|
|
try:
|
|
self.source.close()
|
|
finally:
|
|
try:
|
|
self.cleanup()
|
|
finally:
|
|
release_idle_memory()
|
|
|
|
|
|
def csv_chunks(records):
|
|
yield b'\xef\xbb\xbf'
|
|
stream = io.StringIO(newline='')
|
|
writer = csv.writer(stream)
|
|
writer.writerow(list(LABELS.values()))
|
|
yield stream.getvalue().encode('utf-8')
|
|
stream.seek(0)
|
|
stream.truncate(0)
|
|
for record in records:
|
|
writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS])
|
|
if stream.tell() >= 65536:
|
|
yield stream.getvalue().encode('utf-8')
|
|
stream.seek(0)
|
|
stream.truncate(0)
|
|
if stream.tell():
|
|
yield stream.getvalue().encode('utf-8')
|
|
|
|
|
|
def json_chunks(database, filters):
|
|
pairs = set()
|
|
yield b'{"version":1,"catalog":['
|
|
first = True
|
|
for row in database.iter_catalog(filters):
|
|
pairs.add((row['school_code'], row['major_code']))
|
|
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
|
|
first = False
|
|
yield b'],"scores":['
|
|
first = True
|
|
for row in database.iter_scores(pairs):
|
|
yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8')
|
|
first = False
|
|
yield b']}'
|
|
|
|
|
|
def file_chunks(path):
|
|
with path.open('rb') as stream:
|
|
while chunk := stream.read(65536):
|
|
yield chunk
|