"""Bounded exports, one download at a time, with disconnect cleanup.""" import csv import io import json from fastapi.responses import StreamingResponse from studio.imports import LABELS, spreadsheet_value from studio.memory import release_idle_memory class ExportStream(StreamingResponse): def __init__(self, source, cleanup, **kwargs): super().__init__(source, **kwargs) self.source, self.cleanup = source, cleanup async def __call__(self, scope, receive, send): try: await super().__call__(scope, receive, send) finally: try: self.source.close() finally: try: self.cleanup() finally: release_idle_memory() def csv_chunks(records): yield b'\xef\xbb\xbf' stream = io.StringIO(newline='') writer = csv.writer(stream) writer.writerow(list(LABELS.values())) yield stream.getvalue().encode('utf-8') stream.seek(0) stream.truncate(0) for record in records: writer.writerow([spreadsheet_value(record.get(field)) for field in LABELS]) if stream.tell() >= 65536: yield stream.getvalue().encode('utf-8') stream.seek(0) stream.truncate(0) if stream.tell(): yield stream.getvalue().encode('utf-8') def json_chunks(database, filters): pairs = set() yield b'{"version":1,"catalog":[' first = True for row in database.iter_catalog(filters): pairs.add((row['school_code'], row['major_code'])) yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8') first = False yield b'],"scores":[' first = True for row in database.iter_scores(pairs): yield (('' if first else ',') + json.dumps(row, ensure_ascii=False)).encode('utf-8') first = False yield b']}' def file_chunks(path): with path.open('rb') as stream: while chunk := stream.read(65536): yield chunk