├── .gitignore ├── README.md ├── asyncstream ├── __init__.py ├── async_file_obj.py ├── async_reader.py ├── async_writer.py ├── codecs │ ├── __init__.py │ ├── bzip2_codec.py │ ├── gzip_codec.py │ ├── none_codec.py │ ├── orc_codec.py │ ├── parquet_codec.py │ ├── snappy_codec.py │ └── zstd_codec.py └── enc_types.py ├── examples ├── simple_compress_gzip.py ├── simple_compress_gzip_with_aiofiles.py ├── simple_conversion_gzip_to_snappy.py └── simple_uncompress_bzip2_from_s3.py ├── setup.py └── tests ├── data └── baby_names.csv ├── test_open_read.py ├── test_open_write.py ├── test_readers.py ├── test_utils.py └── test_writers.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | async-stream 2 | ============ 3 | 4 | Simple library to compress/uncompress Async streams using file iterator and readers. 5 | 6 | It supports the following compression format: 7 | 8 | * gzip 9 | * bzip2 10 | * snappy 11 | * zstandard 12 | * parquet (experimental) 13 | * orc (experimental) 14 | 15 | ### Getting started 16 | 17 | Install the library as follows: 18 | 19 | pip install asyncstream 20 | 21 | Compress a regular file to gzip (`examples/simple_compress_gzip.py`): 22 | ```python 23 | import asyncstream 24 | import asyncio 25 | 26 | 27 | async def run(): 28 | async with asyncstream.open('samples/animals.txt', 'rb') as fd: 29 | async with asyncstream.open('samples/animals.txt.gz', compression='gzip') as gzfd: 30 | async for line in fd: 31 | await gzfd.write(line) 32 | 33 | if __name__ == '__main__': 34 | asyncio.run(run()) 35 | ``` 36 | 37 | or you can also open from an async file descriptor using aiofiles (`examples/simple_compress_gzip_with_aiofiles.py`): 38 | 39 | pip install aiofiles 40 | 41 | And then run the following code: 42 | ```python 43 | import aiofiles 44 | import asyncstream 45 | import asyncio 46 | 47 | 48 | async def run(): 49 | async with aiofiles.open('examples/animals.txt', 'rb') as fd: 50 | async with aiofiles.open('/tmp/animals.txt.gz', 'wb') as wfd: 51 | async with asyncstream.open(wfd, 'wb', compression='gzip') as gzfd: 52 | async for line in fd: 53 | await gzfd.write(line) 54 | 55 | 56 | if __name__ == '__main__': 57 | asyncio.run(run()) 58 | ``` 59 | 60 | You can also uncompress an S3 file on the fly using aiobotocore: 61 | 62 | pip install aiobotocore 63 | 64 | And then run the following code (`examples/simple_uncompress_bzip2_from_s3.py`): 65 | ```python 66 | import aiobotocore 67 | import asyncstream 68 | import asyncio 69 | 70 | async def run(): 71 | session = aiobotocore.get_session() 72 | async with session.create_client('s3') as s3: 73 | obj = await s3.get_object(Bucket='test-bucket', Key='path/to/file.gz') 74 | async with asyncstream.open(obj['Body'], 'rt', compression='bzip2') as fd: 75 | async for line in fd: 76 | print(line) 77 | 78 | 79 | if __name__ == '__main__': 80 | asyncio.run(run()) 81 | ``` 82 | 83 | ### Convert a gzip file to a snappy file 84 | 85 | ```python 86 | import asyncstream 87 | import asyncio 88 | 89 | async def run(): 90 | async with asyncstream.open('samples/animals.txt.gz', 'rb', compression='gzip') as inc_fd: 91 | async with asyncstream.open('samples/animals.txt.snappy', 'wb', compression='snappy') as outc_fd: 92 | async for line in inc_fd: 93 | await outc_fd.write(line) 94 | 95 | 96 | if __name__ == '__main__': 97 | asyncio.run(run()) 98 | ``` 99 | 100 | ### Use an async reader and writer to filter and update data on the fly 101 | ```python 102 | import asyncstream 103 | import asyncio 104 | 105 | async def run(): 106 | async with asyncstream.open('/tmp/animals.txt.bz2', 'rb') as in_fd: 107 | async with asyncstream.open('/tmp/animals.txt.snappy', 'wb') as out_fd: 108 | async with asyncstream.reader(in_fd) as reader: 109 | async with asyncstream.writer(out_fd) as writer: 110 | async for name, color, age in reader: 111 | if color != 'white': 112 | await writer.writerow([name, color, age * 2]) 113 | 114 | asyncio.run(run()) 115 | ``` 116 | 117 | ### Simple parquet encoding 118 | 119 | You will need to install `pyarrow` and `pandas`: 120 | 121 | pip install pyarrow pandas 122 | 123 | To compress using `snappy`, you can install `snappy`: 124 | 125 | pip install snappy 126 | 127 | The code below converts a csv file and convert it to parquet 128 | ```python 129 | import asyncstream 130 | import asyncio 131 | 132 | async def run(): 133 | async with asyncstream.open('examples/animals.txt', 'rb') as fd: 134 | async with asyncstream.open('output.parquet', 'wb', encoding='parquet', compression='snappy') as wfd: 135 | async with asyncstream.writer(wfd) as writer: 136 | async for line in fd: 137 | await writer.write(line) 138 | 139 | asyncio.run(run()) 140 | ``` 141 | 142 | ### Simple parquet decoding 143 | ```python 144 | import asyncstream 145 | import asyncio 146 | 147 | async def run(): 148 | async with asyncstream.open('output.parquet', 'rb', encoding='parquet') as fd: 149 | async with asyncstream.reader(fd) as reader: 150 | async for line in reader: 151 | print(line) 152 | 153 | asyncio.run(run()) 154 | ``` 155 | 156 | ### Simple orc encoding 157 | ```python 158 | import asyncstream 159 | import asyncio 160 | 161 | async def run(): 162 | async with asyncstream.open('examples/animals.txt', 'rb') as fd: 163 | async with asyncstream.open('output.orc.snappy', 'wb', encoding='orc', compression='snappy') as wfd: 164 | async with asyncstream.writer(wfd) as writer: 165 | async for line in fd: 166 | await writer.write(line) 167 | 168 | asyncio.run(run()) 169 | ``` 170 | ### Simple orc decoding 171 | 172 | ```python 173 | import asyncstream 174 | import asyncio 175 | 176 | async def run(): 177 | async with asyncstream.open('output.orc.snappy', 'rb', encoding='orc') as fd: 178 | async with asyncstream.reader(fd) as reader: 179 | async for line in reader: 180 | print(line) 181 | 182 | asyncio.run(run()) 183 | ``` 184 | 185 | Other compression scheme are supported: `zlib`, `brotli`. 186 | 187 | 188 | ## Compression supported 189 | 190 | Compression | Status 191 | -------------------------------------- | :-----: 192 | `gzip` / `zlib` | :white_check_mark: 193 | `bzip2` | :white_check_mark: 194 | `snappy` | :white_check_mark: 195 | `zstd` | :white_check_mark: 196 | 197 | ### Parquet 198 | Compression | Status 199 | -------------------------------------- | :-----: 200 | none | :white_check_mark: 201 | `brotli` | :white_check_mark: 202 | `bzip2` | :x: 203 | `gzip` | :x: 204 | `snappy` | :white_check_mark: 205 | `zstd` | :x: 206 | `zlib` | :white_check_mark: 207 | 208 | ### Orc 209 | 210 | Compression | Status 211 | -------------------------------------- | :-----: 212 | none | :white_check_mark: 213 | `bzip2` | :x: 214 | `gzip` / `zlib` | :white_check_mark: 215 | `snappy` | :white_check_mark: 216 | `zlib` | :white_check_mark: 217 | `zstd` | :white_check_mark: 218 | -------------------------------------------------------------------------------- /asyncstream/__init__.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import Optional, Iterable, Union 3 | 4 | import aiofiles 5 | from aiofiles.threadpool import AsyncBufferedIOBase 6 | 7 | from asyncstream.async_file_obj import AsyncFileObj 8 | from asyncstream.async_reader import AsyncReader 9 | from asyncstream.async_writer import AsyncWriter 10 | 11 | def open(afd: Union[str, AsyncBufferedIOBase], mode=None, encoding=None, compression=None, compress_level=-1): 12 | if encoding is None and compression is None: 13 | from asyncstream.codecs.none_codec import NoneDecompressor, NoneCompressor 14 | compressor = NoneCompressor() 15 | decompressor = NoneDecompressor() 16 | elif encoding == 'parquet': 17 | from asyncstream.codecs.parquet_codec import ParquetCompressor, ParquetDecompressor 18 | compressor = ParquetCompressor() 19 | decompressor = ParquetDecompressor() 20 | elif encoding == 'orc': 21 | from asyncstream.codecs.orc_codec import OrcCompressor, OrcDecompressor 22 | compressor = OrcCompressor() 23 | decompressor = OrcDecompressor() 24 | elif compression == 'gzip': 25 | from asyncstream.codecs.gzip_codec import get_gzip_encoder, get_gzip_decoder 26 | compressor = get_gzip_encoder() 27 | decompressor = get_gzip_decoder() 28 | elif compression == 'bzip2': 29 | from asyncstream.codecs.bzip2_codec import get_bzip2_encoder, get_bzip2_decoder 30 | compressor = get_bzip2_encoder() 31 | decompressor = get_bzip2_decoder() 32 | elif compression == 'zstd': 33 | from asyncstream.codecs.zstd_codec import get_zstd_encoder, get_zstd_decoder 34 | compressor = get_zstd_encoder() 35 | decompressor = get_zstd_decoder() 36 | elif compression == 'snappy': 37 | from asyncstream.codecs.snappy_codec import get_snappy_encoder, get_snappy_decoder 38 | compressor = get_snappy_encoder() 39 | decompressor = get_snappy_decoder() 40 | else: 41 | raise ValueError('Unsupported compression %s' % compression) 42 | 43 | return AsyncFileObj(afd, mode, compressor, decompressor) 44 | 45 | 46 | def reader(afd: AsyncFileObj, has_header: bool = True, columns: Optional[Iterable[str]] = None, 47 | column_types: Optional[Iterable[str]] = None, delimiter=','): 48 | return AsyncReader(afd, columns, column_types, has_header) 49 | 50 | 51 | def writer(afd: AsyncFileObj, has_header: bool = True, columns: Optional[Iterable[str]] = None, 52 | column_types: Optional[Iterable[str]] = None): 53 | return AsyncWriter(afd, has_header=has_header) 54 | -------------------------------------------------------------------------------- /asyncstream/async_file_obj.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from asyncstream.codecs import error_import_usage 4 | 5 | 6 | class AsyncFileObj(object): 7 | MODE_DEFAULT = 0 8 | MODE_BINARY = 1 9 | MODE_TEXT = 2 10 | 11 | def __init__(self, afd, mode, compressor, decompressor, ignore_header=False, buffer_size=1024 * 1024): 12 | self._afd = afd 13 | self._mode = mode 14 | self._compressor = compressor 15 | self._decompressor = decompressor 16 | self._ignore_header = ignore_header 17 | self._header = None 18 | self._buffer = b'' 19 | self._buffer_size = buffer_size 20 | self._eof = False 21 | self._lines = [] 22 | self._index = 0 23 | self._filename = None 24 | self._has_flushed = True 25 | self._is_closed = False 26 | 27 | if 'b' in mode: 28 | self._file_type = self.MODE_BINARY 29 | 30 | if 'w' in mode: 31 | self.write = self._write 32 | else: 33 | self.write = self._cannot_write 34 | 35 | async def _cannot_write(self, buffer: bytes): 36 | raise IOError('Cannot write because mode is not set to write') 37 | 38 | async def read(self, n: Optional[int] = None): 39 | buffer_size = n if n else 1024 * 1024 40 | while True: 41 | if self._eof or len(self._buffer) >= buffer_size: 42 | result = self._buffer[:buffer_size] 43 | self._buffer = self._buffer[buffer_size:] 44 | return result 45 | 46 | data = await self._afd.read(buffer_size) 47 | if data: 48 | self._buffer += self._decompressor.decompress(data) 49 | else: 50 | if hasattr(self._decompressor, 'flush'): 51 | data = self._decompressor.flush() 52 | if data: 53 | self._buffer += data 54 | self._eof = True 55 | 56 | return self._buffer 57 | 58 | async def _write(self, buffer: bytes): 59 | self._has_flushed = False 60 | compressed_data = self._compressor.compress(buffer) 61 | if compressed_data: 62 | return await self._afd.write(compressed_data) 63 | else: 64 | return 0 65 | 66 | def __aiter__(self): 67 | return self 68 | 69 | async def __anext__(self): 70 | while True: 71 | if not self._lines and self._eof: 72 | raise StopAsyncIteration 73 | 74 | if self._index >= len(self._lines) - 1: 75 | tmp = await self.read(self._buffer_size) 76 | if tmp: 77 | lines = tmp.splitlines(True) 78 | self._index = 0 79 | if self._lines: 80 | result = self._lines[0] + lines[0] 81 | else: 82 | result = lines[0] 83 | self._lines = lines[1:] 84 | return result 85 | else: 86 | self._eof = True 87 | if not self._lines: 88 | raise StopAsyncIteration 89 | result = self._lines[-1] 90 | self._lines = [] 91 | return result 92 | else: 93 | result = self._lines[self._index] 94 | self._index += 1 95 | return result 96 | 97 | # 98 | # tokens = self._buffer.split(b'\n', 1) 99 | # if len(tokens) == 2: 100 | # self._buffer = tokens[1] 101 | # return tokens[0] 102 | # else: 103 | # tmp = await self.read(self._buffer_size) 104 | # if tmp: 105 | # self._buffer += tmp 106 | # else: 107 | # if self._buffer: 108 | # return self._buffer 109 | # else: 110 | # raise StopAsyncIteration 111 | 112 | async def flush(self): 113 | if self._has_flushed: 114 | return b'' 115 | 116 | self._has_flushed = True 117 | compressed_data = self._compressor.flush() 118 | if compressed_data: 119 | return await self._afd.write(compressed_data) 120 | 121 | async def close(self): 122 | # TODO add test for read and write 123 | if self._is_closed: 124 | return 125 | 126 | await self.flush() 127 | if hasattr(self._decompressor, 'close'): 128 | self._decompressor.close() 129 | 130 | if hasattr(self._compressor, 'close'): 131 | buf = self._compressor.close() 132 | if buf: 133 | self._afd.write(buf) 134 | 135 | if hasattr(self._afd, 'flush'): 136 | await self._afd.flush() 137 | # If we pass a filename, then we close the file otherwise it's the 138 | # responsibility of the caller 139 | if self._filename: 140 | self._afd.close() 141 | 142 | self._is_closed = True 143 | 144 | async def __aenter__(self): 145 | if isinstance(self._afd, str): 146 | try: 147 | from aiofiles.threadpool import _open 148 | except ImportError: 149 | error_import_usage('aiofiles') 150 | 151 | fd = await _open(self._afd, self._mode) 152 | self._filename = self._afd 153 | self._afd = fd 154 | return self 155 | 156 | async def __aexit__(self, *exc_info): 157 | await self.close() 158 | -------------------------------------------------------------------------------- /asyncstream/async_reader.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Iterable 2 | 3 | from asyncstream import AsyncFileObj 4 | 5 | 6 | class AsyncReader(object): 7 | def __init__(self, afd: AsyncFileObj, columns=Optional[Iterable[str]], column_types=Optional[Iterable[str]], has_header=False, sep=',', eol='\n'): 8 | self._afd = afd 9 | self._sep = sep 10 | self._eol = eol 11 | self._columns = columns 12 | self._column_types = column_types 13 | self._has_header = has_header 14 | 15 | async def __aenter__(self): 16 | return self 17 | 18 | def __aiter__(self): 19 | return self 20 | 21 | async def __aexit__(self, *exc): 22 | return None 23 | 24 | async def __anext__(self): 25 | next_line = await self._afd.__anext__() 26 | if next_line: 27 | return next_line.decode('utf-8').rstrip(self._eol).split(self._sep) 28 | else: 29 | raise StopAsyncIteration 30 | 31 | -------------------------------------------------------------------------------- /asyncstream/async_writer.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Iterable, List, Any 2 | 3 | from asyncstream import AsyncFileObj 4 | 5 | 6 | class AsyncWriter(object): 7 | def __init__(self, afd: AsyncFileObj, columns=Optional[Iterable[str]], column_types=Optional[Iterable[str]], has_header=False, sep=b',', eol=b'\n'): 8 | self._afd = afd 9 | self._sep = sep 10 | self._eol = eol 11 | self._columns = columns 12 | self._column_types = column_types 13 | self._has_header = has_header 14 | 15 | async def __aenter__(self): 16 | return self 17 | 18 | async def __aexit__(self, *exc): 19 | return await self._afd.close() 20 | 21 | async def writeheader(self): 22 | await self._afd.write(self._sep.join(self._columns) + self._eol) 23 | 24 | async def writerow(self, row: List[Any]): 25 | await self._afd.write(self._sep.join(row) + self._eol) 26 | 27 | async def writerows(self, rows: List[List[Any]]): 28 | for row in rows: 29 | await self.writerow(row) 30 | 31 | async def flush(self): 32 | await self._afd.flush() 33 | -------------------------------------------------------------------------------- /asyncstream/codecs/__init__.py: -------------------------------------------------------------------------------- 1 | def error_import_usage(package: str): 2 | raise Exception('Package {package} missing. Please install it: pip install {package}'.format(package=package)) 3 | -------------------------------------------------------------------------------- /asyncstream/codecs/bzip2_codec.py: -------------------------------------------------------------------------------- 1 | import bz2 2 | 3 | 4 | def get_bzip2_encoder(): 5 | return bz2.BZ2Compressor() 6 | 7 | 8 | def get_bzip2_decoder(): 9 | return bz2.BZ2Decompressor() 10 | -------------------------------------------------------------------------------- /asyncstream/codecs/gzip_codec.py: -------------------------------------------------------------------------------- 1 | import zlib 2 | 3 | 4 | def get_gzip_encoder(compress_level: int = -1): 5 | return zlib.compressobj( 6 | compress_level, # level: 0-9 7 | zlib.DEFLATED, # method: must be DEFLATED 8 | 16 + zlib.MAX_WBITS, # window size in bits: 9 | # -15..-8: negate, suppress header 10 | # 8..15: normal 11 | # 16..30: subtract 16, gzip header 12 | zlib.DEF_MEM_LEVEL, # mem level: 1..8/9 13 | 0 # strategy: 14 | # 0 = Z_DEFAULT_STRATEGY 15 | # 1 = Z_FILTERED 16 | # 2 = Z_HUFFMAN_ONLY 17 | # 3 = Z_RLE 18 | # 4 = Z_FIXED 19 | ) 20 | 21 | 22 | def get_gzip_decoder(): 23 | return zlib.decompressobj( 24 | 16 + zlib.MAX_WBITS # see above 25 | ) 26 | -------------------------------------------------------------------------------- /asyncstream/codecs/none_codec.py: -------------------------------------------------------------------------------- 1 | class NoneDecompressor(object): 2 | def decompress(self, data: bytes) -> bytes: 3 | return data 4 | 5 | def flush(self) -> bytes: 6 | return b'' 7 | 8 | class NoneCompressor(object): 9 | def compress(self, data: bytes) -> bytes: 10 | return data 11 | 12 | def flush(self) -> bytes: 13 | return b'' 14 | -------------------------------------------------------------------------------- /asyncstream/codecs/orc_codec.py: -------------------------------------------------------------------------------- 1 | import io 2 | from tempfile import SpooledTemporaryFile 3 | from typing import Any, Iterable 4 | 5 | from asyncstream.codecs import error_import_usage 6 | 7 | try: 8 | import pandas as pd 9 | except ModuleNotFoundError: 10 | error_import_usage('pandas') 11 | 12 | try: 13 | import pyorc 14 | except ModuleNotFoundError: 15 | error_import_usage('pyorc') 16 | 17 | from asyncstream import AsyncFileObj 18 | 19 | 20 | class OrcDecompressor(object): 21 | def __init__(self): 22 | # self._buffer = TemporaryFile(mode='wb') 23 | # TODO try to make SpooledTemporaryFile work 24 | self._buffer = io.BytesIO() 25 | self._result = SpooledTemporaryFile(mode='wt') 26 | 27 | def decompress(self, data: bytes): 28 | self._buffer.write(data) 29 | return b'' 30 | 31 | def flush(self): 32 | self._buffer.seek(0) 33 | # TODO: to optimize it 34 | reader = pyorc.Reader(self._buffer) 35 | columns = reader.schema.fields.keys() 36 | return (','.join(columns) + '\n' + '\n'.join( 37 | ','.join( 38 | [str(c) for c in row] 39 | ) 40 | for row in reader 41 | )).encode('utf-8') + b'\n' 42 | 43 | def __del__(self): 44 | self._buffer.close() 45 | self._result.close() 46 | 47 | 48 | class OrcCompressor(object): 49 | def __init__(self): 50 | self._buffer = SpooledTemporaryFile(mode='wb') 51 | 52 | def compress(self, data: bytes): 53 | self._buffer.write(data) 54 | 55 | def flush(self): 56 | if self._buffer: 57 | self._buffer.seek(0) 58 | df = pd.read_csv(self._buffer) 59 | with SpooledTemporaryFile(mode='wb') as wfd: 60 | df.to_parquet(wfd, engine='pyarrow') 61 | wfd.seek(0) 62 | return wfd.read() 63 | 64 | def __del__(self): 65 | self._buffer.close() 66 | 67 | 68 | # async def parquet_write(afd, , columns: Iterable[str] = None, column_types: Iterable[str] = None, compression = None, buffer_memory = 1024 * 1024): 69 | 70 | async def parquet_write(afd, rows: Iterable[Iterable[Any]], columns: Iterable[str] = None, 71 | column_types: Iterable[str] = None, compression=None, buffer_memory=1024 * 1024): 72 | with SpooledTemporaryFile(mode='w+b', max_size=buffer_memory) as wfd: 73 | df = pd.DataFrame(rows, columns=columns, dtype=column_types) 74 | df.to_parquet(wfd, engine='pyarrow', compression=compression) 75 | wfd.seek(0) 76 | 77 | 78 | async def get_parquet_reader(afd: AsyncFileObj, buffer_memory=1024 * 1024): 79 | with SpooledTemporaryFile(mode='w+b', max_size=buffer_memory) as wfd: 80 | async for buf in afd: 81 | wfd.write(buf) 82 | 83 | wfd.flush() 84 | wfd.seek(0) 85 | table = pd.read_parquet(wfd, engine='pyarrow') 86 | 87 | for row in table.itertuples(index=False): 88 | yield row 89 | # [col.decode('utf-8') if isinstance(col, bytes) else str(col) for col in row] 90 | -------------------------------------------------------------------------------- /asyncstream/codecs/parquet_codec.py: -------------------------------------------------------------------------------- 1 | from tempfile import SpooledTemporaryFile 2 | from typing import Any, Iterable 3 | from asyncstream import AsyncFileObj 4 | from asyncstream.codecs import error_import_usage 5 | 6 | try: 7 | import pandas as pd 8 | except ImportError as e: 9 | error_import_usage('pandas') 10 | 11 | try: 12 | import pyarrow 13 | except ModuleNotFoundError: 14 | error_import_usage('pyarrow') 15 | 16 | 17 | 18 | class ParquetDecompressor(object): 19 | def __init__(self, sep=b','): 20 | self._buffer = SpooledTemporaryFile(mode='wb') 21 | self._result = SpooledTemporaryFile(mode='wb') 22 | self._sep = sep 23 | self.eof = False 24 | 25 | def decompress(self, data: bytes): 26 | self._buffer.write(data) 27 | return b'' 28 | 29 | def flush(self): 30 | self._buffer.seek(0) 31 | # TODO: to optimize it 32 | df = pd.read_parquet(self._buffer, engine='pyarrow') 33 | return df.to_csv(index=False).encode('utf-8') 34 | 35 | def __del__(self): 36 | self._buffer.close() 37 | self._result.close() 38 | 39 | 40 | class ParquetCompressor(object): 41 | def __init__(self): 42 | self._buffer = None 43 | 44 | def compress(self, data: bytes): 45 | if self._buffer is None: 46 | self._buffer = SpooledTemporaryFile(mode='wb') 47 | self._buffer.write(data) 48 | 49 | def flush(self): 50 | if self._buffer: 51 | self._buffer.seek(0) 52 | df = pd.read_csv(self._buffer) 53 | with SpooledTemporaryFile(mode='wb') as wfd: 54 | df.to_parquet(wfd, engine='pyarrow') 55 | wfd.seek(0) 56 | return wfd.read() 57 | 58 | def __del__(self): 59 | if self._buffer: 60 | self._buffer.close() 61 | 62 | 63 | # async def parquet_write(afd, , columns: Iterable[str] = None, column_types: Iterable[str] = None, compression = None, buffer_memory = 1024 * 1024): 64 | 65 | async def parquet_write(afd, rows: Iterable[Iterable[Any]], columns: Iterable[str] = None, 66 | column_types: Iterable[str] = None, compression=None, buffer_memory=1024 * 1024): 67 | with SpooledTemporaryFile(mode='w+b', max_size=buffer_memory) as wfd: 68 | df = pd.DataFrame(rows, columns=columns, dtype=column_types) 69 | df.to_parquet(wfd, engine='pyarrow', compression=compression) 70 | wfd.seek(0) 71 | 72 | 73 | async def get_parquet_reader(afd: AsyncFileObj, buffer_memory=1024 * 1024): 74 | with SpooledTemporaryFile(mode='w+b', max_size=buffer_memory) as wfd: 75 | async for buf in afd: 76 | wfd.write(buf) 77 | 78 | wfd.flush() 79 | wfd.seek(0) 80 | table = pd.read_parquet(wfd, engine='pyarrow') 81 | 82 | for row in table.itertuples(index=False): 83 | yield row 84 | -------------------------------------------------------------------------------- /asyncstream/codecs/snappy_codec.py: -------------------------------------------------------------------------------- 1 | from asyncstream.codecs import error_import_usage 2 | 3 | try: 4 | from snappy import snappy 5 | except ImportError as e: 6 | error_import_usage('snappy') 7 | 8 | 9 | def get_snappy_encoder(): 10 | return snappy.StreamCompressor() 11 | 12 | 13 | def get_snappy_decoder(): 14 | return snappy.StreamDecompressor() 15 | -------------------------------------------------------------------------------- /asyncstream/codecs/zstd_codec.py: -------------------------------------------------------------------------------- 1 | from asyncstream.codecs import error_import_usage 2 | 3 | try: 4 | import zstd 5 | except ImportError as e: 6 | error_import_usage('zstd') 7 | 8 | 9 | def get_zstd_encoder(): 10 | dctx = zstd.ZstdCompressor() 11 | return dctx.compressobj() 12 | 13 | def get_zstd_decoder(): 14 | dctx = zstd.ZstdDecompressor() 15 | return dctx.decompressobj() 16 | -------------------------------------------------------------------------------- /asyncstream/enc_types.py: -------------------------------------------------------------------------------- 1 | # # https://orc.apache.org/docs/types.html 2 | # ORC_TYPES = [ 3 | # 'boolean', 4 | # 'tinyint', 5 | # 'smallint', 6 | # 'int', 7 | # 'bigint', 8 | # 'float', 9 | # 'double', 10 | # 'string', 11 | # 'char', 12 | # 'varchar', 13 | # 'binary', 14 | # 'timestamp', 15 | # 'timestamp with local time zone' 16 | # # TODO add compound types 17 | # ] 18 | # 19 | # # https://github.com/apache/parquet-format/blob/master/LogicalTypes.md 20 | # PARQUET_TYPES = [ 21 | # 'string', 22 | # 'enum', 23 | # 'uuid', 24 | # 'int' 25 | # 'boolean', 26 | # 'int32', 27 | # 'int64', 28 | # 'int96', 29 | # 'float', 30 | # 'double', 31 | # 'byte_array', 32 | # ] 33 | # 34 | -------------------------------------------------------------------------------- /examples/simple_compress_gzip.py: -------------------------------------------------------------------------------- 1 | import aiofiles 2 | import asyncstream 3 | import asyncio 4 | 5 | 6 | async def run(): 7 | async with asyncstream.open('samples/animals.txt', mode='rb') as fd: 8 | async with asyncstream.open('samples/animals.txt.gz', mode='wb', compression='gzip') as gzfd: 9 | async for line in fd: 10 | await gzfd.write(line) 11 | 12 | 13 | if __name__ == '__main__': 14 | asyncio.run(run()) 15 | -------------------------------------------------------------------------------- /examples/simple_compress_gzip_with_aiofiles.py: -------------------------------------------------------------------------------- 1 | import aiofiles 2 | import asyncstream 3 | import asyncio 4 | 5 | 6 | async def run(): 7 | async with aiofiles.open('samples/animals.txt', 'rb') as fd: 8 | async with aiofiles.open('samples/animals2.txt.gz', 'wb') as wfd: 9 | async with asyncstream.open(wfd, 'wb', compression='gzip') as gzfd: 10 | async for line in fd: 11 | await gzfd.write(line) 12 | 13 | 14 | if __name__ == '__main__': 15 | asyncio.run(run()) 16 | -------------------------------------------------------------------------------- /examples/simple_conversion_gzip_to_snappy.py: -------------------------------------------------------------------------------- 1 | import asyncstream 2 | import asyncio 3 | 4 | async def run(): 5 | async with asyncstream.open('samples/animals.txt.gz', 'rb', compression='gzip') as inc_fd: 6 | async with asyncstream.open('samples/animals.txt.snappy', 'wb', compression='snappy') as outc_fd: 7 | async for line in inc_fd: 8 | await outc_fd.write(line) 9 | 10 | 11 | if __name__ == '__main__': 12 | asyncio.run(run()) 13 | -------------------------------------------------------------------------------- /examples/simple_uncompress_bzip2_from_s3.py: -------------------------------------------------------------------------------- 1 | import aiobotocore 2 | import asyncstream 3 | import asyncio 4 | 5 | async def run(): 6 | session = aiobotocore.get_session() 7 | async with session.create_client('s3') as s3: 8 | obj = await s3.get_object(Bucket='test-bucket', Key='path/to/file.gz') 9 | async with asyncstream.open(obj['Body'], 'rt', compression='bzip2') as fd: 10 | async for line in fd: 11 | print(line) 12 | 13 | 14 | if __name__ == '__main__': 15 | asyncio.run(run()) 16 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | 4 | from setuptools import find_packages 5 | from setuptools import setup 6 | from setuptools.command.test import test as TestCommand 7 | 8 | 9 | class PyTestCommand(TestCommand): 10 | user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] 11 | 12 | def initialize_options(self): 13 | TestCommand.initialize_options(self) 14 | self.pytest_args = [] 15 | 16 | def finalize_options(self): 17 | TestCommand.finalize_options(self) 18 | self.test_args = [] 19 | self.test_suite = True 20 | 21 | def run_tests(self): 22 | import pytest 23 | errno = pytest.main(self.pytest_args) 24 | sys.exit(errno) 25 | 26 | 27 | setup( 28 | name='async-stream', 29 | version='1.0.0', 30 | description='Async s3 stream', 31 | long_description='Async s3 stream', 32 | keywords='async stream compression gzip bzip2 zstd parquet orc', 33 | author='Francois Dang Ngoc', 34 | url='http://github.com/chimpler/async-stream/', 35 | classifiers=[ 36 | 'License :: OSI Approved :: Apache Software License', 37 | 'Topic :: Software Development :: Libraries :: Python Modules', 38 | 'Programming Language :: Python', 39 | 'Programming Language :: Python :: 3' 40 | ], 41 | install_requires=[ 42 | 'zstandard', 43 | 'uvloop', 44 | 'aiofiles', 45 | 'pyarrow==0.14.1', 46 | 'pyorc', 47 | 'pandas', 48 | 'python-snappy' 49 | ], 50 | packages=find_packages(), 51 | tests_require=[ 52 | 'pytest-asyncio', 53 | 'pytest-cov' 54 | ], 55 | entry_points={ 56 | 'console_scripts': [ 57 | ] 58 | }, 59 | cmdclass={ 60 | 'test': PyTestCommand 61 | } 62 | ) 63 | -------------------------------------------------------------------------------- /tests/data/baby_names.csv: -------------------------------------------------------------------------------- 1 | birth_year,gender,ethnicity,child_name,count,rank 2 | 2011,FEMALE,HISPANIC,IRIS,10,78 3 | 2011,FEMALE,HISPANIC,JAYLEEN,51,40 4 | 2011,FEMALE,HISPANIC,KAILEY,20,68 5 | 2011,FEMALE,HISPANIC,KHLOE,57,35 6 | 2011,FEMALE,HISPANIC,LILIANA,28,60 7 | 2011,FEMALE,HISPANIC,MELANY,14,74 8 | 2011,FEMALE,HISPANIC,NATHALIA,10,78 9 | 2011,FEMALE,HISPANIC,ROSELYN,16,72 10 | 2011,FEMALE,HISPANIC,SKYLA,21,67 11 | 2011,FEMALE,HISPANIC,VIVIANA,11,77 12 | 2011,FEMALE,WHITE NON HISPANIC,ALICE,50,43 13 | 2011,FEMALE,WHITE NON HISPANIC,ARIANNA,28,63 14 | 2011,FEMALE,WHITE NON HISPANIC,BIANCA,10,81 15 | 2011,FEMALE,WHITE NON HISPANIC,CHANA,132,10 16 | 2011,FEMALE,WHITE NON HISPANIC,DEVORAH,44,48 17 | 2011,FEMALE,WHITE NON HISPANIC,EMMA,213,2 18 | 2011,FEMALE,WHITE NON HISPANIC,GEMMA,15,76 19 | 2011,FEMALE,WHITE NON HISPANIC,HENNY,18,73 20 | 2011,FEMALE,WHITE NON HISPANIC,JULIANNA,28,63 21 | 2011,FEMALE,WHITE NON HISPANIC,LEILA,10,81 22 | 2011,FEMALE,WHITE NON HISPANIC,MADELINE,39,52 23 | 2011,FEMALE,WHITE NON HISPANIC,MILA,31,60 24 | 2011,FEMALE,WHITE NON HISPANIC,PENELOPE,38,53 25 | 2011,FEMALE,WHITE NON HISPANIC,RUCHY,12,79 26 | 2011,FEMALE,WHITE NON HISPANIC,SIENA,16,75 27 | 2011,FEMALE,WHITE NON HISPANIC,TOBY,39,52 28 | 2011,FEMALE,WHITE NON HISPANIC,YIDES,25,66 29 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,ALLEN,14,52 30 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,CHARLES,17,49 31 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,FARHAN,17,49 32 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,JASON,98,6 33 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,LEO,40,27 34 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,NELSON,12,54 35 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,STEVEN,35,32 36 | 2011,MALE,BLACK NON HISPANIC,ADAM,28,46 37 | 2011,MALE,BLACK NON HISPANIC,ANTONIO,15,59 38 | 2011,MALE,BLACK NON HISPANIC,CHASE,41,33 39 | 2011,MALE,BLACK NON HISPANIC,ELIJAH,156,3 40 | 2011,MALE,BLACK NON HISPANIC,JACKSON,11,63 41 | 2011,MALE,BLACK NON HISPANIC,JERMAINE,16,58 42 | 2011,MALE,BLACK NON HISPANIC,KENNETH,16,58 43 | 2011,MALE,BLACK NON HISPANIC,MARQUIS,10,64 44 | 2011,MALE,BLACK NON HISPANIC,NIGEL,10,64 45 | 2011,MALE,BLACK NON HISPANIC,SINCERE,24,50 46 | 2011,MALE,HISPANIC,ABRAHAM,22,82 47 | 2011,MALE,HISPANIC,AMIR,21,83 48 | 2011,MALE,HISPANIC,BRADLEY,24,80 49 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ALICE,27,21 50 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ARIANNA,11,37 51 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,DIYA,12,36 52 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,HANNAH,23,25 53 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,JULIA,24,24 54 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,MARYAM,17,31 55 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,SARA,24,24 56 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,VIVIAN,44,11 57 | 2011,FEMALE,BLACK NON HISPANIC,AMANDA,17,38 58 | 2011,FEMALE,BLACK NON HISPANIC,ASHLEY,16,39 59 | 2011,FEMALE,BLACK NON HISPANIC,EMMA,15,40 60 | 2011,FEMALE,BLACK NON HISPANIC,JADA,52,10 61 | 2011,FEMALE,BLACK NON HISPANIC,KENYA,10,45 62 | 2011,FEMALE,BLACK NON HISPANIC,MARIAM,32,23 63 | 2011,FEMALE,BLACK NON HISPANIC,NYLAH,42,14 64 | 2011,FEMALE,BLACK NON HISPANIC,SARIAH,17,38 65 | 2011,FEMALE,BLACK NON HISPANIC,TIANNA,13,42 66 | 2011,FEMALE,HISPANIC,ALANA,11,77 67 | 2011,FEMALE,HISPANIC,ANA,33,55 68 | 2011,FEMALE,HISPANIC,ASHLY,13,75 69 | 2011,FEMALE,HISPANIC,CAROLINE,12,76 70 | 2011,FEMALE,HISPANIC,DELILAH,29,59 71 | 2011,FEMALE,HISPANIC,EVELYN,63,30 72 | 2011,MALE,HISPANIC,DANTE,21,83 73 | 2011,MALE,HISPANIC,ELIAN,17,87 74 | 2011,MALE,HISPANIC,FRANK,17,87 75 | 2011,MALE,HISPANIC,ISMAEL,17,87 76 | 2011,FEMALE,WHITE NON HISPANIC,SIMI,10,81 77 | 2011,MALE,HISPANIC,JOHNNY,18,86 78 | 2011,MALE,HISPANIC,KENNETH,33,71 79 | 2011,MALE,HISPANIC,MARCUS,33,71 80 | 2011,MALE,HISPANIC,MOISES,18,86 81 | 2011,MALE,HISPANIC,RANDY,48,59 82 | 2011,MALE,HISPANIC,STEPHEN,10,94 83 | 2011,MALE,HISPANIC,YANIEL,10,94 84 | 2011,MALE,WHITE NON HISPANIC,ALI,33,74 85 | 2011,MALE,WHITE NON HISPANIC,AVROHOM,27,80 86 | 2011,MALE,WHITE NON HISPANIC,BRYCE,10,97 87 | 2011,MALE,WHITE NON HISPANIC,DASHIELL,12,95 88 | 2011,MALE,WHITE NON HISPANIC,ELIYAHU,36,71 89 | 2011,MALE,WHITE NON HISPANIC,GRAYSON,19,88 90 | 2011,MALE,WHITE NON HISPANIC,JACKSON,77,42 91 | 2011,MALE,WHITE NON HISPANIC,KAI,15,92 92 | 2011,MALE,WHITE NON HISPANIC,LUKAS,16,91 93 | 2011,MALE,WHITE NON HISPANIC,MENACHEM,103,27 94 | 2011,MALE,WHITE NON HISPANIC,NICO,13,94 95 | 2011,MALE,WHITE NON HISPANIC,RAPHAEL,21,86 96 | 2011,MALE,WHITE NON HISPANIC,SHANE,21,86 97 | 2011,MALE,WHITE NON HISPANIC,SPENCER,15,92 98 | 2011,MALE,WHITE NON HISPANIC,YECHEZKEL,11,96 99 | 2011,FEMALE,WHITE NON HISPANIC,SURY,20,71 100 | 2011,FEMALE,WHITE NON HISPANIC,VIVIAN,30,61 101 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,AHMED,21,45 102 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,BENSON,19,47 103 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,EDISON,14,52 104 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,JACK,21,45 105 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,JULIAN,11,55 106 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,MILES,10,56 107 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,RYAN,150,3 108 | 2011,MALE,ASIAN AND PACIFIC ISLANDER,XAVIER,10,56 109 | 2011,MALE,BLACK NON HISPANIC,AMARE,15,59 110 | 2011,MALE,BLACK NON HISPANIC,CAMERON,47,29 111 | 2011,MALE,BLACK NON HISPANIC,DOMINIC,17,57 112 | 2011,MALE,BLACK NON HISPANIC,IBRAHIM,25,49 113 | 2011,MALE,BLACK NON HISPANIC,JASON,44,31 114 | 2011,MALE,BLACK NON HISPANIC,KAI,14,60 115 | 2011,MALE,BLACK NON HISPANIC,MAKAI,10,64 116 | 2011,MALE,BLACK NON HISPANIC,NANA,10,64 117 | 2011,MALE,BLACK NON HISPANIC,RYAN,41,33 118 | 2011,MALE,BLACK NON HISPANIC,ZACHARY,22,52 119 | 2011,MALE,HISPANIC,ALEX,68,47 120 | 2011,MALE,HISPANIC,ARTURO,10,94 121 | 2011,MALE,HISPANIC,CESAR,29,75 122 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ANGELINA,26,22 123 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,CHLOE,106,2 124 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,ESHAL,11,37 125 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,JENNY,27,21 126 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,LAUREN,13,35 127 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,PHOEBE,18,30 128 | 2011,FEMALE,ASIAN AND PACIFIC ISLANDER,SYEDA,16,32 129 | 2011,FEMALE,BLACK NON HISPANIC,AICHA,14,41 130 | 2011,FEMALE,BLACK NON HISPANIC,ANGELINA,11,44 131 | 2011,FEMALE,BLACK NON HISPANIC,DAKOTA,15,40 132 | 2011,FEMALE,BLACK NON HISPANIC,GRACE,15,40 133 | 2011,FEMALE,BLACK NON HISPANIC,KAITLYN,21,34 134 | 2011,FEMALE,BLACK NON HISPANIC,LONDYN,25,30 135 | 2011,FEMALE,BLACK NON HISPANIC,NADIA,16,39 136 | 2011,FEMALE,BLACK NON HISPANIC,SAMARA,11,44 137 | 2011,FEMALE,BLACK NON HISPANIC,SORAYA,10,45 138 | 2011,FEMALE,BLACK NON HISPANIC,ZURI,10,45 139 | 2011,FEMALE,HISPANIC,ALLYSON,19,69 140 | 2011,FEMALE,HISPANIC,ANNA,13,75 141 | 2011,FEMALE,HISPANIC,BRIANA,32,56 142 | 2011,FEMALE,HISPANIC,CRYSTAL,26,62 143 | 2011,FEMALE,HISPANIC,ELISA,17,71 144 | 2011,FEMALE,HISPANIC,GIA,21,67 145 | 2011,FEMALE,HISPANIC,ISABELA,10,78 146 | 2011,FEMALE,HISPANIC,JAYLIN,13,75 147 | 2011,FEMALE,HISPANIC,KAITLYN,31,57 148 | 2011,FEMALE,HISPANIC,KIMBERLY,103,13 149 | 2011,FEMALE,HISPANIC,LINDSAY,12,76 150 | 2011,FEMALE,HISPANIC,MELODY,23,65 151 | 2011,FEMALE,HISPANIC,NATHALY,20,68 152 | 2011,FEMALE,HISPANIC,SABRINA,25,63 153 | 2011,FEMALE,HISPANIC,SOFIA,165,6 154 | 2011,FEMALE,HISPANIC,XIMENA,18,70 155 | 2011,FEMALE,WHITE NON HISPANIC,ALINA,11,80 156 | 2011,FEMALE,WHITE NON HISPANIC,ARIELA,10,81 157 | 2011,FEMALE,WHITE NON HISPANIC,BLIMA,20,71 158 | 2011,FEMALE,WHITE NON HISPANIC,CHARLIE,14,77 159 | 2011,FEMALE,WHITE NON HISPANIC,DINA,41,50 160 | 2011,FEMALE,WHITE NON HISPANIC,ESTER,14,77 161 | 2011,FEMALE,WHITE NON HISPANIC,GEORGIA,26,65 162 | 2011,FEMALE,WHITE NON HISPANIC,IDY,10,81 163 | 2011,FEMALE,WHITE NON HISPANIC,JULIE,10,81 164 | 2011,FEMALE,WHITE NON HISPANIC,LEORA,12,79 165 | 2011,FEMALE,WHITE NON HISPANIC,MADELYN,15,76 166 | 2011,FEMALE,WHITE NON HISPANIC,MILENA,13,78 167 | 2011,FEMALE,WHITE NON HISPANIC,PEREL,15,76 168 | 2011,FEMALE,WHITE NON HISPANIC,RUTH,11,80 169 | 2011,MALE,HISPANIC,DANIEL,221,8 170 | 2011,MALE,HISPANIC,EDWIN,71,44 171 | 2011,MALE,HISPANIC,FERNANDO,28,76 172 | 2011,MALE,HISPANIC,ISAAC,73,42 173 | 2011,MALE,HISPANIC,JAY,18,86 174 | 2011,MALE,HISPANIC,JOHN,96,37 175 | 2011,MALE,HISPANIC,KAYDEN,25,79 176 | 2011,MALE,HISPANIC,MARCO,22,82 177 | 2011,MALE,HISPANIC,MIGUEL,101,34 178 | 2011,MALE,HISPANIC,PRINCE,14,90 179 | 2011,MALE,HISPANIC,SHANE,10,94 180 | 2011,MALE,HISPANIC,YAIR,10,94 181 | 2011,MALE,WHITE NON HISPANIC,ALEX,34,73 182 | 2011,MALE,WHITE NON HISPANIC,AVI,12,95 183 | 2011,MALE,WHITE NON HISPANIC,BRODY,19,88 184 | 2011,MALE,WHITE NON HISPANIC,COOPER,30,77 185 | 2011,MALE,WHITE NON HISPANIC,ELIJAH,35,72 186 | 2011,MALE,WHITE NON HISPANIC,GRAHAM,21,86 187 | 2011,MALE,WHITE NON HISPANIC,IVAN,13,94 188 | 2011,MALE,WHITE NON HISPANIC,JUSTIN,46,62 189 | 2011,MALE,WHITE NON HISPANIC,LUCAS,128,22 190 | 2011,MALE,WHITE NON HISPANIC,MEILECH,11,96 191 | 2011,MALE,WHITE NON HISPANIC,NATHANIEL,46,62 192 | 2011,MALE,WHITE NON HISPANIC,QUINN,13,94 193 | 2011,MALE,WHITE NON HISPANIC,SEBASTIAN,71,46 194 | 2011,MALE,WHITE NON HISPANIC,SIMON,47,61 195 | 2011,MALE,WHITE NON HISPANIC,YAKOV,60,54 196 | 2011,MALE,WHITE NON HISPANIC,ZEV,47,61 197 | 2012,FEMALE,ASIAN AND PACI,ANGELA,52,7 198 | 2012,FEMALE,ASIAN AND PACI,BONNIE,12,43 199 | 2012,FEMALE,ASIAN AND PACI,ELEANOR,15,40 200 | 2012,FEMALE,ASIAN AND PACI,HANNAH,46,13 201 | 2012,FEMALE,ASIAN AND PACI,JUDY,10,45 202 | 2012,FEMALE,ASIAN AND PACI,LYDIA,13,42 203 | 2012,FEMALE,ASIAN AND PACI,PHOEBE,22,33 204 | 2012,FEMALE,ASIAN AND PACI,STEPHANIE,19,36 205 | 2012,FEMALE,BLACK NON HISP,AALIYAH,55,10 206 | 2012,FEMALE,BLACK NON HISP,AMIRAH,27,31 207 | 2012,FEMALE,BLACK NON HISP,BRIANNA,32,26 208 | 2012,FEMALE,BLACK NON HISP,GABRIELLE,42,17 209 | 2012,FEMALE,BLACK NON HISP,JORDAN,10,47 210 | 2012,FEMALE,BLACK NON HISP,LONDON,87,2 211 | 2012,FEMALE,BLACK NON HISP,MILAN,23,34 212 | 2012,FEMALE,BLACK NON HISP,SANAA,22,35 213 | 2012,FEMALE,BLACK NON HISP,TAYLOR,71,3 214 | 2012,FEMALE,HISPANIC,ALEXA,66,28 215 | 2012,FEMALE,HISPANIC,AMY,68,26 216 | 2012,FEMALE,HISPANIC,ATHENA,17,72 217 | 2012,FEMALE,HISPANIC,CAROLINE,14,75 218 | 2012,FEMALE,HISPANIC,DULCE,10,79 219 | 2012,FEMALE,HISPANIC,FERNANDA,10,79 220 | 2012,FEMALE,HISPANIC,ISABELA,13,76 221 | 2012,FEMALE,HISPANIC,JAZZLYN,10,79 222 | 2012,FEMALE,HISPANIC,KASSANDRA,13,76 223 | 2012,FEMALE,HISPANIC,LEILA,27,62 224 | 2012,FEMALE,HISPANIC,MAKAYLA,21,68 225 | 2012,FEMALE,HISPANIC,MILEY,17,72 226 | 2012,FEMALE,HISPANIC,PAULA,10,79 227 | 2012,FEMALE,HISPANIC,SCARLET,20,69 228 | 2012,FEMALE,HISPANIC,VALERIE,53,37 229 | 2012,FEMALE,WHITE NON HISP,ADRIANNA,23,70 230 | 2012,FEMALE,WHITE NON HISP,AMINA,16,77 231 | 2012,FEMALE,WHITE NON HISP,AVA,165,8 232 | 2012,FEMALE,WHITE NON HISP,CAROLINE,48,45 233 | 2012,FEMALE,WHITE NON HISP,DAISY,11,82 234 | 2012,FEMALE,WHITE NON HISP,ELLE,20,73 235 | 2012,FEMALE,WHITE NON HISP,FAIGY,70,34 236 | 2012,FEMALE,WHITE NON HISP,GIULIA,12,81 237 | 2012,FEMALE,WHITE NON HISP,INES,10,83 238 | 2012,FEMALE,WHITE NON HISP,JUDY,13,80 239 | 2012,FEMALE,WHITE NON HISP,LANA,14,79 240 | 2012,FEMALE,WHITE NON HISP,LINA,12,81 241 | 2012,FEMALE,WHITE NON HISP,MARIA,44,49 242 | 2012,FEMALE,WHITE NON HISP,NAOMI,30,63 243 | 2012,FEMALE,WHITE NON HISP,PHOEBE,20,73 244 | 2012,FEMALE,WHITE NON HISP,ROSIE,10,83 245 | 2012,FEMALE,WHITE NON HISP,SHOSHANA,15,78 246 | 2012,FEMALE,WHITE NON HISP,TOBY,52,43 247 | 2012,FEMALE,WHITE NON HISP,YOCHEVED,32,61 248 | 2012,MALE,ASIAN AND PACI,ALLEN,35,38 249 | 2012,MALE,ASIAN AND PACI,BRUCE,10,61 250 | 2012,MALE,ASIAN AND PACI,EASON,30,41 251 | 2012,MALE,ASIAN AND PACI,IAN,43,30 252 | 2012,MALE,ASIAN AND PACI,JOHNNY,14,57 253 | 2012,MALE,ASIAN AND PACI,LOUIS,17,54 254 | 2012,MALE,ASIAN AND PACI,NICHOLAS,35,38 255 | 2012,MALE,ASIAN AND PACI,SHAYAN,10,61 256 | 2012,MALE,ASIAN AND PACI,ZACHARY,35,38 257 | 2012,MALE,BLACK NON HISP,AMIR,66,17 258 | 2012,MALE,BLACK NON HISP,CALEB,41,33 259 | 2012,MALE,BLACK NON HISP,DOMINIC,18,53 260 | 2012,MALE,BLACK NON HISP,JACOB,58,20 261 | 2012,MALE,BLACK NON HISP,JOSHUA,124,5 262 | 2012,MALE,BLACK NON HISP,KYLE,44,30 263 | 2012,MALE,BLACK NON HISP,MICAH,29,42 264 | 2012,MALE,BLACK NON HISP,PRESTON,12,59 265 | 2012,MALE,BLACK NON HISP,WILLIAM,22,49 266 | 2012,MALE,HISPANIC,ALDO,14,96 267 | 2012,MALE,HISPANIC,ASHTON,10,100 268 | 2012,MALE,HISPANIC,CARMELO,10,100 269 | 2012,MALE,HISPANIC,DENZEL,14,96 270 | 2012,MALE,HISPANIC,EMILIO,25,85 271 | 2012,MALE,HISPANIC,GAVIN,24,86 272 | 2012,MALE,HISPANIC,JACOB,293,2 273 | 2012,MALE,HISPANIC,JEFFREY,32,79 274 | 2012,MALE,HISPANIC,JUAN,96,36 275 | 2012,MALE,HISPANIC,LIAM,140,24 276 | 2012,MALE,HISPANIC,MATHEW,22,88 277 | 2012,MALE,HISPANIC,NICOLAS,61,55 278 | 2012,MALE,HISPANIC,ROBERTO,18,92 279 | 2012,MALE,HISPANIC,URIEL,13,97 280 | 2012,MALE,WHITE NON HISP,AHMED,34,70 281 | 2012,MALE,WHITE NON HISP,ARIEL,20,84 282 | 2012,MALE,WHITE NON HISP,BINYOMIN,11,93 283 | 2012,MALE,WHITE NON HISP,CHRIS,14,90 284 | 2012,MALE,WHITE NON HISP,EFRAIM,19,85 285 | 2012,MALE,WHITE NON HISP,EZRA,53,54 286 | 2012,MALE,WHITE NON HISP,HARRY,13,91 287 | 2012,MALE,WHITE NON HISP,JACKSON,85,35 288 | 2012,MALE,WHITE NON HISP,JULIAN,70,45 289 | 2012,MALE,WHITE NON HISP,LUKA,23,81 290 | 2012,MALE,WHITE NON HISP,MEIR,50,56 291 | 2012,MALE,WHITE NON HISP,NATHAN,76,40 292 | 2012,MALE,WHITE NON HISP,PINCHAS,13,91 293 | 2012,MALE,WHITE NON HISP,SAM,29,75 294 | 2012,MALE,WHITE NON HISP,SHOLOM,12,92 295 | 2012,MALE,WHITE NON HISP,WESLEY,12,92 296 | 2012,MALE,WHITE NON HISP,YUSUF,14,90 297 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Anaya,16,36 298 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Anya,15,37 299 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Arya,26,26 300 | 2013,FEMALE,HISPANIC,Esther,21,67 301 | 2014,MALE,BLACK NON HISPANIC,Caden,10,60 302 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Charlotte,21,31 303 | 2013,FEMALE,HISPANIC,Katelyn,19,69 304 | 2012,MALE,HISPANIC,MAXIMUS,26,84 305 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Emaan,10,42 306 | 2014,FEMALE,WHITE NON HISPANIC,Goldy,17,79 307 | 2013,MALE,WHITE NON HISPANIC,Yossi,18,92 308 | 2012,FEMALE,WHITE NON HISP,CHAYA,174,7 309 | 2013,FEMALE,BLACK NON HISPANIC,Shania,11,46 310 | 2012,MALE,ASIAN AND PACI,MICHAEL,37,36 311 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Karina,11,41 312 | 2014,MALE,WHITE NON HISPANIC,Berl,12,100 313 | 2012,FEMALE,WHITE NON HISP,AYLA,18,75 314 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Madelyn,14,38 315 | 2012,MALE,WHITE NON HISP,YONA,10,94 316 | 2012,MALE,HISPANIC,ELI,48,65 317 | 2012,FEMALE,WHITE NON HISP,MALKA,57,39 318 | 2012,FEMALE,WHITE NON HISP,SALMA,15,78 319 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Carter,11,56 320 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Sydney,13,39 321 | 2014,FEMALE,WHITE NON HISPANIC,Brianna,16,80 322 | 2013,MALE,HISPANIC,Pablo,21,84 323 | 2014,FEMALE,WHITE NON HISPANIC,Alessia,20,76 324 | 2012,FEMALE,ASIAN AND PACI,IVY,35,20 325 | 2013,FEMALE,BLACK NON HISPANIC,Amelia,22,35 326 | 2013,FEMALE,BLACK NON HISPANIC,Amiyah,18,39 327 | 2013,FEMALE,HISPANIC,Jazmin,26,62 328 | 2012,FEMALE,HISPANIC,SIENNA,16,73 329 | 2014,FEMALE,WHITE NON HISPANIC,Avigail,12,84 330 | 2014,FEMALE,WHITE NON HISPANIC,Devorah,47,49 331 | 2012,FEMALE,HISPANIC,ISABEL,40,49 332 | 2013,FEMALE,BLACK NON HISPANIC,Fatoumata,44,14 333 | 2013,FEMALE,BLACK NON HISPANIC,Giselle,11,46 334 | 2013,FEMALE,BLACK NON HISPANIC,Imani,22,35 335 | 2013,FEMALE,BLACK NON HISPANIC,Janiyah,14,43 336 | 2014,MALE,WHITE NON HISPANIC,Alex,27,85 337 | 2013,FEMALE,BLACK NON HISPANIC,Kyla,10,47 338 | 2013,FEMALE,BLACK NON HISPANIC,Lyric,14,43 339 | 2013,FEMALE,BLACK NON HISPANIC,Malia,28,29 340 | 2013,FEMALE,WHITE NON HISPANIC,Emerson,15,77 341 | 2013,FEMALE,WHITE NON HISPANIC,Blimy,34,58 342 | 2012,MALE,ASIAN AND PACI,OWEN,54,24 343 | 2013,FEMALE,BLACK NON HISPANIC,Oumou,27,30 344 | 2013,FEMALE,BLACK NON HISPANIC,Princess,11,46 345 | 2013,FEMALE,BLACK NON HISPANIC,Saniyah,11,46 346 | 2013,FEMALE,BLACK NON HISPANIC,Scarlett,11,46 347 | 2013,FEMALE,BLACK NON HISPANIC,Sydney,21,36 348 | 2013,FEMALE,WHITE NON HISPANIC,Sarah,182,5 349 | 2012,MALE,ASIAN AND PACI,ANGUS,16,55 350 | 2014,MALE,WHITE NON HISPANIC,Ellis,12,100 351 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Alina,41,10 352 | 2014,FEMALE,WHITE NON HISPANIC,Marielle,11,85 353 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Jennifer,17,33 354 | 2014,FEMALE,HISPANIC,Nicole,48,39 355 | 2012,FEMALE,BLACK NON HISP,ELISE,10,47 356 | 2014,FEMALE,WHITE NON HISPANIC,Atara,13,83 357 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Eleanor,12,38 358 | 2013,FEMALE,HISPANIC,Arielle,11,77 359 | 2013,FEMALE,HISPANIC,Ayleen,16,72 360 | 2013,FEMALE,HISPANIC,Britney,14,74 361 | 2013,FEMALE,HISPANIC,Chloe,54,35 362 | 2013,MALE,HISPANIC,Kaiden,33,72 363 | 2013,FEMALE,HISPANIC,Eileen,10,78 364 | 2013,MALE,WHITE NON HISPANIC,Aharon,19,91 365 | 2013,FEMALE,HISPANIC,Faith,23,65 366 | 2013,MALE,WHITE NON HISPANIC,Menachem,72,49 367 | 2012,MALE,BLACK NON HISP,LUCAS,15,56 368 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Carson,38,30 369 | 2012,MALE,ASIAN AND PACI,CARTER,10,61 370 | 2014,MALE,WHITE NON HISPANIC,Hershy,21,91 371 | 2013,MALE,HISPANIC,Zion,10,95 372 | 2012,FEMALE,WHITE NON HISP,MELANIE,10,83 373 | 2013,FEMALE,HISPANIC,Sherlyn,26,62 374 | 2012,MALE,HISPANIC,LEO,16,94 375 | 2013,FEMALE,HISPANIC,Karina,15,73 376 | 2013,FEMALE,HISPANIC,Kaylee,81,24 377 | 2013,FEMALE,HISPANIC,Kelsey,15,73 378 | 2013,FEMALE,HISPANIC,Nina,10,78 379 | 2013,MALE,WHITE NON HISPANIC,Elias,23,87 380 | 2013,FEMALE,HISPANIC,Liliana,29,59 381 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Aayan,13,54 382 | 2013,FEMALE,HISPANIC,Nyah,10,78 383 | 2013,FEMALE,HISPANIC,Jocelyn,37,51 384 | 2013,MALE,WHITE NON HISPANIC,Sam,36,74 385 | 2012,FEMALE,ASIAN AND PACI,OLIVIA,132,3 386 | 2013,FEMALE,HISPANIC,Nayeli,13,75 387 | 2013,FEMALE,HISPANIC,Paris,11,77 388 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Benson,23,45 389 | 2014,FEMALE,WHITE NON HISPANIC,Leila,14,82 390 | 2012,FEMALE,ASIAN AND PACI,ALEXIS,18,37 391 | 2013,FEMALE,HISPANIC,Summer,11,77 392 | 2013,FEMALE,HISPANIC,Valery,15,73 393 | 2012,FEMALE,WHITE NON HISP,FRANCES,13,80 394 | 2013,FEMALE,WHITE NON HISPANIC,Adina,17,75 395 | 2013,FEMALE,WHITE NON HISPANIC,Gia,21,71 396 | 2014,FEMALE,WHITE NON HISPANIC,Mackenzie,26,70 397 | 2012,MALE,HISPANIC,GABRIEL,132,26 398 | 2013,FEMALE,WHITE NON HISPANIC,Anastasia,33,59 399 | 2013,FEMALE,WHITE NON HISPANIC,Ariana,34,58 400 | 2012,MALE,ASIAN AND PACI,IBRAHIM,29,42 401 | 2012,FEMALE,ASIAN AND PACI,MAGGIE,16,39 402 | 2012,MALE,ASIAN AND PACI,MILES,18,53 403 | 2014,FEMALE,WHITE NON HISPANIC,Lila,42,54 404 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Hailey,23,27 405 | 2013,FEMALE,HISPANIC,Savannah,46,42 406 | 2013,FEMALE,BLACK NON HISPANIC,Alana,16,41 407 | 2013,FEMALE,WHITE NON HISPANIC,Talia,41,52 408 | 2013,FEMALE,WHITE NON HISPANIC,Edith,12,80 409 | 2012,FEMALE,WHITE NON HISP,TZIVIA,11,82 410 | 2013,FEMALE,WHITE NON HISPANIC,Emma,228,2 411 | 2012,FEMALE,WHITE NON HISP,EMMA,228,1 412 | 2013,FEMALE,WHITE NON HISPANIC,Evelyn,56,39 413 | 2013,FEMALE,WHITE NON HISPANIC,Farah,11,81 414 | 2012,MALE,HISPANIC,AVERY,12,98 415 | 2013,FEMALE,WHITE NON HISPANIC,Gemma,18,74 416 | 2014,FEMALE,WHITE NON HISPANIC,Aziza,11,85 417 | 2012,MALE,WHITE NON HISP,KYLE,15,89 418 | 2013,FEMALE,WHITE NON HISPANIC,Iris,16,76 419 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Benjamin,48,24 420 | 2014,MALE,HISPANIC,Lukas,18,90 421 | 2012,FEMALE,ASIAN AND PACI,JESSICA,40,17 422 | 2012,FEMALE,WHITE NON HISP,NICOLETTE,14,79 423 | 2013,FEMALE,WHITE NON HISPANIC,Leah,179,6 424 | 2013,FEMALE,WHITE NON HISPANIC,Liliana,21,71 425 | 2013,FEMALE,WHITE NON HISPANIC,Luciana,11,81 426 | 2014,FEMALE,WHITE NON HISPANIC,Abigail,124,13 427 | 2013,FEMALE,WHITE NON HISPANIC,Maeve,24,68 428 | 2014,FEMALE,HISPANIC,Tiffany,20,63 429 | 2013,FEMALE,WHITE NON HISPANIC,Maryam,11,81 430 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Amelia,32,20 431 | 2012,MALE,WHITE NON HISP,BRODY,15,89 432 | 2013,FEMALE,HISPANIC,Alexandra,30,58 433 | 2014,MALE,WHITE NON HISPANIC,Landon,23,89 434 | 2012,MALE,BLACK NON HISP,JACE,20,51 435 | 2012,MALE,BLACK NON HISP,MARCUS,29,42 436 | 2014,MALE,WHITE NON HISPANIC,Emmett,12,100 437 | 2013,FEMALE,WHITE NON HISPANIC,Lia,28,64 438 | 2013,FEMALE,WHITE NON HISPANIC,Riley,41,52 439 | 2013,MALE,HISPANIC,Benjamin,61,52 440 | 2013,FEMALE,BLACK NON HISPANIC,Grace,32,25 441 | 2012,FEMALE,BLACK NON HISP,MYA,21,36 442 | 2012,FEMALE,WHITE NON HISP,CLAIRE,45,48 443 | 2014,MALE,HISPANIC,Moises,10,98 444 | 2014,FEMALE,HISPANIC,Annabella,15,68 445 | 2012,MALE,BLACK NON HISP,AMARI,44,30 446 | 2014,MALE,HISPANIC,Noel,31,77 447 | 2013,FEMALE,WHITE NON HISPANIC,Tzipora,16,76 448 | 2013,FEMALE,WHITE NON HISPANIC,Vivian,46,47 449 | 2013,MALE,WHITE NON HISPANIC,Baruch,15,95 450 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Abdullah,18,49 451 | 2014,MALE,HISPANIC,Raymond,17,91 452 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Alvin,38,30 453 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Ayan,25,42 454 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Brian,49,21 455 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Kyle,37,31 456 | 2012,MALE,WHITE NON HISP,JACE,10,94 457 | 2013,MALE,WHITE NON HISPANIC,Ethan,125,23 458 | 2012,FEMALE,WHITE NON HISP,CHAVA,38,55 459 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Jimmy,12,55 460 | 2014,MALE,BLACK NON HISPANIC,Logan,70,16 461 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Ibrahim,25,42 462 | 2012,FEMALE,HISPANIC,MELODY,30,59 463 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Jimmy,12,55 464 | 2014,FEMALE,HISPANIC,Jasmine,34,49 465 | 2012,MALE,HISPANIC,EMMANUEL,83,42 466 | 2013,FEMALE,WHITE NON HISPANIC,Adrianna,11,81 467 | 2013,FEMALE,HISPANIC,Taylor,17,71 468 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Miles,11,56 469 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Owen,54,18 470 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Ricky,13,54 471 | 2013,FEMALE,WHITE NON HISPANIC,Bridget,12,80 472 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Tristan,14,53 473 | 2014,MALE,WHITE NON HISPANIC,Maxim,26,86 474 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Vincent,47,23 475 | 2012,MALE,WHITE NON HISP,SHANE,24,80 476 | 2013,MALE,BLACK NON HISPANIC,Aidan,31,42 477 | 2014,FEMALE,HISPANIC,Penelope,58,34 478 | 2013,MALE,BLACK NON HISPANIC,Austin,28,44 479 | 2013,MALE,BLACK NON HISPANIC,Blake,25,47 480 | 2014,FEMALE,HISPANIC,Julissa,15,68 481 | 2013,MALE,HISPANIC,Andres,29,76 482 | 2012,FEMALE,WHITE NON HISP,LILIANA,24,69 483 | 2013,MALE,WHITE NON HISPANIC,Gabriel,116,28 484 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Tenzin,31,21 485 | 2014,FEMALE,HISPANIC,Aliyah,19,64 486 | 2013,FEMALE,HISPANIC,Nathalie,20,68 487 | 2012,FEMALE,BLACK NON HISP,AMINATA,20,37 488 | 2014,FEMALE,BLACK NON HISPANIC,Autumn,41,12 489 | 2013,MALE,BLACK NON HISPANIC,Jeremy,16,56 490 | 2014,FEMALE,WHITE NON HISPANIC,Elizabeth,101,19 491 | 2012,MALE,HISPANIC,ERICK,76,45 492 | 2014,FEMALE,WHITE NON HISPANIC,Isabel,30,66 493 | 2012,FEMALE,WHITE NON HISP,LANA,14,79 494 | 2014,FEMALE,WHITE NON HISPANIC,Kayla,42,54 495 | 2012,MALE,BLACK NON HISP,ALI,14,57 496 | 2013,MALE,BLACK NON HISPANIC,Matthew,56,21 497 | 2013,FEMALE,WHITE NON HISPANIC,Adelaide,12,80 498 | 2013,MALE,BLACK NON HISPANIC,Nathan,53,24 499 | 2013,MALE,BLACK NON HISPANIC,Joel,20,52 500 | 2013,MALE,BLACK NON HISPANIC,Patrick,10,62 501 | 2012,MALE,HISPANIC,SAMUEL,83,42 502 | 2012,FEMALE,HISPANIC,BRITNEY,15,74 503 | 2013,MALE,BLACK NON HISPANIC,Travis,13,59 504 | 2013,MALE,HISPANIC,Alex,52,57 505 | 2012,FEMALE,HISPANIC,JAZZLYN,10,79 506 | 2013,MALE,HISPANIC,Aiden,182,14 507 | 2013,MALE,HISPANIC,Allan,11,94 508 | 2014,FEMALE,HISPANIC,Valeria,34,49 509 | 2013,MALE,HISPANIC,Thiago,38,67 510 | 2013,MALE,HISPANIC,Brayden,10,95 511 | 2014,FEMALE,HISPANIC,Riley,15,68 512 | 2014,FEMALE,WHITE NON HISPANIC,Sadie,43,53 513 | 2013,MALE,HISPANIC,Cristian,47,61 514 | 2013,MALE,HISPANIC,Dariel,29,76 515 | 2013,MALE,HISPANIC,Devin,24,81 516 | 2013,MALE,HISPANIC,Edison,15,90 517 | 2013,MALE,HISPANIC,Eli,46,62 518 | 2014,MALE,WHITE NON HISPANIC,Rocco,15,97 519 | 2012,FEMALE,WHITE NON HISP,KAYLEE,10,83 520 | 2012,FEMALE,HISPANIC,HAZEL,15,74 521 | 2014,FEMALE,WHITE NON HISPANIC,Ruchel,10,86 522 | 2013,MALE,HISPANIC,Hunter,18,87 523 | 2013,MALE,HISPANIC,Felix,20,85 524 | 2013,MALE,HISPANIC,Jaiden,20,85 525 | 2012,MALE,WHITE NON HISP,LEVI,58,51 526 | 2013,MALE,HISPANIC,Jefferson,13,92 527 | 2014,MALE,WHITE NON HISPANIC,Shane,19,93 528 | 2012,FEMALE,ASIAN AND PACI,CRYSTAL,17,38 529 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Stephanie,13,39 530 | 2014,FEMALE,HISPANIC,Isabel,42,44 531 | 2013,MALE,HISPANIC,Kelvin,17,88 532 | 2013,MALE,HISPANIC,Kevin,139,18 533 | 2013,MALE,HISPANIC,Leonardo,51,58 534 | 2012,FEMALE,WHITE NON HISP,SELMA,10,83 535 | 2012,FEMALE,HISPANIC,CATALINA,25,64 536 | 2012,MALE,WHITE NON HISP,ANTHONY,133,19 537 | 2014,FEMALE,HISPANIC,Angie,19,64 538 | 2014,FEMALE,BLACK NON HISPANIC,Brianna,24,27 539 | 2013,MALE,HISPANIC,Milan,14,91 540 | 2014,FEMALE,HISPANIC,Natalie,28,55 541 | 2013,MALE,HISPANIC,Paul,12,93 542 | 2013,MALE,HISPANIC,Richard,22,83 543 | 2012,MALE,ASIAN AND PACI,KRISH,15,56 544 | 2012,FEMALE,HISPANIC,JAYLENE,24,65 545 | 2014,MALE,WHITE NON HISPANIC,Myles,10,102 546 | 2014,FEMALE,BLACK NON HISPANIC,Amara,12,39 547 | 2013,MALE,WHITE NON HISPANIC,Adrian,40,70 548 | 2013,MALE,WHITE NON HISPANIC,Thomas,107,31 549 | 2014,FEMALE,BLACK NON HISPANIC,Sanai,15,36 550 | 2013,MALE,WHITE NON HISPANIC,Aron,78,44 551 | 2013,MALE,WHITE NON HISPANIC,Baruch,15,95 552 | 2013,MALE,WHITE NON HISPANIC,Bradley,10,100 553 | 2012,FEMALE,WHITE NON HISP,DIANA,20,73 554 | 2013,MALE,WHITE NON HISPANIC,Christian,50,62 555 | 2012,MALE,WHITE NON HISP,YOEL,23,81 556 | 2014,FEMALE,WHITE NON HISPANIC,Emmeline,11,85 557 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Gordon,13,54 558 | 2013,MALE,WHITE NON HISPANIC,Elimelech,18,92 559 | 2014,MALE,HISPANIC,Ashton,13,95 560 | 2013,MALE,WHITE NON HISPANIC,Francesco,12,98 561 | 2013,MALE,WHITE NON HISPANIC,Gavin,54,60 562 | 2013,FEMALE,WHITE NON HISPANIC,Suri,13,79 563 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Vihaan,17,50 564 | 2014,FEMALE,WHITE NON HISPANIC,Sarah,185,7 565 | 2013,MALE,WHITE NON HISPANIC,Ilan,12,98 566 | 2013,FEMALE,BLACK NON HISPANIC,Brianna,24,33 567 | 2013,FEMALE,WHITE NON HISPANIC,Grace,65,33 568 | 2013,MALE,WHITE NON HISPANIC,Joel,18,92 569 | 2012,MALE,WHITE NON HISP,MARCO,30,74 570 | 2014,FEMALE,WHITE NON HISPANIC,Suri,13,83 571 | 2014,MALE,HISPANIC,Max,13,95 572 | 2012,FEMALE,ASIAN AND PACI,JUDY,10,45 573 | 2013,MALE,WHITE NON HISPANIC,Lorenzo,25,85 574 | 2013,MALE,WHITE NON HISPANIC,Lucas,125,23 575 | 2012,FEMALE,ASIAN AND PACI,EVA,28,27 576 | 2013,MALE,HISPANIC,Yadiel,20,85 577 | 2013,FEMALE,HISPANIC,Arianna,76,27 578 | 2014,MALE,WHITE NON HISPANIC,Dov,38,74 579 | 2013,MALE,WHITE NON HISPANIC,Muhammad,21,89 580 | 2013,MALE,WHITE NON HISPANIC,Nicolas,28,82 581 | 2013,MALE,WHITE NON HISPANIC,Oliver,121,26 582 | 2013,MALE,WHITE NON HISPANIC,Peter,65,53 583 | 2012,FEMALE,HISPANIC,DANIELLA,21,68 584 | 2012,FEMALE,HISPANIC,SUMMER,13,76 585 | 2012,FEMALE,HISPANIC,CATHERINE,15,74 586 | 2012,MALE,ASIAN AND PACI,ELIJAH,19,52 587 | 2012,FEMALE,HISPANIC,JASMINE,30,59 588 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Evan,76,11 589 | 2013,MALE,WHITE NON HISPANIC,Stephen,13,97 590 | 2014,MALE,WHITE NON HISPANIC,Aiden,78,45 591 | 2013,MALE,WHITE NON HISPANIC,Usher,25,85 592 | 2013,MALE,BLACK NON HISPANIC,Karter,17,55 593 | 2012,FEMALE,BLACK NON HISP,SANAI,19,38 594 | 2014,FEMALE,HISPANIC,Ivanna,11,72 595 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Abby,27,23 596 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Scarlett,19,31 597 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Amanda,15,35 598 | 2012,MALE,WHITE NON HISP,JACK,180,11 599 | 2012,MALE,WHITE NON HISP,DREW,15,89 600 | 2012,FEMALE,WHITE NON HISP,SARA,133,14 601 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Victoria,36,16 602 | 2014,FEMALE,WHITE NON HISPANIC,Kira,13,83 603 | 2012,MALE,WHITE NON HISP,MILES,53,54 604 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Joseph,18,50 605 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Eshaal,12,38 606 | 2014,MALE,WHITE NON HISPANIC,Samir,13,99 607 | 2012,MALE,WHITE NON HISP,SEBASTIAN,74,42 608 | 2012,FEMALE,BLACK NON HISP,SARAH,29,29 609 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Jenny,20,30 610 | 2014,FEMALE,WHITE NON HISPANIC,Lilah,11,85 611 | 2012,FEMALE,HISPANIC,JULIA,20,69 612 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Kylie,14,36 613 | 2013,MALE,BLACK NON HISPANIC,Ibrahima,16,56 614 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Maryam,20,30 615 | 2012,FEMALE,ASIAN AND PACI,ARIEL,12,43 616 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Samantha,18,32 617 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Elaine,17,35 618 | 2012,FEMALE,WHITE NON HISP,MIRIAM,150,10 619 | 2013,MALE,WHITE NON HISPANIC,Felix,22,88 620 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Zoe,34,16 621 | 2013,FEMALE,HISPANIC,Carla,14,74 622 | 2014,FEMALE,BLACK NON HISPANIC,Ella,11,40 623 | 2012,MALE,WHITE NON HISP,BENNETT,10,94 624 | 2014,FEMALE,BLACK NON HISPANIC,Ariana,24,27 625 | 2013,MALE,WHITE NON HISPANIC,Mohammed,30,80 626 | 2013,MALE,BLACK NON HISPANIC,Ibrahim,34,39 627 | 2012,FEMALE,WHITE NON HISP,SHIRA,16,77 628 | 2014,MALE,HISPANIC,Gael,38,70 629 | 2013,FEMALE,HISPANIC,Briana,21,67 630 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Calvin,10,58 631 | 2013,FEMALE,BLACK NON HISPANIC,Zoey,36,21 632 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Erin,16,36 633 | 2012,MALE,HISPANIC,LORENZO,15,95 634 | 2014,FEMALE,BLACK NON HISPANIC,Kristen,10,41 635 | 2014,FEMALE,BLACK NON HISPANIC,Lauren,37,15 636 | 2014,FEMALE,BLACK NON HISPANIC,Londyn,26,25 637 | 2012,MALE,WHITE NON HISP,CHAIM,169,12 638 | 2014,FEMALE,BLACK NON HISPANIC,Melanie,11,40 639 | 2013,MALE,WHITE NON HISPANIC,Alan,25,85 640 | 2014,FEMALE,BLACK NON HISPANIC,Nylah,34,17 641 | 2013,MALE,BLACK NON HISPANIC,Amir,102,7 642 | 2014,FEMALE,BLACK NON HISPANIC,Saniyah,10,41 643 | 2012,FEMALE,ASIAN AND PACI,STELLA,33,22 644 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Aaron,63,14 645 | 2014,FEMALE,BLACK NON HISPANIC,Taylor,45,11 646 | 2012,MALE,HISPANIC,YARIEL,10,100 647 | 2013,MALE,ASIAN AND PACIFIC ISLANDER,Nathan,48,22 648 | 2012,FEMALE,WHITE NON HISP,REBECCA,45,48 649 | 2014,FEMALE,WHITE NON HISPANIC,Michaela,13,83 650 | 2014,FEMALE,HISPANIC,Alice,19,64 651 | 2012,FEMALE,WHITE NON HISP,MINDY,30,63 652 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Kimi,24,44 653 | 2013,MALE,BLACK NON HISPANIC,Kai,20,52 654 | 2014,FEMALE,HISPANIC,April,12,71 655 | 2012,FEMALE,WHITE NON HISP,TZIPORA,13,80 656 | 2013,MALE,HISPANIC,Logan,80,38 657 | 2014,FEMALE,HISPANIC,Aylin,45,41 658 | 2014,FEMALE,HISPANIC,Bryanna,12,71 659 | 2013,FEMALE,HISPANIC,Amaya,30,58 660 | 2014,FEMALE,HISPANIC,Danna,31,52 661 | 2014,MALE,BLACK NON HISPANIC,Ibrahim,22,48 662 | 2014,FEMALE,HISPANIC,Emilia,21,62 663 | 2014,FEMALE,HISPANIC,Evangeline,11,72 664 | 2014,FEMALE,HISPANIC,Giselle,32,51 665 | 2013,FEMALE,HISPANIC,Avery,21,67 666 | 2014,MALE,HISPANIC,Ezekiel,10,98 667 | 2014,FEMALE,HISPANIC,Jasmine,34,49 668 | 2014,FEMALE,HISPANIC,Jazlyn,22,61 669 | 2014,FEMALE,BLACK NON HISPANIC,Kaylee,24,27 670 | 2013,FEMALE,WHITE NON HISPANIC,Renee,10,82 671 | 2013,FEMALE,ASIAN AND PACIFIC ISLANDER,Arya,26,26 672 | 2012,MALE,HISPANIC,ROMEO,15,95 673 | 2014,FEMALE,HISPANIC,Katie,12,71 674 | 2014,FEMALE,HISPANIC,Keyla,15,68 675 | 2014,FEMALE,HISPANIC,Gabriela,45,41 676 | 2014,FEMALE,BLACK NON HISPANIC,Sarai,15,36 677 | 2014,FEMALE,WHITE NON HISPANIC,Aria,45,51 678 | 2014,FEMALE,HISPANIC,Lucia,19,64 679 | 2014,FEMALE,HISPANIC,Makayla,16,67 680 | 2014,FEMALE,HISPANIC,Maya,45,41 681 | 2012,FEMALE,WHITE NON HISP,GEORGIA,16,77 682 | 2014,FEMALE,ASIAN AND PACIFIC ISLANDER,Ayesha,28,22 683 | 2012,FEMALE,ASIAN AND PACI,MANHA,15,40 684 | 2014,MALE,HISPANIC,Ayden,91,36 685 | 2014,MALE,WHITE NON HISPANIC,Frank,14,98 686 | 2014,MALE,HISPANIC,Isaiah,47,61 687 | 2014,FEMALE,HISPANIC,Skylar,25,58 688 | 2014,MALE,WHITE NON HISPANIC,Menashe,12,100 689 | 2012,MALE,BLACK NON HISP,ABDUL,15,56 690 | 2014,FEMALE,WHITE NON HISPANIC,Ruby,48,48 691 | 2012,MALE,WHITE NON HISP,VINCENT,41,64 692 | 2013,MALE,HISPANIC,Maximilian,12,93 693 | 2014,FEMALE,WHITE NON HISPANIC,Adriana,23,73 694 | 2013,MALE,HISPANIC,William,66,48 695 | 2014,FEMALE,WHITE NON HISPANIC,Ana,11,85 696 | 2012,FEMALE,BLACK NON HISP,SHAYLA,10,47 697 | 2014,FEMALE,WHITE NON HISPANIC,Athena,10,86 698 | 2014,FEMALE,WHITE NON HISPANIC,Aviva,14,82 699 | 2014,FEMALE,WHITE NON HISPANIC,Beatrice,44,52 700 | 2014,FEMALE,WHITE NON HISPANIC,Brielle,12,84 701 | 2014,FEMALE,WHITE NON HISPANIC,Charlie,20,76 702 | 2014,FEMALE,WHITE NON HISPANIC,Dina,28,68 703 | 2014,FEMALE,WHITE NON HISPANIC,Erin,10,86 704 | 2014,FEMALE,WHITE NON HISPANIC,Francesca,39,57 705 | 2014,FEMALE,WHITE NON HISPANIC,Hailey,22,74 706 | 2014,FEMALE,WHITE NON HISPANIC,Jana,10,86 707 | 2014,FEMALE,WHITE NON HISPANIC,Kaylee,15,81 708 | 2014,FEMALE,WHITE NON HISPANIC,Liba,23,73 709 | 2014,FEMALE,WHITE NON HISPANIC,Mackenzie,26,70 710 | 2014,FEMALE,WHITE NON HISPANIC,Melanie,13,83 711 | 2014,FEMALE,WHITE NON HISPANIC,Nechama,30,66 712 | 2014,FEMALE,WHITE NON HISPANIC,Poppy,14,82 713 | 2014,FEMALE,WHITE NON HISPANIC,Ruchy,17,79 714 | 2014,FEMALE,WHITE NON HISPANIC,Siena,16,80 715 | 2014,FEMALE,WHITE NON HISPANIC,Tessa,13,83 716 | 2014,FEMALE,WHITE NON HISPANIC,Yocheved,28,68 717 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Alfred,18,50 718 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Azaan,10,58 719 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Daniel,87,7 720 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Gabriel,17,51 721 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Jake,14,54 722 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Kevin,62,15 723 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Michael,44,27 724 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Ricky,10,58 725 | 2014,MALE,ASIAN AND PACIFIC ISLANDER,Tony,11,57 726 | 2014,MALE,BLACK NON HISPANIC,Adonis,11,59 727 | 2014,MALE,BLACK NON HISPANIC,Ayden,59,21 728 | 2014,MALE,BLACK NON HISPANIC,Christian,76,14 729 | 2014,MALE,BLACK NON HISPANIC,Greyson,11,59 730 | 2014,MALE,BLACK NON HISPANIC,Jayson,11,59 731 | 2014,MALE,BLACK NON HISPANIC,Karter,35,37 732 | 2014,MALE,BLACK NON HISPANIC,Mamadou,23,47 733 | 2014,MALE,BLACK NON HISPANIC,Nolan,16,54 734 | 2014,MALE,BLACK NON HISPANIC,Steven,10,60 735 | 2014,MALE,HISPANIC,Adam,51,59 736 | 2014,MALE,HISPANIC,Andy,33,75 737 | 2014,MALE,HISPANIC,Bryce,13,95 738 | 2014,MALE,HISPANIC,Dean,12,96 739 | 2014,MALE,HISPANIC,Emiliano,11,97 740 | 2014,MALE,HISPANIC,Gavin,18,90 741 | 2014,MALE,HISPANIC,Jacob,292,2 742 | 2014,MALE,HISPANIC,Jeffrey,18,90 743 | 2014,MALE,HISPANIC,Julian,106,30 744 | 2014,MALE,HISPANIC,Leonardo,61,53 745 | 2014,MALE,HISPANIC,Marlon,11,97 746 | 2014,MALE,HISPANIC,Milan,32,76 747 | 2014,MALE,HISPANIC,Peter,17,91 748 | 2014,MALE,HISPANIC,Skyler,11,97 749 | 2014,MALE,HISPANIC,Zachary,27,81 750 | 2014,MALE,WHITE NON HISPANIC,Anthony,110,29 751 | 2014,MALE,WHITE NON HISPANIC,Aydin,11,101 752 | 2014,MALE,WHITE NON HISPANIC,Caleb,28,84 753 | 2014,MALE,WHITE NON HISPANIC,Dashiell,10,102 754 | 2014,MALE,WHITE NON HISPANIC,Elijah,28,84 755 | 2014,MALE,WHITE NON HISPANIC,Gabriel,105,32 756 | 2014,MALE,WHITE NON HISPANIC,Hillel,12,100 757 | 2014,MALE,WHITE NON HISPANIC,Jax,11,101 758 | 2014,MALE,WHITE NON HISPANIC,Kevin,17,95 759 | 2014,MALE,WHITE NON HISPANIC,Luka,24,88 760 | 2014,MALE,WHITE NON HISPANIC,Meir,66,51 761 | 2014,MALE,WHITE NON HISPANIC,Naftali,22,90 762 | 2014,MALE,WHITE NON HISPANIC,Philip,28,84 763 | 2014,MALE,WHITE NON HISPANIC,Rowan,15,97 764 | 2014,MALE,WHITE NON HISPANIC,Shlomo,55,59 765 | 2014,MALE,WHITE NON HISPANIC,Usher,20,92 766 | 2014,MALE,WHITE NON HISPANIC,Yisroel,83,43 767 | 2015,MALE,BLACK NON HISPANIC,Major,15,58 768 | 2015,FEMALE,ASIAN AND PACIFIC ISLANDER,Riley,10,43 769 | 2016,FEMALE,HISPANIC,Ximena,30,54 770 | 2015,MALE,HISPANIC,Nelson,12,97 771 | 2016,MALE,WHITE NON HISPANIC,Ryan,129,21 772 | 2016,MALE,BLACK NON HISPANIC,Kyree,10,57 773 | 2016,FEMALE,HISPANIC,Olivia,108,13 774 | 2016,MALE,WHITE NON HISPANIC,Dominick,13,96 775 | 2016,MALE,ASIAN AND PACIFIC ISLANDER,Kyle,44,27 776 | 2015,FEMALE,WHITE NON HISPANIC,Michelle,25,68 777 | 2015,FEMALE,HISPANIC,Ella,26,65 778 | 2016,MALE,WHITE NON HISPANIC,Walter,11,98 779 | 2015,FEMALE,HISPANIC,Victoria,118,11 780 | 2015,MALE,HISPANIC,Edward,38,71 781 | 2016,MALE,BLACK NON HISPANIC,Mason,91,9 782 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Livia,10,41 783 | 2015,FEMALE,WHITE NON HISPANIC,Michaela,11,82 784 | 2015,FEMALE,ASIAN AND PACIFIC ISLANDER,Lucy,13,40 785 | 2016,FEMALE,HISPANIC,Chelsea,27,57 786 | 2016,MALE,WHITE NON HISPANIC,Jeremy,11,98 787 | 2016,MALE,WHITE NON HISPANIC,Walker,10,99 788 | 2015,FEMALE,WHITE NON HISPANIC,Noelle,13,80 789 | 2016,FEMALE,WHITE NON HISPANIC,Alexis,22,72 790 | 2016,FEMALE,WHITE NON HISPANIC,Arianna,16,78 791 | 2015,MALE,HISPANIC,Dariel,28,81 792 | 2016,MALE,ASIAN AND PACIFIC ISLANDER,Rehan,13,56 793 | 2016,FEMALE,HISPANIC,Celine,13,71 794 | 2016,FEMALE,WHITE NON HISPANIC,Layla,58,40 795 | 2015,MALE,BLACK NON HISPANIC,Christian,65,16 796 | 2016,MALE,WHITE NON HISPANIC,Josef,11,98 797 | 2015,MALE,BLACK NON HISPANIC,Ibrahima,11,62 798 | 2016,FEMALE,HISPANIC,Maia,21,63 799 | 2015,MALE,BLACK NON HISPANIC,Ibrahim,18,55 800 | 2016,FEMALE,WHITE NON HISPANIC,Sonia,10,84 801 | 2016,MALE,WHITE NON HISPANIC,Greyson,13,96 802 | 2016,FEMALE,WHITE NON HISPANIC,Zara,11,83 803 | 2015,FEMALE,HISPANIC,Ariana,93,17 804 | 2016,FEMALE,HISPANIC,Alicia,15,69 805 | 2016,FEMALE,HISPANIC,Alexis,11,73 806 | 2015,FEMALE,WHITE NON HISPANIC,Brielle,13,80 807 | 2016,MALE,HISPANIC,Josue,32,70 808 | 2016,MALE,HISPANIC,Omar,12,89 809 | 2015,MALE,HISPANIC,Abel,21,88 810 | 2016,MALE,WHITE NON HISPANIC,Maximilian,24,85 811 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Allison,20,31 812 | 2016,FEMALE,WHITE NON HISPANIC,Gemma,27,67 813 | 2015,MALE,BLACK NON HISPANIC,Khalil,14,59 814 | 2015,MALE,WHITE NON HISPANIC,Leo,117,26 815 | 2015,FEMALE,HISPANIC,Annalise,14,77 816 | 2016,MALE,HISPANIC,Jeffrey,16,85 817 | 2015,MALE,ASIAN AND PACIFIC ISLANDER,Ayan,17,48 818 | 2015,FEMALE,WHITE NON HISPANIC,Kiera,13,80 819 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Carina,12,39 820 | 2015,FEMALE,BLACK NON HISPANIC,Hailey,26,24 821 | 2015,FEMALE,BLACK NON HISPANIC,Danielle,11,39 822 | 2016,MALE,ASIAN AND PACIFIC ISLANDER,Xavier,19,50 823 | 2015,FEMALE,ASIAN AND PACIFIC ISLANDER,Leela,14,39 824 | 2016,FEMALE,HISPANIC,Leah,116,10 825 | 2015,MALE,HISPANIC,Logan,133,21 826 | 2016,FEMALE,WHITE NON HISPANIC,Camille,12,82 827 | 2015,MALE,WHITE NON HISPANIC,Richard,20,84 828 | 2015,MALE,HISPANIC,Damien,13,96 829 | 2016,FEMALE,WHITE NON HISPANIC,Zelda,10,84 830 | 2015,FEMALE,WHITE NON HISPANIC,Sophie,58,37 831 | 2016,MALE,WHITE NON HISPANIC,Zain,11,98 832 | 2015,MALE,ASIAN AND PACIFIC ISLANDER,Ian,29,37 833 | 2015,FEMALE,HISPANIC,Riley,36,56 834 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Mila,17,34 835 | 2015,MALE,HISPANIC,Saul,15,94 836 | 2016,MALE,BLACK NON HISPANIC,Ayden,47,23 837 | 2015,FEMALE,BLACK NON HISPANIC,Aminata,29,21 838 | 2016,MALE,WHITE NON HISPANIC,Brian,16,93 839 | 2015,FEMALE,HISPANIC,Maia,25,66 840 | 2016,MALE,BLACK NON HISPANIC,Karter,29,39 841 | 2016,FEMALE,WHITE NON HISPANIC,Natalie,43,51 842 | 2015,MALE,BLACK NON HISPANIC,Benjamin,21,52 843 | 2015,FEMALE,WHITE NON HISPANIC,Sabina,12,81 844 | 2016,FEMALE,BLACK NON HISPANIC,Hope,10,43 845 | 2015,MALE,HISPANIC,Erick,60,54 846 | 2016,MALE,BLACK NON HISPANIC,Major,20,48 847 | 2016,FEMALE,HISPANIC,Giselle,18,66 848 | 2016,MALE,ASIAN AND PACIFIC ISLANDER,Gavin,32,38 849 | 2016,MALE,ASIAN AND PACIFIC ISLANDER,Devin,10,59 850 | 2016,MALE,WHITE NON HISPANIC,Martin,23,86 851 | 2016,MALE,HISPANIC,Bryan,46,60 852 | 2016,MALE,ASIAN AND PACIFIC ISLANDER,Mustafa,14,55 853 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Leela,10,41 854 | 2015,MALE,WHITE NON HISPANIC,Ariel,23,81 855 | 2015,FEMALE,WHITE NON HISPANIC,Lily,84,26 856 | 2016,MALE,HISPANIC,Lionel,10,91 857 | 2016,FEMALE,WHITE NON HISPANIC,Dina,39,55 858 | 2016,FEMALE,WHITE NON HISPANIC,Gianna,40,54 859 | 2016,FEMALE,HISPANIC,Ashley,100,14 860 | 2015,FEMALE,BLACK NON HISPANIC,Emma,17,33 861 | 2015,MALE,WHITE NON HISPANIC,Elimelech,19,85 862 | 2015,MALE,ASIAN AND PACIFIC ISLANDER,Brandon,37,29 863 | 2015,FEMALE,WHITE NON HISPANIC,Valerie,12,81 864 | 2015,MALE,HISPANIC,Romeo,13,96 865 | 2016,MALE,WHITE NON HISPANIC,Moses,37,72 866 | 2015,MALE,ASIAN AND PACIFIC ISLANDER,Syed,29,37 867 | 2015,FEMALE,WHITE NON HISPANIC,Adeline,22,71 868 | 2016,MALE,BLACK NON HISPANIC,Israel,13,54 869 | 2016,FEMALE,WHITE NON HISPANIC,Esther,209,3 870 | 2016,FEMALE,HISPANIC,Aubrey,38,46 871 | 2015,FEMALE,HISPANIC,Alanis,13,78 872 | 2015,FEMALE,ASIAN AND PACIFIC ISLANDER,Raina,11,42 873 | 2016,MALE,HISPANIC,Erick,63,45 874 | 2015,FEMALE,ASIAN AND PACIFIC ISLANDER,Winnie,13,40 875 | 2016,MALE,HISPANIC,Ezequiel,11,90 876 | 2015,FEMALE,WHITE NON HISPANIC,Clementine,11,82 877 | 2015,FEMALE,WHITE NON HISPANIC,Anastasia,32,61 878 | 2015,MALE,WHITE NON HISPANIC,Jesse,14,90 879 | 2015,FEMALE,WHITE NON HISPANIC,Thea,15,78 880 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Zoey,29,22 881 | 2016,FEMALE,WHITE NON HISPANIC,Harper,67,36 882 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Alice,28,23 883 | 2016,MALE,HISPANIC,Kyrie,14,87 884 | 2016,MALE,ASIAN AND PACIFIC ISLANDER,Wesley,20,49 885 | 2015,MALE,WHITE NON HISPANIC,Jason,28,76 886 | 2016,FEMALE,WHITE NON HISPANIC,Ashley,10,84 887 | 2016,FEMALE,HISPANIC,Briana,21,63 888 | 2015,MALE,BLACK NON HISPANIC,Jordan,40,34 889 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Hannah,56,8 890 | 2016,MALE,WHITE NON HISPANIC,Malek,10,99 891 | 2016,MALE,ASIAN AND PACIFIC ISLANDER,Kai,34,36 892 | 2016,MALE,HISPANIC,Thiago,55,51 893 | 2016,MALE,WHITE NON HISPANIC,Chaim,140,18 894 | 2015,MALE,ASIAN AND PACIFIC ISLANDER,George,16,49 895 | 2016,FEMALE,BLACK NON HISPANIC,Alani,15,38 896 | 2015,FEMALE,WHITE NON HISPANIC,Anisa,10,83 897 | 2016,MALE,HISPANIC,Ryan,103,27 898 | 2016,FEMALE,WHITE NON HISPANIC,Yocheved,38,56 899 | 2016,FEMALE,HISPANIC,Alana,26,58 900 | 2016,MALE,WHITE NON HISPANIC,Victor,24,85 901 | 2015,FEMALE,WHITE NON HISPANIC,Aria,34,59 902 | 2016,FEMALE,HISPANIC,Keyla,29,55 903 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Jessica,29,22 904 | 2016,FEMALE,WHITE NON HISPANIC,Alexa,36,58 905 | 2015,FEMALE,BLACK NON HISPANIC,Erin,11,39 906 | 2016,FEMALE,WHITE NON HISPANIC,Kate,15,79 907 | 2016,MALE,HISPANIC,Nathan,83,34 908 | 2015,FEMALE,WHITE NON HISPANIC,Lucy,46,49 909 | 2015,FEMALE,WHITE NON HISPANIC,Lara,18,75 910 | 2016,MALE,HISPANIC,Emmanuel,56,50 911 | 2016,MALE,ASIAN AND PACIFIC ISLANDER,Aydin,11,58 912 | 2015,FEMALE,BLACK NON HISPANIC,Peyton,44,12 913 | 2015,MALE,BLACK NON HISPANIC,Jeremy,10,63 914 | 2015,MALE,BLACK NON HISPANIC,Mohamed,46,28 915 | 2015,FEMALE,HISPANIC,Zoey,49,43 916 | 2016,MALE,WHITE NON HISPANIC,Parker,27,82 917 | 2015,FEMALE,HISPANIC,Londyn,10,81 918 | 2016,MALE,WHITE NON HISPANIC,Enzo,12,97 919 | 2016,FEMALE,HISPANIC,Trinity,10,74 920 | 2016,MALE,WHITE NON HISPANIC,Edward,47,64 921 | 2015,MALE,ASIAN AND PACIFIC ISLANDER,Ayden,22,43 922 | 2016,MALE,HISPANIC,Damien,19,82 923 | 2015,MALE,WHITE NON HISPANIC,Ethan,138,19 924 | 2016,FEMALE,BLACK NON HISPANIC,Amira,15,38 925 | 2015,FEMALE,HISPANIC,Alicia,15,76 926 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Irene,20,31 927 | 2016,FEMALE,HISPANIC,Alessandra,16,68 928 | 2016,MALE,HISPANIC,Andy,28,73 929 | 2015,FEMALE,BLACK NON HISPANIC,Jade,20,30 930 | 2015,MALE,BLACK NON HISPANIC,Michael,78,12 931 | 2015,FEMALE,WHITE NON HISPANIC,Autumn,11,82 932 | 2015,FEMALE,ASIAN AND PACIFIC ISLANDER,Julia,17,36 933 | 2015,FEMALE,HISPANIC,Layla,50,42 934 | 2016,MALE,BLACK NON HISPANIC,Henry,10,57 935 | 2015,MALE,ASIAN AND PACIFIC ISLANDER,Jacob,46,23 936 | 2015,MALE,HISPANIC,Carter,24,85 937 | 2015,FEMALE,BLACK NON HISPANIC,Zahra,12,38 938 | 2016,MALE,HISPANIC,Julius,25,76 939 | 2016,FEMALE,WHITE NON HISPANIC,Sloane,41,53 940 | 2016,FEMALE,HISPANIC,Alison,24,60 941 | 2015,MALE,BLACK NON HISPANIC,Nathan,50,25 942 | 2015,MALE,WHITE NON HISPANIC,Shmiel,38,67 943 | 2015,MALE,WHITE NON HISPANIC,Crosby,11,93 944 | 2016,MALE,WHITE NON HISPANIC,Usher,21,88 945 | 2015,MALE,WHITE NON HISPANIC,Brandon,33,71 946 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Iris,18,33 947 | 2016,MALE,HISPANIC,Anderson,20,81 948 | 2015,MALE,BLACK NON HISPANIC,Abdoul,10,63 949 | 2015,MALE,ASIAN AND PACIFIC ISLANDER,Damon,10,55 950 | 2016,MALE,WHITE NON HISPANIC,Boruch,13,96 951 | 2015,FEMALE,WHITE NON HISPANIC,Maryam,16,77 952 | 2015,MALE,WHITE NON HISPANIC,Spencer,21,83 953 | 2016,MALE,BLACK NON HISPANIC,Camren,10,57 954 | 2015,MALE,ASIAN AND PACIFIC ISLANDER,Gary,15,50 955 | 2016,MALE,ASIAN AND PACIFIC ISLANDER,Caden,18,51 956 | 2015,MALE,HISPANIC,Jonah,22,87 957 | 2016,MALE,WHITE NON HISPANIC,Reid,21,88 958 | 2016,FEMALE,WHITE NON HISPANIC,Tzipora,16,78 959 | 2016,FEMALE,ASIAN AND PACIFIC ISLANDER,Jacqueline,14,37 960 | 2016,FEMALE,HISPANIC,Adelyn,11,73 961 | 2015,FEMALE,BLACK NON HISPANIC,Madison,128,1 962 | 2015,MALE,BLACK NON HISPANIC,Josiah,111,5 963 | 2015,FEMALE,HISPANIC,Sara,23,68 964 | 2016,FEMALE,WHITE NON HISPANIC,Zissy,15,79 965 | 2015,MALE,ASIAN AND PACIFIC ISLANDER,Jack,14,51 966 | 2015,MALE,HISPANIC,Justin,104,28 967 | 2015,MALE,WHITE NON HISPANIC,Dovid,35,69 968 | 2016,MALE,HISPANIC,Dante,10,91 969 | 2015,FEMALE,HISPANIC,Violet,26,65 970 | 2016,FEMALE,WHITE NON HISPANIC,Diyora,11,83 971 | 2015,FEMALE,ASIAN AND PACIFIC ISLANDER,Christine,10,43 972 | -------------------------------------------------------------------------------- /tests/test_open_read.py: -------------------------------------------------------------------------------- 1 | import os 2 | from tempfile import NamedTemporaryFile 3 | 4 | import aiofiles 5 | import pytest 6 | 7 | import asyncstream 8 | from tests.test_utils import async_gen_to_list, compress, get_raw_lines, encode_parquet, encode_orc 9 | 10 | 11 | @pytest.mark.parametrize( 12 | "compression", [ 13 | None, 14 | 'gzip', 15 | 'bzip2', 16 | 'zstd', 17 | 'snappy' 18 | ] 19 | ) 20 | @pytest.mark.asyncio 21 | async def test_open_read(compression: str): 22 | baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 23 | with NamedTemporaryFile() as tmpfd: 24 | tmpfd.write(compress(baby_name_filename, compression)) 25 | tmpfd.flush() 26 | async with aiofiles.open(tmpfd.name, 'rb') as cfd: 27 | async with asyncstream.open(cfd, mode='rb', compression=compression) as fd: 28 | assert get_raw_lines(baby_name_filename) == await async_gen_to_list(fd) 29 | 30 | 31 | @pytest.mark.parametrize( 32 | "compression", [ 33 | None, 34 | 'gzip', 35 | 'bzip2', 36 | 'zstd', 37 | 'snappy' 38 | ] 39 | ) 40 | @pytest.mark.asyncio 41 | async def test_open_read_with_filename(compression: str): 42 | baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 43 | with NamedTemporaryFile() as tmpfd: 44 | tmpfd.write(compress(baby_name_filename, compression)) 45 | tmpfd.flush() 46 | async with asyncstream.open(tmpfd.name, mode='rb', compression=compression) as fd: 47 | assert get_raw_lines(baby_name_filename) == await async_gen_to_list(fd) 48 | 49 | @pytest.mark.parametrize( 50 | "compression", [ 51 | None, 52 | 'gzip', 53 | 'bzip2', 54 | 'zstd', 55 | 'snappy' 56 | ] 57 | ) 58 | @pytest.mark.asyncio 59 | async def test_open_read_parquet(compression: str): 60 | baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 61 | with NamedTemporaryFile(delete=False) as tmpfd: 62 | tmpfd.write(encode_parquet(baby_name_filename, compression)) 63 | tmpfd.flush() 64 | async with aiofiles.open(tmpfd.name, 'rb') as cfd: 65 | async with asyncstream.open(cfd, mode='rb', encoding='parquet') as fd: 66 | assert get_raw_lines(baby_name_filename) == await async_gen_to_list(fd) 67 | 68 | @pytest.mark.parametrize( 69 | "compression", [ 70 | None, 71 | 'gzip', 72 | 'bzip2', 73 | 'zstd', 74 | 'snappy' 75 | ] 76 | ) 77 | @pytest.mark.asyncio 78 | async def test_open_read_parquet_with_filename(compression: str): 79 | baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 80 | with NamedTemporaryFile(delete=False) as tmpfd: 81 | tmpfd.write(encode_parquet(baby_name_filename, compression)) 82 | tmpfd.flush() 83 | async with asyncstream.open(tmpfd.name, mode='rb', encoding='parquet') as fd: 84 | assert get_raw_lines(baby_name_filename) == await async_gen_to_list(fd) 85 | 86 | 87 | @pytest.mark.parametrize( 88 | "compression", [ 89 | None, 90 | # 'gzip', 91 | # 'bzip2', 92 | # 'zstd', 93 | # 'snappy' 94 | ] 95 | ) 96 | @pytest.mark.asyncio 97 | async def test_open_read_orc(compression: str): 98 | baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 99 | with NamedTemporaryFile(delete=False) as tmpfd: 100 | tmpfd.write(encode_orc(baby_name_filename, compression)) 101 | tmpfd.flush() 102 | async with aiofiles.open(tmpfd.name, 'rb') as cfd: 103 | async with asyncstream.open(cfd, mode='rb', encoding='orc', compression=compression) as fd: 104 | assert get_raw_lines(baby_name_filename) == await async_gen_to_list(fd) 105 | -------------------------------------------------------------------------------- /tests/test_open_write.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | from tempfile import NamedTemporaryFile 4 | 5 | import aiofiles 6 | import pyorc 7 | import pytest 8 | 9 | import asyncstream 10 | from tests import test_utils 11 | from tests.test_utils import async_gen_to_list, encode_parquet, encode_orc, get_raw_rows, get_raw_lines, compress 12 | 13 | 14 | @pytest.mark.parametrize( 15 | "compression", [ 16 | None, 17 | 'gzip', 18 | 'bzip2', 19 | 'zstd', 20 | 'snappy' 21 | ] 22 | ) 23 | @pytest.mark.asyncio 24 | async def test_open_write(compression: str): 25 | baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 26 | with NamedTemporaryFile() as tmpfd: 27 | async with aiofiles.open(tmpfd.name, 'wb') as rfd: 28 | async with aiofiles.open(baby_name_filename, 'rb') as cfd: 29 | async with asyncstream.open(rfd, mode='wb', compression=compression) as fd: 30 | await fd.write(await cfd.read()) 31 | assert get_raw_lines(baby_name_filename) == get_raw_lines(tmpfd.name, compression) 32 | 33 | @pytest.mark.parametrize( 34 | "compression", [ 35 | None, 36 | 'gzip', 37 | 'bzip2', 38 | 'zstd', 39 | 'snappy' 40 | ] 41 | ) 42 | @pytest.mark.asyncio 43 | async def test_open_read_with_filename(compression: str): 44 | baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 45 | with NamedTemporaryFile() as tmpfd: 46 | async with aiofiles.open(baby_name_filename, 'rb') as cfd: 47 | async with asyncstream.open(tmpfd.name, mode='wb', compression=compression) as fd: 48 | await fd.write(await cfd.read()) 49 | assert get_raw_lines(baby_name_filename) == get_raw_lines(tmpfd.name, compression) 50 | -------------------------------------------------------------------------------- /tests/test_readers.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | from tempfile import NamedTemporaryFile 4 | 5 | import aiofiles 6 | import pyorc 7 | import pytest 8 | 9 | import asyncstream 10 | from tests import test_utils 11 | from tests.test_utils import async_gen_to_list, encode_parquet, encode_orc, get_raw_rows, compress 12 | 13 | 14 | @pytest.mark.parametrize( 15 | "compression", [ 16 | None, 17 | 'gzip', 18 | 'bzip2', 19 | 'zstd' 20 | ] 21 | ) 22 | @pytest.mark.asyncio 23 | async def test_reader(compression: str): 24 | baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 25 | with NamedTemporaryFile() as tmpfd: 26 | tmpfd.write(compress(baby_name_filename, compression)) 27 | tmpfd.flush() 28 | async with asyncstream.open(tmpfd.name, mode='rb', compression=compression) as fd: 29 | async with asyncstream.reader(fd) as reader: 30 | assert get_raw_rows(baby_name_filename) == await async_gen_to_list(reader) 31 | 32 | # 33 | # @pytest.mark.parametrize( 34 | # "compression", [ 35 | # None, 36 | # 'gzip', 37 | # 'bzip2', 38 | # 'zstd' 39 | # ] 40 | # ) 41 | # @pytest.mark.asyncio 42 | # async def test_open(compression: str): 43 | # baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 44 | # with NamedTemporaryFile() as tmpfd: 45 | # compressed_fd = getattr(test_utils, 'compress_' + str(compression).lower())(baby_name_filename) 46 | # tmpfd.write(compressed_fd) 47 | # tmpfd.flush() 48 | # async with aiofiles.open(tmpfd.name, 'rb') as fd: 49 | # assert get_raw_lines(baby_name_filename) == await async_gen_to_list(asyncstream.open(fd, compression=compression)) 50 | # 51 | # @pytest.mark.parametrize( 52 | # "compression", [ 53 | # None, 54 | # 'snappy', 55 | # 'gzip', 56 | # 'brotli' 57 | # ] 58 | # ) 59 | # @pytest.mark.asyncio 60 | # async def test_read_parquet(compression: str): 61 | # baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 62 | # with NamedTemporaryFile() as tmpfd: 63 | # tmpfd.write(encode_parquet(baby_name_filename, compression=compression)) 64 | # tmpfd.flush() 65 | # 66 | # async with aiofiles.open(tmpfd.name, 'rb') as parquet_fd: 67 | # async with asyncstream.open(parquet_fd, encoding='parquet', compression=compression) as fd: 68 | # async with asyncstream.reader(fd, has_header=False) as reader: 69 | # assert get_raw_rows(baby_name_filename) == [(await reader.header())] + await async_gen_to_list(reader) 70 | # 71 | # 72 | # @pytest.mark.parametrize( 73 | # "compression", [ 74 | # None, 75 | # # 'zlib', 76 | # # 'zstd' 77 | # ] 78 | # ) 79 | # @pytest.mark.asyncio 80 | # async def test_read_orc_builtin(compression: str): 81 | # baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 82 | # with NamedTemporaryFile() as tmpfd: 83 | # columns = ['birth_year', 'gender', 'ethnicity', "child_name", 'count', 'rank'] 84 | # column_types = ['string', 'string', 'string', 'string', 'string', 'string'] 85 | # tmpfd.write(encode_orc(baby_name_filename, compression=compression, columns=columns, column_types=column_types)) 86 | # 87 | # async with aiofiles.open(tmpfd.name, 'rb') as fd: 88 | # async with asyncstream.reader(fd, encoding='orc', ignore_header=False) as reader: 89 | # assert get_raw_rows(baby_name_filename) == [(await reader.header())] + await async_gen_to_list(reader) 90 | 91 | # @pytest.mark.parametrize( 92 | # "compression", [ 93 | # # 'lz4', 94 | # # 'lzo', 95 | # 'snappy', 96 | # 'gzip' 97 | # ] 98 | # ) 99 | # @pytest.mark.asyncio 100 | # async def test_read_orc_nonbuiltin(compression: str): 101 | # baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 102 | # 103 | # with NamedTemporaryFile() as tmpfd: 104 | # columns = ['birth_year', 'gender', 'ethnicity', "child_name", 'count', 'rank'] 105 | # column_types = ['string', 'string', 'string', 'string', 'string', 'string'] 106 | # tmpfd.write(encode_orc(baby_name_filename, compression=compression, columns=columns, column_types=column_types)) 107 | # 108 | # async with aiofiles.open(tmpfd.name, 'rb') as fd: 109 | # async with asyncstream.reader(fd, encoding='orc', compression=compression, ignore_header=False) as reader: 110 | # assert get_raw_rows(baby_name_filename) == [(await reader.header())] + await async_gen_to_list(reader) 111 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import bz2 2 | import csv 3 | import gzip 4 | import io 5 | from contextlib import contextmanager 6 | from itertools import zip_longest 7 | from typing import AsyncGenerator, Iterable, Union 8 | 9 | import pandas as pd 10 | import pyorc 11 | import zstd 12 | from snappy import snappy 13 | 14 | 15 | async def async_gen_to_list(async_gen: AsyncGenerator): 16 | result = [] 17 | async for obj in async_gen: 18 | result.append(obj) 19 | 20 | return result 21 | 22 | 23 | def uncompress_none(filename: str): 24 | with open(filename, 'rb') as fd: 25 | return list(fd) 26 | 27 | 28 | def uncompress_gzip(filename: str): 29 | with gzip.open(filename) as gzfd: 30 | return list(gzfd) 31 | 32 | 33 | def uncompress_bzip2(filename: str): 34 | with bz2.open(filename) as bz2fd: 35 | return list(bz2fd) 36 | 37 | 38 | def uncompress_zstd(filename: str): 39 | with open(filename, 'rb') as fd: 40 | dctx = zstd.ZstdDecompressor() 41 | return dctx.decompress(fd.read(), max_output_size=1048576).splitlines(True) 42 | 43 | def uncompress_snappy(filename: str): 44 | with open(filename, 'rb') as fd: 45 | dctx = snappy.StreamDecompressor() 46 | return dctx.decompress(fd.read()).splitlines(True) 47 | 48 | @contextmanager 49 | def open_file_or_bytesio(file: Union[str, io.BytesIO]): 50 | if isinstance(file, str): 51 | with open(file, 'rb') as fd: 52 | yield fd 53 | else: 54 | yield file 55 | 56 | 57 | def compress_none(filename: str): 58 | with open(filename, 'rb') as fd: 59 | return fd.read() 60 | 61 | 62 | def compress_gzip(file: Union[str, io.BytesIO]): 63 | buffer = io.BytesIO() 64 | with open_file_or_bytesio(file) as fd: 65 | with gzip.open(buffer, 'wb') as gzfd: 66 | gzfd.write(fd.read()) 67 | return buffer.getvalue() 68 | 69 | 70 | def compress_bzip2(file: Union[str, io.BytesIO]): 71 | buffer = io.BytesIO() 72 | with open_file_or_bytesio(file) as fd: 73 | with bz2.open(buffer, 'wb') as bz2fd: 74 | bz2fd.write(fd.read()) 75 | return buffer.getvalue() 76 | 77 | 78 | def compress_zstd(filename: str): 79 | with open(filename, 'rb') as fd: 80 | dctx = zstd.ZstdCompressor() 81 | return dctx.compress(fd.read()) 82 | 83 | 84 | def compress_snappy(file: Union[str, io.BytesIO]): 85 | with open_file_or_bytesio(file) as fd: 86 | comp = snappy.StreamCompressor() 87 | return comp.compress(fd.read()) 88 | 89 | 90 | def compress(file, compression=None): 91 | if compression is None: 92 | return compress_none(file) 93 | elif compression == 'gzip': 94 | return compress_gzip(file) 95 | elif compression == 'bzip2': 96 | return compress_bzip2(file) 97 | elif compression == 'zstd': 98 | return compress_zstd(file) 99 | elif compression == 'snappy': 100 | return compress_snappy(file) 101 | else: 102 | raise Exception('Compression {compression} not supported'.format(compression=compression)) 103 | 104 | def uncompress(file, compression=None): 105 | if compression is None: 106 | return uncompress_none(file) 107 | elif compression == 'gzip': 108 | return uncompress_gzip(file) 109 | elif compression == 'bzip2': 110 | return uncompress_bzip2(file) 111 | elif compression == 'zstd': 112 | return uncompress_zstd(file) 113 | elif compression == 'snappy': 114 | return uncompress_snappy(file) 115 | else: 116 | raise Exception('Compression {compression} not supported'.format(compression=compression)) 117 | 118 | 119 | def encode_parquet(filename: str, compression: str = None): 120 | with open(filename, 'rt') as fd: 121 | reader = csv.reader(fd) 122 | header = next(reader) 123 | df = pd.DataFrame(list(reader), columns=header) 124 | buffer = io.BytesIO() 125 | df.to_parquet(buffer, compression=compression) 126 | return buffer.getvalue() 127 | 128 | 129 | def decode_parquet(filename: str): 130 | with open(filename, 'rb') as fd: 131 | df = pd.read_parquet(fd, engine='pyarrow') 132 | return df.to_csv(index=False).rstrip('\n') 133 | 134 | 135 | def encode_orc(filename: str, compression: str = None, columns: Iterable[str] = None, 136 | column_types: Iterable[str] = None, skip_header=True): 137 | buffer = io.BytesIO() 138 | with open(filename, 'rt') as fd: 139 | reader = csv.reader(fd) 140 | 141 | # If columns not provided try to read header from the file 142 | if skip_header or columns is None: 143 | columns = next(reader) 144 | column_types = ['string'] * len(columns) 145 | 146 | struct = 'struct<{columns}>'.format( 147 | columns=','.join( 148 | name + ':' + (col_type if col_type else 'string') 149 | for name, col_type in zip_longest(columns, column_types) 150 | ) 151 | ) 152 | 153 | if compression in (None, 'zlib', 'zstd'): 154 | compression_type = getattr(pyorc.CompressionKind, str(compression).upper()) 155 | else: 156 | compression_type = pyorc.CompressionKind.NONE 157 | with pyorc.Writer(buffer, struct, compression=compression_type) as writer: 158 | for row in reader: 159 | writer.write(tuple(row)) 160 | 161 | if compression in (None, 'zlib', 'zstd'): 162 | return buffer.getvalue() 163 | else: 164 | buffer.seek(0) 165 | return compress(buffer, compression) 166 | 167 | 168 | def decode_orc(filename: str): 169 | with open(filename, 'rb') as data: 170 | reader = pyorc.Reader(data) 171 | return '\n'.join(','.join([str(c) for c in row]) for row in reader) 172 | 173 | 174 | def get_raw_rows(filename: str, compression=None): 175 | return list(csv.reader([line.decode('utf-8') for line in uncompress(filename, compression)])) 176 | 177 | 178 | def get_raw_lines(filename: str, compression=None): 179 | return uncompress(filename, compression) 180 | -------------------------------------------------------------------------------- /tests/test_writers.py: -------------------------------------------------------------------------------- 1 | # import csv 2 | # import os 3 | # import tempfile 4 | # 5 | # import aiofiles 6 | # import pytest 7 | # 8 | # import asyncstream 9 | # from tests import test_utils 10 | # from tests.test_utils import decode_parquet, decode_orc 11 | # 12 | import os 13 | from tempfile import NamedTemporaryFile 14 | 15 | import aiofiles 16 | import pytest 17 | 18 | import asyncstream 19 | from tests.test_utils import get_raw_lines 20 | 21 | 22 | @pytest.mark.parametrize( 23 | "compression", [ 24 | None, 25 | 'gzip', 26 | 'bzip2', 27 | 'zstd' 28 | ] 29 | ) 30 | @pytest.mark.asyncio 31 | async def test_writer(compression: str): 32 | baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 33 | with NamedTemporaryFile(mode='wb', delete=False) as tmpfd: 34 | async with aiofiles.open(baby_name_filename, 'rb') as rfd: 35 | async with aiofiles.open(tmpfd.name, 'wb') as owfd: 36 | async with asyncstream.open(owfd, mode='wb', compression=compression) as wfd: 37 | async for line in rfd: 38 | await wfd.write(line) 39 | assert get_raw_lines(baby_name_filename) == get_raw_lines(tmpfd.name, compression) 40 | 41 | # 42 | # def get_raw_rows(filename: str): 43 | # with open(filename) as fd: 44 | # csv_reader = csv.reader(fd) 45 | # return list(csv_reader) 46 | # 47 | # 48 | # def get_raw_lines(filename: str): 49 | # with open(filename, 'rb') as fd: 50 | # return list(fd) 51 | # 52 | # 53 | # @pytest.mark.parametrize( 54 | # "compression,extension", [ 55 | # (None, ''), 56 | # ('gzip', '.gz'), 57 | # ('bzip2', '.bz2'), 58 | # ('zstd', '.zst') 59 | # ] 60 | # ) 61 | # @pytest.mark.asyncio 62 | # async def test_open(compression: str, extension: str): 63 | # baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 64 | # decomp_func = getattr(test_utils, 'uncompress_' + str(compression).lower()) 65 | # async with aiofiles.open(baby_name_filename, 'rb') as fd: 66 | # with tempfile.NamedTemporaryFile('wb') as tmpfd: 67 | # async with aiofiles.open(tmpfd.name, 'wb') as atmpfd: 68 | # async with asyncstream.open(atmpfd, 'wb', compression=compression) as gzfd: 69 | # async for line in fd: 70 | # await gzfd.write(line) 71 | # result = decomp_func(tmpfd.name) 72 | # assert get_raw_lines(baby_name_filename) == result 73 | # 74 | # @pytest.mark.parametrize( 75 | # "compression", [ 76 | # None, 77 | # 'snappy', 78 | # 'gzip', 79 | # 'brotli' 80 | # ] 81 | # ) 82 | # @pytest.mark.asyncio 83 | # async def test_write_parquet(compression: str): 84 | # baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 85 | # async with aiofiles.open(baby_name_filename, 'rb') as fd: 86 | # async with asyncstream.reader(fd, has_header=True) as reader: 87 | # with tempfile.NamedTemporaryFile() as tmpfd: 88 | # async with aiofiles.open(tmpfd.name, 'wb') as wfd: 89 | # writer = asyncstream.writer(wfd, encoding='parquet', columns=await reader.header()) 90 | # async for row in reader: 91 | # writer.writerow(row) 92 | # await writer.close() 93 | # assert get_raw_lines(baby_name_filename) == decode_parquet(tmpfd.name).encode('utf-8').splitlines(True) 94 | # 95 | # # @pytest.mark.parametrize( 96 | # # "compression", [ 97 | # # None, 98 | # # 'snappy', 99 | # # 'gzip' 100 | # # ] 101 | # # ) 102 | # # @pytest.mark.asyncio 103 | # # async def test_write_orc(compression: str): 104 | # # baby_name_filename = os.path.join(os.path.dirname(__file__), 'data', 'baby_names.csv') 105 | # # async with aiofiles.open(baby_name_filename, 'rb') as fd: 106 | # # async with asyncstream.reader(fd, ignore_header=False) as reader: 107 | # # with tempfile.NamedTemporaryFile() as tmpfd: 108 | # # async with aiofiles.open(tmpfd.name, 'wb') as wfd: 109 | # # writer = asyncstream.writer(wfd, encoding='orc', columns=await reader.header()) 110 | # # async for row in reader: 111 | # # writer.writerow(row) 112 | # # await writer.close() 113 | # # assert get_raw_lines(baby_name_filename) == decode_orc(tmpfd.name).encode('utf-8').splitlines(True) 114 | --------------------------------------------------------------------------------