├── LICENSE
├── README.md
├── example.py
├── reddit_data.zst
├── requirements.txt
└── zreader.py


/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2019 Jason Michael Baumgartner
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # zreader
2 | Read compressed NDJSON .zst files easily
3 | 
4 | Usage:
5 | 
6 | Take a look at the example.py script. When creating a new zreader, you can pass a chunk size or let it default to 16k.
7 | 
8 | 
9 | 


--------------------------------------------------------------------------------
/example.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | 
 3 | import zreader
 4 | import ujson as json
 5 | 
 6 | 
 7 | 
 8 | # Adjust chunk_size as necessary -- defaults to 16,384 if not specified
 9 | reader = zreader.Zreader("reddit_data.zst", chunk_size=8192)
10 | 
11 | # Read each line from the reader
12 | for line in reader.readlines():
13 |     obj = json.loads(line)
14 |     print (obj['author'], obj['subreddit'], sep=",")
15 | 
16 | 


--------------------------------------------------------------------------------
/reddit_data.zst:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/pushshift/zreader/48626c3bb36f33ec02d79d94f62c0458b8e928b8/reddit_data.zst


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | ujson==1.35
2 | zstandard==0.11.0
3 | 


--------------------------------------------------------------------------------
/zreader.py:
--------------------------------------------------------------------------------
 1 | import zstandard as zstd
 2 | 
 3 | 
 4 | 
 5 | class Zreader:
 6 | 
 7 |     def __init__(self, file, chunk_size=16384):
 8 |         '''Init method'''
 9 |         self.fh = open(file,'rb')
10 |         self.chunk_size = chunk_size
11 |         self.dctx = zstd.ZstdDecompressor()
12 |         self.reader = self.dctx.stream_reader(self.fh)
13 |         self.buffer = ''
14 | 
15 | 
16 |     def readlines(self):
17 |         '''Generator method that creates an iterator for each line of JSON'''
18 |         while True:
19 |             chunk = self.reader.read(self.chunk_size).decode()
20 |             if not chunk:
21 |                 break
22 |             lines = (self.buffer + chunk).split("\n")
23 | 
24 |             for line in lines[:-1]:
25 |                 yield line
26 | 
27 |             self.buffer = lines[-1]
28 | 
29 | 


--------------------------------------------------------------------------------