├── .github └── workflows │ └── ci.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── pyproject.toml ├── requirements.txt ├── src └── orjsonl │ ├── __init__.py │ ├── orjsonl.py │ └── py.typed └── tests ├── helpers.py └── test_orjsonl.py /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] 16 | 17 | steps: 18 | - name: Check-out repository 19 | uses: actions/checkout@v3 20 | 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v3 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip 29 | python -m pip install pytest 30 | python -m pip install pytest-cov 31 | 32 | - name: Install orjsonl 33 | run: | 34 | python -m pip install . 35 | 36 | - name: Test with pytest 37 | run: | 38 | pytest --cov=orjsonl --cov-report=xml 39 | 40 | - name: Use Codecov to track coverage 41 | uses: codecov/codecov-action@v3 42 | with: 43 | files: ./coverage.xml 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 5 | 6 | ## [1.0.0] - 2023-11-09 7 | ### Changed 8 | - Fixed links to version numbers in this changelog. 9 | - Version bump to 1.0.0 to flag that `orjsonl` is stable. 10 | 11 | ## [0.3.1] - 2023-10-08 12 | ### Changed 13 | - Added tests for `append()` and `extend()` with `newline` set to `False` to achieve 100% coverage. 14 | 15 | ## [0.3.0] - 2023-10-08 16 | ### Added 17 | - Added the `extend()` function, which serializes and appends an iterable of Python objects to a UTF-8-encoded jsonl file. 18 | 19 | ### Changed 20 | - Changed the `append()` function to serialize and append a single Python object to a UTF-8-encoded jsonl file. 21 | 22 | ## [0.2.2] - 2022-12-24 23 | ### Fixed 24 | - Updated the link to the GitHub Actions shield as per badges/shields#8671. 25 | 26 | ## [0.2.1] - 2022-11-23 27 | ### Added 28 | - Added new keywords to the project metadata. 29 | 30 | ### Changed 31 | - Rephrased and simplifed the README file. 32 | - Changed the order of `stream()` and `load()` in `orjsonl.py`. 33 | 34 | ### Fixed 35 | - Corrected typos in the README file. 36 | - Corrected typos in the changelog. 37 | - Corrected typos in docstrings. 38 | 39 | ## [0.2.0] - 2022-11-22 40 | ### Added 41 | - Added support for gzip, bzip2, xz and Zstandard compression to `load()`, `stream()`, `save()` and `append()` as requested in [#1](https://github.com/umarbutler/orjsonl/issues/1). 42 | - Created `py.typed`. 43 | - Ensured that `load()`, `stream()`, `save()` and `append()` are tested with compressed jsonl files. 44 | 45 | ### Changed 46 | - Changed `stream()` to return a `generator` rather than a `map`. 47 | - Changed `load()`, `stream()`, `save()` and `append()` to rely on [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) rather than [`open()`](https://docs.python.org/3/library/functions.html#open). 48 | - Updated the package description and README file to reflect the fact that `orjsonl` now supports compression. 49 | 50 | ### Fixed 51 | - Fixed [#1](https://github.com/umarbutler/orjsonl/issues/1) by ensuring that `stream()` closes jsonl files whenever a `generator` has been exhuasted. 52 | - Corrected typos in the changelog. 53 | - Corrected typos in docstrings. 54 | - Ensured that optional arguments are type hinted as such. 55 | - Updated dependencies to prevent the use of versions of [`orjson`](https://github.com/ijl/orjson) older than 3.7.7. 56 | 57 | ### Removed 58 | - Removed support for integer file descriptors. 59 | 60 | ## [0.1.3] - 2022-11-20 61 | ### Removed 62 | - Removed unnecessary links to `load()`, `stream()`, `save()` and `append()` in the README file. 63 | 64 | ## [0.1.2] - 2022-11-20 65 | ### Added 66 | - Allowed for the `default` and `option` arguments to be passed to `orjson.dumps()` through `save()` and `append()`. 67 | - Added 'ndjson' as a keyword in the project metadata. 68 | 69 | ## [0.1.1] - 2022-11-19 70 | ### Added 71 | - Created a changelog. 72 | - Added 'lines', 'json lines' and 'fast' as keywords in the project metadata. 73 | 74 | ### Changed 75 | - Renamed the 'Bug Tracker' url to 'Issues' in the project metadata. 76 | - Specified `orjsonl`'s license to be the MIT License in the project metadata. 77 | 78 | ### Fixed 79 | - Fixed typos in the README file. 80 | - Fixed typos in the tests script. 81 | 82 | ## [0.1.0] - 2022-11-18 83 | ### Added 84 | - Added the `load()` function, which deserializes a UTF-8-encoded jsonl file to a list of Python objects. 85 | - Added the `stream()` function, which creates a map object that deserializes a UTF-8-encoded jsonl file to Python objects. 86 | - Added the `save()` function, which serializes an iterable of Python objects to a UTF-8-encoded jsonl file. 87 | - Added the `append()` function, which serializes and appends an iterable of Python objects to a UTF-8-encoded jsonl file. 88 | 89 | [1.0.0]: https://github.com/umarbutler/orjsonl/compare/v0.3.1...v1.0.0 90 | [0.3.1]: https://github.com/umarbutler/orjsonl/compare/v0.3.0...v0.3.1 91 | [0.3.0]: https://github.com/umarbutler/orjsonl/compare/v0.2.2...v0.3.0 92 | [0.2.2]: https://github.com/umarbutler/orjsonl/compare/v0.2.1...v0.2.2 93 | [0.2.1]: https://github.com/umarbutler/orjsonl/compare/v0.2.0...v0.2.1 94 | [0.2.0]: https://github.com/umarbutler/orjsonl/compare/v0.1.3...v0.2.0 95 | [0.1.3]: https://github.com/umarbutler/orjsonl/compare/v0.1.2...v0.1.3 96 | [0.1.2]: https://github.com/umarbutler/orjsonl/compare/v0.1.1...v0.1.2 97 | [0.1.1]: https://github.com/umarbutler/orjsonl/compare/v0.1.0...v0.1.1 98 | [0.1.0]: https://github.com/umarbutler/orjsonl/releases/tag/v0.1.0 99 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022 Umar Butler 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # orjsonl 2 | 3 | 4 | `orjsonl` is a lightweight, high-performance Python library for parsing jsonl files. It supports a wide variety of compression formats, including gzip, bzip2, xz and Zstandard. It is powered by [`orjson`](https://github.com/ijl/orjson), the fastest and most accurate json serializer for Python. 5 | 6 | ## Installation 7 | `orjsonl` may be installed with `pip`: 8 | ```bash 9 | pip install orjsonl 10 | ``` 11 | 12 | To read or write Zstandard files, install either [`zstd`](https://github.com/facebook/zstd) or the [`zstandard`](https://pypi.org/project/zstandard/) Python package. 13 | 14 | ## Usage 15 | The code snippet below demonstrates how jsonl files can be saved, loaded, streamed, appended and extended with `orjsonl`: 16 | ```python 17 | >>> import orjsonl 18 | >>> # Create an iterable of Python objects. 19 | >>> data = [ 20 | 'hello world', 21 | ['fizz', 'buzz'], 22 | ] 23 | >>> # Save the iterable to a jsonl file. 24 | >>> orjsonl.save('test.jsonl', data) 25 | >>> # Append a Python object to the jsonl file. 26 | >>> orjsonl.append('test.jsonl', {42 : 3.14}) 27 | >>> # Extend the jsonl file with an iterable of Python objects. 28 | >>> orjsonl.extend('test.jsonl', [True, False]) 29 | >>> # Load the jsonl file. 30 | >>> orjsonl.load('test.jsonl') 31 | ['hello world', ['fizz', 'buzz'], {42 : 3.14}, True, False] 32 | >>> # Stream the jsonl file. 33 | >>> list(orjsonl.stream('test.jsonl')) 34 | ['hello world', ['fizz', 'buzz'], {42 : 3.14}, True, False] 35 | ``` 36 | 37 | `orjsonl` can also be used to process jsonl files compressed with gzip, bzip2, xz and Zstandard: 38 | ```python 39 | >>> orjsonl.save('test.jsonl.gz', data) 40 | >>> orjsonl.append('test.jsonl.gz', {42 : 3.14}) 41 | >>> orjsonl.extend('test.jsonl.gz', [True, False]) 42 | >>> orjsonl.load('test.jsonl.gz') 43 | ['hello world', ['fizz', 'buzz'], {42 : 3.14}, True, False] 44 | >>> [obj for obj in orjsonl.stream('test.jsonl.gz')] 45 | ['hello world', ['fizz', 'buzz'], {42 : 3.14}, True, False] 46 | ``` 47 | 48 | ### Load 49 | ```python 50 | def load( 51 | path: str | bytes | int | os.PathLike, 52 | decompression_threads: Optional[int] = None, 53 | compression_format: Optional[str] = None 54 | ) -> list[dict | list | int | float | str | bool | None] 55 | ``` 56 | 57 | `load()` deserializes a compressed or uncompressed UTF-8-encoded jsonl file to a list of Python objects. 58 | 59 | `path` is a path-like object giving the pathname (absolute or relative to the current working directory) of the compressed or uncompressed UTF-8-encoded jsonl file to be deserialized. 60 | 61 | `decompression_threads` is an optional integer passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`threads`](https://github.com/pycompression/xopen/#xopen) argument that specifies the number of threads that should be used for decompression. 62 | 63 | `compression_format` is an optional string passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`format`](https://github.com/pycompression/xopen/#v130-2022-01-10) argument that overrides the autodetection of the file’s compression format based on its extension or content. Possible values are ‘gz’, ‘xz’, ‘bz2’ and ‘zst’. 64 | 65 | This function returns a `list` object comprised of deserialized `dict`, `list`, `int`, `float`, `str`, `bool` or `None` objects. 66 | 67 | ### Stream 68 | ```python 69 | def stream( 70 | path: str | bytes | int | os.PathLike, 71 | decompression_threads: Optional[int] = None, 72 | compression_format: Optional[str] = None 73 | ) -> Generator[dict | list | int | float | str | bool | None, None, None] 74 | ``` 75 | 76 | `stream()` creates a `generator` that deserializes a compressed or uncompressed UTF-8-encoded jsonl file to Python objects. 77 | 78 | `path` is a path-like object giving the pathname (absolute or relative to the current working directory) of the compressed or uncompressed UTF-8-encoded jsonl file to be deserialized by the `generator`. 79 | 80 | `decompression_threads` is an optional integer passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`threads`](https://github.com/pycompression/xopen/#xopen) argument that specifies the number of threads that should be used for decompression. 81 | 82 | `compression_format` is an optional string passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`format`](https://github.com/pycompression/xopen/#v130-2022-01-10) argument that overrides the autodetection of the file’s compression format based on its extension or content. Possible values are ‘gz’, ‘xz’, ‘bz2’ and ‘zst’. 83 | 84 | This function returns a `generator` that deserializes the file to `dict`, `list`, `int`, `float`, `str`, `bool` or `None` objects. 85 | 86 | ### Save 87 | ```python 88 | def save( 89 | path: str | bytes | int | os.PathLike, 90 | data: Iterable, 91 | default: Optional[Callable] = None, 92 | option: int = 0, 93 | compression_level: Optional[int] = None, 94 | compression_threads: Optional[int] = None, 95 | compression_format: Optional[str] = None 96 | ) -> None 97 | ``` 98 | 99 | `save()` serializes an iterable of Python objects to a compressed or uncompressed UTF-8-encoded jsonl file. 100 | 101 | `path` is a path-like object giving the pathname (absolute or relative to the current working directory) of the compressed or uncompressed UTF-8-encoded jsonl file to be saved. 102 | 103 | `data` is an iterable of Python objects to be serialized to the file. 104 | 105 | `default` is an optional callable passed to [`orjson.dumps()`](https://github.com/ijl/orjson#serialize) as the [`default`](https://github.com/ijl/orjson#default) argument that serializes subclasses or arbitrary types to supported types. 106 | 107 | `option` is an optional integer passed to [`orjson.dumps()`](https://github.com/ijl/orjson#serialize) as the [`option`](https://github.com/ijl/orjson#option) argument that modifies how data is serialized. 108 | 109 | `compression_level` is an optional integer passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`compresslevel`](https://github.com/pycompression/xopen/#usage) argument that determines the compression level for writing to gzip, xz and Zstandard files. 110 | 111 | `compression_threads` is an optional integer passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`threads`](https://github.com/pycompression/xopen/#xopen) argument that specifies the number of threads that should be used for compression. 112 | 113 | `compression_format` is an optional string passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`format`](https://github.com/pycompression/xopen/#v130-2022-01-10) argument that overrides the autodetection of the file’s compression format based on its extension. Possible values are ‘gz’, ‘xz’, ‘bz2’ and ‘zst’. 114 | 115 | ### Append 116 | ```python 117 | def append( 118 | path: str | bytes | int | os.PathLike, 119 | data: Any, 120 | newline: bool = True, 121 | default: Optional[Callable] = None, 122 | option: int = 0, 123 | compression_level: Optional[int] = None, 124 | compression_threads: Optional[int] = None, 125 | compression_format: Optional[str] = None 126 | ) -> None 127 | ``` 128 | 129 | `append()` serializes and appends a Python object to a compressed or uncompressed UTF-8-encoded jsonl file. 130 | 131 | `path` is a path-like object giving the pathname (absolute or relative to the current working directory) of the compressed or uncompressed UTF-8-encoded jsonl file to be appended. 132 | 133 | `data` is a Python object to be serialized and appended to the file. 134 | 135 | `newline` is an optional Boolean flag that, if set to `False`, indicates that the file does not end with a newline and should, therefore, have one added before data is appended. 136 | 137 | `default` is an optional callable passed to [`orjson.dumps()`](https://github.com/ijl/orjson#serialize) as the [`default`](https://github.com/ijl/orjson#default) argument that serializes subclasses or arbitrary types to supported types. 138 | 139 | `option` is an optional integer passed to [`orjson.dumps()`](https://github.com/ijl/orjson#serialize) as the [`option`](https://github.com/ijl/orjson#option) argument that modifies how data is serialized. 140 | 141 | `compression_level` is an optional integer passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`compresslevel`](https://github.com/pycompression/xopen/#usage) argument that determines the compression level for writing to gzip, xz and Zstandard files. 142 | 143 | `compression_threads` is an optional integer passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`threads`](https://github.com/pycompression/xopen/#xopen) argument that specifies the number of threads that should be used for compression. 144 | 145 | `compression_format` is an optional string passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`format`](https://github.com/pycompression/xopen/#v130-2022-01-10) argument that overrides the autodetection of the file’s compression format based on its extension or content. Possible values are ‘gz’, ‘xz’, ‘bz2’ and ‘zst’. 146 | 147 | ### Extend 148 | ```python 149 | def extend( 150 | path: str | bytes | int | os.PathLike, 151 | data: Iterable, 152 | newline: bool = True, 153 | default: Optional[Callable] = None, 154 | option: int = 0, 155 | compression_level: Optional[int] = None, 156 | compression_threads: Optional[int] = None, 157 | compression_format: Optional[str] = None 158 | ) -> None 159 | ``` 160 | 161 | `extend()` serializes and appends an iterable of Python objects to a compressed or uncompressed UTF-8-encoded jsonl file. 162 | 163 | `path` is a path-like object giving the pathname (absolute or relative to the current working directory) of the compressed or uncompressed UTF-8-encoded jsonl file to be extended. 164 | 165 | `data` is an iterable of Python objects to be serialized and appended to the file. 166 | 167 | `newline` is an optional Boolean flag that, if set to `False`, indicates that the file does not end with a newline and should, therefore, have one added before data is extended. 168 | 169 | `default` is an optional callable passed to [`orjson.dumps()`](https://github.com/ijl/orjson#serialize) as the [`default`](https://github.com/ijl/orjson#default) argument that serializes subclasses or arbitrary types to supported types. 170 | 171 | `option` is an optional integer passed to [`orjson.dumps()`](https://github.com/ijl/orjson#serialize) as the [`option`](https://github.com/ijl/orjson#option) argument that modifies how data is serialized. 172 | 173 | `compression_level` is an optional integer passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`compresslevel`](https://github.com/pycompression/xopen/#usage) argument that determines the compression level for writing to gzip, xz and Zstandard files. 174 | 175 | `compression_threads` is an optional integer passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`threads`](https://github.com/pycompression/xopen/#xopen) argument that specifies the number of threads that should be used for compression. 176 | 177 | `compression_format` is an optional string passed to [`xopen.xopen()`](https://github.com/pycompression/xopen/#xopen) as the [`format`](https://github.com/pycompression/xopen/#v130-2022-01-10) argument that overrides the autodetection of the file’s compression format based on its extension or content. Possible values are ‘gz’, ‘xz’, ‘bz2’ and ‘zst’. 178 | 179 | ## License 180 | This library is licensed under the [MIT License](https://github.com/umarbutler/orjsonl/blob/main/LICENSE). 181 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "orjsonl" 7 | version = "1.0.0" 8 | authors = [ 9 | {name="Umar Butler", email="umar@umar.au"}, 10 | ] 11 | description = "A lightweight, high-performance Python library for parsing jsonl files." 12 | readme = "README.md" 13 | requires-python = ">=3.7" 14 | license = {text="MIT"} 15 | keywords = [ 16 | "json", 17 | "jsonl", 18 | "jsonlines", 19 | "orjson", 20 | "lines", 21 | "json lines", 22 | "fast", 23 | "ndjson", 24 | "gzip", 25 | "bzip2", 26 | "xz", 27 | "zstandard", 28 | "xopen" 29 | ] 30 | classifiers = [ 31 | "Development Status :: 5 - Production/Stable", 32 | "Intended Audience :: Developers", 33 | "Intended Audience :: Information Technology", 34 | "Intended Audience :: Science/Research", 35 | "License :: OSI Approved :: MIT License", 36 | "Operating System :: OS Independent", 37 | "Programming Language :: Python :: 3", 38 | "Programming Language :: Python :: 3.7", 39 | "Programming Language :: Python :: 3.8", 40 | "Programming Language :: Python :: 3.9", 41 | "Programming Language :: Python :: 3.10", 42 | "Programming Language :: Python :: 3.11", 43 | "Programming Language :: Python :: 3.12", 44 | "Programming Language :: Python :: Implementation :: CPython", 45 | "Topic :: Scientific/Engineering", 46 | "Topic :: Software Development :: Libraries :: Python Modules", 47 | "Topic :: Utilities", 48 | "Typing :: Typed" 49 | ] 50 | dependencies = [ 51 | "orjson>=3.7.7", 52 | "xopen>=1.7.0" 53 | ] 54 | 55 | [project.urls] 56 | Homepage = "https://github.com/umarbutler/orjsonl" 57 | Documentation = "https://github.com/umarbutler/orjsonl/blob/main/README.md" 58 | Issues = "https://github.com/umarbutler/orjsonl/issues" 59 | Source = "https://github.com/umarbutler/orjsonl" 60 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | orjson>=3.7.7 2 | xopen>=1.7.0 -------------------------------------------------------------------------------- /src/orjsonl/__init__.py: -------------------------------------------------------------------------------- 1 | """A lightweight, high-performance Python library for parsing jsonl files.""" 2 | 3 | from .orjsonl import ( 4 | stream, 5 | load, 6 | save, 7 | append, 8 | extend, 9 | ) 10 | -------------------------------------------------------------------------------- /src/orjsonl/orjsonl.py: -------------------------------------------------------------------------------- 1 | from typing import ( 2 | Union, 3 | Generator, 4 | List, 5 | Optional, 6 | Callable, 7 | Any 8 | ) 9 | import os 10 | from xopen import xopen 11 | import orjson 12 | from collections.abc import Iterable 13 | 14 | 15 | def load( 16 | path: Union[str, bytes, os.PathLike], 17 | decompression_threads: Optional[int] = None, 18 | compression_format: Optional[str] = None 19 | ) -> List[Union[dict, list, int, float, str, bool, None]]: 20 | """Deserialize a compressed or uncompressed UTF-8-encoded jsonl file to a list of Python objects. 21 | 22 | Args: 23 | path (str | bytes | os.PathLike): A path-like object giving the pathname (absolute or relative to the current working directory) of the compressed or uncompressed UTF-8-encoded jsonl file to be deserialized. 24 | decompression_threads (int, optional): An optional integer passed to xopen.xopen() as the 'threads' argument that specifies the number of threads that should be used for decompression. Defaults to None. 25 | compression_format (str, optional): An optional string passed to xopen.xopen() as the 'format' argument that overrides the autodetection of the file's compression format based on its extension or content. Possible values are 'gz', 'xz', 'bz2' and 'zst'. Defaults to None. 26 | 27 | Returns: 28 | list[dict | list | int | float | str | bool | None]: A list of deserialized objects.""" 29 | 30 | with xopen(path, 'rb', threads=decompression_threads, format=compression_format) as file: 31 | return [orjson.loads(json) for json in file] 32 | 33 | 34 | def stream( 35 | path: Union[str, bytes, os.PathLike], 36 | decompression_threads: Optional[int] = None, 37 | compression_format: Optional[str] = None 38 | ) -> Generator[Union[dict, list, int, float, str, bool, None], None, None]: 39 | """Create a generator that deserializes a compressed or uncompressed UTF-8-encoded jsonl file to Python objects. 40 | 41 | Args: 42 | path (str | bytes | os.PathLike): A path-like object giving the pathname (absolute or relative to the current working directory) of the compressed or uncompressed UTF-8-encoded jsonl file to be deserialized by the generator. 43 | decompression_threads (int, optional): An optional integer passed to xopen.xopen() as the 'threads' argument that specifies the number of threads that should be used for decompression. Defaults to None. 44 | compression_format (str, optional): An optional string passed to xopen.xopen() as the 'format' argument that overrides the autodetection of the file's compression format based on its extension or content. Possible values are 'gz', 'xz', 'bz2' and 'zst'. Defaults to None. 45 | 46 | Returns: 47 | Generator[dict | list | int | float | str | bool | None, None, None]: A generator that deserializes the file to Python objects.""" 48 | 49 | with xopen(path, 'rb', threads=decompression_threads, format=compression_format) as file: 50 | for json in file: 51 | yield orjson.loads(json) 52 | 53 | 54 | def save( 55 | path: Union[str, bytes, os.PathLike], 56 | data: Iterable, 57 | default: Optional[Callable] = None, 58 | option: int = 0, 59 | compression_level: Optional[int] = None, 60 | compression_threads: Optional[int] = None, 61 | compression_format: Optional[str] = None 62 | ) -> None: 63 | """Serialize an iterable of Python objects to a compressed or uncompressed UTF-8-encoded jsonl file. 64 | 65 | Args: 66 | path (str | bytes | os.PathLike): A path-like object giving the pathname (absolute or relative to the current working directory) of the compressed or uncompressed UTF-8-encoded jsonl file to be saved. 67 | data (Iterable): An iterable of Python objects to be serialized to the file. 68 | default (Callable, optional): An optional callable passed to orjson.dumps() as the 'default' argument that serializes subclasses or arbitrary types to supported types. Defaults to None. 69 | option (int, optional): An optional integer passed to orjson.dumps() as the 'option' argument that modifies how data is serialized. Defaults to 0. 70 | compression_level (int, optional): An optional integer passed to xopen.xopen() as the 'compresslevel' argument that determines the compression level for writing to gzip, xz and Zstandard files. Defaults to None. 71 | compression_threads (int, optional): An optional integer passed to xopen.xopen() as the 'threads' argument that specifies the number of threads that should be used for compression. Defaults to None. 72 | compression_format (str, optional): An optional string passed to xopen.xopen() as the 'format' argument that overrides the autodetection of the file's compression format based on its extension. Possible values are 'gz', 'xz', 'bz2' and 'zst'. Defaults to None.""" 73 | 74 | with xopen(path, 'wb', compresslevel=compression_level, threads=compression_threads, format=compression_format) as writer: 75 | for item in data: 76 | writer.write(orjson.dumps(item, default=default, option=option)) 77 | writer.write(b'\n') 78 | 79 | 80 | def append( 81 | path: Union[str, bytes, os.PathLike], 82 | data: Any, 83 | newline: Optional[bool] = True, 84 | default: Optional[Callable] = None, 85 | option: int = 0, 86 | compression_level: Optional[int] = None, 87 | compression_threads: Optional[int] = None, 88 | compression_format: Optional[str] = None 89 | ) -> None: 90 | """Serialize and append a Python object to a compressed or uncompressed UTF-8-encoded jsonl file. 91 | 92 | Args: 93 | path (str | bytes | os.PathLike): A path-like object giving the pathname (absolute or relative to the current working directory) of the compressed or uncompressed UTF-8-encoded jsonl file to be appended. 94 | data (Iterable): A Python object to be serialized and appended to the file. 95 | newline (bool, optional): An optional Boolean flag that, if set to False, indicates that the file does not end with a newline and should, therefore, have one added before data is appended. Defaults to True. 96 | default (Callable, optional): An optional callable passed to orjson.dumps() as the 'default' argument that serializes subclasses or arbitrary types to supported types. Defaults to None. 97 | option (int, optional): An optional integer passed to orjson.dumps() as the 'option' argument that modifies how data is serialized. Defaults to 0. 98 | compression_level (int, optional): An optional integer passed to xopen.xopen() as the 'compresslevel' argument that determines the compression level for writing to gzip, xz and Zstandard files. Defaults to None. 99 | compression_threads (int, optional): An optional integer passed to xopen.xopen() as the 'threads' argument that specifies the number of threads that should be used for compression. Defaults to None. 100 | compression_format (str, optional): An optional string passed to xopen.xopen() as the 'format' argument that overrides the autodetection of the file's compression format based on its extension or content. Possible values are 'gz', 'xz', 'bz2' and 'zst'. Defaults to None.""" 101 | 102 | with xopen(path, 'ab', compresslevel=compression_level, threads=compression_threads, format=compression_format) as writer: 103 | if not newline: 104 | writer.write(b'\n') 105 | 106 | writer.write(orjson.dumps(data, default=default, option=option)) 107 | writer.write(b'\n') 108 | 109 | 110 | def extend( 111 | path: Union[str, bytes, os.PathLike], 112 | data: Iterable, 113 | newline: Optional[bool] = True, 114 | default: Optional[Callable] = None, 115 | option: int = 0, 116 | compression_level: Optional[int] = None, 117 | compression_threads: Optional[int] = None, 118 | compression_format: Optional[str] = None 119 | ) -> None: 120 | """Serialize and append an iterable of Python objects to a compressed or uncompressed UTF-8-encoded jsonl file. 121 | 122 | Args: 123 | path (str | bytes | os.PathLike): A path-like object giving the pathname (absolute or relative to the current working directory) of the compressed or uncompressed UTF-8-encoded jsonl file to be extended. 124 | data (Iterable): An iterable of Python objects to be serialized and appended to the file. 125 | newline (bool, optional): An optional Boolean flag that, if set to False, indicates that the file does not end with a newline and should, therefore, have one added before data is extended. Defaults to True. 126 | default (Callable, optional): An optional callable passed to orjson.dumps() as the 'default' argument that serializes subclasses or arbitrary types to supported types. Defaults to None. 127 | option (int, optional): An optional integer passed to orjson.dumps() as the 'option' argument that modifies how data is serialized. Defaults to 0. 128 | compression_level (int, optional): An optional integer passed to xopen.xopen() as the 'compresslevel' argument that determines the compression level for writing to gzip, xz and Zstandard files. Defaults to None. 129 | compression_threads (int, optional): An optional integer passed to xopen.xopen() as the 'threads' argument that specifies the number of threads that should be used for compression. Defaults to None. 130 | compression_format (str, optional): An optional string passed to xopen.xopen() as the 'format' argument that overrides the autodetection of the file's compression format based on its extension or content. Possible values are 'gz', 'xz', 'bz2' and 'zst'. Defaults to None.""" 131 | 132 | with xopen(path, 'ab', compresslevel=compression_level, threads=compression_threads, format=compression_format) as writer: 133 | if not newline: 134 | writer.write(b'\n') 135 | 136 | for item in data: 137 | writer.write(orjson.dumps(item, default=default, option=option)) 138 | writer.write(b'\n') 139 | -------------------------------------------------------------------------------- /src/orjsonl/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/umarbutler/orjsonl/055e18597983ab82e263988a751b5f28c3f023b5/src/orjsonl/py.typed -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | def save(path: str, data: Any) -> None: 4 | """Save a file.""" 5 | 6 | with open(path, 'wb') as file: 7 | file.write(data) 8 | 9 | 10 | def load(path: str) -> Any: 11 | """Load a file.""" 12 | 13 | with open(path, 'rb') as file: 14 | return file.read() -------------------------------------------------------------------------------- /tests/test_orjsonl.py: -------------------------------------------------------------------------------- 1 | """Test orjsonl.""" 2 | 3 | import orjsonl 4 | import helpers 5 | import orjson 6 | 7 | DATA = [ 8 | 'hello world', 9 | ['fizz', 'buzz'], 10 | ] 11 | 12 | JSONL = """\ 13 | "hello world" 14 | ["fizz","buzz"] 15 | """ 16 | 17 | 18 | def test_save(tmp_path): 19 | """Test `orjsonl.save()`.""" 20 | 21 | orjsonl.save(tmp_path / 'test_orjsonl.jsonl', data=DATA) 22 | assert helpers.load(tmp_path / 'test_orjsonl.jsonl') == JSONL.encode('utf-8') 23 | 24 | 25 | def test_load(tmp_path) -> None: 26 | """Test `orjsonl.load()`.""" 27 | 28 | helpers.save(tmp_path / 'test_orjsonl.jsonl', JSONL.encode('utf-8')) 29 | assert orjsonl.load(tmp_path / 'test_orjsonl.jsonl') == DATA 30 | 31 | 32 | def test_stream(tmp_path) -> None: 33 | """Test `orjsonl.stream()`.""" 34 | 35 | helpers.save(tmp_path / 'test_orjsonl.jsonl', JSONL.encode('utf-8')) 36 | assert list(orjsonl.stream(tmp_path / 'test_orjsonl.jsonl')) == DATA 37 | 38 | 39 | def test_append(tmp_path) -> None: 40 | """Test `orjsonl.append()`.""" 41 | 42 | helpers.save(tmp_path / 'test_orjsonl.jsonl', JSONL.encode('utf-8')) 43 | orjsonl.append(tmp_path / 'test_orjsonl.jsonl', data=DATA[-1]) 44 | assert helpers.load(tmp_path / 'test_orjsonl.jsonl') == JSONL.encode('utf-8') + orjson.dumps(DATA[-1]) + b'\n' 45 | 46 | helpers.save(tmp_path / 'test_orjsonl.jsonl', JSONL.encode('utf-8')) 47 | orjsonl.append(tmp_path / 'test_orjsonl.jsonl', data=DATA[-1], newline=False) 48 | assert helpers.load(tmp_path / 'test_orjsonl.jsonl') == JSONL.encode('utf-8') + b'\n' + orjson.dumps(DATA[-1]) + b'\n' 49 | 50 | 51 | def test_extend(tmp_path) -> None: 52 | """Test `orjsonl.extend()`.""" 53 | 54 | helpers.save(tmp_path / 'test_orjsonl.jsonl', JSONL.encode('utf-8')) 55 | orjsonl.extend(tmp_path / 'test_orjsonl.jsonl', data=DATA[-2:]) 56 | assert helpers.load(tmp_path / 'test_orjsonl.jsonl') == JSONL.encode('utf-8') + orjson.dumps(DATA[-2]) + b'\n' + orjson.dumps(DATA[-1]) + b'\n' 57 | 58 | 59 | helpers.save(tmp_path / 'test_orjsonl.jsonl', JSONL.encode('utf-8')) 60 | orjsonl.extend(tmp_path / 'test_orjsonl.jsonl', data=DATA[-2:], newline=False) 61 | assert helpers.load(tmp_path / 'test_orjsonl.jsonl') == JSONL.encode('utf-8') + b'\n' + orjson.dumps(DATA[-2]) + b'\n' + orjson.dumps(DATA[-1]) + b'\n' --------------------------------------------------------------------------------