├── .codecov.yml ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── documentation.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml ├── stale.yml └── workflows │ ├── check-typing.yml │ ├── ci-checks.yml │ ├── ci-testing.yml │ ├── cleanup-caches.yml │ ├── docs-build.yml │ ├── greetings.yml │ └── release-pypi.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .prettierignore ├── .readthedocs.yml ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── docs ├── .build_docs.sh ├── Makefile ├── make.bat └── source │ ├── _static │ ├── copybutton.js │ └── images │ │ ├── icon.svg │ │ ├── logo-large.svg │ │ ├── logo-small.svg │ │ ├── logo.png │ │ └── logo.svg │ ├── _templates │ └── theme_variables.jinja │ ├── conf.py │ └── index.rst ├── pyproject.toml ├── requirements.txt ├── requirements ├── docs.txt ├── extras.txt └── test.txt ├── setup.py ├── src └── litdata │ ├── CHANGELOG.md │ ├── __about__.py │ ├── __init__.py │ ├── constants.py │ ├── imports.py │ ├── processing │ ├── __init__.py │ ├── data_processor.py │ ├── functions.py │ ├── readers.py │ └── utilities.py │ ├── streaming │ ├── __init__.py │ ├── cache.py │ ├── client.py │ ├── combined.py │ ├── compression.py │ ├── config.py │ ├── dataloader.py │ ├── dataset.py │ ├── downloader.py │ ├── item_loader.py │ ├── reader.py │ ├── resolver.py │ ├── sampler.py │ ├── serializers.py │ ├── shuffle.py │ └── writer.py │ └── utilities │ ├── __init__.py │ ├── broadcast.py │ ├── env.py │ ├── format.py │ ├── packing.py │ └── shuffle.py └── tests ├── __init__.py ├── conftest.py ├── processing ├── __init__.py ├── test_data_processor.py ├── test_functions.py ├── test_readers.py └── test_utilities.py ├── streaming ├── __init__.py ├── test_cache.py ├── test_client.py ├── test_combined.py ├── test_dataloader.py ├── test_dataset.py ├── test_downloader.py ├── test_reader.py ├── test_resolver.py ├── test_sampler.py ├── test_serializer.py └── test_writer.py └── utilities ├── __init__.py ├── test_broadcast.py ├── test_format.py ├── test_packing.py └── test_shuffle.py /.codecov.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.codecov.yml -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/CODEOWNERS -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/ISSUE_TEMPLATE/bug_report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/documentation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/ISSUE_TEMPLATE/documentation.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/ISSUE_TEMPLATE/feature_request.md -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/PULL_REQUEST_TEMPLATE.md -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/stale.yml -------------------------------------------------------------------------------- /.github/workflows/check-typing.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/workflows/check-typing.yml -------------------------------------------------------------------------------- /.github/workflows/ci-checks.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/workflows/ci-checks.yml -------------------------------------------------------------------------------- /.github/workflows/ci-testing.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/workflows/ci-testing.yml -------------------------------------------------------------------------------- /.github/workflows/cleanup-caches.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/workflows/cleanup-caches.yml -------------------------------------------------------------------------------- /.github/workflows/docs-build.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/workflows/docs-build.yml -------------------------------------------------------------------------------- /.github/workflows/greetings.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/workflows/greetings.yml -------------------------------------------------------------------------------- /.github/workflows/release-pypi.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.github/workflows/release-pypi.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | # Ignore all MD files: 2 | **/*.md 3 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/.readthedocs.yml -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/README.md -------------------------------------------------------------------------------- /docs/.build_docs.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/.build_docs.sh -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/make.bat -------------------------------------------------------------------------------- /docs/source/_static/copybutton.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/source/_static/copybutton.js -------------------------------------------------------------------------------- /docs/source/_static/images/icon.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/source/_static/images/icon.svg -------------------------------------------------------------------------------- /docs/source/_static/images/logo-large.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/source/_static/images/logo-large.svg -------------------------------------------------------------------------------- /docs/source/_static/images/logo-small.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/source/_static/images/logo-small.svg -------------------------------------------------------------------------------- /docs/source/_static/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/source/_static/images/logo.png -------------------------------------------------------------------------------- /docs/source/_static/images/logo.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/source/_static/images/logo.svg -------------------------------------------------------------------------------- /docs/source/_templates/theme_variables.jinja: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/source/_templates/theme_variables.jinja -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/source/conf.py -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/docs/source/index.rst -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/pyproject.toml -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch >=2.1.0 2 | filelock 3 | numpy 4 | boto3[crt] 5 | requests 6 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/requirements/docs.txt -------------------------------------------------------------------------------- /requirements/extras.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/requirements/extras.txt -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/requirements/test.txt -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/setup.py -------------------------------------------------------------------------------- /src/litdata/CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/CHANGELOG.md -------------------------------------------------------------------------------- /src/litdata/__about__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/__about__.py -------------------------------------------------------------------------------- /src/litdata/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/__init__.py -------------------------------------------------------------------------------- /src/litdata/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/constants.py -------------------------------------------------------------------------------- /src/litdata/imports.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/imports.py -------------------------------------------------------------------------------- /src/litdata/processing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/processing/__init__.py -------------------------------------------------------------------------------- /src/litdata/processing/data_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/processing/data_processor.py -------------------------------------------------------------------------------- /src/litdata/processing/functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/processing/functions.py -------------------------------------------------------------------------------- /src/litdata/processing/readers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/processing/readers.py -------------------------------------------------------------------------------- /src/litdata/processing/utilities.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/processing/utilities.py -------------------------------------------------------------------------------- /src/litdata/streaming/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/__init__.py -------------------------------------------------------------------------------- /src/litdata/streaming/cache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/cache.py -------------------------------------------------------------------------------- /src/litdata/streaming/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/client.py -------------------------------------------------------------------------------- /src/litdata/streaming/combined.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/combined.py -------------------------------------------------------------------------------- /src/litdata/streaming/compression.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/compression.py -------------------------------------------------------------------------------- /src/litdata/streaming/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/config.py -------------------------------------------------------------------------------- /src/litdata/streaming/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/dataloader.py -------------------------------------------------------------------------------- /src/litdata/streaming/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/dataset.py -------------------------------------------------------------------------------- /src/litdata/streaming/downloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/downloader.py -------------------------------------------------------------------------------- /src/litdata/streaming/item_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/item_loader.py -------------------------------------------------------------------------------- /src/litdata/streaming/reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/reader.py -------------------------------------------------------------------------------- /src/litdata/streaming/resolver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/resolver.py -------------------------------------------------------------------------------- /src/litdata/streaming/sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/sampler.py -------------------------------------------------------------------------------- /src/litdata/streaming/serializers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/serializers.py -------------------------------------------------------------------------------- /src/litdata/streaming/shuffle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/shuffle.py -------------------------------------------------------------------------------- /src/litdata/streaming/writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/streaming/writer.py -------------------------------------------------------------------------------- /src/litdata/utilities/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/utilities/__init__.py -------------------------------------------------------------------------------- /src/litdata/utilities/broadcast.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/utilities/broadcast.py -------------------------------------------------------------------------------- /src/litdata/utilities/env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/utilities/env.py -------------------------------------------------------------------------------- /src/litdata/utilities/format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/utilities/format.py -------------------------------------------------------------------------------- /src/litdata/utilities/packing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/utilities/packing.py -------------------------------------------------------------------------------- /src/litdata/utilities/shuffle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/src/litdata/utilities/shuffle.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/processing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/processing/test_data_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/processing/test_data_processor.py -------------------------------------------------------------------------------- /tests/processing/test_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/processing/test_functions.py -------------------------------------------------------------------------------- /tests/processing/test_readers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/processing/test_readers.py -------------------------------------------------------------------------------- /tests/processing/test_utilities.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/processing/test_utilities.py -------------------------------------------------------------------------------- /tests/streaming/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/streaming/test_cache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/streaming/test_cache.py -------------------------------------------------------------------------------- /tests/streaming/test_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/streaming/test_client.py -------------------------------------------------------------------------------- /tests/streaming/test_combined.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/streaming/test_combined.py -------------------------------------------------------------------------------- /tests/streaming/test_dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/streaming/test_dataloader.py -------------------------------------------------------------------------------- /tests/streaming/test_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/streaming/test_dataset.py -------------------------------------------------------------------------------- /tests/streaming/test_downloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/streaming/test_downloader.py -------------------------------------------------------------------------------- /tests/streaming/test_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/streaming/test_reader.py -------------------------------------------------------------------------------- /tests/streaming/test_resolver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/streaming/test_resolver.py -------------------------------------------------------------------------------- /tests/streaming/test_sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/streaming/test_sampler.py -------------------------------------------------------------------------------- /tests/streaming/test_serializer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/streaming/test_serializer.py -------------------------------------------------------------------------------- /tests/streaming/test_writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/streaming/test_writer.py -------------------------------------------------------------------------------- /tests/utilities/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/utilities/test_broadcast.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/utilities/test_broadcast.py -------------------------------------------------------------------------------- /tests/utilities/test_format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/utilities/test_format.py -------------------------------------------------------------------------------- /tests/utilities/test_packing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/utilities/test_packing.py -------------------------------------------------------------------------------- /tests/utilities/test_shuffle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rasbt/litdata/HEAD/tests/utilities/test_shuffle.py --------------------------------------------------------------------------------