├── .codecov.yml ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md ├── benchmark │ ├── benchmark.py │ └── greet.md ├── dependabot.yml ├── markdown-links-config.json ├── stale.yml └── workflows │ ├── ci-benchmark.yml │ ├── ci-checks.yml │ ├── ci-testing.yml │ ├── cleanup-caches.yml │ ├── docs-build.yml │ ├── greetings.yml │ └── release-pypi.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── benchmarks ├── .gitignore ├── README.md ├── ffcv │ ├── README.md │ ├── convert_imagenet.py │ ├── install_ffcv.sh │ ├── stream_imagenet.py │ └── write_imagenet.py ├── litdata │ ├── README.md │ ├── optimize_imagenet.py │ └── stream_imagenet.py └── stream_raw_imagenet.py ├── docs ├── .build_docs.sh ├── Makefile ├── make.bat └── source │ ├── _static │ ├── copybutton.js │ └── images │ │ ├── icon.svg │ │ ├── logo-large.svg │ │ ├── logo-small.svg │ │ ├── logo.png │ │ └── logo.svg │ ├── _templates │ └── theme_variables.jinja │ ├── conf.py │ └── index.rst ├── examples ├── getting_started │ ├── README.md │ ├── generate.py │ └── stream.py ├── multi_modal │ ├── README.md │ ├── cl_run.py │ ├── config.py │ ├── convert.py │ ├── create_labelencoder.py │ ├── dataloader.py │ ├── generate.py │ ├── loop.py │ ├── model_arc.py │ ├── requirements.txt │ └── train.py └── sine_function_model_prediction │ ├── 01-optimize.py │ ├── 02-model_training.py │ ├── README.md │ └── main.ipynb ├── pyproject.toml ├── requirements.txt ├── requirements ├── docs.txt ├── extras.txt └── test.txt ├── setup.py ├── src └── litdata │ ├── CHANGELOG.md │ ├── __about__.py │ ├── __init__.py │ ├── __main__.py │ ├── cli │ ├── __init__.py │ ├── commands.py │ ├── handler │ │ ├── cache.py │ │ └── optimize.py │ └── parser.py │ ├── constants.py │ ├── debugger.py │ ├── helpers.py │ ├── imports.py │ ├── processing │ ├── __init__.py │ ├── data_processor.py │ ├── functions.py │ ├── readers.py │ └── utilities.py │ ├── raw │ ├── __init__.py │ ├── dataset.py │ ├── indexer.py │ └── types.py │ ├── streaming │ ├── __init__.py │ ├── cache.py │ ├── client.py │ ├── combined.py │ ├── compression.py │ ├── config.py │ ├── dataloader.py │ ├── dataset.py │ ├── downloader.py │ ├── fs_provider.py │ ├── item_loader.py │ ├── parallel.py │ ├── reader.py │ ├── resolver.py │ ├── sampler.py │ ├── serializers.py │ ├── shuffle.py │ └── writer.py │ └── utilities │ ├── __init__.py │ ├── _pytree.py │ ├── base.py │ ├── breakpoint.py │ ├── broadcast.py │ ├── dataset_utilities.py │ ├── encryption.py │ ├── env.py │ ├── format.py │ ├── hf_dataset.py │ ├── packing.py │ ├── parquet.py │ ├── shuffle.py │ ├── subsample.py │ ├── torch_utils.py │ └── train_test_split.py └── tests ├── __init__.py ├── conftest.py ├── processing ├── __init__.py ├── test_data_processor.py ├── test_functions.py ├── test_readers.py └── test_utilities.py ├── raw ├── __init__.py ├── test_dataset.py └── test_indexer.py ├── streaming ├── __init__.py ├── test_cache.py ├── test_client.py ├── test_combined.py ├── test_config.py ├── test_dataloader.py ├── test_dataset.py ├── test_downloader.py ├── test_fs_provider.py ├── test_item_loader.py ├── test_lock_cleanup.py ├── test_parallel.py ├── test_parquet.py ├── test_reader.py ├── test_resolver.py ├── test_sampler.py ├── test_serializer.py ├── test_writer.py └── utils.py ├── test_cli.py ├── test_debugger.py ├── test_helper.py └── utilities ├── __init__.py ├── test_broadcast.py ├── test_dataset_utilities.py ├── test_encryption.py ├── test_env.py ├── test_format.py ├── test_packing.py ├── test_shuffle.py ├── test_subsample.py └── test_train_test_split.py /.codecov.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.codecov.yml -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/CODEOWNERS -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/ISSUE_TEMPLATE/bug_report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/ISSUE_TEMPLATE/feature_request.md -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/PULL_REQUEST_TEMPLATE.md -------------------------------------------------------------------------------- /.github/benchmark/benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/benchmark/benchmark.py -------------------------------------------------------------------------------- /.github/benchmark/greet.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/benchmark/greet.md -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/markdown-links-config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/markdown-links-config.json -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/stale.yml -------------------------------------------------------------------------------- /.github/workflows/ci-benchmark.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/workflows/ci-benchmark.yml -------------------------------------------------------------------------------- /.github/workflows/ci-checks.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/workflows/ci-checks.yml -------------------------------------------------------------------------------- /.github/workflows/ci-testing.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/workflows/ci-testing.yml -------------------------------------------------------------------------------- /.github/workflows/cleanup-caches.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/workflows/cleanup-caches.yml -------------------------------------------------------------------------------- /.github/workflows/docs-build.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/workflows/docs-build.yml -------------------------------------------------------------------------------- /.github/workflows/greetings.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/workflows/greetings.yml -------------------------------------------------------------------------------- /.github/workflows/release-pypi.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.github/workflows/release-pypi.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/README.md -------------------------------------------------------------------------------- /benchmarks/.gitignore: -------------------------------------------------------------------------------- 1 | data 2 | -------------------------------------------------------------------------------- /benchmarks/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/benchmarks/README.md -------------------------------------------------------------------------------- /benchmarks/ffcv/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/benchmarks/ffcv/README.md -------------------------------------------------------------------------------- /benchmarks/ffcv/convert_imagenet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/benchmarks/ffcv/convert_imagenet.py -------------------------------------------------------------------------------- /benchmarks/ffcv/install_ffcv.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/benchmarks/ffcv/install_ffcv.sh -------------------------------------------------------------------------------- /benchmarks/ffcv/stream_imagenet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/benchmarks/ffcv/stream_imagenet.py -------------------------------------------------------------------------------- /benchmarks/ffcv/write_imagenet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/benchmarks/ffcv/write_imagenet.py -------------------------------------------------------------------------------- /benchmarks/litdata/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/benchmarks/litdata/README.md -------------------------------------------------------------------------------- /benchmarks/litdata/optimize_imagenet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/benchmarks/litdata/optimize_imagenet.py -------------------------------------------------------------------------------- /benchmarks/litdata/stream_imagenet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/benchmarks/litdata/stream_imagenet.py -------------------------------------------------------------------------------- /benchmarks/stream_raw_imagenet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/benchmarks/stream_raw_imagenet.py -------------------------------------------------------------------------------- /docs/.build_docs.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/.build_docs.sh -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/make.bat -------------------------------------------------------------------------------- /docs/source/_static/copybutton.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/source/_static/copybutton.js -------------------------------------------------------------------------------- /docs/source/_static/images/icon.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/source/_static/images/icon.svg -------------------------------------------------------------------------------- /docs/source/_static/images/logo-large.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/source/_static/images/logo-large.svg -------------------------------------------------------------------------------- /docs/source/_static/images/logo-small.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/source/_static/images/logo-small.svg -------------------------------------------------------------------------------- /docs/source/_static/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/source/_static/images/logo.png -------------------------------------------------------------------------------- /docs/source/_static/images/logo.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/source/_static/images/logo.svg -------------------------------------------------------------------------------- /docs/source/_templates/theme_variables.jinja: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/source/_templates/theme_variables.jinja -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/source/conf.py -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/docs/source/index.rst -------------------------------------------------------------------------------- /examples/getting_started/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/getting_started/README.md -------------------------------------------------------------------------------- /examples/getting_started/generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/getting_started/generate.py -------------------------------------------------------------------------------- /examples/getting_started/stream.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/getting_started/stream.py -------------------------------------------------------------------------------- /examples/multi_modal/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/multi_modal/README.md -------------------------------------------------------------------------------- /examples/multi_modal/cl_run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/multi_modal/cl_run.py -------------------------------------------------------------------------------- /examples/multi_modal/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/multi_modal/config.py -------------------------------------------------------------------------------- /examples/multi_modal/convert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/multi_modal/convert.py -------------------------------------------------------------------------------- /examples/multi_modal/create_labelencoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/multi_modal/create_labelencoder.py -------------------------------------------------------------------------------- /examples/multi_modal/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/multi_modal/dataloader.py -------------------------------------------------------------------------------- /examples/multi_modal/generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/multi_modal/generate.py -------------------------------------------------------------------------------- /examples/multi_modal/loop.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/multi_modal/loop.py -------------------------------------------------------------------------------- /examples/multi_modal/model_arc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/multi_modal/model_arc.py -------------------------------------------------------------------------------- /examples/multi_modal/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/multi_modal/requirements.txt -------------------------------------------------------------------------------- /examples/multi_modal/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/multi_modal/train.py -------------------------------------------------------------------------------- /examples/sine_function_model_prediction/01-optimize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/sine_function_model_prediction/01-optimize.py -------------------------------------------------------------------------------- /examples/sine_function_model_prediction/02-model_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/sine_function_model_prediction/02-model_training.py -------------------------------------------------------------------------------- /examples/sine_function_model_prediction/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/sine_function_model_prediction/README.md -------------------------------------------------------------------------------- /examples/sine_function_model_prediction/main.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/examples/sine_function_model_prediction/main.ipynb -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/pyproject.toml -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/requirements.txt -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/requirements/docs.txt -------------------------------------------------------------------------------- /requirements/extras.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/requirements/extras.txt -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/requirements/test.txt -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/setup.py -------------------------------------------------------------------------------- /src/litdata/CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/CHANGELOG.md -------------------------------------------------------------------------------- /src/litdata/__about__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/__about__.py -------------------------------------------------------------------------------- /src/litdata/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/__init__.py -------------------------------------------------------------------------------- /src/litdata/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/__main__.py -------------------------------------------------------------------------------- /src/litdata/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/cli/__init__.py -------------------------------------------------------------------------------- /src/litdata/cli/commands.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/cli/commands.py -------------------------------------------------------------------------------- /src/litdata/cli/handler/cache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/cli/handler/cache.py -------------------------------------------------------------------------------- /src/litdata/cli/handler/optimize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/cli/handler/optimize.py -------------------------------------------------------------------------------- /src/litdata/cli/parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/cli/parser.py -------------------------------------------------------------------------------- /src/litdata/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/constants.py -------------------------------------------------------------------------------- /src/litdata/debugger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/debugger.py -------------------------------------------------------------------------------- /src/litdata/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/helpers.py -------------------------------------------------------------------------------- /src/litdata/imports.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/imports.py -------------------------------------------------------------------------------- /src/litdata/processing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/processing/__init__.py -------------------------------------------------------------------------------- /src/litdata/processing/data_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/processing/data_processor.py -------------------------------------------------------------------------------- /src/litdata/processing/functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/processing/functions.py -------------------------------------------------------------------------------- /src/litdata/processing/readers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/processing/readers.py -------------------------------------------------------------------------------- /src/litdata/processing/utilities.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/processing/utilities.py -------------------------------------------------------------------------------- /src/litdata/raw/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/raw/__init__.py -------------------------------------------------------------------------------- /src/litdata/raw/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/raw/dataset.py -------------------------------------------------------------------------------- /src/litdata/raw/indexer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/raw/indexer.py -------------------------------------------------------------------------------- /src/litdata/raw/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/raw/types.py -------------------------------------------------------------------------------- /src/litdata/streaming/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/__init__.py -------------------------------------------------------------------------------- /src/litdata/streaming/cache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/cache.py -------------------------------------------------------------------------------- /src/litdata/streaming/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/client.py -------------------------------------------------------------------------------- /src/litdata/streaming/combined.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/combined.py -------------------------------------------------------------------------------- /src/litdata/streaming/compression.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/compression.py -------------------------------------------------------------------------------- /src/litdata/streaming/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/config.py -------------------------------------------------------------------------------- /src/litdata/streaming/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/dataloader.py -------------------------------------------------------------------------------- /src/litdata/streaming/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/dataset.py -------------------------------------------------------------------------------- /src/litdata/streaming/downloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/downloader.py -------------------------------------------------------------------------------- /src/litdata/streaming/fs_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/fs_provider.py -------------------------------------------------------------------------------- /src/litdata/streaming/item_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/item_loader.py -------------------------------------------------------------------------------- /src/litdata/streaming/parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/parallel.py -------------------------------------------------------------------------------- /src/litdata/streaming/reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/reader.py -------------------------------------------------------------------------------- /src/litdata/streaming/resolver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/resolver.py -------------------------------------------------------------------------------- /src/litdata/streaming/sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/sampler.py -------------------------------------------------------------------------------- /src/litdata/streaming/serializers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/serializers.py -------------------------------------------------------------------------------- /src/litdata/streaming/shuffle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/shuffle.py -------------------------------------------------------------------------------- /src/litdata/streaming/writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/streaming/writer.py -------------------------------------------------------------------------------- /src/litdata/utilities/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/__init__.py -------------------------------------------------------------------------------- /src/litdata/utilities/_pytree.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/_pytree.py -------------------------------------------------------------------------------- /src/litdata/utilities/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/base.py -------------------------------------------------------------------------------- /src/litdata/utilities/breakpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/breakpoint.py -------------------------------------------------------------------------------- /src/litdata/utilities/broadcast.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/broadcast.py -------------------------------------------------------------------------------- /src/litdata/utilities/dataset_utilities.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/dataset_utilities.py -------------------------------------------------------------------------------- /src/litdata/utilities/encryption.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/encryption.py -------------------------------------------------------------------------------- /src/litdata/utilities/env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/env.py -------------------------------------------------------------------------------- /src/litdata/utilities/format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/format.py -------------------------------------------------------------------------------- /src/litdata/utilities/hf_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/hf_dataset.py -------------------------------------------------------------------------------- /src/litdata/utilities/packing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/packing.py -------------------------------------------------------------------------------- /src/litdata/utilities/parquet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/parquet.py -------------------------------------------------------------------------------- /src/litdata/utilities/shuffle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/shuffle.py -------------------------------------------------------------------------------- /src/litdata/utilities/subsample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/subsample.py -------------------------------------------------------------------------------- /src/litdata/utilities/torch_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/torch_utils.py -------------------------------------------------------------------------------- /src/litdata/utilities/train_test_split.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/src/litdata/utilities/train_test_split.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/conftest.py -------------------------------------------------------------------------------- /tests/processing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/processing/test_data_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/processing/test_data_processor.py -------------------------------------------------------------------------------- /tests/processing/test_functions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/processing/test_functions.py -------------------------------------------------------------------------------- /tests/processing/test_readers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/processing/test_readers.py -------------------------------------------------------------------------------- /tests/processing/test_utilities.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/processing/test_utilities.py -------------------------------------------------------------------------------- /tests/raw/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/raw/test_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/raw/test_dataset.py -------------------------------------------------------------------------------- /tests/raw/test_indexer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/raw/test_indexer.py -------------------------------------------------------------------------------- /tests/streaming/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/streaming/test_cache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_cache.py -------------------------------------------------------------------------------- /tests/streaming/test_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_client.py -------------------------------------------------------------------------------- /tests/streaming/test_combined.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_combined.py -------------------------------------------------------------------------------- /tests/streaming/test_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_config.py -------------------------------------------------------------------------------- /tests/streaming/test_dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_dataloader.py -------------------------------------------------------------------------------- /tests/streaming/test_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_dataset.py -------------------------------------------------------------------------------- /tests/streaming/test_downloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_downloader.py -------------------------------------------------------------------------------- /tests/streaming/test_fs_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_fs_provider.py -------------------------------------------------------------------------------- /tests/streaming/test_item_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_item_loader.py -------------------------------------------------------------------------------- /tests/streaming/test_lock_cleanup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_lock_cleanup.py -------------------------------------------------------------------------------- /tests/streaming/test_parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_parallel.py -------------------------------------------------------------------------------- /tests/streaming/test_parquet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_parquet.py -------------------------------------------------------------------------------- /tests/streaming/test_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_reader.py -------------------------------------------------------------------------------- /tests/streaming/test_resolver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_resolver.py -------------------------------------------------------------------------------- /tests/streaming/test_sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_sampler.py -------------------------------------------------------------------------------- /tests/streaming/test_serializer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_serializer.py -------------------------------------------------------------------------------- /tests/streaming/test_writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/test_writer.py -------------------------------------------------------------------------------- /tests/streaming/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/streaming/utils.py -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/test_cli.py -------------------------------------------------------------------------------- /tests/test_debugger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/test_debugger.py -------------------------------------------------------------------------------- /tests/test_helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/test_helper.py -------------------------------------------------------------------------------- /tests/utilities/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/utilities/test_broadcast.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/utilities/test_broadcast.py -------------------------------------------------------------------------------- /tests/utilities/test_dataset_utilities.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/utilities/test_dataset_utilities.py -------------------------------------------------------------------------------- /tests/utilities/test_encryption.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/utilities/test_encryption.py -------------------------------------------------------------------------------- /tests/utilities/test_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/utilities/test_env.py -------------------------------------------------------------------------------- /tests/utilities/test_format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/utilities/test_format.py -------------------------------------------------------------------------------- /tests/utilities/test_packing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/utilities/test_packing.py -------------------------------------------------------------------------------- /tests/utilities/test_shuffle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/utilities/test_shuffle.py -------------------------------------------------------------------------------- /tests/utilities/test_subsample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/utilities/test_subsample.py -------------------------------------------------------------------------------- /tests/utilities/test_train_test_split.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lightning-AI/litData/HEAD/tests/utilities/test_train_test_split.py --------------------------------------------------------------------------------