├── .flake8 ├── .github └── workflows │ ├── black.yml │ ├── ci.yml │ └── lint.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── deltatorch ├── __init__.py ├── deltadataset.py ├── id_based_deltadataset.py ├── pytorch.py └── skip_batch_deltadataset.py ├── examples ├── __init__.py ├── caltech │ ├── caltech_data_prep.py │ ├── cv_caltech256_sparktorchdistributor_native_pytoch.py │ ├── cv_caltech256_sparktorchdistributor_pytochlightning.py │ └── cv_caltech256_sparktorchdistributor_torchvision_dataset.py ├── diffusers_unconditional_image_generation │ ├── data_prep.py │ ├── run.py │ └── train_unconditional.py ├── difusers_stable_diffusion │ ├── data_prep.py │ ├── run.py │ └── train_text_to_image_lora.py └── imagenet │ ├── data_preparation.py │ └── ddp_spark_pytorch_distributor.py ├── old_examples ├── cifar_data_prep.py ├── cv_caltech256_petastorm.py ├── cv_caltech256_ptl_singlenode.py ├── cv_cifar10_singlenode.py └── nlp_agnews_torchtext_singlenode.py ├── poetry.lock ├── pyproject.toml └── tests ├── data └── ag_news.parquet ├── test_id_based_deltadataset.py └── test_skip_deltadataset.py /.flake8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/.flake8 -------------------------------------------------------------------------------- /.github/workflows/black.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/.github/workflows/black.yml -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/.github/workflows/ci.yml -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/.github/workflows/lint.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/LICENSE -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/README.md -------------------------------------------------------------------------------- /deltatorch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/deltatorch/__init__.py -------------------------------------------------------------------------------- /deltatorch/deltadataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/deltatorch/deltadataset.py -------------------------------------------------------------------------------- /deltatorch/id_based_deltadataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/deltatorch/id_based_deltadataset.py -------------------------------------------------------------------------------- /deltatorch/pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/deltatorch/pytorch.py -------------------------------------------------------------------------------- /deltatorch/skip_batch_deltadataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/deltatorch/skip_batch_deltadataset.py -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/caltech/caltech_data_prep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/caltech/caltech_data_prep.py -------------------------------------------------------------------------------- /examples/caltech/cv_caltech256_sparktorchdistributor_native_pytoch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/caltech/cv_caltech256_sparktorchdistributor_native_pytoch.py -------------------------------------------------------------------------------- /examples/caltech/cv_caltech256_sparktorchdistributor_pytochlightning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/caltech/cv_caltech256_sparktorchdistributor_pytochlightning.py -------------------------------------------------------------------------------- /examples/caltech/cv_caltech256_sparktorchdistributor_torchvision_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/caltech/cv_caltech256_sparktorchdistributor_torchvision_dataset.py -------------------------------------------------------------------------------- /examples/diffusers_unconditional_image_generation/data_prep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/diffusers_unconditional_image_generation/data_prep.py -------------------------------------------------------------------------------- /examples/diffusers_unconditional_image_generation/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/diffusers_unconditional_image_generation/run.py -------------------------------------------------------------------------------- /examples/diffusers_unconditional_image_generation/train_unconditional.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/diffusers_unconditional_image_generation/train_unconditional.py -------------------------------------------------------------------------------- /examples/difusers_stable_diffusion/data_prep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/difusers_stable_diffusion/data_prep.py -------------------------------------------------------------------------------- /examples/difusers_stable_diffusion/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/difusers_stable_diffusion/run.py -------------------------------------------------------------------------------- /examples/difusers_stable_diffusion/train_text_to_image_lora.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/difusers_stable_diffusion/train_text_to_image_lora.py -------------------------------------------------------------------------------- /examples/imagenet/data_preparation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/imagenet/data_preparation.py -------------------------------------------------------------------------------- /examples/imagenet/ddp_spark_pytorch_distributor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/examples/imagenet/ddp_spark_pytorch_distributor.py -------------------------------------------------------------------------------- /old_examples/cifar_data_prep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/old_examples/cifar_data_prep.py -------------------------------------------------------------------------------- /old_examples/cv_caltech256_petastorm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/old_examples/cv_caltech256_petastorm.py -------------------------------------------------------------------------------- /old_examples/cv_caltech256_ptl_singlenode.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/old_examples/cv_caltech256_ptl_singlenode.py -------------------------------------------------------------------------------- /old_examples/cv_cifar10_singlenode.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/old_examples/cv_cifar10_singlenode.py -------------------------------------------------------------------------------- /old_examples/nlp_agnews_torchtext_singlenode.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/old_examples/nlp_agnews_torchtext_singlenode.py -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/poetry.lock -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/pyproject.toml -------------------------------------------------------------------------------- /tests/data/ag_news.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/tests/data/ag_news.parquet -------------------------------------------------------------------------------- /tests/test_id_based_deltadataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/tests/test_id_based_deltadataset.py -------------------------------------------------------------------------------- /tests/test_skip_deltadataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/delta-incubator/deltatorch/HEAD/tests/test_skip_deltadataset.py --------------------------------------------------------------------------------