├── .devcontainer └── devcontainer.json ├── .dockerignore ├── .gitattributes ├── .github └── workflows │ └── ecr.yml ├── .gitignore ├── .vscode ├── settings.json └── sftp.json ├── LICENSE.md ├── README.md ├── docker-compose.yaml ├── experiments ├── configuration.py ├── prepare_cifar10.sh ├── prepare_coco.sh ├── prepare_random.sh ├── run_all.sh ├── run_benchmarks.py ├── run_nomodel.py ├── run_remote.py └── setup.py ├── infrastructure ├── Dockerfile ├── install.sh └── requirements │ └── req.txt ├── params.yaml ├── scripts ├── build.sh ├── entrypoint.sh ├── get_ecr.sh ├── push-ecr.sh ├── run.sh └── start_minio.sh ├── settings.toml └── src ├── __init__.py ├── analysis ├── extract_metrics.py ├── extract_overview.py ├── plots.py ├── proc_pcap.py └── process_traces.py ├── config.py ├── dataloaders ├── base.py ├── cifar10 │ ├── deep_lake.py │ ├── ffcv.py │ ├── hub.py │ ├── index.py │ ├── nvidia_dali.py │ ├── pytorch.py │ ├── squirrel.py │ ├── torchdata.py │ └── webdataset.py ├── coco │ ├── deep_lake.py │ ├── hub.py │ ├── index.py │ ├── nvidia_dali.py │ ├── pytorch.py │ ├── squirrel.py │ ├── torchdata.py │ └── webdataset.py └── random │ ├── deep_lake.py │ ├── ffcv.py │ ├── hub.py │ ├── index.py │ ├── nvidia_dali.py │ ├── pytorch.py │ ├── squirrel.py │ ├── torchdata.py │ └── webdataset.py ├── datasets ├── base.py ├── cifar10 │ ├── base.py │ ├── deep_lake.py │ ├── ffcv.py │ ├── hub.py │ ├── index.py │ ├── nvidia_dali.py │ ├── pytorch.py │ ├── squirrel.py │ ├── torchdata.py │ └── webdataset.py ├── coco │ ├── base.py │ ├── deep_lake.py │ ├── hub.py │ ├── index.py │ ├── labels.py │ ├── nvidia_dali.py │ ├── pytorch.py │ ├── squirrel.py │ ├── torchdata.py │ └── webdataset.py ├── prepare.py ├── random │ ├── base.py │ ├── deep_lake.py │ ├── ffcv.py │ ├── hub.py │ ├── index.py │ ├── nvidia_dali.py │ ├── pytorch.py │ ├── squirrel.py │ ├── torchdata.py │ └── webdataset.py └── transformations │ ├── base.py │ ├── custom.py │ ├── cutout.py │ ├── identity.py │ ├── normalize.py │ └── to_tensor.py ├── libraries ├── deep_lake.py ├── hub.py ├── pytorch.py ├── squirrel.py ├── torchdata.py └── webdataset.py ├── models ├── base.py ├── cnn.py ├── coco │ ├── all.py │ ├── engine.py │ ├── presets.py │ ├── train_complete.py │ ├── transforms.py │ └── utils.py ├── faster_rcnn.py └── index.py ├── plots ├── collect_experiments.py ├── download_results.py ├── generate_plots.py ├── interest_over_time.csv ├── nomodel_experiments.py ├── preprocess_results.py └── remote_experiments.py ├── profiling ├── cpu_usage.sh ├── gpu_usage.sh ├── metric_logger.py ├── metrics.py ├── networking.py └── pytorch.py ├── run.py ├── train.py └── utils ├── distributed.py ├── experiments.py ├── general.py ├── persist.py ├── setup_s3cmd.py └── time.py /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/.devcontainer/devcontainer.json -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/.dockerignore -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/.gitattributes -------------------------------------------------------------------------------- /.github/workflows/ecr.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/.github/workflows/ecr.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/.gitignore -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.formatting.provider": "autopep8" 3 | } -------------------------------------------------------------------------------- /.vscode/sftp.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/.vscode/sftp.json -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/LICENSE.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/README.md -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/docker-compose.yaml -------------------------------------------------------------------------------- /experiments/configuration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/experiments/configuration.py -------------------------------------------------------------------------------- /experiments/prepare_cifar10.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/experiments/prepare_cifar10.sh -------------------------------------------------------------------------------- /experiments/prepare_coco.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/experiments/prepare_coco.sh -------------------------------------------------------------------------------- /experiments/prepare_random.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/experiments/prepare_random.sh -------------------------------------------------------------------------------- /experiments/run_all.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/experiments/run_all.sh -------------------------------------------------------------------------------- /experiments/run_benchmarks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/experiments/run_benchmarks.py -------------------------------------------------------------------------------- /experiments/run_nomodel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/experiments/run_nomodel.py -------------------------------------------------------------------------------- /experiments/run_remote.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/experiments/run_remote.py -------------------------------------------------------------------------------- /experiments/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/experiments/setup.py -------------------------------------------------------------------------------- /infrastructure/Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/infrastructure/Dockerfile -------------------------------------------------------------------------------- /infrastructure/install.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/infrastructure/install.sh -------------------------------------------------------------------------------- /infrastructure/requirements/req.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/infrastructure/requirements/req.txt -------------------------------------------------------------------------------- /params.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/params.yaml -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/scripts/build.sh -------------------------------------------------------------------------------- /scripts/entrypoint.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/scripts/entrypoint.sh -------------------------------------------------------------------------------- /scripts/get_ecr.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/scripts/get_ecr.sh -------------------------------------------------------------------------------- /scripts/push-ecr.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/scripts/push-ecr.sh -------------------------------------------------------------------------------- /scripts/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/scripts/run.sh -------------------------------------------------------------------------------- /scripts/start_minio.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/scripts/start_minio.sh -------------------------------------------------------------------------------- /settings.toml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/analysis/extract_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/analysis/extract_metrics.py -------------------------------------------------------------------------------- /src/analysis/extract_overview.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/analysis/extract_overview.py -------------------------------------------------------------------------------- /src/analysis/plots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/analysis/plots.py -------------------------------------------------------------------------------- /src/analysis/proc_pcap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/analysis/proc_pcap.py -------------------------------------------------------------------------------- /src/analysis/process_traces.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/analysis/process_traces.py -------------------------------------------------------------------------------- /src/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/config.py -------------------------------------------------------------------------------- /src/dataloaders/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/base.py -------------------------------------------------------------------------------- /src/dataloaders/cifar10/deep_lake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/cifar10/deep_lake.py -------------------------------------------------------------------------------- /src/dataloaders/cifar10/ffcv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/cifar10/ffcv.py -------------------------------------------------------------------------------- /src/dataloaders/cifar10/hub.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/cifar10/hub.py -------------------------------------------------------------------------------- /src/dataloaders/cifar10/index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/cifar10/index.py -------------------------------------------------------------------------------- /src/dataloaders/cifar10/nvidia_dali.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/cifar10/nvidia_dali.py -------------------------------------------------------------------------------- /src/dataloaders/cifar10/pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/cifar10/pytorch.py -------------------------------------------------------------------------------- /src/dataloaders/cifar10/squirrel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/cifar10/squirrel.py -------------------------------------------------------------------------------- /src/dataloaders/cifar10/torchdata.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/cifar10/torchdata.py -------------------------------------------------------------------------------- /src/dataloaders/cifar10/webdataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/cifar10/webdataset.py -------------------------------------------------------------------------------- /src/dataloaders/coco/deep_lake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/coco/deep_lake.py -------------------------------------------------------------------------------- /src/dataloaders/coco/hub.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/coco/hub.py -------------------------------------------------------------------------------- /src/dataloaders/coco/index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/coco/index.py -------------------------------------------------------------------------------- /src/dataloaders/coco/nvidia_dali.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/coco/nvidia_dali.py -------------------------------------------------------------------------------- /src/dataloaders/coco/pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/coco/pytorch.py -------------------------------------------------------------------------------- /src/dataloaders/coco/squirrel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/coco/squirrel.py -------------------------------------------------------------------------------- /src/dataloaders/coco/torchdata.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/coco/torchdata.py -------------------------------------------------------------------------------- /src/dataloaders/coco/webdataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/coco/webdataset.py -------------------------------------------------------------------------------- /src/dataloaders/random/deep_lake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/random/deep_lake.py -------------------------------------------------------------------------------- /src/dataloaders/random/ffcv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/random/ffcv.py -------------------------------------------------------------------------------- /src/dataloaders/random/hub.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/random/hub.py -------------------------------------------------------------------------------- /src/dataloaders/random/index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/random/index.py -------------------------------------------------------------------------------- /src/dataloaders/random/nvidia_dali.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/random/nvidia_dali.py -------------------------------------------------------------------------------- /src/dataloaders/random/pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/random/pytorch.py -------------------------------------------------------------------------------- /src/dataloaders/random/squirrel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/random/squirrel.py -------------------------------------------------------------------------------- /src/dataloaders/random/torchdata.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/random/torchdata.py -------------------------------------------------------------------------------- /src/dataloaders/random/webdataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/dataloaders/random/webdataset.py -------------------------------------------------------------------------------- /src/datasets/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/base.py -------------------------------------------------------------------------------- /src/datasets/cifar10/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/cifar10/base.py -------------------------------------------------------------------------------- /src/datasets/cifar10/deep_lake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/cifar10/deep_lake.py -------------------------------------------------------------------------------- /src/datasets/cifar10/ffcv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/cifar10/ffcv.py -------------------------------------------------------------------------------- /src/datasets/cifar10/hub.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/cifar10/hub.py -------------------------------------------------------------------------------- /src/datasets/cifar10/index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/cifar10/index.py -------------------------------------------------------------------------------- /src/datasets/cifar10/nvidia_dali.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/cifar10/nvidia_dali.py -------------------------------------------------------------------------------- /src/datasets/cifar10/pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/cifar10/pytorch.py -------------------------------------------------------------------------------- /src/datasets/cifar10/squirrel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/cifar10/squirrel.py -------------------------------------------------------------------------------- /src/datasets/cifar10/torchdata.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/cifar10/torchdata.py -------------------------------------------------------------------------------- /src/datasets/cifar10/webdataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/cifar10/webdataset.py -------------------------------------------------------------------------------- /src/datasets/coco/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/coco/base.py -------------------------------------------------------------------------------- /src/datasets/coco/deep_lake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/coco/deep_lake.py -------------------------------------------------------------------------------- /src/datasets/coco/hub.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/coco/hub.py -------------------------------------------------------------------------------- /src/datasets/coco/index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/coco/index.py -------------------------------------------------------------------------------- /src/datasets/coco/labels.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/coco/labels.py -------------------------------------------------------------------------------- /src/datasets/coco/nvidia_dali.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/coco/nvidia_dali.py -------------------------------------------------------------------------------- /src/datasets/coco/pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/coco/pytorch.py -------------------------------------------------------------------------------- /src/datasets/coco/squirrel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/coco/squirrel.py -------------------------------------------------------------------------------- /src/datasets/coco/torchdata.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/coco/torchdata.py -------------------------------------------------------------------------------- /src/datasets/coco/webdataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/coco/webdataset.py -------------------------------------------------------------------------------- /src/datasets/prepare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/prepare.py -------------------------------------------------------------------------------- /src/datasets/random/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/random/base.py -------------------------------------------------------------------------------- /src/datasets/random/deep_lake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/random/deep_lake.py -------------------------------------------------------------------------------- /src/datasets/random/ffcv.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/random/ffcv.py -------------------------------------------------------------------------------- /src/datasets/random/hub.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/random/hub.py -------------------------------------------------------------------------------- /src/datasets/random/index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/random/index.py -------------------------------------------------------------------------------- /src/datasets/random/nvidia_dali.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/random/nvidia_dali.py -------------------------------------------------------------------------------- /src/datasets/random/pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/random/pytorch.py -------------------------------------------------------------------------------- /src/datasets/random/squirrel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/random/squirrel.py -------------------------------------------------------------------------------- /src/datasets/random/torchdata.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/random/torchdata.py -------------------------------------------------------------------------------- /src/datasets/random/webdataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/random/webdataset.py -------------------------------------------------------------------------------- /src/datasets/transformations/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/transformations/base.py -------------------------------------------------------------------------------- /src/datasets/transformations/custom.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/transformations/custom.py -------------------------------------------------------------------------------- /src/datasets/transformations/cutout.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/transformations/cutout.py -------------------------------------------------------------------------------- /src/datasets/transformations/identity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/transformations/identity.py -------------------------------------------------------------------------------- /src/datasets/transformations/normalize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/transformations/normalize.py -------------------------------------------------------------------------------- /src/datasets/transformations/to_tensor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/datasets/transformations/to_tensor.py -------------------------------------------------------------------------------- /src/libraries/deep_lake.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/libraries/deep_lake.py -------------------------------------------------------------------------------- /src/libraries/hub.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/libraries/hub.py -------------------------------------------------------------------------------- /src/libraries/pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/libraries/pytorch.py -------------------------------------------------------------------------------- /src/libraries/squirrel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/libraries/squirrel.py -------------------------------------------------------------------------------- /src/libraries/torchdata.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/libraries/torchdata.py -------------------------------------------------------------------------------- /src/libraries/webdataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/libraries/webdataset.py -------------------------------------------------------------------------------- /src/models/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/models/base.py -------------------------------------------------------------------------------- /src/models/cnn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/models/cnn.py -------------------------------------------------------------------------------- /src/models/coco/all.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/models/coco/all.py -------------------------------------------------------------------------------- /src/models/coco/engine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/models/coco/engine.py -------------------------------------------------------------------------------- /src/models/coco/presets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/models/coco/presets.py -------------------------------------------------------------------------------- /src/models/coco/train_complete.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/models/coco/train_complete.py -------------------------------------------------------------------------------- /src/models/coco/transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/models/coco/transforms.py -------------------------------------------------------------------------------- /src/models/coco/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/models/coco/utils.py -------------------------------------------------------------------------------- /src/models/faster_rcnn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/models/faster_rcnn.py -------------------------------------------------------------------------------- /src/models/index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/models/index.py -------------------------------------------------------------------------------- /src/plots/collect_experiments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/plots/collect_experiments.py -------------------------------------------------------------------------------- /src/plots/download_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/plots/download_results.py -------------------------------------------------------------------------------- /src/plots/generate_plots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/plots/generate_plots.py -------------------------------------------------------------------------------- /src/plots/interest_over_time.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/plots/interest_over_time.csv -------------------------------------------------------------------------------- /src/plots/nomodel_experiments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/plots/nomodel_experiments.py -------------------------------------------------------------------------------- /src/plots/preprocess_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/plots/preprocess_results.py -------------------------------------------------------------------------------- /src/plots/remote_experiments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/plots/remote_experiments.py -------------------------------------------------------------------------------- /src/profiling/cpu_usage.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/profiling/cpu_usage.sh -------------------------------------------------------------------------------- /src/profiling/gpu_usage.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/profiling/gpu_usage.sh -------------------------------------------------------------------------------- /src/profiling/metric_logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/profiling/metric_logger.py -------------------------------------------------------------------------------- /src/profiling/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/profiling/metrics.py -------------------------------------------------------------------------------- /src/profiling/networking.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/profiling/networking.py -------------------------------------------------------------------------------- /src/profiling/pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/profiling/pytorch.py -------------------------------------------------------------------------------- /src/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/run.py -------------------------------------------------------------------------------- /src/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/train.py -------------------------------------------------------------------------------- /src/utils/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/utils/distributed.py -------------------------------------------------------------------------------- /src/utils/experiments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/utils/experiments.py -------------------------------------------------------------------------------- /src/utils/general.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/utils/general.py -------------------------------------------------------------------------------- /src/utils/persist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/utils/persist.py -------------------------------------------------------------------------------- /src/utils/setup_s3cmd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/utils/setup_s3cmd.py -------------------------------------------------------------------------------- /src/utils/time.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/smartnets/dataloader-benchmarks/HEAD/src/utils/time.py --------------------------------------------------------------------------------