├── .deepsource.toml ├── .github └── workflows │ ├── pypi.yml │ └── test.yml ├── .gitignore ├── LICENSE ├── README.md ├── VERSION ├── conftest.py ├── docs ├── 404.html ├── api │ ├── index.html │ ├── search.json │ ├── webdataset.html │ └── webdataset │ │ ├── autodecode.html │ │ ├── bench.html │ │ ├── checks.html │ │ ├── composable.html │ │ ├── dataset.html │ │ ├── dbcache.html │ │ ├── dsspecs.html │ │ ├── extradatasets.html │ │ ├── filters.html │ │ ├── fluid.html │ │ ├── gopen.html │ │ ├── handlers.html │ │ ├── iterators.html │ │ ├── multi.html │ │ ├── pytorch.html │ │ ├── shardcache.html │ │ ├── shardlists.html │ │ ├── tariterators.html │ │ ├── tenbin.html │ │ ├── utils.html │ │ └── writer.html ├── convert-falling-things │ └── index.html ├── convert-objectron │ └── index.html ├── creating │ └── index.html ├── css │ ├── theme.css │ └── theme_extra.css ├── decoding │ └── index.html ├── desktop │ └── index.html ├── falling-things-make-shards │ └── index.html ├── fonts │ ├── Lato │ │ ├── lato-bold.eot │ │ ├── lato-bold.ttf │ │ ├── lato-bold.woff │ │ ├── lato-bold.woff2 │ │ ├── lato-bolditalic.eot │ │ ├── lato-bolditalic.ttf │ │ ├── lato-bolditalic.woff │ │ ├── lato-bolditalic.woff2 │ │ ├── lato-italic.eot │ │ ├── lato-italic.ttf │ │ ├── lato-italic.woff │ │ ├── lato-italic.woff2 │ │ ├── lato-regular.eot │ │ ├── lato-regular.ttf │ │ ├── lato-regular.woff │ │ └── lato-regular.woff2 │ ├── RobotoSlab │ │ ├── roboto-slab-v7-bold.eot │ │ ├── roboto-slab-v7-bold.ttf │ │ ├── roboto-slab-v7-bold.woff │ │ ├── roboto-slab-v7-bold.woff2 │ │ ├── roboto-slab-v7-regular.eot │ │ ├── roboto-slab-v7-regular.ttf │ │ ├── roboto-slab-v7-regular.woff │ │ ├── roboto-slab-v7-regular.woff2 │ │ └── roboto-slab.eot │ ├── fontawesome-webfont.eot │ ├── fontawesome-webfont.svg │ ├── fontawesome-webfont.ttf │ ├── fontawesome-webfont.woff │ └── fontawesome-webfont.woff2 ├── gettingstarted │ └── index.html ├── howitworks │ └── index.html ├── img │ └── favicon.ico ├── index.html ├── js │ ├── jquery-2.1.1.min.js │ ├── modernizr-2.8.3.min.js │ ├── theme.js │ └── theme_extra.js ├── module.html.jinja2 ├── multinode-test │ └── index.html ├── multinode │ └── index.html ├── objectron-conversion │ └── index.html ├── search.html ├── search │ ├── lunr.js │ ├── main.js │ ├── search_index.json │ └── worker.js ├── sharding │ └── index.html ├── sitemap.xml ├── sitemap.xml.gz ├── sources │ └── index.html ├── video-loading-example │ └── index.html ├── writing │ └── index.html └── ytsamples-split │ └── index.html ├── docsrc ├── index.md └── module.html.jinja2 ├── environment.yml ├── mkdocs.yml ├── mypy.ini ├── notebooks ├── convert-falling-things.ipynb ├── convert-objectron.ipynb ├── creating.ipynb ├── decoding.ipynb ├── desktop.ipynb ├── gettingstarted.ipynb ├── howitworks.ipynb ├── multinode-test.ipynb ├── multinode.ipynb ├── old │ ├── coco-show.ipynb │ ├── coco-split.ipynb │ ├── index.md │ └── oldreadme.ipynb ├── sharding.ipynb ├── sources.ipynb ├── video-loading-example.ipynb ├── writing.ipynb └── ytsamples-split.ipynb ├── pytest.ini ├── readme.ipynb ├── requirements.dev.txt ├── requirements.txt ├── run-jupyterlab ├── setup.py ├── tasks.py ├── testdata ├── augment.py ├── compressed.tar ├── crop.py ├── imagenet-000000.tgz ├── imagenet-extra.json ├── imagenet-shards.json ├── mpdata.tar ├── sample.tgz └── tendata.tar └── webdataset ├── __init__.py ├── autodecode.py ├── bench.py ├── checks.py ├── composable.py ├── dataset.py ├── dbcache.py ├── dsspecs.py ├── extradatasets.py ├── filters.py ├── fluid.py ├── gopen.py ├── handlers.py ├── iterators.py ├── multi.py ├── pipeline.py ├── pytorch.py ├── shardcache.py ├── shardlists.py ├── tariterators.py ├── tenbin.py ├── tests ├── test_dataset.py ├── test_gopen.py ├── test_pipeline.py ├── test_utils.py └── test_writer.py ├── utils.py └── writer.py /.deepsource.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/.deepsource.toml -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/.github/workflows/pypi.yml -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/.github/workflows/test.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/README.md -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.1.103 -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/404.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/404.html -------------------------------------------------------------------------------- /docs/api/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/index.html -------------------------------------------------------------------------------- /docs/api/search.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/search.json -------------------------------------------------------------------------------- /docs/api/webdataset.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset.html -------------------------------------------------------------------------------- /docs/api/webdataset/autodecode.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/autodecode.html -------------------------------------------------------------------------------- /docs/api/webdataset/bench.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/bench.html -------------------------------------------------------------------------------- /docs/api/webdataset/checks.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/checks.html -------------------------------------------------------------------------------- /docs/api/webdataset/composable.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/composable.html -------------------------------------------------------------------------------- /docs/api/webdataset/dataset.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/dataset.html -------------------------------------------------------------------------------- /docs/api/webdataset/dbcache.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/dbcache.html -------------------------------------------------------------------------------- /docs/api/webdataset/dsspecs.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/dsspecs.html -------------------------------------------------------------------------------- /docs/api/webdataset/extradatasets.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/extradatasets.html -------------------------------------------------------------------------------- /docs/api/webdataset/filters.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/filters.html -------------------------------------------------------------------------------- /docs/api/webdataset/fluid.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/fluid.html -------------------------------------------------------------------------------- /docs/api/webdataset/gopen.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/gopen.html -------------------------------------------------------------------------------- /docs/api/webdataset/handlers.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/handlers.html -------------------------------------------------------------------------------- /docs/api/webdataset/iterators.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/iterators.html -------------------------------------------------------------------------------- /docs/api/webdataset/multi.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/multi.html -------------------------------------------------------------------------------- /docs/api/webdataset/pytorch.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/pytorch.html -------------------------------------------------------------------------------- /docs/api/webdataset/shardcache.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/shardcache.html -------------------------------------------------------------------------------- /docs/api/webdataset/shardlists.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/shardlists.html -------------------------------------------------------------------------------- /docs/api/webdataset/tariterators.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/tariterators.html -------------------------------------------------------------------------------- /docs/api/webdataset/tenbin.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/tenbin.html -------------------------------------------------------------------------------- /docs/api/webdataset/utils.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/utils.html -------------------------------------------------------------------------------- /docs/api/webdataset/writer.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/api/webdataset/writer.html -------------------------------------------------------------------------------- /docs/convert-falling-things/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/convert-falling-things/index.html -------------------------------------------------------------------------------- /docs/convert-objectron/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/convert-objectron/index.html -------------------------------------------------------------------------------- /docs/creating/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/creating/index.html -------------------------------------------------------------------------------- /docs/css/theme.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/css/theme.css -------------------------------------------------------------------------------- /docs/css/theme_extra.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/css/theme_extra.css -------------------------------------------------------------------------------- /docs/decoding/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/decoding/index.html -------------------------------------------------------------------------------- /docs/desktop/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/desktop/index.html -------------------------------------------------------------------------------- /docs/falling-things-make-shards/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/falling-things-make-shards/index.html -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-bold.eot -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-bold.ttf -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-bold.woff -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-bold.woff2 -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-bolditalic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-bolditalic.eot -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-bolditalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-bolditalic.ttf -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-bolditalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-bolditalic.woff -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-bolditalic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-bolditalic.woff2 -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-italic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-italic.eot -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-italic.ttf -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-italic.woff -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-italic.woff2 -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-regular.eot -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-regular.ttf -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-regular.woff -------------------------------------------------------------------------------- /docs/fonts/Lato/lato-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/Lato/lato-regular.woff2 -------------------------------------------------------------------------------- /docs/fonts/RobotoSlab/roboto-slab-v7-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/RobotoSlab/roboto-slab-v7-bold.eot -------------------------------------------------------------------------------- /docs/fonts/RobotoSlab/roboto-slab-v7-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/RobotoSlab/roboto-slab-v7-bold.ttf -------------------------------------------------------------------------------- /docs/fonts/RobotoSlab/roboto-slab-v7-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/RobotoSlab/roboto-slab-v7-bold.woff -------------------------------------------------------------------------------- /docs/fonts/RobotoSlab/roboto-slab-v7-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/RobotoSlab/roboto-slab-v7-bold.woff2 -------------------------------------------------------------------------------- /docs/fonts/RobotoSlab/roboto-slab-v7-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/RobotoSlab/roboto-slab-v7-regular.eot -------------------------------------------------------------------------------- /docs/fonts/RobotoSlab/roboto-slab-v7-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/RobotoSlab/roboto-slab-v7-regular.ttf -------------------------------------------------------------------------------- /docs/fonts/RobotoSlab/roboto-slab-v7-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/RobotoSlab/roboto-slab-v7-regular.woff -------------------------------------------------------------------------------- /docs/fonts/RobotoSlab/roboto-slab-v7-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/RobotoSlab/roboto-slab-v7-regular.woff2 -------------------------------------------------------------------------------- /docs/fonts/RobotoSlab/roboto-slab.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/RobotoSlab/roboto-slab.eot -------------------------------------------------------------------------------- /docs/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /docs/fonts/fontawesome-webfont.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/fontawesome-webfont.svg -------------------------------------------------------------------------------- /docs/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /docs/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /docs/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /docs/gettingstarted/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/gettingstarted/index.html -------------------------------------------------------------------------------- /docs/howitworks/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/howitworks/index.html -------------------------------------------------------------------------------- /docs/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/img/favicon.ico -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/index.html -------------------------------------------------------------------------------- /docs/js/jquery-2.1.1.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/js/jquery-2.1.1.min.js -------------------------------------------------------------------------------- /docs/js/modernizr-2.8.3.min.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/js/modernizr-2.8.3.min.js -------------------------------------------------------------------------------- /docs/js/theme.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/js/theme.js -------------------------------------------------------------------------------- /docs/js/theme_extra.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/js/theme_extra.js -------------------------------------------------------------------------------- /docs/module.html.jinja2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/module.html.jinja2 -------------------------------------------------------------------------------- /docs/multinode-test/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/multinode-test/index.html -------------------------------------------------------------------------------- /docs/multinode/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/multinode/index.html -------------------------------------------------------------------------------- /docs/objectron-conversion/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/objectron-conversion/index.html -------------------------------------------------------------------------------- /docs/search.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/search.html -------------------------------------------------------------------------------- /docs/search/lunr.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/search/lunr.js -------------------------------------------------------------------------------- /docs/search/main.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/search/main.js -------------------------------------------------------------------------------- /docs/search/search_index.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/search/search_index.json -------------------------------------------------------------------------------- /docs/search/worker.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/search/worker.js -------------------------------------------------------------------------------- /docs/sharding/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/sharding/index.html -------------------------------------------------------------------------------- /docs/sitemap.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/sitemap.xml -------------------------------------------------------------------------------- /docs/sitemap.xml.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/sitemap.xml.gz -------------------------------------------------------------------------------- /docs/sources/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/sources/index.html -------------------------------------------------------------------------------- /docs/video-loading-example/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/video-loading-example/index.html -------------------------------------------------------------------------------- /docs/writing/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/writing/index.html -------------------------------------------------------------------------------- /docs/ytsamples-split/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docs/ytsamples-split/index.html -------------------------------------------------------------------------------- /docsrc/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docsrc/index.md -------------------------------------------------------------------------------- /docsrc/module.html.jinja2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/docsrc/module.html.jinja2 -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/environment.yml -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/mkdocs.yml -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/mypy.ini -------------------------------------------------------------------------------- /notebooks/convert-falling-things.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/convert-falling-things.ipynb -------------------------------------------------------------------------------- /notebooks/convert-objectron.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/convert-objectron.ipynb -------------------------------------------------------------------------------- /notebooks/creating.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/creating.ipynb -------------------------------------------------------------------------------- /notebooks/decoding.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/decoding.ipynb -------------------------------------------------------------------------------- /notebooks/desktop.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/desktop.ipynb -------------------------------------------------------------------------------- /notebooks/gettingstarted.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/gettingstarted.ipynb -------------------------------------------------------------------------------- /notebooks/howitworks.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/howitworks.ipynb -------------------------------------------------------------------------------- /notebooks/multinode-test.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/multinode-test.ipynb -------------------------------------------------------------------------------- /notebooks/multinode.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/multinode.ipynb -------------------------------------------------------------------------------- /notebooks/old/coco-show.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/old/coco-show.ipynb -------------------------------------------------------------------------------- /notebooks/old/coco-split.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/old/coco-split.ipynb -------------------------------------------------------------------------------- /notebooks/old/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/old/index.md -------------------------------------------------------------------------------- /notebooks/old/oldreadme.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/old/oldreadme.ipynb -------------------------------------------------------------------------------- /notebooks/sharding.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/sharding.ipynb -------------------------------------------------------------------------------- /notebooks/sources.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/sources.ipynb -------------------------------------------------------------------------------- /notebooks/video-loading-example.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/video-loading-example.ipynb -------------------------------------------------------------------------------- /notebooks/writing.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/writing.ipynb -------------------------------------------------------------------------------- /notebooks/ytsamples-split.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/notebooks/ytsamples-split.ipynb -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/pytest.ini -------------------------------------------------------------------------------- /readme.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/readme.ipynb -------------------------------------------------------------------------------- /requirements.dev.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/requirements.dev.txt -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | braceexpand 2 | numpy 3 | pyyaml 4 | -------------------------------------------------------------------------------- /run-jupyterlab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/run-jupyterlab -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/setup.py -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/tasks.py -------------------------------------------------------------------------------- /testdata/augment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/testdata/augment.py -------------------------------------------------------------------------------- /testdata/compressed.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/testdata/compressed.tar -------------------------------------------------------------------------------- /testdata/crop.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/testdata/crop.py -------------------------------------------------------------------------------- /testdata/imagenet-000000.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/testdata/imagenet-000000.tgz -------------------------------------------------------------------------------- /testdata/imagenet-extra.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/testdata/imagenet-extra.json -------------------------------------------------------------------------------- /testdata/imagenet-shards.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/testdata/imagenet-shards.json -------------------------------------------------------------------------------- /testdata/mpdata.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/testdata/mpdata.tar -------------------------------------------------------------------------------- /testdata/sample.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/testdata/sample.tgz -------------------------------------------------------------------------------- /testdata/tendata.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/testdata/tendata.tar -------------------------------------------------------------------------------- /webdataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/__init__.py -------------------------------------------------------------------------------- /webdataset/autodecode.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/autodecode.py -------------------------------------------------------------------------------- /webdataset/bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/bench.py -------------------------------------------------------------------------------- /webdataset/checks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/checks.py -------------------------------------------------------------------------------- /webdataset/composable.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/composable.py -------------------------------------------------------------------------------- /webdataset/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/dataset.py -------------------------------------------------------------------------------- /webdataset/dbcache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/dbcache.py -------------------------------------------------------------------------------- /webdataset/dsspecs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/dsspecs.py -------------------------------------------------------------------------------- /webdataset/extradatasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/extradatasets.py -------------------------------------------------------------------------------- /webdataset/filters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/filters.py -------------------------------------------------------------------------------- /webdataset/fluid.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/fluid.py -------------------------------------------------------------------------------- /webdataset/gopen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/gopen.py -------------------------------------------------------------------------------- /webdataset/handlers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/handlers.py -------------------------------------------------------------------------------- /webdataset/iterators.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/iterators.py -------------------------------------------------------------------------------- /webdataset/multi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/multi.py -------------------------------------------------------------------------------- /webdataset/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/pipeline.py -------------------------------------------------------------------------------- /webdataset/pytorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/pytorch.py -------------------------------------------------------------------------------- /webdataset/shardcache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/shardcache.py -------------------------------------------------------------------------------- /webdataset/shardlists.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/shardlists.py -------------------------------------------------------------------------------- /webdataset/tariterators.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/tariterators.py -------------------------------------------------------------------------------- /webdataset/tenbin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/tenbin.py -------------------------------------------------------------------------------- /webdataset/tests/test_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/tests/test_dataset.py -------------------------------------------------------------------------------- /webdataset/tests/test_gopen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/tests/test_gopen.py -------------------------------------------------------------------------------- /webdataset/tests/test_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/tests/test_pipeline.py -------------------------------------------------------------------------------- /webdataset/tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/tests/test_utils.py -------------------------------------------------------------------------------- /webdataset/tests/test_writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/tests/test_writer.py -------------------------------------------------------------------------------- /webdataset/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/utils.py -------------------------------------------------------------------------------- /webdataset/writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/webdataset/HEAD/webdataset/writer.py --------------------------------------------------------------------------------