├── runtime.txt ├── beginner_source ├── examples_nn │ ├── README.txt │ ├── two_layer_net_module.py │ ├── two_layer_net_optim.py │ ├── two_layer_net_nn.py │ └── dynamic_net.py ├── examples_tensor │ ├── README.txt │ ├── two_layer_net_numpy.py │ └── two_layer_net_tensor.py ├── examples_autograd │ ├── README.txt │ ├── two_layer_net_autograd.py │ ├── two_layer_net_custom_function.py │ └── tf_two_layer_net.py ├── hybrid_frontend │ └── README.txt ├── former_torchies │ ├── README.txt │ ├── autograd_tutorial_old.py │ ├── tensor_tutorial_old.py │ └── parallelism_tutorial.py ├── hybrid_frontend_tutorial.rst ├── blitz │ ├── README.txt │ ├── tensor_tutorial.py │ └── autograd_tutorial.py ├── nlp │ └── README.txt ├── README.txt ├── former_torchies_tutorial.rst ├── deep_learning_60min_blitz.rst ├── colab.rst ├── deep_learning_nlp_tutorial.rst └── PyTorch Cheat.md ├── cleanup.sh ├── _static ├── down.png ├── file.png ├── plus.png ├── up.png ├── minus.png ├── comment.png ├── img │ ├── cat.jpg │ ├── chat.png │ ├── mnist.png │ ├── panda.png │ ├── cifar10.png │ ├── hybrid.png │ ├── stn │ │ ├── tr.png │ │ ├── FSeq.png │ │ ├── Five.gif │ │ └── stn-arch.png │ ├── ONNXLive.png │ ├── SRResNet.png │ ├── Variable.png │ ├── autodiff.png │ ├── cartpole.gif │ ├── cpp_logo.png │ ├── rnnclass.png │ ├── scipynumpy.png │ ├── torch.nn.png │ ├── cat_224x224.jpg │ ├── cat_output1.png │ ├── chatbot │ │ ├── bot.png │ │ ├── diff.png │ │ ├── attn1.png │ │ ├── attn2.png │ │ ├── scores.png │ │ ├── grad_clip.png │ │ ├── global_attn.png │ │ ├── seq2seq_ts.png │ │ ├── RNN-bidirectional.png │ │ ├── pytorch_workflow.png │ │ └── seq2seq_batches.png │ ├── cpp-pytorch.png │ ├── seq2seq_flat.png │ ├── data_parallel.png │ ├── dcgan_generator.png │ ├── dynamic_graph.gif │ ├── thumbnails │ │ ├── eye.png │ │ ├── babel.jpg │ │ ├── floppy.png │ │ ├── default.png │ │ ├── examples.png │ │ ├── torch-logo.png │ │ ├── landmarked_face2.png │ │ ├── pytorch-logo-flat.png │ │ ├── 220px-KnnClassification.png │ │ └── sphx_glr_transfer_learning_tutorial_001.png │ ├── fgsm_panda_image.png │ ├── landmarked_face2.png │ ├── pytorch-logo-dark.png │ ├── char_rnn_generation.png │ ├── cpp-frontend │ │ └── digits.png │ ├── distributed │ │ ├── gather.png │ │ ├── reduce.png │ │ ├── scatter.png │ │ ├── all_gather.pdf │ │ ├── all_gather.png │ │ ├── all_reduce.pdf │ │ ├── all_reduce.png │ │ ├── broadcast.png │ │ ├── send_recv.png │ │ ├── DistPyTorch.jpg │ │ └── send_recv_big.png │ ├── neural-style │ │ ├── dancing.jpg │ │ ├── picasso.jpg │ │ ├── neuralstyle.png │ │ ├── sphx_glr_neural_style_tutorial_001.png │ │ ├── sphx_glr_neural_style_tutorial_002.png │ │ ├── sphx_glr_neural_style_tutorial_003.png │ │ └── sphx_glr_neural_style_tutorial_004.png │ ├── tensor_illustration.png │ ├── seq-seq-images │ │ ├── decoder.png │ │ ├── seq2seq.png │ │ ├── decoder@2x.png │ │ ├── seq2seq@2x.png │ │ ├── decoder-network.png │ │ ├── encoder-network.png │ │ ├── word-encoding.png │ │ ├── word-encoding@2x.png │ │ ├── attention-decoder-network.png │ │ ├── encoder-network.dot │ │ ├── decoder-network.dot │ │ └── attention-decoder-network.dot │ ├── torch-nn-vs-pytorch-nn.png │ ├── hybrid_frontend │ │ ├── iris_pic.jpg │ │ ├── 220px-KnnClassification.png │ │ └── pytorch_workflow_small.jpg │ ├── tensor_illustration_flat.png │ ├── model-parallel-images │ │ ├── mp_vs_rn.png │ │ ├── mp_vs_rn_vs_pp.png │ │ └── split_size_tradeoff.png │ ├── reinforcement_learning_diagram.jpg │ └── pytorch-logo-dark.svg ├── no_image.png ├── ajax-loader.gif ├── up-pressed.png ├── broken_example.png ├── comment-bright.png ├── comment-close.png ├── down-pressed.png ├── images │ ├── home-footer-background.jpg │ ├── logo-facebook-dark.svg │ ├── view-page-source-icon.svg │ ├── logo-icon.svg │ ├── chevron-right-orange.svg │ ├── chevron-right-white.svg │ ├── chevron-down-grey.svg │ ├── arrow-right-with-tail.svg │ ├── arrow-down-orange.svg │ ├── logo-twitter-dark.svg │ ├── icon-close.svg │ ├── logo-dark.svg │ ├── logo.svg │ └── icon-menu-dots-dark.svg ├── fonts │ ├── IBMPlexMono │ │ ├── IBMPlexMono-Light.woff │ │ ├── IBMPlexMono-Light.woff2 │ │ ├── IBMPlexMono-Medium.woff │ │ ├── IBMPlexMono-Medium.woff2 │ │ ├── IBMPlexMono-Regular.woff │ │ ├── IBMPlexMono-Regular.woff2 │ │ ├── IBMPlexMono-SemiBold.woff │ │ └── IBMPlexMono-SemiBold.woff2 │ └── FreightSans │ │ ├── freight-sans-light.woff │ │ ├── freight-sans-light.woff2 │ │ ├── freight-sans-regular.woff │ │ └── freight-sans-regular.woff2 ├── documentation_options.js ├── pytorch-logo-dark.svg ├── css │ └── pytorch_theme.css ├── pygments.css └── gallery.css ├── .gitmodules ├── .jenkins ├── delete_html_file_with_runnable_code_removed.py ├── remove_invisible_code_block_from_ipynb.py ├── remove_invisible_code_block_from_html.py ├── remove_invisible_code_block_from_rst_txt.py ├── remove_invisible_code_block_from_py.py ├── remove_invisible_code_block_batch.sh ├── replace_tutorial_html_content.py ├── remove_runnable_code.py └── build.sh ├── requirements.txt ├── advanced_source ├── README.txt ├── numpy_extensions_tutorial.py └── ONNXLive.rst ├── intermediate_source ├── README.txt └── spatial_transformer_tutorial.py ├── LICENSE ├── .gitignore ├── README.md ├── _templates └── layout.html ├── Makefile ├── custom_directives.py └── conf.py /runtime.txt: -------------------------------------------------------------------------------- 1 | 3.6 2 | -------------------------------------------------------------------------------- /beginner_source/examples_nn/README.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /beginner_source/examples_tensor/README.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /beginner_source/examples_autograd/README.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cleanup.sh: -------------------------------------------------------------------------------- 1 | rm -rf __pycache__/ _build/ advanced/ beginner/ intermediate/ 2 | -------------------------------------------------------------------------------- /_static/down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/down.png -------------------------------------------------------------------------------- /_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/file.png -------------------------------------------------------------------------------- /_static/plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/plus.png -------------------------------------------------------------------------------- /_static/up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/up.png -------------------------------------------------------------------------------- /_static/minus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/minus.png -------------------------------------------------------------------------------- /_static/comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/comment.png -------------------------------------------------------------------------------- /_static/img/cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/cat.jpg -------------------------------------------------------------------------------- /_static/img/chat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chat.png -------------------------------------------------------------------------------- /_static/img/mnist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/mnist.png -------------------------------------------------------------------------------- /_static/img/panda.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/panda.png -------------------------------------------------------------------------------- /_static/no_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/no_image.png -------------------------------------------------------------------------------- /_static/ajax-loader.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/ajax-loader.gif -------------------------------------------------------------------------------- /_static/img/cifar10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/cifar10.png -------------------------------------------------------------------------------- /_static/img/hybrid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/hybrid.png -------------------------------------------------------------------------------- /_static/img/stn/tr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/stn/tr.png -------------------------------------------------------------------------------- /_static/up-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/up-pressed.png -------------------------------------------------------------------------------- /_static/broken_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/broken_example.png -------------------------------------------------------------------------------- /_static/comment-bright.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/comment-bright.png -------------------------------------------------------------------------------- /_static/comment-close.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/comment-close.png -------------------------------------------------------------------------------- /_static/down-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/down-pressed.png -------------------------------------------------------------------------------- /_static/img/ONNXLive.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/ONNXLive.png -------------------------------------------------------------------------------- /_static/img/SRResNet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/SRResNet.png -------------------------------------------------------------------------------- /_static/img/Variable.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/Variable.png -------------------------------------------------------------------------------- /_static/img/autodiff.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/autodiff.png -------------------------------------------------------------------------------- /_static/img/cartpole.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/cartpole.gif -------------------------------------------------------------------------------- /_static/img/cpp_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/cpp_logo.png -------------------------------------------------------------------------------- /_static/img/rnnclass.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/rnnclass.png -------------------------------------------------------------------------------- /_static/img/scipynumpy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/scipynumpy.png -------------------------------------------------------------------------------- /_static/img/stn/FSeq.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/stn/FSeq.png -------------------------------------------------------------------------------- /_static/img/stn/Five.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/stn/Five.gif -------------------------------------------------------------------------------- /_static/img/torch.nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/torch.nn.png -------------------------------------------------------------------------------- /_static/img/cat_224x224.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/cat_224x224.jpg -------------------------------------------------------------------------------- /_static/img/cat_output1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/cat_output1.png -------------------------------------------------------------------------------- /_static/img/chatbot/bot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chatbot/bot.png -------------------------------------------------------------------------------- /_static/img/chatbot/diff.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chatbot/diff.png -------------------------------------------------------------------------------- /_static/img/cpp-pytorch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/cpp-pytorch.png -------------------------------------------------------------------------------- /_static/img/seq2seq_flat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/seq2seq_flat.png -------------------------------------------------------------------------------- /_static/img/stn/stn-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/stn/stn-arch.png -------------------------------------------------------------------------------- /_static/img/chatbot/attn1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chatbot/attn1.png -------------------------------------------------------------------------------- /_static/img/chatbot/attn2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chatbot/attn2.png -------------------------------------------------------------------------------- /_static/img/chatbot/scores.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chatbot/scores.png -------------------------------------------------------------------------------- /_static/img/data_parallel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/data_parallel.png -------------------------------------------------------------------------------- /_static/img/dcgan_generator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/dcgan_generator.png -------------------------------------------------------------------------------- /_static/img/dynamic_graph.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/dynamic_graph.gif -------------------------------------------------------------------------------- /_static/img/thumbnails/eye.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/thumbnails/eye.png -------------------------------------------------------------------------------- /_static/img/chatbot/grad_clip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chatbot/grad_clip.png -------------------------------------------------------------------------------- /_static/img/fgsm_panda_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/fgsm_panda_image.png -------------------------------------------------------------------------------- /_static/img/landmarked_face2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/landmarked_face2.png -------------------------------------------------------------------------------- /_static/img/pytorch-logo-dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/pytorch-logo-dark.png -------------------------------------------------------------------------------- /_static/img/thumbnails/babel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/thumbnails/babel.jpg -------------------------------------------------------------------------------- /_static/img/thumbnails/floppy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/thumbnails/floppy.png -------------------------------------------------------------------------------- /_static/img/char_rnn_generation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/char_rnn_generation.png -------------------------------------------------------------------------------- /_static/img/chatbot/global_attn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chatbot/global_attn.png -------------------------------------------------------------------------------- /_static/img/chatbot/seq2seq_ts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chatbot/seq2seq_ts.png -------------------------------------------------------------------------------- /_static/img/cpp-frontend/digits.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/cpp-frontend/digits.png -------------------------------------------------------------------------------- /_static/img/distributed/gather.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/distributed/gather.png -------------------------------------------------------------------------------- /_static/img/distributed/reduce.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/distributed/reduce.png -------------------------------------------------------------------------------- /_static/img/distributed/scatter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/distributed/scatter.png -------------------------------------------------------------------------------- /_static/img/neural-style/dancing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/neural-style/dancing.jpg -------------------------------------------------------------------------------- /_static/img/neural-style/picasso.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/neural-style/picasso.jpg -------------------------------------------------------------------------------- /_static/img/tensor_illustration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/tensor_illustration.png -------------------------------------------------------------------------------- /_static/img/thumbnails/default.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/thumbnails/default.png -------------------------------------------------------------------------------- /_static/img/thumbnails/examples.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/thumbnails/examples.png -------------------------------------------------------------------------------- /_static/img/distributed/all_gather.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/distributed/all_gather.pdf -------------------------------------------------------------------------------- /_static/img/distributed/all_gather.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/distributed/all_gather.png -------------------------------------------------------------------------------- /_static/img/distributed/all_reduce.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/distributed/all_reduce.pdf -------------------------------------------------------------------------------- /_static/img/distributed/all_reduce.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/distributed/all_reduce.png -------------------------------------------------------------------------------- /_static/img/distributed/broadcast.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/distributed/broadcast.png -------------------------------------------------------------------------------- /_static/img/distributed/send_recv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/distributed/send_recv.png -------------------------------------------------------------------------------- /_static/img/seq-seq-images/decoder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/seq-seq-images/decoder.png -------------------------------------------------------------------------------- /_static/img/seq-seq-images/seq2seq.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/seq-seq-images/seq2seq.png -------------------------------------------------------------------------------- /_static/img/thumbnails/torch-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/thumbnails/torch-logo.png -------------------------------------------------------------------------------- /_static/img/torch-nn-vs-pytorch-nn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/torch-nn-vs-pytorch-nn.png -------------------------------------------------------------------------------- /_static/images/home-footer-background.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/images/home-footer-background.jpg -------------------------------------------------------------------------------- /_static/img/chatbot/RNN-bidirectional.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chatbot/RNN-bidirectional.png -------------------------------------------------------------------------------- /_static/img/chatbot/pytorch_workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chatbot/pytorch_workflow.png -------------------------------------------------------------------------------- /_static/img/chatbot/seq2seq_batches.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/chatbot/seq2seq_batches.png -------------------------------------------------------------------------------- /_static/img/distributed/DistPyTorch.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/distributed/DistPyTorch.jpg -------------------------------------------------------------------------------- /_static/img/distributed/send_recv_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/distributed/send_recv_big.png -------------------------------------------------------------------------------- /_static/img/hybrid_frontend/iris_pic.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/hybrid_frontend/iris_pic.jpg -------------------------------------------------------------------------------- /_static/img/neural-style/neuralstyle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/neural-style/neuralstyle.png -------------------------------------------------------------------------------- /_static/img/seq-seq-images/decoder@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/seq-seq-images/decoder@2x.png -------------------------------------------------------------------------------- /_static/img/seq-seq-images/seq2seq@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/seq-seq-images/seq2seq@2x.png -------------------------------------------------------------------------------- /_static/img/tensor_illustration_flat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/tensor_illustration_flat.png -------------------------------------------------------------------------------- /_static/img/thumbnails/landmarked_face2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/thumbnails/landmarked_face2.png -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "src/pytorch-sphinx-theme"] 2 | path = src/pytorch-sphinx-theme 3 | url = https://github.com/pytorch/pytorch_sphinx_theme 4 | -------------------------------------------------------------------------------- /_static/img/model-parallel-images/mp_vs_rn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/model-parallel-images/mp_vs_rn.png -------------------------------------------------------------------------------- /_static/img/reinforcement_learning_diagram.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/reinforcement_learning_diagram.jpg -------------------------------------------------------------------------------- /_static/img/seq-seq-images/decoder-network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/seq-seq-images/decoder-network.png -------------------------------------------------------------------------------- /_static/img/seq-seq-images/encoder-network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/seq-seq-images/encoder-network.png -------------------------------------------------------------------------------- /_static/img/seq-seq-images/word-encoding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/seq-seq-images/word-encoding.png -------------------------------------------------------------------------------- /_static/img/thumbnails/pytorch-logo-flat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/thumbnails/pytorch-logo-flat.png -------------------------------------------------------------------------------- /_static/fonts/IBMPlexMono/IBMPlexMono-Light.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/IBMPlexMono/IBMPlexMono-Light.woff -------------------------------------------------------------------------------- /_static/img/seq-seq-images/word-encoding@2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/seq-seq-images/word-encoding@2x.png -------------------------------------------------------------------------------- /_static/fonts/FreightSans/freight-sans-light.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/FreightSans/freight-sans-light.woff -------------------------------------------------------------------------------- /_static/fonts/FreightSans/freight-sans-light.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/FreightSans/freight-sans-light.woff2 -------------------------------------------------------------------------------- /_static/fonts/FreightSans/freight-sans-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/FreightSans/freight-sans-regular.woff -------------------------------------------------------------------------------- /_static/fonts/IBMPlexMono/IBMPlexMono-Light.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/IBMPlexMono/IBMPlexMono-Light.woff2 -------------------------------------------------------------------------------- /_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff -------------------------------------------------------------------------------- /_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2 -------------------------------------------------------------------------------- /_static/fonts/IBMPlexMono/IBMPlexMono-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/IBMPlexMono/IBMPlexMono-Regular.woff -------------------------------------------------------------------------------- /_static/fonts/IBMPlexMono/IBMPlexMono-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/IBMPlexMono/IBMPlexMono-Regular.woff2 -------------------------------------------------------------------------------- /_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff -------------------------------------------------------------------------------- /_static/img/thumbnails/220px-KnnClassification.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/thumbnails/220px-KnnClassification.png -------------------------------------------------------------------------------- /_static/fonts/FreightSans/freight-sans-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/FreightSans/freight-sans-regular.woff2 -------------------------------------------------------------------------------- /_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2 -------------------------------------------------------------------------------- /_static/img/model-parallel-images/mp_vs_rn_vs_pp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/model-parallel-images/mp_vs_rn_vs_pp.png -------------------------------------------------------------------------------- /_static/img/hybrid_frontend/220px-KnnClassification.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/hybrid_frontend/220px-KnnClassification.png -------------------------------------------------------------------------------- /_static/img/hybrid_frontend/pytorch_workflow_small.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/hybrid_frontend/pytorch_workflow_small.jpg -------------------------------------------------------------------------------- /_static/img/seq-seq-images/attention-decoder-network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/seq-seq-images/attention-decoder-network.png -------------------------------------------------------------------------------- /_static/img/model-parallel-images/split_size_tradeoff.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/model-parallel-images/split_size_tradeoff.png -------------------------------------------------------------------------------- /_static/img/neural-style/sphx_glr_neural_style_tutorial_001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/neural-style/sphx_glr_neural_style_tutorial_001.png -------------------------------------------------------------------------------- /_static/img/neural-style/sphx_glr_neural_style_tutorial_002.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/neural-style/sphx_glr_neural_style_tutorial_002.png -------------------------------------------------------------------------------- /_static/img/neural-style/sphx_glr_neural_style_tutorial_003.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/neural-style/sphx_glr_neural_style_tutorial_003.png -------------------------------------------------------------------------------- /_static/img/neural-style/sphx_glr_neural_style_tutorial_004.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/neural-style/sphx_glr_neural_style_tutorial_004.png -------------------------------------------------------------------------------- /_static/img/thumbnails/sphx_glr_transfer_learning_tutorial_001.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/justinpinkney/tutorials/master/_static/img/thumbnails/sphx_glr_transfer_learning_tutorial_001.png -------------------------------------------------------------------------------- /_static/documentation_options.js: -------------------------------------------------------------------------------- 1 | var DOCUMENTATION_OPTIONS = { 2 | URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), 3 | VERSION: '0.5.0a0+a24163a', 4 | LANGUAGE: 'None', 5 | COLLAPSE_INDEX: false, 6 | FILE_SUFFIX: '.html', 7 | HAS_SOURCE: true, 8 | SOURCELINK_SUFFIX: '.txt' 9 | }; -------------------------------------------------------------------------------- /.jenkins/delete_html_file_with_runnable_code_removed.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | 4 | html_file_path = sys.argv[1] 5 | 6 | with open(html_file_path, 'r', encoding='utf-8') as html_file: 7 | html = html_file.read() 8 | 9 | if "%%%%%%RUNNABLE_CODE_REMOVED%%%%%%" in html: 10 | print("Removing " + html_file_path) 11 | os.remove(html_file_path) 12 | -------------------------------------------------------------------------------- /beginner_source/hybrid_frontend/README.txt: -------------------------------------------------------------------------------- 1 | Hybrid Frontend Tutorials 2 | ------------------------- 3 | 4 | 1. learning_hybrid_frontend_through_example_tutorial.py 5 | Learning Hybrid Frontend Through Example 6 | https://pytorch.org/tutorials/beginner/hybrid_frontend/learning_hybrid_frontend_through_example_tutorial.html 7 | 8 | 2. introduction_to_hybrid_frontend_tutorial.py 9 | Introduction to Hybrid Frontend 10 | https://pytorch.org/tutorials/beginner/hybrid_frontend/introduction_to_hybrid_frontend_tutorial.html 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Refer to ./jenkins/build.sh for tutorial build instructions 2 | 3 | sphinx 4 | sphinx-gallery 5 | tqdm 6 | numpy 7 | matplotlib 8 | torchvision 9 | PyHamcrest 10 | bs4 11 | awscli==1.16.35 12 | 13 | # PyTorch Theme 14 | -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme 15 | 16 | ipython 17 | 18 | # to run examples 19 | pandas 20 | scikit-image 21 | # pillow >= 4.2 will throw error when trying to write mode RGBA as JPEG, 22 | # this is a workaround to the issue. 23 | pillow==4.1.1 24 | wget 25 | -------------------------------------------------------------------------------- /.jenkins/remove_invisible_code_block_from_ipynb.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from bs4 import BeautifulSoup 3 | 4 | ipynb_file_path = sys.argv[1] 5 | output_file_path = sys.argv[2] 6 | 7 | with open(ipynb_file_path, 'r', encoding='utf-8') as ipynb_file: 8 | ipynb_lines = ipynb_file.readlines() 9 | 10 | ipynb_out_lines = [] 11 | 12 | for line in ipynb_lines: 13 | if not '%%%%%%INVISIBLE_CODE_BLOCK%%%%%%' in line: 14 | ipynb_out_lines.append(line) 15 | 16 | with open(output_file_path, "w", encoding='utf-8') as output_file: 17 | for line in ipynb_out_lines: 18 | output_file.write(line) 19 | -------------------------------------------------------------------------------- /.jenkins/remove_invisible_code_block_from_html.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from bs4 import BeautifulSoup 3 | 4 | html_file_path = sys.argv[1] 5 | output_file_path = sys.argv[2] 6 | 7 | with open(html_file_path, 'r', encoding='utf-8') as html_file: 8 | html = html_file.read() 9 | html_soup = BeautifulSoup(html, 'html.parser') 10 | 11 | elems = html_soup.find_all("div", {"class": "highlight-python"}) 12 | for elem in elems: 13 | if "%%%%%%INVISIBLE_CODE_BLOCK%%%%%%" in str(elem): 14 | elem.decompose() 15 | 16 | with open(output_file_path, "w", encoding='utf-8') as output_file: 17 | output_file.write(str(html_soup)) 18 | -------------------------------------------------------------------------------- /_static/img/seq-seq-images/encoder-network.dot: -------------------------------------------------------------------------------- 1 | digraph G { 2 | 3 | // Main styles 4 | nodesep=0.3; ranksep=0.15; 5 | 6 | node [shape=rect, fillcolor=darkorange, color=white, style=filled, fontsize=11, fontname="arial", height=0.2]; 7 | edge [color=gray, arrowsize=0.5]; 8 | 9 | // Layout 10 | {rank=same;input;prev_hidden} 11 | 12 | input -> embedding; 13 | embedding -> embedded; 14 | embedded -> gru; 15 | prev_hidden -> gru; 16 | gru -> output; 17 | gru -> hidden; 18 | 19 | embedding [fillcolor=dodgerblue, fontcolor=white]; 20 | gru [fillcolor=dodgerblue, fontcolor=white]; 21 | 22 | } 23 | -------------------------------------------------------------------------------- /beginner_source/former_torchies/README.txt: -------------------------------------------------------------------------------- 1 | PyTorch for former Torch users 2 | ------------------------------ 3 | 4 | 1. tensor_tutorial_old.py 5 | Tensors 6 | https://pytorch.org/tutorials/beginner/former_torchies/tensor_tutorial_old.html 7 | 8 | 2. autograd_tutorial_old.py 9 | Autograd 10 | https://pytorch.org/tutorials/beginner/former_torchies/autograd_tutorial_old.html 11 | 12 | 3. nnft_tutorial.py 13 | nn package 14 | https://pytorch.org/tutorials/beginner/former_torchies/nnft_tutorial.html 15 | 16 | 4. parallelism_tutorial.py 17 | Multi-GPU examples 18 | https://pytorch.org/tutorials/beginner/former_torchies/parallelism_tutorial.html 19 | -------------------------------------------------------------------------------- /advanced_source/README.txt: -------------------------------------------------------------------------------- 1 | Advanced Tutorials 2 | ------------------ 3 | 4 | 1. neural_style_tutorial.py 5 | Neural Transfer with PyTorch 6 | https://pytorch.org/tutorials/advanced/neural_style_tutorial.html 7 | 8 | 2. numpy_extensions_tutorial.py 9 | Creating Extensions Using numpy and scipy 10 | https://pytorch.org/tutorials/advanced/numpy_extensions_tutorial.html 11 | 12 | 3. c_extension.rst 13 | Custom C Extensions for PyTorch 14 | https://pytorch.org/tutorials/advanced/c_extension.html 15 | 16 | 4. super_resolution_with_caffe2.py 17 | Transfering a Model from PyTorch to Caffe2 and Mobile using ONNX 18 | https://pytorch.org/tutorials/advanced/super_resolution_with_caffe2.html 19 | -------------------------------------------------------------------------------- /beginner_source/hybrid_frontend_tutorial.rst: -------------------------------------------------------------------------------- 1 | Hybrid Frontend Tutorials 2 | ------------------------- 3 | **Authors**: `Nathan Inkawhich `_ and `Matthew Inkawhich `_ 4 | 5 | In this set of tutorials, you will learn the following: 6 | 7 | 1. What the hybrid frontend is and the suggested workflow 8 | 2. Basic syntax 9 | 3. How to transition an eager model to graph mode 10 | 11 | 12 | .. toctree:: 13 | :hidden: 14 | 15 | /beginner/hybrid_frontend/learning_hybrid_frontend_through_example_tutorial 16 | 17 | .. galleryitem:: /beginner/hybrid_frontend/learning_hybrid_frontend_through_example_tutorial.py 18 | 19 | .. raw:: html 20 | 21 |
22 | -------------------------------------------------------------------------------- /_static/images/logo-facebook-dark.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 8 | 9 | -------------------------------------------------------------------------------- /_static/images/view-page-source-icon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /.jenkins/remove_invisible_code_block_from_rst_txt.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from bs4 import BeautifulSoup 3 | 4 | rst_txt_file_path = sys.argv[1] 5 | output_file_path = sys.argv[2] 6 | 7 | with open(rst_txt_file_path, 'r', encoding='utf-8') as rst_txt_file: 8 | rst_txt = rst_txt_file.read() 9 | 10 | splits = rst_txt.split('.. code-block:: python\n\n\n # %%%%%%INVISIBLE_CODE_BLOCK%%%%%%\n') 11 | if len(splits) == 2: 12 | code_before_invisible_block = splits[0] 13 | code_after_invisible_block = splits[1].split(' # %%%%%%INVISIBLE_CODE_BLOCK%%%%%%\n')[1] 14 | rst_txt_out = code_before_invisible_block + code_after_invisible_block 15 | else: 16 | rst_txt_out = rst_txt 17 | 18 | with open(output_file_path, "w", encoding='utf-8') as output_file: 19 | output_file.write(rst_txt_out) 20 | -------------------------------------------------------------------------------- /.jenkins/remove_invisible_code_block_from_py.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from bs4 import BeautifulSoup 3 | 4 | py_file_path = sys.argv[1] 5 | output_file_path = sys.argv[2] 6 | 7 | with open(py_file_path, 'r', encoding='utf-8') as py_file: 8 | py_lines = py_file.readlines() 9 | 10 | py_out_lines = [] 11 | 12 | in_invisible_block = False 13 | for line in py_lines: 14 | if not in_invisible_block: 15 | if '%%%%%%INVISIBLE_CODE_BLOCK%%%%%%' in line: 16 | in_invisible_block = True 17 | else: 18 | py_out_lines.append(line) 19 | else: 20 | if '%%%%%%INVISIBLE_CODE_BLOCK%%%%%%' in line: 21 | in_invisible_block = False 22 | 23 | with open(output_file_path, "w", encoding='utf-8') as output_file: 24 | for line in py_out_lines: 25 | output_file.write(line) 26 | -------------------------------------------------------------------------------- /beginner_source/blitz/README.txt: -------------------------------------------------------------------------------- 1 | Deep Learning with PyTorch: A 60 Minute Blitz 2 | --------------------------------------------- 3 | 4 | 1. tensor_tutorial.py 5 | What is PyTorch? 6 | https://pytorch.org/tutorials/beginner/blitz/tensor_tutorial.html 7 | 8 | 2. autograd_tutorial.py 9 | Autograd: Automatic Differentiation 10 | https://pytorch.org/tutorials/beginner/blitz/autograd_tutorial.html 11 | 12 | 3. neural_networks_tutorial.py 13 | Neural Networks 14 | https://pytorch.org/tutorials/beginner/blitz/neural_networks_tutorial.html# 15 | 16 | 4. autograd_tutorial.py 17 | Automatic Differentiation 18 | https://pytorch.org/tutorials/beginner/blitz/autograd_tutorial.html 19 | 20 | 5. cifar10_tutorial.py 21 | Training a Classifier 22 | https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html 23 | 24 | 25 | -------------------------------------------------------------------------------- /_static/images/logo-icon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /beginner_source/nlp/README.txt: -------------------------------------------------------------------------------- 1 | Deep Learning for NLP with Pytorch 2 | ---------------------------------- 3 | 4 | 1. pytorch_tutorial.py 5 | Introduction to PyTorch 6 | https://pytorch.org/tutorials/beginner/nlp/pytorch_tutorial.html 7 | 8 | 2. deep_learning_tutorial.py 9 | Deep Learning with PyTorch 10 | https://pytorch.org/tutorials/beginner/nlp/deep_learning_tutorial.html 11 | 12 | 3. word_embeddings_tutorial.py 13 | Word Embeddings: Encoding Lexical Semantics 14 | https://pytorch.org/tutorials/beginner/nlp/word_embeddings_tutorial.html 15 | 16 | 4. sequence_models_tutorial.py 17 | Sequence Models and Long-Short Term Memory Networks 18 | https://pytorch.org/tutorials/beginner/nlp/sequence_models_tutorial.html 19 | 20 | 5. advanced_tutorial.py 21 | Advanced: Making Dynamic Decisions and the Bi-LSTM CRF 22 | https://pytorch.org/tutorials/beginner/nlp/advanced_tutorial.html -------------------------------------------------------------------------------- /beginner_source/README.txt: -------------------------------------------------------------------------------- 1 | Beginner Tutorials 2 | ------------------ 3 | 4 | 1. blitz/* and deep_learning_60min_blitz.rst 5 | Deep Learning with PyTorch: A 60 Minute Blitz 6 | https://pytorch.org/tutorials/beginner/deep_learning_60min_blitz.html 7 | 8 | 2. former_torches/* and former_torchies_tutorial.rst 9 | PyTorch for Former Torch Users 10 | https://pytorch.org/tutorials/beginner/former_torchies_tutorial.html 11 | 12 | 3. examples_*/* and pytorch_with_examples.rst 13 | Learning PyTorch with Examples 14 | https://pytorch.org/tutorials/beginner/pytorch_with_examples.html 15 | 16 | 4. transfer_learning_tutorial.py 17 | Transfer Learning Tutorial 18 | https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html 19 | 20 | 5. nlp/* and deep_learning_nlp_tutorial.rst 21 | Deep Learning for NLP with Pytorch 22 | https://pytorch.org/tutorials/beginner/deep_learning_nlp_tutorial.html 23 | -------------------------------------------------------------------------------- /_static/img/seq-seq-images/decoder-network.dot: -------------------------------------------------------------------------------- 1 | digraph G { 2 | 3 | // Main styles 4 | nodesep=0.3; ranksep=0.15; 5 | 6 | node [shape=rect, fillcolor=darkorange, color=white, style=filled, fontsize=11, fontname="arial", height=0.2]; 7 | edge [color=gray, arrowsize=0.5]; 8 | 9 | // Layout 10 | {rank=same;input;prev_hidden} 11 | 12 | input -> embedding; 13 | embedding -> relu; 14 | relu -> gru; 15 | 16 | prev_hidden -> gru; 17 | gru -> out; 18 | gru -> hidden; 19 | 20 | out -> softmax; 21 | softmax -> output; 22 | 23 | {rank=same;output;hidden} 24 | 25 | // Layer nodes 26 | embedding [fillcolor=dodgerblue, fontcolor=white]; 27 | gru [fillcolor=dodgerblue, fontcolor=white]; 28 | out [fillcolor=dodgerblue, fontcolor=white]; 29 | 30 | // Function nodes 31 | relu [fillcolor=palegreen]; 32 | softmax [fillcolor=palegreen]; 33 | 34 | } 35 | -------------------------------------------------------------------------------- /_static/images/chevron-right-orange.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | Page 1 7 | Created with Sketch. 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /_static/images/chevron-right-white.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | Page 1 7 | Created with Sketch. 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /_static/images/chevron-down-grey.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | Created with Sketch. 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /intermediate_source/README.txt: -------------------------------------------------------------------------------- 1 | Intermediate tutorials 2 | ---------------------- 3 | 4 | 1. char_rnn_classification_tutorial.py 5 | Classifying Names with a Character-Level RNN 6 | https://pytorch.org/tutorials/intermediate/char_rnn_classification_tutorial.html 7 | 8 | 2. char_rnn_generation_tutorial.py 9 | Generating Names with a Character-Level RNN 10 | https://pytorch.org/tutorials/intermediate/char_rnn_generation_tutorial.html 11 | 12 | 3. seq2seq_translation_tutorial.py 13 | Translation with a Sequence to Sequence Network and Attention 14 | https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html 15 | 16 | 4. reinforcement_q_learning.py 17 | Reinforcement Learning (DQN) Tutorial 18 | https://pytorch.org/tutorials/intermediate/reinforcement_q_learning.html 19 | 20 | 5. dist_tuto.rst 21 | Writing Distributed Applications with PyTorch 22 | https://pytorch.org/tutorials/intermediate/dist_tuto.html 23 | 24 | 6. spatial_transformer_tutorial 25 | Spatial Transformer Networks Tutorial 26 | https://pytorch.org/tutorials/intermediate/spatial_transformer_tutorial.html 27 | -------------------------------------------------------------------------------- /.jenkins/remove_invisible_code_block_batch.sh: -------------------------------------------------------------------------------- 1 | BUILDDIR=$1 2 | 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 4 | 5 | # Remove INVISIBLE_CODE_BLOCK from .html/.rst/.rst.txt/.ipynb/.py files 6 | for filename in $(find $BUILDDIR/beginner $BUILDDIR/intermediate $BUILDDIR/advanced -name '*.html'); do 7 | echo "Removing INVISIBLE_CODE_BLOCK from " $filename 8 | python $DIR/remove_invisible_code_block_from_html.py $filename $filename 9 | done 10 | for filename in $(find $BUILDDIR/_sources/beginner $BUILDDIR/_sources/intermediate $BUILDDIR/_sources/advanced -name '*.rst.txt'); do 11 | echo "Removing INVISIBLE_CODE_BLOCK from " $filename 12 | python $DIR/remove_invisible_code_block_from_rst_txt.py $filename $filename 13 | done 14 | for filename in $(find $BUILDDIR/_downloads -name '*.ipynb'); do 15 | echo "Removing INVISIBLE_CODE_BLOCK from " $filename 16 | python $DIR/remove_invisible_code_block_from_ipynb.py $filename $filename 17 | done 18 | for filename in $(find $BUILDDIR/_downloads -name '*.py'); do 19 | echo "Removing INVISIBLE_CODE_BLOCK from " $filename 20 | python $DIR/remove_invisible_code_block_from_py.py $filename $filename 21 | done 22 | -------------------------------------------------------------------------------- /beginner_source/former_torchies_tutorial.rst: -------------------------------------------------------------------------------- 1 | PyTorch for Former Torch Users 2 | ------------------------------ 3 | **Author**: `Soumith Chintala `_ 4 | 5 | In this tutorial, you will learn the following: 6 | 7 | 1. Using torch Tensors, and important difference against (Lua)Torch 8 | 2. Using the autograd package 9 | 3. Building neural networks 10 | 11 | - Building a ConvNet 12 | - Building a Recurrent Net 13 | 14 | 4. Use multiple GPUs 15 | 16 | 17 | .. toctree:: 18 | :hidden: 19 | 20 | /beginner/former_torchies/tensor_tutorial_old 21 | /beginner/former_torchies/autograd_tutorial_old 22 | /beginner/former_torchies/nnft_tutorial 23 | /beginner/former_torchies/parallelism_tutorial 24 | 25 | .. galleryitem:: /beginner/former_torchies/tensor_tutorial_old.py 26 | :figure: /_static/img/tensor_illustration_flat.png 27 | 28 | .. galleryitem:: /beginner/former_torchies/autograd_tutorial_old.py 29 | 30 | .. galleryitem:: /beginner/former_torchies/nnft_tutorial.py 31 | :figure: /_static/img/torch-nn-vs-pytorch-nn.png 32 | 33 | .. galleryitem:: /beginner/former_torchies/parallelism_tutorial.py 34 | 35 | .. raw:: html 36 | 37 |
38 | -------------------------------------------------------------------------------- /.jenkins/replace_tutorial_html_content.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | noplot_html_file_path = sys.argv[1] 4 | hasplot_html_file_path = sys.argv[2] 5 | output_html_file_path = sys.argv[3] 6 | 7 | from bs4 import BeautifulSoup 8 | with open(noplot_html_file_path, 'r', encoding='utf-8') as noplot_html_file: 9 | noplot_html = noplot_html_file.read() 10 | with open(hasplot_html_file_path, 'r', encoding='utf-8') as hasplot_html_file: 11 | hasplot_html = hasplot_html_file.read() 12 | 13 | noplot_html_soup = BeautifulSoup(noplot_html, 'html.parser') 14 | elems = noplot_html_soup.find_all("div", {"class": "sphx-glr-example-title"}) 15 | if len(elems) == 0: 16 | print("No match found, not replacing HTML content in "+noplot_html_file_path) 17 | elif len(elems) == 1: 18 | print("Match found in "+noplot_html_file_path+". Replacing its content.") 19 | elem = elems[0] 20 | elem.replace_with(BeautifulSoup(hasplot_html, 'html.parser').find_all("div", {"class": "sphx-glr-example-title"})[0]) 21 | with open(output_html_file_path, "w", encoding='utf-8') as output_html_file: 22 | output_html_file.write(str(noplot_html_soup)) 23 | else: 24 | raise Exception("Found more than one match in "+noplot_html_file_path+". Aborting.") 25 | -------------------------------------------------------------------------------- /_static/images/arrow-right-with-tail.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Page 1 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /_static/images/arrow-down-orange.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Group 5 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /_static/images/logo-twitter-dark.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8 | 9 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /_static/img/seq-seq-images/attention-decoder-network.dot: -------------------------------------------------------------------------------- 1 | digraph G { 2 | 3 | // Main styles 4 | nodesep=0.3; ranksep=0.15; 5 | 6 | node [shape=rect, fillcolor=darkorange, color=white, style=filled, fontsize=11, fontname="arial", height=0.2]; 7 | edge [color=gray, arrowsize=0.5]; 8 | 9 | // Layout 10 | {rank=same;input;prev_hidden;encoder_outputs} 11 | 12 | 13 | input -> embedding; 14 | embedding -> dropout; 15 | dropout -> embedded; 16 | 17 | embedded -> attn; 18 | prev_hidden -> attn; 19 | attn -> attn_softmax; 20 | attn_softmax -> attn_weights; 21 | attn_weights -> bmm; 22 | encoder_outputs -> bmm; 23 | bmm -> attn_applied; 24 | attn_applied -> attn_combine; 25 | embedded -> attn_combine; 26 | 27 | attn_combine -> relu -> gru; 28 | prev_hidden -> gru; 29 | gru -> out; 30 | gru -> hidden; 31 | 32 | out -> softmax; 33 | softmax -> output; 34 | 35 | {rank=same;output;hidden} 36 | 37 | // Layer nodes 38 | embedding [fillcolor=dodgerblue, fontcolor=white]; 39 | attn [fillcolor=dodgerblue, fontcolor=white]; 40 | attn_combine [fillcolor=dodgerblue, fontcolor=white]; 41 | bmm [fillcolor=dodgerblue, fontcolor=white]; 42 | gru [fillcolor=dodgerblue, fontcolor=white]; 43 | out [fillcolor=dodgerblue, fontcolor=white]; 44 | 45 | // Function nodes 46 | dropout [fillcolor=palegreen]; 47 | relu [fillcolor=palegreen]; 48 | softmax [fillcolor=palegreen]; 49 | attn_softmax [fillcolor=palegreen]; 50 | 51 | } 52 | -------------------------------------------------------------------------------- /beginner_source/deep_learning_60min_blitz.rst: -------------------------------------------------------------------------------- 1 | Deep Learning with PyTorch: A 60 Minute Blitz 2 | --------------------------------------------- 3 | **Author**: `Soumith Chintala `_ 4 | 5 | Goal of this tutorial: 6 | 7 | - Understand PyTorch’s Tensor library and neural networks at a high 8 | level. 9 | - Train a small neural network to classify images 10 | 11 | *This tutorial assumes that you have a basic familiarity of numpy* 12 | 13 | .. Note:: 14 | Make sure you have the `torch`_ and `torchvision`_ packages installed. 15 | 16 | .. _torch: https://github.com/pytorch/pytorch 17 | .. _torchvision: https://github.com/pytorch/vision 18 | 19 | 20 | .. toctree:: 21 | :hidden: 22 | 23 | /beginner/blitz/tensor_tutorial 24 | /beginner/blitz/autograd_tutorial 25 | /beginner/blitz/neural_networks_tutorial 26 | /beginner/blitz/cifar10_tutorial 27 | /beginner/blitz/data_parallel_tutorial 28 | 29 | .. galleryitem:: /beginner/blitz/tensor_tutorial.py 30 | :figure: /_static/img/tensor_illustration_flat.png 31 | 32 | .. galleryitem:: /beginner/blitz/autograd_tutorial.py 33 | :figure: /_static/img/autodiff.png 34 | 35 | .. galleryitem:: /beginner/blitz/neural_networks_tutorial.py 36 | :figure: /_static/img/mnist.png 37 | 38 | .. galleryitem:: /beginner/blitz/cifar10_tutorial.py 39 | :figure: /_static/img/cifar10.png 40 | 41 | .. galleryitem:: /beginner/blitz/data_parallel_tutorial.py 42 | :figure: /_static/img/data_parallel.png 43 | 44 | .. raw:: html 45 | 46 |
47 | -------------------------------------------------------------------------------- /_static/images/icon-close.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Page 1 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Pytorch contributors 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /beginner_source/examples_tensor/two_layer_net_numpy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Warm-up: numpy 4 | -------------- 5 | 6 | A fully-connected ReLU network with one hidden layer and no biases, trained to 7 | predict y from x using Euclidean error. 8 | 9 | This implementation uses numpy to manually compute the forward pass, loss, and 10 | backward pass. 11 | 12 | A numpy array is a generic n-dimensional array; it does not know anything about 13 | deep learning or gradients or computational graphs, and is just a way to perform 14 | generic numeric computations. 15 | """ 16 | import numpy as np 17 | 18 | # N is batch size; D_in is input dimension; 19 | # H is hidden dimension; D_out is output dimension. 20 | N, D_in, H, D_out = 64, 1000, 100, 10 21 | 22 | # Create random input and output data 23 | x = np.random.randn(N, D_in) 24 | y = np.random.randn(N, D_out) 25 | 26 | # Randomly initialize weights 27 | w1 = np.random.randn(D_in, H) 28 | w2 = np.random.randn(H, D_out) 29 | 30 | learning_rate = 1e-6 31 | for t in range(500): 32 | # Forward pass: compute predicted y 33 | h = x.dot(w1) 34 | h_relu = np.maximum(h, 0) 35 | y_pred = h_relu.dot(w2) 36 | 37 | # Compute and print loss 38 | loss = np.square(y_pred - y).sum() 39 | print(t, loss) 40 | 41 | # Backprop to compute gradients of w1 and w2 with respect to loss 42 | grad_y_pred = 2.0 * (y_pred - y) 43 | grad_w2 = h_relu.T.dot(grad_y_pred) 44 | grad_h_relu = grad_y_pred.dot(w2.T) 45 | grad_h = grad_h_relu.copy() 46 | grad_h[h < 0] = 0 47 | grad_w1 = x.T.dot(grad_h) 48 | 49 | # Update weights 50 | w1 -= learning_rate * grad_w1 51 | w2 -= learning_rate * grad_w2 52 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # sphinx things 2 | beginner 3 | intermediate 4 | advanced 5 | pytorch_basics 6 | 7 | #data things 8 | _data/ 9 | beginner_source/hymenoptera_data 10 | beginner_source/blitz/data 11 | beginner_source/faces 12 | beginner_source/hybrid_frontend/data 13 | intermediate_source/data/ 14 | advanced_source/images/ 15 | *data.zip 16 | faces.zip 17 | 18 | #builds 19 | _build/ 20 | _static/thumbs/ 21 | 22 | # Byte-compiled / optimized / DLL files 23 | __pycache__/ 24 | *.py[cod] 25 | *$py.class 26 | 27 | # C extensions 28 | *.so 29 | 30 | # Distribution / packaging 31 | .Python 32 | env/ 33 | build/ 34 | develop-eggs/ 35 | dist/ 36 | downloads/ 37 | eggs/ 38 | .eggs/ 39 | lib/ 40 | lib64/ 41 | parts/ 42 | sdist/ 43 | var/ 44 | *.egg-info/ 45 | .installed.cfg 46 | *.egg 47 | 48 | # PyInstaller 49 | # Usually these files are written by a python script from a template 50 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 51 | *.manifest 52 | *.spec 53 | 54 | # Installer logs 55 | pip-log.txt 56 | pip-delete-this-directory.txt 57 | 58 | # Unit test / coverage reports 59 | htmlcov/ 60 | .tox/ 61 | .coverage 62 | .coverage.* 63 | .cache 64 | nosetests.xml 65 | coverage.xml 66 | *,cover 67 | .hypothesis/ 68 | 69 | # Translations 70 | *.mo 71 | *.pot 72 | 73 | # Django stuff: 74 | *.log 75 | local_settings.py 76 | 77 | # Flask stuff: 78 | instance/ 79 | .webassets-cache 80 | 81 | # Scrapy stuff: 82 | .scrapy 83 | 84 | # Sphinx documentation 85 | docs/_build/ 86 | 87 | # PyBuilder 88 | target/ 89 | 90 | # IPython Notebook 91 | .ipynb_checkpoints 92 | 93 | # pyenv 94 | .python-version 95 | 96 | # celery beat schedule file 97 | celerybeat-schedule 98 | # dotenv 99 | .env 100 | 101 | # virtualenv 102 | venv/ 103 | ENV/ 104 | 105 | # Spyder project settings 106 | .spyderproject 107 | 108 | # Rope project settings 109 | .ropeproject 110 | 111 | # Mac things 112 | .DS_Store 113 | cleanup.sh 114 | *.swp 115 | -------------------------------------------------------------------------------- /.jenkins/remove_runnable_code.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | STATE_IN_MULTILINE_COMMENT_BLOCK_DOUBLE_QUOTE = "STATE_IN_MULTILINE_COMMENT_BLOCK_DOUBLE_QUOTE" 4 | STATE_IN_MULTILINE_COMMENT_BLOCK_SINGLE_QUOTE = "STATE_IN_MULTILINE_COMMENT_BLOCK_SINGLE_QUOTE" 5 | STATE_NORMAL = "STATE_NORMAL" 6 | 7 | python_file_path = sys.argv[1] 8 | output_file_path = sys.argv[2] 9 | 10 | with open(python_file_path, 'r', encoding='utf-8') as file: 11 | lines = file.readlines() 12 | ret_lines = [] 13 | state = STATE_NORMAL 14 | for line in lines: 15 | if state == STATE_NORMAL: 16 | if line.startswith('#'): 17 | ret_lines.append(line) 18 | state = STATE_NORMAL 19 | elif line.startswith('"""') or line.startswith('r"""'): 20 | ret_lines.append(line) 21 | state = STATE_IN_MULTILINE_COMMENT_BLOCK_DOUBLE_QUOTE 22 | elif line.startswith("'''") or line.startswith("r'''"): 23 | ret_lines.append(line) 24 | state = STATE_IN_MULTILINE_COMMENT_BLOCK_SINGLE_QUOTE 25 | else: 26 | ret_lines.append("\n") 27 | state = STATE_NORMAL 28 | elif state == STATE_IN_MULTILINE_COMMENT_BLOCK_DOUBLE_QUOTE: 29 | if line.startswith('"""'): 30 | ret_lines.append(line) 31 | state = STATE_NORMAL 32 | else: 33 | ret_lines.append(line) 34 | state = STATE_IN_MULTILINE_COMMENT_BLOCK_DOUBLE_QUOTE 35 | elif state == STATE_IN_MULTILINE_COMMENT_BLOCK_SINGLE_QUOTE: 36 | if line.startswith("'''"): 37 | ret_lines.append(line) 38 | state = STATE_NORMAL 39 | else: 40 | ret_lines.append(line) 41 | state = STATE_IN_MULTILINE_COMMENT_BLOCK_SINGLE_QUOTE 42 | 43 | ret_lines.append("\n# %%%%%%RUNNABLE_CODE_REMOVED%%%%%%") 44 | 45 | with open(output_file_path, 'w', encoding='utf-8') as file: 46 | for line in ret_lines: 47 | file.write(line) 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyTorch Tutorials 2 | 3 | 4 | All the tutorials are now presented as sphinx style documentation at: 5 | 6 | ## [https://pytorch.org/tutorials](https://pytorch.org/tutorials) 7 | 8 | 9 | 10 | # Contributing 11 | 12 | We use sphinx-gallery's [notebook styled examples](https://sphinx-gallery.github.io/tutorials/plot_notebook.html#sphx-glr-tutorials-plot-notebook-py) to create the tutorials. Syntax is very simple. In essence, you write a slightly well formatted python file and it shows up as documentation page. 13 | 14 | Here's how to create a new tutorial: 15 | 1. Create a notebook styled python file. If you want it executed while inserted into documentation, save the file with suffix `tutorial` so that file name is `your_tutorial.py`. 16 | 2. Put it in one of the beginner_source, intermediate_source, advanced_source based on the level. 17 | 2. Include it in the right TOC tree at index.rst 18 | 3. Create a thumbnail in the index file using a command like `.. galleryitem:: beginner/your_tutorial.py`. (This is a custom directive. See `custom_directives.py` for more info.) 19 | 20 | In case you prefer to write your tutorial in jupyter, you can use [this script](https://gist.github.com/chsasank/7218ca16f8d022e02a9c0deb94a310fe) to convert the notebook to python file. After conversion and addition to the project, please make sure the sections headings etc are in logical order. 21 | 22 | ## Building 23 | 24 | - Start with installing torch, torchvision, and your GPUs latest drivers. Install other requirements using `pip install -r requirements.txt` 25 | 26 | > If you want to use `virtualenv`, make your environment in a `venv` directory like: `virtualenv ./venv`, then `source ./venv/bin/activate`. 27 | 28 | - Then you can build using `make docs`. This will download the data, execute the tutorials and build the documentation to `docs/` directory. This will take about 60-120 min for systems with GPUs. If you do not have a GPU installed on your system, then see next step. 29 | - You can skip the computationally intensive graph generation by running `make html-noplot` to build basic html documentation to `_build/html`. This way, you can quickly preview your tutorial. 30 | -------------------------------------------------------------------------------- /_static/pytorch-logo-dark.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 9 | 10 | 12 | 13 | 14 | 15 | 16 | 18 | 20 | 21 | 24 | 26 | 29 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /_static/img/pytorch-logo-dark.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 9 | 10 | 12 | 13 | 14 | 15 | 16 | 18 | 20 | 21 | 24 | 26 | 29 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /beginner_source/examples_tensor/two_layer_net_tensor.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | PyTorch: Tensors 4 | ---------------- 5 | 6 | A fully-connected ReLU network with one hidden layer and no biases, trained to 7 | predict y from x by minimizing squared Euclidean distance. 8 | 9 | This implementation uses PyTorch tensors to manually compute the forward pass, 10 | loss, and backward pass. 11 | 12 | A PyTorch Tensor is basically the same as a numpy array: it does not know 13 | anything about deep learning or computational graphs or gradients, and is just 14 | a generic n-dimensional array to be used for arbitrary numeric computation. 15 | 16 | The biggest difference between a numpy array and a PyTorch Tensor is that 17 | a PyTorch Tensor can run on either CPU or GPU. To run operations on the GPU, 18 | just cast the Tensor to a cuda datatype. 19 | """ 20 | 21 | import torch 22 | 23 | 24 | dtype = torch.float 25 | device = torch.device("cpu") 26 | # device = torch.device("cuda:0") # Uncomment this to run on GPU 27 | 28 | # N is batch size; D_in is input dimension; 29 | # H is hidden dimension; D_out is output dimension. 30 | N, D_in, H, D_out = 64, 1000, 100, 10 31 | 32 | # Create random input and output data 33 | x = torch.randn(N, D_in, device=device, dtype=dtype) 34 | y = torch.randn(N, D_out, device=device, dtype=dtype) 35 | 36 | # Randomly initialize weights 37 | w1 = torch.randn(D_in, H, device=device, dtype=dtype) 38 | w2 = torch.randn(H, D_out, device=device, dtype=dtype) 39 | 40 | learning_rate = 1e-6 41 | for t in range(500): 42 | # Forward pass: compute predicted y 43 | h = x.mm(w1) 44 | h_relu = h.clamp(min=0) 45 | y_pred = h_relu.mm(w2) 46 | 47 | # Compute and print loss 48 | loss = (y_pred - y).pow(2).sum().item() 49 | print(t, loss) 50 | 51 | # Backprop to compute gradients of w1 and w2 with respect to loss 52 | grad_y_pred = 2.0 * (y_pred - y) 53 | grad_w2 = h_relu.t().mm(grad_y_pred) 54 | grad_h_relu = grad_y_pred.mm(w2.t()) 55 | grad_h = grad_h_relu.clone() 56 | grad_h[h < 0] = 0 57 | grad_w1 = x.t().mm(grad_h) 58 | 59 | # Update weights using gradient descent 60 | w1 -= learning_rate * grad_w1 61 | w2 -= learning_rate * grad_w2 62 | -------------------------------------------------------------------------------- /_static/images/logo-dark.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 15 | 17 | 18 | 21 | 23 | 26 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /beginner_source/examples_nn/two_layer_net_module.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | PyTorch: Custom nn Modules 4 | -------------------------- 5 | 6 | A fully-connected ReLU network with one hidden layer, trained to predict y from x 7 | by minimizing squared Euclidean distance. 8 | 9 | This implementation defines the model as a custom Module subclass. Whenever you 10 | want a model more complex than a simple sequence of existing Modules you will 11 | need to define your model this way. 12 | """ 13 | import torch 14 | 15 | 16 | class TwoLayerNet(torch.nn.Module): 17 | def __init__(self, D_in, H, D_out): 18 | """ 19 | In the constructor we instantiate two nn.Linear modules and assign them as 20 | member variables. 21 | """ 22 | super(TwoLayerNet, self).__init__() 23 | self.linear1 = torch.nn.Linear(D_in, H) 24 | self.linear2 = torch.nn.Linear(H, D_out) 25 | 26 | def forward(self, x): 27 | """ 28 | In the forward function we accept a Tensor of input data and we must return 29 | a Tensor of output data. We can use Modules defined in the constructor as 30 | well as arbitrary operators on Tensors. 31 | """ 32 | h_relu = self.linear1(x).clamp(min=0) 33 | y_pred = self.linear2(h_relu) 34 | return y_pred 35 | 36 | 37 | # N is batch size; D_in is input dimension; 38 | # H is hidden dimension; D_out is output dimension. 39 | N, D_in, H, D_out = 64, 1000, 100, 10 40 | 41 | # Create random Tensors to hold inputs and outputs 42 | x = torch.randn(N, D_in) 43 | y = torch.randn(N, D_out) 44 | 45 | # Construct our model by instantiating the class defined above 46 | model = TwoLayerNet(D_in, H, D_out) 47 | 48 | # Construct our loss function and an Optimizer. The call to model.parameters() 49 | # in the SGD constructor will contain the learnable parameters of the two 50 | # nn.Linear modules which are members of the model. 51 | criterion = torch.nn.MSELoss(reduction='sum') 52 | optimizer = torch.optim.SGD(model.parameters(), lr=1e-4) 53 | for t in range(500): 54 | # Forward pass: Compute predicted y by passing x to the model 55 | y_pred = model(x) 56 | 57 | # Compute and print loss 58 | loss = criterion(y_pred, y) 59 | print(t, loss.item()) 60 | 61 | # Zero gradients, perform a backward pass, and update the weights. 62 | optimizer.zero_grad() 63 | loss.backward() 64 | optimizer.step() 65 | -------------------------------------------------------------------------------- /_static/images/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 9 | 10 | 11 | 12 | 13 | 16 | 18 | 19 | 22 | 24 | 27 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /beginner_source/examples_nn/two_layer_net_optim.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | PyTorch: optim 4 | -------------- 5 | 6 | A fully-connected ReLU network with one hidden layer, trained to predict y from x 7 | by minimizing squared Euclidean distance. 8 | 9 | This implementation uses the nn package from PyTorch to build the network. 10 | 11 | Rather than manually updating the weights of the model as we have been doing, 12 | we use the optim package to define an Optimizer that will update the weights 13 | for us. The optim package defines many optimization algorithms that are commonly 14 | used for deep learning, including SGD+momentum, RMSProp, Adam, etc. 15 | """ 16 | import torch 17 | 18 | # N is batch size; D_in is input dimension; 19 | # H is hidden dimension; D_out is output dimension. 20 | N, D_in, H, D_out = 64, 1000, 100, 10 21 | 22 | # Create random Tensors to hold inputs and outputs 23 | x = torch.randn(N, D_in) 24 | y = torch.randn(N, D_out) 25 | 26 | # Use the nn package to define our model and loss function. 27 | model = torch.nn.Sequential( 28 | torch.nn.Linear(D_in, H), 29 | torch.nn.ReLU(), 30 | torch.nn.Linear(H, D_out), 31 | ) 32 | loss_fn = torch.nn.MSELoss(reduction='sum') 33 | 34 | # Use the optim package to define an Optimizer that will update the weights of 35 | # the model for us. Here we will use Adam; the optim package contains many other 36 | # optimization algoriths. The first argument to the Adam constructor tells the 37 | # optimizer which Tensors it should update. 38 | learning_rate = 1e-4 39 | optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) 40 | for t in range(500): 41 | # Forward pass: compute predicted y by passing x to the model. 42 | y_pred = model(x) 43 | 44 | # Compute and print loss. 45 | loss = loss_fn(y_pred, y) 46 | print(t, loss.item()) 47 | 48 | # Before the backward pass, use the optimizer object to zero all of the 49 | # gradients for the variables it will update (which are the learnable 50 | # weights of the model). This is because by default, gradients are 51 | # accumulated in buffers( i.e, not overwritten) whenever .backward() 52 | # is called. Checkout docs of torch.autograd.backward for more details. 53 | optimizer.zero_grad() 54 | 55 | # Backward pass: compute gradient of the loss with respect to model 56 | # parameters 57 | loss.backward() 58 | 59 | # Calling the step function on an Optimizer makes an update to its 60 | # parameters 61 | optimizer.step() 62 | -------------------------------------------------------------------------------- /beginner_source/colab.rst: -------------------------------------------------------------------------------- 1 | Using Tutorial Data from Google Drive in Colab 2 | ============================================== 3 | 4 | We've added a new feature to tutorials that allows users to open the 5 | notebook associated with a tutorial in Google Colab. You may need to 6 | copy data to your Google drive account to get the more complex tutorials 7 | to work. 8 | 9 | In this example, we'll demonstrate how to change the notebook in Colab 10 | to work with the Chatbot Tutorial. To do this, you'll first need to be 11 | logged into Google Drive. (For a full description of how to access data 12 | in Colab, you can view their example notebook 13 | `here `__.) 14 | 15 | To get started open the `Chatbot 16 | Tutorial `__ 17 | in your browser. 18 | 19 | At the top of the page click **Run in Google Colab**. 20 | 21 | The file will open in Colab. 22 | 23 | If you choose, **Runtime** then **Run All**, you'll get an error as the 24 | file can't be found. 25 | 26 | To fix this, we'll copy the required file into our Google Drive account. 27 | 28 | 1. Log into Google Drive. 29 | 2. In Google Drive, make a folder named **data**, with a subfolder named 30 | **cornell**. 31 | 3. Visit the Cornell Movie Dialogs Corpus and download the ZIP file. 32 | 4. Unzip the file on your local machine. 33 | 5. Copy the file **movie\_lines.txt** to **data/cornell** folder you 34 | created in Google Drive. 35 | 36 | Now we'll need to edit the file in\_ \_Colab to point to the file on 37 | Google Drive. 38 | 39 | In Colab, add the following to top of the code section over the line 40 | that begins *corpus\_name*: 41 | 42 | :: 43 | 44 | from google.colab import drive 45 | drive.mount('/content/gdrive') 46 | 47 | Change the two lines that follow: 48 | 49 | 1. Change the **corpus\_name** value to **"cornell"**. 50 | 2. Change the line that begins with **corpus** to this: 51 | 52 | :: 53 | 54 | corpus = os.path.join("/content/gdrive/My Drive/data", corpus_name) 55 | 56 | We're now pointing to the file we uploaded to Drive. 57 | 58 | Now when you click on the **Run cell** button for the code section, 59 | you'll be prompted to authorize Google Drive and you'll get an 60 | authorization code. Paste the code into the prompt in Colab and you 61 | should be set. 62 | 63 | Rerun the notebook from **Runtime** / **Run All** menu command and 64 | you'll see it process. (Note that this tutorial takes a long time to 65 | run.) 66 | 67 | Hopefully this example will give you a good starting point for running 68 | some of the more complex tutorials in Colab. As we evolve our use of 69 | Colab on the PyTorch tutorials site, we'll look at ways to make this 70 | easier for users. 71 | -------------------------------------------------------------------------------- /beginner_source/deep_learning_nlp_tutorial.rst: -------------------------------------------------------------------------------- 1 | Deep Learning for NLP with Pytorch 2 | ********************************** 3 | **Author**: `Robert Guthrie `_ 4 | 5 | This tutorial will walk you through the key ideas of deep learning 6 | programming using Pytorch. Many of the concepts (such as the computation 7 | graph abstraction and autograd) are not unique to Pytorch and are 8 | relevant to any deep learning toolkit out there. 9 | 10 | I am writing this tutorial to focus specifically on NLP for people who 11 | have never written code in any deep learning framework (e.g, TensorFlow, 12 | Theano, Keras, Dynet). It assumes working knowledge of core NLP 13 | problems: part-of-speech tagging, language modeling, etc. It also 14 | assumes familiarity with neural networks at the level of an intro AI 15 | class (such as one from the Russel and Norvig book). Usually, these 16 | courses cover the basic backpropagation algorithm on feed-forward neural 17 | networks, and make the point that they are chains of compositions of 18 | linearities and non-linearities. This tutorial aims to get you started 19 | writing deep learning code, given you have this prerequisite knowledge. 20 | 21 | Note this is about *models*, not data. For all of the models, I just 22 | create a few test examples with small dimensionality so you can see how 23 | the weights change as it trains. If you have some real data you want to 24 | try, you should be able to rip out any of the models from this notebook 25 | and use them on it. 26 | 27 | 28 | .. toctree:: 29 | :hidden: 30 | 31 | /beginner/nlp/pytorch_tutorial 32 | /beginner/nlp/deep_learning_tutorial 33 | /beginner/nlp/word_embeddings_tutorial 34 | /beginner/nlp/sequence_models_tutorial 35 | /beginner/nlp/advanced_tutorial 36 | 37 | 38 | .. galleryitem:: /beginner/nlp/pytorch_tutorial.py 39 | :intro: All of deep learning is computations on tensors, which are generalizations of a matrix that can be 40 | 41 | .. galleryitem:: /beginner/nlp/deep_learning_tutorial.py 42 | :intro: Deep learning consists of composing linearities with non-linearities in clever ways. The introduction of non-linearities allows 43 | 44 | .. galleryitem:: /beginner/nlp/word_embeddings_tutorial.py 45 | :intro: Word embeddings are dense vectors of real numbers, one per word in your vocabulary. In NLP, it is almost always the case that your features are 46 | 47 | .. galleryitem:: /beginner/nlp/sequence_models_tutorial.py 48 | :intro: At this point, we have seen various feed-forward networks. That is, there is no state maintained by the network at all. 49 | 50 | .. galleryitem:: /beginner/nlp/advanced_tutorial.py 51 | :intro: Dynamic versus Static Deep Learning Toolkits. Pytorch is a *dynamic* neural network kit. 52 | 53 | 54 | .. raw:: html 55 | 56 |
57 | -------------------------------------------------------------------------------- /_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | 3 | {% block footer %} 4 | {{ super() }} 5 | 15 | 16 | 17 | 18 | 28 | 29 | 77 | 78 | 83 | 84 | {% endblock %} 85 | -------------------------------------------------------------------------------- /beginner_source/examples_nn/two_layer_net_nn.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | PyTorch: nn 4 | ----------- 5 | 6 | A fully-connected ReLU network with one hidden layer, trained to predict y from x 7 | by minimizing squared Euclidean distance. 8 | 9 | This implementation uses the nn package from PyTorch to build the network. 10 | PyTorch autograd makes it easy to define computational graphs and take gradients, 11 | but raw autograd can be a bit too low-level for defining complex neural networks; 12 | this is where the nn package can help. The nn package defines a set of Modules, 13 | which you can think of as a neural network layer that has produces output from 14 | input and may have some trainable weights. 15 | """ 16 | import torch 17 | 18 | # N is batch size; D_in is input dimension; 19 | # H is hidden dimension; D_out is output dimension. 20 | N, D_in, H, D_out = 64, 1000, 100, 10 21 | 22 | # Create random Tensors to hold inputs and outputs 23 | x = torch.randn(N, D_in) 24 | y = torch.randn(N, D_out) 25 | 26 | # Use the nn package to define our model as a sequence of layers. nn.Sequential 27 | # is a Module which contains other Modules, and applies them in sequence to 28 | # produce its output. Each Linear Module computes output from input using a 29 | # linear function, and holds internal Tensors for its weight and bias. 30 | model = torch.nn.Sequential( 31 | torch.nn.Linear(D_in, H), 32 | torch.nn.ReLU(), 33 | torch.nn.Linear(H, D_out), 34 | ) 35 | 36 | # The nn package also contains definitions of popular loss functions; in this 37 | # case we will use Mean Squared Error (MSE) as our loss function. 38 | loss_fn = torch.nn.MSELoss(reduction='sum') 39 | 40 | learning_rate = 1e-4 41 | for t in range(500): 42 | # Forward pass: compute predicted y by passing x to the model. Module objects 43 | # override the __call__ operator so you can call them like functions. When 44 | # doing so you pass a Tensor of input data to the Module and it produces 45 | # a Tensor of output data. 46 | y_pred = model(x) 47 | 48 | # Compute and print loss. We pass Tensors containing the predicted and true 49 | # values of y, and the loss function returns a Tensor containing the 50 | # loss. 51 | loss = loss_fn(y_pred, y) 52 | print(t, loss.item()) 53 | 54 | # Zero the gradients before running the backward pass. 55 | model.zero_grad() 56 | 57 | # Backward pass: compute gradient of the loss with respect to all the learnable 58 | # parameters of the model. Internally, the parameters of each Module are stored 59 | # in Tensors with requires_grad=True, so this call will compute gradients for 60 | # all learnable parameters in the model. 61 | loss.backward() 62 | 63 | # Update the weights using gradient descent. Each parameter is a Tensor, so 64 | # we can access its gradients like we did before. 65 | with torch.no_grad(): 66 | for param in model.parameters(): 67 | param -= learning_rate * param.grad 68 | -------------------------------------------------------------------------------- /beginner_source/examples_nn/dynamic_net.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | PyTorch: Control Flow + Weight Sharing 4 | -------------------------------------- 5 | 6 | To showcase the power of PyTorch dynamic graphs, we will implement a very strange 7 | model: a fully-connected ReLU network that on each forward pass randomly chooses 8 | a number between 1 and 4 and has that many hidden layers, reusing the same 9 | weights multiple times to compute the innermost hidden layers. 10 | """ 11 | import random 12 | import torch 13 | 14 | 15 | class DynamicNet(torch.nn.Module): 16 | def __init__(self, D_in, H, D_out): 17 | """ 18 | In the constructor we construct three nn.Linear instances that we will use 19 | in the forward pass. 20 | """ 21 | super(DynamicNet, self).__init__() 22 | self.input_linear = torch.nn.Linear(D_in, H) 23 | self.middle_linear = torch.nn.Linear(H, H) 24 | self.output_linear = torch.nn.Linear(H, D_out) 25 | 26 | def forward(self, x): 27 | """ 28 | For the forward pass of the model, we randomly choose either 0, 1, 2, or 3 29 | and reuse the middle_linear Module that many times to compute hidden layer 30 | representations. 31 | 32 | Since each forward pass builds a dynamic computation graph, we can use normal 33 | Python control-flow operators like loops or conditional statements when 34 | defining the forward pass of the model. 35 | 36 | Here we also see that it is perfectly safe to reuse the same Module many 37 | times when defining a computational graph. This is a big improvement from Lua 38 | Torch, where each Module could be used only once. 39 | """ 40 | h_relu = self.input_linear(x).clamp(min=0) 41 | for _ in range(random.randint(0, 3)): 42 | h_relu = self.middle_linear(h_relu).clamp(min=0) 43 | y_pred = self.output_linear(h_relu) 44 | return y_pred 45 | 46 | 47 | # N is batch size; D_in is input dimension; 48 | # H is hidden dimension; D_out is output dimension. 49 | N, D_in, H, D_out = 64, 1000, 100, 10 50 | 51 | # Create random Tensors to hold inputs and outputs 52 | x = torch.randn(N, D_in) 53 | y = torch.randn(N, D_out) 54 | 55 | # Construct our model by instantiating the class defined above 56 | model = DynamicNet(D_in, H, D_out) 57 | 58 | # Construct our loss function and an Optimizer. Training this strange model with 59 | # vanilla stochastic gradient descent is tough, so we use momentum 60 | criterion = torch.nn.MSELoss(reduction='sum') 61 | optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9) 62 | for t in range(500): 63 | # Forward pass: Compute predicted y by passing x to the model 64 | y_pred = model(x) 65 | 66 | # Compute and print loss 67 | loss = criterion(y_pred, y) 68 | print(t, loss.item()) 69 | 70 | # Zero gradients, perform a backward pass, and update the weights. 71 | optimizer.zero_grad() 72 | loss.backward() 73 | optimizer.step() 74 | -------------------------------------------------------------------------------- /_static/css/pytorch_theme.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: "Lato","proxima-nova","Helvetica Neue",Arial,sans-serif; 3 | } 4 | 5 | /* Default header fonts are ugly */ 6 | h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend, p.caption { 7 | font-family: "Lato","proxima-nova","Helvetica Neue",Arial,sans-serif; 8 | } 9 | 10 | /* Use white for docs background */ 11 | .wy-side-nav-search { 12 | background-color: #fff; 13 | } 14 | 15 | .wy-nav-content-wrap, .wy-menu li.current > a { 16 | background-color: #fff; 17 | } 18 | 19 | @media screen and (min-width: 1400px) { 20 | .wy-nav-content-wrap { 21 | background-color: rgba(0, 0, 0, 0.0470588); 22 | } 23 | 24 | .wy-nav-content { 25 | background-color: #fff; 26 | } 27 | } 28 | 29 | /* Fixes for mobile */ 30 | .wy-nav-top { 31 | background-color: #fff; 32 | background-image: url('../img/pytorch-logo-dark.svg'); 33 | background-repeat: no-repeat; 34 | background-position: center; 35 | padding: 0; 36 | margin: 0.4045em 0.809em; 37 | color: #333; 38 | } 39 | 40 | .wy-nav-top > a { 41 | display: none; 42 | } 43 | 44 | @media screen and (max-width: 768px) { 45 | .wy-side-nav-search>a img.logo { 46 | height: 60px; 47 | } 48 | } 49 | 50 | /* This is needed to ensure that logo above search scales properly */ 51 | .wy-side-nav-search a { 52 | display: block; 53 | } 54 | 55 | /* This ensures that multiple constructors will remain in separate lines. */ 56 | .rst-content dl:not(.docutils) dt { 57 | display: table; 58 | } 59 | 60 | /* Use our red for literals (it's very similar to the original color) */ 61 | .rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal { 62 | color: #F05732; 63 | } 64 | 65 | .rst-content tt.xref, a .rst-content tt, .rst-content tt.xref, 66 | .rst-content code.xref, a .rst-content tt, a .rst-content code { 67 | color: #404040; 68 | } 69 | 70 | /* Change link colors (except for the menu) */ 71 | 72 | a { 73 | color: #F05732; 74 | } 75 | 76 | a:hover { 77 | color: #F05732; 78 | } 79 | 80 | 81 | a:visited { 82 | color: #D44D2C; 83 | } 84 | 85 | .wy-menu a { 86 | color: #b3b3b3; 87 | } 88 | 89 | .wy-menu a:hover { 90 | color: #b3b3b3; 91 | } 92 | 93 | a.icon.icon-home { 94 | color: #D44D2C; 95 | } 96 | 97 | .version{ 98 | color: #D44D2C !important; 99 | } 100 | 101 | /* Default footer text is quite big */ 102 | footer { 103 | font-size: 80%; 104 | } 105 | 106 | footer .rst-footer-buttons { 107 | font-size: 125%; /* revert footer settings - 1/80% = 125% */ 108 | } 109 | 110 | footer p { 111 | font-size: 100%; 112 | } 113 | 114 | /* For hidden headers that appear in TOC tree */ 115 | /* see https://stackoverflow.com/a/32363545/3343043 */ 116 | .rst-content .hidden-section { 117 | display: none; 118 | } 119 | 120 | nav .hidden-section { 121 | display: inherit; 122 | } 123 | 124 | /* Make code blocks have a background */ 125 | .codeblock,pre.literal-block,.rst-content .literal-block,.rst-content pre.literal-block,div[class^='highlight'] { 126 | background: rgba(0, 0, 0, 0.0470588); 127 | } 128 | -------------------------------------------------------------------------------- /beginner_source/examples_autograd/two_layer_net_autograd.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | PyTorch: Tensors and autograd 4 | ------------------------------- 5 | 6 | A fully-connected ReLU network with one hidden layer and no biases, trained to 7 | predict y from x by minimizing squared Euclidean distance. 8 | 9 | This implementation computes the forward pass using operations on PyTorch 10 | Tensors, and uses PyTorch autograd to compute gradients. 11 | 12 | 13 | A PyTorch Tensor represents a node in a computational graph. If ``x`` is a 14 | Tensor that has ``x.requires_grad=True`` then ``x.grad`` is another Tensor 15 | holding the gradient of ``x`` with respect to some scalar value. 16 | """ 17 | import torch 18 | 19 | dtype = torch.float 20 | device = torch.device("cpu") 21 | # device = torch.device("cuda:0") # Uncomment this to run on GPU 22 | 23 | # N is batch size; D_in is input dimension; 24 | # H is hidden dimension; D_out is output dimension. 25 | N, D_in, H, D_out = 64, 1000, 100, 10 26 | 27 | # Create random Tensors to hold input and outputs. 28 | # Setting requires_grad=False indicates that we do not need to compute gradients 29 | # with respect to these Tensors during the backward pass. 30 | x = torch.randn(N, D_in, device=device, dtype=dtype) 31 | y = torch.randn(N, D_out, device=device, dtype=dtype) 32 | 33 | # Create random Tensors for weights. 34 | # Setting requires_grad=True indicates that we want to compute gradients with 35 | # respect to these Tensors during the backward pass. 36 | w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True) 37 | w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True) 38 | 39 | learning_rate = 1e-6 40 | for t in range(500): 41 | # Forward pass: compute predicted y using operations on Tensors; these 42 | # are exactly the same operations we used to compute the forward pass using 43 | # Tensors, but we do not need to keep references to intermediate values since 44 | # we are not implementing the backward pass by hand. 45 | y_pred = x.mm(w1).clamp(min=0).mm(w2) 46 | 47 | # Compute and print loss using operations on Tensors. 48 | # Now loss is a Tensor of shape (1,) 49 | # loss.item() gets the a scalar value held in the loss. 50 | loss = (y_pred - y).pow(2).sum() 51 | print(t, loss.item()) 52 | 53 | # Use autograd to compute the backward pass. This call will compute the 54 | # gradient of loss with respect to all Tensors with requires_grad=True. 55 | # After this call w1.grad and w2.grad will be Tensors holding the gradient 56 | # of the loss with respect to w1 and w2 respectively. 57 | loss.backward() 58 | 59 | # Manually update weights using gradient descent. Wrap in torch.no_grad() 60 | # because weights have requires_grad=True, but we don't need to track this 61 | # in autograd. 62 | # An alternative way is to operate on weight.data and weight.grad.data. 63 | # Recall that tensor.data gives a tensor that shares the storage with 64 | # tensor, but doesn't track history. 65 | # You can also use torch.optim.SGD to achieve this. 66 | with torch.no_grad(): 67 | w1 -= learning_rate * w1.grad 68 | w2 -= learning_rate * w2.grad 69 | 70 | # Manually zero the gradients after updating weights 71 | w1.grad.zero_() 72 | w2.grad.zero_() 73 | -------------------------------------------------------------------------------- /beginner_source/examples_autograd/two_layer_net_custom_function.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | PyTorch: Defining New autograd Functions 4 | ---------------------------------------- 5 | 6 | A fully-connected ReLU network with one hidden layer and no biases, trained to 7 | predict y from x by minimizing squared Euclidean distance. 8 | 9 | This implementation computes the forward pass using operations on PyTorch 10 | Variables, and uses PyTorch autograd to compute gradients. 11 | 12 | In this implementation we implement our own custom autograd function to perform 13 | the ReLU function. 14 | """ 15 | import torch 16 | 17 | 18 | class MyReLU(torch.autograd.Function): 19 | """ 20 | We can implement our own custom autograd Functions by subclassing 21 | torch.autograd.Function and implementing the forward and backward passes 22 | which operate on Tensors. 23 | """ 24 | 25 | @staticmethod 26 | def forward(ctx, input): 27 | """ 28 | In the forward pass we receive a Tensor containing the input and return 29 | a Tensor containing the output. ctx is a context object that can be used 30 | to stash information for backward computation. You can cache arbitrary 31 | objects for use in the backward pass using the ctx.save_for_backward method. 32 | """ 33 | ctx.save_for_backward(input) 34 | return input.clamp(min=0) 35 | 36 | @staticmethod 37 | def backward(ctx, grad_output): 38 | """ 39 | In the backward pass we receive a Tensor containing the gradient of the loss 40 | with respect to the output, and we need to compute the gradient of the loss 41 | with respect to the input. 42 | """ 43 | input, = ctx.saved_tensors 44 | grad_input = grad_output.clone() 45 | grad_input[input < 0] = 0 46 | return grad_input 47 | 48 | 49 | dtype = torch.float 50 | device = torch.device("cpu") 51 | # device = torch.device("cuda:0") # Uncomment this to run on GPU 52 | 53 | # N is batch size; D_in is input dimension; 54 | # H is hidden dimension; D_out is output dimension. 55 | N, D_in, H, D_out = 64, 1000, 100, 10 56 | 57 | # Create random Tensors to hold input and outputs. 58 | x = torch.randn(N, D_in, device=device, dtype=dtype) 59 | y = torch.randn(N, D_out, device=device, dtype=dtype) 60 | 61 | # Create random Tensors for weights. 62 | w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True) 63 | w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True) 64 | 65 | learning_rate = 1e-6 66 | for t in range(500): 67 | # To apply our Function, we use Function.apply method. We alias this as 'relu'. 68 | relu = MyReLU.apply 69 | 70 | # Forward pass: compute predicted y using operations; we compute 71 | # ReLU using our custom autograd operation. 72 | y_pred = relu(x.mm(w1)).mm(w2) 73 | 74 | # Compute and print loss 75 | loss = (y_pred - y).pow(2).sum() 76 | print(t, loss.item()) 77 | 78 | # Use autograd to compute the backward pass. 79 | loss.backward() 80 | 81 | # Update weights using gradient descent 82 | with torch.no_grad(): 83 | w1 -= learning_rate * w1.grad 84 | w2 -= learning_rate * w2.grad 85 | 86 | # Manually zero the gradients after updating weights 87 | w1.grad.zero_() 88 | w2.grad.zero_() 89 | -------------------------------------------------------------------------------- /_static/images/icon-menu-dots-dark.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Page 1 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /beginner_source/examples_autograd/tf_two_layer_net.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | TensorFlow: Static Graphs 4 | ------------------------- 5 | 6 | A fully-connected ReLU network with one hidden layer and no biases, trained to 7 | predict y from x by minimizing squared Euclidean distance. 8 | 9 | This implementation uses basic TensorFlow operations to set up a computational 10 | graph, then executes the graph many times to actually train the network. 11 | 12 | One of the main differences between TensorFlow and PyTorch is that TensorFlow 13 | uses static computational graphs while PyTorch uses dynamic computational 14 | graphs. 15 | 16 | In TensorFlow we first set up the computational graph, then execute the same 17 | graph many times. 18 | """ 19 | import tensorflow as tf 20 | import numpy as np 21 | 22 | # First we set up the computational graph: 23 | 24 | # N is batch size; D_in is input dimension; 25 | # H is hidden dimension; D_out is output dimension. 26 | N, D_in, H, D_out = 64, 1000, 100, 10 27 | 28 | # Create placeholders for the input and target data; these will be filled 29 | # with real data when we execute the graph. 30 | x = tf.placeholder(tf.float32, shape=(None, D_in)) 31 | y = tf.placeholder(tf.float32, shape=(None, D_out)) 32 | 33 | # Create Variables for the weights and initialize them with random data. 34 | # A TensorFlow Variable persists its value across executions of the graph. 35 | w1 = tf.Variable(tf.random_normal((D_in, H))) 36 | w2 = tf.Variable(tf.random_normal((H, D_out))) 37 | 38 | # Forward pass: Compute the predicted y using operations on TensorFlow Tensors. 39 | # Note that this code does not actually perform any numeric operations; it 40 | # merely sets up the computational graph that we will later execute. 41 | h = tf.matmul(x, w1) 42 | h_relu = tf.maximum(h, tf.zeros(1)) 43 | y_pred = tf.matmul(h_relu, w2) 44 | 45 | # Compute loss using operations on TensorFlow Tensors 46 | loss = tf.reduce_sum((y - y_pred) ** 2.0) 47 | 48 | # Compute gradient of the loss with respect to w1 and w2. 49 | grad_w1, grad_w2 = tf.gradients(loss, [w1, w2]) 50 | 51 | # Update the weights using gradient descent. To actually update the weights 52 | # we need to evaluate new_w1 and new_w2 when executing the graph. Note that 53 | # in TensorFlow the the act of updating the value of the weights is part of 54 | # the computational graph; in PyTorch this happens outside the computational 55 | # graph. 56 | learning_rate = 1e-6 57 | new_w1 = w1.assign(w1 - learning_rate * grad_w1) 58 | new_w2 = w2.assign(w2 - learning_rate * grad_w2) 59 | 60 | # Now we have built our computational graph, so we enter a TensorFlow session to 61 | # actually execute the graph. 62 | with tf.Session() as sess: 63 | # Run the graph once to initialize the Variables w1 and w2. 64 | sess.run(tf.global_variables_initializer()) 65 | 66 | # Create numpy arrays holding the actual data for the inputs x and targets 67 | # y 68 | x_value = np.random.randn(N, D_in) 69 | y_value = np.random.randn(N, D_out) 70 | for _ in range(500): 71 | # Execute the graph many times. Each time it executes we want to bind 72 | # x_value to x and y_value to y, specified with the feed_dict argument. 73 | # Each time we execute the graph we want to compute the values for loss, 74 | # new_w1, and new_w2; the values of these Tensors are returned as numpy 75 | # arrays. 76 | loss_value, _, _ = sess.run([loss, new_w1, new_w2], 77 | feed_dict={x: x_value, y: y_value}) 78 | print(loss_value) 79 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # Locale 5 | export LC_ALL=C 6 | 7 | # You can set these variables from the command line. 8 | SPHINXOPTS = 9 | SPHINXBUILD = sphinx-build 10 | SPHINXPROJ = PyTorchTutorials 11 | SOURCEDIR = . 12 | BUILDDIR = _build 13 | DATADIR = _data 14 | GH_PAGES_SOURCES = $(SOURCEDIR) Makefile 15 | 16 | # Put it first so that "make" without argument is like "make help". 17 | help: 18 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 19 | 20 | .PHONY: help Makefile docs 21 | 22 | # Catch-all target: route all unknown targets to Sphinx using the new 23 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 24 | %: Makefile 25 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -v 26 | 27 | download: 28 | # IMPORTANT NOTE: Please make sure your dataset is downloaded to *_source/data folder, 29 | # otherwise CI might silently break. 30 | 31 | # NOTE: Please consider using the Step1 and one of Step2 for new dataset, 32 | # [something] should be replaced with the actual value. 33 | # Step1. DOWNLOAD: wget -N [SOURCE_FILE] -P $(DATADIR) 34 | # Step2-1. UNZIP: unzip -o $(DATADIR)/[SOURCE_FILE] -d [*_source/data/] 35 | # Step2-2. UNTAR: tar -xzf $(DATADIR)/[SOURCE_FILE] -C [*_source/data/] 36 | # Step2-3. AS-IS: cp $(DATADIR)/[SOURCE_FILE] [*_source/data/] 37 | 38 | # make data directories 39 | mkdir -p $(DATADIR) 40 | mkdir -p advanced_source/data 41 | mkdir -p beginner_source/data 42 | mkdir -p intermediate_source/data 43 | 44 | # transfer learning tutorial data 45 | wget -N https://download.pytorch.org/tutorial/hymenoptera_data.zip -P $(DATADIR) 46 | unzip -o $(DATADIR)/hymenoptera_data.zip -d beginner_source/data/ 47 | 48 | # nlp tutorial data 49 | wget -N https://download.pytorch.org/tutorial/data.zip -P $(DATADIR) 50 | unzip -o $(DATADIR)/data.zip -d intermediate_source/ # This will unzip all files in data.zip to intermediate_source/data/ folder 51 | 52 | # data loader tutorial 53 | wget -N https://download.pytorch.org/tutorial/faces.zip -P $(DATADIR) 54 | unzip -o $(DATADIR)/faces.zip -d beginner_source/data/ 55 | 56 | wget -N https://download.pytorch.org/models/tutorials/4000_checkpoint.tar -P $(DATADIR) 57 | cp $(DATADIR)/4000_checkpoint.tar beginner_source/data/ 58 | 59 | # neural style images 60 | rm -rf advanced_source/data/images/ || true 61 | mkdir -p advanced_source/data/images/ 62 | cp -r _static/img/neural-style/ advanced_source/data/images/ 63 | 64 | # Download dataset for beginner_source/dcgan_faces_tutorial.py 65 | wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/img_align_celeba.zip -P $(DATADIR) 66 | unzip -q -o $(DATADIR)/img_align_celeba.zip -d beginner_source/data/celeba 67 | 68 | # Download dataset for beginner_source/hybrid_frontend/introduction_to_hybrid_frontend_tutorial.py 69 | wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/iris.data -P $(DATADIR) 70 | cp $(DATADIR)/iris.data beginner_source/data/ 71 | 72 | # Download dataset for beginner_source/chatbot_tutorial.py 73 | wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/cornell_movie_dialogs_corpus.zip -P $(DATADIR) 74 | unzip -q -o $(DATADIR)/cornell_movie_dialogs_corpus.zip -d beginner_source/data/ 75 | 76 | # Download dataset for beginner_source/audio_classifier_tutorial.py 77 | wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/UrbanSound8K.tar.gz -P $(DATADIR) 78 | tar -xzf $(DATADIR)/UrbanSound8K.tar.gz -C ./beginner_source/data/ 79 | 80 | # Download model for beginner_source/fgsm_tutorial.py 81 | wget -N https://s3.amazonaws.com/pytorch-tutorial-assets/lenet_mnist_model.pth -P $(DATADIR) 82 | cp $(DATADIR)/lenet_mnist_model.pth ./beginner_source/data/lenet_mnist_model.pth 83 | 84 | docs: 85 | make download 86 | make html 87 | rm -rf docs 88 | cp -r $(BUILDDIR)/html docs 89 | touch docs/.nojekyll 90 | 91 | html-noplot: 92 | $(SPHINXBUILD) -D plot_gallery=0 -b html $(SPHINXOPTS) "$(SOURCEDIR)" "$(BUILDDIR)/html" 93 | bash .jenkins/remove_invisible_code_block_batch.sh "$(BUILDDIR)/html" 94 | @echo 95 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 96 | 97 | clean-cache: 98 | make clean 99 | rm -rf advanced beginner intermediate 100 | -------------------------------------------------------------------------------- /beginner_source/former_torchies/autograd_tutorial_old.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Autograd 4 | ======== 5 | 6 | Autograd is now a core torch package for automatic differentiation. 7 | It uses a tape based system for automatic differentiation. 8 | 9 | In the forward phase, the autograd tape will remember all the operations 10 | it executed, and in the backward phase, it will replay the operations. 11 | 12 | Tensors that track history 13 | -------------------------- 14 | 15 | In autograd, if any input ``Tensor`` of an operation has ``requires_grad=True``, 16 | the computation will be tracked. After computing the backward pass, a gradient 17 | w.r.t. this tensor is accumulated into ``.grad`` attribute. 18 | 19 | There’s one more class which is very important for autograd 20 | implementation - a ``Function``. ``Tensor`` and ``Function`` are 21 | interconnected and build up an acyclic graph, that encodes a complete 22 | history of computation. Each variable has a ``.grad_fn`` attribute that 23 | references a function that has created a function (except for Tensors 24 | created by the user - these have ``None`` as ``.grad_fn``). 25 | 26 | If you want to compute the derivatives, you can call ``.backward()`` on 27 | a ``Tensor``. If ``Tensor`` is a scalar (i.e. it holds a one element 28 | tensor), you don’t need to specify any arguments to ``backward()``, 29 | however if it has more elements, you need to specify a ``grad_output`` 30 | argument that is a tensor of matching shape. 31 | """ 32 | 33 | import torch 34 | 35 | ############################################################### 36 | # Create a tensor and set requires_grad=True to track computation with it 37 | x = torch.ones(2, 2, requires_grad=True) 38 | print(x) 39 | 40 | ############################################################### 41 | # 42 | print(x.data) 43 | 44 | ############################################################### 45 | # 46 | print(x.grad) 47 | 48 | ############################################################### 49 | # 50 | 51 | print(x.grad_fn) # we've created x ourselves 52 | 53 | ############################################################### 54 | # Do an operation of x: 55 | 56 | y = x + 2 57 | print(y) 58 | 59 | ############################################################### 60 | # y was created as a result of an operation, 61 | # so it has a grad_fn 62 | print(y.grad_fn) 63 | 64 | ############################################################### 65 | # More operations on y: 66 | 67 | z = y * y * 3 68 | out = z.mean() 69 | 70 | print(z, out) 71 | 72 | ################################################################ 73 | # ``.requires_grad_( ... )`` changes an existing Tensor's ``requires_grad`` 74 | # flag in-place. The input flag defaults to ``True`` if not given. 75 | a = torch.randn(2, 2) 76 | a = ((a * 3) / (a - 1)) 77 | print(a.requires_grad) 78 | a.requires_grad_(True) 79 | print(a.requires_grad) 80 | b = (a * a).sum() 81 | print(b.grad_fn) 82 | 83 | ############################################################### 84 | # Gradients 85 | # --------- 86 | # 87 | # let's backprop now and print gradients d(out)/dx 88 | 89 | out.backward() 90 | print(x.grad) 91 | 92 | 93 | ############################################################### 94 | # By default, gradient computation flushes all the internal buffers 95 | # contained in the graph, so if you even want to do the backward on some 96 | # part of the graph twice, you need to pass in ``retain_variables = True`` 97 | # during the first pass. 98 | 99 | x = torch.ones(2, 2, requires_grad=True) 100 | y = x + 2 101 | y.backward(torch.ones(2, 2), retain_graph=True) 102 | # the retain_variables flag will prevent the internal buffers from being freed 103 | print(x.grad) 104 | 105 | ############################################################### 106 | # 107 | z = y * y 108 | print(z) 109 | 110 | ############################################################### 111 | # 112 | # just backprop random gradients 113 | 114 | gradient = torch.randn(2, 2) 115 | 116 | # this would fail if we didn't specify 117 | # that we want to retain variables 118 | y.backward(gradient) 119 | 120 | print(x.grad) 121 | 122 | ############################################################### 123 | # You can also stops autograd from tracking history on Tensors 124 | # with requires_grad=True by wrapping the code block in 125 | # ``with torch.no_grad():`` 126 | print(x.requires_grad) 127 | print((x ** 2).requires_grad) 128 | 129 | with torch.no_grad(): 130 | print((x ** 2).requires_grad) 131 | -------------------------------------------------------------------------------- /beginner_source/former_torchies/tensor_tutorial_old.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tensors 3 | ======= 4 | 5 | Tensors behave almost exactly the same way in PyTorch as they do in 6 | Torch. 7 | 8 | Create a tensor of size (5 x 7) with uninitialized memory: 9 | 10 | """ 11 | 12 | import torch 13 | a = torch.empty(5, 7, dtype=torch.float) 14 | 15 | ############################################################### 16 | # Initialize a double tensor randomized with a normal distribution with mean=0, 17 | # var=1: 18 | 19 | a = torch.randn(5, 7, dtype=torch.double) 20 | print(a) 21 | print(a.size()) 22 | 23 | ############################################################### 24 | # .. note:: 25 | # ``torch.Size`` is in fact a tuple, so it supports the same operations 26 | # 27 | # Inplace / Out-of-place 28 | # ---------------------- 29 | # 30 | # The first difference is that ALL operations on the tensor that operate 31 | # in-place on it will have an ``_`` postfix. For example, ``add`` is the 32 | # out-of-place version, and ``add_`` is the in-place version. 33 | 34 | a.fill_(3.5) 35 | # a has now been filled with the value 3.5 36 | 37 | b = a.add(4.0) 38 | # a is still filled with 3.5 39 | # new tensor b is returned with values 3.5 + 4.0 = 7.5 40 | 41 | print(a, b) 42 | 43 | ############################################################### 44 | # Some operations like ``narrow`` do not have in-place versions, and 45 | # hence, ``.narrow_`` does not exist. Similarly, some operations like 46 | # ``fill_`` do not have an out-of-place version, so ``.fill`` does not 47 | # exist. 48 | # 49 | # Zero Indexing 50 | # ------------- 51 | # 52 | # Another difference is that Tensors are zero-indexed. (In lua, tensors are 53 | # one-indexed) 54 | 55 | b = a[0, 3] # select 1st row, 4th column from a 56 | 57 | ############################################################### 58 | # Tensors can be also indexed with Python's slicing 59 | 60 | b = a[:, 3:5] # selects all rows, 4th column and 5th column from a 61 | 62 | ############################################################### 63 | # No camel casing 64 | # --------------- 65 | # 66 | # The next small difference is that all functions are now NOT camelCase 67 | # anymore. For example ``indexAdd`` is now called ``index_add_`` 68 | 69 | 70 | x = torch.ones(5, 5) 71 | print(x) 72 | 73 | ############################################################### 74 | # 75 | 76 | z = torch.empty(5, 2) 77 | z[:, 0] = 10 78 | z[:, 1] = 100 79 | print(z) 80 | 81 | ############################################################### 82 | # 83 | x.index_add_(1, torch.tensor([4, 0], dtype=torch.long), z) 84 | print(x) 85 | 86 | ############################################################### 87 | # Numpy Bridge 88 | # ------------ 89 | # 90 | # Converting a torch Tensor to a numpy array and vice versa is a breeze. 91 | # The torch Tensor and numpy array will share their underlying memory 92 | # locations, and changing one will change the other. 93 | # 94 | # Converting torch Tensor to numpy Array 95 | # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 96 | 97 | a = torch.ones(5) 98 | print(a) 99 | 100 | ############################################################### 101 | # 102 | 103 | b = a.numpy() 104 | print(b) 105 | 106 | ############################################################### 107 | # 108 | a.add_(1) 109 | print(a) 110 | print(b) # see how the numpy array changed in value 111 | 112 | 113 | ############################################################### 114 | # Converting numpy Array to torch Tensor 115 | # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 116 | 117 | import numpy as np 118 | a = np.ones(5) 119 | b = torch.from_numpy(a) 120 | np.add(a, 1, out=a) 121 | print(a) 122 | print(b) # see how changing the np array changed the torch Tensor automatically 123 | 124 | ############################################################### 125 | # All the Tensors on the CPU except a CharTensor support converting to 126 | # NumPy and back. 127 | # 128 | # CUDA Tensors 129 | # ------------ 130 | # 131 | # CUDA Tensors are nice and easy in pytorch, and transfering a CUDA tensor 132 | # from the CPU to GPU will retain its underlying type. 133 | 134 | # let us run this cell only if CUDA is available 135 | if torch.cuda.is_available(): 136 | 137 | # creates a LongTensor and transfers it 138 | # to GPU as torch.cuda.LongTensor 139 | a = torch.full((10,), 3, device=torch.device("cuda")) 140 | print(type(a)) 141 | b = a.to(torch.device("cpu")) 142 | # transfers it to CPU, back to 143 | # being a torch.LongTensor 144 | -------------------------------------------------------------------------------- /_static/pygments.css: -------------------------------------------------------------------------------- 1 | .highlight .hll { background-color: #ffffcc } 2 | .highlight { background: #eeffcc; } 3 | .highlight .c { color: #408090; font-style: italic } /* Comment */ 4 | .highlight .err { border: 1px solid #FF0000 } /* Error */ 5 | .highlight .k { color: #007020; font-weight: bold } /* Keyword */ 6 | .highlight .o { color: #666666 } /* Operator */ 7 | .highlight .ch { color: #408090; font-style: italic } /* Comment.Hashbang */ 8 | .highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */ 9 | .highlight .cp { color: #007020 } /* Comment.Preproc */ 10 | .highlight .cpf { color: #408090; font-style: italic } /* Comment.PreprocFile */ 11 | .highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */ 12 | .highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ 13 | .highlight .gd { color: #A00000 } /* Generic.Deleted */ 14 | .highlight .ge { font-style: italic } /* Generic.Emph */ 15 | .highlight .gr { color: #FF0000 } /* Generic.Error */ 16 | .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 17 | .highlight .gi { color: #00A000 } /* Generic.Inserted */ 18 | .highlight .go { color: #333333 } /* Generic.Output */ 19 | .highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ 20 | .highlight .gs { font-weight: bold } /* Generic.Strong */ 21 | .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 22 | .highlight .gt { color: #0044DD } /* Generic.Traceback */ 23 | .highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ 24 | .highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ 25 | .highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ 26 | .highlight .kp { color: #007020 } /* Keyword.Pseudo */ 27 | .highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ 28 | .highlight .kt { color: #902000 } /* Keyword.Type */ 29 | .highlight .m { color: #208050 } /* Literal.Number */ 30 | .highlight .s { color: #4070a0 } /* Literal.String */ 31 | .highlight .na { color: #4070a0 } /* Name.Attribute */ 32 | .highlight .nb { color: #007020 } /* Name.Builtin */ 33 | .highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ 34 | .highlight .no { color: #60add5 } /* Name.Constant */ 35 | .highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ 36 | .highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ 37 | .highlight .ne { color: #007020 } /* Name.Exception */ 38 | .highlight .nf { color: #06287e } /* Name.Function */ 39 | .highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ 40 | .highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ 41 | .highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ 42 | .highlight .nv { color: #bb60d5 } /* Name.Variable */ 43 | .highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ 44 | .highlight .w { color: #bbbbbb } /* Text.Whitespace */ 45 | .highlight .mb { color: #208050 } /* Literal.Number.Bin */ 46 | .highlight .mf { color: #208050 } /* Literal.Number.Float */ 47 | .highlight .mh { color: #208050 } /* Literal.Number.Hex */ 48 | .highlight .mi { color: #208050 } /* Literal.Number.Integer */ 49 | .highlight .mo { color: #208050 } /* Literal.Number.Oct */ 50 | .highlight .sa { color: #4070a0 } /* Literal.String.Affix */ 51 | .highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ 52 | .highlight .sc { color: #4070a0 } /* Literal.String.Char */ 53 | .highlight .dl { color: #4070a0 } /* Literal.String.Delimiter */ 54 | .highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ 55 | .highlight .s2 { color: #4070a0 } /* Literal.String.Double */ 56 | .highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ 57 | .highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ 58 | .highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ 59 | .highlight .sx { color: #c65d09 } /* Literal.String.Other */ 60 | .highlight .sr { color: #235388 } /* Literal.String.Regex */ 61 | .highlight .s1 { color: #4070a0 } /* Literal.String.Single */ 62 | .highlight .ss { color: #517918 } /* Literal.String.Symbol */ 63 | .highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ 64 | .highlight .fm { color: #06287e } /* Name.Function.Magic */ 65 | .highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ 66 | .highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ 67 | .highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ 68 | .highlight .vm { color: #bb60d5 } /* Name.Variable.Magic */ 69 | .highlight .il { color: #208050 } /* Literal.Number.Integer.Long */ -------------------------------------------------------------------------------- /_static/gallery.css: -------------------------------------------------------------------------------- 1 | /* 2 | Sphinx-Gallery has compatible CSS to fix default sphinx themes 3 | Tested for Sphinx 1.3.1 for all themes: default, alabaster, sphinxdoc, 4 | scrolls, agogo, traditional, nature, haiku, pyramid 5 | Tested for Read the Docs theme 0.1.7 */ 6 | .sphx-glr-thumbcontainer { 7 | background: #fff; 8 | border: solid #fff 1px; 9 | -moz-border-radius: 5px; 10 | -webkit-border-radius: 5px; 11 | border-radius: 5px; 12 | box-shadow: none; 13 | float: left; 14 | margin: 5px; 15 | min-height: 230px; 16 | padding-top: 5px; 17 | position: relative; 18 | } 19 | .sphx-glr-thumbcontainer:hover { 20 | border: solid #b4ddfc 1px; 21 | box-shadow: 0 0 15px rgba(142, 176, 202, 0.5); 22 | } 23 | .sphx-glr-thumbcontainer a.internal { 24 | bottom: 0; 25 | display: block; 26 | left: 0; 27 | padding: 150px 10px 0; 28 | position: absolute; 29 | right: 0; 30 | top: 0; 31 | } 32 | /* Next one is to avoid Sphinx traditional theme to cover all the 33 | thumbnail with its default link Background color */ 34 | .sphx-glr-thumbcontainer a.internal:hover { 35 | background-color: transparent; 36 | } 37 | 38 | .sphx-glr-thumbcontainer p { 39 | margin: 0 0 .1em 0; 40 | } 41 | .sphx-glr-thumbcontainer .figure { 42 | margin: 10px; 43 | width: 160px; 44 | } 45 | .sphx-glr-thumbcontainer img { 46 | display: inline; 47 | max-height: 160px; 48 | width: 160px; 49 | } 50 | .sphx-glr-thumbcontainer[tooltip]:hover:after { 51 | background: rgba(0, 0, 0, 0.8); 52 | -webkit-border-radius: 5px; 53 | -moz-border-radius: 5px; 54 | border-radius: 5px; 55 | color: #fff; 56 | content: attr(tooltip); 57 | left: 95%; 58 | padding: 5px 15px; 59 | position: absolute; 60 | z-index: 98; 61 | width: 220px; 62 | bottom: 52%; 63 | } 64 | .sphx-glr-thumbcontainer[tooltip]:hover:before { 65 | border: solid; 66 | border-color: #333 transparent; 67 | border-width: 18px 0 0 20px; 68 | bottom: 58%; 69 | content: ''; 70 | left: 85%; 71 | position: absolute; 72 | z-index: 99; 73 | } 74 | 75 | .highlight-pytb pre { 76 | background-color: #ffe4e4; 77 | border: 1px solid #f66; 78 | margin-top: 10px; 79 | padding: 7px; 80 | } 81 | 82 | .sphx-glr-script-out { 83 | color: #888; 84 | margin: 0; 85 | } 86 | .sphx-glr-script-out .highlight { 87 | background-color: transparent; 88 | margin-left: 2.5em; 89 | margin-top: -1.4em; 90 | } 91 | .sphx-glr-script-out .highlight pre { 92 | background-color: #fafae2; 93 | border: 0; 94 | max-height: 30em; 95 | overflow: auto; 96 | padding-left: 1ex; 97 | margin: 0px; 98 | word-break: break-word; 99 | } 100 | .sphx-glr-script-out + p { 101 | margin-top: 1.8em; 102 | } 103 | blockquote.sphx-glr-script-out { 104 | margin-left: 0pt; 105 | } 106 | 107 | div.sphx-glr-footer { 108 | text-align: center; 109 | } 110 | 111 | div.binder-badge { 112 | margin: 1em auto; 113 | vertical-align: middle; 114 | } 115 | 116 | div.sphx-glr-download { 117 | margin: 1em auto; 118 | vertical-align: middle; 119 | } 120 | 121 | div.sphx-glr-download a { 122 | background-color: #ffc; 123 | background-image: linear-gradient(to bottom, #FFC, #d5d57e); 124 | border-radius: 4px; 125 | border: 1px solid #c2c22d; 126 | color: #000; 127 | display: inline-block; 128 | font-weight: bold; 129 | padding: 1ex; 130 | text-align: center; 131 | } 132 | 133 | div.sphx-glr-download code.download { 134 | display: inline-block; 135 | white-space: normal; 136 | word-break: normal; 137 | overflow-wrap: break-word; 138 | /* border and background are given by the enclosing 'a' */ 139 | border: none; 140 | background: none; 141 | } 142 | 143 | div.sphx-glr-download a:hover { 144 | box-shadow: inset 0 1px 0 rgba(255,255,255,.1), 0 1px 5px rgba(0,0,0,.25); 145 | text-decoration: none; 146 | background-image: none; 147 | background-color: #d5d57e; 148 | } 149 | 150 | .sphx-glr-example-title > :target::before { 151 | display: block; 152 | content: ""; 153 | margin-top: -50px; 154 | height: 50px; 155 | visibility: hidden; 156 | } 157 | 158 | ul.sphx-glr-horizontal { 159 | list-style: none; 160 | padding: 0; 161 | } 162 | ul.sphx-glr-horizontal li { 163 | display: inline; 164 | } 165 | ul.sphx-glr-horizontal img { 166 | height: auto !important; 167 | } 168 | 169 | .sphx-glr-single-img { 170 | margin: auto; 171 | display: block; 172 | max-width: 100%; 173 | } 174 | 175 | .sphx-glr-multi-img { 176 | max-width: 42%; 177 | height: auto; 178 | } 179 | 180 | p.sphx-glr-signature a.reference.external { 181 | -moz-border-radius: 5px; 182 | -webkit-border-radius: 5px; 183 | border-radius: 5px; 184 | padding: 3px; 185 | font-size: 75%; 186 | text-align: right; 187 | margin-left: auto; 188 | display: table; 189 | } 190 | -------------------------------------------------------------------------------- /advanced_source/numpy_extensions_tutorial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Creating Extensions Using numpy and scipy 4 | ========================================= 5 | **Author**: `Adam Paszke `_ 6 | 7 | **Updated by**: `Adam Dziedzic `_ 8 | 9 | In this tutorial, we shall go through two tasks: 10 | 11 | 1. Create a neural network layer with no parameters. 12 | 13 | - This calls into **numpy** as part of its implementation 14 | 15 | 2. Create a neural network layer that has learnable weights 16 | 17 | - This calls into **SciPy** as part of its implementation 18 | """ 19 | 20 | import torch 21 | from torch.autograd import Function 22 | 23 | ############################################################### 24 | # Parameter-less example 25 | # ---------------------- 26 | # 27 | # This layer doesn’t particularly do anything useful or mathematically 28 | # correct. 29 | # 30 | # It is aptly named BadFFTFunction 31 | # 32 | # **Layer Implementation** 33 | 34 | from numpy.fft import rfft2, irfft2 35 | 36 | 37 | class BadFFTFunction(Function): 38 | 39 | def forward(self, input): 40 | numpy_input = input.detach().numpy() 41 | result = abs(rfft2(numpy_input)) 42 | return input.new(result) 43 | 44 | def backward(self, grad_output): 45 | numpy_go = grad_output.numpy() 46 | result = irfft2(numpy_go) 47 | return grad_output.new(result) 48 | 49 | # since this layer does not have any parameters, we can 50 | # simply declare this as a function, rather than as an nn.Module class 51 | 52 | 53 | def incorrect_fft(input): 54 | return BadFFTFunction()(input) 55 | 56 | ############################################################### 57 | # **Example usage of the created layer:** 58 | 59 | input = torch.randn(8, 8, requires_grad=True) 60 | result = incorrect_fft(input) 61 | print(result) 62 | result.backward(torch.randn(result.size())) 63 | print(input) 64 | 65 | ############################################################### 66 | # Parametrized example 67 | # -------------------- 68 | # 69 | # In deep learning literature, this layer is confusingly referred 70 | # to as convolution while the actual operation is cross-correlation 71 | # (the only difference is that filter is flipped for convolution, 72 | # which is not the case for cross-correlation). 73 | # 74 | # Implementation of a layer with learnable weights, where cross-correlation 75 | # has a filter (kernel) that represents weights. 76 | # 77 | # The backward pass computes the gradient wrt the input and the gradient wrt the filter. 78 | 79 | from numpy import flip 80 | import numpy as np 81 | from scipy.signal import convolve2d, correlate2d 82 | from torch.nn.modules.module import Module 83 | from torch.nn.parameter import Parameter 84 | 85 | 86 | class ScipyConv2dFunction(Function): 87 | @staticmethod 88 | def forward(ctx, input, filter, bias): 89 | # detach so we can cast to NumPy 90 | input, filter, bias = input.detach(), filter.detach(), bias.detach() 91 | result = correlate2d(input.numpy(), filter.numpy(), mode='valid') 92 | result += bias.numpy() 93 | ctx.save_for_backward(input, filter, bias) 94 | return torch.as_tensor(result, dtype=input.dtype) 95 | 96 | @staticmethod 97 | def backward(ctx, grad_output): 98 | grad_output = grad_output.detach() 99 | input, filter, bias = ctx.saved_tensors 100 | grad_output = grad_output.numpy() 101 | grad_bias = np.sum(grad_output, keepdims=True) 102 | grad_input = convolve2d(grad_output, filter.numpy(), mode='full') 103 | # the previous line can be expressed equivalently as: 104 | # grad_input = correlate2d(grad_output, flip(flip(filter.numpy(), axis=0), axis=1), mode='full') 105 | grad_filter = correlate2d(input.numpy(), grad_output, mode='valid') 106 | return torch.from_numpy(grad_input), torch.from_numpy(grad_filter).to(torch.float), torch.from_numpy(grad_bias).to(torch.float) 107 | 108 | 109 | class ScipyConv2d(Module): 110 | def __init__(self, filter_width, filter_height): 111 | super(ScipyConv2d, self).__init__() 112 | self.filter = Parameter(torch.randn(filter_width, filter_height)) 113 | self.bias = Parameter(torch.randn(1, 1)) 114 | 115 | def forward(self, input): 116 | return ScipyConv2dFunction.apply(input, self.filter, self.bias) 117 | 118 | 119 | ############################################################### 120 | # **Example usage:** 121 | 122 | module = ScipyConv2d(3, 3) 123 | print("Filter and bias: ", list(module.parameters())) 124 | input = torch.randn(10, 10, requires_grad=True) 125 | output = module(input) 126 | print("Output from the convolution: ", output) 127 | output.backward(torch.randn(8, 8)) 128 | print("Gradient for the input map: ", input.grad) 129 | 130 | ############################################################### 131 | # **Check the gradients:** 132 | 133 | from torch.autograd.gradcheck import gradcheck 134 | 135 | moduleConv = ScipyConv2d(3, 3) 136 | 137 | input = [torch.randn(20, 20, dtype=torch.double, requires_grad=True)] 138 | test = gradcheck(moduleConv, input, eps=1e-6, atol=1e-4) 139 | print("Are the gradients correct: ", test) 140 | -------------------------------------------------------------------------------- /beginner_source/former_torchies/parallelism_tutorial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Multi-GPU Examples 4 | ================== 5 | 6 | Data Parallelism is when we split the mini-batch of samples into 7 | multiple smaller mini-batches and run the computation for each of the 8 | smaller mini-batches in parallel. 9 | 10 | Data Parallelism is implemented using ``torch.nn.DataParallel``. 11 | One can wrap a Module in ``DataParallel`` and it will be parallelized 12 | over multiple GPUs in the batch dimension. 13 | 14 | 15 | DataParallel 16 | ------------- 17 | """ 18 | import torch 19 | import torch.nn as nn 20 | 21 | 22 | class DataParallelModel(nn.Module): 23 | 24 | def __init__(self): 25 | super().__init__() 26 | self.block1 = nn.Linear(10, 20) 27 | 28 | # wrap block2 in DataParallel 29 | self.block2 = nn.Linear(20, 20) 30 | self.block2 = nn.DataParallel(self.block2) 31 | 32 | self.block3 = nn.Linear(20, 20) 33 | 34 | def forward(self, x): 35 | x = self.block1(x) 36 | x = self.block2(x) 37 | x = self.block3(x) 38 | return x 39 | 40 | ######################################################################## 41 | # The code does not need to be changed in CPU-mode. 42 | # 43 | # The documentation for DataParallel can be found 44 | # `here `_. 45 | # 46 | # **Attributes of the wrapped module** 47 | # 48 | # After wrapping a Module with ``DataParallel``, the attributes of the module 49 | # (e.g. custom methods) became inaccessible. This is because ``DataParallel`` 50 | # defines a few new members, and allowing other attributes might lead to 51 | # clashes in their names. For those who still want to access the attributes, 52 | # a workaround is to use a subclass of ``DataParallel`` as below. 53 | 54 | class MyDataParallel(nn.DataParallel): 55 | def __getattr__(self, name): 56 | return getattr(self.module, name) 57 | 58 | ######################################################################## 59 | # **Primitives on which DataParallel is implemented upon:** 60 | # 61 | # 62 | # In general, pytorch’s `nn.parallel` primitives can be used independently. 63 | # We have implemented simple MPI-like primitives: 64 | # 65 | # - replicate: replicate a Module on multiple devices 66 | # - scatter: distribute the input in the first-dimension 67 | # - gather: gather and concatenate the input in the first-dimension 68 | # - parallel\_apply: apply a set of already-distributed inputs to a set of 69 | # already-distributed models. 70 | # 71 | # To give a better clarity, here function ``data_parallel`` composed using 72 | # these collectives 73 | 74 | 75 | def data_parallel(module, input, device_ids, output_device=None): 76 | if not device_ids: 77 | return module(input) 78 | 79 | if output_device is None: 80 | output_device = device_ids[0] 81 | 82 | replicas = nn.parallel.replicate(module, device_ids) 83 | inputs = nn.parallel.scatter(input, device_ids) 84 | replicas = replicas[:len(inputs)] 85 | outputs = nn.parallel.parallel_apply(replicas, inputs) 86 | return nn.parallel.gather(outputs, output_device) 87 | 88 | ######################################################################## 89 | # Part of the model on CPU and part on the GPU 90 | # -------------------------------------------- 91 | # 92 | # Let’s look at a small example of implementing a network where part of it 93 | # is on the CPU and part on the GPU 94 | 95 | device = torch.device("cuda:0") 96 | 97 | class DistributedModel(nn.Module): 98 | 99 | def __init__(self): 100 | super().__init__( 101 | embedding=nn.Embedding(1000, 10), 102 | rnn=nn.Linear(10, 10).to(device), 103 | ) 104 | 105 | def forward(self, x): 106 | # Compute embedding on CPU 107 | x = self.embedding(x) 108 | 109 | # Transfer to GPU 110 | x = x.to(device) 111 | 112 | # Compute RNN on GPU 113 | x = self.rnn(x) 114 | return x 115 | 116 | ######################################################################## 117 | # 118 | # This was a small introduction to PyTorch for former Torch users. 119 | # There’s a lot more to learn. 120 | # 121 | # Look at our more comprehensive introductory tutorial which introduces 122 | # the ``optim`` package, data loaders etc.: :doc:`/beginner/deep_learning_60min_blitz`. 123 | # 124 | # Also look at 125 | # 126 | # - :doc:`Train neural nets to play video games ` 127 | # - `Train a state-of-the-art ResNet network on imagenet`_ 128 | # - `Train an face generator using Generative Adversarial Networks`_ 129 | # - `Train a word-level language model using Recurrent LSTM networks`_ 130 | # - `More examples`_ 131 | # - `More tutorials`_ 132 | # - `Discuss PyTorch on the Forums`_ 133 | # - `Chat with other users on Slack`_ 134 | # 135 | # .. _`Deep Learning with PyTorch: a 60-minute blitz`: https://github.com/pytorch/tutorials/blob/master/Deep%20Learning%20with%20PyTorch.ipynb 136 | # .. _Train a state-of-the-art ResNet network on imagenet: https://github.com/pytorch/examples/tree/master/imagenet 137 | # .. _Train an face generator using Generative Adversarial Networks: https://github.com/pytorch/examples/tree/master/dcgan 138 | # .. _Train a word-level language model using Recurrent LSTM networks: https://github.com/pytorch/examples/tree/master/word_language_model 139 | # .. _More examples: https://github.com/pytorch/examples 140 | # .. _More tutorials: https://github.com/pytorch/tutorials 141 | # .. _Discuss PyTorch on the Forums: https://discuss.pytorch.org/ 142 | # .. _Chat with other users on Slack: https://pytorch.slack.com/messages/beginner/ 143 | -------------------------------------------------------------------------------- /beginner_source/blitz/tensor_tutorial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | What is PyTorch? 4 | ================ 5 | 6 | It’s a Python-based scientific computing package targeted at two sets of 7 | audiences: 8 | 9 | - A replacement for NumPy to use the power of GPUs 10 | - a deep learning research platform that provides maximum flexibility 11 | and speed 12 | 13 | Getting Started 14 | --------------- 15 | 16 | Tensors 17 | ^^^^^^^ 18 | 19 | Tensors are similar to NumPy’s ndarrays, with the addition being that 20 | Tensors can also be used on a GPU to accelerate computing. 21 | """ 22 | 23 | from __future__ import print_function 24 | import torch 25 | 26 | ############################################################### 27 | # Construct a 5x3 matrix, uninitialized: 28 | 29 | x = torch.empty(5, 3) 30 | print(x) 31 | 32 | ############################################################### 33 | # Construct a randomly initialized matrix: 34 | 35 | x = torch.rand(5, 3) 36 | print(x) 37 | 38 | ############################################################### 39 | # Construct a matrix filled zeros and of dtype long: 40 | 41 | x = torch.zeros(5, 3, dtype=torch.long) 42 | print(x) 43 | 44 | ############################################################### 45 | # Construct a tensor directly from data: 46 | 47 | x = torch.tensor([5.5, 3]) 48 | print(x) 49 | 50 | ############################################################### 51 | # or create a tensor based on an existing tensor. These methods 52 | # will reuse properties of the input tensor, e.g. dtype, unless 53 | # new values are provided by user 54 | 55 | x = x.new_ones(5, 3, dtype=torch.double) # new_* methods take in sizes 56 | print(x) 57 | 58 | x = torch.randn_like(x, dtype=torch.float) # override dtype! 59 | print(x) # result has the same size 60 | 61 | ############################################################### 62 | # Get its size: 63 | 64 | print(x.size()) 65 | 66 | ############################################################### 67 | # .. note:: 68 | # ``torch.Size`` is in fact a tuple, so it supports all tuple operations. 69 | # 70 | # Operations 71 | # ^^^^^^^^^^ 72 | # There are multiple syntaxes for operations. In the following 73 | # example, we will take a look at the addition operation. 74 | # 75 | # Addition: syntax 1 76 | y = torch.rand(5, 3) 77 | print(x + y) 78 | 79 | ############################################################### 80 | # Addition: syntax 2 81 | 82 | print(torch.add(x, y)) 83 | 84 | ############################################################### 85 | # Addition: providing an output tensor as argument 86 | result = torch.empty(5, 3) 87 | torch.add(x, y, out=result) 88 | print(result) 89 | 90 | ############################################################### 91 | # Addition: in-place 92 | 93 | # adds x to y 94 | y.add_(x) 95 | print(y) 96 | 97 | ############################################################### 98 | # .. note:: 99 | # Any operation that mutates a tensor in-place is post-fixed with an ``_``. 100 | # For example: ``x.copy_(y)``, ``x.t_()``, will change ``x``. 101 | # 102 | # You can use standard NumPy-like indexing with all bells and whistles! 103 | 104 | print(x[:, 1]) 105 | 106 | ############################################################### 107 | # Resizing: If you want to resize/reshape tensor, you can use ``torch.view``: 108 | x = torch.randn(4, 4) 109 | y = x.view(16) 110 | z = x.view(-1, 8) # the size -1 is inferred from other dimensions 111 | print(x.size(), y.size(), z.size()) 112 | 113 | ############################################################### 114 | # If you have a one element tensor, use ``.item()`` to get the value as a 115 | # Python number 116 | x = torch.randn(1) 117 | print(x) 118 | print(x.item()) 119 | 120 | ############################################################### 121 | # **Read later:** 122 | # 123 | # 124 | # 100+ Tensor operations, including transposing, indexing, slicing, 125 | # mathematical operations, linear algebra, random numbers, etc., 126 | # are described 127 | # `here `_. 128 | # 129 | # NumPy Bridge 130 | # ------------ 131 | # 132 | # Converting a Torch Tensor to a NumPy array and vice versa is a breeze. 133 | # 134 | # The Torch Tensor and NumPy array will share their underlying memory 135 | # locations (if the Torch Tensor is on CPU), and changing one will change 136 | # the other. 137 | # 138 | # Converting a Torch Tensor to a NumPy Array 139 | # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 140 | 141 | a = torch.ones(5) 142 | print(a) 143 | 144 | ############################################################### 145 | # 146 | 147 | b = a.numpy() 148 | print(b) 149 | 150 | ############################################################### 151 | # See how the numpy array changed in value. 152 | 153 | a.add_(1) 154 | print(a) 155 | print(b) 156 | 157 | ############################################################### 158 | # Converting NumPy Array to Torch Tensor 159 | # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 160 | # See how changing the np array changed the Torch Tensor automatically 161 | 162 | import numpy as np 163 | a = np.ones(5) 164 | b = torch.from_numpy(a) 165 | np.add(a, 1, out=a) 166 | print(a) 167 | print(b) 168 | 169 | ############################################################### 170 | # All the Tensors on the CPU except a CharTensor support converting to 171 | # NumPy and back. 172 | # 173 | # CUDA Tensors 174 | # ------------ 175 | # 176 | # Tensors can be moved onto any device using the ``.to`` method. 177 | 178 | # let us run this cell only if CUDA is available 179 | # We will use ``torch.device`` objects to move tensors in and out of GPU 180 | if torch.cuda.is_available(): 181 | device = torch.device("cuda") # a CUDA device object 182 | y = torch.ones_like(x, device=device) # directly create a tensor on GPU 183 | x = x.to(device) # or just use strings ``.to("cuda")`` 184 | z = x + y 185 | print(z) 186 | print(z.to("cpu", torch.double)) # ``.to`` can also change dtype together! 187 | -------------------------------------------------------------------------------- /custom_directives.py: -------------------------------------------------------------------------------- 1 | from docutils.parsers.rst import Directive, directives 2 | from docutils.statemachine import StringList 3 | from docutils import nodes 4 | import re 5 | import os 6 | import sphinx_gallery 7 | 8 | try: 9 | FileNotFoundError 10 | except NameError: 11 | FileNotFoundError = IOError 12 | 13 | 14 | class IncludeDirective(Directive): 15 | """Include source file without docstring at the top of file. 16 | 17 | Implementation just replaces the first docstring found in file 18 | with '' once. 19 | 20 | Example usage: 21 | 22 | .. includenodoc:: /beginner/examples_tensor/two_layer_net_tensor.py 23 | 24 | """ 25 | 26 | # defines the parameter the directive expects 27 | # directives.unchanged means you get the raw value from RST 28 | required_arguments = 1 29 | optional_arguments = 0 30 | final_argument_whitespace = True 31 | has_content = False 32 | add_index = False 33 | 34 | docstring_pattern = r'"""(?P(?:.|[\r\n])*?)"""\n' 35 | docstring_regex = re.compile(docstring_pattern) 36 | 37 | def run(self): 38 | document = self.state.document 39 | env = document.settings.env 40 | rel_filename, filename = env.relfn2path(self.arguments[0]) 41 | 42 | try: 43 | text = open(filename).read() 44 | text_no_docstring = self.docstring_regex.sub('', text, count=1) 45 | 46 | code_block = nodes.literal_block(text=text_no_docstring) 47 | return [code_block] 48 | except FileNotFoundError as e: 49 | print(e) 50 | return [] 51 | 52 | 53 | class GalleryItemDirective(Directive): 54 | """ 55 | Create a sphinx gallery thumbnail for insertion anywhere in docs. 56 | 57 | Optionally, you can specify the custom figure and intro/tooltip for the 58 | thumbnail. 59 | 60 | Example usage: 61 | 62 | .. galleryitem:: intermediate/char_rnn_generation_tutorial.py 63 | :figure: _static/img/char_rnn_generation.png 64 | :intro: Put your custom intro here. 65 | 66 | If figure is specified, a thumbnail will be made out of it and stored in 67 | _static/thumbs. Therefore, consider _static/thumbs as a 'built' directory. 68 | """ 69 | 70 | required_arguments = 1 71 | optional_arguments = 0 72 | final_argument_whitespace = True 73 | option_spec = {'figure': directives.unchanged, 74 | 'intro': directives.unchanged} 75 | has_content = False 76 | add_index = False 77 | 78 | def run(self): 79 | args = self.arguments 80 | fname = args[-1] 81 | 82 | env = self.state.document.settings.env 83 | fname, abs_fname = env.relfn2path(fname) 84 | basename = os.path.basename(fname) 85 | dirname = os.path.dirname(fname) 86 | 87 | try: 88 | if 'intro' in self.options: 89 | intro = self.options['intro'][:195] + '...' 90 | else: 91 | _, blocks = sphinx_gallery.gen_rst.split_code_and_text_blocks(abs_fname) 92 | intro, _ = sphinx_gallery.gen_rst.extract_intro_and_title(abs_fname, blocks[0][1]) 93 | 94 | thumbnail_rst = sphinx_gallery.backreferences._thumbnail_div( 95 | dirname, basename, intro) 96 | 97 | if 'figure' in self.options: 98 | rel_figname, figname = env.relfn2path(self.options['figure']) 99 | save_figname = os.path.join('_static/thumbs/', 100 | os.path.basename(figname)) 101 | 102 | try: 103 | os.makedirs('_static/thumbs') 104 | except OSError: 105 | pass 106 | 107 | sphinx_gallery.gen_rst.scale_image(figname, save_figname, 108 | 400, 280) 109 | # replace figure in rst with simple regex 110 | thumbnail_rst = re.sub(r'..\sfigure::\s.*\.png', 111 | '.. figure:: /{}'.format(save_figname), 112 | thumbnail_rst) 113 | 114 | thumbnail = StringList(thumbnail_rst.split('\n')) 115 | thumb = nodes.paragraph() 116 | self.state.nested_parse(thumbnail, self.content_offset, thumb) 117 | 118 | return [thumb] 119 | except FileNotFoundError as e: 120 | print(e) 121 | return [] 122 | 123 | 124 | GALLERY_TEMPLATE = """ 125 | .. raw:: html 126 | 127 |
128 | 129 | .. only:: html 130 | 131 | .. figure:: {thumbnail} 132 | 133 | {description} 134 | 135 | .. raw:: html 136 | 137 |
138 | """ 139 | 140 | 141 | class CustomGalleryItemDirective(Directive): 142 | """Create a sphinx gallery style thumbnail. 143 | 144 | tooltip and figure are self explanatory. Description could be a link to 145 | a document like in below example. 146 | 147 | Example usage: 148 | 149 | .. customgalleryitem:: 150 | :tooltip: I am writing this tutorial to focus specifically on NLP for people who have never written code in any deep learning framework 151 | :figure: /_static/img/thumbnails/babel.jpg 152 | :description: :doc:`/beginner/deep_learning_nlp_tutorial` 153 | 154 | If figure is specified, a thumbnail will be made out of it and stored in 155 | _static/thumbs. Therefore, consider _static/thumbs as a 'built' directory. 156 | """ 157 | 158 | required_arguments = 0 159 | optional_arguments = 0 160 | final_argument_whitespace = True 161 | option_spec = {'tooltip': directives.unchanged, 162 | 'figure': directives.unchanged, 163 | 'description': directives.unchanged} 164 | 165 | has_content = False 166 | add_index = False 167 | 168 | def run(self): 169 | try: 170 | if 'tooltip' in self.options: 171 | tooltip = self.options['tooltip'][:195] + '...' 172 | else: 173 | raise ValueError('tooltip not found') 174 | 175 | if 'figure' in self.options: 176 | env = self.state.document.settings.env 177 | rel_figname, figname = env.relfn2path(self.options['figure']) 178 | thumbnail = os.path.join('_static/thumbs/', os.path.basename(figname)) 179 | 180 | try: 181 | os.makedirs('_static/thumbs') 182 | except FileExistsError: 183 | pass 184 | 185 | sphinx_gallery.gen_rst.scale_image(figname, thumbnail, 400, 280) 186 | else: 187 | thumbnail = '_static/img/thumbnails/default.png' 188 | 189 | if 'description' in self.options: 190 | description = self.options['description'] 191 | else: 192 | raise ValueError('description not doc found') 193 | 194 | except FileNotFoundError as e: 195 | print(e) 196 | return [] 197 | except ValueError as e: 198 | print(e) 199 | raise 200 | return [] 201 | 202 | thumbnail_rst = GALLERY_TEMPLATE.format(tooltip=tooltip, 203 | thumbnail=thumbnail, 204 | description=description) 205 | thumbnail = StringList(thumbnail_rst.split('\n')) 206 | thumb = nodes.paragraph() 207 | self.state.nested_parse(thumbnail, self.content_offset, thumb) 208 | return [thumb] 209 | -------------------------------------------------------------------------------- /beginner_source/blitz/autograd_tutorial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Autograd: Automatic Differentiation 4 | =================================== 5 | 6 | Central to all neural networks in PyTorch is the ``autograd`` package. 7 | Let’s first briefly visit this, and we will then go to training our 8 | first neural network. 9 | 10 | 11 | The ``autograd`` package provides automatic differentiation for all operations 12 | on Tensors. It is a define-by-run framework, which means that your backprop is 13 | defined by how your code is run, and that every single iteration can be 14 | different. 15 | 16 | Let us see this in more simple terms with some examples. 17 | 18 | Tensor 19 | -------- 20 | 21 | ``torch.Tensor`` is the central class of the package. If you set its attribute 22 | ``.requires_grad`` as ``True``, it starts to track all operations on it. When 23 | you finish your computation you can call ``.backward()`` and have all the 24 | gradients computed automatically. The gradient for this tensor will be 25 | accumulated into ``.grad`` attribute. 26 | 27 | To stop a tensor from tracking history, you can call ``.detach()`` to detach 28 | it from the computation history, and to prevent future computation from being 29 | tracked. 30 | 31 | To prevent tracking history (and using memory), you can also wrap the code block 32 | in ``with torch.no_grad():``. This can be particularly helpful when evaluating a 33 | model because the model may have trainable parameters with 34 | ``requires_grad=True``, but for which we don't need the gradients. 35 | 36 | There’s one more class which is very important for autograd 37 | implementation - a ``Function``. 38 | 39 | ``Tensor`` and ``Function`` are interconnected and build up an acyclic 40 | graph, that encodes a complete history of computation. Each tensor has 41 | a ``.grad_fn`` attribute that references a ``Function`` that has created 42 | the ``Tensor`` (except for Tensors created by the user - their 43 | ``grad_fn is None``). 44 | 45 | If you want to compute the derivatives, you can call ``.backward()`` on 46 | a ``Tensor``. If ``Tensor`` is a scalar (i.e. it holds a one element 47 | data), you don’t need to specify any arguments to ``backward()``, 48 | however if it has more elements, you need to specify a ``gradient`` 49 | argument that is a tensor of matching shape. 50 | """ 51 | 52 | import torch 53 | 54 | ############################################################### 55 | # Create a tensor and set ``requires_grad=True`` to track computation with it 56 | x = torch.ones(2, 2, requires_grad=True) 57 | print(x) 58 | 59 | ############################################################### 60 | # Do a tensor operation: 61 | y = x + 2 62 | print(y) 63 | 64 | ############################################################### 65 | # ``y`` was created as a result of an operation, so it has a ``grad_fn``. 66 | print(y.grad_fn) 67 | 68 | ############################################################### 69 | # Do more operations on ``y`` 70 | z = y * y * 3 71 | out = z.mean() 72 | 73 | print(z, out) 74 | 75 | ################################################################ 76 | # ``.requires_grad_( ... )`` changes an existing Tensor's ``requires_grad`` 77 | # flag in-place. The input flag defaults to ``False`` if not given. 78 | a = torch.randn(2, 2) 79 | a = ((a * 3) / (a - 1)) 80 | print(a.requires_grad) 81 | a.requires_grad_(True) 82 | print(a.requires_grad) 83 | b = (a * a).sum() 84 | print(b.grad_fn) 85 | 86 | ############################################################### 87 | # Gradients 88 | # --------- 89 | # Let's backprop now. 90 | # Because ``out`` contains a single scalar, ``out.backward()`` is 91 | # equivalent to ``out.backward(torch.tensor(1.))``. 92 | 93 | out.backward() 94 | 95 | ############################################################### 96 | # Print gradients d(out)/dx 97 | # 98 | 99 | print(x.grad) 100 | 101 | ############################################################### 102 | # You should have got a matrix of ``4.5``. Let’s call the ``out`` 103 | # *Tensor* “:math:`o`”. 104 | # We have that :math:`o = \frac{1}{4}\sum_i z_i`, 105 | # :math:`z_i = 3(x_i+2)^2` and :math:`z_i\bigr\rvert_{x_i=1} = 27`. 106 | # Therefore, 107 | # :math:`\frac{\partial o}{\partial x_i} = \frac{3}{2}(x_i+2)`, hence 108 | # :math:`\frac{\partial o}{\partial x_i}\bigr\rvert_{x_i=1} = \frac{9}{2} = 4.5`. 109 | 110 | ############################################################### 111 | # Mathematically, if you have a vector valued function :math:`\vec{y}=f(\vec{x})`, 112 | # then the gradient of :math:`\vec{y}` with respect to :math:`\vec{x}` 113 | # is a Jacobian matrix: 114 | # 115 | # .. math:: 116 | # J=\left(\begin{array}{ccc} 117 | # \frac{\partial y_{1}}{\partial x_{1}} & \cdots & \frac{\partial y_{1}}{\partial x_{n}}\\ 118 | # \vdots & \ddots & \vdots\\ 119 | # \frac{\partial y_{m}}{\partial x_{1}} & \cdots & \frac{\partial y_{m}}{\partial x_{n}} 120 | # \end{array}\right) 121 | # 122 | # Generally speaking, ``torch.autograd`` is an engine for computing 123 | # vector-Jacobian product. That is, given any vector 124 | # :math:`v=\left(\begin{array}{cccc} v_{1} & v_{2} & \cdots & v_{m}\end{array}\right)^{T}`, 125 | # compute the product :math:`v^{T}\cdot J`. If :math:`v` happens to be 126 | # the gradient of a scalar function :math:`l=g\left(\vec{y}\right)`, 127 | # that is, 128 | # :math:`v=\left(\begin{array}{ccc}\frac{\partial l}{\partial y_{1}} & \cdots & \frac{\partial l}{\partial y_{m}}\end{array}\right)^{T}`, 129 | # then by the chain rule, the vector-Jacobian product would be the 130 | # gradient of :math:`l` with respect to :math:`\vec{x}`: 131 | # 132 | # .. math:: 133 | # J^{T}\cdot v=\left(\begin{array}{ccc} 134 | # \frac{\partial y_{1}}{\partial x_{1}} & \cdots & \frac{\partial y_{m}}{\partial x_{1}}\\ 135 | # \vdots & \ddots & \vdots\\ 136 | # \frac{\partial y_{1}}{\partial x_{n}} & \cdots & \frac{\partial y_{m}}{\partial x_{n}} 137 | # \end{array}\right)\left(\begin{array}{c} 138 | # \frac{\partial l}{\partial y_{1}}\\ 139 | # \vdots\\ 140 | # \frac{\partial l}{\partial y_{m}} 141 | # \end{array}\right)=\left(\begin{array}{c} 142 | # \frac{\partial l}{\partial x_{1}}\\ 143 | # \vdots\\ 144 | # \frac{\partial l}{\partial x_{n}} 145 | # \end{array}\right) 146 | # 147 | # (Note that :math:`v^{T}\cdot J` gives a row vector which can be 148 | # treated as a column vector by taking :math:`J^{T}\cdot v`.) 149 | # 150 | # This characteristic of vector-Jacobian product makes it very 151 | # convenient to feed external gradients into a model that has 152 | # non-scalar output. 153 | 154 | ############################################################### 155 | # Now let's take a look at an example of vector-Jacobian product: 156 | 157 | x = torch.randn(3, requires_grad=True) 158 | 159 | y = x * 2 160 | while y.data.norm() < 1000: 161 | y = y * 2 162 | 163 | print(y) 164 | 165 | ############################################################### 166 | # Now in this case ``y`` is no longer a scalar. ``torch.autograd`` 167 | # could not compute the full Jacobian directly, but if we just 168 | # want the vector-Jacobian product, simply pass the vector to 169 | # ``backward`` as argument: 170 | v = torch.tensor([0.1, 1.0, 0.0001], dtype=torch.float) 171 | y.backward(v) 172 | 173 | print(x.grad) 174 | 175 | ############################################################### 176 | # You can also stop autograd from tracking history on Tensors 177 | # with ``.requires_grad=True`` by wrapping the code block in 178 | # ``with torch.no_grad():`` 179 | print(x.requires_grad) 180 | print((x ** 2).requires_grad) 181 | 182 | with torch.no_grad(): 183 | print((x ** 2).requires_grad) 184 | 185 | ############################################################### 186 | # **Read Later:** 187 | # 188 | # Documentation of ``autograd`` and ``Function`` is at 189 | # https://pytorch.org/docs/autograd 190 | -------------------------------------------------------------------------------- /conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # PyTorch Tutorials documentation build configuration file, created by 5 | # sphinx-quickstart on Wed Mar 8 22:38:10 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | import os 21 | import sys 22 | sys.path.insert(0, os.path.abspath('.')) 23 | import pytorch_sphinx_theme 24 | import torch 25 | import glob 26 | import shutil 27 | from custom_directives import IncludeDirective, GalleryItemDirective, CustomGalleryItemDirective 28 | 29 | 30 | try: 31 | import torchvision 32 | except ImportError: 33 | import warnings 34 | warnings.warn('unable to load "torchvision" package') 35 | import pytorch_sphinx_theme 36 | 37 | # -- General configuration ------------------------------------------------ 38 | 39 | # If your documentation needs a minimal Sphinx version, state it here. 40 | # 41 | # needs_sphinx = '1.0' 42 | 43 | # Add any Sphinx extension module names here, as strings. They can be 44 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 45 | # ones. 46 | extensions = ['sphinx.ext.mathjax', 47 | 'sphinx_gallery.gen_gallery'] 48 | 49 | 50 | # -- Sphinx-gallery configuration -------------------------------------------- 51 | 52 | 53 | sphinx_gallery_conf = { 54 | 'examples_dirs': ['beginner_source', 'intermediate_source', 55 | 'advanced_source'], 56 | 'gallery_dirs': ['beginner', 'intermediate', 'advanced'], 57 | 'filename_pattern': 'tutorial.py', 58 | 'backreferences_dir': False 59 | } 60 | 61 | for i in range(len(sphinx_gallery_conf['examples_dirs'])): 62 | gallery_dir = sphinx_gallery_conf['gallery_dirs'][i] 63 | source_dir = sphinx_gallery_conf['examples_dirs'][i] 64 | # Create gallery dirs if it doesn't exist 65 | try: 66 | os.mkdir(gallery_dir) 67 | except OSError: 68 | pass 69 | 70 | # Copy rst files from source dir to gallery dir 71 | for f in glob.glob(os.path.join(source_dir, '*.rst')): 72 | shutil.copy(f, gallery_dir) 73 | 74 | 75 | # Add any paths that contain templates here, relative to this directory. 76 | 77 | 78 | templates_path = ['_templates'] 79 | 80 | # The suffix(es) of source filenames. 81 | # You can specify multiple suffix as a list of string: 82 | # 83 | # source_suffix = ['.rst', '.md'] 84 | source_suffix = '.rst' 85 | 86 | # The master toctree document. 87 | master_doc = 'index' 88 | 89 | # General information about the project. 90 | project = 'PyTorch Tutorials' 91 | copyright = '2017, PyTorch' 92 | author = 'PyTorch contributors' 93 | 94 | # The version info for the project you're documenting, acts as replacement for 95 | # |version| and |release|, also used in various other places throughout the 96 | # built documents. 97 | # 98 | # The short X.Y version. 99 | version = torch.__version__ 100 | # The full version, including alpha/beta/rc tags. 101 | release = torch.__version__ 102 | 103 | # The language for content autogenerated by Sphinx. Refer to documentation 104 | # for a list of supported languages. 105 | # 106 | # This is also used if you do content translation via gettext catalogs. 107 | # Usually you set "language" from the command line for these cases. 108 | language = None 109 | 110 | # List of patterns, relative to source directory, that match files and 111 | # directories to ignore when looking for source files. 112 | # This patterns also effect to html_static_path and html_extra_path 113 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 114 | exclude_patterns += sphinx_gallery_conf['examples_dirs'] 115 | exclude_patterns += ['*/index.rst'] 116 | 117 | # The name of the Pygments (syntax highlighting) style to use. 118 | pygments_style = 'sphinx' 119 | 120 | # If true, `todo` and `todoList` produce output, else they produce nothing. 121 | todo_include_todos = False 122 | 123 | 124 | # -- Options for HTML output ---------------------------------------------- 125 | 126 | # The theme to use for HTML and HTML Help pages. See the documentation for 127 | # a list of builtin themes. 128 | # 129 | # html_theme = 'alabaster' 130 | 131 | # # Theme options are theme-specific and customize the look and feel of a theme 132 | # # further. For a list of options available for each theme, see the 133 | # # documentation. 134 | # # 135 | 136 | # html_theme_options = { 137 | # 'page_width': '1000px', 138 | # 'fixed_sidebar': True, 139 | # 'code_font_size': '0.87em', 140 | # 'sidebar_includehidden': True 141 | # } 142 | 143 | # # Add any paths that contain custom static files (such as style sheets) here, 144 | # # relative to this directory. They are copied after the builtin static files, 145 | # # so a file named "default.css" will overwrite the builtin "default.css". 146 | html_static_path = ['_static'] 147 | 148 | # # Custom sidebar templates, maps document names to template names. 149 | # html_sidebars = { 150 | # 'index': ['sidebarlogo.html', 'globaltoc.html', 'searchbox.html', 'sourcelink.html'], 151 | # '**': ['sidebarlogo.html', 'globaltoc.html', 'searchbox.html', 'sourcelink.html'] 152 | # } 153 | 154 | 155 | html_theme = 'pytorch_sphinx_theme' 156 | html_theme_path = [pytorch_sphinx_theme.get_html_theme_path()] 157 | html_logo = '_static/img/pytorch-logo-dark.svg' 158 | html_theme_options = { 159 | 'pytorch_project': 'tutorials', 160 | 'collapse_navigation': False, 161 | 'display_version': True, 162 | 'logo_only': False, 163 | } 164 | 165 | 166 | # -- Options for HTMLHelp output ------------------------------------------ 167 | 168 | # Output file base name for HTML help builder. 169 | htmlhelp_basename = 'PyTorchTutorialsdoc' 170 | 171 | 172 | # -- Options for LaTeX output --------------------------------------------- 173 | 174 | latex_elements = { 175 | # The paper size ('letterpaper' or 'a4paper'). 176 | # 177 | # 'papersize': 'letterpaper', 178 | 179 | # The font size ('10pt', '11pt' or '12pt'). 180 | # 181 | # 'pointsize': '10pt', 182 | 183 | # Additional stuff for the LaTeX preamble. 184 | # 185 | # 'preamble': '', 186 | 187 | # Latex figure (float) alignment 188 | # 189 | # 'figure_align': 'htbp', 190 | } 191 | 192 | # Grouping the document tree into LaTeX files. List of tuples 193 | # (source start file, target name, title, 194 | # author, documentclass [howto, manual, or own class]). 195 | latex_documents = [ 196 | (master_doc, 'PyTorchTutorials.tex', 'PyTorch Tutorials', 197 | 'Sasank, PyTorch contributors', 'manual'), 198 | ] 199 | 200 | 201 | # -- Options for manual page output --------------------------------------- 202 | 203 | # One entry per manual page. List of tuples 204 | # (source start file, name, description, authors, manual section). 205 | man_pages = [ 206 | (master_doc, 'pytorchtutorials', 'PyTorch Tutorials', 207 | [author], 1) 208 | ] 209 | 210 | 211 | # -- Options for Texinfo output ------------------------------------------- 212 | 213 | # Grouping the document tree into Texinfo files. List of tuples 214 | # (source start file, target name, title, author, 215 | # dir menu entry, description, category) 216 | texinfo_documents = [ 217 | (master_doc, 'PyTorchTutorials', 'PyTorch Tutorials', 218 | author, 'PyTorchTutorials', 'One line description of project.', 219 | 'Miscellaneous'), 220 | ] 221 | 222 | 223 | def setup(app): 224 | # Custom CSS 225 | # app.add_stylesheet('css/pytorch_theme.css') 226 | # app.add_stylesheet('https://fonts.googleapis.com/css?family=Lato') 227 | # Custom directives 228 | app.add_directive('includenodoc', IncludeDirective) 229 | app.add_directive('galleryitem', GalleryItemDirective) 230 | app.add_directive('customgalleryitem', CustomGalleryItemDirective) 231 | -------------------------------------------------------------------------------- /.jenkins/build.sh: -------------------------------------------------------------------------------- 1 | set -ex 2 | 3 | if [[ "$COMMIT_SOURCE" == master ]]; then 4 | export BUCKET_NAME=pytorch-tutorial-build-master 5 | else 6 | export BUCKET_NAME=pytorch-tutorial-build-pull-request 7 | fi 8 | 9 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 10 | 11 | sudo apt-get update 12 | sudo apt-get install -y --no-install-recommends unzip p7zip-full sox libsox-dev libsox-fmt-all rsync 13 | 14 | export PATH=/opt/conda/bin:$PATH 15 | rm -rf src 16 | pip install -r $DIR/../requirements.txt 17 | pip uninstall -y torchvision || true 18 | 19 | export PATH=/opt/conda/bin:$PATH 20 | conda install -y sphinx==1.8.2 pandas 21 | # PyTorch Theme 22 | rm -rf src 23 | pip install -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme 24 | # pillow >= 4.2 will throw error when trying to write mode RGBA as JPEG, 25 | # this is a workaround to the issue. 26 | pip install sphinx-gallery tqdm matplotlib ipython pillow==4.1.1 27 | 28 | # Install torchvision from source 29 | git clone https://github.com/pytorch/vision --quiet 30 | pushd vision 31 | pip install . --no-deps # We don't want it to install the stock PyTorch version from pip 32 | popd 33 | 34 | # Install torchaudio from source 35 | git clone https://github.com/pytorch/audio --quiet 36 | pushd audio 37 | python setup.py install 38 | popd 39 | 40 | aws configure set default.s3.multipart_threshold 5120MB 41 | 42 | if [[ $(pip show torch) ]]; then 43 | # Clean up previous PyTorch installations 44 | pip uninstall -y torch || true 45 | pip uninstall -y torch || true 46 | fi 47 | 48 | # Install a nightly build of pytorch 49 | 50 | # GPU, requires CUDA version 9.0 51 | pip install cython torch_nightly -f https://download.pytorch.org/whl/nightly/cu90/torch_nightly.html 52 | 53 | # Decide whether to parallelize tutorial builds, based on $JOB_BASE_NAME 54 | export NUM_WORKERS=20 55 | if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then 56 | # Step 1: Remove runnable code from tutorials that are not supposed to be run 57 | python $DIR/remove_runnable_code.py beginner_source/aws_distributed_training_tutorial.py beginner_source/aws_distributed_training_tutorial.py 58 | # TODO: Fix bugs in these tutorials to make them runnable again 59 | python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py 60 | 61 | # Step 2: Keep certain tutorials based on file count, and remove runnable code in all other tutorials 62 | # IMPORTANT NOTE: We assume that each tutorial has a UNIQUE filename. 63 | export WORKER_ID=$(echo "${JOB_BASE_NAME}" | tr -dc '0-9') 64 | count=0 65 | FILES_TO_RUN=() 66 | for filename in $(find beginner_source/ -name '*.py' -not -path '*/data/*'); do 67 | if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then 68 | echo "Removing runnable code from "$filename 69 | python $DIR/remove_runnable_code.py $filename $filename 70 | else 71 | echo "Keeping "$filename 72 | FILES_TO_RUN+=($(basename $filename .py)) 73 | fi 74 | count=$((count+1)) 75 | done 76 | for filename in $(find intermediate_source/ -name '*.py' -not -path '*/data/*'); do 77 | if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then 78 | echo "Removing runnable code from "$filename 79 | python $DIR/remove_runnable_code.py $filename $filename 80 | else 81 | echo "Keeping "$filename 82 | FILES_TO_RUN+=($(basename $filename .py)) 83 | fi 84 | count=$((count+1)) 85 | done 86 | for filename in $(find advanced_source/ -name '*.py' -not -path '*/data/*'); do 87 | if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then 88 | echo "Removing runnable code from "$filename 89 | python $DIR/remove_runnable_code.py $filename $filename 90 | else 91 | echo "Keeping "$filename 92 | FILES_TO_RUN+=($(basename $filename .py)) 93 | fi 94 | count=$((count+1)) 95 | done 96 | echo "FILES_TO_RUN: " ${FILES_TO_RUN[@]} 97 | 98 | # Step 3: Run `make docs` to generate HTML files and static files for these tutorials 99 | make docs 100 | 101 | # Step 4: If any of the generated files are not related the tutorial files we want to run, 102 | # then we remove them 103 | for filename in $(find docs/beginner docs/intermediate docs/advanced -name '*.html'); do 104 | file_basename=$(basename $filename .html) 105 | if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then 106 | rm $filename 107 | fi 108 | done 109 | for filename in $(find docs/beginner docs/intermediate docs/advanced -name '*.rst'); do 110 | file_basename=$(basename $filename .rst) 111 | if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then 112 | rm $filename 113 | fi 114 | done 115 | for filename in $(find docs/_downloads -name '*.py'); do 116 | file_basename=$(basename $filename .py) 117 | if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then 118 | rm $filename 119 | fi 120 | done 121 | for filename in $(find docs/_downloads -name '*.ipynb'); do 122 | file_basename=$(basename $filename .ipynb) 123 | if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then 124 | rm $filename 125 | fi 126 | done 127 | for filename in $(find docs/_sources/beginner docs/_sources/intermediate docs/_sources/advanced -name '*.rst.txt'); do 128 | file_basename=$(basename $filename .rst.txt) 129 | if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then 130 | rm $filename 131 | fi 132 | done 133 | for filename in $(find docs/.doctrees/beginner docs/.doctrees/intermediate docs/.doctrees/advanced -name '*.doctree'); do 134 | file_basename=$(basename $filename .doctree) 135 | if [[ ! " ${FILES_TO_RUN[@]} " =~ " ${file_basename} " ]]; then 136 | rm $filename 137 | fi 138 | done 139 | 140 | # Step 5: Remove INVISIBLE_CODE_BLOCK from .html/.rst.txt/.ipynb/.py files 141 | bash $DIR/remove_invisible_code_block_batch.sh docs 142 | 143 | # Step 6: Copy generated files to S3, tag with commit ID 144 | 7z a worker_${WORKER_ID}.7z docs 145 | aws s3 cp worker_${WORKER_ID}.7z s3://${BUCKET_NAME}/${COMMIT_ID}/worker_${WORKER_ID}.7z --acl public-read 146 | elif [[ "${JOB_BASE_NAME}" == *manager ]]; then 147 | # Step 1: Generate no-plot HTML pages for all tutorials 148 | make html-noplot 149 | cp -r _build/html docs 150 | 151 | # Step 2: Wait for all workers to finish 152 | set +e 153 | for ((worker_id=0;worker_id (a,b,1,c) tensor 90 | ``` 91 | See [tensor](https://pytorch.org/docs/stable/tensors.html) 92 | 93 | ### Algebra 94 | 95 | ``` 96 | A.mm(B) # matrix multiplication 97 | A.mv(x) # matrix-vector multiplication 98 | x.t() # matrix transpose 99 | ``` 100 | See [math operations](https://pytorch.org/docs/stable/torch.html?highlight=mm#math-operations) 101 | 102 | ### GPU Usage 103 | 104 | ``` 105 | torch.cuda.is_available # check for cuda 106 | x.cuda() # move x's data from CPU to GPU and return new object 107 | x.cpu() # move x's data from GPU to CPU and return new object 108 | 109 | if not args.disable_cuda and torch.cuda.is_available(): # device agnostic code and modularity 110 | args.device = torch.device('cuda') # 111 | else: # 112 | args.device = torch.device('cpu') # 113 | 114 | net.to(device) # recursively convert their parameters and buffers to device specific tensors 115 | mytensor.to(device) # copy your tensors to a device (gpu, cpu) 116 | ``` 117 | See [cuda](https://pytorch.org/docs/stable/cuda.html) 118 | 119 | 120 | # Deep Learning 121 | ``` 122 | nn.Linear(m,n) # fully connected layer from m to n units 123 | nn.ConvXd(m,n,s) # X dimensional conv layer from m to n channels where X⍷{1,2,3} and the kernel size is s 124 | nn.MaxPoolXd(s) # X dimension pooling layer (notation as above) 125 | nn.BatchNorm # batch norm layer 126 | nn.RNN/LSTM/GRU # recurrent layers 127 | nn.Dropout(p=0.5, inplace=False) # dropout layer for any dimensional input 128 | nn.Dropout2d(p=0.5, inplace=False) # 2-dimensional channel-wise dropout 129 | nn.Embedding(num_embeddings, embedding_dim) # (tensor-wise) mapping from indices to embedding vectors 130 | ``` 131 | See [nn](https://pytorch.org/docs/stable/nn.html) 132 | 133 | ### Loss Functions 134 | 135 | ``` 136 | nn.X where for example X is ... # BCELoss, CrossEntropyLoss, L1Loss, MSELoss, NLLLoss, SoftMarginLoss, MultiLabelSoftMarginLoss, CosineEmbeddingLoss, KLDivLoss, MarginRankingLoss, HingeEmbeddingLoss or CosineEmbeddingLoss 137 | ``` 138 | See [loss functions](https://pytorch.org/docs/stable/nn.html#loss-functions) 139 | 140 | ### Activation Functions 141 | 142 | ``` 143 | nn.X where for example X is ... # ReLU, ReLU6, ELU, SELU, PReLU, LeakyReLU, Threshold, HardTanh, Sigmoid, Tanh, LogSigmoid, Softplus, SoftShrink, Softsign, TanhShrink, Softmin, Softmax, Softmax2d or LogSoftmax 144 | ``` 145 | See [activation functions](https://pytorch.org/docs/stable/nn.html#non-linear-activations-weighted-sum-nonlinearity) 146 | 147 | ### Optimizers 148 | 149 | ``` 150 | opt = optim.x(model.parameters(), ...) # create optimizer 151 | opt.step() # update weights 152 | optim.X where for example X is ... # SGD, Adadelta, Adagrad, Adam, SparseAdam, Adamax, ASGD, LBFGS, RMSProp or Rprop 153 | ``` 154 | See [optimizers](https://pytorch.org/docs/stable/optim.html) 155 | 156 | ### Learning rate scheduling 157 | 158 | ``` 159 | scheduler = optim.X(optimizer,...) # create lr scheduler 160 | scheduler.step() # update lr at start of epoch 161 | optim.lr_scheduler.X where ... # LambdaLR, StepLR, MultiStepLR, ExponentialLR or ReduceLROnPLateau 162 | ``` 163 | See [learning rate scheduler](https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate) 164 | 165 | 166 | # Data Utilities 167 | 168 | ### Datasets 169 | 170 | ``` 171 | Dataset # abstract class representing dataset 172 | TensorDataset # labelled dataset in the form of tensors 173 | ConcatDataset # concatenation of Datasets 174 | ``` 175 | See [datasets](https://pytorch.org/docs/stable/data.html?highlight=dataset#torch.utils.data.Dataset) 176 | 177 | ### Dataloaders and DataSamplers 178 | 179 | ``` 180 | DataLoader(dataset, batch_size=1, ...) # loads data batches agnostic of structure of individual data points 181 | sampler.Sampler(dataset,...) # abstract class dealing with ways to sample from dataset 182 | sampler.XSampler where ... # Sequential, Random, Subset, WeightedRandom or Distributed 183 | ``` 184 | See [dataloader](https://pytorch.org/docs/stable/data.html?highlight=dataloader#torch.utils.data.DataLoader) 185 | 186 | 187 | ## Also see 188 | 189 | * [Deep Learning with PyTorch: A 60 Minute Blitz](https://pytorch.org/tutorials/beginner/deep_learning_60min_blitz.html) _(pytorch.org)_ 190 | * [PyTorch Forums](https://discuss.pytorch.org/) _(discuss.pytorch.org)_ 191 | * [PyTorch for Numpy users](https://github.com/wkentaro/pytorch-for-numpy-users) _(github.com/wkentaro/pytorch-for-numpy-users)_ 192 | -------------------------------------------------------------------------------- /advanced_source/ONNXLive.rst: -------------------------------------------------------------------------------- 1 | 2 | ONNX Live Tutorial 3 | ================== 4 | 5 | This tutorial will show you to convert a neural style transfer model that has been exported from PyTorch into the Apple CoreML format using ONNX. This will allow you to easily run deep learning models on Apple devices and, in this case, live stream from the camera. 6 | 7 | What is ONNX? 8 | ------------- 9 | 10 | ONNX (Open Neural Network Exchange) is an open format to represent deep learning models. With ONNX, AI developers can more easily move models between state-of-the-art tools and choose the combination that is best for them. ONNX is developed and supported by a community of partners. You can learn more about ONNX and what tools are supported by going to `onnx.ai `_. 11 | 12 | Tutorial Overview 13 | ----------------- 14 | 15 | This tutorial will walk you through 4 main steps: 16 | 17 | 18 | #. `Download (or train) PyTorch style transfer models`_ 19 | #. `Convert the PyTorch models to ONNX models`_ 20 | #. `Convert the ONNX models to CoreML models`_ 21 | #. `Run the CoreML models in a style transfer iOS App`_ 22 | 23 | Preparing the Environment 24 | ------------------------- 25 | 26 | We will be working in a virtualenv in order to avoid conflicts with your local packages. 27 | We are also using Python 3.6 for this tutorial, but other versions should work as well. 28 | 29 | .. code-block:: python 30 | 31 | python3.6 -m venv venv 32 | source ./venv/bin/activate 33 | 34 | 35 | You need to install pytorch and the onnx->coreml converter: 36 | 37 | .. code-block:: bash 38 | 39 | pip install torchvision onnx-coreml 40 | 41 | 42 | You will also need to install XCode if you want to run the iOS style transfer app on your iPhone. 43 | You can also convert models in Linux, however to run the iOS app itself, you will need a Mac. 44 | 45 | Download (or train) PyTorch style transfer models 46 | ------------------------------------------------- 47 | 48 | For this tutorial, we will use the style transfer models that are published with pytorch in https://github.com/pytorch/examples/tree/master/fast_neural_style . 49 | If you would like to use a different PyTorch or ONNX model, feel free to skip this step. 50 | 51 | These models are meant for applying style transfer on still images and really not optimized to be fast enough for video. However if we reduce the resolution low enough, they can also work well on videos. 52 | 53 | Let's download the models: 54 | 55 | .. code-block:: bash 56 | 57 | git clone https://github.com/pytorch/examples 58 | cd examples/fast_neural_style 59 | 60 | 61 | If you would like to train the models yourself, the pytorch/examples repository you just cloned has more information on how to do this. 62 | For now, we'll just download pre-trained models with the script provided by the repository: 63 | 64 | .. code-block:: bash 65 | 66 | ./download_saved_models.sh 67 | 68 | 69 | This script downloads the pre-trained PyTorch models and puts them into the ``saved_models`` folder. 70 | There should now be 4 files, ``candy.pth``\ , ``mosaic.pth``\ , ``rain_princess.pth`` and ``udnie.pth`` in your directory. 71 | 72 | Convert the PyTorch models to ONNX models 73 | ----------------------------------------- 74 | 75 | Now that we have the pre-trained PyTorch models as ``.pth`` files in the ``saved_models`` folder, we will need to convert them to ONNX format. 76 | The model definition is in the pytorch/examples repository we cloned previously, and with a few lines of python we can export it to ONNX. 77 | In this case, instead of actually running the neural net, we will call ``torch.onnx._export``\ , which is provided with PyTorch as an api to directly export ONNX formatted models from PyTorch. 78 | However, in this case we don't even need to do that, because a script already exists ``neural_style/neural_style.py`` that will do this for us. 79 | You can also take a look at that script if you would like to apply it to other models. 80 | 81 | Exporting the ONNX format from PyTorch is essentially tracing your neural network so this api call will internally run the network on 'dummy data' in order to generate the graph. 82 | For this, it needs an input image to apply the style transfer to which can simply be a blank image. 83 | However, the pixel size of this image is important, as this will be the size for the exported style transfer model. 84 | To get good performance, we'll use a resolution of 250x540. Feel free to take a larger resolution if you care less about 85 | FPS and more about style transfer quality. 86 | 87 | Let's use `ImageMagick `_ to create a blank image of the resolution we want: 88 | 89 | .. code-block:: bash 90 | 91 | convert -size 250x540 xc:white png24:dummy.jpg 92 | 93 | 94 | and use that to export the PyTorch models: 95 | 96 | .. code-block:: bash 97 | 98 | python ./neural_style/neural_style.py eval --content-image dummy.jpg --output-image dummy-out.jpg --model ./saved_models/candy.pth --cuda 0 --export_onnx ./saved_models/candy.onnx 99 | python ./neural_style/neural_style.py eval --content-image dummy.jpg --output-image dummy-out.jpg --model ./saved_models/udnie.pth --cuda 0 --export_onnx ./saved_models/udnie.onnx 100 | python ./neural_style/neural_style.py eval --content-image dummy.jpg --output-image dummy-out.jpg --model ./saved_models/rain_princess.pth --cuda 0 --export_onnx ./saved_models/rain_princess.onnx 101 | python ./neural_style/neural_style.py eval --content-image dummy.jpg --output-image dummy-out.jpg --model ./saved_models/mosaic.pth --cuda 0 --export_onnx ./saved_models/mosaic.onnx 102 | 103 | 104 | You should end up with 4 files, ``candy.onnx``\ , ``mosaic.onnx``\ , ``rain_princess.onnx`` and ``udnie.onnx``\ , 105 | created from the corresponding ``.pth`` files. 106 | 107 | Convert the ONNX models to CoreML models 108 | ---------------------------------------- 109 | 110 | Now that we have ONNX models, we can convert them to CoreML models in order to run them on Apple devices. 111 | For this, we use the onnx-coreml converter we installed previously. 112 | The converter comes with a ``convert-onnx-to-coreml`` script, which the installation steps above added to our path. Unfortunately that won't work for us as we need to mark the input and output of the network as an image 113 | and, while this is supported by the converter, it is only supported when calling the converter from python. 114 | 115 | Looking at the style transfer model (for example opening the .onnx file in an application like `Netron `_\ ), 116 | we see that the input is named '0' and the output is named '186'. These are just numeric ids assigned by PyTorch. 117 | We will need to mark these as images. 118 | 119 | So let's create a small python file and call it ``onnx_to_coreml.py``. This can be created by using the touch command and edited with your favorite editor to add the following lines of code. 120 | 121 | .. code-block:: python 122 | 123 | import sys 124 | from onnx import onnx_pb 125 | from onnx_coreml import convert 126 | 127 | model_in = sys.argv[1] 128 | model_out = sys.argv[2] 129 | 130 | model_file = open(model_in, 'rb') 131 | model_proto = onnx_pb.ModelProto() 132 | model_proto.ParseFromString(model_file.read()) 133 | coreml_model = convert(model_proto, image_input_names=['0'], image_output_names=['186']) 134 | coreml_model.save(model_out) 135 | 136 | 137 | we now run it: 138 | 139 | .. code-block:: bash 140 | 141 | python onnx_to_coreml.py ./saved_models/candy.onnx ./saved_models/candy.mlmodel 142 | python onnx_to_coreml.py ./saved_models/udnie.onnx ./saved_models/udnie.mlmodel 143 | python onnx_to_coreml.py ./saved_models/rain_princess.onnx ./saved_models/rain_princess.mlmodel 144 | python onnx_to_coreml.py ./saved_models/mosaic.onnx ./saved_models/mosaic.mlmodel 145 | 146 | 147 | Now, there should be 4 CoreML models in your ``saved_models`` directory: ``candy.mlmodel``\ , ``mosaic.mlmodel``\ , ``rain_princess.mlmodel`` and ``udnie.mlmodel``. 148 | 149 | Run the CoreML models in a style transfer iOS App 150 | ------------------------------------------------- 151 | 152 | This repository (i.e. the one you're currently reading the README.md of) contains an iOS app able to run CoreML style transfer models on a live camera stream from your phone camera. Let's clone the repository: 153 | 154 | .. code-block:: bash 155 | 156 | git clone https://github.com/onnx/tutorials 157 | 158 | 159 | and open the ``tutorials/examples/CoreML/ONNXLive/ONNXLive.xcodeproj`` project in XCode. 160 | We recommend using XCode 9.3 and an iPhone X. There might be issues running on older devices or XCode versions. 161 | 162 | In the ``Models/`` folder, the project contains some .mlmodel files. We're going to replace them with the models we just created. 163 | 164 | You then run the app on your iPhone and you are all set. Tapping on the screen switches through the models. 165 | 166 | Conclusion 167 | ---------- 168 | 169 | We hope this tutorial gave you an overview of what ONNX is about and how you can use it to convert neural networks 170 | between frameworks, in this case neural style transfer models moving from PyTorch to CoreML. 171 | 172 | Feel free to experiment with these steps and test them on your own models. 173 | Please let us know if you hit any issues or want to give feedback. We'd like to hear what you think. 174 | -------------------------------------------------------------------------------- /intermediate_source/spatial_transformer_tutorial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Spatial Transformer Networks Tutorial 4 | ===================================== 5 | **Author**: `Ghassen HAMROUNI `_ 6 | 7 | .. figure:: /_static/img/stn/FSeq.png 8 | 9 | In this tutorial, you will learn how to augment your network using 10 | a visual attention mechanism called spatial transformer 11 | networks. You can read more about the spatial transformer 12 | networks in the `DeepMind paper `__ 13 | 14 | Spatial transformer networks are a generalization of differentiable 15 | attention to any spatial transformation. Spatial transformer networks 16 | (STN for short) allow a neural network to learn how to perform spatial 17 | transformations on the input image in order to enhance the geometric 18 | invariance of the model. 19 | For example, it can crop a region of interest, scale and correct 20 | the orientation of an image. It can be a useful mechanism because CNNs 21 | are not invariant to rotation and scale and more general affine 22 | transformations. 23 | 24 | One of the best things about STN is the ability to simply plug it into 25 | any existing CNN with very little modification. 26 | """ 27 | # License: BSD 28 | # Author: Ghassen Hamrouni 29 | 30 | from __future__ import print_function 31 | import torch 32 | import torch.nn as nn 33 | import torch.nn.functional as F 34 | import torch.optim as optim 35 | import torchvision 36 | from torchvision import datasets, transforms 37 | import matplotlib.pyplot as plt 38 | import numpy as np 39 | 40 | plt.ion() # interactive mode 41 | 42 | ###################################################################### 43 | # Loading the data 44 | # ---------------- 45 | # 46 | # In this post we experiment with the classic MNIST dataset. Using a 47 | # standard convolutional network augmented with a spatial transformer 48 | # network. 49 | 50 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 51 | 52 | # Training dataset 53 | train_loader = torch.utils.data.DataLoader( 54 | datasets.MNIST(root='.', train=True, download=True, 55 | transform=transforms.Compose([ 56 | transforms.ToTensor(), 57 | transforms.Normalize((0.1307,), (0.3081,)) 58 | ])), batch_size=64, shuffle=True, num_workers=4) 59 | # Test dataset 60 | test_loader = torch.utils.data.DataLoader( 61 | datasets.MNIST(root='.', train=False, transform=transforms.Compose([ 62 | transforms.ToTensor(), 63 | transforms.Normalize((0.1307,), (0.3081,)) 64 | ])), batch_size=64, shuffle=True, num_workers=4) 65 | 66 | ###################################################################### 67 | # Depicting spatial transformer networks 68 | # -------------------------------------- 69 | # 70 | # Spatial transformer networks boils down to three main components : 71 | # 72 | # - The localization network is a regular CNN which regresses the 73 | # transformation parameters. The transformation is never learned 74 | # explicitly from this dataset, instead the network learns automatically 75 | # the spatial transformations that enhances the global accuracy. 76 | # - The grid generator generates a grid of coordinates in the input 77 | # image corresponding to each pixel from the output image. 78 | # - The sampler uses the parameters of the transformation and applies 79 | # it to the input image. 80 | # 81 | # .. figure:: /_static/img/stn/stn-arch.png 82 | # 83 | # .. Note:: 84 | # We need the latest version of PyTorch that contains 85 | # affine_grid and grid_sample modules. 86 | # 87 | 88 | 89 | class Net(nn.Module): 90 | def __init__(self): 91 | super(Net, self).__init__() 92 | self.conv1 = nn.Conv2d(1, 10, kernel_size=5) 93 | self.conv2 = nn.Conv2d(10, 20, kernel_size=5) 94 | self.conv2_drop = nn.Dropout2d() 95 | self.fc1 = nn.Linear(320, 50) 96 | self.fc2 = nn.Linear(50, 10) 97 | 98 | # Spatial transformer localization-network 99 | self.localization = nn.Sequential( 100 | nn.Conv2d(1, 8, kernel_size=7), 101 | nn.MaxPool2d(2, stride=2), 102 | nn.ReLU(True), 103 | nn.Conv2d(8, 10, kernel_size=5), 104 | nn.MaxPool2d(2, stride=2), 105 | nn.ReLU(True) 106 | ) 107 | 108 | # Regressor for the 3 * 2 affine matrix 109 | self.fc_loc = nn.Sequential( 110 | nn.Linear(10 * 3 * 3, 32), 111 | nn.ReLU(True), 112 | nn.Linear(32, 3 * 2) 113 | ) 114 | 115 | # Initialize the weights/bias with identity transformation 116 | self.fc_loc[2].weight.data.zero_() 117 | self.fc_loc[2].bias.data.copy_(torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float)) 118 | 119 | # Spatial transformer network forward function 120 | def stn(self, x): 121 | xs = self.localization(x) 122 | xs = xs.view(-1, 10 * 3 * 3) 123 | theta = self.fc_loc(xs) 124 | theta = theta.view(-1, 2, 3) 125 | 126 | grid = F.affine_grid(theta, x.size()) 127 | x = F.grid_sample(x, grid) 128 | 129 | return x 130 | 131 | def forward(self, x): 132 | # transform the input 133 | x = self.stn(x) 134 | 135 | # Perform the usual forward pass 136 | x = F.relu(F.max_pool2d(self.conv1(x), 2)) 137 | x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2)) 138 | x = x.view(-1, 320) 139 | x = F.relu(self.fc1(x)) 140 | x = F.dropout(x, training=self.training) 141 | x = self.fc2(x) 142 | return F.log_softmax(x, dim=1) 143 | 144 | 145 | model = Net().to(device) 146 | 147 | ###################################################################### 148 | # Training the model 149 | # ------------------ 150 | # 151 | # Now, let's use the SGD algorithm to train the model. The network is 152 | # learning the classification task in a supervised way. In the same time 153 | # the model is learning STN automatically in an end-to-end fashion. 154 | 155 | 156 | optimizer = optim.SGD(model.parameters(), lr=0.01) 157 | 158 | 159 | def train(epoch): 160 | model.train() 161 | for batch_idx, (data, target) in enumerate(train_loader): 162 | data, target = data.to(device), target.to(device) 163 | 164 | optimizer.zero_grad() 165 | output = model(data) 166 | loss = F.nll_loss(output, target) 167 | loss.backward() 168 | optimizer.step() 169 | if batch_idx % 500 == 0: 170 | print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( 171 | epoch, batch_idx * len(data), len(train_loader.dataset), 172 | 100. * batch_idx / len(train_loader), loss.item())) 173 | # 174 | # A simple test procedure to measure STN the performances on MNIST. 175 | # 176 | 177 | 178 | def test(): 179 | with torch.no_grad(): 180 | model.eval() 181 | test_loss = 0 182 | correct = 0 183 | for data, target in test_loader: 184 | data, target = data.to(device), target.to(device) 185 | output = model(data) 186 | 187 | # sum up batch loss 188 | test_loss += F.nll_loss(output, target, size_average=False).item() 189 | # get the index of the max log-probability 190 | pred = output.max(1, keepdim=True)[1] 191 | correct += pred.eq(target.view_as(pred)).sum().item() 192 | 193 | test_loss /= len(test_loader.dataset) 194 | print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n' 195 | .format(test_loss, correct, len(test_loader.dataset), 196 | 100. * correct / len(test_loader.dataset))) 197 | 198 | ###################################################################### 199 | # Visualizing the STN results 200 | # --------------------------- 201 | # 202 | # Now, we will inspect the results of our learned visual attention 203 | # mechanism. 204 | # 205 | # We define a small helper function in order to visualize the 206 | # transformations while training. 207 | 208 | 209 | def convert_image_np(inp): 210 | """Convert a Tensor to numpy image.""" 211 | inp = inp.numpy().transpose((1, 2, 0)) 212 | mean = np.array([0.485, 0.456, 0.406]) 213 | std = np.array([0.229, 0.224, 0.225]) 214 | inp = std * inp + mean 215 | inp = np.clip(inp, 0, 1) 216 | return inp 217 | 218 | # We want to visualize the output of the spatial transformers layer 219 | # after the training, we visualize a batch of input images and 220 | # the corresponding transformed batch using STN. 221 | 222 | 223 | def visualize_stn(): 224 | with torch.no_grad(): 225 | # Get a batch of training data 226 | data = next(iter(test_loader))[0].to(device) 227 | 228 | input_tensor = data.cpu() 229 | transformed_input_tensor = model.stn(data).cpu() 230 | 231 | in_grid = convert_image_np( 232 | torchvision.utils.make_grid(input_tensor)) 233 | 234 | out_grid = convert_image_np( 235 | torchvision.utils.make_grid(transformed_input_tensor)) 236 | 237 | # Plot the results side-by-side 238 | f, axarr = plt.subplots(1, 2) 239 | axarr[0].imshow(in_grid) 240 | axarr[0].set_title('Dataset Images') 241 | 242 | axarr[1].imshow(out_grid) 243 | axarr[1].set_title('Transformed Images') 244 | 245 | for epoch in range(1, 20 + 1): 246 | train(epoch) 247 | test() 248 | 249 | # Visualize the STN transformation on some input batch 250 | visualize_stn() 251 | 252 | plt.ioff() 253 | plt.show() 254 | --------------------------------------------------------------------------------