├── chap7 ├── python │ ├── __init__.py │ └── extend.py ├── csrc │ ├── CMakeLists.txt │ ├── gc_layer.cpp │ └── setup.py ├── check_profile.py └── readme.md ├── chap8 ├── python │ ├── __init__.py │ └── AutoGrad.py ├── check.py └── readme.md ├── pics ├── chap5-LeNet-5.png ├── chap5-compute-graph1.png └── chap5-compute-graph2.png ├── chap5 ├── include │ ├── CMakeLists.txt │ ├── Lenet5.h │ ├── GCN.h │ ├── Lenet5.cpp │ └── GCN.cpp ├── CMakeLists.txt ├── data │ ├── prepare_cora.sh │ └── prepare_mnist.sh ├── PracticeModels.cpp ├── python │ ├── utils.py │ └── PracticeModels.py └── readme.md ├── chap4 ├── include │ ├── CMakeLists.txt │ ├── LR.h │ ├── LR.cpp │ ├── MLP.h │ ├── CNN.h │ ├── MLP.cpp │ └── CNN.cpp ├── CMakeLists.txt ├── BasicModels.cpp ├── python │ └── BasicModels.py └── readme.md ├── .gitignore ├── chap3 ├── CMakeLists.txt ├── python │ └── AutoGrad.py ├── AutoGrad.cpp └── readme.md ├── chap1 ├── CMakeLists.txt ├── python │ └── HelloWorld.py ├── HelloWorld.cpp └── readme.md ├── chap2 ├── CMakeLists.txt ├── TensorBasics.cpp ├── include │ ├── TensorAttribute.h │ ├── TensorCuda.h │ ├── TensorCalculate.h │ ├── TensorIndexSlice.h │ ├── TensorInit.h │ └── TensorTransform.h ├── python │ └── TensorBasics.py └── readme.md ├── chap6 ├── CMakeLists.txt ├── readme.md ├── TorchScript.cpp └── python │ └── export_model.py ├── .github └── workflows │ ├── cl.yml │ └── test.yml ├── chap4_2 └── readme.md ├── LICENSE ├── readme.md └── chap0 └── readme.md /chap7/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chap8/python/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pics/chap5-LeNet-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clearhanhui/LearnLibTorch/HEAD/pics/chap5-LeNet-5.png -------------------------------------------------------------------------------- /chap5/include/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(libchap5 Lenet5.cpp) 2 | target_link_libraries(libchap5 ${TORCH_LIBRARIES}) -------------------------------------------------------------------------------- /chap4/include/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(libchap4 MLP.cpp CNN.cpp) 2 | target_link_libraries(libchap4 ${TORCH_LIBRARIES}) -------------------------------------------------------------------------------- /pics/chap5-compute-graph1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clearhanhui/LearnLibTorch/HEAD/pics/chap5-compute-graph1.png -------------------------------------------------------------------------------- /pics/chap5-compute-graph2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clearhanhui/LearnLibTorch/HEAD/pics/chap5-compute-graph2.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | .vscode 3 | chap5/data/cora 4 | chap5/data/MNIST 5 | chap5/python/__pycache__ 6 | chap7/python/__pycache__ 7 | chap8/python/__pycache__ 8 | chap7/csrc/dist/* 9 | chap7/csrc/gc_cpp.egg-info/* -------------------------------------------------------------------------------- /chap3/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.21) 2 | project(AutoGrad) 3 | 4 | find_package(Torch REQUIRED) 5 | add_executable(AutoGrad AutoGrad.cpp) 6 | target_link_libraries(AutoGrad ${TORCH_LIBRARIES}) 7 | -------------------------------------------------------------------------------- /chap1/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.21) 2 | project(HelloWorld) 3 | 4 | find_package(Torch REQUIRED) 5 | add_executable(HelloWorld HelloWorld.cpp) 6 | target_link_libraries(HelloWorld ${TORCH_LIBRARIES}) 7 | -------------------------------------------------------------------------------- /chap2/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.21) 2 | project(TensorBasics) 3 | 4 | find_package(Torch REQUIRED) 5 | add_executable(TensorBasics TensorBasics.cpp) 6 | target_link_libraries(TensorBasics ${TORCH_LIBRARIES}) 7 | -------------------------------------------------------------------------------- /chap6/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.21) 2 | project(TorchScript) 3 | 4 | find_package(Torch REQUIRED) 5 | add_executable(TorchScript TorchScript.cpp) 6 | target_link_libraries(TorchScript ${TORCH_LIBRARIES}) 7 | 8 | -------------------------------------------------------------------------------- /.github/workflows/cl.yml: -------------------------------------------------------------------------------- 1 | name: Commline Line 2 | 3 | on: workflow_dispatch 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | 11 | - name: Debugging with tmate 12 | uses: mxschmitt/action-tmate@v3.11 -------------------------------------------------------------------------------- /chap4/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.21) 2 | project(BasicModels) 3 | 4 | find_package(Torch REQUIRED) 5 | add_subdirectory(include) 6 | add_executable(BasicModels BasicModels.cpp) 7 | target_link_libraries(BasicModels ${TORCH_LIBRARIES} libchap4) 8 | -------------------------------------------------------------------------------- /chap4_2/readme.md: -------------------------------------------------------------------------------- 1 | # 数据集和数据加载 2 | 3 | 关于自定义 DataSet 可以参考 AllentDan 的博客，[中文版](https://zhuanlan.zhihu.com/p/369930525)，[英文版](https://github.com/AllentDan/LibtorchTutorials/tree/main/lesson4-DatasetUtilization)。 4 | DataLoader 一般不需要改动，用法在[第五章](../chap5)中可以找到，主要是 `make_data_loader` 函数。 -------------------------------------------------------------------------------- /chap5/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.21) 2 | project(PracticeModels) 3 | 4 | find_package(Torch REQUIRED) 5 | add_subdirectory(include) 6 | add_executable(PracticeModels PracticeModels.cpp) 7 | target_link_libraries(PracticeModels ${TORCH_LIBRARIES} libchap5) 8 | -------------------------------------------------------------------------------- /chap5/data/prepare_cora.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Download from https://github.com/tkipf/pygcn/tree/master/data/cora " 3 | mkdir cora 4 | wget -q https://github.com/tkipf/pygcn/raw/master/data/cora/cora.cites -O cora/cora.cites 5 | wget -q https://github.com/tkipf/pygcn/raw/master/data/cora/cora.content -O cora/cora.content 6 | -------------------------------------------------------------------------------- /chap1/python/HelloWorld.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | @File : HelloWorld.py 5 | @Time : 2022/07/06 16:15:32 6 | @Author : Han Hui 7 | @Contact : clearhanhui@gmail.com 8 | ''' 9 | 10 | 11 | import torch 12 | 13 | tensor = torch.zeros((2, 3)) 14 | print(tensor) 15 | print("\nWelcome to LibTorch!") 16 | -------------------------------------------------------------------------------- /chap4/include/LR.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : LR.h 3 | * @Time : 2022/07/09 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | 10 | class LinearRegression : public torch::nn::Module { 11 | public: 12 | LinearRegression(int in_dim, int out_dim); 13 | torch::Tensor forward(torch::Tensor); 14 | 15 | private: 16 | torch::nn::Linear lin{nullptr}; 17 | }; -------------------------------------------------------------------------------- /chap1/HelloWorld.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : HelloWorld.cpp 3 | * @Time : 2022/07/06 16:10:05 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | int main() { 12 | // 创建一个(2,3)张量 13 | torch::Tensor tensor = torch::zeros({2, 3}); 14 | std::cout << tensor << std::endl; 15 | std::cout << "\nWelcome to LibTorch!" << std::endl; 16 | 17 | return 0; 18 | } -------------------------------------------------------------------------------- /chap4/include/LR.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : LR.cpp 3 | * @Time : 2022/07/11 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include "LR.h" 9 | 10 | LinearRegression::LinearRegression(int in_dim, int out_dim){ 11 | lin = torch::nn::Linear(in_dim, out_dim); 12 | lin = register_module("lin", lin); 13 | } 14 | 15 | torch::Tensor LinearRegression::forward(torch::Tensor x){ 16 | x = lin->forward(x); 17 | return x; 18 | } -------------------------------------------------------------------------------- /chap4/include/MLP.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : MLP.h 3 | * @Time : 2022/07/07 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | class MLP : public torch::nn::Module { 12 | public: 13 | MLP(int in_dim, int hidden_dim,int out_dim); 14 | torch::Tensor forward(torch::Tensor x); 15 | 16 | private: 17 | torch::nn::Linear lin1{nullptr}; 18 | torch::nn::Linear lin2{nullptr}; 19 | torch::nn::Linear lin3{nullptr}; 20 | }; -------------------------------------------------------------------------------- /chap4/include/CNN.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : CNN.h 3 | * @Time : 2022/07/11 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | 10 | class CNN : public torch::nn::Module { 11 | public: 12 | CNN(int num_classes); 13 | torch::Tensor forward(torch::Tensor x); 14 | 15 | private: 16 | torch::nn::Conv2d conv1{nullptr}; 17 | torch::nn::Conv2d conv2{nullptr}; 18 | torch::nn::ReLU relu{nullptr}; 19 | torch::nn::MaxPool2d max_pool{nullptr}; 20 | torch::nn::BatchNorm2d bn{nullptr}; 21 | torch::nn::Linear lin{nullptr}; 22 | }; -------------------------------------------------------------------------------- /chap2/TensorBasics.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : TensorBasics.cpp 3 | * @Time : 2022/07/06 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include "include/TensorIndexSlice.h" 9 | #include "include/TensorInit.h" 10 | #include "include/TensorAttribute.h" 11 | #include "include/TensorTransform.h" 12 | #include "include/TensorCalculate.h" 13 | #include "include/TensorCuda.h" 14 | 15 | int main() { 16 | tensor_init(); 17 | tensor_index_slice(); 18 | tensor_attribute(); 19 | tensor_transform(); 20 | tensor_calculate(); 21 | tensor_cuda(); 22 | 23 | return 0; 24 | } -------------------------------------------------------------------------------- /chap5/include/Lenet5.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : Lenet5.h 3 | * @Time : 2022/07/13 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | 10 | class Lenet5 : public torch::nn::Module { 11 | private: 12 | torch::nn::Conv2d conv1{nullptr}; 13 | torch::nn::Conv2d conv2{nullptr}; 14 | torch::nn::ReLU relu{nullptr}; 15 | torch::nn::MaxPool2d max_pool{nullptr}; 16 | torch::nn::Linear lin1{nullptr}; 17 | torch::nn::Linear lin2{nullptr}; 18 | torch::nn::Linear lin3{nullptr}; 19 | 20 | public: 21 | Lenet5(); 22 | torch::Tensor forward(torch::Tensor x); 23 | }; 24 | -------------------------------------------------------------------------------- /chap2/include/TensorAttribute.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : TensorAttribute.h 3 | * @Time : 2022/07/06 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | void tensor_attribute() { 12 | torch::Tensor a = torch::randn({2, 3}); 13 | std::cout << a.size(1) << std::endl; 14 | std::cout << a.sizes() << std::endl; 15 | std::cout << a[0].sizes() << std::endl; 16 | std::cout << a[0][0].item() << std::endl; 17 | std::cout << a.data() << std::endl; 18 | std::cout << a.dtype() << std::endl; 19 | std::cout << a.device() << std::endl; 20 | } 21 | -------------------------------------------------------------------------------- /chap6/readme.md: -------------------------------------------------------------------------------- 1 | 2 | # TorchScript 3 | 4 | TorchScript 允许对 Python 中定义的 PyTorch 模型进行序列化，然后在 C++ 中加载和运行，通过 `torch.jit.script`或 `torch.jit.trace` 其执行来捕获模型代码。 5 | 6 | > The C++ interface to TorchScript encompasses three primary pieces of functionality: 7 | > * A mechanism for loading and executing serialized TorchScript models defined in Python; 8 | > * An API for defining custom operators that extend the TorchScript standard library of operations; 9 | > * Just-in-time compilation of TorchScript programs from C++. 10 | 11 | 12 | 13 | ------------ 14 | 15 | 内容主要来自于这篇官方文档，内容稍有修改，感兴趣可以阅读原文档。 16 | 17 | [LOADING A TORCHSCRIPT MODEL IN C++](https://pytorch.org/tutorials/advanced/cpp_export.html) -------------------------------------------------------------------------------- /chap4/include/MLP.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : MLP.cpp 3 | * @Time : 2022/07/11 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include "MLP.h" 9 | 10 | MLP::MLP(int in_dim, int hidden_dim, int out_dim) { 11 | lin1 = torch::nn::Linear(in_dim, hidden_dim); 12 | lin2 = torch::nn::Linear(hidden_dim, hidden_dim); 13 | lin3 = torch::nn::Linear(hidden_dim, out_dim); 14 | 15 | register_module("lin1", lin1); 16 | register_module("lin2", lin2); 17 | register_module("lin3", lin3); 18 | }; 19 | 20 | torch::Tensor MLP::forward(torch::Tensor x) { 21 | x = lin1(x); 22 | x = torch::relu(x); 23 | x = lin2(x); 24 | x = torch::relu(x); 25 | x = lin3(x); 26 | return x; 27 | } 28 | -------------------------------------------------------------------------------- /chap2/include/TensorCuda.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : TensorCuda.h 3 | * @Time : 2023/08/31 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | void tensor_cuda() { 12 | torch::Device device = torch::Device(torch::kCPU); 13 | if (torch::cuda::is_available()) { 14 | device = torch::Device(torch::kCUDA); 15 | } 16 | // torch::Tensor a = torch::randn({3, 3}).cuda(); 17 | torch::Tensor b = torch::randn({3, 3}).to(device); 18 | torch::Tensor c = torch::randn({3, 3}, device); // 似乎是发生了 Device -> TensorOptions 的隐式转换 19 | 20 | // std::cout << a << std::endl; 21 | std::cout << b << std::endl; 22 | std::cout << c << std::endl; 23 | } -------------------------------------------------------------------------------- /chap2/include/TensorCalculate.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : TensorCalculate.h 3 | * @Time : 2022/07/07 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | void tensor_calculate() { 12 | torch::Tensor a = torch::ones({3, 3}); 13 | torch::Tensor b = torch::randn({3, 3}); 14 | torch::Tensor c = a.matmul(b); 15 | torch::Tensor d = a.mul(b); 16 | torch::Tensor e = torch::cat({a, b}, 0); 17 | torch::Tensor f = torch::stack({a, b}); 18 | 19 | std::cout << a << std::endl; 20 | std::cout << b << std::endl; 21 | std::cout << c << std::endl; 22 | std::cout << d << std::endl; 23 | std::cout << e << std::endl; 24 | std::cout << f << std::endl; 25 | } -------------------------------------------------------------------------------- /chap5/data/prepare_mnist.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Download from http://yann.lecun.com/exdb/mnist" 3 | mkdir -p MNIST/raw 4 | wget -q http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz -O MNIST/raw/train-images-idx3-ubyte.gz 5 | wget -q http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz -O MNIST/raw/train-labels-idx1-ubyte.gz 6 | wget -q http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz -O MNIST/raw/t10k-images-idx3-ubyte.gz 7 | wget -q http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz -O MNIST/raw/t10k-labels-idx1-ubyte.gz 8 | gzip -dfk MNIST/raw/train-images-idx3-ubyte.gz 9 | gzip -dfk MNIST/raw/train-labels-idx1-ubyte.gz 10 | gzip -dfk MNIST/raw/t10k-images-idx3-ubyte.gz 11 | gzip -dfk MNIST/raw/t10k-labels-idx1-ubyte.gz 12 | -------------------------------------------------------------------------------- /chap3/python/AutoGrad.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | @File : AutoGrad.py 5 | @Time : 2022/07/07 6 | @Author : Han Hui 7 | @Contact : clearhanhui@gmail.com 8 | ''' 9 | 10 | 11 | import torch 12 | 13 | x = torch.tensor(2.0) 14 | w = torch.tensor(3.0, requires_grad=True) 15 | b = torch.tensor(4.0, requires_grad=True) 16 | y = w * x + b 17 | y.backward(retain_graph=True) 18 | print(w.grad) 19 | print(b.grad) 20 | print(x.requires_grad) 21 | x.requires_grad_(True) # x.requires_grad=True 22 | print(x.requires_grad) 23 | 24 | xx = torch.randn(3, requires_grad=True) 25 | yy = xx * xx + 2 * xx 26 | yy.backward(torch.tensor([1,2,3])) 27 | print(xx) 28 | print(xx.grad) 29 | print((xx*torch.tensor([2,4,6])+torch.tensor([2,4,6])).data) 30 | -------------------------------------------------------------------------------- /chap7/csrc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.21) 2 | project(lltm_cpp) 3 | 4 | 5 | 6 | ## c++ 7 | # find_package(Python3 COMPONENTS Development) 8 | # find_package(Torch REQUIRED) 9 | # add_library(lltm_cpp SHARED lltm.cpp) 10 | # target_compile_features(lltm_cpp PRIVATE cxx_std_14) 11 | # target_link_libraries(lltm_cpp ${TORCH_LIBRARIES} Python3::Python) 12 | 13 | 14 | ## python 15 | ## https://github.com/pytorch/pytorch/issues/38122 16 | find_package(pybind11 REQUIRED) 17 | find_package(Torch REQUIRED) 18 | find_library(TORCH_PYTHON_LIBRARY torch_python PATHS "${TORCH_INSTALL_PREFIX}/lib") 19 | message(${TORCH_PYTHON_LIBRARY}) 20 | pybind11_add_module(lltm_cpp MODULE lltm.cpp) 21 | target_link_libraries(lltm_cpp 22 | PRIVATE ${TORCH_LIBRARIES} 23 | PRIVATE ${TORCH_PYTHON_LIBRARY} 24 | ) 25 | -------------------------------------------------------------------------------- /chap7/csrc/gc_layer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | torch::Tensor gc_forward(torch::Tensor a, 5 | torch::Tensor x, 6 | torch::Tensor w, 7 | torch::Tensor b) { 8 | return a.mm(x).mm(w) + b; 9 | } 10 | 11 | torch::Tensor gc_backward(torch::Tensor a, 12 | torch::Tensor x, 13 | torch::Tensor g) { 14 | return a.mm(x).t().mm(g); 15 | } 16 | 17 | PYBIND11_MODULE(gc_cpp, m) { 18 | m.def("forward", &gc_forward, "gc forward"); 19 | m.def("backward", &gc_backward, "gc backward"); 20 | } 21 | 22 | // PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { 23 | // m.def("forward", &gc_forward, "gc forward"); 24 | // m.def("backward", &gc_backward, "gc backward"); 25 | // } -------------------------------------------------------------------------------- /chap5/include/GCN.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : GCN.h 3 | * @Time : 2022/07/21 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | namespace nn = torch::nn; 10 | namespace F = torch::nn::functional; 11 | 12 | class GCNLayerImpl : public nn::Module { 13 | public: 14 | 15 | GCNLayerImpl(int in_features, int out_features); 16 | torch::Tensor forward(torch::Tensor x, torch::Tensor a); 17 | 18 | private: 19 | torch::Tensor w; 20 | torch::Tensor b; 21 | }; 22 | TORCH_MODULE(GCNLayer); // 注册 23 | 24 | 25 | class GCN : public nn::Module { 26 | public: 27 | GCN(); 28 | torch::Tensor forward(torch::Tensor x, torch::Tensor a); 29 | 30 | private: 31 | torch::nn::Dropout dropout = nullptr; 32 | GCNLayer gc1 = nullptr; 33 | GCNLayer gc2 = nullptr; 34 | }; -------------------------------------------------------------------------------- /chap6/TorchScript.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : TorchScript.cpp 3 | * @Time : 2022/07/18 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | int main() { 14 | std::string module_path = "../python/traced_resnet_model_pos.pt"; 15 | 16 | torch::jit::script::Module module; 17 | try { 18 | module = torch::jit::load(module_path); 19 | } catch (const c10::Error &e) { 20 | std::cout << "error loading the model\n"; 21 | return -1; 22 | } 23 | 24 | std::vector x; 25 | x.push_back(torch::ones({1, 1, 28, 28})); 26 | 27 | at::Tensor output = module.forward(x).toTensor(); 28 | std::cout << output.sum() << std::endl; 29 | 30 | return 0; 31 | } -------------------------------------------------------------------------------- /chap2/include/TensorIndexSlice.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : TensorIndexSlice.h 3 | * @Time : 2022/07/07 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | #include 10 | using namespace torch::indexing; //不然一行代码真长啊 11 | 12 | void tensor_index_slice() { 13 | torch::Tensor a = torch::randn({2, 3, 4}); 14 | torch::Tensor b = a[1]; 15 | torch::Tensor c = a.index({1, 2, 3}); // 等同 a[1][2][3] 16 | torch::Tensor d = a.index({Slice(None), 2}); // 等同 a.index({"...", 2}) 17 | torch::Tensor e = a.index({Slice(None), Slice(None), Slice(None, None, 2)}); 18 | torch::Tensor f = a.index_select(-1, torch::tensor({1, 1, 0})); 19 | 20 | std::cout << a << std::endl; 21 | std::cout << b << std::endl; 22 | std::cout << c << std::endl; 23 | std::cout << d << std::endl; 24 | std::cout << e << std::endl; 25 | std::cout << f << std::endl; 26 | } -------------------------------------------------------------------------------- /chap7/csrc/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import Extension, setup 2 | from torch.utils.cpp_extension import BuildExtension, CppExtension, include_paths, library_paths 3 | 4 | 5 | # libraries = ['c10', 'torch', 'torch_cpu', 'torch_python'] 6 | # modules = [Extension( 7 | # name='gc_cpp', # package name 8 | # sources=['gc_layer.cpp'], 9 | # include_dirs=include_paths(), 10 | # library_dirs=library_paths(), 11 | # libraries=libraries, 12 | # language='c++')] 13 | 14 | 15 | # The next line is equivalent to the above code snippets 16 | modules = [CppExtension('gc_cpp', ['gc_layer.cpp'])] 17 | 18 | setup( 19 | name='gc_cpp', # pypi name, not package name. 20 | version='0.0.1', 21 | description='GCN layer forward and backward extension of pytorch', 22 | author='clearhanhui', 23 | author_email='clearhanhui@gmail.com', 24 | 25 | ext_modules=modules, 26 | cmdclass={'build_ext': BuildExtension} 27 | ) 28 | -------------------------------------------------------------------------------- /chap2/include/TensorInit.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : TensorInit.h 3 | * @Time : 2022/07/06 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | #include 10 | torch::TensorOptions opt; 11 | 12 | void tensor_init() { 13 | torch::Tensor a = torch::zeros({2, 3}); 14 | std::cout << (a > 0); 15 | torch::Tensor b = torch::ones({2, 3}); 16 | torch::Tensor c = torch::eye(3); 17 | torch::Tensor d = torch::full_like(a, 10); 18 | torch::Tensor e = torch::randn({2, 3}); 19 | torch::Tensor f = torch::arange(10); 20 | torch::Tensor g = torch::tensor({{1, 2}, {3, 4}}, opt.dtype(torch::kFloat32)); 21 | // torch::Tensor h = torch::from_blob({1, 2, 3, 4}, {4}); // 大坑 22 | 23 | std::cout << a << std::endl; 24 | std::cout << b << std::endl; 25 | std::cout << c << std::endl; 26 | std::cout << d << std::endl; 27 | std::cout << e << std::endl; 28 | std::cout << f << std::endl; 29 | std::cout << g << std::endl; 30 | // std::cout << h << std::endl; 31 | } -------------------------------------------------------------------------------- /chap4/include/CNN.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : CNN.cpp 3 | * @Time : 2022/07/11 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include "CNN.h" 9 | #include 10 | 11 | CNN::CNN(int num_classes) { 12 | conv1 = torch::nn::Conv2d(torch::nn::Conv2dOptions(1, 16, 3).padding(1)); 13 | conv2 = torch::nn::Conv2d(torch::nn::Conv2dOptions(16, 16, 3).padding(1)); 14 | bn = torch::nn::BatchNorm2d(16); 15 | relu = torch::nn::ReLU(); 16 | max_pool = torch::nn::MaxPool2d(2); 17 | lin = torch::nn::Linear(7 * 7 * 16, num_classes); 18 | 19 | register_module("conv1", conv1); 20 | register_module("conv2", conv2); 21 | register_module("bn", bn); 22 | register_module("relu", relu); 23 | register_module("max_pool", max_pool); 24 | register_module("lin", lin); 25 | } 26 | 27 | torch::Tensor CNN::forward(torch::Tensor x) { 28 | x = conv1(x); 29 | x = bn(x); 30 | x = relu(x); 31 | x = max_pool2d(x, 2); 32 | 33 | x = conv2(x); 34 | x = bn(x); 35 | x = relu(x); 36 | x = max_pool2d(x, 2); 37 | x = lin(x.reshape({x.size(0), -1})); 38 | 39 | return x; 40 | } 41 | -------------------------------------------------------------------------------- /chap3/AutoGrad.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : AutoGrad.cpp 3 | * @Time : 2022/07/07 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | int main() { 12 | // y = w * x + b 13 | torch::Tensor x = torch::tensor({2.0}); 14 | torch::Tensor w = torch::tensor({3.0}, torch::requires_grad()); 15 | torch::Tensor b = torch::tensor({4.0}, torch::requires_grad()); 16 | torch::Tensor y = w * x + b; 17 | y.backward(); 18 | std::cout << w.grad() << std::endl; 19 | std::cout << b.grad() << std::endl; 20 | std::cout << x.requires_grad() << std::endl; 21 | x.requires_grad_(true); 22 | std::cout << x.requires_grad() << std::endl; 23 | 24 | // yy = xx * xx + 2 * xx 25 | torch::Tensor xx = torch::randn({3}, torch::requires_grad()); 26 | torch::Tensor yy = xx * xx + 2 * xx; 27 | yy.backward(torch::tensor({1, 2, 3})); 28 | std::cout << xx << std::endl; 29 | std::cout << xx.grad() << std::endl; 30 | std::cout << xx * torch::tensor({2, 4, 6}) + torch::tensor({2, 4, 6}) 31 | << std::endl; 32 | 33 | return 0; 34 | } -------------------------------------------------------------------------------- /chap5/include/Lenet5.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : Lenet5.cpp 3 | * @Time : 2022/07/13 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include "Lenet5.h" 9 | 10 | Lenet5::Lenet5(){ 11 | conv1 = torch::nn::Conv2d(torch::nn::Conv2dOptions(1, 6, 5).padding(2)); 12 | conv2 = torch::nn::Conv2d(torch::nn::Conv2dOptions(6, 16, 5)); 13 | relu = torch::nn::ReLU(); 14 | max_pool = torch::nn::MaxPool2d(2); 15 | lin1 = torch::nn::Linear(5 * 5 * 16, 120); 16 | lin2 = torch::nn::Linear(120, 84); 17 | lin3 = torch::nn::Linear(84, 10); 18 | 19 | register_module("conv1", conv1); 20 | register_module("conv2", conv2); 21 | register_module("relu", relu); 22 | register_module("max_pool", max_pool); 23 | register_module("lin1", lin1); 24 | register_module("lin2", lin2); 25 | register_module("lin3", lin3); 26 | } 27 | 28 | torch::Tensor Lenet5::forward(torch::Tensor x){ 29 | x = max_pool(relu(conv1(x))); 30 | x = max_pool(relu(conv2(x))); 31 | x = x.reshape({x.size(0), -1}); 32 | x = lin1(x); 33 | x = lin2(x); 34 | x = lin3(x); 35 | 36 | return x; 37 | } 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 HanHui 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /chap8/check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | @File : check.py 5 | @Time : 2022/07/28 6 | @Author : Han Hui 7 | @Contact : clearhanhui@gmail.com 8 | ''' 9 | 10 | 11 | import torch 12 | import numpy as np 13 | from python.AutoGrad import make_array, mm, mul, add, BackwardEngine 14 | 15 | 16 | a = make_array(np.random.randn(2,2), requires_grad=True) 17 | b = make_array(np.random.randn(2,2), requires_grad=True) 18 | c = make_array(np.random.randn(2,2), requires_grad=True) 19 | 20 | d = mul(add(a, b), c) 21 | e = mm(d, add(a, c)) 22 | engine = BackwardEngine(e) 23 | engine.run_backward(np.ones_like(e)) 24 | 25 | # print(a.grad) 26 | # print(b.grad) 27 | # print(c.grad) 28 | 29 | 30 | aa = torch.tensor(a, requires_grad=True) 31 | bb = torch.tensor(b, requires_grad=True) 32 | cc = torch.tensor(c, requires_grad=True) 33 | 34 | dd = torch.mul(torch.add(aa, bb), cc) 35 | ee = torch.mm(dd, torch.add(aa, cc)) 36 | ee.backward(torch.ones_like(ee)) 37 | 38 | # print(aa.grad.numpy()) 39 | # print(bb.grad.numpy()) 40 | # print(cc.grad.numpy()) 41 | 42 | print(np.allclose(a.grad, aa.grad.numpy())) 43 | print(np.allclose(b.grad, bb.grad.numpy())) 44 | print(np.allclose(c.grad, cc.grad.numpy())) 45 | -------------------------------------------------------------------------------- /chap5/include/GCN.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : GCN.cpp 3 | * @Time : 2022/07/21 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include "GCN.h" 9 | #include 10 | #include 11 | 12 | GCNLayerImpl::GCNLayerImpl(int in_features, int out_features){ 13 | w = torch::randn((in_features, out_features)); 14 | b = torch::randn((out_features)); 15 | w.requires_grad_(); 16 | b.requires_grad_(); 17 | float dev = sqrt(out_features); 18 | torch::nn::init::uniform_(w, -dev, dev); 19 | torch::nn::init::uniform_(b, -dev, dev); 20 | register_parameter("w", w); 21 | register_parameter("b", b); 22 | } 23 | 24 | torch::Tensor GCNLayerImpl::forward(torch::Tensor x, torch::Tensor a){ 25 | torch::Tensor out = torch::mm(a, torch::mm(x, w)); 26 | return out; 27 | } 28 | 29 | GCN::GCN(){ 30 | dropout = torch::nn::Dropout(torch::nn::DropoutOptions().p(0.5)); 31 | gc1 = GCNLayer(1433, 16); 32 | gc2 = GCNLayer(16, 7); 33 | register_module("gc1", gc1); 34 | register_module("gc2", gc2); 35 | register_module("dropout", dropout); 36 | } 37 | 38 | torch::Tensor GCN::forward(torch::Tensor x, torch::Tensor a){ 39 | x = F::relu(gc1(x, a)); 40 | x = dropout(x); 41 | x = gc2(x, a); 42 | return x; 43 | } -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | - [LibTorch 教程](#libtorch-教程) 2 | - [简介](#简介) 3 | - [目录](#目录) 4 | - [软件环境](#软件环境) 5 | 6 | 7 | # LibTorch 教程 8 | 9 | ## 简介 10 | 11 | LibTorch 是什么呢，简单来讲可以认为它提供了一种 c++ 前端（同理 PyTorch 是一种 python 前端）。在其[设计哲学](https://pytorch.org/cppdocs/frontend.html#philosophy)中提到 12 | 13 | > PyTorch’s C++ frontend was designed with the idea that the Python frontend is great, and should be used when possible; but in some settings, performance and portability requirements make the use of the Python interpreter infeasible. 14 | 15 | 简单来说它可以提供更高的效率，此外由于在部署和拓展上面也可以和 PyTorch 很好的结合。然而网上关于 LibTorch 的中文教程和资料太少了，最近也要用到就好好学习总结下。 16 | 17 | 本教程每个目录是独立一个 [CMake](https://cmake.org/) 项目，每个项目主要参考内容是 [LibTorch 官方文档](https://pytorch.org/cppdocs/)和网上的一些中英文博客资料。 18 | 19 | 20 | ## 目录 21 | * [0--LibTorch 配置](./chap0/) 22 | * [1--HelloWorld](./chap1/) 23 | * [2--张量基础](./chap2/) 24 | * [3--自动微分](./chap3/) 25 | * [4--基本模型](./chap4/) 26 | * [4_2--数据集和数据加载](./chap4_2/) 27 | * [5--模型实践](./chap5/) 28 | * [6--TorchScript](./chap6/) 29 | * [7--PyTorch拓展](./chap7/) 30 | * [8--自动微分引擎demo](./chap8/) 31 | 32 | 33 | ## 软件环境 34 | 35 | 必要的软件和详细的环境配置可以参考[chap0](./chap0/)。 36 | 37 | 38 | ---------------- 39 | ---------------- 40 | 41 | 如有 bug 欢迎 [issue](https://github.com/clearhanhui/LearnLibTorch/issues)，喜欢的话给个免费的 star 。 42 | 43 | 对在 CV 的应用 LibTorch 感兴趣可以去看 AllentDan 大佬的 LibTorch 系列，我也从中学习到很多。 44 | * [https://github.com/AllentDan/LibtorchTutorials](https://github.com/AllentDan/LibtorchTutorials) 45 | * [https://github.com/AllentDan/LibtorchDetection](https://github.com/AllentDan/LibtorchDetection) 46 | * [https://github.com/AllentDan/LibtorchSegmentation](https://github.com/AllentDan/LibtorchSegmentation) 47 | 48 | -------------------------------------------------------------------------------- /chap1/readme.md: -------------------------------------------------------------------------------- 1 | - [HelloWorld](#helloworld) 2 | - [1. 创建 CMake 项目](#1-创建-cmake-项目) 3 | - [2. 编写 HelloWorld.cpp](#2-编写-helloworldcpp) 4 | - [3. 编译运行](#3-编译运行) 5 | 6 | # HelloWorld 7 | 8 | ## 1. 创建 CMake 项目 9 | 10 | `CMakelists.txt` 文件如下： 11 | 12 | ```cmake 13 | cmake_minimum_required(VERSION 3.21) 14 | project(HelloWorld) 15 | 16 | find_package(Torch REQUIRED) 17 | add_executable(HelloWorld HelloWorld.cpp) 18 | target_link_libraries(HelloWorld $TORCH_LIBRARIES) 19 | ``` 20 | 逐行解释一下 21 | `cmake_minimum_required` 设置了最小的 cmake 版本。 22 | `project` 声明了项目名称，会在全局增加一个 `$PROJECT_NAME` 的变量。 23 | `find_package` 用于查找 LibTorch，并且会根据 LibTorch 中的内容生成对应的变量如 `$TORCH_LIBRARIES`。 24 | `add_executable` 声明了生成的可执行文件和对应的源文件。 25 | `target_link_libraries` 链接库。 26 | 27 | 28 | ## 2. 编写 HelloWorld.cpp 29 | 30 | 创建并打印一个全零的张量，代码如下： 31 | ```cpp 32 | #include 33 | #include 34 | 35 | int main() { 36 | // 创建一个(2,3)张量 37 | torch::Tensor tensor = torch::zeros({2, 3}); 38 | std::cout << tensor << std::endl; 39 | std::cout << "Welcome to LibTorch" << std::endl; 40 | 41 | return 0; 42 | } 43 | ``` 44 | 45 | 46 | ## 3. 编译运行 47 | 48 | 依次执行下列命令 49 | 50 | ```bash 51 | mkdir build && cd build 52 | cmake .. 53 | make 54 | ./HelloWorld 55 | ``` 56 | 57 | 输出如下： 58 | 59 | ``` 60 | 0 0 0 61 | 0 0 0 62 | [ CPUFloatType{2,3} ] 63 | 64 | Welcome to LibTorch! 65 | ``` 66 | 67 | 如果可以正确得到以上输出就说明基本环境没问题了。 68 | 69 | > 大坑：执行 `cmake ..` 的时候一直报错找不到 `TorchConfig.cmake`，但可以通过 `-DCMAKE_PREFIX_PATH=/path/to/libtorch` 编译通过，说明 LibTorch 没问题。我是通过 `apt` 命令安装的 3.18，`find_package()` [官方文档](https://cmake.org/cmake/help/latest/command/find_package.html)上面说 3.12 之后的版本可以通过 `_ROOT` 变量查找，按理说版本应该没问题，百思不得其解，最后无奈从官网手动下载了 3.23 竟然可以了，实测 3.21 也可以。 -------------------------------------------------------------------------------- /chap6/python/export_model.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | @File : train_export_model.py 5 | @Time : 2022/07/18 6 | @Author : Han Hui 7 | @Contact : clearhanhui@gmail.com 8 | ''' 9 | 10 | import sys 11 | import torch 12 | import torch.nn as nn 13 | import torch.nn.functional as F 14 | 15 | 16 | class LeNet5(nn.Module): 17 | def __init__(self): 18 | super().__init__() 19 | self.conv1 = nn.Conv2d(1, 6, (5, 5), padding=2) 20 | self.conv2 = nn.Conv2d(6, 16, (5, 5)) 21 | self.fc1 = torch.nn.Linear(16*5*5, 120) 22 | self.fc2 = torch.nn.Linear(120, 84) 23 | self.fc3 = torch.nn.Linear(84, 10) 24 | 25 | def infer(self, x): 26 | x = F.max_pool2d(F.relu(self.conv1(x)), 2) 27 | x = F.max_pool2d(F.relu(self.conv2(x)), 2) 28 | x = x.reshape((x.size(0), -1)) 29 | x = F.relu(self.fc1(x)) 30 | x = F.relu(self.fc2(x)) 31 | x = self.fc3(x) 32 | return x 33 | 34 | def forward(self, x): 35 | x = self.infer(x) 36 | return x 37 | 38 | 39 | class LeNet5_pos(LeNet5): 40 | def __init__(self): 41 | super().__init__() 42 | def forward(self, x): 43 | x = self.infer(x) 44 | if x.sum() > 0: 45 | return x 46 | else: 47 | return -x 48 | 49 | 50 | model = LeNet5() 51 | model_pos = LeNet5_pos() 52 | example = torch.rand(1, 1, 28, 28) 53 | 54 | traced_script_module = torch.jit.trace(model, example) 55 | # traced_script_module_pos = torch.jit.trace(model_pos, example) # cause warning 56 | traced_script_module_pos = torch.jit.script(model_pos) 57 | 58 | save_path = sys.path[0] + "/" 59 | traced_script_module.save(save_path+"traced_resnet_model.pt") 60 | traced_script_module_pos.save(save_path+"traced_resnet_model_pos.pt") 61 | -------------------------------------------------------------------------------- /chap7/check_profile.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | @File : check_profile.py 5 | @Time : 2022/07/28 6 | @Author : Han Hui 7 | @Contact : clearhanhui@gmail.com 8 | ''' 9 | 10 | 11 | import time 12 | import torch 13 | import torch.nn as nn 14 | import torch.nn.functional as F 15 | from torch.autograd import gradcheck 16 | from python.extend import GCNLayerFunction, LinearFunction, GCNLayer, AddSelf 17 | 18 | 19 | addself = AddSelf.apply 20 | x = torch.randn(1, dtype=torch.double, requires_grad=True) 21 | y = addself(x) 22 | print(gradcheck(addself, x, eps=1e-6, atol=1e-4)) 23 | 24 | 25 | linear = LinearFunction.apply 26 | input = torch.randn(20, 20, dtype=torch.double, requires_grad=True) 27 | weight = torch.randn(20, 30, dtype=torch.double, requires_grad=True) 28 | print(gradcheck(linear, (input, weight), eps=1e-6, atol=1e-4)) 29 | 30 | 31 | use_cpp = False 32 | gc = GCNLayerFunction.apply 33 | a = torch.randn((10, 10), dtype=torch.double) 34 | x = torch.randn((10, 20), dtype=torch.double) 35 | w = torch.randn((20, 30), dtype=torch.double, requires_grad=True) 36 | b = torch.randn(30, dtype=torch.double, requires_grad=True) 37 | print(gradcheck(gc, (a, x, w, b, use_cpp), eps=1e-6, atol=1e-4)) 38 | 39 | 40 | class GCN(nn.Module): 41 | def __init__(self): 42 | super().__init__() 43 | self.gc1 = GCNLayer(1433, 16) 44 | self.gc2 = GCNLayer(16, 7) 45 | self.dropout = nn.Dropout(0.5) 46 | 47 | def forward(self, a, x): 48 | x = F.relu(self.gc1(a, x, use_cpp)) 49 | x = self.dropout(x) 50 | x = self.gc2(a, x, use_cpp) 51 | return x 52 | 53 | start_t = time.time() 54 | gcn = GCN() 55 | loss_fn = torch.nn.CrossEntropyLoss() 56 | adam = torch.optim.Adam(gcn.parameters(), lr=0.01, weight_decay=5e-4) 57 | a = torch.randn((2708, 2708)) 58 | x = torch.randn((2708, 1433)) 59 | y = torch.randint(0, 2, (2708,), dtype=torch.long) 60 | for i in range(200): 61 | y_prob = gcn(a, x) 62 | loss = loss_fn(y_prob, y) 63 | adam.zero_grad() 64 | loss.backward() 65 | adam.step() 66 | end_t = time.time() 67 | print(end_t - start_t) 68 | -------------------------------------------------------------------------------- /chap2/include/TensorTransform.h: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : TensorTransform.h 3 | * @Time : 2022/07/07 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | void tensor_transform() { 12 | torch::Tensor a = torch::randn({2, 3}); 13 | torch::Tensor b = a.transpose(0, 1); 14 | torch::Tensor c = a.reshape({-1, 1}); 15 | torch::Tensor d = a.view({3, 2}); 16 | torch::Tensor e = a.toType(torch::kFloat32); 17 | 18 | std::cout < f_accessor = f.accessor(); 32 | std::cout << sizeof(f_accessor.data()) << std::endl; // 输出 8，表示指针的长度 33 | std::cout << sizeof(f_accessor[0][2]) << std::endl; // 同上，表示 long 的长度 34 | // 下面打印的地址一致 35 | std::cout << f_accessor.data() << std::endl 36 | << &f_accessor[0][0] << std::endl 37 | << &f_accessor[0][1] << std::endl; 38 | // 连续地址输出结果，可以发现仍然是转置之前的顺序， 39 | // contiguous() 可以强行把内存顺序修改。 40 | for (int i = 0; i < 9; ++i){ 41 | std::cout << *(f_accessor.data() + i) << " "; 42 | if (i == 8) { 43 | std::cout << std::endl; 44 | } 45 | } 46 | f_accessor[2][0] = 100; 47 | f[2][1] = 101; // &f[2][1] 会报错 48 | std::cout << f << std::endl; 49 | 50 | torch::Tensor g = torch::tensor({{3},{1}}); 51 | torch::Tensor gg = g.expand({2, 2}); 52 | auto g_accessor = g.accessor(); 53 | auto gg_accessor = gg.accessor(); 54 | // 下面打印的地址一致，都是数字 3 对应的地址。 55 | std::cout << g_accessor.data() << std::endl 56 | << gg_accessor.data() << std::endl; 57 | std::cout << &g_accessor[0][0] << std::endl 58 | << &gg_accessor[0][1] << std::endl 59 | << &gg_accessor[0][1] << std::endl; 60 | 61 | } 62 | -------------------------------------------------------------------------------- /chap3/readme.md: -------------------------------------------------------------------------------- 1 | - [自动微分](#自动微分) 2 | - [1. 普通微分](#1-普通微分) 3 | - [2. vector-Jacobian product(vjp)](#2-vector-jacobian-productvjp) 4 | 5 | 6 | # 自动微分 7 | 8 | 9 | ## 1. 普通微分 10 | 11 | LibTorch 支持自动微分, 一般情况下可以调用 `backward()` 函数直接计算，和 python 操作基本一致。 12 | 13 | $$ 14 | \begin{aligned} 15 | & y = wx+b \\ 16 | & \frac{\partial y}{\partial w} = x 17 | \end{aligned} 18 | $$ 19 | 20 | ```cpp 21 | torch::Tensor x = torch::tensor({2.0}); 22 | torch::Tensor w = torch::tensor({3.0}, torch::requires_grad()); 23 | torch::Tensor b = torch::tensor({4.0}, torch::requires_grad()); 24 | torch::Tensor y = w * x + b; 25 | y.backward(); 26 | std::cout << w.grad() << std::endl; 27 | std::cout << b.grad() << std::endl; 28 | ``` 29 | 30 | > 注意：只有浮点和复数可以获取梯度。 31 | 32 | 已经计算的张量也是可以通过修改 `required_grad` 属性，但获取梯度需要重新执行计算，接着上面的代码： 33 | 34 | ```cpp 35 | std::cout << x.requires_grad() << std::endl; 36 | x.requires_grad_(true); 37 | std::cout << x.grad() << std::endl; 38 | y.backward(); 39 | std::cout << x.requires_grad() << std::endl; 40 | ``` 41 | 42 | > 注意：libtorch 默认不保留计算图，如果想要得到正确的结果需要第一次计算梯度时候设置相关参数，并且清除已有梯度。 43 | 44 | 45 | ## 2. vector-Jacobian product(vjp) 46 | 47 | 雅可比矩阵 $J$ 计算的是向量 $y$ 对于向量 $w$ 的导数，这里假设向量 $w=[w_1, w_2, w_3]$ 是当前某个中间层的权重，$y=[y_1, y_2, y_3]$ 由 $w$ 经过某个可导函数产生。反向传播的时候，实际的梯度向量就是本层的导数 $J$ 与上层的梯度向量 $v$ 的乘积。 48 | 49 | 对于 $y = x^2+2x$，雅可比矩阵为： 50 | $$ 51 | J=\left( 52 | \begin{array}{ccc} 53 | \frac{\partial y_{1}}{\partial x_{1}} & \frac{\partial y_{1}}{\partial x_{2}} & \frac{\partial y_{1}}{\partial x_{3}} \\ 54 | \frac{\partial y_{2}}{\partial x_{1}} & \frac{\partial y_{2}}{\partial x_{2}} & \frac{\partial y_{2}}{\partial x_{3}} \\ 55 | \frac{\partial y_{3}}{\partial x_{1}} & \frac{\partial y_{3}}{\partial x_{2}} & \frac{\partial y_{3}}{\partial x_{3}} 56 | \end{array} 57 | \right) = 58 | \left( 59 | \begin{array}{ccc} 60 | 2 x_{1}+2 & 0 & 0 \\ 61 | 0 & 2 x_{2}+2 & 0 \\ 62 | 0 & 0 & 2 x_{3}+2 63 | \end{array}\right) 64 | $$ 65 | 66 | 如果向量是$v=[1, 2, 3]$，那么实际的梯度值为： 67 | 68 | $$ 69 | vJ=[2 x_{1}+2, 4 x_{2}+4, 6 x_{3}+6] 70 | $$ 71 | 72 | 这个向量其实就可以理解为通过**链式法则**传递的上一层的梯度值，也可以理解为在投影方向，实现的时候传入 `backward()` 函数中即可。在 python 也有同样的特性，但一般各种运算算子一般都是封装好的，所以很少用到。查看下面的代码： 73 | 74 | ```cpp 75 | torch::Tensor xx = torch::randn({3}, torch::requires_grad()); 76 | torch::Tensor yy = xx * xx + 2 * xx; 77 | yy.backward(torch::tensor({1, 2, 3})); 78 | ``` 79 | -------------------------------------------------------------------------------- /chap4/BasicModels.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : LR.cpp 3 | * @Time : 2022/07/11 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | // #include "include/CNN.h" 9 | #include "include/CNN.h" 10 | #include "include/MLP.h" 11 | #include 12 | #include 13 | 14 | int main() { 15 | // 生成数据 16 | torch::Tensor w = torch::tensor({{1.0, 2.0}}); 17 | torch::Tensor x = torch::rand({20, 2}); 18 | torch::Tensor b = torch::randn({20, 1}) + 3; 19 | torch::Tensor y = x.mm(w.t()) + b; 20 | 21 | torch::Tensor img0 = torch::randn({10, 1, 28, 28}) * 100 + 100; 22 | torch::Tensor label0 = torch::zeros({10}, torch::kLong); 23 | torch::Tensor img1 = torch::randn({10, 1, 28, 28}) * 100 + 150; 24 | torch::Tensor label1 = torch::ones({10}, torch::kLong); 25 | torch::Tensor img = torch::cat({img0, img1}); 26 | torch::Tensor label = torch::cat({label0, label1}); 27 | 28 | // 线性回归 29 | std::cout << "\n============= train_lr ==============\n"; 30 | torch::nn::Linear lin(2, 1); 31 | torch::optim::SGD sgd(lin->parameters(), 0.1); 32 | for (int i = 0; i < 10; i++) { 33 | torch::Tensor y_ = lin(x); 34 | torch::Tensor loss = torch::mse_loss(y_, y); 35 | sgd.zero_grad(); 36 | loss.backward(); 37 | sgd.step(); 38 | std::cout << "Epoch " << i << " loss=" << loss.item() << std::endl; 39 | } 40 | 41 | //多层感知机 42 | std::cout << "\n============= train_mlp ==============\n"; 43 | MLP mlp(x.size(1), 4, 1); 44 | torch::optim::RMSprop rms_prop(mlp.parameters(), 0.1); 45 | for (int i = 0; i < 10; i++) { 46 | torch::Tensor y_ = mlp.forward(x); 47 | torch::Tensor loss = torch::mse_loss(y_, y); 48 | rms_prop.zero_grad(); 49 | loss.backward(); 50 | rms_prop.step(); 51 | std::cout << "Epoch " << i << " loss=" << loss.item() << std::endl; 52 | } 53 | 54 | // 卷积网络 55 | std::cout << "\n============= train_cnn ==============\n"; 56 | CNN cnn(2); 57 | torch::optim::Adam adam(cnn.parameters(), 0.01); 58 | torch::nn::CrossEntropyLoss cross_entropy; 59 | for (int i = 0; i < 10; i++) { 60 | torch::Tensor label_ = cnn.forward(img); 61 | torch::Tensor loss = cross_entropy(label_, label); 62 | adam.zero_grad(); 63 | loss.backward(); 64 | adam.step(); 65 | std::cout << "Epoch " << i << " loss=" << loss.item() << std::endl; 66 | } 67 | 68 | return 0; 69 | } 70 | -------------------------------------------------------------------------------- /chap2/python/TensorBasics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | @File : CreateTensor.py 5 | @Time : 2022/07/06 6 | @Author : Han Hui 7 | @Contact : clearhanhui@gmail.com 8 | ''' 9 | 10 | 11 | import torch 12 | 13 | # 创建张量 14 | def tensor_init(): 15 | a = torch.zeros((2,3)) 16 | b = torch.ones((2,3)) 17 | c = torch.eye(4) 18 | d = torch.full_like(a, 10) 19 | e = torch.randn((2,3)) 20 | f = torch.arange(10) 21 | g = torch.tensor([[1,2], [3,4]]) 22 | print(a) 23 | print(b) 24 | print(c) 25 | print(d) 26 | print(e) 27 | print(f) 28 | print(g) 29 | 30 | 31 | # 张量索引切片 32 | def tensor_index_slice(): 33 | a = torch.randn((2,3,4)) 34 | b = a[1] 35 | c = a[1,2,3] 36 | d = a[:, 2] 37 | e = a[:, :, ::2] 38 | f = a.index_select(-1, torch.tensor([1,1,0])) 39 | print(a) 40 | print(b) 41 | print(c) 42 | print(d) 43 | print(e) 44 | print(f) 45 | 46 | # 张量属性 47 | def tensor_attribute(): 48 | a = torch.randn(2,3) 49 | print(a.size(1)) 50 | print(a.shape) 51 | print(a[0].shape) 52 | print(a[0][0].item()) 53 | print(a.data) 54 | print(a.dtype) 55 | print(a.device) 56 | 57 | # 张量变换 58 | def tensor_transform(): 59 | a = torch.randn((2,3)) 60 | b = a.T 61 | c = a.reshape(-1, 1) 62 | d = a.view((3,2)) 63 | print(a) 64 | print(b) 65 | print(c) 66 | print(d) 67 | 68 | 69 | # 张量计算 70 | def tensor_calculate(): 71 | a = torch.ones((3,3)) 72 | b = torch.randn((3,3)) 73 | c = a.matmul(b) 74 | d = a.mul(b) 75 | e = torch.cat([a,b], 1) 76 | f = torch.stack([a,b]) 77 | print(a) 78 | print(b) 79 | print(c) 80 | print(d) 81 | print(e) 82 | print(f) 83 | 84 | # CUDA 85 | def tensor_cuda(): 86 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 87 | # a = torch.randn((3,3)).cuda() 88 | b = torch.randn((3,3)).to(device) 89 | c = torch.randn((3,3), device=device) # more efficient 90 | # print(a) 91 | print(b) 92 | print(c) 93 | 94 | 95 | if __name__ == "__main__": 96 | tensor_init() 97 | tensor_index_slice() 98 | tensor_attribute() 99 | tensor_transform() 100 | tensor_calculate() 101 | tensor_cuda() 102 | -------------------------------------------------------------------------------- /chap5/PracticeModels.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * @File : PracticeModels.cpp 3 | * @Time : 2022/07/13 4 | * @Author : Han Hui 5 | * @Contact : clearhanhui@gmail.com 6 | */ 7 | 8 | #include "include/Lenet5.h" 9 | #include 10 | 11 | int main() { 12 | /// 下面是 LibTorch 的官方注释，另外需要注意运行的目录 13 | /// c++我还没有找到一种能优雅的够获取当前绝对路径的方法 14 | /// The supplied `root` path should contain the *content* of the unzipped 15 | /// MNIST dataset, available from http://yann.lecun.com/exdb/mnist. 16 | auto train_dataset = 17 | torch::data::datasets::MNIST("../data/MNIST/raw", 18 | torch::data::datasets::MNIST::Mode::kTrain) 19 | .map(torch::data::transforms::Stack<>()); 20 | 21 | auto test_dataset = 22 | torch::data::datasets::MNIST("../data/MNIST/raw", 23 | torch::data::datasets::MNIST::Mode::kTest) 24 | .map(torch::data::transforms::Stack<>()); 25 | 26 | auto train_loader = 27 | torch::data::make_data_loader( 28 | std::move(train_dataset), 128); 29 | auto test_loader = 30 | torch::data::make_data_loader( 31 | std::move(test_dataset), 128); 32 | 33 | Lenet5 lenet5; 34 | torch::optim::Adam adam(lenet5.parameters(), 0.001); 35 | torch::nn::CrossEntropyLoss cross_entropy; 36 | for (int i = 0; i < 5; i++) { 37 | float total_loss = 0.0; 38 | for (auto &batch : *train_loader) { 39 | torch::Tensor x = batch.data; 40 | torch::Tensor y = batch.target; 41 | torch::Tensor y_prob = lenet5.forward(x); 42 | torch::Tensor loss = cross_entropy(y_prob, y); 43 | total_loss += loss.item(); 44 | adam.zero_grad(); 45 | loss.backward(); 46 | adam.step(); 47 | } 48 | std::cout << "Epoch " << i << " total_loss = " << total_loss << std::endl; 49 | } 50 | 51 | // torch::serialize::OutputArchive output_archive; 52 | // lenet5.save(output_archive); 53 | // output_archive.save_to("lenet5.pt"); 54 | lenet5.eval(); 55 | 56 | int correct = 0; 57 | int total = 0; 58 | for (auto &batch : *test_loader) { 59 | torch::Tensor x = batch.data; 60 | torch::Tensor y = batch.target; 61 | torch::Tensor y_prob = lenet5.forward(x); 62 | correct += y_prob.argmax(1).eq(y).sum().item(); 63 | total += y.size(0); 64 | } 65 | std::cout << "Test Accuracy = " << (float)correct / (float)total * 100 << " %" 66 | << std::endl; 67 | return 0; 68 | } -------------------------------------------------------------------------------- /chap0/readme.md: -------------------------------------------------------------------------------- 1 | - [LibTorch 配置](#libtorch-配置) 2 | - [1. 必要软件](#1-必要软件) 3 | - [2. 安装 PyTorch](#2-安装-pytorch) 4 | - [3. 下载解压 LibTorch](#3-下载解压-libtorch) 5 | - [4. 配置环境](#4-配置环境) 6 | - [5. 运行 demo](#5-运行-demo) 7 | 8 | # LibTorch 配置 9 | 10 | > 推荐直接使用 Docker 镜像： 11 | > ```shell 12 | > # cpu 13 | > docker pull clearhanhui/ubuntu-libtorch-cpu:latest 14 | > docker run -ti clearhanhui/ubuntu-libtorch-cpu /bin/bash 15 | > # cuda 16 | > docker pull clearhanhui/ubuntu-libtorch-cuda:latest 17 | > docker run -ti --gpus all clearhanhui/ubuntu-libtorch-cuda /bin/bash 18 | > ``` 19 | > 注意1：由于 cuda 镜像使用的较新的版本，`docker run` 的时候可以增加 `NVIDIA_DISABLE_REQUIRE=1` 来禁止版本检查（[参考](https://github.com/NVIDIA/nvidia-docker/issues/1409#issuecomment-778089784)）。 20 | > 注意2：仅学习推荐使用 cpu 版，依赖更少，文件更小。 21 | > 22 | 23 | ## 1. 必要软件 24 | 25 | * wget 26 | * unzip 27 | * python3 28 | * pip3 29 | * git 30 | * cmake 31 | * gcc (or clang) 32 | * make (or ninja) 33 | * build-essential 34 | 35 | 软件版本尽量用新的。 36 | 根据 (issue)[https://github.com/clearhanhui/LearnLibTorch/issues/1] 的提醒，可能某些 LibTorch 版本使用了C++17语法，编译的时候加上 flag `-std=c++17`。 37 | 38 | 39 | ## 2. 安装 PyTorch 40 | 41 | ```bash 42 | pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu 43 | ``` 44 | 推荐使用 [miniconda](https://docs.conda.io/en/latest/miniconda.html) 管理 python 环境。 45 | 46 | 47 | ## 3. 下载解压 LibTorch 48 | 49 | ```bash 50 | wget https://download.pytorch.org/libtorch/nightly/cpu/libtorch-shared-with-deps-latest.zip 51 | unzip libtorch-shared-with-deps-latest.zip 52 | ``` 53 | 54 | > 注意1：本教程适用 Linux 系统，Windows 等我有时间，MacOS 等我有钱。 55 | 56 | > 注意2：以上均是 CPU 版本，有 GPU 并配置了 CUDA 的土豪移步[官网](https://pytorch.org/get-started/locally/)根据版本下载。 57 | 58 | 59 | ## 4. 配置环境 60 | 61 | 四种方式（影响范围由大到小）： 62 | 1. 将解压目录 `/path/to/libtorch`（注意替换）添加到系统 `PATH` 变量中： 63 | 64 | ```bash 65 | # 临时使用 66 | export PATH=/path/to/libtorch:$PATH 67 | 68 | # 永久使用 69 | echo "export PATH=/path/to/libtorch:\$PATH" >> ~/.bashrc && . ~/.bashrc 70 | ``` 71 | 72 | 2. 设置环境变量 `Torch_ROOT`，方法参考上面。 73 | 3. 在 `CMakeLists.txt` 中通过 `set` 函数临时设置。 74 | 4. 执行 `cmake` 命令的时候，设置参数 `-DCMAKE_PREFIX_PATH=/path/to/libtorch`。 75 | 76 | 推荐使用第2种。 77 | 78 | 79 | ## 5. 运行 demo 80 | 81 | 下载项目 82 | ```bash 83 | git clone https://github.com/clearhanhui/LearnLibTorch.git 84 | cd LearnLibTorch/chap1/ 85 | ``` 86 | 87 | 使用 make 构建 88 | ```bash 89 | mkdir build-make && cd build-make 90 | cmake .. 91 | make 92 | ./HelloWorld 93 | ``` 94 | 95 | 使用 ninja 构建 96 | ```bash 97 | mkdir build-ninja && cd build-ninja 98 | cmake .. -G Ninja 99 | ninja -v 100 | ./HelloWorld 101 | ``` 102 | 103 | 输出： 104 | ``` 105 | 0 0 0 106 | 0 0 0 107 | [ CPUFloatType{2,3} ] 108 | 109 | Welcome to LibTorch! 110 | ``` -------------------------------------------------------------------------------- /chap7/readme.md: -------------------------------------------------------------------------------- 1 | - [PyTorch拓展](#pytorch拓展) 2 | 3 | 4 | # PyTorch拓展 5 | 这一章节介绍用 LibTorch 拓展 PyTorch 的算子或者模块，提升性能。 6 | 7 | 首先需要写一个 `gc_layer.cpp`，里面包含了用 LibTorch 写的前向传播和反向传播函数。理论上计算过程是不需要必须用 LibTorch 实现的，但是需要返回对应的输入和输出。代码如下： 8 | ```cpp 9 | torch::Tensor gc_forward(torch::Tensor a, 10 | torch::Tensor x, 11 | torch::Tensor w, 12 | torch::Tensor b) { 13 | return a.mm(x).mm(w) + b; 14 | } 15 | 16 | torch::Tensor gc_backward(torch::Tensor a, 17 | torch::Tensor x, 18 | torch::Tensor g) { 19 | return a.mm(x).t().mm(g); 20 | } 21 | ``` 22 | 23 | 然后需要使用 PyBind11 的一个包，需要提前安装，安装过程可以参考[这个文档](https://pybind11.readthedocs.io/en/stable/installing.html)。在上面的文件下面写下面的几行代码： 24 | ```cpp 25 | PYBIND11_MODULE(gc_cpp, m) { 26 | m.def("forward", &gc_forward, "gc forward"); 27 | m.def("backward", &gc_backward, "gc backward"); 28 | } 29 | ``` 30 | `PYBIND11_MODULE` 的第一个参数 `gc_cpp` 是我们后面需要 `import` 的包名，`m` 是一个 `library` 对象，`def` 的三个参数分别是暴露给 python 的函数名，函数的指针，和描述。关于宏 `PYBIND11_MODULE` 的详细文档可以看[官网描述](https://pybind11.readthedocs.io/en/stable/reference.html#c.PYBIND11_MODULE)。 31 | > 提一句，这个宏其实帮我们做了很多事情，如果不使用 PyBind11 的话，可以参考这两个文档：[链接1](https://docs.python.org/3/extending/extending.html)，[链接2](https://docs.microsoft.com/zh-cn/visualstudio/python/working-with-c-cpp-python-in-visual-studio?view=vs-2022#use-cpython-extensions)。当然这个除了封装函数，像类、属性、等都可以封装，这其实利用了就是 python 一切皆对象的特性。 32 | 33 | 然后在同一个目录下创建 `setup.py` ，最简单的只需要这几行代码（对我提供的代码做了简化）： 34 | ```python 35 | from setuptools import setup 36 | from torch.utils.cpp_extension import CppExtension, BuildExtension 37 | setup( 38 | name='gc_cpp', 39 | ext_modules=[CppExtension('gc_cpp', ['gc_layer.cpp'])], 40 | cmdclass={'build_ext': BuildExtension} 41 | ) 42 | ``` 43 | 这里需要特别注意，CppExtension 中的 `gc_cpp` 必须要和 `PYBIND11_MODULE` 中保持一致，而 `setup` 函数的第一个名字并不需要，这是安装包的名字。如果我们不想维护两个名字可以在 c++ 的代码中使用 `TORCH_EXTENSION_NAME` 宏代替 `gc_cpp`，前提是使用 `BuildExtension`，他会帮我们在 c++ 的世界里创建一个系统变量。此外 `setup` 还有很多其他的参数，例如可以指定作者和版本信息等。`CppExtension` 是 PyTorch 中提供的继承了 `setuptools.Extension` 的一个类，为我们节省了一些工作。 44 | 45 | 然后执行下面的命令就可以在当前 python 环境中安装我们的 `gc_cpp` 包了。 46 | ```bash 47 | python3 setup.py install 48 | ``` 49 | 可以通过下面的命令测试安装是否成功 50 | ```bash 51 | python3 -c "import torch; import gc_cpp" 52 | ``` 53 | > 注意：上述过程虽然编写代码引用到了 LibTorch，但是安装和运行并不依赖 LibTorch。 54 | 55 | 简单做了一个测试： 56 | * PyTorch: 27.52 s 57 | * Py_ext: 88.47 s 58 | * Cpp_ext: 78.84 s 59 | 60 | 我们的 c++ 拓展虽然比用 python 写的拓展速度快一些，但是和直接使用 PyTorch 的相比还是查了很多，我猜测可能 PyTorch 本身优化就很好，使用 c++ 写的拓展反而增加了加载动态链接库和数据从 python 转换到 c++ 的开销。 61 | 62 | 我还提供了一个 `CMakeLists.txt` 文件，可以使用 `cmake` 构建出和 `setup.py` 同样的动态链接库，并且也可以创建为 c++ 使用的库。 63 | 64 | 关于类和注册算子的拓展，总体上逻辑差不多，可以去查看 PyTorch 官网文档（[注册算子](https://pytorch.org/tutorials/advanced/torch_script_custom_ops.html)，[注册类](https://pytorch.org/tutorials/advanced/torch_script_custom_classes.html)），和上面的区别在于注册的方式，我们可以直接在 PyTorch 中使用，而不需要额外导入一个包。 -------------------------------------------------------------------------------- /chap2/readme.md: -------------------------------------------------------------------------------- 1 | - [张量基础](#张量基础) 2 | - [1. 创建张量](#1-创建张量) 3 | - [2. 张量索引和切片](#2-张量索引和切片) 4 | - [3. 张量属性](#3-张量属性) 5 | - [4. 张量变换](#4-张量变换) 6 | - [5. 张量计算](#5-张量计算) 7 | - [6. CUDA](#6-cuda) 8 | 9 | # 张量基础 10 | 11 | `include` 目录中包含了各种头文件，每个头文件对一类操作。 12 | `python` 目录中有对应的 python 脚本。 13 | 14 | ## 1. 创建张量 15 | 16 | 大部分接口的用法和 PyTorch 相比可以说极为相似，唯一需要注意就是用 `{}` 代替 `()` 作为参数 `size` 的输入。 17 | 18 | ```cpp 19 | torch::Tensor a = torch::zeros({2, 3}); 20 | torch::Tensor b = torch::ones({2, 3}); 21 | torch::Tensor c = torch::eye(3); 22 | torch::Tensor d = torch::full_like(a, 10); 23 | torch::Tensor e = torch::randn({2, 3}); 24 | torch::Tensor f = torch::arange(10); 25 | torch::Tensor g = torch::tensor({{1, 2}, {3, 4}}); 26 | ``` 27 | 28 | 29 | ## 2. 张量索引和切片 30 | 张量支持运算符 `[]` 但是不能如 `[i,j,k]` 这样使用。如果想要更加灵活的切片需要使用 `torch::indexing` 空间中的相关类如 `Slice`。`Slice(None)` 类似 Numpy 中的 `:` 或 `...` 操作符，表示所有。 31 | 32 | ```cpp 33 | torch::Tensor a = torch::randn({2, 3, 4}); 34 | torch::Tensor b = a[1]; 35 | torch::Tensor c = a.index({1, 2, 3}); // 等同 a[1][2][3] 36 | torch::Tensor d = a.index({Slice(None), 2}); // 等同 a.index({"...", 2}) 37 | torch::Tensor e = a.index({Slice(None), Slice(None), Slice(None, None, 2)}); 38 | torch::Tensor f = a.index_select(-1, torch::tensor({1, 1, 0})); 39 | ``` 40 | 41 | 42 | ## 3. 张量属性 43 | 44 | 获取张量属性的方法，除了 `shape` 属性（或者 `size()` 函数）变为了 `sizes()` 函数，其余常用的基本上和 PyTorch 一致。 45 | 46 | ```cpp 47 | torch::Tensor a = torch::randn({2, 3}); 48 | std::cout << a.size(1) << std::endl; 49 | std::cout << a.sizes() << std::endl; 50 | std::cout << a[0].sizes() << std::endl; 51 | std::cout << a[0][0].item() << std::endl; 52 | std::cout << a.data() << std::endl; 53 | std::cout << a.dtype() << std::endl; 54 | std::cout << a.device() << std::endl; 55 | ``` 56 | 57 | 58 | ## 4. 张量变换 59 | 60 | 不支持形如 `.T` 的操作，另外 `transpose()` 函数的用法也与 PyTorch 稍有区别，但是都支持 `.t()`。 61 | 62 | ```cpp 63 | torch::Tensor a = torch::randn({2, 3}); 64 | torch::Tensor b = a.transpose(0, 1); 65 | torch::Tensor c = a.reshape({-1, 1}); 66 | torch::Tensor d = a.view({3, 2}); 67 | torch::Tensor e = a.toType(torch::kFloat32); 68 | ``` 69 | 70 | 71 | ## 5. 张量计算 72 | 73 | 各种常见的 API 用法基本与 PyTorch 一致。 74 | 75 | ```cpp 76 | torch::Tensor a = torch::ones({3, 3}); 77 | torch::Tensor b = torch::randn({3, 3}); 78 | torch::Tensor c = a.matmul(b); 79 | torch::Tensor d = a.mul(b); 80 | torch::Tensor e = torch::cat({a, b}, 0); 81 | torch::Tensor f = torch::stack({a, b}); 82 | ``` 83 | 84 | 85 | ## 6. CUDA 86 | 87 | 注意 `torch::randn` 的默认参数是 `size` 和 `options`，但是 `TensorOptions` 类支持从 `ScalarType`、`Device` 的隐式转换，所以这里可以编译通过，并不是重载了有 `device` 参数的函数。 88 | 89 | ```cpp 90 | torch::Device device = torch::Device(torch::kCPU); 91 | if (torch::cuda::is_available()) { 92 | device = torch::Device(torch::kCUDA); 93 | } 94 | torch::Tensor a = torch::randn({3, 3}).cuda(); 95 | torch::Tensor b = torch::randn({3, 3}).to(device); 96 | torch::Tensor c = torch::randn({3, 3}, device); 97 | ``` 98 | 99 | ----------- 100 | 101 | LibTorch 大部分 API 和 PyTorch 命名和使用逻辑都基本一致，稍加熟悉就可以了。 -------------------------------------------------------------------------------- /chap5/python/utils.py: -------------------------------------------------------------------------------- 1 | # This utils script is from [pygcn](https://github.com/tkipf/pygcn) 2 | 3 | import numpy as np 4 | import scipy.sparse as sp 5 | import torch 6 | 7 | 8 | def encode_onehot(labels): 9 | classes = set(labels) 10 | classes_dict = {c: np.identity(len(classes))[i, :] for i, c in 11 | enumerate(classes)} 12 | labels_onehot = np.array(list(map(classes_dict.get, labels)), 13 | dtype=np.int32) 14 | return labels_onehot 15 | 16 | 17 | def load_data(path="../data/cora/", dataset="cora"): 18 | """Load citation network dataset (cora only for now)""" 19 | print('Loading {} dataset...'.format(dataset)) 20 | 21 | idx_features_labels = np.genfromtxt("{}{}.content".format(path, dataset), 22 | dtype=np.dtype(str)) 23 | features = sp.csr_matrix(idx_features_labels[:, 1:-1], dtype=np.float32) 24 | labels = encode_onehot(idx_features_labels[:, -1]) 25 | 26 | # build graph 27 | idx = np.array(idx_features_labels[:, 0], dtype=np.int32) 28 | idx_map = {j: i for i, j in enumerate(idx)} 29 | edges_unordered = np.genfromtxt("{}{}.cites".format(path, dataset), 30 | dtype=np.int32) 31 | edges = np.array(list(map(idx_map.get, edges_unordered.flatten())), 32 | dtype=np.int32).reshape(edges_unordered.shape) 33 | adj = sp.coo_matrix((np.ones(edges.shape[0]), (edges[:, 0], edges[:, 1])), 34 | shape=(labels.shape[0], labels.shape[0]), 35 | dtype=np.float32) 36 | 37 | # build symmetric adjacency matrix 38 | adj = adj + adj.T.multiply(adj.T > adj) - adj.multiply(adj.T > adj) 39 | 40 | features = normalize(features) 41 | adj = normalize(adj + sp.eye(adj.shape[0])) 42 | 43 | idx_train = range(140) 44 | idx_val = range(200, 500) 45 | idx_test = range(500, 1500) 46 | 47 | features = torch.FloatTensor(np.array(features.todense())) 48 | labels = torch.LongTensor(np.where(labels)[1]) 49 | adj = sparse_mx_to_torch_sparse_tensor(adj) 50 | 51 | idx_train = torch.LongTensor(idx_train) 52 | idx_val = torch.LongTensor(idx_val) 53 | idx_test = torch.LongTensor(idx_test) 54 | 55 | return adj, features, labels, idx_train, idx_val, idx_test 56 | 57 | 58 | def normalize(mx): 59 | """Row-normalize sparse matrix""" 60 | rowsum = np.array(mx.sum(1)) 61 | r_inv = np.power(rowsum, -1).flatten() 62 | r_inv[np.isinf(r_inv)] = 0. 63 | r_mat_inv = sp.diags(r_inv) 64 | mx = r_mat_inv.dot(mx) 65 | return mx 66 | 67 | 68 | def accuracy(output, labels): 69 | preds = output.max(1)[1].type_as(labels) 70 | correct = preds.eq(labels).double() 71 | correct = correct.sum() 72 | return (correct / len(labels)).item() 73 | 74 | 75 | def sparse_mx_to_torch_sparse_tensor(sparse_mx): 76 | """Convert a scipy sparse matrix to a torch sparse tensor.""" 77 | sparse_mx = sparse_mx.tocoo().astype(np.float32) 78 | indices = torch.from_numpy( 79 | np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64)) 80 | values = torch.from_numpy(sparse_mx.data) 81 | shape = torch.Size(sparse_mx.shape) 82 | return torch.sparse.FloatTensor(indices, values, shape) -------------------------------------------------------------------------------- /chap4/python/BasicModels.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | @File : BasicModels.py 5 | @Time : 2022/07/07 6 | @Author : Han Hui 7 | @Contact : clearhanhui@gmail.com 8 | ''' 9 | 10 | import torch 11 | 12 | # 生成线性数据 13 | w = torch.tensor([[1.0, 2.0]]) 14 | x = torch.rand((20, 2)) 15 | b = torch.randn((20, 1)) + 3 16 | y = x.mm(w.t()) + b 17 | 18 | # 生成图像数据 19 | img0 = torch.randn((10, 1, 28, 28)) * 100 + 100 20 | label0 = torch.zeros(10, dtype=torch.long) 21 | img1 = torch.randn((10, 1, 28, 28)) * 100 + 150 22 | label1 = torch.ones(10, dtype=torch.long) 23 | img = torch.cat([img0, img1])# .clamp(0, 255) 24 | label = torch.cat([label0, label1]) 25 | 26 | 27 | # 线性回归 28 | def train_lr(x, y): 29 | lin = torch.nn.Linear(2, 1) 30 | loss_fn = torch.nn.MSELoss() 31 | sgd = torch.optim.SGD(lin.parameters(), lr=0.1) 32 | 33 | for i in range(10): 34 | y_ = lin(x) 35 | loss = loss_fn(y_, y) 36 | sgd.zero_grad() 37 | loss.backward() 38 | sgd.step() 39 | print("Epoch [{:0>2d}] loss={:.4f}".format(i, loss.item())) 40 | 41 | 42 | # 非线性激活的多层感知机，三层 43 | def train_mlp(x, y): 44 | class MLP(torch.nn.Module): 45 | def __init__(self, in_dim, hidden_dim, out_dim): 46 | super().__init__() 47 | self.lin1 = torch.nn.Linear(in_dim, hidden_dim) 48 | self.lin2 = torch.nn.Linear(hidden_dim, hidden_dim) 49 | self.lin3 = torch.nn.Linear(hidden_dim, out_dim) 50 | self.relu = torch.nn.ReLU() 51 | 52 | def forward(self, x): 53 | x = self.lin1(x) 54 | x = self.relu(x) 55 | x = self.lin2(x) 56 | x = self.relu(x) 57 | x = self.lin3(x) 58 | return x 59 | 60 | mlp = MLP(x.shape[1], 4, 1) 61 | rms_prop = torch.optim.RMSprop(mlp.parameters(), lr=0.1) 62 | loss_fn = torch.nn.MSELoss() 63 | 64 | for i in range(10): 65 | y_ = mlp(x) 66 | loss = loss_fn(y_, y) 67 | rms_prop.zero_grad() 68 | loss.backward() 69 | rms_prop.step() 70 | print("Epoch [{:0>2d}] loss={:.4f}".format(i, loss.item())) 71 | 72 | 73 | # 2层 卷积网络 74 | def train_cnn(img, label): 75 | class CNN(torch.nn.Module): 76 | def __init__(self, num_classes): 77 | super().__init__() 78 | 79 | self.conv1 = torch.nn.Conv2d(1, 16, 3, padding=1) 80 | self.conv2 = torch.nn.Conv2d(16, 16, 3, padding=1) 81 | self.bn = torch.nn.BatchNorm2d(16) 82 | self.max_pool = torch.nn.MaxPool2d(2) 83 | self.relu = torch.nn.ReLU() 84 | self.lin = torch.nn.Linear(7*7*16, num_classes) 85 | 86 | def forward(self, x): 87 | x = self.conv1(x) 88 | x = self.bn(x) 89 | x = self.relu(x) 90 | x = self.max_pool(x) 91 | 92 | x = self.conv2(x) 93 | x = self.bn(x) 94 | x = self.relu(x) 95 | x = self.max_pool(x) 96 | 97 | x = self.lin(x.reshape(x.size(0), -1)) 98 | return x 99 | 100 | cnn = CNN(2) 101 | loss_fn = torch.nn.CrossEntropyLoss() 102 | adam = torch.optim.Adam(cnn.parameters(), lr=0.01) 103 | 104 | for i in range(10): 105 | label_ = cnn(img) 106 | loss = loss_fn(label_, label) 107 | adam.zero_grad() 108 | loss.backward() 109 | adam.step() 110 | print("Epoch [{:0>2d}] loss={:.4f}".format(i, loss.item())) 111 | 112 | 113 | if __name__ == "__main__": 114 | print("\n============= train_lr ==============") 115 | train_lr(x, y) 116 | print("\n============= train_mlp ==============") 117 | train_mlp(x, y) 118 | # print("\n============ train_mlpp ==============") 119 | # train_mlp(img.reshape(20, -1), label.float()) 120 | print("\n============= train_cnn ==============") 121 | train_cnn(img, label) 122 | -------------------------------------------------------------------------------- /chap7/python/extend.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | @File : extend.py 5 | @Time : 2022/07/21 6 | @Author : Han Hui 7 | @Contact : clearhanhui@gmail.com 8 | ''' 9 | 10 | 11 | import torch 12 | import torch.nn as nn 13 | from torch.autograd.function import Function 14 | try: 15 | import gc_cpp # must after `import torch` 16 | except: 17 | print("\nModule gc_cpp not installed.\n") 18 | 19 | class AddSelf(Function): 20 | @staticmethod 21 | def forward(ctx, x, n=1): 22 | ctx.n = n 23 | return torch.mul(x, n) 24 | 25 | @staticmethod 26 | def backward(ctx, gx): 27 | return gx * ctx.n 28 | 29 | 30 | class LinearFunction(Function): 31 | @staticmethod 32 | def forward(ctx, input, weight, bias=None): 33 | ctx.save_for_backward(input, weight, bias) 34 | output = input.mm(weight) 35 | if bias is not None: 36 | output += bias.unsqueeze(0).expand_as(output) 37 | return output 38 | 39 | @staticmethod 40 | def backward(ctx, grad_output): 41 | input, weight, bias = ctx.saved_tensors 42 | grad_input = grad_weight = grad_bias = None 43 | 44 | if ctx.needs_input_grad[0]: 45 | grad_input = grad_output.mm(weight.t()) 46 | if ctx.needs_input_grad[1]: 47 | grad_weight = input.t().mm(grad_output) 48 | if bias is not None and ctx.needs_input_grad[2]: 49 | grad_bias = grad_output.sum(0) 50 | 51 | return grad_input, grad_weight, grad_bias 52 | 53 | 54 | class Linear(nn.Module): 55 | def __init__(self, in_features, out_features, bias=True): 56 | super(Linear, self).__init__() 57 | self.in_features = in_features 58 | self.out_features = out_features 59 | 60 | self.weight = nn.Parameter( 61 | torch.empty(in_features, out_features)) 62 | if bias: 63 | self.bias = nn.Parameter(torch.empty(out_features)) 64 | else: 65 | self.register_parameter('bias', None) 66 | 67 | nn.init.uniform_(self.weight, -0.1, 0.1) 68 | if self.bias is not None: 69 | nn.init.uniform_(self.bias, -0.1, 0.1) 70 | 71 | def forward(self, input): 72 | return LinearFunction.apply(input, self.weight, self.bias) 73 | 74 | def extra_repr(self): 75 | return 'in_features={}, out_features={}, bias={}'.format( 76 | self.in_features, self.out_features, self.bias is not None 77 | ) 78 | 79 | 80 | class GCNLayerFunction(Function): 81 | @staticmethod 82 | def forward(ctx, a, x, w, b, use_cpp=False): 83 | ctx.save_for_backward(a, x, w, b) 84 | ctx.use_cpp = use_cpp 85 | output = gc_cpp.forward(a,x,w,b) if use_cpp else a.mm(x).mm(w) + b 86 | return output 87 | 88 | @staticmethod 89 | def backward(ctx, grad_output): 90 | a, x, w, b = ctx.saved_tensors 91 | grad_a = grad_x = grad_w = grad_b = grad_use_cpp = None 92 | grad_w = gc_cpp.backward(a,x, grad_output) if ctx.use_cpp else a.mm(x).t().mm(grad_output) 93 | grad_b = grad_output.sum(0) 94 | return grad_a, grad_x, grad_w, grad_b, grad_use_cpp 95 | 96 | 97 | class GCNLayer(nn.Module): 98 | def __init__(self, in_features, out_features, bias=True): 99 | super().__init__() 100 | a = torch.rand((2,3)) 101 | self.weight = nn.Parameter( 102 | torch.empty(in_features, out_features)) 103 | if bias: 104 | self.bias = nn.Parameter(torch.empty(out_features)) 105 | else: 106 | self.register_parameter('bias', None) 107 | 108 | nn.init.uniform_(self.weight, -0.1, 0.1) 109 | if self.bias is not None: 110 | nn.init.uniform_(self.bias, -0.1, 0.1) 111 | 112 | def forward(self, a, x, use_cpp=False): 113 | return GCNLayerFunction.apply(a, x, self.weight, self.bias, use_cpp) 114 | -------------------------------------------------------------------------------- /chap5/python/PracticeModels.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | @File : Lenet5MNIST.py 5 | @Time : 2022/07/13 6 | @Author : Han Hui 7 | @Contact : clearhanhui@gmail.com 8 | ''' 9 | 10 | 11 | import sys 12 | import math 13 | import torch 14 | import torch.nn as nn 15 | import torch.nn.functional as F 16 | import torchvision 17 | import torchvision.transforms as transforms 18 | from torch.nn.parameter import Parameter 19 | from utils import load_data, accuracy 20 | 21 | 22 | class LeNet5(nn.Module): 23 | def __init__(self): 24 | super(LeNet5, self).__init__() 25 | self.conv1 = nn.Conv2d(1, 6, (5, 5), padding=2) 26 | self.conv2 = nn.Conv2d(6, 16, (5, 5)) 27 | self.fc1 = torch.nn.Linear(16*5*5, 120) 28 | self.fc2 = torch.nn.Linear(120, 84) 29 | self.fc3 = torch.nn.Linear(84, 10) 30 | 31 | def forward(self, x): 32 | x = F.max_pool2d(F.relu(self.conv1(x)), 2) 33 | x = F.max_pool2d(F.relu(self.conv2(x)), 2) 34 | x = x.reshape((x.size(0), -1)) 35 | x = F.relu(self.fc1(x)) 36 | x = F.relu(self.fc2(x)) 37 | x = self.fc3(x) 38 | return x 39 | 40 | 41 | def train_lenet5(): 42 | datapath = sys.path[0] + "/../data/" 43 | train_dataset = torchvision.datasets.MNIST(root=datapath, 44 | train=True, 45 | transform=transforms.ToTensor(), 46 | download=True) 47 | 48 | test_dataset = torchvision.datasets.MNIST(root=datapath, 49 | train=False, 50 | transform=transforms.ToTensor(), 51 | download=True) 52 | 53 | train_loader = torch.utils.data.DataLoader(dataset=train_dataset, 54 | batch_size=128, 55 | shuffle=True) 56 | 57 | test_loader = torch.utils.data.DataLoader(dataset=test_dataset, 58 | batch_size=128, 59 | shuffle=False) 60 | 61 | lenet5 = LeNet5() 62 | loss_fn = nn.CrossEntropyLoss() 63 | optimizer = torch.optim.Adam(lenet5.parameters(), lr=0.001) 64 | 65 | for i in range(5): 66 | total_loss = 0 67 | for x, y in train_loader: 68 | y_prob = lenet5(x) 69 | loss = loss_fn(y_prob, y) 70 | total_loss += loss.item() 71 | optimizer.zero_grad() 72 | loss.backward() 73 | optimizer.step() 74 | 75 | print("Epoch [{:0>2d}] total_loss = {:.4f}".format(i, total_loss)) 76 | 77 | # torch.save(lenet5.state_dict(), 'lenet5.pt') 78 | # lenet5.load_state_dict(torch.load('lenet5.pt')) 79 | lenet5.eval() 80 | 81 | correct = 0 82 | total = 0 83 | for x, y in test_loader: 84 | y_prob = lenet5(x) 85 | y_ = y_prob.argmax(1) 86 | correct += (y_ == y).sum().item() 87 | total += y.size(0) 88 | 89 | print('Test Accuracy = {:.2f} %'.format(100 * correct / total)) 90 | 91 | 92 | class GCNLayer(nn.Module): 93 | def __init__(self, in_features, out_features): 94 | super().__init__() 95 | w = torch.empty(in_features, out_features) 96 | b = torch.empty(out_features) 97 | stdv = 1. / math.sqrt(out_features) 98 | w.uniform_(-stdv, stdv) 99 | b.uniform_(-stdv, stdv) 100 | self.w = Parameter(w) 101 | self.b = Parameter(b) 102 | # self.w = Parameter(nn.init.trunc_normal_( 103 | # torch.empty(in_features, out_features), std=0.05)) 104 | # self.b = Parameter(nn.init.trunc_normal_( 105 | # torch.empty(out_features), std=0.05)) 106 | 107 | def forward(self, x, a): 108 | out = torch.spmm(a, torch.mm(x, self.w)) + self.b 109 | # x = a@self.w@x + self.b 110 | return out 111 | 112 | 113 | class GCN(nn.Module): 114 | def __init__(self): 115 | super().__init__() 116 | self.gc1 = GCNLayer(1433, 16) 117 | self.gc2 = GCNLayer(16, 7) 118 | self.dropout = nn.Dropout(0.5) 119 | 120 | def forward(self, x, a): 121 | x = F.relu(self.gc1(x, a)) 122 | x = self.dropout(x) 123 | x = self.gc2(x, a) 124 | return x 125 | 126 | 127 | def train_gcn(): 128 | datapath = sys.path[0] + "/../data/cora/" 129 | a, x, y, idx_train, idx_val, idx_test = load_data(path=datapath) 130 | gcn = GCN() 131 | loss_fn = torch.nn.CrossEntropyLoss() 132 | adam = torch.optim.Adam(gcn.parameters(), lr=0.01, weight_decay=5e-4) 133 | 134 | for i in range(200): 135 | y_prob = gcn(x, a) 136 | loss = loss_fn(y_prob[idx_train], y[idx_train]) 137 | adam.zero_grad() 138 | loss.backward() 139 | adam.step() 140 | print("Epoch [{:0>3d}] loss={:.4f} train_acc={:.2f}% val_acc={:.2f}%" 141 | .format(i, loss.item(), 100*accuracy(y_prob[idx_train], y[idx_train]), 142 | 100*accuracy(y_prob[idx_val], y[idx_val]))) 143 | y_prob = gcn(x, a) 144 | print("test_acc={:.2f}%".format( 145 | 100*accuracy(y_prob[idx_test], y[idx_test]))) 146 | 147 | 148 | if __name__ == "__main__": 149 | train_lenet5() 150 | train_gcn() 151 | -------------------------------------------------------------------------------- /chap5/readme.md: -------------------------------------------------------------------------------- 1 | - [模型实践](#模型实践) 2 | - [1. LeNet-5](#1-lenet-5) 3 | - [2. GCN](#2-gcn) 4 | 5 | # 模型实践 6 | 这里复现两个模型，分别是 Lecun Yann 的 LeNet-5 和 Thomas Kipf 的 GCN 。 7 | 8 | 9 | ## 1. LeNet-5 10 | 11 | [LeNet-5](https://en.wikipedia.org/wiki/LeNet) 是一个经典的卷积网络，定义了许多卷积神经网络的经典结构（距今快30年了），包含了两层卷积层，两层激活层和三层线形层，框架如下 12 | 13 | ![LeNet-5](../pics/chap5-LeNet-5.png) 14 | 15 | 我们只需要将 chap4 中的 CNN 模型稍作修改即可，完整代码可以在 [include](./include/) 目录查看，这里给出部分关键代码 16 | 17 | ```cpp 18 | // 构建函数 19 | Lenet5::Lenet5(){ 20 | conv1 = torch::nn::Conv2d(torch::nn::Conv2dOptions(1, 6, 5).padding(2)); 21 | conv2 = torch::nn::Conv2d(torch::nn::Conv2dOptions(6, 16, 5)); 22 | relu = torch::nn::ReLU(); 23 | max_pool = torch::nn::MaxPool2d(2); 24 | lin1 = torch::nn::Linear(5 * 5 * 16, 120); 25 | lin2 = torch::nn::Linear(120, 84); 26 | lin3 = torch::nn::Linear(84, 10); 27 | } 28 | 29 | // 前向传播 30 | torch::Tensor Lenet5::forward(torch::Tensor x){ 31 | x = max_pool(relu(conv1(x))); 32 | x = max_pool(relu(conv2(x))); 33 | x = x.reshape({x.size(0), -1}); 34 | x = lin1(x); 35 | x = lin2(x); 36 | x = lin3(x); 37 | return x; 38 | } 39 | ``` 40 | 41 | 定义好模型之后，需要准备 [MNIST](http://yann.lecun.com/exdb/mnist/) 数据集，在官网上下载四个压缩包并解压，将解压后的文件存放在 `${workspace}/chap5/data/MNIST/raw` 中。或者可以运行 [python 脚本](./python/PracticeModels.py) 的前45行，PyTorch 的 API 提供了 `download` 参数可以直接下载，但是 LibTorch 没有此选项，只能手动下载。 42 | 43 | 准备好之后，利用 `MNIST` 数据集类加载，第一个参数是路径，第二个参数是训练or测试数据。 44 | 45 | ```cpp 46 | auto train_dataset = 47 | torch::data::datasets::MNIST("../data/MNIST/raw", 48 | torch::data::datasets::MNIST::Mode::kTrain) 49 | .map(torch::data::transforms::Stack<>()); 50 | 51 | auto test_dataset = 52 | torch::data::datasets::MNIST("../data/MNIST/raw", 53 | torch::data::datasets::MNIST::Mode::kTest) 54 | .map(torch::data::transforms::Stack<>()); 55 | ``` 56 | 57 | `map()` 函数执行的功能是根据输入的转换类型进行格式转换，返回一个 `MapDataset`。对于 `Stack` 结构官网的解释是 58 | 59 | ```cpp 60 | /// A `Collation` for `Example` types that stacks all data 61 | /// tensors into one tensor, and all target (label) tensors into one tensor. 62 | ``` 63 | 64 | 个人理解这里是的意思是一种打包类型，将 `data` 和 `target` 分别合并，并通过 `Example` 结构模板向外提供获取接口。 65 | 66 | 然后通过 `make_data_loader` 模板函数创建 dataloader ，对训练数据采用 `RandomSampler` 以给模型引入噪声避免标签泄露和过拟合，对测试数据采用 `SequentialSampler` 提升数据访问性能。 67 | 68 | ```cpp 69 | auto train_loader = 70 | torch::data::make_data_loader( 71 | std::move(train_dataset), 128); 72 | auto test_loader = 73 | torch::data::make_data_loader( 74 | std::move(test_dataset), 128); 75 | ``` 76 | 77 | 训练过程和 chap4 中基本一致，但增加了内循环，以进行数据集分批的遍历，并计算总损失 `total_loss`。 78 | 79 | ```cpp 80 | for (int i = 0; i < 5; i++) { 81 | torch::Tensor total_loss = torch::tensor(0.0); 82 | for (auto &batch : *train_loader) { 83 | torch::Tensor x = batch.data; 84 | torch::Tensor y = batch.target; 85 | torch::Tensor y_prob = lenet5.forward(x); 86 | torch::Tensor loss = cross_entropy(y_prob, y); 87 | total_loss += loss; 88 | adam.zero_grad(); 89 | loss.backward(); 90 | adam.step(); 91 | } 92 | std::cout << "Epoch " << i << " total_loss = " << total_loss.item() << std::endl; 93 | } 94 | ``` 95 | 96 | 随便跑一次的结果，注意看时间对比： 97 | 98 | | | LibTorch | PyTorch | 99 | | ------------- | --------- | -------- | 100 | | Test Accuracy | 98.56 % | 98.64 % | 101 | | Total Loss | 21.33 | 20.3501 | 102 | | Time Duration | 0m36.685s | 1m7.996s | 103 | 104 | （请忽略我电脑垃圾的性能🤣） 105 | 106 | 107 | ## 2. GCN 108 | 109 | > 由于目前 LibTorch 或者 PyTorch 并没有为图数据集提供支持，GCN 的 c++ 版本目前只完成了模型部分，数据集加载部分和训练部分我还没有实现，等哪天来填坑。 110 | 111 | GCN 是在图数据上进行卷积的模型，也是采用了局部卷积的方式，即对相邻的节点卷积，但是和 CNN 不同，GCN 的滤波核是一个固定的低通稍微带一点高通的滤波核（证明可以看 SGC 那篇文章，最近还有一篇讲为什么 GCN 在异配图表现也不错的文章也可以看）。模型的架构如下，有两层卷积和两层非线性变换，但是作者实现的时候并没有第二层 ReLU。 112 | 113 | ![gcn](https://github.com/tkipf/pygcn/raw/master/figure.png) 114 | 115 | 我们首先需要实现 `GCNLayer` 类，但是为了和 LibTorch 用法统一，我们需要实现 `GCNLayerImpl` 并通过 `TORCH_MODULE` 函数注册（或者 封装？），在使用的时候，便可以直接使用 `GCNLayer`，用法和 `Linear` 等相同。头文件中需要这样写，注意最后一行。 116 | 117 | ```cpp 118 | class GCNLayerImpl : public nn::Module { 119 | public: 120 | 121 | GCNLayerImpl(int in_features, int out_features); 122 | torch::Tensor forward(torch::Tensor x, torch::Tensor a); 123 | 124 | private: 125 | torch::Tensor w; 126 | torch::Tensor b; 127 | }; 128 | TORCH_MODULE(GCNLayer); // 注册 129 | ``` 130 | 131 | 然后需要实现构建函数和前向传播函数，需要注意两点，一点是 `sqrt` 函数是在 `cmath` 头文件中提供，另一点是 `LibTorch` 目前似乎还没有提供稀疏矩阵的支持，这里就先用 `mm` 了。 132 | 133 | ```cpp 134 | GCNLayerImpl::GCNLayerImpl(int in_features, int out_features){ 135 | w = torch::randn((in_features, out_features)); 136 | b = torch::randn((out_features)); 137 | w.requires_grad_(); 138 | b.requires_grad_(); 139 | float dev = sqrt(out_features); 140 | torch::nn::init::uniform_(w, -dev, dev); 141 | torch::nn::init::uniform_(b, -dev, dev); 142 | register_parameter("w", w); 143 | register_parameter("b", b); 144 | } 145 | 146 | torch::Tensor GCNLayerImpl::forward(torch::Tensor x, torch::Tensor a){ 147 | torch::Tensor out = torch::mm(a, torch::mm(x, w)); 148 | return out; 149 | } 150 | ``` 151 | 152 | 这样我们就能像使用 `Linear` 或者 `Conv2d` 一样使用 `GCNLayer` 了，关于 GCN 模型的实现，这里就不展开说了，如果仔细看了前面的教程，其实套路都一样的，相关的代码在 [`include`](./include/) 目录中可以直接查看 153 | 154 | 155 | ------------------------- 156 | 157 | 到这里，得益于 LibTorch 良好的设计，其实会你发现 LibTorch 和 PyTorch 似乎没什么区别，甚至在一些语法上是相同的，而且在网上也都能找到类似博客或者教程。 158 | 159 | 后面的内容每章内容比较独立，我也在学习。 160 | -------------------------------------------------------------------------------- /chap8/readme.md: -------------------------------------------------------------------------------- 1 | - [自动微分引擎demo](#自动微分引擎demo) 2 | - [预备知识](#预备知识) 3 | - [原型实现（python）](#原型实现python) 4 | - [1. 数据结构](#1-数据结构) 5 | - [2. 计算节点](#2-计算节点) 6 | - [3. 计算引擎](#3-计算引擎) 7 | - [原型实现（c++）](#原型实现c) 8 | 9 | # 自动微分引擎demo 10 | 11 | 这一章是手写自动微分引擎完成自动微分。其实这一章本来不在我的计划中，但是最近工作中遇到了一点构建自动微分引擎需求，所以突然打算在这里写一个原型。 12 | 13 | 14 | ## 预备知识 15 | 16 | 首先一定要明白的是自动微分的原理——链式法则和 vjp，链式法则可以在网上找到 N 多的博客，vjp 在[第三章](../chap3/readme.md#2-vector-jacobian-productvjp)也有简单介绍，简单来说就是输出梯度为输入梯度和雅各比矩阵的乘积。 17 | 18 | 关于自动微分引擎，通过我的调研找到两种原型，一种将张量当作计算图中的节点，另一种将算子当作计算图中的节点。前者很容易理解，计算图的连边是算子，如 `Sum`， `Mul` 等。可能是受到 陈天奇 大佬的 [CSE 599W](http://dlsys.cs.washington.edu)（[课程github](https://github.com/dlsys-course)）影响，在网上很多流行的 demo 都是类似的实现，在他的[第一个作业](https://github.com/dlsys-course/assignment1-2018)中，采用的就是这种方法。这里给出一个例子，对于 $A$，$B$，$C$，完成如下计算： 19 | $$ 20 | \begin{aligned} 21 | &D = (A+B)\cdot C \\ 22 | &E = D (A+C) 23 | \end{aligned} 24 | $$ 25 | （注意 $D$，$E$ 分别是哈达玛积和矩阵乘法）计算图可以用下面的方式表示： 26 | ![compute-graph-1](../pics/chap5-compute-graph1.png) 27 | 28 | 对于这种方式实现的原型可以参考[这个博客](https://zhuanlan.zhihu.com/p/161635270)，我的实现也参考了他的部分代码。 29 | 30 | 这种实现中虽然容易理解，实现起来也不困那，但是这种原型中张量（或者数据）承担了部分维护上下文的功能，使得张量的功能太臃肿，其实最主要是不满足我的需求。我实现的是另一种方式，即将加减乘除等算子当作节点，可以看下面的计算图： 31 | ![compute-graph-2](../pics/chap5-compute-graph2.png) 32 | 33 | 这种实现有一个很好的特性，上下文的维护可以独立于张量，很方便拓展和后续开发，如多线程。（我猜的，因为PyTorch底层的逻辑就是采用的第二种，每个节点正好能对应一次反向传播计算） 34 | 35 | 36 | ## 原型实现（python） 37 | 38 | ### 1. 数据结构 39 | 首先要做的事情就是定义一个数据结构代替张量，用 python 的话就很简单了，直接集成 `numpy.ndarray` 给他增加一个 grad 属性。为了尽量维持父类的特征，所以直接增加了一个 `__setattr__` 方法： 40 | ```python 41 | class MyArray(np.ndarray): 42 | """ 43 | Same with `np.ndarray` but `__setattr__` fucntion. 44 | """ 45 | def __setattr__(self, __name, __value): 46 | self.__dict__[__name] = __value 47 | ``` 48 | 因为 `numpy.ndarray` 的构造函数接受一个 `shape` 参数返回一个零 array。我写了一个丑陋的函数，通过 `requires_grad` 表示是否需要计算计梯度。 49 | ```python 50 | def make_array(input_list, requires_grad=False): 51 | nparray = np.array(input_list) 52 | myarray = MyArray(nparray.shape) 53 | myarray[:] = nparray[:] 54 | if requires_grad: 55 | myarray.requires_grad = True 56 | return myarray 57 | ``` 58 | 59 | ### 2. 计算节点 60 | 然后定义了一个计算节点基类，这里省略了部分注释和部分未实现的函数。先介绍每个变量的意义，`_edges` 计算节点的输入节点，即计算图的中边的“逆边”。`_depends` 是一个整形变量，用来标识当前计算节点被多少下游节点依赖，也即计算图的出边。`_inputs` 和 `_outputs` 分别保存了计算节点的输入和输出。两个 `_save` 函数分别用来保存输入和输出。 61 | 62 | ```python 63 | class NodeBase: 64 | def __init__(self): 65 | self._edges = [] 66 | self._depends = 0 67 | self._inputs = [] 68 | self._outputs = [] 69 | 70 | def __call__(self, *args, **kwds): 71 | return self.forward(*args, **kwds) 72 | 73 | def _save_inputs(self, *args): 74 | for a in args: 75 | self._inputs.append(a) 76 | if not hasattr(a, "requires_grad"): 77 | self._edges.append(a.fn) 78 | 79 | def _save_outputs(self, *args): 80 | for a in args: 81 | self._outputs.append(a) 82 | ``` 83 | 具体实现以哈达玛积 `Mul` 为例，继承 `NodeBase` 类并实现 `forward` 和 `_backward` 函数。需要注意 `forward` 中除了计算结果还需要保存输入，输出和当前计算节点。 `_backward` 中根据 vjp 实现对应的功能即可。这里可以延申一下，如上图中存在两个 `Add` 计算节点都对应了变量 `A`，PyTorch 采用了 拓扑排序+累加梯度 的策略，也就是对两个节点的计算出来 `A` 的梯度求和，这也就是为什么在使用 PyTorch 的时候需要执行 `optmizer.zero_grad()`，否则梯度值会在上一个 epoch 的基础上继续累积。 84 | ```python 85 | class MulOP(NodeBase): 86 | def __init__(self): 87 | super().__init__() 88 | 89 | def forward(self, a, b): 90 | self._save_inputs(a, b) 91 | output = a * b 92 | output.fn = self 93 | self._save_outputs(output) 94 | return output 95 | 96 | def _backward(self): 97 | if not hasattr(self._inputs[0], "grad"): 98 | self._inputs[0].grad = 0.0 99 | self._inputs[0].grad += self._outputs[0].grad * self._inputs[1] 100 | if not hasattr(self._inputs[1], "grad"): 101 | self._inputs[1].grad = 0.0 102 | self._inputs[1].grad += self._outputs[0].grad * self._inputs[0] 103 | ``` 104 | `_forward` 中还将当前节点的计算 `function` 给保存了下来，这一点是仿照的 PyTorch，形如注意 `b` 的输出，包含两项： 105 | ```bash 106 | >>> a = torch.ones(2, requires_grad=True) 107 | >>> b = a*a 108 | >>> b 109 | tensor([1., 1.], grad_fn=) 110 | ``` 111 | 然后在 `_save_inputs` 中将输入的结果保存，这一点在 PyTorch 源代码中也有体现，但是 PyTorch 源代码中构建了一个 `Edge` 类来抽象这个行为，这里简化了。对于 PyTorch 源码感兴趣可以查看[罗西大佬的博客](https://www.cnblogs.com/rossiXYZ/p/15421453.html)。 112 | 113 | ### 3. 计算引擎 114 | 完成了以上基础类的实现之后，我们需要实现一个计算引擎，来控制协调整个反向计算过程。以上面提到的计算过程为例，反向执行的过程分别是 `MatMul`，`Mul`，`Add`，`Add`。如何要让机器完成上述过程呢，其实就是拓扑排序，因为计算图必定是一个有向无环图，而且反向计算依赖必定是下游计算节点，所以可以使用拓扑排序，从目标节点出发，根据拓扑次序依次完成反向计算。所以我们的计算引擎只需要完成一个拓扑排序并按照拓扑顺序依次执行计算节点的 `_backward` 函数即可。 115 | 116 | 关键代码如下，省略了部分检查和注释。初始化的时候保存了一个拓扑序列，使用 dfs 算法生成改拓扑顺序（对于反向计算图来说是拓扑逆序），同时没遇到一个节点就将其 `_depends` 加 1，除了根节点为 0。在反向计算函数中如果当前计算节点的 `_depends` 为 0，则说明该节点可以被执行，并将依赖该节点的 `_depends` 减 1。 117 | 118 | ```python 119 | class BackwardEngine: 120 | def __init__(self, arr): 121 | self._target_arr = arr 122 | self._root = arr.fn 123 | self.topo_queue = [] 124 | self._topological_sorting() 125 | 126 | def _topological_sorting(self): 127 | def _dfs(node): 128 | for next_node in node._edges: 129 | _dfs(next_node) 130 | node._depends += 1 131 | self.topo_queue.append(node) 132 | _dfs(self._root) 133 | self.root._depends = 0 134 | 135 | def run_backward(self, gradient=None): 136 | while len(self.topo_queue): 137 | task_node = self.topo_queue.pop() 138 | if task_node._depends == 0: 139 | task_node._backward() 140 | for n in task_node._edges: 141 | n._depends -= 1 142 | ``` 143 | 144 | 145 | ## 原型实现（c++） 146 | （留坑） 147 | -------------------------------------------------------------------------------- /chap8/python/AutoGrad.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | @File : AutoGrad.py 5 | @Time : 2022/07/27 6 | @Author : Han Hui 7 | @Contact : clearhanhui@gmail.com 8 | ''' 9 | 10 | 11 | import numpy as np 12 | 13 | 14 | class MyArray(np.ndarray): 15 | """ 16 | Same with `np.ndarray` but `__setattr__` fucntion. 17 | """ 18 | def __setattr__(self, __name, __value): 19 | self.__dict__[__name] = __value 20 | 21 | 22 | def make_array(input_list, requires_grad=False): 23 | nparray = np.array(input_list) 24 | myarray = MyArray(nparray.shape) 25 | myarray[:] = nparray[:] 26 | if requires_grad: 27 | myarray.requires_grad = True 28 | return myarray 29 | 30 | 31 | class NodeBase: 32 | """ 33 | Base class of Function 34 | """ 35 | 36 | def __init__(self): 37 | """ 38 | initialize the function. 39 | """ 40 | self._edges = [] 41 | self._depends = 0 42 | self._inputs = [] 43 | self._outputs = [] 44 | 45 | def __call__(self, *args, **kwds): 46 | return self.forward(*args, **kwds) 47 | 48 | def _backward(self): 49 | """ 50 | calculate the gradient of inputs. 51 | this is local. 52 | """ 53 | raise NotImplementedError 54 | 55 | def forward(self, inputs): 56 | """ 57 | forward propagation 58 | """ 59 | raise NotImplementedError 60 | 61 | def _save_inputs(self, *args): 62 | for a in args: 63 | self._inputs.append(a) 64 | if not hasattr(a, "requires_grad"): 65 | self._edges.append(a.fn) 66 | 67 | def _save_outputs(self, *args): 68 | for a in args: 69 | self._outputs.append(a) 70 | 71 | 72 | class MulOP(NodeBase): 73 | def __init__(self): 74 | super().__init__() 75 | 76 | def forward(self, a, b): 77 | self._save_inputs(a, b) 78 | output = a * b 79 | output.fn = self 80 | self._save_outputs(output) 81 | return output 82 | 83 | def _backward(self): 84 | if not hasattr(self._inputs[0], "grad"): 85 | self._inputs[0].grad = 0.0 86 | self._inputs[0].grad += self._outputs[0].grad * self._inputs[1] 87 | if not hasattr(self._inputs[1], "grad"): 88 | self._inputs[1].grad = 0.0 89 | self._inputs[1].grad += self._outputs[0].grad * self._inputs[0] 90 | 91 | 92 | class MatMulOP(NodeBase): 93 | def __init__(self): 94 | super().__init__() 95 | 96 | def forward(self, a, b): 97 | self._save_inputs(a, b) 98 | output = a @ b 99 | output.fn = self 100 | self._save_outputs(output) 101 | return output 102 | 103 | def _backward(self): 104 | if not hasattr(self._inputs[0], "grad"): 105 | self._inputs[0].grad = np.zeros( 106 | self._inputs[0].shape, dtype=np.float32) 107 | self._inputs[0].grad += self._outputs[0].grad @ self._inputs[1].T 108 | if not hasattr(self._inputs[1], "grad"): 109 | self._inputs[1].grad = np.zeros( 110 | self._inputs[1].shape, dtype=np.float32) 111 | self._inputs[1].grad += self._inputs[0].T @ self._outputs[0].grad 112 | 113 | 114 | class AddOP(NodeBase): 115 | def __init__(self): 116 | super().__init__() 117 | 118 | def forward(self, a, b): 119 | self._save_inputs(a, b) 120 | output = a + b 121 | output.fn = self 122 | self._save_outputs(output) 123 | return output 124 | 125 | def _backward(self): 126 | if not hasattr(self._inputs[0], "grad"): 127 | self._inputs[0].grad = 0.0 128 | self._inputs[0].grad += self._outputs[0].grad 129 | if not hasattr(self._inputs[1], "grad"): 130 | self._inputs[1].grad = 0.0 131 | self._inputs[1].grad += self._outputs[0].grad 132 | 133 | 134 | class ExpOP(NodeBase): 135 | def __init__(self): 136 | super().__init__() 137 | 138 | def forward(self, x): 139 | pass 140 | 141 | def _backward(self): 142 | pass 143 | 144 | 145 | class BackwardEngine: 146 | """ 147 | Single thread backward. 148 | """ 149 | 150 | def __init__(self, arr): 151 | self._target_arr = arr 152 | self._root = arr.fn 153 | self.topo_queue = [] 154 | self._topological_sorting() 155 | 156 | def _topological_sorting(self): 157 | def _dfs(node): 158 | for next_node in node._edges: 159 | _dfs(next_node) 160 | node._depends += 1 161 | self.topo_queue.append(node) 162 | _dfs(self._root) 163 | self._root._depends = 0 164 | 165 | def run_backward(self, gradient=None): 166 | # Following check codes are simulated to PyTorch 167 | if not hasattr(self._root, "grad"): 168 | if isinstance(gradient, np.ndarray): 169 | assert gradient.shape == self._target_arr.shape 170 | self._target_arr.grad = gradient 171 | elif gradient is None: 172 | if self._target_arr.size == 1: 173 | self._target_arr.grad = np.ones_like(self._target_arr) 174 | else: 175 | raise RuntimeError( 176 | "grad can be implicitly created only for scalar outputs") 177 | else: 178 | raise TypeError("gradients can be either `np.ndarray` or `None`, but got `{}`" 179 | .format(type(gradient).__name__)) 180 | else: 181 | raise Exception("root node cannot has grad attribute") 182 | 183 | 184 | while len(self.topo_queue): 185 | task_node = self.topo_queue.pop() 186 | if task_node._depends == 0: 187 | task_node._backward() 188 | for n in task_node._edges: 189 | n._depends -= 1 190 | 191 | 192 | # Wrap again, every time call the func will create an NODE instance. 193 | def add(a, b): 194 | op = AddOP() 195 | return op(a, b) 196 | 197 | 198 | def mul(a, b): 199 | op = MulOP() 200 | return op(a, b) 201 | 202 | 203 | def mm(a, b): 204 | op = MatMulOP() 205 | return op(a, b) 206 | -------------------------------------------------------------------------------- /chap4/readme.md: -------------------------------------------------------------------------------- 1 | - [基本模型](#基本模型) 2 | - [1. 数据准备](#1-数据准备) 3 | - [2. 线性回归](#2-线性回归) 4 | - [3. 多层感知机](#3-多层感知机) 5 | - [4. 卷积网络](#4-卷积网络) 6 | - [5. 长短时记忆网络](#5-长短时记忆网络) 7 | 8 | 9 | # 基本模型 10 | 11 | 这一章介绍几种使用 LibTorch 和 C++ 实现的基础模型，其实大部分的操作和 PyTorch 相比来说都是很相似的。 12 | 13 | > torch::nn::Module 的第一句注释：The design and implementation of this class is largely based on the Python API. 14 | 15 | 16 | ## 1. 数据准备 17 | 生成两个数据集，每个数据集各20个样本。第一个是一个带有噪声的线性分布数据集，然后分别应用[线性回归](#2-线性回归)和[多层感知机](#3-多层感知机)拟合这一条曲线，其实后者添加了非线性激活，更适合拟合非线性曲线。 18 | 19 | ``` cpp 20 | torch::Tensor w = torch::tensor({{1.0, 2.0}}); 21 | torch::Tensor x = torch::rand({20, 2}); 22 | torch::Tensor b = torch::randn({20, 1}) + 3; 23 | torch::Tensor y = x.mm(w.t()) + b; 24 | ``` 25 | 26 | 第二个数据集是图像数据集，在[卷积网络](#4-卷积网络)中应用，这里数据仿照的是 [MNIST](http://yann.lecun.com/exdb/mnist/) 将数据维度设定为 [1, 28, 28] 。 27 | 28 | ```cpp 29 | torch::Tensor img0 = torch::randn({10, 1, 28, 28}) * 100 + 100; 30 | torch::Tensor label0 = torch::zeros({10}, torch::kLong); 31 | torch::Tensor img1 = torch::randn({10, 1, 28, 28}) * 100 + 150; 32 | torch::Tensor label1 = torch::ones({10}, torch::kLong); 33 | torch::Tensor img = torch::cat({img0, img1}); 34 | torch::Tensor label = torch::cat({label0, label1}); 35 | ``` 36 | 37 | > 注意1：保证数据类型正确。 38 | 39 | 40 | ## 2. 线性回归 41 | 高中知识，最小二乘法可以求精确解，这里采用梯度下降法拟合。如果采用 Sigmoid 激活，并将激活更换为负对数损失函数，就变成了逻辑回归模型。由于模型很简单，这里可以直接应用 `torch::nn::Linear`，下面是采用 `MSE` 损失函数和 `SGD` 优化器。 42 | 43 | ```cpp 44 | torch::nn::Linear lin(2, 1); 45 | torch::optim::SGD sgd(lin->parameters(), 0.1); 46 | for (int i = 0; i < 10; i++) { 47 | torch::Tensor y_ = lin(x); 48 | torch::Tensor loss = torch::mse_loss(y_, y); 49 | sgd.zero_grad(); 50 | loss.backward(); 51 | sgd.step(); 52 | std::cout << "Epoch " << i << " loss=" << loss.item() << std::endl; 53 | } 54 | ``` 55 | 56 | 在默认情况下 `torch::nn::Linear` 附带 `bias`，如果不需要可以用对应的 `(ModuleName)Options` 类，在卷积网络中也有同样的设置。 57 | 58 | ```cpp 59 | torch::nn::Linear lin_no_bias(torch::nn::LinearOptions(2,1).bias(false)); 60 | ``` 61 | 62 | > 注意2： `torch::nn::Linear` 会声明**一个经由封装过的指针，而不是对象**，所以语法上调用 `forward()` 或者 `parameters()` 函数的时候用 `->` 而不是 `.` ，在 [include](./include/) 目录中也实现了对应的 LR 类，可以应用 `.` 调用相关方法。 63 | 64 | > 注意3：在 LibTorch 中可以像 PyTorch 语法那样直接通过形如 `lin(x)` 方式前向传播（仅限 LibTorch 原生定义的模块）。PyTorch 中是因为在 `__call__()` 函数中调用了 `forward()` ,LibTorch 是因为重写了运算符 `()` 相关内容在 TORCH_MODULE_IMPL 和 ModuleHolder 中。 65 | 66 | 67 | ## 3. 多层感知机 68 | 69 | 多层感知机 MLP 的结构设计如下： 70 | 71 | ``` 72 | (lin1): Linear(2, 4) 73 | (relu): ReLU() 74 | (lin2): Linear(4, 4) 75 | (relu): ReLU() 76 | (lin3): Linear(4, 1) 77 | ``` 78 | 79 | 在 [`include`](./include/) 目录下创建两个文件 `MLP.h` 和 `MLP.cpp`，创建 `MLP` 类并实现构建函数和 `forward()`（其实套路和 PyTorch 也差不多）。 80 | 81 | ```cpp 82 | // MLP.h 83 | class MLP : public torch::nn::Module { 84 | public: 85 | MLP(int in_dim, int hidden_dim,int out_dim); 86 | torch::Tensor forward(torch::Tensor x); 87 | 88 | private: 89 | torch::nn::Linear lin1{nullptr}; 90 | torch::nn::Linear lin2{nullptr}; 91 | torch::nn::Linear lin3{nullptr}; 92 | }; 93 | ``` 94 | 95 | 头文件中声明类名，继承自 `torch::nn::Module`，并在类内的 `public` 关键字下声明构建函数和 `forward()`，`private` 关键字下面声明三个空指针。 96 | 97 | ```cpp 98 | // MLP.cpp 99 | MLP::MLP(int in_dim, int hidden_dim, int out_dim) { 100 | lin1 = torch::nn::Linear(in_dim, hidden_dim); 101 | lin2 = torch::nn::Linear(hidden_dim, hidden_dim); 102 | lin3 = torch::nn::Linear(hidden_dim, out_dim); 103 | 104 | register_module("lin1", lin1); 105 | register_module("lin2", lin2); 106 | register_module("lin3", lin3); 107 | }; 108 | 109 | torch::Tensor MLP::forward(torch::Tensor x) { 110 | x = lin1(x); 111 | x = torch::relu(x); 112 | x = lin2(x); 113 | x = torch::relu(x); 114 | x = lin3(x); 115 | return x; 116 | } 117 | ``` 118 | 119 | 源文件分别实现两个函数。`register_module` 会为创建 键-值 对以供可以嵌套调用。PyTorch 中不需要显示的声明是因为 python 的类天然支持通过 键-值 的方式查找内部对象。 120 | 121 | 此外，为了能在目录外部调用子文件夹的内容，工程上需要在 include 目录下创建一个 `CMakeLists.txt`。将需要的源文件添加到 `libchap4` 库中。 122 | 123 | ```cmake 124 | # chap4/include/CMakeLists.txt 125 | add_library(libchap4 MLP.cpp CNN.cpp) 126 | target_link_libraries(libchap4 ${TORCH_LIBRARIES}) 127 | ``` 128 | 129 | 主目录下面的 `CMakeLists.txt` 也需要做一些修改。 130 | ```cmake 131 | # chap4/CMakeLists.txt 132 | cmake_minimum_required(VERSION 3.21) 133 | project(BasicModels) 134 | 135 | find_package(Torch REQUIRED) 136 | add_subdirectory(include) 137 | add_executable(BasicModels BasicModels.cpp) 138 | target_link_libraries(BasicModels ${TORCH_LIBRARIES} libchap4) 139 | ``` 140 | 141 | 训练过程基本和 [2.](#2-线性回归) 中一致。 142 | 143 | 144 | ## 4. 卷积网络 145 | 146 | 卷积网络 CNN 的结构设计如下， 主要包含两层 3 $\times$ 3 卷积层和一层线性变换层。每层卷积层之后一次执行归一、池化、激活的操作，由于他们没有科学系参数，只需定义一个即可，也可以用 `torch::nn::functional` 中对应的函数。 147 | 148 | ``` 149 | (conv1): Conv2d(1, 16, kernel_size=(3, 3), padding=(1, 1)) 150 | (bn): BatchNorm2d(16) 151 | (max_pool): MaxPool2d(2) 152 | (relu): ReLU() 153 | (conv2): Conv2d(16, 16, kernel_size=(3, 3), padding=(1, 1)) 154 | (bn): BatchNorm2d(16) 155 | (max_pool): MaxPool2d(2) 156 | (relu): ReLU() 157 | (lin): Linear(783, 2) 158 | ``` 159 | 160 | 同样创建两个文件 `CNN.h` 和 `CNN.cpp` 。 161 | 162 | ```cpp 163 | // CNN.h 164 | class CNN : public torch::nn::Module { 165 | public: 166 | CNN(int num_classes); 167 | torch::Tensor forward(torch::Tensor x); 168 | 169 | private: 170 | torch::nn::Conv2d conv1{nullptr}; 171 | torch::nn::Conv2d conv2{nullptr}; 172 | torch::nn::ReLU relu{nullptr}; 173 | torch::nn::MaxPool2d max_pool{nullptr}; 174 | torch::nn::BatchNorm2d bn{nullptr}; 175 | torch::nn::Linear lin{nullptr}; 176 | }; 177 | ``` 178 | 179 | ```cpp 180 | // CNN.cpp 181 | CNN::CNN(int num_classes) { 182 | conv1 = torch::nn::Conv2d(torch::nn::Conv2dOptions(1, 16, 3).padding(1)); 183 | conv2 = torch::nn::Conv2d(torch::nn::Conv2dOptions(16, 16, 3).padding(1)); 184 | bn = torch::nn::BatchNorm2d(16); 185 | relu = torch::nn::ReLU(); 186 | max_pool = torch::nn::MaxPool2d(2); 187 | lin = torch::nn::Linear(7 * 7 * 16, num_classes); 188 | 189 | register_module("conv1", conv1); 190 | register_module("conv2", conv2); 191 | register_module("bn", bn); 192 | register_module("relu", relu); 193 | register_module("max_pool", max_pool); 194 | register_module("lin", lin); 195 | } 196 | 197 | torch::Tensor CNN::forward(torch::Tensor x) { 198 | x = conv1(x); 199 | x = bn(x); 200 | x = relu(x); 201 | x = max_pool2d(x, 2); 202 | 203 | x = conv2(x); 204 | x = bn(x); 205 | x = relu(x); 206 | x = max_pool2d(x, 2); 207 | x = lin(x.reshape({x.size(0), -1})); 208 | 209 | return x; 210 | } 211 | ``` 212 | 213 | 卷积类需要通过 `Conv2dOptions` 设置 `padding` 参数，默认值是 0。 `BatchNorm2d` 的 `stride` 参数默认会和 `kernel_size` 保持一致。 214 | 215 | > 注意4：在 LibTorch 和 PyTorch 中，不同损失函数对输入数据要求不同，如 `MSE` 一般可以交换 `target` 和 `input` ，虽然留了自动广播机制，但要求输入维度匹配且都是浮点型。`CrossEntropy` 要求 `target` 是长整型，如果输入 `target` 维度是 1，会自动生成对应的 onehot 张量。 216 | 217 | 218 | ## 5. 长短时记忆网络 219 | （留坑） 220 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test Every Chapter 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | env: 10 | BUILD_TYPE: Release 11 | 12 | jobs: 13 | chap1: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | - name: Install essential lib 18 | run: | 19 | wget -q https://download.pytorch.org/libtorch/nightly/cpu/libtorch-shared-with-deps-latest.zip 20 | unzip -q libtorch-shared-with-deps-latest.zip 21 | sudo mv libtorch / 22 | export Torch_ROOT=/libtorch 23 | python3 -m pip -q install torch torchvision 24 | 25 | - name: Configure CMake 26 | run: | 27 | cmake -B ${{github.workspace}}/chap1/build -S ${{github.workspace}}/chap1 -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_PREFIX_PATH=/libtorch 28 | 29 | - name: Build 30 | run: | 31 | cmake --build ${{github.workspace}}/chap1/build --config ${{env.BUILD_TYPE}} 32 | 33 | - name: Run python 34 | run: | 35 | python3 ${{github.workspace}}/chap1/python/HelloWorld.py 36 | 37 | - name: Run cpp 38 | run: | 39 | ${{github.workspace}}/chap1/build/HelloWorld 40 | 41 | 42 | chap2: 43 | runs-on: ubuntu-latest 44 | steps: 45 | - uses: actions/checkout@v3 46 | - name: Install essential lib 47 | run: | 48 | wget -q https://download.pytorch.org/libtorch/nightly/cpu/libtorch-shared-with-deps-latest.zip 49 | unzip -q libtorch-shared-with-deps-latest.zip 50 | sudo mv libtorch / 51 | export Torch_ROOT=/libtorch 52 | python3 -m pip -q install torch torchvision 53 | 54 | - name: Configure CMake 55 | run: | 56 | cmake -B ${{github.workspace}}/chap2/build -S ${{github.workspace}}/chap2 -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_PREFIX_PATH=/libtorch 57 | 58 | - name: Build 59 | run: | 60 | cmake --build ${{github.workspace}}/chap2/build --config ${{env.BUILD_TYPE}} 61 | 62 | - name: Run python 63 | run: | 64 | python3 ${{github.workspace}}/chap2/python/TensorBasics.py 65 | 66 | - name: Run cpp 67 | run: | 68 | ${{github.workspace}}/chap2/build/TensorBasics 69 | 70 | 71 | chap3: 72 | runs-on: ubuntu-latest 73 | steps: 74 | - uses: actions/checkout@v3 75 | - name: Install essential lib 76 | run: | 77 | wget -q https://download.pytorch.org/libtorch/nightly/cpu/libtorch-shared-with-deps-latest.zip 78 | unzip -q libtorch-shared-with-deps-latest.zip 79 | sudo mv libtorch / 80 | export Torch_ROOT=/libtorch 81 | python3 -m pip -q install torch torchvision 82 | 83 | - name: Configure CMake 84 | run: | 85 | cmake -B ${{github.workspace}}/chap3/build -S ${{github.workspace}}/chap3 -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_PREFIX_PATH=/libtorch 86 | 87 | - name: Build 88 | run: | 89 | cmake --build ${{github.workspace}}/chap3/build --config ${{env.BUILD_TYPE}} 90 | 91 | - name: Run python 92 | run: | 93 | python3 ${{github.workspace}}/chap3/python/AutoGrad.py 94 | 95 | - name: Run cpp 96 | run: | 97 | ${{github.workspace}}/chap3/build/AutoGrad 98 | 99 | 100 | chap4: 101 | runs-on: ubuntu-latest 102 | steps: 103 | - uses: actions/checkout@v3 104 | - name: Install essential lib 105 | run: | 106 | wget -q https://download.pytorch.org/libtorch/nightly/cpu/libtorch-shared-with-deps-latest.zip 107 | unzip -q libtorch-shared-with-deps-latest.zip 108 | sudo mv libtorch / 109 | export Torch_ROOT=/libtorch 110 | python3 -m pip -q install torch torchvision 111 | 112 | - name: Configure CMake 113 | run: | 114 | cmake -B ${{github.workspace}}/chap4/build -S ${{github.workspace}}/chap4 -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_PREFIX_PATH=/libtorch 115 | 116 | - name: Build 117 | run: | 118 | cmake --build ${{github.workspace}}/chap4/build --config ${{env.BUILD_TYPE}} 119 | 120 | - name: Run python 121 | run: | 122 | python3 ${{github.workspace}}/chap4/python/BasicModels.py 123 | 124 | - name: Run cpp 125 | run: | 126 | ${{github.workspace}}/chap4/build/BasicModels 127 | 128 | 129 | chap5: 130 | runs-on: ubuntu-latest 131 | steps: 132 | - uses: actions/checkout@v3 133 | - name: Install essential lib 134 | run: | 135 | wget -q https://download.pytorch.org/libtorch/nightly/cpu/libtorch-shared-with-deps-latest.zip 136 | unzip -q libtorch-shared-with-deps-latest.zip 137 | sudo mv libtorch / 138 | export Torch_ROOT=/libtorch 139 | python3 -m pip -q install torch torchvision scipy 140 | cd ${{github.workspace}}/chap5/data 141 | bash prepare_cora.sh 142 | bash prepare_mnist.sh 143 | cd ${{github.workspace}} 144 | 145 | - name: Configure CMake 146 | run: | 147 | cmake -B ${{github.workspace}}/chap5/build -S ${{github.workspace}}/chap5 -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_PREFIX_PATH=/libtorch 148 | 149 | - name: Build 150 | run: | 151 | cmake --build ${{github.workspace}}/chap5/build --config ${{env.BUILD_TYPE}} 152 | 153 | - name: Run python 154 | run: | 155 | python3 ${{github.workspace}}/chap5/python/PracticeModels.py 156 | 157 | - name: Run cpp 158 | run: | 159 | cd ${{github.workspace}}/chap5/build/ 160 | ./PracticeModels 161 | cd ${{github.workspace}} 162 | 163 | 164 | chap6: 165 | runs-on: ubuntu-latest 166 | steps: 167 | - uses: actions/checkout@v3 168 | - name: Install essential lib 169 | run: | 170 | wget -q https://download.pytorch.org/libtorch/nightly/cpu/libtorch-shared-with-deps-latest.zip 171 | unzip -q libtorch-shared-with-deps-latest.zip 172 | sudo mv libtorch / 173 | export Torch_ROOT=/libtorch 174 | python3 -m pip -q install torch torchvision 175 | 176 | - name: Configure CMake 177 | run: | 178 | cmake -B ${{github.workspace}}/chap6/build -S ${{github.workspace}}/chap6 -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_PREFIX_PATH=/libtorch 179 | 180 | - name: Build 181 | run: | 182 | cmake --build ${{github.workspace}}/chap6/build --config ${{env.BUILD_TYPE}} 183 | 184 | - name: Run python 185 | run: | 186 | python3 ${{github.workspace}}/chap6/python/export_model.py 187 | 188 | - name: Run cpp 189 | run: | 190 | cd ${{github.workspace}}/chap6/build/ 191 | ./TorchScript 192 | cd ${{github.workspace}} 193 | 194 | chap7: 195 | runs-on: ubuntu-latest 196 | steps: 197 | - uses: actions/checkout@v3 198 | - name: Install essential lib 199 | run: | 200 | wget -q https://download.pytorch.org/libtorch/nightly/cpu/libtorch-shared-with-deps-latest.zip 201 | unzip -q libtorch-shared-with-deps-latest.zip 202 | sudo mv libtorch / 203 | export Torch_ROOT=/libtorch 204 | sudo python3 -m pip install torch torchvision 205 | 206 | - name: Run python 207 | run: | 208 | cd ${{github.workspace}}/chap7/csrc 209 | sudo python3 setup.py install 210 | cd .. 211 | python3 check_profile.py 212 | 213 | 214 | chap8: 215 | runs-on: ubuntu-latest 216 | steps: 217 | - uses: actions/checkout@v3 218 | - name: Install essential lib 219 | run: | 220 | wget -q https://download.pytorch.org/libtorch/nightly/cpu/libtorch-shared-with-deps-latest.zip 221 | unzip -q libtorch-shared-with-deps-latest.zip 222 | sudo mv libtorch / 223 | export Torch_ROOT=/libtorch 224 | python3 -m pip -q install torch torchvision 225 | 226 | - name: Run python 227 | run: | 228 | cd ${{github.workspace}}/chap8 229 | python3 check.py 230 | 231 | 232 | --------------------------------------------------------------------------------