├── .clang-format ├── .gitignore ├── .gitmodules ├── .pre-commit-config.yaml ├── CMakeLists.txt ├── README.md ├── doc └── images │ └── computation_graph.png └── src ├── CMakeLists.txt ├── backward.cc ├── example ├── CMakeLists.txt ├── cifar │ ├── CMakeLists.txt │ ├── create_cifar_recordio.py │ ├── fluid_cifar_benchmark.py │ └── test_cifar.cc └── mnist │ ├── CMakeLists.txt │ ├── create_mnist_recordio.py │ └── test_mnist.cc ├── function.cc ├── function.h ├── optimizer.cc ├── optimizer.h ├── parameter.cc ├── parameter.h ├── tape.cc ├── tape.h ├── test_backward.cc ├── test_parameter.cc ├── test_tape.cc └── variable.cc /.clang-format: -------------------------------------------------------------------------------- 1 | # This file is used by clang-format to autoformat paddle source code 2 | # 3 | # The clang-format is part of llvm toolchain. 4 | # It need to install llvm and clang to format source code style. 5 | # 6 | # The basic usage is, 7 | # clang-format -i -style=file PATH/TO/SOURCE/CODE 8 | # 9 | # The -style=file implicit use ".clang-format" file located in one of 10 | # parent directory. 11 | # The -i means inplace change. 12 | # 13 | # The document of clang-format is 14 | # http://clang.llvm.org/docs/ClangFormat.html 15 | # http://clang.llvm.org/docs/ClangFormatStyleOptions.html 16 | --- 17 | Language: Cpp 18 | BasedOnStyle: Google 19 | IndentWidth: 2 20 | TabWidth: 2 21 | ContinuationIndentWidth: 4 22 | AccessModifierOffset: -1 # The private/protected/public has no indent in class 23 | Standard: Cpp11 24 | AllowAllParametersOfDeclarationOnNextLine: true 25 | BinPackParameters: false 26 | BinPackArguments: false 27 | ... 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | *~ -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "paddle"] 2 | path = paddle 3 | url = https://github.com/kexinzhao/Paddle 4 | branch = tape_submodule 5 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/Lucas-C/pre-commit-hooks.git 3 | sha: v1.0.1 4 | hooks: 5 | - id: remove-crlf 6 | files: (?!.*third_party)^.*$ | (?!.*book)^.*$ 7 | - repo: https://github.com/PaddlePaddle/mirrors-yapf.git 8 | sha: 0d79c0c469bab64f7229c9aca2b1186ef47f0e37 9 | hooks: 10 | - id: yapf 11 | files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$ 12 | - repo: https://github.com/pre-commit/pre-commit-hooks 13 | sha: 5bf6c09bfa1297d3692cadd621ef95f1284e33c0 14 | hooks: 15 | - id: check-added-large-files 16 | - id: check-merge-conflict 17 | - id: check-symlinks 18 | - id: detect-private-key 19 | files: (?!.*third_party)^.*$ | (?!.*book)^.*$ 20 | - id: end-of-file-fixer 21 | - repo: local 22 | hooks: 23 | - id: clang-format-with-version-check 24 | name: clang-format 25 | description: Format files with ClangFormat. 26 | entry: bash ./paddle/tools/codestyle/clang_format.hook -i 27 | language: system 28 | files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto)$ 29 | - repo: local 30 | hooks: 31 | - id: cpplint-cpp-source 32 | name: cpplint 33 | description: Check C++ code style using cpplint.py. 34 | entry: bash ./paddle/tools/codestyle/cpplint_pre_commit.hook 35 | language: system 36 | files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx)$ 37 | - repo: local 38 | hooks: 39 | - id: pylint-doc-string 40 | name: pylint 41 | description: Check python docstring style using docstring_checker. 42 | entry: bash ./paddle/tools/codestyle/pylint_pre_commit.hook 43 | language: system 44 | files: \.(py)$ 45 | - repo: https://github.com/PaddlePaddle/pre-commit-golang 46 | sha: 8337620115c25ff8333f1b1a493bd031049bd7c0 47 | hooks: 48 | - id: go-fmt 49 | types: 50 | - go 51 | - repo: local 52 | hooks: 53 | - id: copyright_checker 54 | name: copyright_checker 55 | entry: python ./paddle/tools/codestyle/copyright.hook 56 | language: system 57 | files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|py)$ 58 | exclude: (?!.*third_party)^.*$ | (?!.*book)^.*$ 59 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0) 2 | 3 | # CMAKE_BUILD_TYPE 4 | if(NOT CMAKE_BUILD_TYPE) 5 | set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING 6 | "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel" 7 | FORCE) 8 | endif() 9 | 10 | set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -DNDEBUG") 11 | set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g -DNDEBUG") 12 | 13 | include_directories(${CMAKE_SOURCE_DIR}) 14 | add_subdirectory(paddle) 15 | add_subdirectory(src) 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dynamic Graph on Fluid 2 | 3 | PaddlePaddle Fluid is targeting the autodiff without tape, which, however, is very 4 | challenging and we are still way from there. DyNet and PyTorch provide a good design 5 | idea, the *tape*, that significantly eases the challenge. Also, DyNet provides 6 | a C++ API that is as convenient as Python but with higher efficiency and could 7 | conveniently integrate with industrial/production systems. This package, `tape`, 8 | combines the good of 9 | 10 | 1. tape from PyTorch and DyNet 11 | 2. C++ API and core from DyNet 12 | 3. rich set of operators from PaddlePaddle 13 | 14 | ## Overview 15 | 16 | We can implement Dynet-like Tape(See this [survey](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/survey/dynamic_graph.md)) 17 | by wrapping Paddle Fluid's `Operator` and `Variable`. 18 | 19 | The user API is straight forward since 20 | 21 | 1. it is imperative. And it uses host language's control flow logic. 22 | 1. it avoids extra concepts such as `Scope` and `Executor`. 23 | 24 | All of these benefits come at the cost of just adding one line `reset_global_tape` 25 | at every iteration. 26 | 27 | ## Code Structure 28 | 29 | In short, the `Tape` contains a vector of `OpHandle`s. And an `OpHandle` contains its 30 | `type`, the pointers to the `Variable`s, and necessary attributes. 31 | 32 | ```c++ 33 | class Variable { 34 | public: 35 | VriableHandle Grad(); // returns its gradient variable 36 | private: 37 | framework::VarDesc desc_; // compile time infershape, necessary for lazy execution 38 | framework::Variable var_; // run time variable, holds data memory 39 | }; 40 | 41 | using VariableHandle = shared_ptr; 42 | 43 | struct OpHandle { 44 | string type_; 45 | map> inputs_; 46 | map> outputs_; 47 | AttributeMap attrs_; 48 | }; 49 | 50 | class Tape { 51 | public: 52 | void AddOp(OpHandle); // add op 53 | void Forward(); // execute the tape_ 54 | void Backward(); // execute the backward of the tape_ 55 | private: 56 | vector tape_; 57 | }; 58 | ``` 59 | 60 | We uses `Function` to indicate layers. It takes care of parameter 61 | initialization and `AddOp` to the Tape when it is called. 62 | 63 | ```c++ 64 | class Linear { 65 | public: 66 | Linear(int in_dim, int out_dim, const std::string &act) 67 | : w_(new Variable("LinearWeight")), 68 | b_(new Variable("LinearBias")), 69 | act_(act) { 70 | Tape init_tape; 71 | 72 | std::string initializer = "fill_constant"; 73 | framework::AttributeMap attrs; 74 | attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; 75 | attrs["shape"] = std::vector{in_dim, out_dim}; 76 | attrs["value"] = 1.0f; 77 | init_tape.AddOp(initializer, {}, {{"Out", {w_}}}, attrs); 78 | 79 | attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; 80 | attrs["shape"] = std::vector{out_dim}; 81 | attrs["value"] = 1.0f; 82 | init_tape.AddOp(initializer, {}, {{"Out", {b_}}}, attrs); 83 | 84 | init_tape.Forward(); 85 | } 86 | 87 | VariableHandle operator()(VariableHandle input) { 88 | VariableHandle pre_bias(new Variable("linear")); 89 | get_global_tape().AddOp("mul", 90 | {{"X", {input}}, {"Y", {w_}}}, 91 | {{"Out", {pre_bias}}}, 92 | {{"x_num_col_dims", 1}, {"y_num_col_dims", 1}}); 93 | VariableHandle pre_act(new Variable("linear")); 94 | get_global_tape().AddOp("elementwise_add", 95 | {{"X", {pre_bias}}, {"Y", {b_}}}, 96 | {{"Out", {pre_act}}}, 97 | {{"axis", 1}}); 98 | VariableHandle post_act(new Variable("linear")); 99 | get_global_tape().AddOp(act_, 100 | {{"X", {pre_act}}}, 101 | {{"Out", {post_act}}}, 102 | {}); 103 | return post_act; 104 | } 105 | 106 | std::vector Params() { return {w_, b_}; } 107 | 108 | private: 109 | VariableHandle w_; 110 | VariableHandle b_; 111 | std::string act_; 112 | }; 113 | ``` 114 | 115 | ## User API 116 | 117 | ```c++ 118 | // Model function 119 | paddle::tape::Linear linear1(3, 3, "relu"); // init weight and bias 120 | paddle::tape::Linear linear2(3, 3, "relu"); // init weight and bias 121 | paddle::tape::Mean mean; 122 | 123 | // Optimizer 124 | paddle::tape::SGD sgd(0.001); 125 | 126 | // Data Feeder 127 | paddle::tape::Fill data_feeder(...); 128 | VariableHandle input(new paddle::tape::Variable("input")); 129 | VariableHandle label(new paddle::tape::Variable("label")); 130 | 131 | for (int i = 0; i < 2; ++i) { 132 | reset_global_tape(); 133 | 134 | data_feeder(input, label); 135 | 136 | auto loss = softmax(linear2(linear1(input)), label); // compile time InferShape & InferVarType 137 | LOG(INFO) << loss.value(); // Run forward up to loss 138 | 139 | // Run backward, store gradient of w at w->Grad() 140 | get_global_tape.Backward(loss); 141 | 142 | // Update w 143 | sgd(linear1.Params()); 144 | sgd(linear2.Params()); 145 | } 146 | ``` 147 | 148 |
149 | 150 | digraph G { 151 | 152 | subgraph cluster_0 { 153 | node [shape=record,style=filled]; 154 | style=filled; 155 | color=lightgrey; 156 | linear1 [label="{type: mul | {input | {X: before_mul1 | Y: weight1}} | {output | Out: before_bias1}}"]; 157 | elementwise_add1 [label="{type: elementwise_add | {input | {X: before_bias1 | Y: bias1}} | {output | Out: before_act1}}"]; 158 | relu1 [label="{type: relu | {input | {X: before_act1 }} | {output | Out: after_act1}}"]; 159 | 160 | linear1 -> elementwise_add1->relu1; 161 | label = "forward tape"; 162 | } 163 | 164 | linear1:before_mul1->before_mul1 165 | linear1:weight1->weight1 166 | linear1:before_bias1->before_bias1 167 | 168 | elementwise_add1:bias1->bias1 169 | elementwise_add1:before_bias1->before_bias1 170 | elementwise_add1:before_act1->before_act1 171 | 172 | relu1:before_act1->before_act1 173 | relu1:after_act1->after_act1 174 | 175 | subgraph cluster_1 { 176 | node [shape=record,style=filled]; 177 | style=filled; 178 | color=lightgrey; 179 | linear1_grad [label="{type: mul_grad | {input | {X: before_mul1 | Y: weight1| Out_grad: before_bias1_grad}} | {output |{X_grad: before_mul1_grad | Y_grad: weight1_grad}}}"]; 180 | 181 | elementwise_add1_grad [label="{type: elementwise_add_grad | {input | Out_grad: before_act1_grad} | {output |{X_grad: before_bias1_grad | Y_grad: bias1_grad}}}"]; 182 | 183 | relu1_grad [label="{type: relu_grad | {input | Out_grad: after_act1_grad} | {ouput | {X_grad: before_act1_grad }}}"]; 184 | 185 | linear1_grad -> elementwise_add1_grad ->relu1_grad [dir=back]; 186 | label = "backward tape"; 187 | } 188 | 189 | relu1_grad:after_act1_grad->after_act1_grad 190 | relu1_grad:before_act1_grad->before_act1_grad 191 | 192 | elementwise_add1_grad:before_act1_grad->before_act1_grad 193 | elementwise_add1_grad:before_bias1_grad->before_bias1_grad 194 | elementwise_add1_grad:bias1_grad->bias1_grad 195 | 196 | linear1_grad:before_mul1->before_mul1 197 | linear1_grad:weight1->weight1 198 | linear1_grad:before_bias1_grad->before_bias1_grad 199 | linear1_grad:before_mul1_grad->before_mul1_grad 200 | linear1_grad:weight1_grad->weight1_grad 201 | 202 | 203 | subgraph cluster_2 { 204 | node [shape=record]; 205 | label = "Linear1"; 206 | weight1 207 | bias1 208 | } 209 | 210 | weight1 -> weight1_grad [ label="Grad()", style="dashed" ]; 211 | bias1 -> bias1_grad [ label="Grad()", style="dashed"]; 212 | 213 | 214 | 215 | } 216 |
217 | 218 | ![Image](https://github.com/PaddlePaddle/tape/blob/develop/doc/images/computation_graph.png) 219 | 220 | ## Code Reuse 221 | 222 | We want to stay close to Paddle Fluid as much as possible. 223 | 224 | ### Reuse All Operators 225 | 226 | As all Ops are registered at `OpInfoMap`, the effort of adding a new `Function` 227 | is about 10 lines of code, similar to expose an operator to Python. 228 | 229 | ### Reuse Compile Time InferShape and InferVarType 230 | 231 | Note that all the symbolic information is stored at `tape::Varaible::desc_`, instead 232 | of `ProgramDesc.block.vars`, we create a temporary `BlockDesc` to do `InferShape` and 233 | `InferVarType` every time we `AddOp` to the tape. 234 | 235 | ### Reuse Operator::Run 236 | 237 | We use smart pointer, instead of `Scope`, to manage memory. So we create a temporary 238 | `Scope` for every `Operator::Run()`. 239 | 240 | ## Possible Feature 241 | 242 | ### Release Memory on Backward 243 | 244 | We can release memory aggressively. During backward, we can delete the OpHandle once 245 | we have finished its backward. Since all the variable is managed by smart pointer, the 246 | memory is automatically released when its `ref_count` goes to 0. 247 | 248 | ### Kernel Fusion 249 | 250 | As a symbolic representation of the Tape is constructed first before the actual 251 | execution, it would be possible to perform graph optimization. One use case is kernel 252 | fusion. 253 | -------------------------------------------------------------------------------- /doc/images/computation_graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PaddlePaddle/tape/ee6b62022944a64826f7f5fa6df36e5fef5cbbaf/doc/images/computation_graph.png -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | if(APPLE) 17 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=pessimizing-move") 18 | endif(APPLE) 19 | 20 | CheckCompilerCXX11Flag() 21 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 22 | 23 | # Some Paddle Operators uses OMP, e.g. 24 | # https://github.com/PaddlePaddle/Paddle/blob/d00a0436b1c562060b49aa2981be094d78bcbdf5/paddle/fluid/operators/top_k_op.h#L58-L60 25 | find_package(OpenMP) 26 | if(OPENMP_FOUND) 27 | set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") 28 | set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 29 | set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") 30 | endif() 31 | 32 | # Include cuda and cudnn 33 | if(WITH_GPU) 34 | add_definitions(-DPADDLE_WITH_CUDA) 35 | include_directories(${CUDNN_INCLUDE_DIR}) 36 | include_directories(${CUDA_TOOLKIT_INCLUDE}) 37 | endif() 38 | 39 | include_directories(${CMAKE_SOURCE_DIR}/paddle) # for header files in paddle/ 40 | include_directories(${CMAKE_BINARY_DIR}/paddle) # for *.pb.h 41 | include_directories(${CMAKE_BINARY_DIR}/third_party/install/glog/include) 42 | include_directories(${CMAKE_BINARY_DIR}/third_party/install/gflags/include) 43 | include_directories(${CMAKE_BINARY_DIR}/third_party/install/protobuf/include) 44 | include_directories(${CMAKE_BINARY_DIR}/third_party/install/gtest/include) 45 | include_directories(${CMAKE_BINARY_DIR}/third_party/boost/src/extern_boost/boost_1_41_0/) # hack in paddle/cmake/external/boost.cmake 46 | include_directories(${CMAKE_BINARY_DIR}/third_party/eigen3/src/extern_eigen3) 47 | # hack in paddle/cmake/external/eigen.cmake 48 | 49 | cc_library(tape SRCS variable.cc tape.cc backward.cc DEPS ${GLOB_OP_LIB}) 50 | cc_library(tape_parameter SRCS parameter.cc DEPS tape) 51 | cc_library(tape_optimizer SRCS optimizer.cc DEPS tape tape_parameter) 52 | cc_library(tape_function SRCS function.cc DEPS ${GLOB_OP_LIB} tape tape_parameter) 53 | 54 | cc_test(test_parameter 55 | SRCS test_parameter.cc 56 | DEPS tape_parameter) 57 | cc_test(test_tape 58 | SRCS test_tape.cc 59 | DEPS tape tape_function tape_optimizer) 60 | cc_test(test_backward 61 | SRCS test_backward.cc 62 | DEPS tape tape_function) 63 | 64 | add_subdirectory(example) 65 | -------------------------------------------------------------------------------- /src/backward.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "src/tape.h" 16 | 17 | namespace paddle { 18 | namespace tape {} // namespace tape 19 | } // namespace paddle 20 | -------------------------------------------------------------------------------- /src/example/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | add_subdirectory(mnist) 17 | add_subdirectory(cifar) 18 | -------------------------------------------------------------------------------- /src/example/cifar/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | cc_test(test_cifar 17 | SRCS test_cifar.cc 18 | DEPS tape tape_function tape_optimizer) 19 | -------------------------------------------------------------------------------- /src/example/cifar/create_cifar_recordio.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import paddle 16 | import paddle.fluid as fluid 17 | import paddle.dataset.cifar as cifar 18 | 19 | 20 | def create_cifar_recordio_files(): 21 | # Convert cifar training set to recordio files 22 | with fluid.program_guard(fluid.Program(), fluid.Program()): 23 | reader = paddle.batch(cifar.train10(), batch_size=1) 24 | feeder = fluid.DataFeeder( 25 | feed_list=[ # order is image and label 26 | fluid.layers.data( 27 | name='image', shape=[3, 32, 32], dtype='float32'), 28 | fluid.layers.data( 29 | name='label', shape=[1], dtype='int64'), 30 | ], 31 | place=fluid.CPUPlace()) 32 | fluid.recordio_writer.convert_reader_to_recordio_file( 33 | '/tmp/cifar10_train.recordio', reader, feeder) 34 | 35 | # Convert cifar testing set to recordio files 36 | with fluid.program_guard(fluid.Program(), fluid.Program()): 37 | reader = paddle.batch(cifar.test10(), batch_size=1) 38 | feeder = fluid.DataFeeder( 39 | feed_list=[ # order is image and label 40 | fluid.layers.data( 41 | name='image', shape=[3, 32, 32], dtype='float32'), 42 | fluid.layers.data( 43 | name='label', shape=[1], dtype='int64'), 44 | ], 45 | place=fluid.CPUPlace()) 46 | fluid.recordio_writer.convert_reader_to_recordio_file( 47 | '/tmp/cifar10_test.recordio', reader, feeder) 48 | 49 | 50 | if __name__ == "__main__": 51 | create_cifar_recordio_files() 52 | -------------------------------------------------------------------------------- /src/example/cifar/fluid_cifar_benchmark.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import print_function 16 | 17 | import paddle 18 | import paddle.fluid as fluid 19 | import contextlib 20 | import math 21 | import sys 22 | import numpy 23 | import unittest 24 | import os 25 | import numpy as np 26 | import time 27 | 28 | 29 | def resnet_cifar10(input, depth=32): 30 | def conv_bn_layer(input, 31 | ch_out, 32 | filter_size, 33 | stride, 34 | padding, 35 | act='relu', 36 | bias_attr=False): 37 | tmp = fluid.layers.conv2d( 38 | input=input, 39 | filter_size=filter_size, 40 | num_filters=ch_out, 41 | stride=stride, 42 | padding=padding, 43 | act=None, 44 | bias_attr=bias_attr) 45 | return fluid.layers.batch_norm(input=tmp, act=act) 46 | 47 | def shortcut(input, ch_in, ch_out, stride): 48 | if ch_in != ch_out: 49 | return conv_bn_layer(input, ch_out, 1, stride, 0, None) 50 | else: 51 | return input 52 | 53 | def basicblock(input, ch_in, ch_out, stride): 54 | tmp = conv_bn_layer(input, ch_out, 3, stride, 1) 55 | tmp = conv_bn_layer(tmp, ch_out, 3, 1, 1, act=None, bias_attr=True) 56 | short = shortcut(input, ch_in, ch_out, stride) 57 | return fluid.layers.elementwise_add(x=tmp, y=short, act='relu') 58 | 59 | def layer_warp(block_func, input, ch_in, ch_out, count, stride): 60 | tmp = block_func(input, ch_in, ch_out, stride) 61 | for i in range(1, count): 62 | tmp = block_func(tmp, ch_out, ch_out, 1) 63 | return tmp 64 | 65 | assert (depth - 2) % 6 == 0 66 | n = (depth - 2) / 6 67 | conv1 = conv_bn_layer( 68 | input=input, ch_out=16, filter_size=3, stride=1, padding=1) 69 | res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) 70 | res2 = layer_warp(basicblock, res1, 16, 32, n, 2) 71 | res3 = layer_warp(basicblock, res2, 32, 64, n, 2) 72 | pool = fluid.layers.pool2d( 73 | input=res3, pool_size=8, pool_type='avg', pool_stride=1) 74 | return pool 75 | 76 | 77 | def vgg16_bn_drop(input): 78 | def conv_block(input, num_filter, groups, dropouts): 79 | return fluid.nets.img_conv_group( 80 | input=input, 81 | pool_size=2, 82 | pool_stride=2, 83 | conv_num_filter=[num_filter] * groups, 84 | conv_filter_size=3, 85 | conv_act='relu', 86 | conv_with_batchnorm=True, 87 | conv_batchnorm_drop_rate=dropouts, 88 | pool_type='max') 89 | 90 | conv1 = conv_block(input, 64, 2, [0.3, 0]) 91 | conv2 = conv_block(conv1, 128, 2, [0.4, 0]) 92 | conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0]) 93 | conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0]) 94 | conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0]) 95 | 96 | drop = fluid.layers.dropout(x=conv5, dropout_prob=0.5) 97 | fc1 = fluid.layers.fc(input=drop, size=512, act=None) 98 | bn = fluid.layers.batch_norm(input=fc1, act='relu') 99 | drop2 = fluid.layers.dropout(x=bn, dropout_prob=0.5) 100 | fc2 = fluid.layers.fc(input=drop2, size=512, act=None) 101 | return fc2 102 | 103 | 104 | def train(net_type, use_cuda, use_reader_op): 105 | classdim = 10 106 | data_shape = [3, 32, 32] 107 | BATCH_SIZE = 128 108 | print("Batch size is {}".format(BATCH_SIZE)) 109 | 110 | train_file_path = "/tmp/cifar10_train.recordio" 111 | 112 | if use_reader_op: 113 | print("use reader op from {}".format(train_file_path)) 114 | train_data_file = fluid.layers.open_recordio_file( 115 | filename=train_file_path, 116 | shapes=[[-1, 3, 32, 32], [-1, 1]], 117 | lod_levels=[0, 0], 118 | dtypes=["float32", "int64"], 119 | pass_num=50, 120 | for_parallel=False) 121 | train_data_file = fluid.layers.double_buffer( 122 | fluid.layers.batch( 123 | train_data_file, batch_size=BATCH_SIZE)) 124 | images, label = fluid.layers.read_file(train_data_file) 125 | else: 126 | images = fluid.layers.data( 127 | name='pixel', shape=data_shape, dtype='float32') 128 | label = fluid.layers.data(name='label', shape=[1], dtype='int64') 129 | 130 | if net_type == "vgg": 131 | print("train vgg net") 132 | net = vgg16_bn_drop(images) 133 | else: 134 | raise ValueError("%s network is not supported" % net_type) 135 | 136 | predict = fluid.layers.fc(input=net, size=classdim, act='softmax') 137 | cost = fluid.layers.cross_entropy(input=predict, label=label) 138 | avg_cost = fluid.layers.mean(cost) 139 | acc = fluid.layers.accuracy(input=predict, label=label) 140 | 141 | optimizer = fluid.optimizer.Adam(learning_rate=0.001) 142 | optimizer.minimize(avg_cost) 143 | 144 | # train_reader = paddle.batch( 145 | # paddle.reader.shuffle( 146 | # paddle.dataset.cifar.train10(), buf_size=BATCH_SIZE * 10), 147 | # batch_size=BATCH_SIZE) 148 | 149 | train_reader = paddle.batch( 150 | paddle.dataset.cifar.train10(), batch_size=BATCH_SIZE) 151 | 152 | test_reader = paddle.batch( 153 | paddle.dataset.cifar.test10(), batch_size=BATCH_SIZE) 154 | 155 | place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() 156 | exe = fluid.Executor(place) 157 | 158 | if not use_reader_op: 159 | feeder = fluid.DataFeeder(place=place, feed_list=[images, label]) 160 | 161 | exe.run(fluid.default_startup_program()) 162 | 163 | PASS = 50 164 | iters = 1050 165 | skip_batch_num = 50 166 | 167 | print('start') 168 | 169 | iter_num, num_samples, start = 0, 0, time.time() 170 | for pass_id in range(PASS): 171 | if not use_reader_op: 172 | reader_generator = train_reader() 173 | data = None 174 | while True: 175 | if not use_reader_op: 176 | data = next(reader_generator, None) 177 | if data is None: 178 | break 179 | if iter_num == iters: 180 | break 181 | if iter_num == skip_batch_num: 182 | start = time.time() 183 | num_samples = 0 184 | if use_reader_op: 185 | try: 186 | exe.run(fluid.default_main_program(), 187 | use_program_cache=True) 188 | except fluid.core.EnforceNotMet as ex: 189 | break 190 | else: 191 | exe.run(fluid.default_main_program(), feed=feeder.feed(data)) 192 | iter_num += 1 193 | if use_reader_op: 194 | num_samples += BATCH_SIZE 195 | else: 196 | num_samples += len(data) 197 | print("Pass: %d, Iter: %d" % (pass_id, iter_num)) 198 | if iter_num == iters: 199 | break 200 | 201 | end = time.time() 202 | elapsed_time = end - start 203 | print('{} iteratios takes {} seconds wall clock time'.format( 204 | iters - skip_batch_num, elapsed_time)) 205 | print('Total examples: %d; Throughput: %.5f examples per sec' % 206 | (num_samples, num_samples / elapsed_time)) 207 | 208 | 209 | if __name__ == '__main__': 210 | train('vgg', use_cuda=True, use_reader_op=True) 211 | -------------------------------------------------------------------------------- /src/example/cifar/test_cifar.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include // NOLINT 16 | #include 17 | #include 18 | #include 19 | 20 | #include "gtest/gtest.h" 21 | #include "paddle/fluid/platform/device_context.h" 22 | #include "paddle/fluid/platform/gpu_info.h" 23 | #include "paddle/fluid/platform/place.h" 24 | #include "src/function.h" 25 | #include "src/optimizer.h" 26 | 27 | using paddle::tape::VariableHandle; 28 | using paddle::tape::Linear; 29 | using Conv2D = paddle::tape::Convolution2D; 30 | using paddle::tape::BatchNorm; 31 | using paddle::tape::SGD; 32 | using paddle::tape::Adam; 33 | using paddle::tape::accuracy; 34 | using paddle::tape::mean; 35 | using paddle::tape::softmax; 36 | using paddle::tape::cross_entropy; 37 | using paddle::tape::reset_global_tape; 38 | using paddle::tape::get_global_tape; 39 | using paddle::tape::OptimizableParameters; 40 | using paddle::tape::BackwardAndUpdate; 41 | using paddle::tape::ParameterCollection; 42 | using paddle::tape::ParameterHandle; 43 | using paddle::tape::GlobalParameterCollection; 44 | 45 | using paddle::tape::CreateRecordioFileReader; 46 | using paddle::tape::CreateBatchReader; 47 | using paddle::tape::CreateDoubleBufferReader; 48 | using paddle::tape::ReadNext; 49 | using paddle::tape::ResetReader; 50 | 51 | TEST(Cifar, TestGPU) { 52 | // auto place = paddle::platform::CPUPlace(); 53 | auto place = paddle::platform::CUDAPlace(0); 54 | reset_global_tape(place); 55 | 56 | const int batch_size = 128; 57 | LOG(INFO) << "Batch size is " << batch_size << std::endl; 58 | 59 | std::string save_model_path = "/tmp/cifar_model/"; 60 | std::string filename1 = "/tmp/cifar10_train.recordio"; 61 | std::string filename2 = "/tmp/cifar10_test.recordio"; 62 | auto train_reader = CreateRecordioFileReader( 63 | filename1, {-1, 3, 32, 32, -1, 1}, {4, 2}, {0, 0}); 64 | auto test_reader = CreateRecordioFileReader( 65 | filename2, {-1, 3, 32, 32, -1, 1}, {4, 2}, {0, 0}); 66 | train_reader = 67 | CreateDoubleBufferReader(CreateBatchReader(train_reader, batch_size)); 68 | test_reader = 69 | CreateDoubleBufferReader(CreateBatchReader(test_reader, batch_size)); 70 | 71 | // input 3x32x32 72 | // after conv1_1 64x32x32 73 | // after conv1_2 64x32x32 74 | // after pool1 64x16x16 75 | Conv2D conv1_1(3, 64, 3); 76 | BatchNorm bn1_1(64, "relu"); 77 | Conv2D conv1_2(64, 64, 3); 78 | BatchNorm bn1_2(64, "relu"); 79 | 80 | // after conv2_1 128x16x16 81 | // after conv2_2 128x16x16 82 | // after pool2 128x8x8 83 | Conv2D conv2_1(64, 128, 3); 84 | BatchNorm bn2_1(128, "relu"); 85 | Conv2D conv2_2(128, 128, 3); 86 | BatchNorm bn2_2(128, "relu"); 87 | 88 | // after conv3_1 256x8x8 89 | // after conv3_2 256x8x8 90 | // after conv3_3 256x8x8 91 | // after pool3 256x4x4 92 | Conv2D conv3_1(128, 256, 3); 93 | BatchNorm bn3_1(256, "relu"); 94 | Conv2D conv3_2(256, 256, 3); 95 | BatchNorm bn3_2(256, "relu"); 96 | Conv2D conv3_3(256, 256, 3); 97 | BatchNorm bn3_3(256, "relu"); 98 | 99 | // after conv4_1 512x4x4 100 | // after conv4_2 512x4x4 101 | // after conv4_3 512x4x4 102 | // after pool4 512x2x2 103 | Conv2D conv4_1(256, 512, 3); 104 | BatchNorm bn4_1(512, "relu"); 105 | Conv2D conv4_2(512, 512, 3); 106 | BatchNorm bn4_2(512, "relu"); 107 | Conv2D conv4_3(512, 512, 3); 108 | BatchNorm bn4_3(512, "relu"); 109 | 110 | // after conv5_1 512x2x2 111 | // after conv5_2 512x2x2 112 | // after conv5_3 512x2x2 113 | // after pool5 512x1x1 114 | Conv2D conv5_1(512, 512, 3); 115 | BatchNorm bn5_1(512, "relu"); 116 | Conv2D conv5_2(512, 512, 3); 117 | BatchNorm bn5_2(512, "relu"); 118 | Conv2D conv5_3(512, 512, 3); 119 | BatchNorm bn5_3(512, "relu"); 120 | 121 | // Input dim 512x1x1 = 512 122 | Linear fc1(512, 512); 123 | BatchNorm bn6(512, "relu"); 124 | Linear fc2(512, 512); 125 | Linear fc3(512, 10, "softmax"); 126 | 127 | Adam adam(0.001); 128 | 129 | auto vgg16_forward = [&](VariableHandle input, 130 | bool is_test) -> VariableHandle { 131 | // Set attribute for batchnorm and dropout op 132 | paddle::framework::AttributeMap bn_attrs, d_attrs; 133 | bn_attrs["is_test"] = is_test; 134 | d_attrs["is_test"] = is_test; 135 | d_attrs["dropout_prob"] = 0.3f; 136 | auto temp1 = dropout(bn1_1(conv1_1(input), bn_attrs), d_attrs); 137 | auto pool1 = pool2d(bn1_2(conv1_2(temp1), bn_attrs)); 138 | 139 | d_attrs["dropout_prob"] = 0.4f; 140 | auto temp2 = dropout(bn2_1(conv2_1(pool1), bn_attrs), d_attrs); 141 | auto pool2 = pool2d(bn2_2(conv2_2(temp2), bn_attrs)); 142 | 143 | auto temp3_1 = dropout(bn3_1(conv3_1(pool2), bn_attrs), d_attrs); 144 | auto temp3_2 = dropout(bn3_2(conv3_2(temp3_1), bn_attrs), d_attrs); 145 | auto pool3 = pool2d(bn3_3(conv3_3(temp3_2), bn_attrs)); 146 | 147 | auto temp4_1 = dropout(bn4_1(conv4_1(pool3), bn_attrs), d_attrs); 148 | auto temp4_2 = dropout(bn4_2(conv4_2(temp4_1), bn_attrs), d_attrs); 149 | auto pool4 = pool2d(bn4_3(conv4_3(temp4_2), bn_attrs)); 150 | 151 | auto temp5_1 = dropout(bn5_1(conv5_1(pool4), bn_attrs), d_attrs); 152 | auto temp5_2 = dropout(bn5_2(conv5_2(temp5_1), bn_attrs), d_attrs); 153 | auto pool5 = pool2d(bn5_3(conv5_3(temp5_2), bn_attrs)); 154 | 155 | d_attrs["dropout_prob"] = 0.5f; 156 | auto temp6 = bn6(fc1(dropout(pool5, d_attrs)), bn_attrs); 157 | return fc3(fc2(dropout(temp6, d_attrs))); 158 | }; 159 | 160 | int total_steps = 100000; 161 | int test_steps = 1000; 162 | int print_step = 100; 163 | float threshold = 0.8f; 164 | 165 | int iter_num = 1050; 166 | int skip_batch_num = 50; 167 | bool model_saved = false; 168 | bool do_benchmark = true; 169 | 170 | auto start = std::chrono::system_clock::now(); 171 | int num_samples = 0; 172 | // Training 173 | for (int i = 0; i < total_steps; ++i) { 174 | LOG(INFO) << "Train step #" << i; 175 | 176 | if (do_benchmark && i == iter_num) { 177 | break; 178 | } 179 | 180 | if (do_benchmark && i == skip_batch_num) { 181 | start = std::chrono::system_clock::now(); 182 | num_samples = 0; 183 | } 184 | 185 | reset_global_tape(place); 186 | auto data_label = ReadNext(train_reader, true); 187 | auto data = data_label[0]; 188 | auto label = data_label[1]; 189 | 190 | auto predict = vgg16_forward(data, false); 191 | auto loss = mean(cross_entropy(predict, label)); 192 | auto precision = accuracy(predict, label); 193 | 194 | BackwardAndUpdate(loss, &adam); 195 | 196 | num_samples += batch_size; 197 | 198 | // Every time certain amount of batches have been processed, 199 | // we test the average loss and accuracy on the test data set, 200 | // we stop training when the accuracy hit some threshold 201 | if (!do_benchmark && (i + 1) % print_step == 0) { 202 | std::vector losses; 203 | std::vector accuracies; 204 | ResetReader(test_reader); 205 | 206 | LOG(INFO) << "Start testing"; 207 | for (int i = 0; i < test_steps; ++i) { 208 | reset_global_tape(place); 209 | 210 | auto data_label = ReadNext(test_reader, false); 211 | if (data_label.empty()) { 212 | LOG(INFO) << "Full test set has been traversed"; 213 | break; 214 | } 215 | 216 | auto data = data_label[0]; 217 | auto label = data_label[1]; 218 | 219 | auto predict = vgg16_forward(data, true); 220 | auto loss = mean(cross_entropy(predict, label)); 221 | auto precision = accuracy(predict, label); 222 | 223 | get_global_tape().Forward(); 224 | 225 | losses.push_back(loss->FetchValue() 226 | ->Get() 227 | .data()[0]); 228 | accuracies.push_back(precision->FetchValue() 229 | ->Get() 230 | .data()[0]); 231 | } 232 | 233 | float avg_loss = 234 | std::accumulate(losses.begin(), losses.end(), 0.0f) / losses.size(); 235 | float avg_accu = 236 | std::accumulate(accuracies.begin(), accuracies.end(), 0.0f) / 237 | accuracies.size(); 238 | 239 | LOG(INFO) << "Batch #" << i 240 | << ", test set evaluation result: Avg loss is " << avg_loss 241 | << ", Avg accuracy is " << avg_accu; 242 | 243 | if (avg_accu >= threshold) { 244 | LOG(INFO) << "Meets target accuracy, stop training and save parameters"; 245 | GlobalParameterCollection().SaveAllParameters(save_model_path); 246 | model_saved = true; 247 | break; 248 | } 249 | } 250 | } 251 | 252 | if (do_benchmark) { 253 | auto end = std::chrono::system_clock::now(); 254 | std::chrono::duration elapsed_time = end - start; 255 | LOG(INFO) << "Total wall clock time for iteration num " 256 | << (iter_num - skip_batch_num) << " is " << elapsed_time.count() 257 | << " seconds" << std::endl; 258 | LOG(INFO) << "Total samples: " << num_samples 259 | << "; Throughput: " << num_samples / elapsed_time.count() 260 | << std::endl; 261 | } 262 | 263 | if (!model_saved) { 264 | return; 265 | } 266 | 267 | // Inference using test set 268 | LOG(INFO) << "Start inferencing and load parameters"; 269 | ParameterCollection loaded_pc(save_model_path); 270 | 271 | // Reconstruct layers by loading the saved parameters 272 | Conv2D inf_conv1_1(loaded_pc.LookUp(conv1_1.ParamNames()), conv1_1.ActName()); 273 | BatchNorm inf_bn1_1(loaded_pc.LookUp(bn1_1.ParamNames()), bn1_1.ActName()); 274 | Conv2D inf_conv1_2(loaded_pc.LookUp(conv1_2.ParamNames()), conv1_2.ActName()); 275 | BatchNorm inf_bn1_2(loaded_pc.LookUp(bn1_2.ParamNames()), bn1_2.ActName()); 276 | 277 | Conv2D inf_conv2_1(loaded_pc.LookUp(conv2_1.ParamNames()), conv2_1.ActName()); 278 | BatchNorm inf_bn2_1(loaded_pc.LookUp(bn2_1.ParamNames()), bn2_1.ActName()); 279 | Conv2D inf_conv2_2(loaded_pc.LookUp(conv2_2.ParamNames()), conv2_2.ActName()); 280 | BatchNorm inf_bn2_2(loaded_pc.LookUp(bn2_2.ParamNames()), bn2_2.ActName()); 281 | 282 | Conv2D inf_conv3_1(loaded_pc.LookUp(conv3_1.ParamNames()), conv3_1.ActName()); 283 | BatchNorm inf_bn3_1(loaded_pc.LookUp(bn3_1.ParamNames()), bn3_1.ActName()); 284 | Conv2D inf_conv3_2(loaded_pc.LookUp(conv3_2.ParamNames()), conv3_2.ActName()); 285 | BatchNorm inf_bn3_2(loaded_pc.LookUp(bn3_2.ParamNames()), bn3_2.ActName()); 286 | Conv2D inf_conv3_3(loaded_pc.LookUp(conv3_3.ParamNames()), conv3_3.ActName()); 287 | BatchNorm inf_bn3_3(loaded_pc.LookUp(bn3_3.ParamNames()), bn3_3.ActName()); 288 | 289 | Conv2D inf_conv4_1(loaded_pc.LookUp(conv4_1.ParamNames()), conv4_1.ActName()); 290 | BatchNorm inf_bn4_1(loaded_pc.LookUp(bn4_1.ParamNames()), bn4_1.ActName()); 291 | Conv2D inf_conv4_2(loaded_pc.LookUp(conv4_2.ParamNames()), conv4_2.ActName()); 292 | BatchNorm inf_bn4_2(loaded_pc.LookUp(bn4_2.ParamNames()), bn4_2.ActName()); 293 | Conv2D inf_conv4_3(loaded_pc.LookUp(conv4_3.ParamNames()), conv4_3.ActName()); 294 | BatchNorm inf_bn4_3(loaded_pc.LookUp(bn4_3.ParamNames()), bn4_3.ActName()); 295 | 296 | Conv2D inf_conv5_1(loaded_pc.LookUp(conv5_1.ParamNames()), conv5_1.ActName()); 297 | BatchNorm inf_bn5_1(loaded_pc.LookUp(bn5_1.ParamNames()), bn5_1.ActName()); 298 | Conv2D inf_conv5_2(loaded_pc.LookUp(conv5_2.ParamNames()), conv5_2.ActName()); 299 | BatchNorm inf_bn5_2(loaded_pc.LookUp(bn5_2.ParamNames()), bn5_2.ActName()); 300 | Conv2D inf_conv5_3(loaded_pc.LookUp(conv5_3.ParamNames()), conv5_3.ActName()); 301 | BatchNorm inf_bn5_3(loaded_pc.LookUp(bn5_3.ParamNames()), bn5_3.ActName()); 302 | 303 | Linear inf_fc1(loaded_pc.LookUp(fc1.ParamNames()), fc1.ActName()); 304 | BatchNorm inf_bn6(loaded_pc.LookUp(bn6.ParamNames()), bn6.ActName()); 305 | Linear inf_fc2(loaded_pc.LookUp(fc2.ParamNames()), fc2.ActName()); 306 | Linear inf_fc3(loaded_pc.LookUp(fc3.ParamNames()), fc3.ActName()); 307 | 308 | auto vgg16_inference = [&](VariableHandle input, 309 | bool is_test) -> VariableHandle { 310 | // Set attribute for batchnorm and dropout op 311 | paddle::framework::AttributeMap bn_attrs, d_attrs; 312 | bn_attrs["is_test"] = is_test; 313 | d_attrs["is_test"] = is_test; 314 | d_attrs["dropout_prob"] = 0.3f; 315 | auto temp1 = dropout(inf_bn1_1(inf_conv1_1(input), bn_attrs), d_attrs); 316 | auto pool1 = pool2d(inf_bn1_2(inf_conv1_2(temp1), bn_attrs)); 317 | 318 | d_attrs["dropout_prob"] = 0.4f; 319 | auto temp2 = dropout(inf_bn2_1(inf_conv2_1(pool1), bn_attrs), d_attrs); 320 | auto pool2 = pool2d(inf_bn2_2(inf_conv2_2(temp2), bn_attrs)); 321 | 322 | auto temp3_1 = dropout(inf_bn3_1(inf_conv3_1(pool2), bn_attrs), d_attrs); 323 | auto temp3_2 = dropout(inf_bn3_2(inf_conv3_2(temp3_1), bn_attrs), d_attrs); 324 | auto pool3 = pool2d(inf_bn3_3(inf_conv3_3(temp3_2), bn_attrs)); 325 | 326 | auto temp4_1 = dropout(inf_bn4_1(inf_conv4_1(pool3), bn_attrs), d_attrs); 327 | auto temp4_2 = dropout(inf_bn4_2(inf_conv4_2(temp4_1), bn_attrs), d_attrs); 328 | auto pool4 = pool2d(inf_bn4_3(inf_conv4_3(temp4_2), bn_attrs)); 329 | 330 | auto temp5_1 = dropout(inf_bn5_1(inf_conv5_1(pool4), bn_attrs), d_attrs); 331 | auto temp5_2 = dropout(inf_bn5_2(inf_conv5_2(temp5_1), bn_attrs), d_attrs); 332 | auto pool5 = pool2d(inf_bn5_3(inf_conv5_3(temp5_2), bn_attrs)); 333 | 334 | d_attrs["dropout_prob"] = 0.5f; 335 | auto temp6 = inf_bn6(inf_fc1(dropout(pool5, d_attrs)), bn_attrs); 336 | return inf_fc3(inf_fc2(dropout(temp6, d_attrs))); 337 | }; 338 | 339 | std::vector losses; 340 | std::vector accuracies; 341 | ResetReader(test_reader); 342 | 343 | while (true) { 344 | reset_global_tape(place); 345 | 346 | auto data_label = ReadNext(test_reader, false); 347 | if (data_label.empty()) { 348 | break; 349 | } 350 | 351 | auto data = data_label[0]; 352 | auto label = data_label[1]; 353 | 354 | auto predict = vgg16_inference(data, true); 355 | auto loss = mean(cross_entropy(predict, label)); 356 | auto precision = accuracy(predict, label); 357 | 358 | get_global_tape().Forward(); 359 | 360 | losses.push_back(loss->FetchValue() 361 | ->Get() 362 | .data()[0]); 363 | accuracies.push_back(precision->FetchValue() 364 | ->Get() 365 | .data()[0]); 366 | } 367 | 368 | float avg_loss = 369 | std::accumulate(losses.begin(), losses.end(), 0.0f) / losses.size(); 370 | float avg_accu = std::accumulate(accuracies.begin(), accuracies.end(), 0.0f) / 371 | accuracies.size(); 372 | 373 | LOG(INFO) << "Inference on test set result: Avg loss is " << avg_loss 374 | << ", Avg accuracy is " << avg_accu; 375 | 376 | PADDLE_ENFORCE_EQ(system(std::string("rm -r " + save_model_path).c_str()), 0); 377 | } 378 | 379 | int main(int argc, char** argv) { 380 | std::vector places; 381 | 382 | places.emplace_back(paddle::platform::CPUPlace()); 383 | int count = paddle::platform::GetCUDADeviceCount(); 384 | for (int i = 0; i < count; ++i) { 385 | places.emplace_back(paddle::platform::CUDAPlace(i)); 386 | } 387 | LOG(INFO) << "DeviceCount " << count; 388 | paddle::platform::DeviceContextPool::Init(places); 389 | 390 | testing::InitGoogleTest(&argc, argv); 391 | return RUN_ALL_TESTS(); 392 | } 393 | -------------------------------------------------------------------------------- /src/example/mnist/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | cc_test(test_mnist 17 | SRCS test_mnist.cc 18 | DEPS tape tape_function tape_optimizer) 19 | -------------------------------------------------------------------------------- /src/example/mnist/create_mnist_recordio.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import paddle.fluid as fluid 16 | import paddle.v2 as paddle 17 | import paddle.v2.dataset.mnist as mnist 18 | 19 | 20 | def create_mnist_recordio_files(): 21 | # Convert mnist training set to recordio files 22 | with fluid.program_guard(fluid.Program(), fluid.Program()): 23 | reader = paddle.batch(mnist.train(), batch_size=32) 24 | feeder = fluid.DataFeeder( 25 | feed_list=[ # order is image and label 26 | fluid.layers.data( 27 | name='image', shape=[1, 28, 28], dtype='float32'), 28 | fluid.layers.data( 29 | name='label', shape=[1], dtype='int64'), 30 | ], 31 | place=fluid.CPUPlace()) 32 | fluid.recordio_writer.convert_reader_to_recordio_file( 33 | '/tmp/mnist_train.recordio', reader, feeder) 34 | 35 | # Convert mnist testing set to recordio files 36 | with fluid.program_guard(fluid.Program(), fluid.Program()): 37 | reader = paddle.batch(mnist.test(), batch_size=32) 38 | feeder = fluid.DataFeeder( 39 | feed_list=[ # order is image and label 40 | fluid.layers.data( 41 | name='image', shape=[1, 28, 28], dtype='float32'), 42 | fluid.layers.data( 43 | name='label', shape=[1], dtype='int64'), 44 | ], 45 | place=fluid.CPUPlace()) 46 | fluid.recordio_writer.convert_reader_to_recordio_file( 47 | '/tmp/mnist_test.recordio', reader, feeder) 48 | 49 | 50 | if __name__ == "__main__": 51 | create_mnist_recordio_files() 52 | -------------------------------------------------------------------------------- /src/example/mnist/test_mnist.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "gtest/gtest.h" 19 | #include "src/function.h" 20 | #include "src/optimizer.h" 21 | 22 | using paddle::tape::VariableHandle; 23 | using paddle::tape::Linear; 24 | using paddle::tape::SGD; 25 | using paddle::tape::Adam; 26 | using paddle::tape::accuracy; 27 | using paddle::tape::mean; 28 | using paddle::tape::softmax; 29 | using paddle::tape::cross_entropy; 30 | using paddle::tape::reset_global_tape; 31 | using paddle::tape::get_global_tape; 32 | using paddle::tape::OptimizableParameters; 33 | using paddle::tape::BackwardAndUpdate; 34 | using paddle::tape::ParameterCollection; 35 | using paddle::tape::ParameterHandle; 36 | using paddle::tape::GlobalParameterCollection; 37 | 38 | using paddle::tape::CreateRecordioFileReader; 39 | using paddle::tape::ReadNext; 40 | 41 | TEST(Mnist, TestCPU) { 42 | std::string save_model_path = "/tmp/mnist_model/"; 43 | std::string filename1 = "/tmp/mnist_train.recordio"; 44 | std::string filename2 = "/tmp/mnist_test.recordio"; 45 | auto train_reader = CreateRecordioFileReader( 46 | filename1, {32, 1, 28, 28, 32, 1}, {4, 2}, {0, 0}); 47 | auto test_reader = CreateRecordioFileReader( 48 | filename2, {32, 1, 28, 28, 32, 1}, {4, 2}, {0, 0}); 49 | 50 | Linear linear1(784, 200, "tanh"); 51 | Linear linear2(200, 200, "tanh"); 52 | Linear linear3(200, 10, "softmax"); 53 | Adam adam(0.001); 54 | 55 | auto forward = [&](VariableHandle input) -> VariableHandle { 56 | return linear3(linear2(linear1(input))); 57 | }; 58 | 59 | int total_steps = 10000; 60 | int print_step = 100; 61 | float threshold = 0.90f; 62 | 63 | // Training 64 | for (int i = 0; i < total_steps; ++i) { 65 | reset_global_tape(); 66 | auto data_label = ReadNext(train_reader, true); 67 | auto data = data_label[0]; 68 | auto label = data_label[1]; 69 | 70 | auto predict = forward(data); 71 | auto loss = mean(cross_entropy(predict, label)); 72 | auto precision = accuracy(predict, label); 73 | 74 | BackwardAndUpdate(loss, &adam); 75 | 76 | // Every time certain amount of batches have been processed, 77 | // we test the average loss and accuracy on the test data set, 78 | // we stop training when the accuracy hit some threshold 79 | if ((i + 1) % print_step == 0) { 80 | std::vector losses; 81 | std::vector accuracies; 82 | 83 | while (true) { 84 | reset_global_tape(); 85 | 86 | auto data_label = ReadNext(test_reader, false); 87 | if (data_label.empty()) { 88 | break; 89 | } 90 | 91 | auto data = data_label[0]; 92 | auto label = data_label[1]; 93 | 94 | auto predict = forward(data); 95 | auto loss = mean(cross_entropy(predict, label)); 96 | auto precision = accuracy(predict, label); 97 | 98 | get_global_tape().Forward(); 99 | 100 | losses.push_back( 101 | loss->Get().data()[0]); 102 | accuracies.push_back( 103 | precision->Get().data()[0]); 104 | } 105 | 106 | float avg_loss = 107 | std::accumulate(losses.begin(), losses.end(), 0.0f) / losses.size(); 108 | float avg_accu = 109 | std::accumulate(accuracies.begin(), accuracies.end(), 0.0f) / 110 | accuracies.size(); 111 | 112 | LOG(INFO) << "Pass #" << (i + 1) / print_step 113 | << ", test set evaluation result: Avg loss is " << avg_loss 114 | << ", Avg accuracy is " << avg_accu; 115 | 116 | if (avg_accu >= threshold) { 117 | LOG(INFO) << "Meets target accuracy, stop training and save parameters"; 118 | GlobalParameterCollection().SaveAllParameters(save_model_path); 119 | break; 120 | } 121 | } 122 | } 123 | 124 | // Inference using test set 125 | LOG(INFO) << "Start inferencing and load parameters"; 126 | ParameterCollection loaded_pc(save_model_path); 127 | Linear inf_linear1(loaded_pc.LookUp(linear1.ParamNames()), linear1.ActName()); 128 | Linear inf_linear2(loaded_pc.LookUp(linear2.ParamNames()), linear2.ActName()); 129 | Linear inf_linear3(loaded_pc.LookUp(linear3.ParamNames()), linear3.ActName()); 130 | 131 | auto inference = [&](VariableHandle input) -> VariableHandle { 132 | return inf_linear3(inf_linear2(inf_linear1(input))); 133 | }; 134 | 135 | std::vector losses; 136 | std::vector accuracies; 137 | 138 | while (true) { 139 | reset_global_tape(); 140 | 141 | auto data_label = ReadNext(test_reader, false); 142 | if (data_label.empty()) { 143 | break; 144 | } 145 | 146 | auto data = data_label[0]; 147 | auto label = data_label[1]; 148 | 149 | auto predict = inference(data); 150 | auto loss = mean(cross_entropy(predict, label)); 151 | auto precision = accuracy(predict, label); 152 | 153 | get_global_tape().Forward(); 154 | 155 | losses.push_back( 156 | loss->Get().data()[0]); 157 | accuracies.push_back( 158 | precision->Get().data()[0]); 159 | } 160 | 161 | float avg_loss = 162 | std::accumulate(losses.begin(), losses.end(), 0.0f) / losses.size(); 163 | float avg_accu = std::accumulate(accuracies.begin(), accuracies.end(), 0.0f) / 164 | accuracies.size(); 165 | 166 | LOG(INFO) << "Inference on test set result: Avg loss is " << avg_loss 167 | << ", Avg accuracy is " << avg_accu; 168 | 169 | PADDLE_ENFORCE_EQ(system(std::string("rm -r " + save_model_path).c_str()), 0); 170 | } 171 | 172 | int main(int argc, char** argv) { 173 | std::vector places; 174 | places.emplace_back(paddle::platform::CPUPlace()); 175 | paddle::platform::DeviceContextPool::Init(places); 176 | 177 | testing::InitGoogleTest(&argc, argv); 178 | return RUN_ALL_TESTS(); 179 | } 180 | -------------------------------------------------------------------------------- /src/function.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "src/function.h" 16 | -------------------------------------------------------------------------------- /src/function.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "paddle/fluid/framework/ddim.h" 24 | #include "paddle/fluid/framework/op_registry.h" 25 | #include "paddle/fluid/framework/reader.h" 26 | #include "paddle/fluid/framework/type_defs.h" 27 | #include "paddle/fluid/platform/place.h" 28 | 29 | #include "src/parameter.h" 30 | #include "src/tape.h" 31 | 32 | namespace paddle { 33 | namespace tape { 34 | 35 | class Function {}; 36 | 37 | class RandomSeed { 38 | public: 39 | static int GetRandomSeed() { return 0; } 40 | }; 41 | 42 | class Fill { 43 | public: 44 | Fill(const std::string &initializer, const framework::AttributeMap &attrs) 45 | : initializer_(initializer), attrs_(attrs) {} 46 | 47 | VariableHandle operator()() { 48 | if (initializer_ == "fill_constant") { 49 | PADDLE_THROW( 50 | "fill_constant is not supported, since it is not of type " 51 | "OperatorWithKernel"); 52 | } 53 | 54 | VariableHandle var(new Variable(initializer_)); 55 | get_global_tape().AddOp(initializer_, {}, {{"Out", {var}}}, attrs_); 56 | return var; 57 | } 58 | 59 | private: 60 | const std::string initializer_; 61 | const framework::AttributeMap attrs_; 62 | }; 63 | 64 | class Linear { 65 | public: 66 | Linear(int in_dim, int out_dim, const std::string &act = "") : act_(act) { 67 | // Use Xavier to initialize Weight 68 | float limit = sqrt(6.0 / static_cast(in_dim + out_dim)); 69 | framework::AttributeMap attrs; 70 | attrs["shape"] = std::vector{in_dim, out_dim}; 71 | attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; 72 | attrs["min"] = -limit; 73 | attrs["max"] = limit; 74 | attrs["seed"] = RandomSeed::GetRandomSeed(); 75 | w_ = GlobalParameterCollection().AddParameter( 76 | "LinearWeight", "uniform_random", attrs); 77 | 78 | // Use fill zero to initialize Bias 79 | attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; 80 | attrs["shape"] = std::vector{out_dim}; 81 | attrs["value"] = 0.0f; 82 | b_ = GlobalParameterCollection().AddParameter( 83 | "LinearBias", "fill_constant", attrs); 84 | } 85 | 86 | Linear(const std::vector ¶ms, const std::string &act) 87 | : act_(act), w_(params[0]), b_(params[1]) {} 88 | 89 | VariableHandle operator()(VariableHandle input, 90 | const framework::AttributeMap &mul_op_attrs = {}, 91 | const framework::AttributeMap &add_op_attrs = {}) { 92 | VariableHandle pre_bias(new Variable("linear")); 93 | get_global_tape().AddOp("mul", 94 | {{"X", {input}}, {"Y", {w_}}}, 95 | {{"Out", {pre_bias}}}, 96 | mul_op_attrs); 97 | VariableHandle pre_act(new Variable("linear")); 98 | get_global_tape().AddOp("elementwise_add", 99 | {{"X", {pre_bias}}, {"Y", {b_}}}, 100 | {{"Out", {pre_act}}}, 101 | add_op_attrs); 102 | if (act_.empty()) { 103 | return pre_act; 104 | } 105 | VariableHandle post_act(new Variable("linear")); 106 | get_global_tape().AddOp( 107 | act_, {{"X", {pre_act}}}, {{"Out", {post_act}}}, {}); 108 | return post_act; 109 | } 110 | 111 | std::vector ParamNames() { return {w_->Name(), b_->Name()}; } 112 | 113 | std::string ActName() { return act_; } 114 | 115 | private: 116 | ParameterHandle w_; 117 | ParameterHandle b_; 118 | std::string act_; 119 | }; 120 | 121 | class Convolution2D { 122 | public: 123 | Convolution2D(int c_in, int c_out, int f_size, const std::string &act = "") 124 | : act_(act) { 125 | // Use Xavier to initialize Weight 126 | float fan_in = c_in * f_size * f_size, fan_out = c_out * f_size * f_size; 127 | float limit = sqrt(6.0 / (fan_in + fan_out)); 128 | framework::AttributeMap attrs; 129 | attrs["shape"] = std::vector{c_out, c_in, f_size, f_size}; 130 | attrs["min"] = -limit; 131 | attrs["max"] = limit; 132 | attrs["seed"] = RandomSeed::GetRandomSeed(); 133 | w_ = GlobalParameterCollection().AddParameter( 134 | "ConvolutionWeight", "uniform_random", attrs); 135 | 136 | // Use fill zero to initialize Bias 137 | attrs["shape"] = std::vector{c_out}; 138 | attrs["value"] = 0.0f; 139 | b_ = GlobalParameterCollection().AddParameter( 140 | "ConvolutionBias", "fill_constant", attrs); 141 | } 142 | 143 | Convolution2D(const std::vector ¶ms, 144 | const std::string &act) 145 | : act_(act), w_(params[0]), b_(params[1]) {} 146 | 147 | VariableHandle operator()( 148 | VariableHandle input, 149 | const framework::AttributeMap &conv_op_attrs = {{"paddings", 150 | std::vector{1, 1}}, 151 | {"use_cudnn", true}}, 152 | const framework::AttributeMap &add_op_attrs = {{"axis", 1}}) { 153 | VariableHandle pre_bias(new Variable("conv")); 154 | get_global_tape().AddOp("conv2d", 155 | {{"Input", {input}}, {"Filter", {w_}}}, 156 | {{"Output", {pre_bias}}}, 157 | conv_op_attrs); 158 | VariableHandle pre_act(new Variable("conv")); 159 | get_global_tape().AddOp("elementwise_add", 160 | {{"X", {pre_bias}}, {"Y", {b_}}}, 161 | {{"Out", {pre_act}}}, 162 | add_op_attrs); 163 | if (act_.empty()) { 164 | return pre_act; 165 | } 166 | VariableHandle post_act(new Variable("conv")); 167 | get_global_tape().AddOp( 168 | act_, {{"X", {pre_act}}}, {{"Out", {post_act}}}, {}); 169 | return post_act; 170 | } 171 | 172 | std::vector ParamNames() { return {w_->Name(), b_->Name()}; } 173 | 174 | std::string ActName() { return act_; } 175 | 176 | private: 177 | ParameterHandle w_; 178 | ParameterHandle b_; 179 | std::string act_; 180 | }; 181 | 182 | class BatchNorm { 183 | public: 184 | explicit BatchNorm(int channel_in, const std::string &act = "") : act_(act) { 185 | // Use fill one to initialize scale and variance 186 | framework::AttributeMap attrs; 187 | attrs["shape"] = std::vector{channel_in}; 188 | attrs["value"] = 1.0f; 189 | scale_ = GlobalParameterCollection().AddParameter( 190 | "BatchNormScale", "fill_constant", attrs); 191 | variance_ = GlobalParameterCollection().AddParameter( 192 | "BatchNormVariance", "fill_constant", attrs); 193 | 194 | // Use fill zero to initialize bias and mean 195 | attrs["value"] = 0.0f; 196 | bias_ = GlobalParameterCollection().AddParameter( 197 | "BatchNormBias", "fill_constant", attrs); 198 | mean_ = GlobalParameterCollection().AddParameter( 199 | "BatchNormMean", "fill_constant", attrs); 200 | 201 | GlobalParameterCollection().MarkNoGrad(variance_->Name()); 202 | GlobalParameterCollection().MarkNoGrad(mean_->Name()); 203 | } 204 | 205 | BatchNorm(const std::vector ¶ms, const std::string &act) 206 | : act_(act), 207 | scale_(params[0]), 208 | bias_(params[1]), 209 | mean_(params[2]), 210 | variance_(params[3]) {} 211 | 212 | VariableHandle operator()(VariableHandle x, 213 | const framework::AttributeMap &attrs = {}) { 214 | VariableHandle pre_act(new Variable("batch_norm")); 215 | VariableHandle tmp_mean(new Variable("tmp_mean")); 216 | VariableHandle tmp_var(new Variable("tmp_var")); 217 | get_global_tape().AddOp("batch_norm", 218 | {{"X", {x}}, 219 | {"Scale", {scale_}}, 220 | {"Bias", {bias_}}, 221 | {"Mean", {mean_}}, 222 | {"Variance", {variance_}}}, 223 | {{"Y", {pre_act}}, 224 | {"MeanOut", {mean_}}, 225 | {"VarianceOut", {variance_}}, 226 | {"SavedMean", {tmp_mean}}, 227 | {"SavedVariance", {tmp_var}}}, 228 | attrs); 229 | if (act_.empty()) { 230 | return pre_act; 231 | } 232 | VariableHandle post_act(new Variable("batch_norm")); 233 | get_global_tape().AddOp( 234 | act_, {{"X", {pre_act}}}, {{"Out", {post_act}}}, {}); 235 | return post_act; 236 | } 237 | 238 | std::vector ParamNames() { 239 | return {scale_->Name(), bias_->Name(), mean_->Name(), variance_->Name()}; 240 | } 241 | 242 | std::string ActName() { return act_; } 243 | 244 | private: 245 | ParameterHandle scale_; 246 | ParameterHandle bias_; 247 | ParameterHandle mean_; 248 | ParameterHandle variance_; 249 | std::string act_; 250 | }; 251 | 252 | // Calculate the top k accuracy of the prediction against the label 253 | VariableHandle accuracy(VariableHandle prediction, 254 | VariableHandle label, 255 | int k = 1) { 256 | // Use top_k op to get top k prediction class labels 257 | VariableHandle topk_values(new Variable("accuracy")); 258 | VariableHandle topk_indices(new Variable("accuracy")); 259 | get_global_tape().AddOp("top_k", 260 | {{"X", {prediction}}}, 261 | {{"Out", {topk_values}}, {"Indices", {topk_indices}}}, 262 | {{"k", k}}); 263 | 264 | VariableHandle acc_out(new Variable("accuracy")); 265 | VariableHandle correct(new Variable("accuracy")); 266 | VariableHandle total(new Variable("accuracy")); 267 | get_global_tape().AddOp( 268 | "accuracy", 269 | {{"Out", {topk_values}}, {"Indices", {topk_indices}}, {"Label", {label}}}, 270 | {{"Accuracy", {acc_out}}, {"Correct", {correct}}, {"Total", {total}}}, 271 | {}); 272 | return acc_out; 273 | } 274 | 275 | VariableHandle pool2d(VariableHandle x, 276 | const framework::AttributeMap &attrs = { 277 | {"strides", std::vector{2, 2}}, 278 | {"use_cudnn", true}}) { 279 | VariableHandle out(new Variable("pool2d")); 280 | get_global_tape().AddOp("pool2d", {{"X", {x}}}, {{"Out", {out}}}, attrs); 281 | return out; 282 | } 283 | 284 | VariableHandle dropout(VariableHandle x, 285 | const framework::AttributeMap &attrs = {}) { 286 | VariableHandle out(new Variable("dropout")); 287 | VariableHandle mask(new Variable("mask")); 288 | get_global_tape().AddOp( 289 | "dropout", {{"X", {x}}}, {{"Out", {out}}, {"Mask", {mask}}}, attrs); 290 | return out; 291 | } 292 | 293 | VariableHandle mean(VariableHandle x) { 294 | VariableHandle out(new Variable("mean")); 295 | get_global_tape().AddOp("mean", {{"X", {x}}}, {{"Out", {out}}}, {}); 296 | return out; 297 | } 298 | 299 | VariableHandle relu(VariableHandle x) { 300 | VariableHandle out(new Variable("relu")); 301 | get_global_tape().AddOp("relu", {{"X", {x}}}, {{"Out", {out}}}, {}); 302 | return out; 303 | } 304 | 305 | VariableHandle softmax(VariableHandle x) { 306 | VariableHandle out(new Variable("softmax")); 307 | get_global_tape().AddOp("softmax", {{"X", {x}}}, {{"Out", {out}}}, {}); 308 | return out; 309 | } 310 | 311 | VariableHandle cross_entropy(VariableHandle x, VariableHandle label) { 312 | VariableHandle out(new Variable("cross_entropy")); 313 | get_global_tape().AddOp( 314 | "cross_entropy", {{"X", {x}}, {"Label", {label}}}, {{"Y", {out}}}, {}); 315 | return out; 316 | } 317 | 318 | VariableHandle add(VariableHandle x, VariableHandle y) { 319 | VariableHandle out(new Variable("add")); 320 | get_global_tape().AddOp( 321 | "elementwise_add", {{"X", {x}}, {"Y", {y}}}, {{"Out", {out}}}, {}); 322 | return out; 323 | } 324 | 325 | VariableHandle CreateRecordioFileReader(std::string filename, 326 | std::vector shape_concat, 327 | std::vector ranks, 328 | std::vector lod_levels) { 329 | std::ifstream infile(filename); 330 | PADDLE_ENFORCE( 331 | infile.good(), 332 | "%s doesn't exist; have you run the corresponding create_recordio.py?", 333 | filename); 334 | 335 | VariableHandle reader(new paddle::tape::Variable("reader")); 336 | 337 | RunOperator("create_recordio_file_reader", 338 | {}, 339 | {{"Out", {reader}}}, 340 | {{"filename", filename}, 341 | {"shape_concat", shape_concat}, 342 | {"ranks", ranks}, 343 | {"lod_levels", lod_levels}}); 344 | 345 | return reader; 346 | } 347 | 348 | VariableHandle CreateBatchReader(VariableHandle reader, int batch_size) { 349 | VariableHandle batch_reader(new paddle::tape::Variable("reader")); 350 | 351 | RunOperator("create_batch_reader", 352 | {{"UnderlyingReader", {reader}}}, 353 | {{"Out", {batch_reader}}}, 354 | {{"batch_size", batch_size}}); 355 | 356 | return batch_reader; 357 | } 358 | 359 | VariableHandle CreateDoubleBufferReader(VariableHandle reader) { 360 | VariableHandle db_reader(new paddle::tape::Variable("reader")); 361 | 362 | RunOperator("create_double_buffer_reader", 363 | {{"UnderlyingReader", {reader}}}, 364 | {{"Out", {db_reader}}}, 365 | {}); 366 | 367 | return db_reader; 368 | } 369 | 370 | void ResetReader(VariableHandle reader) { 371 | PADDLE_ENFORCE(reader->Var().IsType()); 372 | reader->GetMutable()->ReInit(); 373 | } 374 | 375 | std::vector ReadNext(VariableHandle reader, bool repeat) { 376 | PADDLE_ENFORCE(reader->Var().IsType()); 377 | 378 | framework::LoDTensorArray data_holder; 379 | reader->GetMutable()->ReadNext(&data_holder); 380 | if (data_holder.empty()) { 381 | VLOG(5) << "ReInit reader"; 382 | reader->GetMutable()->ReInit(); 383 | reader->GetMutable()->ReadNext(&data_holder); 384 | PADDLE_ENFORCE(!data_holder.empty(), "Error reading file."); 385 | if (!repeat) { 386 | // FIXME(kexinzhao): Doing ReInit here will cause error when using 387 | // double_buffer reader 388 | // reader->GetMutable()->ReInit(); 389 | return {}; 390 | } 391 | } 392 | 393 | std::vector rval; 394 | for (size_t i = 0; i < data_holder.size(); ++i) { 395 | rval.emplace_back(new Variable("data" + std::to_string(i))); 396 | auto *lod_tensor = rval.back()->GetMutable(); 397 | lod_tensor->ShareDataWith(data_holder[i]); 398 | lod_tensor->set_lod(data_holder[i].lod()); 399 | } 400 | 401 | return rval; 402 | } 403 | 404 | } // namespace tape 405 | } // namespace paddle 406 | -------------------------------------------------------------------------------- /src/optimizer.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "src/optimizer.h" 16 | 17 | namespace paddle { 18 | namespace tape { 19 | 20 | SGD::SGD(float learning_rate) : learning_rate_(new Variable("sgd")) { 21 | std::string initializer = "fill_constant"; 22 | framework::AttributeMap attrs; 23 | attrs["shape"] = std::vector{1}; 24 | attrs["value"] = learning_rate; 25 | RunOperator("fill_constant", {}, {{"Out", {learning_rate_}}}, attrs); 26 | } 27 | 28 | void SGD::Step() { 29 | for (auto &w : OptimizableParameters()) { 30 | Update(w); 31 | } 32 | } 33 | 34 | void SGD::Update(ParameterHandle param) { 35 | PADDLE_ENFORCE(get_global_tape().HasBeenBackwarded(), 36 | "optimization must happen after the backward"); 37 | RunOperatorWithKernel("sgd", 38 | {{"Param", {param}}, 39 | {"LearningRate", {learning_rate_}}, 40 | {"Grad", {param->Grad()}}}, 41 | {{"ParamOut", {param}}}, 42 | {}); 43 | } 44 | 45 | Adam::Adam(float learning_rate, float beta1, float beta2) 46 | : learning_rate_(new Variable("adam")), 47 | beta1_pow_var_(new Variable("adam")), 48 | beta2_pow_var_(new Variable("adam")), 49 | beta1_(beta1), 50 | beta2_(beta2) { 51 | std::string initializer = "fill_constant"; 52 | framework::AttributeMap attrs; 53 | attrs["shape"] = std::vector{1}; 54 | attrs["value"] = learning_rate; 55 | RunOperator("fill_constant", {}, {{"Out", {learning_rate_}}}, attrs); 56 | } 57 | 58 | void Adam::Step() { 59 | // Update optimizer parameters 60 | beta1_pow_ *= beta1_; 61 | beta2_pow_ *= beta2_; 62 | std::string initializer = "fill_constant"; 63 | framework::AttributeMap attrs; 64 | attrs["shape"] = std::vector{1}; 65 | attrs["value"] = beta1_pow_; 66 | RunOperator("fill_constant", {}, {{"Out", {beta1_pow_var_}}}, attrs); 67 | attrs["value"] = beta2_pow_; 68 | RunOperator("fill_constant", {}, {{"Out", {beta2_pow_var_}}}, attrs); 69 | // Update model parameters 70 | for (auto &w : OptimizableParameters()) { 71 | Update(w); 72 | } 73 | } 74 | 75 | void Adam::Update(ParameterHandle param) { 76 | PADDLE_ENFORCE(get_global_tape().HasBeenBackwarded(), 77 | "optimization must happen after the backward"); 78 | auto *hyperparams = param->MutableHyperParams("adam"); 79 | // initialize states if they haven't been created 80 | if (hyperparams->empty()) { 81 | framework::AttributeMap attrs; 82 | attrs["shape"] = paddle::framework::vectorize2int( 83 | param->Get().dims()); 84 | attrs["value"] = 0.0f; 85 | ParameterHandle moment1(new Parameter("adam")); 86 | ParameterHandle moment2(new Parameter("adam")); 87 | RunOperator("fill_constant", {}, {{"Out", {moment1}}}, attrs); 88 | RunOperator("fill_constant", {}, {{"Out", {moment2}}}, attrs); 89 | 90 | hyperparams->emplace_back(moment1); 91 | hyperparams->emplace_back(moment2); 92 | } 93 | 94 | PADDLE_ENFORCE_EQ( 95 | hyperparams->size(), 2, "Adam should have two hyperparameters"); 96 | auto moment1 = hyperparams->at(0); 97 | auto moment2 = hyperparams->at(1); 98 | 99 | RunOperatorWithKernel("adam", 100 | {{"Param", {param}}, 101 | {"LearningRate", {learning_rate_}}, 102 | {"Grad", {param->Grad()}}, 103 | {"Moment1", {moment1}}, 104 | {"Moment2", {moment2}}, 105 | {"Beta1Pow", {beta1_pow_var_}}, 106 | {"Beta2Pow", {beta2_pow_var_}}}, 107 | {{"ParamOut", {param}}, 108 | {"Moment1Out", {moment1}}, 109 | {"Moment2Out", {moment2}}}, 110 | {{"beta1", beta1_}, {"beta2", beta2_}}); 111 | } 112 | 113 | void BackwardAndUpdate(VariableHandle target, Optimizer *optimizer) { 114 | get_global_tape().Backward(target); 115 | optimizer->Step(); 116 | } 117 | 118 | } // namespace tape 119 | } // namespace paddle 120 | -------------------------------------------------------------------------------- /src/optimizer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | #include "src/parameter.h" 21 | #include "src/tape.h" 22 | 23 | namespace paddle { 24 | namespace tape { 25 | 26 | class Optimizer { 27 | public: 28 | virtual void Step() = 0; 29 | 30 | virtual void Update(ParameterHandle param) = 0; 31 | 32 | virtual ~Optimizer() {} 33 | }; 34 | 35 | class SGD : public Optimizer { 36 | public: 37 | explicit SGD(float learning_rate); 38 | 39 | void Step() override; 40 | 41 | void Update(ParameterHandle) override; 42 | 43 | private: 44 | VariableHandle learning_rate_; 45 | }; 46 | 47 | class Adam : public Optimizer { 48 | public: 49 | explicit Adam(float learning_rate, float beta1 = 0.9f, float beta2 = 0.999f); 50 | 51 | void Step() override; 52 | 53 | void Update(ParameterHandle param) override; 54 | 55 | private: 56 | VariableHandle learning_rate_; 57 | VariableHandle beta1_pow_var_; 58 | VariableHandle beta2_pow_var_; 59 | float beta1_; 60 | float beta2_; 61 | float beta1_pow_{1.0f}; 62 | float beta2_pow_{1.0f}; 63 | }; 64 | 65 | void BackwardAndUpdate(VariableHandle target, Optimizer *optimizer); 66 | 67 | } // namespace tape 68 | } // namespace paddle 69 | -------------------------------------------------------------------------------- /src/parameter.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "src/parameter.h" 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | namespace paddle { 25 | namespace tape { 26 | 27 | using std::vector; 28 | using std::string; 29 | 30 | constexpr char kTensorFileSuffix[] = ".pd"; 31 | 32 | // borrowed from 33 | // https://stackoverflow.com/questions/874134/find-if-string-ends-with-another-string-in-c 34 | bool ends_with(const string &value, const string &ending) { 35 | if (ending.size() > value.size()) return false; 36 | return std::equal(ending.rbegin(), ending.rend(), value.rbegin()); 37 | } 38 | 39 | ParameterCollection::ParameterCollection(string directory_name) { 40 | PADDLE_ENFORCE_EQ(directory_name.back(), '/'); 41 | DIR *dir = opendir(directory_name.c_str()); 42 | PADDLE_ENFORCE_NOT_NULL(dir); 43 | struct dirent *ent; 44 | while ((ent = readdir(dir)) != nullptr) { 45 | string filename = ent->d_name; 46 | if (ends_with(filename, kTensorFileSuffix)) { 47 | string param_name( 48 | filename, 0, filename.size() - strlen(kTensorFileSuffix)); 49 | ParameterHandle param(new Parameter(param_name, Variable::Suffix::NONE)); 50 | RunOperator("load", 51 | {}, 52 | {{"Out", {param}}}, 53 | {{"file_path", directory_name + filename}}); 54 | // TODO(tonyyang-svail): batchnorm's Varaince and Mean should be added to 55 | // no_grad_ 56 | params_[param->Name()] = param; 57 | } 58 | } 59 | } 60 | 61 | ParameterHandle ParameterCollection::AddParameter( 62 | const string &name, 63 | const string &initializer, 64 | const framework::AttributeMap &attrs) { 65 | ParameterHandle param(new Parameter(name)); 66 | InitParameter(param, initializer, attrs); 67 | params_[param->Name()] = param; 68 | 69 | return param; 70 | } 71 | 72 | vector ParameterCollection::LookUp(vector names) { 73 | vector params; 74 | for (auto &name : names) { 75 | params.push_back(params_[name]); 76 | } 77 | return params; 78 | } 79 | 80 | vector ParameterCollection::OptimizableParameters() { 81 | vector rval; 82 | for (auto &pair : params_) { 83 | if (no_grad_name_.find(pair.first) == no_grad_name_.end()) { 84 | rval.emplace_back(pair.second); 85 | } 86 | } 87 | return rval; 88 | } 89 | 90 | void ParameterCollection::SaveAllParameters(string directory_name) { 91 | if (directory_name.empty()) { 92 | auto t = std::time(nullptr); 93 | auto tm = *std::localtime(&t); 94 | std::stringstream ss; 95 | ss << std::put_time(&tm, "%d-%m-%Y %H-%M-%S"); 96 | directory_name = "/tmp/" + ss.str() + "/"; 97 | } 98 | VLOG(3) << directory_name; 99 | PADDLE_ENFORCE_EQ(directory_name.back(), '/'); 100 | PADDLE_ENFORCE_EQ(mkdir(directory_name.c_str(), 0664), 101 | 0, 102 | "directory %s already exists", 103 | directory_name); 104 | for (auto &pair : params_) { 105 | auto ¶m = pair.second; 106 | VLOG(10) << pair.first; 107 | VLOG(10) << param->Name(); 108 | string file_path = directory_name + param->Name() + kTensorFileSuffix; 109 | RunOperator("save", {{"X", {param}}}, {}, {{"file_path", file_path}}); 110 | } 111 | } 112 | 113 | void ParameterCollection::InitParameter(ParameterHandle param, 114 | const string &initializer, 115 | const framework::AttributeMap &attrs) { 116 | if (initializer == "fill_constant") { 117 | // fill_constant is an Operator instead of OperatorWithKernel 118 | RunOperator(initializer, {}, {{"Out", {param}}}, attrs); 119 | } else { 120 | RunOperatorWithKernel(initializer, {}, {{"Out", {param}}}, attrs); 121 | } 122 | } 123 | 124 | ParameterCollection &GlobalParameterCollection() { 125 | static ParameterCollection pc; 126 | return pc; 127 | } 128 | 129 | vector OptimizableParameters() { 130 | return GlobalParameterCollection().OptimizableParameters(); 131 | } 132 | 133 | } // namespace tape 134 | } // namespace paddle 135 | -------------------------------------------------------------------------------- /src/parameter.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #pragma once 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include "src/tape.h" 23 | 24 | namespace paddle { 25 | namespace tape { 26 | 27 | class Parameter; 28 | using ParameterHandle = std::shared_ptr; 29 | 30 | class Parameter : public Variable { 31 | public: 32 | explicit Parameter(const std::string &name, 33 | Variable::Suffix suffix = Variable::Suffix::COUNT) 34 | : Variable(name, suffix) {} 35 | 36 | std::vector *MutableHyperParams( 37 | const std::string &optimizer) { 38 | PADDLE_ENFORCE(hyperparams_.find(optimizer) != hyperparams_.end(), 39 | "%s optimizer is not supported", 40 | optimizer); 41 | return &hyperparams_[optimizer]; 42 | } 43 | 44 | private: 45 | std::unordered_map> hyperparams_{ 46 | {"adam", {}}}; 47 | }; 48 | 49 | class ParameterCollection { 50 | public: 51 | ParameterCollection() {} 52 | 53 | // Load Parameter from a directory 54 | explicit ParameterCollection(std::string directory_name); 55 | 56 | ParameterHandle AddParameter(const std::string &name, 57 | const std::string &initializer, 58 | const framework::AttributeMap &attrs); 59 | 60 | void MarkNoGrad(const std::string &name) { no_grad_name_.insert(name); } 61 | 62 | std::vector LookUp(std::vector names); 63 | 64 | // All parameters excluding no_grad_name_ 65 | std::vector OptimizableParameters(); 66 | 67 | void SaveAllParameters(std::string directory_name = ""); 68 | 69 | private: 70 | void InitParameter(ParameterHandle param, 71 | const std::string &initializer, 72 | const framework::AttributeMap &attrs); 73 | 74 | std::set no_grad_name_; 75 | std::map params_; 76 | }; 77 | 78 | ParameterCollection &GlobalParameterCollection(); 79 | std::vector OptimizableParameters(); 80 | 81 | } // namespace tape 82 | } // namespace paddle 83 | -------------------------------------------------------------------------------- /src/tape.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "src/tape.h" 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "paddle/fluid/framework/data_type.h" 25 | #include "paddle/fluid/framework/dim.h" 26 | #include "paddle/fluid/framework/op_registry.h" 27 | #include "paddle/fluid/framework/operator.h" 28 | #include "paddle/fluid/framework/scope.h" 29 | #include "paddle/fluid/platform/place.h" 30 | #include "paddle/fluid/pybind/pybind.h" 31 | 32 | namespace paddle { 33 | namespace tape { 34 | 35 | using std::map; 36 | using std::pair; 37 | using std::unordered_map; 38 | using std::string; 39 | using std::unique_ptr; 40 | using std::vector; 41 | 42 | // Temporary Scope for Operator::Run() 43 | class ScopeWrapper : public framework::Scope { 44 | public: 45 | ScopeWrapper(const VariableHandleMap &in_vars, 46 | const VariableHandleMap &out_vars) { 47 | for (auto &v : in_vars) { 48 | for (auto &vv : v.second) { 49 | if (!vars_.count(vv->Name())) { 50 | vars_[vv->Name()].reset(vv->MutableVar()); 51 | } 52 | } 53 | } 54 | for (auto &v : out_vars) { 55 | for (auto &vv : v.second) { 56 | if (!vars_.count(vv->Name())) { 57 | vars_[vv->Name()].reset(vv->MutableVar()); 58 | } 59 | } 60 | } 61 | } 62 | 63 | ~ScopeWrapper() { 64 | for (auto &pair : vars_) { 65 | pair.second.release(); 66 | } 67 | } 68 | }; 69 | 70 | // borrowed from 71 | // https://stackoverflow.com/questions/874134/find-if-string-ends-with-another-string-in-c 72 | inline bool ends_with(string const &value, string const &ending) { 73 | if (ending.size() > value.size()) return false; 74 | return std::equal(ending.rbegin(), ending.rend(), value.rbegin()); 75 | } 76 | 77 | string to_string(const string &type, 78 | const VariableHandleMap &in_vars, 79 | const VariableHandleMap &out_vars, 80 | const framework::AttributeMap &attrs) { 81 | std::stringstream ss; 82 | ss << type << " "; 83 | for (auto ¶m_name : in_vars) { 84 | for (auto &var : param_name.second) { 85 | ss << param_name.first << ":(" << var->Name() << ") "; 86 | } 87 | } 88 | for (auto ¶m_name : out_vars) { 89 | for (auto &var : param_name.second) { 90 | ss << param_name.first << ":(" << var->Name() << ") "; 91 | } 92 | } 93 | return ss.str(); 94 | } 95 | 96 | framework::OpDesc CreateOpDesc(const string &type, 97 | const VariableHandleMap &in_vars, 98 | const VariableHandleMap &out_vars, 99 | const framework::AttributeMap &attrs) { 100 | framework::VariableNameMap inputs; 101 | for (auto ¶m_name : in_vars) { 102 | for (auto &var : param_name.second) { 103 | inputs[param_name.first].emplace_back(var->Name()); 104 | } 105 | } 106 | framework::VariableNameMap outputs; 107 | for (auto ¶m_name : out_vars) { 108 | for (auto &var : param_name.second) { 109 | outputs[param_name.first].emplace_back(var->Name()); 110 | } 111 | } 112 | framework::OpDesc op_desc(type, inputs, outputs, attrs); 113 | op_desc.CheckAttrs(); 114 | return op_desc; 115 | } 116 | 117 | void InferShapeAndVarType(const string &type, 118 | const VariableHandleMap &in_vars, 119 | const VariableHandleMap &out_vars, 120 | const framework::AttributeMap &attrs) { 121 | // Tape only supports LoDTensor 122 | for (auto ¶m2var : out_vars) { 123 | for (auto &var : param2var.second) { 124 | var->GetMutable(); 125 | } 126 | } 127 | 128 | framework::OpDesc op_desc = CreateOpDesc(type, in_vars, out_vars, attrs); 129 | ScopeWrapper scope(in_vars, out_vars); 130 | 131 | // Tape only supports OperatorWithKernel 132 | auto op = framework::OpRegistry::CreateOp(op_desc); 133 | auto *op_with_kernel = 134 | dynamic_cast(op.get()); 135 | PADDLE_ENFORCE_NOT_NULL(op_with_kernel, "%s doesn't have kernel", type); 136 | paddle::framework::RuntimeInferShapeContext infer_shape_ctx(*op_with_kernel, 137 | scope); 138 | op_with_kernel->InferShape(&infer_shape_ctx); 139 | } 140 | 141 | void Tape::AddOp(const string &type, 142 | const VariableHandleMap &in_vars, 143 | const VariableHandleMap &out_vars, 144 | const framework::AttributeMap &attrs) { 145 | PADDLE_ENFORCE(!has_been_backwarded_); 146 | VLOG(6) << "AddOp " << to_string(type, in_vars, out_vars, attrs); 147 | InferShapeAndVarType(type, in_vars, out_vars, attrs); 148 | ops_.emplace_back(type, in_vars, out_vars, attrs); 149 | } 150 | 151 | void Tape::Forward() { 152 | VLOG(3) << "Starting forward -------------------------"; 153 | while (current_position_ < ops_.size()) { 154 | PADDLE_ENFORCE(!has_been_backwarded_); 155 | OpRecord &op = ops_[current_position_]; 156 | framework::OpDesc op_desc = 157 | CreateOpDesc(op.type_, op.inputs_, op.outputs_, op.attrs_); 158 | ScopeWrapper scope(op.inputs_, op.outputs_); 159 | framework::OpRegistry::CreateOp(op_desc)->Run(scope, place_); 160 | current_position_++; 161 | } 162 | VLOG(3) << "Finishing forward -------------------------"; 163 | } 164 | 165 | /* 166 | * Only used in backward 167 | * 168 | * Construct vhm based on name2var, variable_name_map 169 | * 170 | * During the construction, record duplicated gradient and 171 | * uninitialzied gradient. 172 | */ 173 | void DescMapToVarMap(const unordered_map &name2var, 174 | const framework::VariableNameMap &variable_name_map, 175 | VariableHandleMap *vhm, 176 | vector> *dup_grad, 177 | vector> *init_grad, 178 | bool is_output) { 179 | for (auto &p2a : variable_name_map) { 180 | for (auto &arg : p2a.second) { 181 | auto ¶m = p2a.first; 182 | if (name2var.count(arg)) { 183 | (*vhm)[param].push_back(name2var.at(arg)); 184 | } else { 185 | PADDLE_ENFORCE( 186 | ends_with(arg, framework::kGradVarSuffix), 187 | "Backward can only add gradient variable. %s not end with %s", 188 | arg, 189 | framework::kGradVarSuffix); 190 | string name = 191 | arg.substr(0, arg.size() - strlen(framework::kGradVarSuffix)); 192 | PADDLE_ENFORCE(name2var.count(name), "%s not found", name); 193 | if (is_output && 194 | name2var.at(name)->GradExist()) { // Sum duplicated grad 195 | VariableHandle temp_grad(new Variable( 196 | name + framework::kGradVarSuffix + framework::kTempVarName)); 197 | // we want sum duplicated grad to be in-place 198 | // since sum_op uses X[0] == Out to determine inplace 199 | // we assign name2var[name]->Grad to be the first element 200 | dup_grad->emplace_back(name2var.at(name)->Grad(), temp_grad); 201 | (*vhm)[param].emplace_back(temp_grad); 202 | } else if (!is_output && 203 | !name2var.at(name) 204 | ->GradExist()) { // zero initialize empty grad 205 | auto var = name2var.at(name); 206 | init_grad->emplace_back(var, var->Grad()); 207 | (*vhm)[param].push_back(var->Grad()); 208 | } else { 209 | (*vhm)[param].push_back(name2var.at(name)->Grad()); 210 | } 211 | } 212 | } 213 | } 214 | } 215 | 216 | void Tape::Backward(VariableHandle target) { 217 | PADDLE_ENFORCE(!has_been_backwarded_, "A tape can only backward once."); 218 | 219 | Forward(); 220 | 221 | // TODO(tonyyang-svail): check output of last op is target 222 | backward_tape_.reset(new Tape(Place())); 223 | 224 | backward_tape_->AddOp( 225 | "fill_ones_like", {{"X", {target}}}, {{"Out", {target->Grad()}}}, {}); 226 | 227 | for (auto it = ops_.rbegin(); it != ops_.rend(); ++it) { 228 | framework::OpDesc op_desc = 229 | CreateOpDesc(it->type_, it->inputs_, it->outputs_, it->attrs_); 230 | unordered_map grad_to_var; 231 | vector> grad_op_descs = 232 | framework::OpInfoMap::Instance() 233 | .Get(op_desc.Type()) 234 | .GradOpMaker()(op_desc, {}, &grad_to_var, {}); 235 | 236 | for (auto &op_grad_desc : grad_op_descs) { 237 | unordered_map name2var; 238 | for (auto ¶m2vars : it->inputs_) { 239 | for (auto &a : param2vars.second) { 240 | name2var[a->Name()] = a; 241 | } 242 | } 243 | for (auto ¶m2vars : it->outputs_) { 244 | for (auto &a : param2vars.second) { 245 | name2var[a->Name()] = a; 246 | } 247 | } 248 | 249 | vector> 250 | dup_grad; // {grad, grad@temp} 251 | vector> 252 | init_grad; // {var, var_grad} 253 | VariableHandleMap in_vars, out_vars; 254 | DescMapToVarMap(name2var, 255 | op_grad_desc->Inputs(), 256 | &in_vars, 257 | &dup_grad, 258 | &init_grad, 259 | false); 260 | DescMapToVarMap(name2var, 261 | op_grad_desc->Outputs(), 262 | &out_vars, 263 | &dup_grad, 264 | &init_grad, 265 | true); 266 | 267 | for (auto &pair : init_grad) { 268 | backward_tape_->AddOp("fill_zeros_like", 269 | {{"X", {pair.first}}}, 270 | {{"Out", {pair.second}}}, 271 | {}); 272 | } 273 | backward_tape_->AddOp( 274 | op_grad_desc->Type(), in_vars, out_vars, op_grad_desc->GetAttrMap()); 275 | for (auto &pair : dup_grad) { 276 | backward_tape_->AddOp("sum", 277 | {{"X", {pair.first, pair.second}}}, 278 | {{"Out", {pair.first}}}, 279 | {}); 280 | } 281 | } 282 | } 283 | 284 | backward_tape_->Forward(); 285 | has_been_backwarded_ = true; 286 | } 287 | 288 | void GraphVizHelper(std::ostream &ss, 289 | const std::vector &ops, 290 | bool is_forward) { 291 | std::string node_prefix = is_forward ? "_op" : "op_grad"; 292 | ss << "subgraph cluster_" << std::to_string(is_forward ? 0 : 1) << " {\n"; 293 | ss << "node [shape=record,style=filled];\n"; 294 | ss << "style=filled;\n"; 295 | ss << "color=lightgrey;\n"; 296 | for (size_t i = 0; i < ops.size(); ++i) { 297 | auto &op = ops[i]; 298 | std::string op_node = node_prefix + std::to_string(i); 299 | ss << op_node << " [label=" << op.type_ << ", shape=box]\n"; 300 | } 301 | if (is_forward) { 302 | ss << node_prefix + std::to_string(0); 303 | for (size_t i = 1; i < ops.size(); ++i) { 304 | ss << "->" << node_prefix + std::to_string(i); 305 | } 306 | ss << "\nlabel=\"forward tape\""; 307 | } else { 308 | ss << node_prefix + std::to_string(ops.size() - 1); 309 | for (int i = ops.size() - 2; i >= 0; --i) { 310 | ss << "->" << node_prefix + std::to_string(i); 311 | } 312 | ss << " [dir=back]\nlabel=\"backward tape\""; 313 | } 314 | ss << "\n}\n"; 315 | for (size_t i = 0; i < ops.size(); ++i) { 316 | auto &op = ops[i]; 317 | std::string op_node = node_prefix + std::to_string(i); 318 | for (auto &p2a : op.inputs_) { 319 | for (auto &arg : p2a.second) { 320 | ss << "\"" << arg->Name() << "\" -> " << op_node << "\n"; 321 | } 322 | } 323 | for (auto &p2a : op.outputs_) { 324 | for (auto &arg : p2a.second) { 325 | ss << op_node << " -> \"" << arg->Name() << "\"\n"; 326 | } 327 | } 328 | } 329 | } 330 | 331 | std::string Tape::GraphVizString(bool with_backward) { 332 | std::stringstream ss; 333 | 334 | ss << "Please copy to http://www.webgraphviz.com/ for better visualization\n"; 335 | ss << "digraph G {\n"; 336 | GraphVizHelper(ss, ops_, true); 337 | if (with_backward) { 338 | GraphVizHelper(ss, backward_tape_->ops_, false); 339 | } 340 | ss << "}\n"; 341 | 342 | return ss.str(); 343 | } 344 | 345 | void RunOperator(const std::string &type, 346 | const VariableHandleMap &in_vars, 347 | const VariableHandleMap &out_vars, 348 | const framework::AttributeMap &attrs) { 349 | framework::OpDesc op_desc = CreateOpDesc(type, in_vars, out_vars, attrs); 350 | ScopeWrapper scope(in_vars, out_vars); 351 | framework::OpRegistry::CreateOp(op_desc)->Run(scope, 352 | get_global_tape().Place()); 353 | } 354 | 355 | void RunOperatorWithKernel(const std::string &type, 356 | const VariableHandleMap &in_vars, 357 | const VariableHandleMap &out_vars, 358 | const framework::AttributeMap &attrs) { 359 | Tape temp_tape(get_global_tape().Place()); 360 | temp_tape.AddOp(type, in_vars, out_vars, attrs); 361 | temp_tape.Forward(); 362 | } 363 | 364 | Tape &get_global_tape() { 365 | static Tape T; 366 | return T; 367 | } 368 | 369 | void reset_global_tape(const platform::Place &place) { 370 | get_global_tape() = Tape(place); 371 | } 372 | 373 | } // namespace tape 374 | } // namespace paddle 375 | -------------------------------------------------------------------------------- /src/tape.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #pragma once 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "paddle/fluid/framework/operator.h" // framework::kGradVarSuffix 24 | #include "paddle/fluid/framework/variable.h" 25 | #include "paddle/fluid/platform/place.h" 26 | 27 | namespace paddle { 28 | namespace tape { 29 | 30 | class Variable; 31 | using VariableHandle = std::shared_ptr; 32 | using VariableHandleMap = std::map>; 33 | 34 | std::ostream &operator<<(std::ostream &, const Variable &); 35 | 36 | class Variable { 37 | public: 38 | explicit Variable(const std::string &pre_fix); 39 | 40 | enum Suffix { COUNT, GRAD, NONE }; 41 | Variable(const std::string &pre_fix, Suffix suffix); 42 | 43 | ~Variable() { VLOG(10) << "Deleting " << Name(); } 44 | 45 | bool GradExist() { return !grad_.expired(); } 46 | 47 | // Return the gradient of a Variable 48 | VariableHandle Grad(); 49 | 50 | // Evaluate a variable by running Forward() on the global tape 51 | const Variable &Value(); 52 | 53 | // Evaluate and make a copy of Variable data on CPU 54 | VariableHandle FetchValue(); 55 | 56 | std::string Name() const { return name_; } 57 | 58 | const framework::Variable &Var() const { return var_; } 59 | framework::Variable *MutableVar() { return &var_; } 60 | 61 | template 62 | const T &Get() const { 63 | return var_.Get(); 64 | } 65 | 66 | template 67 | T *GetMutable() { 68 | return var_.GetMutable(); 69 | } 70 | 71 | private: 72 | std::string name_; 73 | framework::Variable var_; 74 | 75 | // Not own 76 | std::weak_ptr grad_; 77 | }; 78 | 79 | struct OpRecord { 80 | OpRecord(const std::string &type, 81 | const VariableHandleMap &in_vars, 82 | const VariableHandleMap &out_vars, 83 | const framework::AttributeMap &attrs) 84 | : type_(type), inputs_(in_vars), outputs_(out_vars), attrs_(attrs) {} 85 | 86 | std::string type_; 87 | VariableHandleMap inputs_; 88 | VariableHandleMap outputs_; 89 | framework::AttributeMap attrs_; 90 | }; 91 | 92 | class Tape { 93 | public: 94 | Tape() : place_(platform::CPUPlace()) {} 95 | explicit Tape(const platform::Place &place) : place_(place) {} 96 | 97 | void AddOp(const std::string &type, 98 | const VariableHandleMap &in_vars, 99 | const VariableHandleMap &out_vars, 100 | const framework::AttributeMap &attrs); 101 | void Forward(); 102 | void Backward(VariableHandle target); 103 | 104 | bool HasBeenBackwarded() { return has_been_backwarded_; } 105 | 106 | std::string GraphVizString(bool print_backward = true); 107 | 108 | const platform::Place &Place() { return place_; } 109 | 110 | private: 111 | bool has_been_backwarded_ = false; 112 | size_t current_position_ = 0; 113 | platform::Place place_; 114 | 115 | std::vector ops_; 116 | std::shared_ptr backward_tape_; 117 | }; 118 | 119 | void RunOperator(const std::string &type, 120 | const VariableHandleMap &in_vars, 121 | const VariableHandleMap &out_vars, 122 | const framework::AttributeMap &attrs); 123 | void RunOperatorWithKernel(const std::string &type, 124 | const VariableHandleMap &in_vars, 125 | const VariableHandleMap &out_vars, 126 | const framework::AttributeMap &attrs); 127 | 128 | Tape &get_global_tape(); 129 | 130 | void reset_global_tape(const platform::Place &place = platform::CPUPlace()); 131 | } // namespace tape 132 | } // namespace paddle 133 | -------------------------------------------------------------------------------- /src/test_backward.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include "gtest/gtest.h" 18 | #include "src/function.h" 19 | 20 | using paddle::tape::reset_global_tape; 21 | using paddle::tape::get_global_tape; 22 | 23 | using paddle::framework::LoDTensor; 24 | 25 | using paddle::tape::add; 26 | using paddle::tape::mean; 27 | using paddle::tape::Variable; 28 | using paddle::tape::VariableHandle; 29 | 30 | VariableHandle fill_with_vector(std::vector data) { 31 | VariableHandle var(new Variable("fill")); 32 | auto* tensor = var->GetMutable(); 33 | tensor->Resize({static_cast(data.size())}); 34 | LOG(INFO) << tensor->dims(); 35 | auto* ptr = tensor->mutable_data(paddle::platform::CPUPlace()); 36 | for (size_t i = 0; i < data.size(); ++i) { 37 | ptr[i] = data[i]; 38 | } 39 | return var; 40 | } 41 | 42 | /* 43 | * y = op(x) 44 | * z = op(x) 45 | * loss = y + z 46 | */ 47 | TEST(Backward, TestMultipleAssignment) { 48 | reset_global_tape(); 49 | 50 | auto x = fill_with_vector({42}); 51 | auto y = mean(x); 52 | auto z = mean(x); 53 | auto loss = add(y, z); 54 | 55 | get_global_tape().Backward(loss); 56 | 57 | LOG(INFO) << x->Value(); 58 | LOG(INFO) << x->Grad()->Value(); 59 | PADDLE_ENFORCE_EQ(x->Grad()->Get().data()[0], 2.0); 60 | } 61 | 62 | /* 63 | * loss = x + x 64 | */ 65 | TEST(Backward, TestInplaceSum) { 66 | reset_global_tape(); 67 | 68 | auto x = fill_with_vector({42}); 69 | auto loss = add(x, x); 70 | 71 | get_global_tape().Backward(loss); 72 | 73 | PADDLE_ENFORCE_EQ(x->Grad()->Get().data()[0], 2.0); 74 | } 75 | 76 | /* 77 | * y = op(x) // y@grad is not initialized 78 | * loss = op(z) 79 | */ 80 | TEST(Backward, TestEmptyGrad) { 81 | reset_global_tape(); 82 | auto x = fill_with_vector({42}); 83 | auto y = mean(x); 84 | 85 | auto z = fill_with_vector({42}); 86 | auto loss = mean(z); 87 | 88 | get_global_tape().Backward(loss); 89 | 90 | PADDLE_ENFORCE_EQ(x->Grad()->Get().data()[0], 0.0); 91 | PADDLE_ENFORCE_EQ(y->Grad()->Get().data()[0], 0.0); 92 | PADDLE_ENFORCE_EQ(z->Grad()->Get().data()[0], 1.0); 93 | } 94 | 95 | /* 96 | * vector<> v 97 | * for i in dim(x, 0): 98 | * y = x.Slice(i) 99 | * out = linear(y) 100 | * v.push_back(out) 101 | * loss = v.back() 102 | */ 103 | TEST(Backward, TestForLoop) { reset_global_tape(); } 104 | 105 | int main(int argc, char** argv) { 106 | std::vector places; 107 | places.emplace_back(paddle::platform::CPUPlace()); 108 | paddle::platform::DeviceContextPool::Init(places); 109 | 110 | testing::InitGoogleTest(&argc, argv); 111 | return RUN_ALL_TESTS(); 112 | } 113 | -------------------------------------------------------------------------------- /src/test_parameter.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "src/parameter.h" 19 | 20 | using paddle::framework::LoDTensor; 21 | using paddle::tape::ParameterCollection; 22 | using paddle::tape::ParameterHandle; 23 | 24 | template 25 | bool EnforceClose(ParameterHandle p1, ParameterHandle p2, T epsilon) { 26 | auto& t1 = p1->Get(); 27 | auto& t2 = p2->Get(); 28 | 29 | PADDLE_ENFORCE(t1.numel() == t2.numel()); 30 | for (int i = 0; i < t1.numel(); ++i) { 31 | T d1 = t1.data()[i], d2 = t2.data()[i]; 32 | PADDLE_ENFORCE(d1 - d2 <= epsilon); 33 | PADDLE_ENFORCE(d2 - d1 <= epsilon); 34 | } 35 | } 36 | 37 | TEST(ParameterCollection, TestAddParameter) { 38 | ParameterCollection pc; 39 | pc.AddParameter("w", "fill_constant", {{"shape", std::vector{3}}}); 40 | PADDLE_ENFORCE_EQ(pc.OptimizableParameters().size(), 1); 41 | 42 | auto param = pc.OptimizableParameters()[0]; 43 | for (int i = 0; i < 3; ++i) { 44 | PADDLE_ENFORCE_EQ( 45 | param->Get().data()[i], 0.0); 46 | } 47 | } 48 | 49 | TEST(ParameterCollection, TestLookUp) { 50 | ParameterCollection pc; 51 | pc.AddParameter( 52 | "w", "fill_constant", {{"shape", std::vector{3}}, {"value", 42.0f}}); 53 | auto params = pc.OptimizableParameters(); 54 | auto looked_up_params = pc.LookUp({params[0]->Name()}); 55 | EnforceClose(params[0], looked_up_params[0], 0.0001); 56 | } 57 | 58 | TEST(ParameterCollection, TestSaveLoadAllParameters) { 59 | std::string file_path = "/tmp/test_parameter_save/"; 60 | ParameterCollection pc; 61 | pc.AddParameter("w", "fill_constant", {{"shape", std::vector{3}}}); 62 | pc.SaveAllParameters(file_path); 63 | 64 | ParameterCollection loaded_pc(file_path); 65 | PADDLE_ENFORCE_EQ(loaded_pc.OptimizableParameters().size(), 1); 66 | auto param = loaded_pc.OptimizableParameters()[0]; 67 | EnforceClose(pc.OptimizableParameters()[0], param, 0.0001); 68 | 69 | PADDLE_ENFORCE_EQ(system(std::string("rm -r " + file_path).c_str()), 0); 70 | } 71 | 72 | int main(int argc, char** argv) { 73 | std::vector places; 74 | places.emplace_back(paddle::platform::CPUPlace()); 75 | paddle::platform::DeviceContextPool::Init(places); 76 | 77 | testing::InitGoogleTest(&argc, argv); 78 | return RUN_ALL_TESTS(); 79 | } 80 | -------------------------------------------------------------------------------- /src/test_tape.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "gtest/gtest.h" 16 | #include "src/function.h" 17 | #include "src/optimizer.h" 18 | 19 | using paddle::tape::VariableHandle; 20 | using paddle::tape::Variable; 21 | using paddle::tape::Linear; 22 | using paddle::tape::Convolution2D; 23 | using paddle::tape::SGD; 24 | using paddle::tape::Adam; 25 | using paddle::tape::Fill; 26 | using paddle::tape::BatchNorm; 27 | using paddle::tape::dropout; 28 | using paddle::tape::mean; 29 | using paddle::tape::softmax; 30 | using paddle::tape::cross_entropy; 31 | using paddle::tape::reset_global_tape; 32 | using paddle::tape::get_global_tape; 33 | using paddle::tape::CreateRecordioFileReader; 34 | using paddle::tape::ReadNext; 35 | using paddle::tape::BackwardAndUpdate; 36 | using paddle::framework::LoDTensor; 37 | using paddle::tape::ParameterCollection; 38 | using paddle::tape::ParameterHandle; 39 | using paddle::tape::GlobalParameterCollection; 40 | 41 | template 42 | bool EnforceClose(ParameterHandle p1, ParameterHandle p2, T epsilon) { 43 | auto& t1 = p1->Get(); 44 | auto& t2 = p2->Get(); 45 | 46 | PADDLE_ENFORCE(t1.numel() == t2.numel()); 47 | for (int i = 0; i < t1.numel(); ++i) { 48 | T d1 = t1.data()[i], d2 = t2.data()[i]; 49 | PADDLE_ENFORCE(d1 - d2 <= epsilon); 50 | PADDLE_ENFORCE(d2 - d1 <= epsilon); 51 | } 52 | } 53 | 54 | TEST(Tape, TestDropout) { 55 | std::string initializer = "uniform_random"; 56 | paddle::framework::AttributeMap attrs; 57 | attrs["min"] = -1.0f; 58 | attrs["max"] = 1.0f; 59 | attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; 60 | attrs["seed"] = 123; 61 | attrs["shape"] = std::vector{3, 3}; 62 | Fill filler(initializer, attrs); 63 | 64 | reset_global_tape(); 65 | 66 | auto input = filler(); 67 | auto loss = dropout(input); 68 | LOG(INFO) << input->Value(); 69 | LOG(INFO) << loss->Value(); 70 | 71 | get_global_tape().Backward(loss); 72 | LOG(INFO) << input->Grad()->Value(); 73 | } 74 | 75 | TEST(Tape, TestPool2d) { 76 | std::string initializer = "uniform_random"; 77 | paddle::framework::AttributeMap attrs; 78 | attrs["min"] = -1.0f; 79 | attrs["max"] = 1.0f; 80 | attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; 81 | attrs["seed"] = 123; 82 | attrs["shape"] = std::vector{1, 1, 3, 3}; 83 | Fill filler(initializer, attrs); 84 | 85 | reset_global_tape(); 86 | 87 | auto input = filler(); 88 | auto loss = pool2d(input); 89 | LOG(INFO) << input->Value(); 90 | LOG(INFO) << loss->Value(); 91 | 92 | get_global_tape().Backward(loss); 93 | LOG(INFO) << input->Grad()->Value(); 94 | } 95 | 96 | TEST(Tape, TestBatchNorm) { 97 | BatchNorm bn(4, "relu"); 98 | 99 | Adam adam(0.001); 100 | 101 | std::string initializer = "uniform_random"; 102 | paddle::framework::AttributeMap attrs; 103 | attrs["min"] = -1.0f; 104 | attrs["max"] = 1.0f; 105 | attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; 106 | attrs["seed"] = 123; 107 | attrs["shape"] = std::vector{32, 4, 8, 8}; 108 | Fill filler(initializer, attrs); 109 | 110 | for (int i = 0; i < 5; ++i) { 111 | reset_global_tape(); 112 | 113 | auto input = filler(); 114 | auto loss = bn(input); 115 | 116 | LOG(INFO) << loss->Value(); 117 | BackwardAndUpdate(loss, &adam); 118 | } 119 | } 120 | 121 | TEST(Tape, TestGraph) { 122 | Convolution2D conv1(3, 16, 3, "relu"); 123 | Convolution2D conv2(16, 1, 3, "relu"); 124 | 125 | std::string initializer = "uniform_random"; 126 | paddle::framework::AttributeMap attrs; 127 | attrs["min"] = -1.0f; 128 | attrs["max"] = 1.0f; 129 | attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; 130 | attrs["seed"] = 123; 131 | attrs["shape"] = std::vector{32, 3, 8, 8}; 132 | Fill filler(initializer, attrs); 133 | 134 | reset_global_tape(); 135 | 136 | auto input = filler(); 137 | auto loss = mean(conv2(conv1(input))); 138 | get_global_tape().Backward(loss); 139 | 140 | LOG(INFO) << get_global_tape().GraphVizString(false); 141 | } 142 | 143 | TEST(Tape, TestConv) { 144 | Convolution2D conv1(3, 16, 3, "relu"); 145 | Convolution2D conv2(16, 1, 3, "relu"); 146 | 147 | Adam adam(0.001); 148 | 149 | std::string initializer = "uniform_random"; 150 | paddle::framework::AttributeMap attrs; 151 | attrs["min"] = -1.0f; 152 | attrs["max"] = 1.0f; 153 | attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; 154 | attrs["seed"] = 123; 155 | attrs["shape"] = std::vector{32, 3, 8, 8}; 156 | Fill filler(initializer, attrs); 157 | 158 | for (int i = 0; i < 2; ++i) { 159 | reset_global_tape(); 160 | 161 | auto input = filler(); 162 | auto loss = mean(conv2(conv1(input))); 163 | LOG(INFO) << loss->Value(); 164 | 165 | BackwardAndUpdate(loss, &adam); 166 | } 167 | } 168 | 169 | TEST(Tape, TestMLP) { 170 | Linear linear1(3, 3, "relu"); 171 | Linear linear2(3, 3, "relu"); 172 | 173 | SGD sgd(0.001); 174 | 175 | std::string initializer = "uniform_random"; 176 | paddle::framework::AttributeMap attrs; 177 | attrs["min"] = -1.0f; 178 | attrs["max"] = 1.0f; 179 | attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32; 180 | attrs["seed"] = 123; 181 | attrs["shape"] = std::vector{3, 3}; 182 | Fill filler(initializer, attrs); 183 | 184 | for (int i = 0; i < 2; ++i) { 185 | reset_global_tape(); 186 | 187 | auto input = filler(); 188 | auto loss = mean(linear2(linear1(input))); 189 | LOG(INFO) << loss->Value(); 190 | 191 | BackwardAndUpdate(loss, &sgd); 192 | } 193 | } 194 | 195 | TEST(Tape, TestSaveLoadLayer) { 196 | std::string file_path = "/tmp/test_layer_save_load/"; 197 | Linear linear1(3, 6, "relu"); 198 | Linear linear2(6, 3); 199 | 200 | auto& global_pc = GlobalParameterCollection(); 201 | global_pc.SaveAllParameters(file_path); 202 | 203 | ParameterCollection loaded_pc(file_path); 204 | PADDLE_ENFORCE_EQ(global_pc.OptimizableParameters().size(), 205 | loaded_pc.OptimizableParameters().size()); 206 | 207 | Linear loaded_linear1(loaded_pc.LookUp(linear1.ParamNames()), 208 | linear1.ActName()); 209 | Linear loaded_linear2(loaded_pc.LookUp(linear2.ParamNames()), 210 | linear2.ActName()); 211 | 212 | for (size_t i = 0; i < global_pc.OptimizableParameters().size(); ++i) { 213 | auto old_param = global_pc.OptimizableParameters()[i]; 214 | auto new_param = loaded_pc.OptimizableParameters()[i]; 215 | EnforceClose(old_param, new_param, 0.0001); 216 | PADDLE_ENFORCE_NE(old_param.get(), 217 | new_param.get(), 218 | "Loaded parameter should be in different memory"); 219 | } 220 | 221 | PADDLE_ENFORCE_EQ(linear1.ActName(), loaded_linear1.ActName()); 222 | PADDLE_ENFORCE_EQ(linear2.ActName(), loaded_linear2.ActName()); 223 | 224 | PADDLE_ENFORCE_EQ(system(std::string("rm -r " + file_path).c_str()), 0); 225 | } 226 | 227 | int main(int argc, char** argv) { 228 | std::vector places; 229 | places.emplace_back(paddle::platform::CPUPlace()); 230 | paddle::platform::DeviceContextPool::Init(places); 231 | 232 | testing::InitGoogleTest(&argc, argv); 233 | return RUN_ALL_TESTS(); 234 | } 235 | -------------------------------------------------------------------------------- /src/variable.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "src/tape.h" 16 | 17 | #include "paddle/fluid/framework/lod_tensor_array.h" 18 | #include "paddle/fluid/framework/tensor_util.h" 19 | 20 | namespace paddle { 21 | namespace tape { 22 | 23 | std::ostream& operator<<(std::ostream& os, const Variable& var) { 24 | LOG(INFO) << "Printing " << var.Name(); 25 | if (var.Var().IsType()) { 26 | os << var.Get(); 27 | } else if (var.Var().IsType()) { 28 | framework::LoDTensorArray array = var.Get(); 29 | for (size_t i = 0; i < array.size(); ++i) { 30 | os << "Printing lod_tensor #" << i << " in lod_tensor_array " 31 | << var.Name() << "\n"; 32 | os << array[i] << "\n"; 33 | } 34 | } else { 35 | PADDLE_THROW("Variable type is not in [LOD_TENSOR, LOD_TENSOR_ARRAY]"); 36 | } 37 | return os; 38 | } 39 | 40 | int64_t count() { 41 | static int64_t counter = 0; 42 | return counter++; 43 | } 44 | 45 | Variable::Variable(const std::string& pre_fix) { 46 | name_ = pre_fix + std::to_string(count()); 47 | } 48 | 49 | Variable::Variable(const std::string& pre_fix, Suffix suffix) { 50 | if (suffix == Suffix::COUNT) { 51 | name_ = pre_fix + std::to_string(count()); 52 | } else if (suffix == Suffix::GRAD) { 53 | name_ = pre_fix + framework::kGradVarSuffix; 54 | } else { 55 | name_ = pre_fix; 56 | } 57 | } 58 | 59 | VariableHandle Variable::Grad() { 60 | if (grad_.expired()) { 61 | VariableHandle new_grad(new Variable(name_, Suffix::GRAD)); 62 | grad_ = new_grad; 63 | return new_grad; 64 | } else { 65 | return VariableHandle(grad_); 66 | } 67 | } 68 | 69 | VariableHandle Variable::FetchValue() { 70 | get_global_tape().Forward(); 71 | auto place = this->Get().place(); 72 | auto context = platform::DeviceContextPool::Instance().Get(place); 73 | context->Wait(); 74 | VariableHandle cpu_copy(new Variable("temp")); 75 | framework::TensorCopy(this->Get(), 76 | platform::CPUPlace(), 77 | *context, 78 | cpu_copy->GetMutable()); 79 | context->Wait(); 80 | return cpu_copy; 81 | } 82 | 83 | const Variable& Variable::Value() { 84 | get_global_tape().Forward(); 85 | auto place = this->Get().place(); 86 | if (platform::is_gpu_place(place)) { 87 | auto* context = platform::DeviceContextPool::Instance().Get(place); 88 | context->Wait(); 89 | } 90 | return *this; 91 | } 92 | 93 | } // namespace tape 94 | } // namespace paddle 95 | --------------------------------------------------------------------------------