├── .gitignore ├── .vim-template:.cc ├── .vim-template:.cpp ├── .vim-template:.h ├── .ycm_extra_conf.py ├── LICENSE ├── Makefile ├── README.md ├── developer.txt ├── examples ├── 1_test_dtype │ ├── main.cc │ └── run.sh ├── 2_test_tensor │ ├── main.cc │ └── run.sh ├── 2_test_tensor_pickle │ ├── main.cc │ ├── run.sh │ └── test.bin ├── 3_test_op_arith │ ├── main.cc │ └── run.sh ├── 3_test_op_reduce │ ├── main.cc │ └── run.sh ├── 3_test_op_shape │ ├── main.cc │ └── run.sh ├── 3_test_op_slice │ ├── main.cc │ └── run.sh ├── 4_test_graph_arith │ ├── main.cc │ └── run.sh ├── 4_test_graph_matrix │ ├── main.cc │ └── run.sh ├── 5_test_backward │ ├── main.cc │ └── run.sh ├── mnist │ ├── .gitignore │ ├── Makefile │ ├── download-data.sh │ ├── dumps │ │ └── .gitignore │ ├── main.cc │ ├── mnist.hpp │ └── src ├── newton_method │ ├── in.txt │ ├── main.cc │ └── run.sh ├── stage1 │ ├── Makefile │ ├── data │ │ ├── 1.txt │ │ ├── 2.txt │ │ ├── 3.txt │ │ ├── 4.txt │ │ ├── 5.txt │ │ ├── 6.txt │ │ └── 7.txt │ ├── main.cc │ └── run.sh └── stage2 │ ├── .gitignore │ ├── Makefile │ ├── data │ ├── 1.txt │ ├── 10.txt │ ├── 11.txt │ ├── 2.txt │ ├── 3.txt │ ├── 4.txt │ ├── 5.txt │ ├── 6.txt │ ├── 7.txt │ ├── 8.txt │ └── 9.txt │ ├── main.cc │ └── src └── src ├── core.h ├── core ├── common.h ├── datatype.h ├── op.cc ├── op.h ├── ops │ ├── elemwise.h │ ├── linalg.h │ ├── reduction.h │ ├── shape.h │ └── slice.h ├── pickle.h ├── tensor.cc ├── tensor.h ├── tensor_desc.cc ├── tensor_desc.h ├── tensor_extra_ops.cc ├── tensor_extra_ops.h ├── tensor_impl.h ├── tensor_storage.cc └── tensor_storage.h ├── custom_ops ├── assert.h ├── bind.h └── print.h ├── graph.h ├── graph ├── graph.cc ├── graph.h ├── op.cc ├── op.h ├── ops │ ├── elemwise.cc │ ├── elemwise.h │ ├── grad.h │ ├── linalg.cc │ ├── linalg.h │ ├── netsrc.h │ ├── reduction.cc │ ├── reduction.h │ ├── shape.cc │ ├── shape.h │ ├── slice.cc │ ├── slice.h │ └── update.h ├── tensor.cc └── tensor.h ├── ncg.h ├── nn.h └── nn ├── ops.cc └── ops.h /.gitignore: -------------------------------------------------------------------------------- 1 | /.vscode 2 | 3 | # Prerequisites 4 | *.d 5 | 6 | # Compiled Object files 7 | *.slo 8 | *.lo 9 | *.o 10 | *.obj 11 | 12 | # Precompiled Headers 13 | *.gch 14 | *.pch 15 | 16 | # Compiled Dynamic libraries 17 | *.so 18 | *.dylib 19 | *.dll 20 | 21 | # Fortran module files 22 | *.mod 23 | *.smod 24 | 25 | # Compiled Static libraries 26 | *.lai 27 | *.la 28 | *.a 29 | *.lib 30 | 31 | # Executables 32 | *.exe 33 | *.out 34 | *.app 35 | -------------------------------------------------------------------------------- /.vim-template:.cc: -------------------------------------------------------------------------------- 1 | .vim-template:.cpp -------------------------------------------------------------------------------- /.vim-template:.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * %FFILE% 3 | * Copyright (C) %YEAR% 4 | * 5 | * Distributed under terms of the %LICENSE% license. 6 | */ 7 | 8 | #include "%FILE%.h" 9 | 10 | namespace ncg { 11 | 12 | %HERE% 13 | 14 | } /* !namespace ncg */ 15 | -------------------------------------------------------------------------------- /.vim-template:.h: -------------------------------------------------------------------------------- 1 | /* 2 | * %FFILE% 3 | * Copyright (C) %YEAR% 4 | * 5 | * Distributed under terms of the %LICENSE% license. 6 | */ 7 | 8 | #pragma once 9 | 10 | namespace ncg { 11 | 12 | %HERE% 13 | 14 | } /* !namespace ncg */ 15 | 16 | -------------------------------------------------------------------------------- /.ycm_extra_conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as osp 3 | import ycm_core 4 | 5 | flags = [ 6 | '-std=c++17', 7 | '-I' + osp.join(osp.realpath(osp.dirname(__file__)), 'src') 8 | ] 9 | 10 | SOURCE_EXTENSIONS = ['.cpp', '.cxx', '.cc', '.c'] 11 | 12 | def FlagsForFile(filename, **kwargs): 13 | return { 14 | 'flags': flags, 15 | 'do_cache': True 16 | } 17 | 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 NaiveCompGraph authors. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # $File: Makefile 3 | # $Date: Fri Sep 12 22:51:24 2014 +0800 4 | # 5 | # A single output portable Makefile for 6 | # simple c++ project 7 | 8 | 9 | SRC_DIR = src 10 | INC_DIR = src 11 | OBJ_DIR = obj 12 | 13 | TARGET1 = main1 14 | TARGET2 = main2 15 | TARGET3 = main3 16 | 17 | CXX = $(ENVIRONMENT_OPTIONS) g++ 18 | BIN_TARGET1 = $(TARGET1) 19 | BIN_TARGET2 = $(TARGET2) 20 | BIN_TARGET3 = $(TARGET3) 21 | 22 | INCLUDE_DIR = -I $(SRC_DIR) -I $(INC_DIR) 23 | 24 | CXXFLAGS = -O2 -w 25 | # CXXFLAGS = -O2 -w -fopenmp 26 | # CXXFLAGS = -g 27 | # CXXFLAGS = -pg 28 | 29 | # CXXFLAGS += $(DEFINES) 30 | CXXFLAGS += -std=c++17 31 | # CXXFLAGS += -ansi 32 | # CXXFLAGS += -Wall -Wextra 33 | CXXFLAGS += $(INCLUDE_DIR) 34 | # CXXFLAGS += $(LDFLAGS) 35 | # CXXFLAGS += -pthread -lpthread 36 | # CXXFLAGS += -fPIC 37 | 38 | CXXSOURCES = $(shell find $(SRC_DIR) -name *.cc) 39 | CXXSOURCES_MAIN1 = examples/stage2/main.cc 40 | CXXSOURCES_MAIN2 = examples/stage2/main.cc 41 | CXXSOURCES_MAIN3 = examples/newton_method/main.cc 42 | 43 | OBJS = $(addprefix $(OBJ_DIR)/,$(CXXSOURCES:.cc=.o)) 44 | OBJS_MAIN1 = $(addprefix $(OBJ_DIR)/,$(CXXSOURCES_MAIN1:.cc=.o)) 45 | OBJS_MAIN2 = $(addprefix $(OBJ_DIR)/,$(CXXSOURCES_MAIN2:.cc=.o)) 46 | OBJS_MAIN3 = $(addprefix $(OBJ_DIR)/,$(CXXSOURCES_MAIN3:.cc=.o)) 47 | 48 | OBJS_ALL = $(OBJS) $(OBJS_MAIN1) $(OBJS_MAIN2) $(OBJS_MAIN3) 49 | DEPFILES_ALL = $(OBJS_ALL:.o=.d) 50 | 51 | .PHONY: all clean run rebuild gdb 52 | 53 | all: $(BIN_TARGET1) $(BIN_TARGET2) $(BIN_TARGET3) 54 | 55 | $(OBJ_DIR)/%.d: %.cc 56 | @mkdir -pv $(dir $@) 57 | @echo "[dep] $< ..." 58 | @$(CXX) $(INCLUDE_DIR) $(CXXFLAGS) -MM -MT "$(OBJ_DIR)/$(<:.cc=.o) $(OBJ_DIR)/$(<:.cc=.d)" "$<" > "$@" 59 | 60 | sinclude $(DEPFILES_ALL) 61 | 62 | $(OBJ_DIR)/%.o: %.cc 63 | @echo "[CC] $< ..." 64 | @$(CXX) -c $< $(CXXFLAGS) -o $@ 65 | 66 | $(BIN_TARGET1): $(OBJS) $(OBJS_MAIN1) 67 | @echo $(OBJS) $(OBJS_MAIN1) 68 | @echo "[link] $< ..." 69 | @$(CXX) $(OBJS) $(OBJS_MAIN1) -o $@ $(CXXFLAGS) 70 | 71 | $(BIN_TARGET2): $(OBJS) $(OBJS_MAIN2) 72 | @echo $(OBJS) $(OBJS_MAIN2) 73 | @echo "[link] $< ..." 74 | @$(CXX) $(OBJS) $(OBJS_MAIN2) -o $@ $(CXXFLAGS) 75 | 76 | $(BIN_TARGET3): $(OBJS) $(OBJS_MAIN3) 77 | @echo $(OBJS) $(OBJS_MAIN3) 78 | @echo "[link] $< ..." 79 | @$(CXX) $(OBJS) $(OBJS_MAIN3) -o $@ $(CXXFLAGS) 80 | 81 | clean: 82 | rm -rf $(OBJ_DIR) $(BIN_TARGET1) $(BIN_TARGET2) $(BIN_TARGET3) 83 | 84 | rebuild: 85 | +@make clean 86 | +@make 87 | 88 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NaiveCompGraph 2 | Naive Computation Graph. 3 | 4 | This project is the school project Computation Graph of the course Object-Oriented Programming at Tsinghua University. 5 | 6 | ## Implemented Features 7 | - Tensor (Max Dimension = 15). 8 | - Basic tensor operations: unary arithmetic, binary arithmatic, matrix multiplication, reshaping and broadcasting, slicing, indexing, reducing (min, max, sum, mean). 9 | - Reshaping, broadcasting and slicing are implemtented using the stride trick. No actual data copy needed. Tensor are made contiguous only when necessary. 10 | - Most operations supports non-contiguous input (e.g., except matmul, for performance perpose). Many operations (e.g., arithmatic operations) are optimized when the input view is contiguous. 11 | - Complete (and dynamic) data type support. 12 | - Computation graph. 13 | - Session for storing shared tensors (i.e., variables in Tensorflow or Parameters/Buffers in PyTorch). 14 | - Assign Op for updating variables. 15 | - Gradient for all operations are implemented. 16 | - Second-order gradient is supported. 17 | - Graph operations allow dynamic shapes. E.g., `G::reshape(x, G::shape_cat({x.shape(0), -1}))`. Note that `x.shape(0)` returns a graph tensor (an int64-typed scalar). 18 | - Complete MNIST example. 19 | 20 | ## MNIST Example 21 | 构建了一个两层的神经网络： 22 | Flatten input: `(batch_size, 784)` -> Hidden layer `(batch_size, 512)` -> Logits `(batch_size, 10)`. 23 | 网络使用Tanh激活。SGD learning rate = 0.01。网络初始化用Normal(0, 0.01)初始化Weights，全0初始化Bias。 24 | 25 | 网络结构部分代码： 26 | ``` 27 | MLPModel(std::mt19937 &rng) : rng(rng) { 28 | image = G::placeholder("image", {100, 784}, DTypeName::Float32); 29 | label = G::placeholder("label", {100}, DTypeName::Int64); 30 | linear1 = G::linear("linear1", image, 512, rng); 31 | activation1 = G::tanh(linear1); 32 | logits = G::linear("linear2", activation1, 10, rng); 33 | pred = logits.max(-1)[1]; 34 | 35 | prob = G::softmax(logits, -1); 36 | loss = G::xent_sparse(prob, label, -1).mean(0); 37 | accuracy = (pred.eq(label)).float32().mean(0); 38 | } 39 | ``` 40 | 41 | Linear Layer初始化 42 | ``` 43 | GTensorPtr linear(std::string name, GTensorPtr x, ssize_t output_dim, std::mt19937 &rng, double stddev) { 44 | auto W = variable(name + ":W", ::ncg::rand_normal(rng, x->desc().dtype(), {x->desc().shape(1), output_dim}, 0, stddev)); 45 | auto b = variable(name + ":b", ::ncg::zeros(x->desc().dtype(), {output_dim})); 46 | return matmul(x, W) + b.unsqueeze(0); 47 | } 48 | ``` 49 | 50 | SGD部分代码： 51 | ``` 52 | GTensorVec train_ops(float lr=0.01) { 53 | GTensorVec ops; 54 | auto &graph = get_default_graph(); 55 | 56 | graph.backward(loss); 57 | for (const auto &name : {"linear1:W", "linear2:W", "linear1:b", "linear2:b"}) { 58 | auto W = graph.find_op(name)->outputs()[0]; 59 | auto G = W->grad(loss); 60 | auto new_W = W - G * lr; 61 | ops.push_back(G::assign(W, new_W)); 62 | } 63 | 64 | return ops; 65 | } 66 | ``` 67 | 68 | 训练50Epoch后可以达到准确度97.5%。 69 | 70 | ## Stage 1 71 | ``` 72 | cd examples/stage1 73 | ./run.sh 74 | ``` 75 | 76 | Alternative using Makefile: 77 | ``` 78 | cd examples/stage1 79 | make 80 | ./main < data/.txt 81 | ``` 82 | 83 | ## Stage 2 84 | ``` 85 | cd examples/stage2 86 | make 87 | ./main < data/.txt 88 | ``` 89 | 90 | ## Newton Method 91 | ``` 92 | cd examples/newton_method 93 | make 94 | ./main < in.txt 95 | ``` 96 | 97 | # MNIST MLP 98 | ``` 99 | cd examples/mnist 100 | ./download-data.sh 101 | make 102 | ./main 103 | ``` 104 | 105 | ## Tutorial Examples 106 | 建议阅读顺序（除了stage1的代码需要指定example id，其他所有示例直接运行`./run.sh`即可观察结果）： 107 | 108 | 1. `examples/1_test_dtype` 理解数据类型（data type系统）。 109 | 1. `examples/2_test_tensor` 理解Tensor类型，包括定义，shape，取值。 110 | 1. `examples/2_test_tensor_pickle` 理解数据持久化（Pickle,Unpickle）。 111 | 1. `examples/3_test_op_arith` 理解Op系统，学会创建一个Op（OpAdd），进行运算。 112 | 1. `examples/3_test_op_shape` 深入理解Shape, Axes，学习Reshape，Permute, Expand操作。 113 | 1. `examples/3_test_op_slice` 理解Slice操作，包括Narrow（即Python Slice），IndexSelect和Gather。 114 | 1. `examples/3_test_op_reduce` 理解各种reduce操作，比如`reduce_sum`。 115 | 1. `examples/4_test_graph_arith` 理解Graph系统，学会用`G::op_name`创建Op，用`GraphForwardContext`进行Eval。 116 | 1. `examples/4_test_graph_matrix` 理解Graph系统，进行矩阵运算。 117 | 1. `examples/stage1/print_op.h`, `examples/stage1/cond_op.h`，定义自己的Op。 118 | 119 | ## Manual 120 | 121 | 结构： 122 | ``` 123 | core/ 包含基础定义文件(core.h datatype.h), tensor的实现(tensor.h, tensor.cc), op基础(op.h, op.cc). 124 | ops/ 包含基本op，目前只有基础算数运算(e.g., +, -, *, /) 125 | graph/ 包含图定义相关。包括图op基础(op.h/cc, tensor.h/cc), 拓扑排序相关(graph.h, graph.cc), 和图op(graph/ops/)。 126 | ``` 127 | 128 | Tensor： 129 | 130 | 1. `Tensor`是存放数据的基本单位。 131 | 2. `TensorDesc`定义了Tensor的shape（标量是0维，shape={}, 向量是1维，shape={n}，矩阵是2维，shape={n, m}）。 132 | 3. `TensorStorage`存放数据。 133 | 4. 使用时直接使用`TensorPtr=std::shared_ptr`。 134 | 5. 获取数据使用`tensor_ptr->as()->data_ptr()` (返回`const float *`)或者`tensor_ptr->as()->mutable_data_ptr()` (返回`float *`)。 135 | 6. 可以使用`tensor(DTypeName::Float32, {})`创建空tensor，第二个参数是shape。可以使用`scalar(DTypeName::Float32, 1.0)`创建数值内容为1.0，数据类型为float32的标量tensor。 136 | 137 | Op: 138 | 139 | 1. `Op`是关于Tensor的操作，使用时需要重载两个函数`check_inputs`和`compute`。 140 | 2. `check_inputs(OpContext &ctx, const TensorVec &inputs)`检查用户输入的tensors，`TensorVec=std::vector`。 141 | 3. `TensorVec compute(OpContext &ctx, const TensorVec &inputs)`返回计算结果。 142 | 4. `Op`所有的运算都是真实的基于数据的运算，和计算图无关。 143 | 144 | Graph: 145 | 1. `GraphTensor`是定义计算图使用的Tensor类型(类比Tensor)。只有Shape和数据类型信息，没有实际数据。 146 | 2. `GraphOp`是定义计算图使用的Op类型，需要调用实际的`Op`才能进行运算。 147 | 148 | ## Officially Supported Ops 149 | ``` 150 | // Defined in graph/tensor.h 151 | 152 | // elemwise::misc 153 | GTensorPtr cast(TensorPtr a, DTypeName dtype); 154 | GTensorPtr cond(TensorPtr a, TensorPtr b, TensorPtr c); 155 | 156 | // elemwise::unary 157 | GTensorPtr neg(GTensorPtr a); 158 | GTensorPtr sin(GTensorPtr a); 159 | GTensorPtr cos(GTensorPtr a); 160 | GTensorPtr tan(GTensorPtr a); 161 | GTensorPtr log(GTensorPtr a); 162 | GTensorPtr exp(GTensorPtr a); 163 | GTensorPtr tanh(GTensorPtr a); 164 | GTensorPtr sigmoid(GTensorPtr a); 165 | GTensorPtr reciprocal(GTensorPtr a); 166 | 167 | GTensorPtr add(GTensorPtr a, GTensorPtr b); 168 | GTensorPtr sub(GTensorPtr a, GTensorPtr b); 169 | GTensorPtr mul(GTensorPtr a, GTensorPtr b); 170 | GTensorPtr div(GTensorPtr a, GTensorPtr b); 171 | GTensorPtr ge(GTensorPtr a, GTensorPtr b); 172 | GTensorPtr le(GTensorPtr a, GTensorPtr b); 173 | GTensorPtr geq(GTensorPtr a, GTensorPtr b); 174 | GTensorPtr leq(GTensorPtr a, GTensorPtr b); 175 | GTensorPtr eq(GTensorPtr a, GTensorPtr b); 176 | GTensorPtr neq(GTensorPtr a, GTensorPtr b); 177 | GTensorPtr pow(GTensorPtr a, GTensorPtr b); 178 | GTensorPtr min(GTensorPtr a, GTensorPtr b); 179 | GTensorPtr max(GTensorPtr a, GTensorPtr b); 180 | 181 | // netsrc 182 | GTensorPtr placeholder(std::string name, const ShapeVec &shape, DTypeName dtype=DTypeName::Float32); 183 | GTensorPtr constant(TensorPtr value); 184 | GTensorPtr variable(std::string name, TensorPtr init_value); 185 | GTensorPtr zeros(const ShapeVec &shape, DTypeName dtype=DTypeName::Float32); 186 | GTensorPtr ones(const ShapeVec &shape, DTypeName dtype=DTypeName::Float32); 187 | 188 | // linalg 189 | GTensorPtr matmul(GTensorPtr a, GTensorPtr b, bool transpose_a=false, bool transpose_b=false); 190 | 191 | // update 192 | GTensorPtr assign(GTensorPtr a, GTensorPtr b); 193 | 194 | // reduce 195 | GTensorVec reduce_min(GTensorPtr a, ssize_t axis, bool keepdims=false); 196 | GTensorVec reduce_max(GTensorPtr a, ssize_t axis, bool keepdims=false); 197 | GTensorPtr reduce_sum(GTensorPtr a, ssize_t axis, bool keepdims=false); 198 | GTensorPtr reduce_mean(GTensorPtr a, ssize_t axis, bool keepdims=false); 199 | 200 | // shape 201 | GTensorPtr reshape(GTensorPtr a, const ShapeVec &shape); 202 | GTensorPtr permute(GTensorPtr a, const ShapeVec &axes); 203 | GTensorPtr expand(GTensorPtr a, const ShapeVec &shape); 204 | GTensorPtr squeeze(GTensorPtr a, ssize_t axis); 205 | GTensorPtr unsqueeze(GTensorPtr a, ssize_t axis); 206 | 207 | // shape 208 | GTensorPtr shape_of(GTensorPtr a); 209 | GTensorPtr shape_of(GTensorPtr a, ssize_t axis); 210 | GTensorPtr shape_cat(const GTensorVec &a); 211 | 212 | // slice 213 | GTensorPtr concat(const GTensorVec &a, ssize_t axis); 214 | GTensorVec split(GTensorPtr a, ssize_t axis, const ShapeVec &splits); 215 | GTensorPtr narrow(GTensorPtr a, ssize_t axis, ssize_t start, ssize_t length); 216 | GTensorPtr index_select(GTensorPtr a, ssize_t axis, GTensorPtr b); 217 | GTensorPtr gather(GTensorPtr a, ssize_t axis, GTensorPtr b); 218 | ``` 219 | -------------------------------------------------------------------------------- /developer.txt: -------------------------------------------------------------------------------- 1 | 2014011431 茅佳源 2 | -------------------------------------------------------------------------------- /examples/1_test_dtype/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core/common.h" 9 | #include "core/datatype.h" 10 | #include 11 | #include 12 | 13 | namespace ncg { 14 | 15 | } /* !namespace ncg */ 16 | 17 | template 18 | typename ncg::DType
::cctype f() { 19 | return static_cast::cctype>(1.345); 20 | } 21 | 22 | int main() { 23 | std::cout << ncg::DType::name << " " << f() << std::endl; 24 | std::cout << ncg::DType::name << " " << f() << std::endl; 25 | std::cout << ncg::CCType::name << std::endl; 26 | std::cout << ncg::CCType::name << std::endl; 27 | 28 | return 0; 29 | } 30 | -------------------------------------------------------------------------------- /examples/1_test_dtype/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc -I ../../src/ -o main -std=c++17 && ./main && rm -f main 2 | -------------------------------------------------------------------------------- /examples/2_test_tensor/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core/tensor.h" 9 | #include 10 | 11 | namespace ncg { 12 | 13 | } /* !namespace ncg */ 14 | 15 | using namespace ncg; 16 | using namespace std; 17 | 18 | int main() { 19 | auto t1 = empty(DTypeName::Float32, {3, 3}); 20 | cerr << t1 << endl; 21 | auto t2 = empty(DTypeName::Float32, {}); 22 | cerr << t2 << endl; 23 | 24 | return 0; 25 | } 26 | -------------------------------------------------------------------------------- /examples/2_test_tensor/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc -I ../../src/ -o main -std=c++17 && ./main && rm -f main 2 | -------------------------------------------------------------------------------- /examples/2_test_tensor_pickle/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core/tensor.h" 9 | #include "core/tensor_impl.h" 10 | #include 11 | 12 | namespace ncg { 13 | 14 | } /* !namespace ncg */ 15 | 16 | using namespace ncg; 17 | using namespace std; 18 | 19 | ostream &print_tensor(ostream &out, TensorPtr tensor) { 20 | ncg_assert(tensor->desc().dim() == 2); 21 | out << "["; 22 | for (ssize_t i = 0; i < tensor->desc().shape(0); ++i) { 23 | if (i != 0) out << endl << " "; 24 | out << "["; 25 | for (ssize_t j = 0; j < tensor->desc().shape(1); ++j) { 26 | if (j != 0) out << ", "; 27 | out << tensor->as()->at(i, j); 28 | } 29 | out << "],"; 30 | } 31 | 32 | return out; 33 | } 34 | 35 | int main() { 36 | auto t1 = arange(DTypeName::Float32, 18).reshape({9, 2}); 37 | print_tensor(cerr << "t1:" << std::endl, t1) << std::endl; 38 | 39 | NCGPickler pkl("test.bin"); 40 | t1->pickle(pkl); 41 | pkl.close(); 42 | NCGUnpickler unpkl("test.bin"); 43 | auto t2 = tensor(unpkl); 44 | unpkl.close(); 45 | 46 | print_tensor(cerr << "t2:" << std::endl, t2) << std::endl; 47 | 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /examples/2_test_tensor_pickle/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc -I ../../src/ -o main -std=c++17 && ./main && rm -f main 2 | -------------------------------------------------------------------------------- /examples/2_test_tensor_pickle/test.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vacancy/NaiveCompGraph/2315b85d576bd205d47d1210ddd315f841d39b3e/examples/2_test_tensor_pickle/test.bin -------------------------------------------------------------------------------- /examples/3_test_op_arith/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core.h" 9 | #include 10 | 11 | namespace ncg { 12 | 13 | } /* !namespace ncg */ 14 | 15 | using namespace ncg; 16 | using namespace std; 17 | 18 | int main() { 19 | auto t1 = scalar(DTypeName::Float32, 1); 20 | auto t2 = scalar(DTypeName::Float32, 2); 21 | auto t3 = t1 + t2; 22 | 23 | cerr << *t1->as() << endl; 24 | cerr << *t2->as() << endl; 25 | cerr << *t3->as() << endl; 26 | 27 | return 0; 28 | } 29 | -------------------------------------------------------------------------------- /examples/3_test_op_arith/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc -I ../../src/ -o main -std=c++17 && ./main && rm -f main 2 | -------------------------------------------------------------------------------- /examples/3_test_op_reduce/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core.h" 9 | #include 10 | 11 | namespace ncg { 12 | 13 | } /* !namespace ncg */ 14 | 15 | using namespace ncg; 16 | using namespace std; 17 | 18 | ostream &print_tensor(ostream &out, TensorPtr tensor) { 19 | ncg_assert(tensor->desc().dim() == 2); 20 | out << "["; 21 | for (ssize_t i = 0; i < tensor->desc().shape(0); ++i) { 22 | if (i != 0) out << endl << " "; 23 | out << "["; 24 | for (ssize_t j = 0; j < tensor->desc().shape(1); ++j) { 25 | if (j != 0) out << ", "; 26 | out << tensor->as()->at(i, j); 27 | } 28 | out << "],"; 29 | } 30 | 31 | return out; 32 | } 33 | 34 | int main() { 35 | #define P(tensor) do { \ 36 | auto tensor_val = (tensor); \ 37 | cerr << #tensor << ":" << tensor_val << endl; \ 38 | if (tensor_val->desc().dim() == 2) { \ 39 | print_tensor(cerr, tensor_val) << endl; \ 40 | } \ 41 | } while(0) 42 | 43 | auto t1 = arange(DTypeName::Float32, 24); 44 | P(t1); 45 | auto t2 = t1.reshape({2, 3, 4}); 46 | P(t2); 47 | 48 | P(t2.min(2)[0]); 49 | P(t2.max(-1)[0]); 50 | P(t2.sum(2)); 51 | P(t2.mean(-1)); 52 | 53 | return 0; 54 | } 55 | -------------------------------------------------------------------------------- /examples/3_test_op_reduce/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc -I ../../src/ -o main -std=c++17 && ./main && rm -f main 2 | -------------------------------------------------------------------------------- /examples/3_test_op_shape/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core.h" 9 | #include 10 | 11 | namespace ncg { 12 | 13 | } /* !namespace ncg */ 14 | 15 | using namespace ncg; 16 | using namespace std; 17 | 18 | ostream &print_tensor(ostream &out, TensorPtr tensor) { 19 | ncg_assert(tensor->desc().dim() == 2); 20 | out << "["; 21 | for (ssize_t i = 0; i < tensor->desc().shape(0); ++i) { 22 | if (i != 0) out << endl << " "; 23 | out << "["; 24 | for (ssize_t j = 0; j < tensor->desc().shape(1); ++j) { 25 | if (j != 0) out << ", "; 26 | out << tensor->as()->at(i, j); 27 | } 28 | out << "],"; 29 | } 30 | 31 | return out; 32 | } 33 | 34 | int main() { 35 | #define P(tensor) do { \ 36 | cerr << #tensor << ":" << tensor << endl; \ 37 | if (tensor->desc().dim() == 2) { \ 38 | print_tensor(cerr, tensor) << endl; \ 39 | } \ 40 | } while(0) 41 | 42 | auto t1 = arange(DTypeName::Float32, 12); 43 | P(t1); 44 | auto t2 = t1.reshape({3, 4}); 45 | P(t2); 46 | auto t3 = t2.permute({1, 0}); 47 | P(t3); 48 | auto t4 = t3.reshape({4, 3, 1}); 49 | P(t4); 50 | auto t5 = t4.expand({4, 3, 2}); 51 | P(t5); 52 | auto t6 = t5.reshape({4, 6}); 53 | P(t6); 54 | 55 | return 0; 56 | } 57 | -------------------------------------------------------------------------------- /examples/3_test_op_shape/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc -I ../../src/ -o main -std=c++17 && ./main && rm -f main 2 | -------------------------------------------------------------------------------- /examples/3_test_op_slice/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core.h" 9 | #include 10 | 11 | namespace ncg { 12 | 13 | } /* !namespace ncg */ 14 | 15 | using namespace ncg; 16 | using namespace std; 17 | 18 | ostream &print_tensor(ostream &out, TensorPtr tensor) { 19 | ncg_assert(tensor->desc().dim() == 2); 20 | out << "["; 21 | for (ssize_t i = 0; i < tensor->desc().shape(0); ++i) { 22 | if (i != 0) out << endl << " "; 23 | out << "["; 24 | for (ssize_t j = 0; j < tensor->desc().shape(1); ++j) { 25 | if (j != 0) out << ", "; 26 | out << tensor->as()->at(i, j); 27 | } 28 | out << "],"; 29 | } 30 | 31 | return out; 32 | } 33 | 34 | int main() { 35 | #define P(tensor) do { \ 36 | cerr << #tensor << ":" << tensor << endl; \ 37 | if (tensor->desc().dim() == 2) { \ 38 | print_tensor(cerr, tensor) << endl; \ 39 | } \ 40 | } while(0) 41 | 42 | auto t1 = arange(DTypeName::Float32, 12); 43 | P(t1); 44 | auto t2 = t1.reshape({3, 4}); 45 | P(t2); 46 | auto t3 = t2.narrow(1, 1, 2); 47 | P(t3); 48 | auto t4 = concat({t3, t3, t3}, 0); 49 | P(t4); 50 | auto t5 = split(t4, 0, {1, 4, 4}); 51 | P(t5[0]); P(t5[1]); P(t5[2]); 52 | 53 | auto tindex = as_tensor({0, 2, 2, 2, 4}); 54 | auto t6 = t4.index_select(0, tindex); 55 | P(t6); 56 | 57 | auto tgindex = fromcc(DTypeName::Int64, {0, 1, 1, 1, 0, 1, 1, 1, 0}); 58 | auto t7 = t4.gather(1, tgindex.reshape({9, 1})); 59 | P(t7); 60 | 61 | return 0; 62 | } 63 | -------------------------------------------------------------------------------- /examples/3_test_op_slice/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc -I ../../src/ -o main -std=c++17 && ./main && rm -f main 2 | -------------------------------------------------------------------------------- /examples/4_test_graph_arith/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "ncg.h" 9 | #include 10 | 11 | namespace ncg { 12 | 13 | } /* !namespace ncg */ 14 | 15 | using namespace ncg; 16 | using namespace std; 17 | 18 | int main() { 19 | auto x = G::placeholder("x", {}, DTypeName::Float32); 20 | auto y = G::placeholder("y", {}, DTypeName::Float32); 21 | auto z = x + y; 22 | 23 | cout << z << endl; 24 | cout << *(z->owner_op()) << endl; 25 | 26 | GraphForwardContext ctx; 27 | ctx.feed("x", scalar(DTypeName::Float32, 1)); 28 | ctx.feed("y", scalar(DTypeName::Float32, 2)); 29 | auto outputs = ctx.eval({z}); 30 | 31 | if (ctx.ok()) { 32 | cout << outputs[0] << endl; 33 | } else { 34 | cerr << ctx.error_str() << endl; 35 | } 36 | 37 | return 0; 38 | } 39 | 40 | -------------------------------------------------------------------------------- /examples/4_test_graph_arith/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc ../../src/graph/*.cc ../../src/graph/ops/*.cc -I ../../src/ -o main -std=c++17 && ./main && rm -f main 2 | -------------------------------------------------------------------------------- /examples/4_test_graph_matrix/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "ncg.h" 9 | #include 10 | 11 | namespace ncg { 12 | 13 | } /* !namespace ncg */ 14 | 15 | using namespace ncg; 16 | using namespace std; 17 | 18 | int main() { 19 | auto x = G::placeholder("x", {3, 4}); 20 | auto y = G::placeholder("y", {4, 5}); 21 | auto bias = as_gtensor({1, 2, 3, 4, 5}); 22 | auto z = G::matmul(x, y) + bias.unsqueeze(0) + float(1); 23 | 24 | cout << z << endl; 25 | cout << *(z->owner_op()) << endl; 26 | 27 | GraphForwardContext ctx; 28 | ctx.feed("x", ones(DTypeName::Float32, {3, 4})); 29 | ctx.feed("y", ones(DTypeName::Float32, {4, 5})); 30 | auto outputs = ctx.eval({z}); 31 | 32 | if (ctx.ok()) { 33 | cout << outputs[0] << endl; 34 | } else { 35 | cerr << ctx.error_str() << endl; 36 | } 37 | 38 | return 0; 39 | } 40 | 41 | -------------------------------------------------------------------------------- /examples/4_test_graph_matrix/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc ../../src/graph/*.cc ../../src/graph/ops/*.cc -I ../../src/ -o main -std=c++17 && ./main && rm -f main 2 | -------------------------------------------------------------------------------- /examples/5_test_backward/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "ncg.h" 9 | #include 10 | 11 | namespace ncg { 12 | 13 | } /* !namespace ncg */ 14 | 15 | using namespace ncg; 16 | using namespace std; 17 | 18 | int main() { 19 | Graph graph; 20 | auto x = graph.op("x", OpDescPtr(new GOpPlaceholderDesc(DTypeName::Float32, {}))); 21 | auto y = graph.op("y", OpDescPtr(new GOpConstantDesc(scalar(DTypeName::Float32, 2)))); 22 | 23 | auto z1 = graph.op(nullptr, x, x); 24 | auto z2 = graph.op(nullptr, x, x); 25 | auto z3 = graph.op(nullptr, x, y); 26 | 27 | graph.backward(z1); 28 | graph.backward(z2); 29 | graph.backward(z3); 30 | auto gx1 = x->grad(z1); 31 | auto gx2 = x->grad(z2); 32 | auto gx3 = x->grad(z3); 33 | 34 | if (!graph.ok()) { 35 | cerr << graph.error_str() << endl; 36 | return 0; 37 | } 38 | 39 | cout << *gx1 << ", " << *gx2 << ", " << *gx3 << endl; 40 | 41 | Session session(graph); 42 | GraphForwardContext ctx(session); 43 | ctx.feed("x", scalar(DTypeName::Float32, 3)); 44 | auto outputs = ctx.eval({gx1, gx2, gx3}); 45 | 46 | if (ctx.ok()) { 47 | cout << *(outputs[0]->as()) << endl; 48 | cout << *(outputs[1]->as()) << endl; 49 | cout << *(outputs[2]->as()) << endl; 50 | } else { 51 | cerr << ctx.error_str() << endl; 52 | } 53 | 54 | return 0; 55 | } 56 | 57 | -------------------------------------------------------------------------------- /examples/5_test_backward/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc ../../src/graph/*.cc ../../src/graph/ops/*.cc -I ../../src/ -o main -std=c++17 && ./main && rm -f main 2 | -------------------------------------------------------------------------------- /examples/mnist/.gitignore: -------------------------------------------------------------------------------- 1 | /data 2 | /main 3 | /bin 4 | /obj 5 | -------------------------------------------------------------------------------- /examples/mnist/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # $File: Makefile 3 | # $Date: Fri Sep 12 22:51:24 2014 +0800 4 | # 5 | # A single output portable Makefile for 6 | # simple c++ project 7 | 8 | 9 | SRC_DIR = src 10 | INC_DIR = src 11 | OBJ_DIR = obj 12 | TARGET = main 13 | 14 | CXX = $(ENVIRONMENT_OPTIONS) g++ 15 | #CXX = clang++ 16 | 17 | BIN_TARGET = $(TARGET) 18 | #PROF_FILE = $(BIN_TARGET).prof 19 | 20 | INCLUDE_DIR = -I $(SRC_DIR) -I $(INC_DIR) 21 | 22 | # CXXFLAGS = -O3 -w 23 | CXXFLAGS = -O3 -w 24 | # CXXFLAGS = -g 25 | # CXXFLAGS = -pg 26 | 27 | #CXXFLAGS += $(DEFINES) 28 | CXXFLAGS += -std=c++17 29 | #CXXFLAGS += -ansi 30 | #CXXFLAGS += -Wall -Wextra 31 | CXXFLAGS += $(INCLUDE_DIR) 32 | #CXXFLAGS += $(LDFLAGS) 33 | #CXXFLAGS += -pthread -lpthread 34 | #CXXFLAGS += -lopencv_core -lopencv_highgui -lopencv_imgproc 35 | #CXXFLAGS += `pkg-config --libs --cflags hdf5` -lhdf5_hl -lhdf5_cpp 36 | 37 | # CXXFLAGS += -fPIC 38 | 39 | 40 | #CC = /usr/share/clang/scan-build/ccc-analyzer 41 | #CXX = /usr/share/clang/scan-build/c++-analyzer 42 | CXXSOURCES = $(shell find $(SRC_DIR)/ -name "*.cc") 43 | CXXSOURCES += $(shell find . -name "*.cc") 44 | 45 | OBJS = $(addprefix $(OBJ_DIR)/,$(CXXSOURCES:.cc=.o)) 46 | DEPFILES = $(OBJS:.o=.d) 47 | 48 | .PHONY: all clean run rebuild gdb 49 | 50 | all: $(BIN_TARGET) 51 | 52 | $(LIB_DIR): 53 | mkdir $(LIB_DIR) 54 | 55 | $(OBJ_DIR)/%.o: %.cc 56 | @echo "[CC] $< ..." 57 | @$(CXX) -c $< $(CXXFLAGS) -o $@ 58 | 59 | $(OBJ_DIR)/%.d: %.cc 60 | @mkdir -pv $(dir $@) 61 | @echo "[dep] $< ..." 62 | @$(CXX) $(INCLUDE_DIR) $(CXXFLAGS) -MM -MT "$(OBJ_DIR)/$(<:.cc=.o) $(OBJ_DIR)/$(<:.cc=.d)" "$<" > "$@" 63 | 64 | sinclude $(DEPFILES) 65 | 66 | $(BIN_TARGET): $(OBJS) 67 | @echo $(OBJS) 68 | @echo "[link] $< ..." 69 | @$(CXX) $(OBJS) -o $@ $(CXXFLAGS) 70 | @echo have a nice day! 71 | 72 | clean: 73 | rm -rf $(OBJ_DIR) $(BIN_TARGET) 74 | 75 | run: $(BIN_TARGET) 76 | ./$(BIN_TARGET) 77 | 78 | rebuild: 79 | +@make clean 80 | +@make 81 | 82 | gdb: $(BIN_TARGET) 83 | gdb ./$(BIN_TARGET) 84 | -------------------------------------------------------------------------------- /examples/mnist/download-data.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # download-data.sh 4 | # Copyright (C) 2019 Jiayuan Mao 5 | # 6 | # Distributed under terms of the MIT license. 7 | # 8 | 9 | set +e +x 10 | 11 | mkdir -p data 12 | 13 | for fname in train-images-idx3-ubyte.gz train-labels-idx1-ubyte.gz t10k-images-idx3-ubyte.gz t10k-labels-idx1-ubyte.gz 14 | do 15 | curl -o data/$fname http://yann.lecun.com/exdb/mnist/$fname 16 | cd data; gunzip $fname; cd ..; 17 | done 18 | 19 | -------------------------------------------------------------------------------- /examples/mnist/dumps/.gitignore: -------------------------------------------------------------------------------- 1 | /* 2 | !/.gitignore 3 | -------------------------------------------------------------------------------- /examples/mnist/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "mnist.hpp" 9 | #include "ncg.h" 10 | #include "nn/ops.h" 11 | 12 | #include 13 | #include 14 | 15 | namespace ncg { 16 | 17 | std::ostream &print_matrix(std::ostream &out, ncg::TensorPtr tensor) { 18 | ncg_assert(tensor->desc().dim() == 2); 19 | out << "["; 20 | for (ssize_t i = 0; i < tensor->desc().shape(0); ++i) { 21 | if (i != 0) out << std::endl << " "; 22 | out << "["; 23 | for (ssize_t j = 0; j < tensor->desc().shape(1); ++j) { 24 | if (j != 0) out << ", "; 25 | out << tensor->as()->at(i, j); 26 | } 27 | out << "],"; 28 | } 29 | 30 | return out; 31 | } 32 | 33 | } /* !namespace ncg */ 34 | 35 | namespace mnist { 36 | 37 | ncg::TensorPtr ncg_read_mnist_image(std::string filename) { 38 | auto images = ncg::fromcc(ncg::DTypeName::Float32, mnist::read_mnist_image(filename)); 39 | 40 | ncg::OpContext ctx; 41 | auto reshape_op = ncg::OpReshape(); 42 | reshape_op.set_desc(ncg::OpDescPtr(new ncg::OpReshapeDesc({images->desc().shape(0), 1, 28, 28}))); 43 | auto output_vec = reshape_op.execute(ctx, {images}); 44 | ncg_assert_msg(!ctx.is_error(), ctx.error_str()); 45 | return output_vec[0]; 46 | } 47 | 48 | ncg::TensorPtr ncg_read_mnist_label(std::string filename) { 49 | return ncg::fromcc(ncg::DTypeName::Int32, mnist::read_mnist_label(filename)); 50 | } 51 | 52 | void print_data(ncg::TensorPtr raw_images, ncg::TensorPtr raw_labels, ssize_t index) { 53 | auto images = raw_images->as(); 54 | auto labels = raw_labels->as(); 55 | 56 | std::cerr << "Image #" << index << " (Label: " << labels->at(index) << ")" << std::endl; 57 | for (ssize_t i = 0; i < 28; ++i) { 58 | for (ssize_t j = 0; j < 28; ++j) { 59 | if (j != 0) std::cerr << " "; 60 | std::cerr << ((images->at(index, ssize_t(0), i, j) > 0) ? 'X' : ' '); 61 | } 62 | std::cerr << std::endl; 63 | } 64 | } 65 | 66 | class DataLoader { 67 | public: 68 | DataLoader(const ncg::TensorVec &data, ssize_t batch_size, std::mt19937 &rng) : m_data(data), m_batch_size(batch_size), m_shuffle_indices(), m_index(0), m_rng(rng) {} 69 | 70 | ncg::TensorVec next() { 71 | if (m_shuffle_indices == nullptr) { 72 | m_shuffle_indices = ncg::rand_permutation(m_rng, m_data[0]->desc().shape(0)); 73 | } 74 | 75 | ssize_t begin = m_index, end = std::min(m_index + m_batch_size, m_data[0]->desc().shape(0)); 76 | m_index += m_batch_size; 77 | 78 | ncg::TensorVec outputs; 79 | outputs.push_back(m_shuffle_indices.narrow(0, begin, end - begin)); 80 | for (const auto &i : m_data) { 81 | outputs.push_back(i.index_select(0, m_shuffle_indices.narrow(0, begin, end - begin))); 82 | } 83 | 84 | if (m_index >= m_data[0]->desc().shape(0)) { 85 | m_shuffle_indices = nullptr; 86 | m_index = 0; 87 | } 88 | 89 | return outputs; 90 | } 91 | 92 | ssize_t epoch_size() const { 93 | ssize_t n = m_data[0]->desc().shape(0); 94 | return n / m_batch_size + (n % m_batch_size != 0); 95 | } 96 | 97 | private: 98 | ncg::TensorVec m_data; 99 | ssize_t m_batch_size; 100 | 101 | ncg::TensorPtr m_shuffle_indices; 102 | ssize_t m_index; 103 | 104 | std::mt19937 &m_rng; 105 | }; 106 | 107 | } /* !namespace mnist */ 108 | 109 | namespace mnist_model { 110 | 111 | using namespace ncg; 112 | 113 | struct MLPModel { 114 | MLPModel(std::mt19937 &rng) : rng(rng) { 115 | image = G::placeholder("image", {100, 784}, DTypeName::Float32); 116 | label = G::placeholder("label", {100}, DTypeName::Int64); 117 | linear1 = G::linear("linear1", image, 512, rng); 118 | activation1 = G::tanh(linear1); 119 | logits = G::linear("linear2", activation1, 10, rng); 120 | pred = logits.max(-1)[1]; 121 | 122 | prob = G::softmax(logits, -1); 123 | loss = G::xent_sparse(prob, label, -1).mean(0); 124 | accuracy = (pred.eq(label)).float32().mean(0); 125 | } 126 | 127 | GTensorVec train_ops(float lr=0.01) { 128 | GTensorVec ops; 129 | auto &graph = get_default_graph(); 130 | 131 | graph.backward(loss); 132 | for (const auto &name : {"linear1:W", "linear2:W", "linear1:b", "linear2:b"}) { 133 | auto W = graph.find_op(name)->outputs()[0]; 134 | auto G = W->grad(loss); 135 | auto new_W = W - G * lr; 136 | ops.push_back(G::assign(W, new_W)); 137 | } 138 | 139 | return ops; 140 | } 141 | 142 | std::mt19937 &rng; 143 | GTensorPtr image, label, linear1, activation1, logits, prob, pred, loss, accuracy; 144 | }; 145 | 146 | } /* !namespace mnist_model */ 147 | 148 | int main() { 149 | std::cout << std::fixed << std::setprecision(4); 150 | std::cerr << std::fixed << std::setprecision(4); 151 | 152 | std::random_device rd{}; 153 | std::mt19937 rng{rd()}; 154 | 155 | std::cerr << "Loading training data..." << std::endl; 156 | auto train_images = mnist::ncg_read_mnist_image("./data/train-images-idx3-ubyte"); 157 | auto train_labels = mnist::ncg_read_mnist_label("./data/train-labels-idx1-ubyte"); 158 | std::cerr << "Loading test data..." << std::endl; 159 | auto test_images = mnist::ncg_read_mnist_image("./data/t10k-images-idx3-ubyte"); 160 | auto test_labels = mnist::ncg_read_mnist_label("./data/t10k-labels-idx1-ubyte"); 161 | 162 | // for (ssize_t i = 0; i < 10; ++i) { 163 | // mnist::print_data(test_images, test_labels, i); 164 | // } 165 | 166 | std::cerr << "Building data loaders..." << std::endl; 167 | auto train_loader = std::unique_ptr(new mnist::DataLoader({train_images, train_labels}, 100, rng)); 168 | auto test_loader = std::unique_ptr(new mnist::DataLoader({test_images, test_labels}, 100, rng)); 169 | 170 | std::cerr << "Building the MLP model..." << std::endl; 171 | auto model = std::make_unique(rng); 172 | auto train_ops = model->train_ops(); 173 | train_ops.insert(train_ops.begin() + 0, model->loss); 174 | train_ops.insert(train_ops.begin() + 1, model->accuracy); 175 | auto test_ops = {model->loss, model->accuracy}; 176 | 177 | ncg::get_default_session().save_shared_tensors("dumps/saved_model_0.bin"); 178 | 179 | for (int i = 1; i <= train_loader->epoch_size() * 200; ++i) { 180 | auto start = std::chrono::steady_clock::now(); 181 | 182 | auto inputs = train_loader->next(); 183 | ncg::GraphForwardContext ctx; 184 | ctx.feed("image", inputs[1].reshape({-1, 784})); 185 | ctx.feed("label", inputs[2].cast(ncg::DTypeName::Int64)); 186 | auto outputs = ctx.eval(train_ops); 187 | ncg_assert_msg(ctx.ok(), ctx.error_str()); 188 | 189 | auto end = std::chrono::steady_clock::now(); 190 | 191 | auto i_epoch = (i - 1) / train_loader->epoch_size() + 1; 192 | auto i_iter = (i - 1) % train_loader->epoch_size() + 1; 193 | 194 | std::cerr << "Iteration " << i << " [" << i_epoch << "::" << i_iter << "/" << train_loader->epoch_size() << "]: " 195 | << "loss = " << ncg::tocc_scalar(outputs[0]) << ", " 196 | << "accuracy = " << ncg::tocc_scalar(outputs[1]) << ", " 197 | << "time = " << std::chrono::duration_cast(end - start).count() << "(ms)" << "."; 198 | if (i % 100 == 0) std::cerr << std::endl; else std::cerr << "\r"; 199 | 200 | if (i % train_loader->epoch_size() == 0) { 201 | double loss = 0, accuracy = 0; 202 | ssize_t tot = 0; 203 | 204 | for (int j = 1; j <= test_loader->epoch_size(); ++j) { 205 | auto inputs = test_loader->next(); 206 | ncg::GraphForwardContext ctx; 207 | ctx.feed("image", inputs[1].reshape({-1, 784})); 208 | ctx.feed("label", inputs[2].cast(ncg::DTypeName::Int64)); 209 | auto outputs = ctx.eval(test_ops); 210 | ncg_assert_msg(ctx.ok(), ctx.error_str()); 211 | 212 | std::cerr << "Evaluation [" << i_epoch << "::" << j << "/" << test_loader->epoch_size() << "]: " 213 | << "loss = " << ncg::tocc_scalar(outputs[0]) << ", " 214 | << "accuracy = " << ncg::tocc_scalar(outputs[1]) << "."; 215 | if (j % 100 == 0) std::cerr << std::endl; else std::cerr << "\r"; 216 | 217 | auto batch_size = inputs[1]->desc().shape(0); 218 | loss += ncg::tocc_scalar(outputs[0]) * batch_size; 219 | accuracy += ncg::tocc_scalar(outputs[1]) * batch_size; 220 | tot += batch_size; 221 | } 222 | 223 | std::cerr << "Evaluation [" << i << "]: " 224 | << "loss = " << loss / tot << ", " 225 | << "accuracy = " << accuracy / tot << "." 226 | << std::endl; 227 | 228 | ncg::get_default_session().save_shared_tensors(std::string("dumps/saved_model_") + std::to_string(i_epoch) + ".bin"); 229 | } 230 | } 231 | 232 | return 0; 233 | } 234 | 235 | -------------------------------------------------------------------------------- /examples/mnist/mnist.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * mnist.hpp 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | namespace mnist { 14 | 15 | int32_t reverse_int(int32_t i) { 16 | unsigned char ch1, ch2, ch3, ch4; 17 | ch1 = i & 255; 18 | ch2 = (i >> 8) & 255; 19 | ch3 = (i >> 16) & 255; 20 | ch4 = (i >> 24) & 255; 21 | return (int32_t(ch1) << 24) + (int32_t(ch2) << 16) + (int32_t(ch3) << 8) + ch4; 22 | } 23 | 24 | int32_t read_int(std::istream &file) { 25 | int32_t int_value = 0; 26 | file.read(reinterpret_cast(&int_value), sizeof(int_value)); 27 | return reverse_int(int_value); 28 | } 29 | 30 | std::vector> read_mnist_image(std::string filename) { 31 | std::ifstream file(filename, std::ios::binary); 32 | std::vector> output; 33 | if (file.is_open()) { 34 | int32_t magic_number = read_int(file); 35 | int32_t nr_images = read_int(file); 36 | int32_t nr_rows = read_int(file); 37 | int32_t nr_cols = read_int(file); 38 | 39 | output.resize(nr_images, std::vector(nr_rows * nr_cols)); 40 | 41 | for (int i = 0; i < nr_images; ++i) { 42 | for (int r = 0; r < nr_rows; ++r) { 43 | for (int c = 0; c < nr_cols; ++c) { 44 | unsigned char temp = 0; 45 | file.read(reinterpret_cast(&temp), sizeof(temp)); 46 | output[i][(nr_rows * r) + c] = (static_cast(temp) / 255) - 0.5; 47 | } 48 | } 49 | } 50 | } 51 | 52 | return output; 53 | } 54 | 55 | std::vector read_mnist_label(std::string filename) { 56 | std::ifstream file(filename, std::ios::binary); 57 | std::vector output; 58 | if (file.is_open()) { 59 | int32_t magic_number = read_int(file); 60 | int32_t nr_images = read_int(file); 61 | output.resize(nr_images); 62 | 63 | for (int i = 0; i < nr_images; ++i) { 64 | unsigned char temp = 0; 65 | file.read(reinterpret_cast(&temp), sizeof(temp)); 66 | output[i] = int32_t(temp); 67 | } 68 | } 69 | 70 | return output; 71 | } 72 | 73 | void print_data(std::vector> images, std::vector labels, ssize_t index) { 74 | using namespace std; 75 | 76 | cerr << "Image #" << index << " (Label: " << labels[index] << ")" << endl; 77 | for (ssize_t i = 0; i < 28; ++i) { 78 | for (ssize_t j = 0; j < 28; ++j) { 79 | if (j != 0) cerr << " "; 80 | cerr << ((images[index][i * 28 + j] > 0) ? 'X' : ' '); 81 | } 82 | cerr << endl; 83 | } 84 | } 85 | 86 | } /* !namespace mnist */ 87 | 88 | -------------------------------------------------------------------------------- /examples/mnist/src: -------------------------------------------------------------------------------- 1 | ../../src -------------------------------------------------------------------------------- /examples/newton_method/in.txt: -------------------------------------------------------------------------------- 1 | 2 2 | 2 2.0 1.0 0.0 -2.0 3 | 5 1.0 0.0 3.0 2.0 1.0 1.0 -1.0 4 | 5 | -------------------------------------------------------------------------------- /examples/newton_method/main.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * main.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "ncg.h" 9 | #include 10 | #include 11 | 12 | namespace ncg { 13 | 14 | } /* !namespace ncg */ 15 | 16 | using namespace ncg; 17 | using namespace std; 18 | 19 | void solve() { 20 | Graph graph; 21 | Session session(graph); 22 | as_default_graph(graph); 23 | as_default_session(session); 24 | 25 | auto x = G::placeholder("x", {}), y = as_gtensor(float(0)); 26 | 27 | int k; 28 | cin >> k; 29 | for (int i = k; i >= 0; --i) { 30 | double j; 31 | cin >> j; 32 | 33 | float pi = i, ai = j; 34 | y = y + ai * G::pow(x, pi); 35 | } 36 | 37 | graph.backward(y); 38 | auto new_x = x - y / x->grad(y); 39 | 40 | double x_val; 41 | cin >> x_val; 42 | 43 | for (int i = 0; i < 5; ++i) { 44 | GraphForwardContext ctx; 45 | ctx.feed("x", scalar(DTypeName::Float32, x_val)); 46 | auto outputs = ctx.eval({new_x, x, y, x->grad(y)}); 47 | 48 | x_val = outputs[0]->as()->elat(0); 49 | cout << x_val << " "; 50 | 51 | // cerr << "Iter " << i << endl; 52 | // cerr << std::string(80, '=') << endl; 53 | // cerr << " x = " << outputs[1]->as()->elat(0) << endl; 54 | // cerr << " y = " << outputs[2]->as()->elat(0) << endl; 55 | // cerr << "gy/gx = " << outputs[3]->as()->elat(0) << endl; 56 | // cerr << "new_x = " << outputs[0]->as()->elat(0) << endl; 57 | // cerr << endl; 58 | } 59 | cout << endl; 60 | 61 | restore_default_session(); 62 | restore_default_graph(); 63 | } 64 | 65 | int main() { 66 | int k; 67 | cin >> k; 68 | cout << fixed << setprecision(4); 69 | 70 | while (k--) { 71 | solve(); 72 | } 73 | 74 | return 0; 75 | } 76 | 77 | -------------------------------------------------------------------------------- /examples/newton_method/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc ../../src/graph/*.cc ../../src/graph/ops/*.cc -I ../../src/ -o main -std=c++17 && ./main < in.txt && rm -f main 2 | -------------------------------------------------------------------------------- /examples/stage1/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | g++ main.cc ../../src/core/*.cc ../../src/graph/*.cc ../../src/graph/ops/*.cc -I ../../src/ -o main -std=c++17 3 | 4 | clean: 5 | rm main 6 | 7 | # vim:ft=make 8 | # 9 | -------------------------------------------------------------------------------- /examples/stage1/data/1.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | y P 4 | z C 3.0 5 | 4 6 | a = x + y 7 | b = a - z 8 | c = b * a 9 | res = z / c 10 | 5 11 | EVAL x 1 x 1.0 12 | EVAL a 2 x 1.0 y 2.0 13 | EVAL b 2 x 1.0 y 0.0 14 | EVAL c 1 y 1.0 15 | EVAL res 2 x -1.0 y 1.0 16 | 17 | -------------------------------------------------------------------------------- /examples/stage1/data/2.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | y P 4 | z C 3.0 5 | 7 6 | a = SIN z 7 | b = LOG y 8 | c = EXP x 9 | d = SIGMOID c 10 | e = TANH d 11 | t = a + b 12 | res = t * e 13 | 4 14 | EVAL res 2 x 1.0 y 2.0 15 | EVAL res 2 x 1.0 y -1.0 16 | EVAL c 2 x 1.0 y 2.0 17 | EVAL d 2 x 1.0 y 1.0 18 | -------------------------------------------------------------------------------- /examples/stage1/data/3.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | y P 4 | z C 3.0 5 | 5 6 | a = x + y 7 | b = PRINT a 8 | c = b * z 9 | d = PRINT c 10 | res = b * d 11 | 5 12 | EVAL x 1 x 1.0 13 | EVAL a 2 x 1.0 y 2.0 14 | EVAL b 2 x 1.0 y 0.0 15 | EVAL c 2 x 2.0 y 1.0 16 | EVAL res 2 x -1.0 y 1.0 17 | 18 | -------------------------------------------------------------------------------- /examples/stage1/data/4.txt: -------------------------------------------------------------------------------- 1 | 2 2 | x P 3 | z C 3.0 4 | 4 5 | t = PRINT x 6 | a = t + z 7 | b = t + a 8 | c = t + b 9 | 1 10 | EVAL c 1 x 1.0 11 | -------------------------------------------------------------------------------- /examples/stage1/data/5.txt: -------------------------------------------------------------------------------- 1 | 2 2 | x P 3 | y V 1.0 4 | 1 5 | res = x + y 6 | 5 7 | EVAL res 1 x 1.0 8 | SETCONSTANT y 2.0 9 | EVAL res 1 x 2.0 10 | SETANSWER y 3 11 | EVAL res 1 x 1.0 12 | -------------------------------------------------------------------------------- /examples/stage1/data/6.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | y P 4 | z C 3.0 5 | 7 6 | a = x < y 7 | b = a <= y 8 | c = b > a 9 | d = c >= b 10 | e = d == c 11 | t = a + e 12 | res = t * z 13 | 1 14 | EVAL res 2 x 1.0 y 2.0 15 | -------------------------------------------------------------------------------- /examples/stage1/data/7.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | y P 4 | z C 3.0 5 | 5 6 | a = x + y 7 | b = a - z 8 | c = x > y 9 | t = COND c a b 10 | res = COND b a c 11 | 2 12 | EVAL t 2 x 1.0 y -1.0 13 | EVAL res 2 x 1.0 y -1.0 14 | -------------------------------------------------------------------------------- /examples/stage1/run.sh: -------------------------------------------------------------------------------- 1 | g++ main.cc ../../src/core/*.cc ../../src/graph/*.cc ../../src/graph/ops/*.cc -I ../../src/ -o main -std=c++17 && ./main < data/$1.txt && rm -f main 2 | -------------------------------------------------------------------------------- /examples/stage2/.gitignore: -------------------------------------------------------------------------------- 1 | /bin 2 | /obj 3 | -------------------------------------------------------------------------------- /examples/stage2/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # $File: Makefile 3 | # $Date: Fri Sep 12 22:51:24 2014 +0800 4 | # 5 | # A single output portable Makefile for 6 | # simple c++ project 7 | 8 | 9 | SRC_DIR = src 10 | INC_DIR = src 11 | OBJ_DIR = obj 12 | BIN_DIR = bin 13 | LIB_DIR = lib 14 | TARGET = main 15 | 16 | CXX = $(ENVIRONMENT_OPTIONS) g++ 17 | #CXX = clang++ 18 | 19 | BIN_TARGET = $(BIN_DIR)/$(TARGET) 20 | #PROF_FILE = $(BIN_TARGET).prof 21 | 22 | INCLUDE_DIR = -I $(SRC_DIR) -I $(INC_DIR) 23 | 24 | 25 | CXXFLAGS = -O2 -w 26 | # CXXFLAGS = -O2 -w -fopenmp 27 | # CXXFLAGS = -g 28 | # CXXFLAGS = -pg 29 | 30 | 31 | #CXXFLAGS += $(DEFINES) 32 | CXXFLAGS += -std=c++17 33 | #CXXFLAGS += -ansi 34 | #CXXFLAGS += -Wall -Wextra 35 | CXXFLAGS += $(INCLUDE_DIR) 36 | #CXXFLAGS += $(LDFLAGS) 37 | #CXXFLAGS += -pthread -lpthread 38 | #CXXFLAGS += -lopencv_core -lopencv_highgui -lopencv_imgproc 39 | #CXXFLAGS += `pkg-config --libs --cflags hdf5` -lhdf5_hl -lhdf5_cpp 40 | 41 | # CXXFLAGS += -fPIC 42 | 43 | 44 | #CC = /usr/share/clang/scan-build/ccc-analyzer 45 | #CXX = /usr/share/clang/scan-build/c++-analyzer 46 | CXXSOURCES = $(shell find $(SRC_DIR)/ -name "*.cc") 47 | CXXSOURCES += $(shell find . -name "*.cc") 48 | 49 | OBJS = $(addprefix $(OBJ_DIR)/,$(CXXSOURCES:.cc=.o)) 50 | DEPFILES = $(OBJS:.o=.d) 51 | 52 | .PHONY: all clean run rebuild gdb 53 | 54 | all: $(BIN_TARGET) 55 | 56 | $(LIB_DIR): 57 | mkdir $(LIB_DIR) 58 | 59 | $(OBJ_DIR)/%.o: %.cc 60 | @echo "[CC] $< ..." 61 | @$(CXX) -c $< $(CXXFLAGS) -o $@ 62 | 63 | $(OBJ_DIR)/%.d: %.cc 64 | @mkdir -pv $(dir $@) 65 | @echo "[dep] $< ..." 66 | @$(CXX) $(INCLUDE_DIR) $(CXXFLAGS) -MM -MT "$(OBJ_DIR)/$(<:.cc=.o) $(OBJ_DIR)/$(<:.cc=.d)" "$<" > "$@" 67 | 68 | sinclude $(DEPFILES) 69 | 70 | $(BIN_TARGET): $(OBJS) 71 | @echo $(OBJS) 72 | @echo "[link] $< ..." 73 | @mkdir -p $(BIN_DIR) 74 | @$(CXX) $(OBJS) -o $@ $(CXXFLAGS) 75 | @echo have a nice day! 76 | 77 | clean: 78 | rm -rf $(OBJ_DIR) $(BIN_DIR) $(LIB_DIR) 79 | 80 | run: $(BIN_TARGET) 81 | ./$(BIN_TARGET) 82 | 83 | rebuild: 84 | +@make clean 85 | +@make 86 | 87 | gdb: $(BIN_TARGET) 88 | gdb ./$(BIN_TARGET) 89 | -------------------------------------------------------------------------------- /examples/stage2/data/1.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | y P 4 | z C 3.0 5 | 4 6 | a = x + y 7 | b = a - z 8 | c = b * a 9 | res = z / c 10 | 5 11 | EVAL x 1 x 1.0 12 | EVAL a 2 x 1.0 y 2.0 13 | EVAL b 2 x 1.0 y 0.0 14 | EVAL c 1 y 1.0 15 | EVAL res 2 x -1.0 y 1.0 16 | 17 | -------------------------------------------------------------------------------- /examples/stage2/data/10.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | y P 4 | z C 2 5 | 6 6 | t = x * x 7 | L = t * y 8 | g = GRAD L 9 | gt = g AT t 10 | gx = g AT x 11 | gy = g AT y 12 | 3 13 | EVAL gt 2 x 3.0 y 2.0 14 | EVAL gx 2 x 3.0 y 2.0 15 | EVAL gy 2 x 3.0 y 2.0 16 | -------------------------------------------------------------------------------- /examples/stage2/data/11.txt: -------------------------------------------------------------------------------- 1 | 2 2 | x V 1.0 3 | y V 2.0 4 | 5 5 | a = ASSIGN x y 6 | b = a * y 7 | c = ASSIGN y b 8 | d = c * x 9 | res = d * y 10 | 3 11 | EVAL res 0 12 | EVAL x 0 13 | EVAL y 0 14 | -------------------------------------------------------------------------------- /examples/stage2/data/2.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | y P 4 | z C 3.0 5 | 7 6 | a = SIN z 7 | b = LOG y 8 | c = EXP x 9 | d = SIGMOID c 10 | e = TANH d 11 | t = a + b 12 | res = t * e 13 | 4 14 | EVAL res 2 x 1.0 y 2.0 15 | EVAL res 2 x 1.0 y -1.0 16 | EVAL c 2 x 1.0 y 2.0 17 | EVAL d 2 x 1.0 y 1.0 18 | -------------------------------------------------------------------------------- /examples/stage2/data/3.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | y P 4 | z C 3.0 5 | 5 6 | a = x + y 7 | b = PRINT a 8 | c = b * z 9 | d = PRINT c 10 | res = b * d 11 | 5 12 | EVAL x 1 x 1.0 13 | EVAL a 2 x 1.0 y 2.0 14 | EVAL b 2 x 1.0 y 0.0 15 | EVAL c 2 x 2.0 y 1.0 16 | EVAL res 2 x -1.0 y 1.0 17 | 18 | -------------------------------------------------------------------------------- /examples/stage2/data/4.txt: -------------------------------------------------------------------------------- 1 | 2 2 | x P 3 | z C 3.0 4 | 4 5 | t = PRINT x 6 | a = t + z 7 | b = t + a 8 | c = t + b 9 | 1 10 | EVAL c 1 x 1.0 11 | -------------------------------------------------------------------------------- /examples/stage2/data/5.txt: -------------------------------------------------------------------------------- 1 | 2 2 | x P 3 | y V 1.0 4 | 1 5 | res = x + y 6 | 5 7 | EVAL res 1 x 1.0 8 | SETCONSTANT y 2.0 9 | EVAL res 1 x 2.0 10 | SETANSWER y 3 11 | EVAL res 1 x 1.0 12 | -------------------------------------------------------------------------------- /examples/stage2/data/6.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | y P 4 | z C 3.0 5 | 7 6 | a = x < y 7 | b = a <= y 8 | c = b > a 9 | d = c >= b 10 | e = d == c 11 | t = a + e 12 | res = t * z 13 | 1 14 | EVAL res 2 x 1.0 y 2.0 15 | -------------------------------------------------------------------------------- /examples/stage2/data/7.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | y P 4 | z C 3.0 5 | 5 6 | a = x + y 7 | b = a - z 8 | c = x > y 9 | t = COND c a b 10 | res = COND b a c 11 | 2 12 | EVAL t 2 x 1.0 y -1.0 13 | EVAL res 2 x 1.0 y -1.0 14 | -------------------------------------------------------------------------------- /examples/stage2/data/8.txt: -------------------------------------------------------------------------------- 1 | 4 2 | x P 3 | y P 4 | z C 3.0 5 | t C 2.0 6 | 5 7 | a = x + y 8 | b = x + t 9 | d = b - y 10 | c = ASSERT d 11 | res = BIND a c 12 | 2 13 | EVAL res 2 x 1.0 y 2.0 14 | EVAL res 2 x 1.0 y 4.0 15 | -------------------------------------------------------------------------------- /examples/stage2/data/9.txt: -------------------------------------------------------------------------------- 1 | 3 2 | x P 3 | a C 3 4 | b C 2 5 | 7 6 | y = x * a 7 | z = x * b 8 | L = y + z 9 | g = GRAD L 10 | gx = g AT x 11 | gy = g AT y 12 | t = gx + gy 13 | 3 14 | EVAL gx 1 x 3.0 15 | EVAL gy 1 x 3.0 16 | EVAL t 1 x 3.0 17 | -------------------------------------------------------------------------------- /examples/stage2/src: -------------------------------------------------------------------------------- 1 | ../../src -------------------------------------------------------------------------------- /src/core.h: -------------------------------------------------------------------------------- 1 | /* 2 | * core.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core/common.h" 9 | #include "core/datatype.h" 10 | #include "core/tensor.h" 11 | #include "core/tensor_impl.h" 12 | #include "core/tensor_extra_ops.h" 13 | #include "core/op.h" 14 | #include "core/ops/elemwise.h" 15 | #include "core/ops/linalg.h" 16 | #include "core/ops/reduction.h" 17 | #include "core/ops/shape.h" 18 | #include "core/ops/slice.h" 19 | 20 | -------------------------------------------------------------------------------- /src/core/common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * common.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | namespace ncg { 24 | 25 | typedef unsigned int uint; 26 | typedef long long int64; 27 | typedef unsigned long long uint64; 28 | 29 | inline bool is_a_gt_zero_and_lt_b(int a, int b) { 30 | return static_cast(a) < static_cast(b); 31 | } 32 | 33 | // Begin OpenMP {{ 34 | 35 | #ifdef USE_OPENMP 36 | #include 37 | #if defined(_WIN32) || defined(__WIN32__) 38 | #define ompfor __pragma(omp parallel for) for 39 | #define omplock __pragma(omp critical) 40 | #else 41 | #define ompfor _Pragma("omp parallel for") for 42 | #define omplock _Pragma("omp critical") 43 | #endif 44 | const int kNumThreads = omp_get_max_threads(); 45 | inline int omp_thread_id() { return omp_get_thread_num(); } 46 | #else // USE_OPENMP 47 | #define ompfor for 48 | #define omplock 49 | const int kNumThreads = 1; 50 | inline int omp_thread_id() { return 0; } 51 | #endif // USE_OPENMP 52 | 53 | // End OpenMP }} 54 | 55 | // Begin Alignment {{ 56 | 57 | #if defined(_WIN32) || defined(__WIN32__) 58 | #define align_attrib(typ, siz) __declspec(align(siz)) typ 59 | #else 60 | #define align_attrib(typ, siz) typ __attribute__((aligned(siz))) 61 | #endif 62 | 63 | #if defined(_WIN32) || defined(__WIN32__) 64 | inline void* align_alloc(size_t size, size_t alignsize) { 65 | return _aligned_malloc(size, alignsize); 66 | } 67 | inline void align_free(void* mem) { _aligned_free(mem); } 68 | #else 69 | inline void* align_alloc(size_t size, size_t alignsize) { 70 | void* mem = nullptr; 71 | int ret = posix_memalign((void**)&mem, alignsize, size); 72 | return (ret == 0) ? mem : nullptr; 73 | } 74 | inline void align_free(void* mem) { free(mem); } 75 | #endif 76 | 77 | // End Alignment }} 78 | 79 | // Begin Arithmetic {{ 80 | 81 | #if defined(_WIN32) || defined(__WIN32__) 82 | #if _MSC_VER <= 1600 83 | #define isnan(x) _isnan(x) 84 | #define isinf(x) (!_finite(x)) 85 | #endif 86 | #endif 87 | 88 | // End Arithmetic }} 89 | 90 | // Begin Assertion {{ 91 | 92 | #ifndef __FUNCTION_NAME__ 93 | #if defined(_WIN32) || defined(__WIN32__) 94 | #define __FUNCTION_NAME__ __FUNCTION__ 95 | #else 96 | #define __FUNCTION_NAME__ __func__ 97 | #endif 98 | #endif 99 | 100 | #define ncg_assert_msg(PREDICATE, MSG) \ 101 | do { \ 102 | if (!(PREDICATE)) { \ 103 | std::cerr << "Asssertion \"" \ 104 | << #PREDICATE << "\" failed in " << __FILE__ \ 105 | << " line " << __LINE__ \ 106 | << " in function \"" << (__FUNCTION_NAME__) << "\"" \ 107 | << " : " << (MSG) << std::endl; \ 108 | std::abort(); \ 109 | } \ 110 | } while (false) 111 | 112 | #define ncg_assert(PREDICATE) \ 113 | do { \ 114 | if (!(PREDICATE)) { \ 115 | std::cerr << "Asssertion \"" \ 116 | << #PREDICATE << "\" failed in " << __FILE__ \ 117 | << " line " << __LINE__ \ 118 | << " in function \"" << (__FUNCTION_NAME__) << "\"" << std::endl; \ 119 | std::abort(); \ 120 | } \ 121 | } while (false) 122 | 123 | #ifndef NDEBUG 124 | #define ncg_dassert_msg(PREDICATE, MSG) ncg_assert_msg(PREDICATE, MSG) 125 | #define ncg_dassert(PREDICATE) ncg_assert(PREDICATE) 126 | #else // NDEBUG 127 | #define ncg_dassert_msg(PREDICATE, MSG) do {} while(false) 128 | #define ncg_dassert(PREDICATE) do {} while (false) 129 | #endif // NDEBUG 130 | 131 | // End Assertion }} 132 | 133 | class RuntimeContext { 134 | public: 135 | RuntimeContext() : m_is_error(false), m_error() {} 136 | virtual ~RuntimeContext() = default; 137 | 138 | bool ok() const { return !m_is_error; } 139 | bool is_error() const { return m_is_error; } 140 | std::ostringstream &error() { m_is_error = true; return m_error; } 141 | 142 | std::string error_str() const { return m_error.str(); } 143 | void reset_error() { m_is_error = false; m_error.clear(); } 144 | 145 | private: 146 | bool m_is_error; 147 | std::ostringstream m_error; 148 | }; 149 | 150 | template 151 | class DefaultManager { 152 | public: 153 | DefaultManager() : m_stack(), m_has_default(false), m_default_element(nullptr) { 154 | // pass 155 | } 156 | 157 | explicit DefaultManager(bool has_default) : m_stack(), m_has_default(has_default), m_default_element(nullptr) { 158 | if (has_default) { 159 | m_default_element = std::make_unique(); 160 | m_stack.push(m_default_element.get()); 161 | } 162 | } 163 | ~DefaultManager() = default; 164 | 165 | T &get_default() { 166 | ncg_assert(m_stack.size() > 0); 167 | return *(m_stack.top()); 168 | } 169 | void as_default(T *new_default) { 170 | m_stack.push(new_default); 171 | } 172 | void restore_default() { 173 | ncg_assert(m_stack.size() > int(m_has_default)); 174 | m_stack.pop(); 175 | } 176 | 177 | private: 178 | std::stack m_stack; 179 | 180 | bool m_has_default; 181 | std::unique_ptr m_default_element; 182 | }; 183 | 184 | } /* !namespace ncg */ 185 | 186 | -------------------------------------------------------------------------------- /src/core/datatype.h: -------------------------------------------------------------------------------- 1 | /* 2 | * datatype.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/common.h" 11 | 12 | namespace ncg { 13 | 14 | enum class DTypeName : int { 15 | Int8, 16 | UInt8, 17 | Int32, 18 | UInt32, 19 | Int64, 20 | UInt64, 21 | Float32, 22 | Float64, 23 | }; 24 | 25 | template 26 | struct DType { 27 | }; 28 | 29 | template ::value>::type* = nullptr> 30 | struct CCType { 31 | }; 32 | 33 | #define DEF_DTYPE_CCTYPE(identifier_, cctype_) template<> \ 34 | struct DType { \ 35 | using cctype = cctype_; \ 36 | static constexpr char name[] = #identifier_; \ 37 | static constexpr DTypeName identifier = DTypeName::identifier_; \ 38 | }; \ 39 | template<> \ 40 | struct CCType { \ 41 | using cctype = cctype_; \ 42 | static constexpr char name[] = #identifier_; \ 43 | static constexpr DTypeName identifier = DTypeName::identifier_; \ 44 | } 45 | 46 | DEF_DTYPE_CCTYPE(Int8, int8_t); 47 | DEF_DTYPE_CCTYPE(UInt8, uint8_t); 48 | DEF_DTYPE_CCTYPE(Int32, int32_t); 49 | DEF_DTYPE_CCTYPE(UInt32, uint32_t); 50 | DEF_DTYPE_CCTYPE(Int64, int64_t); 51 | DEF_DTYPE_CCTYPE(UInt64, uint64_t); 52 | DEF_DTYPE_CCTYPE(Float32, float); 53 | DEF_DTYPE_CCTYPE(Float64, double); 54 | 55 | #define NCG_DTYPE_SWITCH(dtype_, MACRO_) case DTypeName::dtype_: MACRO_(dtype_); break; 56 | 57 | #define NCG_DTYPE_SWITCH_ALL(dtype_var, MACRO) switch(dtype_var) { \ 58 | NCG_DTYPE_SWITCH(Int8, MACRO); \ 59 | NCG_DTYPE_SWITCH(UInt8, MACRO); \ 60 | NCG_DTYPE_SWITCH(Int32, MACRO); \ 61 | NCG_DTYPE_SWITCH(UInt32, MACRO); \ 62 | NCG_DTYPE_SWITCH(Int64, MACRO); \ 63 | NCG_DTYPE_SWITCH(UInt64, MACRO); \ 64 | NCG_DTYPE_SWITCH(Float32, MACRO); \ 65 | NCG_DTYPE_SWITCH(Float64, MACRO); \ 66 | } 67 | 68 | #define NCG_DTYPE_SWITCH_FLOAT(dtype_var, MACRO) switch(dtype_var) { \ 69 | NCG_DTYPE_SWITCH(Float32, MACRO); \ 70 | NCG_DTYPE_SWITCH(Float64, MACRO); \ 71 | default: break; \ 72 | } 73 | 74 | #define NCG_INSTANTIATE_DTYPE(dtype_, MACRO_) template MACRO_(dtype_) 75 | 76 | #define NCG_DTYPE_INSTANTIATE_ALL(MACRO) \ 77 | NCG_INSTANTIATE_DTYPE(Int8, MACRO); \ 78 | NCG_INSTANTIATE_DTYPE(UInt8, MACRO); \ 79 | NCG_INSTANTIATE_DTYPE(Int32, MACRO); \ 80 | NCG_INSTANTIATE_DTYPE(UInt32, MACRO); \ 81 | NCG_INSTANTIATE_DTYPE(Int64, MACRO); \ 82 | NCG_INSTANTIATE_DTYPE(UInt64, MACRO); \ 83 | NCG_INSTANTIATE_DTYPE(Float32, MACRO); \ 84 | NCG_INSTANTIATE_DTYPE(Float64, MACRO) 85 | 86 | #define NCG_DTYPE_INSTANTIATE_CLASS(dtype_, class_name) template class class_name 87 | 88 | #define NCG_DTYPE_INSTANTIATE_CLASS_ALL(class_name) \ 89 | NCG_DTYPE_INSTANTIATE_CLASS(Int8, class_name); \ 90 | NCG_DTYPE_INSTANTIATE_CLASS(UInt8, class_name); \ 91 | NCG_DTYPE_INSTANTIATE_CLASS(Int32, class_name); \ 92 | NCG_DTYPE_INSTANTIATE_CLASS(UInt32, class_name); \ 93 | NCG_DTYPE_INSTANTIATE_CLASS(Int64, class_name); \ 94 | NCG_DTYPE_INSTANTIATE_CLASS(UInt64, class_name); \ 95 | NCG_DTYPE_INSTANTIATE_CLASS(Float32, class_name); \ 96 | NCG_DTYPE_INSTANTIATE_CLASS(Float64, class_name) 97 | 98 | inline const char *get_dtype_name(DTypeName dtype) { 99 | #define GET_NAME_DTYPE_CASE(dtype_name) return #dtype_name; 100 | NCG_DTYPE_SWITCH_ALL(dtype, GET_NAME_DTYPE_CASE); 101 | #undef GET_NAME_DTYPE_CASE 102 | } 103 | 104 | 105 | } /* !namespace ncg */ 106 | 107 | -------------------------------------------------------------------------------- /src/core/op.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * op.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "op.h" 9 | 10 | namespace ncg { 11 | 12 | TensorVec Op::execute(OpContext &ctx, const TensorVec &inputs) { 13 | check_inputs(ctx, inputs); 14 | if (ctx.is_error()) { 15 | return TensorVec(); 16 | } 17 | return compute(ctx, inputs); 18 | } 19 | 20 | void Op::set_desc(OpDescPtr desc) { 21 | m_desc = desc; 22 | } 23 | 24 | std::ostream & operator << (std::ostream &out, const Op &op) { 25 | return out << op.op_name() << "@" << &op; 26 | } 27 | 28 | std::ostringstream &OpContext::error(const Op *op) { 29 | auto &error = RuntimeContext::error(); 30 | // error << op->op_name() << ": "; 31 | return error; 32 | } 33 | 34 | } /* !namespace ncg */ 35 | -------------------------------------------------------------------------------- /src/core/op.h: -------------------------------------------------------------------------------- 1 | /* 2 | * op.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/tensor.h" 11 | 12 | namespace ncg { 13 | 14 | class OpContext; 15 | 16 | class OpDesc { 17 | public: 18 | virtual ~OpDesc() = default; 19 | }; 20 | 21 | typedef std::shared_ptr OpDescPtr; 22 | 23 | class Op { 24 | public: 25 | Op() : m_desc() {} 26 | virtual ~Op() = default; 27 | 28 | TensorVec execute(OpContext &ctx, const TensorVec &inputs); 29 | virtual void check_inputs(OpContext &ctx, const TensorVec &inputs) = 0; 30 | virtual TensorVec compute(OpContext &ctx, const TensorVec &inputs) = 0; 31 | virtual const char *op_name() const = 0; 32 | 33 | template 34 | const DescT &desc() const { 35 | auto p = dynamic_cast(m_desc.get()); 36 | ncg_assert(p != nullptr); 37 | return *p; 38 | } 39 | 40 | void set_desc(OpDescPtr); 41 | 42 | friend std::ostream & operator << (std::ostream &, const Op &); 43 | 44 | protected: 45 | OpDescPtr m_desc; 46 | }; 47 | 48 | #define NCG_OP_DEF_NAME(op_name_) virtual const char *op_name() const { return #op_name_; } 49 | 50 | #define NCG_OP_CHECK_CTX_CLEAN(ctx) if (ctx.is_error()) return; 51 | 52 | #define NCG_OP_CHECK_NR_INPUTS(ctx, inputs, n) do { \ 53 | if (inputs.size() != n) { \ 54 | ctx.error(this) << this->op_name() << " requires " << n << " input(s), but got " << inputs.size() << " input(s)."; \ 55 | return; \ 56 | } \ 57 | } while (0) 58 | 59 | #define NCG_OP_CHECK_NR_INPUTS2(ctx, inputs, n1, n2) do { \ 60 | if (inputs.size() != n1 && inputs.size() != n2) { \ 61 | ctx.error(this) << this->op_name() << " requires " << n1 << " or " << n2 << " input(s), but got " << inputs.size() << " input(s)."; \ 62 | return; \ 63 | } \ 64 | } while (0) 65 | 66 | #define NCG_OP_CHECK_EMPTY_INPUTS(ctx, inputs) NCG_OP_CHECK_NR_INPUTS(ctx, inputs, 0) 67 | 68 | #define NCG_OP_CHECK_NONEMPTY_INPUTS(ctx, inputs) do { \ 69 | if (inputs.size() == 0) { \ 70 | ctx.error(this) << this->op_name() << " requires at least one input, but got zero."; \ 71 | return; \ 72 | } \ 73 | } while (0) 74 | 75 | #define NCG_OP_CHECK_COMPATIBLE_DTYPE(ctx, inputs) do { \ 76 | if (inputs.size() > 0) { \ 77 | for (ssize_t i = 1; i < inputs.size(); ++i) { \ 78 | if (inputs[i]->desc().dtype() != inputs[0]->desc().dtype()) { \ 79 | auto &err_stream = ctx.error(this) << this->op_name() << " requires all inputs to have the same dtype, but got: "; \ 80 | for (ssize_t j = 0; j < inputs.size(); ++j) { \ 81 | if (j != 0) err_stream << ", "; \ 82 | err_stream << get_dtype_name(inputs[j]->desc().dtype()); \ 83 | } \ 84 | err_stream << "."; \ 85 | return; \ 86 | } \ 87 | } \ 88 | } \ 89 | } while (0) 90 | 91 | #define NCG_OP_CHECK_COMPATIBLE_DIM(ctx, inputs) do { \ 92 | if (inputs.size() > 0) { \ 93 | for (ssize_t i = 1; i < inputs.size(); ++i) { \ 94 | if (inputs[i]->desc().dim() != inputs[0]->desc().dim()) { \ 95 | auto &err_stream = ctx.error(this) << this->op_name() << " requires all inputs to have the same dimension, but got: "; \ 96 | for (ssize_t j = 0; j < inputs.size(); ++j) { \ 97 | if (j != 0) err_stream << ", "; \ 98 | err_stream << inputs[j]->desc().dim(); \ 99 | } \ 100 | err_stream << "."; \ 101 | return; \ 102 | } \ 103 | } \ 104 | } \ 105 | } while (0) 106 | 107 | #define NCG_OP_CHECK_COMPATIBLE_SHAPE_PRINT(ctx, inputs, err_stream) do { \ 108 | for (ssize_t j = 0; j < inputs.size(); ++j) { \ 109 | if (j != 0) err_stream << ", "; \ 110 | err_stream << inputs[j]->desc().shape_vec(); \ 111 | } \ 112 | } while(0) 113 | 114 | #define NCG_OP_CHECK_COMPATIBLE_SHAPE(ctx, inputs) do { \ 115 | if (inputs.size() > 0) { \ 116 | for (ssize_t i = 1; i < inputs.size(); ++i) { \ 117 | if (!inputs[i]->desc().is_compatible(inputs[0]->desc())) { \ 118 | auto &err_stream = ctx.error(this) << this->op_name() << " requires all inputs to have compatible shapes, but got: "; \ 119 | NCG_OP_CHECK_COMPATIBLE_SHAPE_PRINT(ctx, inputs, err_stream); \ 120 | err_stream << "."; \ 121 | return; \ 122 | } \ 123 | } \ 124 | } \ 125 | } while (0) 126 | 127 | #define NCG_OP_CHECK_BROADCASTABLE_SHAPE(ctx, inputs) do { \ 128 | if (inputs.size() > 0) { \ 129 | for (ssize_t i = 1; i < inputs.size(); ++i) { \ 130 | if (!inputs[i]->desc().is_compatible(inputs[0]->desc(), true)) { \ 131 | auto &err_stream = ctx.error(this) << this->op_name() << " requires all inputs to have broadcastable shapes, but got: "; \ 132 | NCG_OP_CHECK_COMPATIBLE_SHAPE_PRINT(ctx, inputs, err_stream); \ 133 | err_stream << "."; \ 134 | return; \ 135 | } \ 136 | } \ 137 | } \ 138 | } while (0) 139 | 140 | #define NCG_OP_CHECK_INPUT_DTYPE(ctx, inputs, idx, dtype_name) do { \ 141 | auto idx_value = (idx); \ 142 | if (inputs[idx_value]->desc().dtype() != DTypeName::dtype_name) { \ 143 | ctx.error(this) << this->op_name() << " requires the input " << (idx_value + 1) << " to have dtype " << #dtype_name << ", but got " << get_dtype_name(inputs[idx_value]->desc().dtype()) << "."; \ 144 | return; \ 145 | } \ 146 | } while (0) 147 | 148 | #define NCG_OP_CHECK_INPUT_DTYPE_INT(ctx, inputs, idx) do { \ 149 | auto idx_value = (idx); \ 150 | if (inputs[idx_value]->desc().dtype() != DTypeName::Int32 && inputs[idx_value]->desc().dtype() != DTypeName::Int64) { \ 151 | ctx.error(this) << this->op_name() << " requires the input " << (idx_value + 1) << " to have dtype Int32 or Int64, but got " << get_dtype_name(inputs[idx_value]->desc().dtype()) << "."; \ 152 | return; \ 153 | } \ 154 | } while (0) 155 | 156 | #define NCG_OP_CHECK_INPUT_DTYPE_FLOAT(ctx, inputs, idx) do { \ 157 | auto idx_value = (idx); \ 158 | if (inputs[idx_value]->desc().dtype() != DTypeName::Float32 && inputs[idx_value]->desc().dtype() != DTypeName::Float64) { \ 159 | ctx.error(this) << this->op_name() << " requires the input " << (idx_value + 1) << " to have dtype Float32 or Float64, but got " << get_dtype_name(inputs[idx_value]->desc().dtype()) << "."; \ 160 | return; \ 161 | } \ 162 | } while (0) 163 | 164 | #define NCG_OP_CHECK_INPUT_DIM(ctx, inputs, idx, dim_expr) do { \ 165 | auto idx_value = (idx); \ 166 | auto dim_value = (dim_expr); \ 167 | if (inputs[idx_value]->desc().dim() != dim_value) { \ 168 | ctx.error(this) << this->op_name() << " requires the input " << (idx_value + 1) << " to have dimension " << dim_value << ", but got " << inputs[idx_value]->desc().dim() << "."; \ 169 | return; \ 170 | } \ 171 | } while (0) 172 | 173 | #define NCG_OP_CHECK_INPUT_DIM_GEQ(ctx, inputs, idx, dim_expr) do { \ 174 | auto idx_value = (idx); \ 175 | auto dim_value = (dim_expr); \ 176 | if (dim_value > 0 && inputs[idx_value]->desc().dim() < dim_value) { \ 177 | ctx.error(this) << this->op_name() << " requires the input " << (idx_value + 1) << " to have a dimension of at least " << dim_value << ", but got " << inputs[idx_value]->desc().dim() << "."; \ 178 | return; \ 179 | } \ 180 | } while (0) 181 | 182 | #define NCG_OP_CHECK_INPUT_SCALAR(ctx, inputs, idx) do { \ 183 | auto idx_value = (idx); \ 184 | if (inputs[idx_value]->desc().dim() != 0) { \ 185 | ctx.error(this) << this->op_name() << " requires the input " << (idx_value + 1) << " to be a scalar, but got a " << inputs[idx_value]->desc().dim() << " dimensional input."; \ 186 | return; \ 187 | } \ 188 | } while (0) 189 | 190 | #define NCG_OP_CHECK_INPUT_VECTOR(ctx, inputs, idx) do { \ 191 | auto idx_value = (idx); \ 192 | if (inputs[idx_value]->desc().dim() != 1) { \ 193 | ctx.error(this) << this->op_name() << " requires the input " << (idx_value + 1) << " to be a vector, but got a " << inputs[idx_value]->desc().dim() << " dimensional input."; \ 194 | return; \ 195 | } \ 196 | } while (0) 197 | 198 | #define NCG_OP_CHECK_INPUT_SCALAR_VECTOR(ctx, inputs, idx) do { \ 199 | auto idx_value = (idx); \ 200 | if (inputs[idx_value]->desc().dim() > 1) { \ 201 | ctx.error(this) << this->op_name() << " requires the input " << (idx_value + 1) << " to be a scalar or a vector, but got a " << inputs[idx_value]->desc().dim() << " dimensional input."; \ 202 | return; \ 203 | } \ 204 | } while (0) 205 | 206 | class OpContext : public RuntimeContext { 207 | public: 208 | OpContext() = default; 209 | virtual ~OpContext() = default; 210 | 211 | std::ostringstream &error(const Op *); 212 | }; 213 | 214 | } /* !namespace ncg */ 215 | 216 | -------------------------------------------------------------------------------- /src/core/ops/linalg.h: -------------------------------------------------------------------------------- 1 | /* 2 | * linalg.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/op.h" 11 | 12 | namespace ncg { 13 | 14 | class OpMatMulDesc : public OpDesc { 15 | public: 16 | OpMatMulDesc() : transpose_a(false), transpose_b(false) {} 17 | OpMatMulDesc(bool transpose_a, bool transpose_b) : transpose_a(transpose_a), transpose_b(transpose_b) {} 18 | virtual ~OpMatMulDesc() = default; 19 | 20 | bool transpose_a, transpose_b; 21 | }; 22 | 23 | class OpMatMul : public Op { 24 | public: 25 | NCG_OP_DEF_NAME(OpMatMul); 26 | 27 | virtual void check_inputs(OpContext &ctx, const TensorVec &inputs) { 28 | NCG_OP_CHECK_NR_INPUTS(ctx, inputs, 2); 29 | NCG_OP_CHECK_COMPATIBLE_DTYPE(ctx, inputs); 30 | NCG_OP_CHECK_INPUT_DIM(ctx, inputs, 0, 2); 31 | NCG_OP_CHECK_INPUT_DIM(ctx, inputs, 1, 2); 32 | 33 | const auto &desc = this->template desc(); 34 | ssize_t k1 = !desc.transpose_a ? inputs[0]->desc().shape(1) : inputs[0]->desc().shape(0); 35 | ssize_t k2 = !desc.transpose_b ? inputs[1]->desc().shape(0) : inputs[1]->desc().shape(1); 36 | if (k1 != k2) { 37 | ctx.error(this) << "Invalid shape: " << inputs[0]->desc().shape_vec() << " vs. " << inputs[1]->desc().shape_vec() << "."; 38 | } 39 | } 40 | 41 | virtual TensorVec compute(OpContext &ctx, const TensorVec &inputs) { 42 | const auto &desc = this->template desc(); 43 | const auto &a = inputs[0], &b = inputs[1]; 44 | ssize_t N = !desc.transpose_a ? a->desc().shape(0) : a->desc().shape(1); 45 | ssize_t M = !desc.transpose_b ? b->desc().shape(1) : b->desc().shape(0); 46 | 47 | auto output = zeros(inputs[0]->desc().dtype(), {N, M}); 48 | #define MATMUL_DTYPE_CASE(dtype_name) kernel_(ctx, inputs[0]->template as(), inputs[1]->template as(), output->template as()); 49 | NCG_DTYPE_SWITCH_ALL(inputs[0]->desc().dtype(), MATMUL_DTYPE_CASE); 50 | #undef MATMUL_DTYPE_CASE 51 | 52 | return {output}; 53 | } 54 | 55 | private: 56 | template 57 | void kernel_(OpContext &ctx, TensorImpl
*a, TensorImpl
*b, TensorImpl
*c) { 58 | const auto &desc = this->template desc(); 59 | 60 | a->make_contiguous(); 61 | b->make_contiguous(); 62 | c->make_contiguous(); 63 | 64 | ssize_t N = !desc.transpose_a ? a->desc().shape(0) : a->desc().shape(1); 65 | ssize_t M = !desc.transpose_b ? b->desc().shape(1) : b->desc().shape(0); 66 | ssize_t K = !desc.transpose_a ? a->desc().shape(1) : a->desc().shape(0); 67 | 68 | auto a_ptr = a->data_ptr(), b_ptr = b->data_ptr(); 69 | auto c_ptr = c->mutable_data_ptr(); 70 | 71 | if (!desc.transpose_a && !desc.transpose_b) { 72 | for (ssize_t k = 0; k < K; ++k) { 73 | for (ssize_t i = 0; i < N; ++i) { 74 | for (ssize_t j = 0; j < M; ++j) { 75 | c_ptr[i * M + j] += a_ptr[i * K + k] * b_ptr[k * M + j]; 76 | } 77 | } 78 | } 79 | } else if (!desc.transpose_a && desc.transpose_b) { 80 | for (ssize_t i = 0; i < N; ++i) { 81 | for (ssize_t j = 0; j < M; ++j) { 82 | for (ssize_t k = 0; k < K; ++k) { 83 | c_ptr[i * M + j] += a_ptr[i * K + k] * b_ptr[j * K + k]; 84 | } 85 | } 86 | } 87 | } else if (desc.transpose_a && !desc.transpose_b) { 88 | for (ssize_t k = 0; k < K; ++k) { 89 | for (ssize_t i = 0; i < N; ++i) { 90 | for (ssize_t j = 0; j < M; ++j) { 91 | c_ptr[i * M + j] += a_ptr[k * N + i] * b_ptr[k * M + j]; 92 | } 93 | } 94 | } 95 | } else if (desc.transpose_a && desc.transpose_b) { 96 | for (ssize_t k = 0; k < K; ++k) { 97 | for (ssize_t i = 0; i < N; ++i) { 98 | for (ssize_t j = 0; j < M; ++j) { 99 | c_ptr[i * M + j] += a_ptr[k * N + i] * b_ptr[j * K + k]; 100 | } 101 | } 102 | } 103 | } 104 | } 105 | }; 106 | 107 | } /* !namespace ncg */ 108 | 109 | -------------------------------------------------------------------------------- /src/core/ops/reduction.h: -------------------------------------------------------------------------------- 1 | /* 2 | * reduction.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/op.h" 11 | 12 | namespace ncg { 13 | 14 | enum class ReduceType1 : int { 15 | Max, 16 | Min 17 | }; 18 | 19 | enum class ReduceType2 : int { 20 | Sum, 21 | Mean, 22 | Prod 23 | }; 24 | 25 | class OpReduceDesc : public OpDesc { 26 | public: 27 | OpReduceDesc(ssize_t axis = 0, bool keepdims = false) : axis(axis), keepdims(keepdims) {} 28 | virtual ~OpReduceDesc() = default; 29 | 30 | ssize_t axis; 31 | bool keepdims; 32 | }; 33 | 34 | class OpReduceBase : public Op { 35 | public: 36 | virtual void check_inputs(OpContext &ctx, const TensorVec &inputs) { 37 | const auto &desc = this->template desc(); 38 | auto axis = desc.axis; 39 | if (axis < 0) axis += inputs[0]->desc().dim(); 40 | 41 | NCG_OP_CHECK_NR_INPUTS(ctx, inputs, 1); 42 | NCG_OP_CHECK_INPUT_DIM_GEQ(ctx, inputs, 0, axis); 43 | } 44 | }; 45 | 46 | template 47 | class OpReduceType1Base : public OpReduceBase { 48 | public: 49 | virtual TensorVec compute(OpContext &ctx, const TensorVec &inputs) { 50 | const auto input = inputs[0]; 51 | const auto &desc = this->template desc(); 52 | auto axis = desc.axis; 53 | if (axis < 0) axis += inputs[0]->desc().dim(); 54 | 55 | TensorVec outputs; 56 | 57 | #define REDUCE_DTYPE_CASE(dtype_name) outputs = kernel_(ctx, input->template as(), axis, desc.keepdims) 58 | NCG_DTYPE_SWITCH_ALL(input->desc().dtype(), REDUCE_DTYPE_CASE); 59 | #undef REDUCE_DTYPE_CASE 60 | 61 | return outputs; 62 | } 63 | 64 | private: 65 | template 66 | TensorVec kernel_(OpContext &ctx, TensorImpl
*input, ssize_t axis, bool keepdims) { 67 | auto output_shape = input->desc().shape_vec(); 68 | if (keepdims) { 69 | output_shape[axis] = 1; 70 | } else { 71 | output_shape.erase(output_shape.begin() + axis); 72 | } 73 | 74 | TensorPtr output_ptr, indices_ptr; 75 | if (ReduceType == ReduceType1::Min) { 76 | output_ptr = fill(DT, output_shape, std::numeric_limits::cctype>::max()); 77 | } else { 78 | output_ptr = fill(DT, output_shape, std::numeric_limits::cctype>::lowest()); 79 | } 80 | indices_ptr = empty(DTypeName::Int64, output_shape); 81 | 82 | auto output = output_ptr->template as
(); 83 | auto indices = indices_ptr->template as(); 84 | 85 | auto input_data_ptr = input->data_ptr(); 86 | bool input_con = input->desc().is_contiguous(); 87 | auto output_data_ptr = output->mutable_data_ptr(); 88 | auto indices_data_ptr = indices->mutable_data_ptr(); 89 | 90 | const auto &input_default_stride = input->desc().get_default_stride(); 91 | const auto &output_default_stride = output->desc().get_default_stride(); 92 | for (ssize_t i = 0; i < input->desc().numel(); ++i) { 93 | ssize_t j1, j2, j3; 94 | if (axis != 0) { 95 | j1 = i / input_default_stride[axis - 1]; 96 | j2 = (i % input_default_stride[axis - 1]) / input_default_stride[axis]; 97 | j3 = i % input_default_stride[axis]; 98 | } else { 99 | j1 = 0; 100 | j2 = i / input_default_stride[axis]; 101 | j3 = i % input_default_stride[axis]; 102 | } 103 | 104 | ssize_t j; 105 | if (axis != 0) { 106 | j = j1 * output_default_stride[axis - 1] + j3; 107 | } else { 108 | j = j3; 109 | } 110 | 111 | const auto input_val = input_con ? input_data_ptr[i] : input->elat(i); 112 | if (ReduceType == ReduceType1::Min) { 113 | if (input_val < output_data_ptr[j]) { 114 | output_data_ptr[j] = input_val; 115 | indices_data_ptr[j] = static_cast::cctype>(j2); 116 | } 117 | } else { 118 | if (input->elat(i) > output_data_ptr[j]) { 119 | output_data_ptr[j] = input_val; 120 | indices_data_ptr[j] = static_cast::cctype>(j2); 121 | } 122 | } 123 | } 124 | 125 | return {output_ptr, indices_ptr}; 126 | } 127 | }; 128 | 129 | class OpReduceMax : public OpReduceType1Base { 130 | public: 131 | NCG_OP_DEF_NAME(OpReduceMax); 132 | }; 133 | 134 | class OpReduceMin : public OpReduceType1Base { 135 | public: 136 | NCG_OP_DEF_NAME(OpReduceMin); 137 | }; 138 | 139 | template 140 | class OpReduceType2Base : public OpReduceBase { 141 | public: 142 | virtual TensorVec compute(OpContext &ctx, const TensorVec &inputs) { 143 | const auto input = inputs[0]; 144 | const auto &desc = this->template desc(); 145 | auto axis = desc.axis; 146 | if (axis < 0) axis += inputs[0]->desc().dim(); 147 | 148 | TensorVec outputs; 149 | 150 | #define REDUCE_DTYPE_CASE(dtype_name) outputs = kernel_(ctx, input->template as(), axis, desc.keepdims) 151 | NCG_DTYPE_SWITCH_ALL(input->desc().dtype(), REDUCE_DTYPE_CASE); 152 | #undef REDUCE_DTYPE_CASE 153 | 154 | return outputs; 155 | } 156 | 157 | private: 158 | template 159 | TensorVec kernel_(OpContext &ctx, TensorImpl
*input, ssize_t axis, bool keepdims) { 160 | auto output_shape = input->desc().shape_vec(); 161 | auto axis_size = static_cast::cctype>(output_shape[axis]); 162 | if (keepdims) { 163 | output_shape[axis] = 1; 164 | } else { 165 | output_shape.erase(output_shape.begin() + axis); 166 | } 167 | 168 | TensorPtr output_ptr; 169 | if (ReduceType == ReduceType2::Sum || ReduceType == ReduceType2::Mean) { 170 | output_ptr = fill(DT, output_shape, 0); 171 | } else { 172 | output_ptr = fill(DT, output_shape, 1); 173 | } 174 | auto output = output_ptr->template as
(); 175 | 176 | auto input_data_ptr = input->data_ptr(); 177 | bool input_con = input->desc().is_contiguous(); 178 | auto output_data_ptr = output->mutable_data_ptr(); 179 | 180 | const auto &input_default_stride = input->desc().get_default_stride(); 181 | const auto &output_default_stride = output->desc().get_default_stride(); 182 | for (ssize_t i = 0; i < input->desc().numel(); ++i) { 183 | ssize_t j1, j2, j3; 184 | if (axis != 0) { 185 | j1 = i / input_default_stride[axis - 1]; 186 | j2 = (i % input_default_stride[axis - 1]) / input_default_stride[axis]; 187 | j3 = i % input_default_stride[axis]; 188 | } else { 189 | j1 = 0; 190 | j2 = i / input_default_stride[axis]; 191 | j3 = i % input_default_stride[axis]; 192 | } 193 | 194 | ssize_t j; 195 | if (axis != 0) { 196 | j = j1 * output_default_stride[axis - 1] + j3; 197 | } else { 198 | j = j3; 199 | } 200 | 201 | const auto input_val = input_con ? input_data_ptr[i] : input->elat(i); 202 | if (ReduceType == ReduceType2::Sum) { 203 | output_data_ptr[j] += input_val; 204 | } else if (ReduceType == ReduceType2::Mean) { 205 | output_data_ptr[j] += input_val / axis_size; 206 | } else if (ReduceType == ReduceType2::Prod) { 207 | output_data_ptr[j] *= input_val; 208 | } 209 | } 210 | 211 | return {output_ptr}; 212 | } 213 | }; 214 | 215 | class OpReduceSum : public OpReduceType2Base { 216 | public: 217 | NCG_OP_DEF_NAME(OpReduceSum); 218 | }; 219 | 220 | class OpReduceMean : public OpReduceType2Base { 221 | public: 222 | NCG_OP_DEF_NAME(OpReduceMean); 223 | }; 224 | 225 | class OpReduceProd : public OpReduceType2Base { 226 | public: 227 | NCG_OP_DEF_NAME(OpReduceProd); 228 | }; 229 | 230 | } /* !namespace ncg */ 231 | 232 | -------------------------------------------------------------------------------- /src/core/ops/shape.h: -------------------------------------------------------------------------------- 1 | /* 2 | * shape.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/op.h" 11 | 12 | #include 13 | 14 | namespace ncg { 15 | 16 | class OpReshapeDesc : public OpDesc { 17 | public: 18 | OpReshapeDesc() : shape() {} 19 | OpReshapeDesc(const ShapeVec &shape) : shape(shape) {} 20 | virtual ~OpReshapeDesc() = default; 21 | 22 | ShapeVec shape; 23 | }; 24 | 25 | class OpReshape : public Op { 26 | public: 27 | NCG_OP_DEF_NAME(OpReshape); 28 | 29 | virtual void check_inputs(OpContext &ctx, const TensorVec &inputs) { 30 | NCG_OP_CHECK_NR_INPUTS(ctx, inputs, 1); 31 | 32 | #define RESHAPE_INVALID_SHAPE do {\ 33 | ctx.error(this) << "Invalid target shape " << shape << "; input tensor shape is " << inputs[0]->desc().shape_vec(); \ 34 | return; \ 35 | } while (0) 36 | 37 | auto &shape = this->template desc().shape; 38 | 39 | int nr_negative = 0; 40 | int nr_total = 1; 41 | for (int i = 0; i < shape.size(); ++i) { 42 | if (shape[i] == -1) { 43 | nr_negative += 1; 44 | } else if (shape[i] == NewAxis) { 45 | // pass 46 | } else if (shape[i] > 0) { 47 | nr_total *= shape[i]; 48 | } else { 49 | RESHAPE_INVALID_SHAPE; 50 | } 51 | } 52 | 53 | if (nr_negative == 1) { 54 | if (inputs[0]->desc().numel() % nr_total != 0) { 55 | RESHAPE_INVALID_SHAPE; 56 | } 57 | } else if (nr_negative == 0) { 58 | if (inputs[0]->desc().numel() != nr_total) { 59 | RESHAPE_INVALID_SHAPE; 60 | } 61 | } else { 62 | RESHAPE_INVALID_SHAPE; 63 | } 64 | 65 | #undef RESHAPE_INVALID_SHAPE 66 | } 67 | 68 | virtual TensorVec compute(OpContext &ctx, const TensorVec &inputs) { 69 | const auto input = inputs[0]; 70 | auto shape = this->template desc().shape; 71 | 72 | int negative_idx = -1; 73 | int nr_total = 1; 74 | for (ssize_t i = 0; i < shape.size(); ++i) { 75 | if (shape[i] == -1) { 76 | negative_idx = 0; 77 | } else if (shape[i] == NewAxis) { 78 | shape[i] = 1; 79 | } else if (shape[i] > 0) { 80 | nr_total *= shape[i]; 81 | } 82 | } 83 | 84 | if (negative_idx != -1) { 85 | shape[negative_idx] = input->desc().numel() / nr_total; 86 | } 87 | 88 | inputs[0]->make_contiguous(); 89 | TensorDesc desc(input->desc().dtype(), shape); 90 | TensorPtr output = tensor(desc, input->storage(), false); 91 | 92 | return {output}; 93 | } 94 | }; 95 | 96 | class OpPermuteDesc : public OpDesc { 97 | public: 98 | OpPermuteDesc() : axes() {} 99 | OpPermuteDesc(const ShapeVec &axes) : axes(axes) {} 100 | virtual ~OpPermuteDesc() = default; 101 | 102 | ShapeVec axes; 103 | }; 104 | 105 | class OpPermute: public Op { 106 | public: 107 | NCG_OP_DEF_NAME(OpPermute); 108 | 109 | virtual void check_inputs(OpContext &ctx, const TensorVec &inputs) { 110 | NCG_OP_CHECK_NR_INPUTS(ctx, inputs, 1); 111 | 112 | auto &axes = this->template desc().axes; 113 | std::vector used(axes.size()); 114 | for (ssize_t i = 0; i < axes.size(); ++i) { 115 | int j = axes[i]; 116 | 117 | if (j >= 0 && j < axes.size() && !used[j]) { 118 | used[j] = true; 119 | } else { 120 | ctx.error(this) << "Invalid permute axes " << axes << "."; 121 | return; 122 | } 123 | } 124 | } 125 | 126 | virtual TensorVec compute(OpContext &ctx, const TensorVec &inputs) { 127 | const auto input = inputs[0]; 128 | auto output = tensor(input->desc(), input->storage(), false); 129 | auto &axes = this->template desc().axes; 130 | 131 | auto stride = input->desc().stride(); 132 | for (ssize_t i = 0; i < input->desc().dim(); ++i) { 133 | output->desc().stride(i) = input->desc().stride(axes[i]); 134 | output->desc().shape(i) = input->desc().shape(axes[i]); 135 | } 136 | 137 | return {output}; 138 | } 139 | }; 140 | 141 | class OpExpandDesc : public OpDesc { 142 | public: 143 | OpExpandDesc() : shape() {}; 144 | OpExpandDesc(const ShapeVec &shape) : shape(shape) {} 145 | virtual ~OpExpandDesc() = default; 146 | 147 | ShapeVec shape; 148 | }; 149 | 150 | class OpExpand : public Op { 151 | public: 152 | NCG_OP_DEF_NAME(OpExpand); 153 | 154 | virtual void check_inputs(OpContext &ctx, const TensorVec &inputs) { 155 | NCG_OP_CHECK_NR_INPUTS(ctx, inputs, 1); 156 | 157 | const auto input = inputs[0]; 158 | auto &shape = this->template desc().shape; 159 | 160 | if (shape.size() != input->desc().dim()) { 161 | ctx.error(this) << "Expand op cannot change the dimension."; 162 | return; 163 | } 164 | for (ssize_t i = 0; i < shape.size(); ++i) { 165 | if (shape[i] != input->desc().shape(i) && input->desc().shape(i) != 1) { 166 | ctx.error(this) << "Invalid target shape " << shape << "; input tensor shape is " << input->desc().shape_vec() << "."; 167 | return; 168 | } 169 | } 170 | } 171 | 172 | virtual TensorVec compute(OpContext &ctx, const TensorVec &inputs) { 173 | const auto input = inputs[0]; 174 | auto &shape = this->template desc().shape; 175 | auto output = tensor(input->desc(), input->storage(), false); 176 | 177 | for (ssize_t i = 0; i < input->desc().dim(); ++i) { 178 | if (shape[i] != input->desc().shape(i)) { 179 | output->desc().shape(i) = shape[i]; 180 | output->desc().stride(i) = 0; 181 | } 182 | } 183 | 184 | return {output}; 185 | } 186 | }; 187 | 188 | class OpSqueezeDesc : public OpDesc { 189 | public: 190 | OpSqueezeDesc() : axis(0) {}; 191 | OpSqueezeDesc(ssize_t axis) : axis(axis) {} 192 | virtual ~OpSqueezeDesc() = default; 193 | 194 | ssize_t axis; 195 | }; 196 | 197 | class OpSqueeze : public Op { 198 | public: 199 | NCG_OP_DEF_NAME(OpSqueeze); 200 | 201 | virtual void check_inputs(OpContext &ctx, const TensorVec &inputs) { 202 | NCG_OP_CHECK_NR_INPUTS(ctx, inputs, 1); 203 | 204 | const auto input = inputs[0]; 205 | auto axis = this->template desc().axis; 206 | if (axis < 0) axis += input->desc().dim(); 207 | 208 | NCG_OP_CHECK_INPUT_DIM_GEQ(ctx, inputs, 0, axis); 209 | if (input->desc().shape(axis) != 1) { 210 | ctx.error(this) << "Only size 1 dimensions can be squeezed."; 211 | } 212 | } 213 | 214 | virtual TensorVec compute(OpContext &ctx, const TensorVec &inputs) { 215 | const auto input = inputs[0]; 216 | auto axis = this->template desc().axis; 217 | if (axis < 0) axis += input->desc().dim(); 218 | auto output = tensor(input->desc(), input->storage(), false); 219 | 220 | auto shape_vec = input->desc().shape_vec(); 221 | shape_vec.erase(shape_vec.begin() + axis); 222 | auto stride_vec = input->desc().stride_vec(); 223 | stride_vec.erase(stride_vec.begin() + axis); 224 | 225 | output->desc().set_shape_vec(shape_vec); 226 | output->desc().set_stride_vec(stride_vec); 227 | return {output}; 228 | } 229 | }; 230 | 231 | class OpUnsqueezeDesc : public OpDesc { 232 | public: 233 | OpUnsqueezeDesc() : axis(0) {}; 234 | OpUnsqueezeDesc(ssize_t axis) : axis(axis) {} 235 | virtual ~OpUnsqueezeDesc() = default; 236 | 237 | ssize_t axis; 238 | }; 239 | 240 | class OpUnsqueeze : public Op { 241 | public: 242 | NCG_OP_DEF_NAME(OpUnsqueeze); 243 | 244 | virtual void check_inputs(OpContext &ctx, const TensorVec &inputs) { 245 | NCG_OP_CHECK_NR_INPUTS(ctx, inputs, 1); 246 | 247 | const auto input = inputs[0]; 248 | auto axis = this->template desc().axis; 249 | if (axis < 0) axis += input->desc().dim(); 250 | 251 | NCG_OP_CHECK_INPUT_DIM_GEQ(ctx, inputs, 0, axis - 1); 252 | } 253 | 254 | virtual TensorVec compute(OpContext &ctx, const TensorVec &inputs) { 255 | const auto input = inputs[0]; 256 | auto axis = this->template desc().axis; 257 | if (axis < 0) axis += input->desc().dim(); 258 | auto output = tensor(input->desc(), input->storage(), false); 259 | 260 | auto shape_vec = input->desc().shape_vec(); 261 | shape_vec.insert(shape_vec.begin() + axis, 1); 262 | auto stride_vec = input->desc().stride_vec(); 263 | stride_vec.insert(stride_vec.begin() + axis, 0); 264 | 265 | output->desc().set_shape_vec(shape_vec); 266 | output->desc().set_stride_vec(stride_vec); 267 | return {output}; 268 | } 269 | }; 270 | 271 | } /* !namespace ncg */ 272 | 273 | -------------------------------------------------------------------------------- /src/core/pickle.h: -------------------------------------------------------------------------------- 1 | /* 2 | * pickle.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/common.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | namespace ncg { 19 | 20 | enum class NCGPickleTypes : int32_t { 21 | Int64 = 0x0001, 22 | String = 0x0002, 23 | Int64Array = 0x0003, 24 | CharArray = 0x0004 25 | }; 26 | 27 | #define WP(x) reinterpret_cast(x) 28 | #define RP(x) reinterpret_cast(x) 29 | 30 | class NCGPickler { 31 | public: 32 | NCGPickler(std::ostream &out) : m_out(out), m_own_stream(false) { ncg_assert(bool(m_out)); } 33 | NCGPickler(std::string fname) : m_own_out(fname, std::ios::out | std::ios::binary), m_own_stream(true), m_out(m_own_out) { ncg_assert(bool(m_out)); } 34 | 35 | NCGPickler(const NCGPickler &) = delete; 36 | NCGPickler(NCGPickler &&) = delete; 37 | 38 | void close() { 39 | if (m_own_stream) { 40 | m_own_out.close(); 41 | m_own_stream = false; 42 | } 43 | } 44 | 45 | virtual ~NCGPickler() { close(); } 46 | 47 | void write(const int64_t &val) { 48 | int32_t type_val = static_cast(NCGPickleTypes::Int64); 49 | m_out.write(WP(&type_val), sizeof(type_val)); 50 | m_out.write(WP(&val), sizeof(val)); 51 | } 52 | 53 | void write(const std::string &val) { 54 | int32_t type_val = static_cast(NCGPickleTypes::String); 55 | m_out.write(WP(&type_val), sizeof(type_val)); 56 | int64_t size_val = val.size(); 57 | m_out.write(WP(&size_val), sizeof(size_val)); 58 | const char *cstr_val = val.c_str(); 59 | m_out.write(WP(cstr_val), sizeof(char) * (size_val + 1)); 60 | } 61 | 62 | void write_ssize_array(const ssize_t *arr_val, int64_t size_val) { 63 | int32_t type_val = static_cast(NCGPickleTypes::Int64Array); 64 | m_out.write(WP(&type_val), sizeof(type_val)); 65 | m_out.write(WP(&size_val), sizeof(size_val)); 66 | 67 | for (ssize_t i = 0; i < size_val; ++i) { 68 | int64_t val = static_cast(arr_val[i]); 69 | m_out.write(WP(&val), sizeof(val)); 70 | } 71 | } 72 | 73 | template 74 | void write_char_array(const T *arr_val, int64_t size_val) { 75 | int32_t type_val = static_cast(NCGPickleTypes::CharArray); 76 | m_out.write(WP(&type_val), sizeof(type_val)); 77 | m_out.write(WP(&size_val), sizeof(size_val)); 78 | m_out.write(WP(arr_val), sizeof(T) * size_val); 79 | } 80 | 81 | protected: 82 | std::ostream &m_out; 83 | 84 | std::ofstream m_own_out; 85 | bool m_own_stream; 86 | }; 87 | 88 | class NCGUnpickler { 89 | public: 90 | NCGUnpickler(std::istream &in) : m_in(in) { ncg_assert(bool(m_in)); } 91 | NCGUnpickler(std::string fname) : m_own_in(fname, std::ios::in | std::ios::binary), m_own_stream(true), m_in(m_own_in) { ncg_assert(bool(m_in)); } 92 | 93 | NCGUnpickler(const NCGUnpickler &) = delete; 94 | NCGUnpickler(NCGUnpickler &&) = delete; 95 | 96 | void close() { 97 | if (m_own_stream) { 98 | m_own_in.close(); 99 | m_own_stream = false; 100 | } 101 | } 102 | 103 | virtual ~NCGUnpickler() { close(); } 104 | 105 | int64_t read_int64() { 106 | int32_t type_val = 0; 107 | m_in.read(RP(&type_val), sizeof(type_val)); 108 | ncg_assert(type_val == static_cast(NCGPickleTypes::Int64)); 109 | 110 | int64_t val = 0; 111 | m_in.read(RP(&val), sizeof(val)); 112 | return val; 113 | } 114 | 115 | std::string read_string() { 116 | int32_t type_val = 0; 117 | m_in.read(RP(&type_val), sizeof(type_val)); 118 | ncg_assert(type_val == static_cast(NCGPickleTypes::String)); 119 | 120 | int64_t size_val = 0; 121 | m_in.read(RP(&size_val), sizeof(size_val)); 122 | char *cstr_val = new char[size_val + 1]; 123 | m_in.read(RP(cstr_val), sizeof(char) * (size_val + 1)); 124 | std::string val(cstr_val, size_val); 125 | delete []cstr_val; 126 | 127 | return val; 128 | } 129 | 130 | std::pair, size_t> read_ssize_array() { 131 | int32_t type_val = 0; 132 | m_in.read(RP(&type_val), sizeof(type_val)); 133 | ncg_assert(type_val == static_cast(NCGPickleTypes::Int64Array)); 134 | 135 | int64_t size_val = 0; 136 | m_in.read(RP(&size_val), sizeof(size_val)); 137 | int64_t *i64_val = new int64_t[size_val]; 138 | m_in.read(RP(i64_val), sizeof(int64_t) * size_val); 139 | ssize_t *arr_val = new ssize_t[size_val]; 140 | for (int64_t i = 0; i < size_val; ++i) { 141 | arr_val[i] = i64_val[i]; 142 | } 143 | delete []i64_val; 144 | 145 | return std::make_pair(std::unique_ptr(arr_val), static_cast(size_val)); 146 | } 147 | 148 | template 149 | std::pair, size_t> read_char_array() { 150 | int32_t type_val = 0; 151 | m_in.read(RP(&type_val), sizeof(type_val)); 152 | ncg_assert(type_val == static_cast(NCGPickleTypes::CharArray)); 153 | 154 | int64_t size_val = 0; 155 | m_in.read(RP(&size_val), sizeof(size_val)); 156 | char *arr_val = new char[size_val * sizeof(T)]; 157 | m_in.read(RP(arr_val), sizeof(T) * size_val); 158 | 159 | return std::make_pair(std::unique_ptr(reinterpret_cast(arr_val)), static_cast(size_val)); 160 | } 161 | 162 | protected: 163 | std::istream &m_in; 164 | 165 | std::ifstream m_own_in; 166 | bool m_own_stream; 167 | }; 168 | 169 | #undef RP 170 | #undef WP 171 | 172 | 173 | } /* !namespace ncg */ 174 | 175 | -------------------------------------------------------------------------------- /src/core/tensor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * tensor.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/common.h" 11 | #include "core/datatype.h" 12 | #include "core/tensor_desc.h" 13 | #include "core/tensor_storage.h" 14 | #include "core/pickle.h" 15 | 16 | #include 17 | #include 18 | 19 | namespace ncg { 20 | 21 | template 22 | class TensorImpl; 23 | 24 | class Tensor { 25 | public: 26 | Tensor(); 27 | Tensor(const TensorDesc &desc, std::shared_ptr storage, bool own_data=true, ssize_t data_ptr_offset=0); 28 | 29 | template ::value>::type* = nullptr> 30 | Tensor(T value); 31 | template ::value>::type* = nullptr> 32 | Tensor(std::vector value); 33 | template ::value>::type* = nullptr> 34 | Tensor(std::vector> value); 35 | 36 | virtual ~Tensor() = default; 37 | 38 | void pickle(NCGPickler &pickler) const; 39 | 40 | TensorDesc &desc(); 41 | const TensorDesc &desc() const; 42 | 43 | template TensorImpl
*as(); 44 | template const TensorImpl
*as() const; 45 | 46 | std::shared_ptr storage(); 47 | std::shared_ptr storage() const; 48 | 49 | template 50 | const ssize_t index(Ints... args) const; 51 | const ssize_t elindex(ssize_t elindex) const; 52 | 53 | bool own_data() const; 54 | ssize_t data_ptr_offset() const; 55 | virtual void make_own_data() = 0; 56 | virtual void make_contiguous() = 0; 57 | 58 | friend std::ostream &operator << (std::ostream &out, const Tensor &tensor); 59 | 60 | protected: 61 | TensorDesc m_desc; 62 | std::shared_ptr m_storage; 63 | bool m_own_data; 64 | ssize_t m_data_ptr_offset; 65 | }; 66 | 67 | class TensorPtr : public std::shared_ptr { 68 | public: 69 | using std::shared_ptr::shared_ptr; 70 | using super = std::shared_ptr; 71 | 72 | TensorPtr() : super(nullptr) {} 73 | TensorPtr(const TensorPtr &other) : super(other) {} 74 | 75 | TensorPtr(const std::shared_ptr &other) : super(other) {} 76 | TensorPtr(std::shared_ptr &&other) : super(std::forward>(other)) {} 77 | 78 | template ::value>::type* = nullptr> 79 | TensorPtr(T value); 80 | template ::value>::type* = nullptr> 81 | TensorPtr(std::vector value); 82 | template ::value>::type* = nullptr> 83 | TensorPtr(std::vector> value); 84 | 85 | TensorPtr eq(const TensorPtr &rhs) const; 86 | TensorPtr neq(const TensorPtr &rhs) const; 87 | 88 | TensorPtr cast(DTypeName dtype) const; 89 | TensorPtr int8() const; 90 | TensorPtr uint8() const; 91 | TensorPtr int32() const; 92 | TensorPtr uint32() const; 93 | TensorPtr int64() const; 94 | TensorPtr uint64() const; 95 | TensorPtr float32() const; 96 | TensorPtr float64() const; 97 | 98 | std::vector min(ssize_t axis, bool keepdims=false) const; 99 | std::vector max(ssize_t axis, bool keepdims=false) const; 100 | TensorPtr sum(ssize_t axis, bool keepdims=false) const; 101 | TensorPtr mean(ssize_t axis, bool keepdims=false) const; 102 | 103 | TensorPtr reshape(const ShapeVec &shape) const; 104 | TensorPtr permute(const ShapeVec &axes) const; 105 | TensorPtr expand(const ShapeVec &shape) const; 106 | TensorPtr squeeze(ssize_t axis) const; 107 | TensorPtr unsqueeze(ssize_t axis) const; 108 | 109 | TensorPtr narrow(ssize_t axis, ssize_t start, ssize_t length) const; 110 | TensorPtr index_select(ssize_t axis, const TensorPtr &indices) const; 111 | TensorPtr gather(ssize_t axis, const TensorPtr &indices) const; 112 | 113 | friend std::ostream &operator << (std::ostream &out, const TensorPtr &tensor); 114 | }; 115 | 116 | typedef std::vector TensorVec; 117 | 118 | template 119 | class TensorImpl : public Tensor { 120 | public: 121 | using cctype = typename DType
::cctype; 122 | 123 | TensorImpl(); 124 | explicit TensorImpl(const TensorDesc &desc, std::shared_ptr storage, bool own_data=true, ssize_t data_ptr_offset=0); 125 | explicit TensorImpl(const TensorDesc &desc, TensorStorage *storage, bool own_data=true, ssize_t data_ptr_offset=0); 126 | explicit TensorImpl(const TensorDesc &desc, cctype *data_ptr, bool own_data=true, ssize_t data_ptr_offset=0); 127 | virtual ~TensorImpl() = default; 128 | 129 | virtual void make_own_data(); 130 | virtual void make_contiguous(); 131 | 132 | template 133 | const cctype &at(Ints... args) const; 134 | template 135 | cctype &mutable_at(Ints... args); 136 | const cctype &elat(ssize_t i) const; 137 | cctype &mutable_elat(ssize_t i); 138 | const cctype *data_ptr() const; 139 | cctype *mutable_data_ptr(); 140 | 141 | protected: 142 | const TensorStorageImpl
*storage_impl_() const; 143 | TensorStorageImpl
*storage_impl_(); 144 | }; 145 | 146 | TensorPtr tensor(NCGUnpickler &unpickler); 147 | TensorPtr tensor(const TensorDesc &desc, std::shared_ptr storage, bool own_data=true, ssize_t data_ptr_offset=0); 148 | TensorPtr empty(DTypeName dtype, const ShapeVec &shape); 149 | 150 | template 151 | typename std::enable_if::value, TensorPtr>::type 152 | fill(DTypeName dtype, const ShapeVec &shape, ValueT value); 153 | 154 | TensorPtr zeros(DTypeName dtype, const ShapeVec &shape); 155 | TensorPtr ones(DTypeName dtype, const ShapeVec &shape); 156 | 157 | template 158 | typename std::enable_if::value, TensorPtr>::type 159 | scalar(DTypeName dtype, ValueT value); 160 | 161 | template 162 | typename std::enable_if::value, TensorPtr>::type 163 | fromcc(DTypeName dtype, ValueT value); 164 | 165 | template 166 | typename std::enable_if::value, TensorPtr>::type 167 | fromcc(DTypeName dtype, std::vector values); 168 | 169 | template 170 | typename std::enable_if::value, TensorPtr>::type 171 | fromcc(DTypeName dtype, std::vector> values); 172 | 173 | template 174 | ValueT tocc_scalar(TensorPtr tensor); 175 | 176 | template 177 | std::vector tocc_vector(TensorPtr tensor); 178 | 179 | TensorPtr arange(DTypeName dtype, int64_t begin, int64_t end = std::numeric_limits::min(), int64_t step=1); 180 | 181 | // elemwise::misc 182 | TensorPtr cast(TensorPtr a, DTypeName dtype); 183 | TensorPtr cond(TensorPtr a, TensorPtr b, TensorPtr c); 184 | 185 | // elemwise::unary 186 | TensorPtr neg(TensorPtr a); 187 | TensorPtr sin(TensorPtr a); 188 | TensorPtr cos(TensorPtr a); 189 | TensorPtr tan(TensorPtr a); 190 | TensorPtr log(TensorPtr a); 191 | TensorPtr exp(TensorPtr a); 192 | TensorPtr tanh(TensorPtr a); 193 | TensorPtr sigmoid(TensorPtr a); 194 | TensorPtr reciprocal(TensorPtr a); 195 | 196 | // elemwise::binary 197 | TensorPtr add(TensorPtr a, TensorPtr b); 198 | TensorPtr sub(TensorPtr a, TensorPtr b); 199 | TensorPtr mul(TensorPtr a, TensorPtr b); 200 | TensorPtr div(TensorPtr a, TensorPtr b); 201 | TensorPtr ge(TensorPtr a, TensorPtr b); 202 | TensorPtr le(TensorPtr a, TensorPtr b); 203 | TensorPtr geq(TensorPtr a, TensorPtr b); 204 | TensorPtr leq(TensorPtr a, TensorPtr b); 205 | TensorPtr eq(TensorPtr a, TensorPtr b); 206 | TensorPtr neq(TensorPtr a, TensorPtr b); 207 | TensorPtr pow(TensorPtr a, TensorPtr b); 208 | TensorPtr min(TensorPtr a, TensorPtr b); 209 | TensorPtr max(TensorPtr a, TensorPtr b); 210 | 211 | // linalg 212 | TensorPtr matmul(TensorPtr a, TensorPtr b, bool transpose_a=false, bool transpose_b=false); 213 | 214 | // reduce 215 | TensorVec reduce_min(TensorPtr a, ssize_t axis, bool keepdims=false); 216 | TensorVec reduce_max(TensorPtr a, ssize_t axis, bool keepdims=false); 217 | TensorPtr reduce_sum(TensorPtr a, ssize_t axis, bool keepdims=false); 218 | TensorPtr reduce_mean(TensorPtr a, ssize_t axis, bool keepdims=false); 219 | 220 | // shape 221 | TensorPtr reshape(TensorPtr a, const ShapeVec &shape); 222 | TensorPtr permute(TensorPtr a, const ShapeVec &axes); 223 | TensorPtr expand(TensorPtr a, const ShapeVec &shape); 224 | TensorPtr squeeze(TensorPtr a, ssize_t axis); 225 | TensorPtr unsqueeze(TensorPtr a, ssize_t axis); 226 | 227 | // slice 228 | TensorPtr concat(const TensorVec &a, ssize_t axis); 229 | TensorVec split(TensorPtr a, ssize_t axis, const ShapeVec &splits); 230 | TensorPtr narrow(TensorPtr a, ssize_t axis, ssize_t start, ssize_t length); 231 | TensorPtr index_select(TensorPtr a, ssize_t axis, TensorPtr b); 232 | TensorPtr gather(TensorPtr a, ssize_t axis, TensorPtr b); 233 | 234 | TensorPtr narrow_backward(TensorPtr a, ssize_t axis, ssize_t start, ssize_t input_size); 235 | TensorPtr index_select_backward(TensorPtr a, ssize_t axis, TensorPtr b, ssize_t input_size); 236 | TensorPtr gather_backward(TensorPtr a, ssize_t axis, TensorPtr b, ssize_t input_size); 237 | 238 | TensorPtr operator + (const TensorPtr &a, const TensorPtr &b); 239 | TensorPtr operator - (const TensorPtr &a, const TensorPtr &b); 240 | TensorPtr operator * (const TensorPtr &a, const TensorPtr &b); 241 | TensorPtr operator / (const TensorPtr &a, const TensorPtr &b); 242 | TensorPtr operator > (const TensorPtr &a, const TensorPtr &b); 243 | TensorPtr operator < (const TensorPtr &a, const TensorPtr &b); 244 | TensorPtr operator >= (const TensorPtr &a, const TensorPtr &b); 245 | TensorPtr operator <= (const TensorPtr &a, const TensorPtr &b); 246 | 247 | TensorPtr as_tensor(const TensorPtr &value); 248 | template 249 | typename std::enable_if::value, TensorPtr>::type 250 | as_tensor(T value); 251 | template 252 | typename std::enable_if::value, TensorPtr>::type 253 | as_tensor(std::vector value); 254 | template 255 | typename std::enable_if::value, TensorPtr>::type 256 | as_tensor(std::vector> value); 257 | 258 | } /* !namespace ncg */ 259 | 260 | -------------------------------------------------------------------------------- /src/core/tensor_desc.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * tensor_desc.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core/tensor_desc.h" 9 | 10 | #include 11 | 12 | namespace ncg { 13 | 14 | std::ostream &operator << (std::ostream &out, const ShapeVec &shape) { 15 | out << "["; 16 | for (ssize_t i = 0; i < shape.size() ; ++i) { 17 | if (i != 0) out << ", "; 18 | out << shape[i]; 19 | } 20 | out << "]"; 21 | return out; 22 | } 23 | 24 | TensorDesc::TensorDesc() { 25 | m_dtype = DTypeName::UInt8; 26 | memset(m_shape, 0, sizeof(m_shape)); 27 | memset(m_stride, 0, sizeof(m_stride)); 28 | } 29 | 30 | TensorDesc::TensorDesc(DTypeName dtype, const ShapeVec &shape, const ShapeVec &stride) : m_dtype(dtype) { 31 | memset(m_shape, 0, sizeof(m_shape)); 32 | memset(m_stride, 0, sizeof(m_stride)); 33 | 34 | ncg_assert(shape.size() <= TensorMaxDim); 35 | ncg_assert(stride.size() <= TensorMaxDim); 36 | 37 | set_shape_vec(shape); 38 | if (stride.size() > 0) { 39 | ncg_assert(shape.size() == stride.size()); 40 | set_stride_vec(stride); 41 | } else { 42 | set_default_stride(); 43 | } 44 | } 45 | 46 | TensorDesc::TensorDesc(NCGUnpickler &unpickler) { 47 | m_dtype = static_cast(unpickler.read_int64()); 48 | auto shape = unpickler.read_ssize_array(); 49 | ncg_assert(shape.second == TensorMaxDim + 1); 50 | memcpy(m_shape, shape.first.get(), sizeof(m_shape)); 51 | auto stride = unpickler.read_ssize_array(); 52 | ncg_assert(stride.second == TensorMaxDim + 1); 53 | memcpy(m_stride, stride.first.get(), sizeof(m_stride)); 54 | } 55 | 56 | void TensorDesc::pickle(NCGPickler &pickler) const { 57 | pickler.write(static_cast(m_dtype)); 58 | pickler.write_ssize_array(m_shape, TensorMaxDim + 1); 59 | pickler.write_ssize_array(m_stride, TensorMaxDim + 1); 60 | } 61 | 62 | DTypeName TensorDesc::dtype() const { 63 | return m_dtype; 64 | } 65 | 66 | size_t TensorDesc::dim() const { 67 | size_t i; 68 | for (i = 0; i <= TensorMaxDim; ++i) 69 | if (m_shape[i] == TensorShape0) break; 70 | return i; 71 | } 72 | 73 | ShapeVec TensorDesc::shape_vec() const { 74 | return ShapeVec(m_shape, m_shape + dim()); 75 | } 76 | 77 | void TensorDesc::set_shape_vec(const ShapeVec &shape) { 78 | ncg_assert(shape.size() <= TensorMaxDim); 79 | size_t i = 0; 80 | for (auto it = shape.begin(); it != shape.end(); ++it) m_shape[i++] = *it; 81 | m_shape[i++] = TensorShape0; 82 | } 83 | 84 | ssize_t *TensorDesc::shape() { 85 | return m_shape; 86 | } 87 | 88 | const ssize_t *TensorDesc::shape() const { 89 | return m_shape; 90 | } 91 | 92 | ShapeVec TensorDesc::stride_vec() const { 93 | return ShapeVec(m_stride, m_stride + dim()); 94 | } 95 | 96 | void TensorDesc::set_stride_vec(const ShapeVec &stride) { 97 | ncg_assert(stride.size() <= TensorMaxDim); 98 | size_t i = 0; 99 | for (auto it = stride.begin(); it != stride.end(); ++it) m_stride[i++] = *it; 100 | m_stride[i++] = TensorShape0; 101 | } 102 | 103 | ssize_t *TensorDesc::stride() { 104 | return m_stride; 105 | } 106 | 107 | const ssize_t *TensorDesc::stride() const { 108 | return m_stride; 109 | } 110 | 111 | ssize_t &TensorDesc::shape(ssize_t i) { 112 | return m_shape[i]; 113 | } 114 | 115 | ssize_t TensorDesc::shape(ssize_t i) const { 116 | return m_shape[i]; 117 | } 118 | 119 | ssize_t &TensorDesc::stride(ssize_t i) { 120 | return m_stride[i]; 121 | } 122 | 123 | ssize_t TensorDesc::stride(ssize_t i) const { 124 | return m_stride[i]; 125 | } 126 | 127 | ShapeVec TensorDesc::get_default_stride() const { 128 | size_t d = dim(); 129 | ShapeVec stride_vec(d); 130 | 131 | if (d == 0) { 132 | // pass 133 | } else { 134 | stride_vec[d - 1] = 1; 135 | for (ssize_t i = d - 2; i >= 0; --i) { 136 | stride_vec[i] = stride_vec[i + 1] * m_shape[i + 1]; 137 | } 138 | } 139 | 140 | return stride_vec; 141 | } 142 | 143 | void TensorDesc::set_default_stride() { 144 | size_t d = dim(); 145 | 146 | memset(m_stride, 0, sizeof(m_stride)); 147 | if (d == 0) { 148 | // pass 149 | } else { 150 | m_stride[d - 1] = 1; 151 | for (ssize_t i = d - 2; i >= 0; --i) { 152 | m_stride[i] = m_stride[i + 1] * m_shape[i + 1]; 153 | } 154 | } 155 | } 156 | 157 | bool TensorDesc::is_contiguous() const { 158 | size_t d = dim(); 159 | if (d == 0) { 160 | return true; 161 | } 162 | if (m_stride[d - 1] != 1) return false; 163 | for (ssize_t i = d - 2; i >= 0; --i) { 164 | if (m_stride[i] != m_stride[i + 1] * m_shape[i + 1]) 165 | return false; 166 | } 167 | return true; 168 | } 169 | 170 | bool TensorDesc::is_scalar_broadcasted() const { 171 | size_t d = dim(); 172 | if (d == 0) { 173 | return true; 174 | } 175 | for (ssize_t i = 0; i < d; ++i) { 176 | if (m_shape[i] > 1 && m_stride[i] != 0) { 177 | return false; 178 | } 179 | } 180 | return true; 181 | } 182 | 183 | 184 | size_t TensorDesc::numel() const { 185 | size_t n = 1; 186 | for (ssize_t i = 0; i < TensorMaxDim; ++i) { 187 | if (m_shape[i] == TensorShape0) break; 188 | n *= m_shape[i]; 189 | } 190 | return n; 191 | } 192 | 193 | bool TensorDesc::is_compatible(const TensorDesc &rhs, bool allow_broadcast) const { 194 | if (dim() != rhs.dim()) { 195 | return false; 196 | } 197 | 198 | for (ssize_t i = 0; i < dim(); ++i) { 199 | if (allow_broadcast) { 200 | if (m_shape[i] != rhs.m_shape[i] && !(m_shape[i] == 1 || rhs.m_shape[i] == 1)) return false; 201 | } else { 202 | if (m_shape[i] != rhs.m_shape[i]) return false; 203 | } 204 | } 205 | return true; 206 | } 207 | 208 | std::ostream &operator << (std::ostream &out, const TensorDesc &desc) { 209 | size_t d = desc.dim(); 210 | out << "TensorDesc(dtype=" << get_dtype_name(desc.m_dtype) << ", dim=" << d << ", shape=["; 211 | for (ssize_t i = 0; i < d; ++i) out << desc.m_shape[i] << (i == d - 1 ? "" : ", "); 212 | out << "], stride=["; 213 | for (ssize_t i = 0; i < d; ++i) out << desc.m_stride[i] << (i == d - 1 ? "" : ", "); 214 | out << "])"; 215 | return out; 216 | } 217 | 218 | } /* !namespace ncg */ 219 | -------------------------------------------------------------------------------- /src/core/tensor_desc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * tensor_desc.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/common.h" 11 | #include "core/datatype.h" 12 | #include "core/pickle.h" 13 | 14 | #include 15 | 16 | namespace ncg { 17 | 18 | const size_t TensorMaxDim = 15; 19 | 20 | const ssize_t TensorShape0 = std::numeric_limits::min(); 21 | const ssize_t NoneShape = std::numeric_limits::min() + 1; 22 | const ssize_t NewAxis = std::numeric_limits::max(); 23 | 24 | class ShapeVec : public std::vector { 25 | public: 26 | using std::vector::vector; 27 | virtual ~ShapeVec() = default; 28 | friend std::ostream &operator << (std::ostream &out, const ShapeVec &shape); 29 | }; 30 | 31 | class TensorDesc { 32 | public: 33 | TensorDesc(); 34 | TensorDesc(DTypeName dtype, const ShapeVec &shape, const ShapeVec &stride = {}); 35 | virtual ~TensorDesc() = default; 36 | 37 | TensorDesc(NCGUnpickler &unpickler); 38 | void pickle(NCGPickler &pickler) const; 39 | 40 | DTypeName dtype() const; 41 | 42 | size_t dim() const; 43 | 44 | ShapeVec shape_vec() const; 45 | void set_shape_vec(const ShapeVec &); 46 | ssize_t *shape(); 47 | const ssize_t *shape() const; 48 | ShapeVec stride_vec() const; 49 | void set_stride_vec(const ShapeVec &); 50 | ssize_t *stride(); 51 | const ssize_t *stride() const; 52 | 53 | ssize_t &shape(ssize_t i); 54 | ssize_t shape(ssize_t i) const; 55 | ssize_t &stride(ssize_t i); 56 | ssize_t stride(ssize_t i) const; 57 | 58 | ShapeVec get_default_stride() const; 59 | void set_default_stride(); 60 | 61 | bool is_contiguous() const; 62 | bool is_scalar_broadcasted() const; 63 | size_t numel() const; 64 | bool is_compatible(const TensorDesc &rhs, bool allow_broadcast=false) const; 65 | 66 | friend std::ostream &operator << (std::ostream &out, const TensorDesc &desc); 67 | 68 | protected: 69 | DTypeName m_dtype; 70 | ssize_t m_shape[TensorMaxDim + 1]; 71 | ssize_t m_stride[TensorMaxDim + 1]; 72 | }; 73 | 74 | class TensorDescVec : public std::vector { 75 | public: 76 | using std::vector::vector; 77 | }; 78 | 79 | } /* !namespace ncg */ 80 | 81 | -------------------------------------------------------------------------------- /src/core/tensor_extra_ops.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * tensor_extra_ops.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core/tensor_extra_ops.h" 9 | #include "core/tensor_impl.h" 10 | 11 | namespace ncg { 12 | 13 | TensorPtr rand_uniform(URBG& rng, DTypeName dtype, const ShapeVec &shape, double low, double high) { 14 | ncg_assert(dtype == DTypeName::Float32 || dtype == DTypeName::Float64); 15 | auto s = empty(dtype, shape); 16 | 17 | #define RAND_DTYPE_CASE(dtype_name) do { \ 18 | auto s_dtype = s->template as();\ 19 | std::uniform_real_distribution::cctype> dis(low, high); \ 20 | for (ssize_t i = 0; i < s->desc().numel(); ++i) { s_dtype->mutable_elat(i) = dis(rng); } \ 21 | } while(0) 22 | NCG_DTYPE_SWITCH_FLOAT(dtype, RAND_DTYPE_CASE); 23 | #undef RAND_DTYPE_CASE 24 | 25 | return s; 26 | } 27 | 28 | TensorPtr rand_normal(URBG& rng, DTypeName dtype, const ShapeVec &shape, double mean, double stddev) { 29 | ncg_assert(dtype == DTypeName::Float32 || dtype == DTypeName::Float64); 30 | auto s = empty(dtype, shape); 31 | 32 | #define RAND_DTYPE_CASE(dtype_name) do { \ 33 | auto s_dtype = s->template as();\ 34 | std::normal_distribution::cctype> dis(mean, stddev); \ 35 | for (ssize_t i = 0; i < s->desc().numel(); ++i) { s_dtype->mutable_elat(i) = dis(rng); } \ 36 | } while(0) 37 | NCG_DTYPE_SWITCH_FLOAT(dtype, RAND_DTYPE_CASE); 38 | #undef RAND_DTYPE_CASE 39 | 40 | return s; 41 | } 42 | 43 | TensorPtr rand_permutation(URBG& rng, ssize_t size) { 44 | auto s = empty(DTypeName::Int64, {size}); 45 | auto s_dtype = s->template as(); 46 | for (ssize_t i = 0; i < size; ++i) { s_dtype->mutable_elat(i) = static_cast::cctype>(i); } 47 | std::shuffle(s_dtype->mutable_data_ptr(), s_dtype->mutable_data_ptr() + size, rng); 48 | return s; 49 | } 50 | 51 | } /* !namespace ncg */ 52 | -------------------------------------------------------------------------------- /src/core/tensor_extra_ops.h: -------------------------------------------------------------------------------- 1 | /* 2 | * tensor_extra_ops.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "tensor.h" 11 | #include 12 | 13 | namespace ncg { 14 | 15 | using URBG = std::mt19937; 16 | 17 | TensorPtr rand_uniform(URBG& rng, DTypeName dtype, const ShapeVec &shape, double low = 0.0, double high = 1.0); 18 | TensorPtr rand_normal(URBG& rng, DTypeName dtype, const ShapeVec &shape, double mean = 0.0, double stddev = 1.0); 19 | TensorPtr rand_permutation(URBG& rng, ssize_t size); 20 | 21 | } /* !namespace ncg */ 22 | 23 | -------------------------------------------------------------------------------- /src/core/tensor_impl.h: -------------------------------------------------------------------------------- 1 | /* 2 | * tensor_impl.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/tensor.h" 11 | 12 | namespace ncg { 13 | 14 | template 15 | TensorImpl
*Tensor::as() { 16 | return (dynamic_cast *>(this)); 17 | } 18 | 19 | #define INSTANTIATE_FUNC(dtype_name) TensorImpl *Tensor::as() 20 | NCG_DTYPE_INSTANTIATE_ALL(INSTANTIATE_FUNC); 21 | #undef INSTANTIATE_FUNC 22 | 23 | template 24 | const TensorImpl
*Tensor::as() const { 25 | return (dynamic_cast *>(this)); 26 | } 27 | 28 | #define INSTANTIATE_FUNC(dtype_name) const TensorImpl *Tensor::as() const 29 | NCG_DTYPE_INSTANTIATE_ALL(INSTANTIATE_FUNC); 30 | #undef INSTANTIATE_FUNC 31 | 32 | template 33 | const ssize_t Tensor::index(Ints... args) const { 34 | auto indices = {args...}; 35 | ncg_assert(indices.size() == m_desc.dim()); 36 | ssize_t j = 0, i = 0; 37 | for (auto it = indices.begin(); it != indices.end(); ++it) j += (*it) * m_desc.stride(i++); 38 | return j; 39 | } 40 | 41 | template 42 | TensorImpl
::TensorImpl() {} 43 | 44 | template 45 | TensorImpl
::TensorImpl(const TensorDesc &desc, std::shared_ptr storage, bool own_data, ssize_t data_ptr_offset) : Tensor(desc, storage, own_data, data_ptr_offset) { 46 | ncg_assert(storage->dtype() == desc.dtype()); 47 | } 48 | 49 | template 50 | TensorImpl
::TensorImpl(const TensorDesc &desc, TensorStorage *storage, bool own_data, ssize_t data_ptr_offset) : Tensor(desc, nullptr, own_data, data_ptr_offset) { 51 | ncg_assert(storage->dtype() == desc.dtype()); 52 | m_storage = std::shared_ptr(storage); 53 | } 54 | 55 | template 56 | TensorImpl
::TensorImpl(const TensorDesc &desc, typename TensorImpl
::cctype *data_ptr, bool own_data, ssize_t data_ptr_offset) : Tensor(desc, nullptr, own_data, data_ptr_offset) { 57 | auto storage = new TensorStorageImpl
(data_ptr); 58 | m_storage = std::shared_ptr(storage); 59 | } 60 | 61 | template 62 | void TensorImpl
::make_own_data() { 63 | if (!m_own_data) { 64 | if (m_desc.is_contiguous()) { 65 | m_storage = std::shared_ptr(m_storage->clone(m_data_ptr_offset, m_desc.numel())); 66 | m_own_data = true; 67 | m_data_ptr_offset = 0; 68 | } else { 69 | make_contiguous(); 70 | } 71 | } 72 | } 73 | 74 | template 75 | void TensorImpl
::make_contiguous() { 76 | if (m_desc.is_contiguous()) { 77 | return; 78 | } else { 79 | auto storage = new TensorStorageImpl
(m_desc.numel()); 80 | for (ssize_t i = 0; i < m_desc.numel(); ++i) { 81 | storage->mutable_data_ptr()[i] = elat(i); 82 | } 83 | 84 | m_desc.set_default_stride(); 85 | m_storage = std::shared_ptr(static_cast(storage)); 86 | m_own_data = true; 87 | m_data_ptr_offset = 0; 88 | } 89 | } 90 | 91 | template 92 | template 93 | const typename TensorImpl
::cctype &TensorImpl
::at(Ints... args) const { 94 | return data_ptr()[index(args...)]; 95 | } 96 | 97 | template 98 | template 99 | typename TensorImpl
::cctype &TensorImpl
::mutable_at(Ints... args) { 100 | return mutable_data_ptr()[index(args...)]; 101 | } 102 | 103 | template 104 | const typename TensorImpl
::cctype &TensorImpl
::elat(ssize_t i) const { 105 | return data_ptr()[elindex(i)]; 106 | } 107 | 108 | template 109 | typename TensorImpl
::cctype &TensorImpl
::mutable_elat(ssize_t i) { 110 | return mutable_data_ptr()[elindex(i)]; 111 | } 112 | 113 | template 114 | const typename TensorImpl
::cctype *TensorImpl
::data_ptr() const { 115 | return storage_impl_()->data_ptr() + m_data_ptr_offset; 116 | } 117 | 118 | template 119 | typename TensorImpl
::cctype *TensorImpl
::mutable_data_ptr() { 120 | make_own_data(); 121 | return storage_impl_()->mutable_data_ptr() + m_data_ptr_offset; 122 | } 123 | 124 | template 125 | const TensorStorageImpl
*TensorImpl
::storage_impl_() const { 126 | return dynamic_cast *>(m_storage.get()); 127 | } 128 | 129 | template 130 | TensorStorageImpl
*TensorImpl
::storage_impl_() { 131 | return dynamic_cast *>(m_storage.get()); 132 | } 133 | 134 | template 135 | typename std::enable_if::value, TensorPtr>::type 136 | as_tensor(T value) { 137 | return TensorPtr(fromcc(::ncg::CCType::identifier, value)); 138 | } 139 | template 140 | typename std::enable_if::value, TensorPtr>::type 141 | as_tensor(std::vector value) { 142 | return TensorPtr(fromcc(::ncg::CCType::identifier, value)); 143 | } 144 | template 145 | typename std::enable_if::value, TensorPtr>::type 146 | as_tensor(std::vector> value) { 147 | return TensorPtr(fromcc(::ncg::CCType::identifier, value)); 148 | } 149 | 150 | template ::value>::type*> 151 | TensorPtr::TensorPtr(T value) : TensorPtr(as_tensor(value)) {} 152 | template ::value>::type*> 153 | TensorPtr::TensorPtr(std::vector value) : TensorPtr(as_tensor(value)) {} 154 | template ::value>::type*> 155 | TensorPtr::TensorPtr(std::vector> value) : TensorPtr(as_tensor(value)) {} 156 | 157 | template 158 | typename std::enable_if::value, TensorPtr>::type 159 | fill(DTypeName dtype, const ShapeVec &shape, ValueT value) { 160 | auto s = empty(dtype, shape); 161 | 162 | #define FILL_DTYPE_CASE(dtype_name) do { \ 163 | auto s_dtype = s->template as();\ 164 | for (ssize_t i = 0; i < s_dtype->desc().numel(); ++i) s_dtype->mutable_elat(i) = value; \ 165 | } while(0) 166 | NCG_DTYPE_SWITCH_ALL(dtype, FILL_DTYPE_CASE); 167 | #undef FILL_DTYPE_CASE 168 | 169 | return s; 170 | } 171 | 172 | template 173 | typename std::enable_if::value, TensorPtr>::type 174 | scalar(DTypeName dtype, ValueT value) { 175 | auto s = empty(dtype, {}); 176 | 177 | #define SCALAR_DTYPE_CASE(dtype_name) s->template as()->mutable_data_ptr()[0] = value 178 | NCG_DTYPE_SWITCH_ALL(dtype, SCALAR_DTYPE_CASE); 179 | #undef SCALAR_DTYPE_CASE 180 | 181 | return s; 182 | } 183 | 184 | template 185 | typename std::enable_if::value, TensorPtr>::type 186 | fromcc(DTypeName dtype, ValueT value) { 187 | return scalar(dtype, value); 188 | } 189 | 190 | template 191 | typename std::enable_if::value, TensorPtr>::type 192 | fromcc(DTypeName dtype, std::vector values) { 193 | auto s = empty(dtype, {static_cast(values.size())}); 194 | 195 | #define FROMCC_DTYPE_CASE(dtype_name) do { \ 196 | auto data_ptr = s->template as()->mutable_data_ptr(); \ 197 | for (ssize_t i = 0; i < values.size(); ++i) { data_ptr[i] = values[i]; } \ 198 | } while(0) 199 | NCG_DTYPE_SWITCH_ALL(dtype, FROMCC_DTYPE_CASE); 200 | #undef FROMCC_DTYPE_CASE 201 | 202 | return s; 203 | } 204 | 205 | template 206 | typename std::enable_if::value, TensorPtr>::type 207 | fromcc(DTypeName dtype, std::vector> values) { 208 | ncg_assert(values.size() > 0); 209 | for (ssize_t i = 0; i < values.size(); ++i) { 210 | ncg_assert(values[i].size() == values[0].size()); 211 | } 212 | auto s = empty(dtype, {static_cast(values.size()), static_cast(values[0].size())}); 213 | 214 | #define FROMCC_DTYPE_CASE(dtype_name) do { \ 215 | auto data_ptr = s->template as()->mutable_data_ptr(); \ 216 | for (ssize_t i = 0, k = 0; i < values.size(); ++i) { \ 217 | for (ssize_t j = 0; j < values[i].size(); ++j) data_ptr[k++] = values[i][j]; \ 218 | } \ 219 | } while(0) 220 | NCG_DTYPE_SWITCH_ALL(dtype, FROMCC_DTYPE_CASE); 221 | #undef FROMCC_DTYPE_CASE 222 | 223 | return s; 224 | } 225 | 226 | template 227 | ValueT tocc_scalar(TensorPtr tensor) { 228 | ncg_assert(tensor->desc().dim() == 0); 229 | 230 | #define TOCC_DTYPE_CASE(dtype_name) do { \ 231 | auto data_ptr = tensor->template as()->data_ptr(); \ 232 | return data_ptr[0]; \ 233 | } while(0) 234 | NCG_DTYPE_SWITCH_ALL(tensor->desc().dtype(), TOCC_DTYPE_CASE); 235 | #undef TOCC_DTYPE_CASE 236 | } 237 | 238 | template 239 | std::vector tocc_vector(TensorPtr tensor) { 240 | ncg_assert(tensor->desc().dim() == 1); 241 | std::vector output(tensor->desc().shape(0)); 242 | 243 | #define TOCC_DTYPE_CASE(dtype_name) do { \ 244 | auto tensor_dtype = tensor->template as(); \ 245 | for (ssize_t i = 0; i < output.size(); ++i) output[i] = tensor_dtype->at(i); \ 246 | } while(0) 247 | NCG_DTYPE_SWITCH_ALL(tensor->desc().dtype(), TOCC_DTYPE_CASE); 248 | #undef TOCC_DTYPE_CASE 249 | 250 | return output; 251 | } 252 | 253 | } /* !namespace ncg */ 254 | 255 | -------------------------------------------------------------------------------- /src/core/tensor_storage.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * tensor_storage.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core/datatype.h" 9 | #include "core/tensor_storage.h" 10 | 11 | namespace ncg { 12 | 13 | TensorStorage::TensorStorage(DTypeName dtype) : m_dtype(dtype) { 14 | 15 | } 16 | 17 | DTypeName TensorStorage::dtype() const { 18 | return m_dtype; 19 | } 20 | 21 | std::ostream &operator << (std::ostream &out, const TensorStorage &storage) { 22 | #define COUT_STORAGE_DTYPE_CASE(dtype_name) out << dynamic_cast &>(storage); 23 | NCG_DTYPE_SWITCH_ALL(storage.dtype(), COUT_STORAGE_DTYPE_CASE) 24 | #undef COUT_STORAGE_DTYPE_CASE 25 | 26 | return out; 27 | } 28 | 29 | template 30 | TensorStorageImpl
::TensorStorageImpl() : TensorStorage(DT) { 31 | 32 | } 33 | 34 | template 35 | TensorStorageImpl
::TensorStorageImpl(cctype *data_ptr, size_t size) : TensorStorage(DT), m_data_ptr(data_ptr), m_size(size) { 36 | 37 | } 38 | 39 | template 40 | TensorStorageImpl
::TensorStorageImpl(size_t size) : TensorStorage(DT), m_size(size) { 41 | /* TODO: use aligned allocation. */ 42 | m_data_ptr = new cctype[size]; 43 | } 44 | 45 | template 46 | TensorStorageImpl
::~TensorStorageImpl() { 47 | if (m_data_ptr != nullptr) { 48 | delete []m_data_ptr; 49 | m_data_ptr = nullptr; 50 | } 51 | } 52 | 53 | template 54 | TensorStorage *TensorStorageImpl
::clone(ssize_t start, ssize_t length) const { 55 | auto ret = new TensorStorageImpl
(); 56 | 57 | if (length > m_size - start) { 58 | length = m_size - start; 59 | } 60 | 61 | if (m_data_ptr != nullptr) { 62 | ret->m_data_ptr = new cctype[length]; 63 | ret->m_size = length; 64 | 65 | memcpy(ret->m_data_ptr, m_data_ptr + start, m_size * sizeof(cctype)); 66 | } 67 | return static_cast(ret); 68 | } 69 | 70 | template 71 | void TensorStorageImpl
::pickle(NCGPickler &pickler) const { 72 | pickler.write(static_cast(m_dtype)); 73 | pickler.write_char_array(m_data_ptr, m_size); 74 | } 75 | 76 | template 77 | size_t TensorStorageImpl
::size() const { 78 | return m_size; 79 | } 80 | 81 | template 82 | size_t TensorStorageImpl
::memsize() const { 83 | return m_size * sizeof(cctype); 84 | } 85 | 86 | template 87 | const typename TensorStorageImpl
::cctype *TensorStorageImpl
::data_ptr() const { 88 | return m_data_ptr; 89 | } 90 | 91 | template 92 | typename TensorStorageImpl
::cctype *TensorStorageImpl
::mutable_data_ptr() { 93 | return m_data_ptr; 94 | } 95 | 96 | template 97 | std::ostream &operator << (std::ostream &out, const TensorStorageImpl
&storage) { 98 | out << "TensorStorage(dtype=" << get_dtype_name(DT) << ", " << "size=" << storage.m_size << ", data_ptr=" << storage.m_data_ptr << ", values=["; 99 | for (ssize_t i = 0; i < std::min(TensorValueMaxPrint, storage.m_size); ++i) { 100 | out << (i == 0 ? "" : ", ") << storage.m_data_ptr[i]; 101 | } 102 | if (storage.m_size > TensorValueMaxPrint) { 103 | out << ", ..."; 104 | } 105 | out << "])"; 106 | return out; 107 | } 108 | 109 | NCG_DTYPE_INSTANTIATE_CLASS_ALL(TensorStorageImpl); 110 | 111 | #define INSTANTIATE_FUNC(dtype_name) std::ostream &operator << (std::ostream &out, const TensorStorageImpl &storage) 112 | NCG_DTYPE_INSTANTIATE_ALL(INSTANTIATE_FUNC); 113 | #undef INSTANTIATE_FUNC 114 | 115 | std::shared_ptr tensor_storage(NCGUnpickler &unpickler) { 116 | auto dtype = static_cast(unpickler.read_int64()); 117 | 118 | #define TS_UNPICKLE_DTYPE_CASE(dtype_name) do { \ 119 | auto data = unpickler.read_char_array::cctype>(); \ 120 | return std::shared_ptr(new TensorStorageImpl( \ 121 | data.first.release(), data.second \ 122 | )); \ 123 | } while (0) 124 | NCG_DTYPE_SWITCH_ALL(dtype, TS_UNPICKLE_DTYPE_CASE); 125 | #undef TS_UNPICKLE_DTYPE_CASE 126 | } 127 | 128 | } /* !namespace ncg */ 129 | -------------------------------------------------------------------------------- /src/core/tensor_storage.h: -------------------------------------------------------------------------------- 1 | /* 2 | * tensor_storage.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/common.h" 11 | #include "core/datatype.h" 12 | #include "core/pickle.h" 13 | 14 | #include 15 | 16 | namespace ncg { 17 | 18 | const size_t TensorValueMaxPrint = 16; 19 | 20 | class TensorStorage { 21 | public: 22 | TensorStorage(DTypeName dtype); 23 | virtual ~TensorStorage() = default; 24 | 25 | DTypeName dtype() const; 26 | virtual size_t size() const = 0; 27 | virtual size_t memsize() const = 0; 28 | 29 | virtual TensorStorage *clone(ssize_t start = 0, ssize_t length = std::numeric_limits::max()) const = 0; 30 | virtual void pickle(NCGPickler &pickler) const = 0; 31 | 32 | friend std::ostream &operator << (std::ostream &out, const TensorStorage &storage); 33 | 34 | protected: 35 | DTypeName m_dtype; 36 | }; 37 | 38 | template class TensorStorageImpl; 39 | template std::ostream &operator << (std::ostream &out, const TensorStorageImpl
&storage); 40 | 41 | template 42 | class TensorStorageImpl : public TensorStorage { 43 | public: 44 | using cctype = typename DType
::cctype; 45 | 46 | TensorStorageImpl(); 47 | explicit TensorStorageImpl(cctype *data_ptr, size_t size); 48 | explicit TensorStorageImpl(size_t size); 49 | 50 | /* NB: delete the copy-constructor and move-constructor */ 51 | TensorStorageImpl(const TensorStorageImpl
&) = delete; 52 | TensorStorageImpl(TensorStorageImpl
&&) = delete; 53 | 54 | virtual ~TensorStorageImpl(); 55 | 56 | virtual size_t size() const; 57 | virtual size_t memsize() const; 58 | 59 | const cctype *data_ptr() const; 60 | cctype *mutable_data_ptr(); 61 | 62 | virtual TensorStorage *clone(ssize_t start = 0, ssize_t length = std::numeric_limits::max()) const; 63 | virtual void pickle(NCGPickler &pickler) const; 64 | 65 | friend std::ostream &operator << <> (std::ostream &out, const TensorStorageImpl
&storage); 66 | 67 | protected: 68 | cctype *m_data_ptr; 69 | size_t m_size; 70 | }; 71 | 72 | std::shared_ptr tensor_storage(NCGUnpickler &unpickler); 73 | 74 | } /* !namespace ncg */ 75 | 76 | -------------------------------------------------------------------------------- /src/custom_ops/assert.h: -------------------------------------------------------------------------------- 1 | /* 2 | * assert.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/tensor_impl.h" 11 | #include "graph/op.h" 12 | 13 | namespace ncg { 14 | 15 | class OpAssert : public Op { 16 | NCG_OP_DEF_NAME(OpAssert); 17 | 18 | virtual void check_inputs(OpContext &ctx, const TensorVec &inputs) { 19 | NCG_OP_CHECK_NR_INPUTS(ctx, inputs, 1); 20 | NCG_OP_CHECK_INPUT_SCALAR(ctx, inputs, 0); 21 | } 22 | 23 | virtual TensorVec compute(OpContext &ctx, const TensorVec &inputs) { 24 | bool rv = true; 25 | 26 | #define ASSERT_COMPUTE_DTYPE(dtype_name) rv = compute_inner_(ctx, inputs[0]) 27 | NCG_DTYPE_SWITCH_ALL(inputs[0]->desc().dtype(), ASSERT_COMPUTE_DTYPE); 28 | #undef ASSERT_COMPUTE_DTYPE 29 | 30 | if (!rv) { 31 | ctx.error(this) << "Assertion failed"; 32 | } 33 | 34 | return {inputs[0]}; 35 | } 36 | 37 | private: 38 | template 39 | bool compute_inner_(OpContext &ctx, const TensorPtr &input) { 40 | auto a = input->as
(); 41 | return a->elat(0) > 0; 42 | } 43 | }; 44 | 45 | class GOpAssert : public GraphOpWrapper, public GraphSingleOutputOp { 46 | public: 47 | NCG_GOP_DEF_NAME(GOpAssert); 48 | 49 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 50 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 1); 51 | NCG_OP_CHECK_INPUT_SCALAR(graph, inputs, 0); 52 | } 53 | 54 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 55 | return {make_tensor(0, inputs[0]->desc())}; 56 | } 57 | 58 | virtual void backward(Graph &graph, GTensorPtr loss) { 59 | auto output_grad = m_outputs[0]->grad(loss); 60 | m_inputs[0]->set_grad(graph, loss, output_grad); 61 | } 62 | }; 63 | 64 | } /* !namespace ncg */ 65 | 66 | -------------------------------------------------------------------------------- /src/custom_ops/bind.h: -------------------------------------------------------------------------------- 1 | /* 2 | * bind.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "graph/op.h" 11 | 12 | namespace ncg { 13 | 14 | class GOpBind : public GraphOp, public GraphSingleOutputOp { 15 | public: 16 | NCG_GOP_DEF_NAME(GOpBind); 17 | 18 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 19 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 2); 20 | } 21 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 22 | return {make_tensor(0, inputs[0]->desc())}; 23 | } 24 | 25 | virtual void forward(GraphForwardContext &ctx) const { 26 | TensorPtr tensor = ctx.tensor(m_inputs[0]); 27 | ctx.set_tensor(m_outputs[0], tensor); 28 | } 29 | 30 | virtual void backward(Graph &graph, GTensorPtr loss) { 31 | auto output_grad = m_outputs[0]->grad(loss); 32 | m_inputs[0]->set_grad(graph, loss, output_grad); 33 | m_inputs[1]->set_grad(graph, loss, nullptr); 34 | } 35 | }; 36 | 37 | } /* !namespace ncg */ 38 | 39 | -------------------------------------------------------------------------------- /src/custom_ops/print.h: -------------------------------------------------------------------------------- 1 | /* 2 | * print.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/tensor_impl.h" 11 | #include "graph/op.h" 12 | 13 | namespace ncg { 14 | 15 | struct OpPrintDesc : OpDesc { 16 | public: 17 | OpPrintDesc(const std::string &name = "") : name(name) {} 18 | std::string name; 19 | }; 20 | 21 | class GOpPrint : public GraphOp, public GraphSingleOutputOp { 22 | public: 23 | NCG_GOP_DEF_NAME(GOpPrint); 24 | 25 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 26 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 1); 27 | } 28 | 29 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 30 | return {make_tensor(0, inputs[0]->desc())}; 31 | } 32 | 33 | virtual void forward(GraphForwardContext &ctx) const { 34 | TensorPtr tensor = ctx.tensor(m_inputs[0]); 35 | const std::string &name = this->template desc().name; 36 | std::cout << "PRINT operator: " << name << " = " << tensor->as()->data_ptr()[0] << std::endl; 37 | ctx.set_tensor(m_outputs[0], tensor); 38 | } 39 | 40 | virtual void backward(Graph &graph, GTensorPtr loss) { 41 | auto output_grad = m_outputs[0]->grad(loss); 42 | m_inputs[0]->set_grad(graph, loss, output_grad); 43 | } 44 | }; 45 | 46 | } /* !namespace ncg */ 47 | 48 | -------------------------------------------------------------------------------- /src/graph.h: -------------------------------------------------------------------------------- 1 | /* 2 | * graph.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "graph/graph.h" 9 | #include "graph/op.h" 10 | #include "graph/ops/elemwise.h" 11 | #include "graph/ops/grad.h" 12 | #include "graph/ops/linalg.h" 13 | #include "graph/ops/netsrc.h" 14 | #include "graph/ops/reduction.h" 15 | #include "graph/ops/shape.h" 16 | #include "graph/ops/slice.h" 17 | #include "graph/ops/update.h" 18 | 19 | -------------------------------------------------------------------------------- /src/graph/graph.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * graph.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "graph/tensor.h" 9 | #include "graph/op.h" 10 | #include "graph/graph.h" 11 | #include "graph/ops/grad.h" 12 | 13 | namespace ncg { 14 | 15 | GraphTopoSorter::GraphTopoSorter(Graph &graph) : m_graph(graph) { 16 | // Pass 17 | } 18 | 19 | void GraphTopoSorter::sort(const GTensorVec &target) { 20 | m_sorted.clear(); 21 | m_visited.clear(); 22 | 23 | for (const auto &t : target) { 24 | mark_(t); 25 | } 26 | } 27 | 28 | const std::vector &GraphTopoSorter::sorted() const { 29 | return m_sorted; 30 | } 31 | 32 | void GraphTopoSorter::mark_(const GTensorPtr &t) { 33 | auto op = t->owner_op(); 34 | std::uintptr_t opi = reinterpret_cast(op); 35 | 36 | if (m_visited.find(opi) != m_visited.end()) { 37 | return; 38 | } 39 | for (const auto &input : op->inputs()) { 40 | mark_(input); 41 | } 42 | m_sorted.emplace_back(op); 43 | m_visited.emplace(opi); 44 | } 45 | 46 | Graph::Graph() : m_ops(), m_backproped_tensors() { 47 | // pass 48 | } 49 | 50 | std::ostringstream &Graph::error(const GraphOp *op) { 51 | auto &error = RuntimeContext::error(); 52 | // error << op->op_name() << ": "; 53 | return error; 54 | } 55 | 56 | const std::vector &Graph::ops() const { 57 | return m_ops; 58 | } 59 | 60 | GOpPtr Graph::find_op(const std::string &name) { 61 | for (auto &op : m_ops) { 62 | if (op->name() == name) { 63 | return op; 64 | } 65 | } 66 | return nullptr; 67 | } 68 | 69 | void Graph::backward(GTensorPtr loss) { 70 | auto loss_identifier = reinterpret_cast(loss.get()); 71 | if (m_backproped_tensors.find(loss_identifier) != m_backproped_tensors.end()) { 72 | return; 73 | } 74 | 75 | auto sorter = std::make_unique(*this); 76 | sorter->sort({loss}); 77 | const auto &sorted = sorter->sorted(); 78 | 79 | loss->set_grad(*this, loss, this->op(nullptr, loss)); 80 | for (auto it = sorted.rbegin(); it != sorted.rend(); ++it) { 81 | (*it)->backward(*this, loss); 82 | } 83 | 84 | m_backproped_tensors.emplace(loss_identifier); 85 | } 86 | 87 | Session::Session(Graph &graph) : m_graph(graph) { 88 | // pass 89 | } 90 | 91 | Graph &Session::Session::graph() { 92 | return m_graph; 93 | } 94 | 95 | const Graph &Session::graph() const { 96 | return m_graph; 97 | } 98 | 99 | bool Session::is_shared_tensor_initialized(const GTensorPtr >ensor) const { 100 | auto it = m_shared_tensors.find(reinterpret_cast(gtensor.get())); 101 | return it != m_shared_tensors.end(); 102 | } 103 | 104 | TensorPtr Session::shared_tensor(const GTensorPtr >ensor) const { 105 | auto it = m_shared_tensors.find(reinterpret_cast(gtensor.get())); 106 | ncg_assert(it != m_shared_tensors.end()); 107 | return it->second; 108 | } 109 | 110 | void Session::set_shared_tensor(const GTensorPtr >ensor, const TensorPtr &tensor) { 111 | m_shared_tensors[reinterpret_cast(gtensor.get())] = tensor; 112 | } 113 | 114 | void Session::save_shared_tensors(std::string filename) { 115 | NCGPickler pickler(filename); 116 | 117 | pickler.write(static_cast(m_shared_tensors.size())); 118 | for (auto &it : m_shared_tensors) { 119 | pickler.write(reinterpret_cast(it.first)->owner_op()->name()); 120 | it.second->pickle(pickler); 121 | } 122 | 123 | pickler.close(); 124 | } 125 | 126 | void Session::load_shared_tensors(std::string filename) { 127 | NCGUnpickler unpickler(filename); 128 | 129 | size_t size = static_cast(unpickler.read_int64()); 130 | for (ssize_t i = 0; i < size; ++i) { 131 | std::string name = unpickler.read_string(); 132 | auto tensor = ::ncg::tensor(unpickler); 133 | auto op = m_graph.find_op(name); 134 | if (op != nullptr) { 135 | set_shared_tensor(op->outputs()[0], tensor); 136 | } 137 | } 138 | 139 | unpickler.close(); 140 | } 141 | 142 | GraphForwardContext::GraphForwardContext() : m_session(get_default_session()), m_feed_dict(), m_storage() { 143 | } 144 | 145 | GraphForwardContext::GraphForwardContext(Session &session) : m_session(session), m_feed_dict(), m_storage() { 146 | // Pass 147 | } 148 | 149 | Session &GraphForwardContext::session() { 150 | return m_session; 151 | } 152 | 153 | const Session &GraphForwardContext::session() const { 154 | return m_session; 155 | } 156 | 157 | void GraphForwardContext::feed(const std::string &name, TensorPtr tensor) { 158 | m_feed_dict.emplace(name, tensor); 159 | } 160 | 161 | TensorPtr GraphForwardContext::feed_dict(const std::string &name) { 162 | auto it = m_feed_dict.find(name); 163 | if (it == m_feed_dict.end()) { 164 | return nullptr; 165 | } 166 | return it->second; 167 | } 168 | 169 | TensorVec GraphForwardContext::eval(const GTensorVec &targets) { 170 | TensorVec outputs; 171 | auto sorter = std::make_unique(m_session.graph()); 172 | sorter->sort(targets); 173 | 174 | for (const GraphOp *op: sorter->sorted()) { 175 | op->forward(*this); 176 | if (!ok()) { 177 | return outputs; 178 | } 179 | } 180 | 181 | for (const GraphOp *op: sorter->sorted()) { 182 | op->forward_hook_post(*this); 183 | if (!ok()) { 184 | return outputs; 185 | } 186 | } 187 | 188 | for (const auto &t: targets) { 189 | outputs.emplace_back(tensor(t)); 190 | } 191 | return outputs; 192 | } 193 | 194 | TensorPtr GraphForwardContext::tensor(const GTensorPtr >ensor) { 195 | auto it = m_storage.find(reinterpret_cast(gtensor.get())); 196 | if (it == m_storage.end()) { 197 | RuntimeContext::error() << "Can not found tensor: " << gtensor.get() << " " << gtensor << "."; 198 | } 199 | ncg_assert_msg(ok(), error_str()); 200 | return it->second; 201 | } 202 | 203 | void GraphForwardContext::set_tensor(const GTensorPtr >ensor, const TensorPtr &tensor) { 204 | m_storage.emplace(reinterpret_cast(gtensor.get()), tensor); 205 | } 206 | 207 | std::ostringstream &GraphForwardContext::error(const GraphOp *op) { 208 | auto &error = RuntimeContext::error(); 209 | // error << op->op_name() << ": "; 210 | return error; 211 | } 212 | 213 | namespace { 214 | 215 | static auto default_graph_manager = std::make_unique>(true); 216 | static auto default_session_manager = std::make_unique>(); 217 | static auto default_session = std::make_unique(default_graph_manager->get_default()); 218 | 219 | struct DefaultSessionInitializer { 220 | DefaultSessionInitializer() { 221 | default_session_manager->as_default(default_session.get()); 222 | } 223 | }; 224 | 225 | static DefaultSessionInitializer _default_session_initializer; 226 | 227 | } /* !namespace */ 228 | 229 | void as_default_graph(Graph &graph) { 230 | default_graph_manager->as_default(&graph); 231 | } 232 | 233 | Graph &get_default_graph() { 234 | return default_graph_manager->get_default(); 235 | } 236 | 237 | void restore_default_graph() { 238 | default_graph_manager->restore_default(); 239 | } 240 | 241 | void as_default_session(Session &session) { 242 | default_session_manager->as_default(&session); 243 | } 244 | 245 | Session &get_default_session() { 246 | return default_session_manager->get_default(); 247 | } 248 | 249 | void restore_default_session() { 250 | default_session_manager->restore_default(); 251 | } 252 | 253 | } /* !namespace ncg */ 254 | -------------------------------------------------------------------------------- /src/graph/graph.h: -------------------------------------------------------------------------------- 1 | /* 2 | * graph.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/op.h" 11 | #include "graph/tensor.h" 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | namespace ncg { 18 | 19 | class GraphOp; 20 | class GraphSingleOutputOp; 21 | typedef std::shared_ptr GOpPtr; 22 | 23 | class GraphTopoSorter final { 24 | public: 25 | GraphTopoSorter(Graph &graph); 26 | 27 | void sort(const GTensorVec &target); 28 | const std::vector &sorted() const; 29 | 30 | protected: 31 | Graph &m_graph; 32 | std::vector m_sorted; 33 | std::unordered_set m_visited; 34 | 35 | private: 36 | void mark_(const GTensorPtr &t); 37 | }; 38 | 39 | class Graph : public RuntimeContext { 40 | public: 41 | Graph(); 42 | virtual ~Graph() = default; 43 | 44 | std::ostringstream &error(const GraphOp *op); 45 | 46 | virtual void backward(GTensorPtr loss); 47 | 48 | template 49 | GOpPtr make_op(OpDescPtr desc, const TensorVec &inputs) { 50 | auto op = new OpClass(); 51 | (*op)(*this, desc, inputs); 52 | ncg_assert_msg(ok(), error_str()); 53 | auto op_ptr = GOpPtr(op); 54 | m_ops.push_back(op_ptr); 55 | return op_ptr; 56 | } 57 | 58 | template 59 | GOpPtr make_op(OpDescPtr desc, Tensors &&... args) { 60 | auto op = new OpClass(); 61 | (*op)(*this, desc, {std::forward(args)...}); 62 | ncg_assert_msg(ok(), error_str()); 63 | auto op_ptr = GOpPtr(op); 64 | m_ops.push_back(op_ptr); 65 | return op_ptr; 66 | } 67 | 68 | template 69 | GOpPtr make_op(const std::string &name, OpDescPtr desc, const TensorVec &inputs) { 70 | auto op = new OpClass(); 71 | op->set_name(name); 72 | (*op)(*this, desc, inputs); 73 | ncg_assert_msg(ok(), error_str()); 74 | auto op_ptr = GOpPtr(op); 75 | m_ops.push_back(op_ptr); 76 | return op_ptr; 77 | } 78 | 79 | template 80 | GOpPtr make_op(const std::string &name, OpDescPtr desc, Tensors &&... args) { 81 | auto op = new OpClass(); 82 | op->set_name(name); 83 | (*op)(*this, desc, {std::forward(args)...}); 84 | ncg_assert_msg(ok(), error_str()); 85 | auto op_ptr = GOpPtr(op); 86 | m_ops.push_back(op_ptr); 87 | return op_ptr; 88 | } 89 | 90 | const std::vector &ops() const; 91 | GOpPtr find_op(const std::string &name); 92 | 93 | template 94 | typename std::enable_if::value, GTensorPtr>::type 95 | op(OpDescPtr desc, Tensors &&... args) { 96 | auto op = make_op(desc, std::forward(args)...); 97 | ncg_assert(op->outputs().size() == 1); 98 | return op->outputs()[0]; 99 | } 100 | 101 | template 102 | typename std::enable_if::value, GTensorPtr>::type 103 | op(const std::string &name, OpDescPtr desc, Tensors &&... args) { 104 | auto op = make_op(name, desc, std::forward(args)...); 105 | ncg_assert(op->outputs().size() == 1); 106 | return op->outputs()[0]; 107 | } 108 | 109 | template 110 | typename std::enable_if::value, GTensorVec>::type 111 | op(OpDescPtr desc, Tensors &&... args) { 112 | auto op = make_op(desc, std::forward(args)...); 113 | return op->outputs(); 114 | } 115 | 116 | template 117 | typename std::enable_if::value, GTensorVec>::type 118 | op(const std::string &name, OpDescPtr desc, Tensors &&... args) { 119 | auto op = make_op(name, desc, std::forward(args)...); 120 | return op->outputs(); 121 | } 122 | 123 | protected: 124 | std::vector m_ops; 125 | std::unordered_set m_backproped_tensors; 126 | }; 127 | 128 | class Session { 129 | public: 130 | Session(Graph &graph); 131 | 132 | Graph &graph(); 133 | const Graph &graph() const; 134 | 135 | bool is_shared_tensor_initialized(const GTensorPtr &) const; 136 | TensorPtr shared_tensor(const GTensorPtr &) const; 137 | void set_shared_tensor(const GTensorPtr >ensor, const TensorPtr &tensor); 138 | 139 | void save_shared_tensors(std::string filename); 140 | void load_shared_tensors(std::string filename); 141 | 142 | protected: 143 | Graph &m_graph; 144 | std::unordered_map m_shared_tensors; 145 | }; 146 | 147 | class GraphForwardContext : public OpContext { 148 | public: 149 | GraphForwardContext(); 150 | GraphForwardContext(Session &session); 151 | 152 | Session &session(); 153 | const Session &session() const; 154 | 155 | void feed(const std::string &name, TensorPtr tensor); 156 | TensorPtr feed_dict(const std::string &name); 157 | std::vector eval(const GTensorVec &); 158 | 159 | TensorPtr tensor(const GTensorPtr &); 160 | void set_tensor(const GTensorPtr >ensor, const TensorPtr &tensor); 161 | 162 | std::ostringstream &error(const GraphOp *); 163 | 164 | protected: 165 | Session &m_session; 166 | std::unordered_map m_storage; 167 | std::unordered_map m_feed_dict; 168 | }; 169 | 170 | void as_default_graph(Graph &); 171 | Graph &get_default_graph(); 172 | void restore_default_graph(); 173 | 174 | void as_default_session(Session &); 175 | Session &get_default_session(); 176 | void restore_default_session(); 177 | 178 | } /* !namespace ncg */ 179 | 180 | -------------------------------------------------------------------------------- /src/graph/op.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * graph_op.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "graph/tensor.h" 9 | #include "graph/op.h" 10 | #include "graph/graph.h" 11 | 12 | namespace ncg { 13 | 14 | GraphOp::GraphOp() : 15 | m_desc(), m_inputs(), m_outputs(), 16 | m_initialized(false), 17 | m_name(), m_name_initialized(false) { 18 | // Pass 19 | } 20 | 21 | std::string GraphOp::name() const { 22 | if (m_name_initialized) { 23 | return m_name; 24 | } 25 | return auto_name(); 26 | } 27 | 28 | std::string GraphOp::auto_name() const { 29 | std::ostringstream ss; 30 | ss << op_name() << "@" << this; 31 | return ss.str(); 32 | } 33 | 34 | void GraphOp::set_name(const std::string &name) { 35 | m_name_initialized = true; 36 | m_name = name; 37 | } 38 | 39 | const GTensorVec &GraphOp::inputs() const { 40 | return m_inputs; 41 | } 42 | 43 | const GTensorVec &GraphOp::outputs() const { 44 | return m_outputs; 45 | } 46 | 47 | const GTensorVec &GraphOp::operator () (Graph &graph, OpDescPtr desc, const GTensorVec &inputs) { 48 | if (graph.is_error()) { 49 | return m_outputs; 50 | } 51 | 52 | ncg_assert_msg(!m_initialized, std::string("Op ") + name() + " initialized twice."); 53 | m_initialized = true; 54 | m_desc = desc; 55 | 56 | check_inputs(graph, inputs); 57 | m_inputs = inputs; 58 | if (graph.is_error()) { 59 | return m_outputs; 60 | } 61 | 62 | m_outputs = init_outputs(graph, inputs); 63 | return m_outputs; 64 | } 65 | 66 | void GraphOp::backward(Graph &graph, GTensorPtr loss) { 67 | graph.error(this) << "Backward is not implemented for " << op_name() << "."; 68 | } 69 | 70 | GTensorPtr GraphOp::make_tensor(ssize_t index, const TensorDesc &desc) { 71 | return GTensorPtr(new GraphTensor(this, index, desc)); 72 | } 73 | 74 | std::ostream & operator << (std::ostream &out, const GraphOp &op) { 75 | out << op.name() << "(\n\t"; 76 | for (ssize_t i = 0; i < op.m_inputs.size(); ++i) { 77 | out << (i == 0 ? "" : ", \n\t") << *(op.m_inputs[i]); 78 | } 79 | out << "\n)"; 80 | return out; 81 | } 82 | 83 | } /* !namespace ncg */ 84 | -------------------------------------------------------------------------------- /src/graph/op.h: -------------------------------------------------------------------------------- 1 | /* 2 | * graph_op.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/op.h" 11 | #include "graph/tensor.h" 12 | #include "graph/graph.h" 13 | 14 | #include 15 | #include 16 | 17 | namespace ncg { 18 | 19 | class GraphOp { 20 | public: 21 | GraphOp(); 22 | virtual ~GraphOp() = default; 23 | 24 | virtual const char *op_name() const = 0; 25 | std::string name() const; 26 | std::string auto_name() const; 27 | void set_name(const std::string &name); 28 | 29 | template 30 | const DescT &desc() const { 31 | auto p = dynamic_cast(m_desc.get()); 32 | ncg_assert(p != nullptr); 33 | return *p; 34 | } 35 | const GTensorVec &inputs() const; 36 | const GTensorVec &outputs() const; 37 | 38 | const GTensorVec & operator () (Graph &graph, OpDescPtr desc, const GTensorVec &inputs); 39 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) = 0; 40 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) = 0; 41 | 42 | virtual void forward_hook_pre(GraphForwardContext &ctx) const {} 43 | virtual void forward(GraphForwardContext &ctx) const = 0; 44 | virtual void forward_hook_post(GraphForwardContext &ctx) const {} 45 | virtual void backward(Graph &graph, GTensorPtr loss); 46 | 47 | GTensorPtr make_tensor(ssize_t index, const TensorDesc &desc); 48 | friend std::ostream & operator << (std::ostream &, const GraphOp &); 49 | 50 | protected: 51 | std::string m_name; 52 | bool m_name_initialized; 53 | 54 | OpDescPtr m_desc; 55 | GTensorVec m_inputs; 56 | GTensorVec m_outputs; 57 | bool m_initialized; 58 | }; 59 | 60 | #define NCG_GOP_DEF_NAME(op_name_) virtual const char *op_name() const { return #op_name_; } 61 | 62 | #define NCG_GOP_DEF_NO_GRAD(op_name_) void op_name_::backward(Graph &graph, GTensorPtr loss) { \ 63 | for (auto &tensor : m_inputs) { tensor->set_grad(graph, loss, nullptr); } \ 64 | } 65 | 66 | #define NCG_GOP_DEF_NO_GRAD_INLINE virtual void backward(Graph &graph, GTensorPtr loss) { \ 67 | for (auto &tensor : m_inputs) { tensor->set_grad(graph, loss, nullptr); } \ 68 | } 69 | 70 | class GraphSingleOutputOp { 71 | // Pass 72 | }; 73 | 74 | template 75 | class GraphOpWrapper : public GraphOp { 76 | public: 77 | virtual void forward(GraphForwardContext &ctx) const { 78 | TensorVec inputs; 79 | for (const auto >ensor : m_inputs) { 80 | inputs.push_back(ctx.tensor(gtensor)); 81 | } 82 | auto op = std::make_unique(); 83 | op->set_desc(m_desc); 84 | TensorVec outputs = op->execute(ctx, inputs); 85 | if (ctx.is_error()) { 86 | return; 87 | } 88 | ncg_assert(m_outputs.size() == outputs.size()); 89 | for (ssize_t i = 0; i < m_outputs.size(); ++i) { 90 | ctx.set_tensor(m_outputs[i], outputs[i]); 91 | } 92 | } 93 | }; 94 | 95 | } /* !namespace ncg */ 96 | 97 | -------------------------------------------------------------------------------- /src/graph/ops/elemwise.h: -------------------------------------------------------------------------------- 1 | /* 2 | * elemwise.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/tensor_impl.h" 11 | #include "core/ops/elemwise.h" 12 | #include "graph/op.h" 13 | 14 | namespace ncg { 15 | 16 | template 17 | class GOpElemwiseBase : public GraphOpWrapper { 18 | public: 19 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 20 | NCG_OP_CHECK_NONEMPTY_INPUTS(graph, inputs); 21 | NCG_OP_CHECK_COMPATIBLE_DTYPE(graph, inputs); 22 | NCG_OP_CHECK_COMPATIBLE_SHAPE(graph, inputs); 23 | } 24 | }; 25 | 26 | class GOpCast : public GOpElemwiseBase, public GraphSingleOutputOp { 27 | public: 28 | NCG_GOP_DEF_NAME(GOpCast); 29 | 30 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 31 | GOpElemwiseBase::check_inputs(graph, inputs); 32 | NCG_OP_CHECK_CTX_CLEAN(graph); 33 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 1); 34 | } 35 | 36 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 37 | auto dtype = this->template desc().dtype; 38 | TensorDesc desc(dtype, inputs[0]->desc().shape_vec()); 39 | return {this->make_tensor(0, desc)}; 40 | } 41 | 42 | virtual void backward(Graph &graph, GTensorPtr loss); 43 | }; 44 | 45 | class GOpCond : public GOpElemwiseBase, public GraphSingleOutputOp { 46 | public: 47 | NCG_GOP_DEF_NAME(GOpCond); 48 | 49 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 50 | GOpElemwiseBase::check_inputs(graph, inputs); 51 | NCG_OP_CHECK_CTX_CLEAN(graph); 52 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 3); 53 | } 54 | 55 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 56 | return {make_tensor(0, inputs[2]->desc())}; 57 | } 58 | 59 | virtual void backward(Graph &graph, GTensorPtr loss); 60 | }; 61 | 62 | template 63 | class GOpUnaryElemwiseBase : public GOpElemwiseBase, public GraphSingleOutputOp { 64 | public: 65 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 66 | GOpElemwiseBase::check_inputs(graph, inputs); 67 | NCG_OP_CHECK_CTX_CLEAN(graph); 68 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 1); 69 | } 70 | 71 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 72 | TensorDesc desc(inputs[0]->desc().dtype(), inputs[0]->desc().shape_vec()); 73 | return {this->make_tensor(0, desc)}; 74 | } 75 | }; 76 | 77 | template 78 | class GOpBinaryElemwiseBase : public GOpElemwiseBase, public GraphSingleOutputOp { 79 | public: 80 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 81 | GOpElemwiseBase::check_inputs(graph, inputs); 82 | NCG_OP_CHECK_CTX_CLEAN(graph); 83 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 2); 84 | } 85 | 86 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 87 | TensorDesc desc(inputs[0]->desc().dtype(), inputs[0]->desc().shape_vec()); 88 | return {this->make_tensor(0, desc)}; 89 | } 90 | }; 91 | 92 | #define DEF_GOP_UNARY(name) \ 93 | class GOp##name : public GOpUnaryElemwiseBase { \ 94 | public: \ 95 | NCG_GOP_DEF_NAME(GOp##name); \ 96 | virtual void backward(Graph &graph, GTensorPtr loss); \ 97 | } 98 | 99 | DEF_GOP_UNARY(Neg); 100 | DEF_GOP_UNARY(Sin); 101 | DEF_GOP_UNARY(Cos); 102 | DEF_GOP_UNARY(Tan); 103 | DEF_GOP_UNARY(Log); 104 | DEF_GOP_UNARY(Exp); 105 | DEF_GOP_UNARY(Tanh); 106 | DEF_GOP_UNARY(Sigmoid); 107 | DEF_GOP_UNARY(Reciprocal); 108 | 109 | #undef DEF_GOP_UNARY 110 | 111 | #define DEF_GOP_BINARY(name) \ 112 | class GOp##name : public GOpBinaryElemwiseBase { \ 113 | public: \ 114 | NCG_GOP_DEF_NAME(GOp##name); \ 115 | virtual void backward(Graph &graph, GTensorPtr loss); \ 116 | } 117 | 118 | DEF_GOP_BINARY(Add); 119 | DEF_GOP_BINARY(Sub); 120 | DEF_GOP_BINARY(Mul); 121 | DEF_GOP_BINARY(Div); 122 | DEF_GOP_BINARY(Ge); 123 | DEF_GOP_BINARY(Le); 124 | DEF_GOP_BINARY(Geq); 125 | DEF_GOP_BINARY(Leq); 126 | DEF_GOP_BINARY(Eq); 127 | DEF_GOP_BINARY(Neq); 128 | DEF_GOP_BINARY(Pow); 129 | DEF_GOP_BINARY(Min); 130 | DEF_GOP_BINARY(Max); 131 | 132 | #undef DEF_GOP_BINARY 133 | 134 | } /* !namespace ncg */ 135 | 136 | -------------------------------------------------------------------------------- /src/graph/ops/grad.h: -------------------------------------------------------------------------------- 1 | /* 2 | * grad.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/tensor_impl.h" 11 | #include "graph/op.h" 12 | 13 | namespace ncg { 14 | 15 | class GOpGradLoss : public GraphOp, public GraphSingleOutputOp { 16 | public: 17 | NCG_GOP_DEF_NAME(GOpGradLoss); 18 | 19 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 20 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 1); 21 | NCG_OP_CHECK_INPUT_SCALAR(graph, inputs, 0); 22 | } 23 | 24 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 25 | return {make_tensor(0, TensorDesc(inputs[0]->desc().dtype(), {}))}; 26 | } 27 | 28 | virtual void forward(GraphForwardContext &ctx) const { 29 | auto loss = ctx.tensor(m_inputs[0]); 30 | auto grad = scalar(loss->desc().dtype(), 1); 31 | ctx.set_tensor(m_outputs[0], grad); 32 | } 33 | 34 | NCG_GOP_DEF_NO_GRAD_INLINE; 35 | }; 36 | 37 | } /* !namespace ncg */ 38 | 39 | -------------------------------------------------------------------------------- /src/graph/ops/linalg.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * linalg.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "graph/ops/linalg.h" 9 | 10 | namespace ncg { 11 | 12 | void GOpMatMul::backward(Graph &graph, GTensorPtr loss) { 13 | const auto &desc = this->template desc(); 14 | 15 | auto output_grad = m_outputs[0]->grad(loss); 16 | if (output_grad == nullptr) { 17 | m_inputs[0]->set_grad(graph, loss, nullptr); 18 | m_inputs[1]->set_grad(graph, loss, nullptr); 19 | return; 20 | } 21 | 22 | m_inputs[0]->set_grad(graph, loss, 23 | graph.op(OpDescPtr(new OpMatMulDesc( 24 | desc.transpose_a, !desc.transpose_b 25 | )), output_grad, m_inputs[1]) 26 | ); 27 | m_inputs[1]->set_grad(graph, loss, 28 | graph.op(OpDescPtr(new OpMatMulDesc( 29 | !desc.transpose_a, desc.transpose_b 30 | )), m_inputs[0], output_grad) 31 | ); 32 | } 33 | 34 | } /* !namespace ncg */ 35 | -------------------------------------------------------------------------------- /src/graph/ops/linalg.h: -------------------------------------------------------------------------------- 1 | /* 2 | * linalg.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/tensor_impl.h" 11 | #include "core/ops/linalg.h" 12 | #include "graph/op.h" 13 | 14 | namespace ncg { 15 | 16 | class GOpMatMul : public GraphOpWrapper, GraphSingleOutputOp { 17 | public: 18 | NCG_GOP_DEF_NAME(GOPMatMul); 19 | 20 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 21 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 2); 22 | NCG_OP_CHECK_COMPATIBLE_DTYPE(graph, inputs); 23 | NCG_OP_CHECK_INPUT_DIM(graph, inputs, 0, 2); 24 | NCG_OP_CHECK_INPUT_DIM(graph, inputs, 1, 2); 25 | 26 | const auto &desc = this->template desc(); 27 | ssize_t k1 = !desc.transpose_a ? inputs[0]->desc().shape(1) : inputs[0]->desc().shape(0); 28 | ssize_t k2 = !desc.transpose_b ? inputs[1]->desc().shape(0) : inputs[1]->desc().shape(1); 29 | if (k1 != k2) { 30 | graph.error(this) << "Invalid shape: " << inputs[0]->desc().shape_vec() << " vs. " << inputs[1]->desc().shape_vec() << "."; 31 | } 32 | } 33 | 34 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 35 | const auto &desc = this->template desc(); 36 | const auto &a = inputs[0], &b = inputs[1]; 37 | ssize_t N = !desc.transpose_a ? a->desc().shape(0) : a->desc().shape(1); 38 | ssize_t M = !desc.transpose_b ? b->desc().shape(1) : b->desc().shape(0); 39 | 40 | return {make_tensor(0, TensorDesc(inputs[0]->desc().dtype(), {N, M}))}; 41 | } 42 | 43 | virtual void backward(Graph &graph, GTensorPtr loss); 44 | }; 45 | 46 | } /* !namespace ncg */ 47 | 48 | -------------------------------------------------------------------------------- /src/graph/ops/netsrc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * netsrc.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "graph/op.h" 11 | 12 | namespace ncg { 13 | 14 | class GraphNetSrcOp : public GraphOp, public GraphSingleOutputOp { 15 | public: 16 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 17 | NCG_OP_CHECK_EMPTY_INPUTS(graph, inputs); 18 | } 19 | 20 | NCG_GOP_DEF_NO_GRAD_INLINE; 21 | }; 22 | 23 | class GOpPlaceholderDesc : public OpDesc { 24 | public: 25 | GOpPlaceholderDesc() : desc() {} 26 | GOpPlaceholderDesc(DTypeName dtype, const ShapeVec &shape) : desc(dtype, shape) {} 27 | virtual ~GOpPlaceholderDesc() = default; 28 | 29 | TensorDesc desc; 30 | }; 31 | 32 | class GOpPlaceholder : public GraphNetSrcOp { 33 | public: 34 | NCG_GOP_DEF_NAME(GOpPlaceholder); 35 | 36 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 37 | return {make_tensor(0, this->template desc().desc)}; 38 | } 39 | virtual void forward(GraphForwardContext &ctx) const { 40 | TensorPtr tensor = ctx.feed_dict(name()); 41 | if (!tensor) { 42 | ctx.error(this) << "Placeholder missing"; 43 | } 44 | ctx.set_tensor(m_outputs[0], tensor); 45 | } 46 | }; 47 | 48 | class GOpConstantDesc : public OpDesc { 49 | public: 50 | GOpConstantDesc() : tensor() {} 51 | GOpConstantDesc(const TensorPtr &tensor) : tensor(tensor) {} 52 | virtual ~GOpConstantDesc() = default; 53 | 54 | TensorPtr tensor; 55 | }; 56 | 57 | class GOpConstant : public GraphNetSrcOp { 58 | public: 59 | NCG_GOP_DEF_NAME(GOpConstant); 60 | 61 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 62 | return {make_tensor(0, this->template desc().tensor->desc())}; 63 | } 64 | virtual void forward(GraphForwardContext &ctx) const { 65 | ctx.set_tensor(m_outputs[0], this->template desc().tensor); 66 | } 67 | }; 68 | 69 | class GOpVariableDesc : public OpDesc { 70 | public: 71 | GOpVariableDesc() : tensor() {} 72 | GOpVariableDesc(const TensorPtr &tensor) : tensor(tensor) {} 73 | virtual ~GOpVariableDesc() = default; 74 | 75 | TensorPtr tensor; 76 | }; 77 | 78 | class GOpVariable : public GraphNetSrcOp { 79 | public: 80 | NCG_GOP_DEF_NAME(GOpVariable); 81 | 82 | void set_value(Session &session, const TensorPtr &tensor) { 83 | session.set_shared_tensor(m_outputs[0], tensor); 84 | } 85 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 86 | return {make_tensor(0, this->template desc().tensor->desc())}; 87 | } 88 | virtual void forward(GraphForwardContext &ctx) const { 89 | if (!ctx.session().is_shared_tensor_initialized(m_outputs[0])) { 90 | ctx.session().set_shared_tensor(m_outputs[0], this->template desc().tensor); 91 | } 92 | ctx.set_tensor(m_outputs[0], ctx.session().shared_tensor(m_outputs[0])); 93 | } 94 | }; 95 | 96 | class GraphNetSrcOpDynamicShape : public GraphOp, public GraphSingleOutputOp { 97 | public: 98 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 99 | NCG_OP_CHECK_NR_INPUTS2(graph, inputs, 0, 1); 100 | if (inputs.size() == 1) { 101 | NCG_OP_CHECK_INPUT_VECTOR(graph, inputs, 0); 102 | } 103 | } 104 | 105 | NCG_GOP_DEF_NO_GRAD_INLINE; 106 | }; 107 | 108 | class OpZerosDesc : public OpDesc { 109 | public: 110 | OpZerosDesc() : desc() {} 111 | OpZerosDesc(DTypeName dtype, const ShapeVec &shape) : desc(dtype, shape) {} 112 | virtual ~OpZerosDesc() = default; 113 | 114 | TensorDesc desc; 115 | }; 116 | 117 | class GOpZeros: public GraphNetSrcOpDynamicShape { 118 | public: 119 | NCG_GOP_DEF_NAME(GOpZeros); 120 | 121 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 122 | return {make_tensor(0, this->template desc().desc)}; 123 | } 124 | virtual void forward(GraphForwardContext &ctx) const { 125 | auto &desc = this->template desc().desc; 126 | 127 | ShapeVec shape; 128 | if (m_inputs.size() == 0) { 129 | shape = desc.shape_vec(); 130 | } else { 131 | auto tmp_shape = tocc_vector(ctx.tensor(m_inputs[0])); 132 | shape = ShapeVec(tmp_shape.begin(), tmp_shape.end()); 133 | } 134 | auto tensor = zeros(desc.dtype(), shape); 135 | ctx.set_tensor(m_outputs[0], tensor); 136 | } 137 | }; 138 | 139 | class OpOnesDesc : public OpDesc { 140 | public: 141 | OpOnesDesc() : desc() {} 142 | OpOnesDesc(DTypeName dtype, const ShapeVec &shape) : desc(dtype, shape) {} 143 | virtual ~OpOnesDesc() = default; 144 | 145 | TensorDesc desc; 146 | }; 147 | 148 | class GOpOnes: public GraphNetSrcOpDynamicShape { 149 | public: 150 | NCG_GOP_DEF_NAME(GOpZeros); 151 | 152 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 153 | return {make_tensor(0, this->template desc().desc)}; 154 | } 155 | virtual void forward(GraphForwardContext &ctx) const { 156 | auto &desc = this->template desc().desc; 157 | ShapeVec shape; 158 | if (m_inputs.size() == 0) { 159 | shape = desc.shape_vec(); 160 | } else { 161 | auto tmp_shape = tocc_vector(ctx.tensor(m_inputs[0])); 162 | shape = ShapeVec(tmp_shape.begin(), tmp_shape.end()); 163 | } 164 | auto tensor = ones(desc.dtype(), shape); 165 | ctx.set_tensor(m_outputs[0], tensor); 166 | } 167 | }; 168 | 169 | } /* !namespace ncg */ 170 | 171 | -------------------------------------------------------------------------------- /src/graph/ops/reduction.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * reduction.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "graph/ops/elemwise.h" 9 | #include "graph/ops/reduction.h" 10 | #include "graph/ops/shape.h" 11 | #include "graph/ops/slice.h" 12 | 13 | namespace ncg { 14 | 15 | void GOpReduceMax::backward(Graph &graph, GTensorPtr loss) { 16 | auto output_grad = m_outputs[0]->grad(loss); 17 | if (output_grad == nullptr) { 18 | m_inputs[0]->set_grad(graph, loss, nullptr); 19 | return; 20 | } 21 | 22 | const auto &desc = this->template desc(); 23 | 24 | if (!desc.keepdims) { 25 | output_grad = graph.op( 26 | OpDescPtr(new OpUnsqueezeDesc(desc.axis)), 27 | output_grad 28 | ); 29 | } 30 | 31 | m_inputs[0]->set_grad(graph, loss, graph.op( 32 | OpDescPtr(new OpGatherBackwardDesc(desc.axis, m_inputs[0]->desc().shape(desc.axis))), 33 | output_grad, m_outputs[1], 34 | graph.op(OpDescPtr(new OpShapeOfIndexDesc(desc.axis)), m_inputs[0]) 35 | )); 36 | } 37 | 38 | void GOpReduceMin::backward(Graph &graph, GTensorPtr loss) { 39 | auto output_grad = m_outputs[0]->grad(loss); 40 | if (output_grad == nullptr) { 41 | m_inputs[0]->set_grad(graph, loss, nullptr); 42 | return; 43 | } 44 | 45 | const auto &desc = this->template desc(); 46 | 47 | if (!desc.keepdims) { 48 | output_grad = graph.op( 49 | OpDescPtr(new OpUnsqueezeDesc(desc.axis)), 50 | output_grad 51 | ); 52 | } 53 | 54 | m_inputs[0]->set_grad(graph, loss, graph.op( 55 | OpDescPtr(new OpGatherBackwardDesc(desc.axis, m_inputs[0]->desc().shape(desc.axis))), 56 | output_grad, m_outputs[1], 57 | graph.op(OpDescPtr(new OpShapeOfIndexDesc(desc.axis)), m_inputs[0]) 58 | )); 59 | } 60 | 61 | void GOpReduceSum::backward(Graph &graph, GTensorPtr loss) { 62 | auto output_grad = m_outputs[0]->grad(loss); 63 | if (output_grad == nullptr) { 64 | m_inputs[0]->set_grad(graph, loss, nullptr); 65 | return; 66 | } 67 | 68 | const auto &desc = this->template desc(); 69 | 70 | if (!desc.keepdims) { 71 | output_grad = graph.op( 72 | OpDescPtr(new OpUnsqueezeDesc(desc.axis)), 73 | output_grad 74 | ); 75 | } 76 | 77 | m_inputs[0]->set_grad(graph, loss, graph.op( 78 | OpDescPtr(new OpExpandDesc(m_inputs[0]->desc().shape_vec())), 79 | output_grad, 80 | graph.op(nullptr, m_inputs[0]) 81 | )); 82 | } 83 | 84 | void GOpReduceMean::backward(Graph &graph, GTensorPtr loss) { 85 | auto output_grad = m_outputs[0]->grad(loss); 86 | if (output_grad == nullptr) { 87 | m_inputs[0]->set_grad(graph, loss, nullptr); 88 | return; 89 | } 90 | 91 | const auto &desc = this->template desc(); 92 | 93 | if (!desc.keepdims) { 94 | output_grad = graph.op( 95 | OpDescPtr(new OpUnsqueezeDesc(desc.axis)), 96 | output_grad 97 | ); 98 | } 99 | 100 | m_inputs[0]->set_grad(graph, loss, graph.op(nullptr, 101 | G::auto_broadcast(graph, { 102 | graph.op( 103 | OpDescPtr(new OpExpandDesc(m_inputs[0]->desc().shape_vec())), 104 | output_grad, 105 | graph.op(nullptr, m_inputs[0]) 106 | ), 107 | graph.op( 108 | OpDescPtr(new OpCastDesc(m_inputs[0]->desc().dtype())), 109 | graph.op( 110 | OpDescPtr(new OpShapeOfIndexDesc(desc.axis)), 111 | m_inputs[0] 112 | ) 113 | ) 114 | }) 115 | )); 116 | } 117 | 118 | } /* !namespace ncg */ 119 | -------------------------------------------------------------------------------- /src/graph/ops/reduction.h: -------------------------------------------------------------------------------- 1 | /* 2 | * reduction.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/tensor_impl.h" 11 | #include "core/ops/reduction.h" 12 | #include "graph/op.h" 13 | 14 | namespace ncg { 15 | 16 | template 17 | class GOpReduceBase : public GraphOpWrapper { 18 | public: 19 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 20 | const auto &desc = this->template desc(); 21 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 1); 22 | NCG_OP_CHECK_INPUT_DIM_GEQ(graph, inputs, 0, desc.axis); 23 | } 24 | }; 25 | 26 | template 27 | class GOpReduceType1Base : public GOpReduceBase { 28 | public: 29 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 30 | const auto &desc = this->template desc(); 31 | auto axis = desc.axis; 32 | if (axis < 0) axis += inputs[0]->desc().dim(); 33 | 34 | auto output_shape = inputs[0]->desc().shape_vec(); 35 | 36 | if (desc.keepdims) { 37 | output_shape[axis] = 1; 38 | } else { 39 | output_shape.erase(output_shape.begin() + axis); 40 | } 41 | 42 | GTensorVec outputs = { 43 | this->make_tensor(0, TensorDesc(inputs[0]->desc().dtype(), output_shape)), 44 | this->make_tensor(1, TensorDesc(DTypeName::Int64, output_shape)) 45 | }; 46 | 47 | return outputs; 48 | } 49 | }; 50 | 51 | template 52 | class GOpReduceType2Base : public GOpReduceBase, public GraphSingleOutputOp { 53 | public: 54 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 55 | const auto &desc = this->template desc(); 56 | auto axis = desc.axis; 57 | if (axis < 0) axis += inputs[0]->desc().dim(); 58 | 59 | auto output_shape = inputs[0]->desc().shape_vec(); 60 | if (desc.keepdims) { 61 | output_shape[axis] = 1; 62 | } else { 63 | output_shape.erase(output_shape.begin() + axis); 64 | } 65 | 66 | return {this->make_tensor(0, TensorDesc(inputs[0]->desc().dtype(), output_shape))}; 67 | } 68 | }; 69 | 70 | #define DEF_GOP_REDUCE(name, type_id) \ 71 | class GOpReduce##name : public GOpReduceType##type_id##Base { \ 72 | public: \ 73 | NCG_GOP_DEF_NAME(GOp##name); \ 74 | virtual void backward(Graph &graph, GTensorPtr loss); \ 75 | } 76 | 77 | DEF_GOP_REDUCE(Max, 1); 78 | DEF_GOP_REDUCE(Min, 1); 79 | DEF_GOP_REDUCE(Sum, 2); 80 | DEF_GOP_REDUCE(Mean, 2); 81 | 82 | #undef DEF_GOP_REDUCE 83 | 84 | } /* !namespace ncg */ 85 | 86 | -------------------------------------------------------------------------------- /src/graph/ops/shape.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * shape.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "graph/ops/shape.h" 9 | #include "graph/ops/reduction.h" 10 | 11 | namespace ncg { 12 | 13 | 14 | void GOpReshape::backward(Graph &graph, GTensorPtr loss) { 15 | if (m_inputs.size() == 2) { 16 | m_inputs[1]->set_grad(graph, loss, nullptr); 17 | } 18 | 19 | auto output_grad = m_outputs[0]->grad(loss); 20 | if (output_grad == nullptr) { 21 | m_inputs[0]->set_grad(graph, loss, nullptr); 22 | return; 23 | } 24 | 25 | m_inputs[0]->set_grad(graph, loss, graph.op( 26 | OpDescPtr(new OpReshapeDesc(m_inputs[0]->desc().shape_vec())), 27 | output_grad, 28 | graph.op(nullptr, m_inputs[0]) 29 | )); 30 | } 31 | 32 | void GOpPermute::backward(Graph &graph, GTensorPtr loss) { 33 | auto output_grad = m_outputs[0]->grad(loss); 34 | if (output_grad == nullptr) { 35 | m_inputs[0]->set_grad(graph, loss, nullptr); 36 | return; 37 | } 38 | 39 | const auto &axes = this->template desc().axes; 40 | auto inverse_axes = ShapeVec(axes.size()); 41 | for (ssize_t i = 0; i < m_inputs[0]->desc().dim(); ++i){ 42 | inverse_axes[axes[i]] = i; 43 | } 44 | m_inputs[0]->set_grad(graph, loss, graph.op( 45 | OpDescPtr(new OpPermuteDesc(inverse_axes)), 46 | output_grad 47 | )); 48 | } 49 | 50 | void GOpExpand::backward(Graph &graph, GTensorPtr loss) { 51 | if (m_inputs.size() == 2) { 52 | m_inputs[1]->set_grad(graph, loss, nullptr); 53 | } 54 | 55 | auto output_grad = m_outputs[0]->grad(loss); 56 | if (output_grad == nullptr) { 57 | m_inputs[0]->set_grad(graph, loss, nullptr); 58 | return; 59 | } 60 | 61 | auto grad = output_grad; 62 | for (ssize_t i = 0; i < m_outputs[0]->desc().dim(); ++i) { 63 | if (m_inputs[0]->desc().shape(i) != m_outputs[0]->desc().shape(i)) { 64 | grad = graph.op( 65 | OpDescPtr(new OpReduceDesc(i, true)), 66 | grad 67 | ); 68 | } 69 | } 70 | 71 | m_inputs[0]->set_grad(graph, loss, grad); 72 | } 73 | 74 | void GOpSqueeze::backward(Graph &graph, GTensorPtr loss) { 75 | auto output_grad = m_outputs[0]->grad(loss); 76 | if (output_grad == nullptr) { 77 | m_inputs[0]->set_grad(graph, loss, nullptr); 78 | return; 79 | } 80 | 81 | const auto &desc = this->template desc(); 82 | 83 | m_inputs[0]->set_grad(graph, loss, graph.op( 84 | OpDescPtr(new OpUnsqueezeDesc(desc.axis)), 85 | output_grad 86 | )); 87 | } 88 | 89 | void GOpUnsqueeze::backward(Graph &graph, GTensorPtr loss) { 90 | auto output_grad = m_outputs[0]->grad(loss); 91 | if (output_grad == nullptr) { 92 | m_inputs[0]->set_grad(graph, loss, nullptr); 93 | return; 94 | } 95 | 96 | const auto &desc = this->template desc(); 97 | 98 | m_inputs[0]->set_grad(graph, loss, graph.op( 99 | OpDescPtr(new OpSqueezeDesc(desc.axis)), 100 | output_grad 101 | )); 102 | } 103 | 104 | } /* !namespace ncg */ 105 | -------------------------------------------------------------------------------- /src/graph/ops/shape.h: -------------------------------------------------------------------------------- 1 | /* 2 | * shape.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/ops/shape.h" 11 | #include "graph/op.h" 12 | 13 | namespace ncg { 14 | 15 | class GOpShapeOf : public GraphOp, public GraphSingleOutputOp { 16 | public: 17 | NCG_GOP_DEF_NAME(GOpShapeOf); 18 | 19 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 20 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 1); 21 | } 22 | 23 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 24 | return {make_tensor(0, TensorDesc(DTypeName::Int64, {static_cast(inputs[0]->desc().dim())}))}; 25 | } 26 | 27 | virtual void forward(GraphForwardContext &ctx) const { 28 | auto input = ctx.tensor(m_inputs[0]); 29 | auto shape_tensor = fromcc(DTypeName::Int64, input->desc().shape_vec()); 30 | ctx.set_tensor(m_outputs[0], shape_tensor); 31 | } 32 | 33 | NCG_GOP_DEF_NO_GRAD_INLINE; 34 | }; 35 | 36 | class OpShapeOfIndexDesc : public OpDesc { 37 | public: 38 | OpShapeOfIndexDesc() : axis(0) {} 39 | OpShapeOfIndexDesc(ssize_t axis) : axis(axis) {} 40 | virtual ~OpShapeOfIndexDesc() = default; 41 | 42 | ssize_t axis; 43 | }; 44 | 45 | class GOpShapeOfIndex : public GraphOp, public GraphSingleOutputOp { 46 | public: 47 | NCG_GOP_DEF_NAME(GOpShapeOfIndex); 48 | 49 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 50 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 1); 51 | const auto &desc = this->template desc(); 52 | auto axis = desc.axis; 53 | if (axis < 0) axis += inputs[0]->desc().dim(); 54 | NCG_OP_CHECK_INPUT_DIM_GEQ(graph, inputs, 0, axis); 55 | } 56 | 57 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 58 | return {make_tensor(0, TensorDesc(DTypeName::Int64, {}))}; 59 | } 60 | 61 | virtual void forward(GraphForwardContext &ctx) const { 62 | auto input = ctx.tensor(m_inputs[0]); 63 | const auto &desc = this->template desc(); 64 | auto axis = desc.axis; 65 | if (axis < 0) axis += input->desc().dim(); 66 | 67 | auto shape_tensor = fromcc(DTypeName::Int64, input->desc().shape(axis)); 68 | ctx.set_tensor(m_outputs[0], shape_tensor); 69 | } 70 | 71 | NCG_GOP_DEF_NO_GRAD_INLINE; 72 | }; 73 | 74 | class GOpShapeConcat : public GraphOp, public GraphSingleOutputOp { 75 | public: 76 | NCG_GOP_DEF_NAME(GOpShapeConcat); 77 | 78 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 79 | NCG_OP_CHECK_NONEMPTY_INPUTS(graph, inputs); 80 | for (ssize_t i = 0; i < inputs.size(); ++i) { 81 | NCG_OP_CHECK_INPUT_SCALAR_VECTOR(graph, inputs, i); 82 | NCG_OP_CHECK_INPUT_DTYPE_INT(graph, inputs, i); 83 | } 84 | } 85 | 86 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 87 | ssize_t tot = 0; 88 | for (const auto &i: inputs) { 89 | if (i->desc().dim() == 0) ++tot; 90 | else tot += i->desc().shape(0); 91 | } 92 | return {make_tensor(0, TensorDesc(DTypeName::Int64, {tot}))}; 93 | } 94 | 95 | virtual void forward(GraphForwardContext &ctx) const { 96 | ssize_t tot = 0; 97 | for (const auto &gi: m_inputs) { 98 | auto i = ctx.tensor(gi); 99 | if (i->desc().dim() == 0) ++tot; 100 | else tot += i->desc().shape(0); 101 | } 102 | 103 | auto shape_tensor = empty(DTypeName::Int64, {tot}); 104 | auto shape_tensor_dtype = shape_tensor->template as(); 105 | tot = 0; 106 | for (const auto &gi: m_inputs) { 107 | auto i = ctx.tensor(gi); 108 | if (i->desc().dim() == 0) { 109 | auto i_value = tocc_scalar(i); 110 | shape_tensor_dtype->mutable_at(tot) = i_value; 111 | ++tot; 112 | } else { 113 | auto i_value = tocc_vector(i); 114 | for (ssize_t j = 0; j < i->desc().shape(0); ++j) { 115 | shape_tensor_dtype->mutable_at(tot) = i_value[j]; 116 | ++tot; 117 | } 118 | } 119 | } 120 | 121 | ctx.set_tensor(m_outputs[0], shape_tensor); 122 | } 123 | 124 | NCG_GOP_DEF_NO_GRAD_INLINE; 125 | }; 126 | 127 | class GOpReshape : public GraphOp, public GraphSingleOutputOp { 128 | public: 129 | NCG_GOP_DEF_NAME(GOpReshape); 130 | 131 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 132 | NCG_OP_CHECK_NR_INPUTS2(graph, inputs, 1, 2); 133 | 134 | if (inputs.size() == 2) { 135 | NCG_OP_CHECK_INPUT_DTYPE_INT(graph, inputs, 1); 136 | NCG_OP_CHECK_INPUT_DIM(graph, inputs, 1, 1); 137 | } 138 | } 139 | 140 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 141 | const auto &desc = this->template desc(); 142 | return {make_tensor(0, TensorDesc(inputs[0]->desc().dtype(), desc.shape))}; 143 | } 144 | 145 | virtual void forward(GraphForwardContext &ctx) const { 146 | const auto &desc = this->template desc(); 147 | auto input = ctx.tensor(m_inputs[0]); 148 | 149 | if (m_inputs.size() == 1) { 150 | auto op = std::make_unique(); 151 | op->set_desc(m_desc); 152 | auto output = op->execute(ctx, {input}); 153 | ctx.set_tensor(m_outputs[0], output[0]); 154 | } else { 155 | auto shape_vector = tocc_vector(ctx.tensor(m_inputs[1])); 156 | auto tmp_desc = OpDescPtr(new OpReshapeDesc(ShapeVec(shape_vector.begin(), shape_vector.end()))); 157 | auto op = std::make_unique(); 158 | op->set_desc(tmp_desc); 159 | auto output = op->execute(ctx, {input}); 160 | ctx.set_tensor(m_outputs[0], output[0]); 161 | } 162 | } 163 | 164 | virtual void backward(Graph &graph, GTensorPtr loss); 165 | }; 166 | 167 | class GOpPermute : public GraphOp, public GraphSingleOutputOp { 168 | public: 169 | NCG_GOP_DEF_NAME(GOpPermute); 170 | 171 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 172 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 1); 173 | } 174 | 175 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 176 | const auto input = inputs[0]; 177 | const auto &axes = this->template desc().axes; 178 | 179 | ShapeVec output_shape = input->desc().shape_vec(); 180 | for (ssize_t i = 0; i < input->desc().dim(); ++i) { 181 | output_shape[i] = input->desc().shape(axes[i]); 182 | } 183 | return {make_tensor(0, TensorDesc(input->desc().dtype(), output_shape))}; 184 | } 185 | 186 | virtual void forward(GraphForwardContext &ctx) const { 187 | const auto &desc = this->template desc(); 188 | auto input = ctx.tensor(m_inputs[0]); 189 | auto op = std::make_unique(); 190 | op->set_desc(m_desc); 191 | auto output = op->execute(ctx, {input}); 192 | ctx.set_tensor(m_outputs[0], output[0]); 193 | } 194 | 195 | virtual void backward(Graph &graph, GTensorPtr loss); 196 | }; 197 | 198 | class GOpExpand : public GraphOp, public GraphSingleOutputOp { 199 | public: 200 | NCG_GOP_DEF_NAME(GOpExpand); 201 | 202 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 203 | NCG_OP_CHECK_NR_INPUTS2(graph, inputs, 1, 2); 204 | 205 | if (inputs.size() == 2) { 206 | NCG_OP_CHECK_INPUT_DTYPE_INT(graph, inputs, 1); 207 | NCG_OP_CHECK_INPUT_DIM(graph, inputs, 1, 1); 208 | } 209 | } 210 | 211 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 212 | const auto &desc = this->template desc(); 213 | return {make_tensor(0, TensorDesc(inputs[0]->desc().dtype(), desc.shape))}; 214 | } 215 | 216 | virtual void forward(GraphForwardContext &ctx) const { 217 | const auto &desc = this->template desc(); 218 | auto input = ctx.tensor(m_inputs[0]); 219 | 220 | if (m_inputs.size() == 1) { 221 | auto op = std::make_unique(); 222 | op->set_desc(m_desc); 223 | auto output = op->execute(ctx, {input}); 224 | ctx.set_tensor(m_outputs[0], output[0]); 225 | } else { 226 | auto shape_vector = tocc_vector(ctx.tensor(m_inputs[1])); 227 | auto tmp_desc = OpDescPtr(new OpExpandDesc(ShapeVec(shape_vector.begin(), shape_vector.end()))); 228 | auto op = std::make_unique(); 229 | op->set_desc(tmp_desc); 230 | auto output = op->execute(ctx, {input}); 231 | ctx.set_tensor(m_outputs[0], output[0]); 232 | } 233 | } 234 | 235 | virtual void backward(Graph &graph, GTensorPtr loss); 236 | }; 237 | 238 | class GOpSqueeze : public GraphOpWrapper, public GraphSingleOutputOp { 239 | public: 240 | NCG_GOP_DEF_NAME(GOpSqueeze); 241 | 242 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 243 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 1); 244 | 245 | const auto &desc = this->template desc(); 246 | auto axis = desc.axis; 247 | if (axis < 0) axis += inputs[0]->desc().dim(); 248 | NCG_OP_CHECK_INPUT_DIM_GEQ(graph, inputs, 0, axis); 249 | } 250 | 251 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 252 | const auto &desc = this->template desc(); 253 | auto axis = desc.axis; 254 | if (axis < 0) axis += inputs[0]->desc().dim(); 255 | 256 | auto shape = inputs[0]->desc().shape_vec(); 257 | shape.erase(shape.begin() + axis); 258 | 259 | return {make_tensor(0, TensorDesc(inputs[0]->desc().dtype(), shape))}; 260 | } 261 | 262 | virtual void backward(Graph &graph, GTensorPtr loss); 263 | }; 264 | 265 | class GOpUnsqueeze : public GraphOpWrapper, public GraphSingleOutputOp { 266 | public: 267 | NCG_GOP_DEF_NAME(GOpUnsqueeze); 268 | 269 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 270 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 1); 271 | 272 | const auto &desc = this->template desc(); 273 | auto axis = desc.axis; 274 | if (axis < 0) axis += inputs[0]->desc().dim(); 275 | NCG_OP_CHECK_INPUT_DIM_GEQ(graph, inputs, 0, axis - 1); 276 | } 277 | 278 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 279 | const auto &desc = this->template desc(); 280 | auto axis = desc.axis; 281 | if (axis < 0) axis += inputs[0]->desc().dim(); 282 | 283 | auto shape = inputs[0]->desc().shape_vec(); 284 | shape.insert(shape.begin() + axis, 1); 285 | 286 | return {make_tensor(0, TensorDesc(inputs[0]->desc().dtype(), shape))}; 287 | } 288 | 289 | virtual void backward(Graph &graph, GTensorPtr loss); 290 | }; 291 | 292 | // auto broadcast 293 | 294 | } /* !namespace ncg */ 295 | 296 | -------------------------------------------------------------------------------- /src/graph/ops/slice.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * slice.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "graph/ops/slice.h" 9 | #include "graph/ops/netsrc.h" 10 | #include "graph/ops/shape.h" 11 | 12 | namespace ncg { 13 | 14 | void GOpConcat::backward(Graph &graph, GTensorPtr loss) { 15 | auto output_grad = m_outputs[0]->grad(loss); 16 | if (output_grad == nullptr) { 17 | for (auto &i : m_inputs) { 18 | i->set_grad(graph, loss, nullptr); 19 | } 20 | return; 21 | } 22 | 23 | const auto &desc = this->template desc(); 24 | auto sta_splits = ShapeVec(); 25 | auto dyn_splits = GTensorVec(); 26 | for (auto &i : m_inputs) { 27 | sta_splits.emplace_back(i->desc().shape(desc.axis)); 28 | dyn_splits.push_back(graph.op(OpDescPtr(new OpShapeOfIndexDesc(desc.axis)), i)); 29 | } 30 | 31 | auto input_grads = graph.op( 32 | OpDescPtr(new OpSplitDesc(desc.axis, sta_splits)), 33 | output_grad, graph.op(nullptr, dyn_splits) 34 | ); 35 | 36 | for (ssize_t i = 0; i < m_inputs.size(); ++i) { 37 | m_inputs[i]->set_grad(graph, loss, input_grads[i]); 38 | } 39 | } 40 | 41 | void GOpSplit::backward(Graph &graph, GTensorPtr loss) { 42 | if (m_inputs.size() == 2) { 43 | m_inputs[1]->set_grad(graph, loss, nullptr); 44 | } 45 | 46 | GTensorVec output_grads; 47 | bool is_null = true; 48 | for (auto &o : m_outputs) { 49 | auto grad = o->grad(loss); 50 | if (grad != nullptr) { 51 | is_null = false; 52 | } 53 | } 54 | 55 | if (is_null) { 56 | m_inputs[0]->set_grad(graph, loss, nullptr); 57 | return; 58 | } 59 | 60 | for (auto &o : m_outputs) { 61 | auto grad = o->grad(loss); 62 | if (grad == nullptr) { 63 | grad = graph.op( 64 | OpDescPtr(new OpZerosDesc(o->desc().dtype(), o->desc().shape_vec())), 65 | graph.op(nullptr, o) 66 | ); 67 | } 68 | output_grads.push_back(grad); 69 | } 70 | 71 | const auto &desc = this->template desc(); 72 | m_inputs[0]->set_grad(graph, loss, graph.op( 73 | OpDescPtr(new OpConcatDesc(desc.axis)), 74 | output_grads 75 | )); 76 | } 77 | 78 | void GOpNarrow::backward(Graph &graph, GTensorPtr loss) { 79 | if (m_inputs.size() == 3) { 80 | m_inputs[1]->set_grad(graph, loss, nullptr); 81 | m_inputs[2]->set_grad(graph, loss, nullptr); 82 | } 83 | 84 | auto output_grad = m_outputs[0]->grad(loss); 85 | if (output_grad == nullptr) { 86 | m_inputs[0]->set_grad(graph, loss, nullptr); 87 | return; 88 | } 89 | 90 | const auto &desc = this->template desc(); 91 | m_inputs[0]->set_grad(graph, loss, graph.op( 92 | OpDescPtr(new OpNarrowBackwardDesc(desc.axis, desc.start, m_inputs[0]->desc().shape(desc.axis))), 93 | output_grad, 94 | as_gtensor(graph, desc.start), 95 | graph.op(OpDescPtr(new OpShapeOfIndexDesc(desc.axis)), m_inputs[0]) 96 | )); 97 | } 98 | 99 | void GOpNarrowBackward::backward(Graph &graph, GTensorPtr loss) { 100 | if (m_inputs.size() == 3) { 101 | m_inputs[1]->set_grad(graph, loss, nullptr); 102 | m_inputs[2]->set_grad(graph, loss, nullptr); 103 | } 104 | 105 | auto output_grad = m_outputs[0]->grad(loss); 106 | if (output_grad == nullptr) { 107 | m_inputs[0]->set_grad(graph, loss, nullptr); 108 | return; 109 | } 110 | 111 | const auto &desc = this->template desc(); 112 | m_inputs[0]->set_grad(graph, loss, graph.op( 113 | OpDescPtr(new OpNarrowDesc(desc.axis, desc.start, m_inputs[0]->desc().shape(desc.axis))), 114 | output_grad, 115 | as_gtensor(graph, desc.start), 116 | graph.op(OpDescPtr(new OpShapeOfIndexDesc(desc.axis)), m_inputs[0]) 117 | )); 118 | } 119 | 120 | void GOpIndexSelect::backward(Graph &graph, GTensorPtr loss) { 121 | m_inputs[1]->set_grad(graph, loss, nullptr); 122 | 123 | auto output_grad = m_outputs[0]->grad(loss); 124 | if (output_grad == nullptr) { 125 | m_inputs[0]->set_grad(graph, loss, nullptr); 126 | return; 127 | } 128 | 129 | const auto &desc = this->template desc(); 130 | m_inputs[0]->set_grad(graph, loss, graph.op( 131 | OpDescPtr(new OpIndexSelectBackwardDesc(desc.axis, m_inputs[0]->desc().shape(desc.axis))), 132 | output_grad, m_inputs[1], 133 | graph.op(OpDescPtr(new OpShapeOfIndexDesc(desc.axis)), m_inputs[0]) 134 | )); 135 | } 136 | 137 | void GOpIndexSelectBackward::backward(Graph &graph, GTensorPtr loss) { 138 | m_inputs[1]->set_grad(graph, loss, nullptr); 139 | 140 | if (m_inputs.size() == 3) { 141 | m_inputs[2]->set_grad(graph, loss, nullptr); 142 | } 143 | 144 | auto output_grad = m_outputs[0]->grad(loss); 145 | if (output_grad == nullptr) { 146 | m_inputs[0]->set_grad(graph, loss, nullptr); 147 | return; 148 | } 149 | 150 | const auto &desc = this->template desc(); 151 | m_inputs[0]->set_grad(graph, loss, graph.op( 152 | OpDescPtr(new OpIndexSelectDesc(desc.axis)), 153 | output_grad, m_inputs[1] 154 | )); 155 | } 156 | 157 | void GOpGather::backward(Graph &graph, GTensorPtr loss) { 158 | m_inputs[1]->set_grad(graph, loss, nullptr); 159 | 160 | auto output_grad = m_outputs[0]->grad(loss); 161 | if (output_grad == nullptr) { 162 | m_inputs[0]->set_grad(graph, loss, nullptr); 163 | return; 164 | } 165 | 166 | const auto &desc = this->template desc(); 167 | m_inputs[0]->set_grad(graph, loss, graph.op( 168 | OpDescPtr(new OpGatherBackwardDesc(desc.axis, m_inputs[0]->desc().shape(desc.axis))), 169 | output_grad, m_inputs[1] 170 | )); 171 | } 172 | 173 | void GOpGatherBackward::backward(Graph &graph, GTensorPtr loss) { 174 | m_inputs[1]->set_grad(graph, loss, nullptr); 175 | 176 | if (m_inputs.size() == 3) { 177 | m_inputs[2]->set_grad(graph, loss, nullptr); 178 | } 179 | 180 | auto output_grad = m_outputs[0]->grad(loss); 181 | if (output_grad == nullptr) { 182 | m_inputs[0]->set_grad(graph, loss, nullptr); 183 | return; 184 | } 185 | 186 | const auto &desc = this->template desc(); 187 | m_inputs[0]->set_grad(graph, loss, graph.op( 188 | OpDescPtr(new OpGatherDesc(desc.axis)), 189 | output_grad, m_inputs[1], 190 | graph.op(OpDescPtr(new OpShapeOfIndexDesc(desc.axis)), m_inputs[0]) 191 | )); 192 | } 193 | 194 | } /* !namespace ncg */ 195 | 196 | -------------------------------------------------------------------------------- /src/graph/ops/update.h: -------------------------------------------------------------------------------- 1 | /* 2 | * update.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "graph/tensor.h" 11 | #include "graph/op.h" 12 | 13 | namespace ncg { 14 | 15 | class GOpAssign : public GraphOp, public GraphSingleOutputOp { 16 | public: 17 | NCG_GOP_DEF_NAME(GOpAssign); 18 | 19 | virtual void check_inputs(Graph &graph, const GTensorVec &inputs) { 20 | NCG_OP_CHECK_NR_INPUTS(graph, inputs, 2); 21 | NCG_OP_CHECK_COMPATIBLE_DTYPE(graph, inputs); 22 | NCG_OP_CHECK_COMPATIBLE_SHAPE(graph, inputs); 23 | 24 | auto var_op = inputs[0]->template owner_op(); 25 | if (var_op == nullptr) { 26 | graph.error(this) << "The first input to " << this->op_name() << " must be a variable."; 27 | } 28 | } 29 | 30 | virtual GTensorVec init_outputs(Graph &graph, const GTensorVec &inputs) { 31 | return {make_tensor(0, TensorDesc(inputs[0]->desc().dtype(), {}))}; 32 | } 33 | 34 | virtual void forward(GraphForwardContext &ctx) const { 35 | auto variable = ctx.tensor(m_inputs[0]); 36 | auto new_value = ctx.tensor(m_inputs[1]); 37 | ctx.set_tensor(m_outputs[0], new_value); 38 | } 39 | 40 | virtual void forward_hook_post(GraphForwardContext &ctx) const { 41 | auto new_variable = ctx.tensor(m_inputs[1]); 42 | ctx.session().set_shared_tensor(m_inputs[0], new_variable); 43 | } 44 | 45 | NCG_GOP_DEF_NO_GRAD_INLINE; 46 | }; 47 | 48 | } /* !namespace ncg */ 49 | 50 | -------------------------------------------------------------------------------- /src/graph/tensor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * tensor.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core/tensor.h" 11 | 12 | #include 13 | #include 14 | 15 | namespace ncg { 16 | 17 | // From graph/op.h 18 | class GraphOp; 19 | 20 | // From graph/graph.h 21 | class Graph; 22 | 23 | // Forward define the GTensor type. 24 | class GraphTensor; 25 | 26 | class GTensorPtr : public std::shared_ptr { 27 | public: 28 | using std::shared_ptr::shared_ptr; 29 | using super = std::shared_ptr; 30 | 31 | GTensorPtr() : super(nullptr) {} 32 | GTensorPtr(const GTensorPtr &other) : super(other) {} 33 | GTensorPtr(const std::shared_ptr &other) : super(other) {} 34 | GTensorPtr(std::shared_ptr &&other) : super(std::forward(other)) {} 35 | 36 | template ::value>::type* = nullptr> 37 | GTensorPtr(T value); 38 | template ::value>::type* = nullptr> 39 | GTensorPtr(std::vector value); 40 | template ::value>::type* = nullptr> 41 | GTensorPtr(std::vector> value); 42 | GTensorPtr(const TensorPtr &value); 43 | 44 | GTensorPtr eq(const GTensorPtr &rhs) const; 45 | GTensorPtr neq(const GTensorPtr &rhs) const; 46 | 47 | GTensorPtr cast(DTypeName dtype) const; 48 | GTensorPtr int8() const; 49 | GTensorPtr uint8() const; 50 | GTensorPtr int32() const; 51 | GTensorPtr uint32() const; 52 | GTensorPtr int64() const; 53 | GTensorPtr uint64() const; 54 | GTensorPtr float32() const; 55 | GTensorPtr float64() const; 56 | 57 | std::vector min(ssize_t axis, bool keepdims=false) const; 58 | std::vector max(ssize_t axis, bool keepdims=false) const; 59 | GTensorPtr sum(ssize_t axis, bool keepdims=false) const; 60 | GTensorPtr mean(ssize_t axis, bool keepdims=false) const; 61 | 62 | GTensorPtr reshape(const ShapeVec &shape) const; 63 | GTensorPtr permute(const ShapeVec &axes) const; 64 | GTensorPtr expand(const ShapeVec &shape) const; 65 | GTensorPtr squeeze(ssize_t axis) const; 66 | GTensorPtr unsqueeze(ssize_t axis) const; 67 | 68 | GTensorPtr narrow(ssize_t axis, ssize_t start, ssize_t length) const; 69 | GTensorPtr index_select(ssize_t axis, const GTensorPtr &indices) const; 70 | GTensorPtr gather(ssize_t axis, const GTensorPtr &indices) const; 71 | 72 | GTensorPtr shape() const; 73 | GTensorPtr shape(ssize_t axis) const; 74 | 75 | friend std::ostream &operator << (std::ostream &out, const GTensorPtr &tensor); 76 | }; 77 | 78 | GTensorPtr operator - (const GTensorPtr &a); 79 | GTensorPtr operator + (const GTensorPtr &a, const GTensorPtr &b); 80 | GTensorPtr operator - (const GTensorPtr &a, const GTensorPtr &b); 81 | GTensorPtr operator * (const GTensorPtr &a, const GTensorPtr &b); 82 | GTensorPtr operator / (const GTensorPtr &a, const GTensorPtr &b); 83 | GTensorPtr operator > (const GTensorPtr &a, const GTensorPtr &b); 84 | GTensorPtr operator < (const GTensorPtr &a, const GTensorPtr &b); 85 | GTensorPtr operator >= (const GTensorPtr &a, const GTensorPtr &b); 86 | GTensorPtr operator <= (const GTensorPtr &a, const GTensorPtr &b); 87 | 88 | GTensorPtr as_gtensor(const GTensorPtr &value); 89 | GTensorPtr as_gtensor(const TensorPtr &value); 90 | 91 | template 92 | typename std::enable_if::value, GTensorPtr>::type 93 | as_gtensor(T value) { 94 | return as_gtensor(fromcc(CCType::identifier, value)); 95 | } 96 | template 97 | typename std::enable_if::value, GTensorPtr>::type 98 | as_gtensor(std::vector value) { 99 | return as_gtensor(fromcc(CCType::identifier, value)); 100 | } 101 | template 102 | typename std::enable_if::value, GTensorPtr>::type 103 | as_gtensor(std::vector> value) { 104 | return as_gtensor(fromcc(CCType::identifier, value)); 105 | } 106 | 107 | GTensorPtr as_gtensor(Graph &graph, const TensorPtr &value); 108 | 109 | template 110 | typename std::enable_if::value, GTensorPtr>::type 111 | as_gtensor(Graph &graph, T value) { 112 | return as_gtensor(graph, fromcc(CCType::identifier, value)); 113 | } 114 | template 115 | typename std::enable_if::value, GTensorPtr>::type 116 | as_gtensor(Graph &graph, std::vector value) { 117 | return as_gtensor(graph, fromcc(CCType::identifier, value)); 118 | } 119 | template 120 | typename std::enable_if::value, GTensorPtr>::type 121 | as_gtensor(Graph &graph, std::vector> value) { 122 | return as_gtensor(graph, fromcc(CCType::identifier, value)); 123 | } 124 | 125 | template ::value>::type*> 126 | GTensorPtr::GTensorPtr(T value) : GTensorPtr(as_gtensor(value)) {} 127 | template ::value>::type*> 128 | GTensorPtr::GTensorPtr(std::vector value) : GTensorPtr(as_gtensor(value)) {} 129 | template ::value>::type*> 130 | GTensorPtr::GTensorPtr(std::vector> value) : GTensorPtr(as_gtensor(value)) {} 131 | 132 | typedef std::vector GTensorVec; 133 | 134 | class GraphTensor { 135 | public: 136 | GraphTensor(); 137 | GraphTensor(GraphOp *owner_op, ssize_t index, const TensorDesc &desc); 138 | virtual ~GraphTensor() = default; 139 | 140 | template 141 | OpType *owner_op() { 142 | return dynamic_cast(m_owner_op); 143 | } 144 | template 145 | const OpType *owner_op() const { 146 | return dynamic_cast(m_owner_op); 147 | } 148 | 149 | ssize_t owner_op_index() const; 150 | TensorDesc &desc(); 151 | const TensorDesc &desc() const; 152 | 153 | GTensorPtr grad(GTensorPtr loss) const; 154 | void set_grad(Graph &graph, GTensorPtr loss, GTensorPtr grad); 155 | 156 | friend std::ostream & operator << (std::ostream &, const GraphTensor &); 157 | 158 | protected: 159 | GraphOp *m_owner_op; 160 | ssize_t m_owner_op_index; 161 | TensorDesc m_desc; 162 | 163 | std::unordered_map m_grads; 164 | }; 165 | 166 | namespace G { 167 | 168 | GTensorVec auto_broadcast(Graph &graph, const GTensorVec &a); 169 | GTensorVec auto_broadcast(const GTensorVec &a); 170 | 171 | // elemwise::misc 172 | GTensorPtr cast(TensorPtr a, DTypeName dtype); 173 | GTensorPtr cond(TensorPtr a, TensorPtr b, TensorPtr c); 174 | 175 | // elemwise::unary 176 | GTensorPtr neg(GTensorPtr a); 177 | GTensorPtr sin(GTensorPtr a); 178 | GTensorPtr cos(GTensorPtr a); 179 | GTensorPtr tan(GTensorPtr a); 180 | GTensorPtr log(GTensorPtr a); 181 | GTensorPtr exp(GTensorPtr a); 182 | GTensorPtr tanh(GTensorPtr a); 183 | GTensorPtr sigmoid(GTensorPtr a); 184 | GTensorPtr reciprocal(GTensorPtr a); 185 | 186 | GTensorPtr add(GTensorPtr a, GTensorPtr b); 187 | GTensorPtr sub(GTensorPtr a, GTensorPtr b); 188 | GTensorPtr mul(GTensorPtr a, GTensorPtr b); 189 | GTensorPtr div(GTensorPtr a, GTensorPtr b); 190 | GTensorPtr ge(GTensorPtr a, GTensorPtr b); 191 | GTensorPtr le(GTensorPtr a, GTensorPtr b); 192 | GTensorPtr geq(GTensorPtr a, GTensorPtr b); 193 | GTensorPtr leq(GTensorPtr a, GTensorPtr b); 194 | GTensorPtr eq(GTensorPtr a, GTensorPtr b); 195 | GTensorPtr neq(GTensorPtr a, GTensorPtr b); 196 | GTensorPtr pow(GTensorPtr a, GTensorPtr b); 197 | GTensorPtr min(GTensorPtr a, GTensorPtr b); 198 | GTensorPtr max(GTensorPtr a, GTensorPtr b); 199 | 200 | // netsrc 201 | GTensorPtr placeholder(std::string name, const ShapeVec &shape, DTypeName dtype=DTypeName::Float32); 202 | GTensorPtr constant(TensorPtr value); 203 | GTensorPtr variable(std::string name, TensorPtr init_value); 204 | GTensorPtr zeros(const ShapeVec &shape, DTypeName dtype=DTypeName::Float32); 205 | GTensorPtr ones(const ShapeVec &shape, DTypeName dtype=DTypeName::Float32); 206 | 207 | // linalg 208 | GTensorPtr matmul(GTensorPtr a, GTensorPtr b, bool transpose_a=false, bool transpose_b=false); 209 | 210 | // update 211 | GTensorPtr assign(GTensorPtr a, GTensorPtr b); 212 | 213 | // reduce 214 | GTensorVec reduce_min(GTensorPtr a, ssize_t axis, bool keepdims=false); 215 | GTensorVec reduce_max(GTensorPtr a, ssize_t axis, bool keepdims=false); 216 | GTensorPtr reduce_sum(GTensorPtr a, ssize_t axis, bool keepdims=false); 217 | GTensorPtr reduce_mean(GTensorPtr a, ssize_t axis, bool keepdims=false); 218 | 219 | // shape 220 | GTensorPtr reshape(GTensorPtr a, const ShapeVec &shape); 221 | GTensorPtr permute(GTensorPtr a, const ShapeVec &axes); 222 | GTensorPtr expand(GTensorPtr a, const ShapeVec &shape); 223 | GTensorPtr squeeze(GTensorPtr a, ssize_t axis); 224 | GTensorPtr unsqueeze(GTensorPtr a, ssize_t axis); 225 | 226 | // shape 227 | GTensorPtr shape_of(GTensorPtr a); 228 | GTensorPtr shape_of(GTensorPtr a, ssize_t axis); 229 | GTensorPtr shape_cat(const GTensorVec &a); 230 | 231 | // slice 232 | GTensorPtr concat(const GTensorVec &a, ssize_t axis); 233 | GTensorVec split(GTensorPtr a, ssize_t axis, const ShapeVec &splits); 234 | GTensorPtr narrow(GTensorPtr a, ssize_t axis, ssize_t start, ssize_t length); 235 | GTensorPtr index_select(GTensorPtr a, ssize_t axis, GTensorPtr b); 236 | GTensorPtr gather(GTensorPtr a, ssize_t axis, GTensorPtr b); 237 | 238 | } /* !namespace graph */ 239 | 240 | } /* !namespace ncg */ 241 | 242 | -------------------------------------------------------------------------------- /src/ncg.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ncg.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "core.h" 9 | #include "graph.h" 10 | #include "nn.h" 11 | 12 | -------------------------------------------------------------------------------- /src/nn.h: -------------------------------------------------------------------------------- 1 | /* 2 | * nn.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "nn/ops.h" 9 | -------------------------------------------------------------------------------- /src/nn/ops.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * ops.cc 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #include "nn/ops.h" 9 | 10 | #include 11 | 12 | namespace ncg { 13 | 14 | namespace G { 15 | 16 | GTensorPtr linear(std::string name, GTensorPtr x, ssize_t output_dim, std::mt19937 &rng, double stddev) { 17 | auto W = variable(name + ":W", ::ncg::rand_normal(rng, x->desc().dtype(), {x->desc().shape(1), output_dim}, 0, stddev)); 18 | auto b = variable(name + ":b", ::ncg::zeros(x->desc().dtype(), {output_dim})); 19 | return matmul(x, W) + b.unsqueeze(0); 20 | } 21 | 22 | GTensorPtr softmax(GTensorPtr logits, ssize_t axis) { 23 | if (axis < 0) axis += logits->desc().dim(); 24 | 25 | auto x = logits - logits.max(axis, true)[0]; 26 | auto exp_x = exp(x); 27 | return exp_x / exp_x.sum(axis, true); 28 | } 29 | 30 | GTensorPtr xent_sparse(GTensorPtr probs, GTensorPtr indices, ssize_t axis) { 31 | if (axis < 0) axis += probs->desc().dim(); 32 | 33 | auto mlog_probs = -log(probs); 34 | return mlog_probs.gather(axis, indices.unsqueeze(axis)).squeeze(axis); 35 | } 36 | 37 | }; /* !namespace G */ 38 | 39 | } /* !namespace ncg */ 40 | -------------------------------------------------------------------------------- /src/nn/ops.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ops.h 3 | * Copyright (C) 2019 4 | * 5 | * Distributed under terms of the MIT license. 6 | */ 7 | 8 | #pragma once 9 | 10 | #include "core.h" 11 | #include "graph.h" 12 | #include 13 | 14 | namespace ncg { 15 | 16 | namespace G { 17 | 18 | GTensorPtr linear(std::string name, GTensorPtr x, ssize_t output_dim, std::mt19937 &rng, double stddev=0.01); 19 | GTensorPtr softmax(GTensorPtr logits, ssize_t axis); 20 | GTensorPtr xent_sparse(GTensorPtr probs, GTensorPtr indices, ssize_t axis); 21 | 22 | }; /* !namespace G */ 23 | 24 | } /* !namespace ncg */ 25 | 26 | --------------------------------------------------------------------------------