├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── README.md ├── include ├── format.hpp ├── index_generator.hpp ├── layout.hpp ├── slice.hpp ├── storage.hpp ├── stride_generator.hpp ├── tensor.hpp ├── tensor_ops.hpp ├── tensor_slice.hpp └── types.hpp └── test ├── test_indexing.cpp ├── test_storage.cpp ├── test_tensor.cpp ├── test_tensor_ops.cpp └── tests.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | /build/* 2 | .vscode/* 3 | /docs/_build 4 | /docs/_templates 5 | /docs/_static 6 | /docs/doxyoutput 7 | /docs/api -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | 3 | set(CMAKE_CXX_STANDARD 17) 4 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 5 | 6 | set(CMAKE_CXX_FLAGS "-Wall -Wextra") 7 | set(CMAKE_CXX_FLAGS_DEBUG "-g") 8 | set(CMAKE_CXX_FLAGS_RELEASE "-O0") 9 | 10 | 11 | # Locate GTest 12 | find_package(GTest REQUIRED) 13 | include_directories(${GTEST_INCLUDE_DIRS} include) 14 | 15 | find_package(fmt) 16 | 17 | add_executable(TensorTests 18 | test/tests.cpp 19 | test/test_indexing.cpp 20 | test/test_storage.cpp 21 | test/test_tensor.cpp 22 | test/test_tensor_ops.cpp 23 | ) 24 | target_link_libraries(TensorTests ${GTEST_LIBRARIES} pthread fmt::fmt) 25 | set_target_properties(TensorTests PROPERTIES CXX_STANDARD 17) 26 | 27 | include(GoogleTest) 28 | gtest_discover_tests(TensorTests) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 abeschneider 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tensor Library for C++ 2 | 3 | ## Goal 4 | 5 | To create a modern c++ tensor library for instructional purposes. 6 | 7 | ### Why c++? 8 | 9 | The c++11 language is a very different than c++98. The soon to be finished c++20 language has continued in that vein, and has added a large set of features that make c++ a good choice for both math and ML applications. 10 | 11 | ### Why not use Python? 12 | 13 | It's not possible to write the library entirely in Python, so some compiled language has to be used. So the question is what does Python provide that isn't already available in c++? Conventional wisdom is that Python is easy to use and c++ is difficult. And while to some extent that's true, modern c++ has simplified a lot of aspects of the language, to the point that if you're not writing a library, often the code itself is often close to what you would write in Python. 14 | 15 | For example here is some simple code in Python: 16 | ```python 17 | # create a 2d tensor 18 | a = np.array([[1, 2, 3], [4, 5, 6]) 19 | print("a:\n{}".format(a)) 20 | 21 | # slicing the tensor with a range 22 | b = a[0:2, 1:3] 23 | print("b:\n{}".format(b)) 24 | 25 | # broadcast 26 | c = np.array([[1], [2]]) 27 | print("c:\n{}".format(c)) 28 | 29 | d = a + c 30 | print("d:\n{}".format(d)) 31 | ``` 32 | 33 | and the corresponding code for the tensor library: 34 | ```c++ 35 | // create a 2d tensor 36 | auto a = tensor({{1, 2, 3}, {4, 5, 6}}); 37 | fmt::print("a:\n{}\n", a); 38 | //[[ 1, 2, 3], 39 | // [ 4, 5, 6]] 40 | 41 | // slicing the tensor with a range 42 | Tensor b = a[{0, 2}][{1, 3}]; 43 | fmt::print("b: \n{}\n", b); 44 | // [[ 2, 3], 45 | // [ 5, 6]] 46 | 47 | // broadcasting also works 48 | auto c = tensor({{1}, {2}}); 49 | fmt::print("c:\n{}\n", c); 50 | // [[ 1], 51 | // [ 2]] 52 | 53 | auto d = a + c; 54 | fmt::print("d: \n{}\n", d); 55 | // [[ 2, 3, 4], 56 | // [ 6, 7, 8]] 57 | ``` 58 | 59 | Additionally, c++ has features that python doesn't have: 60 | 1. **Strong typing**: It's easy to know the type of every variable. For large projects this can radically decrease development time. Additionally, IDEs can help automatically detect errors for you. 61 | 2. **Compiled**: A compiled language allows you to catch many errors before you run. Especially in cases where you may train a model for long periods of time, it's better to know your code doesn't have errors. 62 | 3. **Simplicity**: If you need to develop new features, you will either have to write bridge code to Python, or use a library that automates that. Additionally, this makes it more difficult to both understand how the code works, as well as debug your code. -------------------------------------------------------------------------------- /include/format.hpp: -------------------------------------------------------------------------------- 1 | #ifndef FORMAT_HPP 2 | #define FORMAT_HPP 3 | 4 | #include 5 | 6 | template 7 | void repeat(OutputIterator write, const char ch, std::size_t count) { 8 | for (std::size_t i = 0; i < count; i++) { 9 | fmt::format_to(write, "{}", ch); 10 | } 11 | } 12 | 13 | // displays the inner most dimension as a row of values 14 | template 15 | void format_inner(OutputIterator write, Sliceable const &slice) { 16 | index_t last_dim_size = slice.shape(slice.num_dims()-1); 17 | 18 | fmt::format_to(write, "["); 19 | for (index_t i = 0; i < last_dim_size; i++) { 20 | auto offset = calculate_offset(slice.view(), i); 21 | 22 | fmt::format_to(write, "{:3}", slice.storage()[offset]); 23 | 24 | if (i < last_dim_size-1) { 25 | fmt::format_to(write, ", "); 26 | } 27 | } 28 | 29 | fmt::format_to(write, "]"); 30 | } 31 | 32 | template 33 | void format_outer(OutputIterator write, Sliceable const &slice, index_t index) { 34 | for (index_t i = 0; i < slice.shape(index); i++) { 35 | if (i == 0) { 36 | fmt::format_to(write, "["); 37 | } else { 38 | repeat(write, ' ', index+1); 39 | } 40 | 41 | auto next_slice = slice[i]; 42 | if (next_slice.index() == next_slice.num_dims()-2) { 43 | format_inner(write, next_slice); 44 | } else { 45 | format_outer(write, next_slice, next_slice.index()+1); 46 | } 47 | 48 | if (i == slice.shape(index)-1) { 49 | fmt::format_to(write, "]"); 50 | } else { 51 | fmt::format_to(write, ","); 52 | repeat(write, '\n', (slice.num_dims() - index)-1); 53 | } 54 | } 55 | } 56 | 57 | template 58 | void format_tensor(OutputIterator write, Tensor const &tensor) { 59 | if (tensor.num_dims() > 1) { 60 | format_outer(write, tensor, 0); 61 | } else { 62 | format_inner(write, tensor); 63 | } 64 | } 65 | 66 | namespace fmt { 67 | 68 | template 69 | struct formatter> { 70 | template 71 | constexpr auto parse(ParseContext &ctx) { 72 | return ctx.begin(); 73 | } 74 | 75 | template 76 | auto format(const Tensor &t, FormatContext &ctx) { 77 | format_tensor(ctx.out(), t); 78 | return ctx.out(); 79 | } 80 | }; 81 | 82 | } // namespace fmt 83 | 84 | 85 | template 86 | std::ostream& operator<<(std::ostream &os, Tensor const &tensor) { 87 | fmt::print(os, "{}", tensor); 88 | return os; 89 | } 90 | 91 | #endif -------------------------------------------------------------------------------- /include/index_generator.hpp: -------------------------------------------------------------------------------- 1 | #ifndef INDEX_GENERATOR_HPP 2 | #define INDEX_GENERATOR_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "types.hpp" 9 | #include "layout.hpp" 10 | 11 | class index_generator: public ranges::view_facade { 12 | public: 13 | friend ranges::range_access; 14 | 15 | index_generator() = default; 16 | 17 | index_generator(extent const &shape, indices const &strides): 18 | shape_(shape), 19 | strides_(strides), 20 | index_(shape.size(), 0), 21 | count_(0), 22 | max_count_(::num_elements(shape_)) {} 23 | 24 | explicit index_generator(extent const &shape): 25 | index_generator(shape, make_strides(shape, make_row_major_order(shape.size()))) {} 26 | 27 | explicit index_generator(Layout const &layout): 28 | index_generator(layout.shape, layout.strides) {} 29 | 30 | explicit index_generator(View const &view): 31 | index_generator(view.shape, make_strides(view.shape, view.order)) {} 32 | private: 33 | void update_index(index_t value) { 34 | auto write = std::begin(index_); 35 | auto extent = std::begin(shape_); 36 | 37 | for (auto const &stride : strides_) { 38 | *write++ = index_t(value / stride) % *extent++; 39 | } 40 | } 41 | public: 42 | void next() { 43 | ++count_; 44 | update_index(count_); 45 | } 46 | 47 | bool equal(ranges::default_sentinel_t) const { 48 | return count_ == max_count_; 49 | } 50 | 51 | const extent &read() const { return index_; } 52 | private: 53 | extent shape_; 54 | indices strides_; 55 | indices index_; 56 | 57 | index_t count_; 58 | index_t max_count_; 59 | }; 60 | 61 | #endif -------------------------------------------------------------------------------- /include/layout.hpp: -------------------------------------------------------------------------------- 1 | #ifndef LAYOUT_HPP 2 | #define LAYOUT_HPP 3 | 4 | #include "types.hpp" 5 | #include "stride_generator.hpp" 6 | 7 | struct Layout { 8 | Layout() = default; 9 | 10 | Layout(Layout const &layout): 11 | shape(layout.shape), strides(layout.strides) {} 12 | 13 | Layout(extent const &shape, indices const &strides): 14 | shape(shape), strides(strides) {} 15 | 16 | std::size_t num_elements() const { return ::num_elements(shape); } 17 | std::size_t size() const { return shape.size(); } 18 | 19 | offset_t get_offset(index_t dim, index_t index) const { 20 | return index*strides[dim]; 21 | } 22 | 23 | extent shape; 24 | indices strides; 25 | }; 26 | 27 | inline indices make_offset(std::size_t dims) { 28 | return indices(dims, 0); 29 | } 30 | 31 | inline indices increasing_order(std::size_t num_dims) { 32 | indices result = ranges::view::iota(0) | ranges::view::take(num_dims); 33 | return result; 34 | } 35 | 36 | struct View: public Layout { 37 | View() = default; 38 | 39 | View(extent const &shape, 40 | indices const &offset, 41 | indices const &order, 42 | indices const &strides): 43 | Layout(shape, strides), 44 | offset(offset), 45 | order(order) {} 46 | 47 | View(extent const &shape, 48 | indices const &order, 49 | indices const &strides): 50 | View(shape, make_offset(shape.size()), order, strides) {} 51 | 52 | View(extent const &shape, 53 | extent const &order): 54 | View(shape, make_offset(shape.size()), order, make_strides(shape, order)) {} 55 | 56 | View(Layout const &layout, 57 | extent const &shape, 58 | indices const &offset, 59 | indices const &order): 60 | Layout(shape, layout.strides), 61 | offset(offset), 62 | order(order) {} 63 | 64 | View(Layout const &layout, 65 | extent const &shape, 66 | indices const &offset): 67 | Layout(shape, layout.strides), 68 | offset(offset), 69 | order(increasing_order(shape.size())) {} 70 | 71 | View(Layout const &layout, 72 | extent const &shape): 73 | View(layout, shape, make_offset(shape.size()), 74 | increasing_order(shape.size())) {} 75 | 76 | explicit View(Layout const &layout): 77 | View(layout, layout.shape, make_offset(layout.shape.size()), 78 | increasing_order(layout.shape.size())) {} 79 | 80 | explicit View(View const &view): 81 | Layout(view), offset(view.offset), order(view.order) {} 82 | 83 | offset_t get_offset(index_t dim, index_t index) const { 84 | return (index+offset[dim])*strides[dim]; 85 | } 86 | 87 | indices offset; 88 | indices order; 89 | }; 90 | 91 | inline std::size_t num_dims(Layout const &layout) { return layout.size(); } 92 | inline std::size_t num_dims(View const &view) { return view.size(); } 93 | 94 | 95 | template 96 | offset_t calculate_offset(const LayoutType &layout, 97 | ContainerType const &index) 98 | { 99 | offset_t offset = 0; 100 | 101 | for (std::size_t d = 0; d < num_dims(layout); d++) { 102 | offset += layout.get_offset(d, index[d]); 103 | } 104 | 105 | return offset; 106 | } 107 | 108 | // TODO: provide check that all but one dimension is a singleton 109 | template 110 | offset_t calculate_offset(const LayoutType &layout, index_t index) { 111 | offset_t offset = 0; 112 | 113 | for (index_t i = 0; i < layout.size(); i++) { 114 | if (layout.shape[i] == 1) { 115 | offset += layout.get_offset(i, 0); 116 | } else { 117 | offset += layout.get_offset(i, index); 118 | } 119 | } 120 | 121 | return offset; 122 | } 123 | 124 | #endif -------------------------------------------------------------------------------- /include/slice.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SLICE_HPP 2 | #define SLICE_HPP 3 | 4 | #include 5 | 6 | #include 7 | 8 | #include "types.hpp" 9 | #include "layout.hpp" 10 | 11 | using index_range_t = std::pair; 12 | 13 | template 14 | class Slice { 15 | public: 16 | using NumericType = T; 17 | using Device = D; 18 | 19 | Slice(index_t dim_index, 20 | index_t index, 21 | View const &view, 22 | std::shared_ptr> storage): 23 | dim_index_(dim_index), view_(view), storage_(storage) 24 | { 25 | view_.shape[dim_index] = 1; 26 | view_.offset[dim_index_] = index + view.offset[dim_index_]; 27 | } 28 | 29 | Slice(index_t dim_index, 30 | index_range_t index_range, 31 | View const &view, 32 | std::shared_ptr> storage): 33 | dim_index_(dim_index), view_(view), storage_(storage) 34 | { 35 | view_.shape[dim_index] = index_range.second - index_range.first; 36 | view_.offset[dim_index_] = index_range.first; 37 | } 38 | 39 | Slice operator [](index_t index) const { 40 | return Slice(dim_index_+1, index, view_, storage_); 41 | } 42 | 43 | Slice operator [](index_range_t index_range) const { 44 | return Slice(dim_index_+1, index_range, view_, storage_); 45 | } 46 | 47 | View const &view() const { return view_; } 48 | View &view() { return view_; } 49 | 50 | extent const &shape() const { return view_.shape; } 51 | index_t shape(index_t dim) const { return view_.shape[dim]; } 52 | 53 | Storage const &storage() const { return *storage_; } 54 | std::shared_ptr> storage_ptr() const { return storage_; } 55 | 56 | // TODO: rename dim_index, confusing otherwise 57 | index_t index() const { return dim_index_; } 58 | index_t num_dims() const { return view_.shape.size(); } 59 | private: 60 | index_t dim_index_; 61 | View view_; 62 | std::shared_ptr> storage_; 63 | }; 64 | 65 | #endif -------------------------------------------------------------------------------- /include/storage.hpp: -------------------------------------------------------------------------------- 1 | #ifndef STORAGE_HPP 2 | #define STORAGE_HPP 3 | 4 | #include 5 | 6 | #include "types.hpp" 7 | 8 | // namespace tensor { 9 | 10 | struct CPU {}; 11 | struct CPU_BLAS {}; 12 | struct GPU_CUDA {}; 13 | 14 | /** 15 | * @brief Storage type for `Tensor`s. 16 | * 17 | * @tparam T Element type of storage 18 | * @tparam Device Device storage lives on 19 | */ 20 | template 21 | struct Storage {}; 22 | 23 | /** 24 | * @brief CPU specialization of `Storage` 25 | * 26 | * @tparam T Element type of storage 27 | */ 28 | template 29 | struct Storage { 30 | using element_type = T; 31 | using storage_type = std::vector; 32 | using iterator = typename storage_type::iterator; 33 | 34 | Storage(std::size_t size): data(size, 0) {} 35 | 36 | T &operator [](index_t i) { return data[i]; } 37 | const T &operator [](index_t i) const { return data[i]; } 38 | 39 | auto begin() { return data.begin(); } 40 | auto end() { return data.end(); } 41 | 42 | auto cbegin() const { return data.cbegin(); } 43 | auto cend() const { return data.cend(); } 44 | 45 | std::size_t size() const { return data.size(); } 46 | 47 | std::vector data; 48 | }; 49 | 50 | #endif -------------------------------------------------------------------------------- /include/stride_generator.hpp: -------------------------------------------------------------------------------- 1 | #ifndef STRIDE_GENERATOR_HPP 2 | #define STRIDE_GENERATOR_HPP 3 | 4 | #include 5 | 6 | #include "types.hpp" 7 | 8 | class ordered_shape_view: public ranges::view_facade { 9 | public: 10 | ordered_shape_view() = default; 11 | ordered_shape_view(extent const &shape, indices const &order): 12 | shape_(&shape), order_(&order), index_(0) {} 13 | 14 | void next() { ++index_; } 15 | const index_t &read() const { return (*shape_)[(*order_)[index_]]; } 16 | std::size_t size() const { return shape_->size(); } 17 | 18 | bool equal(ranges::default_sentinel_t) const { 19 | return index_ >= shape_->size(); 20 | } 21 | private: 22 | extent const *shape_; 23 | indices const *order_; 24 | index_t index_; 25 | }; 26 | 27 | 28 | class stride_generator: public ranges::view_facade { 29 | public: 30 | friend ranges::range_access; 31 | 32 | stride_generator() = default; 33 | explicit stride_generator(ordered_shape_view const &shape): 34 | shape_(shape), stride_(1), count_(0) {} 35 | 36 | void next() { 37 | stride_ *= shape_.read(); 38 | shape_.next(); 39 | ++count_; 40 | } 41 | 42 | const index_t &read() const { return stride_; } 43 | 44 | bool equal(ranges::default_sentinel_t) const { 45 | return count_ >= shape_.size(); 46 | } 47 | 48 | private: 49 | ordered_shape_view shape_; 50 | index_t stride_; 51 | index_t count_; 52 | }; 53 | 54 | inline indices make_strides(extent const &shape, const indices &order) { 55 | indices result(shape.size()); 56 | 57 | ordered_shape_view shape_view(shape, order); 58 | stride_generator strides(shape_view); 59 | 60 | std::size_t i = 0; 61 | for (auto stride : strides) { 62 | result[order[i++]] = stride; 63 | } 64 | 65 | return result; 66 | } 67 | 68 | inline indices make_row_major_order(std::size_t num_dims) { 69 | indices result = ranges::view::iota(0) | ranges::view::take(num_dims) | ranges::view::reverse; 70 | return result; 71 | } 72 | 73 | inline indices make_col_major_order(std::size_t num_dims) { 74 | indices result = ranges::view::iota(0) | ranges::view::take(num_dims); 75 | return result; 76 | } 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /include/tensor.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TENSOR_HPP 2 | #define TENSOR_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "types.hpp" 18 | #include "index_generator.hpp" 19 | #include "stride_generator.hpp" 20 | #include "storage.hpp" 21 | #include "layout.hpp" 22 | #include "slice.hpp" 23 | 24 | enum class TensorOrder { 25 | RowMajor, 26 | ColumnMajor 27 | }; 28 | 29 | inline indices make_order(std::size_t dims, TensorOrder order) { 30 | if (order == TensorOrder::RowMajor) { 31 | return make_row_major_order(dims); 32 | } else { 33 | return make_col_major_order(dims); 34 | } 35 | } 36 | 37 | template 38 | constexpr std::array tuple_to_array(std::tuple &&tup) { 39 | std::array result{}; 40 | std::size_t i = 0; 41 | std::apply([&result, &i](auto &&... v) { 42 | ((result[i++] = v), ...); 43 | }, tup); 44 | 45 | return result; 46 | } 47 | 48 | template 49 | class Tensor; 50 | 51 | template 52 | Tensor tensor(const T &value) { 53 | Tensor result{1}; 54 | result(0) = value; 55 | 56 | return result; 57 | } 58 | 59 | template 60 | Tensor tensor(std::initializer_list values) { 61 | Tensor result({values.size()}); 62 | 63 | std::size_t i = 0; 64 | for (auto value : values) { 65 | result(i++) = value; 66 | } 67 | return result; 68 | } 69 | 70 | template 71 | Tensor tensor(std::initializer_list> values) { 72 | std::size_t d0 = values.size(); 73 | auto l1 = *std::begin(values); 74 | std::size_t d1 = l1.size(); 75 | 76 | Tensor result(extent{d0, d1}, TensorOrder::RowMajor); 77 | std::size_t i = 0; 78 | for (auto row : values) { 79 | std::size_t j = 0; 80 | for (auto value : row) { 81 | result(i, j) = value; 82 | ++j; 83 | } 84 | ++i; 85 | } 86 | 87 | return result; 88 | } 89 | 90 | template 91 | Tensor tensor(std::initializer_list>> values) { 92 | std::size_t d0 = values.size(); 93 | auto l1 = *std::begin(values); 94 | std::size_t d1 = l1.size(); 95 | auto l2 = *std::begin(l1); 96 | std::size_t d2 = l2.size(); 97 | 98 | Tensor result({d0, d1, d2}); 99 | std::size_t i = 0; 100 | for (auto row : values) { 101 | std::size_t j = 0; 102 | for (auto col : row) { 103 | std::size_t k = 0; 104 | for (auto value : col) { 105 | result(i, j, k) = value; 106 | ++k; 107 | } 108 | ++j; 109 | } 110 | ++i; 111 | } 112 | 113 | return result; 114 | } 115 | 116 | template 117 | Tensor tensor(std::initializer_list>>> values) { 118 | std::size_t d0 = values.size(); 119 | auto l1 = *std::begin(values); 120 | std::size_t d1 = l1.size(); 121 | auto l2 = *std::begin(l1); 122 | std::size_t d2 = l2.size(); 123 | auto l3 = *std::begin(l2); 124 | std::size_t d3 = l3.size(); 125 | 126 | Tensor result({d0, d1, d2, d3}); 127 | std::size_t b = 0; 128 | for (auto batch : values) { 129 | std::size_t i = 0; 130 | for (auto row : batch) { 131 | std::size_t j = 0; 132 | for (auto col : row) { 133 | std::size_t k = 0; 134 | for (auto value : col) { 135 | result(b, i, j, k) = value; 136 | ++k; 137 | } 138 | ++j; 139 | } 140 | ++i; 141 | } 142 | ++b; 143 | } 144 | 145 | return result; 146 | } 147 | 148 | template 149 | struct ElementTensor; 150 | 151 | /** 152 | * \class Tensor tensor.hpp include/tensor.hpp 153 | * \brief The Tensor class holds a pointer to data held in storage and a view of that data. 154 | * \tparam T The type of the Tensor (e.g. ``int``, ``float``, ``double``, etc.) 155 | * \tparam D The device the data is stored on (e.g. `CPU`, `GPU`, etc.) 156 | */ 157 | template 158 | class Tensor { 159 | public: 160 | using NumericType = T; 161 | using Device = D; 162 | 163 | /** 164 | * \brief Copy constructor 165 | * \param Tensor to copy 166 | */ 167 | Tensor(Tensor const &t): 168 | view_(t.view_), storage_(t.storage_), order_(t.order_) {} 169 | 170 | /** 171 | * \brief Constructs a Tensor with given a shape and order 172 | * \param shape The extent of the Tensor in each dimension 173 | * \param order Either TensorOrder::RowMajor or TensorOrder::ColumnMajor 174 | */ 175 | Tensor(extent shape, TensorOrder order): 176 | view_(shape, make_order(shape.size(), order)), 177 | storage_(std::make_shared>(view_.num_elements())), 178 | order_(order) {} 179 | 180 | /** 181 | * \brief Constructs a Tensor with a given shape 182 | * \param shape The extent of the Tensor in each dimension 183 | */ 184 | explicit Tensor(extent const &shape): 185 | Tensor(shape, TensorOrder::RowMajor) {} 186 | 187 | /** 188 | * \brief Constructs a Tensor with the given `storage` and `view` 189 | * \param storage The storage containing the Tensor's data 190 | * \param view The view of the storage 191 | */ 192 | Tensor(std::shared_ptr> storage, const View &view): 193 | view_(view), storage_(storage), 194 | order_(TensorOrder::RowMajor) {} 195 | 196 | /** 197 | * \brief Constructs a Tensor from a Slice 198 | * \param slice The slice of another Tensor 199 | */ 200 | Tensor(Slice const &slice): 201 | view_(slice.view()), 202 | storage_(slice.storage_ptr()), 203 | order_(TensorOrder::RowMajor) {} 204 | 205 | /** 206 | * \brief Index operator 207 | * \param cindices A sequence of index's used to calculate an offset 208 | * \return The value in the tensor for the given index 209 | */ 210 | template 211 | T &operator ()(Args... cindices) { 212 | auto array_indices = tuple_to_array(std::tuple(cindices...)); 213 | auto pos = calculate_offset(view_, array_indices); 214 | return (*storage_)[pos]; 215 | } 216 | 217 | /** 218 | * \brief Index operator 219 | * \param cindices A sequence of index's used to calculate an offset 220 | * \return The value in the tensor for the given index 221 | */ 222 | template 223 | T &operator ()(Args... cindices) const { 224 | auto array_indices = tuple_to_array(std::tuple(cindices...)); 225 | auto pos = calculate_offset(view_, array_indices); 226 | return (*storage_)[pos]; 227 | } 228 | 229 | 230 | /** 231 | * \brief Index operator 232 | * \param cindices A sequence of index's used to calculate an offset 233 | * \return The value in the tensor for the given index 234 | */ 235 | T &operator ()(indices const &index) { 236 | auto offset = calculate_offset(view_, index); 237 | return (*storage_)[offset]; 238 | } 239 | 240 | 241 | /** 242 | * \brief Index operator 243 | * \param cindices A sequence of index's used to calculate an offset 244 | * \return The value in the tensor for the given index 245 | */ 246 | T const &operator ()(indices const &index) const { 247 | auto offset = calculate_offset(view_, index); 248 | return (*storage_)[offset]; 249 | } 250 | 251 | // think about ways of storing results for faster indexing 252 | index_generator indices() const { 253 | return index_generator(view_); 254 | } 255 | 256 | /** 257 | * \brief Slice operator 258 | * \param index Index to set first dimension of Slice to 259 | * \return A Slice with the same storage as this Tensor 260 | */ 261 | Slice operator [](index_t index) const { 262 | return Slice(0, index, view_, storage_); 263 | } 264 | 265 | /** 266 | * \brief Slice operator 267 | * \param index_index Range to set first dimension of Slice to 268 | * \return A Slice with the same storage as this Tensor 269 | */ 270 | Slice operator [](index_range_t index_range) const { 271 | return Slice(0, index_range, view_, storage_); 272 | } 273 | 274 | /** 275 | * \brief Returns the Tensor's view 276 | * \return View 277 | */ 278 | const View &view() const { return view_; } 279 | 280 | /** 281 | * \brief Returns the Tensor's view 282 | * \return View 283 | */ 284 | View &view() { return view_; } 285 | 286 | /** 287 | * \brief Creates a new Tensor using the same storage but 288 | * a new view 289 | * \return Tensor 290 | */ 291 | Tensor view(const View &new_view) const { 292 | return Tensor{storage_, new_view}; 293 | } 294 | 295 | /** 296 | * \brief Returns shape of tensor (from view) 297 | * \return extent 298 | */ 299 | extent const &shape() const { return view_.shape; } 300 | 301 | /** 302 | * \brief Returns shape of tensor (from view) 303 | * \return extent 304 | */ 305 | index_t shape(index_t dim) const { return view_.shape[dim]; } 306 | 307 | std::size_t num_dims() const { return view_.shape.size(); } 308 | 309 | Storage const &storage() const { return *storage_; } 310 | Storage &storage() { return *storage_; } 311 | std::shared_ptr> storage_ptr() { return storage_; } 312 | 313 | /** 314 | * \brief Returns true if Tensor is contiguous 315 | * \return 316 | */ 317 | bool contiguous() const { 318 | auto contiguous_order = make_row_major_order(num_dims()); 319 | return view_.order == contiguous_order; 320 | } 321 | 322 | ElementTensor el() { 323 | return ElementTensor{*this}; 324 | } 325 | private: 326 | View view_; 327 | std::shared_ptr> storage_; 328 | 329 | // row-major or column major 330 | TensorOrder order_; 331 | 332 | // if true, tensor elements can be changed 333 | bool mutable_; 334 | }; 335 | 336 | /** 337 | * @brief Provides a wrapper around Tensor that causes elementwise operations to 338 | * be used. Do not use this struct directly, but instead use `Tensor.el()` 339 | * @tparam T 340 | * @tparam Device 341 | */ 342 | template 343 | struct ElementTensor { 344 | Tensor &tensor; 345 | }; 346 | 347 | template 348 | index_t num_dims(Tensor const &tensor) { 349 | return tensor.shape().size(); 350 | } 351 | 352 | template 353 | index_t num_elements(Tensor const &tensor) { 354 | return num_elements(tensor.shape()); 355 | } 356 | 357 | template 358 | Tensor make_contiguous(Tensor const &t) { 359 | if (t.contiguous()) return t; 360 | return Tensor(t); 361 | } 362 | 363 | template 364 | Tensor copy(Tensor const &tensor) { 365 | auto new_storage = std::make_shared>(tensor.view().num_elements()); 366 | for (index_t i = 0; i < tensor.view().num_elements(); i++) { 367 | (*new_storage)[i] = tensor.storage()[i]; 368 | } 369 | 370 | return Tensor(new_storage, tensor.view()); 371 | } 372 | 373 | // template 374 | // Tensor copy(Tensor const &tensor, View view) { 375 | // auto new_storage = std::make_shared>(view.num_elements()); 376 | // for (index_t i = 0; i < view.num_elements(); i++) { 377 | // (*new_storage)[i] = tensor.storage()[i]; 378 | // } 379 | // return Tensor(new_storage, view); 380 | // } 381 | 382 | #endif -------------------------------------------------------------------------------- /include/tensor_ops.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TENSOR_OPS_H 2 | #define TENSOR_OPS_H 3 | 4 | #include 5 | 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | constexpr index_t expand = std::numeric_limits::max(); 14 | 15 | struct TensorError: public std::runtime_error { 16 | TensorError(std::string const &msg): std::runtime_error(msg) {} 17 | }; 18 | 19 | struct MismatchedNumberOfElements: public TensorError { 20 | MismatchedNumberOfElements(std::size_t lhs, std::size_t rhs): 21 | TensorError(fmt::format("Number of elements are not the same: {} and {}", lhs, rhs)) {} 22 | }; 23 | 24 | struct MismatchedDimensions: public TensorError { 25 | MismatchedDimensions(extent const &shape1, extent const &shape2): 26 | TensorError(fmt::format("Mismatched dimensions: {} and {}", shape1, shape2)) {} 27 | }; 28 | 29 | struct CannotBroadcast: public TensorError { 30 | CannotBroadcast(extent const &shape1, extent const &shape2): 31 | TensorError(fmt::format("Cannot broadcast: {} and {}", shape1, shape2)) {} 32 | }; 33 | 34 | struct NotEnoughDimensions: public TensorError { 35 | NotEnoughDimensions(extent const &shape): 36 | TensorError(fmt::format("Not enough dimensions: {}", shape)) {} 37 | }; 38 | 39 | /** 40 | * @brief Returns a tensor fill with `0`s 41 | * 42 | * @tparam T The tensor type 43 | * @tparam Device=CPU The device tensor is stored on 44 | * @param shape The shape of the tensor 45 | * @return Tensor 46 | */ 47 | template 48 | Tensor zeros(extent const &shape) { 49 | Tensor result(shape); 50 | fill(result, 0); 51 | return result; 52 | } 53 | 54 | /** 55 | * @brief Returns a tensor fill with `1`s 56 | * 57 | * @tparam T The tensor type 58 | * @tparam Device=CPU The device tensor is stored on 59 | * @param shape The shape of the tensor 60 | * @return Tensor 61 | */ 62 | template 63 | Tensor ones(extent const &shape) { 64 | Tensor result(shape); 65 | fill(result, 0); 66 | return result; 67 | } 68 | 69 | template 70 | void iota(Tensor &t, T start=0, T stride=1) { 71 | T value = start; 72 | fill(t, [&value, stride]() { 73 | T tmp = value; 74 | value += stride; 75 | return tmp; 76 | }); 77 | } 78 | 79 | template 80 | Tensor range(T start, T end, T stride=1) { 81 | std::size_t size = std::floor((end - start) / stride); 82 | Tensor result({size}); 83 | 84 | iota(result, start, end, stride); 85 | return result; 86 | } 87 | 88 | /** 89 | * @brief Re-orders dimensions of Tensor 90 | * 91 | * @tparam T The tensor type 92 | * @tparam Device The device tensor is stored on 93 | * @param tensor The tensor to re-order 94 | * @param order The new order of dimensions 95 | * @return Tensor 96 | */ 97 | template 98 | Tensor transpose( 99 | Tensor const &tensor, 100 | indices const &order) 101 | { 102 | extent shape(tensor.shape().size()); 103 | indices offset(tensor.shape().size()); 104 | indices strides(tensor.shape().size()); 105 | 106 | for (index_t i = 0; i < tensor.shape().size(); i++) { 107 | shape[i] = tensor.view().shape[order[i]]; 108 | offset[i] = tensor.view().offset[order[i]]; 109 | strides[i] = tensor.view().strides[order[i]]; 110 | } 111 | 112 | return tensor.view({shape, offset, order, strides}); 113 | } 114 | 115 | // TODO: move to source 116 | inline boost::optional get_inferred_dimension(extent const &shape) { 117 | boost::optional result; 118 | 119 | for (index_t i = 0; i < shape.size(); i++) { 120 | if (shape[i] == expand) { 121 | if (!result) { 122 | result = i; 123 | } else { 124 | throw TensorError("Cannot infer more than one dimension."); 125 | } 126 | } 127 | } 128 | 129 | return result; 130 | } 131 | 132 | // TODO: move to source 133 | inline void calculate_reshape(extent const &from_shape, extent &to_shape) { 134 | auto inferred_dimension = get_inferred_dimension(to_shape); 135 | 136 | if (inferred_dimension) { 137 | // number of elements for the new shape 138 | to_shape[*inferred_dimension] = 1; 139 | std::size_t new_size = num_elements(to_shape); 140 | 141 | // number of elements for the old shape 142 | std::size_t total_size = num_elements(from_shape); 143 | 144 | // calculate the size of the inferred dimension 145 | std::size_t inferred_size = total_size / new_size; 146 | to_shape[*inferred_dimension] = inferred_size; 147 | } 148 | } 149 | 150 | // TODO: make this either respect order, or take order f strides 151 | template 152 | Tensor reshape(Tensor const &tensor, extent const &shape) { 153 | extent new_shape = shape; 154 | calculate_reshape(tensor.shape(), new_shape); 155 | 156 | // we can only reshape if the number of elements is conserved 157 | if (num_elements(new_shape) != num_elements(tensor.shape())) { 158 | throw MismatchedNumberOfElements(num_elements(new_shape), 159 | num_elements(tensor.shape())); 160 | } 161 | 162 | auto order = make_row_major_order(new_shape.size()); 163 | auto strides = make_strides(new_shape, order); 164 | auto offset = make_offset(new_shape.size()); 165 | 166 | if (tensor.contiguous()) { 167 | return tensor.view({new_shape, offset, order, strides}); 168 | } 169 | 170 | return copy(tensor.view({new_shape, offset, order, strides})); 171 | } 172 | 173 | template 174 | bool is_broadcastable_to(Tensor const &tensor, extent const &shape) { 175 | if (shape.size() < tensor.shape().size()) return false; 176 | 177 | auto shape_it = shape.rbegin(); 178 | auto tensor_shape_it = tensor.shape().rbegin(); 179 | for (; tensor_shape_it != tensor.shape().rend(); ++shape_it, ++tensor_shape_it) { 180 | if (*shape_it != *tensor_shape_it && *tensor_shape_it != 1) return false; 181 | } 182 | 183 | return true; 184 | } 185 | 186 | namespace detail { 187 | 188 | template 189 | Tensor broadcast_to(Tensor const &tensor, extent const &shape) { 190 | extent strides(shape.size(), 0); 191 | auto offset = make_offset(shape.size()); 192 | auto order = make_row_major_order(shape.size()); 193 | 194 | std::size_t diff = shape.size() - tensor.shape().size(); 195 | for (index_t i = 0; i < tensor.shape().size(); i++) { 196 | if (tensor.shape()[i] > 1) { 197 | strides[diff+i] = tensor.view().strides[i]; 198 | } else { 199 | strides[diff+i] = 0; 200 | } 201 | } 202 | 203 | return tensor.view({shape, offset, order, strides}); 204 | } 205 | 206 | } // namespace detail 207 | 208 | template 209 | Tensor broadcast_to(Tensor const &tensor, extent const &shape) { 210 | if (tensor.shape() == shape) return tensor; 211 | 212 | if (!is_broadcastable_to(tensor, shape)) { 213 | throw CannotBroadcast(tensor.shape(), shape); 214 | } 215 | 216 | return detail::broadcast_to(tensor, shape); 217 | } 218 | 219 | template 220 | std::pair, Tensor> 221 | broadcast(Tensor const &t1, Tensor const &t2) { 222 | if (t1.shape() == t2.shape()) return std::make_pair(t1, t2); 223 | 224 | bool t1_to_t2 = is_broadcastable_to(t1, t2.shape()); 225 | bool t2_to_t1 = is_broadcastable_to(t2, t1.shape()); 226 | 227 | if (!t1_to_t2 && !t2_to_t1) { 228 | throw CannotBroadcast(t1.shape(), t2.shape()); 229 | } 230 | 231 | auto result1 = t1_to_t2 ? detail::broadcast_to(t1, t2.shape()) : t1; 232 | auto result2 = t2_to_t1 ? detail::broadcast_to(t2, t1.shape()) : t2; 233 | return std::make_pair(result1, result2); 234 | } 235 | 236 | template 237 | Tensor apply(Tensor const &lhs, 238 | Tensor const &rhs, 239 | F fn) 240 | { 241 | if (lhs.shape() != rhs.shape()) { 242 | // if the dimensions don't match, try to broadcast 243 | auto [lhs_broadcast, rhs_broadcast] = broadcast(lhs, rhs); 244 | return apply(lhs_broadcast, rhs_broadcast, fn); 245 | } 246 | 247 | Tensor result(lhs.shape()); 248 | 249 | // can optimize if both lhs and rhs are contiguous 250 | for (auto const &index : lhs.indices()) { 251 | result(index) = fn(lhs(index), rhs(index)); 252 | } 253 | 254 | return result; 255 | } 256 | 257 | template 258 | void iapply(Tensor &lhs, Tensor const &rhs, F fn) { 259 | if (lhs.shape() != rhs.shape()) { 260 | throw MismatchedDimensions(lhs.shape(), rhs.shape()); 261 | } 262 | 263 | // can optimize if both lhs and rhs are contiguous 264 | for (auto const &index : lhs.indices()) { 265 | lhs(index) = fn(lhs(index), rhs(index)); 266 | } 267 | } 268 | 269 | template 270 | Tensor apply(Tensor const &t, F fn) { 271 | Tensor result(t.shape()); 272 | for (auto const &index : t.indices()) { 273 | result(index) = fn(t(index)); 274 | } 275 | 276 | return result; 277 | } 278 | 279 | template 280 | void iapply(Tensor &t, F fn) { 281 | for (auto const &index : t.indices()) { 282 | t(index) = fn(t(index)); 283 | } 284 | } 285 | 286 | /** 287 | * @brief Sets all the elements of a tensor to the given value 288 | * 289 | * @tparam T The tensor type 290 | * @tparam Device The device tensor is stored on 291 | * @param t The tensor to fill 292 | * @param value The value use 293 | */ 294 | template 295 | void fill(Tensor &t, T const &value) { 296 | iapply(t, [value](T const &) { return value; }); 297 | } 298 | 299 | /** 300 | * @brief Fills tensor using a successive calls to passed in function 301 | * 302 | * @tparam T The tensor type 303 | * @tparam Device The device tensor is stored on 304 | * @tparam F Function type 305 | * @param t The tensor to fill 306 | * @param fn The function used to fill tensor 307 | */ 308 | template 309 | void fill(Tensor &t, F fn) { 310 | iapply(t, [fn](T const &) { return fn(); }); 311 | } 312 | 313 | template 314 | Tensor operator +(Tensor const &lhs, Tensor const &rhs) { 315 | return apply(lhs, rhs, [](T const &lop, T const &rop) { return lop + rop; }); 316 | } 317 | 318 | template 319 | Tensor &operator +=(Tensor &lhs, Tensor const &rhs) { 320 | iapply(lhs, rhs, [](T const &lop, T const &rop) { return lop + rop; }); 321 | return lhs; 322 | } 323 | 324 | template 325 | Tensor operator -(Tensor const &lhs, Tensor const &rhs) { 326 | return apply(lhs, rhs, [](T const &lop, T const &rop) { return lop - rop; }); 327 | } 328 | 329 | template 330 | Tensor &operator -=(Tensor &lhs, Tensor const &rhs) { 331 | iapply(lhs, rhs, [](T const &lop, T const &rop) { return lop - rop; }); 332 | return lhs; 333 | } 334 | 335 | template 336 | Tensor operator *(ElementTensor lhs, ElementTensor rhs) { 337 | return apply(lhs.tensor, rhs.tensor, [](T const &lop, T const &rop) { 338 | return lop * rop; 339 | }); 340 | } 341 | 342 | template 343 | Tensor &operator *=(ElementTensor lhs, ElementTensor rhs) { 344 | iapply(lhs.tensor, rhs.tensor, [](T const &lop, T const &rop) { return lop * rop; }); 345 | return lhs.tensor; 346 | } 347 | 348 | template 349 | Tensor operator /(Tensor const &lhs, Tensor const &rhs) { 350 | return apply(lhs, rhs, [](T const &lop, T const &rop) { return lop / rop; }); 351 | } 352 | 353 | template 354 | Tensor &operator /=(Tensor &lhs, Tensor const &rhs) { 355 | iapply(lhs, rhs, [](T const &lop, T const &rop) { return lop / rop; }); 356 | return lhs; 357 | } 358 | 359 | template 360 | Tensor sin(Tensor &t) { 361 | return apply(t, [](T const &v) { return std::sin(v); }); 362 | } 363 | 364 | template 365 | void isin(Tensor &t) { 366 | iapply(t, [](T const &v) { return std::sin(v); }); 367 | } 368 | 369 | // move to detail 370 | template 371 | Tensor vector_vector_product(Tensor const &lhs, 372 | Tensor const &rhs) 373 | { 374 | if (num_elements(lhs) != num_elements(rhs)) { 375 | throw MismatchedNumberOfElements(num_elements(lhs), num_elements(rhs)); 376 | } 377 | 378 | T result(0); 379 | for (index_t i = 0; i < num_elements(lhs); i++) { 380 | result += lhs(i)*rhs(i); 381 | } 382 | 383 | return tensor({result}); 384 | } 385 | 386 | template 387 | Tensor matrix_vector_product(Tensor const &lhs, 388 | Tensor const &rhs) 389 | { 390 | // verify inner dimensions match: (Nx1, 1) 391 | Tensor result({lhs.shape()[0]}); 392 | fill(result, 0); 393 | 394 | for (index_t i = 0; i < lhs.shape()[0]; i++) { 395 | for (index_t j = 0; j < lhs.shape()[1]; j++) { 396 | result(i) += lhs(i, j)*rhs(j); 397 | } 398 | } 399 | 400 | return result; 401 | } 402 | 403 | // need batch_matrix_vector 404 | 405 | template 406 | Tensor matrix_matrix_product(Tensor const &lhs, 407 | Tensor const &rhs) 408 | { 409 | Tensor result({lhs.shape()[0], rhs.shape()[1]}); 410 | fill(result, 0); 411 | 412 | for (index_t i = 0; i < lhs.shape()[0]; i++) { 413 | for (index_t j = 0; j < lhs.shape()[1]; j++) { 414 | for (index_t k = 0; k < rhs.shape()[1]; k++) { 415 | result(i, k) += lhs(i, j)*rhs(j, k); 416 | } 417 | } 418 | } 419 | 420 | return result; 421 | } 422 | 423 | template 424 | Tensor batch_matrix_matrix_product(Tensor const &lhs, 425 | Tensor const &rhs) 426 | { 427 | // BxMxN * BxNxP 428 | Tensor result({lhs.shape()[0], lhs.shape()[1], rhs.shape()[2]}); 429 | fill(result, 0); 430 | 431 | for (index_t b = 0; b < lhs.shape()[0]; b++) { 432 | for (index_t i = 0; i < lhs.shape()[1]; i++) { 433 | for (index_t j = 0; j < lhs.shape()[2]; j++) { 434 | for (index_t k = 0; k < rhs.shape()[2]; k++) { 435 | result(b, i, k) += lhs(b, i, j)*rhs(b, j, k); 436 | } 437 | } 438 | } 439 | } 440 | 441 | return result; 442 | } 443 | 444 | 445 | inline extent get_batch_shape(extent const &shape) { 446 | if (shape.size() < 3) throw NotEnoughDimensions(shape); 447 | std::size_t count = shape.size() - 2; 448 | 449 | extent batch_shape(count); 450 | std::copy_n(std::begin(shape), count, std::begin(batch_shape)); 451 | return batch_shape; 452 | } 453 | 454 | inline extent calculate_batch_shape(extent const &shape, index_t d0, index_t d1) { 455 | auto new_shape = get_batch_shape(shape); 456 | new_shape.push_back(d0); 457 | new_shape.push_back(d1); 458 | return new_shape; 459 | } 460 | 461 | template 462 | Tensor product(Tensor const &lhs, 463 | Tensor const &rhs) 464 | { 465 | auto lhs_dims = num_dims(lhs); 466 | auto rhs_dims = num_dims(rhs); 467 | 468 | // 5x3x6 * 5x 469 | if (lhs_dims > 2 || rhs_dims > 2) { 470 | auto lhs_copy = lhs; 471 | auto rhs_copy = rhs; 472 | 473 | // flatten batch dimension if either tensor is > 3 dims 474 | if (lhs_dims > 3) { 475 | lhs_copy = reshape(lhs_copy, {expand, lhs.shape()[lhs_dims-2], lhs.shape()[lhs_dims-1]}); 476 | } 477 | 478 | if (rhs_dims > 3) { 479 | rhs_copy = reshape(rhs_copy, {expand, rhs.shape()[rhs_dims-2], rhs.shape()[rhs_dims-1]}); 480 | } 481 | 482 | // if (lhs_dims == 2) { 483 | // // add batch dimension 484 | // } else if (lhs_dims == 1) { 485 | // // treat as batch matrix-vector multiplication 486 | // } 487 | 488 | // 2. either lhs or rhs > 3, in which case need to flatten 489 | auto result = batch_matrix_matrix_product(lhs_copy, rhs_copy); 490 | 491 | // for now, assume both lhs and rhs start off as the same shape 492 | // auto new_shape = get_batch_size(orig_lhs_shape); 493 | auto new_shape = calculate_batch_shape(lhs.shape(), result.shape()[1], result.shape()[2]); 494 | 495 | // reshape into original batch dims 496 | return reshape(result, new_shape); 497 | } 498 | 499 | if (lhs_dims == 1 && rhs_dims == 1) { 500 | return vector_vector_product(lhs, rhs); 501 | } 502 | 503 | if (rhs_dims == 1) { 504 | return matrix_vector_product(lhs, rhs); 505 | } 506 | 507 | if (lhs_dims == 1) { 508 | // TODO: follow pytorch convention and add a dimension 509 | // to lhs and make matrix_matrix? 510 | return matrix_vector_product(rhs, lhs); 511 | } 512 | 513 | // matrix output 514 | return matrix_matrix_product(lhs, rhs); 515 | } 516 | 517 | template 518 | Tensor operator *(Tensor const &lhs, Tensor const &rhs) { 519 | return product(lhs, rhs); 520 | } 521 | 522 | 523 | template 524 | Tensor operator ==(Tensor const &lhs, 525 | Tensor const &rhs) 526 | { 527 | return apply(lhs, rhs, [](T const &lop, T const &rop) { 528 | return lop == rop; 529 | }); 530 | } 531 | 532 | template 533 | Tensor operator !=(Tensor const &lhs, 534 | Tensor const &rhs) 535 | { 536 | return !(lhs == rhs); 537 | } 538 | 539 | template 540 | bool all(Tensor const &op, F fn) { 541 | for (auto const &index : op.indices()) { 542 | if (!fn(op(index))) return false; 543 | } 544 | 545 | return true; 546 | } 547 | 548 | template 549 | bool any(Tensor const &op, F fn) { 550 | for (auto const &index : op.indices()) { 551 | if (fn(op(index))) return true; 552 | } 553 | 554 | return false; 555 | } 556 | 557 | template 558 | bool equals(Tensor const &lhs, Tensor const &rhs) { 559 | return all(lhs == rhs, [](std::uint8_t v) { return v == true; }); 560 | } 561 | 562 | #endif -------------------------------------------------------------------------------- /include/tensor_slice.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TENSOR_SLICE_HPP 2 | #define TENSOR_SLICE_HPP 3 | 4 | #include "types.hpp" 5 | #include "layout.hpp" 6 | #include "tensor_data.hpp" 7 | 8 | template 9 | class Slice { 10 | public: 11 | 12 | Slice(index_t dim_index, 13 | index_t index, 14 | View const &view, 15 | std::shared_ptr> &data): 16 | dim_index_(dim_index), view_(view), data_(data) 17 | { 18 | view_.shape[dim_index] = 1; 19 | view.offset[dim_index] = index; 20 | } 21 | 22 | Slice(index_t dim_index, 23 | index_range range, 24 | const View const &view, 25 | std::shared_ptr> &data): 26 | dim_index_(dim_index), view_(view), data_(data) 27 | { 28 | view_.shape[dim_index] = range.second - range.first; 29 | view_.offset[dim_index] = range.first; 30 | } 31 | private: 32 | extent_t dim_index_; 33 | View view_; 34 | std::shared_ptr> &data_; 35 | }; 36 | 37 | #endif -------------------------------------------------------------------------------- /include/types.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TYPES_HPP 2 | #define TYPES_HPP 3 | 4 | #include 5 | #include 6 | 7 | using offset_t = std::size_t; 8 | using index_t = std::size_t; 9 | 10 | // make extent_t 11 | using extent = std::vector; 12 | 13 | // make indices_t 14 | using indices = std::vector; 15 | using index_range = std::pair; 16 | 17 | inline std::size_t num_elements(const extent &shape) { 18 | return std::accumulate( 19 | std::begin(shape), std::end(shape), 1, std::multiplies<>()); 20 | } 21 | 22 | #endif -------------------------------------------------------------------------------- /test/test_indexing.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "layout.hpp" 8 | #include "index_generator.hpp" 9 | #include "stride_generator.hpp" 10 | 11 | TEST(IndexTestSuite, TestMakeRowMajorOrder) { 12 | auto order = make_row_major_order(4); 13 | 14 | indices expected = {3, 2, 1, 0}; 15 | auto expected_value = std::begin(expected); 16 | for (auto index : order) { 17 | ASSERT_EQ(*expected_value++, index); 18 | } 19 | } 20 | 21 | TEST(IndexTestSuite, TestMakeColMajorOrder) { 22 | indices order = make_col_major_order(4); 23 | 24 | indices expected = {0, 1, 2, 3}; 25 | auto expected_value = std::begin(expected); 26 | for (auto index : order) { 27 | ASSERT_EQ(*expected_value++, index); 28 | } 29 | } 30 | 31 | TEST(IndexTestSuite, TestOrderedViewRowMajor) { 32 | extent shape{2, 3, 4}; 33 | indices order = make_row_major_order(shape.size()); 34 | ordered_shape_view shape_view(shape, order); 35 | 36 | indices expected = {4, 3, 2}; 37 | auto expected_value = std::begin(expected); 38 | for (auto e : shape_view) { 39 | ASSERT_EQ(*expected_value++, e); 40 | } 41 | } 42 | 43 | TEST(IndexTestSuite, TestOrderedViewColMajor) { 44 | extent shape{2, 3, 4}; 45 | indices order = make_col_major_order(shape.size()); 46 | ordered_shape_view shape_view(shape, order); 47 | 48 | indices expected = {2, 3, 4}; 49 | auto expected_value = std::begin(expected); 50 | for (auto e : shape_view) { 51 | ASSERT_EQ(*expected_value++, e); 52 | } 53 | } 54 | 55 | TEST(IndexTestSuite, TestStridesRowMajor1) { 56 | extent shape{2, 3, 4}; 57 | indices order = make_row_major_order(shape.size()); 58 | indices strides = make_strides(shape, order); 59 | 60 | ASSERT_EQ((indices{12, 4, 1}), strides); 61 | } 62 | 63 | TEST(IndexTestSuite, TestStridesRowMajor2) { 64 | { 65 | extent shape{4, 1}; 66 | indices order = make_row_major_order(shape.size()); 67 | indices strides = make_strides(shape, order); 68 | 69 | ASSERT_EQ((indices{1, 1}), strides); 70 | } 71 | 72 | { 73 | extent shape{1, 4}; 74 | indices order = make_row_major_order(shape.size()); 75 | indices strides = make_strides(shape, order); 76 | 77 | ASSERT_EQ((indices{4, 1}), strides); 78 | } 79 | } 80 | 81 | TEST(IndexTestSuite, TestStridesColMajor) { 82 | extent shape{2, 3, 4}; 83 | indices order = make_col_major_order(shape.size()); 84 | indices strides = make_strides(shape, order); 85 | 86 | ASSERT_EQ((indices{1, 2, 6}), strides); 87 | } 88 | 89 | TEST(IndexTestSuite, TestIndexGenerator1dRowMajor) { 90 | extent shape{5}; 91 | indices order = make_row_major_order(shape.size()); 92 | indices strides = make_strides(shape, order); 93 | index_generator indices(shape, strides); 94 | 95 | std::list expected = { {0}, {1}, {2}, {3}, {4} }; 96 | } 97 | 98 | TEST(IndexTestSuite, TestIndexGenerator1dColMajor) { 99 | extent shape{5}; 100 | indices order = make_col_major_order(shape.size()); 101 | indices strides = make_strides(shape, order); 102 | index_generator indices(shape, strides); 103 | 104 | std::list expected = { {0}, {1}, {2}, {3}, {4} }; 105 | auto expected_value = std::begin(expected); 106 | 107 | for (auto &index : indices) { 108 | ASSERT_EQ(*expected_value++, index); 109 | } 110 | } 111 | 112 | TEST(IndexTestSuite, TestIndexGenerator2dRowMajor) { 113 | extent shape{3, 3}; 114 | indices order = make_row_major_order(shape.size()); 115 | indices strides = make_strides(shape, order); 116 | index_generator indices(shape, strides); 117 | 118 | std::list expected = { 119 | {0, 0}, {0, 1}, {0, 2}, 120 | {1, 0}, {1, 1}, {1, 2}, 121 | {2, 0}, {2, 1}, {2, 2} 122 | }; 123 | auto expected_value = std::begin(expected); 124 | 125 | for (auto &index : indices) { 126 | ASSERT_EQ(*expected_value++, index); 127 | } 128 | } 129 | 130 | TEST(IndexTestSuite, TestIndexGenerator2dColMajor) { 131 | extent shape{3, 3}; 132 | indices order = make_col_major_order(shape.size()); 133 | indices strides = make_strides(shape, order); 134 | index_generator indices(shape, strides); 135 | 136 | std::list expected = { 137 | {0, 0}, {1, 0}, {2, 0}, 138 | {0, 1}, {1, 1}, {2, 1}, 139 | {0, 2}, {1, 2}, {2, 2} 140 | }; 141 | auto expected_value = std::begin(expected); 142 | 143 | for (auto &index : indices) { 144 | ASSERT_EQ(*expected_value++, index); 145 | } 146 | } 147 | 148 | TEST(IndexTestSuite, TestIndexGenerator3dRowMajor) { 149 | extent shape{3, 3, 3}; 150 | indices order = make_row_major_order(shape.size()); 151 | indices strides = make_strides(shape, order); 152 | index_generator indices(shape, strides); 153 | 154 | std::list expected = { 155 | {0, 0, 0}, {0, 0, 1}, {0, 0, 2}, 156 | {0, 1, 0}, {0, 1, 1}, {0, 1, 2}, 157 | {0, 2, 0}, {0, 2, 1}, {0, 2, 2}, 158 | 159 | {1, 0, 0}, {1, 0, 1}, {1, 0, 2}, 160 | {1, 1, 0}, {1, 1, 1}, {1, 1, 2}, 161 | {1, 2, 0}, {1, 2, 1}, {1, 2, 2}, 162 | 163 | {2, 0, 0}, {2, 0, 1}, {2, 0, 2}, 164 | {2, 1, 0}, {2, 1, 1}, {2, 1, 2}, 165 | {2, 2, 0}, {2, 2, 1}, {2, 2, 2} 166 | }; 167 | auto expected_value = std::begin(expected); 168 | 169 | for (auto &index : indices) { 170 | ASSERT_EQ(*expected_value++, index); 171 | } 172 | } 173 | 174 | TEST(IndexTestSuite, TestIndexGenerator3dColMajor) { 175 | extent shape{3, 3, 3}; 176 | indices order = make_col_major_order(shape.size()); 177 | indices strides = make_strides(shape, order); 178 | index_generator indices(shape, strides); 179 | 180 | std::list expected = { 181 | {0, 0, 0}, {1, 0, 0}, {2, 0, 0}, 182 | {0, 1, 0}, {1, 1, 0}, {2, 1, 0}, 183 | {0, 2, 0}, {1, 2, 0}, {2, 2, 0}, 184 | {0, 0, 1}, {1, 0, 1}, {2, 0, 1}, 185 | {0, 1, 1}, {1, 1, 1}, {2, 1, 1}, 186 | {0, 2, 1}, {1, 2, 1}, {2, 2, 1}, 187 | {0, 0, 2}, {1, 0, 2}, {2, 0, 2}, 188 | {0, 1, 2}, {1, 1, 2}, {2, 1, 2}, 189 | {0, 2, 2}, {1, 2, 2}, {2, 2, 2}, 190 | }; 191 | auto expected_value = std::begin(expected); 192 | 193 | for (auto &index : indices) { 194 | ASSERT_EQ(*expected_value++, index); 195 | } 196 | } 197 | 198 | TEST(IndexTestSuite, TestLayoutCalculateOffset1d) { 199 | extent shape{10}; 200 | indices order = make_row_major_order(shape.size()); 201 | Layout layout(shape, make_strides(shape, order)); 202 | 203 | indices expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; 204 | auto expected_value = std::begin(expected); 205 | 206 | for (auto &index : index_generator(layout)) { 207 | offset_t offset = calculate_offset(layout, index); 208 | ASSERT_EQ(*expected_value++, offset); 209 | } 210 | } 211 | 212 | TEST(IndexTestSuite, TestLayoutCalculateOffset2dRowMajor) { 213 | extent shape{5, 2}; 214 | indices order = make_row_major_order(shape.size()); 215 | Layout layout(shape, make_strides(shape, order)); 216 | 217 | indices expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; 218 | auto expected_value = std::begin(expected); 219 | 220 | for (auto &index : index_generator(layout)) { 221 | offset_t offset = calculate_offset(layout, index); 222 | ASSERT_EQ(*expected_value++, offset); 223 | } 224 | } 225 | 226 | TEST(IndexTestSuite, TestLayoutCalculateOffset2dColMajor) { 227 | extent shape{5, 2}; 228 | indices order = make_col_major_order(shape.size()); 229 | Layout layout(shape, make_strides(shape, order)); 230 | 231 | indices expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; 232 | auto expected_value = std::begin(expected); 233 | 234 | for (auto &index : index_generator(layout)) { 235 | offset_t offset = calculate_offset(layout, index); 236 | ASSERT_EQ(*expected_value++, offset); 237 | } 238 | } 239 | 240 | TEST(IndexTestSuite, TestLayoutCalculateOffset3dRowMajor) { 241 | extent shape{2, 3, 2}; 242 | indices order = make_row_major_order(shape.size()); 243 | Layout layout(shape, make_strides(shape, order)); 244 | 245 | indices expected = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; 246 | auto expected_value = std::begin(expected); 247 | 248 | for (auto &index : index_generator(layout)) { 249 | offset_t offset = calculate_offset(layout, index); 250 | ASSERT_EQ(*expected_value++, offset); 251 | } 252 | } 253 | 254 | TEST(IndexTestSuite, TestViewCalculateOffset1dRowMajor) { 255 | extent shape{10}; 256 | indices order = make_row_major_order(shape.size()); 257 | Layout layout(shape, make_strides(shape, order)); 258 | 259 | extent view_shape{8}; 260 | indices view_offset{2}; 261 | View view(layout, view_shape, view_offset); 262 | 263 | indices expected = {2, 3, 4, 5, 6, 7, 8, 9}; 264 | auto expected_value = std::begin(expected); 265 | 266 | for (auto &index : index_generator(view)) { 267 | offset_t offset = calculate_offset(view, index); 268 | ASSERT_EQ(*expected_value++, offset); 269 | } 270 | } 271 | 272 | TEST(IndexTestSuite, TestViewCalculateOffset2dRowMajor) { 273 | extent shape{3, 5}; 274 | indices order = make_row_major_order(shape.size()); 275 | Layout layout(shape, make_strides(shape, order)); 276 | 277 | extent view_shape{2, 3}; 278 | indices view_offset{1, 2}; 279 | View view(layout, view_shape, view_offset); 280 | 281 | index_t expected[3][5] = { 282 | { 0, 1, 2, 3, 4}, 283 | { 5, 6, 7, 8, 9}, 284 | {10, 11, 12, 13, 14} 285 | }; 286 | 287 | auto index = index_generator(view); 288 | for (index_t j = 0; j < view.shape[1]; j++) { 289 | for (index_t i = 0; i < view.shape[0]; i++) { 290 | offset_t offset = calculate_offset(view, index.read()); 291 | auto expected_value = expected[i + view.offset[0]][j + view.offset[1]]; 292 | ASSERT_EQ(expected_value, offset); 293 | index.next(); 294 | } 295 | } 296 | } 297 | 298 | TEST(IndexTestSuite, TestViewCalculateOffset2dColMajor) { 299 | extent shape{3, 5}; 300 | indices order = make_col_major_order(shape.size()); 301 | Layout layout(shape, make_strides(shape, order)); 302 | 303 | extent view_shape{2, 3}; 304 | indices view_offset{1, 2}; 305 | View view(layout, view_shape, view_offset); 306 | 307 | index_t expected[3][5] = { 308 | {0, 3, 6, 9 , 12}, 309 | {1, 4, 7, 10, 13}, 310 | {2, 5, 8, 11, 14} 311 | }; 312 | 313 | // show view.stride is necessary for making the indices, but layout.stride is 314 | // necessary for storage? 315 | auto index = index_generator(view); 316 | for (index_t j = 0; j < view.shape[1]; j++) { 317 | for (index_t i = 0; i < view.shape[0]; i++) { 318 | offset_t offset = calculate_offset(view, index.read()); 319 | auto expected_value = expected[i + view.offset[0]][j + view.offset[1]]; 320 | ASSERT_EQ(expected_value, offset); 321 | index.next(); 322 | } 323 | } 324 | } 325 | 326 | TEST(IndexTestSuite, TestViewCalculateOffsetWithSingletons) { 327 | extent shape{1, 1, 1, 5}; 328 | auto order = make_row_major_order(shape.size()); 329 | auto strides = make_strides(shape, order); 330 | 331 | ASSERT_EQ((indices{5, 5, 5, 1}), strides); 332 | View view(Layout(shape, strides)); 333 | 334 | indices expected = {0, 1, 2, 3, 4}; 335 | auto expected_value = std::begin(expected); 336 | 337 | auto index = index_generator(view); 338 | for (index_t i = 0; i < view.shape[3]; i++) { 339 | offset_t offset = calculate_offset(view, index.read()); 340 | ASSERT_EQ(*expected_value, offset); 341 | index.next(); 342 | ++expected_value; 343 | } 344 | 345 | // verify it also works using this method 346 | expected_value = std::begin(expected); 347 | for (index_t i = 0; i < view.shape[3]; i++) { 348 | offset_t offset = calculate_offset(view, i); 349 | ASSERT_EQ(*expected_value, offset); 350 | ++expected_value; 351 | } 352 | } -------------------------------------------------------------------------------- /test/test_storage.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | #include "storage.hpp" 8 | 9 | template 10 | class StorageTestSuite: public ::testing::Test {}; 11 | 12 | using StorageTypes = ::testing::Types; 13 | TYPED_TEST_CASE(StorageTestSuite, StorageTypes); 14 | 15 | TYPED_TEST(StorageTestSuite, TestCPUCreate) { 16 | Storage storage(10); 17 | 18 | ASSERT_EQ(10, storage.size()); 19 | 20 | for (auto e : storage) { 21 | ASSERT_EQ(0, e); 22 | } 23 | 24 | ranges::fill(storage, 1); 25 | 26 | for (auto e : storage) { 27 | ASSERT_EQ(1, e); 28 | } 29 | 30 | for (index_t i = 0; i < storage.size(); i++) { 31 | storage[i] = 2; 32 | } 33 | 34 | for (index_t i = 0; i < storage.size(); i++) { 35 | ASSERT_EQ(2, storage[i]); 36 | } 37 | } -------------------------------------------------------------------------------- /test/test_tensor.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | #include "tensor.hpp" 8 | #include "tensor_ops.hpp" 9 | #include "format.hpp" 10 | 11 | #define ASSERT_TENSORS_EQ(expected, result) \ 12 | ASSERT_TRUE(equals(expected, result)) 13 | 14 | 15 | template 16 | void fill_tensor(Tensor &tensor) { 17 | index_t i = 0; 18 | for (auto &index : index_generator(tensor.shape())) { 19 | tensor(index) = i++; 20 | } 21 | } 22 | 23 | TEST(TensorTestSuite, TestMakeRowMajorTensor2d) { 24 | extent shape{3, 3}; 25 | Tensor t(shape, TensorOrder::RowMajor); 26 | 27 | fill_tensor(t); 28 | 29 | int expected[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; 30 | 31 | // verify storage contents are row-major 32 | for (index_t i = 0; i < t.storage().size(); i++) { 33 | ASSERT_EQ(expected[i], t.storage()[i]); 34 | } 35 | } 36 | 37 | TEST(TensorTestSuite, TestMakeColMajorTensor2d) { 38 | Tensor t({3, 3}, TensorOrder::ColumnMajor); 39 | 40 | fill_tensor(t); 41 | 42 | int expected[] = {0, 3, 6, 1, 4, 7, 2, 5, 8}; 43 | 44 | // verify storage contents are column-major 45 | for (index_t i = 0; i < t.storage().size(); i++) { 46 | ASSERT_EQ(expected[i], t.storage()[i]); 47 | } 48 | } 49 | 50 | // tests that even though t1 and t2 store their contents in a different ordering, 51 | // the same indexing can be used 52 | TEST(TensorTestSuite, TestOrdering) { 53 | Tensor t1({3, 5}, TensorOrder::RowMajor); 54 | Tensor t2({3, 5}, TensorOrder::ColumnMajor); 55 | 56 | int count = 0; 57 | for (auto &index : index_generator(t1.shape(), t1.view().strides)) { 58 | t1(index) = count; 59 | t2(index) = count; 60 | ++count; 61 | } 62 | 63 | for (auto &index : index_generator(t1.shape(), t1.view().strides)) { 64 | ASSERT_EQ(t1(index), t2(index)); 65 | } 66 | } 67 | 68 | TEST(TensorTestSuite, TestSliceTensor) { 69 | Tensor t({3, 3}); 70 | 71 | { 72 | auto slice = t[0]; 73 | ASSERT_EQ((extent{1, 3}), slice.view().shape); 74 | ASSERT_EQ(0, slice.view().offset[0]); 75 | 76 | auto slice2 = slice[0]; 77 | ASSERT_EQ((extent{1, 1}), slice2.view().shape); 78 | ASSERT_EQ(0, slice.view().offset[0]); 79 | } 80 | 81 | { 82 | auto slice = t[1]; 83 | ASSERT_EQ((extent{1, 3}), slice.view().shape); 84 | ASSERT_EQ(1, slice.view().offset[0]); 85 | 86 | auto slice2 = slice[2]; 87 | ASSERT_EQ((extent{1, 1}), slice2.view().shape); 88 | ASSERT_EQ(2, slice2.view().offset[1]); 89 | } 90 | } 91 | 92 | TEST(TensorTestSuite, TestSliceRangeTensor) { 93 | Tensor t({5, 3}); 94 | fill_tensor(t); 95 | 96 | { 97 | Tensor t2 = t[{0, 3}]; 98 | 99 | auto expected = tensor({ 100 | {0, 1, 2}, 101 | {3, 4, 5}, 102 | {6, 7, 8} 103 | }); 104 | 105 | 106 | ASSERT_TENSORS_EQ(expected, t2); 107 | } 108 | 109 | { 110 | Tensor t2 = t[{1, 4}]; 111 | 112 | auto expected = tensor({ 113 | {3, 4, 5}, 114 | {6, 7, 8}, 115 | {9, 10, 11} 116 | }); 117 | 118 | ASSERT_TENSORS_EQ(expected, t2); 119 | } 120 | 121 | { 122 | Tensor t2 = t[{1, 3}][{1, 3}]; 123 | 124 | auto expected = tensor({ 125 | {4, 5}, 126 | {7, 8} 127 | }); 128 | 129 | ASSERT_TENSORS_EQ(expected, t2); 130 | } 131 | } 132 | 133 | TEST(TensorTestSuite, TestMakeView) { 134 | auto t = Tensor({5, 3}); 135 | fill_tensor(t); 136 | 137 | { 138 | View view(t.view(), {2, 2}, {0, 0}); 139 | auto t2 = t.view(view); 140 | 141 | index_t expected[] = {0, 3, 1, 4}; 142 | 143 | index_t i = 0; 144 | for (auto index : t2.indices()) { 145 | ASSERT_EQ(expected[i++], t2(index)); 146 | } 147 | } 148 | 149 | { 150 | View view(t.view(), {2, 2}, {1, 1}); 151 | auto t2 = t.view(view); 152 | 153 | index_t expected[] = {4, 7, 5, 8}; 154 | 155 | index_t i = 0; 156 | for (auto index : t2.indices()) { 157 | ASSERT_EQ(expected[i++], t2(index)); 158 | } 159 | } 160 | 161 | { 162 | View view(t.view(), {5, 1}, {0, 1}); 163 | auto t2 = t.view(view); 164 | 165 | index_t expected[] = {1, 4, 7, 10, 13}; 166 | 167 | index_t i = 0; 168 | for (auto index : t2.indices()) { 169 | ASSERT_EQ(expected[i++], t2(index)); 170 | } 171 | } 172 | 173 | { 174 | View view(t.view(), {1, 3}, {1, 0}); 175 | auto t2 = t.view(view); 176 | 177 | index_t expected[] = {3, 4, 5}; 178 | 179 | index_t i = 0; 180 | for (auto index : t2.indices()) { 181 | ASSERT_EQ(expected[i++], t2(index)); 182 | } 183 | } 184 | } 185 | 186 | TEST(TensorTestSuite, TestTranspose) { 187 | auto t = Tensor({2, 3, 4}); 188 | fill_tensor(t); 189 | 190 | { 191 | auto t2 = transpose(t, {1, 0, 2}); 192 | 193 | auto expected = tensor({ 194 | {{0, 1, 2, 3}, 195 | {12, 13, 14, 15}}, 196 | 197 | {{4, 5, 6, 7}, 198 | {16, 17, 18, 19}}, 199 | 200 | {{8, 9, 10, 11}, 201 | {20, 21, 22, 23}} 202 | }); 203 | 204 | ASSERT_EQ((std::vector{4ul, 12ul, 1ul}), t2.view().strides); 205 | ASSERT_TENSORS_EQ(expected, t2); 206 | } 207 | 208 | { 209 | auto t2 = transpose(t, {0, 2, 1}); 210 | 211 | auto expected = tensor({ 212 | {{ 0, 4, 8}, 213 | { 1, 5, 9}, 214 | { 2, 6, 10}, 215 | { 3, 7, 11}}, 216 | 217 | {{12, 16, 20}, 218 | {13, 17, 21}, 219 | {14, 18, 22}, 220 | {15, 19, 23}} 221 | }); 222 | 223 | ASSERT_EQ((std::vector{12ul, 1ul, 4ul}), t2.view().strides); 224 | ASSERT_TENSORS_EQ(expected, t2); 225 | } 226 | 227 | { 228 | auto t2 = transpose(t, {1, 2, 0}); 229 | 230 | auto expected = tensor({ 231 | {{ 0, 12}, 232 | { 1, 13}, 233 | { 2, 14}, 234 | { 3, 15}}, 235 | 236 | {{ 4, 16}, 237 | { 5, 17}, 238 | { 6, 18}, 239 | { 7, 19}}, 240 | 241 | {{ 8, 20}, 242 | { 9, 21}, 243 | {10, 22}, 244 | {11, 23}} 245 | }); 246 | 247 | ASSERT_EQ((std::vector{4ul, 1ul, 12ul}), t2.view().strides); 248 | ASSERT_TENSORS_EQ(expected, t2); 249 | } 250 | } 251 | 252 | TEST(TensorTestSuite, TestTransposeWithView) { 253 | auto t = Tensor({2, 3, 4}); 254 | fill_tensor(t); 255 | 256 | { 257 | Tensor t2 = t[{0, 2}][{0, 3}][{0, 2}]; 258 | auto t3 = transpose(t2, {1, 0, 2}); 259 | 260 | auto expected = tensor({ 261 | {{ 0, 1}, 262 | {12, 13}}, 263 | 264 | {{ 4, 5}, 265 | {16, 17}}, 266 | 267 | {{ 8, 9}, 268 | {20, 21}}, 269 | }); 270 | 271 | ASSERT_EQ((std::vector{4ul, 12ul, 1ul}), t3.view().strides); 272 | ASSERT_TENSORS_EQ(expected, t3); 273 | } 274 | 275 | { 276 | Tensor t2 = t[{0, 2}][{0, 3}][{1, 3}]; 277 | auto t3 = transpose(t2, {1, 0, 2}); 278 | 279 | auto expected = tensor({ 280 | {{ 1, 2}, 281 | {13, 14}}, 282 | 283 | {{ 5, 6}, 284 | {17, 18}}, 285 | 286 | {{ 9, 10}, 287 | {21, 22}}, 288 | }); 289 | 290 | ASSERT_EQ((std::vector{4ul, 12ul, 1ul}), t3.view().strides); 291 | ASSERT_TENSORS_EQ(expected, t3); 292 | } 293 | 294 | } 295 | 296 | // TODO: move to test_format.cpp 297 | TEST(TensorTestSuite, TestFormatTensor1d) { 298 | Tensor t({5}, TensorOrder::RowMajor); 299 | 300 | fill_tensor(t); 301 | 302 | std::string expected = "[ 0, 1, 2, 3, 4]"; 303 | 304 | fmt::memory_buffer buffer; 305 | fmt::format_to(buffer, "{}", t); 306 | ASSERT_EQ(fmt::to_string(buffer), expected); 307 | } 308 | 309 | TEST(TensorTestSuite, TestFormatTensor2d_1) { 310 | Tensor t({1, 5}, TensorOrder::RowMajor); 311 | 312 | fill_tensor(t); 313 | 314 | std::string expected = "[[ 0, 1, 2, 3, 4]]"; 315 | 316 | fmt::memory_buffer buffer; 317 | fmt::format_to(buffer, "{}", t); 318 | ASSERT_EQ(fmt::to_string(buffer), expected); 319 | } 320 | 321 | TEST(TensorTestSuite, TestFormatTensor2d_2) { 322 | Tensor t({5, 1}, TensorOrder::RowMajor); 323 | 324 | fill_tensor(t); 325 | 326 | std::string expected = "[[ 0],\n" 327 | " [ 1],\n" 328 | " [ 2],\n" 329 | " [ 3],\n" 330 | " [ 4]]"; 331 | 332 | fmt::memory_buffer buffer; 333 | fmt::format_to(buffer, "{}", t); 334 | ASSERT_EQ(fmt::to_string(buffer), expected); 335 | } 336 | 337 | TEST(TensorTestSuite, TestFormatTensor2d_3) { 338 | Tensor t({3, 4}, TensorOrder::RowMajor); 339 | 340 | fill_tensor(t); 341 | 342 | std::string expected = "[[ 0, 1, 2, 3],\n" 343 | " [ 4, 5, 6, 7],\n" 344 | " [ 8, 9, 10, 11]]"; 345 | 346 | fmt::memory_buffer buffer; 347 | fmt::format_to(buffer, "{}", t); 348 | ASSERT_EQ(fmt::to_string(buffer), expected); 349 | } 350 | 351 | TEST(TensorTestSuite, TestFormatTensor3d_1) { 352 | Tensor t({2, 3, 4}, TensorOrder::RowMajor); 353 | 354 | fill_tensor(t); 355 | 356 | std::string expected = "[[[ 0, 1, 2, 3],\n" 357 | " [ 4, 5, 6, 7],\n" 358 | " [ 8, 9, 10, 11]],\n" 359 | "\n" 360 | " [[ 12, 13, 14, 15],\n" 361 | " [ 16, 17, 18, 19],\n" 362 | " [ 20, 21, 22, 23]]]"; 363 | 364 | fmt::memory_buffer buffer; 365 | fmt::format_to(buffer, "{}", t); 366 | ASSERT_EQ(fmt::to_string(buffer), expected); 367 | } 368 | 369 | TEST(TensorTestSuite, TestFormatTensor4d_1) { 370 | Tensor t({2, 2, 3, 4}, TensorOrder::RowMajor); 371 | 372 | fill_tensor(t); 373 | 374 | std::string expected = "[[[[ 0, 1, 2, 3],\n" 375 | " [ 4, 5, 6, 7],\n" 376 | " [ 8, 9, 10, 11]],\n" 377 | "\n" 378 | " [[ 12, 13, 14, 15],\n" 379 | " [ 16, 17, 18, 19],\n" 380 | " [ 20, 21, 22, 23]]],\n" 381 | "\n" 382 | "\n" 383 | " [[[ 24, 25, 26, 27],\n" 384 | " [ 28, 29, 30, 31],\n" 385 | " [ 32, 33, 34, 35]],\n" 386 | "\n" 387 | " [[ 36, 37, 38, 39],\n" 388 | " [ 40, 41, 42, 43],\n" 389 | " [ 44, 45, 46, 47]]]]"; 390 | 391 | fmt::memory_buffer buffer; 392 | fmt::format_to(buffer, "{}", t); 393 | ASSERT_EQ(fmt::to_string(buffer), expected); 394 | } 395 | -------------------------------------------------------------------------------- /test/test_tensor_ops.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | #include "tensor.hpp" 8 | #include "tensor_ops.hpp" 9 | #include "format.hpp" 10 | 11 | #define ASSERT_TENSORS_EQ(expected, result) \ 12 | ASSERT_TRUE(equals(expected, result)) 13 | 14 | TEST(TensorOpsTestSuite, TestIsContiguous) { 15 | Tensor t({2, 3, 4}); 16 | iota(t); 17 | 18 | ASSERT_TRUE(t.contiguous()); 19 | 20 | auto t2 = transpose(t, {1, 0, 2}); 21 | ASSERT_FALSE(t2.contiguous()); 22 | } 23 | 24 | TEST(TensorOpsTestSuite, TestReshape) { 25 | auto t = Tensor({2, 3, 4}); 26 | iota(t, 0); 27 | 28 | { 29 | auto t2 = reshape(t, {24}); 30 | 31 | auto expected = tensor({ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 32 | 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}); 33 | 34 | ASSERT_EQ((std::vector{24UL}), t2.shape()); 35 | ASSERT_EQ((std::vector{1UL}), t2.view().strides); 36 | ASSERT_TENSORS_EQ(expected, t2); 37 | 38 | // contiguous, so should have same memory address 39 | ASSERT_EQ(t.storage_ptr(), t2.storage_ptr()); 40 | } 41 | 42 | { 43 | auto t2 = reshape(t, {6, 4}); 44 | 45 | auto expected = tensor({ 46 | { 0, 1, 2, 3}, 47 | { 4, 5, 6, 7}, 48 | { 8, 9, 10, 11}, 49 | { 12, 13, 14, 15}, 50 | { 16, 17, 18, 19}, 51 | { 20, 21, 22, 23} 52 | }); 53 | 54 | ASSERT_EQ((std::vector{6UL, 4UL}), t2.shape()); 55 | ASSERT_EQ((std::vector{4UL, 1UL}), t2.view().strides); 56 | ASSERT_TENSORS_EQ(expected, t2); 57 | 58 | // contiguous, so should have same memory address 59 | ASSERT_EQ(t.storage_ptr(), t2.storage_ptr()); 60 | } 61 | 62 | { 63 | auto t2 = reshape(t, {4, 6}); 64 | 65 | auto expected = tensor({ 66 | { 0, 1, 2, 3, 4, 5}, 67 | { 6, 7, 8, 9, 10, 11}, 68 | { 12, 13, 14, 15, 16, 17}, 69 | { 18, 19, 20, 21, 22, 23} 70 | }); 71 | 72 | 73 | ASSERT_EQ((std::vector{4UL, 6UL}), t2.shape()); 74 | ASSERT_EQ((std::vector{6UL, 1UL}), t2.view().strides); 75 | ASSERT_TENSORS_EQ(expected, t2); 76 | 77 | // contiguous, so should have same memory address 78 | ASSERT_EQ(t.storage_ptr(), t2.storage_ptr()); 79 | } 80 | 81 | { 82 | EXPECT_THROW(reshape(t, {2, 3}), MismatchedNumberOfElements); 83 | } 84 | } 85 | 86 | TEST(TensorOpTestSuite, TestReshapeWithNonContiguousTensor) { 87 | Tensor t({2, 3, 4}); 88 | iota(t); 89 | 90 | auto t2 = transpose(t, {1, 0, 2}); 91 | auto t3 = reshape(t2, {6, 4}); 92 | 93 | auto expected = tensor({ 94 | { 0, 1, 2, 3}, 95 | { 4, 5, 6, 7}, 96 | { 8, 9, 10, 11}, 97 | { 12, 13, 14, 15}, 98 | { 16, 17, 18, 19}, 99 | { 20, 21, 22, 23} 100 | }); 101 | 102 | ASSERT_EQ((extent{6, 4}), t3.shape()); 103 | ASSERT_TENSORS_EQ(expected, t3); 104 | 105 | // t2 was non-contiguous, so verify it's a different memory address 106 | ASSERT_NE(t3.storage_ptr(), t2.storage_ptr()); 107 | } 108 | 109 | TEST(TensorOpTestSuite, TestReshapeWithInferredDimension) { 110 | Tensor t({2, 3, 4}); 111 | iota(t); 112 | 113 | auto t2 = reshape(t, {expand, 4}); 114 | ASSERT_EQ((extent{6, 4}), t2.shape()); 115 | 116 | auto expected = tensor({ 117 | { 0, 1, 2, 3}, 118 | { 4, 5, 6, 7}, 119 | { 8, 9, 10, 11}, 120 | { 12, 13, 14, 15}, 121 | { 16, 17, 18, 19}, 122 | { 20, 21, 22, 23} 123 | }); 124 | 125 | ASSERT_TENSORS_EQ(expected, t2); 126 | } 127 | 128 | TEST(TensorOpsTestSuite, TestBroadcast1) { 129 | { 130 | auto t = tensor({1, 2, 3, 4}); 131 | auto t2 = broadcast_to(t, {3, 4}); 132 | 133 | auto expected = tensor({ 134 | { 1, 2, 3, 4}, 135 | { 1, 2, 3, 4}, 136 | { 1, 2, 3, 4} 137 | }); 138 | 139 | ASSERT_EQ((extent{3, 4}), t2.shape()); 140 | ASSERT_EQ((indices{0, 1}), t2.view().strides); 141 | ASSERT_TENSORS_EQ(expected, t2); 142 | } 143 | 144 | { 145 | auto t = tensor({1, 2, 3, 4}); 146 | EXPECT_THROW(broadcast_to(t, {4, 3}), CannotBroadcast); 147 | } 148 | 149 | { 150 | auto t = tensor({3}); 151 | auto t2 = broadcast_to(t, {3, 3}); 152 | 153 | auto expected = tensor({ 154 | {3, 3, 3}, 155 | {3, 3, 3}, 156 | {3, 3, 3} 157 | }); 158 | 159 | 160 | ASSERT_EQ((extent{3, 3}), t2.shape()); 161 | ASSERT_EQ((indices{0, 0}), t2.view().strides); 162 | ASSERT_TENSORS_EQ(expected, t2); 163 | } 164 | } 165 | 166 | TEST(TensorOpsTestSuite, TestBroadcast2) { 167 | auto t1 = tensor({ 168 | {1, 1, 1, 1, 1}, 169 | {2, 2, 2, 2, 2}, 170 | {3, 3, 3, 3, 3} 171 | }); 172 | 173 | auto t2 = tensor({4, 4, 4, 4, 4}); 174 | 175 | auto [t3, t4] = broadcast(t1, t2); 176 | 177 | auto expected = tensor({ 178 | {5, 5, 5, 5, 5}, 179 | {6, 6, 6, 6, 6}, 180 | {7, 7, 7, 7, 7} 181 | }); 182 | 183 | auto result = t3 + t4; 184 | 185 | ASSERT_EQ((extent{3, 5}), t3.shape()); 186 | ASSERT_EQ((extent{3, 5}), t4.shape()); 187 | ASSERT_TENSORS_EQ(expected, result); 188 | } 189 | 190 | TEST(TensorOpsTestSuite, TestBroadcast3) { 191 | auto t1 = tensor({ 192 | {1, 1, 1, 1, 1}, 193 | {2, 2, 2, 2, 2}, 194 | {3, 3, 3, 3, 3} 195 | }); 196 | 197 | auto t2 = tensor({{4, 4, 4, 4, 4}}); 198 | 199 | auto [t3, t4] = broadcast(t1, t2); 200 | 201 | auto expected = tensor({ 202 | {5, 5, 5, 5, 5}, 203 | {6, 6, 6, 6, 6}, 204 | {7, 7, 7, 7, 7} 205 | }); 206 | 207 | auto result = t3 + t4; 208 | 209 | ASSERT_EQ((extent{3, 5}), t3.shape()); 210 | ASSERT_EQ((extent{3, 5}), t4.shape()); 211 | ASSERT_TENSORS_EQ(expected, result); 212 | } 213 | 214 | TEST(TensorOpsTestSuite, TestAddTensors1) { 215 | auto lhs = tensor({1, 2, 3}); 216 | auto rhs = tensor({1, 1, 1}); 217 | auto expected = tensor({2, 3, 4}); 218 | 219 | auto result = lhs + rhs; 220 | ASSERT_TENSORS_EQ(expected, result); 221 | } 222 | 223 | TEST(TensorOpsTestSuite, TestAddTensorsWithBroadcast1) { 224 | auto lhs = tensor({1, 2, 3}); 225 | auto rhs = tensor({1}); 226 | auto expected = tensor({2, 3, 4}); 227 | 228 | auto result = lhs + rhs; 229 | ASSERT_TENSORS_EQ(expected, result); 230 | } 231 | 232 | TEST(TensorOpsTestSuite, TestAddTensorsWithBroadcast2) { 233 | auto lhs = tensor({ 234 | {1, 2, 3}, 235 | {4, 5, 6}, 236 | {7, 8, 9} 237 | }); 238 | 239 | auto rhs = tensor({1}); 240 | auto expected = tensor({ 241 | {2, 3, 4}, 242 | {5, 6, 7}, 243 | {8, 9, 10} 244 | }); 245 | 246 | auto result = lhs + rhs; 247 | ASSERT_TENSORS_EQ(expected, result); 248 | } 249 | 250 | TEST(TensorOpsTestSuite, TestAddTensorsWithBroadcast3) { 251 | auto lhs = tensor({ 252 | {1, 2, 3}, 253 | {4, 5, 6}, 254 | {7, 8, 9}, 255 | {10, 11, 12} 256 | }); 257 | 258 | auto rhs = tensor({1, 2, 3}); 259 | auto expected = tensor({ 260 | {2, 4, 6}, 261 | {5, 7, 9}, 262 | {8, 10, 12}, 263 | {11, 13, 15} 264 | }); 265 | 266 | auto result = lhs + rhs; 267 | ASSERT_TENSORS_EQ(expected, result); 268 | } 269 | 270 | TEST(TensorOpsTestSuite, TestAddTensorsWithBroadcast4) { 271 | auto lhs = tensor({ 272 | {1, 2, 3}, 273 | {4, 5, 6}, 274 | {7, 8, 9}, 275 | {10, 11, 12} 276 | }); 277 | 278 | auto rhs = tensor({{1}, {2}, {3}, {4}}); 279 | 280 | auto expected = tensor({ 281 | {2, 3, 4}, 282 | {6, 7, 8}, 283 | {10, 11, 12}, 284 | {14, 15, 16} 285 | }); 286 | 287 | auto result = lhs + rhs; 288 | ASSERT_TENSORS_EQ(expected, result); 289 | } 290 | 291 | TEST(TensorOpsTestSuite, TestAddInplaceTensors1) { 292 | auto lhs = tensor({1, 2, 3}); 293 | auto rhs = tensor({1, 1, 1}); 294 | auto expected = tensor({2, 3, 4}); 295 | 296 | lhs += rhs; 297 | ASSERT_TENSORS_EQ(expected, lhs); 298 | } 299 | 300 | TEST(TensorOpsTestSuite, TestAddInvalidTensors) { 301 | auto lhs = tensor({1, 2, 3}); 302 | auto rhs = tensor({{2, 3}, {4, 5}}); 303 | 304 | EXPECT_THROW(lhs + rhs, CannotBroadcast); 305 | } 306 | 307 | TEST(TensorOpsTestSuite, TestSubTensors1) { 308 | auto lhs = tensor({1, 2, 3}); 309 | auto rhs = tensor({1, 1, 1}); 310 | auto expected = tensor({0, 1, 2}); 311 | 312 | auto result = lhs - rhs; 313 | ASSERT_TENSORS_EQ(expected, result); 314 | } 315 | 316 | TEST(TensorOpsTestSuite, TestSubInplaceTensors1) { 317 | auto lhs = tensor({1, 2, 3}); 318 | auto rhs = tensor({1, 1, 1}); 319 | auto expected = tensor({0, 1, 2}); 320 | 321 | lhs -= rhs; 322 | ASSERT_TENSORS_EQ(expected, lhs); 323 | } 324 | 325 | TEST(TensorOpsTestSuite, TestMulTensors1) { 326 | auto lhs = tensor({1, 2, 3}); 327 | auto rhs = tensor({2, 2, 2}); 328 | auto expected = tensor({2, 4, 6}); 329 | 330 | auto result = lhs.el() * rhs.el(); 331 | ASSERT_TENSORS_EQ(expected, result); 332 | } 333 | 334 | TEST(TensorOpsTestSuite, TestMulInplaceTensors1) { 335 | auto lhs = tensor({1, 2, 3}); 336 | auto rhs = tensor({2, 2, 2}); 337 | auto expected = tensor({2, 4, 6}); 338 | 339 | lhs.el() *= rhs.el(); 340 | ASSERT_TENSORS_EQ(expected, lhs); 341 | } 342 | 343 | TEST(TensorOpsTestSuite, TestDivTensors1) { 344 | auto lhs = tensor({2, 4, 6}); 345 | auto rhs = tensor({2, 2, 2}); 346 | auto expected = tensor({1, 2, 3}); 347 | 348 | auto result = lhs / rhs; 349 | ASSERT_TENSORS_EQ(expected, result); 350 | } 351 | 352 | TEST(TensorOpsTestSuite, TestDivInplaceTensors1) { 353 | auto lhs = tensor({2, 4, 6}); 354 | auto rhs = tensor({2, 2, 2}); 355 | auto expected = tensor({1, 2, 3}); 356 | 357 | lhs /= rhs; 358 | ASSERT_TENSORS_EQ(expected, lhs); 359 | } 360 | 361 | TEST(TensorOpsTestSuite, TestAddTensors2) { 362 | auto lhs = tensor({ 363 | {1, 2, 3}, 364 | {4, 5, 6}, 365 | {7, 8, 9} 366 | }); 367 | 368 | auto rhs = tensor({ 369 | {1, 1, 1}, 370 | {0, 0, 0}, 371 | {1 ,1, 1} 372 | }); 373 | 374 | auto expected = tensor({ 375 | {2, 3, 4}, 376 | {4, 5, 6}, 377 | {8, 9, 10} 378 | }); 379 | 380 | auto result = lhs + rhs; 381 | ASSERT_TENSORS_EQ(expected, result); 382 | } 383 | 384 | TEST(TensorOpsTestSuite, TestAddInplaceTensors2) { 385 | auto lhs = tensor({ 386 | {1, 2, 3}, 387 | {4, 5, 6}, 388 | {7, 8, 9} 389 | }); 390 | 391 | auto rhs = tensor({ 392 | {1, 1, 1}, 393 | {0, 0, 0}, 394 | {1 ,1, 1} 395 | }); 396 | 397 | auto expected = tensor({ 398 | {2, 3, 4}, 399 | {4, 5, 6}, 400 | {8, 9, 10} 401 | }); 402 | 403 | lhs += rhs; 404 | ASSERT_TENSORS_EQ(expected, lhs); 405 | } 406 | 407 | TEST(TensorOpsTestSuite, TestSubTensors2) { 408 | auto lhs = tensor({ 409 | {1, 2, 3}, 410 | {4, 5, 6}, 411 | {7, 8, 9} 412 | }); 413 | 414 | auto rhs = tensor({ 415 | {1, 1, 1}, 416 | {0, 0, 0}, 417 | {1 ,1, 1} 418 | }); 419 | 420 | auto expected = tensor({ 421 | {0, 1, 2}, 422 | {4, 5, 6}, 423 | {6, 7, 8} 424 | }); 425 | 426 | auto result = lhs - rhs; 427 | ASSERT_TENSORS_EQ(expected, result); 428 | } 429 | 430 | TEST(TensorOpsTestSuite, TestSubInplaceTensors2) { 431 | auto lhs = tensor({ 432 | {1, 2, 3}, 433 | {4, 5, 6}, 434 | {7, 8, 9} 435 | }); 436 | 437 | auto rhs = tensor({ 438 | {1, 1, 1}, 439 | {0, 0, 0}, 440 | {1 ,1, 1} 441 | }); 442 | 443 | auto expected = tensor({ 444 | {0, 1, 2}, 445 | {4, 5, 6}, 446 | {6, 7, 8} 447 | }); 448 | 449 | lhs -= rhs; 450 | ASSERT_TENSORS_EQ(expected, lhs); 451 | } 452 | 453 | TEST(TensorOpsTestSuite, TestProductVectorVector) { 454 | auto lhs = tensor({1, 1, 1, 1, 1}); 455 | auto rhs = tensor({2, 2, 2, 2, 2}); 456 | 457 | auto expected = tensor({10}); 458 | auto result = lhs*rhs; 459 | 460 | ASSERT_EQ(expected.shape(), result.shape()); 461 | ASSERT_TENSORS_EQ(expected, result); 462 | } 463 | 464 | TEST(TensorOpsTestSuite, TestProductMatrixVector) { 465 | auto lhs = tensor({{1, 1, 1, 1, 1}, 466 | {1, 1, 1, 1, 1}, 467 | {1, 1, 1, 1, 1}}); 468 | 469 | auto rhs = tensor({2, 2, 2, 2, 2}); 470 | 471 | auto expected = tensor({10, 10, 10}); 472 | auto result = lhs*rhs; 473 | 474 | // 3x5 * 5 => 3 475 | ASSERT_EQ(expected.shape(), result.shape()); 476 | ASSERT_TENSORS_EQ(expected, result); 477 | } 478 | 479 | TEST(TensorOpsTestSuite, TestProductMatrixMatrix) { 480 | auto lhs = tensor({ 481 | {1, 2, 3}, 482 | {4, 5, 6}, 483 | {7, 8, 9}, 484 | {10, 11, 12} 485 | }); 486 | 487 | auto rhs = tensor({ 488 | {2, 2, 2, 2}, 489 | {2, 2, 2, 2}, 490 | {2, 2, 2, 2} 491 | }); 492 | 493 | auto expected = tensor({ 494 | {12, 12, 12, 12}, 495 | {30, 30, 30, 30}, 496 | {48, 48, 48, 48}, 497 | {66, 66, 66, 66} 498 | }); 499 | 500 | auto result = lhs*rhs; 501 | 502 | // 4x3 * 3x4 => 4x4 503 | ASSERT_EQ(expected.shape(), result.shape()); 504 | ASSERT_TENSORS_EQ(expected, result); 505 | } 506 | 507 | TEST(TensorOpsTestSuite, TestProductBatchMatrixMatrix) { 508 | Tensor lhs({2, 3, 4}); 509 | Tensor rhs({2, 4, 6}); 510 | 511 | iota(lhs); 512 | iota(rhs); 513 | 514 | auto expected = tensor({ 515 | {{84, 90, 96, 102, 108, 114}, 516 | {228, 250, 272, 294, 316, 338}, 517 | {372, 410, 448, 486, 524, 562}}, 518 | 519 | {{1812, 1866, 1920, 1974, 2028, 2082}, 520 | {2340, 2410, 2480, 2550, 2620, 2690}, 521 | {2868, 2954, 3040, 3126, 3212, 3298}} 522 | }); 523 | 524 | auto result = lhs*rhs; 525 | 526 | // 2x3x4 * 2x4x6 = 2x2x6 (2, 3, 6) 527 | ASSERT_EQ((extent{2, 3, 6}), result.shape()); 528 | ASSERT_TENSORS_EQ(expected, result); 529 | } 530 | 531 | TEST(TensorOpsTestSuite, TestProductBatchMatrixMatrix2) { 532 | Tensor lhs({1, 2, 3, 4}); 533 | Tensor rhs({1, 2, 4, 6}); 534 | 535 | iota(lhs); 536 | iota(rhs); 537 | 538 | auto expected = tensor({ 539 | {{{84, 90, 96, 102, 108, 114}, 540 | {228, 250, 272, 294, 316, 338}, 541 | {372, 410, 448, 486, 524, 562}}, 542 | 543 | {{1812, 1866, 1920, 1974, 2028, 2082}, 544 | {2340, 2410, 2480, 2550, 2620, 2690}, 545 | {2868, 2954, 3040, 3126, 3212, 3298}}} 546 | }); 547 | 548 | auto result = lhs*rhs; 549 | 550 | ASSERT_EQ((extent{1, 2, 3, 6}), result.shape()); 551 | ASSERT_TENSORS_EQ(expected, result); 552 | } 553 | 554 | TEST(TensorOpsTestSuite, TestSin) { 555 | auto t = tensor({ 556 | {0.0f, 3.145f/2, 3.145f, 2*3.145f}, 557 | {0.0f, 3.145f/2, 3.145f, 2*3.145f}, 558 | {0.0f, 3.145f/2, 3.145f, 2*3.145f}, 559 | {0.0f, 3.145f/2, 3.145f, 2*3.145f} 560 | }); 561 | 562 | auto expected = tensor({ 563 | {std::sin(0.0f), std::sin(3.145f/2), std::sin(3.145f), std::sin(2*3.145f)}, 564 | {std::sin(0.0f), std::sin(3.145f/2), std::sin(3.145f), std::sin(2*3.145f)}, 565 | {std::sin(0.0f), std::sin(3.145f/2), std::sin(3.145f), std::sin(2*3.145f)}, 566 | {std::sin(0.0f), std::sin(3.145f/2), std::sin(3.145f), std::sin(2*3.145f)} 567 | }); 568 | 569 | auto t2 = sin(t); 570 | ASSERT_TENSORS_EQ(expected, t2); 571 | } -------------------------------------------------------------------------------- /test/tests.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "layout.hpp" 5 | 6 | // TEST(LayoutTestSuite, TestView) { 7 | // View test{{2, 3}, {1, 2}, {0, 1}, {0, 0}}; 8 | 9 | // { 10 | // indices index{0, 0}; 11 | // fmt::print("offset = {}\n", test.calculate_offset(std::begin(index), std::end(index))); 12 | // } 13 | 14 | // { 15 | // indices index{0, 1}; 16 | // fmt::print("offset = {}\n", test.calculate_offset(std::begin(index), std::end(index))); 17 | // } 18 | 19 | // { 20 | // indices index{1, 0}; 21 | // fmt::print("offset = {}\n", test.calculate_offset(std::begin(index), std::end(index))); 22 | // } 23 | 24 | // View test2{{2, 3}, {1, 2}, {0, 1}, {1, 1}}; 25 | 26 | // { 27 | // indices index{0, 0}; 28 | // fmt::print("offset = {}\n", test2.calculate_offset(std::begin(index), std::end(index))); 29 | // } 30 | 31 | // { 32 | // indices index{0, 1}; 33 | // fmt::print("offset = {}\n", test2.calculate_offset(std::begin(index), std::end(index))); 34 | // } 35 | 36 | // { 37 | // indices index{1, 0}; 38 | // fmt::print("offset = {}\n", test2.calculate_offset(std::begin(index), std::end(index))); 39 | // } 40 | 41 | // fmt::print("singleton test\n"); 42 | // View test3{{2, 1, 3}, {1, 1, 2}, {0, 1, 2}, {0, 0, 0}}; 43 | // { 44 | // indices index{1, 2}; 45 | // fmt::print("offset = {}\n", test3.calculate_offset(std::begin(index), std::end(index))); 46 | // } 47 | // } 48 | 49 | // #include "tensor.hpp" 50 | /* 51 | TEST(TensorTestSuite, MakeCPUTensor0d) { 52 | auto t = tensor(42); 53 | ASSERT_EQ(1, t.shape().size()); 54 | ASSERT_EQ(1, t.shape()[0]); 55 | } 56 | 57 | TEST(TensorTestSuite, MakeCPUTensor1d) { 58 | auto t = tensor({1, 2, 3}); 59 | ASSERT_EQ(1, t.shape().size()); 60 | ASSERT_EQ(3, t.shape()[0]); 61 | 62 | ASSERT_EQ(1, t(0)); 63 | ASSERT_EQ(2, t(1)); 64 | ASSERT_EQ(3, t(2)); 65 | } 66 | 67 | TEST(TensorTestSuite, MakeCPUTensor2d) { 68 | auto t = tensor({ 69 | {1, 2, 3}, 70 | {4, 5, 6} 71 | }); 72 | ASSERT_EQ(2, t.shape().size()); 73 | ASSERT_EQ(2, t.shape()[0]); 74 | ASSERT_EQ(3, t.shape()[1]); 75 | 76 | // row 0 77 | ASSERT_EQ(1, t(0, 0)); 78 | ASSERT_EQ(2, t(0, 1)); 79 | ASSERT_EQ(3, t(0, 2)); 80 | 81 | // row 1 82 | ASSERT_EQ(4, t(1, 0)); 83 | ASSERT_EQ(5, t(1, 1)); 84 | ASSERT_EQ(6, t(1, 2)); 85 | } 86 | 87 | TEST(TensorTestSuite, MakeCPUTensor3d) { 88 | auto t = tensor({ 89 | { 90 | {1, 2, 3}, 91 | {4, 5, 6} 92 | }, 93 | { 94 | {7, 8, 9}, 95 | {10, 11, 12} 96 | } 97 | }); 98 | ASSERT_EQ(3, t.shape().size()); 99 | ASSERT_EQ(2, t.shape()[0]); 100 | ASSERT_EQ(2, t.shape()[1]); 101 | ASSERT_EQ(3, t.shape()[2]); 102 | 103 | ASSERT_EQ(1, t(0, 0, 0)); 104 | ASSERT_EQ(2, t(0, 0, 1)); 105 | ASSERT_EQ(3, t(0, 0, 2)); 106 | 107 | ASSERT_EQ(4, t(0, 1, 0)); 108 | ASSERT_EQ(5, t(0, 1, 1)); 109 | ASSERT_EQ(6, t(0, 1, 2)); 110 | 111 | ASSERT_EQ(7, t(1, 0, 0)); 112 | ASSERT_EQ(8, t(1, 0, 1)); 113 | ASSERT_EQ(9, t(1, 0, 2)); 114 | 115 | ASSERT_EQ(10, t(1, 1, 0)); 116 | ASSERT_EQ(11, t(1, 1, 1)); 117 | ASSERT_EQ(12, t(1, 1, 2)); 118 | } 119 | 120 | TEST(TensorTestSuite, TensorIndex1d) { 121 | TensorIndex index(1, {10}); 122 | 123 | for (auto i = 0; i < 10; i++) { 124 | ASSERT_EQ(i, index[0]); 125 | ++index; 126 | } 127 | 128 | // the index will wrap back to 0 129 | ASSERT_EQ(0, index[0]); 130 | } 131 | 132 | TEST(TensorTestSuite, TensorIndex2d) { 133 | TensorIndex index(2, {5, 5}); 134 | 135 | for (auto i = 0; i < 5; i++) { 136 | for (auto j = 0; j < 5; j++) { 137 | ASSERT_EQ(i, index[0]); 138 | ASSERT_EQ(j, index[1]); 139 | ++index; 140 | } 141 | } 142 | 143 | ASSERT_EQ(0, index[0]); 144 | ASSERT_EQ(0, index[1]); 145 | } 146 | 147 | TEST(TensorTestSuite, TensorIndex3d) { 148 | TensorIndex index(3, {5, 5, 5}); 149 | 150 | for (auto i = 0; i < 5; i++) { 151 | for (auto j = 0; j < 5; j++) { 152 | for (auto k = 0; k < 5; k++) { 153 | ASSERT_EQ(i, index[0]); 154 | ASSERT_EQ(j, index[1]); 155 | ASSERT_EQ(k, index[2]); 156 | ++index; 157 | } 158 | } 159 | } 160 | 161 | ASSERT_EQ(0, index[0]); 162 | ASSERT_EQ(0, index[1]); 163 | ASSERT_EQ(0, index[2]); 164 | } 165 | 166 | TEST(TensorTestSuite, IteratorCPUTensor1d) { 167 | auto t = tensor({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); 168 | 169 | // test reading 170 | int i = 0; 171 | for (auto const &el : t) { 172 | ASSERT_EQ(i++, el); 173 | } 174 | 175 | // test writing 176 | for (auto &el : t) { 177 | el = 0; 178 | } 179 | 180 | // verify write 181 | for (auto const &el : t) { 182 | ASSERT_EQ(0, el); 183 | } 184 | } 185 | 186 | TEST(TensorTestSuite, IteratorCPUTensor1dView) { 187 | auto t = tensor({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); 188 | 189 | // create a view of t 190 | auto t2 = t.view({{8}, {1}, t.view().strides, t.view().order}); 191 | 192 | ASSERT_EQ(8, t2.shape()[0]); 193 | 194 | // test read 195 | int i = 1; 196 | for (auto const &el : t2) { 197 | ASSERT_EQ(i++, el); 198 | } 199 | 200 | // test write 201 | for (auto &el : t2) { 202 | el = -1; 203 | } 204 | 205 | // verify write 206 | for (auto const &el : t2) { 207 | ASSERT_EQ(-1, el); 208 | } 209 | 210 | // verify we didn't write outside the view 211 | ASSERT_EQ(0, t(0)); 212 | ASSERT_EQ(9, t(9)); 213 | 214 | // verify we wrote inside the view 215 | for (int i = 1; i < 9; i++) { 216 | ASSERT_EQ(-1, t(i)); 217 | } 218 | } 219 | 220 | TEST(TensorTestSuite, IteratorCPUTensor2d) { 221 | auto t = tensor({ 222 | {1, 4, 7}, 223 | {2, 5, 8}, 224 | {3, 6, 9} 225 | }); 226 | 227 | // expected order for values to be read 228 | int expected[] = {1, 4, 7, 2, 5, 8, 3, 6, 9}; 229 | 230 | // test reading 231 | int i = 0; 232 | for (auto const &el : t) { 233 | ASSERT_EQ(expected[i++], el); 234 | } 235 | 236 | // test writing 237 | for (auto &el : t) { 238 | el = 0; 239 | } 240 | 241 | // verify writing 242 | for (auto const &el : t) { 243 | ASSERT_EQ(el, 0); 244 | } 245 | } 246 | 247 | TEST(TensorTestSuite, IteratorCPUTensor2dView) { 248 | auto t = tensor({ 249 | {1, 2, 3}, 250 | {4, 5, 6}, 251 | {7, 8, 9} 252 | }); 253 | 254 | // create a view of t 255 | auto t2 = t.view({{2, 2}, {1, 1}, t.view().strides, t.view().order}); 256 | 257 | ASSERT_EQ(2, t2.shape()[0]); 258 | ASSERT_EQ(2, t2.shape()[1]); 259 | 260 | int i = 0; 261 | int values[] = {5, 6, 8, 9}; 262 | for (auto const &el : t2) { 263 | ASSERT_EQ(values[i++], el); 264 | } 265 | 266 | // test write 267 | for (auto &el : t2) { 268 | el = -1; 269 | } 270 | 271 | // verify write 272 | for (auto const &el : t2) { 273 | ASSERT_EQ(-1, el); 274 | } 275 | 276 | // verify we didn't write outside the view 277 | ASSERT_EQ(1, t(0, 0)); 278 | ASSERT_EQ(2, t(0, 1)); 279 | ASSERT_EQ(3, t(0, 2)); 280 | ASSERT_EQ(4, t(1, 0)); 281 | ASSERT_EQ(7, t(2, 0)); 282 | 283 | // verify we did write inside the view 284 | ASSERT_EQ(-1, t(1, 1)); 285 | ASSERT_EQ(-1, t(1, 2)); 286 | ASSERT_EQ(-1, t(2, 1)); 287 | ASSERT_EQ(-1, t(2, 2)); 288 | } 289 | 290 | TEST(TensorTestSuite, TranposeCPUTensor2dWithViewOffset) { 291 | auto t = tensor({ 292 | {0, 1, 2, 3}, 293 | {4, 5, 6, 7}, 294 | {8, 9, 10, 11}, 295 | {12, 13, 14, 15} 296 | }); 297 | 298 | auto t2 = t.view({{3, 2}, {0, 1}, t.view().strides, t.view().order}); 299 | 300 | ASSERT_EQ(1, t2(0, 0)); 301 | ASSERT_EQ(2, t2(0, 1)); 302 | ASSERT_EQ(5, t2(1, 0)); 303 | ASSERT_EQ(6, t2(1, 1)); 304 | ASSERT_EQ(9, t2(2, 0)); 305 | ASSERT_EQ(10, t2(2, 1)); 306 | 307 | auto t3 = transpose(t2); 308 | 309 | ASSERT_EQ(1, t3(0, 0)); 310 | ASSERT_EQ(5, t3(0, 1)); 311 | ASSERT_EQ(9, t3(0, 2)); 312 | ASSERT_EQ(2, t3(1, 0)); 313 | ASSERT_EQ(6, t3(1, 1)); 314 | ASSERT_EQ(10, t3(1, 2)); 315 | } 316 | 317 | TEST(TensorTestSuite, SliceCPUTensor2d) { 318 | auto t = tensor({ 319 | {0, 1, 2}, 320 | {3, 4, 5}, 321 | {6, 7, 8} 322 | }); 323 | 324 | // [3, 4, 5] 325 | Tensor t2 = t[1]; 326 | 327 | // verify the values match 328 | ASSERT_EQ(3, t2(0)); 329 | ASSERT_EQ(4, t2(1)); 330 | ASSERT_EQ(5, t2(2)); 331 | } 332 | 333 | TEST(TensorTestSuite, SliceCPUTensor3d) { 334 | auto t = tensor({ 335 | { 336 | {0, 1, 2}, 337 | {3, 4, 5}, 338 | }, 339 | { 340 | {6, 7, 8}, 341 | {9, 10, 11} 342 | } 343 | }); 344 | 345 | // [ [ 6, 7, 8] 346 | // [ 9, 10, 11] ] 347 | Tensor t2 = t[1]; 348 | 349 | ASSERT_EQ(6, t2(0, 0)); 350 | ASSERT_EQ(7, t2(0, 1)); 351 | ASSERT_EQ(8, t2(0, 2)); 352 | ASSERT_EQ(9, t2(1, 0)); 353 | ASSERT_EQ(10, t2(1, 1)); 354 | ASSERT_EQ(11, t2(1, 2)); 355 | 356 | // [9, 10, 11] 357 | Tensor t3 = t2[1]; 358 | 359 | ASSERT_EQ(9, t3(0)); 360 | ASSERT_EQ(10, t3(1)); 361 | ASSERT_EQ(11, t3(2)); 362 | } 363 | 364 | TEST(TensorTestSuite, MultiSliceCPUTensor3d) { 365 | auto t = tensor({ 366 | { 367 | {0, 1, 2}, 368 | {3, 4, 5}, 369 | }, 370 | { 371 | {6, 7, 8}, 372 | {9, 10, 11} 373 | } 374 | }); 375 | 376 | // [10] 377 | Tensor t2 = t[1][1][1]; 378 | ASSERT_EQ(10, t2(0)); 379 | 380 | // verify that we can still slice the single value 381 | // [10] 382 | Tensor t3 = t2[0]; 383 | ASSERT_EQ(10, t3(0)); 384 | } 385 | 386 | TEST(TensorTestSuite, RangeSliceCPUTensor3d) { 387 | auto t = tensor({ 388 | { 389 | {0, 1, 2}, 390 | {3, 4, 5}, 391 | }, 392 | { 393 | {6, 7, 8}, 394 | {9, 10, 11} 395 | } 396 | }); 397 | 398 | // [ [ 0, 1, 2] 399 | // [ 3, 4, 5] ] 400 | // 401 | // auto slice = t[0]; 402 | Tensor t2 = t[0]; //slice; //[{0, 2}][{0, 3}]; 403 | fmt::print("\nt2 = {}\n", t2); 404 | 405 | ASSERT_EQ(0, t2(0, 0)); 406 | ASSERT_EQ(1, t2(0, 1)); 407 | ASSERT_EQ(2, t2(0, 2)); 408 | ASSERT_EQ(3, t2(1, 0)); 409 | ASSERT_EQ(4, t2(1, 1)); 410 | ASSERT_EQ(5, t2(1, 2)); 411 | 412 | // [ [1, 2], 413 | // [4, 5] ] 414 | Tensor t3 = t[0][{0, 2}][{1, 3}]; 415 | fmt::print("\nt2 = {}\n", t3); 416 | 417 | ASSERT_EQ(1, t3(0, 0)); 418 | ASSERT_EQ(2, t3(0, 1)); 419 | ASSERT_EQ(4, t3(1, 0)); 420 | ASSERT_EQ(5, t3(1, 1)); 421 | 422 | // [1 4] 423 | Tensor t4 = t[0][{0, 2}][{1, 2}]; 424 | fmt::print("\nt4 = {}\n", t4); 425 | ASSERT_EQ(1, t4(0)); 426 | ASSERT_EQ(4, t4(1)); 427 | }*/ 428 | 429 | 430 | 431 | int main(int argc, char **argv) { 432 | testing::InitGoogleTest(&argc, argv); 433 | return RUN_ALL_TESTS(); 434 | } --------------------------------------------------------------------------------